fix: docker setup for usercode
This commit is contained in:
11
.dockerignore
Normal file
11
.dockerignore
Normal file
@@ -0,0 +1,11 @@
|
||||
.venv
|
||||
__pycache__
|
||||
*.pyc
|
||||
.git
|
||||
.github
|
||||
.pytest_cache
|
||||
*.egg-info
|
||||
dbt/target
|
||||
dbt/logs
|
||||
dagster_home/storage
|
||||
dagster_home/logs
|
||||
8
Makefile
8
Makefile
@@ -1,5 +1,5 @@
|
||||
.PHONY: help install sync build up down restart logs ps \
|
||||
build-code reload-code \
|
||||
build-code reload-code reload-usercode \
|
||||
dbt-parse dbt-run dbt-test dbt-build dbt-seed dbt-clean dbt-deps dbt-docs \
|
||||
dagster-dev \
|
||||
lint lint-fix lint-python lint-sql lint-format \
|
||||
@@ -11,7 +11,7 @@ COMPOSE := docker compose
|
||||
DBT_DIR := dbt
|
||||
DBT := uv run dbt
|
||||
# Container(s) that carry user code
|
||||
CODE_SERVICES := dagster-webserver dagster-daemon
|
||||
CODE_SERVICES := dagster-usercode dagster-webserver dagster-daemon
|
||||
|
||||
## —— Help ——————————————————————————————————————————————————————
|
||||
help: ## Show this help
|
||||
@@ -51,6 +51,10 @@ reload-code: ## Rebuild and restart the user-code containers (no downtime for po
|
||||
$(COMPOSE) build $(CODE_SERVICES)
|
||||
$(COMPOSE) up -d --no-deps $(CODE_SERVICES)
|
||||
|
||||
reload-usercode: ## Rebuild and restart only the gRPC usercode server
|
||||
$(COMPOSE) build dagster-usercode
|
||||
$(COMPOSE) up -d --no-deps --force-recreate dagster-usercode
|
||||
|
||||
## —— dbt (local, via uv) —————————————————————————————————————
|
||||
dbt-parse: ## Parse the dbt project and generate manifest.json
|
||||
$(DBT) parse --project-dir $(DBT_DIR) --profiles-dir $(DBT_DIR)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
load_from:
|
||||
- python_package:
|
||||
package_name: data_platform
|
||||
attribute: defs
|
||||
- grpc_server:
|
||||
host: dagster-usercode
|
||||
port: 4000
|
||||
location_name: data_platform
|
||||
|
||||
@@ -8,15 +8,12 @@ x-dagster: &dagster-common
|
||||
env_file: .env
|
||||
environment:
|
||||
DAGSTER_HOME: /app/dagster_home
|
||||
volumes:
|
||||
- dbt-target:/app/dbt/target
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
services:
|
||||
|
||||
# Metadata storage and dbt target
|
||||
postgres:
|
||||
image: postgres:16
|
||||
@@ -37,19 +34,37 @@ services:
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# User code gRPC server
|
||||
dagster-usercode:
|
||||
<<: *dagster-common
|
||||
container_name: dagster-usercode
|
||||
command: ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-m", "data_platform"]
|
||||
volumes:
|
||||
- dbt-target:/app/dbt/target
|
||||
expose:
|
||||
- "4000"
|
||||
|
||||
# Web UI
|
||||
dagster-webserver:
|
||||
<<: *dagster-common
|
||||
container_name: dagster-webserver
|
||||
entrypoint: []
|
||||
command: ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000"]
|
||||
ports:
|
||||
- "3000:3000"
|
||||
depends_on:
|
||||
dagster-usercode:
|
||||
condition: service_started
|
||||
|
||||
# Schedules, sensors and run queuing
|
||||
dagster-daemon:
|
||||
<<: *dagster-common
|
||||
container_name: dagster-daemon
|
||||
entrypoint: []
|
||||
command: ["dagster-daemon", "run"]
|
||||
depends_on:
|
||||
dagster-usercode:
|
||||
condition: service_started
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
|
||||
@@ -9,7 +9,6 @@ dependencies = [
|
||||
"dagster-dbt",
|
||||
"dbt-core",
|
||||
"dbt-postgres",
|
||||
"pyfunda",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
@@ -23,39 +22,8 @@ packages = ["data_platform"]
|
||||
module_name = "data_platform"
|
||||
attribute = "defs"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
[tool.uv]
|
||||
dev-dependencies = [
|
||||
"pytest",
|
||||
"dagster-webserver",
|
||||
"pre-commit",
|
||||
"ruff",
|
||||
"sqlfluff",
|
||||
"sqlfluff-templater-dbt",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py311"
|
||||
line-length = 100
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
"E", # pycodestyle errors
|
||||
"W", # pycodestyle warnings
|
||||
"F", # pyflakes
|
||||
"I", # isort
|
||||
"UP", # pyupgrade
|
||||
"B", # flake8-bugbear
|
||||
"SIM", # flake8-simplify
|
||||
"C4", # flake8-comprehensions
|
||||
]
|
||||
ignore = [
|
||||
"E501", # line too long – handled by formatter
|
||||
"B008", # function call in default argument (Dagster configs use this)
|
||||
]
|
||||
|
||||
[tool.ruff.lint.isort]
|
||||
known-first-party = ["data_platform"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
addopts = "-v"
|
||||
|
||||
Reference in New Issue
Block a user