fix: docker setup for usercode

This commit is contained in:
Stijnvandenbroek
2026-03-04 10:03:48 +00:00
parent fc43570506
commit 0a0f9d711b
5 changed files with 41 additions and 42 deletions

11
.dockerignore Normal file
View File

@@ -0,0 +1,11 @@
.venv
__pycache__
*.pyc
.git
.github
.pytest_cache
*.egg-info
dbt/target
dbt/logs
dagster_home/storage
dagster_home/logs

View File

@@ -1,5 +1,5 @@
.PHONY: help install sync build up down restart logs ps \
build-code reload-code \
build-code reload-code reload-usercode \
dbt-parse dbt-run dbt-test dbt-build dbt-seed dbt-clean dbt-deps dbt-docs \
dagster-dev \
lint lint-fix lint-python lint-sql lint-format \
@@ -11,7 +11,7 @@ COMPOSE := docker compose
DBT_DIR := dbt
DBT := uv run dbt
# Container(s) that carry user code
CODE_SERVICES := dagster-webserver dagster-daemon
CODE_SERVICES := dagster-usercode dagster-webserver dagster-daemon
## —— Help ——————————————————————————————————————————————————————
help: ## Show this help
@@ -51,6 +51,10 @@ reload-code: ## Rebuild and restart the user-code containers (no downtime for po
$(COMPOSE) build $(CODE_SERVICES)
$(COMPOSE) up -d --no-deps $(CODE_SERVICES)
reload-usercode: ## Rebuild and restart only the gRPC usercode server
$(COMPOSE) build dagster-usercode
$(COMPOSE) up -d --no-deps --force-recreate dagster-usercode
## —— dbt (local, via uv) —————————————————————————————————————
dbt-parse: ## Parse the dbt project and generate manifest.json
$(DBT) parse --project-dir $(DBT_DIR) --profiles-dir $(DBT_DIR)

View File

@@ -1,4 +1,5 @@
load_from:
- python_package:
package_name: data_platform
attribute: defs
- grpc_server:
host: dagster-usercode
port: 4000
location_name: data_platform

View File

@@ -8,15 +8,12 @@ x-dagster: &dagster-common
env_file: .env
environment:
DAGSTER_HOME: /app/dagster_home
volumes:
- dbt-target:/app/dbt/target
depends_on:
postgres:
condition: service_healthy
restart: unless-stopped
services:
# Metadata storage and dbt target
postgres:
image: postgres:16
@@ -37,19 +34,37 @@ services:
timeout: 5s
retries: 5
# User code gRPC server
dagster-usercode:
<<: *dagster-common
container_name: dagster-usercode
command: ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-m", "data_platform"]
volumes:
- dbt-target:/app/dbt/target
expose:
- "4000"
# Web UI
dagster-webserver:
<<: *dagster-common
container_name: dagster-webserver
entrypoint: []
command: ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000"]
ports:
- "3000:3000"
depends_on:
dagster-usercode:
condition: service_started
# Schedules, sensors and run queuing
dagster-daemon:
<<: *dagster-common
container_name: dagster-daemon
entrypoint: []
command: ["dagster-daemon", "run"]
depends_on:
dagster-usercode:
condition: service_started
volumes:
postgres-data:

View File

@@ -9,7 +9,6 @@ dependencies = [
"dagster-dbt",
"dbt-core",
"dbt-postgres",
"pyfunda",
]
[build-system]
@@ -23,39 +22,8 @@ packages = ["data_platform"]
module_name = "data_platform"
attribute = "defs"
[dependency-groups]
dev = [
[tool.uv]
dev-dependencies = [
"pytest",
"dagster-webserver",
"pre-commit",
"ruff",
"sqlfluff",
"sqlfluff-templater-dbt",
]
[tool.ruff]
target-version = "py311"
line-length = 100
[tool.ruff.lint]
select = [
"E", # pycodestyle errors
"W", # pycodestyle warnings
"F", # pyflakes
"I", # isort
"UP", # pyupgrade
"B", # flake8-bugbear
"SIM", # flake8-simplify
"C4", # flake8-comprehensions
]
ignore = [
"E501", # line too long handled by formatter
"B008", # function call in default argument (Dagster configs use this)
]
[tool.ruff.lint.isort]
known-first-party = ["data_platform"]
[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = "-v"