diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c9bd605 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.venv +__pycache__ +*.pyc +.git +.github +.pytest_cache +*.egg-info +dbt/target +dbt/logs +dagster_home/storage +dagster_home/logs diff --git a/Makefile b/Makefile index 9e62300..804395b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: help install sync build up down restart logs ps \ - build-code reload-code \ + build-code reload-code reload-usercode \ dbt-parse dbt-run dbt-test dbt-build dbt-seed dbt-clean dbt-deps dbt-docs \ dagster-dev \ lint lint-fix lint-python lint-sql lint-format \ @@ -11,7 +11,7 @@ COMPOSE := docker compose DBT_DIR := dbt DBT := uv run dbt # Container(s) that carry user code -CODE_SERVICES := dagster-webserver dagster-daemon +CODE_SERVICES := dagster-usercode dagster-webserver dagster-daemon ## —— Help —————————————————————————————————————————————————————— help: ## Show this help @@ -51,6 +51,10 @@ reload-code: ## Rebuild and restart the user-code containers (no downtime for po $(COMPOSE) build $(CODE_SERVICES) $(COMPOSE) up -d --no-deps $(CODE_SERVICES) +reload-usercode: ## Rebuild and restart only the gRPC usercode server + $(COMPOSE) build dagster-usercode + $(COMPOSE) up -d --no-deps --force-recreate dagster-usercode + ## —— dbt (local, via uv) ————————————————————————————————————— dbt-parse: ## Parse the dbt project and generate manifest.json $(DBT) parse --project-dir $(DBT_DIR) --profiles-dir $(DBT_DIR) diff --git a/dagster_home/workspace.yaml b/dagster_home/workspace.yaml index f5ea6fc..4326b16 100644 --- a/dagster_home/workspace.yaml +++ b/dagster_home/workspace.yaml @@ -1,4 +1,5 @@ load_from: - - python_package: - package_name: data_platform - attribute: defs + - grpc_server: + host: dagster-usercode + port: 4000 + location_name: data_platform diff --git a/docker-compose.yaml b/docker-compose.yaml index 799cad1..e00c881 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,15 +8,12 @@ x-dagster: &dagster-common env_file: .env environment: DAGSTER_HOME: /app/dagster_home - volumes: - - dbt-target:/app/dbt/target depends_on: postgres: condition: service_healthy restart: unless-stopped services: - # Metadata storage and dbt target postgres: image: postgres:16 @@ -37,19 +34,37 @@ services: timeout: 5s retries: 5 + # User code gRPC server + dagster-usercode: + <<: *dagster-common + container_name: dagster-usercode + command: ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-m", "data_platform"] + volumes: + - dbt-target:/app/dbt/target + expose: + - "4000" + # Web UI dagster-webserver: <<: *dagster-common container_name: dagster-webserver + entrypoint: [] command: ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000"] ports: - "3000:3000" + depends_on: + dagster-usercode: + condition: service_started # Schedules, sensors and run queuing dagster-daemon: <<: *dagster-common container_name: dagster-daemon + entrypoint: [] command: ["dagster-daemon", "run"] + depends_on: + dagster-usercode: + condition: service_started volumes: postgres-data: diff --git a/pyproject.toml b/pyproject.toml index ec51158..9974305 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,6 @@ dependencies = [ "dagster-dbt", "dbt-core", "dbt-postgres", - "pyfunda", ] [build-system] @@ -23,39 +22,8 @@ packages = ["data_platform"] module_name = "data_platform" attribute = "defs" -[dependency-groups] -dev = [ +[tool.uv] +dev-dependencies = [ "pytest", "dagster-webserver", - "pre-commit", - "ruff", - "sqlfluff", - "sqlfluff-templater-dbt", ] - -[tool.ruff] -target-version = "py311" -line-length = 100 - -[tool.ruff.lint] -select = [ - "E", # pycodestyle errors - "W", # pycodestyle warnings - "F", # pyflakes - "I", # isort - "UP", # pyupgrade - "B", # flake8-bugbear - "SIM", # flake8-simplify - "C4", # flake8-comprehensions -] -ignore = [ - "E501", # line too long – handled by formatter - "B008", # function call in default argument (Dagster configs use this) -] - -[tool.ruff.lint.isort] -known-first-party = ["data_platform"] - -[tool.pytest.ini_options] -testpaths = ["tests"] -addopts = "-v"