diff --git a/.env.example b/.env.example index b5bd60a..04b87a7 100644 --- a/.env.example +++ b/.env.example @@ -1,16 +1,16 @@ -# ── PostgreSQL ─────────────────────────────────────────────────────────────── +# PostgreSQL POSTGRES_HOST=postgres POSTGRES_PORT=5432 POSTGRES_USER=dagster POSTGRES_PASSWORD=changeme POSTGRES_DB=dagster -# ── Dagster metadata storage (uses the same postgres instance) ─────────────── +# Dagster metadata storage (same postgres instance) DAGSTER_POSTGRES_URL=postgresql://dagster:changeme@postgres:5432/dagster -# ── dbt profile target (overrides profiles.yml env_var defaults) ───────────── +# dbt profile target DBT_TARGET=dev -# ── pgAdmin ────────────────────────────────────────────────────────────────── +# pgAdmin PGADMIN_EMAIL=admin@example.com PGADMIN_PASSWORD=changeme diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fe0328..014c99b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: branches: [main] jobs: - # ── Lint ──────────────────────────────────────────────────────────────── + # Lint lint-python: name: Ruff (lint + format) runs-on: ubuntu-latest @@ -67,7 +67,7 @@ jobs: "**/*.yml" "**/*.yaml" "**/*.md" \ --ignore-path .prettierignore - # ── Tests ──────────────────────────────────────────────────────────────── + # Tests test: name: Pytest runs-on: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d66572d..bc10ffd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: - # ── Local hooks (use uv-managed tool versions from uv.lock) ────────────── + # Local hooks using uv-managed tool versions - repo: local hooks: - id: ruff-lint @@ -19,7 +19,7 @@ repos: entry: uv run sqlfluff lint --dialect postgres language: system types: [sql] - files: ^(dbt/models/|data_platform/assets/ingestion/) + files: ^dbt/models/ - id: prettier name: prettier @@ -34,7 +34,7 @@ repos: \.pytest_cache/ ) - # ── General hygiene ─────────────────────────────────────────────────────── + # General hygiene - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/Dockerfile b/Dockerfile index 12b0d28..0762234 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# ── Usercode: full application (gRPC server) ───────────────────────── +# Usercode: full application (gRPC server) FROM python:3.12-slim AS usercode WORKDIR /app @@ -21,15 +21,14 @@ ENV DAGSTER_HOME=/app/dagster_home RUN chmod +x /app/entrypoint.sh ENTRYPOINT ["/app/entrypoint.sh"] -# ── Infrastructure: webserver / daemon (no user code) ──────────────── +# Infrastructure: webserver / daemon (no user code) FROM python:3.12-slim AS dagster-infra WORKDIR /app COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv -# Only the Dagster packages needed to run webserver / daemon and reach -# the metadata store. No application code is installed. +# No application code, only dagster packages for webserver/daemon RUN uv pip install --system dagster dagster-webserver dagster-postgres COPY dagster_home/ ./dagster_home/ diff --git a/Makefile b/Makefile index decb671..c3e12d9 100644 --- a/Makefile +++ b/Makefile @@ -13,18 +13,18 @@ DBT := uv run dbt # Container(s) that carry user code CODE_SERVICES := dagster-usercode -## —— Help —————————————————————————————————————————————————————— +## Help help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*##' $(MAKEFILE_LIST) | \ awk 'BEGIN {FS = ":.*## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' -## —— Python / uv —————————————————————————————————————————————— +## Python / uv install: ## Install all dependencies (including dev) with uv uv sync sync: install ## Alias for install -## —— Docker Compose ——————————————————————————————————————————— +## Docker Compose build: ## Build all Docker images $(COMPOSE) build @@ -43,7 +43,7 @@ logs: ## Tail logs for all services (ctrl-c to quit) ps: ## Show running containers $(COMPOSE) ps -## —— User-code container shortcuts ———————————————————————————— +## User-code container shortcuts build-code: ## Rebuild only the user-code containers $(COMPOSE) build $(CODE_SERVICES) @@ -55,7 +55,7 @@ reload-usercode: ## Rebuild and restart only the gRPC usercode server $(COMPOSE) build dagster-usercode $(COMPOSE) up -d --no-deps --force-recreate dagster-usercode -## —— dbt (local, via uv) ————————————————————————————————————— +## dbt (local, via uv) dbt-parse: ## Parse the dbt project and generate manifest.json $(DBT) parse --project-dir $(DBT_DIR) --profiles-dir $(DBT_DIR) @@ -81,11 +81,11 @@ dbt-docs: ## Generate and serve dbt docs $(DBT) docs generate --project-dir $(DBT_DIR) --profiles-dir $(DBT_DIR) $(DBT) docs serve --project-dir $(DBT_DIR) --profiles-dir $(DBT_DIR) -## —— Dagster (local dev) ————————————————————————————————————— +## Dagster (local dev) dagster-dev: ## Start Dagster webserver locally for development uv run dagster dev -## —— Quality ────────────────────────────────────────────────── +## Quality lint: lint-python lint-sql lint-format ## Run all linters (ruff + sqlfluff + prettier) lint-python: ## Ruff lint + format check @@ -106,7 +106,7 @@ lint-fix: ## Auto-fix all linters (ruff + sqlfluff + prettier) npx --yes prettier --write "**/*.yml" "**/*.yaml" "**/*.md" \ --ignore-path .prettierignore -## —— Pre-commit ──────────────────────────────────────────────── +## Pre-commit pre-commit-install: ## Install pre-commit hooks into .git/hooks uv run pre-commit install @@ -119,7 +119,7 @@ validate: ## Validate Dagster definitions load without errors test: ## Run pytest uv run pytest -## —— Misc ————————————————————————————————————————————————————— +## Misc clean: ## Remove Python caches and build artefacts find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true rm -rf .pytest_cache dist *.egg-info