diff --git a/Dockerfile.code b/Dockerfile.code index fbc5e17..194ce5e 100644 --- a/Dockerfile.code +++ b/Dockerfile.code @@ -13,7 +13,7 @@ RUN uv pip install polars-lts-cpu --system ARG APP ENV APP=$APP -ENV PYTHONPATH=/apps/$APP/src/:/shared/src/ +ENV PYTHONPATH=/apps/$APP/src/:/shared/src/:$PYTHONPATH WORKDIR /opt/dagster/home # Run dagster gRPC server on port 4000 diff --git a/Dockerfile.code.playwright b/Dockerfile.code.playwright index 37d97e5..1ef58d4 100644 --- a/Dockerfile.code.playwright +++ b/Dockerfile.code.playwright @@ -18,7 +18,7 @@ RUN . /venv/bin/activate && \ ARG APP ENV APP=$APP -ENV PYTHONPATH=/apps/$APP/src/:/shared/src/ +ENV PYTHONPATH=/apps/$APP/src/:/shared/src/:$PYTHONPATH WORKDIR /opt/dagster/home # Run dagster gRPC server on port 4000 diff --git a/Dockerfile.system b/Dockerfile.system index 45e94a6..20a2691 100644 --- a/Dockerfile.system +++ b/Dockerfile.system @@ -33,4 +33,6 @@ EOF RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] +ENV PYTHONPATH=/system:$PYTHONPATH + WORKDIR $DAGSTER_HOME diff --git a/apps/stocks/src/assets.py b/apps/stocks/src/assets.py index 3e8c7c9..d0afd1b 100644 --- a/apps/stocks/src/assets.py +++ b/apps/stocks/src/assets.py @@ -105,3 +105,15 @@ def raw_html(context: dg.AssetExecutionContext) -> Iterator[dg.Output[str]]: partitions_def=daily_partitions_def, ) def daily_html() -> str: ... + + +class MyAssetConfig(dg.Config): + image: str = "bla" + + +@asset(metadata={"docker_image": "my-other-image:latest"}) +def my_asset(context: dg.AssetExecutionContext, config: MyAssetConfig) -> None: + ic(context.op_config) + ic(config) + # image = context.op_config.image + # context.log.info(f"This asset wants to use Docker image: {image}") diff --git a/apps/stocks/src/definitions.py b/apps/stocks/src/definitions.py index 5d12b11..ddbe2e2 100644 --- a/apps/stocks/src/definitions.py +++ b/apps/stocks/src/definitions.py @@ -1,4 +1,5 @@ import assets +import jobs import sensors from dagster_polars import PolarsParquetIOManager from icecream import install @@ -21,5 +22,6 @@ definitions = dg.Definitions( "html_io_manager": HtmlIOManager(base_dir=STORAGE_DIR), "polars_parquet_io_manager": PolarsParquetIOManager(base_dir=STORAGE_DIR), }, + jobs=[jobs.test_job], sensors=[sensors.check_update], ) diff --git a/apps/stocks/src/jobs.py b/apps/stocks/src/jobs.py index d7c3099..593159a 100644 --- a/apps/stocks/src/jobs.py +++ b/apps/stocks/src/jobs.py @@ -3,3 +3,9 @@ import assets import dagster as dg raw_html_job = dg.define_asset_job("deals_job", selection=[assets.raw_html.key]) + +test_job = dg.define_asset_job( + "my_asset_job", + selection=[assets.my_asset.key], + tags={"docker/image": "my.registry/image:from-config"}, +) diff --git a/compose.code.yaml b/compose.code.yaml index dd53ac2..6b73ec8 100644 --- a/compose.code.yaml +++ b/compose.code.yaml @@ -59,6 +59,15 @@ services: networks: - dagster + dagster-code-stocks-playwright: + build: + context: apps/stocks + dockerfile: ../../Dockerfile.code.playwright + args: + - APP=stocks + image: dagster-code-stocks-playwright + profiles: [ "never" ] + dagster-code-tesla: build: context: apps/tesla diff --git a/compose.system.yaml b/compose.system.yaml index f30e7fa..4cf6901 100644 --- a/compose.system.yaml +++ b/compose.system.yaml @@ -25,6 +25,7 @@ x-volumes: &volumes volumes: - /opt/dagster/dagster.yaml:/opt/dagster/home/dagster.yaml.template:ro - /opt/dagster/workspace.yaml:/opt/dagster/home/workspace.yaml:ro + - /opt/dagster/system/:/system/:ro - /opt/dagster/storage/:/storage/:rw - /opt/dagster/logs/:/logs:rw - /var/run/docker.sock:/var/run/docker.sock:rw diff --git a/dagster.yaml b/dagster.yaml index eef6d42..1f35a74 100644 --- a/dagster.yaml +++ b/dagster.yaml @@ -11,8 +11,8 @@ run_coordinator: class: QueuedRunCoordinator run_launcher: - module: dagster_docker - class: DockerRunLauncher + module: run_launcher + class: CustomDockerRunLauncher config: env_vars: - DAGSTER_POSTGRES_HOST