custom run launcher for custom docker image

Dockerfile.code

@@ -13,7 +13,7 @@ RUN uv pip install polars-lts-cpu --system
 
 ARG APP
 ENV APP=$APP
-ENV PYTHONPATH=/apps/$APP/src/:/shared/src/
+ENV PYTHONPATH=/apps/$APP/src/:/shared/src/:$PYTHONPATH
 WORKDIR /opt/dagster/home
 
 # Run dagster gRPC server on port 4000

Dockerfile.code.playwright

@@ -18,7 +18,7 @@ RUN . /venv/bin/activate && \
 
 ARG APP
 ENV APP=$APP
-ENV PYTHONPATH=/apps/$APP/src/:/shared/src/
+ENV PYTHONPATH=/apps/$APP/src/:/shared/src/:$PYTHONPATH
 WORKDIR /opt/dagster/home
 
 # Run dagster gRPC server on port 4000

Dockerfile.system

@@ -33,4 +33,6 @@ EOF
 RUN chmod +x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]
 
+ENV PYTHONPATH=/system:$PYTHONPATH
+
 WORKDIR $DAGSTER_HOME

apps/stocks/src/assets.py

@@ -105,3 +105,15 @@ def raw_html(context: dg.AssetExecutionContext) -> Iterator[dg.Output[str]]:
     partitions_def=daily_partitions_def,
 )
 def daily_html() -> str: ...
+
+
+class MyAssetConfig(dg.Config):
+    image: str = "bla"
+
+
+@asset(metadata={"docker_image": "my-other-image:latest"})
+def my_asset(context: dg.AssetExecutionContext, config: MyAssetConfig) -> None:
+    ic(context.op_config)
+    ic(config)
+    # image = context.op_config.image
+    # context.log.info(f"This asset wants to use Docker image: {image}")

apps/stocks/src/definitions.py

@@ -1,4 +1,5 @@
 import assets
+import jobs
 import sensors
 from dagster_polars import PolarsParquetIOManager
 from icecream import install
@@ -21,5 +22,6 @@ definitions = dg.Definitions(
         "html_io_manager": HtmlIOManager(base_dir=STORAGE_DIR),
         "polars_parquet_io_manager": PolarsParquetIOManager(base_dir=STORAGE_DIR),
     },
+    jobs=[jobs.test_job],
     sensors=[sensors.check_update],
 )

apps/stocks/src/jobs.py

@@ -3,3 +3,9 @@ import assets
 import dagster as dg
 
 raw_html_job = dg.define_asset_job("deals_job", selection=[assets.raw_html.key])
+
+test_job = dg.define_asset_job(
+    "my_asset_job",
+    selection=[assets.my_asset.key],
+    tags={"docker/image": "my.registry/image:from-config"},
+)

compose.code.yaml

@@ -59,6 +59,15 @@ services:
     networks:
       - dagster
 
+  dagster-code-stocks-playwright:
+    build:
+      context: apps/stocks
+      dockerfile: ../../Dockerfile.code.playwright
+      args:
+        - APP=stocks
+    image: dagster-code-stocks-playwright
+    profiles: [ "never" ]
+
   dagster-code-tesla:
     build:
       context: apps/tesla

compose.system.yaml

@@ -25,6 +25,7 @@ x-volumes: &volumes
   volumes:
     - /opt/dagster/dagster.yaml:/opt/dagster/home/dagster.yaml.template:ro
     - /opt/dagster/workspace.yaml:/opt/dagster/home/workspace.yaml:ro
+    - /opt/dagster/system/:/system/:ro
     - /opt/dagster/storage/:/storage/:rw
     - /opt/dagster/logs/:/logs:rw
     - /var/run/docker.sock:/var/run/docker.sock:rw

dagster.yaml

@@ -11,8 +11,8 @@ run_coordinator:
   class: QueuedRunCoordinator
 
 run_launcher:
-  module: dagster_docker
+  module: run_launcher
-  class: DockerRunLauncher
+  class: CustomDockerRunLauncher
   config:
     env_vars:
       - DAGSTER_POSTGRES_HOST