mount all apps in container

.pre-commit-config.yaml

@@ -10,7 +10,6 @@ repos:
         exclude: .*template.*
       - id: debug-statements
       - id: detect-aws-credentials
-        args: [ "--allow-missing-credentials" ]
       - id: detect-private-key
       - id: end-of-file-fixer
       - id: requirements-txt-fixer

Dockerfile.code

@@ -9,7 +9,8 @@ RUN pip install uv
 RUN uv pip install -r requirements.txt --system
 RUN uv pip install polars-lts-cpu --system
 
-# Add repository code
+ARG APP
+ENV PYTHONPATH=/apps/$APP/src/
 WORKDIR /opt/dagster/home
 
 # Run dagster gRPC server on port 4000
@@ -17,4 +18,4 @@ EXPOSE 4000
 
 # CMD allows this to be overridden from run launchers or executors that want
 # to run other commands against your repository
-CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-m", "app.definitions"]
+CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-m", "definitions"]

apps/other/src/assets.py

@@ -1,3 +1,4 @@
+import pandas as pd
 import polars as pl
 
 from dagster import (
@@ -22,6 +23,16 @@ partitions_def_multi = MultiPartitionsDefinition(
 )
 
 
+@asset(
+    # tags={
+    #     "dagster/executor": "other_executor"
+    # },
+)
+def dummy_asset():
+    """A dummy asset to ensure the module is recognized by Dagster."""
+    return pd.DataFrame({"dummy": [1, 2, 3]})
+
+
 @asset(
     io_manager_key="polars_parquet_io_manager",
     partitions_def=partitions_def_single,

apps/other/src/definitions.py

@@ -1,13 +1,22 @@
-from assets import asset_multi_1, asset_multi_2, asset_single_1, asset_single_2
+import assets
 from dagster_polars import PolarsParquetIOManager
 
-from dagster import Definitions, define_asset_job
+from dagster import Definitions, load_assets_from_modules
 
 # Define a job that includes both assets
-daily_job = define_asset_job("daily_job", selection=[asset_multi_1, asset_multi_2])
+# daily_job = define_asset_job("daily_job", selection=[dummy_asset, asset_multi_1, asset_multi_2])
 
 definitions = Definitions(
-    assets=[asset_single_1, asset_multi_1, asset_single_2, asset_multi_2],
+    assets=load_assets_from_modules([assets]),
+    # [dummy_asset],  # , asset_single_1, asset_multi_1, asset_single_2, asset_multi_2],
     resources={"polars_parquet_io_manager": PolarsParquetIOManager()},
-    jobs=[daily_job],
+    # jobs=[daily_job],
+    # executor=docker_executor.configured({"container_kwargs": {
+    #     "volumes": [
+    #         "/opt/dagster/apps/other/src/:/opt/dagster/home/app/",
+    #         "/opt/dagster/storage/:/storage/"
+    #     ]
+    # }
+    # }
+    # )
 )

apps/vinyl/src/assets.py

@@ -54,9 +54,12 @@ partition_mapping = MultiPartitionMapping(
     metadata={
         "partition_by": ["date", "source"],
     },
-    config_schema={
+    config_schema={"import_dir": Field(str, default_value="/storage/import")},
-        "import_dir": Field(str, default_value="/opt/dagster/home/storage/import")
+    # tags={
-    },
+    #     "dagster/executor": "vinyl_executor",
+    #     "app": "vinyl"
+    # },
+    # group_name="vinylllll"
 )
 def deals(context):
     ic()

apps/vinyl/src/definitions.py

@@ -18,8 +18,25 @@ class PandasDuckDBIOManager(DuckDBIOManager):
         return [DuckDBPandasTypeHandler()]
 
 
+deals.with_attributes()
+assets = []
+for asset in [deals, new_deals, works]:
+    print(asset.tags_by_key)
+    # for k, v in {
+    #     "dagster/executor": "vinyl_executor",
+    #     "app": "vinyl"
+    # }.items():
+    #     pass
+    # asset._specs_by_key[asset.key][k] = v
+    assets.append(
+        asset.with_attributes(
+            tags_by_key={
+                asset.key: {"dagster/executor": "vinyl_executor", "app": "vinyl"}
+            }
+        )
+    )
 definitions = Definitions(
-    assets=[deals, new_deals, works],
+    assets=assets,
     resources={
         "polars_parquet_io_manager": PolarsParquetIOManager(),
         "duckdb_io_manager": PandasDuckDBIOManager(database="vinyl"),

compose.code.yaml

@@ -15,17 +15,19 @@ services:
     build:
       context: apps/vinyl
       dockerfile: ../../Dockerfile.code
+      args:
+        - APP=vinyl
     container_name: user_code_vinyl
     image: user_code_vinyl
     restart: always
     environment:
       <<: *dagster_env
       DAGSTER_CURRENT_IMAGE: user_code_vinyl
-      PYTHONPATH: app
     volumes:
-      - /opt/dagster/storage/import/:/opt/dagster/home/storage/import/
+      - /opt/dagster/apps/:/apps/:ro
-      - /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/
+      - /opt/dagster/storage/import/:/storage/import/:ro
-      - /opt/dagster/apps/vinyl/src/:/opt/dagster/home/app/
+      - /opt/dagster/storage/deals/:/storage/deals/:rw
+    #      - /opt/dagster/apps/vinyl/src/:/opt/dagster/home/app/
     networks:
       - dagster
 
@@ -34,14 +36,16 @@ services:
     build:
       context: apps/other
       dockerfile: ../../Dockerfile.code
+      args:
+        - APP=other
     container_name: user_code_other
     image: user_code_other
     restart: always
     environment:
       <<: *dagster_env
       DAGSTER_CURRENT_IMAGE: user_code_other
-      PYTHONPATH: app
     volumes:
-      - /opt/dagster/apps/other/src/:/opt/dagster/home/app/
+      #      - /opt/dagster/apps/other/src/:/opt/dagster/home/app/
+      - /opt/dagster/apps/:/apps:ro
     networks:
       - dagster

compose.system.yaml

@@ -13,7 +13,6 @@ x-dagster-env: &dagster_env
   DAGSTER_POSTGRES_USER: ${POSTGRES_USER}
   DAGSTER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
   DAGSTER_POSTGRES_DB: ${POSTGRES_DB}
-  DAGSTER_CURRENT_IMAGE: ${DAGSTER_CURRENT_IMAGE}
 
 x-volumes: &volumes
   volumes:
@@ -21,8 +20,8 @@ x-volumes: &volumes
     - /opt/dagster/workspace.yaml:/opt/dagster/home/workspace.yaml
     - /var/run/docker.sock:/var/run/docker.sock
 
-    - /opt/dagster/storage/import/:/opt/dagster/home/storage/import/
+    - /opt/dagster/storage/import/:/storage/import/
-    - /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/
+    - /opt/dagster/storage/deals/:/storage/deals/
 
     - /opt/dagster/src/app/:/opt/dagster/home/app/
     - /opt/dagster/src/repo.py:/opt/dagster/home/repo.py