scaffold other code locations

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
     rev: 25.1.0
     hooks:
       - id: black-jupyter
-        language_version: python3.11
+        language_version: python3.12
   - repo: https://github.com/pycqa/isort
     rev: 6.0.1
     hooks:
@@ -26,7 +26,7 @@ repos:
     rev: v0.11.13
     hooks:
       - id: ruff
-        args: [ "--target-version=py311" ]
+        args: [ "--target-version=py312" ]
   - repo: https://github.com/kynan/nbstripout
     rev: 0.8.1
     hooks:

Dockerfile.code

@@ -10,6 +10,7 @@ RUN uv pip install -r requirements.txt --system
 RUN uv pip install polars-lts-cpu --system
 
 ARG APP
+ENV APP=$APP
 ENV PYTHONPATH=/apps/$APP/src/:/shared/src/
 WORKDIR /opt/dagster/home
 

apps/stocks/src/definitions.py

@@ -0,0 +1,27 @@
+import os
+
+import assets
+from dagster_polars import PolarsParquetIOManager
+from icecream import install
+
+import dagster as dg
+from dagster import load_assets_from_modules
+
+install()
+
+APP = os.environ["APP"]
+
+definitions = dg.Definitions(
+    assets=[
+        asset.with_attributes(
+            group_names_by_key={asset.key: APP},
+            tags_by_key={asset.key: {"app": APP}},
+        )
+        for asset in load_assets_from_modules([assets])
+    ],
+    resources={
+        "polars_parquet_io_manager": PolarsParquetIOManager(
+            base_dir=os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}"
+        ),
+    },
+)

apps/tesla/src/definitions.py

@@ -0,0 +1,27 @@
+import os
+
+import assets
+from dagster_polars import PolarsParquetIOManager
+from icecream import install
+
+import dagster as dg
+from dagster import load_assets_from_modules
+
+install()
+
+APP = os.environ["APP"]
+
+definitions = dg.Definitions(
+    assets=[
+        asset.with_attributes(
+            group_names_by_key={asset.key: APP},
+            tags_by_key={asset.key: {"app": APP}},
+        )
+        for asset in load_assets_from_modules([assets])
+    ],
+    resources={
+        "polars_parquet_io_manager": PolarsParquetIOManager(
+            base_dir=os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}"
+        ),
+    },
+)

apps/vinyl/src/definitions.py

@@ -9,18 +9,20 @@ from utils.email import EmailService
 
 import dagster as dg
 
+APP = os.environ["APP"]
+
 install()
 definitions = dg.Definitions(
     assets=[
         asset.with_attributes(
-            group_names_by_key={asset.key: "vinyl"},
+            group_names_by_key={asset.key: APP},
-            tags_by_key={asset.key: {"app": "vinyl"}},
+            tags_by_key={asset.key: {"app": APP}},
         )
         for asset in dg.load_assets_from_modules([assets])
     ],
     resources={
         "polars_parquet_io_manager": PolarsParquetIOManager(
-            base_dir=os.environ.get("STORAGE_DIR", "/storage") + "/vinyl"
+            base_dir=os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}"
         ),
         "email_service": EmailService(
             smtp_server=dg.EnvVar("SMTP_SERVER"),

apps/weather/src/definitions.py

@@ -0,0 +1,27 @@
+import os
+
+import assets
+from dagster_polars import PolarsParquetIOManager
+from icecream import install
+
+import dagster as dg
+from dagster import load_assets_from_modules
+
+install()
+
+APP = os.environ["APP"]
+
+definitions = dg.Definitions(
+    assets=[
+        asset.with_attributes(
+            group_names_by_key={asset.key: APP},
+            tags_by_key={asset.key: {"app": APP}},
+        )
+        for asset in load_assets_from_modules([assets])
+    ],
+    resources={
+        "polars_parquet_io_manager": PolarsParquetIOManager(
+            base_dir=os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}"
+        ),
+    },
+)

compose.code.yaml

@@ -19,18 +19,18 @@ services:
   # run launcher to use this same image when launching runs in a new container as well.
   # Multiple containers like this can be deployed separately - each just needs to run on
   # its own port, and have its own entry in the workspace.yaml file that's loaded by dagit.
-  user_code_vinyl:
+  dagster-code-vinyl:
     build:
       context: apps/vinyl
       dockerfile: ../../Dockerfile.code
       args:
         - APP=vinyl
-    container_name: user_code_vinyl
+    container_name: dagster-code-vinyl
-    image: user_code_vinyl
+    image: dagster-code-vinyl
     restart: always
     environment:
       <<: [ *dagster_env, *email_env ]
-      DAGSTER_CURRENT_IMAGE: user_code_vinyl
+      DAGSTER_CURRENT_IMAGE: dagster-code-vinyl
     volumes:
       - /opt/dagster/apps/:/apps/:ro
       - /opt/dagster/shared/:/shared/:ro
@@ -40,21 +40,78 @@ services:
     networks:
       - dagster
 
-  user_code_other:
+  dagster-code-stocks:
-    # profiles: [ disabled ]
+    build:
+      context: apps/stocks
+      dockerfile: ../../Dockerfile.code
+      args:
+        - APP=stocks
+    container_name: dagster-code-stocks
+    image: dagster-code-stocks
+    restart: always
+    environment:
+      <<: [ *dagster_env ]
+      DAGSTER_CURRENT_IMAGE: dagster-code-stocks
+    volumes:
+      - /opt/dagster/apps/:/apps/:ro
+      - /opt/dagster/shared/:/shared/:ro
+      - /opt/dagster/logs/:/logs:rw
+    networks:
+      - dagster
+
+  dagster-code-tesla:
+    build:
+      context: apps/tesla
+      dockerfile: ../../Dockerfile.code
+      args:
+        - APP=tesla
+    container_name: dagster-code-tesla
+    image: dagster-code-tesla
+    restart: always
+    environment:
+      <<: [ *dagster_env ]
+      DAGSTER_CURRENT_IMAGE: dagster-code-tesla
+    volumes:
+      - /opt/dagster/apps/:/apps/:ro
+      - /opt/dagster/shared/:/shared/:ro
+      - /opt/dagster/logs/:/logs:rw
+    networks:
+      - dagster
+
+  dagster-code-weather:
+    build:
+      context: apps/weather
+      dockerfile: ../../Dockerfile.code
+      args:
+        - APP=weather
+    container_name: dagster-code-weather
+    image: dagster-code-weather
+    restart: always
+    environment:
+      <<: [ *dagster_env ]
+      DAGSTER_CURRENT_IMAGE: dagster-code-weather
+    volumes:
+      - /opt/dagster/apps/:/apps/:ro
+      - /opt/dagster/shared/:/shared/:ro
+      - /opt/dagster/logs/:/logs:rw
+    networks:
+      - dagster
+
+  dagster-code-other:
     build:
       context: apps/other
       dockerfile: ../../Dockerfile.code
       args:
         - APP=other
-    container_name: user_code_other
+    container_name: dagster-code-other
-    image: user_code_other
+    image: dagster-code-other
     restart: always
     environment:
       <<: [ *dagster_env ]
-      DAGSTER_CURRENT_IMAGE: user_code_other
+      DAGSTER_CURRENT_IMAGE: dagster-code-other
     volumes:
-      - /opt/dagster/apps/:/apps:ro
+      - /opt/dagster/apps/:/apps/:ro
+      - /opt/dagster/shared/:/shared/:ro
       - /opt/dagster/logs/:/logs:rw
     networks:
       - dagster

compose.system.yaml

@@ -23,9 +23,9 @@ x-volumes: &volumes
 services:
   # This service runs the postgres DB used by dagster for run storage, schedule storage,
   # and event log storage.
-  postgresql:
+  dagster-postgres:
     image: postgres:11
-    container_name: postgresql
+    container_name: dagster-postgres
     environment:
       <<: *postgres_env
     networks:
@@ -36,7 +36,7 @@ services:
   # This service runs dagit, which loads your user code from the user code container.
   # Since our instance uses the QueuedRunCoordinator, any runs submitted from dagit will be put on
   # a queue and later dequeued and launched by dagster-daemon.
-  dagit:
+  dagster-dagit:
     build:
       context: .
       dockerfile: Dockerfile.system
@@ -48,7 +48,7 @@ services:
       - "3000"
       - -w
       - workspace.yaml
-    container_name: dagit
+    container_name: dagster-dagit
     expose:
       - "3000"
     ports:
@@ -59,18 +59,18 @@ services:
     networks:
       - dagster
     depends_on:
-      - postgresql
+      - dagster-postgres
 
   # This service runs the dagster-daemon process, which is responsible for taking runs
   # off of the queue and launching them, as well as creating runs from schedules or sensors.
-  daemon:
+  dagster-daemon:
     build:
       context: .
       dockerfile: Dockerfile.system
     entrypoint:
       - dagster-daemon
       - run
-    container_name: daemon
+    container_name: dagster-daemon
     restart: on-failure
     environment:
       <<: *dagster_env
@@ -78,4 +78,4 @@ services:
     networks:
       - dagster
     depends_on:
-      - postgresql
+      - dagster-postgres

pyproject.toml

@@ -54,6 +54,11 @@ dagster = [
     "dagit"
 ]
 vinyl = []
+stocks = [
+    "selenium"
+]
+tesla = []
+weather = []
 other = []
 unknown = [
     "fastapi",

requirements.sh

@@ -2,4 +2,7 @@ uv pip compile pyproject.toml > requirements.txt
 uv pip compile pyproject.toml --extra=dagster > dagster-requirements.txt
 uv pip compile pyproject.toml --extra=dagster --extra=dev > dev-requirements.txt
 uv pip compile pyproject.toml --extra=dagster --extra=vinyl > apps/vinyl/requirements.txt
+uv pip compile pyproject.toml --extra=dagster --extra=stocks > apps/stocks/requirements.txt
+uv pip compile pyproject.toml --extra=dagster --extra=tesla > apps/tesla/requirements.txt
+uv pip compile pyproject.toml --extra=dagster --extra=weather > apps/weather/requirements.txt
 uv pip compile pyproject.toml --extra=dagster --extra=other > apps/other/requirements.txt

workspace.yaml

@@ -1,7 +1,16 @@
 load_from:
   - grpc_server:
-      host: user_code_vinyl
+      host: dagster-code-vinyl
       port: 4000
   - grpc_server:
-      host: user_code_other
+      host: dagster-code-stocks
+      port: 4000
+  - grpc_server:
+      host: dagster-code-tesla
+      port: 4000
+  - grpc_server:
+      host: dagster-code-weather
+      port: 4000
+  - grpc_server:
+      host: dagster-code-other
       port: 4000