From 119583c73073a2c1bbca28b4a33f6d1146deea62 Mon Sep 17 00:00:00 2001 From: Rik Veenboer Date: Mon, 21 Jul 2025 09:18:16 +0200 Subject: [PATCH] mount all apps in container --- .pre-commit-config.yaml | 1 - Dockerfile.code | 5 +++-- apps/other/src/assets.py | 11 +++++++++++ apps/other/src/definitions.py | 19 ++++++++++++++----- apps/vinyl/src/assets.py | 9 ++++++--- apps/vinyl/src/definitions.py | 19 ++++++++++++++++++- compose.code.yaml | 16 ++++++++++------ compose.system.yaml | 5 ++--- 8 files changed, 64 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1f026f1..395f1eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,6 @@ repos: exclude: .*template.* - id: debug-statements - id: detect-aws-credentials - args: [ "--allow-missing-credentials" ] - id: detect-private-key - id: end-of-file-fixer - id: requirements-txt-fixer diff --git a/Dockerfile.code b/Dockerfile.code index 401b7e8..147fcd0 100644 --- a/Dockerfile.code +++ b/Dockerfile.code @@ -9,7 +9,8 @@ RUN pip install uv RUN uv pip install -r requirements.txt --system RUN uv pip install polars-lts-cpu --system -# Add repository code +ARG APP +ENV PYTHONPATH=/apps/$APP/src/ WORKDIR /opt/dagster/home # Run dagster gRPC server on port 4000 @@ -17,4 +18,4 @@ EXPOSE 4000 # CMD allows this to be overridden from run launchers or executors that want # to run other commands against your repository -CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-m", "app.definitions"] +CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-m", "definitions"] diff --git a/apps/other/src/assets.py b/apps/other/src/assets.py index 41374d4..da895fd 100644 --- a/apps/other/src/assets.py +++ b/apps/other/src/assets.py @@ -1,3 +1,4 @@ +import pandas as pd import polars as pl from dagster import ( @@ -22,6 +23,16 @@ partitions_def_multi = MultiPartitionsDefinition( ) +@asset( + # tags={ + # "dagster/executor": "other_executor" + # }, +) +def dummy_asset(): + """A dummy asset to ensure the module is recognized by Dagster.""" + return pd.DataFrame({"dummy": [1, 2, 3]}) + + @asset( io_manager_key="polars_parquet_io_manager", partitions_def=partitions_def_single, diff --git a/apps/other/src/definitions.py b/apps/other/src/definitions.py index 3360257..1de39ed 100644 --- a/apps/other/src/definitions.py +++ b/apps/other/src/definitions.py @@ -1,13 +1,22 @@ -from assets import asset_multi_1, asset_multi_2, asset_single_1, asset_single_2 +import assets from dagster_polars import PolarsParquetIOManager -from dagster import Definitions, define_asset_job +from dagster import Definitions, load_assets_from_modules # Define a job that includes both assets -daily_job = define_asset_job("daily_job", selection=[asset_multi_1, asset_multi_2]) +# daily_job = define_asset_job("daily_job", selection=[dummy_asset, asset_multi_1, asset_multi_2]) definitions = Definitions( - assets=[asset_single_1, asset_multi_1, asset_single_2, asset_multi_2], + assets=load_assets_from_modules([assets]), + # [dummy_asset], # , asset_single_1, asset_multi_1, asset_single_2, asset_multi_2], resources={"polars_parquet_io_manager": PolarsParquetIOManager()}, - jobs=[daily_job], + # jobs=[daily_job], + # executor=docker_executor.configured({"container_kwargs": { + # "volumes": [ + # "/opt/dagster/apps/other/src/:/opt/dagster/home/app/", + # "/opt/dagster/storage/:/storage/" + # ] + # } + # } + # ) ) diff --git a/apps/vinyl/src/assets.py b/apps/vinyl/src/assets.py index ec027d7..9ccda83 100644 --- a/apps/vinyl/src/assets.py +++ b/apps/vinyl/src/assets.py @@ -54,9 +54,12 @@ partition_mapping = MultiPartitionMapping( metadata={ "partition_by": ["date", "source"], }, - config_schema={ - "import_dir": Field(str, default_value="/opt/dagster/home/storage/import") - }, + config_schema={"import_dir": Field(str, default_value="/storage/import")}, + # tags={ + # "dagster/executor": "vinyl_executor", + # "app": "vinyl" + # }, + # group_name="vinylllll" ) def deals(context): ic() diff --git a/apps/vinyl/src/definitions.py b/apps/vinyl/src/definitions.py index 15d55ad..7655713 100644 --- a/apps/vinyl/src/definitions.py +++ b/apps/vinyl/src/definitions.py @@ -18,8 +18,25 @@ class PandasDuckDBIOManager(DuckDBIOManager): return [DuckDBPandasTypeHandler()] +deals.with_attributes() +assets = [] +for asset in [deals, new_deals, works]: + print(asset.tags_by_key) + # for k, v in { + # "dagster/executor": "vinyl_executor", + # "app": "vinyl" + # }.items(): + # pass + # asset._specs_by_key[asset.key][k] = v + assets.append( + asset.with_attributes( + tags_by_key={ + asset.key: {"dagster/executor": "vinyl_executor", "app": "vinyl"} + } + ) + ) definitions = Definitions( - assets=[deals, new_deals, works], + assets=assets, resources={ "polars_parquet_io_manager": PolarsParquetIOManager(), "duckdb_io_manager": PandasDuckDBIOManager(database="vinyl"), diff --git a/compose.code.yaml b/compose.code.yaml index 6ad527a..f643419 100644 --- a/compose.code.yaml +++ b/compose.code.yaml @@ -15,17 +15,19 @@ services: build: context: apps/vinyl dockerfile: ../../Dockerfile.code + args: + - APP=vinyl container_name: user_code_vinyl image: user_code_vinyl restart: always environment: <<: *dagster_env DAGSTER_CURRENT_IMAGE: user_code_vinyl - PYTHONPATH: app volumes: - - /opt/dagster/storage/import/:/opt/dagster/home/storage/import/ - - /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/ - - /opt/dagster/apps/vinyl/src/:/opt/dagster/home/app/ + - /opt/dagster/apps/:/apps/:ro + - /opt/dagster/storage/import/:/storage/import/:ro + - /opt/dagster/storage/deals/:/storage/deals/:rw + # - /opt/dagster/apps/vinyl/src/:/opt/dagster/home/app/ networks: - dagster @@ -34,14 +36,16 @@ services: build: context: apps/other dockerfile: ../../Dockerfile.code + args: + - APP=other container_name: user_code_other image: user_code_other restart: always environment: <<: *dagster_env DAGSTER_CURRENT_IMAGE: user_code_other - PYTHONPATH: app volumes: - - /opt/dagster/apps/other/src/:/opt/dagster/home/app/ + # - /opt/dagster/apps/other/src/:/opt/dagster/home/app/ + - /opt/dagster/apps/:/apps:ro networks: - dagster diff --git a/compose.system.yaml b/compose.system.yaml index 58ebebc..cf8c514 100644 --- a/compose.system.yaml +++ b/compose.system.yaml @@ -13,7 +13,6 @@ x-dagster-env: &dagster_env DAGSTER_POSTGRES_USER: ${POSTGRES_USER} DAGSTER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} DAGSTER_POSTGRES_DB: ${POSTGRES_DB} - DAGSTER_CURRENT_IMAGE: ${DAGSTER_CURRENT_IMAGE} x-volumes: &volumes volumes: @@ -21,8 +20,8 @@ x-volumes: &volumes - /opt/dagster/workspace.yaml:/opt/dagster/home/workspace.yaml - /var/run/docker.sock:/var/run/docker.sock - - /opt/dagster/storage/import/:/opt/dagster/home/storage/import/ - - /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/ + - /opt/dagster/storage/import/:/storage/import/ + - /opt/dagster/storage/deals/:/storage/deals/ - /opt/dagster/src/app/:/opt/dagster/home/app/ - /opt/dagster/src/repo.py:/opt/dagster/home/repo.py