tidy up

apps/other/src/__init__.py

@@ -1,3 +0,0 @@
-from icecream import install
-
-install()

apps/other/src/definitions.py

@@ -1,22 +1,25 @@
 import assets
 from dagster_polars import PolarsParquetIOManager
+from icecream import install
 
-from dagster import Definitions, load_assets_from_modules
+from dagster import Definitions, define_asset_job, load_assets_from_modules
+
+install()
 
 # Define a job that includes both assets
-# daily_job = define_asset_job("daily_job", selection=[dummy_asset, asset_multi_1, asset_multi_2])
+daily_job = define_asset_job(
+    "daily_job",
+    selection=[assets.dummy_asset, assets.asset_multi_1, assets.asset_multi_2],
+)
 
 definitions = Definitions(
-    assets=load_assets_from_modules([assets]),
-    # [dummy_asset],  # , asset_single_1, asset_multi_1, asset_single_2, asset_multi_2],
+    assets=[
+        asset.with_attributes(
+            group_names_by_key={asset.key: "other"},
+            tags_by_key={asset.key: {"app": "other"}},
+        )
+        for asset in load_assets_from_modules([assets])
+    ],
     resources={"polars_parquet_io_manager": PolarsParquetIOManager()},
-    # jobs=[daily_job],
-    # executor=docker_executor.configured({"container_kwargs": {
-    #     "volumes": [
-    #         "/opt/dagster/apps/other/src/:/opt/dagster/home/app/",
-    #         "/opt/dagster/storage/:/storage/"
-    #     ]
-    # }
-    # }
-    # )
+    jobs=[daily_job],
 )

apps/vinyl/src/__init__.py

@@ -1,3 +0,0 @@
-from icecream import install
-
-install()

apps/vinyl/src/assets.py

@@ -55,11 +55,6 @@ partition_mapping = MultiPartitionMapping(
         "partition_by": ["date", "source"],
     },
     config_schema={"import_dir": Field(str, default_value="/storage/import")},
-    # tags={
-    #     "dagster/executor": "vinyl_executor",
-    #     "app": "vinyl"
-    # },
-    # group_name="vinylllll"
 )
 def deals(context):
     ic()

apps/vinyl/src/definitions.py

@@ -1,15 +1,16 @@
 from collections.abc import Sequence
 
-from assets import deals, new_deals, works
+import assets
 from dagster_duckdb import DuckDBIOManager
 from dagster_duckdb.io_manager import DbTypeHandler
 from dagster_duckdb_pandas import DuckDBPandasTypeHandler
 from dagster_polars import PolarsParquetIOManager
+from icecream import install
 from jobs import check_partititions_job, deals_job, musicbrainz_lookup_job
 from schedules import deals_schedule
 from sensors import musicbrainz_lookup_sensor
 
-from dagster import Definitions
+from dagster import Definitions, load_assets_from_modules
 
 
 class PandasDuckDBIOManager(DuckDBIOManager):
@@ -18,25 +19,15 @@ class PandasDuckDBIOManager(DuckDBIOManager):
         return [DuckDBPandasTypeHandler()]
 
 
-deals.with_attributes()
-assets = []
-for asset in [deals, new_deals, works]:
-    print(asset.tags_by_key)
-    # for k, v in {
-    #     "dagster/executor": "vinyl_executor",
-    #     "app": "vinyl"
-    # }.items():
-    #     pass
-    # asset._specs_by_key[asset.key][k] = v
-    assets.append(
-        asset.with_attributes(
-            tags_by_key={
-                asset.key: {"dagster/executor": "vinyl_executor", "app": "vinyl"}
-            }
-        )
-    )
+install()
 definitions = Definitions(
-    assets=assets,
+    assets=[
+        asset.with_attributes(
+            group_names_by_key={asset.key: "vinyl"},
+            tags_by_key={asset.key: {"app": "vinyl"}},
+        )
+        for asset in load_assets_from_modules([assets])
+    ],
     resources={
         "polars_parquet_io_manager": PolarsParquetIOManager(),
         "duckdb_io_manager": PandasDuckDBIOManager(database="vinyl"),
@@ -45,3 +36,5 @@ definitions = Definitions(
     schedules=[deals_schedule],
     sensors=[musicbrainz_lookup_sensor],
 )
+
+ic("jo")

apps/vinyl/src/test.py

@@ -28,7 +28,7 @@ def test_deals(source="sounds", date: str = None):
         resources=resources,
         run_config={
             "loggers": {"console": {"config": {"log_level": "ERROR"}}},
-            "ops": {"deals": {"config": {"import_dir": "/opt/dagster/storage/import"}}},
+            "ops": {"deals": {"config": {"import_dir": "/storage/import"}}},
         },
     )
     assert result.success