add existing partitions

apps/vinyl/src/jobs.py

@@ -1,3 +1,4 @@
+import polars as pl
 from assets import deals, new_deals, works
 
 from dagster import (
@@ -14,27 +15,22 @@ deals_job = define_asset_job(
 )
 
 
-@op
-def check_partititions(context: OpExecutionContext):
-    # Replace with your asset/job name
+@op(required_resource_keys={"polars_parquet_io_manager"})
+def check_partitions(context: OpExecutionContext):
     asset_key = "deals"
 
-    context.log_event(
-        AssetMaterialization(asset_key=asset_key, partition="2024-09-30|sounds")
-    )
-
     # Fetch the materializations for the asset key
     materializations = context.instance.get_materialized_partitions(
         asset_key=AssetKey(asset_key)
     )
-    context.log.info("Existing partitions", extra=dict(partitions=materializations))
+    ic(materializations)
 
-    import polars as pl
-
-    storage_dir = context.instance.storage_directory()
+    storage_dir = context.resources.polars_parquet_io_manager.base_dir
     ic(storage_dir)
     for row in (
-        pl.scan_parquet(f"{storage_dir}/{asset_key}/*/*.parquet")
+        pl.scan_parquet(
+            f"{storage_dir}/{asset_key}/*/*.parquet", extra_columns="ignore"
+        )
         .select(["date", "source"])
         .unique()
         .collect()
@@ -49,8 +45,8 @@ def check_partititions(context: OpExecutionContext):
 
 
 @job
-def check_partititions_job():
-    check_partititions()
+def check_partitions_job():
+    check_partitions()
 
 
 musicbrainz_lookup_job = define_asset_job(