This commit is contained in:
2024-10-14 12:49:32 +02:00
parent a7d8dfdbd3
commit ebef914be6
11 changed files with 138 additions and 142 deletions

View File

@@ -1,9 +1,11 @@
from dagster import job, OpExecutionContext, op, \
AssetMaterialization, AssetKey, define_asset_job
from dagster import (AssetKey, AssetMaterialization, OpExecutionContext,
define_asset_job, job, op)
from .assets import deals
deals_job = define_asset_job("deals_job", selection=[deals], partitions_def=deals.partitions_def)
deals_job = define_asset_job(
"deals_job", selection=[deals], partitions_def=deals.partitions_def
)
@op
@@ -22,11 +24,17 @@ def check_partititions(context: OpExecutionContext):
context.log.info("Existing partitions", extra=dict(partitions=materializations))
import polars as pl
storage_dir = context.instance.storage_directory()
ic(storage_dir)
for row in pl.scan_parquet(f'{storage_dir}/{asset_key}/*/*.parquet').select(
['date', 'source']).unique().collect().iter_rows():
partition = '|'.join(row)
for row in (
pl.scan_parquet(f"{storage_dir}/{asset_key}/*/*.parquet")
.select(["date", "source"])
.unique()
.collect()
.iter_rows()
):
partition = "|".join(row)
if partition not in materializations:
context.log.info(f"Missing partition: {partition}")
context.log_event(