warn for missing partitions

This commit is contained in:
2025-10-29 10:15:10 +01:00
parent 2a4da9abb9
commit e9ad1677ef

View File

@@ -232,7 +232,9 @@ def new_deals(
},
ins={"partitions": dg.AssetIn(key=new_deals.key)},
output_required=False,
automation_condition=dg.AutomationCondition.eager().without(~dg.AutomationCondition.any_deps_missing())
automation_condition=dg.AutomationCondition.eager().without(
~dg.AutomationCondition.any_deps_missing()
),
)
def good_deals(
context: dg.AssetExecutionContext,
@@ -242,6 +244,9 @@ def good_deals(
parsed_partition_keys = parse_partition_keys(context, "partitions")
ic(parsed_partition_keys)
if not partitions:
logger.warning("Partitions are empty!")
return
df = pl.concat(partitions.values(), how="vertical_relaxed").collect()
counts = dict(df.group_by("source").len().iter_rows())