fix works asset
This commit is contained in:
@@ -127,17 +127,18 @@ def cleaned_deals(
|
||||
deps=[cleaned_deals],
|
||||
io_manager_key="polars_parquet_io_manager",
|
||||
automation_condition=dg.AutomationCondition.eager(),
|
||||
output_required=False,
|
||||
)
|
||||
def works(context: dg.AssetExecutionContext) -> pl.DataFrame | None:
|
||||
def works(context: dg.AssetExecutionContext) -> Iterator[dg.Output[pl.DataFrame]]:
|
||||
"""Aggregate works from cleaned deals."""
|
||||
partitions = context.instance.get_materialized_partitions(cleaned_deals.key)
|
||||
ic(partitions)
|
||||
logger.info("Works", partitions=partitions)
|
||||
dfs = list(load_partitions(context, cleaned_deals.key, partitions))
|
||||
ic(len(dfs))
|
||||
if dfs:
|
||||
columns = ["artist", "title", "release"]
|
||||
return pl.concat(dfs, how="vertical_relaxed").select(columns).unique()
|
||||
return None
|
||||
yield dg.Output(pl.concat(dfs, how="vertical_relaxed").select(columns).unique())
|
||||
|
||||
|
||||
@asset(
|
||||
|
||||
Reference in New Issue
Block a user