rewrite to dagster as dg

This commit is contained in:
2025-07-26 10:03:39 +02:00
parent ded11d9e60
commit 62c25b32c8
5 changed files with 51 additions and 66 deletions

View File

@@ -1,27 +1,20 @@
import polars as pl
from assets import deals, new_deals, works
from dagster import (
AssetKey,
AssetMaterialization,
OpExecutionContext,
define_asset_job,
job,
op,
)
import dagster as dg
deals_job = define_asset_job(
"deals_job", selection=[deals], partitions_def=deals.partitions_def
deals_job = dg.define_asset_job(
"deals_job", selection=[deals.key], partitions_def=deals.partitions_def
)
@op(required_resource_keys={"polars_parquet_io_manager"})
def check_partitions(context: OpExecutionContext):
@dg.op(required_resource_keys={"polars_parquet_io_manager"})
def check_partitions(context: dg.OpExecutionContext):
asset_key = "deals"
# Fetch the materializations for the asset key
materializations = context.instance.get_materialized_partitions(
asset_key=AssetKey(asset_key)
asset_key=dg.AssetKey(asset_key)
)
ic(materializations)
@@ -40,15 +33,15 @@ def check_partitions(context: OpExecutionContext):
if partition not in materializations:
context.log.info(f"Missing partition: {partition}")
context.log_event(
AssetMaterialization(asset_key=asset_key, partition=partition)
dg.AssetMaterialization(asset_key=asset_key, partition=partition)
)
@job
@dg.job
def check_partitions_job():
check_partitions()
musicbrainz_lookup_job = define_asset_job(
"musicbrainz_lookup_job", selection=[works, new_deals]
musicbrainz_lookup_job = dg.define_asset_job(
"musicbrainz_lookup_job", selection=[works.key, new_deals.key]
)