rewrite to dagster as dg
This commit is contained in:
@@ -1,27 +1,20 @@
|
||||
import polars as pl
|
||||
from assets import deals, new_deals, works
|
||||
|
||||
from dagster import (
|
||||
AssetKey,
|
||||
AssetMaterialization,
|
||||
OpExecutionContext,
|
||||
define_asset_job,
|
||||
job,
|
||||
op,
|
||||
)
|
||||
import dagster as dg
|
||||
|
||||
deals_job = define_asset_job(
|
||||
"deals_job", selection=[deals], partitions_def=deals.partitions_def
|
||||
deals_job = dg.define_asset_job(
|
||||
"deals_job", selection=[deals.key], partitions_def=deals.partitions_def
|
||||
)
|
||||
|
||||
|
||||
@op(required_resource_keys={"polars_parquet_io_manager"})
|
||||
def check_partitions(context: OpExecutionContext):
|
||||
@dg.op(required_resource_keys={"polars_parquet_io_manager"})
|
||||
def check_partitions(context: dg.OpExecutionContext):
|
||||
asset_key = "deals"
|
||||
|
||||
# Fetch the materializations for the asset key
|
||||
materializations = context.instance.get_materialized_partitions(
|
||||
asset_key=AssetKey(asset_key)
|
||||
asset_key=dg.AssetKey(asset_key)
|
||||
)
|
||||
ic(materializations)
|
||||
|
||||
@@ -40,15 +33,15 @@ def check_partitions(context: OpExecutionContext):
|
||||
if partition not in materializations:
|
||||
context.log.info(f"Missing partition: {partition}")
|
||||
context.log_event(
|
||||
AssetMaterialization(asset_key=asset_key, partition=partition)
|
||||
dg.AssetMaterialization(asset_key=asset_key, partition=partition)
|
||||
)
|
||||
|
||||
|
||||
@job
|
||||
@dg.job
|
||||
def check_partitions_job():
|
||||
check_partitions()
|
||||
|
||||
|
||||
musicbrainz_lookup_job = define_asset_job(
|
||||
"musicbrainz_lookup_job", selection=[works, new_deals]
|
||||
musicbrainz_lookup_job = dg.define_asset_job(
|
||||
"musicbrainz_lookup_job", selection=[works.key, new_deals.key]
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user