linting
This commit is contained in:
@@ -1,9 +1,11 @@
|
||||
from dagster import job, OpExecutionContext, op, \
|
||||
AssetMaterialization, AssetKey, define_asset_job
|
||||
from dagster import (AssetKey, AssetMaterialization, OpExecutionContext,
|
||||
define_asset_job, job, op)
|
||||
|
||||
from .assets import deals
|
||||
|
||||
deals_job = define_asset_job("deals_job", selection=[deals], partitions_def=deals.partitions_def)
|
||||
deals_job = define_asset_job(
|
||||
"deals_job", selection=[deals], partitions_def=deals.partitions_def
|
||||
)
|
||||
|
||||
|
||||
@op
|
||||
@@ -22,11 +24,17 @@ def check_partititions(context: OpExecutionContext):
|
||||
context.log.info("Existing partitions", extra=dict(partitions=materializations))
|
||||
|
||||
import polars as pl
|
||||
|
||||
storage_dir = context.instance.storage_directory()
|
||||
ic(storage_dir)
|
||||
for row in pl.scan_parquet(f'{storage_dir}/{asset_key}/*/*.parquet').select(
|
||||
['date', 'source']).unique().collect().iter_rows():
|
||||
partition = '|'.join(row)
|
||||
for row in (
|
||||
pl.scan_parquet(f"{storage_dir}/{asset_key}/*/*.parquet")
|
||||
.select(["date", "source"])
|
||||
.unique()
|
||||
.collect()
|
||||
.iter_rows()
|
||||
):
|
||||
partition = "|".join(row)
|
||||
if partition not in materializations:
|
||||
context.log.info(f"Missing partition: {partition}")
|
||||
context.log_event(
|
||||
|
||||
Reference in New Issue
Block a user