put vinyl assets under key
This commit is contained in:
@@ -1,10 +1,12 @@
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
from types import SimpleNamespace
|
||||
|
||||
import polars as pl
|
||||
import structlog
|
||||
from config import APP
|
||||
from dagster_polars.patito import patito_model_to_dagster_type
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
from models import Deal
|
||||
@@ -18,10 +20,11 @@ from utils.email import EmailService
|
||||
|
||||
import dagster as dg
|
||||
|
||||
asset = partial(dg.asset, key_prefix=APP)
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
@dg.asset(
|
||||
@asset(
|
||||
io_manager_key="polars_parquet_io_manager",
|
||||
partitions_def=multi_partitions_def,
|
||||
config_schema={"import_dir": dg.Field(str, default_value="/storage/import")},
|
||||
@@ -77,7 +80,7 @@ def deals(context: dg.AssetExecutionContext) -> pl.DataFrame:
|
||||
)
|
||||
|
||||
|
||||
@dg.asset(
|
||||
@asset(
|
||||
io_manager_key="polars_parquet_io_manager",
|
||||
partitions_def=deals.partitions_def,
|
||||
ins={"df": dg.AssetIn(key=deals.key)},
|
||||
@@ -116,7 +119,7 @@ def cleaned_deals(
|
||||
)
|
||||
|
||||
|
||||
@dg.asset(
|
||||
@asset(
|
||||
deps=[cleaned_deals],
|
||||
io_manager_key="polars_parquet_io_manager",
|
||||
automation_condition=dg.AutomationCondition.on_missing().without(
|
||||
@@ -135,7 +138,7 @@ def works(context: dg.AssetExecutionContext) -> pl.DataFrame | None:
|
||||
return None
|
||||
|
||||
|
||||
@dg.asset(
|
||||
@asset(
|
||||
io_manager_key="polars_parquet_io_manager",
|
||||
partitions_def=multi_partitions_def,
|
||||
ins={
|
||||
@@ -202,7 +205,7 @@ def new_deals(
|
||||
context.log.info("No new deals found!")
|
||||
|
||||
|
||||
@dg.asset(
|
||||
@asset(
|
||||
io_manager_key="polars_parquet_io_manager",
|
||||
partitions_def=daily_partitions_def,
|
||||
metadata={
|
||||
|
||||
Reference in New Issue
Block a user