rewrite to dagster as dg

This commit is contained in:
2025-07-26 10:03:39 +02:00
parent ded11d9e60
commit 62c25b32c8
5 changed files with 51 additions and 66 deletions

View File

@@ -10,7 +10,7 @@ from jobs import check_partitions_job, deals_job, musicbrainz_lookup_job
from schedules import deals_schedule
from sensors import musicbrainz_lookup_sensor
from dagster import Definitions, load_assets_from_modules
import dagster as dg
class PandasDuckDBIOManager(DuckDBIOManager):
@@ -20,13 +20,13 @@ class PandasDuckDBIOManager(DuckDBIOManager):
install()
definitions = Definitions(
definitions = dg.Definitions(
assets=[
asset.with_attributes(
group_names_by_key={asset.key: "vinyl"},
tags_by_key={asset.key: {"app": "vinyl"}},
)
for asset in load_assets_from_modules([assets])
for asset in dg.load_assets_from_modules([assets])
],
resources={
"polars_parquet_io_manager": PolarsParquetIOManager(base_dir="/storage"),
@@ -34,5 +34,12 @@ definitions = Definitions(
},
jobs=[deals_job, check_partitions_job, musicbrainz_lookup_job],
schedules=[deals_schedule],
sensors=[musicbrainz_lookup_sensor],
sensors=[
dg.AutomationConditionSensorDefinition(
"run_tags_automation_condition_sensor",
target=dg.AssetSelection.all(),
default_status=dg.DefaultSensorStatus.RUNNING,
),
musicbrainz_lookup_sensor,
],
)