refactor to allow for multiple code locations

This commit is contained in:
2025-07-20 19:49:30 +02:00
parent 9b8cfabee5
commit fd73e1367c
40 changed files with 161 additions and 628 deletions

View File

@@ -0,0 +1,30 @@
from collections.abc import Sequence
from assets import deals, new_deals, works
from dagster_duckdb import DuckDBIOManager
from dagster_duckdb.io_manager import DbTypeHandler
from dagster_duckdb_pandas import DuckDBPandasTypeHandler
from dagster_polars import PolarsParquetIOManager
from jobs import check_partititions_job, deals_job, musicbrainz_lookup_job
from schedules import deals_schedule
from sensors import musicbrainz_lookup_sensor
from dagster import Definitions
class PandasDuckDBIOManager(DuckDBIOManager):
@staticmethod
def type_handlers() -> Sequence[DbTypeHandler]:
return [DuckDBPandasTypeHandler()]
definitions = Definitions(
assets=[deals, new_deals, works],
resources={
"polars_parquet_io_manager": PolarsParquetIOManager(),
"duckdb_io_manager": PandasDuckDBIOManager(database="vinyl"),
},
jobs=[deals_job, check_partititions_job, musicbrainz_lookup_job],
schedules=[deals_schedule],
sensors=[musicbrainz_lookup_sensor],
)