demo of delta lake with forked io manager

This commit is contained in:
2025-07-29 15:37:37 +02:00
parent f59b30b9ea
commit 4c59dcb8ac
6 changed files with 64 additions and 6 deletions

View File

@@ -1,4 +1,5 @@
import assets
from dagster_delta import DeltaLakePyarrowIOManager, LocalConfig, WriteMode
from dagster_polars import PolarsParquetIOManager
from icecream import install
from shared.config import APP, STORAGE_DIR
@@ -11,11 +12,16 @@ definitions = dg.Definitions(
assets=[
asset.with_attributes(
group_names_by_key={asset.key: APP},
tags_by_key={asset.key: {"app": APP}},
)
for asset in dg.load_assets_from_modules([assets])
],
resources={
"polars_parquet_io_manager": PolarsParquetIOManager(base_dir=STORAGE_DIR)
"polars_parquet_io_manager": PolarsParquetIOManager(base_dir=STORAGE_DIR),
"delta_io_manager": DeltaLakePyarrowIOManager(
root_uri=STORAGE_DIR,
storage_options=LocalConfig(),
mode=WriteMode.overwrite,
parquet_read_options={"coerce_int96_timestamp_unit": "us"},
),
},
)