demo of delta lake with forked io manager
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import assets
|
||||
from dagster_delta import DeltaLakePyarrowIOManager, LocalConfig, WriteMode
|
||||
from dagster_polars import PolarsParquetIOManager
|
||||
from icecream import install
|
||||
from shared.config import APP, STORAGE_DIR
|
||||
@@ -11,11 +12,16 @@ definitions = dg.Definitions(
|
||||
assets=[
|
||||
asset.with_attributes(
|
||||
group_names_by_key={asset.key: APP},
|
||||
tags_by_key={asset.key: {"app": APP}},
|
||||
)
|
||||
for asset in dg.load_assets_from_modules([assets])
|
||||
],
|
||||
resources={
|
||||
"polars_parquet_io_manager": PolarsParquetIOManager(base_dir=STORAGE_DIR)
|
||||
"polars_parquet_io_manager": PolarsParquetIOManager(base_dir=STORAGE_DIR),
|
||||
"delta_io_manager": DeltaLakePyarrowIOManager(
|
||||
root_uri=STORAGE_DIR,
|
||||
storage_options=LocalConfig(),
|
||||
mode=WriteMode.overwrite,
|
||||
parquet_read_options={"coerce_int96_timestamp_unit": "us"},
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user