rik/dagster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

polars io manager ignores partition_by metadata

Browse Source
This commit is contained in:
Rik Veenboer
2025-07-27 17:51:52 +02:00
parent f3c2f9ec11
commit 662bc239f9
1 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
apps/other/src/assets.py
View File

@@ -1,5 +1,6 @@
import polars as pl import polars as pl
import dagster as dg
from dagster import ( from dagster import (
AssetIn, AssetIn,
DailyPartitionsDefinition, DailyPartitionsDefinition,
@@ -173,3 +174,11 @@ def asset_multi_3(context, asset_multi_1):
return pl.scan_parquet( return pl.scan_parquet(
f"{storage_dir}/{asset_key}/*/*.parquet", extra_columns="ignore" f"{storage_dir}/{asset_key}/*/*.parquet", extra_columns="ignore"
).filter(pl.col("date").is_in(keep_dates)) ).filter(pl.col("date").is_in(keep_dates))
@asset(io_manager_key="polars_parquet_io_manager", metadata={"partition_by": ["b"]})
def dummy_asset():
yield dg.Output(
pl.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}),
metadata={"partition_by": ["c"]},
)