From cb9a4026109bb4f9f586cf66c9ad3be459a7cf91 Mon Sep 17 00:00:00 2001 From: Rik Veenboer Date: Sat, 26 Jul 2025 09:25:10 +0200 Subject: [PATCH] vinyl repo is not yet using multi partition mapping --- apps/vinyl/src/assets.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/apps/vinyl/src/assets.py b/apps/vinyl/src/assets.py index 9f5b1fb..570a840 100644 --- a/apps/vinyl/src/assets.py +++ b/apps/vinyl/src/assets.py @@ -11,15 +11,11 @@ from utils import parse_date from dagster import ( DailyPartitionsDefinition, - DimensionPartitionMapping, Failure, Field, - IdentityPartitionMapping, - MultiPartitionMapping, MultiPartitionsDefinition, OpExecutionContext, StaticPartitionsDefinition, - TimeWindowPartitionMapping, asset, ) @@ -34,19 +30,6 @@ partitions_def = MultiPartitionsDefinition( } ) -partition_mapping = MultiPartitionMapping( - { - "date": DimensionPartitionMapping( - dimension_name="date", - partition_mapping=TimeWindowPartitionMapping(start_offset=-1, end_offset=0), - ), - "source": DimensionPartitionMapping( - dimension_name="source", - partition_mapping=IdentityPartitionMapping(), - ), - } -) - @asset( io_manager_key="polars_parquet_io_manager", @@ -114,6 +97,8 @@ def new_deals(context: OpExecutionContext) -> pl.DataFrame: storage_dir = context.resources.polars_parquet_io_manager.base_dir asset_key = "deals" + # TODO: can we directly query from the deals input? + with duckdb.connect() as con: con.create_function("PARSE_DATE", parse_date, [VARCHAR], DATE) return con.execute(