fix paths to storage and logs

This commit is contained in:
2025-07-22 12:03:03 +02:00
parent 5b5e962773
commit be8fae969c
8 changed files with 16 additions and 17 deletions

View File

@@ -1,4 +1,3 @@
import pandas as pd
import polars as pl
from dagster import (
@@ -23,16 +22,6 @@ partitions_def_multi = MultiPartitionsDefinition(
)
@asset(
# tags={
# "dagster/executor": "other_executor"
# },
)
def dummy_asset():
"""A dummy asset to ensure the module is recognized by Dagster."""
return pd.DataFrame({"dummy": [1, 2, 3]})
@asset(
io_manager_key="polars_parquet_io_manager",
partitions_def=partitions_def_single,
@@ -58,7 +47,6 @@ def asset_single_1(context):
def asset_multi_1(context):
ic()
ic(context.partition_key)
return pl.DataFrame(
[{"date": context.partition_key, "data": f"Data for {context.partition_key}"}]
)

View File

@@ -9,7 +9,7 @@ install()
# Define a job that includes both assets
daily_job = define_asset_job(
"daily_job",
selection=[assets.dummy_asset, assets.asset_multi_1, assets.asset_multi_2],
selection=[assets.asset_multi_1, assets.asset_multi_2],
)
definitions = Definitions(
@@ -20,6 +20,8 @@ definitions = Definitions(
)
for asset in load_assets_from_modules([assets])
],
resources={"polars_parquet_io_manager": PolarsParquetIOManager()},
resources={
"polars_parquet_io_manager": PolarsParquetIOManager(base_dir="/storage")
},
jobs=[daily_job],
)

View File

@@ -11,7 +11,6 @@ from dagster._core.instance import DynamicPartitionsStore
from dagster._serdes import whitelist_for_serdes
# @whitelist_for_serdes
class LatestTwoPartitionsMapping(PartitionMapping):
def get_upstream_mapped_partitions_result_for_partitions(
self,