fix paths to storage and logs
This commit is contained in:
@@ -1,4 +1,3 @@
|
||||
import pandas as pd
|
||||
import polars as pl
|
||||
|
||||
from dagster import (
|
||||
@@ -23,16 +22,6 @@ partitions_def_multi = MultiPartitionsDefinition(
|
||||
)
|
||||
|
||||
|
||||
@asset(
|
||||
# tags={
|
||||
# "dagster/executor": "other_executor"
|
||||
# },
|
||||
)
|
||||
def dummy_asset():
|
||||
"""A dummy asset to ensure the module is recognized by Dagster."""
|
||||
return pd.DataFrame({"dummy": [1, 2, 3]})
|
||||
|
||||
|
||||
@asset(
|
||||
io_manager_key="polars_parquet_io_manager",
|
||||
partitions_def=partitions_def_single,
|
||||
@@ -58,7 +47,6 @@ def asset_single_1(context):
|
||||
def asset_multi_1(context):
|
||||
ic()
|
||||
ic(context.partition_key)
|
||||
|
||||
return pl.DataFrame(
|
||||
[{"date": context.partition_key, "data": f"Data for {context.partition_key}"}]
|
||||
)
|
||||
|
||||
@@ -9,7 +9,7 @@ install()
|
||||
# Define a job that includes both assets
|
||||
daily_job = define_asset_job(
|
||||
"daily_job",
|
||||
selection=[assets.dummy_asset, assets.asset_multi_1, assets.asset_multi_2],
|
||||
selection=[assets.asset_multi_1, assets.asset_multi_2],
|
||||
)
|
||||
|
||||
definitions = Definitions(
|
||||
@@ -20,6 +20,8 @@ definitions = Definitions(
|
||||
)
|
||||
for asset in load_assets_from_modules([assets])
|
||||
],
|
||||
resources={"polars_parquet_io_manager": PolarsParquetIOManager()},
|
||||
resources={
|
||||
"polars_parquet_io_manager": PolarsParquetIOManager(base_dir="/storage")
|
||||
},
|
||||
jobs=[daily_job],
|
||||
)
|
||||
|
||||
@@ -11,7 +11,6 @@ from dagster._core.instance import DynamicPartitionsStore
|
||||
from dagster._serdes import whitelist_for_serdes
|
||||
|
||||
|
||||
# @whitelist_for_serdes
|
||||
class LatestTwoPartitionsMapping(PartitionMapping):
|
||||
def get_upstream_mapped_partitions_result_for_partitions(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user