shared config

This commit is contained in:
2025-07-29 11:38:41 +02:00
parent cc6beeffed
commit f59b30b9ea
6 changed files with 17 additions and 44 deletions

View File

@@ -1,20 +1,15 @@
import os
import assets
import sensors
from dagster_polars import PolarsParquetIOManager
from icecream import install
from resources import HtmlIOManager
from shared.config import APP, STORAGE_DIR
import dagster as dg
from dagster import load_assets_from_modules
install()
APP = os.environ["APP"]
storage_dir = os.environ.get("STORAGE_DIR", "/storage")
definitions = dg.Definitions(
assets=[
asset.with_attributes(
@@ -23,10 +18,8 @@ definitions = dg.Definitions(
for asset in load_assets_from_modules([assets])
],
resources={
"html_io_manager": HtmlIOManager(base_dir=storage_dir),
"polars_parquet_io_manager": PolarsParquetIOManager(
base_dir=os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}"
),
"html_io_manager": HtmlIOManager(base_dir=STORAGE_DIR),
"polars_parquet_io_manager": PolarsParquetIOManager(base_dir=STORAGE_DIR),
},
sensors=[sensors.check_update],
)