first version of weather loader

This commit is contained in:
2025-07-27 14:54:19 +02:00
parent d3d3911609
commit 131912b70d
9 changed files with 309 additions and 7 deletions

View File

@@ -1,27 +1,30 @@
import os
import assets
import sensors
from dagster_polars import PolarsParquetIOManager
from icecream import install
from resoures import JsonIOManager
import dagster as dg
from dagster import load_assets_from_modules
install()
APP = os.environ["APP"]
storage_dir = os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}"
definitions = dg.Definitions(
assets=[
asset.with_attributes(
group_names_by_key={asset.key: APP},
tags_by_key={asset.key: {"app": APP}},
)
for asset in load_assets_from_modules([assets])
for asset in dg.load_assets_from_modules([assets])
],
resources={
"polars_parquet_io_manager": PolarsParquetIOManager(
base_dir=os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}"
),
"json_io_manager": JsonIOManager(base_dir=storage_dir),
"polars_parquet_io_manager": PolarsParquetIOManager(base_dir=storage_dir),
},
sensors=[sensors.list_locations],
)