first version of weather loader
This commit is contained in:
@@ -1,27 +1,30 @@
|
||||
import os
|
||||
|
||||
import assets
|
||||
import sensors
|
||||
from dagster_polars import PolarsParquetIOManager
|
||||
from icecream import install
|
||||
from resoures import JsonIOManager
|
||||
|
||||
import dagster as dg
|
||||
from dagster import load_assets_from_modules
|
||||
|
||||
install()
|
||||
|
||||
APP = os.environ["APP"]
|
||||
|
||||
storage_dir = os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}"
|
||||
|
||||
definitions = dg.Definitions(
|
||||
assets=[
|
||||
asset.with_attributes(
|
||||
group_names_by_key={asset.key: APP},
|
||||
tags_by_key={asset.key: {"app": APP}},
|
||||
)
|
||||
for asset in load_assets_from_modules([assets])
|
||||
for asset in dg.load_assets_from_modules([assets])
|
||||
],
|
||||
resources={
|
||||
"polars_parquet_io_manager": PolarsParquetIOManager(
|
||||
base_dir=os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}"
|
||||
),
|
||||
"json_io_manager": JsonIOManager(base_dir=storage_dir),
|
||||
"polars_parquet_io_manager": PolarsParquetIOManager(base_dir=storage_dir),
|
||||
},
|
||||
sensors=[sensors.list_locations],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user