add asset key prefix

This commit is contained in:
2025-07-27 18:06:47 +02:00
parent e5cb15d1ba
commit 1e9078fed5
4 changed files with 14 additions and 16 deletions

View File

@@ -1,7 +1,9 @@
from datetime import datetime, timezone
from functools import partial
from typing import Any
import requests_cache
from config import APP
from partitions import location_partitions_def
from requests import Request
from retry_requests import retry
@@ -9,20 +11,13 @@ from utils import parse_coord
import dagster as dg
@dg.asset(
io_manager_key="json_io_manager",
)
def organised():
yield dg.Output(
{"interesting": "data"},
metadata={"where": "something", "path_prefix": "a/b", "path_suffix": "c/d"},
)
asset = partial(dg.asset, key_prefix=APP)
@dg.asset(
@asset(
io_manager_key="json_io_manager",
partitions_def=location_partitions_def,
name="raw",
)
def raw_weather(context: dg.AssetExecutionContext) -> Any:
"""Asset to fetch raw weather data for each location."""

View File

@@ -0,0 +1,4 @@
import os
from pathlib import Path
APP = os.environ.get("APP", Path(__file__).parent.parent.name)

View File

@@ -1,8 +1,8 @@
import os
from pathlib import Path
import assets
import sensors
from config import APP
from dagster_polars import PolarsParquetIOManager
from icecream import install
from resoures import JsonIOManager
@@ -11,9 +11,7 @@ import dagster as dg
install()
APP = os.environ.get("APP", Path(__file__).parent.parent.name)
storage_dir = os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}"
storage_dir = os.environ.get("STORAGE_DIR", "/storage")
definitions = dg.Definitions(
assets=[

View File

@@ -4,7 +4,7 @@ from datetime import datetime
from dotenv import find_dotenv, load_dotenv
import dagster as dg
from apps.weather.src.assets import organised
from apps.weather.src.assets import raw_weather
logging.getLogger().setLevel(logging.INFO)
@@ -25,7 +25,8 @@ if __name__ == "__main__":
case 1:
dg.materialize(
assets=definitions.assets,
selection=[organised.key],
selection=[raw_weather.key],
partition_key="N5100E0300",
resources=resources,
)
case _: