organize weather data by retrieval time

This commit is contained in:
2025-07-27 17:52:31 +02:00
parent 662bc239f9
commit e5cb15d1ba
4 changed files with 83 additions and 9 deletions

View File

@@ -1,3 +1,4 @@
from datetime import datetime, timezone
from typing import Any
import requests_cache
@@ -9,6 +10,16 @@ from utils import parse_coord
import dagster as dg
@dg.asset(
io_manager_key="json_io_manager",
)
def organised():
yield dg.Output(
{"interesting": "data"},
metadata={"where": "something", "path_prefix": "a/b", "path_suffix": "c/d"},
)
@dg.asset(
io_manager_key="json_io_manager",
partitions_def=location_partitions_def,
@@ -79,4 +90,14 @@ def raw_weather(context: dg.AssetExecutionContext) -> Any:
raise ValueError(
f"Error (data['error']) fetching weather data: {data.get('reason', '')}"
)
return response.json()
now = datetime.now(tz=timezone.utc)
date_str = now.strftime("%Y-%m-%d")
time_str = now.strftime("%H:%M:%S")
yield dg.Output(
data,
metadata={
"date": dg.MetadataValue.timestamp(now),
"path_suffix": [date_str, time_str],
},
)