organize weather data by retrieval time
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import requests_cache
|
||||
@@ -9,6 +10,16 @@ from utils import parse_coord
|
||||
import dagster as dg
|
||||
|
||||
|
||||
@dg.asset(
|
||||
io_manager_key="json_io_manager",
|
||||
)
|
||||
def organised():
|
||||
yield dg.Output(
|
||||
{"interesting": "data"},
|
||||
metadata={"where": "something", "path_prefix": "a/b", "path_suffix": "c/d"},
|
||||
)
|
||||
|
||||
|
||||
@dg.asset(
|
||||
io_manager_key="json_io_manager",
|
||||
partitions_def=location_partitions_def,
|
||||
@@ -79,4 +90,14 @@ def raw_weather(context: dg.AssetExecutionContext) -> Any:
|
||||
raise ValueError(
|
||||
f"Error (data['error']) fetching weather data: {data.get('reason', '')}"
|
||||
)
|
||||
return response.json()
|
||||
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
date_str = now.strftime("%Y-%m-%d")
|
||||
time_str = now.strftime("%H:%M:%S")
|
||||
yield dg.Output(
|
||||
data,
|
||||
metadata={
|
||||
"date": dg.MetadataValue.timestamp(now),
|
||||
"path_suffix": [date_str, time_str],
|
||||
},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user