first version of weather loader

apps/weather/src/assets.py

@@ -0,0 +1,73 @@
+from typing import Any
+
+import requests_cache
+from partitions import location_partitions_def
+from retry_requests import retry
+from utils import parse_coord
+
+import dagster as dg
+
+
+@dg.asset(
+    io_manager_key="json_io_manager",
+    partitions_def=location_partitions_def,
+)
+def raw_weather(context: dg.AssetExecutionContext) -> Any:
+    """Asset to fetch raw weather data for each location."""
+    partition_key = context.partition_key
+    lat, lon = parse_coord(partition_key)
+    context.log.info(
+        f"Fetching weather data for location ({partition_key}): {lat}, {lon}"
+    )
+
+    # Setup the Open-Meteo API client with cache and retry on error
+    cache_session = requests_cache.CachedSession("/cache/open-meteo", expire_after=3600)
+    retry_session = retry(cache_session, retries=5, backoff_factor=0.2)
+    # openmeteo = openmeteo_requests.Client(session=retry_session)
+
+    # Make sure all required weather variables are listed here
+    # The order of variables in hourly or daily is important to assign them correctly below
+    url = "https://api.open-meteo.com/v1/forecast"
+    params = {
+        "latitude": 52.5,
+        "longitude": 4.5,
+        "hourly": [
+            "temperature_2m",
+            "shortwave_radiation",
+            "shortwave_radiation_instant",
+            "direct_radiation",
+            "rain",
+            "precipitation",
+            "showers",
+            "weather_code",
+            "pressure_msl",
+            "surface_pressure",
+            "wind_speed_80m",
+            "wind_direction_80m",
+            "wind_gusts_10m",
+            "temperature_80m",
+            "wind_direction_10m",
+            "wind_speed_10m",
+            "surface_temperature",
+        ],
+        "models": "gfs_global",
+        "current": [
+            "wind_speed_10m",
+            "wind_gusts_10m",
+            "temperature_2m",
+            "relative_humidity_2m",
+            "apparent_temperature",
+            "precipitation",
+            "rain",
+            "snowfall",
+        ],
+        "forecast_days": 16,
+    }
+    # responses = openmeteo.weather_api(url, params=params)
+    # print(len(responses))
+    # print(responses)
+
+    response = retry_session.get(url, params=params)
+    response.raise_for_status()
+    response.json()
+    return response.json()