towards batch download

apps/weather/src/assets.py

@@ -4,10 +4,14 @@ from typing import Any
 
 import requests_cache
 from config import APP
-from partitions import location_partitions_def
+from partitions import (
+    latitude_partitions_def,
+    location_partitions_def,
+    longitude_partitions_def,
+)
 from requests import Request
 from retry_requests import retry
-from utils import parse_coord
+from utils import parse_coordinate_str
 
 import dagster as dg
 
@@ -22,7 +26,7 @@ asset = partial(dg.asset, key_prefix=APP)
 def raw_weather(context: dg.AssetExecutionContext) -> Any:
     """Asset to fetch raw weather data for each location."""
     partition_key = context.partition_key
-    lat, lon = parse_coord(partition_key)
+    lat, lon = parse_coordinate_str(partition_key)
     context.log.info(
         f"Fetching weather data for location ({partition_key}): {lat}, {lon}"
     )
@@ -96,3 +100,20 @@ def raw_weather(context: dg.AssetExecutionContext) -> Any:
             "path_suffix": [date_str, time_str],
         },
     )
+
+
+@asset(
+    io_manager_key="json_io_manager",
+    partitions_def=latitude_partitions_def,
+    name="raw_batch",
+)
+def raw_weather_batch(context: dg.AssetExecutionContext) -> None:
+    for partitions_def in [
+        location_partitions_def,
+        latitude_partitions_def,
+        longitude_partitions_def,
+    ]:
+        existing_keys = set(
+            context.instance.get_dynamic_partitions(partitions_def.name)
+        )
+        ic(partitions_def.name, len(existing_keys), existing_keys)