first version of weather loader

apps/weather/src/sensors.py

@@ -0,0 +1,34 @@
+import jobs
+import numpy as np
+from partitions import location_partitions_def
+from utils import format_coord
+
+import dagster as dg
+
+
+@dg.sensor(job=jobs.raw_weather_job)
+def list_locations(context: dg.SensorEvaluationContext) -> dg.SensorResult:
+    """Sensor that emits RunRequests for new 0.25-degree grid locations not yet seen as partitions."""
+    existing_keys = set(
+        context.instance.get_dynamic_partitions(location_partitions_def.name)
+    )
+
+    lat_range = np.arange(51.0, 53.01, 0.25, dtype=float)
+    lon_range = np.arange(3.0, 7.01, 0.25, dtype=float)
+
+    locations = [format_coord(lat, lon) for lat in lat_range for lon in lon_range]
+
+    new_locations = [
+        location for location in locations if location not in existing_keys
+    ]
+    if new_locations:
+        context.log.info(f"Discovered {len(new_locations)} new locations.")
+
+    # Limit to 3 new locations
+    selected = new_locations[:3]
+    return dg.SensorResult(
+        run_requests=[dg.RunRequest(partition_key=loc) for loc in selected],
+        dynamic_partitions_requests=[
+            location_partitions_def.build_add_request(selected)
+        ],
+    )