rik/dagster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use batch weather fetcher

Browse Source
This commit is contained in:
Rik Veenboer
2025-08-04 15:15:53 +02:00
parent d6737cc327
commit 4a44d4c3cd
4 changed files with 153 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
apps/weather/src/sensors.py
View File

@@ -51,7 +51,7 @@ def list_locations(context: dg.SensorEvaluationContext) -> dg.SensorResult:
)
@dg.sensor(job=jobs.raw_weather_batch_job)
@dg.sensor(job=jobs.raw_weather_batch_latitude_job)
def list_latitudes(context: dg.SensorEvaluationContext) -> dg.SensorResult:
existing_latitudes = set(
context.instance.get_dynamic_partitions(location_partitions_def.name)
@@ -69,7 +69,25 @@ def list_latitudes(context: dg.SensorEvaluationContext) -> dg.SensorResult:
)
@dg.sensor(job=jobs.raw_weather_job, minimum_interval_seconds=60 * 60)
@dg.sensor(job=jobs.raw_weather_batch_longitude_job)
def list_longitudes(context: dg.SensorEvaluationContext) -> dg.SensorResult:
existing_longitudes = set(
context.instance.get_dynamic_partitions(location_partitions_def.name)
)
longitudes = [longitude_to_str(lon) for lon in lon_range]
new_longitudes = [lon for lon in longitudes if lon not in existing_longitudes]
return dg.SensorResult(
run_requests=[
dg.RunRequest(partition_key=partition_key)
for partition_key in new_longitudes
],
dynamic_partitions_requests=[
longitude_partitions_def.build_add_request(new_longitudes)
],
)
@dg.sensor(job=jobs.raw_weather_batch_latitude_job, minimum_interval_seconds=60 * 60)
def retrieve_weather(context: dg.SensorEvaluationContext) -> dg.SensorResult:
"""
Retrieve weather sensor function.
@@ -86,7 +104,7 @@ def retrieve_weather(context: dg.SensorEvaluationContext) -> dg.SensorResult:
The result of the sensor's evaluation, containing run requests for each existing partition key.
"""
existing_keys = set(
context.instance.get_dynamic_partitions(location_partitions_def.name)
context.instance.get_dynamic_partitions(latitude_partitions_def.name)
)
return dg.SensorResult(
run_requests=[