feat: expand testing
This commit is contained in:
11
data_platform/schedules/__init__.py
Normal file
11
data_platform/schedules/__init__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
from data_platform.schedules.elementary import elementary_refresh_schedule
|
||||
from data_platform.schedules.funda import (
|
||||
funda_ingestion_schedule,
|
||||
funda_raw_quality_schedule,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"funda_ingestion_schedule",
|
||||
"funda_raw_quality_schedule",
|
||||
"elementary_refresh_schedule",
|
||||
]
|
||||
13
data_platform/schedules/elementary.py
Normal file
13
data_platform/schedules/elementary.py
Normal file
@@ -0,0 +1,13 @@
|
||||
"""Elementary schedules."""
|
||||
|
||||
from dagster import DefaultScheduleStatus, ScheduleDefinition
|
||||
|
||||
from data_platform.jobs.elementary import elementary_refresh_job
|
||||
|
||||
elementary_refresh_schedule = ScheduleDefinition(
|
||||
name="elementary_refresh_schedule",
|
||||
job=elementary_refresh_job,
|
||||
cron_schedule="0 9 * * *",
|
||||
description="Regenerate the Elementary report daily at 09:00 UTC.",
|
||||
default_status=DefaultScheduleStatus.RUNNING,
|
||||
)
|
||||
32
data_platform/schedules/funda.py
Normal file
32
data_platform/schedules/funda.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""Funda schedules."""
|
||||
|
||||
from dagster import DefaultScheduleStatus, RunConfig, ScheduleDefinition
|
||||
|
||||
from data_platform.assets.ingestion.funda.funda import (
|
||||
FundaDetailsConfig,
|
||||
FundaPriceHistoryConfig,
|
||||
FundaSearchConfig,
|
||||
)
|
||||
from data_platform.jobs.funda import funda_ingestion_job, funda_raw_quality_job
|
||||
|
||||
funda_ingestion_schedule = ScheduleDefinition(
|
||||
name="funda_ingestion_schedule",
|
||||
job=funda_ingestion_job,
|
||||
cron_schedule="0 */4 * * *",
|
||||
run_config=RunConfig(
|
||||
ops={
|
||||
"funda_search_results": FundaSearchConfig(),
|
||||
"funda_listing_details": FundaDetailsConfig(),
|
||||
"funda_price_history": FundaPriceHistoryConfig(),
|
||||
}
|
||||
),
|
||||
default_status=DefaultScheduleStatus.RUNNING,
|
||||
)
|
||||
|
||||
funda_raw_quality_schedule = ScheduleDefinition(
|
||||
name="funda_raw_quality_schedule",
|
||||
job=funda_raw_quality_job,
|
||||
cron_schedule="0 8 * * *",
|
||||
description="Daily quality checks on all raw Funda tables.",
|
||||
default_status=DefaultScheduleStatus.RUNNING,
|
||||
)
|
||||
Reference in New Issue
Block a user