feat: expand testing
This commit is contained in:
8
data_platform/jobs/__init__.py
Normal file
8
data_platform/jobs/__init__.py
Normal file
@@ -0,0 +1,8 @@
|
||||
from data_platform.jobs.elementary import elementary_refresh_job
|
||||
from data_platform.jobs.funda import funda_ingestion_job, funda_raw_quality_job
|
||||
|
||||
__all__ = [
|
||||
"funda_ingestion_job",
|
||||
"funda_raw_quality_job",
|
||||
"elementary_refresh_job",
|
||||
]
|
||||
10
data_platform/jobs/elementary.py
Normal file
10
data_platform/jobs/elementary.py
Normal file
@@ -0,0 +1,10 @@
|
||||
"""Elementary jobs."""
|
||||
|
||||
from dagster import job
|
||||
|
||||
from data_platform.ops.elementary import elementary_generate_report
|
||||
|
||||
|
||||
@job(description="Regenerate the Elementary data observability report.")
|
||||
def elementary_refresh_job():
|
||||
elementary_generate_report()
|
||||
28
data_platform/jobs/funda.py
Normal file
28
data_platform/jobs/funda.py
Normal file
@@ -0,0 +1,28 @@
|
||||
"""Funda jobs."""
|
||||
|
||||
from dagster import AssetSelection, RunConfig, define_asset_job, job
|
||||
|
||||
from data_platform.ops.check_source_freshness import (
|
||||
SourceFreshnessConfig,
|
||||
check_source_freshness,
|
||||
)
|
||||
|
||||
funda_ingestion_job = define_asset_job(
|
||||
name="funda_ingestion",
|
||||
selection=AssetSelection.assets(
|
||||
"funda_search_results",
|
||||
"funda_listing_details",
|
||||
"funda_price_history",
|
||||
),
|
||||
description="Full Funda ingestion pipeline.",
|
||||
)
|
||||
|
||||
|
||||
@job(
|
||||
description="dbt source freshness checks for all raw_funda sources.",
|
||||
config=RunConfig(
|
||||
ops={"check_source_freshness": SourceFreshnessConfig(source_name="raw_funda")}
|
||||
),
|
||||
)
|
||||
def funda_raw_quality_job():
|
||||
check_source_freshness()
|
||||
Reference in New Issue
Block a user