feat: add funda ingest and staging models
This commit is contained in:
@@ -1,12 +1,27 @@
|
||||
version: 2
|
||||
|
||||
models:
|
||||
- name: stg_example
|
||||
- name: stg_funda_listings
|
||||
description: >
|
||||
A placeholder staging model. Replace with your actual source tables.
|
||||
Cleaned Funda listing details – one row per property.
|
||||
columns:
|
||||
- name: id
|
||||
description: Primary key.
|
||||
- name: global_id
|
||||
description: Funda internal listing ID.
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: city
|
||||
description: City name.
|
||||
- name: price
|
||||
description: Asking or rental price in euros.
|
||||
|
||||
- name: stg_funda_price_history
|
||||
description: >
|
||||
Historical price events per listing (asking prices, WOZ assessments, sales).
|
||||
columns:
|
||||
- name: global_id
|
||||
description: Funda internal listing ID.
|
||||
tests:
|
||||
- not_null
|
||||
- name: price
|
||||
description: Price at this point in time.
|
||||
|
||||
51
dbt/models/staging/sources.yml
Normal file
51
dbt/models/staging/sources.yml
Normal file
@@ -0,0 +1,51 @@
|
||||
version: 2
|
||||
|
||||
sources:
|
||||
- name: raw_funda
|
||||
schema: raw_funda
|
||||
description: >
|
||||
Raw Funda real-estate data ingested by Dagster assets.
|
||||
tables:
|
||||
- name: search_results
|
||||
description: Funda search results (broad overview of matching listings).
|
||||
columns:
|
||||
- name: global_id
|
||||
description: Funda internal listing ID.
|
||||
- name: title
|
||||
description: Property address / title.
|
||||
- name: city
|
||||
description: City name.
|
||||
- name: price
|
||||
description: Asking or rental price in euros.
|
||||
- name: ingested_at
|
||||
description: Timestamp when the row was written.
|
||||
|
||||
- name: listing_details
|
||||
description: >
|
||||
Full listing details fetched per search result (50+ fields).
|
||||
columns:
|
||||
- name: global_id
|
||||
description: Funda internal listing ID.
|
||||
- name: tiny_id
|
||||
description: Public ID used in Funda URLs.
|
||||
- name: price
|
||||
description: Asking or rental price in euros.
|
||||
- name: status
|
||||
description: Listing status (available or sold).
|
||||
- name: ingested_at
|
||||
description: Timestamp when the row was written.
|
||||
|
||||
- name: price_history
|
||||
description: >
|
||||
Historical price data per listing (asking prices, WOZ, sales).
|
||||
columns:
|
||||
- name: global_id
|
||||
description: Funda internal listing ID.
|
||||
- name: price
|
||||
description: Price at this point in time.
|
||||
- name: source
|
||||
description: Price data source (Funda or WOZ).
|
||||
- name: status
|
||||
description: Price event type (asking_price, sold, or woz).
|
||||
- name: ingested_at
|
||||
description: Timestamp when the row was written.
|
||||
@@ -1,4 +0,0 @@
|
||||
-- Placeholder staging model.
|
||||
-- Replace with your actual source query using the source() macro.
|
||||
|
||||
select 1 as id, 'example' as name
|
||||
48
dbt/models/staging/stg_funda_listings.sql
Normal file
48
dbt/models/staging/stg_funda_listings.sql
Normal file
@@ -0,0 +1,48 @@
|
||||
-- Staging model: one row per Funda listing with cleaned core fields.
|
||||
|
||||
with source as (
|
||||
select * from {{ source('raw_funda', 'listing_details') }}
|
||||
),
|
||||
|
||||
staged as (
|
||||
select
|
||||
global_id,
|
||||
tiny_id,
|
||||
title,
|
||||
city,
|
||||
postcode,
|
||||
province,
|
||||
neighbourhood,
|
||||
municipality,
|
||||
price,
|
||||
price_formatted,
|
||||
status,
|
||||
offering_type,
|
||||
object_type,
|
||||
house_type,
|
||||
construction_type,
|
||||
construction_year,
|
||||
energy_label,
|
||||
living_area,
|
||||
plot_area,
|
||||
bedrooms,
|
||||
rooms,
|
||||
publication_date,
|
||||
latitude,
|
||||
longitude,
|
||||
has_garden,
|
||||
has_balcony,
|
||||
has_solar_panels,
|
||||
has_heat_pump,
|
||||
has_roof_terrace,
|
||||
is_energy_efficient,
|
||||
is_monument,
|
||||
url,
|
||||
photo_count,
|
||||
views,
|
||||
saves,
|
||||
ingested_at
|
||||
from source
|
||||
)
|
||||
|
||||
select * from staged
|
||||
20
dbt/models/staging/stg_funda_price_history.sql
Normal file
20
dbt/models/staging/stg_funda_price_history.sql
Normal file
@@ -0,0 +1,20 @@
|
||||
-- Staging model: one row per price-history event per listing.
|
||||
|
||||
with source as (
|
||||
select * from {{ source('raw_funda', 'price_history') }}
|
||||
),
|
||||
|
||||
staged as (
|
||||
select
|
||||
global_id,
|
||||
price,
|
||||
human_price,
|
||||
date as price_date,
|
||||
timestamp as price_timestamp,
|
||||
source as price_source,
|
||||
status as price_status,
|
||||
ingested_at
|
||||
from source
|
||||
)
|
||||
|
||||
select * from staged
|
||||
Reference in New Issue
Block a user