mirror/data-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: clean up dbt

Browse Source
This commit is contained in:
Stijnvandenbroek
2026-03-06 11:55:44 +00:00
parent 732e50924e
commit 19f35d4eb5
6 changed files with 49 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
dbt/models/intermediate/int_funda_listings_enriched.yml
View File

@@ -18,7 +18,7 @@ models:
constraints: constraints:
- type: not_null - type: not_null
- type: unique - type: unique
tests: data_tests:
- unique - unique
- not_null - not_null
- name: tiny_id - name: tiny_id
@@ -30,7 +30,7 @@ models:
- name: city - name: city
description: City name. description: City name.
data_type: text data_type: text
tests: data_tests:
- not_null - not_null
- name: postcode - name: postcode
description: Dutch postal code. description: Dutch postal code.
@@ -53,12 +53,12 @@ models:
- name: status - name: status
description: Listing status. description: Listing status.
data_type: text data_type: text
tests: data_tests:
- not_null - not_null
- name: offering_type - name: offering_type
description: Buy or rent. description: Buy or rent.
data_type: text data_type: text
tests: data_tests:
- not_null - not_null
- name: object_type - name: object_type
description: Property type. description: Property type.
@@ -146,12 +146,12 @@ models:
data_type: boolean data_type: boolean
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- name: ingested_at - name: ingested_at
description: Timestamp when the raw row was first written. description: Timestamp when the raw row was first written.
data_type: timestamptz data_type: timestamptz
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null

3
dbt/models/marts/elo_sample_listings.yml
View File

@@ -11,6 +11,7 @@ models:
unique_key: global_id unique_key: global_id
on_schema_change: fail on_schema_change: fail
schema: elo schema: elo
alias: sample_listings
contract: contract:
enforced: true enforced: true
meta: meta:
@@ -23,7 +24,7 @@ models:
constraints: constraints:
- type: not_null - type: not_null
- type: unique - type: unique
tests: data_tests:
- unique - unique
- not_null - not_null
- relationships: - relationships:

18
dbt/models/marts/funda_listings.yml
View File

@@ -18,7 +18,7 @@ models:
constraints: constraints:
- type: not_null - type: not_null
- type: unique - type: unique
tests: data_tests:
- unique - unique
- not_null - not_null
- name: tiny_id - name: tiny_id
@@ -35,7 +35,7 @@ models:
data_type: text data_type: text
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- name: postcode - name: postcode
description: Dutch postal code. description: Dutch postal code.
@@ -66,7 +66,7 @@ models:
data_type: text data_type: text
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- name: construction_type - name: construction_type
description: Construction method. description: Construction method.
@@ -80,7 +80,7 @@ models:
- name: living_area - name: living_area
description: Interior floor area in m². description: Interior floor area in m².
data_type: integer data_type: integer
tests: data_tests:
- dbt_utils.expression_is_true: - dbt_utils.expression_is_true:
expression: "> 0" expression: "> 0"
where: "living_area is not null" where: "living_area is not null"
@@ -117,7 +117,7 @@ models:
- name: current_price - name: current_price
description: Current asking or rental price in euros. description: Current asking or rental price in euros.
data_type: bigint data_type: bigint
tests: data_tests:
- dbt_utils.expression_is_true: - dbt_utils.expression_is_true:
expression: "> 0" expression: "> 0"
where: "current_price is not null" where: "current_price is not null"
@@ -138,7 +138,7 @@ models:
data_type: boolean data_type: boolean
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- name: photo_count - name: photo_count
description: Number of photos on the listing. description: Number of photos on the listing.
@@ -154,7 +154,7 @@ models:
data_type: text data_type: text
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- name: publication_date - name: publication_date
description: Listing publication date. description: Listing publication date.
@@ -164,12 +164,12 @@ models:
data_type: timestamptz data_type: timestamptz
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- name: price_per_sqm - name: price_per_sqm
description: Current price divided by living area in m². description: Current price divided by living area in m².
data_type: numeric data_type: numeric
tests: data_tests:
- dbt_utils.expression_is_true: - dbt_utils.expression_is_true:
expression: "> 0" expression: "> 0"
where: "price_per_sqm is not null" where: "price_per_sqm is not null"

34
dbt/models/staging/sources.yml
View File

@@ -18,32 +18,32 @@ sources:
columns: columns:
- name: global_id - name: global_id
description: Funda internal listing ID. description: Funda internal listing ID.
tests: data_tests:
- unique - unique
- not_null - not_null
- name: title - name: title
description: Property address / title. description: Property address / title.
tests: data_tests:
- not_null - not_null
- name: city - name: city
description: City name. description: City name.
tests: data_tests:
- not_null - not_null
- name: price - name: price
description: Asking or rental price in euros. description: Asking or rental price in euros.
- name: is_active - name: is_active
description: False when the listing has not appeared in search results for 7+ days. description: False when the listing has not appeared in search results for 7+ days.
tests: data_tests:
- not_null - not_null
- accepted_values: - accepted_values:
values: [true, false] values: [true, false]
- name: last_seen_at - name: last_seen_at
description: Timestamp the listing was last returned by the Funda search API. description: Timestamp the listing was last returned by the Funda search API.
tests: data_tests:
- not_null - not_null
- name: ingested_at - name: ingested_at
description: Timestamp when the row was first written. description: Timestamp when the row was first written.
tests: data_tests:
- not_null - not_null
- name: listing_details - name: listing_details
@@ -59,36 +59,36 @@ sources:
columns: columns:
- name: global_id - name: global_id
description: Funda internal listing ID. description: Funda internal listing ID.
tests: data_tests:
- not_null - not_null
- relationships: - relationships:
to: source('raw_funda', 'search_results') to: source('raw_funda', 'search_results')
field: global_id field: global_id
- name: tiny_id - name: tiny_id
description: Public ID used in Funda URLs. description: Public ID used in Funda URLs.
tests: data_tests:
- not_null - not_null
- name: price - name: price
description: Asking or rental price in euros. description: Asking or rental price in euros.
- name: status - name: status
description: Listing status (available or sold). description: Listing status (available or sold).
tests: data_tests:
- not_null - not_null
- accepted_values: - accepted_values:
values: ["available", "sold", "withdrawn", "under_negotiation"] values: ["available", "sold", "withdrawn", "under_negotiation"]
- name: is_stale - name: is_stale
description: True when the parent search listing is no longer active. description: True when the parent search listing is no longer active.
tests: data_tests:
- not_null - not_null
- accepted_values: - accepted_values:
values: [true, false] values: [true, false]
- name: last_fetched_at - name: last_fetched_at
description: Timestamp of the most recent detail fetch. description: Timestamp of the most recent detail fetch.
tests: data_tests:
- not_null - not_null
- name: ingested_at - name: ingested_at
description: Timestamp when the row was first written. description: Timestamp when the row was first written.
tests: data_tests:
- not_null - not_null
- name: price_history - name: price_history
@@ -104,7 +104,7 @@ sources:
columns: columns:
- name: global_id - name: global_id
description: Funda internal listing ID. description: Funda internal listing ID.
tests: data_tests:
- not_null - not_null
- relationships: - relationships:
to: source('raw_funda', 'listing_details') to: source('raw_funda', 'listing_details')
@@ -113,21 +113,21 @@ sources:
description: Price at this point in time. description: Price at this point in time.
- name: date - name: date
description: Date of this price event. description: Date of this price event.
tests: data_tests:
- not_null - not_null
- name: source - name: source
description: Price data source (funda or woz). description: Price data source (funda or woz).
tests: data_tests:
- not_null - not_null
- accepted_values: - accepted_values:
values: ["Funda", "WOZ"] values: ["Funda", "WOZ"]
- name: status - name: status
description: Price event type (asking_price, sold, or woz). description: Price event type (asking_price, sold, or woz).
tests: data_tests:
- not_null - not_null
- accepted_values: - accepted_values:
values: ["asking_price", "sold", "woz", "withdrawn"] values: ["asking_price", "sold", "woz", "withdrawn"]
- name: ingested_at - name: ingested_at
description: Timestamp when the row was written. description: Timestamp when the row was written.
tests: data_tests:
- not_null - not_null

18
dbt/models/staging/stg_funda_listings.yml
View File

@@ -16,7 +16,7 @@ models:
constraints: constraints:
- type: not_null - type: not_null
- type: unique - type: unique
tests: data_tests:
- unique - unique
- not_null - not_null
- name: tiny_id - name: tiny_id
@@ -24,7 +24,7 @@ models:
data_type: text data_type: text
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- name: title - name: title
description: Property address / title. description: Property address / title.
@@ -34,7 +34,7 @@ models:
data_type: text data_type: text
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- name: postcode - name: postcode
description: Dutch postal code. description: Dutch postal code.
@@ -51,7 +51,7 @@ models:
- name: price - name: price
description: Asking or rental price in euros. description: Asking or rental price in euros.
data_type: bigint data_type: bigint
tests: data_tests:
- dbt_utils.expression_is_true: - dbt_utils.expression_is_true:
expression: ">= 0" expression: ">= 0"
where: "price is not null" where: "price is not null"
@@ -63,14 +63,14 @@ models:
data_type: text data_type: text
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- accepted_values: - accepted_values:
values: ["available", "sold", "withdrawn", "under_negotiation"] values: ["available", "sold", "withdrawn", "under_negotiation"]
- name: offering_type - name: offering_type
description: Buy or rent. description: Buy or rent.
data_type: text data_type: text
tests: data_tests:
- not_null - not_null
- name: object_type - name: object_type
description: Property type (house, apartment, etc.). description: Property type (house, apartment, etc.).
@@ -87,14 +87,14 @@ models:
- name: energy_label - name: energy_label
description: Dutch energy performance label (A–G). description: Dutch energy performance label (A–G).
data_type: text data_type: text
tests: data_tests:
- accepted_values: - accepted_values:
values: ["A4", "A3", "A2", "A1", "A", "B", "C", "D", "E", "F", "G"] values: ["A4", "A3", "A2", "A1", "A", "B", "C", "D", "E", "F", "G"]
where: "energy_label is not null" where: "energy_label is not null"
- name: living_area - name: living_area
description: Interior floor area in m². description: Interior floor area in m².
data_type: integer data_type: integer
tests: data_tests:
- dbt_utils.expression_is_true: - dbt_utils.expression_is_true:
expression: "> 0" expression: "> 0"
where: "living_area is not null" where: "living_area is not null"
@@ -154,5 +154,5 @@ models:
data_type: timestamptz data_type: timestamptz
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null

12
dbt/models/staging/stg_funda_price_history.yml
View File

@@ -15,7 +15,7 @@ models:
data_type: text data_type: text
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- relationships: - relationships:
to: ref('stg_funda_listings') to: ref('stg_funda_listings')
@@ -23,7 +23,7 @@ models:
- name: price - name: price
description: Price at this point in time. description: Price at this point in time.
data_type: bigint data_type: bigint
tests: data_tests:
- dbt_utils.expression_is_true: - dbt_utils.expression_is_true:
expression: ">= 0" expression: ">= 0"
where: "price is not null" where: "price is not null"
@@ -35,7 +35,7 @@ models:
data_type: text data_type: text
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- name: price_timestamp - name: price_timestamp
description: Timestamp of this price event. description: Timestamp of this price event.
@@ -45,7 +45,7 @@ models:
data_type: text data_type: text
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- accepted_values: - accepted_values:
values: ["Funda", "WOZ"] values: ["Funda", "WOZ"]
@@ -54,7 +54,7 @@ models:
data_type: text data_type: text
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null
- accepted_values: - accepted_values:
values: ["asking_price", "sold", "woz", "withdrawn"] values: ["asking_price", "sold", "woz", "withdrawn"]
@@ -63,5 +63,5 @@ models:
data_type: timestamptz data_type: timestamptz
constraints: constraints:
- type: not_null - type: not_null
tests: data_tests:
- not_null - not_null