chore: refactor ingestion

data_platform/assets/ingestion/funda/sql/ddl/create_schema.sql

@@ -0,0 +1 @@
+create schema if not exists {{ schema }};

data_platform/assets/ingestion/funda/sql/dml/mark_inactive_search_results.sql

@@ -0,0 +1,4 @@
+update {{ schema }}.search_results
+set is_active = false
+where last_seen_at < now() - interval '7 days'
+returning global_id

data_platform/assets/ingestion/funda/sql/dml/mark_stale_listing_details.sql

@@ -0,0 +1,7 @@
+update {{ schema }}.listing_details d
+set is_stale = true
+from {{ schema }}.search_results as s
+where
+    d.global_id = s.global_id
+    and s.is_active = false
+    and d.is_stale = false

data_platform/assets/ingestion/funda/sql/dml/select_all_detail_ids.sql

@@ -0,0 +1,2 @@
+select distinct global_id
+from {{ schema }}.search_results

data_platform/assets/ingestion/funda/sql/dml/select_all_price_history_listings.sql

@@ -0,0 +1,6 @@
+select distinct
+    d.global_id,
+    d.url,
+    d.title,
+    d.postcode
+from {{ schema }}.listing_details as d

data_platform/assets/ingestion/funda/sql/dml/select_new_detail_ids.sql

@@ -0,0 +1,6 @@
+select distinct s.global_id
+from {{ schema }}.search_results as s
+left join {{ schema }}.listing_details as d on s.global_id = d.global_id
+where
+    s.is_active = true
+    and (d.global_id is null or d.is_stale = true)

data_platform/assets/ingestion/funda/sql/dml/select_new_price_history_listings.sql

@@ -0,0 +1,17 @@
+select distinct
+    d.global_id,
+    d.url,
+    d.title,
+    d.postcode
+from {{ schema }}.listing_details as d
+inner join {{ schema }}.search_results as s on d.global_id = s.global_id
+where s.is_active = true
+union
+select distinct
+    d.global_id,
+    d.url,
+    d.title,
+    d.postcode
+from {{ schema }}.listing_details as d
+left join {{ schema }}.price_history as p on d.global_id = p.global_id
+where p.global_id is null