feat: improve ingestion incremental strategy

data_platform/assets/ingestion/funda/sql/ddl/create_listing_details.sql

@@ -37,5 +37,7 @@ CREATE TABLE IF NOT EXISTS {{ schema }}.listing_details (
     saves INT,
     raw_json JSONB,
     ingested_at TIMESTAMPTZ DEFAULT now(),
+    last_fetched_at TIMESTAMPTZ DEFAULT now(),
+    is_stale BOOLEAN DEFAULT FALSE,
     UNIQUE (global_id, status)
 );

data_platform/assets/ingestion/funda/sql/ddl/create_search_results.sql

@@ -19,5 +19,7 @@ CREATE TABLE IF NOT EXISTS {{ schema }}.search_results (
     broker_name TEXT,
     raw_json JSONB,
     ingested_at TIMESTAMPTZ DEFAULT now(),
+    last_seen_at TIMESTAMPTZ DEFAULT now(),
+    is_active BOOLEAN DEFAULT TRUE,
     UNIQUE (global_id)
 );

data_platform/assets/ingestion/funda/sql/dml/insert_listing_details.sql

@@ -56,4 +56,6 @@ ON CONFLICT (global_id, status) DO UPDATE SET
     views = excluded.views,
     saves = excluded.saves,
     raw_json = excluded.raw_json,
-    ingested_at = now()
+    ingested_at = now(),
+    last_fetched_at = now(),
+    is_stale = FALSE

data_platform/assets/ingestion/funda/sql/dml/insert_search_results.sql

@@ -29,4 +29,6 @@ ON CONFLICT (global_id) DO UPDATE SET
     broker_id = excluded.broker_id,
     broker_name = excluded.broker_name,
     raw_json = excluded.raw_json,
-    ingested_at = now()
+    ingested_at = now(),
+    last_seen_at = now(),
+    is_active = TRUE