chore: refactor ingestion

2026-03-05 21:36:53 +00:00
parent d5d3c2b649
commit 2ab1d71b53
13 changed files with 156 additions and 71 deletions
--- a/dagster_home/dagster.yaml
+++ b/dagster_home/dagster.yaml
@@ -4,3 +4,12 @@ storage:
  postgres:
    postgres_url:
      env: DAGSTER_POSTGRES_URL
+    pool_size: 5
+    max_overflow: 5
+
+# Limit concurrent runs to avoid overwhelming the VM and database.
+concurrency:
+  default_op_concurrency_limit: 1
+
+run_queue:
+  max_concurrent_runs: 1