snapshot

.gitignore

@@ -4,5 +4,9 @@ storage/
 .ipynb_checkpoints/
 __pycache__/
 trash/
+src/.telemetry/
+src/history/
+src/logs/
+src/schedules/
 db/
 .DS_Store

Dockerfile.code

@@ -6,6 +6,7 @@ FROM python:3.12-slim
 COPY dagster-requirements.txt requirements.txt
 RUN pip install uv
 RUN uv pip install -r requirements.txt --system
+RUN uv pip install polars-lts-cpu --system
 
 # Add repository code
 WORKDIR /opt/dagster/home

Dockerfile.system

@@ -6,6 +6,7 @@ FROM python:3.12-slim
 COPY dagster-requirements.txt requirements.txt
 RUN pip install uv
 RUN uv pip install -r requirements.txt --system
+RUN uv pip install polars-lts-cpu --system
 
 # Set $DAGSTER_HOME and copy dagster instance and workspace YAML there
 ENV DAGSTER_HOME=/opt/dagster/home/

dagster.yaml

@@ -1,6 +1,10 @@
 telemetry:
   enabled: false
 
+
+concurrency:
+  default_op_concurrency_limit: 2
+
 run_coordinator:
   module: dagster.core.run_coordinator
   class: QueuedRunCoordinator
@@ -19,9 +23,9 @@ run_launcher:
         - /opt/dagster/src/app/:/opt/dagster/home/app/
         - /opt/dagster/src/repo.py:/opt/dagster/home/repo.py
 
-        # - /opt/dagster/storage/:/opt/dagster/home/storage/
         - /opt/dagster/storage/import/:/opt/dagster/home/storage/import/
         - /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/
+
         - /opt/dagster/storage/:/opt/dagster/home/storage/
 
 run_storage:

docker-compose.system.yaml

@@ -17,18 +17,11 @@ x-volumes: &volumes
     - /opt/dagster/workspace.yaml:/opt/dagster/home/workspace.yaml
     - /var/run/docker.sock:/var/run/docker.sock
 
-    # - /opt/dagster/storage/:/opt/dagster/home/storage/
     - /opt/dagster/storage/import/:/opt/dagster/home/storage/import/
     - /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/
 
     - /opt/dagster/src/app/:/opt/dagster/home/app/
     - /opt/dagster/src/repo.py:/opt/dagster/home/repo.py
-    # - /opt/homebrew/Caskroom/mambaforge/base/envs:/opt/homebrew/Caskroom/mambaforge/base/envs
-
-    # Towel
-    # - /opt/dagster/src/towel.py:/opt/dagster/home/towel.py
-    # - /Users/rik/Seafile/Code/company/navara/Klanten/Eneco/towel/towel:/opt/dagster/home/app/towel
-    # - /Users/rik/Library/Caches/pypoetry/virtualenvs/towel-V7mtCF2c-py3.9:/venv/towel
 
 services:
   # This service runs the postgres DB used by dagster for run storage, schedule storage,

workspace.yaml

@@ -1,27 +1,4 @@
 load_from:
    - grpc_server:
-      location_name: example
       host: user_code
       port: 4000
-
-#   - python_file:
-#      location_name: local
-#      relative_path: repo.py
-#      working_directory: /opt/dagster/home
-
-#   - grpc_server:
-#      location_name: other
-#      host: other_image
-#      port: 4000
-
-#   - python_file:
-#      location_name: towel
-#      relative_path: towel.py
-#      working_directory: /opt/dagster/home
-#      executable_path: /venv/towel/bin/python
-
-#   - python_file:
-#       location_name: alliander
-#       relative_path: towel.py
-#       working_directory: /opt/dagster/home
-#       executable_path: /venv/alliander/bin/python