use envsubst to configure stuff in dagster.yaml

Dockerfile.system

@@ -1,8 +1,11 @@
+FROM python:3.12-slim
+
+# Install envsubst
+RUN apt-get update && apt-get install -y gettext
+
 # Dagster libraries to run both dagit and the dagster-daemon. Does not
 # need to have access to any pipeline code.
 
-FROM python:3.12-slim
-
 COPY dagster-requirements.txt requirements.txt
 
 RUN pip install uv
@@ -14,6 +17,20 @@ ENV DAGSTER_HOME=/opt/dagster/home/
 
 RUN mkdir -p $DAGSTER_HOME
 
-COPY dagster.yaml workspace.yaml $DAGSTER_HOME
+# COPY dagster.yaml workspace.yaml $DAGSTER_HOME
+
+# Create entrypoint that renders the dagster.yaml from a template
+RUN cat << 'EOF' > /entrypoint.sh
+#!/bin/sh
+set -e
+
+echo "Rendering dagster.yaml from template..."
+envsubst < dagster.yaml.template > dagster.yaml
+
+echo "Starting Dagster: $@"
+exec "$@"
+EOF
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
 
 WORKDIR $DAGSTER_HOME

compose.system.yaml

@@ -4,8 +4,10 @@ x-postgres-env: &postgres_env
   POSTGRES_USER: ${POSTGRES_USER}
   POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
   POSTGRES_DB: ${POSTGRES_DB}
-x-dagster-env: &dagster_env
+x-system-env: &system_env
   TZ: Europe/Amsterdam
+  CACHE_DIR: /tmp/cache
+x-dagster-env: &dagster_env
   DAGSTER_POSTGRES_HOST: ${POSTGRES_HOST}
   DAGSTER_POSTGRES_PORT: ${POSTGRES_PORT}
   DAGSTER_POSTGRES_USER: ${POSTGRES_USER}
@@ -21,7 +23,7 @@ x-email-env: &email_env
 
 x-volumes: &volumes
   volumes:
-    - /opt/dagster/dagster.yaml:/opt/dagster/home/dagster.yaml:ro
+    - /opt/dagster/dagster.yaml:/opt/dagster/home/dagster.yaml.template:ro
     - /opt/dagster/workspace.yaml:/opt/dagster/home/workspace.yaml:ro
     - /opt/dagster/storage/:/storage/:rw
     - /opt/dagster/logs/:/logs:rw
@@ -47,7 +49,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.system
-    entrypoint:
+    command:
       - dagster-webserver
       - -h
       - "0.0.0.0"
@@ -74,13 +76,13 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.system
-    entrypoint:
+    command:
       - dagster-daemon
       - run
     container_name: dagster-daemon
     restart: on-failure
     environment:
-      <<: [ *dagster_env, *email_env ]
+      <<: [ *system_env, *dagster_env, *email_env ]
     <<: *volumes
     networks:
       - dagster

dagster.yaml

@@ -31,7 +31,7 @@ run_launcher:
         - /opt/dagster/apps/:/apps:ro
         - /opt/dagster/storage/:/storage/:rw
         - /opt/dagster/logs/:/logs:rw
-        - /opt/dagster/cache/:/cache:rw
+        - ${CACHE_DIR}:/cache:rw
 
 run_storage:
   module: dagster_postgres.run_storage