95 lines
2.6 KiB
YAML
95 lines
2.6 KiB
YAML
x-postgres-env: &postgres_env
|
|
DAGSTER_POSTGRES_HOST: ${POSTGRES_HOST}
|
|
DAGSTER_POSTGRES_PORT: ${POSTGRES_PORT}
|
|
POSTGRES_USER: ${POSTGRES_USER}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
|
POSTGRES_DB: ${POSTGRES_DB}
|
|
x-system-env: &system_env
|
|
TZ: Europe/Amsterdam
|
|
DATA_DIR: ${DATA_DIR}
|
|
CACHE_DIR: /tmp/cache
|
|
x-dagster-env: &dagster_env
|
|
DAGSTER_POSTGRES_HOST: ${POSTGRES_HOST}
|
|
DAGSTER_POSTGRES_PORT: ${POSTGRES_PORT}
|
|
DAGSTER_POSTGRES_USER: ${POSTGRES_USER}
|
|
DAGSTER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
|
DAGSTER_POSTGRES_DB: ${POSTGRES_DB}
|
|
x-email-env: &email_env
|
|
SMTP_SERVER: ${SMTP_SERVER}
|
|
SMTP_PORT: ${SMTP_PORT}
|
|
SMTP_USERNAME: ${SMTP_USERNAME}
|
|
SMTP_PASSWORD: ${SMTP_PASSWORD}
|
|
SENDER_EMAIL: ${SENDER_EMAIL}
|
|
RECEIVER_EMAIL: ${RECEIVER_EMAIL}
|
|
|
|
x-volumes: &volumes
|
|
volumes:
|
|
- /opt/dagster/dagster.yaml:/opt/dagster/home/dagster.yaml.template:ro
|
|
- /opt/dagster/workspace.yaml:/opt/dagster/home/workspace.yaml:ro
|
|
- /opt/dagster/system/:/code/system/:ro
|
|
- /mnt/mezzo/scratch/dagster/:/storage/:rw
|
|
- /opt/dagster/logs/:/logs:rw
|
|
- /var/run/docker.sock:/var/run/docker.sock:rw
|
|
|
|
services:
|
|
# This service runs the postgres DB used by dagster for run storage, schedule storage,
|
|
# and event log storage.
|
|
dagster-postgres:
|
|
image: postgres:11
|
|
container_name: dagster-postgres
|
|
environment:
|
|
<<: *postgres_env
|
|
networks:
|
|
- dagster
|
|
ports:
|
|
- '25432:5432'
|
|
volumes:
|
|
- /opt/dagster/db/:/var/lib/postgresql/data/
|
|
|
|
# This service runs dagit, which loads your user code from the user code container.
|
|
# Since our instance uses the QueuedRunCoordinator, any runs submitted from dagit will be put on
|
|
# a queue and later dequeued and launched by dagster-daemon.
|
|
dagster-dagit:
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile.system
|
|
command:
|
|
- dagster-webserver
|
|
- -h
|
|
- "0.0.0.0"
|
|
- -p
|
|
- "3000"
|
|
- -w
|
|
- workspace.yaml
|
|
container_name: dagster-dagit
|
|
expose:
|
|
- "3000"
|
|
ports:
|
|
- "3000:3000"
|
|
environment:
|
|
<<: *dagster_env
|
|
<<: *volumes
|
|
networks:
|
|
- dagster
|
|
depends_on:
|
|
- dagster-postgres
|
|
|
|
# This service runs the dagster-daemon process, which is responsible for taking runs
|
|
# off of the queue and launching them, as well as creating runs from schedules or sensors.
|
|
dagster-daemon:
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile.system
|
|
command:
|
|
- dagster-daemon
|
|
- run
|
|
container_name: dagster-daemon
|
|
restart: on-failure
|
|
environment:
|
|
<<: [ *system_env, *dagster_env, *email_env ]
|
|
<<: *volumes
|
|
networks:
|
|
- dagster
|
|
depends_on:
|
|
- dagster-postgres
|