add backup repo to dagster and docker files

This commit is contained in:
2025-08-22 09:42:19 +02:00
parent bf537c86a4
commit 316fe03be9
6 changed files with 37 additions and 1 deletions

View File

@@ -64,12 +64,21 @@ services:
dagster-code-stocks-playwright: dagster-code-stocks-playwright:
build: build:
context: apps/stocks context: apps/stocks
dockerfile: ../../Dockerfile.code.playwright dockerfile: Dockerfile.code.playwright
args: args:
- APP=stocks - APP=stocks
image: dagster-code-stocks-playwright image: dagster-code-stocks-playwright
profiles: [ "never" ] profiles: [ "never" ]
dagster-code-backup-base:
build:
context: apps/backup
dockerfile: ../../Dockerfile.code
args:
- APP=backup
image: dagster-code-backup-base
profiles: [ "never" ]
dagster-code-tesla: dagster-code-tesla:
build: build:
context: apps/tesla context: apps/tesla
@@ -108,6 +117,22 @@ services:
networks: networks:
- dagster - dagster
dagster-code-backup:
build:
context: apps/backup
container_name: dagster-code-backup
image: dagster-code-backup
restart: always
environment:
<<: [ *dagster_env ]
DAGSTER_CURRENT_IMAGE: dagster-code-backup
volumes:
- /opt/dagster/apps/:/code/apps/:ro
- /opt/dagster/shared/:/code/shared/:ro
- /opt/dagster/logs/:/logs:rw
networks:
- dagster
dagster-code-other: dagster-code-other:
build: build:
context: apps/other context: apps/other

View File

@@ -40,6 +40,8 @@ services:
<<: *postgres_env <<: *postgres_env
networks: networks:
- dagster - dagster
ports:
- '15432:5432'
volumes: volumes:
- /opt/dagster/db/:/var/lib/postgresql/data/ - /opt/dagster/db/:/var/lib/postgresql/data/

View File

@@ -15,6 +15,7 @@ run_launcher:
class: CustomDockerRunLauncher class: CustomDockerRunLauncher
config: config:
env_vars: env_vars:
- TZ
- DAGSTER_POSTGRES_HOST - DAGSTER_POSTGRES_HOST
- DAGSTER_POSTGRES_PORT - DAGSTER_POSTGRES_PORT
- DAGSTER_POSTGRES_USER - DAGSTER_POSTGRES_USER

View File

@@ -65,6 +65,9 @@ weather = [
"requests_cache", "requests_cache",
"retry_requests" "retry_requests"
] ]
backup = [
"paramiko"
]
other = [ other = [
# "deltalake>=1.0.0", # "deltalake>=1.0.0",
# "dagster-deltalake-pandas", # "dagster-deltalake-pandas",

View File

@@ -5,4 +5,5 @@ uv pip compile pyproject.toml --extra=dagster --extra=vinyl > apps/vinyl/require
uv pip compile pyproject.toml --extra=dagster --extra=stocks > apps/stocks/requirements.txt uv pip compile pyproject.toml --extra=dagster --extra=stocks > apps/stocks/requirements.txt
uv pip compile pyproject.toml --extra=dagster --extra=tesla > apps/tesla/requirements.txt uv pip compile pyproject.toml --extra=dagster --extra=tesla > apps/tesla/requirements.txt
uv pip compile pyproject.toml --extra=dagster --extra=weather > apps/weather/requirements.txt uv pip compile pyproject.toml --extra=dagster --extra=weather > apps/weather/requirements.txt
uv pip compile pyproject.toml --extra=dagster --extra=backup > apps/backup/requirements.txt
uv pip compile pyproject.toml --extra=dagster --extra=other > apps/other/requirements.txt uv pip compile pyproject.toml --extra=dagster --extra=other > apps/other/requirements.txt

View File

@@ -15,6 +15,10 @@ load_from:
location_name: weather location_name: weather
host: dagster-code-weather host: dagster-code-weather
port: 4000 port: 4000
- grpc_server:
location_name: backup
host: dagster-code-backup
port: 4000
- grpc_server: - grpc_server:
location_name: other location_name: other
host: dagster-code-other host: dagster-code-other