add backup repo to dagster and docker files
This commit is contained in:
@@ -64,12 +64,21 @@ services:
|
|||||||
dagster-code-stocks-playwright:
|
dagster-code-stocks-playwright:
|
||||||
build:
|
build:
|
||||||
context: apps/stocks
|
context: apps/stocks
|
||||||
dockerfile: ../../Dockerfile.code.playwright
|
dockerfile: Dockerfile.code.playwright
|
||||||
args:
|
args:
|
||||||
- APP=stocks
|
- APP=stocks
|
||||||
image: dagster-code-stocks-playwright
|
image: dagster-code-stocks-playwright
|
||||||
profiles: [ "never" ]
|
profiles: [ "never" ]
|
||||||
|
|
||||||
|
dagster-code-backup-base:
|
||||||
|
build:
|
||||||
|
context: apps/backup
|
||||||
|
dockerfile: ../../Dockerfile.code
|
||||||
|
args:
|
||||||
|
- APP=backup
|
||||||
|
image: dagster-code-backup-base
|
||||||
|
profiles: [ "never" ]
|
||||||
|
|
||||||
dagster-code-tesla:
|
dagster-code-tesla:
|
||||||
build:
|
build:
|
||||||
context: apps/tesla
|
context: apps/tesla
|
||||||
@@ -108,6 +117,22 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- dagster
|
- dagster
|
||||||
|
|
||||||
|
dagster-code-backup:
|
||||||
|
build:
|
||||||
|
context: apps/backup
|
||||||
|
container_name: dagster-code-backup
|
||||||
|
image: dagster-code-backup
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
<<: [ *dagster_env ]
|
||||||
|
DAGSTER_CURRENT_IMAGE: dagster-code-backup
|
||||||
|
volumes:
|
||||||
|
- /opt/dagster/apps/:/code/apps/:ro
|
||||||
|
- /opt/dagster/shared/:/code/shared/:ro
|
||||||
|
- /opt/dagster/logs/:/logs:rw
|
||||||
|
networks:
|
||||||
|
- dagster
|
||||||
|
|
||||||
dagster-code-other:
|
dagster-code-other:
|
||||||
build:
|
build:
|
||||||
context: apps/other
|
context: apps/other
|
||||||
|
|||||||
@@ -40,6 +40,8 @@ services:
|
|||||||
<<: *postgres_env
|
<<: *postgres_env
|
||||||
networks:
|
networks:
|
||||||
- dagster
|
- dagster
|
||||||
|
ports:
|
||||||
|
- '15432:5432'
|
||||||
volumes:
|
volumes:
|
||||||
- /opt/dagster/db/:/var/lib/postgresql/data/
|
- /opt/dagster/db/:/var/lib/postgresql/data/
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ run_launcher:
|
|||||||
class: CustomDockerRunLauncher
|
class: CustomDockerRunLauncher
|
||||||
config:
|
config:
|
||||||
env_vars:
|
env_vars:
|
||||||
|
- TZ
|
||||||
- DAGSTER_POSTGRES_HOST
|
- DAGSTER_POSTGRES_HOST
|
||||||
- DAGSTER_POSTGRES_PORT
|
- DAGSTER_POSTGRES_PORT
|
||||||
- DAGSTER_POSTGRES_USER
|
- DAGSTER_POSTGRES_USER
|
||||||
|
|||||||
@@ -65,6 +65,9 @@ weather = [
|
|||||||
"requests_cache",
|
"requests_cache",
|
||||||
"retry_requests"
|
"retry_requests"
|
||||||
]
|
]
|
||||||
|
backup = [
|
||||||
|
"paramiko"
|
||||||
|
]
|
||||||
other = [
|
other = [
|
||||||
# "deltalake>=1.0.0",
|
# "deltalake>=1.0.0",
|
||||||
# "dagster-deltalake-pandas",
|
# "dagster-deltalake-pandas",
|
||||||
|
|||||||
@@ -5,4 +5,5 @@ uv pip compile pyproject.toml --extra=dagster --extra=vinyl > apps/vinyl/require
|
|||||||
uv pip compile pyproject.toml --extra=dagster --extra=stocks > apps/stocks/requirements.txt
|
uv pip compile pyproject.toml --extra=dagster --extra=stocks > apps/stocks/requirements.txt
|
||||||
uv pip compile pyproject.toml --extra=dagster --extra=tesla > apps/tesla/requirements.txt
|
uv pip compile pyproject.toml --extra=dagster --extra=tesla > apps/tesla/requirements.txt
|
||||||
uv pip compile pyproject.toml --extra=dagster --extra=weather > apps/weather/requirements.txt
|
uv pip compile pyproject.toml --extra=dagster --extra=weather > apps/weather/requirements.txt
|
||||||
|
uv pip compile pyproject.toml --extra=dagster --extra=backup > apps/backup/requirements.txt
|
||||||
uv pip compile pyproject.toml --extra=dagster --extra=other > apps/other/requirements.txt
|
uv pip compile pyproject.toml --extra=dagster --extra=other > apps/other/requirements.txt
|
||||||
|
|||||||
@@ -15,6 +15,10 @@ load_from:
|
|||||||
location_name: weather
|
location_name: weather
|
||||||
host: dagster-code-weather
|
host: dagster-code-weather
|
||||||
port: 4000
|
port: 4000
|
||||||
|
- grpc_server:
|
||||||
|
location_name: backup
|
||||||
|
host: dagster-code-backup
|
||||||
|
port: 4000
|
||||||
- grpc_server:
|
- grpc_server:
|
||||||
location_name: other
|
location_name: other
|
||||||
host: dagster-code-other
|
host: dagster-code-other
|
||||||
|
|||||||
Reference in New Issue
Block a user