diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d552a07..10aec82 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: rev: 25.1.0 hooks: - id: black-jupyter - language_version: python3.11 + language_version: python3.12 - repo: https://github.com/pycqa/isort rev: 6.0.1 hooks: @@ -26,7 +26,7 @@ repos: rev: v0.11.13 hooks: - id: ruff - args: [ "--target-version=py311" ] + args: [ "--target-version=py312" ] - repo: https://github.com/kynan/nbstripout rev: 0.8.1 hooks: diff --git a/Dockerfile.code b/Dockerfile.code index b61f987..2cc8ce7 100644 --- a/Dockerfile.code +++ b/Dockerfile.code @@ -10,6 +10,7 @@ RUN uv pip install -r requirements.txt --system RUN uv pip install polars-lts-cpu --system ARG APP +ENV APP=$APP ENV PYTHONPATH=/apps/$APP/src/:/shared/src/ WORKDIR /opt/dagster/home diff --git a/apps/stocks/src/__init__.py b/apps/stocks/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/stocks/src/assets.py b/apps/stocks/src/assets.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/stocks/src/definitions.py b/apps/stocks/src/definitions.py new file mode 100644 index 0000000..7e0e20b --- /dev/null +++ b/apps/stocks/src/definitions.py @@ -0,0 +1,27 @@ +import os + +import assets +from dagster_polars import PolarsParquetIOManager +from icecream import install + +import dagster as dg +from dagster import load_assets_from_modules + +install() + +APP = os.environ["APP"] + +definitions = dg.Definitions( + assets=[ + asset.with_attributes( + group_names_by_key={asset.key: APP}, + tags_by_key={asset.key: {"app": APP}}, + ) + for asset in load_assets_from_modules([assets]) + ], + resources={ + "polars_parquet_io_manager": PolarsParquetIOManager( + base_dir=os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}" + ), + }, +) diff --git a/apps/tesla/src/__init__.py b/apps/tesla/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/tesla/src/assets.py b/apps/tesla/src/assets.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/tesla/src/definitions.py b/apps/tesla/src/definitions.py new file mode 100644 index 0000000..7e0e20b --- /dev/null +++ b/apps/tesla/src/definitions.py @@ -0,0 +1,27 @@ +import os + +import assets +from dagster_polars import PolarsParquetIOManager +from icecream import install + +import dagster as dg +from dagster import load_assets_from_modules + +install() + +APP = os.environ["APP"] + +definitions = dg.Definitions( + assets=[ + asset.with_attributes( + group_names_by_key={asset.key: APP}, + tags_by_key={asset.key: {"app": APP}}, + ) + for asset in load_assets_from_modules([assets]) + ], + resources={ + "polars_parquet_io_manager": PolarsParquetIOManager( + base_dir=os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}" + ), + }, +) diff --git a/apps/vinyl/src/definitions.py b/apps/vinyl/src/definitions.py index ba00d5e..2e79eaf 100644 --- a/apps/vinyl/src/definitions.py +++ b/apps/vinyl/src/definitions.py @@ -9,18 +9,20 @@ from utils.email import EmailService import dagster as dg +APP = os.environ["APP"] + install() definitions = dg.Definitions( assets=[ asset.with_attributes( - group_names_by_key={asset.key: "vinyl"}, - tags_by_key={asset.key: {"app": "vinyl"}}, + group_names_by_key={asset.key: APP}, + tags_by_key={asset.key: {"app": APP}}, ) for asset in dg.load_assets_from_modules([assets]) ], resources={ "polars_parquet_io_manager": PolarsParquetIOManager( - base_dir=os.environ.get("STORAGE_DIR", "/storage") + "/vinyl" + base_dir=os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}" ), "email_service": EmailService( smtp_server=dg.EnvVar("SMTP_SERVER"), diff --git a/apps/weather/src/__init__.py b/apps/weather/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/weather/src/assets.py b/apps/weather/src/assets.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/weather/src/definitions.py b/apps/weather/src/definitions.py new file mode 100644 index 0000000..7e0e20b --- /dev/null +++ b/apps/weather/src/definitions.py @@ -0,0 +1,27 @@ +import os + +import assets +from dagster_polars import PolarsParquetIOManager +from icecream import install + +import dagster as dg +from dagster import load_assets_from_modules + +install() + +APP = os.environ["APP"] + +definitions = dg.Definitions( + assets=[ + asset.with_attributes( + group_names_by_key={asset.key: APP}, + tags_by_key={asset.key: {"app": APP}}, + ) + for asset in load_assets_from_modules([assets]) + ], + resources={ + "polars_parquet_io_manager": PolarsParquetIOManager( + base_dir=os.environ.get("STORAGE_DIR", "/storage") + f"/{APP}" + ), + }, +) diff --git a/compose.code.yaml b/compose.code.yaml index 949d10e..3bbf892 100644 --- a/compose.code.yaml +++ b/compose.code.yaml @@ -19,18 +19,18 @@ services: # run launcher to use this same image when launching runs in a new container as well. # Multiple containers like this can be deployed separately - each just needs to run on # its own port, and have its own entry in the workspace.yaml file that's loaded by dagit. - user_code_vinyl: + dagster-code-vinyl: build: context: apps/vinyl dockerfile: ../../Dockerfile.code args: - APP=vinyl - container_name: user_code_vinyl - image: user_code_vinyl + container_name: dagster-code-vinyl + image: dagster-code-vinyl restart: always environment: <<: [ *dagster_env, *email_env ] - DAGSTER_CURRENT_IMAGE: user_code_vinyl + DAGSTER_CURRENT_IMAGE: dagster-code-vinyl volumes: - /opt/dagster/apps/:/apps/:ro - /opt/dagster/shared/:/shared/:ro @@ -40,21 +40,78 @@ services: networks: - dagster - user_code_other: - # profiles: [ disabled ] + dagster-code-stocks: + build: + context: apps/stocks + dockerfile: ../../Dockerfile.code + args: + - APP=stocks + container_name: dagster-code-stocks + image: dagster-code-stocks + restart: always + environment: + <<: [ *dagster_env ] + DAGSTER_CURRENT_IMAGE: dagster-code-stocks + volumes: + - /opt/dagster/apps/:/apps/:ro + - /opt/dagster/shared/:/shared/:ro + - /opt/dagster/logs/:/logs:rw + networks: + - dagster + + dagster-code-tesla: + build: + context: apps/tesla + dockerfile: ../../Dockerfile.code + args: + - APP=tesla + container_name: dagster-code-tesla + image: dagster-code-tesla + restart: always + environment: + <<: [ *dagster_env ] + DAGSTER_CURRENT_IMAGE: dagster-code-tesla + volumes: + - /opt/dagster/apps/:/apps/:ro + - /opt/dagster/shared/:/shared/:ro + - /opt/dagster/logs/:/logs:rw + networks: + - dagster + + dagster-code-weather: + build: + context: apps/weather + dockerfile: ../../Dockerfile.code + args: + - APP=weather + container_name: dagster-code-weather + image: dagster-code-weather + restart: always + environment: + <<: [ *dagster_env ] + DAGSTER_CURRENT_IMAGE: dagster-code-weather + volumes: + - /opt/dagster/apps/:/apps/:ro + - /opt/dagster/shared/:/shared/:ro + - /opt/dagster/logs/:/logs:rw + networks: + - dagster + + dagster-code-other: build: context: apps/other dockerfile: ../../Dockerfile.code args: - APP=other - container_name: user_code_other - image: user_code_other + container_name: dagster-code-other + image: dagster-code-other restart: always environment: <<: [ *dagster_env ] - DAGSTER_CURRENT_IMAGE: user_code_other + DAGSTER_CURRENT_IMAGE: dagster-code-other volumes: - - /opt/dagster/apps/:/apps:ro + - /opt/dagster/apps/:/apps/:ro + - /opt/dagster/shared/:/shared/:ro - /opt/dagster/logs/:/logs:rw networks: - dagster diff --git a/compose.system.yaml b/compose.system.yaml index 8d829b0..9dc3d03 100644 --- a/compose.system.yaml +++ b/compose.system.yaml @@ -23,9 +23,9 @@ x-volumes: &volumes services: # This service runs the postgres DB used by dagster for run storage, schedule storage, # and event log storage. - postgresql: + dagster-postgres: image: postgres:11 - container_name: postgresql + container_name: dagster-postgres environment: <<: *postgres_env networks: @@ -36,7 +36,7 @@ services: # This service runs dagit, which loads your user code from the user code container. # Since our instance uses the QueuedRunCoordinator, any runs submitted from dagit will be put on # a queue and later dequeued and launched by dagster-daemon. - dagit: + dagster-dagit: build: context: . dockerfile: Dockerfile.system @@ -48,7 +48,7 @@ services: - "3000" - -w - workspace.yaml - container_name: dagit + container_name: dagster-dagit expose: - "3000" ports: @@ -59,18 +59,18 @@ services: networks: - dagster depends_on: - - postgresql + - dagster-postgres # This service runs the dagster-daemon process, which is responsible for taking runs # off of the queue and launching them, as well as creating runs from schedules or sensors. - daemon: + dagster-daemon: build: context: . dockerfile: Dockerfile.system entrypoint: - dagster-daemon - run - container_name: daemon + container_name: dagster-daemon restart: on-failure environment: <<: *dagster_env @@ -78,4 +78,4 @@ services: networks: - dagster depends_on: - - postgresql + - dagster-postgres diff --git a/pyproject.toml b/pyproject.toml index 7a26a35..ce8ff78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,11 @@ dagster = [ "dagit" ] vinyl = [] +stocks = [ + "selenium" +] +tesla = [] +weather = [] other = [] unknown = [ "fastapi", diff --git a/requirements.sh b/requirements.sh index f2a1151..12fe8ca 100755 --- a/requirements.sh +++ b/requirements.sh @@ -2,4 +2,7 @@ uv pip compile pyproject.toml > requirements.txt uv pip compile pyproject.toml --extra=dagster > dagster-requirements.txt uv pip compile pyproject.toml --extra=dagster --extra=dev > dev-requirements.txt uv pip compile pyproject.toml --extra=dagster --extra=vinyl > apps/vinyl/requirements.txt +uv pip compile pyproject.toml --extra=dagster --extra=stocks > apps/stocks/requirements.txt +uv pip compile pyproject.toml --extra=dagster --extra=tesla > apps/tesla/requirements.txt +uv pip compile pyproject.toml --extra=dagster --extra=weather > apps/weather/requirements.txt uv pip compile pyproject.toml --extra=dagster --extra=other > apps/other/requirements.txt diff --git a/workspace.yaml b/workspace.yaml index fc8db27..c937a53 100644 --- a/workspace.yaml +++ b/workspace.yaml @@ -1,7 +1,16 @@ load_from: - grpc_server: - host: user_code_vinyl + host: dagster-code-vinyl port: 4000 - grpc_server: - host: user_code_other + host: dagster-code-stocks + port: 4000 + - grpc_server: + host: dagster-code-tesla + port: 4000 + - grpc_server: + host: dagster-code-weather + port: 4000 + - grpc_server: + host: dagster-code-other port: 4000