From fd73e1367c68a52bc493ada35c8a17d6f6f82b01 Mon Sep 17 00:00:00 2001 From: Rik Veenboer Date: Sun, 20 Jul 2025 19:49:30 +0200 Subject: [PATCH] refactor to allow for multiple code locations --- .dockerignore | 13 + Dockerfile.code | 6 +- Dockerfile.system | 1 + __init__.py => apps/other/__init__.py | 0 {src/app => apps/other/src}/__init__.py | 0 .../partitions => apps/other/src}/assets.py | 0 .../repo.py => apps/other/src/definitions.py | 5 +- .../partitions => apps/other/src}/mapping.py | 0 {src/app/partitions => apps/other}/test.py | 7 +- apps/test/__init__.py | 3 + {src/app => apps/test}/sync.sh | 0 {src/app => apps/test}/test.py | 0 {src => apps/vinyl}/__init__.py | 0 apps/vinyl/src/__init__.py | 3 + {src/app/vinyl => apps/vinyl/src}/assets.py | 6 +- .../repo.py => apps/vinyl/src/definitions.py | 11 +- {src/app/vinyl => apps/vinyl/src}/jobs.py | 4 +- .../vinyl/src/plato}/__init__.py | 0 .../vinyl => apps/vinyl/src}/plato/deals.py | 0 .../vinyl => apps/vinyl/src}/plato/fetch.py | 0 .../vinyl => apps/vinyl/src}/plato/scrape.py | 0 .../vinyl => apps/vinyl/src}/plato/utils.py | 0 .../app/vinyl => apps/vinyl/src}/schedules.py | 4 +- {src/app/vinyl => apps/vinyl/src}/sensors.py | 5 +- .../vinyl/src/sounds}/__init__.py | 0 .../vinyl => apps/vinyl/src}/sounds/deals.py | 2 +- .../vinyl => apps/vinyl/src}/sounds/fetch.py | 0 {src/app/vinyl => apps/vinyl/src}/test.py | 9 +- {src/app/vinyl => apps/vinyl/src}/utils.py | 0 compose.code.yaml | 44 +- compose.system.yaml | 1 - dagster-requirements.txt | 398 ------------------ dagster.yaml | 7 +- pyproject.toml | 51 +-- requirements.sh | 5 + requirements.txt | 196 +++------ src/app/vinyl/plato/__init__.py | 0 src/app/vinyl/sounds/__init__.py | 0 src/repo.py | 1 - workspace.yaml | 7 +- 40 files changed, 161 insertions(+), 628 deletions(-) create mode 100644 .dockerignore rename __init__.py => apps/other/__init__.py (100%) rename {src/app => apps/other/src}/__init__.py (100%) rename {src/app/partitions => apps/other/src}/assets.py (100%) rename src/app/partitions/repo.py => apps/other/src/definitions.py (78%) rename {src/app/partitions => apps/other/src}/mapping.py (100%) rename {src/app/partitions => apps/other}/test.py (90%) create mode 100644 apps/test/__init__.py rename {src/app => apps/test}/sync.sh (100%) rename {src/app => apps/test}/test.py (100%) rename {src => apps/vinyl}/__init__.py (100%) create mode 100644 apps/vinyl/src/__init__.py rename {src/app/vinyl => apps/vinyl/src}/assets.py (97%) rename src/app/vinyl/repo.py => apps/vinyl/src/definitions.py (77%) rename {src/app/vinyl => apps/vinyl/src}/jobs.py (97%) rename {src/app/partitions => apps/vinyl/src/plato}/__init__.py (100%) rename {src/app/vinyl => apps/vinyl/src}/plato/deals.py (100%) rename {src/app/vinyl => apps/vinyl/src}/plato/fetch.py (100%) rename {src/app/vinyl => apps/vinyl/src}/plato/scrape.py (100%) rename {src/app/vinyl => apps/vinyl/src}/plato/utils.py (100%) rename {src/app/vinyl => apps/vinyl/src}/schedules.py (87%) rename {src/app/vinyl => apps/vinyl/src}/sensors.py (84%) rename {src/app/vinyl => apps/vinyl/src/sounds}/__init__.py (100%) rename {src/app/vinyl => apps/vinyl/src}/sounds/deals.py (98%) rename {src/app/vinyl => apps/vinyl/src}/sounds/fetch.py (100%) rename {src/app/vinyl => apps/vinyl/src}/test.py (91%) rename {src/app/vinyl => apps/vinyl/src}/utils.py (100%) delete mode 100644 dagster-requirements.txt create mode 100755 requirements.sh delete mode 100644 src/app/vinyl/plato/__init__.py delete mode 100644 src/app/vinyl/sounds/__init__.py delete mode 100644 src/repo.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..42fa855 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +Dockerfile.code +Dockerfile.system +Makefile +__pycache__ +build +compose*.yaml +db +poetry.lock +pyproject.toml +requirements.sh +src +storage +trash diff --git a/Dockerfile.code b/Dockerfile.code index ac97ab0..401b7e8 100644 --- a/Dockerfile.code +++ b/Dockerfile.code @@ -3,7 +3,8 @@ FROM python:3.12-slim # Checkout and install dagster libraries needed to run the gRPC server # exposing your repository to dagit and dagster-daemon, and to load the DagsterInstance -COPY dagster-requirements.txt requirements.txt +COPY requirements.txt . + RUN pip install uv RUN uv pip install -r requirements.txt --system RUN uv pip install polars-lts-cpu --system @@ -16,5 +17,4 @@ EXPOSE 4000 # CMD allows this to be overridden from run launchers or executors that want # to run other commands against your repository -#CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-f", "repo.py"] -CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-f", "repo.py"] +CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-m", "app.definitions"] diff --git a/Dockerfile.system b/Dockerfile.system index a23e6ff..2cad6ea 100644 --- a/Dockerfile.system +++ b/Dockerfile.system @@ -4,6 +4,7 @@ FROM python:3.12-slim COPY dagster-requirements.txt requirements.txt + RUN pip install uv RUN uv pip install -r requirements.txt --system RUN uv pip install polars-lts-cpu --system diff --git a/__init__.py b/apps/other/__init__.py similarity index 100% rename from __init__.py rename to apps/other/__init__.py diff --git a/src/app/__init__.py b/apps/other/src/__init__.py similarity index 100% rename from src/app/__init__.py rename to apps/other/src/__init__.py diff --git a/src/app/partitions/assets.py b/apps/other/src/assets.py similarity index 100% rename from src/app/partitions/assets.py rename to apps/other/src/assets.py diff --git a/src/app/partitions/repo.py b/apps/other/src/definitions.py similarity index 78% rename from src/app/partitions/repo.py rename to apps/other/src/definitions.py index c94e3c4..3360257 100644 --- a/src/app/partitions/repo.py +++ b/apps/other/src/definitions.py @@ -1,13 +1,12 @@ +from assets import asset_multi_1, asset_multi_2, asset_single_1, asset_single_2 from dagster_polars import PolarsParquetIOManager from dagster import Definitions, define_asset_job -from .assets import asset_multi_1, asset_multi_2, asset_single_1, asset_single_2 - # Define a job that includes both assets daily_job = define_asset_job("daily_job", selection=[asset_multi_1, asset_multi_2]) -vinyl = Definitions( +definitions = Definitions( assets=[asset_single_1, asset_multi_1, asset_single_2, asset_multi_2], resources={"polars_parquet_io_manager": PolarsParquetIOManager()}, jobs=[daily_job], diff --git a/src/app/partitions/mapping.py b/apps/other/src/mapping.py similarity index 100% rename from src/app/partitions/mapping.py rename to apps/other/src/mapping.py diff --git a/src/app/partitions/test.py b/apps/other/test.py similarity index 90% rename from src/app/partitions/test.py rename to apps/other/test.py index e9ea00a..307ee4d 100644 --- a/src/app/partitions/test.py +++ b/apps/other/test.py @@ -1,11 +1,6 @@ +from assets import asset_multi_1, asset_multi_2, asset_single_1, asset_single_2 from dagster_polars import PolarsParquetIOManager -from app.vinyl.assets import ( - asset_multi_1, - asset_multi_2, - asset_single_1, - asset_single_2, -) from dagster import materialize resources = { diff --git a/apps/test/__init__.py b/apps/test/__init__.py new file mode 100644 index 0000000..ef37bac --- /dev/null +++ b/apps/test/__init__.py @@ -0,0 +1,3 @@ +from icecream import install + +install() diff --git a/src/app/sync.sh b/apps/test/sync.sh similarity index 100% rename from src/app/sync.sh rename to apps/test/sync.sh diff --git a/src/app/test.py b/apps/test/test.py similarity index 100% rename from src/app/test.py rename to apps/test/test.py diff --git a/src/__init__.py b/apps/vinyl/__init__.py similarity index 100% rename from src/__init__.py rename to apps/vinyl/__init__.py diff --git a/apps/vinyl/src/__init__.py b/apps/vinyl/src/__init__.py new file mode 100644 index 0000000..ef37bac --- /dev/null +++ b/apps/vinyl/src/__init__.py @@ -0,0 +1,3 @@ +from icecream import install + +install() diff --git a/src/app/vinyl/assets.py b/apps/vinyl/src/assets.py similarity index 97% rename from src/app/vinyl/assets.py rename to apps/vinyl/src/assets.py index b8051f7..ec027d7 100644 --- a/src/app/vinyl/assets.py +++ b/apps/vinyl/src/assets.py @@ -5,10 +5,10 @@ import duckdb import polars as pl import structlog from duckdb.typing import DATE, VARCHAR +from plato.fetch import scrape_plato +from sounds.fetch import fetch_deals +from utils import parse_date -from app.vinyl.plato.fetch import scrape_plato -from app.vinyl.sounds.fetch import fetch_deals -from app.vinyl.utils import parse_date from dagster import ( DailyPartitionsDefinition, DimensionPartitionMapping, diff --git a/src/app/vinyl/repo.py b/apps/vinyl/src/definitions.py similarity index 77% rename from src/app/vinyl/repo.py rename to apps/vinyl/src/definitions.py index 5eb239b..15d55ad 100644 --- a/src/app/vinyl/repo.py +++ b/apps/vinyl/src/definitions.py @@ -1,17 +1,16 @@ from collections.abc import Sequence +from assets import deals, new_deals, works from dagster_duckdb import DuckDBIOManager from dagster_duckdb.io_manager import DbTypeHandler from dagster_duckdb_pandas import DuckDBPandasTypeHandler from dagster_polars import PolarsParquetIOManager +from jobs import check_partititions_job, deals_job, musicbrainz_lookup_job +from schedules import deals_schedule +from sensors import musicbrainz_lookup_sensor from dagster import Definitions -from .assets import deals, new_deals, works -from .jobs import check_partititions_job, deals_job, musicbrainz_lookup_job -from .schedules import deals_schedule -from .sensors import musicbrainz_lookup_sensor - class PandasDuckDBIOManager(DuckDBIOManager): @staticmethod @@ -19,7 +18,7 @@ class PandasDuckDBIOManager(DuckDBIOManager): return [DuckDBPandasTypeHandler()] -vinyl = Definitions( +definitions = Definitions( assets=[deals, new_deals, works], resources={ "polars_parquet_io_manager": PolarsParquetIOManager(), diff --git a/src/app/vinyl/jobs.py b/apps/vinyl/src/jobs.py similarity index 97% rename from src/app/vinyl/jobs.py rename to apps/vinyl/src/jobs.py index b1d6bcd..9ae537a 100644 --- a/src/app/vinyl/jobs.py +++ b/apps/vinyl/src/jobs.py @@ -1,3 +1,5 @@ +from assets import deals, new_deals, works + from dagster import ( AssetKey, AssetMaterialization, @@ -7,8 +9,6 @@ from dagster import ( op, ) -from .assets import deals, new_deals, works - deals_job = define_asset_job( "deals_job", selection=[deals], partitions_def=deals.partitions_def ) diff --git a/src/app/partitions/__init__.py b/apps/vinyl/src/plato/__init__.py similarity index 100% rename from src/app/partitions/__init__.py rename to apps/vinyl/src/plato/__init__.py diff --git a/src/app/vinyl/plato/deals.py b/apps/vinyl/src/plato/deals.py similarity index 100% rename from src/app/vinyl/plato/deals.py rename to apps/vinyl/src/plato/deals.py diff --git a/src/app/vinyl/plato/fetch.py b/apps/vinyl/src/plato/fetch.py similarity index 100% rename from src/app/vinyl/plato/fetch.py rename to apps/vinyl/src/plato/fetch.py diff --git a/src/app/vinyl/plato/scrape.py b/apps/vinyl/src/plato/scrape.py similarity index 100% rename from src/app/vinyl/plato/scrape.py rename to apps/vinyl/src/plato/scrape.py diff --git a/src/app/vinyl/plato/utils.py b/apps/vinyl/src/plato/utils.py similarity index 100% rename from src/app/vinyl/plato/utils.py rename to apps/vinyl/src/plato/utils.py diff --git a/src/app/vinyl/schedules.py b/apps/vinyl/src/schedules.py similarity index 87% rename from src/app/vinyl/schedules.py rename to apps/vinyl/src/schedules.py index 048db31..3947793 100644 --- a/src/app/vinyl/schedules.py +++ b/apps/vinyl/src/schedules.py @@ -1,6 +1,6 @@ -from dagster import DefaultScheduleStatus, build_schedule_from_partitioned_job +from jobs import deals_job -from app.vinyl.repo import deals_job +from dagster import DefaultScheduleStatus, build_schedule_from_partitioned_job deals_schedule = build_schedule_from_partitioned_job( job=deals_job, diff --git a/src/app/vinyl/sensors.py b/apps/vinyl/src/sensors.py similarity index 84% rename from src/app/vinyl/sensors.py rename to apps/vinyl/src/sensors.py index 7068c1f..dc57086 100644 --- a/src/app/vinyl/sensors.py +++ b/apps/vinyl/src/sensors.py @@ -1,5 +1,6 @@ -from app.vinyl.assets import deals -from app.vinyl.jobs import musicbrainz_lookup_job +from assets import deals +from jobs import musicbrainz_lookup_job + from dagster import ( DefaultSensorStatus, EventLogEntry, diff --git a/src/app/vinyl/__init__.py b/apps/vinyl/src/sounds/__init__.py similarity index 100% rename from src/app/vinyl/__init__.py rename to apps/vinyl/src/sounds/__init__.py diff --git a/src/app/vinyl/sounds/deals.py b/apps/vinyl/src/sounds/deals.py similarity index 98% rename from src/app/vinyl/sounds/deals.py rename to apps/vinyl/src/sounds/deals.py index 21729af..0fd9b03 100644 --- a/src/app/vinyl/sounds/deals.py +++ b/apps/vinyl/src/sounds/deals.py @@ -72,7 +72,7 @@ if __name__ == "__main__": done = True if len(deals) > 0: - print(f"Discounted items:") + print("Discounted items:") print(deals) done = True diff --git a/src/app/vinyl/sounds/fetch.py b/apps/vinyl/src/sounds/fetch.py similarity index 100% rename from src/app/vinyl/sounds/fetch.py rename to apps/vinyl/src/sounds/fetch.py diff --git a/src/app/vinyl/test.py b/apps/vinyl/src/test.py similarity index 91% rename from src/app/vinyl/test.py rename to apps/vinyl/src/test.py index 1eb0d95..0d118f4 100644 --- a/src/app/vinyl/test.py +++ b/apps/vinyl/src/test.py @@ -1,16 +1,15 @@ +import logging import warnings from datetime import datetime -from dagster import materialize +from assets import deals from dagster_polars import PolarsParquetIOManager +from jobs import check_partititions_job -from app.vinyl.assets import deals -from app.vinyl.jobs import check_partititions_job +from dagster import materialize warnings.filterwarnings("ignore", category=UserWarning) -import logging - logging.getLogger().setLevel(logging.INFO) resources = { diff --git a/src/app/vinyl/utils.py b/apps/vinyl/src/utils.py similarity index 100% rename from src/app/vinyl/utils.py rename to apps/vinyl/src/utils.py diff --git a/compose.code.yaml b/compose.code.yaml index 5b8b351..6ad527a 100644 --- a/compose.code.yaml +++ b/compose.code.yaml @@ -4,15 +4,6 @@ x-dagster-env: &dagster_env DAGSTER_POSTGRES_USER: ${POSTGRES_USER} DAGSTER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} DAGSTER_POSTGRES_DB: ${POSTGRES_DB} - DAGSTER_CURRENT_IMAGE: ${DAGSTER_CURRENT_IMAGE} - -x-volumes: &volumes - volumes: - #- /opt/dagster/storage/:/opt/dagster/home/storage/ - - /opt/dagster/storage/import/:/opt/dagster/home/storage/import/ - - /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/ - - /opt/dagster/src/app/:/opt/dagster/home/app/ - - /opt/dagster/src/repo.py:/opt/dagster/home/repo.py services: # This service runs the gRPC server that loads your user code, in both dagit @@ -20,30 +11,37 @@ services: # run launcher to use this same image when launching runs in a new container as well. # Multiple containers like this can be deployed separately - each just needs to run on # its own port, and have its own entry in the workspace.yaml file that's loaded by dagit. - user_code: + user_code_vinyl: build: - context: . - dockerfile: Dockerfile.code - container_name: user_code - image: user_code_image + context: apps/vinyl + dockerfile: ../../Dockerfile.code + container_name: user_code_vinyl + image: user_code_vinyl restart: always environment: <<: *dagster_env - <<: *volumes + DAGSTER_CURRENT_IMAGE: user_code_vinyl + PYTHONPATH: app + volumes: + - /opt/dagster/storage/import/:/opt/dagster/home/storage/import/ + - /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/ + - /opt/dagster/apps/vinyl/src/:/opt/dagster/home/app/ networks: - dagster - other_image: - profiles: [ disabled ] + user_code_other: + # profiles: [ disabled ] build: - context: . - dockerfile: Dockerfile - container_name: other_image - image: user_code_image + context: apps/other + dockerfile: ../../Dockerfile.code + container_name: user_code_other + image: user_code_other restart: always environment: <<: *dagster_env - DAGSTER_CURRENT_IMAGE: something_else - <<: *volumes + DAGSTER_CURRENT_IMAGE: user_code_other + PYTHONPATH: app + volumes: + - /opt/dagster/apps/other/src/:/opt/dagster/home/app/ networks: - dagster diff --git a/compose.system.yaml b/compose.system.yaml index 4698f2e..58ebebc 100644 --- a/compose.system.yaml +++ b/compose.system.yaml @@ -67,7 +67,6 @@ services: - dagster depends_on: - postgresql - - user_code # This service runs the dagster-daemon process, which is responsible for taking runs # off of the queue and launching them, as well as creating runs from schedules or sensors. diff --git a/dagster-requirements.txt b/dagster-requirements.txt deleted file mode 100644 index 41e78bf..0000000 --- a/dagster-requirements.txt +++ /dev/null @@ -1,398 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile --constraint=requirements.txt --output-file=dagster-requirements.txt --extra=dagster pyproject.toml -aiobotocore==2.15.1 - # via s3fs -aiohappyeyeballs==2.4.3 - # via aiohttp -aiohttp==3.10.8 - # via - # aiobotocore - # s3fs -aioitertools==0.12.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -alembic==1.13.3 - # via dagster -aniso8601==9.0.1 - # via graphene -annotated-types==0.7.0 - # via pydantic -anyio==4.6.0 - # via - # gql - # starlette - # watchfiles -appdirs==1.4.4 - # via pint -asttokens==2.4.1 - # via icecream -attrs==24.2.0 - # via aiohttp -backoff==2.2.1 - # via gql -beautifulsoup4==4.12.3 -boto3==1.35.23 - # via - # aiobotocore - # dagster-aws -botocore==1.35.23 - # via - # aiobotocore - # boto3 - # s3transfer -cachetools==5.5.0 - # via google-auth -certifi==2024.8.30 - # via - # influxdb-client - # kubernetes - # pyogrio - # pyproj - # requests -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # dagster - # dagster-webserver - # uvicorn -colorama==0.4.6 - # via icecream -coloredlogs==14.0 - # via dagster -contourpy==1.3.0 - # via matplotlib -cramjam==2.8.4 - # via fastparquet -croniter==3.0.3 - # via dagster -cycler==0.12.1 - # via matplotlib -dagit==1.8.9 -dagster==1.8.9 - # via - # dagster-aws - # dagster-docker - # dagster-duckdb - # dagster-duckdb-pandas - # dagster-graphql - # dagster-polars - # dagster-postgres - # dagster-webserver -dagster-aws==0.24.9 -dagster-docker==0.24.9 -dagster-duckdb==0.24.9 - # via dagster-duckdb-pandas -dagster-duckdb-pandas==0.24.9 -dagster-graphql==1.8.9 - # via dagster-webserver -dagster-pipes==1.8.9 - # via dagster -dagster-polars==0.24.9 -dagster-postgres==0.24.9 -dagster-webserver==1.8.9 - # via dagit -dnspython==2.6.1 - # via email-validator -docker==7.1.0 - # via dagster-docker -docker-image-py==0.1.13 - # via dagster-docker -docstring-parser==0.16 - # via dagster -duckdb==1.1.1 - # via dagster-duckdb -durationpy==0.8 - # via kubernetes -email-validator==2.2.0 - # via pydantic -et-xmlfile==1.1.0 - # via openpyxl -executing==2.1.0 - # via icecream -fastapi==0.115.0 -fastparquet==2024.5.0 -filelock==3.16.1 - # via dagster -flexcache==0.3 - # via pint -flexparser==0.3.1 - # via pint -fonttools==4.54.1 - # via matplotlib -frozenlist==1.4.1 - # via - # aiohttp - # aiosignal -fsspec==2024.9.0 - # via - # fastparquet - # s3fs - # universal-pathlib -geopandas==1.0.1 -gitdb==4.0.11 - # via gitpython -gitpython==3.1.43 -google-auth==2.35.0 - # via kubernetes -gql==3.5.0 - # via dagster-graphql -graphene==3.3 - # via dagster-graphql -graphql-core==3.2.4 - # via - # gql - # graphene - # graphql-relay -graphql-relay==3.2.0 - # via graphene -grpcio==1.66.2 - # via - # dagster - # grpcio-health-checking -grpcio-health-checking==1.62.3 - # via dagster -h11==0.14.0 - # via uvicorn -httptools==0.6.1 - # via uvicorn -humanfriendly==10.0 - # via coloredlogs -icecream==2.1.3 -idna==3.10 - # via - # anyio - # email-validator - # requests - # yarl -influxdb-client==1.46.0 -jinja2==3.1.4 - # via dagster -jmespath==1.0.1 - # via - # boto3 - # botocore -kiwisolver==1.4.7 - # via matplotlib -kubernetes==31.0.0 -lxml==5.3.0 -mako==1.3.5 - # via alembic -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.5 - # via - # jinja2 - # mako -matplotlib==3.9.2 - # via seaborn -mdurl==0.1.2 - # via markdown-it-py -multidict==6.1.0 - # via - # aiohttp - # yarl -networkx==3.3 -numpy==2.1.1 - # via - # contourpy - # fastparquet - # geopandas - # matplotlib - # pandas - # pyarrow - # pyogrio - # seaborn - # shapely -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -openpyxl==3.1.5 -packaging==24.1 - # via - # dagster - # dagster-aws - # fastparquet - # geopandas - # matplotlib - # pyogrio -pandas==2.2.3 - # via - # dagster-duckdb-pandas - # fastparquet - # geopandas - # pint-pandas - # seaborn -pillow==10.4.0 - # via matplotlib -pint==0.24.3 - # via pint-pandas -pint-pandas==0.6.2 -polars==1.9.0 - # via dagster-polars -protobuf==4.25.5 - # via - # dagster - # grpcio-health-checking -psycopg2-binary==2.9.9 - # via dagster-postgres -pyarrow==17.0.0 - # via dagster-polars -pyasn1==0.6.1 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.1 - # via google-auth -pydantic==2.9.2 - # via - # dagster - # fastapi - # pydantic-settings -pydantic-core==2.23.4 - # via pydantic -pydantic-settings==2.5.2 -pygments==2.18.0 - # via - # icecream - # rich -pyogrio==0.10.0 - # via geopandas -pyparsing==3.1.4 - # via matplotlib -pyproj==3.7.0 - # via geopandas -pysocks==1.7.1 - # via requests -python-dateutil==2.9.0.post0 - # via - # botocore - # croniter - # influxdb-client - # kubernetes - # matplotlib - # pandas -python-dotenv==1.0.1 - # via - # dagster - # pydantic-settings - # uvicorn -pytz==2024.2 - # via - # croniter - # dagster - # pandas -pyyaml==6.0.2 - # via - # dagster - # kubernetes - # uvicorn -reactivex==4.0.4 - # via influxdb-client -regex==2024.9.11 - # via docker-image-py -requests==2.32.3 - # via - # dagster - # dagster-aws - # dagster-graphql - # docker - # gql - # kubernetes - # requests-oauthlib - # requests-toolbelt -requests-oauthlib==2.0.0 - # via kubernetes -requests-toolbelt==1.0.0 - # via gql -rich==13.8.1 - # via dagster -rsa==4.9 - # via google-auth -s3fs==2024.9.0 -s3transfer==0.10.2 - # via boto3 -seaborn==0.13.2 -setuptools==75.1.0 - # via - # dagster - # influxdb-client -shapely==2.0.6 - # via geopandas -six==1.16.0 - # via - # asttokens - # kubernetes - # python-dateutil -smmap==5.0.1 - # via gitdb -sniffio==1.3.1 - # via anyio -soupsieve==2.6 - # via beautifulsoup4 -sqlalchemy==2.0.35 - # via - # alembic - # dagster -starlette==0.38.6 - # via - # dagster-graphql - # dagster-webserver - # fastapi -structlog==24.4.0 - # via dagster -tabulate==0.9.0 - # via dagster -tomli==2.0.1 - # via dagster -toposort==1.10 - # via dagster -tqdm==4.66.5 - # via dagster -typing-extensions==4.12.2 - # via - # alembic - # dagster - # dagster-polars - # fastapi - # flexcache - # flexparser - # pint - # pydantic - # pydantic-core - # reactivex - # sqlalchemy -tzdata==2024.2 - # via pandas -universal-pathlib==0.2.5 - # via - # dagster - # dagster-polars -urllib3==2.2.3 - # via - # botocore - # docker - # influxdb-client - # kubernetes - # requests -uvicorn==0.31.0 - # via dagster-webserver -uvloop==0.20.0 - # via uvicorn -watchdog==5.0.3 - # via dagster -watchfiles==0.24.0 - # via uvicorn -websocket-client==1.8.0 - # via kubernetes -websockets==13.1 - # via uvicorn -wrapt==1.16.0 - # via aiobotocore -xlsxwriter==3.2.0 -yarl==1.13.1 - # via - # aiohttp - # gql diff --git a/dagster.yaml b/dagster.yaml index 4b16a7c..5010d4a 100644 --- a/dagster.yaml +++ b/dagster.yaml @@ -1,7 +1,6 @@ telemetry: enabled: false - concurrency: default_op_concurrency_limit: 2 @@ -19,15 +18,13 @@ run_launcher: - DAGSTER_POSTGRES_USER - DAGSTER_POSTGRES_PASSWORD - DAGSTER_POSTGRES_DB + - PYTHONPATH=app network: dagster container_kwargs: volumes: - - /opt/dagster/src/app/:/opt/dagster/home/app/ - - /opt/dagster/src/repo.py:/opt/dagster/home/repo.py - - /opt/dagster/storage/import/:/opt/dagster/home/storage/import/ - /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/ - + - /opt/dagster/apps/vinyl/src/:/opt/dagster/home/app/ - /opt/dagster/storage/:/opt/dagster/home/storage/ run_storage: diff --git a/pyproject.toml b/pyproject.toml index 3d48deb..5be0fb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,34 +6,25 @@ authors = [ ] version = "0.1.0" dependencies = [ - "fastapi", + "beautifulsoup4", + "boto3", + "duckdb", + "fastparquet", "gitpython", - "kubernetes", - "matplotlib", - "seaborn", + "icecream", + "lxml", "openpyxl", - "xlsxwriter", "pandas", "pyarrow", "pydantic[email]", "pydantic-settings", "pyyaml", "requests", - "s3fs[boto3]", - "structlog", - "uvicorn", - "duckdb", - "geopandas", - "lxml", - "networkx", - "Pint", - "Pint-Pandas", - "boto3", - "influxdb-client", "requests[socks]", - "beautifulsoup4", - "fastparquet", - "icecream" + "s3fs[boto3]", + "seaborn", + "structlog", + "xlsxwriter" ] [project.optional-dependencies] @@ -61,15 +52,19 @@ dagster = [ "dagster-duckdb-pandas", "dagit" ] - -[tool.poetry] -name = "dev" -version = "0.1.0" -description = "" -authors = ["Rik Veenboer "] - -[tool.poetry.dependencies] -seven = "^1.0.0" +vinyl = [] +other = [] +unknown = [ + "fastapi", + "geopandas", + "influxdb-client", + "kubernetes", + "matplotlib", + "networkx", + "Pint", + "Pint-Pandas", + "uvicorn" +] [tool.ruff] builtins = ["ic"] diff --git a/requirements.sh b/requirements.sh new file mode 100755 index 0000000..641908e --- /dev/null +++ b/requirements.sh @@ -0,0 +1,5 @@ +uv pip compile pyproject.toml +uv pip compile pyproject.toml --extra=dagster --output-file=dagster-requirements.txt +uv pip compile pyproject.toml --extra=dagster --extra=dev --extra=local --output-file=dev-requirements.txt +uv pip compile pyproject.toml --extra=dagster --extra=vinyl --output-file=src/vinyl/requirements.txt +uv pip compile pyproject.toml --extra=dagster --extra=other --output-file=src/other/requirements.txt diff --git a/requirements.txt b/requirements.txt index 85e1c5c..236279a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,238 +1,160 @@ # This file was autogenerated by uv via the following command: # uv pip compile --output-file=requirements.txt pyproject.toml -aiobotocore==2.15.1 - # via s3fs -aiohappyeyeballs==2.4.3 - # via aiohttp -aiohttp==3.10.8 - # via - # aiobotocore - # s3fs -aioitertools==0.12.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp annotated-types==0.7.0 # via pydantic -anyio==4.6.0 - # via starlette -appdirs==1.4.4 - # via pint -asttokens==2.4.1 +asttokens==3.0.0 # via icecream -attrs==24.2.0 - # via aiohttp -beautifulsoup4==4.12.3 -boto3==1.35.23 - # via aiobotocore -botocore==1.35.23 +beautifulsoup4==4.13.4 +boto3==1.39.9 +botocore==1.39.9 # via - # aiobotocore # boto3 + # s3fs # s3transfer -cachetools==5.5.0 +cachetools==5.5.2 # via google-auth -certifi==2024.8.30 +certifi==2025.7.14 # via - # influxdb-client # kubernetes - # pyogrio - # pyproj # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.2 # via requests -click==8.1.7 - # via uvicorn colorama==0.4.6 # via icecream -contourpy==1.3.0 +contourpy==1.3.2 # via matplotlib -cramjam==2.8.4 +cramjam==2.10.0 # via fastparquet cycler==0.12.1 # via matplotlib -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator -duckdb==1.1.1 -durationpy==0.8 +duckdb==1.3.2 +durationpy==0.10 # via kubernetes email-validator==2.2.0 # via pydantic -et-xmlfile==1.1.0 +et-xmlfile==2.0.0 # via openpyxl -executing==2.1.0 +executing==2.2.0 # via icecream -fastapi==0.115.0 -fastparquet==2024.5.0 -flexcache==0.3 - # via pint -flexparser==0.3.1 - # via pint -fonttools==4.54.1 +fastparquet==2024.11.0 +fonttools==4.59.0 # via matplotlib -frozenlist==1.4.1 - # via - # aiohttp - # aiosignal -fsspec==2024.9.0 +fsspec==2025.7.0 # via # fastparquet # s3fs -geopandas==1.0.1 -gitdb==4.0.11 +gitdb==4.0.12 # via gitpython -gitpython==3.1.43 -google-auth==2.35.0 +gitpython==3.1.44 +google-auth==2.40.3 # via kubernetes -h11==0.14.0 - # via uvicorn -icecream==2.1.3 +icecream==2.1.5 idna==3.10 # via - # anyio # email-validator # requests - # yarl -influxdb-client==1.46.0 jmespath==1.0.1 # via # boto3 # botocore -kiwisolver==1.4.7 +kiwisolver==1.4.8 # via matplotlib -kubernetes==31.0.0 -lxml==5.3.0 -matplotlib==3.9.2 +kubernetes==33.1.0 +lxml==6.0.0 +matplotlib==3.10.3 # via seaborn -multidict==6.1.0 - # via - # aiohttp - # yarl -networkx==3.3 -numpy==2.1.1 +numpy==2.3.1 # via # contourpy # fastparquet - # geopandas # matplotlib # pandas - # pyarrow - # pyogrio # seaborn - # shapely -oauthlib==3.2.2 +oauthlib==3.3.1 # via # kubernetes # requests-oauthlib openpyxl==3.1.5 -packaging==24.1 +packaging==25.0 # via # fastparquet - # geopandas # matplotlib - # pyogrio -pandas==2.2.3 +pandas==2.3.1 # via # fastparquet - # geopandas - # pint-pandas # seaborn -pillow==10.4.0 +pillow==11.3.0 # via matplotlib -pint==0.24.3 - # via pint-pandas -pint-pandas==0.6.2 -pyarrow==17.0.0 +pyarrow==21.0.0 pyasn1==0.6.1 # via # pyasn1-modules # rsa -pyasn1-modules==0.4.1 +pyasn1-modules==0.4.2 # via google-auth -pydantic==2.9.2 - # via - # fastapi - # pydantic-settings -pydantic-core==2.23.4 +pydantic==2.11.7 + # via pydantic-settings +pydantic-core==2.33.2 # via pydantic -pydantic-settings==2.5.2 -pygments==2.18.0 +pydantic-settings==2.10.1 +pygments==2.19.2 # via icecream -pyogrio==0.10.0 - # via geopandas -pyparsing==3.1.4 +pyparsing==3.2.3 # via matplotlib -pyproj==3.7.0 - # via geopandas pysocks==1.7.1 # via requests python-dateutil==2.9.0.post0 # via # botocore - # influxdb-client # kubernetes # matplotlib # pandas -python-dotenv==1.0.1 +python-dotenv==1.1.1 # via pydantic-settings -pytz==2024.2 +pytz==2025.2 # via pandas pyyaml==6.0.2 # via kubernetes -reactivex==4.0.4 - # via influxdb-client -requests==2.32.3 +requests==2.32.4 # via # kubernetes # requests-oauthlib requests-oauthlib==2.0.0 # via kubernetes -rsa==4.9 +rsa==4.9.1 # via google-auth -s3fs==2024.9.0 -s3transfer==0.10.2 +s3fs==0.4.2 +s3transfer==0.13.1 # via boto3 seaborn==0.13.2 -setuptools==75.1.0 - # via influxdb-client -shapely==2.0.6 - # via geopandas -six==1.16.0 +six==1.17.0 # via - # asttokens # kubernetes # python-dateutil -smmap==5.0.1 +smmap==5.0.2 # via gitdb -sniffio==1.3.1 - # via anyio -soupsieve==2.6 +soupsieve==2.7 # via beautifulsoup4 -starlette==0.38.6 - # via fastapi -structlog==24.4.0 -typing-extensions==4.12.2 +structlog==25.4.0 +typing-extensions==4.14.1 # via - # fastapi - # flexcache - # flexparser - # pint + # beautifulsoup4 # pydantic # pydantic-core - # reactivex -tzdata==2024.2 + # typing-inspection +typing-inspection==0.4.1 + # via + # pydantic + # pydantic-settings +tzdata==2025.2 # via pandas -urllib3==2.2.3 +urllib3==2.5.0 # via # botocore - # influxdb-client # kubernetes # requests -uvicorn==0.31.0 websocket-client==1.8.0 # via kubernetes -wrapt==1.16.0 - # via aiobotocore -xlsxwriter==3.2.0 -yarl==1.13.1 - # via aiohttp +xlsxwriter==3.2.5 diff --git a/src/app/vinyl/plato/__init__.py b/src/app/vinyl/plato/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/app/vinyl/sounds/__init__.py b/src/app/vinyl/sounds/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/repo.py b/src/repo.py deleted file mode 100644 index de2d58d..0000000 --- a/src/repo.py +++ /dev/null @@ -1 +0,0 @@ -from app.vinyl.repo import vinyl # noqa diff --git a/workspace.yaml b/workspace.yaml index 47124fd..fc8db27 100644 --- a/workspace.yaml +++ b/workspace.yaml @@ -1,4 +1,7 @@ load_from: - - grpc_server: - host: user_code + - grpc_server: + host: user_code_vinyl + port: 4000 + - grpc_server: + host: user_code_other port: 4000