refactor to allow for multiple code locations
This commit is contained in:
13
.dockerignore
Normal file
13
.dockerignore
Normal file
@@ -0,0 +1,13 @@
|
||||
Dockerfile.code
|
||||
Dockerfile.system
|
||||
Makefile
|
||||
__pycache__
|
||||
build
|
||||
compose*.yaml
|
||||
db
|
||||
poetry.lock
|
||||
pyproject.toml
|
||||
requirements.sh
|
||||
src
|
||||
storage
|
||||
trash
|
||||
@@ -3,7 +3,8 @@ FROM python:3.12-slim
|
||||
# Checkout and install dagster libraries needed to run the gRPC server
|
||||
# exposing your repository to dagit and dagster-daemon, and to load the DagsterInstance
|
||||
|
||||
COPY dagster-requirements.txt requirements.txt
|
||||
COPY requirements.txt .
|
||||
|
||||
RUN pip install uv
|
||||
RUN uv pip install -r requirements.txt --system
|
||||
RUN uv pip install polars-lts-cpu --system
|
||||
@@ -16,5 +17,4 @@ EXPOSE 4000
|
||||
|
||||
# CMD allows this to be overridden from run launchers or executors that want
|
||||
# to run other commands against your repository
|
||||
#CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-f", "repo.py"]
|
||||
CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-f", "repo.py"]
|
||||
CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-m", "app.definitions"]
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
COPY dagster-requirements.txt requirements.txt
|
||||
|
||||
RUN pip install uv
|
||||
RUN uv pip install -r requirements.txt --system
|
||||
RUN uv pip install polars-lts-cpu --system
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
from assets import asset_multi_1, asset_multi_2, asset_single_1, asset_single_2
|
||||
from dagster_polars import PolarsParquetIOManager
|
||||
|
||||
from dagster import Definitions, define_asset_job
|
||||
|
||||
from .assets import asset_multi_1, asset_multi_2, asset_single_1, asset_single_2
|
||||
|
||||
# Define a job that includes both assets
|
||||
daily_job = define_asset_job("daily_job", selection=[asset_multi_1, asset_multi_2])
|
||||
|
||||
vinyl = Definitions(
|
||||
definitions = Definitions(
|
||||
assets=[asset_single_1, asset_multi_1, asset_single_2, asset_multi_2],
|
||||
resources={"polars_parquet_io_manager": PolarsParquetIOManager()},
|
||||
jobs=[daily_job],
|
||||
@@ -1,11 +1,6 @@
|
||||
from assets import asset_multi_1, asset_multi_2, asset_single_1, asset_single_2
|
||||
from dagster_polars import PolarsParquetIOManager
|
||||
|
||||
from app.vinyl.assets import (
|
||||
asset_multi_1,
|
||||
asset_multi_2,
|
||||
asset_single_1,
|
||||
asset_single_2,
|
||||
)
|
||||
from dagster import materialize
|
||||
|
||||
resources = {
|
||||
3
apps/test/__init__.py
Normal file
3
apps/test/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from icecream import install
|
||||
|
||||
install()
|
||||
3
apps/vinyl/src/__init__.py
Normal file
3
apps/vinyl/src/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from icecream import install
|
||||
|
||||
install()
|
||||
@@ -5,10 +5,10 @@ import duckdb
|
||||
import polars as pl
|
||||
import structlog
|
||||
from duckdb.typing import DATE, VARCHAR
|
||||
from plato.fetch import scrape_plato
|
||||
from sounds.fetch import fetch_deals
|
||||
from utils import parse_date
|
||||
|
||||
from app.vinyl.plato.fetch import scrape_plato
|
||||
from app.vinyl.sounds.fetch import fetch_deals
|
||||
from app.vinyl.utils import parse_date
|
||||
from dagster import (
|
||||
DailyPartitionsDefinition,
|
||||
DimensionPartitionMapping,
|
||||
@@ -1,17 +1,16 @@
|
||||
from collections.abc import Sequence
|
||||
|
||||
from assets import deals, new_deals, works
|
||||
from dagster_duckdb import DuckDBIOManager
|
||||
from dagster_duckdb.io_manager import DbTypeHandler
|
||||
from dagster_duckdb_pandas import DuckDBPandasTypeHandler
|
||||
from dagster_polars import PolarsParquetIOManager
|
||||
from jobs import check_partititions_job, deals_job, musicbrainz_lookup_job
|
||||
from schedules import deals_schedule
|
||||
from sensors import musicbrainz_lookup_sensor
|
||||
|
||||
from dagster import Definitions
|
||||
|
||||
from .assets import deals, new_deals, works
|
||||
from .jobs import check_partititions_job, deals_job, musicbrainz_lookup_job
|
||||
from .schedules import deals_schedule
|
||||
from .sensors import musicbrainz_lookup_sensor
|
||||
|
||||
|
||||
class PandasDuckDBIOManager(DuckDBIOManager):
|
||||
@staticmethod
|
||||
@@ -19,7 +18,7 @@ class PandasDuckDBIOManager(DuckDBIOManager):
|
||||
return [DuckDBPandasTypeHandler()]
|
||||
|
||||
|
||||
vinyl = Definitions(
|
||||
definitions = Definitions(
|
||||
assets=[deals, new_deals, works],
|
||||
resources={
|
||||
"polars_parquet_io_manager": PolarsParquetIOManager(),
|
||||
@@ -1,3 +1,5 @@
|
||||
from assets import deals, new_deals, works
|
||||
|
||||
from dagster import (
|
||||
AssetKey,
|
||||
AssetMaterialization,
|
||||
@@ -7,8 +9,6 @@ from dagster import (
|
||||
op,
|
||||
)
|
||||
|
||||
from .assets import deals, new_deals, works
|
||||
|
||||
deals_job = define_asset_job(
|
||||
"deals_job", selection=[deals], partitions_def=deals.partitions_def
|
||||
)
|
||||
@@ -1,6 +1,6 @@
|
||||
from dagster import DefaultScheduleStatus, build_schedule_from_partitioned_job
|
||||
from jobs import deals_job
|
||||
|
||||
from app.vinyl.repo import deals_job
|
||||
from dagster import DefaultScheduleStatus, build_schedule_from_partitioned_job
|
||||
|
||||
deals_schedule = build_schedule_from_partitioned_job(
|
||||
job=deals_job,
|
||||
@@ -1,5 +1,6 @@
|
||||
from app.vinyl.assets import deals
|
||||
from app.vinyl.jobs import musicbrainz_lookup_job
|
||||
from assets import deals
|
||||
from jobs import musicbrainz_lookup_job
|
||||
|
||||
from dagster import (
|
||||
DefaultSensorStatus,
|
||||
EventLogEntry,
|
||||
@@ -72,7 +72,7 @@ if __name__ == "__main__":
|
||||
done = True
|
||||
|
||||
if len(deals) > 0:
|
||||
print(f"Discounted items:")
|
||||
print("Discounted items:")
|
||||
print(deals)
|
||||
done = True
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
import logging
|
||||
import warnings
|
||||
from datetime import datetime
|
||||
|
||||
from dagster import materialize
|
||||
from assets import deals
|
||||
from dagster_polars import PolarsParquetIOManager
|
||||
from jobs import check_partititions_job
|
||||
|
||||
from app.vinyl.assets import deals
|
||||
from app.vinyl.jobs import check_partititions_job
|
||||
from dagster import materialize
|
||||
|
||||
warnings.filterwarnings("ignore", category=UserWarning)
|
||||
|
||||
import logging
|
||||
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
resources = {
|
||||
@@ -4,15 +4,6 @@ x-dagster-env: &dagster_env
|
||||
DAGSTER_POSTGRES_USER: ${POSTGRES_USER}
|
||||
DAGSTER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
DAGSTER_POSTGRES_DB: ${POSTGRES_DB}
|
||||
DAGSTER_CURRENT_IMAGE: ${DAGSTER_CURRENT_IMAGE}
|
||||
|
||||
x-volumes: &volumes
|
||||
volumes:
|
||||
#- /opt/dagster/storage/:/opt/dagster/home/storage/
|
||||
- /opt/dagster/storage/import/:/opt/dagster/home/storage/import/
|
||||
- /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/
|
||||
- /opt/dagster/src/app/:/opt/dagster/home/app/
|
||||
- /opt/dagster/src/repo.py:/opt/dagster/home/repo.py
|
||||
|
||||
services:
|
||||
# This service runs the gRPC server that loads your user code, in both dagit
|
||||
@@ -20,30 +11,37 @@ services:
|
||||
# run launcher to use this same image when launching runs in a new container as well.
|
||||
# Multiple containers like this can be deployed separately - each just needs to run on
|
||||
# its own port, and have its own entry in the workspace.yaml file that's loaded by dagit.
|
||||
user_code:
|
||||
user_code_vinyl:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.code
|
||||
container_name: user_code
|
||||
image: user_code_image
|
||||
context: apps/vinyl
|
||||
dockerfile: ../../Dockerfile.code
|
||||
container_name: user_code_vinyl
|
||||
image: user_code_vinyl
|
||||
restart: always
|
||||
environment:
|
||||
<<: *dagster_env
|
||||
<<: *volumes
|
||||
DAGSTER_CURRENT_IMAGE: user_code_vinyl
|
||||
PYTHONPATH: app
|
||||
volumes:
|
||||
- /opt/dagster/storage/import/:/opt/dagster/home/storage/import/
|
||||
- /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/
|
||||
- /opt/dagster/apps/vinyl/src/:/opt/dagster/home/app/
|
||||
networks:
|
||||
- dagster
|
||||
|
||||
other_image:
|
||||
profiles: [ disabled ]
|
||||
user_code_other:
|
||||
# profiles: [ disabled ]
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: other_image
|
||||
image: user_code_image
|
||||
context: apps/other
|
||||
dockerfile: ../../Dockerfile.code
|
||||
container_name: user_code_other
|
||||
image: user_code_other
|
||||
restart: always
|
||||
environment:
|
||||
<<: *dagster_env
|
||||
DAGSTER_CURRENT_IMAGE: something_else
|
||||
<<: *volumes
|
||||
DAGSTER_CURRENT_IMAGE: user_code_other
|
||||
PYTHONPATH: app
|
||||
volumes:
|
||||
- /opt/dagster/apps/other/src/:/opt/dagster/home/app/
|
||||
networks:
|
||||
- dagster
|
||||
|
||||
@@ -67,7 +67,6 @@ services:
|
||||
- dagster
|
||||
depends_on:
|
||||
- postgresql
|
||||
- user_code
|
||||
|
||||
# This service runs the dagster-daemon process, which is responsible for taking runs
|
||||
# off of the queue and launching them, as well as creating runs from schedules or sensors.
|
||||
|
||||
@@ -1,398 +0,0 @@
|
||||
# This file was autogenerated by uv via the following command:
|
||||
# uv pip compile --constraint=requirements.txt --output-file=dagster-requirements.txt --extra=dagster pyproject.toml
|
||||
aiobotocore==2.15.1
|
||||
# via s3fs
|
||||
aiohappyeyeballs==2.4.3
|
||||
# via aiohttp
|
||||
aiohttp==3.10.8
|
||||
# via
|
||||
# aiobotocore
|
||||
# s3fs
|
||||
aioitertools==0.12.0
|
||||
# via aiobotocore
|
||||
aiosignal==1.3.1
|
||||
# via aiohttp
|
||||
alembic==1.13.3
|
||||
# via dagster
|
||||
aniso8601==9.0.1
|
||||
# via graphene
|
||||
annotated-types==0.7.0
|
||||
# via pydantic
|
||||
anyio==4.6.0
|
||||
# via
|
||||
# gql
|
||||
# starlette
|
||||
# watchfiles
|
||||
appdirs==1.4.4
|
||||
# via pint
|
||||
asttokens==2.4.1
|
||||
# via icecream
|
||||
attrs==24.2.0
|
||||
# via aiohttp
|
||||
backoff==2.2.1
|
||||
# via gql
|
||||
beautifulsoup4==4.12.3
|
||||
boto3==1.35.23
|
||||
# via
|
||||
# aiobotocore
|
||||
# dagster-aws
|
||||
botocore==1.35.23
|
||||
# via
|
||||
# aiobotocore
|
||||
# boto3
|
||||
# s3transfer
|
||||
cachetools==5.5.0
|
||||
# via google-auth
|
||||
certifi==2024.8.30
|
||||
# via
|
||||
# influxdb-client
|
||||
# kubernetes
|
||||
# pyogrio
|
||||
# pyproj
|
||||
# requests
|
||||
charset-normalizer==3.3.2
|
||||
# via requests
|
||||
click==8.1.7
|
||||
# via
|
||||
# dagster
|
||||
# dagster-webserver
|
||||
# uvicorn
|
||||
colorama==0.4.6
|
||||
# via icecream
|
||||
coloredlogs==14.0
|
||||
# via dagster
|
||||
contourpy==1.3.0
|
||||
# via matplotlib
|
||||
cramjam==2.8.4
|
||||
# via fastparquet
|
||||
croniter==3.0.3
|
||||
# via dagster
|
||||
cycler==0.12.1
|
||||
# via matplotlib
|
||||
dagit==1.8.9
|
||||
dagster==1.8.9
|
||||
# via
|
||||
# dagster-aws
|
||||
# dagster-docker
|
||||
# dagster-duckdb
|
||||
# dagster-duckdb-pandas
|
||||
# dagster-graphql
|
||||
# dagster-polars
|
||||
# dagster-postgres
|
||||
# dagster-webserver
|
||||
dagster-aws==0.24.9
|
||||
dagster-docker==0.24.9
|
||||
dagster-duckdb==0.24.9
|
||||
# via dagster-duckdb-pandas
|
||||
dagster-duckdb-pandas==0.24.9
|
||||
dagster-graphql==1.8.9
|
||||
# via dagster-webserver
|
||||
dagster-pipes==1.8.9
|
||||
# via dagster
|
||||
dagster-polars==0.24.9
|
||||
dagster-postgres==0.24.9
|
||||
dagster-webserver==1.8.9
|
||||
# via dagit
|
||||
dnspython==2.6.1
|
||||
# via email-validator
|
||||
docker==7.1.0
|
||||
# via dagster-docker
|
||||
docker-image-py==0.1.13
|
||||
# via dagster-docker
|
||||
docstring-parser==0.16
|
||||
# via dagster
|
||||
duckdb==1.1.1
|
||||
# via dagster-duckdb
|
||||
durationpy==0.8
|
||||
# via kubernetes
|
||||
email-validator==2.2.0
|
||||
# via pydantic
|
||||
et-xmlfile==1.1.0
|
||||
# via openpyxl
|
||||
executing==2.1.0
|
||||
# via icecream
|
||||
fastapi==0.115.0
|
||||
fastparquet==2024.5.0
|
||||
filelock==3.16.1
|
||||
# via dagster
|
||||
flexcache==0.3
|
||||
# via pint
|
||||
flexparser==0.3.1
|
||||
# via pint
|
||||
fonttools==4.54.1
|
||||
# via matplotlib
|
||||
frozenlist==1.4.1
|
||||
# via
|
||||
# aiohttp
|
||||
# aiosignal
|
||||
fsspec==2024.9.0
|
||||
# via
|
||||
# fastparquet
|
||||
# s3fs
|
||||
# universal-pathlib
|
||||
geopandas==1.0.1
|
||||
gitdb==4.0.11
|
||||
# via gitpython
|
||||
gitpython==3.1.43
|
||||
google-auth==2.35.0
|
||||
# via kubernetes
|
||||
gql==3.5.0
|
||||
# via dagster-graphql
|
||||
graphene==3.3
|
||||
# via dagster-graphql
|
||||
graphql-core==3.2.4
|
||||
# via
|
||||
# gql
|
||||
# graphene
|
||||
# graphql-relay
|
||||
graphql-relay==3.2.0
|
||||
# via graphene
|
||||
grpcio==1.66.2
|
||||
# via
|
||||
# dagster
|
||||
# grpcio-health-checking
|
||||
grpcio-health-checking==1.62.3
|
||||
# via dagster
|
||||
h11==0.14.0
|
||||
# via uvicorn
|
||||
httptools==0.6.1
|
||||
# via uvicorn
|
||||
humanfriendly==10.0
|
||||
# via coloredlogs
|
||||
icecream==2.1.3
|
||||
idna==3.10
|
||||
# via
|
||||
# anyio
|
||||
# email-validator
|
||||
# requests
|
||||
# yarl
|
||||
influxdb-client==1.46.0
|
||||
jinja2==3.1.4
|
||||
# via dagster
|
||||
jmespath==1.0.1
|
||||
# via
|
||||
# boto3
|
||||
# botocore
|
||||
kiwisolver==1.4.7
|
||||
# via matplotlib
|
||||
kubernetes==31.0.0
|
||||
lxml==5.3.0
|
||||
mako==1.3.5
|
||||
# via alembic
|
||||
markdown-it-py==3.0.0
|
||||
# via rich
|
||||
markupsafe==2.1.5
|
||||
# via
|
||||
# jinja2
|
||||
# mako
|
||||
matplotlib==3.9.2
|
||||
# via seaborn
|
||||
mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
multidict==6.1.0
|
||||
# via
|
||||
# aiohttp
|
||||
# yarl
|
||||
networkx==3.3
|
||||
numpy==2.1.1
|
||||
# via
|
||||
# contourpy
|
||||
# fastparquet
|
||||
# geopandas
|
||||
# matplotlib
|
||||
# pandas
|
||||
# pyarrow
|
||||
# pyogrio
|
||||
# seaborn
|
||||
# shapely
|
||||
oauthlib==3.2.2
|
||||
# via
|
||||
# kubernetes
|
||||
# requests-oauthlib
|
||||
openpyxl==3.1.5
|
||||
packaging==24.1
|
||||
# via
|
||||
# dagster
|
||||
# dagster-aws
|
||||
# fastparquet
|
||||
# geopandas
|
||||
# matplotlib
|
||||
# pyogrio
|
||||
pandas==2.2.3
|
||||
# via
|
||||
# dagster-duckdb-pandas
|
||||
# fastparquet
|
||||
# geopandas
|
||||
# pint-pandas
|
||||
# seaborn
|
||||
pillow==10.4.0
|
||||
# via matplotlib
|
||||
pint==0.24.3
|
||||
# via pint-pandas
|
||||
pint-pandas==0.6.2
|
||||
polars==1.9.0
|
||||
# via dagster-polars
|
||||
protobuf==4.25.5
|
||||
# via
|
||||
# dagster
|
||||
# grpcio-health-checking
|
||||
psycopg2-binary==2.9.9
|
||||
# via dagster-postgres
|
||||
pyarrow==17.0.0
|
||||
# via dagster-polars
|
||||
pyasn1==0.6.1
|
||||
# via
|
||||
# pyasn1-modules
|
||||
# rsa
|
||||
pyasn1-modules==0.4.1
|
||||
# via google-auth
|
||||
pydantic==2.9.2
|
||||
# via
|
||||
# dagster
|
||||
# fastapi
|
||||
# pydantic-settings
|
||||
pydantic-core==2.23.4
|
||||
# via pydantic
|
||||
pydantic-settings==2.5.2
|
||||
pygments==2.18.0
|
||||
# via
|
||||
# icecream
|
||||
# rich
|
||||
pyogrio==0.10.0
|
||||
# via geopandas
|
||||
pyparsing==3.1.4
|
||||
# via matplotlib
|
||||
pyproj==3.7.0
|
||||
# via geopandas
|
||||
pysocks==1.7.1
|
||||
# via requests
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# botocore
|
||||
# croniter
|
||||
# influxdb-client
|
||||
# kubernetes
|
||||
# matplotlib
|
||||
# pandas
|
||||
python-dotenv==1.0.1
|
||||
# via
|
||||
# dagster
|
||||
# pydantic-settings
|
||||
# uvicorn
|
||||
pytz==2024.2
|
||||
# via
|
||||
# croniter
|
||||
# dagster
|
||||
# pandas
|
||||
pyyaml==6.0.2
|
||||
# via
|
||||
# dagster
|
||||
# kubernetes
|
||||
# uvicorn
|
||||
reactivex==4.0.4
|
||||
# via influxdb-client
|
||||
regex==2024.9.11
|
||||
# via docker-image-py
|
||||
requests==2.32.3
|
||||
# via
|
||||
# dagster
|
||||
# dagster-aws
|
||||
# dagster-graphql
|
||||
# docker
|
||||
# gql
|
||||
# kubernetes
|
||||
# requests-oauthlib
|
||||
# requests-toolbelt
|
||||
requests-oauthlib==2.0.0
|
||||
# via kubernetes
|
||||
requests-toolbelt==1.0.0
|
||||
# via gql
|
||||
rich==13.8.1
|
||||
# via dagster
|
||||
rsa==4.9
|
||||
# via google-auth
|
||||
s3fs==2024.9.0
|
||||
s3transfer==0.10.2
|
||||
# via boto3
|
||||
seaborn==0.13.2
|
||||
setuptools==75.1.0
|
||||
# via
|
||||
# dagster
|
||||
# influxdb-client
|
||||
shapely==2.0.6
|
||||
# via geopandas
|
||||
six==1.16.0
|
||||
# via
|
||||
# asttokens
|
||||
# kubernetes
|
||||
# python-dateutil
|
||||
smmap==5.0.1
|
||||
# via gitdb
|
||||
sniffio==1.3.1
|
||||
# via anyio
|
||||
soupsieve==2.6
|
||||
# via beautifulsoup4
|
||||
sqlalchemy==2.0.35
|
||||
# via
|
||||
# alembic
|
||||
# dagster
|
||||
starlette==0.38.6
|
||||
# via
|
||||
# dagster-graphql
|
||||
# dagster-webserver
|
||||
# fastapi
|
||||
structlog==24.4.0
|
||||
# via dagster
|
||||
tabulate==0.9.0
|
||||
# via dagster
|
||||
tomli==2.0.1
|
||||
# via dagster
|
||||
toposort==1.10
|
||||
# via dagster
|
||||
tqdm==4.66.5
|
||||
# via dagster
|
||||
typing-extensions==4.12.2
|
||||
# via
|
||||
# alembic
|
||||
# dagster
|
||||
# dagster-polars
|
||||
# fastapi
|
||||
# flexcache
|
||||
# flexparser
|
||||
# pint
|
||||
# pydantic
|
||||
# pydantic-core
|
||||
# reactivex
|
||||
# sqlalchemy
|
||||
tzdata==2024.2
|
||||
# via pandas
|
||||
universal-pathlib==0.2.5
|
||||
# via
|
||||
# dagster
|
||||
# dagster-polars
|
||||
urllib3==2.2.3
|
||||
# via
|
||||
# botocore
|
||||
# docker
|
||||
# influxdb-client
|
||||
# kubernetes
|
||||
# requests
|
||||
uvicorn==0.31.0
|
||||
# via dagster-webserver
|
||||
uvloop==0.20.0
|
||||
# via uvicorn
|
||||
watchdog==5.0.3
|
||||
# via dagster
|
||||
watchfiles==0.24.0
|
||||
# via uvicorn
|
||||
websocket-client==1.8.0
|
||||
# via kubernetes
|
||||
websockets==13.1
|
||||
# via uvicorn
|
||||
wrapt==1.16.0
|
||||
# via aiobotocore
|
||||
xlsxwriter==3.2.0
|
||||
yarl==1.13.1
|
||||
# via
|
||||
# aiohttp
|
||||
# gql
|
||||
@@ -1,7 +1,6 @@
|
||||
telemetry:
|
||||
enabled: false
|
||||
|
||||
|
||||
concurrency:
|
||||
default_op_concurrency_limit: 2
|
||||
|
||||
@@ -19,15 +18,13 @@ run_launcher:
|
||||
- DAGSTER_POSTGRES_USER
|
||||
- DAGSTER_POSTGRES_PASSWORD
|
||||
- DAGSTER_POSTGRES_DB
|
||||
- PYTHONPATH=app
|
||||
network: dagster
|
||||
container_kwargs:
|
||||
volumes:
|
||||
- /opt/dagster/src/app/:/opt/dagster/home/app/
|
||||
- /opt/dagster/src/repo.py:/opt/dagster/home/repo.py
|
||||
|
||||
- /opt/dagster/storage/import/:/opt/dagster/home/storage/import/
|
||||
- /opt/dagster/storage/deals/:/opt/dagster/home/storage/deals/
|
||||
|
||||
- /opt/dagster/apps/vinyl/src/:/opt/dagster/home/app/
|
||||
- /opt/dagster/storage/:/opt/dagster/home/storage/
|
||||
|
||||
run_storage:
|
||||
|
||||
@@ -6,34 +6,25 @@ authors = [
|
||||
]
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"fastapi",
|
||||
"beautifulsoup4",
|
||||
"boto3",
|
||||
"duckdb",
|
||||
"fastparquet",
|
||||
"gitpython",
|
||||
"kubernetes",
|
||||
"matplotlib",
|
||||
"seaborn",
|
||||
"icecream",
|
||||
"lxml",
|
||||
"openpyxl",
|
||||
"xlsxwriter",
|
||||
"pandas",
|
||||
"pyarrow",
|
||||
"pydantic[email]",
|
||||
"pydantic-settings",
|
||||
"pyyaml",
|
||||
"requests",
|
||||
"s3fs[boto3]",
|
||||
"structlog",
|
||||
"uvicorn",
|
||||
"duckdb",
|
||||
"geopandas",
|
||||
"lxml",
|
||||
"networkx",
|
||||
"Pint",
|
||||
"Pint-Pandas",
|
||||
"boto3",
|
||||
"influxdb-client",
|
||||
"requests[socks]",
|
||||
"beautifulsoup4",
|
||||
"fastparquet",
|
||||
"icecream"
|
||||
"s3fs[boto3]",
|
||||
"seaborn",
|
||||
"structlog",
|
||||
"xlsxwriter"
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
@@ -61,15 +52,19 @@ dagster = [
|
||||
"dagster-duckdb-pandas",
|
||||
"dagit"
|
||||
]
|
||||
|
||||
[tool.poetry]
|
||||
name = "dev"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["Rik Veenboer <rik.veenboer@gmail.com>"]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
seven = "^1.0.0"
|
||||
vinyl = []
|
||||
other = []
|
||||
unknown = [
|
||||
"fastapi",
|
||||
"geopandas",
|
||||
"influxdb-client",
|
||||
"kubernetes",
|
||||
"matplotlib",
|
||||
"networkx",
|
||||
"Pint",
|
||||
"Pint-Pandas",
|
||||
"uvicorn"
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
builtins = ["ic"]
|
||||
|
||||
5
requirements.sh
Executable file
5
requirements.sh
Executable file
@@ -0,0 +1,5 @@
|
||||
uv pip compile pyproject.toml
|
||||
uv pip compile pyproject.toml --extra=dagster --output-file=dagster-requirements.txt
|
||||
uv pip compile pyproject.toml --extra=dagster --extra=dev --extra=local --output-file=dev-requirements.txt
|
||||
uv pip compile pyproject.toml --extra=dagster --extra=vinyl --output-file=src/vinyl/requirements.txt
|
||||
uv pip compile pyproject.toml --extra=dagster --extra=other --output-file=src/other/requirements.txt
|
||||
196
requirements.txt
196
requirements.txt
@@ -1,238 +1,160 @@
|
||||
# This file was autogenerated by uv via the following command:
|
||||
# uv pip compile --output-file=requirements.txt pyproject.toml
|
||||
aiobotocore==2.15.1
|
||||
# via s3fs
|
||||
aiohappyeyeballs==2.4.3
|
||||
# via aiohttp
|
||||
aiohttp==3.10.8
|
||||
# via
|
||||
# aiobotocore
|
||||
# s3fs
|
||||
aioitertools==0.12.0
|
||||
# via aiobotocore
|
||||
aiosignal==1.3.1
|
||||
# via aiohttp
|
||||
annotated-types==0.7.0
|
||||
# via pydantic
|
||||
anyio==4.6.0
|
||||
# via starlette
|
||||
appdirs==1.4.4
|
||||
# via pint
|
||||
asttokens==2.4.1
|
||||
asttokens==3.0.0
|
||||
# via icecream
|
||||
attrs==24.2.0
|
||||
# via aiohttp
|
||||
beautifulsoup4==4.12.3
|
||||
boto3==1.35.23
|
||||
# via aiobotocore
|
||||
botocore==1.35.23
|
||||
beautifulsoup4==4.13.4
|
||||
boto3==1.39.9
|
||||
botocore==1.39.9
|
||||
# via
|
||||
# aiobotocore
|
||||
# boto3
|
||||
# s3fs
|
||||
# s3transfer
|
||||
cachetools==5.5.0
|
||||
cachetools==5.5.2
|
||||
# via google-auth
|
||||
certifi==2024.8.30
|
||||
certifi==2025.7.14
|
||||
# via
|
||||
# influxdb-client
|
||||
# kubernetes
|
||||
# pyogrio
|
||||
# pyproj
|
||||
# requests
|
||||
charset-normalizer==3.3.2
|
||||
charset-normalizer==3.4.2
|
||||
# via requests
|
||||
click==8.1.7
|
||||
# via uvicorn
|
||||
colorama==0.4.6
|
||||
# via icecream
|
||||
contourpy==1.3.0
|
||||
contourpy==1.3.2
|
||||
# via matplotlib
|
||||
cramjam==2.8.4
|
||||
cramjam==2.10.0
|
||||
# via fastparquet
|
||||
cycler==0.12.1
|
||||
# via matplotlib
|
||||
dnspython==2.6.1
|
||||
dnspython==2.7.0
|
||||
# via email-validator
|
||||
duckdb==1.1.1
|
||||
durationpy==0.8
|
||||
duckdb==1.3.2
|
||||
durationpy==0.10
|
||||
# via kubernetes
|
||||
email-validator==2.2.0
|
||||
# via pydantic
|
||||
et-xmlfile==1.1.0
|
||||
et-xmlfile==2.0.0
|
||||
# via openpyxl
|
||||
executing==2.1.0
|
||||
executing==2.2.0
|
||||
# via icecream
|
||||
fastapi==0.115.0
|
||||
fastparquet==2024.5.0
|
||||
flexcache==0.3
|
||||
# via pint
|
||||
flexparser==0.3.1
|
||||
# via pint
|
||||
fonttools==4.54.1
|
||||
fastparquet==2024.11.0
|
||||
fonttools==4.59.0
|
||||
# via matplotlib
|
||||
frozenlist==1.4.1
|
||||
# via
|
||||
# aiohttp
|
||||
# aiosignal
|
||||
fsspec==2024.9.0
|
||||
fsspec==2025.7.0
|
||||
# via
|
||||
# fastparquet
|
||||
# s3fs
|
||||
geopandas==1.0.1
|
||||
gitdb==4.0.11
|
||||
gitdb==4.0.12
|
||||
# via gitpython
|
||||
gitpython==3.1.43
|
||||
google-auth==2.35.0
|
||||
gitpython==3.1.44
|
||||
google-auth==2.40.3
|
||||
# via kubernetes
|
||||
h11==0.14.0
|
||||
# via uvicorn
|
||||
icecream==2.1.3
|
||||
icecream==2.1.5
|
||||
idna==3.10
|
||||
# via
|
||||
# anyio
|
||||
# email-validator
|
||||
# requests
|
||||
# yarl
|
||||
influxdb-client==1.46.0
|
||||
jmespath==1.0.1
|
||||
# via
|
||||
# boto3
|
||||
# botocore
|
||||
kiwisolver==1.4.7
|
||||
kiwisolver==1.4.8
|
||||
# via matplotlib
|
||||
kubernetes==31.0.0
|
||||
lxml==5.3.0
|
||||
matplotlib==3.9.2
|
||||
kubernetes==33.1.0
|
||||
lxml==6.0.0
|
||||
matplotlib==3.10.3
|
||||
# via seaborn
|
||||
multidict==6.1.0
|
||||
# via
|
||||
# aiohttp
|
||||
# yarl
|
||||
networkx==3.3
|
||||
numpy==2.1.1
|
||||
numpy==2.3.1
|
||||
# via
|
||||
# contourpy
|
||||
# fastparquet
|
||||
# geopandas
|
||||
# matplotlib
|
||||
# pandas
|
||||
# pyarrow
|
||||
# pyogrio
|
||||
# seaborn
|
||||
# shapely
|
||||
oauthlib==3.2.2
|
||||
oauthlib==3.3.1
|
||||
# via
|
||||
# kubernetes
|
||||
# requests-oauthlib
|
||||
openpyxl==3.1.5
|
||||
packaging==24.1
|
||||
packaging==25.0
|
||||
# via
|
||||
# fastparquet
|
||||
# geopandas
|
||||
# matplotlib
|
||||
# pyogrio
|
||||
pandas==2.2.3
|
||||
pandas==2.3.1
|
||||
# via
|
||||
# fastparquet
|
||||
# geopandas
|
||||
# pint-pandas
|
||||
# seaborn
|
||||
pillow==10.4.0
|
||||
pillow==11.3.0
|
||||
# via matplotlib
|
||||
pint==0.24.3
|
||||
# via pint-pandas
|
||||
pint-pandas==0.6.2
|
||||
pyarrow==17.0.0
|
||||
pyarrow==21.0.0
|
||||
pyasn1==0.6.1
|
||||
# via
|
||||
# pyasn1-modules
|
||||
# rsa
|
||||
pyasn1-modules==0.4.1
|
||||
pyasn1-modules==0.4.2
|
||||
# via google-auth
|
||||
pydantic==2.9.2
|
||||
# via
|
||||
# fastapi
|
||||
# pydantic-settings
|
||||
pydantic-core==2.23.4
|
||||
pydantic==2.11.7
|
||||
# via pydantic-settings
|
||||
pydantic-core==2.33.2
|
||||
# via pydantic
|
||||
pydantic-settings==2.5.2
|
||||
pygments==2.18.0
|
||||
pydantic-settings==2.10.1
|
||||
pygments==2.19.2
|
||||
# via icecream
|
||||
pyogrio==0.10.0
|
||||
# via geopandas
|
||||
pyparsing==3.1.4
|
||||
pyparsing==3.2.3
|
||||
# via matplotlib
|
||||
pyproj==3.7.0
|
||||
# via geopandas
|
||||
pysocks==1.7.1
|
||||
# via requests
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# botocore
|
||||
# influxdb-client
|
||||
# kubernetes
|
||||
# matplotlib
|
||||
# pandas
|
||||
python-dotenv==1.0.1
|
||||
python-dotenv==1.1.1
|
||||
# via pydantic-settings
|
||||
pytz==2024.2
|
||||
pytz==2025.2
|
||||
# via pandas
|
||||
pyyaml==6.0.2
|
||||
# via kubernetes
|
||||
reactivex==4.0.4
|
||||
# via influxdb-client
|
||||
requests==2.32.3
|
||||
requests==2.32.4
|
||||
# via
|
||||
# kubernetes
|
||||
# requests-oauthlib
|
||||
requests-oauthlib==2.0.0
|
||||
# via kubernetes
|
||||
rsa==4.9
|
||||
rsa==4.9.1
|
||||
# via google-auth
|
||||
s3fs==2024.9.0
|
||||
s3transfer==0.10.2
|
||||
s3fs==0.4.2
|
||||
s3transfer==0.13.1
|
||||
# via boto3
|
||||
seaborn==0.13.2
|
||||
setuptools==75.1.0
|
||||
# via influxdb-client
|
||||
shapely==2.0.6
|
||||
# via geopandas
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
# via
|
||||
# asttokens
|
||||
# kubernetes
|
||||
# python-dateutil
|
||||
smmap==5.0.1
|
||||
smmap==5.0.2
|
||||
# via gitdb
|
||||
sniffio==1.3.1
|
||||
# via anyio
|
||||
soupsieve==2.6
|
||||
soupsieve==2.7
|
||||
# via beautifulsoup4
|
||||
starlette==0.38.6
|
||||
# via fastapi
|
||||
structlog==24.4.0
|
||||
typing-extensions==4.12.2
|
||||
structlog==25.4.0
|
||||
typing-extensions==4.14.1
|
||||
# via
|
||||
# fastapi
|
||||
# flexcache
|
||||
# flexparser
|
||||
# pint
|
||||
# beautifulsoup4
|
||||
# pydantic
|
||||
# pydantic-core
|
||||
# reactivex
|
||||
tzdata==2024.2
|
||||
# typing-inspection
|
||||
typing-inspection==0.4.1
|
||||
# via
|
||||
# pydantic
|
||||
# pydantic-settings
|
||||
tzdata==2025.2
|
||||
# via pandas
|
||||
urllib3==2.2.3
|
||||
urllib3==2.5.0
|
||||
# via
|
||||
# botocore
|
||||
# influxdb-client
|
||||
# kubernetes
|
||||
# requests
|
||||
uvicorn==0.31.0
|
||||
websocket-client==1.8.0
|
||||
# via kubernetes
|
||||
wrapt==1.16.0
|
||||
# via aiobotocore
|
||||
xlsxwriter==3.2.0
|
||||
yarl==1.13.1
|
||||
# via aiohttp
|
||||
xlsxwriter==3.2.5
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
from app.vinyl.repo import vinyl # noqa
|
||||
@@ -1,4 +1,7 @@
|
||||
load_from:
|
||||
- grpc_server:
|
||||
host: user_code
|
||||
- grpc_server:
|
||||
host: user_code_vinyl
|
||||
port: 4000
|
||||
- grpc_server:
|
||||
host: user_code_other
|
||||
port: 4000
|
||||
|
||||
Reference in New Issue
Block a user