55 lines
1.6 KiB
Python
55 lines
1.6 KiB
Python
import os
|
|
|
|
import assets
|
|
from config import APP
|
|
from dagster_polars import PolarsParquetIOManager
|
|
from icecream import install
|
|
from jobs import check_partitions_job, deals_job
|
|
from schedules import deals_schedule
|
|
from utils.email import EmailService
|
|
|
|
import dagster as dg
|
|
|
|
install()
|
|
|
|
|
|
def deep_merge_dicts(base: dict, override: dict) -> dict:
|
|
"""
|
|
Recursively merge two dictionaries.
|
|
Values from `override` will overwrite or be merged into `base`.
|
|
"""
|
|
result = base.copy()
|
|
for key, override_value in override.items():
|
|
base_value = result.get(key)
|
|
if isinstance(base_value, dict) and isinstance(override_value, dict):
|
|
result[key] = deep_merge_dicts(base_value, override_value)
|
|
else:
|
|
result[key] = override_value
|
|
return result
|
|
|
|
|
|
definitions = dg.Definitions(
|
|
assets=[
|
|
asset.with_attributes(
|
|
group_names_by_key={asset.key: APP},
|
|
tags_by_key={asset.key: {"app": APP}},
|
|
)
|
|
for asset in dg.load_assets_from_modules([assets])
|
|
],
|
|
resources={
|
|
"polars_parquet_io_manager": PolarsParquetIOManager(
|
|
base_dir=os.environ.get("STORAGE_DIR", "/storage")
|
|
),
|
|
"email_service": EmailService(
|
|
smtp_server=dg.EnvVar("SMTP_SERVER"),
|
|
smtp_port=dg.EnvVar.int("SMTP_PORT"),
|
|
smtp_username=dg.EnvVar("SMTP_USERNAME"),
|
|
smtp_password=dg.EnvVar("SMTP_PASSWORD"),
|
|
sender_email=dg.EnvVar("SENDER_EMAIL"),
|
|
receiver_email=dg.EnvVar("RECEIVER_EMAIL"),
|
|
),
|
|
},
|
|
jobs=[deals_job, check_partitions_job],
|
|
schedules=[deals_schedule],
|
|
)
|