import os import assets from config import APP from dagster_polars import PolarsParquetIOManager from icecream import install from jobs import check_partitions_job, deals_job from schedules import deals_schedule from utils.email import EmailService import dagster as dg install() def deep_merge_dicts(base: dict, override: dict) -> dict: """ Recursively merge two dictionaries. Values from `override` will overwrite or be merged into `base`. """ result = base.copy() for key, override_value in override.items(): base_value = result.get(key) if isinstance(base_value, dict) and isinstance(override_value, dict): result[key] = deep_merge_dicts(base_value, override_value) else: result[key] = override_value return result definitions = dg.Definitions( assets=[ asset.with_attributes( group_names_by_key={asset.key: APP}, tags_by_key={asset.key: {"app": APP}}, ) for asset in dg.load_assets_from_modules([assets]) ], resources={ "polars_parquet_io_manager": PolarsParquetIOManager( base_dir=os.environ.get("STORAGE_DIR", "/storage") ), "email_service": EmailService( smtp_server=dg.EnvVar("SMTP_SERVER"), smtp_port=dg.EnvVar.int("SMTP_PORT"), smtp_username=dg.EnvVar("SMTP_USERNAME"), smtp_password=dg.EnvVar("SMTP_PASSWORD"), sender_email=dg.EnvVar("SENDER_EMAIL"), receiver_email=dg.EnvVar("RECEIVER_EMAIL"), ), }, jobs=[deals_job, check_partitions_job], schedules=[deals_schedule], )