import logging from datetime import datetime from typing import Any from assets import cleaned_deals, deals, good_deals, new_deals, works from dotenv import find_dotenv, load_dotenv from jobs import check_partitions_job import dagster as dg logging.getLogger().setLevel(logging.INFO) def today_str(): """Returns today's date as a string in the format YYYY-MM-DD.""" return datetime.today().strftime("%Y-%m-%d") def test_deals(resources: dict[str, Any], source="sounds", date: str = None): result = dg.materialize( assets=definitions.assets, selection=[deals.key], partition_key=f"{date or today_str()}|{source}", resources=resources, run_config={ "loggers": {"console": {"config": {"log_level": "ERROR"}}}, "ops": {"deals": {"config": {"import_dir": "/storage/import"}}}, }, ) assert result.success if __name__ == "__main__": load_dotenv(find_dotenv()) from definitions import definitions run = 6 resources = definitions.resources source = "plato" match run: case 1: check_partitions_job.execute_in_process(resources=resources) case 2: test_deals(resources, source=source) case 3: dg.materialize( assets=definitions.assets, selection=[cleaned_deals.key], partition_key=f"{today_str()}|{source}", resources=resources, ) case 4: dg.materialize( assets=definitions.assets, selection=[works.key], resources=resources, ) case 5: dg.materialize( assets=definitions.assets, selection=[new_deals.key], partition_key=f"{today_str()}|{source}", resources=resources, ) case 6: dg.materialize( assets=definitions.assets, selection=[good_deals.key], partition_key=f"{today_str()}", resources=resources, ) case _: raise ValueError(f"Invalid run number: {run}!")