towards detecting new deal

This commit is contained in:
2025-07-26 19:09:06 +02:00
parent 8d06b236b7
commit 66bcb3e2d3
4 changed files with 61 additions and 25 deletions

View File

@@ -2,7 +2,7 @@ import logging
from datetime import datetime
from typing import Any
from assets import cleaned_deals, deals, works
from assets import cleaned_deals, deals, new_deals, works
from dagster_polars import PolarsParquetIOManager
from definitions import definitions
from jobs import check_partitions_job
@@ -34,7 +34,7 @@ def test_deals(resources: dict[str, Any], source="sounds", date: str = None):
if __name__ == "__main__":
run = 3
run = 5
resources = {
"polars_parquet_io_manager": PolarsParquetIOManager(
base_dir="/opt/dagster/storage/vinyl"
@@ -61,5 +61,12 @@ if __name__ == "__main__":
selection=[works.key],
resources=resources,
)
case 5:
dg.materialize(
assets=definitions.assets,
selection=[new_deals.key],
partition_key=f"{today_str()}|{source}",
resources=resources,
)
case _:
raise ValueError(f"Invalid run number: {run}!")