towards detecting new deal
This commit is contained in:
@@ -2,7 +2,7 @@ import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from assets import cleaned_deals, deals, works
|
||||
from assets import cleaned_deals, deals, new_deals, works
|
||||
from dagster_polars import PolarsParquetIOManager
|
||||
from definitions import definitions
|
||||
from jobs import check_partitions_job
|
||||
@@ -34,7 +34,7 @@ def test_deals(resources: dict[str, Any], source="sounds", date: str = None):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run = 3
|
||||
run = 5
|
||||
resources = {
|
||||
"polars_parquet_io_manager": PolarsParquetIOManager(
|
||||
base_dir="/opt/dagster/storage/vinyl"
|
||||
@@ -61,5 +61,12 @@ if __name__ == "__main__":
|
||||
selection=[works.key],
|
||||
resources=resources,
|
||||
)
|
||||
case 5:
|
||||
dg.materialize(
|
||||
assets=definitions.assets,
|
||||
selection=[new_deals.key],
|
||||
partition_key=f"{today_str()}|{source}",
|
||||
resources=resources,
|
||||
)
|
||||
case _:
|
||||
raise ValueError(f"Invalid run number: {run}!")
|
||||
|
||||
Reference in New Issue
Block a user