diff --git a/apps/backup/src/jobs.py b/apps/backup/src/jobs.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/backup/src/sensors.py b/apps/backup/src/sensors.py new file mode 100644 index 0000000..3cf45ac --- /dev/null +++ b/apps/backup/src/sensors.py @@ -0,0 +1,26 @@ +import structlog +from partitions import borg_repo_partitions_def +from utils.borg import get_ssh_client, list_repos + +import dagster as dg + +logger = structlog.get_logger() + + +@dg.sensor() +def borg_repos(context: dg.SensorEvaluationContext) -> dg.SensorResult: + existing_repos = set( + context.instance.get_dynamic_partitions(borg_repo_partitions_def.name) + ) + + with get_ssh_client() as client: + parent = "/mnt/yotta/xenon/borg/" + repos = set(list_repos(client, parent)) + + new_repos = list(set(repos) - existing_repos) + return dg.SensorResult( + run_requests=[dg.RunRequest(partition_key=repo) for repo in new_repos], + dynamic_partitions_requests=[ + borg_repo_partitions_def.build_add_request(new_repos), + ], + ) diff --git a/apps/backup/src/utils/__init__.py b/apps/backup/src/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/backup/src/utils/borg.py b/apps/backup/src/utils/borg.py new file mode 100644 index 0000000..445a953 --- /dev/null +++ b/apps/backup/src/utils/borg.py @@ -0,0 +1,54 @@ +from collections.abc import Iterator +from configparser import ConfigParser +from contextlib import contextmanager +from io import StringIO +from pathlib import Path + +import paramiko + + +@contextmanager +def get_ssh_client(): + ssh_config_file = Path.home() / ".ssh/config" + + with open(ssh_config_file) as f: + ssh_config = paramiko.SSHConfig() + ssh_config.parse(f) + + host_config = ssh_config.lookup("backup") # the host alias in ~/.ssh/config + + hostname = host_config.get("hostname", "localhost") + port = int(host_config.get("port", 22)) + username = host_config.get("user") + key_filename = host_config.get("identityfile", [None])[0] + + # Connect using Paramiko + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect( + hostname=hostname, port=port, username=username, key_filename=key_filename + ) + + yield client + + client.close() + + +def list_repos(client, parent) -> Iterator[str]: + command = f"ls {parent}*/config" + stdin, stdout, stderr = client.exec_command(command) + paths = [line.strip() for line in stdout.readlines()] + + sftp = client.open_sftp() + for path in paths: + with sftp.open(path, "r") as f: + try: + content = f.read().decode() + config = ConfigParser() + config.read_file(StringIO(content)) + config.get("repository", "version") + yield Path(path).parent.name + except Exception as e: + print(f"Not a borg repository: {e}") + + sftp.close() diff --git a/apps/backup/ssh_config b/apps/backup/ssh_config new file mode 100644 index 0000000..d7f6cf3 --- /dev/null +++ b/apps/backup/ssh_config @@ -0,0 +1,4 @@ +Host backup + HostName rik.veenboer.xyz + User backup + StrictHostKeyChecking no