inventory of borg backups

apps/backup/src/sensors.py

@@ -0,0 +1,26 @@
+import structlog
+from partitions import borg_repo_partitions_def
+from utils.borg import get_ssh_client, list_repos
+
+import dagster as dg
+
+logger = structlog.get_logger()
+
+
+@dg.sensor()
+def borg_repos(context: dg.SensorEvaluationContext) -> dg.SensorResult:
+    existing_repos = set(
+        context.instance.get_dynamic_partitions(borg_repo_partitions_def.name)
+    )
+
+    with get_ssh_client() as client:
+        parent = "/mnt/yotta/xenon/borg/"
+        repos = set(list_repos(client, parent))
+
+    new_repos = list(set(repos) - existing_repos)
+    return dg.SensorResult(
+        run_requests=[dg.RunRequest(partition_key=repo) for repo in new_repos],
+        dynamic_partitions_requests=[
+            borg_repo_partitions_def.build_add_request(new_repos),
+        ],
+    )

apps/backup/src/utils/borg.py

@@ -0,0 +1,54 @@
+from collections.abc import Iterator
+from configparser import ConfigParser
+from contextlib import contextmanager
+from io import StringIO
+from pathlib import Path
+
+import paramiko
+
+
+@contextmanager
+def get_ssh_client():
+    ssh_config_file = Path.home() / ".ssh/config"
+
+    with open(ssh_config_file) as f:
+        ssh_config = paramiko.SSHConfig()
+        ssh_config.parse(f)
+
+    host_config = ssh_config.lookup("backup")  # the host alias in ~/.ssh/config
+
+    hostname = host_config.get("hostname", "localhost")
+    port = int(host_config.get("port", 22))
+    username = host_config.get("user")
+    key_filename = host_config.get("identityfile", [None])[0]
+
+    # Connect using Paramiko
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    client.connect(
+        hostname=hostname, port=port, username=username, key_filename=key_filename
+    )
+
+    yield client
+
+    client.close()
+
+
+def list_repos(client, parent) -> Iterator[str]:
+    command = f"ls {parent}*/config"
+    stdin, stdout, stderr = client.exec_command(command)
+    paths = [line.strip() for line in stdout.readlines()]
+
+    sftp = client.open_sftp()
+    for path in paths:
+        with sftp.open(path, "r") as f:
+            try:
+                content = f.read().decode()
+                config = ConfigParser()
+                config.read_file(StringIO(content))
+                config.get("repository", "version")
+                yield Path(path).parent.name
+            except Exception as e:
+                print(f"Not a borg repository: {e}")
+
+    sftp.close()

apps/backup/ssh_config

@@ -0,0 +1,4 @@
+Host backup
+    HostName rik.veenboer.xyz
+    User backup
+    StrictHostKeyChecking no