rearange borgmatic setup and configuration

borgmatic/Dockerfile

@@ -0,0 +1,4 @@
+FROM b3vis/borgmatic:2.0.7
+
+RUN apk update && apk add --no-cache jq mosquitto-clients
+

borgmatic/config/bram.yaml

@@ -1,38 +1,52 @@
-location:
+source_directories:
-    source_directories:
+    - /remote/server/bram
-        - /remote/server/bram
-    repositories:
-        - /repo/bram
 
-storage:
+repositories:
-    encryption_passcommand: cat /keys/bram.key
+    - path: /repo/bram
-    compression: lz4
-    archive_name_format: '{now:%Y-%m-%d}'
 
-retention:
+encryption_passcommand: cat /keys/bram.key
-    keep_daily: 7
-    keep_weekly: 4
-    keep_monthly: 6
-    prefix: '20'
 
-consistency:
+umask: 22
-    checks:
-        - repository
-        - archives
-    check_last: 3
-    prefix: '20'
 
-hooks:
+compression: lz4
-    before_backup:
+archive_name_format: '{now:%Y-%m-%d}'
-        - echo -e "\n\n"
+unknown_unencrypted_repo_access_is_ok: true
-        - echo "#"
+
-        - echo "# `date` - Starting a backup job."
+keep_daily: 7
-        - echo "#"
+keep_weekly: 4
-    after_backup:
+keep_monthly: 6
-        - echo "#"
+
-        - echo "# `date` - Backup created."
+checks:
-        - echo "#"
+    - name: repository
-    on_error:
+    - name: archives
-        - echo "#"
+check_last: 3
-        - echo "# `date` - Error while creating a backup."
+
-        - echo "#"
+commands:
+  - before: action
+    when:
+      - create
+    run:
+      - |
+        set -euo pipefail
+        REMOTE=/remote/server/bram
+        echo "Mounting $REMOTE..."
+        mkdir -p "$REMOTE"
+        sshfs user@bram.veenboer.xyz:/media/helios/Bram "$REMOTE"
+
+  - after: action
+    when:
+      - create
+    run:
+      - |
+        set -eu
+        REMOTE=/remote/server/bram
+        echo "Unmounting $REMOTE..."
+        if mountpoint -q "$REMOTE"; then
+          if command -v fusermount >/dev/null 2>&1; then
+            fusermount -u "$REMOTE"
+          else
+            umount "$REMOTE"
+          fi
+        else
+          echo "$REMOTE is not a mountpoint, nothing to unmount."
+        fi

borgmatic/config/config.yaml.default

@@ -1,170 +0,0 @@
-# Where to look for files to backup, and where to store those backups. See
-# https://borgbackup.readthedocs.io/en/stable/quickstart.html and
-# https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details.
-location:
-    # List of source directories to backup (required). Globs and tildes are expanded.
-    source_directories:
-        - /home
-        - /etc
-        - /var/log/syslog*
-
-    # Stay in same file system (do not cross mount points).
-    one_file_system: true
-
-    # Mode in which to operate the files cache. See
-    # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
-    # details.
-    files_cache: ctime,size,inode
-
-    # Alternate Borg local executable. Defaults to "borg".
-    local_path: borg1
-
-    # Alternate Borg remote executable. Defaults to "borg".
-    remote_path: borg1
-
-    # Paths to local or remote repositories (required). Tildes are expanded. Multiple
-    # repositories are backed up to in sequence. See ssh_command for SSH options like
-    # identity file or port.
-    repositories:
-        - user@backupserver:sourcehostname.borg
-
-    # Any paths matching these patterns are included/excluded from backups. Globs are
-    # expanded. (Tildes are not.) Note that Borg considers this option experimental.
-    # See the output of "borg help patterns" for more details. Quote any value if it
-    # contains leading punctuation, so it parses correctly.
-    patterns:
-        - R /
-        - '- /home/*/.cache'
-        - + /home/susan
-        - '- /home/*'
-
-    # Read include/exclude patterns from one or more separate named files, one pattern
-    # per line. Note that Borg considers this option experimental. See the output of
-    # "borg help patterns" for more details.
-    patterns_from:
-        - /etc/borgmatic/patterns
-
-    # Any paths matching these patterns are excluded from backups. Globs and tildes
-    # are expanded. See the output of "borg help patterns" for more details.
-    exclude_patterns:
-        - '*.pyc'
-        - ~/*/.cache
-        - /etc/ssl
-
-    # Read exclude patterns from one or more separate named files, one pattern per
-    # line. See the output of "borg help patterns" for more details.
-    exclude_from:
-        - /etc/borgmatic/excludes
-
-    # Exclude directories that contain a CACHEDIR.TAG file. See
-    # http://www.brynosaurus.com/cachedir/spec.html for details.
-    exclude_caches: true
-
-    # Exclude directories that contain a file with the given filename.
-    exclude_if_present: .nobackup
-
-# Repository storage options. See
-# https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and
-# https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for
-# details.
-storage:
-    # The standard output of this command is used to unlock the encryption key. Only
-    # use on repositories that were initialized with passcommand/repokey encryption.
-    # Note that if both encryption_passcommand and encryption_passphrase are set,
-    # then encryption_passphrase takes precedence.
-    encryption_passcommand: secret-tool lookup borg-repository repo-name
-
-    # Passphrase to unlock the encryption key with. Only use on repositories that were
-    # initialized with passphrase/repokey encryption. Quote the value if it contains
-    # punctuation, so it parses correctly. And backslash any quote or backslash
-    # literals as well.
-    encryption_passphrase: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
-
-    # Type of compression to use when creating archives. See
-    # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
-    # Defaults to no compression.
-    compression: lz4
-
-    # Remote network upload rate limit in kiBytes/second.
-    remote_rate_limit: 100
-
-    # Command to use instead of just "ssh". This can be used to specify ssh options.
-    ssh_command: ssh -i /path/to/private/key
-
-    # Umask to be used for borg create.
-    umask: 0077
-
-    # Maximum seconds to wait for acquiring a repository/cache lock.
-    lock_wait: 5
-
-    # Name of the archive. Borg placeholders can be used. See the output of
-    # "borg help placeholders" for details. Default is
-    # "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this option, you must
-    # also specify a prefix in the retention section to avoid accidental pruning of
-    # archives with a different archive name format.
-    archive_name_format: '{hostname}-documents-{now}'
-	
-    unknown_unencrypted_repo_access_is_ok: true
-
-# Retention policy for how many backups to keep in each category. See
-# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
-# At least one of the "keep" options is required for pruning to work.
-retention:
-    # Keep all archives within this time interval.
-    keep_within: 3H
-
-    # Number of minutely archives to keep.
-    keep_minutely: 60
-
-    # Number of hourly archives to keep.
-    keep_hourly: 24
-
-    # Number of daily archives to keep.
-    keep_daily: 7
-
-    # Number of weekly archives to keep.
-    keep_weekly: 4
-
-    # Number of monthly archives to keep.
-    keep_monthly: 6
-
-    # Number of yearly archives to keep.
-    keep_yearly: 1
-
-    # When pruning, only consider archive names starting with this prefix.
-    # Borg placeholders can be used. See the output of "borg help placeholders" for
-    # details. Default is "{hostname}-".
-    prefix: sourcehostname
-
-# Consistency checks to run after backups. See
-# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and
-# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details.
-consistency:
-    # List of one or more consistency checks to run: "repository", "archives", and/or
-    # "extract". Defaults to "repository" and "archives". Set to "disabled" to disable
-    # all consistency checks. "repository" checks the consistency of the repository,
-    # "archive" checks all of the archives, and "extract" does an extraction dry-run
-    # of just the most recent archive.
-    checks:
-        - repository
-        - archives
-
-    # Restrict the number of checked archives to the last n. Applies only to the "archives" check.
-    check_last: 3
-
-# Shell commands or scripts to execute before and after a backup or if an error has occurred.
-# IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic.
-# Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to
-# prevent potential shell injection or privilege escalation.
-hooks:
-    # List of one or more shell commands or scripts to execute before creating a backup.
-    before_backup:
-        - echo "`date` - Starting a backup job."
-
-    # List of one or more shell commands or scripts to execute after creating a backup.
-    after_backup:
-        - echo "`date` - Backup created."
-
-    # List of one or more shell commands or scripts to execute in case an exception has occurred.
-    on_error:
-        - echo "`date` - Error while creating a backup."

borgmatic/config/crontab.txt

@@ -1 +0,0 @@
-0 2 * * * /config/run.sh

borgmatic/config/etc.yaml

@@ -1,38 +1,20 @@
-location:
+source_directories:
-    source_directories:
+    - /shuttle/etc
-        - /shuttle/etc
-    repositories:
-        - /repo/etc
 
-storage:
+repositories:
-    compression: lz4
+    - path: /repo/etc
-    archive_name_format: '{now:%Y-%m-%d}'
-    unknown_unencrypted_repo_access_is_ok: true
 
-retention:
+umask: 22
-    keep_daily: 7
-    keep_weekly: 4
-    keep_monthly: 6
-    prefix: '20'
 
-consistency:
+compression: lz4
-    checks:
+archive_name_format: '{now:%Y-%m-%d}'
-        - repository
+unknown_unencrypted_repo_access_is_ok: true
-        - archives
-    check_last: 3
-    prefix: '20'
 
-hooks:
+keep_daily: 7
-    before_backup:
+keep_weekly: 4
-        - echo -e "\n\n"
+keep_monthly: 6
-        - echo "#"
+
-        - echo "# `date` - Starting a backup job."
+checks:
-        - echo "#"
+    - name: repository
-    after_backup:
+    - name: archives
-        - echo "#"
+check_last: 3
-        - echo "# `date` - Backup created."
-        - echo "#"
-    on_error:
-        - echo "#"
-        - echo "# `date` - Error while creating a backup."
-        - echo "#"

borgmatic/config/home.yaml

@@ -1,12 +1,15 @@
-location:
-    source_directories:
-        - /shuttle/home
-    patterns:
-        - '- /shuttle/home/.snapshot'
-        - '- /shuttle/home/*/.cache'
-    repositories:
-        - path: /repo/home
 
+source_directories:
+    - /shuttle/home
+
+exclude_patterns:
+    - /shuttle/home/.snapshot
+    - /shuttle/home/*/.cache
+
+repositories:
+    - path: /repo/home
+
+umask: 22
 
 compression: lz4
 archive_name_format: '{now:%Y-%m-%d}'
@@ -20,11 +23,3 @@ checks:
     - name: repository
     - name: archives
 check_last: 3
-
-# log_json: true
-loki:
-    url: http://host:3100/loki/api/v1/push
-    labels:
-        app: borgmatic
-        config: __config
-        hostname: __hostname

borgmatic/config/opt.yaml

@@ -1,44 +1,27 @@
-location:
+source_directories:
-    source_directories:
+    - /shuttle/opt
-        - /shuttle/opt
-    patterns:
-        - '- /shuttle/opt/.snapshot'
-        - '- /shuttle/opt/jupyter/conda'
-        - '- /shuttle/opt/emby/metadata'
-        - '- /shuttle/opt/btsync/var/*.journal*'
-        - '- /shuttle/opt/openvpn-server/*.log'
-    repositories:
-        - /repo/opt
 
-storage:
+exclude_patterns:
-    compression: lz4
+    - /shuttle/opt/.snapshot
-    archive_name_format: '{now:%Y-%m-%d}'
+    - /shuttle/opt/jupyter/conda
-    unknown_unencrypted_repo_access_is_ok: true
+    - /shuttle/opt/emby/metadata
+    - /shuttle/opt/btsync/var/*.journal*
+    - /shuttle/opt/openvpn-server/*.log
 
-retention:
+repositories:
-    keep_daily: 7
+    - path: /repo/opt
-    keep_weekly: 4
-    keep_monthly: 6
-    prefix: '20'
 
-consistency:
+umask: 22
-    checks:
-        - repository
-        - archives
-    check_last: 3
-    prefix: '20'
 
-hooks:
+compression: lz4
-    before_backup:
+archive_name_format: '{now:%Y-%m-%d}'
-        - echo -e "\n\n"
+unknown_unencrypted_repo_access_is_ok: true
-        - echo "#"
+
-        - echo "# `date` - Starting a backup job."
+keep_daily: 7
-        - echo "#"
+keep_weekly: 4
-    after_backup:
+keep_monthly: 6
-        - echo "#"
+
-        - echo "# `date` - Backup created."
+checks:
-        - echo "#"
+    - name: repository
-    on_error:
+    - name: archives
-        - echo "#"
+check_last: 3
-        - echo "# `date` - Error while creating a backup."
-        - echo "#"

borgmatic/config/root.yaml

@@ -1,40 +1,23 @@
-location:
+source_directories:
-    source_directories:
+    - /shuttle/root
-        - /shuttle/root
-    patterns:
-        - '- /shuttle/root/.cache'
-    repositories:
-        - /repo/root
 
-storage:
+exclude_patterns:
-    compression: lz4
+    - /shuttle/root/.cache
-    archive_name_format: '{now:%Y-%m-%d}'
-    unknown_unencrypted_repo_access_is_ok: true
 
-retention:
+repositories:
-    keep_daily: 7
+    - path: /repo/root
-    keep_weekly: 4
-    keep_monthly: 6
-    prefix: '20'
 
-consistency:
+umask: 22
-    checks:
-        - repository
-        - archives
-    check_last: 3
-    prefix: '20'
 
-hooks:
+compression: lz4
-    before_backup:
+archive_name_format: '{now:%Y-%m-%d}'
-        - echo -e "\n\n"
+unknown_unencrypted_repo_access_is_ok: true
-        - echo "#"
+
-        - echo "# `date` - Starting a backup job."
+keep_daily: 7
-        - echo "#"
+keep_weekly: 4
-    after_backup:
+keep_monthly: 6
-        - echo "#"
+
-        - echo "# `date` - Backup created."
+checks:
-        - echo "#"
+    - name: repository
-    on_error:
+    - name: archives
-        - echo "#"
+check_last: 3
-        - echo "# `date` - Error while creating a backup."
-        - echo "#"

borgmatic/config/test.yaml

@@ -0,0 +1,18 @@
+source_directories:
+    - /tmp/
+
+repositories:
+    - path: /repo/test
+
+umask: 22
+
+compression: lz4
+archive_name_format: '{now:%Y-%m-%d_%H%M%S}'
+
+unknown_unencrypted_repo_access_is_ok: true
+relocated_repo_access_is_ok: true
+
+keep_hourly: 24
+keep_daily: 7
+keep_weekly: 4
+keep_monthly: 6

borgmatic/crontab.txt

@@ -0,0 +1 @@
+0 2 * * * /scripts/run.sh

borgmatic/scripts/mqtt.sh

@@ -0,0 +1,26 @@
+CONFIG_FILE="$1"
+
+MQTT_BROKER="host"
+MQTT_TOPIC="borgmatic/stats/$CONFIG_FILE"
+MQTT_USER="borgmatic"
+MQTT_PASSWORD="eH2dAiG7siCIFdB7qX0MHwEFzcr9aqUi"
+
+# Run Borgmatic and extract archive stats as JSON
+PAYLOAD=$(borgmatic \
+    --stats \
+    -v 0 \
+    -c "/config/$CONFIG_FILE.yaml" \
+    --log-json \
+    --no-color \
+    --json \
+    | tee /dev/tty \
+    | sed --quiet '/^\[/p' \
+    | jq '.[0].archive'
+)
+
+# Publish to MQTT
+mosquitto_pub -h "$MQTT_BROKER" \
+               -t "$MQTT_TOPIC" \
+               -u "$MQTT_USER" \
+               -P "$MQTT_PASSWORD" \
+               -m "$PAYLOAD"

borgmatic/scripts/run.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+for name in home root opt etc bram; do
+    /scripts/mqtt.sh "$NAME" >> "/log/$NAME.log" 2>&1
+done