diff --git a/borgmatic/Dockerfile b/borgmatic/Dockerfile new file mode 100644 index 0000000..f598d3f --- /dev/null +++ b/borgmatic/Dockerfile @@ -0,0 +1,4 @@ +FROM b3vis/borgmatic:2.0.7 + +RUN apk update && apk add --no-cache jq mosquitto-clients + diff --git a/borgmatic/config/bram.yaml b/borgmatic/config/bram.yaml index 5ab785e..be2a7f2 100644 --- a/borgmatic/config/bram.yaml +++ b/borgmatic/config/bram.yaml @@ -1,38 +1,52 @@ -location: - source_directories: - - /remote/server/bram - repositories: - - /repo/bram +source_directories: + - /remote/server/bram -storage: - encryption_passcommand: cat /keys/bram.key - compression: lz4 - archive_name_format: '{now:%Y-%m-%d}' +repositories: + - path: /repo/bram -retention: - keep_daily: 7 - keep_weekly: 4 - keep_monthly: 6 - prefix: '20' +encryption_passcommand: cat /keys/bram.key -consistency: - checks: - - repository - - archives - check_last: 3 - prefix: '20' +umask: 22 -hooks: - before_backup: - - echo -e "\n\n" - - echo "#" - - echo "# `date` - Starting a backup job." - - echo "#" - after_backup: - - echo "#" - - echo "# `date` - Backup created." - - echo "#" - on_error: - - echo "#" - - echo "# `date` - Error while creating a backup." - - echo "#" +compression: lz4 +archive_name_format: '{now:%Y-%m-%d}' +unknown_unencrypted_repo_access_is_ok: true + +keep_daily: 7 +keep_weekly: 4 +keep_monthly: 6 + +checks: + - name: repository + - name: archives +check_last: 3 + +commands: + - before: action + when: + - create + run: + - | + set -euo pipefail + REMOTE=/remote/server/bram + echo "Mounting $REMOTE..." + mkdir -p "$REMOTE" + sshfs user@bram.veenboer.xyz:/media/helios/Bram "$REMOTE" + + - after: action + when: + - create + run: + - | + set -eu + REMOTE=/remote/server/bram + echo "Unmounting $REMOTE..." + if mountpoint -q "$REMOTE"; then + if command -v fusermount >/dev/null 2>&1; then + fusermount -u "$REMOTE" + else + umount "$REMOTE" + fi + else + echo "$REMOTE is not a mountpoint, nothing to unmount." + fi diff --git a/borgmatic/config/config.yaml.default b/borgmatic/config/config.yaml.default deleted file mode 100644 index 5a2c047..0000000 --- a/borgmatic/config/config.yaml.default +++ /dev/null @@ -1,170 +0,0 @@ -# Where to look for files to backup, and where to store those backups. See -# https://borgbackup.readthedocs.io/en/stable/quickstart.html and -# https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details. -location: - # List of source directories to backup (required). Globs and tildes are expanded. - source_directories: - - /home - - /etc - - /var/log/syslog* - - # Stay in same file system (do not cross mount points). - one_file_system: true - - # Mode in which to operate the files cache. See - # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for - # details. - files_cache: ctime,size,inode - - # Alternate Borg local executable. Defaults to "borg". - local_path: borg1 - - # Alternate Borg remote executable. Defaults to "borg". - remote_path: borg1 - - # Paths to local or remote repositories (required). Tildes are expanded. Multiple - # repositories are backed up to in sequence. See ssh_command for SSH options like - # identity file or port. - repositories: - - user@backupserver:sourcehostname.borg - - # Any paths matching these patterns are included/excluded from backups. Globs are - # expanded. (Tildes are not.) Note that Borg considers this option experimental. - # See the output of "borg help patterns" for more details. Quote any value if it - # contains leading punctuation, so it parses correctly. - patterns: - - R / - - '- /home/*/.cache' - - + /home/susan - - '- /home/*' - - # Read include/exclude patterns from one or more separate named files, one pattern - # per line. Note that Borg considers this option experimental. See the output of - # "borg help patterns" for more details. - patterns_from: - - /etc/borgmatic/patterns - - # Any paths matching these patterns are excluded from backups. Globs and tildes - # are expanded. See the output of "borg help patterns" for more details. - exclude_patterns: - - '*.pyc' - - ~/*/.cache - - /etc/ssl - - # Read exclude patterns from one or more separate named files, one pattern per - # line. See the output of "borg help patterns" for more details. - exclude_from: - - /etc/borgmatic/excludes - - # Exclude directories that contain a CACHEDIR.TAG file. See - # http://www.brynosaurus.com/cachedir/spec.html for details. - exclude_caches: true - - # Exclude directories that contain a file with the given filename. - exclude_if_present: .nobackup - -# Repository storage options. See -# https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and -# https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for -# details. -storage: - # The standard output of this command is used to unlock the encryption key. Only - # use on repositories that were initialized with passcommand/repokey encryption. - # Note that if both encryption_passcommand and encryption_passphrase are set, - # then encryption_passphrase takes precedence. - encryption_passcommand: secret-tool lookup borg-repository repo-name - - # Passphrase to unlock the encryption key with. Only use on repositories that were - # initialized with passphrase/repokey encryption. Quote the value if it contains - # punctuation, so it parses correctly. And backslash any quote or backslash - # literals as well. - encryption_passphrase: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" - - # Type of compression to use when creating archives. See - # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details. - # Defaults to no compression. - compression: lz4 - - # Remote network upload rate limit in kiBytes/second. - remote_rate_limit: 100 - - # Command to use instead of just "ssh". This can be used to specify ssh options. - ssh_command: ssh -i /path/to/private/key - - # Umask to be used for borg create. - umask: 0077 - - # Maximum seconds to wait for acquiring a repository/cache lock. - lock_wait: 5 - - # Name of the archive. Borg placeholders can be used. See the output of - # "borg help placeholders" for details. Default is - # "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this option, you must - # also specify a prefix in the retention section to avoid accidental pruning of - # archives with a different archive name format. - archive_name_format: '{hostname}-documents-{now}' - - unknown_unencrypted_repo_access_is_ok: true - -# Retention policy for how many backups to keep in each category. See -# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details. -# At least one of the "keep" options is required for pruning to work. -retention: - # Keep all archives within this time interval. - keep_within: 3H - - # Number of minutely archives to keep. - keep_minutely: 60 - - # Number of hourly archives to keep. - keep_hourly: 24 - - # Number of daily archives to keep. - keep_daily: 7 - - # Number of weekly archives to keep. - keep_weekly: 4 - - # Number of monthly archives to keep. - keep_monthly: 6 - - # Number of yearly archives to keep. - keep_yearly: 1 - - # When pruning, only consider archive names starting with this prefix. - # Borg placeholders can be used. See the output of "borg help placeholders" for - # details. Default is "{hostname}-". - prefix: sourcehostname - -# Consistency checks to run after backups. See -# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and -# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details. -consistency: - # List of one or more consistency checks to run: "repository", "archives", and/or - # "extract". Defaults to "repository" and "archives". Set to "disabled" to disable - # all consistency checks. "repository" checks the consistency of the repository, - # "archive" checks all of the archives, and "extract" does an extraction dry-run - # of just the most recent archive. - checks: - - repository - - archives - - # Restrict the number of checked archives to the last n. Applies only to the "archives" check. - check_last: 3 - -# Shell commands or scripts to execute before and after a backup or if an error has occurred. -# IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic. -# Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to -# prevent potential shell injection or privilege escalation. -hooks: - # List of one or more shell commands or scripts to execute before creating a backup. - before_backup: - - echo "`date` - Starting a backup job." - - # List of one or more shell commands or scripts to execute after creating a backup. - after_backup: - - echo "`date` - Backup created." - - # List of one or more shell commands or scripts to execute in case an exception has occurred. - on_error: - - echo "`date` - Error while creating a backup." diff --git a/borgmatic/config/crontab.txt b/borgmatic/config/crontab.txt deleted file mode 100644 index 0f6b500..0000000 --- a/borgmatic/config/crontab.txt +++ /dev/null @@ -1 +0,0 @@ -0 2 * * * /config/run.sh diff --git a/borgmatic/config/etc.yaml b/borgmatic/config/etc.yaml index 6aac3f3..ac467cb 100644 --- a/borgmatic/config/etc.yaml +++ b/borgmatic/config/etc.yaml @@ -1,38 +1,20 @@ -location: - source_directories: - - /shuttle/etc - repositories: - - /repo/etc +source_directories: + - /shuttle/etc -storage: - compression: lz4 - archive_name_format: '{now:%Y-%m-%d}' - unknown_unencrypted_repo_access_is_ok: true +repositories: + - path: /repo/etc -retention: - keep_daily: 7 - keep_weekly: 4 - keep_monthly: 6 - prefix: '20' +umask: 22 -consistency: - checks: - - repository - - archives - check_last: 3 - prefix: '20' +compression: lz4 +archive_name_format: '{now:%Y-%m-%d}' +unknown_unencrypted_repo_access_is_ok: true -hooks: - before_backup: - - echo -e "\n\n" - - echo "#" - - echo "# `date` - Starting a backup job." - - echo "#" - after_backup: - - echo "#" - - echo "# `date` - Backup created." - - echo "#" - on_error: - - echo "#" - - echo "# `date` - Error while creating a backup." - - echo "#" +keep_daily: 7 +keep_weekly: 4 +keep_monthly: 6 + +checks: + - name: repository + - name: archives +check_last: 3 diff --git a/borgmatic/config/home.yaml b/borgmatic/config/home.yaml index 0cd5248..e1441ce 100644 --- a/borgmatic/config/home.yaml +++ b/borgmatic/config/home.yaml @@ -1,12 +1,15 @@ -location: - source_directories: - - /shuttle/home - patterns: - - '- /shuttle/home/.snapshot' - - '- /shuttle/home/*/.cache' - repositories: - - path: /repo/home +source_directories: + - /shuttle/home + +exclude_patterns: + - /shuttle/home/.snapshot + - /shuttle/home/*/.cache + +repositories: + - path: /repo/home + +umask: 22 compression: lz4 archive_name_format: '{now:%Y-%m-%d}' @@ -20,11 +23,3 @@ checks: - name: repository - name: archives check_last: 3 - -# log_json: true -loki: - url: http://host:3100/loki/api/v1/push - labels: - app: borgmatic - config: __config - hostname: __hostname diff --git a/borgmatic/config/opt.yaml b/borgmatic/config/opt.yaml index 89b04fc..176d587 100644 --- a/borgmatic/config/opt.yaml +++ b/borgmatic/config/opt.yaml @@ -1,44 +1,27 @@ -location: - source_directories: - - /shuttle/opt - patterns: - - '- /shuttle/opt/.snapshot' - - '- /shuttle/opt/jupyter/conda' - - '- /shuttle/opt/emby/metadata' - - '- /shuttle/opt/btsync/var/*.journal*' - - '- /shuttle/opt/openvpn-server/*.log' - repositories: - - /repo/opt +source_directories: + - /shuttle/opt -storage: - compression: lz4 - archive_name_format: '{now:%Y-%m-%d}' - unknown_unencrypted_repo_access_is_ok: true +exclude_patterns: + - /shuttle/opt/.snapshot + - /shuttle/opt/jupyter/conda + - /shuttle/opt/emby/metadata + - /shuttle/opt/btsync/var/*.journal* + - /shuttle/opt/openvpn-server/*.log -retention: - keep_daily: 7 - keep_weekly: 4 - keep_monthly: 6 - prefix: '20' +repositories: + - path: /repo/opt -consistency: - checks: - - repository - - archives - check_last: 3 - prefix: '20' +umask: 22 -hooks: - before_backup: - - echo -e "\n\n" - - echo "#" - - echo "# `date` - Starting a backup job." - - echo "#" - after_backup: - - echo "#" - - echo "# `date` - Backup created." - - echo "#" - on_error: - - echo "#" - - echo "# `date` - Error while creating a backup." - - echo "#" +compression: lz4 +archive_name_format: '{now:%Y-%m-%d}' +unknown_unencrypted_repo_access_is_ok: true + +keep_daily: 7 +keep_weekly: 4 +keep_monthly: 6 + +checks: + - name: repository + - name: archives +check_last: 3 diff --git a/borgmatic/config/root.yaml b/borgmatic/config/root.yaml index bffb083..f9ee6ab 100644 --- a/borgmatic/config/root.yaml +++ b/borgmatic/config/root.yaml @@ -1,40 +1,23 @@ -location: - source_directories: - - /shuttle/root - patterns: - - '- /shuttle/root/.cache' - repositories: - - /repo/root +source_directories: + - /shuttle/root -storage: - compression: lz4 - archive_name_format: '{now:%Y-%m-%d}' - unknown_unencrypted_repo_access_is_ok: true +exclude_patterns: + - /shuttle/root/.cache -retention: - keep_daily: 7 - keep_weekly: 4 - keep_monthly: 6 - prefix: '20' +repositories: + - path: /repo/root -consistency: - checks: - - repository - - archives - check_last: 3 - prefix: '20' +umask: 22 -hooks: - before_backup: - - echo -e "\n\n" - - echo "#" - - echo "# `date` - Starting a backup job." - - echo "#" - after_backup: - - echo "#" - - echo "# `date` - Backup created." - - echo "#" - on_error: - - echo "#" - - echo "# `date` - Error while creating a backup." - - echo "#" +compression: lz4 +archive_name_format: '{now:%Y-%m-%d}' +unknown_unencrypted_repo_access_is_ok: true + +keep_daily: 7 +keep_weekly: 4 +keep_monthly: 6 + +checks: + - name: repository + - name: archives +check_last: 3 diff --git a/borgmatic/config/test.yaml b/borgmatic/config/test.yaml new file mode 100644 index 0000000..2c86f5f --- /dev/null +++ b/borgmatic/config/test.yaml @@ -0,0 +1,18 @@ +source_directories: + - /tmp/ + +repositories: + - path: /repo/test + +umask: 22 + +compression: lz4 +archive_name_format: '{now:%Y-%m-%d_%H%M%S}' + +unknown_unencrypted_repo_access_is_ok: true +relocated_repo_access_is_ok: true + +keep_hourly: 24 +keep_daily: 7 +keep_weekly: 4 +keep_monthly: 6 diff --git a/borgmatic/crontab.txt b/borgmatic/crontab.txt new file mode 100644 index 0000000..76e8d93 --- /dev/null +++ b/borgmatic/crontab.txt @@ -0,0 +1 @@ +0 2 * * * /scripts/run.sh diff --git a/borgmatic/scripts/mqtt.sh b/borgmatic/scripts/mqtt.sh new file mode 100755 index 0000000..07bb33a --- /dev/null +++ b/borgmatic/scripts/mqtt.sh @@ -0,0 +1,26 @@ +CONFIG_FILE="$1" + +MQTT_BROKER="host" +MQTT_TOPIC="borgmatic/stats/$CONFIG_FILE" +MQTT_USER="borgmatic" +MQTT_PASSWORD="eH2dAiG7siCIFdB7qX0MHwEFzcr9aqUi" + +# Run Borgmatic and extract archive stats as JSON +PAYLOAD=$(borgmatic \ + --stats \ + -v 0 \ + -c "/config/$CONFIG_FILE.yaml" \ + --log-json \ + --no-color \ + --json \ + | tee /dev/tty \ + | sed --quiet '/^\[/p' \ + | jq '.[0].archive' +) + +# Publish to MQTT +mosquitto_pub -h "$MQTT_BROKER" \ + -t "$MQTT_TOPIC" \ + -u "$MQTT_USER" \ + -P "$MQTT_PASSWORD" \ + -m "$PAYLOAD" diff --git a/borgmatic/config/run.sh b/borgmatic/scripts/run-old.sh old mode 100755 new mode 100644 similarity index 100% rename from borgmatic/config/run.sh rename to borgmatic/scripts/run-old.sh diff --git a/borgmatic/scripts/run.sh b/borgmatic/scripts/run.sh new file mode 100755 index 0000000..c47f389 --- /dev/null +++ b/borgmatic/scripts/run.sh @@ -0,0 +1,4 @@ +#!/bin/bash +for name in home root opt etc bram; do + /scripts/mqtt.sh "$NAME" >> "/log/$NAME.log" 2>&1 +done