snapshot remaining files

This commit is contained in:
2024-12-19 15:13:29 +01:00
parent d9f760ab2a
commit efac98f87c
28 changed files with 1818 additions and 0 deletions

4
.gitignore vendored
View File

@@ -36,6 +36,10 @@ openvpn-server/pki/
openvpn-server/*.ovpn
pgadmin/
rsnapshot/var/
seafile/seafile-data/
seafile/seafile-server-*
seafile/seahub-data/
seafile/database/
seafile/server/
inverter/data/
grafana/

1077
collectd/etc/collectd.conf Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,5 @@
#LoadPlugin exec
#
#<Plugin "exec">
# Exec nobody "/host/usr/local/bin/btrfs-data"
#</Plugin>

View File

@@ -0,0 +1,6 @@
<Plugin df>
MountPoint "/media/docker"
MountPoint "/media/scratch"
FSType "ext4"
IgnoreSelected false
</Plugin>

View File

@@ -0,0 +1,6 @@
LoadPlugin exec
<Plugin "exec">
Exec nobody "/host/usr/local/bin/du-data"
</Plugin>

View File

@@ -0,0 +1,5 @@
LoadPlugin exec
<Plugin "exec">
Exec collectd "/host/usr/local/bin/power-data"
</Plugin>

View File

@@ -0,0 +1,6 @@
LoadPlugin exec
<Plugin "exec">
Exec nobody "/host/usr/local/bin/speedtest-data"
</Plugin>

180
collectd/usr/local/bin/btrfs-data Executable file
View File

@@ -0,0 +1,180 @@
#!/usr/bin/python
#
# Imports
#
import sys
import time
import commands
import argparse
#
# Misc
#
#sys.tracebacklimit = 0
#
# Global variables
#
size_data_total = 0
size_data_exclusive = 0
size_snapshot_total = 0
size_snapshot_exclusive = 0
#
# Methods
#
def get_subvol_list(path):
command = "btrfs subvolume list -t %s" % (path)
status, output = commands.getstatusoutput(command)
if status is not 0:
raise Exception(command)
# Every line contains the following values: subvol_id, gen, toplevel, path
return output.splitlines()[2:]
def get_id_root(name, path):
lines = get_subvol_list(path)
# Filter lines where toplevel == 5
subvol_ids = filter(lambda x: int(x.split()[2]) == 5, lines)
# Try to retrieve the subvol_id for the root subvolume (if any)
if len(subvol_ids) == 1:
# The path contains a btrfs filesystem without subvolume for data
return int(subvol_ids[0].split()[0])
else:
# The path contains a btrfs filesystem with multiple subvolumes for data
try:
return int(filter(lambda x: x.split()[3] == name, subvol_ids)[0].split()[0])
except IndexError:
pass
# Volume not found, root is probably the btrfs default (5)
return 5
def get_id_subvolumes(path, subvol_id):
lines = get_subvol_list(path)
lines = filter(lambda x: int(x.split()[2]) == subvol_id, lines)
return list(map(lambda x: int(x.split()[0]), lines))
def get_disk_usage(name, path):
id_root = get_id_root(name, path)
id_subvolumes = get_id_subvolumes(path, id_root)
command = "btrfs qgroup show --raw %s" % (path)
status, output = commands.getstatusoutput(command)
if status is not 0:
raise Exception(command)
lines = output.splitlines()[2:]
# Global variables
global size_data_total
global size_data_exclusive
global size_snapshot_total
global size_snapshot_exclusive
# Total data volume in subvolume
size_data_total = 0
# Total data volume in snapshots
# -> this variable is useless
size_snapshot_total = 0
# Data exclusively in subvolume
# -> data that is not (yet) incorporated in a snapshot
size_data_exclusive = 0
# Data exclusively available in snapshots
# -> data that was removed from volume
size_snapshot_exclusive = 0
for line in lines:
split = line.split()
subvol_id = int(split[0].split("/")[1])
size_total = float(split[1])
size_exclusive = float(split[2])
if subvol_id == id_root:
size_data_total = size_total
size_data_exclusive = size_exclusive
elif subvol_id in id_subvolumes:
size_snapshot_total += size_total
size_snapshot_exclusive += size_exclusive
def rescan_quota(path):
command = "btrfs quota rescan %s" % (path)
status, output = commands.getstatusoutput(command)
if status is not 0:
Exception(command)
def print_human_readable(name):
global size_data_total
global size_data_exclusive
global size_snapshot_exclusive
size_data_total = size_data_total / (1024*1e6)
size_data_exclusive = size_data_exclusive / (1024*1e6)
size_snapshot_exclusive = size_snapshot_exclusive / (1024*1e6)
print "%10s: %6.1f Gb, %6.1f Gb, %6.1f Gb" % (name, size_data_total, size_data_exclusive, size_snapshot_exclusive)
def print_rrd(name):
timestamp = int(time.time())
print("PUTVAL {}/exec-btrfs_{}/gauge-data_total {}:{:.1f}".format(hostname, name, timestamp, size_data_total))
print("PUTVAL {}/exec-btrfs_{}/gauge-data_exclusive {}:{:.1f}".format(hostname, name, timestamp, size_data_exclusive))
print("PUTVAL {}/exec-btrfs_{}/gauge-snapshot_total {}:{:.1f}".format(hostname, name, timestamp, size_snapshot_total))
print("PUTVAL {}/exec-btrfs_{}/gauge-snapshot_exclusive {}:{:.1f}".format(hostname, name, timestamp, size_snapshot_exclusive))
#
# Volumes to scan
#
hostname = "sepia"
interval = 10
volumes = list()
# 275 GB SSD
volumes.append(["@", "/host/root/"])
volumes.append(["@home", "/host/root/home"])
volumes.append(["opt", "/host/root/opt"])
# 2x 4TB HDD
volumes.append(["data", "/host/root/media/data"])
#
# Command line arguments
#
parser = argparse.ArgumentParser(description='Get BTRFS disk usage')
parser.add_argument('-s', action='store_true', help='print in human readable format')
args = parser.parse_args()
human_readable = args.s
#
# Main
#
if (human_readable):
for (name, path) in volumes:
get_disk_usage(name, path)
print_human_readable(name)
else:
# RRD mode
while True:
for (name, path) in volumes:
get_disk_usage(name, path)
print_rrd(name)
sys.stdout.flush()
time.sleep(interval)
#rescan_quota(path)

25
collectd/usr/local/bin/du-data Executable file
View File

@@ -0,0 +1,25 @@
#!/bin/bash
COLLECTION=sepia
INTERVAL=90
DIRS=$(cat <<LIST
/host/root/media/data/Inverter
/host/root/media/data/Monique
/host/root/media/data/Music
/host/root/media/data/Peter
/host/root/media/data/Photographs
/host/root/media/data/Raw
/host/root/media/data/Sanne
/host/root/media/data/Wii
LIST
)
while :; do
SECONDS=0
for DIR in $DIRS; do
SIZE=$(du -cs $DIR | tail -1 | awk '{print $1}')
NAME=$(echo $DIR | sed 's/.//' | tr / - )
echo "PUTVAL $COLLECTION/exec-du-$NAME/gauge-size interval=$INTERVAL N:$SIZE"
done
sleep $((INTERVAL-$SECONDS))
done

View File

@@ -0,0 +1,69 @@
#!/usr/bin/python
#
# Imports
#
import sys
import time
import argparse
import pylikwid
#
# Configuration
#
hostname = "sepia"
cpuid = 0
pinfo = pylikwid.getpowerinfo()
domainid = pinfo.get("domains").get("PKG").get("ID")
measurement_duration = 5
measurement_interval = 15
dinfo = pinfo.get("domains")
domain_names = dinfo.keys()
domain_ids = [domain['ID'] for domain in dinfo.values()]
#
# Command line arguments
#
parser = argparse.ArgumentParser(description='Get CPU power consumption')
parser.add_argument('-s', action='store_true', help='print in human readable format')
args = parser.parse_args()
human_readable = args.s
#
# Methods
#
def get_power():
start = list()
end = list()
power = list()
for domain_id in domain_ids:
e_start = pylikwid.startpower(cpuid, domain_id)
start.append(e_start)
time.sleep(measurement_duration)
for domain_id in domain_ids:
e_stop = pylikwid.stoppower(cpuid, domain_id)
end.append(e_stop)
for events in zip(start, end, domain_ids):
joules = pylikwid.getpower(events[0], events[1], events[2])
power.append(joules / measurement_duration)
return dict(zip(domain_names, power))
def print_rrd(measurements):
timestamp = int(time.time())
for measurement in measurements.items():
name = measurement[0].lower()
power = measurement[1]
print("PUTVAL {}/exec-power/gauge-{} {}:{:.1f}".format(hostname, name, timestamp, power))
#
# Main
#
if (human_readable):
print get_power()
else:
while True:
power = get_power()
print_rrd(power)
sys.stdout.flush()
time.sleep(measurement_interval)

View File

@@ -0,0 +1,12 @@
#!/bin/bash
SPEEDTEST=/sbin/speedtest-cli
COLLECTION=sepia
INTERVAL=900
while :; do
SECONDS=0
RESULT=($($SPEEDTEST | grep Mbit | cut -d' ' -f 2))
echo "PUTVAL $COLLECTION/exec-speedtest/gauge-download interval=$INTERVAL N:${RESULT[0]}"
echo "PUTVAL $COLLECTION/exec-speedtest/gauge-upload interval=$INTERVAL N:${RESULT[1]}"
sleep $((INTERVAL-$SECONDS))
done

5
esphome/.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
# Gitignore settings for ESPHome
# This is an example and may include too much for your use-case.
# You can modify this file to suit your needs.
/.esphome/
/secrets.yaml

135
esphome/hub.yaml Normal file
View File

@@ -0,0 +1,135 @@
esphome:
name: hub
esp32:
board: esp32dev
framework:
type: arduino
# Enable logging
logger:
# Enable Home Assistant API
api:
encryption:
key: "7/wHBQqTb27jygfGGgRutGYrEMP1AX4gCIK9drLavVU="
ota:
password: "0bd5b8ad22ef5e32c535261ea979e27e"
wifi:
networks:
- ssid: "DD-WRT"
password: "qwerty123"
- ssid: "H369AC68342 2.4"
password: "6A6277E455C9"
# Enable fallback hotspot (captive portal) in case wifi connection fails
ap:
ssid: "Hub Fallback Hotspot"
password: "uJTE4dJxhcBO"
captive_portal:
# mqtt:
# id: mqtt_client
# broker: rik.veenboer.xyz
# username: test
# password: F0fLd9PWRMlVKffpgREIHpeH
# discovery: False # Only if you use the HA API usually
# topic_prefix: herderin
esp32_ble_tracker:
sensor:
# Sanne
- platform: atc_mithermometer
mac_address: A4:C1:38:5A:18:A7
temperature:
name: "ATC Temperature (18:A7)"
humidity:
name: "ATC Humidity (18:A7)"
battery_level:
name: "ATC Battery-Level (18:A7)"
battery_voltage:
name: "ATC Battery-Voltage (18:A7)"
- platform: atc_mithermometer
mac_address: A4:C1:38:AC:64:AE
temperature:
name: "ATC Temperature (64:AE)"
humidity:
name: "ATC Humidity (64:AE)"
battery_level:
name: "ATC Battery-Level (64:AE)"
battery_voltage:
name: "ATC Battery-Voltage (64:AE)"
- platform: atc_mithermometer
mac_address: A4:C1:38:B4:1E:43
temperature:
name: "ATC Temperature (1E:43)"
humidity:
name: "ATC Humidity (1E:43)"
battery_level:
name: "ATC Battery-Level (1E:43)"
battery_voltage:
name: "ATC Battery-Voltage (1E:43)"
- platform: atc_mithermometer
mac_address: A4:C1:38:99:34:DC
temperature:
name: "ATC Temperature (34:DC)"
humidity:
name: "ATC Humidity (34:DC)"
battery_level:
name: "ATC Battery-Level (34:DC)"
battery_voltage:
name: "ATC Battery-Voltage (34:DC)"
# Herderin
- platform: atc_mithermometer
mac_address: A4:C1:38:9F:94:13
temperature:
name: "ATC Temperature (94:13)"
humidity:
name: "ATC Humidity (94:13)"
battery_level:
name: "ATC Battery-Level (94:13)"
battery_voltage:
name: "ATC Battery-Voltage (94:13)"
- platform: atc_mithermometer
mac_address: A4:C1:38:A1:EA:D2
temperature:
name: "ATC Temperature (EA:D2)"
humidity:
name: "ATC Humidity (EA:D2)"
battery_level:
name: "ATC Battery-Level (EA:D2)"
battery_voltage:
name: "ATC Battery-Voltage (EA:D2)"
- platform: atc_mithermometer
mac_address: A4:C1:38:FD:26:B5
temperature:
name: "ATC Temperature (26:B5)"
humidity:
name: "ATC Humidity (26:B5)"
battery_level:
name: "ATC Battery-Level (26:B5)"
battery_voltage:
name: "ATC Battery-Voltage (26:B5)"
- platform: atc_mithermometer
mac_address: A4:C1:38:51:BB:0D
temperature:
name: "ATC Temperature (BB:0D)"
humidity:
name: "ATC Humidity (BB:0D)"
battery_level:
name: "ATC Battery-Level (BB:0D)"
battery_voltage:
name: "ATC Battery-Voltage (BB:0D)"

88
esphome/sanne.yaml Normal file
View File

@@ -0,0 +1,88 @@
esphome:
name: hub
esp32:
board: esp32dev
framework:
type: arduino
# Enable logging
logger:
# Enable Home Assistant API
api:
encryption:
key: "7/wHBQqTb27jygfGGgRutGYrEMP1AX4gCIK9drLavVU="
ota:
password: "0bd5b8ad22ef5e32c535261ea979e27e"
wifi:
networks:
- ssid: "DD-WRT"
password: "qwerty123"
- ssid: "H369AC68342 2.4"
password: "6A6277E455C9"
# Enable fallback hotspot (captive portal) in case wifi connection fails
ap:
ssid: "Hub Fallback Hotspot"
password: "uJTE4dJxhcBO"
captive_portal:
mqtt:
id: mqtt_client
broker: rik.veenboer.xyz
username: test
password: F0fLd9PWRMlVKffpgREIHpeH
discovery: False # Only if you use the HA API usually
topic_prefix: sanne
esp32_ble_tracker:
sensor:
- platform: atc_mithermometer
mac_address: A4:C1:38:5A:18:A7
temperature:
name: "ATC Temperature (18:A7)"
humidity:
name: "ATC Humidity (18:A7)"
battery_level:
name: "ATC Battery-Level (18:A7)"
battery_voltage:
name: "ATC Battery-Voltage (18:A7)"
- platform: atc_mithermometer
mac_address: A4:C1:38:AC:64:AE
temperature:
name: "ATC Temperature (64:AE)"
humidity:
name: "ATC Humidity (64:AE)"
battery_level:
name: "ATC Battery-Level (64:AE)"
battery_voltage:
name: "ATC Battery-Voltage (64:AE)"
- platform: atc_mithermometer
mac_address: A4:C1:38:B4:1E:43
temperature:
name: "ATC Temperature (1E:43)"
humidity:
name: "ATC Humidity (1E:43)"
battery_level:
name: "ATC Battery-Level (1E:43)"
battery_voltage:
name: "ATC Battery-Voltage (1E:43)"
- platform: atc_mithermometer
mac_address: A4:C1:38:99:34:DC
temperature:
name: "ATC Temperature (34:DC)"
humidity:
name: "ATC Humidity (34:DC)"
battery_level:
name: "ATC Battery-Level (34:DC)"
battery_voltage:
name: "ATC Battery-Voltage (34:DC)"

View File

@@ -0,0 +1 @@
[]

View File

@@ -0,0 +1,28 @@
# Loads default set of integrations. Do not remove.
default_config:
# Load frontend themes from the themes folder
frontend:
themes: !include_dir_merge_named themes
# Text to speech
tts:
- platform: google_translate
automation: !include automations.yaml
script: !include scripts.yaml
scene: !include scenes.yaml
http:
use_x_forwarded_for: true
trusted_proxies:
- 172.0.0.0/8
mqtt:
sensor:
!include mqtt.yaml
ltss:
db_url: postgresql://homeassistant:homeassistant@192.168.2.150:6543/homeassistant
chunk_time_interval: 2592000000000

67
homeassistant/mqtt.yaml Normal file
View File

@@ -0,0 +1,67 @@
- object_id: atc_temperature_18a7
unique_id: atc_temperature_18a7
state_topic: "sanne/sensor/atc_temperature_18a7/state"
unit_of_measurement: "°C"
- object_id: atc_humidity_18a7
unique_id: atc_humidity_18a7
unit_of_measurement: "%"
state_topic: "sanne/sensor/atc_humidity_18a7/state"
- object_id: atc_battery-voltage_18a7
unique_id: atc_battery-voltage_18a7
state_topic: "sanne/sensor/atc_battery-voltage_18a7/state"
unit_of_measurement: "V"
- object_id: atc_battery-level_18a7
unique_id: atc_battery-level_18a7
state_topic: "sanne/sensor/atc_battery-level_18a7/state"
unit_of_measurement: "%"
- object_id: atc_temperature_64ae
unique_id: atc_temperature_64ae
unit_of_measurement: "°C"
state_topic: "sanne/sensor/atc_temperature_64ae/state"
- object_id: atc_humidity_64ae
unique_id: atc_humidity_64ae
state_topic: "sanne/sensor/atc_humidity_64ae/state"
unit_of_measurement: "%"
- object_id: atc_battery-voltage_64ae
unique_id: atc_battery-voltage_64ae
state_topic: "sanne/sensor/atc_battery-voltage_64ae/state"
unit_of_measurement: "V"
- object_id: atc_battery-level_64ae
unique_id: atc_battery-level_64ae
state_topic: "sanne/sensor/atc_battery-level_64ae/state"
unit_of_measurement: "%"
- object_id: atc_temperature_1e43
unique_id: atc_temperature_1e43
unit_of_measurement: "°C"
state_topic: "sanne/sensor/atc_temperature_1e43/state"
- object_id: atc_humidity_1e43
unique_id: atc_humidity_1e43
state_topic: "sanne/sensor/atc_humidity_1e43/state"
unit_of_measurement: "%"
- object_id: atc_battery-voltage_1e43
unique_id: atc_battery-voltage_1e43
state_topic: "sanne/sensor/atc_battery-voltage_1e43/state"
unit_of_measurement: "V"
- object_id: atc_battery-level_1e43
unique_id: atc_battery-level_1e43
state_topic: "sanne/sensor/atc_battery-level_1e43/state"
unit_of_measurement: "%"
- object_id: atc_temperature_34dc
unique_id: atc_temperature_34dc
unit_of_measurement: "°C"
state_topic: "sanne/sensor/atc_temperature_34dc/state"
- object_id: atc_humidity_34dc
unique_id: atc_humidity_34dc
state_topic: "sanne/sensor/atc_humidity_34dc/state"
unit_of_measurement: "%"
- object_id: atc_battery-voltage_34dc
unique_id: atc_battery-voltage_34dc
state_topic: "sanne/sensor/atc_battery-voltage_34dc/state"
unit_of_measurement: "V"
- object_id: atc_battery-level_34dc
unique_id: atc_battery-level_34dc
state_topic: "sanne/sensor/atc_battery-level_34dc/state"
unit_of_measurement: "%"

View File

View File

View File

@@ -0,0 +1,31 @@
server 192.168.255.0 255.255.255.0
verb 3
key /etc/openvpn/pki/private/herderin.veenboer.xyz.key
ca /etc/openvpn/pki/ca.crt
cert /etc/openvpn/pki/issued/herderin.veenboer.xyz.crt
dh /etc/openvpn/pki/dh.pem
tls-auth /etc/openvpn/pki/ta.key
key-direction 0
keepalive 10 60
persist-key
persist-tun
proto tcp
# Rely on Docker to do port mapping, internally always 1194
port 443
dev tun0
status /tmp/openvpn-status.log
user nobody
group nogroup
comp-lzo no
### Route Configurations Below
route 192.168.254.0 255.255.255.0
### Push Configurations Below
push "block-outside-dns"
push "dhcp-option DNS 192.168.2.150"
push "comp-lzo no"
port-share host 444

View File

@@ -0,0 +1,25 @@
declare -x OVPN_AUTH=
declare -x OVPN_CIPHER=
declare -x OVPN_CLIENT_TO_CLIENT=
declare -x OVPN_CN=herderin.veenboer.xyz
declare -x OVPN_COMP_LZO=0
declare -x OVPN_DEFROUTE=1
declare -x OVPN_DEVICE=tun
declare -x OVPN_DEVICEN=0
declare -x OVPN_DISABLE_PUSH_BLOCK_DNS=0
declare -x OVPN_DNS=1
declare -x OVPN_DNS_SERVERS=([0]="8.8.8.8" [1]="8.8.4.4")
declare -x OVPN_ENV=/etc/openvpn/ovpn_env.sh
declare -x OVPN_EXTRA_CLIENT_CONFIG=()
declare -x OVPN_EXTRA_SERVER_CONFIG=()
declare -x OVPN_FRAGMENT=
declare -x OVPN_KEEPALIVE='10 60'
declare -x OVPN_MTU=
declare -x OVPN_NAT=0
declare -x OVPN_PORT=1194
declare -x OVPN_PROTO=tcp
declare -x OVPN_PUSH=()
declare -x OVPN_ROUTES=([0]="192.168.254.0/24")
declare -x OVPN_SERVER=192.168.255.0/24
declare -x OVPN_SERVER_URL=tcp://herderin.veenboer.xyz
declare -x OVPN_TLS_CIPHER=

9
openvpn-server/setup.sh Executable file
View File

@@ -0,0 +1,9 @@
#!/usr/bin/bash
OVPN_DATA=/opt/openvpn-server
VERSION=2.4
CLIENTNAME=herderin
docker run -v /$OVPN_DATA:/etc/openvpn --rm kylemanna/openvpn:$VERSION ovpn_genconfig -u tcp://$CLIENTNAME.veenboer.xyz
docker run -v /$OVPN_DATA:/etc/openvpn --rm -it kylemanna/openvpn:$VERSION ovpn_initpki
docker run -v /$OVPN_DATA:/etc/openvpn -d -p 443:443/tcp --cap-add=NET_ADMIN kylemanna/openvpn:$VERSION
docker run -v /$OVPN_DATA:/etc/openvpn --rm -it kylemanna/openvpn easyrsa build-client-full $CLIENTNAME nopass
docker run -v /$OVPN_DATA:/etc/openvpn --rm kylemanna/openvpn:$VERSION ovpn_getclient $CLIENTNAME > $CLIENTNAME.ovpn

3
seafile/conf/ccnet.conf Normal file
View File

@@ -0,0 +1,3 @@
[General]
SERVICE_URL = https://herderin.veenboer.xyz/seafile

1
seafile/conf/conf Symbolic link
View File

@@ -0,0 +1 @@
/seafile/conf

View File

@@ -0,0 +1,16 @@
import os
daemon = True
workers = 5
# default localhost:8000
bind = "0.0.0.0:8000"
# Pid
pids_dir = '/opt/haiwen/pids'
pidfile = os.path.join(pids_dir, 'seahub.pid')
# for file upload, we need a longer timeout value (default is only 30s, too short)
timeout = 1200
limit_request_line = 8190

View File

@@ -0,0 +1,3 @@
[fileserver]
port=8082
max_download_dir_size=8192

View File

@@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
SECRET_KEY = "by-t3pizx0b7b^x#4q8be4b@4n1g8vxle#bl@+gq8*x!2jt*g7"
SERVE_STATIC = False
MEDIA_URL = '/seafmedia/'
COMPRESS_URL = MEDIA_URL
STATIC_URL = MEDIA_URL + 'assets/'
SITE_ROOT = '/seafile/'
LOGIN_URL = '/seafile/accounts/login/'
FILE_SERVER_ROOT = 'https://herderin.veenboer.xyz/seafhttp'
SERVICE_URL = 'https://herderin.veenboer.xyz/seafile'

1
seafile/version Normal file
View File

@@ -0,0 +1 @@
9.0.4