homeserver/opt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

collectd configuration

Browse Source
This commit is contained in:
Rik Veenboer
2025-01-03 09:47:10 +01:00
parent 02495dde81
commit f7941e8ef5
13 changed files with 321 additions and 1088 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
collectd/usr/local/bin/btrfs-data
View File

@@ -1,18 +1,18 @@
#!/usr/bin/python
#!/usr/bin/python3
#
# Imports
#
import sys
import time
import commands
import subprocess
import argparse
#
# Misc
#
#sys.tracebacklimit = 0
# sys.tracebacklimit = 0
#
@@ -28,21 +28,42 @@ size_snapshot_exclusive = 0
# Methods
#
def get_subvol_list(path):
command = "btrfs subvolume list -t %s" % (path)
status, output = commands.getstatusoutput(command)
command = "sudo btrfs subvolume list -t %s" % (path)
status, output = subprocess.getstatusoutput(command)
if status is not 0:
if status != 0:
raise Exception(command)
# Every line contains the following values: subvol_id, gen, toplevel, path
return output.splitlines()[2:]
def get_filesystem_size(path):
command = "sudo btrfs filesystem show --raw %s" % (path)
status, output = subprocess.getstatusoutput(command)
if status != 0:
# This command fails when running inside Docker container
# return maximum size of any filesystem instead
command = "sudo btrfs filesystem show --raw"
status, output = subprocess.getstatusoutput(command)
lines = output.splitlines()
lines = [x for x in lines if "devid" in x]
sizes = [int(line.split()[3]) for line in lines]
return max(sizes)
# The sizes are on the third line
line = output.splitlines()[2]
# Element 3 and 5 respectively contain total and used sizes
return int(line.split()[3])
def get_id_root(name, path):
lines = get_subvol_list(path)
# Filter lines where toplevel == 5
subvol_ids = filter(lambda x: int(x.split()[2]) == 5, lines)
subvol_ids = [x for x in lines if int(x.split()[2]) == 5]
# Try to retrieve the subvol_id for the root subvolume (if any)
if len(subvol_ids) == 1:
@@ -51,28 +72,30 @@ def get_id_root(name, path):
else:
# The path contains a btrfs filesystem with multiple subvolumes for data
try:
return int(filter(lambda x: x.split()[3] == name, subvol_ids)[0].split()[0])
return int(list(filter(lambda x: x.split()[3] == name, subvol_ids))[0].split()[0])
except IndexError:
pass
# Volume not found, root is probably the btrfs default (5)
return 5
def get_id_subvolumes(path, subvol_id):
lines = get_subvol_list(path)
lines = filter(lambda x: int(x.split()[2]) == subvol_id, lines)
return list(map(lambda x: int(x.split()[0]), lines))
lines = [x for x in lines if int(x.split()[2]) == subvol_id]
return list([int(x.split()[0]) for x in lines])
def get_disk_usage(name, path):
id_root = get_id_root(name, path)
id_subvolumes = get_id_subvolumes(path, id_root)
size_filesystem = get_filesystem_size(path)
command = "btrfs qgroup show --raw %s" % (path)
status, output = commands.getstatusoutput(command)
# Get disk usage from quota
command = "sudo btrfs qgroup show --raw %s" % (path)
status, output = subprocess.getstatusoutput(command)
if status is not 0:
if status != 0:
raise Exception(command)
lines = output.splitlines()[2:]
@@ -94,16 +117,28 @@ def get_disk_usage(name, path):
# -> data that is not (yet) incorporated in a snapshot
size_data_exclusive = 0
# Data exclusively available in snapshots
# -> data that was removed from volume
size_snapshot_exclusive = 0
for line in lines:
split = line.split()
subvol_id = int(split[0].split("/")[1])
size_total = float(split[1])
size_exclusive = float(split[2])
subvol_id = 0
size_total = 0
size_exclusive = 0
try:
subvol_id = int(split[0].split("/")[1])
size_total = float(split[1])
size_exclusive = float(split[2])
except IndexError:
# ignore "WARNING: Quota disabled"
pass
# size_exclusive is incorrect when snapshot is
# removed and qgroups are not updated yet,
# ignore the value when it seems unrealistic
if size_exclusive > size_filesystem:
size_exclusive = 0
if subvol_id == id_root:
size_data_total = size_total
@@ -112,27 +147,57 @@ def get_disk_usage(name, path):
size_snapshot_total += size_total
size_snapshot_exclusive += size_exclusive
def rescan_quota(path):
command = "btrfs quota rescan %s" % (path)
status, output = commands.getstatusoutput(command)
if status is not 0:
command = "sudo btrfs quota rescan %s" % (path)
status, output = subprocess.getstatusoutput(command)
if status != 0:
Exception(command)
def print_human_readable(name):
global size_data_total
global size_data_exclusive
global size_snapshot_exclusive
size_data_total = size_data_total / (1024*1e6)
size_data_exclusive = size_data_exclusive / (1024*1e6)
size_snapshot_exclusive = size_snapshot_exclusive / (1024*1e6)
print "%10s: %6.1f Gb, %6.1f Gb, %6.1f Gb" % (name, size_data_total, size_data_exclusive, size_snapshot_exclusive)
size_data_total = size_data_total / (1024 * 1e6)
size_data_exclusive = size_data_exclusive / (1024 * 1e6)
size_snapshot_exclusive = size_snapshot_exclusive / (1024 * 1e6)
print(
"%10s: %6.1f Gb, %6.1f Gb, %6.1f Gb"
% (name, size_data_total, size_data_exclusive, size_snapshot_exclusive)
)
def print_rrd(name):
timestamp = int(time.time())
print("PUTVAL {}/exec-btrfs_{}/gauge-data_total {}:{:.1f}".format(hostname, name, timestamp, size_data_total))
print("PUTVAL {}/exec-btrfs_{}/gauge-data_exclusive {}:{:.1f}".format(hostname, name, timestamp, size_data_exclusive))
print("PUTVAL {}/exec-btrfs_{}/gauge-snapshot_total {}:{:.1f}".format(hostname, name, timestamp, size_snapshot_total))
print("PUTVAL {}/exec-btrfs_{}/gauge-snapshot_exclusive {}:{:.1f}".format(hostname, name, timestamp, size_snapshot_exclusive))
print(
(
"PUTVAL {}/exec-btrfs_{}/gauge-data_total {}:{:.1f}".format(
hostname, name, timestamp, size_data_total
)
)
)
print(
(
"PUTVAL {}/exec-btrfs_{}/gauge-data_exclusive {}:{:.1f}".format(
hostname, name, timestamp, size_data_exclusive
)
)
)
print(
(
"PUTVAL {}/exec-btrfs_{}/gauge-snapshot_total {}:{:.1f}".format(
hostname, name, timestamp, size_snapshot_total
)
)
)
print(
(
"PUTVAL {}/exec-btrfs_{}/gauge-snapshot_exclusive {}:{:.1f}".format(
hostname, name, timestamp, size_snapshot_exclusive
)
)
)
#
@@ -143,19 +208,18 @@ interval = 10
volumes = list()
# 275 GB SSD
volumes.append(["@", "/host/root/"])
volumes.append(["@home", "/host/root/home"])
volumes.append(["opt", "/host/root/opt"])
volumes.append(["home", "/host/root/home"])
# 2x 4TB HDD
volumes.append(["data", "/host/root/media/data"])
volumes.append(["backup", "/host/root/media/backup"])
volumes.append(["seafile", "/host/root/media/seafile"])
#
# Command line arguments
#
parser = argparse.ArgumentParser(description='Get BTRFS disk usage')
parser.add_argument('-s', action='store_true', help='print in human readable format')
parser = argparse.ArgumentParser(description="Get BTRFS disk usage")
parser.add_argument("-s", action="store_true", help="print in human readable format")
args = parser.parse_args()
human_readable = args.s
@@ -163,7 +227,7 @@ human_readable = args.s
#
# Main
#
if (human_readable):
if human_readable:
for (name, path) in volumes:
get_disk_usage(name, path)
print_human_readable(name)
@@ -172,9 +236,9 @@ else:
while True:
for (name, path) in volumes:
get_disk_usage(name, path)
print_rrd(name)
sys.stdout.flush()
time.sleep(interval)
#rescan_quota(path)
# rescan_quota(path)