homeserver/opt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

configure collectd

Browse Source
This commit is contained in:
Rik Veenboer
2025-01-03 09:42:20 +01:00
parent b8471944b0
commit d967b73db4
18 changed files with 314 additions and 1162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
collectd/usr/local/bin/btrfs-data
View File

@@ -1,18 +1,20 @@
#!/usr/bin/python -u
#!/usr/bin/python3
#
# Imports
#
import sys
import time
import commands
import subprocess
import argparse
#sys.exit(1)
#
# Misc
#
#sys.tracebacklimit = 0
# sys.tracebacklimit = 0
#
@@ -28,26 +30,27 @@ size_snapshot_exclusive = 0
# Methods
#
def get_subvol_list(path):
command = 'sudo btrfs subvolume list -t %s' % (path)
status, output = commands.getstatusoutput(command)
command = "sudo btrfs subvolume list -t %s" % (path)
status, output = subprocess.getstatusoutput(command)
if status is not 0:
if status != 0:
raise Exception(command)
# Every line contains the following values: subvol_id, gen, toplevel, path
return output.splitlines()[2:]
def get_filesystem_size(path):
command = 'sudo btrfs filesystem show --raw %s' % (path)
status, output = commands.getstatusoutput(command)
if status is not 0 or True:
def get_filesystem_size(path):
command = "sudo btrfs filesystem show --raw %s" % (path)
status, output = subprocess.getstatusoutput(command)
if status != 0:
# This command fails when running inside Docker container
# return maximum size of any filesystem instead
command = 'sudo btrfs filesystem show --raw'
status, output = commands.getstatusoutput(command)
command = "sudo btrfs filesystem show --raw"
status, output = subprocess.getstatusoutput(command)
lines = output.splitlines()
lines = filter(lambda x: 'devid' in x, lines)
lines = [x for x in lines if "devid" in x]
sizes = [int(line.split()[3]) for line in lines]
return max(sizes)
@@ -57,11 +60,12 @@ def get_filesystem_size(path):
# Element 3 and 5 respectively contain total and used sizes
return int(line.split()[3])
def get_id_root(name, path):
lines = get_subvol_list(path)
# Filter lines where toplevel == 5
subvol_ids = filter(lambda x: int(x.split()[2]) == 5, lines)
subvol_ids = [x for x in lines if int(x.split()[2]) == 5]
# Try to retrieve the subvol_id for the root subvolume (if any)
if len(subvol_ids) == 1:
@@ -70,18 +74,18 @@ def get_id_root(name, path):
else:
# The path contains a btrfs filesystem with multiple subvolumes for data
try:
return int(filter(lambda x: x.split()[3] == name, subvol_ids)[0].split()[0])
return int(list(filter(lambda x: x.split()[3] == name, subvol_ids))[0].split()[0])
except IndexError:
pass
# Volume not found, root is probably the btrfs default (5)
return 5
def get_id_subvolumes(path, subvol_id):
lines = get_subvol_list(path)
lines = filter(lambda x: int(x.split()[2]) == subvol_id, lines)
return list(map(lambda x: int(x.split()[0]), lines))
lines = [x for x in lines if int(x.split()[2]) == subvol_id]
return list([int(x.split()[0]) for x in lines])
def get_disk_usage(name, path):
@@ -90,10 +94,10 @@ def get_disk_usage(name, path):
size_filesystem = get_filesystem_size(path)
# Get disk usage from quota
command = 'sudo btrfs qgroup show --raw %s' % (path)
status, output = commands.getstatusoutput(command)
command = "sudo btrfs qgroup show --raw %s" % (path)
status, output = subprocess.getstatusoutput(command)
if status is not 0:
if status != 0:
raise Exception(command)
lines = output.splitlines()[2:]
@@ -125,11 +129,11 @@ def get_disk_usage(name, path):
size_total = 0
size_exclusive = 0
try:
subvol_id = int(split[0].split('/')[1])
subvol_id = int(split[0].split("/")[1])
size_total = float(split[1])
size_exclusive = float(split[2])
except IndexError:
# ignore 'WARNING: Quota disabled'
# ignore "WARNING: Quota disabled"
pass
# size_exclusive is incorrect when snapshot is
@@ -145,51 +149,83 @@ def get_disk_usage(name, path):
size_snapshot_total += size_total
size_snapshot_exclusive += size_exclusive
def rescan_quota(path):
command = 'sudo btrfs quota rescan %s' % (path)
status, output = commands.getstatusoutput(command)
if status is not 0:
command = "sudo btrfs quota rescan %s" % (path)
status, output = subprocess.getstatusoutput(command)
if status != 0:
Exception(command)
def print_human_readable(name):
global size_data_total
global size_data_exclusive
global size_snapshot_exclusive
size_data_total = size_data_total / (1024*1e6)
size_data_exclusive = size_data_exclusive / (1024*1e6)
size_snapshot_exclusive = size_snapshot_exclusive / (1024*1e6)
print '%10s: %6.1f Gb, %6.1f Gb, %6.1f Gb' % (name, size_data_total, size_data_exclusive, size_snapshot_exclusive)
size_data_total = size_data_total / (1024 * 1e6)
size_data_exclusive = size_data_exclusive / (1024 * 1e6)
size_snapshot_exclusive = size_snapshot_exclusive / (1024 * 1e6)
print(
"%10s: %6.1f Gb, %6.1f Gb, %6.1f Gb"
% (name, size_data_total, size_data_exclusive, size_snapshot_exclusive)
)
def print_rrd(name):
timestamp = int(time.time())
print('PUTVAL {}/exec-btrfs_{}/gauge-data_total {}:{:.1f}'.format(hostname, name, timestamp, size_data_total))
print('PUTVAL {}/exec-btrfs_{}/gauge-data_exclusive {}:{:.1f}'.format(hostname, name, timestamp, size_data_exclusive))
print('PUTVAL {}/exec-btrfs_{}/gauge-snapshot_total {}:{:.1f}'.format(hostname, name, timestamp, size_snapshot_total))
print('PUTVAL {}/exec-btrfs_{}/gauge-snapshot_exclusive {}:{:.1f}'.format(hostname, name, timestamp, size_snapshot_exclusive))
print(
(
"PUTVAL {}/exec-btrfs_{}/gauge-data_total {}:{:.1f}".format(
hostname, name, timestamp, size_data_total
)
)
)
print(
(
"PUTVAL {}/exec-btrfs_{}/gauge-data_exclusive {}:{:.1f}".format(
hostname, name, timestamp, size_data_exclusive
)
)
)
print(
(
"PUTVAL {}/exec-btrfs_{}/gauge-snapshot_total {}:{:.1f}".format(
hostname, name, timestamp, size_snapshot_total
)
)
)
print(
(
"PUTVAL {}/exec-btrfs_{}/gauge-snapshot_exclusive {}:{:.1f}".format(
hostname, name, timestamp, size_snapshot_exclusive
)
)
)
#
# Volumes to scan
#
hostname = 'server'
hostname = "shuttle"
interval = 10
volumes = [
['mezzo-scratch', '/mnt/mezzo/scratch'],
['mezzo-sync', '/mnt/mezzo/sync'],
['helium-personal', '/mnt/yotta/helium/personal'],
['helium-shared', '/mnt/yotta/helium/shared'],
['neon', '/mnt/yotta/neon'],
['krypton', '/mnt/yotta/krypton'],
['xenon-borg', '/mnt/yotta/xenon/borg'],
['xenon-rsnapshot', '/mnt/yotta/xenon/rsnapshot']
]
volumes = list()
# SSD
volumes.append(["scratch", "/host/root/mnt/mezzo/scratch"])
volumes.append(["sync", "/host/root/mnt/mezzo/sync"])
# HDD
volumes.append(["personal", "/host/root/mnt/yotta/helium/personal"])
volumes.append(["shared", "/host/root/mnt/yotta/helium/shared"])
volumes.append(["neon", "/host/root/mnt/yotta/neon"])
volumes.append(["krypton", "/host/root/mnt/yotta/krypton"])
volumes.append(["borg", "/host/root/mnt/yotta/xenon/borg"])
volumes.append(["rsnapshot", "/host/root/mnt/yotta/xenon/rsnapshot"])
#
# Command line arguments
#
parser = argparse.ArgumentParser(description='Get BTRFS disk usage')
parser.add_argument('-s', action='store_true', help='print in human readable format')
parser = argparse.ArgumentParser(description="Get BTRFS disk usage")
parser.add_argument("-s", action="store_true", help="print in human readable format")
args = parser.parse_args()
human_readable = args.s
@@ -198,15 +234,17 @@ human_readable = args.s
# Main
#
if human_readable:
for name, path in volumes:
for (name, path) in volumes:
get_disk_usage(name, path)
print_human_readable(name)
else:
# RRD mode
while True:
for name, path in volumes:
get_disk_usage(name, path)
for (name, path) in volumes:
get_disk_usage(name, path)
print_rrd(name)
sys.stdout.flush()
time.sleep(interval)
rescan_quota(path)
# rescan_quota(path)