snapshot remaining files

collectd/etc/collectd.conf.d/btrfs-data.conf

@@ -0,0 +1,5 @@
+#LoadPlugin exec
+#
+#<Plugin "exec">
+#  Exec nobody "/host/usr/local/bin/btrfs-data"
+#</Plugin>

collectd/etc/collectd.conf.d/df.conf

@@ -0,0 +1,6 @@
+<Plugin df>
+    MountPoint "/media/docker"
+    MountPoint "/media/scratch"
+    FSType "ext4"
+    IgnoreSelected false
+</Plugin>

collectd/etc/collectd.conf.d/du-data.conf

@@ -0,0 +1,6 @@
+LoadPlugin exec
+
+<Plugin "exec">
+  Exec nobody "/host/usr/local/bin/du-data"
+</Plugin>
+

collectd/etc/collectd.conf.d/power-data.conf

@@ -0,0 +1,5 @@
+LoadPlugin exec
+
+<Plugin "exec">
+  Exec collectd "/host/usr/local/bin/power-data"
+</Plugin>

collectd/etc/collectd.conf.d/speedtest-data.conf

@@ -0,0 +1,6 @@
+LoadPlugin exec
+
+<Plugin "exec">
+  Exec nobody "/host/usr/local/bin/speedtest-data"
+</Plugin>
+

collectd/usr/local/bin/btrfs-data

@@ -0,0 +1,180 @@
+#!/usr/bin/python
+
+#
+# Imports
+#
+import sys
+import time
+import commands
+import argparse
+
+
+#
+# Misc
+#
+#sys.tracebacklimit = 0
+
+
+#
+# Global variables
+#
+size_data_total = 0
+size_data_exclusive = 0
+size_snapshot_total = 0
+size_snapshot_exclusive = 0
+
+
+#
+# Methods
+#
+def get_subvol_list(path):
+    command = "btrfs subvolume list -t %s" % (path)
+    status, output = commands.getstatusoutput(command)
+
+    if status is not 0:
+        raise Exception(command)
+
+    # Every line contains the following values: subvol_id, gen, toplevel, path
+    return output.splitlines()[2:]
+
+
+def get_id_root(name, path):
+    lines = get_subvol_list(path)
+
+    # Filter lines where toplevel == 5
+    subvol_ids = filter(lambda x: int(x.split()[2]) == 5, lines)
+
+    # Try to retrieve the subvol_id for the root subvolume (if any)
+    if len(subvol_ids) == 1:
+        # The path contains a btrfs filesystem without subvolume for data
+        return int(subvol_ids[0].split()[0])
+    else:
+        # The path contains a btrfs filesystem with multiple subvolumes for data
+        try:
+            return int(filter(lambda x: x.split()[3] == name, subvol_ids)[0].split()[0])
+        except IndexError:
+            pass
+            
+
+    # Volume not found, root is probably the btrfs default (5)
+    return 5
+            
+def get_id_subvolumes(path, subvol_id):
+    lines = get_subvol_list(path)
+    lines = filter(lambda x: int(x.split()[2]) == subvol_id, lines)
+    return list(map(lambda x: int(x.split()[0]), lines))
+
+
+def get_disk_usage(name, path):
+    id_root = get_id_root(name, path)
+    id_subvolumes = get_id_subvolumes(path, id_root)
+
+    command = "btrfs qgroup show --raw %s" % (path)
+    status, output = commands.getstatusoutput(command)
+
+    if status is not 0:
+        raise Exception(command)
+
+    lines = output.splitlines()[2:]
+
+    # Global variables
+    global size_data_total
+    global size_data_exclusive
+    global size_snapshot_total
+    global size_snapshot_exclusive
+
+    # Total data volume in subvolume
+    size_data_total = 0
+
+    # Total data volume in snapshots
+    #   -> this variable is useless
+    size_snapshot_total = 0
+
+    # Data exclusively in subvolume
+    #   -> data that is not (yet) incorporated in a snapshot
+    size_data_exclusive = 0
+
+
+    # Data exclusively available in snapshots
+    #   -> data that was removed from volume
+    size_snapshot_exclusive = 0
+
+    for line in lines:
+        split = line.split()
+        subvol_id = int(split[0].split("/")[1])
+        size_total = float(split[1])
+        size_exclusive = float(split[2])
+
+        if subvol_id == id_root:
+            size_data_total = size_total
+            size_data_exclusive = size_exclusive
+        elif subvol_id in id_subvolumes:
+            size_snapshot_total += size_total
+            size_snapshot_exclusive += size_exclusive
+
+def rescan_quota(path):
+    command = "btrfs quota rescan %s" % (path)
+    status, output = commands.getstatusoutput(command)
+    if status is not 0:
+        Exception(command)
+
+def print_human_readable(name):
+    global size_data_total
+    global size_data_exclusive
+    global size_snapshot_exclusive
+    size_data_total = size_data_total / (1024*1e6)
+    size_data_exclusive = size_data_exclusive / (1024*1e6)
+    size_snapshot_exclusive = size_snapshot_exclusive / (1024*1e6)
+    print "%10s: %6.1f Gb, %6.1f Gb, %6.1f Gb" % (name, size_data_total, size_data_exclusive, size_snapshot_exclusive)
+
+def print_rrd(name):
+    timestamp = int(time.time())
+    print("PUTVAL {}/exec-btrfs_{}/gauge-data_total {}:{:.1f}".format(hostname, name, timestamp, size_data_total))
+    print("PUTVAL {}/exec-btrfs_{}/gauge-data_exclusive {}:{:.1f}".format(hostname, name, timestamp, size_data_exclusive))
+    print("PUTVAL {}/exec-btrfs_{}/gauge-snapshot_total {}:{:.1f}".format(hostname, name, timestamp, size_snapshot_total))
+    print("PUTVAL {}/exec-btrfs_{}/gauge-snapshot_exclusive {}:{:.1f}".format(hostname, name, timestamp, size_snapshot_exclusive))
+
+
+#
+# Volumes to scan
+#
+hostname = "sepia"
+interval = 10
+volumes = list()
+
+# 275 GB SSD
+volumes.append(["@",       "/host/root/"])
+volumes.append(["@home",   "/host/root/home"])
+volumes.append(["opt",     "/host/root/opt"])
+
+# 2x 4TB HDD
+volumes.append(["data",    "/host/root/media/data"])
+
+
+#
+# Command line arguments
+#
+parser = argparse.ArgumentParser(description='Get BTRFS disk usage')
+parser.add_argument('-s', action='store_true', help='print in human readable format')
+args = parser.parse_args()
+human_readable = args.s
+
+
+#
+# Main
+#
+if (human_readable):
+    for (name, path) in volumes:
+        get_disk_usage(name, path)
+        print_human_readable(name)
+else:
+    # RRD mode
+    while True:
+        for (name, path) in volumes:
+            get_disk_usage(name, path)
+        
+            print_rrd(name)
+
+            sys.stdout.flush()
+        time.sleep(interval)
+        #rescan_quota(path)

collectd/usr/local/bin/du-data

@@ -0,0 +1,25 @@
+#!/bin/bash
+COLLECTION=sepia
+INTERVAL=90
+
+DIRS=$(cat <<LIST
+/host/root/media/data/Inverter
+/host/root/media/data/Monique
+/host/root/media/data/Music
+/host/root/media/data/Peter
+/host/root/media/data/Photographs
+/host/root/media/data/Raw
+/host/root/media/data/Sanne
+/host/root/media/data/Wii
+LIST
+)
+
+while :; do
+    SECONDS=0    
+    for DIR in $DIRS; do
+        SIZE=$(du -cs $DIR | tail -1 | awk '{print $1}')
+        NAME=$(echo $DIR | sed 's/.//' | tr / - )
+        echo "PUTVAL $COLLECTION/exec-du-$NAME/gauge-size interval=$INTERVAL N:$SIZE"
+    done
+    sleep $((INTERVAL-$SECONDS))
+done

collectd/usr/local/bin/power-data

@@ -0,0 +1,69 @@
+#!/usr/bin/python
+
+#
+# Imports
+#
+import sys
+import time
+import argparse
+import pylikwid
+
+#
+# Configuration
+#
+hostname = "sepia"
+cpuid = 0
+pinfo = pylikwid.getpowerinfo()
+domainid = pinfo.get("domains").get("PKG").get("ID")
+measurement_duration = 5
+measurement_interval = 15
+dinfo = pinfo.get("domains")
+domain_names = dinfo.keys()
+domain_ids = [domain['ID'] for domain in dinfo.values()]
+
+#
+# Command line arguments
+#
+parser = argparse.ArgumentParser(description='Get CPU power consumption')
+parser.add_argument('-s', action='store_true', help='print in human readable format')
+args = parser.parse_args()
+human_readable = args.s
+
+# 
+# Methods
+# 
+def get_power():
+    start = list()
+    end = list()
+    power = list()
+    for domain_id in domain_ids:
+        e_start = pylikwid.startpower(cpuid, domain_id)
+        start.append(e_start)
+    time.sleep(measurement_duration)
+    for domain_id in domain_ids:
+        e_stop = pylikwid.stoppower(cpuid, domain_id)
+        end.append(e_stop)
+    for events in zip(start, end, domain_ids):
+        joules = pylikwid.getpower(events[0], events[1], events[2])
+        power.append(joules / measurement_duration)
+
+    return dict(zip(domain_names, power))
+
+def print_rrd(measurements):
+    timestamp = int(time.time())
+    for measurement in measurements.items():
+        name = measurement[0].lower()
+        power = measurement[1]
+        print("PUTVAL {}/exec-power/gauge-{} {}:{:.1f}".format(hostname, name, timestamp, power))
+
+#
+# Main
+#
+if (human_readable):
+    print get_power()
+else:
+    while True:
+        power = get_power()
+        print_rrd(power)
+        sys.stdout.flush()
+        time.sleep(measurement_interval)

collectd/usr/local/bin/speedtest-data

@@ -0,0 +1,12 @@
+#!/bin/bash
+SPEEDTEST=/sbin/speedtest-cli
+COLLECTION=sepia
+INTERVAL=900
+
+while :; do
+    SECONDS=0 
+    RESULT=($($SPEEDTEST | grep Mbit | cut -d' ' -f 2))
+    echo "PUTVAL $COLLECTION/exec-speedtest/gauge-download interval=$INTERVAL N:${RESULT[0]}"
+    echo "PUTVAL $COLLECTION/exec-speedtest/gauge-upload interval=$INTERVAL N:${RESULT[1]}"
+    sleep $((INTERVAL-$SECONDS))
+done