add untracked files

collectd/docker/Dockerfile.bookworm

@@ -0,0 +1,35 @@
+FROM debian:bookworm
+
+ENV LC_ALL=C
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TIMEZONE=Europe/Amsterdam
+
+RUN apt update
+RUN apt dist-upgrade -y --no-install-recommends
+
+RUN echo $TIMEZONE > /etc/timezone
+RUN dpkg-reconfigure -f noninteractive tzdata
+
+RUN apt install -y libsensors5 liblzo2-2 collectd sudo btrfs-progs libatasmart4 speedtest-cli
+
+RUN apt install -y smartmontools
+
+RUN apt install -y wget git
+
+ENV HDDTEMP_VERSION=0.2.4
+RUN wget https://github.com/slowpeek/hddtemp/archive/refs/tags/${HDDTEMP_VERSION}.tar.gz \
+    && tar xvf ${HDDTEMP_VERSION}.tar.gz && mv hddtemp-${HDDTEMP_VERSION}/hddtemp-lt /usr/sbin/hddtemp
+
+RUN apt install -y gcc python3-dev make
+RUN git clone https://github.com/RRZE-HPC/likwid.git
+RUN cd likwid && make -j && make install
+RUN git clone https://github.com/RRZE-HPC/pylikwid.git
+RUN cd pylikwid && python3 setup.py build_ext && python3 setup.py install
+RUN apt remove -y gcc python-dev make
+RUN apt autoremove -y
+
+RUN useradd collectd
+RUN usermod -aG sudo collectd
+RUN echo 'collectd ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers.d/collectd
+
+CMD /usr/sbin/collectd -f

collectd/docker/Dockerfile.buster

@@ -0,0 +1,31 @@
+FROM debian:buster
+
+ENV LC_ALL=C
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TIMEZONE=Europe/Amsterdam
+
+RUN apt update
+RUN apt dist-upgrade -y --no-install-recommends
+
+RUN echo $TIMEZONE > /etc/timezone
+RUN dpkg-reconfigure -f noninteractive tzdata
+
+RUN apt install -y software-properties-common gpgv dirmngr psmisc wget curl python3-pip git gawk zip gperf unzip lbzip2 inetutils-ping inetutils-telnet rsync
+
+RUN pip3 install argparse
+
+RUN apt install -y libsensors5 liblzo2-2 hddtemp collectd sudo btrfs-progs libatasmart4 speedtest-cli
+
+RUN useradd collectd
+RUN usermod -aG sudo collectd
+RUN echo 'collectd ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers.d/collectd
+
+RUN apt install -y gcc python-dev make
+RUN git clone https://github.com/RRZE-HPC/likwid.git
+RUN cd likwid && make -j && make install
+RUN git clone https://github.com/RRZE-HPC/pylikwid.git
+RUN cd pylikwid && python setup.py build_ext && python setup.py install
+
+ENV LD_LIBRARY_PATH /usr/local/lib
+
+CMD /usr/sbin/collectd -f

collectd/usr/local/bin/btrfs-data

@@ -0,0 +1,212 @@
+#!/usr/bin/python -u
+
+#
+# Imports
+#
+import sys
+import time
+import commands
+import argparse
+
+
+#
+# Misc
+#
+#sys.tracebacklimit = 0
+
+
+#
+# Global variables
+#
+size_data_total = 0
+size_data_exclusive = 0
+size_snapshot_total = 0
+size_snapshot_exclusive = 0
+
+
+#
+# Methods
+#
+def get_subvol_list(path):
+    command = 'sudo btrfs subvolume list -t %s' % (path)
+    status, output = commands.getstatusoutput(command)
+
+    if status is not 0:
+        raise Exception(command)
+
+    # Every line contains the following values: subvol_id, gen, toplevel, path
+    return output.splitlines()[2:]
+
+def get_filesystem_size(path):
+    command = 'sudo btrfs filesystem show --raw %s' % (path)
+    status, output = commands.getstatusoutput(command)
+
+    if status is not 0 or True:
+        # This command fails when running inside Docker container
+        # return maximum size of any filesystem instead
+        command = 'sudo btrfs filesystem show --raw'
+        status, output = commands.getstatusoutput(command)
+        lines = output.splitlines()
+        lines = filter(lambda x: 'devid' in x, lines)
+        sizes = [int(line.split()[3]) for line in lines]
+        return max(sizes)
+
+    # The sizes are on the third line
+    line = output.splitlines()[2]
+
+    # Element 3 and 5 respectively contain total and used sizes
+    return int(line.split()[3])
+
+def get_id_root(name, path):
+    lines = get_subvol_list(path)
+
+    # Filter lines where toplevel == 5
+    subvol_ids = filter(lambda x: int(x.split()[2]) == 5, lines)
+
+    # Try to retrieve the subvol_id for the root subvolume (if any)
+    if len(subvol_ids) == 1:
+        # The path contains a btrfs filesystem without subvolume for data
+        return int(subvol_ids[0].split()[0])
+    else:
+        # The path contains a btrfs filesystem with multiple subvolumes for data
+        try:
+            return int(filter(lambda x: x.split()[3] == name, subvol_ids)[0].split()[0])
+        except IndexError:
+            pass
+            
+
+    # Volume not found, root is probably the btrfs default (5)
+    return 5
+            
+def get_id_subvolumes(path, subvol_id):
+    lines = get_subvol_list(path)
+    lines = filter(lambda x: int(x.split()[2]) == subvol_id, lines)
+    return list(map(lambda x: int(x.split()[0]), lines))
+
+
+def get_disk_usage(name, path):
+    id_root = get_id_root(name, path)
+    id_subvolumes = get_id_subvolumes(path, id_root)
+    size_filesystem = get_filesystem_size(path)
+
+    # Get disk usage from quota
+    command = 'sudo btrfs qgroup show --raw %s' % (path)
+    status, output = commands.getstatusoutput(command)
+
+    if status is not 0:
+        raise Exception(command)
+
+    lines = output.splitlines()[2:]
+
+    # Global variables
+    global size_data_total
+    global size_data_exclusive
+    global size_snapshot_total
+    global size_snapshot_exclusive
+
+    # Total data volume in subvolume
+    size_data_total = 0
+
+    # Total data volume in snapshots
+    #   -> this variable is useless
+    size_snapshot_total = 0
+
+    # Data exclusively in subvolume
+    #   -> data that is not (yet) incorporated in a snapshot
+    size_data_exclusive = 0
+
+    # Data exclusively available in snapshots
+    #   -> data that was removed from volume
+    size_snapshot_exclusive = 0
+
+    for line in lines:
+        split = line.split()
+        subvol_id = 0
+        size_total = 0
+        size_exclusive = 0
+        try:
+            subvol_id = int(split[0].split('/')[1])
+            size_total = float(split[1])
+            size_exclusive = float(split[2])
+        except IndexError:
+            # ignore 'WARNING: Quota disabled'
+            pass
+
+        # size_exclusive is incorrect when snapshot is
+        # removed and qgroups are not updated yet,
+        # ignore the value when it seems unrealistic
+        if size_exclusive > size_filesystem:
+            size_exclusive = 0
+
+        if subvol_id == id_root:
+            size_data_total = size_total
+            size_data_exclusive = size_exclusive
+        elif subvol_id in id_subvolumes:
+            size_snapshot_total += size_total
+            size_snapshot_exclusive += size_exclusive
+
+def rescan_quota(path):
+    command = 'sudo btrfs quota rescan %s' % (path)
+    status, output = commands.getstatusoutput(command)
+    if status is not 0:
+        Exception(command)
+
+def print_human_readable(name):
+    global size_data_total
+    global size_data_exclusive
+    global size_snapshot_exclusive
+    size_data_total = size_data_total / (1024*1e6)
+    size_data_exclusive = size_data_exclusive / (1024*1e6)
+    size_snapshot_exclusive = size_snapshot_exclusive / (1024*1e6)
+    print '%10s: %6.1f Gb, %6.1f Gb, %6.1f Gb' % (name, size_data_total, size_data_exclusive, size_snapshot_exclusive)
+
+def print_rrd(name):
+    timestamp = int(time.time())
+    print('PUTVAL {}/exec-btrfs_{}/gauge-data_total {}:{:.1f}'.format(hostname, name, timestamp, size_data_total))
+    print('PUTVAL {}/exec-btrfs_{}/gauge-data_exclusive {}:{:.1f}'.format(hostname, name, timestamp, size_data_exclusive))
+    print('PUTVAL {}/exec-btrfs_{}/gauge-snapshot_total {}:{:.1f}'.format(hostname, name, timestamp, size_snapshot_total))
+    print('PUTVAL {}/exec-btrfs_{}/gauge-snapshot_exclusive {}:{:.1f}'.format(hostname, name, timestamp, size_snapshot_exclusive))
+
+
+#
+# Volumes to scan
+#
+hostname = 'server'
+interval = 10
+volumes = [
+    ['mezzo-scratch',   '/mnt/mezzo/scratch'],
+    ['mezzo-sync',      '/mnt/mezzo/sync'],
+    ['helium-personal', '/mnt/yotta/helium/personal'],
+    ['helium-shared',   '/mnt/yotta/helium/shared'],
+    ['neon',            '/mnt/yotta/neon'],
+    ['krypton',         '/mnt/yotta/krypton'],
+    ['xenon-borg',      '/mnt/yotta/xenon/borg'],
+    ['xenon-rsnapshot', '/mnt/yotta/xenon/rsnapshot']
+]
+
+
+#
+# Command line arguments
+#
+parser = argparse.ArgumentParser(description='Get BTRFS disk usage')
+parser.add_argument('-s', action='store_true', help='print in human readable format')
+args = parser.parse_args()
+human_readable = args.s
+
+
+#
+# Main
+#
+if human_readable:
+    for name, path in volumes:
+        get_disk_usage(name, path)
+        print_human_readable(name)
+else:
+    # RRD mode
+    while True:
+        for name, path in volumes:
+            get_disk_usage(name, path)        
+            print_rrd(name)
+
+        time.sleep(interval)
+        rescan_quota(path)

collectd/usr/local/bin/du-data

@@ -0,0 +1,77 @@
+#!/usr/bin/python
+
+#
+# Imports
+#
+import os
+import sys
+import time
+import commands
+import argparse
+
+
+#
+# Methods
+#
+def get_disk_usage(path, human_readable):
+    '''disk usage in human readable format (e.g. '2,1GB')'''
+    arguments = '-sh' if human_readable else '-s'
+    path = os.path.realpath(path)
+    command = 'sudo du %s %s' % (arguments, path)
+    status, output = commands.getstatusoutput(command)
+
+    if status is not 0:
+        raise Exception(command)
+
+    disk_usage = output.split()[0]
+    if not human_readable:
+        # du reports in units of 1024 bytes, convert to plain number of bytes
+        disk_usage = int(disk_usage) * 1024
+    return disk_usage
+
+#
+# Directories to scan
+#
+hostname = 'server'
+interval = 10
+directories = [
+   #['bram',        '/media/data/Personal/Bram'],
+    ['rik',         '/media/data/Personal/Rik'],
+    ['books',       '/media/data/Shared/Books'],
+    ['games',       '/media/data/Shared/Games'],
+    ['misc',        '/media/data/Shared/Miscellaneous'],
+    ['shows',       '/media/data/Shared/Video/Shows'],
+    ['movies',      '/media/data/Shared/Video/Movies'],
+    ['music',       '/media/data/Shared/Music'],
+    ['photographs', '/media/data/Shared/Photographs'],
+    ['pictures',    '/media/data/Shared/Pictures'],
+    ['raw',         '/media/data/Shared/Raw'],
+    ['software',    '/media/data/Shared/Software']
+]
+
+#
+# Command line arguments
+#
+parser = argparse.ArgumentParser(description='Get BTRFS disk usage')
+parser.add_argument('-s', action='store_true', help='print in human readable format')
+args = parser.parse_args()
+human_readable = args.s
+
+
+#
+# Main
+#
+if (human_readable):
+    for (name, path) in directories:
+        disk_usage = get_disk_usage(path, human_readable)
+        print('%s: %s' % (name, disk_usage))
+else:
+    # RRD mode
+    while True:
+        for (name, path) in directories:
+            disk_usage = get_disk_usage(path, human_readable)
+            timestamp = int(time.time())
+            size = float(disk_usage)
+            print('PUTVAL {}/exec-du_{}/gauge-size {}:{:.1f}'.format(hostname, name, timestamp, size))
+            sys.stdout.flush()
+        time.sleep(interval)

collectd/usr/local/bin/funds-data

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+import os
+import requests
+import re
+import time
+import datetime
+import sys
+import random
+
+collection = 'funds'
+interval = 3600
+funds_behr = {
+    'robeco_one_defensief': 'rg.one.def',
+    'robeco_one_neutraal' : 'rg.one.neut',
+    'robeco_one_offensief': 'rg.one.offe',
+    # 'robeco_plus_gprec'   : 'rg.bp.gprec',
+    # 'robeco_plus_glconsc' : 'rg.gl.consc',
+    # 'robeco_plus_uspreme' : 'rg.us.preme',
+    # 'robeco_plus_uslcfe'  : 'rg.us.lc.fe'
+}
+funds_morningstar = {
+    'robeco_one_defensief': 'F00000OZ3S',
+    'robeco_one_neutraal' : 'F00000OZ3T',
+    'robeco_one_offensief': 'F00000OZ3U',
+    # 'robeco_plus_gprec'   : 'F00000QDBK', # BP Global Premium Equities C
+    # 'robeco_plus_glconsc' : 'F00000PNRA', # Global Conservative Equities C EUR
+    # 'robeco_plus_uspreme' : 'F00000OWHQ', # BP US Premium Equities F
+    # 'robeco_plus_uslcfe'  : 'F00000QDBI'  # BP US Large Cap Equities F EUR
+}
+# log = open('/host/var/log/funds.log', 'a')
+
+def behr_backlog(funds):
+    for fund,code in funds.items():
+        dir = '/var/lib/collectd/rrd/{}/exec-fund-{}'.format(collection, fund)
+        if not os.path.isdir(dir):
+            url = 'http://www.behr.nl/Beurs/Slotkoersen/slotkoersen.php?fd={}'.format(code)
+            # url = 'http://haggis.no-ip.org/funds/{}.html'.format(code)
+            response = requests.get(url)
+            matches = re.findall('(\d+): ([\d\.]+)', response.text)
+            for match in matches:
+                for i in range(23):
+                    timestamp = int(time.mktime(datetime.datetime.strptime(match[0], '%y%m%d').replace(hour = i + 1).timetuple()))
+                    put_value(fund, match[1], timestamp)
+        sys.stdout.flush()
+
+def morningstar_ticker(funds):
+    for fund,code in funds.items():
+        url = 'http://www.morningstar.nl/nl/funds/snapshot/snapshot.aspx?id={}'.format(code)
+        response = requests.get(url)
+        matches = re.findall('>EUR[\s]*[^\d]*([\d,]+)', response.text)
+        quote = matches[0].replace(',', '.')
+        # quote = 100 + 50 * (random.random() - 1)
+        put_value(fund, quote)
+
+def put_value(fund, value, timestamp = 'N'):
+    print('PUTVAL {}/exec-fund-{}/gauge-ticker interval={} {}:{}'.format(collection, fund, interval, timestamp, value))
+    sys.stdout.flush()
+    if timestamp is 'N':
+        timestamp = int(time.time())
+    # log.write('{},{},{}\n'.format(fund, timestamp, int(value)))
+
+behr_backlog(funds_behr)
+while True:
+    seconds = os.times()[4]
+    morningstar_ticker(funds_morningstar)
+    elapsed = os.times()[4] - seconds
+    time.sleep(interval - elapsed)

collectd/usr/local/bin/power-data

@@ -0,0 +1,69 @@
+#!/usr/bin/python
+
+#
+# Imports
+#
+import sys
+import time
+import argparse
+import pylikwid
+
+#
+# Configuration
+#
+hostname = 'server'
+cpuid = 0
+pinfo = pylikwid.getpowerinfo()
+domainid = pinfo.get('domains').get('PKG').get('ID')
+measurement_duration = 10
+measurement_interval = 60
+dinfo = pinfo.get('domains')
+domain_names = dinfo.keys()
+domain_ids = [domain['ID'] for domain in dinfo.values()]
+
+#
+# Command line arguments
+#
+parser = argparse.ArgumentParser(description='Get CPU power consumption')
+parser.add_argument('-s', action='store_true', help='print in human readable format')
+args = parser.parse_args()
+human_readable = args.s
+
+# 
+# Methods
+# 
+def get_power():
+    #print dict(zip(domain_names, domain_ids))
+    start = list()
+    end = list()
+    power = list()
+    for domain_id in domain_ids:
+        e_start = pylikwid.startpower(cpuid, domain_id)
+        start.append(e_start)
+    time.sleep(measurement_duration)
+    for domain_id in domain_ids:
+        e_stop = pylikwid.stoppower(cpuid, domain_id)
+        end.append(e_stop)
+    for events in zip(start, end, domain_ids):
+        power.append(pylikwid.getpower(events[0], events[1], events[2]))
+
+    return dict(zip(domain_names, power))
+
+def print_rrd(measurements):
+    timestamp = int(time.time())
+    for measurement in measurements.items():
+        name = measurement[0].lower()
+        power = measurement[1]
+        print('PUTVAL {}/exec-power/gauge-{} {}:{:.1f}'.format(hostname, name, timestamp, power))
+
+#
+# Main
+#
+if (human_readable):
+    print get_power()
+else:
+    while True:
+        power = get_power()
+        print_rrd(power)
+        sys.stdout.flush()
+        time.sleep(measurement_interval)

collectd/usr/local/bin/speedtest-data

@@ -0,0 +1,12 @@
+#!/bin/bash
+SPEEDTEST=/sbin/speedtest-cli
+COLLECTION=server
+INTERVAL=1800
+
+while :; do
+    SECONDS=0
+    RESULT=($($SPEEDTEST | grep Mbit | cut -d' ' -f 2))
+    echo "PUTVAL $COLLECTION/exec-speedtest/gauge-download interval=$INTERVAL N:${RESULT[0]}"
+    echo "PUTVAL $COLLECTION/exec-speedtest/gauge-upload interval=$INTERVAL N:${RESULT[1]}"
+    sleep $((INTERVAL-$SECONDS))
+done

collectd/usr/local/bin/temperature-data

@@ -0,0 +1,43 @@
+#!/usr/bin/python -u
+
+#
+# Imports
+#
+import sys
+import time
+import commands
+import argparse
+
+
+#
+# Methods
+#
+def get_temperature(disks):
+    command = "sudo smartctl -a /dev/%s | grep Temperature_Celsius | awk '{print $10}'"  % disk
+    status, output = commands.getstatusoutput(command)
+
+    try:
+        return int(output)
+    except Exception as e:
+        return None
+
+
+#
+# Settings
+#
+hostname = 'server'
+interval = 10
+disks = ['sdd', 'sde', 'sdf']
+
+
+#
+# Main
+#
+while True:
+    timestamp = int(time.time())
+    for disk in disks:
+        temperature = get_temperature(disk)
+        if temperature:
+            print('PUTVAL {}/exec-temperature/gauge-{}_total {}:{}'.format(hostname, disk, timestamp, temperature))
+    time.sleep(interval)
+