From f7941e8ef5c49cffc5f579569bf20b7a81e956e7 Mon Sep 17 00:00:00 2001 From: Rik Veenboer Date: Fri, 3 Jan 2025 09:47:10 +0100 Subject: [PATCH] collectd configuration --- collectd/etc/collectd.conf | 989 +----------------- collectd/etc/collectd.conf.d/btrfs-data.conf | 10 +- .../etc/collectd.conf.d/cpufreq-data.conf | 5 + collectd/etc/collectd.conf.d/df.conf | 1 - collectd/etc/collectd.conf.d/du-data.conf | 3 +- .../etc/collectd.conf.d/speedtest-data.conf | 1 - collectd/usr/local/bin/btrfs-data | 142 ++- collectd/usr/local/bin/cpufreq-data | 57 + collectd/usr/local/bin/du-data | 98 +- collectd/usr/local/bin/power-data | 61 +- collectd/usr/local/bin/speedtest-data | 9 +- docker-compose.collectd.yaml | 32 +- docker-compose.yaml | 1 - 13 files changed, 321 insertions(+), 1088 deletions(-) create mode 100644 collectd/etc/collectd.conf.d/cpufreq-data.conf create mode 100755 collectd/usr/local/bin/cpufreq-data diff --git a/collectd/etc/collectd.conf b/collectd/etc/collectd.conf index af316d9..e29f493 100644 --- a/collectd/etc/collectd.conf +++ b/collectd/etc/collectd.conf @@ -16,8 +16,7 @@ Hostname "sepia" FQDNLookup true #BaseDir "/var/lib/collectd" #PluginDir "/usr/lib/collectd" -#TypesDB "/usr/share/collectd/types.db" "/etc/collectd/my_types.db" - +#TypesDB "/usr/share/collectd/types.db" "/etc/collectd/my_types.db" #----------------------------------------------------------------------------# # When enabled, plugins are loaded automatically with the default options # # when an appropriate block is encountered. # @@ -32,8 +31,7 @@ FQDNLookup true # Interval 60 # # # #----------------------------------------------------------------------------# -#Interval 10 -Interval 60 +Interval 10 #Timeout 2 #ReadThreads 5 @@ -53,121 +51,38 @@ Interval 60 ############################################################################## LoadPlugin logfile -#LoadPlugin syslog LogLevel "info" File "/var/lib/collectd/log/collectd.log" Timestamp true - PrintSeverity false + PrintSeverity true -# -# LogLevel info -# - ############################################################################## # LoadPlugin section # #----------------------------------------------------------------------------# # Specify what features to activate. # ############################################################################## -#LoadPlugin aggregation -#LoadPlugin amqp -#LoadPlugin apache -#LoadPlugin apcups -#LoadPlugin ascent -#LoadPlugin battery -#LoadPlugin bind -#LoadPlugin cgroups -#LoadPlugin conntrack -#LoadPlugin contextswitch LoadPlugin cpu LoadPlugin cpufreq -#LoadPlugin csv -#LoadPlugin curl -#LoadPlugin curl_json -#LoadPlugin curl_xml -#LoadPlugin dbi LoadPlugin df LoadPlugin disk -#LoadPlugin dns -#LoadPlugin email LoadPlugin entropy -#LoadPlugin ethstat -#LoadPlugin exec -#LoadPlugin filecount -#LoadPlugin fscache -#LoadPlugin gmond -#LoadPlugin hddtemp -LoadPlugin interface -#LoadPlugin ipmi -#LoadPlugin iptables -#LoadPlugin ipvs -LoadPlugin irq -#LoadPlugin java -#LoadPlugin libvirt +#LoadPlugin interface +#LoadPlugin irq LoadPlugin load -#LoadPlugin lvm -#LoadPlugin madwifi -#LoadPlugin mbmon LoadPlugin md -#LoadPlugin memcachec -#LoadPlugin memcached LoadPlugin memory -#LoadPlugin modbus -#LoadPlugin multimeter -#LoadPlugin mysql -#LoadPlugin netlink LoadPlugin network -LoadPlugin nfs -#LoadPlugin nginx -#LoadPlugin notify_desktop -#LoadPlugin notify_email -#LoadPlugin ntpd -#LoadPlugin numa -#LoadPlugin nut -#LoadPlugin olsrd -#LoadPlugin openvpn -# -# Globals true -# -#LoadPlugin pinba -#LoadPlugin ping -#LoadPlugin postgresql -#LoadPlugin powerdns LoadPlugin processes -#LoadPlugin protocols -# -# Globals true -# -#LoadPlugin rrdcached LoadPlugin rrdtool LoadPlugin sensors -#LoadPlugin serial -#LoadPlugin snmp -#LoadPlugin statsd LoadPlugin swap -#LoadPlugin table -#LoadPlugin tail -#LoadPlugin tail_csv -#LoadPlugin tcpconns -#LoadPlugin teamspeak2 -#LoadPlugin ted -LoadPlugin thermal -#LoadPlugin tokyotyrant -#LoadPlugin unixsock LoadPlugin uptime LoadPlugin users -#LoadPlugin uuid -#LoadPlugin varnish -#LoadPlugin vmem -#LoadPlugin vserver -#LoadPlugin wireless -#LoadPlugin write_graphite -#LoadPlugin write_http -#LoadPlugin write_riemann -LoadPLugin smart +LoadPlugin smart ############################################################################## # Plugin configuration # @@ -176,640 +91,6 @@ LoadPLugin smart # ription of those options is available in the collectd.conf(5) manual page. # ############################################################################## -# -# -# #Host "unspecified" -# Plugin "cpu" -# PluginInstance "/[0,2,4,6,8]$/" -# Type "cpu" -# #TypeInstance "unspecified" -# -# SetPlugin "cpu" -# SetPluginInstance "even-%{aggregation}" -# -# GroupBy "Host" -# GroupBy "TypeInstance" -# -# CalculateNum false -# CalculateSum false -# CalculateAverage true -# CalculateMinimum false -# CalculateMaximum false -# CalculateStddev false -# -# - -# -# -# Host "localhost" -# Port "5672" -# VHost "/" -# User "guest" -# Password "guest" -# Exchange "amq.fanout" -# RoutingKey "collectd" -# Persistent false -# StoreRates false -# -# - -# -# -# URL "http://localhost/server-status?auto" -# User "www-user" -# Password "secret" -# VerifyPeer false -# VerifyHost false -# CACert "/etc/ssl/ca.crt" -# Server "apache" -# -# -# -# URL "http://some.domain.tld/status?auto" -# Host "some.domain.tld" -# Server "lighttpd" -# -# - -# -# Host "localhost" -# Port "3551" -# ReportSeconds true -# - -# -# URL "http://localhost/ascent/status/" -# User "www-user" -# Password "secret" -# VerifyPeer false -# VerifyHost false -# CACert "/etc/ssl/ca.crt" -# - -# -# URL "http://localhost:8053/" -# -# ParseTime false -# -# OpCodes true -# QTypes true -# ServerStats true -# ZoneMaintStats true -# ResolverStats false -# MemoryStats true -# -# -# QTypes true -# ResolverStats true -# CacheRRSets true -# -# Zone "127.in-addr.arpa/IN" -# -# - -# -# CGroup "libvirt" -# IgnoreSelected false -# - -# -# DataDir "/var/lib/collectd/csv" -# StoreRates false -# - -# -# -# URL "http://finance.google.com/finance?q=NYSE%3AAMD" -# User "foo" -# Password "bar" -# VerifyPeer false -# VerifyHost false -# CACert "/etc/ssl/ca.crt" -# MeasureResponseTime false -# -# Regex "]*> *([0-9]*\\.[0-9]+) *" -# DSType "GaugeAverage" -# Type "stock_value" -# Instance "AMD" -# -# -# - -# -## See: http://wiki.apache.org/couchdb/Runtime_Statistics -# -# Instance "httpd" -# -# Type "http_requests" -# -# -# -# Type "http_request_methods" -# -# -# -# Type "http_response_codes" -# -# -## Database status metrics: -# -# Instance "dbs" -# -# Type "gauge" -# -# -# Type "counter" -# -# -# Type "bytes" -# -# -# - -# -# -# Host "my_host" -# Instance "some_instance" -# User "collectd" -# Password "thaiNg0I" -# VerifyPeer true -# VerifyHost true -# CACert "/path/to/ca.crt" -# -# -# Type "magic_level" -# InstancePrefix "prefix-" -# InstanceFrom "td[1]" -# ValuesFrom "td[2]/span[@class=\"level\"]" -# -# -# - -# -# -# Statement "SELECT 'customers' AS c_key, COUNT(*) AS c_value \ -# FROM customers_tbl" -# MinVersion 40102 -# MaxVersion 50042 -# -# Type "gauge" -# InstancePrefix "customer" -# InstancesFrom "c_key" -# ValuesFrom "c_value" -# -# -# -# -# Driver "mysql" -# DriverOption "host" "localhost" -# DriverOption "username" "collectd" -# DriverOption "password" "secret" -# DriverOption "dbname" "custdb0" -# SelectDB "custdb0" -# Query "num_of_customers" -# Query "..." -# Host "..." -# -# - -# -# Disk "hda" -# Disk "/sda[23]/" -# IgnoreSelected false -# - -# -# Interface "eth0" -# IgnoreSource "192.168.0.1" -# SelectNumericQueryTypes false -# - -# -# SocketFile "/var/run/collectd-email" -# SocketGroup "collectd" -# SocketPerms "0770" -# MaxConns 5 -# - -# -# Interface "eth0" -# Map "rx_csum_offload_errors" "if_rx_errors" "checksum_offload" -# Map "multicast" "if_multicast" -# MappedOnly false -# - -# -# Exec user "/path/to/exec" -# Exec "user:group" "/path/to/exec" -# NotificationExec user "/path/to/exec" -# - -# -# -# Instance "foodir" -# Name "*.conf" -# MTime "-5m" -# Size "+10k" -# Recursive true -# IncludeHidden false -# -# - -# -# MCReceiveFrom "239.2.11.71" "8649" -# -# -# Type "swap" -# TypeInstance "total" -# DataSource "value" -# -# -# -# Type "swap" -# TypeInstance "free" -# DataSource "value" -# -# - -# -# Host "127.0.0.1" -# Port 7634 -# - -# -# Interface "eth0" -# IgnoreSelected false -# - - Interface "veth*" - IgnoreSelected true - - -# -# Sensor "some_sensor" -# Sensor "another_one" -# IgnoreSelected false -# NotifySensorAdd false -# NotifySensorRemove true -# NotifySensorNotPresent false -# - -# -# Chain "table" "chain" -# - -# -# Irq 7 -# Irq 8 -# Irq 9 -# IgnoreSelected true -# - -# -# JVMArg "-verbose:jni" -# JVMArg "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar" -# -# LoadPlugin "org.collectd.java.GenericJMX" -# -# # See /usr/share/doc/collectd/examples/GenericJMX.conf -# # for an example config. -# -# - -# -# Connection "xen:///" -# RefreshInterval 60 -# Domain "name" -# BlockDevice "name:device" -# InterfaceDevice "name:device" -# IgnoreSelected false -# HostnameFormat name -# InterfaceFormat name -# - -# -# Interface "wlan0" -# IgnoreSelected false -# Source "SysFS" -# WatchSet "None" -# WatchAdd "node_octets" -# WatchAdd "node_rssi" -# WatchAdd "is_rx_acl" -# WatchAdd "is_scan_active" -# - -# -# Host "127.0.0.1" -# Port 411 -# - -# -# Device "/dev/md0" -# IgnoreSelected false -# - -# -# -# Server "localhost" -# Key "page_key" -# -# Regex "(\\d+) bytes sent" -# ExcludeRegex "" -# DSType CounterAdd -# Type "ipt_octets" -# Instance "type_instance" -# -# -# - -# -# -# Socket "/var/run/memcached.sock" -# or: -# Host "127.0.0.1" -# Port "11211" -# -# - -# -# -# RegisterBase 1234 -# RegisterType float -# Type gauge -# Instance "..." -# -# -# -# Address "addr" -# Port "1234" -# Interval 60 -# -# -# Instance "foobar" # optional -# Collect "data_name" -# -# -# - -# -# -# Host "database.serv.er" -# Port "3306" -# User "db_user" -# Password "secret" -# Database "db_name" -# MasterStats true -# -# -# -# Host "localhost" -# Socket "/var/run/mysql/mysqld.sock" -# SlaveStats true -# SlaveNotifications true -# -# - -# -# Interface "All" -# VerboseInterface "All" -# QDisc "eth0" "pfifo_fast-1:0" -# Class "ppp0" "htb-1:10" -# Filter "ppp0" "u32-1:0" -# IgnoreSelected false -# - -# -# # client setup: -# Server "ff18::efc0:4a42" "25826" -# -# SecurityLevel Encrypt -# Username "user" -# Password "secret" -# Interface "eth0" -# -# TimeToLive "128" -# -# # server setup: -# Listen "ff18::efc0:4a42" "25826" -# -# SecurityLevel Sign -# AuthFile "/etc/collectd/passwd" -# Interface "eth0" -# -# MaxPacketSize 1024 -# -# # proxy setup (client and server as above): -# Forward true -# -# # statistics about the network plugin itself -# ReportStats false -# -# # "garbage collection" -# CacheFlush 1800 -# - -# -# URL "http://localhost/status?auto" -# User "www-user" -# Password "secret" -# VerifyPeer false -# VerifyHost false -# CACert "/etc/ssl/ca.crt" -# - -# -# OkayTimeout 1000 -# WarningTimeout 5000 -# FailureTimeout 0 -# - -# -# SMTPServer "localhost" -# SMTPPort 25 -# SMTPUser "my-username" -# SMTPPassword "my-password" -# From "collectd@main0server.com" -# # on . -# # Beware! Do not use not more than two placeholders (%)! -# Subject "[collectd] %s on %s!" -# Recipient "email1@domain1.net" -# Recipient "email2@domain2.com" -# - -# -# Host "localhost" -# Port 123 -# ReverseLookups false -# IncludeUnitID true -# - -# -# UPS "upsname@hostname:port" -# - -# -# Host "127.0.0.1" -# Port "2006" -# CollectLinks "Summary" -# CollectRoutes "Summary" -# CollectTopology "Summary" -# - -# -# StatusFile "/etc/openvpn/openvpn-status.log" -# ImprovedNamingSchema false -# CollectCompression true -# CollectIndividualUsers true -# CollectUserCount false -# - -# -# IncludeDir "/my/include/path" -# BaseName "Collectd::Plugins" -# EnableDebugger "" -# LoadPlugin Monitorus -# LoadPlugin OpenVZ -# -# -# Foo "Bar" -# Qux "Baz" -# -# - -# -# Address "::0" -# Port "30002" -# -# Host "host name" -# Server "server name" -# Script "script name" -# -# - -# -# Host "host.foo.bar" -# Host "host.baz.qux" -# Interval 1.0 -# Timeout 0.9 -# TTL 255 -# SourceAddress "1.2.3.4" -# Device "eth0" -# MaxMissed -1 -# - -# -# -# Statement "SELECT magic FROM wizard WHERE host = $1;" -# Param hostname -# -# -# Type gauge -# InstancePrefix "magic" -# ValuesFrom "magic" -# -# -# -# -# Statement "SELECT COUNT(type) AS count, type \ -# FROM (SELECT CASE \ -# WHEN resolved = 'epoch' THEN 'open' \ -# ELSE 'resolved' END AS type \ -# FROM tickets) type \ -# GROUP BY type;" -# -# -# Type counter -# InstancePrefix "rt36_tickets" -# InstancesFrom "type" -# ValuesFrom "count" -# -# -# -# -# # See /usr/share/doc/collectd-core/examples/postgresql/collectd_insert.sql for details -# Statement "SELECT collectd_insert($1, $2, $3, $4, $5, $6, $7, $8, $9);" -# StoreRates true -# -# -# -# Host "hostname" -# Port 5432 -# User "username" -# Password "secret" -# -# SSLMode "prefer" -# KRBSrvName "kerberos_service_name" -# -# Query magic -# -# -# -# Interval 60 -# Service "service_name" -# -# Query backend # predefined -# Query rt36_tickets -# -# -# -# Service "collectd_store" -# Writer sqlstore -# # see collectd.conf(5) for details -# CommitInterval 30 -# -# - -# -# -# Collect "latency" -# Collect "udp-answers" "udp-queries" -# Socket "/var/run/pdns.controlsocket" -# -# -# Collect "questions" -# Collect "cache-hits" "cache-misses" -# Socket "/var/run/pdns_recursor.controlsocket" -# -# LocalSocket "/opt/collectd/var/run/collectd-powerdns" -# - -# -# Process "name" -# ProcessMatch "foobar" "/usr/bin/perl foobar\\.pl.*" -# - -# -# Value "/^Tcp:/" -# IgnoreSelected false -# - -# -# ModulePath "/path/to/your/python/modules" -# LogTraces true -# Interactive true -# Import "spam" -# -# -# spam "wonderful" "lovely" -# -# - -# -# DaemonAddress "unix:/var/run/rrdcached.sock" -# DataDir "/var/lib/rrdcached/db/collectd" -# CreateFiles true -# CreateFilesAsync false -# CollectStatistics true -# -# The following settings are rather advanced -# and should usually not be touched: -# StepSize 10 -# HeartBeat 20 -# RRARows 1200 -# RRATimespan 158112000 -# XFF 0.1 -# - DataDir "/var/lib/collectd/rrd" # CacheTimeout 120 @@ -827,250 +108,24 @@ LoadPLugin smart # XFF 0.1 -# -# SensorConfigFile "/etc/sensors3.conf" -# Sensor "it8712-isa-0290/temperature-temp1" -# Sensor "it8712-isa-0290/fanspeed-fan3" -# Sensor "it8712-isa-0290/voltage-in8" -# IgnoreSelected false + + Disk "sda" + Disk "sdb" + Disk "nvme0" + IgnoreSelected false + + +# +# Device "/dev/md0" +# IgnoreSelected false # -# See /usr/share/doc/collectd/examples/snmp-data.conf.gz for a -# comprehensive sample configuration. -# -# -# Type "voltage" -# Table false -# Instance "input_line1" -# Scale 0.1 -# Values "SNMPv2-SMI::enterprises.6050.5.4.1.1.2.1" -# -# -# Type "users" -# Table false -# Instance "" -# Shift -1 -# Values "HOST-RESOURCES-MIB::hrSystemNumUsers.0" -# -# -# Type "if_octets" -# Table true -# InstancePrefix "traffic" -# Instance "IF-MIB::ifDescr" -# Values "IF-MIB::ifInOctets" "IF-MIB::ifOutOctets" -# -# -# -# Address "192.168.0.2" -# Version 1 -# Community "community_string" -# Collect "std_traffic" -# Inverval 120 -# -# -# Address "192.168.0.42" -# Version 2 -# Community "another_string" -# Collect "std_traffic" "hr_users" -# -# -# Address "192.168.0.3" -# Version 1 -# Community "more_communities" -# Collect "powerplus_voltge_input" -# Interval 300 -# -# - -# -# Host "::" -# Port "8125" -# DeleteCounters false -# DeleteTimers false -# DeleteGauges false -# DeleteSets false -# TimerPercentile 90.0 -# - -# -# ReportByDevice false -# ReportBytes true -# - -# -# -# Instance "slabinfo" -# Separator " " -# -# Type gauge -# InstancePrefix "active_objs" -# InstancesFrom 0 -# ValuesFrom 1 -# -# -# Type gauge -# InstancePrefix "objperslab" -# InstancesFrom 0 -# ValuesFrom 4 -# -#
-# - -# -# -# Instance "exim" -# -# Regex "S=([1-9][0-9]*)" -# DSType "CounterAdd" -# Type "ipt_bytes" -# Instance "total" -# -# -# Regex "\\" -# ExcludeRegex "\\.*mail_spool defer" -# DSType "CounterInc" -# Type "counter" -# Instance "local_user" -# -# -# - -# -# -# Type "percent" -# Instance "dropped" -# ValueFrom 1 -# -# -# Type "bytes" -# Instance "wire-realtime" -# ValueFrom 2 -# -# -# Type "alerts_per_second" -# ValueFrom 3 -# -# -# Type "kpackets_wire_per_sec.realtime" -# ValueFrom 4 -# -# -# Instance "snort-eth0" -# Interval 600 -# Collect "dropped" "mbps" "alerts" "kpps" -# TimeFrom 0 -# -# - -# -# ListeningPorts false -# LocalPort "25" -# RemotePort "25" -# - -# -# Host "127.0.0.1" -# Port "51234" -# Server "8767" -# - -# -# Device "/dev/ttyUSB0" -# Retries 0 -# - -# -# ForceUseProcfs false -# Device "THRM" -# IgnoreSelected false -# - -# -# Host "localhost" -# Port "1978" -# - -# -# SocketFile "/var/run/collectd-unixsock" -# SocketGroup "collectd" -# SocketPerms "0660" -# DeleteSocket false -# - -# -# UUIDFile "/etc/uuid" -# - -# -# -# CollectCache true -# CollectBackend true -# CollectBan false # Varnish 3 only -# CollectConnections true -# CollectDirectorDNS false # Varnish 3 only -# CollectSHM true -# CollectESI false -# CollectFetch false -# CollectHCB false -# CollectObjects false -# CollectPurge false # Varnish 2 only -# CollectSession false -# CollectSMA false # Varnish 2 only -# CollectSMS false -# CollectSM false # Varnish 2 only -# CollectStruct false -# CollectTotals false -# CollectUptime false -# CollectdVCL false -# CollectWorkers false -# -# -# -# CollectCache true -# -# - -# -# Verbose false -# - -# -# -# Host "localhost" -# Port "2003" -# Protocol "udp" -# LogSendErrors true -# Prefix "collectd" -# Postfix "collectd" -# StoreRates true -# AlwaysAppendDS false -# EscapeCharacter "_" -# -# - -# -# -# User "collectd" -# Password "secret" -# VerifyPeer true -# VerifyHost true -# CACert "/etc/ssl/ca.crt" -# Format "Command" -# StoreRates false -# -# - -# -# -# Host "localhost" -# Port 5555 -# Protocol UDP -# StoreRates true -# AlwaysAppendDS false -# TTLFactor 2.0 -# -# Tag "foobar" -# + + Disk "sda" + Disk "sdb" + Disk "nvme0" + IgnoreSelected false + Filter "*.conf" diff --git a/collectd/etc/collectd.conf.d/btrfs-data.conf b/collectd/etc/collectd.conf.d/btrfs-data.conf index 1bc3b43..3e8fe80 100644 --- a/collectd/etc/collectd.conf.d/btrfs-data.conf +++ b/collectd/etc/collectd.conf.d/btrfs-data.conf @@ -1,5 +1,5 @@ -#LoadPlugin exec -# -# -# Exec nobody "/host/usr/local/bin/btrfs-data" -# +LoadPlugin exec + + + Exec collectd "/host/usr/local/bin/btrfs-data" + diff --git a/collectd/etc/collectd.conf.d/cpufreq-data.conf b/collectd/etc/collectd.conf.d/cpufreq-data.conf new file mode 100644 index 0000000..f3b1e15 --- /dev/null +++ b/collectd/etc/collectd.conf.d/cpufreq-data.conf @@ -0,0 +1,5 @@ +LoadPlugin exec + + + Exec collectd "/host/usr/local/bin/cpufreq-data" + diff --git a/collectd/etc/collectd.conf.d/df.conf b/collectd/etc/collectd.conf.d/df.conf index 50822d3..594f9ee 100644 --- a/collectd/etc/collectd.conf.d/df.conf +++ b/collectd/etc/collectd.conf.d/df.conf @@ -1,6 +1,5 @@ MountPoint "/media/docker" - MountPoint "/media/scratch" FSType "ext4" IgnoreSelected false diff --git a/collectd/etc/collectd.conf.d/du-data.conf b/collectd/etc/collectd.conf.d/du-data.conf index 13e7910..3382f7b 100644 --- a/collectd/etc/collectd.conf.d/du-data.conf +++ b/collectd/etc/collectd.conf.d/du-data.conf @@ -1,6 +1,5 @@ LoadPlugin exec - Exec nobody "/host/usr/local/bin/du-data" + Exec collectd "/host/usr/local/bin/du-data" - diff --git a/collectd/etc/collectd.conf.d/speedtest-data.conf b/collectd/etc/collectd.conf.d/speedtest-data.conf index 7eba4e9..1902744 100644 --- a/collectd/etc/collectd.conf.d/speedtest-data.conf +++ b/collectd/etc/collectd.conf.d/speedtest-data.conf @@ -3,4 +3,3 @@ LoadPlugin exec Exec nobody "/host/usr/local/bin/speedtest-data" - diff --git a/collectd/usr/local/bin/btrfs-data b/collectd/usr/local/bin/btrfs-data index 494d1ec..6d84738 100755 --- a/collectd/usr/local/bin/btrfs-data +++ b/collectd/usr/local/bin/btrfs-data @@ -1,18 +1,18 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Imports # import sys import time -import commands +import subprocess import argparse # # Misc # -#sys.tracebacklimit = 0 +# sys.tracebacklimit = 0 # @@ -28,21 +28,42 @@ size_snapshot_exclusive = 0 # Methods # def get_subvol_list(path): - command = "btrfs subvolume list -t %s" % (path) - status, output = commands.getstatusoutput(command) + command = "sudo btrfs subvolume list -t %s" % (path) + status, output = subprocess.getstatusoutput(command) - if status is not 0: + if status != 0: raise Exception(command) # Every line contains the following values: subvol_id, gen, toplevel, path return output.splitlines()[2:] +def get_filesystem_size(path): + command = "sudo btrfs filesystem show --raw %s" % (path) + status, output = subprocess.getstatusoutput(command) + + if status != 0: + # This command fails when running inside Docker container + # return maximum size of any filesystem instead + command = "sudo btrfs filesystem show --raw" + status, output = subprocess.getstatusoutput(command) + lines = output.splitlines() + lines = [x for x in lines if "devid" in x] + sizes = [int(line.split()[3]) for line in lines] + return max(sizes) + + # The sizes are on the third line + line = output.splitlines()[2] + + # Element 3 and 5 respectively contain total and used sizes + return int(line.split()[3]) + + def get_id_root(name, path): lines = get_subvol_list(path) # Filter lines where toplevel == 5 - subvol_ids = filter(lambda x: int(x.split()[2]) == 5, lines) + subvol_ids = [x for x in lines if int(x.split()[2]) == 5] # Try to retrieve the subvol_id for the root subvolume (if any) if len(subvol_ids) == 1: @@ -51,28 +72,30 @@ def get_id_root(name, path): else: # The path contains a btrfs filesystem with multiple subvolumes for data try: - return int(filter(lambda x: x.split()[3] == name, subvol_ids)[0].split()[0]) + return int(list(filter(lambda x: x.split()[3] == name, subvol_ids))[0].split()[0]) except IndexError: pass - # Volume not found, root is probably the btrfs default (5) return 5 - + + def get_id_subvolumes(path, subvol_id): lines = get_subvol_list(path) - lines = filter(lambda x: int(x.split()[2]) == subvol_id, lines) - return list(map(lambda x: int(x.split()[0]), lines)) + lines = [x for x in lines if int(x.split()[2]) == subvol_id] + return list([int(x.split()[0]) for x in lines]) def get_disk_usage(name, path): id_root = get_id_root(name, path) id_subvolumes = get_id_subvolumes(path, id_root) + size_filesystem = get_filesystem_size(path) - command = "btrfs qgroup show --raw %s" % (path) - status, output = commands.getstatusoutput(command) + # Get disk usage from quota + command = "sudo btrfs qgroup show --raw %s" % (path) + status, output = subprocess.getstatusoutput(command) - if status is not 0: + if status != 0: raise Exception(command) lines = output.splitlines()[2:] @@ -94,16 +117,28 @@ def get_disk_usage(name, path): # -> data that is not (yet) incorporated in a snapshot size_data_exclusive = 0 - # Data exclusively available in snapshots # -> data that was removed from volume size_snapshot_exclusive = 0 for line in lines: split = line.split() - subvol_id = int(split[0].split("/")[1]) - size_total = float(split[1]) - size_exclusive = float(split[2]) + subvol_id = 0 + size_total = 0 + size_exclusive = 0 + try: + subvol_id = int(split[0].split("/")[1]) + size_total = float(split[1]) + size_exclusive = float(split[2]) + except IndexError: + # ignore "WARNING: Quota disabled" + pass + + # size_exclusive is incorrect when snapshot is + # removed and qgroups are not updated yet, + # ignore the value when it seems unrealistic + if size_exclusive > size_filesystem: + size_exclusive = 0 if subvol_id == id_root: size_data_total = size_total @@ -112,27 +147,57 @@ def get_disk_usage(name, path): size_snapshot_total += size_total size_snapshot_exclusive += size_exclusive + def rescan_quota(path): - command = "btrfs quota rescan %s" % (path) - status, output = commands.getstatusoutput(command) - if status is not 0: + command = "sudo btrfs quota rescan %s" % (path) + status, output = subprocess.getstatusoutput(command) + if status != 0: Exception(command) + def print_human_readable(name): global size_data_total global size_data_exclusive global size_snapshot_exclusive - size_data_total = size_data_total / (1024*1e6) - size_data_exclusive = size_data_exclusive / (1024*1e6) - size_snapshot_exclusive = size_snapshot_exclusive / (1024*1e6) - print "%10s: %6.1f Gb, %6.1f Gb, %6.1f Gb" % (name, size_data_total, size_data_exclusive, size_snapshot_exclusive) + size_data_total = size_data_total / (1024 * 1e6) + size_data_exclusive = size_data_exclusive / (1024 * 1e6) + size_snapshot_exclusive = size_snapshot_exclusive / (1024 * 1e6) + print( + "%10s: %6.1f Gb, %6.1f Gb, %6.1f Gb" + % (name, size_data_total, size_data_exclusive, size_snapshot_exclusive) + ) + def print_rrd(name): timestamp = int(time.time()) - print("PUTVAL {}/exec-btrfs_{}/gauge-data_total {}:{:.1f}".format(hostname, name, timestamp, size_data_total)) - print("PUTVAL {}/exec-btrfs_{}/gauge-data_exclusive {}:{:.1f}".format(hostname, name, timestamp, size_data_exclusive)) - print("PUTVAL {}/exec-btrfs_{}/gauge-snapshot_total {}:{:.1f}".format(hostname, name, timestamp, size_snapshot_total)) - print("PUTVAL {}/exec-btrfs_{}/gauge-snapshot_exclusive {}:{:.1f}".format(hostname, name, timestamp, size_snapshot_exclusive)) + print( + ( + "PUTVAL {}/exec-btrfs_{}/gauge-data_total {}:{:.1f}".format( + hostname, name, timestamp, size_data_total + ) + ) + ) + print( + ( + "PUTVAL {}/exec-btrfs_{}/gauge-data_exclusive {}:{:.1f}".format( + hostname, name, timestamp, size_data_exclusive + ) + ) + ) + print( + ( + "PUTVAL {}/exec-btrfs_{}/gauge-snapshot_total {}:{:.1f}".format( + hostname, name, timestamp, size_snapshot_total + ) + ) + ) + print( + ( + "PUTVAL {}/exec-btrfs_{}/gauge-snapshot_exclusive {}:{:.1f}".format( + hostname, name, timestamp, size_snapshot_exclusive + ) + ) + ) # @@ -143,19 +208,18 @@ interval = 10 volumes = list() # 275 GB SSD -volumes.append(["@", "/host/root/"]) -volumes.append(["@home", "/host/root/home"]) -volumes.append(["opt", "/host/root/opt"]) +volumes.append(["home", "/host/root/home"]) # 2x 4TB HDD volumes.append(["data", "/host/root/media/data"]) - +volumes.append(["backup", "/host/root/media/backup"]) +volumes.append(["seafile", "/host/root/media/seafile"]) # # Command line arguments # -parser = argparse.ArgumentParser(description='Get BTRFS disk usage') -parser.add_argument('-s', action='store_true', help='print in human readable format') +parser = argparse.ArgumentParser(description="Get BTRFS disk usage") +parser.add_argument("-s", action="store_true", help="print in human readable format") args = parser.parse_args() human_readable = args.s @@ -163,7 +227,7 @@ human_readable = args.s # # Main # -if (human_readable): +if human_readable: for (name, path) in volumes: get_disk_usage(name, path) print_human_readable(name) @@ -172,9 +236,9 @@ else: while True: for (name, path) in volumes: get_disk_usage(name, path) - + print_rrd(name) sys.stdout.flush() time.sleep(interval) - #rescan_quota(path) + # rescan_quota(path) diff --git a/collectd/usr/local/bin/cpufreq-data b/collectd/usr/local/bin/cpufreq-data new file mode 100755 index 0000000..403344b --- /dev/null +++ b/collectd/usr/local/bin/cpufreq-data @@ -0,0 +1,57 @@ +#!/usr/bin/python3 +import argparse +import time +import sys +import os + +hostname = "sepia" +measurement_interval = 5 + + +def get_cpu_frequencies(): + frequencies = [] + try: + cpu_dirs = [ + d + for d in os.listdir("/sys/devices/system/cpu/") + if d.startswith("cpu") and d[3:].isdigit() + ] + for cpu_dir in cpu_dirs: + with open( + f"/sys/devices/system/cpu/{cpu_dir}/cpufreq/scaling_cur_freq", "r" + ) as f: + frequency = int(f.read().strip()) / 1000 # Convert Hz to MHz + frequencies.append((int(cpu_dir[3:]), frequency)) + except Exception as e: + print("Error:", e) + return frequencies + + +def main(): + parser = argparse.ArgumentParser(description="Query CPU frequencies.") + parser.add_argument( + "-s", + "--human-readable", + action="store_true", + help="Print frequencies in human-readable format", + ) + args = parser.parse_args() + + if args.human_readable: + frequencies = get_cpu_frequencies() + for cpu, frequency in frequencies: + print(f"CPU{cpu} Frequency: {frequency:.2f} MHz") + else: + while True: + frequencies = get_cpu_frequencies() + timestamp = int(time.time()) + for cpu, frequency in frequencies: + print( + f"PUTVAL {hostname}/cpu-frequency/gauge-cpu{cpu} {timestamp}:{frequency:.0f}" + ) + sys.stdout.flush() + time.sleep(measurement_interval) + + +if __name__ == "__main__": + main() diff --git a/collectd/usr/local/bin/du-data b/collectd/usr/local/bin/du-data index 0caf2c0..9670dd4 100755 --- a/collectd/usr/local/bin/du-data +++ b/collectd/usr/local/bin/du-data @@ -1,25 +1,77 @@ -#!/bin/bash -COLLECTION=sepia -INTERVAL=90 +#!/usr/bin/python3 -DIRS=$(cat <