Add monitoring

2025-10-10 14:48:15 +01:00
parent 92cf99d41f
commit ec7d08bcb7
17 changed files with 2735 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,8 @@ coolify/sentinel
 syncthing-sparky/index-v2
 syncthing-sparky/index-v0.14.0.db-migrated
 coolify/services/sowk04gs00g444ks4ccs4sw0/data/docker/registry/v2
+monitoring/grafana/data
+monitoring/prometheus/data
+monitoring/prometheus/newdata
+monitoring/victoriametrics/data
+monitoring/pinger/nohup.out
--- a/monitoring/docker-compose.yml
+++ b/monitoring/docker-compose.yml
@@ -0,0 +1,121 @@
+services:
+    prometheus:
+        image: prom/prometheus:v3.5.0
+        container_name: prometheus
+        volumes:
+            - ./prometheus/config:/etc/prometheus
+            - ./prometheus/data:/prometheus
+        command:
+            - "--config.file=/etc/prometheus/prometheus.yml"
+            - "--storage.tsdb.path=/prometheus"
+            - "--web.console.libraries=/etc/prometheus/console_libraries"
+            - "--web.console.templates=/etc/prometheus/consoles"
+            - "--storage.tsdb.retention.time=1d"
+            - "--web.enable-lifecycle"
+            - "--web.listen-address=:43261"
+        restart: unless-stopped
+        expose:
+            - 43261
+        network_mode: "host"
+        labels:
+            org.label-schema.group: "monitoring"
+
+    nodeexporter:
+        image: prom/node-exporter:v1.8.2
+        container_name: nodeexporter
+        volumes:
+            - /proc:/host/proc:ro
+            - /sys:/host/sys:ro
+            - /:/rootfs:ro
+        command:
+            - "--path.procfs=/host/proc"
+            - "--path.rootfs=/rootfs"
+            - "--path.sysfs=/host/sys"
+            - "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)"
+            - "--web.listen-address=:56546"
+        restart: unless-stopped
+        expose:
+            - 56546
+        network_mode: "host"
+        labels:
+            org.label-schema.group: "monitoring"
+
+    # cadvisor:
+    #     image: gcr.io/google-containers/cadvisor:v0.34.0
+    #     container_name: cadvisor
+    #     volumes:
+    #         - /:/rootfs:ro
+    #         - /var/run:/var/run:rw
+    #         - /sys:/sys:ro
+    #         - /var/lib/docker:/var/lib/docker:ro
+    #         #- /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux
+    #     restart: unless-stopped
+    #     expose:
+    #         - 8080
+    #     network_mode: "host"
+    #     labels:
+    #         org.label-schema.group: "monitoring"
+
+    # We run grafana via coolify so that it may be exposed to internet
+    # grafana:
+    #     image: grafana/grafana:11.5
+    #     container_name: grafana
+    #     volumes:
+    #         - ./grafana/data:/var/lib/grafana
+    #         - ./grafana/config:/etc/grafana
+    #     environment:
+    #         - GF_SECURITY_ADMIN_USER=dave
+    #         - GF_SECURITY_ADMIN_PASSWORD=9@^F@93qHv48JXHvi3Bf
+    #         - GF_USERS_ALLOW_SIGN_UP=true
+    #     restart: unless-stopped
+    #     expose:
+    #         - 43621
+    #     network_mode: "host"
+    #     labels:
+    #         org.label-schema.group: "monitoring"
+
+    #loki:
+    #    container_name: loki
+    #    image: grafana/loki:3.3.2
+    #    expose:
+    #      - 43622
+    #    command: -config.file=/etc/loki/config.yaml
+    #    restart: unless-stopped
+    #    volumes:
+    #      - ./loki/config:/etc/loki
+    #      - ./loki/data:/data/loki
+    #    network_mode: "host"
+
+    #promtail:
+    #    container_name: promtail
+    #    image: grafana/promtail:3.3.2
+    #    expose:
+    #      - 43623
+    #    volumes:
+    #      - /var/log:/var/log
+    #      - /mnt/data/docker/containers:/var/lib/docker/containers:ro
+    #      - ./promtail/config:/etc/promtail
+    #    command: -config.file=/etc/promtail/config.yml
+    #    restart: unless-stopped
+    #    network_mode: "host"
+
+    #pinger:
+    #    image: pinger
+    #    container_name: pinger
+    #    restart: unless-stopped
+    #    network_mode: "host"
+    #    volumes:
+    #      - ./pinger/logs:/logs
+    victoriametrics:
+        image: victoriametrics/victoria-metrics
+        container_name: victoriametrics
+        volumes:
+          - ./victoriametrics/data:/victoria-metrics-data
+        command: 
+          - "--retentionPeriod=100y"
+        restart: unless-stopped
+        expose:
+            - 8428
+        network_mode: "host"
+        labels:
+            org.label-schema.group: "monitoring"
--- a/monitoring/grafana/config/grafana.ini
+++ b/monitoring/grafana/config/grafana.ini
--- a/monitoring/grafana/config/ldap.toml
+++ b/monitoring/grafana/config/ldap.toml
@@ -0,0 +1,75 @@
+# To troubleshoot and get more log info enable ldap debug logging in grafana.ini
+# [log]
+# filters = ldap:debug
+
+[[servers]]
+# Ldap server host (specify multiple hosts space separated)
+host = "127.0.0.1"
+# Default port is 389 or 636 if use_ssl = true
+port = 389
+# Set to true if LDAP server should use an encrypted TLS connection (either with STARTTLS or LDAPS)
+use_ssl = false
+# If set to true, use LDAP with STARTTLS instead of LDAPS
+start_tls = false
+# The value of an accepted TLS cipher. By default, this value is empty. Example value: ["TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"])
+# For a complete list of supported ciphers and TLS versions, refer to: https://go.dev/src/crypto/tls/cipher_suites.go
+# Starting with Grafana v11.0 only ciphers with ECDHE support are accepted for TLS 1.2 connections.
+tls_ciphers = []
+# This is the minimum TLS version allowed. By default, this value is empty. Accepted values are: TLS1.1 (only for Grafana v10.4 or older), TLS1.2, TLS1.3.
+min_tls_version = ""
+# set to true if you want to skip ssl cert validation
+ssl_skip_verify = false
+# set to the path to your root CA certificate or leave unset to use system defaults
+# root_ca_cert = "/path/to/certificate.crt"
+# Authentication against LDAP servers requiring client certificates
+# client_cert = "/path/to/client.crt"
+# client_key = "/path/to/client.key"
+
+# Search user bind dn
+bind_dn = "cn=admin,dc=grafana,dc=org"
+# Search user bind password
+# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
+bind_password = 'grafana'
+# We recommend using variable expansion for the bind_password, for more info https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion
+# bind_password = '$__env{LDAP_BIND_PASSWORD}'
+
+# Timeout in seconds (applies to each host specified in the 'host' entry (space separated))
+timeout = 10
+
+# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)"
+search_filter = "(cn=%s)"
+
+# An array of base dns to search through
+search_base_dns = ["dc=grafana,dc=org"]
+
+## For Posix or LDAP setups that does not support member_of attribute you can define the below settings
+## Please check grafana LDAP docs for examples
+# group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))"
+# group_search_base_dns = ["ou=groups,dc=grafana,dc=org"]
+# group_search_filter_user_attribute = "uid"
+
+# Specify names of the ldap attributes your ldap uses
+[servers.attributes]
+name = "givenName"
+surname = "sn"
+username = "cn"
+member_of = "memberOf"
+email =  "email"
+
+# Map ldap groups to grafana org roles
+[[servers.group_mappings]]
+group_dn = "cn=admins,ou=groups,dc=grafana,dc=org"
+org_role = "Admin"
+# To make user an instance admin  (Grafana Admin) uncomment line below
+# grafana_admin = true
+# The Grafana organization database id, optional, if left out the default org (id 1) will be used
+# org_id = 1
+
+[[servers.group_mappings]]
+group_dn = "cn=editors,ou=groups,dc=grafana,dc=org"
+org_role = "Editor"
+
+[[servers.group_mappings]]
+# If you want to match all (or no ldap groups) then you can use wildcard
+group_dn = "*"
+org_role = "Viewer"
--- a/monitoring/loki/config/config.yaml
+++ b/monitoring/loki/config/config.yaml
@@ -0,0 +1,44 @@
+# This is a complete configuration to deploy Loki backed by the filesystem.
+# The index will be shipped to the storage via tsdb-shipper.
+
+auth_enabled: false
+
+server:
+  http_listen_port: 43622
+
+limits_config:
+  ingestion_rate_mb: 100
+  ingestion_burst_size_mb: 200
+  per_stream_rate_limit: 10MB
+  per_stream_rate_limit_burst: 20MB
+
+common:
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+  replication_factor: 1
+  path_prefix: /data/loki
+  storage:
+    filesystem:
+      chunks_directory: /data/loki/chunks
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: tsdb
+    object_store: filesystem
+    schema: v13
+    index:
+      prefix: index_
+      period: 24h
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /data/loki/rules
+
+storage_config:
+  filesystem:
+    directory: /data/loki/chunks
--- a/monitoring/pinger/.gitignore
+++ b/monitoring/pinger/.gitignore
@@ -0,0 +1,4 @@
+deploy.tar
+vendor
+main
+*.log
--- a/monitoring/pinger/build.sh
+++ b/monitoring/pinger/build.sh
@@ -0,0 +1,2 @@
+docker build -t pinger .
+cd .. && docker compose up -d && cd pinger
--- a/monitoring/pinger/captain-definition
+++ b/monitoring/pinger/captain-definition
@@ -0,0 +1,4 @@
+{
+  "schemaVersion": 2,
+  "dockerfilePath": "./dockerfile"
+}
--- a/monitoring/pinger/deploy.sh
+++ b/monitoring/pinger/deploy.sh
@@ -0,0 +1 @@
+tar -cf deploy.tar *.go go.mod go.sum dockerfile captain-definition vendor
--- a/monitoring/pinger/dockerfile
+++ b/monitoring/pinger/dockerfile
@@ -0,0 +1,43 @@
+# Build stage
+FROM golang:bullseye as base
+
+# Install iperf3
+RUN apt-get update && apt-get install -y iperf3 && apt-get clean
+
+RUN adduser \
+  --disabled-password \
+  --gecos "" \
+  --home "/nonexistent" \
+  --shell "/sbin/nologin" \
+  --no-create-home \
+  --uid 65532 \
+  small-user
+
+WORKDIR /app
+
+COPY . .
+
+# RUN go mod download
+# RUN go mod verify
+
+RUN GOFLAGS=-mod=vendor CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o main .
+
+# Final stage
+FROM alpine:latest
+
+# Install iperf3 in Alpine
+RUN apk add --no-cache iperf3
+
+# Copy the binary from the build stage
+COPY --from=base /app/main /main
+
+# Copy necessary files from the build stage
+COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
+COPY --from=base /etc/passwd /etc/passwd
+COPY --from=base /etc/group /etc/group
+
+# Use the non-root user
+USER small-user
+EXPOSE 8090
+
+CMD ["/main"]
--- a/monitoring/pinger/go.mod
+++ b/monitoring/pinger/go.mod
@@ -0,0 +1,31 @@
+module pinger
+
+go 1.22
+
+toolchain go1.22.11
+
+require (
+	github.com/prometheus-community/pro-bing v0.6.1
+	github.com/prometheus/client_golang v1.20.5
+)
+
+require (
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/nxadm/tail v1.4.11
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	golang.org/x/net v0.34.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+)
--- a/monitoring/pinger/go.sum
+++ b/monitoring/pinger/go.sum
@@ -0,0 +1,40 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
+github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
+github.com/prometheus-community/pro-bing v0.6.1 h1:EQukUOma9YFZRPe4DGSscxUf9LH07rpqwisNWjSZrgU=
+github.com/prometheus-community/pro-bing v0.6.1/go.mod h1:jNCOI3D7pmTCeaoF41cNS6uaxeFY/Gmc3ffwbuJVzAQ=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
+golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
--- a/monitoring/pinger/main.go
+++ b/monitoring/pinger/main.go
@@ -0,0 +1,357 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/nxadm/tail"
+	ping "github.com/prometheus-community/pro-bing"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+type IperfResult struct {
+	Start struct {
+		Version     string  `json:"version"`
+		SystemInfo  string  `json:"system_info"`
+		ConnectTime float64 `json:"connect_time"`
+	} `json:"start"`
+	End struct {
+		SumSent struct {
+			Seconds     float64 `json:"seconds"`
+			Bytes       int64   `json:"bytes"`
+			BitsPerSec  float64 `json:"bits_per_second"`
+			Retransmits int     `json:"retransmits"`
+		} `json:"sum_sent"`
+		SumReceived struct {
+			Seconds    float64 `json:"seconds"`
+			Bytes      int64   `json:"bytes"`
+			BitsPerSec float64 `json:"bits_per_second"`
+		} `json:"sum_received"`
+		CPUUtilization struct {
+			HostTotal   float64 `json:"host_total"`
+			RemoteTotal float64 `json:"remote_total"`
+		} `json:"cpu_utilization_percent"`
+	} `json:"end"`
+	Error string `json:"error"`
+}
+
+var Error *log.Logger
+var Warning *log.Logger
+
+var (
+	pingRttHistogram = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "ping_rtt_seconds",
+			Help:    "Round-trip time histogram in seconds",
+			Buckets: []float64{.001, .002, .005, .01, .02, .05, .1, .2, .5, 1},
+		},
+		[]string{"target", "name"},
+	)
+
+	iperfBandwidthGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "iperf_bandwidth_bits_per_second",
+			Help: "Bandwidth measured by iperf3 in bits per second",
+		},
+		[]string{"direction", "server"},
+	)
+
+	httpRttHistogram = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "http_rtt_seconds",
+			Help:    "HTTP request round-trip time histogram in seconds",
+			Buckets: []float64{.001, .002, .005, .01, .02, .05, .1, .2, .5, 1},
+		},
+		[]string{"target", "name"},
+	)
+)
+
+var iperfServer = "138.199.199.16"
+var iperfUploadPort = 3332
+var iperfDownloadPort = 3333
+
+func init() {
+	log.Printf("Initializing application...")
+	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
+	Error = log.New(io.MultiWriter(os.Stderr, os.Stdout),
+		fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
+	Warning = log.New(io.MultiWriter(os.Stdout),
+		fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
+
+	// Register only the essential metrics
+	prometheus.MustRegister(pingRttHistogram)
+
+	// Register new iperf3 metrics
+	prometheus.MustRegister(iperfBandwidthGauge)
+
+	// Register HTTP metrics
+	prometheus.MustRegister(httpRttHistogram)
+
+	log.Printf("Registered all Prometheus metrics")
+}
+
+type PingTarget struct {
+	IP   string
+	Name string
+}
+
+func updateMetrics(stats *ping.Statistics, name string) {
+	target := stats.Addr
+
+	if stats.PacketsRecv > 0 {
+		histogram := pingRttHistogram.WithLabelValues(target, name)
+		histogram.Observe(stats.AvgRtt.Seconds())
+		//log.Printf("Ping metrics for %s (%s): RTT=%v, PacketsRecv=%d/%d",
+		//name, target, stats.AvgRtt, stats.PacketsRecv, stats.PacketsSent)
+	} else {
+		log.Printf("No packets received for %s (%s)", name, target)
+	}
+}
+
+// IperfTest represents a runnable iperf3 test configuration
+type IperfTest struct {
+	ServerHost   string
+	UploadPort   int
+	DownloadPort int
+}
+
+// NewIperfTest creates a new IperfTest with default values
+func NewIperfTest(serverHost string, uploadPort, downloadPort int) *IperfTest {
+	return &IperfTest{
+		ServerHost:   serverHost,
+		UploadPort:   uploadPort,
+		DownloadPort: downloadPort,
+	}
+}
+
+var bitrateRegex = regexp.MustCompile(`(\d+) Mbits/sec`)
+
+// runTest executes an iperf3 test with the given direction
+func (t *IperfTest) runTest(reverse bool) error {
+	direction := "upload"
+	port := t.UploadPort
+	if reverse {
+		direction = "download"
+		port = t.DownloadPort
+	}
+
+	log.Printf("Starting iperf3 %s test to %s:%d", direction, t.ServerHost, port)
+
+	// Check if iperf3 is installed
+	if _, err := exec.LookPath("iperf3"); err != nil {
+		return fmt.Errorf("iperf3 not found: %v", err)
+	}
+
+	title := "upload"
+	if reverse {
+		title = "download"
+	}
+
+	logFile := fmt.Sprintf("iperf3-%s-%s.log", direction, t.ServerHost)
+	_, err := os.Stat(logFile)
+	if err == nil {
+		log.Printf("Removing existing log file %s", logFile)
+		os.Remove(logFile)
+	}
+	log.Printf("Logging iperf3 %s test to %s", direction, logFile)
+
+	args := []string{
+		"-c", t.ServerHost,
+		"-p", fmt.Sprintf("%d", port),
+		"-Z",
+		"-T", title,
+		"-t", "0",
+		"-b", "600M", // Limit bandwidth to 200 Mbits
+		"-f", "m",
+		"-i", "1",
+		"--logfile", logFile,
+	}
+
+	if reverse {
+		args = append(args, "-R")
+	}
+
+	log.Printf("Running iperf3 %s test with args: %v", direction, args)
+
+	cmd := exec.Command("iperf3", args...)
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("failed to start iperf3: %v", err)
+	}
+
+	itail, err := tail.TailFile(
+		logFile, tail.Config{Follow: true, ReOpen: true})
+	if err != nil {
+		return fmt.Errorf("failed to tail iperf3 log file: %v", err)
+	}
+
+	for line := range itail.Lines {
+		bitrate := bitrateRegex.FindStringSubmatch(line.Text)
+		if len(bitrate) > 1 {
+			mbps, err := strconv.ParseFloat(bitrate[1], 64)
+			if err != nil {
+				log.Printf("failed to parse bitrate: %v", err)
+			}
+
+			log.Printf("iperf3 %s bandwidth: %.2f Mbps", direction, mbps)
+			if reverse {
+				iperfBandwidthGauge.WithLabelValues(direction, t.ServerHost).Set(mbps)
+			} else {
+				iperfBandwidthGauge.WithLabelValues(direction, t.ServerHost).Set(mbps)
+			}
+		}
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return fmt.Errorf("iperf3 %s test failed: %v",
+			direction, err)
+	}
+	log.Printf("iperf3 %s test completed successfully", direction)
+
+	return nil
+}
+
+// Start begins continuous iperf3 testing in both directions
+func (t *IperfTest) Start() {
+	log.Printf("Starting continuous iperf3 tests to %s", t.ServerHost)
+
+	// Start upload test goroutine
+	go func() {
+		for {
+			if err := t.runTest(false); err != nil {
+				Error.Printf("Upload test failed: %v", err)
+			}
+			time.Sleep(1 * time.Second)
+			log.Printf("Restarting upload test immediately")
+		}
+	}()
+
+	// Start download test goroutine
+	go func() {
+		for {
+			if err := t.runTest(true); err != nil {
+				Error.Printf("Download test failed: %v", err)
+			}
+			time.Sleep(1 * time.Second)
+			log.Printf("Restarting download test immediately")
+		}
+	}()
+}
+
+type HttpTarget struct {
+	URL  string
+	Name string
+}
+
+func probeHTTP(target HttpTarget) {
+	log.Printf("Starting HTTP probe for %s (%s)", target.Name, target.URL)
+	parsedURL, err := url.Parse(target.URL)
+	if err != nil {
+		Error.Printf("Invalid URL %s: %v", target.URL, err)
+		return
+	}
+
+	client := &http.Client{
+		Timeout: 20 * time.Second,
+	}
+
+	for {
+		start := time.Now()
+		log.Printf("Sending HTTP request to %s", target.URL)
+		resp, err := client.Get(target.URL)
+		duration := time.Since(start)
+
+		if err != nil {
+			Error.Printf("HTTP probe to %s failed: %v (took %v)", target.URL, err, duration)
+		} else {
+			httpRttHistogram.WithLabelValues(parsedURL.Host, target.Name).Observe(duration.Seconds())
+			log.Printf("HTTP probe to %s succeeded in %v", target.URL, duration)
+			resp.Body.Close()
+		}
+
+		log.Printf("Sleeping for 10 seconds before next HTTP probe to %s", target.URL)
+		time.Sleep(10 * time.Second)
+	}
+}
+
+func main() {
+	log.Printf("Starting network monitoring application")
+
+	targets := []PingTarget{
+		{IP: "8.8.8.8", Name: "google-dns-1"},
+		{IP: "1.1.1.1", Name: "cloudflare-dns-1"},
+		{IP: "8.8.4.4", Name: "google-dns-2"},
+		{IP: "1.0.0.1", Name: "cloudflare-dns-2"},
+		{IP: "208.67.222.222", Name: "opendns-1"},
+		{IP: "208.67.220.220", Name: "opendns-2"},
+		{IP: "192.168.1.254", Name: "router"},
+	}
+	log.Printf("Configured ping targets: %v", targets)
+
+	// Add HTTP targets
+	httpTargets := []HttpTarget{
+		{URL: "http://192.168.1.254", Name: "router-http"},
+	}
+	log.Printf("Configured HTTP targets: %v", httpTargets)
+
+	// Start HTTP probes
+	for _, target := range httpTargets {
+		go probeHTTP(target)
+	}
+
+	//Start Prometheus HTTP server
+	go func() {
+		http.Handle("/metrics", promhttp.Handler())
+		if err := http.ListenAndServe(":2112", nil); err != nil {
+			Error.Printf("starting prometheus listener failed: %v", err)
+			os.Exit(1)
+		}
+	}()
+
+	pingers := make([]*ping.Pinger, len(targets))
+	for i, target := range targets {
+		pinger, err := ping.NewPinger(target.IP)
+		if err != nil {
+			Error.Printf("new pinger for %s failed: %v", target.IP, err)
+			return
+		}
+		//pinger.SetPrivileged(true)
+		pinger.RecordRtts = false
+		pinger.RecordTTLs = false
+		pingers[i] = pinger
+		go func(t PingTarget) {
+			log.Printf("Starting pinger for %s (%s)", t.Name, t.IP)
+			err = pinger.Run()
+			if err != nil {
+				Error.Printf("pinger for %s (%s) failed: %v", t.Name, t.IP, err)
+				return
+			}
+		}(target)
+	}
+
+	// Initialize and start iperf3 testing with separate ports
+	iperfTest := NewIperfTest(iperfServer, iperfUploadPort, iperfDownloadPort)
+	iperfTest.Start()
+
+	log.Printf("All monitoring services started, entering main loop")
+	for {
+		time.Sleep(1 * time.Second)
+		for i, pinger := range pingers {
+			if pinger != nil {
+				stats := pinger.Statistics()
+				updateMetrics(stats, targets[i].Name)
+			}
+		}
+	}
+}
--- a/monitoring/prometheus/config/console_libraries
+++ b/monitoring/prometheus/config/console_libraries
@@ -0,0 +1 @@
+/usr/share/prometheus/console_libraries
--- a/monitoring/prometheus/config/consoles
+++ b/monitoring/prometheus/config/consoles
@@ -0,0 +1 @@
+/usr/share/prometheus/consoles/
--- a/monitoring/prometheus/config/prometheus.yml
+++ b/monitoring/prometheus/config/prometheus.yml
@@ -0,0 +1,61 @@
+# 793fc68064b5a5917eccf004ee6f30395a0464e7# my global config
+global:
+  scrape_interval: 5s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+          # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["127.0.0.1:43261"]
+
+  - job_name: 'node_exporter'
+    static_configs:
+      # - targets: ['localhost:56546']
+      - targets: ['nodeexporter:56546']
+    
+  - job_name: 'node_exporter_lilbox'
+    static_configs:
+      - targets: ['192.168.1.78:56546']
+
+  - job_name: 'power-meter-reader'
+    scrape_interval: 30s
+    static_configs:
+      - targets: ['localhost:20132']
+
+#   - job_name: 'windows-exporter'
+#     scrape_interval: 10s
+#     static_configs:
+#       - targets: ['192.168.1.64:9182']
+#
+  - job_name: 'libre-metrics-exporter-dave'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['192.168.1.64:9646']
+  - job_name: 'libre-metrics-exporter-jana'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['192.168.1.68:9646']
+
+#   - job_name: 'libre-metrics-exporter-jana'
+#     scrape_interval: 10s
+#     static_configs:
+#       - targets: ['192.168.1.64:9646']
+
+remote_write:
+  - url: "http://127.0.0.1:8428/api/v1/write"
--- a/monitoring/promtail/config/config.yml
+++ b/monitoring/promtail/config/config.yml
@@ -0,0 +1,35 @@
+server:
+  http_listen_port: 43623
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://localhost:43622/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: dockerlogs
+          __path__: /var/lib/docker/containers/*/*-json.log
+    pipeline_stages:
+      - json:
+          expressions:
+            stream: stream
+            attrs: attrs
+            tag: attrs.tag
+      - labels:
+          stream:
+          tag:
+
+  - job_name: system
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: varlogs
+          __path__: /var/log/*log
				`@@ -0,0 +1 @@`
				`tar -cf deploy.tar *.go go.mod go.sum dockerfile captain-definition vendor`