GETVPN Key Server Monitor

#!/usr/bin/env python3
"""
GETVPN Key Server Monitor

READ THE DOCUMENTATION BEFORE YOU RUN THIS:
    https://permitany.com/scripts/getvpn-ks-monitor

It covers what the script does, whether it is safe to run, the
inventory file and credentials it needs, how to read the output,
and what to do when something fails.

    pip install "netmiko>=4.3" "PyYAML>=6.0" "requests>=2.28"
"""

from __future__ import annotations

import argparse
import json
import os
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

try:
    import yaml
except ImportError:
    yaml = None

from netmiko import ConnectHandler
from netmiko.exceptions import NetmikoAuthenticationException, NetmikoTimeoutException
import re
import time
from dataclasses import dataclass, asdict

try:
    import requests
except ImportError:
    requests = None

# ---------------------------------------------------------------------------
# Inventory loading and connection
# ---------------------------------------------------------------------------

NETMIKO_DRIVER = {
    "ios": "cisco_ios",
    "iosxe": "cisco_ios",
    "iosxr": "cisco_xr",
    "nxos": "cisco_nxos",
    "junos": "juniper_junos",
    "eos": "arista_eos",
    "panos": "paloalto_panos",
    "fortios": "fortinet",
}


def load_inventory(path: str) -> list[dict]:
    """
    Read devices.yaml (or .json) and return a list of device dicts.

    Applies the `defaults:` block to every device, validates the `os` field,
    and fails with a readable message rather than a stack trace.
    """
    p = Path(path)
    if not p.exists():
        raise SystemExit(
            f"Inventory file not found: {path}\n"
            f"Create it with:\n\n"
            f"  devices:\n"
            f"    - host: 192.168.1.1\n"
            f"      os: ios\n"
        )

    raw = p.read_text(encoding="utf-8")
    try:
        if p.suffix.lower() in (".yaml", ".yml"):
            if yaml is None:
                raise SystemExit("PyYAML is not installed. Run:  pip install PyYAML")
            data = yaml.safe_load(raw)
        else:
            data = json.loads(raw)
    except SystemExit:
        raise
    except Exception as exc:
        raise SystemExit(f"Could not parse {path}: {exc}\n"
                         f"Check for tabs (use spaces) and missing spaces after colons.")

    if isinstance(data, dict):
        defaults = data.get("defaults", {}) or {}
        devices = data.get("devices", []) or []
    else:
        defaults, devices = {}, data

    if not devices:
        raise SystemExit(f"No devices found in {path}. The file needs a `devices:` list.")

    out = []
    for i, d in enumerate(devices, 1):
        if not isinstance(d, dict):
            raise SystemExit(f"Device #{i} in {path} is not a mapping. "
                             f"Each entry needs at least `host:` and `os:`.")
        merged = {**defaults, **d}
        if "host" not in merged:
            raise SystemExit(f"Device #{i} in {path} has no `host:` field.")
        os_key = merged.get("os")
        if not os_key:
            raise SystemExit(f"Device '{merged['host']}' has no `os:` field. "
                             f"Valid values: {', '.join(sorted(NETMIKO_DRIVER))}")
        if os_key not in NETMIKO_DRIVER:
            raise SystemExit(
                f"Device '{merged['host']}' has os '{os_key}', which is not supported.\n"
                f"Valid values: {', '.join(sorted(NETMIKO_DRIVER))}\n"
                f"Note: 'cisco' is a vendor, not an OS - use ios, iosxe, iosxr, or nxos."
            )
        merged.setdefault("vrfs", [])
        out.append(merged)
    return out


def credentials() -> tuple[str, str, str]:
    """Read credentials from the environment and fail clearly if missing."""
    user = os.environ.get("NET_USER", "")
    pwd = os.environ.get("NET_PASS", "")
    enable = os.environ.get("NET_ENABLE", "")
    if not user or not pwd:
        raise SystemExit(
            "Set NET_USER and NET_PASS in your environment before running.\n\n"
            "  Linux/macOS:   export NET_USER='admin'\n"
            "                 export NET_PASS='your-password'\n\n"
            "  PowerShell:    $env:NET_USER = 'admin'\n"
            "                 $env:NET_PASS = 'your-password'\n"
        )
    return user, pwd, enable


def connect(dev: dict):
    """Open an SSH session to one device using netmiko."""
    user, pwd, enable = credentials()
    params = {
        "device_type": NETMIKO_DRIVER[dev["os"]],
        "host": dev["host"],
        "username": dev.get("username") or user,
        "password": dev.get("password") or pwd,
        "port": dev.get("port", 22),
        "conn_timeout": 20,
        "fast_cli": False,
    }
    secret = dev.get("secret") or enable
    if secret:
        params["secret"] = secret

    conn = ConnectHandler(**params)
    if secret and dev["os"] in ("ios", "iosxe", "nxos"):
        conn.enable()
    return conn


# ---------------------------------------------------------------------------
# Commands
# ---------------------------------------------------------------------------
# GETVPN (GDOI, RFC 6407) runs on Cisco IOS / IOS-XE only.

KS_COMMANDS = {
    "status": "show crypto gdoi ks",
    "members": "show crypto gdoi ks members",
    "coop": "show crypto gdoi ks coop",
    "rekey": "show crypto gdoi ks rekey",
    "policy": "show crypto gdoi ks policy",
}

GM_COMMANDS = {
    "status": "show crypto gdoi",
    "gm": "show crypto gdoi gm",
    "rekey": "show crypto gdoi gm rekey",
    "acl": "show crypto gdoi gm acl",
}

SUPPORTED = {"ios", "iosxe"}

HEALTHY_GM_STATE = {"REGISTERED", "READY", "ACTIVE"}
HEALTHY_COOP = {"SYNCED", "READY", "UP", "ACTIVE", "ALIVE"}


@dataclass
class Member:
    ip: str
    group: str = ""
    status: str = ""
    active: bool = False


@dataclass
class Finding:
    severity: str          # CRITICAL | HIGH | MEDIUM | LOW
    message: str
    why: str = ""
    fix: str = ""


RANK = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3}


# ---------------------------------------------------------------------------
# Parsers
# ---------------------------------------------------------------------------

def parse_ks_status(output: str) -> dict:
    """
    Parse `show crypto gdoi ks`.

    A KS can serve several groups and hold a different role in each, so role is
    tracked per group rather than collapsed into one value for the device.
    """
    info = {"role": "UNKNOWN", "groups": [], "total_gms": 0}
    cur: dict | None = None

    for line in output.splitlines():
        m = re.search(r"Group Name\s*:\s*(\S+)", line, re.IGNORECASE)
        if m:
            cur = {"name": m.group(1), "role": "UNKNOWN", "registered_gms": 0,
                   "rekey_acks": None}
            info["groups"].append(cur)
            continue
        if cur is None:
            continue

        m = re.search(r"Re-auth|Group Member\s*:\s*(\d+)", line, re.IGNORECASE)
        m2 = re.search(r"(?:Number of Registered GMs|Registered GMs?)\s*:\s*(\d+)",
                       line, re.IGNORECASE)
        if m2:
            cur["registered_gms"] = int(m2.group(1))
            info["total_gms"] += int(m2.group(1))
            continue

        m = re.search(r"(?:Server role|Role)\s*:\s*(.+)", line, re.IGNORECASE)
        if m:
            role = m.group(1).strip()
            cur["role"] = role
            low = role.lower()
            if "primary" in low:
                info["role"] = "PRIMARY"
            elif "secondary" in low:
                info["role"] = "SECONDARY"
            continue

        m = re.search(r"Rekey Acks? Rcvd\s*:\s*(\d+)", line, re.IGNORECASE)
        if m:
            cur["rekey_acks"] = int(m.group(1))

    return info


def parse_ks_coop(output: str) -> dict:
    """`show crypto gdoi ks coop` - cooperative key server state."""
    coop = {"my_role": "UNKNOWN", "peers": [], "failovers": 0}
    peer: dict | None = None

    for line in output.splitlines():
        m = re.search(r"My Role\s*:\s*(.+)", line, re.IGNORECASE)
        if m:
            coop["my_role"] = m.group(1).strip().upper()
            continue
        m = re.search(r"Peer Address\s*:\s*(\d{1,3}(?:\.\d{1,3}){3})", line, re.IGNORECASE)
        if m:
            peer = {"ip": m.group(1), "role": "UNKNOWN", "status": "UNKNOWN"}
            coop["peers"].append(peer)
            continue
        if peer is not None:
            m = re.search(r"Peer Role\s*:\s*(.+)", line, re.IGNORECASE)
            if m:
                peer["role"] = m.group(1).strip().upper()
            m = re.search(r"Peer Status\s*:\s*(.+)", line, re.IGNORECASE)
            if m:
                peer["status"] = m.group(1).strip().upper()
        m = re.search(r"(?:Number of (?:COOP )?failovers?|Failovers?)\s*:\s*(\d+)",
                      line, re.IGNORECASE)
        if m:
            coop["failovers"] = int(m.group(1))
    return coop


def parse_ks_members(output: str) -> list[Member]:
    """`show crypto gdoi ks members` - the GMs this KS believes are registered."""
    out: list[Member] = []
    group = ""
    for line in output.splitlines():
        m = re.search(r"Group Name\s*:\s*(\S+)", line, re.IGNORECASE)
        if m:
            group = m.group(1)
            continue
        m = re.search(r"Group Member ID\s*:\s*(\d{1,3}(?:\.\d{1,3}){3})",
                      line, re.IGNORECASE)
        if m:
            out.append(Member(ip=m.group(1), group=group, status="REGISTERED",
                              active=True))
            continue
        m = re.match(r"\s*(\d{1,3}(?:\.\d{1,3}){3})\s+(\S+)\s+\S+\s+(\S+)", line)
        if m:
            status = m.group(3).upper()
            out.append(Member(ip=m.group(1), group=m.group(2), status=status,
                              active=status in ("ACTIVE", "REGISTERED")))
    return out


def parse_gm_status(output: str) -> dict:
    """`show crypto gdoi` / `show crypto gdoi gm` on a group member."""
    gm = {"group": None, "ks_ip": None, "sa_state": "UNKNOWN", "rekeys_received": 0,
          "tek_lifetime": None, "kek_lifetime": None, "acl_entries": 0}

    for line in output.splitlines():
        m = re.search(r"Group Name\s*:\s*(\S+)", line, re.IGNORECASE)
        if m:
            gm["group"] = m.group(1)
        m = re.search(r"(?:Key Server ID|KS Address|Key Server IP)\s*:\s*"
                      r"(\d{1,3}(?:\.\d{1,3}){3})", line, re.IGNORECASE)
        if m:
            gm["ks_ip"] = m.group(1)
        m = re.search(r"(?:GM State|Group Member State|State)\s*:\s*(\S+)",
                      line, re.IGNORECASE)
        if m:
            gm["sa_state"] = m.group(1).strip().upper()
        m = re.search(r"Rekeys? received\s*:\s*(\d+)", line, re.IGNORECASE)
        if m:
            gm["rekeys_received"] = int(m.group(1))
        m = re.search(r"Remaining (?:lifetime|Key Lifetime)\s*\(sec\)\s*:\s*(\d+)",
                      line, re.IGNORECASE)
        if m and gm["tek_lifetime"] is None:
            gm["tek_lifetime"] = int(m.group(1))
        m = re.search(r"TEK.*?(?:lifetime|remaining).*?(\d+)\s*sec", line, re.IGNORECASE)
        if m:
            gm["tek_lifetime"] = int(m.group(1))
        m = re.search(r"KEK.*?(?:lifetime|remaining).*?(\d+)\s*sec", line, re.IGNORECASE)
        if m:
            gm["kek_lifetime"] = int(m.group(1))
    return gm


# ---------------------------------------------------------------------------
# Per-device check
# ---------------------------------------------------------------------------

def check_device(dev: dict, args) -> dict:
    host = dev["host"]
    os_key = dev["os"]
    role = str(dev.get("role", "gm")).lower()
    r = {"host": host, "os": os_key, "role": role, "findings": [], "ks": {},
         "gm": {}, "members": [], "error": None, "skipped": False}

    if os_key not in SUPPORTED:
        r["skipped"] = True
        r["error"] = f"GETVPN (GDOI) is a Cisco IOS/IOS-XE technology; nothing to check on {os_key}"
        return r
    if role not in ("ks", "gm"):
        r["error"] = f"role must be 'ks' or 'gm', got '{role}'"
        return r

    cmds = KS_COMMANDS if role == "ks" else GM_COMMANDS
    conn = None
    f: list[Finding] = []

    try:
        conn = connect(dev)
        out: dict[str, str] = {}
        for key, cmd in cmds.items():
            try:
                out[key] = conn.send_command(cmd, read_timeout=args.read_timeout)
            except Exception:  # noqa: BLE001 - a missing command must not lose the rest
                out[key] = ""

        if role == "ks":
            ks = parse_ks_status(out.get("status", ""))
            coop = parse_ks_coop(out.get("coop", ""))
            members = parse_ks_members(out.get("members", ""))
            ks["coop"] = coop
            r["ks"] = ks
            r["members"] = [asdict(m) for m in members]

            if not ks["groups"]:
                f.append(Finding("CRITICAL",
                                 "no GDOI group is configured or active on this KS",
                                 "the key server is not serving any group",
                                 "check 'crypto gdoi group' and that the server is "
                                 "running 'server local'"))

            for m in members:
                if not m.active:
                    f.append(Finding("HIGH",
                                     f"GM {m.ip} is in state {m.status}",
                                     "the member de-registered or its rekey failed",
                                     "check reachability from the GM to this KS and "
                                     "that the group config still matches"))

            expected = dev.get("expected_gm_count")
            active = sum(1 for m in members if m.active)
            if expected is not None and active < int(expected):
                f.append(Finding("HIGH",
                                 f"{active}/{expected} group member(s) registered",
                                 f"{int(expected) - active} GM(s) are missing entirely - "
                                 f"a GM that never registered leaves no entry at all",
                                 "check each missing GM's 'crypto gdoi group' config "
                                 "and its path to this KS"))

            state = coop.get("my_role", "UNKNOWN")
            for p in coop.get("peers", []):
                if p["status"] not in HEALTHY_COOP and p["status"] != "UNKNOWN":
                    f.append(Finding("HIGH",
                                     f"cooperative peer {p['ip']} is {p['status']}",
                                     "the secondary key server is not in sync, so it "
                                     "cannot take over cleanly if this one fails",
                                     "check IP reachability between the key servers "
                                     "and that both list each other as COOP peers"))
            if coop.get("failovers", 0) >= args.failover_threshold:
                f.append(Finding("MEDIUM",
                                 f"{coop['failovers']} cooperative failover(s) in history",
                                 "repeated failover points at an unstable key server "
                                 "or a flapping path between the KS pair",
                                 "check the KS logs and the path between key servers"))

        else:
            gm = parse_gm_status(out.get("status", "") + "\n" + out.get("gm", ""))
            r["gm"] = gm

            if gm["sa_state"] not in HEALTHY_GM_STATE:
                f.append(Finding("CRITICAL",
                                 f"GM state is {gm['sa_state']} - not registered to a "
                                 f"key server",
                                 "without a group SA this member cannot decrypt group "
                                 "traffic at all",
                                 "check reachability to the KS, the GDOI group id, and "
                                 "the registration ACL on the KS"))

            tek = gm["tek_lifetime"]
            if tek is not None and tek < args.warn_tek:
                f.append(Finding("HIGH",
                                 f"TEK expires in {tek}s",
                                 "if the TEK expires without a rekey arriving, this GM "
                                 "stops passing encrypted traffic - and so does every "
                                 "other GM on the same TEK, simultaneously",
                                 "confirm the KS is sending rekeys and that multicast "
                                 "or unicast rekey reaches this GM"))

            kek = gm["kek_lifetime"]
            if kek is not None and kek < args.warn_tek:
                f.append(Finding("HIGH",
                                 f"KEK expires in {kek}s",
                                 "the KEK protects the rekey messages themselves. Once "
                                 "it expires this GM can no longer receive rekeys and "
                                 "must re-register from scratch",
                                 "confirm rekey delivery before the KEK lapses"))

            expected_ks = dev.get("expected_ks")
            if expected_ks and gm["ks_ip"] and gm["ks_ip"] != expected_ks:
                f.append(Finding("MEDIUM",
                                 f"registered to KS {gm['ks_ip']}, expected {expected_ks}",
                                 "either a failover happened, or this GM is pointed at "
                                 "the wrong key server",
                                 "confirm whether the primary KS failed over"))

            if (gm["rekeys_received"] == 0
                    and gm["sa_state"] in HEALTHY_GM_STATE):
                f.append(Finding("MEDIUM",
                                 "0 rekeys received since registration",
                                 "the GM registered successfully but has never received "
                                 "a rekey. It is running on its initial SA and will drop "
                                 "traffic when that SA expires",
                                 "check the KS rekey schedule, and that multicast rekey "
                                 "is reaching this GM - unicast rekey is more reliable "
                                 "where multicast is not end-to-end"))

        f.sort(key=lambda x: RANK[x.severity])
        r["findings"] = [asdict(x) for x in f]
        return r

    except NetmikoAuthenticationException as exc:
        r["error"] = f"authentication failed - check NET_USER/NET_PASS ({exc})"
    except NetmikoTimeoutException as exc:
        r["error"] = f"unreachable or SSH timed out ({exc})"
    except Exception as exc:  # noqa: BLE001 - one device must not abort the run
        r["error"] = f"{type(exc).__name__}: {exc}"
    finally:
        if conn:
            try:
                conn.disconnect()
            except Exception:  # noqa: BLE001
                pass
    return r


# ---------------------------------------------------------------------------
# Group-wide analysis
# ---------------------------------------------------------------------------

def analyse_group(results: list[dict], args) -> list[str]:
    """
    Compare the key servers against the group members.

    GETVPN's dangerous failure is silent. If every KS stops serving, the GMs
    keep encrypting happily on the TEK they already hold. Nothing alerts, every
    tunnel looks healthy, and then the TEK expires and the ENTIRE group drops
    at the same instant.

    The precursor is detectable: GMs with a TEK counting down while no KS is
    primary and reachable. That is what this function looks for.
    """
    issues: list[str] = []
    live = [r for r in results if not r["error"] and not r["skipped"]]
    kss = [r for r in live if r["role"] == "ks"]
    gms = [r for r in live if r["role"] == "gm"]

    reachable_ks = {r["host"] for r in kss}
    primary = [r for r in kss
               if "PRIMARY" in str(r["ks"].get("coop", {}).get("my_role", "")).upper()
               or r["ks"].get("role") == "PRIMARY"]

    # The silent-failure precursor
    if kss and not primary:
        soonest = [g for g in gms if g["gm"].get("tek_lifetime") is not None]
        if soonest:
            t = min(g["gm"]["tek_lifetime"] for g in soonest)
            issues.append(
                f"NO key server is PRIMARY, and {len(soonest)} group member(s) are "
                f"running on a TEK with as little as {t}s left. When that TEK expires "
                f"with no KS to rekey, EVERY member stops passing traffic at the same "
                f"moment. This is the GETVPN failure that gives no warning")
        else:
            issues.append("no key server reports itself as PRIMARY - the group has no "
                          "server able to issue rekeys")

    # GMs pointing at a KS that is not in the inventory or not answering
    for g in gms:
        ks_ip = g["gm"].get("ks_ip")
        if ks_ip and reachable_ks and ks_ip not in reachable_ks:
            issues.append(f"{g['host']} is registered to KS {ks_ip}, which is not among "
                          f"the reachable key servers in this inventory - verify that "
                          f"server is alive")

    # Does the KS agree the GMs are registered?
    if kss and gms:
        known = {m["ip"] for r in kss for m in r["members"]}
        for g in gms:
            if g["gm"].get("sa_state") in HEALTHY_GM_STATE:
                own = {g["host"]}
                if known and not (own & known):
                    issues.append(
                        f"{g['host']} believes it is registered, but no key server "
                        f"lists it as a member - the GM is running on a stale SA and "
                        f"will not receive the next rekey")

    # TEK spread: members on wildly different lifetimes means rekey is not
    # reaching all of them.
    teks = [(g["host"], g["gm"]["tek_lifetime"]) for g in gms
            if g["gm"].get("tek_lifetime") is not None]
    if len(teks) >= 2:
        lo = min(t for _, t in teks)
        hi = max(t for _, t in teks)
        if hi - lo > args.tek_spread:
            behind = [h for h, t in teks if t < lo + args.tek_spread / 2]
            issues.append(
                f"TEK lifetimes differ by {hi - lo}s across the group - rekey is not "
                f"reaching everyone. Furthest behind: {', '.join(sorted(behind)[:4])}")
    return issues


def summarise(r: dict) -> str:
    host, role = r["host"], r["role"].upper()
    if r["skipped"]:
        return f"[SKIP]  {host:<22} {r['error']}"
    if r["error"]:
        return f"[ERROR] {host:<22} ({role}) {r['error']}"

    if not r["findings"]:
        if role == "KS":
            ks = r["ks"]
            active = sum(1 for m in r["members"] if m["active"])
            coop_role = ks.get("coop", {}).get("my_role", ks.get("role", "?"))
            return (f"[OK]    {host:<22} (KS/{coop_role}) {active} member(s) registered")
        gm = r["gm"]
        tek = f", TEK {gm['tek_lifetime']}s" if gm.get("tek_lifetime") else ""
        return (f"[OK]    {host:<22} (GM) {gm.get('sa_state', '?')}, "
                f"KS {gm.get('ks_ip', '?')}, {gm.get('rekeys_received', 0)} rekey(s){tek}")

    sev = sorted({x["severity"] for x in r["findings"]}, key=lambda s: RANK[s])[0]
    return f"[{sev:<8}] {host:<22} ({role}) {len(r['findings'])} finding(s)"


def main() -> int:
    ap = argparse.ArgumentParser(
        description="Monitor GETVPN key servers and group members, and detect the "
                    "silent key-server failure before it drops the whole group.",
        epilog="Credentials come from NET_USER and NET_PASS. See the docstring at "
               "the top of this file for the full inventory format.",
    )
    ap.add_argument("--inventory", required=True,
                    help="Path to devices.yaml listing the key servers and group members")
    ap.add_argument("--daemon", action="store_true",
                    help="Run continuously instead of checking once")
    ap.add_argument("--interval", type=int, default=300,
                    help="Seconds between checks in daemon mode. Default: 300")
    ap.add_argument("--warn-tek", type=int, default=3600,
                    help="Warn when a TEK or KEK expires in fewer seconds than this. "
                         "Default: 3600")
    ap.add_argument("--tek-spread", type=int, default=1800,
                    help="Flag the group when member TEK lifetimes differ by more than "
                         "this many seconds. Default: 1800")
    ap.add_argument("--failover-threshold", type=int, default=1,
                    help="Cooperative failover count that triggers a finding. Default: 1")
    ap.add_argument("--min-severity", choices=list(RANK), default="LOW",
                    help="Only report findings at or above this severity. Default: LOW")
    ap.add_argument("--show-members", action="store_true",
                    help="List every group member the key server knows about")
    ap.add_argument("--limit", help="Only check hosts whose name contains this text")
    ap.add_argument("--webhook", default="", help="Slack or Teams webhook URL")
    ap.add_argument("--read-timeout", type=int, default=60,
                    help="Seconds to wait per command. Default: 60")
    ap.add_argument("--workers", type=int, default=10,
                    help="How many devices to contact at once. Default: 10")
    ap.add_argument("--json-out", help="Write machine-readable results here")
    args = ap.parse_args()

    devices = load_inventory(args.inventory)
    if args.limit:
        devices = [d for d in devices if args.limit in d["host"]]
        if not devices:
            raise SystemExit(f"No device name contains '{args.limit}'.")

    credentials()  # fail fast before opening any connection

    def run_once() -> int:
        print(f"\nChecking GETVPN on {len(devices)} device(s)...")
        print("=" * 84)

        results = []
        with ThreadPoolExecutor(max_workers=args.workers) as pool:
            futures = {pool.submit(check_device, d, args): d["host"] for d in devices}
            for fut in as_completed(futures):
                host = futures[fut]
                try:
                    results.append(fut.result())
                except Exception as exc:  # noqa: BLE001
                    results.append({"host": host, "os": "?", "role": "?",
                                    "findings": [], "ks": {}, "gm": {}, "members": [],
                                    "error": f"worker crashed: {exc}", "skipped": False})

        floor = RANK[args.min_severity]
        kss = [r for r in results if r["role"] == "ks"]
        gms = [r for r in results if r["role"] != "ks"]

        for r in kss + gms:
            print(summarise(r))
            if args.show_members and r["members"]:
                for m in sorted(r["members"], key=lambda x: x["ip"]):
                    mark = "ok  " if m["active"] else "DOWN"
                    print(f"           {mark} {m['ip']:<16} {m['group']:<16} {m['status']}")
            for f in r["findings"]:
                if RANK[f["severity"]] > floor:
                    continue
                print(f"           [{f['severity']:<8}] {f['message']}")
                if f["why"]:
                    print(f"                      why: {f['why']}")
                if f["fix"]:
                    print(f"                      fix: {f['fix']}")

        group = analyse_group(results, args)
        print("=" * 84)
        if group:
            print(f"GROUP-WIDE: {len(group)} issue(s)")
            for g in group:
                print(f"  ! {g}")

        issues = sum(len(r["findings"]) for r in results) + len(group)
        errs = sum(1 for r in results if r["error"] and not r["skipped"])
        crit = sum(1 for r in results for f in r["findings"]
                   if f["severity"] == "CRITICAL")

        if issues or errs:
            print(f"\n{issues} issue(s) across {len(results)} device(s)"
                  + (f", {errs} unreachable" if errs else ""))
        else:
            print(f"GETVPN healthy across all {len(results)} device(s).")

        if crit:
            print("\nCRITICAL findings present. In GETVPN a key server problem does not "
                  "break\ntraffic immediately - it breaks all of it at once when the "
                  "current TEK expires.")

        if (issues or errs) and args.webhook and requests is not None:
            lines = [summarise(r) for r in results if r["findings"] or r["error"]]
            lines += [f"! {g}" for g in group]
            try:
                requests.post(args.webhook,
                              json={"text": "*GETVPN alert*\n" + "\n".join(lines)},
                              timeout=10)
            except Exception as exc:  # noqa: BLE001
                print(f"[WARN] webhook failed: {exc}", file=sys.stderr)

        if args.json_out:
            Path(args.json_out).write_text(
                json.dumps({"devices": results, "group_issues": group}, indent=2),
                encoding="utf-8")

        return 1 if (issues or errs) else 0

    if not args.daemon:
        return run_once()

    while True:
        run_once()
        print(f"\nNext check in {args.interval}s. Ctrl-C to stop.")
        time.sleep(args.interval)


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        print("\nStopped.", file=sys.stderr)
        sys.exit(130)