Network Config Redactor

#!/usr/bin/env python3
"""
Network Config Redactor

READ THE DOCUMENTATION BEFORE YOU RUN THIS:
    https://permitany.com/scripts/config-redactor

It covers what the script does, whether it is safe to run, the
inventory file and credentials it needs, how to read the output,
and what to do when something fails.

    # No extra packages — uses Python stdlib only
"""

# SAFE: offline. It opens no socket. Reads files, writes files, nothing else.
#
# EXIT CODES  0 nothing left to worry about   1 the verify pass still sees
#             something   2 the run failed
#
# It never overwrites the input unless you pass --in-place.

import argparse
import base64
import binascii
import hashlib
import ipaddress
import json
import math
import os
import re
import sys

try:
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
except Exception:
    pass


# ----------------------------------------------------------------- mapping
class Pseudonymiser(object):
    """Stable fake values: the same input always becomes the same output.

    This is the whole reason the tool is usable. A redactor that replaces
    every IP with X.X.X.X destroys the only thing that made the config worth
    reading -- you can no longer tell whether two lines talk about the same
    host. Here 10.1.1.1 becomes 10.251.0.7 everywhere in every file of the
    same run, so the topology survives and the addresses do not.

    The salt is what stops the mapping being a lookup table. Without one,
    anybody could hash the whole RFC1918 space and undo it in seconds. With a
    random salt the mapping is meaningless outside the run; with --salt you
    choose a shared secret so two people can redact separately and still get
    comparable output.
    """

    def __init__(self, salt):
        self.salt = salt
        self.maps = {}
        self.counters = {}

    def _n(self, kind, value, limit):
        key = (kind, value)
        if key in self.maps:
            return self.maps[key]
        h = hashlib.sha256((self.salt + "\x00" + kind + "\x00" + value).encode("utf-8")).digest()
        n = int.from_bytes(h[:8], "big") % limit
        # Collisions would merge two distinct hosts into one, which is a
        # worse lie than redacting. Walk forward until the slot is free.
        taken = self.counters.setdefault(kind, set())
        while n in taken:
            n = (n + 1) % limit
        taken.add(n)
        self.maps[key] = n
        return n

    def ipv4(self, value):
        """Public addresses only; see keep_private in Redactor."""
        n = self._n("ipv4", value, 65534) + 1
        return "198.51.100.%d" % (n % 254 + 1) if n % 255 == 0 else \
               "203.0.113.%d" % (n % 254 + 1)

    def ipv4_private(self, value):
        n = self._n("ipv4p", value, 65024)
        return "10.%d.%d.%d" % (250 + n // 65024, (n // 254) % 256, n % 254 + 1)

    def ipv6(self, value):
        n = self._n("ipv6", value, 65535)
        return "2001:db8:%x::%x" % (n >> 8, n & 0xFF)

    def mac(self, value):
        n = self._n("mac", value, 16777215)
        return "0200.%04x.%04x" % ((n >> 8) & 0xFFFF, n & 0xFFFF)

    def hostname(self, value):
        return "host-%03d" % self._n("host", value.lower(), 999)

    def user(self, value):
        return "user%03d" % self._n("user", value.lower(), 999)

    def domain(self, value):
        return "example%d.invalid" % self._n("domain", value.lower(), 99)


# ----------------------------------------------------------------- rules
# (name, compiled regex, which group holds the secret, note)
#
# Every rule redacts a capture GROUP, never the whole line, so the command
# that gave the line meaning stays readable. "snmp-server community XXXX RO"
# still tells you it was read-only.
SECRET_RULES = [
    ("cisco-type7", re.compile(r"(?i)\b(password|secret)\s+7\s+(?P<s>[0-9A-Fa-f]{4,})"), "s",
     "Cisco type 7 is obfuscation, not encryption -- it reverses in one line of Python"),
    ("cisco-hash", re.compile(r"(?i)\b(?:secret|password)\s+(?:[589])\s+(?P<s>\$\S+)"), "s",
     "type 5/8/9 are real hashes but crack offline; a hash in a ticket is still a leak"),
    ("cisco-plain", re.compile(r"(?i)\b(?:password|secret)\s+(?:0\s+)?(?P<s>\S+)\s*$"), "s", ""),
    ("enable", re.compile(r"(?i)^\s*enable\s+(?:secret|password)\s+(?:\d\s+)?(?P<s>\S+)"), "s", ""),
    ("snmp-community", re.compile(r"(?i)\bcommunity(?:-string)?\s+(?P<s>\S+)"), "s",
     "an SNMP community is a password that everybody treats as a label"),
    ("snmp-host", re.compile(r"(?i)^\s*snmp-server\s+host\s+\S+(?:\s+\S+)*?\s+(?P<s>\S+)\s*$"), "s", ""),
    ("snmpv3-auth", re.compile(r"(?i)\b(?:auth|priv)\s+(?:md5|sha|sha256|des|aes|3des)\s+(?P<s>\S+)"), "s", ""),
    ("psk", re.compile(r"(?i)\b(?:pre-shared-key|preshared-key|pre-shared-secret)\s+(?:address\s+\S+\s+)?(?P<s>\S+)"), "s", ""),
    ("isakmp-key", re.compile(r"(?i)^\s*crypto\s+isakmp\s+key\s+(?P<s>\S+)"), "s", ""),
    ("key-string", re.compile(r"(?i)\bkey-string\s+(?:\d\s+)?(?P<s>\S+)"), "s", ""),
    ("bgp-password", re.compile(r"(?i)\bneighbor\s+\S+\s+password\s+(?:\d\s+)?(?P<s>\S+)"), "s", ""),
    # "key" is not adjacent to the keyword: tacacs-server host 10.0.0.1 key 7 X
    ("tacacs-radius", re.compile(r"(?i)^\s*(?:tacacs|radius)[\w-]*\s+.*?\bkey\s+(?:\d\s+)?(?P<s>\S+)"), "s", ""),
    ("key-anywhere", re.compile(r"(?i)\bkey\s+(?:\d\s+)?(?P<s>[0-9A-Fa-f]{8,}|\$\S+)\s*$"), "s",
     "any 'key <hex>' at the end of a line, whatever the command is"),
    ("junos-secret", re.compile(r"(?i)\b(?:encrypted-password|authentication-key|secret)\s+\"?(?P<s>\$\d\$[^\"\s]+)\"?"), "s", ""),
    ("junos-plain", re.compile(r"(?i)\bplain-text-password[\w-]*\s+(?P<s>\S+)"), "s", ""),
    # The (?:ENC\s+)? is not cosmetic. Without it the capture lands on the
    # ENC marker and the blob behind it survives, so the line reads
    # "set password XXXXREDACTEDXXXX <the actual secret>" -- redacted to the
    # eye and fully intact. That is the worst failure this tool can have.
    ("fortios-enc", re.compile(r"(?i)^\s*set\s+(?:password|passwd|psksecret|secondary-secret|key|private-key|auth-password-\w+|priv-password-\w+)\s+(?:ENC\s+)?(?P<s>\S+)"), "s", ""),
    ("panos-phash", re.compile(r"(?i)<(?:phash|passwd|secret|master-key)>(?P<s>[^<]+)</"), "s", ""),
    ("generic-kv", re.compile(r"(?i)(?:^|[^\w])[\w.]*?(?:passwd|password|pwd|secret|token|api[_-]?key|apikey|auth[_-]?token|bearer|client[_-]?secret|private[_-]?key)\s*[:=]\s*[\"']?(?P<s>[^\"'\s,;]{4,})"), "s", ""),
    ("aws-akid", re.compile(r"\b(?P<s>(?:AKIA|ASIA)[0-9A-Z]{16})\b"), "s", ""),
    ("jwt", re.compile(r"\b(?P<s>eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})\b"), "s", ""),
    ("priv-key-line", re.compile(r"^(?P<s>(?!-----)[A-Za-z0-9+/=]{40,})$"), "s",
     "only inside a PEM block; see the block handler"),
]

# FortiOS and Junos put a secret in a generic key inside a named block, so the
# line alone is not enough: "set name" means nothing until you know you are
# inside "config system snmp community".
BLOCK_OPEN = re.compile(r"(?i)^\s*config\s+.*\bsnmp\s+community\b")
BLOCK_SECRET_IN_CTX = re.compile(r"""(?i)^\s*set\s+(?:name|password|secret)\s+"?(?P<s>[^"\s]+)""")
BLOCK_END = re.compile(r"(?i)^\s*end\s*$")

PEM_OPEN = re.compile(r"-----BEGIN ([A-Z0-9 ]+)-----")
PEM_CLOSE = re.compile(r"-----END ([A-Z0-9 ]+)-----")

IPV4 = re.compile(r"\b(?P<s>(?:\d{1,3}\.){3}\d{1,3})\b")
IPV6 = re.compile(r"(?<![:\w])(?P<s>(?:[0-9A-Fa-f]{1,4}:){2,7}[0-9A-Fa-f]{1,4})(?![:\w])")
MAC = re.compile(r"\b(?P<s>(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}|(?:[0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4})\b")
EMAIL = re.compile(r"\b(?P<s>[\w.+-]+)@(?P<d>[\w.-]+\.\w{2,})\b")
HOSTNAME_CMD = re.compile(r"(?i)^\s*(?:set\s+)?(?:system\s+)?host-?name\s+\"?(?P<s>[\w.-]+)\"?")

# Addresses that carry no information about anyone and must survive, or the
# config stops making sense.
KEEP_IPS = {"0.0.0.0", "255.255.255.255", "127.0.0.1", "255.255.255.0",
            "255.255.255.252", "255.255.255.248", "255.255.255.240",
            "255.255.255.224", "255.255.255.192", "255.255.255.128",
            "255.255.0.0", "255.0.0.0", "8.8.8.8", "8.8.4.4", "1.1.1.1"}


def looks_like_mask(text):
    """A netmask or wildcard is structure, not an address."""
    try:
        o = [int(x) for x in text.split(".")]
    except ValueError:
        return False
    if len(o) != 4 or any(x > 255 for x in o):
        return False
    bits = "".join(bin(x)[2:].zfill(8) for x in o)
    return "01" not in bits or "10" not in bits.replace("0", "", 0) and bits.count("01") <= 1


def entropy(text):
    if not text:
        return 0.0
    freq = {}
    for c in text:
        freq[c] = freq.get(c, 0) + 1
    n = float(len(text))
    return -sum((c / n) * math.log(c / n, 2) for c in freq.values())


# ----------------------------------------------------------------- engine
class Finding(object):
    def __init__(self, line_no, rule, sample, action):
        self.line_no = line_no
        self.rule = rule
        self.sample = sample
        self.action = action        # REDACTED / PSEUDONYMISED / FLAGGED


class Redactor(object):
    def __init__(self, args, pseudo):
        self.a = args
        self.p = pseudo
        self.findings = []
        self.in_pem = False
        self.in_community = False

    def note(self, n, rule, sample, action):
        self.findings.append(Finding(n, rule, sample, action))

    # -- secrets ------------------------------------------------------
    def secrets(self, line, n):
        for name, rx, grp, _ in SECRET_RULES:
            if name == "priv-key-line":
                continue
            for m in list(rx.finditer(line)):
                val = m.group(grp)
                if not val or val.upper().startswith("XXXX"):
                    continue
                self.note(n, name, val, "REDACTED")
                line = line[:m.start(grp)] + "XXXXREDACTEDXXXX" + line[m.end(grp):]
        return line

    def pem(self, line, n):
        """A key spans many lines, so it needs state, not a regex."""
        if PEM_OPEN.search(line):
            self.in_pem = True
            return line
        if PEM_CLOSE.search(line):
            self.in_pem = False
            return line
        if self.in_pem and line.strip():
            self.note(n, "pem-body", line.strip()[:16] + "...", "REDACTED")
            return "XXXXREDACTEDXXXX"
        return line

    # -- identifiers --------------------------------------------------
    def addresses(self, line, n):
        def sub_v4(m):
            raw = m.group("s")
            try:
                ip = ipaddress.ip_address(raw)
            except ValueError:
                return raw
            if raw in KEEP_IPS or looks_like_mask(raw):
                return raw
            if ip.is_private:
                if self.a.keep_private:
                    return raw
                self.note(n, "ipv4-private", raw, "PSEUDONYMISED")
                return self.p.ipv4_private(raw)
            self.note(n, "ipv4-public", raw, "PSEUDONYMISED")
            return self.p.ipv4(raw)

        line = IPV4.sub(sub_v4, line)

        if not self.a.keep_private:
            def sub_v6(m):
                raw = m.group("s")
                self.note(n, "ipv6", raw, "PSEUDONYMISED")
                return self.p.ipv6(raw)
            line = IPV6.sub(sub_v6, line)

        if self.a.redact_mac:
            def sub_mac(m):
                raw = m.group("s")
                self.note(n, "mac", raw, "PSEUDONYMISED")
                return self.p.mac(raw)
            line = MAC.sub(sub_mac, line)
        return line

    def names(self, line, n):
        m = HOSTNAME_CMD.search(line)
        if m and self.a.redact_names:
            raw = m.group("s")
            self.note(n, "hostname", raw, "PSEUDONYMISED")
            line = line[:m.start("s")] + self.p.hostname(raw) + line[m.end("s"):]

        def sub_mail(m2):
            self.note(n, "email", m2.group(0), "PSEUDONYMISED")
            return "%s@%s" % (self.p.user(m2.group("s")), self.p.domain(m2.group("d")))
        line = EMAIL.sub(sub_mail, line)
        return line

    # -- the honest part ----------------------------------------------
    def suspects(self, line, n):
        """High-entropy tokens this tool does NOT understand.

        They are reported and left alone. Redacting an unrecognised string
        would corrupt configs -- a base64 certificate fingerprint and an API
        key look identical -- and silently redacting the wrong thing is how
        a tool like this loses trust. "Could not classify" is a result.
        """
        if "XXXXREDACTED" in line:
            return
        for tok in re.findall(r"[A-Za-z0-9+/=_\-]{20,}", line):
            if tok.count(".") > 2 or tok.startswith("XXXX"):
                continue
            if len(tok) >= 32 and re.fullmatch(r"[0-9A-Fa-f]+", tok):
                self.note(n, "long-hex", tok[:24] + "...", "FLAGGED")
                continue
            if entropy(tok) >= self.a.entropy:
                self.note(n, "high-entropy", tok[:24] + ("..." if len(tok) > 24 else ""), "FLAGGED")

    def block(self, line, n):
        if BLOCK_OPEN.search(line):
            self.in_community = True
            return line
        if BLOCK_END.search(line):
            self.in_community = False
            return line
        if self.in_community:
            m = BLOCK_SECRET_IN_CTX.search(line)
            if m and not m.group("s").startswith("XXXX"):
                self.note(n, "snmp-community-block", m.group("s"), "REDACTED")
                return line[:m.start("s")] + "XXXXREDACTEDXXXX" + line[m.end("s"):]
        return line

    def line(self, text, n):
        out = self.pem(text, n)
        if out != text and out == "XXXXREDACTEDXXXX":
            return out
        out = self.block(out, n)
        out = self.secrets(out, n)
        out = self.addresses(out, n)
        out = self.names(out, n)
        self.suspects(out, n)
        return out

    def run(self, text):
        return "\n".join(self.line(l, i + 1) for i, l in enumerate(text.split("\n")))


# ----------------------------------------------------------------- verify
# Run against the OUTPUT. A redactor nobody checks is a redactor nobody
# should trust, and the failure mode is silent by construction.
# Written deliberately from the OTHER direction: these look for the SHAPE of a
# surviving secret, not for the keywords the rules above already know about. A
# verifier that reuses the redactor's assumptions inherits its blind spots and
# rubber-stamps the file. That is not hypothetical -- on the first run here
# "tacacs-server host 10.10.10.9 key 7 070C285F4D06" passed as clean.
VERIFY = [
    ("type 7 survived", re.compile(r"(?i)\b(?:password|secret|key)\s+7\s+[0-9A-Fa-f]{4,}")),
    ("credential keyword still has a value", re.compile(
        r"""(?i)\b(?:key|secret|password|passwd|community|psksecret|token)\b\s+(?:\d\s+)?"""
        r"""(?!XXXX)(?!chain\b)(?!exchange\b)(?!id\b)(?:ENC\s+)?(?!XXXX)[^\s"']{6,}""")),
    # FortiOS marks an encrypted value with ENC and then the blob. Excluding
    # the word ENC outright made the verifier blind to the entire vendor: on
    # the raw FortiOS fixture it reported zero findings while two secrets sat
    # in plain sight.
    # A value left sitting NEXT to the marker. This shape is what a
    # mis-anchored capture group produces, and it defeats every other rule
    # here because the keyword now has "XXXXREDACTEDXXXX" after it and looks
    # handled. It is the rule that caught the FortiOS ENC bug.
    ("value left beside the marker",
     # [ \t] and not \s: \s crosses the newline and matches the first token
     # of the next line, which made every redacted line a false positive.
     re.compile(r"XXXXREDACTEDXXXX[ \t]+(?!XXXX)[A-Za-z0-9+/=._-]{10,}")),
    ("FortiOS ENC blob survived",
     re.compile(r"(?im)^\s*set\s+\S+\s+ENC\s+(?!XXXX)\S{10,}")),
    ("hash survived", re.compile(r"\$[0-9a-z]{1,2}\$[./A-Za-z0-9]{8,}")),
    ("AWS key survived", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")),
    ("JWT survived", re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.")),
    ("PEM body survived", re.compile(r"^(?!-----)[A-Za-z0-9+/=]{60,}$", re.M)),
    ("key=value secret survived",
     re.compile(r"(?i)\b(?:password|secret|api[_-]?key|token)\b\s*[:=]\s*[\"']?(?!XXXX)[^\"'\s,;]{6,}")),
]


def verify(text):
    out = []
    for name, rx in VERIFY:
        for m in rx.finditer(text):
            out.append((name, m.group(0)[:50]))
    return out


# ----------------------------------------------------------------- report
def report(path, red, left, args):
    bucket = {}
    for f in red.findings:
        bucket.setdefault(f.action, []).append(f)

    print("=" * 74)
    print("config-redactor  --  %s" % path)
    print("=" * 74)
    for action in ("REDACTED", "PSEUDONYMISED", "FLAGGED"):
        rows = bucket.get(action, [])
        if not rows:
            continue
        by_rule = {}
        for f in rows:
            by_rule.setdefault(f.rule, []).append(f)
        title = {
            "REDACTED": "SECRETS REMOVED -- replaced with XXXXREDACTEDXXXX",
            "PSEUDONYMISED": "IDENTIFIERS REMAPPED -- consistently, so the topology survives",
            "FLAGGED": "NOT TOUCHED, LOOK AT THESE -- high entropy, could not classify",
        }[action]
        print("\n%s (%d)" % (title, len(rows)))
        print("-" * 74)
        for rule, items in sorted(by_rule.items()):
            lines = ", ".join(str(i.line_no) for i in items[:8])
            more = "" if len(items) <= 8 else " +%d" % (len(items) - 8)
            print("  %-18s %3d  lines %s%s" % (rule, len(items), lines, more))
            if action == "FLAGGED":
                for i in items[:3]:
                    print("       %s" % i.sample)

    print("\nVERIFY PASS -- re-reading the output as if someone else wrote it")
    print("-" * 74)
    if not left:
        print("  clean: no known secret pattern survived")
    else:
        for name, sample in left[:20]:
            print("  STILL THERE  %-26s %s" % (name, sample))

    print("\nWHAT THIS TOOL DOES NOT KNOW")
    print("-" * 74)
    for t in [
        "A secret in a format it has no rule for. The FLAGGED list is where",
        "  those surface -- read it before sharing the file.",
        "Free text: a password written inside a description or a banner.",
        "Private addressing is KEPT by default, because it is the topology and",
        "  it is not a secret. --redact-private changes that.",
        "Nothing here proves a file is safe to publish. It makes it far safer",
        "  to paste into a chat, which is a different claim.",
    ]:
        print("  - %s" % t if not t.startswith("  ") else "   %s" % t)


# ----------------------------------------------------------------- entry
def parse_args(argv):
    p = argparse.ArgumentParser(
        description="Mask secrets and identifiers in network configs and logs, "
                    "consistently, so the file stays readable. Offline.")
    p.add_argument("files", nargs="*", help="Files to redact. Omit to read stdin")
    p.add_argument("-o", "--out-dir", help="Write redacted copies here. Default: alongside, with .redacted")
    p.add_argument("--in-place", action="store_true", help="Overwrite the input. Think twice")
    p.add_argument("--stdout", action="store_true", help="Write the result to stdout, report to stderr")
    p.add_argument("--dry-run", action="store_true", help="Report what WOULD change and write nothing")
    p.add_argument("--salt", help="Shared salt, so two people get the same mapping. Default: random per run")
    p.add_argument("--mapping-out", help="Write the original-to-fake mapping here. TREAT IT AS A SECRET: it undoes the redaction")
    p.add_argument("--keep-private", action="store_true", default=True,
                   help="Keep RFC1918 addresses (default): they are the topology, not a secret")
    p.add_argument("--redact-private", dest="keep_private", action="store_false",
                   help="Also remap private addresses")
    p.add_argument("--redact-mac", action="store_true", help="Also remap MAC addresses")
    p.add_argument("--redact-names", action="store_true", help="Also remap the hostname command")
    p.add_argument("--entropy", type=float, default=4.0,
                   help="Bits per character above which an unknown token is FLAGGED. Default 4.0")
    return p.parse_args(argv)


def main(argv=None):
    args = parse_args(argv)
    salt = args.salt or binascii.hexlify(os.urandom(16)).decode()
    pseudo = Pseudonymiser(salt)
    worst = 0

    sources = args.files or ["-"]
    for path in sources:
        text = sys.stdin.read() if path == "-" else open(path, encoding="utf-8", errors="replace").read()
        red = Redactor(args, pseudo)
        out = red.run(text)
        left = verify(out)

        if args.stdout or path == "-":
            sys.stdout.write(out)
            sys.stderr.write("\n")
        elif not args.dry_run:
            dest = path if args.in_place else (
                os.path.join(args.out_dir, os.path.basename(path) + ".redacted")
                if args.out_dir else path + ".redacted")
            if args.out_dir and not os.path.isdir(args.out_dir):
                os.makedirs(args.out_dir)
            with open(dest, "w", encoding="utf-8") as fh:
                fh.write(out)

        stream = sys.stderr if (args.stdout or path == "-") else sys.stdout
        old = sys.stdout
        sys.stdout = stream
        try:
            report(path, red, left, args)
            if not args.dry_run and not args.stdout and path != "-":
                print("\nWritten: %s" % dest)
        finally:
            sys.stdout = old
        worst = max(worst, 1 if left else 0)

    if args.mapping_out:
        with open(args.mapping_out, "w", encoding="utf-8") as fh:
            json.dump({"salt": salt,
                       "mapping": {"%s:%s" % k: v for k, v in pseudo.maps.items()}}, fh, indent=1)
        sys.stderr.write("\nMapping written to %s -- it REVERSES the redaction. "
                         "Treat it like the config itself.\n" % args.mapping_out)
    return worst


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        sys.stderr.write("\ninterrupted\n")
        sys.exit(2)
    except SystemExit:
        raise
    except (OSError, UnicodeError) as exc:
        sys.stderr.write("failed: %s\n" % exc)
        sys.exit(2)