Infoblox VLAN / Network Provisioner

#!/usr/bin/env python3
"""
Infoblox VLAN / Network Provisioner

READ THE DOCUMENTATION BEFORE YOU RUN THIS:
    https://permitany.com/scripts/infoblox-network-provision

It covers what the script does, whether it is safe to run, the
inventory file and credentials it needs, how to read the output,
and what to do when something fails.

    pip install "requests>=2.28" "PyYAML>=6.0"
"""

# NOT read-only. With --apply this script CREATES networks, DHCP ranges,
# reverse zones and gateway records on the grid. Without --apply it writes
# nothing: it reads, plans and prints. It never sends DELETE.
#
# EXIT CODES  0 plan clean / every write succeeded
#             1 something needs a human: blocked steps, conflicts, failures
#             2 the run could not proceed at all
#
# CREDENTIALS come from the environment, never from an argument:
#   IB_USER / IB_PASS      (not needed at all with --offline-dir)
#
# NIOS / WAPI v2.x. Not Universal DDI.

import argparse
import csv
import ipaddress
import json
import os
import re
import sys
from datetime import datetime, timezone

try:
    import requests
    from requests.auth import HTTPBasicAuth
except ImportError:
    sys.stderr.write("requests is not installed. Run: pip install 'requests>=2.28'\n")
    raise SystemExit(2)

try:
    import yaml
except ImportError:
    sys.stderr.write("PyYAML is not installed. Run: pip install 'PyYAML>=6.0'\n")
    raise SystemExit(2)

try:
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
except Exception:
    pass

DEFAULT_WAPI = "v2.12"

# ---------------------------------------------------------------- outcomes
CREATE = "CREATE"
UPDATE = "UPDATE"
SKIP = "SKIP"
BLOCKED = "BLOCKED"

LABEL_RE = re.compile(r"^(?!-)[A-Za-z0-9_-]{1,63}(?<!-)$")


# ---------------------------------------------------------------- transport
class Wapi(object):
    """Live WAPI, or a replay of saved responses. Same interface either way.

    Offline replay is not a convenience. It is how this script was tested
    without a grid, and it is how you can hand a colleague the exact data a
    finding came from.
    """

    def __init__(self, host=None, version=DEFAULT_WAPI, insecure=False,
                 offline_dir=None, save_dir=None):
        self.offline_dir = offline_dir
        self.save_dir = save_dir
        self.base = None
        self.session = None
        if offline_dir:
            if not os.path.isdir(offline_dir):
                raise SystemExit("--offline-dir does not exist: %s" % offline_dir)
            return
        user = os.environ.get("IB_USER")
        password = os.environ.get("IB_PASS")
        if not user or not password:
            raise SystemExit(
                "IB_USER and IB_PASS must be set in the environment.\n"
                "  Linux/macOS : export IB_USER=admin; read -s IB_PASS; export IB_PASS\n"
                "  PowerShell  : $env:IB_USER='admin'; $env:IB_PASS='...'\n"
                "They are read from the environment on purpose: an argument is "
                "visible in 'ps auxww' and lands in shell history."
            )
        self.base = "https://%s/wapi/%s/" % (host, version)
        self.session = requests.Session()
        self.session.auth = HTTPBasicAuth(user, password)
        self.session.verify = not insecure
        if insecure:
            try:
                import urllib3
                urllib3.disable_warnings()
            except Exception:
                pass

    @staticmethod
    def _slug(obj, params):
        """Filename for one query. Must survive Windows, so everything that is
        not alphanumeric becomes a dash -- '>' and ':' are legal in a WAPI
        search key and illegal in an NTFS filename."""
        def clean(text):
            return "".join(c if c.isalnum() else "-" for c in str(text))

        bits = [clean(obj)]
        for k in sorted(params):
            if k.startswith("_"):
                continue
            bits.append("%s=%s" % (clean(k), clean(params[k])))
        return ("__".join(bits) or "root")[:180] + ".json"

    def get(self, obj, params=None):
        """Return a list of objects, following WAPI paging to the end.

        Paging in WAPI only works with _return_as_object=1 -- without it the
        next_page_id never comes back and you silently get the first page and
        nothing else. _max_results is NEGATIVE on purpose: a positive value
        makes WAPI raise 'Result set too large' instead of truncating, which
        turns a big zone into a failed run rather than a partial one.
        """
        params = dict(params or {})
        slug = self._slug(obj, params)

        if self.offline_dir:
            path = os.path.join(self.offline_dir, slug)
            if not os.path.exists(path):
                return []
            with open(path, encoding="utf-8") as fh:
                return json.load(fh)

        params["_return_as_object"] = 1
        params["_paging"] = 1
        params["_max_results"] = -1000
        out = []
        page = None
        while True:
            call = dict(params)
            if page:
                call = {"_page_id": page, "_return_as_object": 1}
            resp = self.session.get(self.base + obj, params=call, timeout=60)
            if resp.status_code == 401:
                raise SystemExit("WAPI rejected IB_USER/IB_PASS (401).")
            if resp.status_code >= 400:
                raise SystemExit("WAPI %s on %s: %s" % (resp.status_code, obj, resp.text[:300]))
            body = resp.json()
            out.extend(body.get("result", []))
            page = body.get("next_page_id")
            if not page:
                break

        if self.save_dir:
            if not os.path.isdir(self.save_dir):
                os.makedirs(self.save_dir)
            with open(os.path.join(self.save_dir, slug), "w", encoding="utf-8") as fh:
                json.dump(out, fh, indent=1)
        return out

    # -- the write half. Everything above this line is shared verbatim with
    # -- the read-only Infoblox scripts; only this part can change a grid.
    def post(self, obj, payload):
        """Create one object. Returns (ok, ref_or_error).

        One object per call on purpose. WAPI does have a /request endpoint
        that batches, but a batch fails as a unit and leaves you guessing
        which step did it. One call per step costs round trips and buys an
        exact rollback file.
        """
        if self.offline_dir:
            return True, "SIMULATED/%s/%s" % (obj, payload.get("name")
                                              or payload.get("ptrdname")
                                              or payload.get("network")
                                              or payload.get("fqdn")
                                              or payload.get("start_addr"))
        try:
            resp = self.session.post(self.base + obj, json=payload, timeout=60)
        except requests.exceptions.RequestException as exc:
            return False, str(exc)
        if resp.status_code in (200, 201):
            return True, resp.json()     # WAPI returns the new _ref, as a string
        return False, self._error(resp)

    def put(self, ref, payload):
        """Change one existing object in place. Returns (ok, ref_or_error)."""
        if self.offline_dir:
            return True, "SIMULATED/" + ref
        try:
            resp = self.session.put(self.base + ref, json=payload, timeout=60)
        except requests.exceptions.RequestException as exc:
            return False, str(exc)
        if resp.status_code == 200:
            return True, resp.json()
        return False, self._error(resp)

    @staticmethod
    def _error(resp):
        """WAPI puts the useful sentence in a JSON 'text' field and the useless
        one in the HTTP reason. Prefer the former, fall back to the body."""
        try:
            body = resp.json()
            return "%s: %s" % (resp.status_code, body.get("text") or body)
        except ValueError:
            return "%s: %s" % (resp.status_code, resp.text[:200])


# ---------------------------------------------------------------- helpers
def dashed(value):
    return "-" if value in (None, "") else str(value)


def is_suffix_zone(name, zone):
    """True when `name` lives inside `zone`, by label, not by characters.

    Comparing with endswith() alone matches 'evilexample.com' against
    'example.com', which is how a check like this quietly stops working.
    """
    name = (name or "").rstrip(".").lower()
    zone = (zone or "").rstrip(".").lower()
    return bool(zone) and (name == zone or name.endswith("." + zone))


def valid_fqdn(name):
    """Syntax only. A name can be perfectly well formed and still have no zone
    on this grid -- that is a separate check, and it needs the grid."""
    name = (name or "").rstrip(".")
    if not name or len(name) > 253 or "." not in name:
        return False
    return all(LABEL_RE.match(label) for label in name.split("."))


def reverse_zones_for(net):
    """The in-addr.arpa zone names a prefix needs, and whether this script can
    create them at all.

    -> (list_of_zone_fqdns, rfc2317_parent_or_None)

    Three cases, and getting them confused is the whole point of this
    function:
      /24            exactly one zone.
      /23 and shorter  one zone PER /24 inside it. A /22 needs FOUR. This is
                     where people create one zone and wonder why half the
                     addresses have no reverse.
      /25 and longer   RFC 2317 classless. The zone cannot be created
                     unilaterally: the owner of the parent /24 has to
                     delegate it first. The second element of the tuple is
                     that parent, and the caller must say so rather than
                     letting WAPI fail.
    """
    if net.prefixlen > 24:
        parent = ipaddress.ip_network("%s/24" % net.network_address, strict=False)
        o = str(parent.network_address).split(".")
        return [], "%s.%s.%s.in-addr.arpa" % (o[2], o[1], o[0])
    zones = []
    for sub in net.subnets(new_prefix=24) if net.prefixlen < 24 else [net]:
        o = str(sub.network_address).split(".")
        zones.append("%s.%s.%s.in-addr.arpa" % (o[2], o[1], o[0]))
    return zones, None


# ---------------------------------------------------------------- DHCP options
# name -> option code. The names are the ones NIOS itself shows, so a template
# written from the GUI transfers across without a translation table.
DHCP_OPTIONS = {
    "subnet-mask": 1,
    "time-offset": 2,
    "routers": 3,
    "time-servers": 4,
    "domain-name-servers": 6,
    "host-name": 12,
    "domain-name": 15,
    "broadcast-address": 28,
    "ntp-servers": 42,
    "vendor-encapsulated-options": 43,
    "netbios-name-servers": 44,
    "netbios-node-type": 46,
    "dhcp-lease-time": 51,
    "tftp-server-name": 66,
    "bootfile-name": 67,
    "domain-search": 119,
}

# NIOS shows a handful of options separately from the rest and gives them a
# "use" flag. For those, WAPI honours use_option; for every other option the
# field is ignored and sending it is noise. Sending it on the wrong option is
# harmless, omitting it on these is NOT -- the value is stored and not used.
USE_OPTION_FLAG = {
    "routers", "domain-name-servers", "domain-name",
    "broadcast-address", "dhcp-lease-time",
}

IP_LIST_OPTIONS = {"routers", "domain-name-servers", "time-servers",
                   "ntp-servers", "netbios-name-servers", "broadcast-address"}
INT_OPTIONS = {"dhcp-lease-time", "time-offset", "netbios-node-type"}


def option_payload(name, value):
    """One entry of a WAPI `options` list."""
    entry = {"name": name, "num": DHCP_OPTIONS[name],
             "value": value, "vendor_class": "DHCP"}
    if name in USE_OPTION_FLAG:
        entry["use_option"] = True
    return entry


def options_as_map(options):
    """A WAPI options list -> {name: value}, keeping only the DHCP vendor
    class. Comparing the raw lists is useless: NIOS returns the inherited
    options as well as the ones set here, in whatever order it likes."""
    out = {}
    for o in options or []:
        if (o.get("vendor_class") or "DHCP") != "DHCP":
            continue
        name = o.get("name")
        if name:
            out[name] = str(o.get("value"))
    return out


# ---------------------------------------------------------------- the input
TEMPLATE_KEYS = {
    "gateway", "reserve_low", "reserve_high", "dhcp", "dhcp_member",
    "exclusions", "options", "network_comment", "range_comment", "extattrs",
    "gateway_record", "create_reverse_zone", "reverse_zone_ns_group",
    "view", "network_view",
}

TEMPLATE_DEFAULTS = {
    "gateway": "first",
    "reserve_low": 10,
    "reserve_high": 5,
    "dhcp": True,
    "dhcp_member": None,
    "exclusions": [],
    "options": {"network": {}, "range": {}},
    "network_comment": "VLAN {vlan} {name}",
    "range_comment": "VLAN {vlan} {name} DHCP pool",
    "extattrs": {},
    "gateway_record": None,
    "create_reverse_zone": True,
    "reverse_zone_ns_group": None,
    "view": "default",
    "network_view": "default",
}

CSV_OVERRIDES = ("gateway", "reserve_low", "reserve_high", "dhcp",
                 "dhcp_member", "view", "network_view")


def merge_template(base, over):
    """Shallow merge, except `options`, which merges one level deeper so a
    per-network override of options.range does not silently delete
    options.network."""
    out = dict(base)
    for key, val in (over or {}).items():
        if key == "options" and isinstance(val, dict):
            opts = {"network": dict(out["options"].get("network") or {}),
                    "range": dict(out["options"].get("range") or {})}
            for scope in ("network", "range"):
                if isinstance(val.get(scope), dict):
                    opts[scope].update(val[scope])
            out["options"] = opts
        else:
            out[key] = val
    return out


class NetSpec(object):
    """One VLAN to provision, after the template has been applied to it."""

    def __init__(self, where, vlan, name, cidr, extra, tmpl):
        self.where = where          # 'plan.yaml networks[2]' or 'vlans.csv line 4'
        self.vlan = vlan
        self.name = name
        self.cidr = cidr            # ipaddress.IPv4Network
        self.extra = extra          # free-form fields usable in {placeholders}
        self.t = tmpl
        self.view = tmpl["view"]
        self.network_view = tmpl["network_view"]
        self.gateway = None         # filled by geometry()
        self.range_start = None
        self.range_end = None
        self.reverse_zones = []
        self.rfc2317_parent = None
        self.exclusions = []
        self.resolved_options = {"network": {}, "range": {}}
        self.steps = []             # Step objects, in topological order

    # -- placeholders -----------------------------------------------------
    def fields(self):
        f = {
            "vlan": self.vlan,
            "name": self.name,
            "cidr": str(self.cidr),
            "network": str(self.cidr.network_address),
            "prefix": self.cidr.prefixlen,
            "netmask": str(self.cidr.netmask),
            "broadcast": str(self.cidr.broadcast_address),
            "gateway": str(self.gateway) if self.gateway else "",
        }
        f.update(self.extra)
        return f

    def fmt(self, text):
        """Format a template string. An unknown placeholder is an error at
        validation time, not a KeyError in the middle of an apply."""
        if text is None:
            return None
        return str(text).format(**self.fields())

    def label(self):
        return "VLAN %s %s %s" % (dashed(self.vlan), self.name, self.cidr)


def _as_bool(value, default=False):
    if value is None or value == "":
        return default
    if isinstance(value, bool):
        return value
    return str(value).strip().lower() in ("yes", "true", "1", "y", "on")


def load_plan(path, csv_path, errors):
    """Read the template file, and the network list from it or from the CSV.

    Everything in here runs with no grid and no credentials. The whole input
    is checked before the first WAPI call on purpose: failing on the fourth
    VLAN with three already provisioned is the worst outcome this script can
    have, and it is the one the ordering exists to prevent.
    """
    if not os.path.exists(path):
        raise SystemExit("template file not found: %s" % path)
    with open(path, encoding="utf-8-sig") as fh:
        try:
            doc = yaml.safe_load(fh) or {}
        except yaml.YAMLError as exc:
            raise SystemExit("%s is not valid YAML:\n%s" % (path, exc))
    if not isinstance(doc, dict):
        raise SystemExit("%s must be a YAML mapping with a 'template:' key." % path)

    raw_tmpl = doc.get("template") or {}
    if not isinstance(raw_tmpl, dict):
        raise SystemExit("'template:' in %s must be a mapping." % path)
    unknown = sorted(set(raw_tmpl) - TEMPLATE_KEYS)
    if unknown:
        errors.append((path, "unknown template key(s): %s. Valid keys: %s"
                       % (", ".join(unknown), ", ".join(sorted(TEMPLATE_KEYS)))))
    base = merge_template(TEMPLATE_DEFAULTS, raw_tmpl)

    specs = []
    if csv_path:
        specs = _load_csv(csv_path, base, errors)
        if doc.get("networks"):
            errors.append((path, "both --csv and a 'networks:' block are present. "
                                 "Pick one: two lists of VLANs in one run is how "
                                 "the wrong one gets provisioned"))
    else:
        items = doc.get("networks")
        if not isinstance(items, list) or not items:
            raise SystemExit(
                "%s has no 'networks:' list and no --csv was given.\n"
                "Either add a networks: block to the template file, or pass the "
                "VLANs with --csv." % path)
        for n, item in enumerate(items):
            where = "%s networks[%d]" % (os.path.basename(path), n)
            if not isinstance(item, dict):
                errors.append((where, "each entry under networks: must be a mapping"))
                continue
            over = {k: v for k, v in item.items() if k in TEMPLATE_KEYS}
            extra = {k: v for k, v in item.items()
                     if k not in TEMPLATE_KEYS and k not in ("vlan", "name", "cidr")}
            spec = _make_spec(where, item.get("vlan"), item.get("name"),
                              item.get("cidr"), extra, merge_template(base, over),
                              errors)
            if spec:
                specs.append(spec)
    return base, specs


def _load_csv(path, base, errors):
    """The CSV carries the VLANs; the template carries everything else. That
    split is the point of the script: VLAN 30 at the new site comes out
    identical to VLAN 20 at the old one because the same template produced
    both."""
    if not os.path.exists(path):
        raise SystemExit("CSV not found: %s" % path)
    specs = []
    with open(path, encoding="utf-8-sig", newline="") as fh:
        reader = csv.DictReader(fh)
        if not reader.fieldnames:
            raise SystemExit("%s is empty -- it needs a header row." % path)
        headers = [(h or "").strip().lower() for h in reader.fieldnames]
        missing = [h for h in ("vlan", "name", "cidr") if h not in headers]
        if missing:
            raise SystemExit(
                "CSV header is missing: %s\n"
                "Required columns: vlan,name,cidr. Optional: %s, plus any column "
                "you want to use as a {placeholder} in the template."
                % (", ".join(missing), ", ".join(CSV_OVERRIDES)))
        for n, raw in enumerate(reader, start=2):
            clean = {}
            for key, val in raw.items():
                if key is None:
                    continue
                clean[(key or "").strip().lower()] = (val or "").strip()
            if not any(clean.get(k) for k in ("vlan", "name", "cidr")):
                continue        # a blank line, not an error
            over = {}
            for key in CSV_OVERRIDES:
                if clean.get(key):
                    over[key] = _as_bool(clean[key], True) if key == "dhcp" else clean[key]
            for key in ("reserve_low", "reserve_high"):
                if key in over and str(over[key]).isdigit():
                    over[key] = int(over[key])
            extra = {k: v for k, v in clean.items()
                     if k not in CSV_OVERRIDES and k not in ("vlan", "name", "cidr")}
            spec = _make_spec("%s line %d" % (os.path.basename(path), n),
                              clean.get("vlan"), clean.get("name"),
                              clean.get("cidr"), extra, merge_template(base, over),
                              errors)
            if spec:
                specs.append(spec)
    return specs


def _make_spec(where, vlan, name, cidr, extra, tmpl, errors):
    if vlan in (None, ""):
        errors.append((where, "vlan is required"))
        return None
    if not str(vlan).isdigit() or not 1 <= int(vlan) <= 4094:
        errors.append((where, "vlan '%s' is not a VLAN id between 1 and 4094"
                       % dashed(vlan)))
        return None
    if not name:
        errors.append((where, "name is required -- it is what the comments and "
                              "the gateway record are built from"))
        return None
    if not LABEL_RE.match(str(name)):
        errors.append((where, "name '%s' is not usable as a DNS label (letters, "
                              "digits, '-' and '_', 1-63 characters)" % dashed(name)))
        return None
    try:
        net = ipaddress.ip_network(str(cidr), strict=True)
    except ValueError as exc:
        errors.append((where, "cidr '%s' is not a network: %s" % (dashed(cidr), exc)))
        return None
    if net.version != 4:
        errors.append((where, "this script provisions IPv4 only; '%s' is IPv6"
                       % cidr))
        return None
    return NetSpec(where, int(vlan), str(name), net, extra, tmpl)


# ---------------------------------------------------------------- geometry
def geometry(spec, errors):
    """Work out the gateway, the DHCP range and the reverse zones.

    This is where the two mistakes that this script exists for are caught,
    and both of them are arithmetic rather than API trouble:

      a range that starts at .1 eats the gateway, and
      a prefix longer than /24 has no reverse zone you can simply create.
    """
    net = spec.cidr
    t = spec.t
    where = spec.where

    for key in ("reserve_low", "reserve_high"):
        val = t.get(key)
        if isinstance(val, bool) or not isinstance(val, int) or val < 0:
            errors.append((where, "%s must be a whole number of addresses, not '%s'"
                           % (key, dashed(val))))
            return

    # -- the gateway ------------------------------------------------------
    gw = t.get("gateway")
    total = net.num_addresses
    if total < 4:
        errors.append((where, "%s is a /%d -- it has no room for a gateway and a "
                              "DHCP range. Provision it by hand if you really want "
                              "the network object" % (net, net.prefixlen)))
        return
    if gw == "first":
        spec.gateway = net.network_address + 1
    elif gw == "last":
        spec.gateway = net.broadcast_address - 1
    elif isinstance(gw, int) and not isinstance(gw, bool):
        if not 1 <= gw <= total - 2:
            errors.append((where, "gateway offset %d is outside %s" % (gw, net)))
            return
        spec.gateway = net.network_address + gw
    else:
        try:
            addr = ipaddress.ip_address(str(gw))
        except ValueError:
            errors.append((where, "gateway '%s' is not 'first', 'last', an offset "
                                  "or an address" % dashed(gw)))
            return
        if addr not in net:
            errors.append((where, "gateway %s is not inside %s" % (addr, net)))
            return
        if addr in (net.network_address, net.broadcast_address):
            errors.append((where, "gateway %s is the network or broadcast address "
                                  "of %s" % (addr, net)))
            return
        spec.gateway = addr

    # -- the reverse zones -------------------------------------------------
    spec.reverse_zones, spec.rfc2317_parent = reverse_zones_for(net)

    # -- the DHCP range ----------------------------------------------------
    if not _as_bool(t.get("dhcp"), True):
        return
    low, high = t["reserve_low"], t["reserve_high"]
    start = net.network_address + 1 + low
    end = net.broadcast_address - 1 - high
    if int(start) > int(end):
        errors.append((where, "nothing is left for a DHCP range in %s: reserve_low "
                              "%d plus reserve_high %d consumes all %d usable "
                              "addresses. Lower them, or set dhcp: false"
                       % (net, low, high, total - 2)))
        return
    if start <= spec.gateway <= end:
        errors.append((where, "the DHCP range %s-%s would hand out the gateway %s. "
                              "A pool that starts at the first usable address eats "
                              "the gateway, and the symptom is a duplicate-address "
                              "complaint on the router, not a DHCP error. Raise "
                              "reserve_low (or reserve_high for a last-address "
                              "gateway) past it"
                       % (start, end, spec.gateway)))
        return
    spec.range_start, spec.range_end = start, end

    # -- explicit exclusions ----------------------------------------------
    cleaned = []
    for item in t.get("exclusions") or []:
        text = str(item).strip()
        parts = [p.strip() for p in text.split("-")]
        if len(parts) != 2:
            errors.append((where, "exclusion '%s' must be written 'first-last'"
                           % dashed(text)))
            continue
        try:
            a, b = ipaddress.ip_address(parts[0]), ipaddress.ip_address(parts[1])
        except ValueError:
            errors.append((where, "exclusion '%s' is not a pair of addresses" % text))
            continue
        if int(a) > int(b):
            errors.append((where, "exclusion '%s' runs backwards" % text))
            continue
        if not (start <= a <= end and start <= b <= end):
            errors.append((where, "exclusion %s-%s is outside the DHCP range %s-%s. "
                                  "NIOS rejects it, and an exclusion outside the "
                                  "pool excludes nothing anyway"
                           % (a, b, start, end)))
            continue
        cleaned.append({"start_address": str(a), "end_address": str(b),
                        "comment": "reserved by template"})
    spec.exclusions = cleaned


def validate_options(spec, errors):
    """Option names, option values, and the one that costs money later: the
    same option defined on the network AND on the range with the same value."""
    t = spec.t
    scopes = t.get("options") or {}
    if not isinstance(scopes, dict):
        errors.append((spec.where, "options: must be a mapping with 'network' "
                                   "and/or 'range' under it"))
        return
    bad_scope = sorted(set(scopes) - {"network", "range"})
    if bad_scope:
        errors.append((spec.where, "options has unknown scope(s) %s -- only "
                                   "'network' and 'range' exist"
                       % ", ".join(bad_scope)))
    resolved = {}
    for scope in ("network", "range"):
        resolved[scope] = {}
        for name, value in (scopes.get(scope) or {}).items():
            if name not in DHCP_OPTIONS:
                errors.append((spec.where, "DHCP option '%s' is not one this script "
                                           "knows. Known: %s"
                               % (dashed(name), ", ".join(sorted(DHCP_OPTIONS)))))
                continue
            try:
                text = spec.fmt(value)
            except KeyError as exc:
                errors.append((spec.where, "option %s uses the placeholder %s, "
                                           "which nothing provides" % (name, exc)))
                continue
            if name in IP_LIST_OPTIONS:
                for part in str(text).split(","):
                    try:
                        ipaddress.ip_address(part.strip())
                    except ValueError:
                        errors.append((spec.where, "option %s: '%s' is not an IPv4 "
                                                   "address" % (name, part.strip())))
                        break
            elif name in INT_OPTIONS and not str(text).strip().isdigit():
                errors.append((spec.where, "option %s must be a whole number, got "
                                           "'%s'" % (name, text)))
            resolved[scope][name] = str(text)
    spec.resolved_options = resolved


def validate_strings(spec, errors):
    """Every template string, resolved now rather than mid-apply.

    A placeholder nobody provides is a KeyError. Finding it here costs a
    second; finding it after the second VLAN has been created costs a change
    window.
    """
    for key in ("network_comment", "range_comment", "gateway_record"):
        try:
            spec.fmt(spec.t.get(key))
        except (KeyError, IndexError, ValueError) as exc:
            errors.append((spec.where, "%s uses a placeholder nothing provides: "
                                       "%s. Available: %s"
                           % (key, exc, ", ".join(sorted(spec.fields())))))
    for name, value in (spec.t.get("extattrs") or {}).items():
        try:
            spec.fmt(value)
        except (KeyError, IndexError, ValueError) as exc:
            errors.append((spec.where, "extended attribute %s uses a placeholder "
                                       "nothing provides: %s" % (name, exc)))


def cross_spec_errors(specs):
    """Contradictions BETWEEN the networks in one file. The per-network checks
    cannot see these, and they are the ones that half-provision a site."""
    errors = []
    seen_cidr = {}
    seen_vlan = {}
    for spec in specs:
        key = (spec.network_view, spec.cidr)
        if key in seen_cidr:
            errors.append((spec.where, "%s in network view '%s' is already on %s"
                           % (spec.cidr, spec.network_view, seen_cidr[key])))
        else:
            seen_cidr[key] = spec.where
        vkey = (spec.network_view, spec.vlan)
        if vkey in seen_vlan:
            errors.append((spec.where, "VLAN %d appears twice (also on %s). If two "
                                       "subnets really share a VLAN id, provision "
                                       "them in separate runs so the comments do "
                                       "not collide"
                           % (spec.vlan, seen_vlan[vkey])))
        else:
            seen_vlan[vkey] = spec.where
    for i, a in enumerate(specs):
        for b in specs[i + 1:]:
            if a.network_view != b.network_view or a.cidr == b.cidr:
                continue
            if a.cidr.overlaps(b.cidr):
                errors.append((b.where, "%s overlaps %s from %s in the same network "
                                        "view. NIOS will accept the container and "
                                        "the child, which is probably not what this "
                                        "file meant"
                               % (b.cidr, a.cidr, a.where)))
    return errors


# ---------------------------------------------------------------- planning
class Step(object):
    """One WAPI object this run would create or change.

    `needs` is the whole reason this class is not a dict. A DHCP range cannot
    exist before its network, and a PTR cannot exist before its reverse zone.
    The plan is executed in this order and a step whose parent failed is NOT
    attempted -- it is reported as not attempted, which is a different fact
    from failed.
    """

    def __init__(self, spec, key, obj, what, verdict, reason,
                 payload=None, ref=None, previous=None, needs=None):
        self.spec = spec
        self.key = key
        self.obj = obj
        self.what = what
        self.verdict = verdict
        self.reason = reason
        self.payload = payload or {}
        self.ref = ref
        self.previous = previous
        self.needs = needs or []
        self.result = None          # None | 'ok' | 'failed: ...' | 'not attempted: ...'
        self.created_ref = None

    def satisfied(self):
        """True when the object this step is responsible for is on the grid
        afterwards -- whether this run put it there or found it there."""
        if self.verdict == SKIP:
            return True
        return self.result == "ok"


class Planner(object):
    def __init__(self, wapi, args):
        self.w = wapi
        self.args = args
        self.counts = {}
        self.warnings = []

    def fetch(self):
        """Read the current state once and index it. Asking WAPI 'does this
        exist' per object is one round trip per object; reading the lists once
        is the difference between a minute and an afternoon on a big grid."""
        w = self.w
        self.views = [v.get("name") for v in
                      w.get("view", {"_return_fields": "name,is_default"})]
        self.netviews = [v.get("name") for v in
                         w.get("networkview", {"_return_fields": "name,is_default"})]
        self.members = [m.get("host_name") for m in
                        w.get("member", {"_return_fields": "host_name"})]
        self.zones = [z for z in
                      w.get("zone_auth", {"_return_fields":
                                          "fqdn,view,zone_format,disable"})]
        self.networks = w.get("network", {"_return_fields":
                              "network,network_view,comment,options,extattrs"})
        self.ranges = w.get("range", {"_return_fields":
                            "start_addr,end_addr,network,network_view,member,"
                            "server_association_type,options,disable"})
        self.a = w.get("record:a", {"_return_fields": "name,ipv4addr,view"})
        self.ptr = w.get("record:ptr", {"_return_fields":
                         "ptrdname,ipv4addr,name,view"})

        self.net_by_cidr = {}
        for n in self.networks:
            self.net_by_cidr[(n.get("network_view") or "default",
                              n.get("network"))] = n
        self.ranges_by_net = {}
        for r in self.ranges:
            self.ranges_by_net.setdefault(
                (r.get("network_view") or "default", r.get("network")), []).append(r)
        self.zone_by_fqdn = {}
        for z in self.zones:
            self.zone_by_fqdn[((z.get("view") or "default"),
                               (z.get("fqdn") or "").lower())] = z
        self.a_by_name = {}
        for r in self.a:
            self.a_by_name.setdefault(
                ((r.get("view") or "default"),
                 (r.get("name") or "").lower()), []).append(r)
        self.ptr_by_ip = {}
        for p in self.ptr:
            self.ptr_by_ip.setdefault(
                ((p.get("view") or "default"), p.get("ipv4addr")), []).append(p)

        self.counts = {
            "views": len(self.views), "netviews": len(self.netviews),
            "zones": len(self.zones), "networks": len(self.networks),
            "ranges": len(self.ranges), "members": len(self.members),
        }

    # -- forward/reverse zone lookup --------------------------------------
    def zone_for(self, fqdn, view):
        """Longest authoritative FORWARD zone holding this name, or None.

        Longest match, by label. endswith() alone matches 'evilexample.com'
        against 'example.com', which is how a check like this quietly stops
        working.
        """
        best, best_len = None, -1
        for z in self.zones:
            if (z.get("view") or "default") != view:
                continue
            name = (z.get("fqdn") or "")
            if is_suffix_zone(fqdn, name) and len(name) > best_len:
                best, best_len = z, len(name)
        return best

    # -- per VLAN ---------------------------------------------------------
    def plan(self, specs):
        for spec in specs:
            self._plan_one(spec)
        return specs

    def _blocked_all(self, spec, reason):
        spec.steps.append(Step(spec, "%s/all" % spec.cidr, "network",
                               str(spec.cidr), BLOCKED, reason))

    def _plan_one(self, spec):
        if self.netviews and spec.network_view not in self.netviews:
            self._blocked_all(spec, "network view '%s' does not exist on this grid "
                                    "(network views here: %s)"
                              % (spec.network_view, ", ".join(sorted(self.netviews))))
            return
        if self.views and spec.view not in self.views:
            self._blocked_all(spec, "DNS view '%s' does not exist on this grid "
                                    "(DNS views here: %s)"
                              % (spec.view, ", ".join(sorted(self.views))))
            return
        net_key = self._plan_network(spec)
        self._plan_range(spec, net_key)
        zone_keys = self._plan_reverse(spec)
        self._plan_gateway_records(spec, zone_keys)

    # -- step 1: the network ----------------------------------------------
    def _plan_network(self, spec):
        key = "network:%s:%s" % (spec.network_view, spec.cidr)
        wanted_opts = spec.resolved_options.get("network") or {}
        payload = {"network": str(spec.cidr), "network_view": spec.network_view,
                   "comment": spec.fmt(spec.t.get("network_comment")) or ""}
        if wanted_opts:
            payload["options"] = [option_payload(n, v)
                                  for n, v in sorted(wanted_opts.items())]
        ea = {k: {"value": spec.fmt(v)} for k, v in (spec.t.get("extattrs") or {}).items()}
        if ea:
            payload["extattrs"] = ea

        existing = self.net_by_cidr.get((spec.network_view, str(spec.cidr)))
        if not existing:
            spec.steps.append(Step(spec, key, "network", str(spec.cidr), CREATE,
                                   "new network in view '%s'" % spec.network_view,
                                   payload))
            return key

        have = options_as_map(existing.get("options"))
        differs = []
        for name, val in sorted(wanted_opts.items()):
            if have.get(name) != val:
                differs.append("option %s: %s -> %s"
                               % (name, dashed(have.get(name)), val))
        if (existing.get("comment") or "") != payload["comment"]:
            differs.append("comment '%s' -> '%s'"
                           % (dashed(existing.get("comment")), payload["comment"]))
        if not differs:
            spec.steps.append(Step(spec, key, "network", str(spec.cidr), SKIP,
                                   "already exists with the same comment and "
                                   "options", ref=existing.get("_ref")))
            return key
        # The PUT carries only the mutable fields. 'network' and 'network_view'
        # are set at creation and WAPI rejects them on an update -- sending the
        # whole create payload back is how an update turns into a 400.
        patch = {k: v for k, v in payload.items()
                 if k in ("comment", "options", "extattrs")}
        self._conflict(spec, key, "network", str(spec.cidr), payload,
                       existing.get("_ref"), "; ".join(differs), patch=patch)
        return key

    # -- step 2: the DHCP range -------------------------------------------
    def _plan_range(self, spec, net_key):
        if spec.range_start is None:
            if not _as_bool(spec.t.get("dhcp"), True):
                spec.steps.append(Step(spec, "range:%s" % spec.cidr, "range",
                                       str(spec.cidr), SKIP,
                                       "dhcp: false -- no range for this network",
                                       needs=[net_key]))
            return
        key = "range:%s:%s" % (spec.network_view, spec.cidr)
        wanted_opts = spec.resolved_options.get("range") or {}
        net_opts = spec.resolved_options.get("network") or {}
        for name, val in sorted(wanted_opts.items()):
            if net_opts.get(name) == val:
                self.warnings.append(
                    (spec.where, "DHCP option %s is set to the same value on the "
                                 "network and on the range. The range inherits it "
                                 "from the network already; defining it twice "
                                 "changes nothing today and becomes a value nobody "
                                 "dares touch in three years. Keep it in one place"
                     % name))

        payload = {"start_addr": str(spec.range_start),
                   "end_addr": str(spec.range_end),
                   "network": str(spec.cidr),
                   "network_view": spec.network_view,
                   "comment": spec.fmt(spec.t.get("range_comment")) or ""}
        if spec.exclusions:
            payload["exclude"] = spec.exclusions
        if wanted_opts:
            payload["options"] = [option_payload(n, v)
                                  for n, v in sorted(wanted_opts.items())]

        member = spec.t.get("dhcp_member")
        if member:
            if self.members and member not in self.members:
                spec.steps.append(Step(spec, key, "range", "%s-%s"
                                       % (spec.range_start, spec.range_end),
                                       BLOCKED,
                                       "dhcp_member '%s' is not a member of this "
                                       "grid (members: %s)"
                                       % (member, ", ".join(sorted(
                                           m for m in self.members if m))),
                                       needs=[net_key]))
                return
            payload["member"] = {"_struct": "dhcpmember", "name": member}
            payload["server_association_type"] = "MEMBER"
        else:
            msg = ("the range has no dhcp_member. NIOS creates it happily and it "
                   "serves nothing: a range with no member or failover "
                   "association is a pool no DHCP server owns. The symptom is "
                   "clients with no address and no log line anywhere")
            if self.args.require_dhcp_member:
                spec.steps.append(Step(spec, key, "range", "%s-%s"
                                       % (spec.range_start, spec.range_end),
                                       BLOCKED, msg + " (--require-dhcp-member)",
                                       needs=[net_key]))
                return
            self.warnings.append((spec.where, msg))

        what = "%s-%s" % (spec.range_start, spec.range_end)
        existing = None
        for r in self.ranges_by_net.get((spec.network_view, str(spec.cidr))) or []:
            existing = r
            break
        if not existing:
            spec.steps.append(Step(spec, key, "range", what, CREATE,
                                   "new DHCP range in %s" % spec.cidr,
                                   payload, needs=[net_key]))
            return
        if (existing.get("start_addr") == str(spec.range_start)
                and existing.get("end_addr") == str(spec.range_end)):
            spec.steps.append(Step(spec, key, "range", what, SKIP,
                                   "a range with exactly these bounds already "
                                   "exists", ref=existing.get("_ref"),
                                   needs=[net_key]))
            return
        patch = {k: v for k, v in payload.items()
                 if k in ("start_addr", "end_addr", "comment", "options",
                          "exclude", "member", "server_association_type")}
        self._conflict(spec, key, "range", what, payload, existing.get("_ref"),
                       "existing range is %s-%s"
                       % (dashed(existing.get("start_addr")),
                          dashed(existing.get("end_addr"))),
                       patch=patch, needs=[net_key])

    # -- step 3: the reverse zone -----------------------------------------
    def _plan_reverse(self, spec):
        """The step everybody forgets, and the one with a case the API cannot
        rescue you from."""
        keys = []
        if not _as_bool(spec.t.get("create_reverse_zone"), True):
            return keys
        if spec.rfc2317_parent:
            spec.steps.append(Step(
                spec, "zone:%s" % spec.cidr, "zone_auth",
                "reverse of %s" % spec.cidr, BLOCKED,
                "a /%d needs an RFC 2317 classless reverse zone, and this script "
                "will not invent one. Whoever owns %s has to delegate the slice "
                "with CNAMEs first; only then can the zone be created here. "
                "Everything else for this VLAN is unaffected"
                % (spec.cidr.prefixlen, spec.rfc2317_parent)))
            return keys
        for fqdn in spec.reverse_zones:
            key = "zone:%s:%s" % (spec.view, fqdn)
            existing = self.zone_by_fqdn.get((spec.view, fqdn.lower()))
            if existing:
                if existing.get("disable"):
                    spec.steps.append(Step(spec, key, "zone_auth", fqdn, BLOCKED,
                                           "the reverse zone exists in view '%s' "
                                           "but is DISABLED, so it answers nothing"
                                           % spec.view))
                    continue
                spec.steps.append(Step(spec, key, "zone_auth", fqdn, SKIP,
                                       "reverse zone already exists in view '%s'"
                                       % spec.view, ref=existing.get("_ref")))
                keys.append(key)
                continue
            payload = {"fqdn": fqdn, "view": spec.view, "zone_format": "IPV4",
                       "comment": "reverse for %s" % spec.cidr}
            ns_group = spec.t.get("reverse_zone_ns_group")
            if ns_group:
                payload["ns_group"] = ns_group
            else:
                self.warnings.append(
                    (spec.where, "reverse zone %s would be created with no "
                                 "ns_group. NIOS accepts that and the zone has no "
                                 "name servers assigned, so it is never served. "
                                 "Set reverse_zone_ns_group in the template"
                     % fqdn))
            reason = "new reverse zone"
            if len(spec.reverse_zones) > 1:
                reason = ("new reverse zone -- a /%d needs %d of them, one per /24"
                          % (spec.cidr.prefixlen, len(spec.reverse_zones)))
            spec.steps.append(Step(spec, key, "zone_auth", fqdn, CREATE, reason,
                                   payload))
            keys.append(key)
        return keys

    # -- step 4 and 5: the gateway records --------------------------------
    def _plan_gateway_records(self, spec, zone_keys):
        pattern = spec.t.get("gateway_record")
        if not pattern:
            return
        name = spec.fmt(pattern)
        if not valid_fqdn(name):
            spec.steps.append(Step(spec, "a:%s" % name, "record:a", dashed(name),
                                   BLOCKED,
                                   "gateway_record resolved to '%s', which is not a "
                                   "valid FQDN" % dashed(name)))
            return
        zone = self.zone_for(name, spec.view)
        akey = "a:%s:%s" % (spec.view, name.lower())
        if not zone:
            spec.steps.append(Step(spec, akey, "record:a",
                                   "%s -> %s" % (name, spec.gateway), BLOCKED,
                                   "no authoritative forward zone on this grid holds "
                                   "%s in view '%s'. This script creates reverse "
                                   "zones, never forward ones -- a forward zone is a "
                                   "namespace decision, not a side effect"
                                   % (name, spec.view)))
            return
        if zone.get("disable"):
            spec.steps.append(Step(spec, akey, "record:a",
                                   "%s -> %s" % (name, spec.gateway), BLOCKED,
                                   "the forward zone %s exists in view '%s' but is "
                                   "disabled" % (zone.get("fqdn"), spec.view)))
            return

        payload = {"name": name, "ipv4addr": str(spec.gateway), "view": spec.view,
                   "comment": "gateway for VLAN %d" % spec.vlan}
        existing = self.a_by_name.get((spec.view, name.lower())) or []
        same = [r for r in existing if r.get("ipv4addr") == str(spec.gateway)]
        if same:
            spec.steps.append(Step(spec, akey, "record:a",
                                   "%s -> %s" % (name, spec.gateway), SKIP,
                                   "already exists with the same address",
                                   ref=same[0].get("_ref")))
        elif existing:
            self._conflict(spec, akey, "record:a",
                           "%s -> %s" % (name, spec.gateway),
                           payload, existing[0].get("_ref"),
                           "exists pointing at %s"
                           % ", ".join(sorted(r.get("ipv4addr") or "-"
                                              for r in existing)),
                           patch={"ipv4addr": str(spec.gateway)})
        else:
            spec.steps.append(Step(spec, akey, "record:a",
                                   "%s -> %s" % (name, spec.gateway), CREATE,
                                   "gateway record in zone %s" % zone.get("fqdn"),
                                   payload))

        # The PTR. Its parent is the reverse zone, which may be a step in THIS
        # run -- that is the dependency the apply order exists for.
        rev_fqdn = ipaddress.ip_address(spec.gateway).reverse_pointer
        owner = None
        for fqdn in spec.reverse_zones:
            if is_suffix_zone(rev_fqdn, fqdn):
                owner = fqdn
                break
        pkey = "ptr:%s:%s" % (spec.view, spec.gateway)
        if not owner:
            spec.steps.append(Step(spec, pkey, "record:ptr",
                                   "%s -> %s" % (spec.gateway, name), BLOCKED,
                                   "no reverse zone for %s is being created or "
                                   "exists here%s"
                                   % (spec.gateway,
                                      " (RFC 2317 delegation needed from %s)"
                                      % spec.rfc2317_parent
                                      if spec.rfc2317_parent else "")))
            return
        zkey = "zone:%s:%s" % (spec.view, owner)
        needs = [zkey] if zkey in zone_keys else []
        if not needs and not self.zone_by_fqdn.get((spec.view, owner.lower())):
            spec.steps.append(Step(spec, pkey, "record:ptr",
                                   "%s -> %s" % (spec.gateway, name), BLOCKED,
                                   "the reverse zone %s is neither on the grid nor "
                                   "being created by this run" % owner))
            return
        ppayload = {"ptrdname": name, "ipv4addr": str(spec.gateway),
                    "view": spec.view,
                    "comment": "gateway for VLAN %d" % spec.vlan}
        existing = self.ptr_by_ip.get((spec.view, str(spec.gateway))) or []
        same = [p for p in existing
                if (p.get("ptrdname") or "").lower() == name.lower()]
        if same:
            spec.steps.append(Step(spec, pkey, "record:ptr",
                                   "%s -> %s" % (spec.gateway, name), SKIP,
                                   "PTR already points here",
                                   ref=same[0].get("_ref"), needs=needs))
        elif existing:
            self._conflict(spec, pkey, "record:ptr",
                           "%s -> %s" % (spec.gateway, name), ppayload,
                           existing[0].get("_ref"),
                           "exists pointing at %s"
                           % ", ".join(sorted(p.get("ptrdname") or "-"
                                              for p in existing)),
                           patch={"ptrdname": name}, needs=needs)
        else:
            spec.steps.append(Step(spec, pkey, "record:ptr",
                                   "%s -> %s" % (spec.gateway, name), CREATE,
                                   "reverse zone %s" % owner, ppayload,
                                   needs=needs))

    # -- conflicts ---------------------------------------------------------
    def _conflict(self, spec, key, obj, what, payload, ref, detail,
                  patch=None, needs=None):
        """The object exists with DIFFERENT values. This is the normal case on
        a second run or a half-finished change, not the exception, and the
        policy has to be explicit: skipping silently leaves the template and
        the grid disagreeing, updating silently changes production."""
        policy = self.args.on_conflict
        if policy == "skip":
            spec.steps.append(Step(spec, key, obj, what, SKIP,
                                   "CONFLICT: %s -- left alone (--on-conflict skip)"
                                   % detail, ref=ref, needs=needs))
        elif policy == "update":
            spec.steps.append(Step(spec, key, obj, what, UPDATE,
                                   "CONFLICT: %s" % detail,
                                   patch if patch is not None else payload,
                                   ref=ref, previous=detail, needs=needs))
        else:
            spec.steps.append(Step(spec, key, obj, what, BLOCKED,
                                   "CONFLICT: %s (--on-conflict abort)" % detail,
                                   ref=ref, needs=needs))


def dedupe(specs):
    """Two planned operations aiming at the SAME WAPI object are a defect.

    WAPI accepts the first and answers 400 on the second, and no amount of
    per-row validation sees it coming: the two steps are well formed on their
    own. The case that actually happens is the reverse zone. Network views are
    separate namespaces and DNS views are separate namespaces, and they are
    NOT the same thing -- so two VLANs in two different network views can
    perfectly well both ask for 30.30.10.in-addr.arpa in the one DNS view
    'default'. One zone, two steps, one 400.
    """
    first = {}
    for spec in specs:
        for step in spec.steps:
            if step.verdict not in (CREATE, UPDATE):
                continue
            if step.key in first:
                owner = first[step.key]
                step.verdict = SKIP
                step.reason = ("the same %s object is already planned by %s -- "
                               "one write, not two" % (step.obj, owner.spec.where))
            else:
                first[step.key] = step


# ---------------------------------------------------------------- applying
class Rollback(object):
    """One JSON object per line, flushed after every write.

    Flushed, not buffered: the run that needs this file most is the one that
    died halfway, and a buffered file loses exactly the objects nobody knows
    about.
    """

    def __init__(self, path):
        self.path = path
        self.fh = open(path, "w", encoding="utf-8")
        self.count = 0

    def record(self, step, ref):
        entry = {
            "op": "created" if step.verdict == CREATE else "updated",
            "wapi_object": step.obj,
            "ref": ref,
            "what": step.what,
            "vlan": step.spec.vlan,
            "cidr": str(step.spec.cidr),
            "dns_view": step.spec.view,
            "network_view": step.spec.network_view,
            "source": step.spec.where,
            "previous_value": step.previous,
            "at": datetime.now(timezone.utc).isoformat(),
        }
        self.fh.write(json.dumps(entry) + "\n")
        self.fh.flush()
        self.count += 1

    def close(self):
        self.fh.close()


def apply_plan(wapi, steps, rollback):
    """Execute in topological order, and NEVER attempt a step whose parent did
    not happen.

    A DHCP range inside a network that was not created is not a failure worth
    reporting as one -- it is a step that was not attempted, which is a
    different fact. Conflating the two is how a half-finished run reads as a
    catastrophe.
    """
    done = {}
    by_key = {}
    for step in steps:
        by_key.setdefault(step.key, step)
    for step in steps:
        if step.verdict in (SKIP, BLOCKED):
            done[step.key] = step.satisfied()
            continue
        missing = [k for k in step.needs if not done.get(k, False)]
        if missing:
            parent = by_key.get(missing[0])
            step.result = ("not attempted: the %s it lives in (%s) did not get "
                           "created, and this object cannot exist without it"
                           % (parent.obj if parent else "parent object",
                              parent.what if parent else missing[0]))
            done[step.key] = False
            continue
        if step.verdict == CREATE:
            ok, res = wapi.post(step.obj, step.payload)
        else:
            ok, res = wapi.put(step.ref, step.payload)
        if ok:
            step.result = "ok"
            step.created_ref = res if isinstance(res, str) else str(res)
            rollback.record(step, step.created_ref)
        else:
            step.result = "failed: %s" % res
        done[step.key] = step.satisfied()
    return done


# ---------------------------------------------------------------- reporting
RULE = "=" * 74
THIN = "-" * 74

ORDER_LABEL = {
    "network": "1 network",
    "range": "2 DHCP range",
    "zone_auth": "3 reverse zone",
    "record:a": "4 gateway A",
    "record:ptr": "5 gateway PTR",
}


def header(args, planner):
    print(RULE)
    print("infoblox-network-provision            NIOS / WAPI %s" % args.wapi_version)
    print(RULE)
    print("Grid        : %s%s" % (args.offline_dir or args.host,
                                  "  (offline replay)" if args.offline_dir else ""))
    print("Template    : %s" % args.template)
    print("Networks    : %s" % (args.csv or "the networks: block of the template"))
    print("Mode        : %s" % ("APPLY -- this run WRITES to the grid" if args.apply
                                else "DRY RUN -- nothing will be written"))
    print("Conflicts   : --on-conflict %s" % args.on_conflict)
    c = planner.counts
    if c:
        print("Grid holds  : %d network(s), %d range(s), %d zone(s), %d DNS view(s), "
              "%d network view(s)"
              % (c["networks"], c["ranges"], c["zones"], c["views"], c["netviews"]))
    if args.offline_dir:
        print("")
        print("  Everything below came from saved JSON in %s, not from a Grid"
              % args.offline_dir)
        print("  Master. It proves the plan and the arithmetic; it proves nothing")
        print("  about how your grid will answer a POST.")
    print("")


def report_errors(errors, args):
    print("INPUT REJECTED -- %d problem(s), and the whole list at once"
          % len(errors))
    print(THIN)
    for where, msg in errors[:args.limit]:
        print("  %-26s %s" % (where, msg))
    if len(errors) > args.limit:
        print("  ... %d more (raise --limit)" % (len(errors) - args.limit))
    print("")
    print("  No transport was built, nothing was read from the grid and nothing")
    print("  was written. The whole batch is validated first on purpose:")
    print("  stopping on the fourth VLAN with three already provisioned is the")
    print("  worst outcome this script can have.")


def report_plan(specs, planner, args):
    """Per VLAN, in the order the steps will run. That order IS the content:
    it is a dependency, not a preference."""
    for spec in specs:
        print("%s   [%s]" % (spec.label(), spec.where))
        print(THIN)
        print("  gateway     : %s" % dashed(spec.gateway))
        if spec.range_start:
            low = int(spec.range_start) - int(spec.cidr.network_address) - 1
            high = int(spec.cidr.broadcast_address) - int(spec.range_end) - 1
            print("  DHCP range  : %s - %s  (%d addresses)"
                  % (spec.range_start, spec.range_end,
                     int(spec.range_end) - int(spec.range_start) + 1))
            print("  reserved    : %d low (.%s to .%s), %d high -- gateway and "
                  "statics live here"
                  % (low, str(spec.cidr.network_address + 1).split(".")[-1],
                     str(spec.cidr.network_address + low).split(".")[-1], high))
            for ex in spec.exclusions:
                print("  excluded    : %s - %s"
                      % (ex["start_address"], ex["end_address"]))
        else:
            print("  DHCP range  : -   (none planned)")
        print("  reverse zone: %s"
              % (", ".join(spec.reverse_zones) if spec.reverse_zones
                 else "- (RFC 2317, see below)" if spec.rfc2317_parent else "-"))
        for step in spec.steps:
            print("  %-16s %-9s %-14s %s"
                  % (ORDER_LABEL.get(step.obj, step.obj), step.verdict,
                     step.obj, step.what))
            print("        %s" % step.reason)
        print("")

    if planner.warnings:
        # Grouped by message, not by VLAN. A template-level mistake produces
        # one warning per network, and forty copies of the same sentence is
        # how a reader learns to skip the section.
        grouped = {}
        for where, msg in planner.warnings:
            grouped.setdefault(msg, []).append(where)
        print("WARNINGS -- valid, accepted by NIOS, and probably not what you want")
        print(THIN)
        for msg in grouped:
            where = grouped[msg]
            shown = ", ".join(where[:3])
            if len(where) > 3:
                shown += " and %d more" % (len(where) - 3)
            print("  %s" % msg)
            print("      affects: %s" % shown)
        print("")

    steps = [s for spec in specs for s in spec.steps]
    by_obj = {}
    for step in steps:
        by_obj.setdefault(step.obj, {}).setdefault(step.verdict, 0)
        by_obj[step.obj][step.verdict] += 1
    print("THE PLAN")
    print(THIN)
    print("  %-16s %8s %8s %8s %8s" % ("WAPI object", "create", "update",
                                       "skip", "blocked"))
    totals = {CREATE: 0, UPDATE: 0, SKIP: 0, BLOCKED: 0}
    for obj in sorted(by_obj, key=lambda o: ORDER_LABEL.get(o, o)):
        b = by_obj[obj]
        for v in totals:
            totals[v] += b.get(v, 0)
        print("  %-16s %8d %8d %8d %8d"
              % (obj, b.get(CREATE, 0), b.get(UPDATE, 0), b.get(SKIP, 0),
                 b.get(BLOCKED, 0)))
    print("  %-16s %8d %8d %8d %8d"
          % ("TOTAL", totals[CREATE], totals[UPDATE], totals[SKIP],
             totals[BLOCKED]))
    writes = totals[CREATE] + totals[UPDATE]
    print("")
    print("  %d VLAN(s), %d object(s) would be written."
          % (len(specs), writes))
    return writes


def report_apply(steps, rollback, args):
    ok = [s for s in steps if s.result == "ok"]
    bad = [s for s in steps if s.result and s.result.startswith("failed")]
    skipped = [s for s in steps if s.result and s.result.startswith("not attempted")]
    print("")
    print("APPLY")
    print(THIN)
    if args.offline_dir:
        print("  SIMULATED. --offline-dir has no grid behind it, so every write was")
        print("  faked and NOTHING was created. This path exists to exercise the")
        print("  plan end to end; it is not a record of a change.")
    for step in (bad + skipped)[:args.limit]:
        print("  %-14s %s" % (step.obj, step.what))
        print("      %s" % step.result)
    print("  %d written, %d failed, %d not attempted."
          % (len(ok), len(bad), len(skipped)))
    if ok:
        print("  WHAT EXISTS NOW because of this run:")
        for step in ok[:args.limit]:
            print("    %-14s %s" % (step.obj, step.what))
    if rollback and rollback.count and not args.offline_dir:
        print("  Rollback file: %s  (%d entry/entries)"
              % (rollback.path, rollback.count))
        print("  It holds the _ref of everything this run touched, in the order it")
        print("  happened. Undo it in REVERSE order -- a reverse zone will not")
        print("  delete while a PTR still lives in it.")


# ---------------------------------------------------------------- entry
def parse_args(argv):
    p = argparse.ArgumentParser(
        description="Provision a new VLAN on an Infoblox NIOS grid as the five "
                    "objects it really is -- network, DHCP range, DHCP options, "
                    "reverse zone and gateway records -- over WAPI v2.x. Dry run "
                    "unless --apply.")
    p.add_argument("--template", required=True,
                   help="YAML template: the provisioning profile, and optionally "
                        "the networks: list itself")
    p.add_argument("--csv", help="CSV of VLANs to provision, instead of the "
                                 "networks: block. Columns vlan,name,cidr")
    p.add_argument("--host", help="Grid Master hostname or IP. Needs IB_USER and IB_PASS")
    p.add_argument("--wapi-version", default=DEFAULT_WAPI,
                   help="WAPI version path segment. Default %s" % DEFAULT_WAPI)
    p.add_argument("--apply", action="store_true",
                   help="Actually write. Without it the script only plans")
    p.add_argument("--on-conflict", choices=("skip", "update", "abort"),
                   default="skip",
                   help="An object that exists with DIFFERENT values: leave it "
                        "(skip, the default), change it (update), or refuse the "
                        "step (abort)")
    p.add_argument("--max-objects", type=int, default=200,
                   help="Refuse to apply a plan bigger than this unless "
                        "--confirm-large is given. Default 200")
    p.add_argument("--confirm-large", action="store_true",
                   help="Acknowledge a plan over --max-objects")
    p.add_argument("--require-dhcp-member", action="store_true",
                   help="Block any range with no dhcp_member instead of warning. "
                        "The right setting for CI")
    p.add_argument("--rollback-out",
                   help="Where to write the rollback file. Default "
                        "rollback-<UTC>.jsonl next to the template")
    p.add_argument("--validate-only", action="store_true",
                   help="Check the template and the VLAN list and stop. No grid, "
                        "no credentials needed")
    p.add_argument("--limit", type=int, default=50,
                   help="Rows printed per section. Default 50")
    p.add_argument("--insecure", action="store_true",
                   help="Skip TLS verification. Only for a certificate you recognise")
    p.add_argument("--save-dir",
                   help="Save every WAPI response here for later offline runs")
    p.add_argument("--offline-dir",
                   help="Replay a --save-dir. No network, no credentials")
    args = p.parse_args(argv)
    if args.validate_only:
        return args
    if bool(args.host) == bool(args.offline_dir):
        p.error("pass exactly one of --host or --offline-dir (or --validate-only)")
    return args


def main(argv=None):
    args = parse_args(argv)

    # ---- everything offline, before a transport exists at all ----------
    errors = []
    _, specs = load_plan(args.template, args.csv, errors)
    for spec in specs:
        # Geometry first: an option value can be written '{gateway}', and the
        # gateway is not known until the geometry has been worked out. The
        # other order resolved it to an empty string and validated that.
        geometry(spec, errors)
        validate_options(spec, errors)
        validate_strings(spec, errors)
    errors.extend(cross_spec_errors(specs))
    errors.sort(key=lambda e: (str(e[0]), str(e[1])))

    if args.validate_only:
        blamed = set(e[0] for e in errors)
        print("%s: %d VLAN(s) ready, %d problem(s) across the whole input."
              % (args.template,
                 len([s for s in specs if s.where not in blamed]), len(errors)))
        for where, msg in errors[:args.limit]:
            print("  %-26s %s" % (where, msg))
        if errors:
            print("Nothing else was done. Fix the input and run this again.")
            return 1
        print("The arithmetic is sound: gateways, ranges, exclusions and reverse")
        print("zone names all check out. This says NOTHING about whether those")
        print("networks, zones or members exist on your grid -- that needs")
        print("--host or --offline-dir.")
        return 0

    if errors:
        wapi = None
        planner = Planner(None, args)
        header(args, planner)
        report_errors(errors, args)
        return 2

    wapi = Wapi(host=args.host, version=args.wapi_version, insecure=args.insecure,
                offline_dir=args.offline_dir, save_dir=args.save_dir)
    planner = Planner(wapi, args)
    planner.fetch()
    planner.plan(specs)
    dedupe(specs)

    header(args, planner)
    writes = report_plan(specs, planner, args)
    steps = [s for spec in specs for s in spec.steps]
    blocked = [s for s in steps if s.verdict == BLOCKED]
    conflicts = [s for s in steps if s.verdict == SKIP
                 and s.reason.startswith("CONFLICT")]

    if not args.apply:
        print("")
        print("  Dry run. Nothing was written. Re-read the plan above, then add")
        print("  --apply to the same command to execute exactly it.")
        return 1 if (blocked or conflicts) else 0

    if writes > args.max_objects and not args.confirm_large:
        print("")
        print("  REFUSING TO APPLY: %d objects is over --max-objects %d."
              % (writes, args.max_objects))
        print("  A CSV with one wrong column is how a site gets a second set of")
        print("  networks. If %d is really what you meant, add --confirm-large."
              % writes)
        return 2

    if args.offline_dir:
        rollback = Rollback(os.devnull)
    else:
        path = args.rollback_out or os.path.join(
            os.path.dirname(os.path.abspath(args.template)),
            "rollback-%s.jsonl"
            % datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ"))
        rollback = Rollback(path)

    try:
        apply_plan(wapi, steps, rollback)
    finally:
        rollback.close()
    report_apply(steps, rollback, args)
    failed = [s for s in steps if s.result and not s.result == "ok"
              and s.verdict in (CREATE, UPDATE)]
    return 1 if (failed or blocked or conflicts) else 0


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        sys.stderr.write("\ninterrupted\n")
        sys.exit(2)
    except SystemExit as exc:
        # The shared loader raises SystemExit with a sentence, which Python
        # would turn into exit code 1 -- the code this script reserves for
        # "there are findings". A setup failure is a 2.
        if isinstance(exc.code, str):
            sys.stderr.write(exc.code + "\n")
            sys.exit(2)
        raise
    except requests.exceptions.SSLError as exc:
        sys.stderr.write("TLS verification failed: %s\n(use --insecure only for a "
                         "certificate you recognise)\n" % exc)
        sys.exit(2)
    except requests.exceptions.RequestException as exc:
        sys.stderr.write("could not reach the Grid Master: %s\n" % exc)
        sys.exit(2)