"""Runbook decision tests: fixtures shaped like real CX-Tools collector output."""
import sys, os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))

from triagelib import alerts as A, cxbridge, runbooks

# --- stub the CX-Tools boundary so the decision logic can be tested alone ---
class FakeCx:
    def status_pair_ok(self, ih, os_, task):
        ih, os_ = (ih or "").upper(), (os_ or "").upper()
        if ih == os_: return True
        return (ih, os_) in {("HIBERNATED","SHELVED_OFFLOADED"), ("REBOOTING","HARD_REBOOT")}
    def mismatch_parts(self, item):
        return str(item.get("check","Mismatch")), str(item.get("detail",""))
    def normalize_empty(self, v): return "" if v in (None,"","None") else str(v)
    def first_present(self, m, *keys, default=""):
        for k in keys:
            if isinstance(m, dict) and k in m: return m[k]
        return default
    def public_ip_from_server(self, s): return (s or {}).get("_public_ip","")
    def json_safe(self, o): return o
    def map_region(self, r): return {"CANADA-1":"ca1","US-1":"us1","NORWAY-1":"no1"}.get(r,"")

STUB = {"host_health": {}, "gpu_census": {}, "failed_event": {}, "vm": {}, "host": {}}
cxbridge.cx = lambda: FakeCx()
cxbridge.host_health = lambda region, host: STUB["host_health"]
cxbridge.host_gpu_census = lambda region, host: STUB["gpu_census"]
cxbridge.failed_openstack_event = lambda region, osid, scan=5: STUB["failed_event"]
cxbridge.collect_vm = lambda t, **k: STUB["vm"]
cxbridge.collect_host = lambda h, **k: STUB["host"]
cxbridge.json_safe = lambda o: o

HEALTHY = {"ok": True, "nova_state": "up", "nova_status": "enabled", "ovs_alive": True,
           "ovs_state": "UP", "uptime": "17:54", "aggregates": "agg", "bad_signals": []}
SICK = {**HEALTHY, "nova_state": "down", "ovs_state": "DOWN", "ovs_alive": False,
        "bad_signals": ["Nova state is down", "OVS state is DOWN"]}

def vm(**over):
    base = {"ok": True, "exit_code": 0, "mode": "vm", "infrahub_id": "123456",
            "openstack_id": "9ec7a021-d741-484f-9387-4eaf8879fd77", "name": "test-vm",
            "region": "ca1", "region_display": "CANADA-1", "ih_status": "ACTIVE", "os_status": "ACTIVE",
            "task_state": "None", "host": "CA1-ESC8-040", "flavor": "n3-H100x8", "gpu_count": "8",
            "floating_ip": "69.19.140.110", "created": "2026-07-01 10:00:00 UTC", "ssh_text": "reachable",
            "ssh_raw": "reachable", "volumes_summary": "None", "openstack_fault": "None",
            "faults": [], "ih_events": [], "ih_events_all": [], "mismatches": [], "warn_reasons": [],
            "info_notes": [], "org_value": "8463 - Acme Corp", "owners": ["Lars <lars@simli.com>"],
            "server": {"status": "ACTIVE"}, "infrahub": {"floating_ip": "69.19.140.110"}}
    base.update(over); return base

def alert(name, **labels):
    return A.from_labels({"alertname": name, **labels})

def run(name, labels, vmdata=None, hostdata=None, health=None, census=None, fevent=None, prom=None):
    STUB.update({"vm": vmdata or {}, "host": hostdata or {}, "host_health": health or HEALTHY,
                 "gpu_census": census or {}, "failed_event": fevent or {}})
    return runbooks.diagnose(alert(name, **labels), prom)

def check(label, cond, extra=""):
    print(f"  {'PASS' if cond else 'FAIL'}  {label}" + (f"  <- {extra}" if not cond and extra else ""))
    return cond

fails = 0
def expect(label, cond, extra=""):
    global fails
    if not check(label, cond, extra): fails += 1

L_ERR = dict(openstack_id="9ec7a021-d741-484f-9387-4eaf8879fd77", region="CANADA-1",
             instance_name="test-vm", organization="8463 - Acme Corp", status="ERROR")

print("\n[1] ERROR - creation failed, never reached ACTIVE, insufficient stock")
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
        vm(host="N/A", ih_status="ERROR", os_status="ERROR", server={"status": "ERROR"},
           openstack_fault="No valid host was found. There are not enough hosts available"))
expect("matched no_valid_host fault", "scheduler could not place" in d.verdict.lower(), d.verdict)
expect("identified as never-ACTIVE", any("never placed on a host" in f.value for f in d.findings))
expect("chose the insufficient-stock template", any(x.template_id == "error_never_active" for x in d.drafts),
       [x.template_id for x in d.drafts])
expect("mentions 7-day outreach window", any("7 days" in x.when for x in d.drafts))
expect("escalates to Infrastructure", any(a.owner == runbooks.INFRA for a in d.actions))
expect("contacts resolved", d.contacts.get("resolved"))

print("\n[2] ERROR - was ACTIVE, stale LVM on host")
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
        vm(ih_status="ERROR", os_status="ERROR",
           faults=[["500", "Build of instance aborted: Failed to remove volume(s): lvremove -f /dev/nova-vg/x_disk", "2026-07-29"]]))
expect("matched lvremove fault", "stale LVM" in d.verdict, d.verdict)
expect("identified as previously ACTIVE", any("hypervisor is recorded" in f.value for f in d.findings))
expect("chose the was-ACTIVE template", any(x.template_id == "error_was_active" for x in d.drafts),
       [x.template_id for x in d.drafts])
expect("assessment flags possible customer data", "customer data" in d.assessment)

print("\n[3] ERROR - NUMA/PCI fault, host is FULL (proves host fine)")
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
        vm(ih_status="ERROR", os_status="ERROR",
           openstack_fault="Insufficient compute resources: Requested instance NUMA topology together with requested PCI devices cannot fit the given host NUMA topology; Claim pci failed."),
        census={"ok": True, "total_gpus": 8, "instances": [{"name":"a"}]*4})
expect("ran the GPU census", any("GPUs allocated on host" in f.label for f in d.findings))
expect("concluded host is FULL", any("FULL" in f.detail for f in d.findings))
expect("marked the capacity check as already done", any(a.status == "done" for a in d.actions))

print("\n[3b] same fault, host NOT full -> must escalate")
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
        vm(ih_status="ERROR", os_status="ERROR",
           openstack_fault="Claim pci failed."),
        census={"ok": True, "total_gpus": 4, "instances": [{"name":"a"}]*2})
expect("raises a Jira for Infra", any("Jira" in a.text and a.owner == runbooks.INFRA for a in d.actions))

print("\n[4] SHUTOFF - billing notice only")
d = run("Instance in SHUTOFF state in :flag-ca:CA-1 for greater than 30 min",
        dict(openstack_id="dc156762-3fa8-481d-89e0-12c2fb55e046", region="CANADA-1",
             instance_name="energetic-galileo", organization="27358 - Zyra", status="SHUTOFF"),
        vm(name="energetic-galileo", ih_status="SHUTOFF", os_status="SHUTOFF", server={"status":"SHUTOFF"}))
expect("verdict is customer-initiated", "customer-initiated" in d.verdict.lower(), d.verdict)
expect("used the shutoff snippet", any(x.template_id == "shutoff" for x in d.drafts))
expect("substituted the VM name", any("energetic-galileo" in x.body for x in d.drafts))
expect("no Infra escalation", not any(a.owner == runbooks.INFRA for a in d.actions))

print("\n[5] DELETING - server still in OpenStack, delete request found")
d = run("Instance in DELETING state in :flag-ca:CA-1 for greater than 30 min",
        dict(openstack_id="13a855a4-c563-4350-9a8e-ffa9efa27d9e", region="CANADA-1",
             instance_name="external-hs-h100", organization="10420 - Inceptions AI", status="DELETING"),
        vm(ih_status="DELETING", os_status="ACTIVE", server={"status": "ACTIVE"},
           ih_events_all=[["2026-07-29 21:40:00", "InstanceDeleteRequest", "Delete Instance Request Sent."]]))
expect("confirmed customer intent from events", any(a.status == "done" and "intent" in a.text for a in d.actions))
expect("tells CX to delete in OpenStack", any("Delete the server in OpenStack" in a.text for a in d.actions))
expect("offers both the notice and the ticket-closing reply",
       sorted(x.template_id for x in d.drafts) == ["deleting", "deleting_resolved"],
       [x.template_id for x in d.drafts])
expect("tells CX to close the Infrahub record too, not just the server",
       any("InfraInsight" in a.text for a in d.actions), [a.text for a in d.actions])

print("\n[5c] DELETING - server exists but never reached a host")
d = run("Instance in DELETING state in :flag-ca:CA-1 for greater than 30 min",
        dict(openstack_id="x", region="CANADA-1", status="DELETING"),
        vm(ih_status="DELETING", os_status="ERROR", host="N/A",
           server={"status": "ERROR"}, openstack_fault="No valid host was found."))
expect("flags that the build never completed", any("build never completed" in f.value for f in d.findings))
expect("still requires the InfraInsight close-out", any("InfraInsight" in a.text for a in d.actions))

print("\n[5b] DELETING - already gone from OpenStack, no delete event")
d = run("Instance in DELETING state in :flag-ca:CA-1 for greater than 30 min",
        dict(openstack_id="x", region="CANADA-1", status="DELETING"),
        vm(ih_status="DELETING", os_status="N/A", server={}, ih_events_all=[]))
expect("notes the server is already gone", any("already gone" in f.value for f in d.findings))
expect("routes attribution to DevOps", any(a.owner == runbooks.DEVOPS for a in d.actions))

print("\n[6] CREATING - never got an OpenStack ID")
d = run("Instance in CREATING state in :flag-ca:CA-1 for greater than 30min",
        dict(openstack_id="None", region="CANADA-1", instance_name="vm-28070520-5d1f2",
             organization="5574 - Nexgen", status="CREATING"),
        vm(openstack_id="N/A", ih_status="CREATING", os_status="N/A", server={}, host="N/A"))
expect("verdict says never got an OpenStack ID", "never got an OpenStack ID" in d.verdict, d.verdict)
expect("used the creating template", any(x.template_id == "creating" for x in d.drafts))
expect("instructs deletion", any("Delete the stuck instance" in a.text for a in d.actions))

print("\n[7] HIBERNATING - sick host")
d = run("Instance in HIBERNATING state in :flag-ca:CA-1 for greater than 120min",
        dict(openstack_id="28171e6f", region="CANADA-1", instance="CA1-ESC812-211",
             instance_name="apt25-prod", status="HIBERNATING"),
        vm(ih_status="HIBERNATING", os_status="ACTIVE"), health=SICK)
expect("verdict blames the host", "host problem" in d.verdict, d.verdict)
expect("escalates to Infra with the bad signals", any(a.owner == runbooks.INFRA and "OVS" in a.text for a in d.actions))
expect("still drives the shelve", any("shelve" in a.text.lower() for a in d.actions))

print("\n[8] Suspected Rogue VM - host with two different mismatches")
host_result = {"ok": True, "mode": "host", "host": "CA1-ESC8-068", "region": "ca1", "server_count": 3,
    "hypervisor": {"state": "up", "status": "enabled"}, "ovs": {"alive": True, "state": "UP"},
    "instances": [
        {"idx": 1, "name": "vm-hib-shutoff", "infrahub_id": "1", "openstack_id": "a", "ih_status": "HIBERNATED",
         "os_status": "SHUTOFF", "host": "CA1-ESC8-068", "mismatches": [{"check": "State", "detail": "IH HIBERNATED vs OS SHUTOFF"}],
         "warn_reasons": ["mismatch detected"], "org_value": "1 - A", "owners": ["a@x.com"]},
        {"idx": 2, "name": "vm-hib-active", "infrahub_id": "2", "openstack_id": "b", "ih_status": "HIBERNATED",
         "os_status": "ACTIVE", "host": "CA1-ESC8-068", "mismatches": [{"check": "State", "detail": "IH HIBERNATED vs OS ACTIVE"}],
         "warn_reasons": ["mismatch detected"], "org_value": "2 - B", "owners": ["b@x.com"]},
        {"idx": 3, "name": "tempest-thing", "tempest": True, "ih_status": "N/A", "os_status": "ACTIVE",
         "mismatches": [{"check": "Infrahub Missing", "detail": "not in Infrahub"}], "warn_reasons": []},
    ]}
d = run(":ninja:Suspected Rogue VM", dict(instance="CA1-ESC8-068"), hostdata=host_result)
expect("counted 2 of 3 as mismatched", "2 of 3" in d.verdict, d.verdict)
expect("ignored the tempest instance", d.evidence.get("ignored_tempest") == 1)
expect("HIBERNATED/SHUTOFF -> Windmill stale-image cleanup", any("Windmill" in a.text for a in d.actions))
expect("HIBERNATED/ACTIVE -> sync-error comms", any(x.template_id == "sync_state" for x in d.drafts))
expect("actions are scoped per instance", any(a.text.startswith("[vm-hib-active]") for a in d.actions))

print("\n[9] Suspected Rogue VM - clean host")
d = run(":ninja:Suspected Rogue VM", dict(instance="CA1-ESC8-068"),
        hostdata={**host_result, "instances": [{"idx":1,"name":"ok","ih_status":"ACTIVE","os_status":"ACTIVE",
                  "mismatches": [], "warn_reasons": []}]})
expect("verdict says no mismatch", "No Infrahub/OpenStack mismatch" in d.verdict, d.verdict)
expect("redirects to the InfraInsight host query", any("InfraInsight" in a.text for a in d.actions))

print("\n[10] Duplicated IPs - one DELETING claimant, one with a wrong Infrahub IP")
multi = {"ok": True, "mode": "multi_vm", "instances": [
    {"name": "old-vm", "infrahub_id": "1", "openstack_id": "a", "ih_status": "DELETING", "os_status": "ACTIVE",
     "server": {"status": "ACTIVE", "_public_ip": "69.19.137.135"}, "infrahub": {"floating_ip": "69.19.137.135"},
     "org_value": "1 - A", "owners": ["a@x.com"]},
    {"name": "new-vm", "infrahub_id": "2", "openstack_id": "b", "ih_status": "ACTIVE", "os_status": "ACTIVE",
     "server": {"status": "ACTIVE", "_public_ip": "69.19.140.9"}, "infrahub": {"floating_ip": "69.19.137.135"},
     "org_value": "2 - B", "owners": ["b@x.com"]},
]}
class FakeProm:
    def resources_by_floating_ip(self, fip):
        return [{"instance_name": "preprod-vm", "status": "ACTIVE", "region": "CANADA-1", "environment": "preprod"}]
d = run(":awkward:Duplicated IPs", dict(floating_ip="69.19.137.135"), vmdata=multi, prom=FakeProm())
expect("found 2 claimants needing correction", "2 of 2" in d.verdict, d.verdict)
expect("DELETING claimant -> delete it", any("[old-vm]" in a.text and "stuck DELETING" in a.text for a in d.actions))
expect("wrong-IP claimant -> Scenario #2", any("Scenario #2" in f.value for f in d.findings))
expect("Scenario #2 comms filled with the real IP", any(
    x.template_id == "dupip_corrected" and "69.19.140.9" in x.body for x in d.drafts))
expect("an unset sign-off name is reported rather than left as a placeholder", any(
    "AGENT_NAME" in x.unfilled for x in d.drafts if x.template_id == "dupip_corrected"))
expect("surfaced the PreProd claimant from Prometheus", any("preprod-vm" in f.label for f in d.findings))
expect("asks for a re-check after 5-10 min", any("5-10 minutes" in a.text for a in d.actions))

print("\n[11] Problem with Total GPUs - customers on host")
d = run("Problem with Total GPUs in a System", dict(instance="CA1-ESC8-111", region="CANADA-1", gpu_name="B200-SXM"),
        census={"ok": True, "total_gpus": 6, "instances": [
            {"name": "cust-vm", "status": "ACTIVE", "flavor": "n3-B200x6", "gpus": "6", "openstack_id": "z"}]})
expect("verdict names the host", "CA1-ESC8-111" in d.verdict, d.verdict)
expect("lists the affected instance", any("cust-vm" in f.label for f in d.findings))
expect("flags host-maintenance comms", any(a.kind == "comms" for a in d.actions))
expect("escalates a Jira to Infra", any("Jira" in a.text and a.owner == runbooks.INFRA for a in d.actions))
expect("notes there is no approved template", any("no approved customer template" in n for n in d.notes))
expect("drafts nothing", not d.drafts)

print("\n[12] K8s instance name is flagged")
d = run("Instance in ERROR state in :flag-ca:CA-1", {**L_ERR, "instance_name": "hyperstack-minion-lydhjev"},
        vm(host="N/A", ih_status="ERROR", os_status="ERROR"))
expect("noted the likely K8s node", any("Kubernetes" in n for n in d.notes))


print("\n[13] GPU sockets - every physical GPU accounted for")
from triagelib.runbooks import _gpu_slots
import collections as _c

R289 = [{"name": "luminous-hubble", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"},
        {"name": "vm832adbe203242", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"},
        {"name": "noble-maxwell", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"},
        {"name": "clever-schrodinger", "gpus": "2", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"}]

# Real numbers from CA1-ESC812-289: 8 physical, 7 reported in use, 4 VMs on 5 GPUs.
s = _gpu_slots({"physical": 8, "in_use_metric": 7, "spare_capacity_artifact": False}, R289)
c = _c.Counter(x["kind"] for x in s)
expect("8 sockets drawn for an 8-GPU host", len(s) == 8, len(s))
expect("5 named + 2 unaccounted + 1 free", (c["vm"], c["unaccounted"], c["free"]) == (5, 2, 1), dict(c))

s = _gpu_slots({"physical": 8, "in_use_metric": 8, "spare_capacity_artifact": True}, R289)
c = _c.Counter(x["kind"] for x in s)
expect("artifact host shows spare sockets as free, not unaccounted",
       (c["vm"], c["unaccounted"], c["free"]) == (5, 0, 3), dict(c))

s = _gpu_slots({"physical": None, "in_use_metric": 2, "spare_capacity_artifact": False},
               [{"name": "basilica", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"}])
c = _c.Counter(x["kind"] for x in s)
expect("unknown socket count degrades gracefully", (c["vm"], c["unaccounted"]) == (1, 1), dict(c))

s = _gpu_slots({"physical": 8, "in_use_metric": 8, "spare_capacity_artifact": False},
               [{"name": "ghost", "gpus": "8", "linked": False, "match": False,
                 "ih_status": "not in Infrahub", "os_status": "ACTIVE"}])
expect("a VM with no Infrahub record still fills its sockets and is flagged",
       len(s) == 8 and all(x["kind"] == "vm" and not x["linked"] for x in s))

s = _gpu_slots({"physical": 8, "in_use_metric": 5, "spare_capacity_artifact": False}, R289)
expect("no negative slots when in_use is below what VMs claim", len(s) >= 5 and all(
    x["kind"] in ("vm", "free", "unaccounted") for x in s), len(s))

print(f"\n{'ALL CHECKS PASSED' if not fails else str(fails) + ' CHECK(S) FAILED'}")
sys.exit(1 if fails else 0)