CX Triage: alert diagnosis over the CX-Tools collectors
Read-only triage for the Infrahub error alerts. Pulls the Prometheus alert queue, re-checks each alert's condition against live state to separate real work from noise, diagnoses it using the CX runbooks, and drafts the customer comms with contacts resolved from Infrahub. Findings from validating against production: - "Suspected Rogue VM" fires on spare GPU capacity, not rogue VMs: In_Use_Gpus equals the physical count on 71 of 75 firing hosts, so the rule reduces to "this host has a free GPU". Verified against OpenStack on 10 hosts. - "Exists in Infrahub but does not exist in OpenStack" matches every VM because openstack_nova_server_status returns no series; excluded as a rule defect. - Prometheus activeAt is reset several times a day by dips in the Resources metric, so alert ages are recovered from ALERTS history instead. Takes ~2,650 firing alerts down to ~20 that need a decision. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
268
tests/test_runbooks.py
Normal file
268
tests/test_runbooks.py
Normal file
@@ -0,0 +1,268 @@
|
||||
"""Runbook decision tests: fixtures shaped like real CX-Tools collector output."""
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
from triagelib import alerts as A, cxbridge, runbooks
|
||||
|
||||
# --- stub the CX-Tools boundary so the decision logic can be tested alone ---
|
||||
class FakeCx:
|
||||
def status_pair_ok(self, ih, os_, task):
|
||||
ih, os_ = (ih or "").upper(), (os_ or "").upper()
|
||||
if ih == os_: return True
|
||||
return (ih, os_) in {("HIBERNATED","SHELVED_OFFLOADED"), ("REBOOTING","HARD_REBOOT")}
|
||||
def mismatch_parts(self, item):
|
||||
return str(item.get("check","Mismatch")), str(item.get("detail",""))
|
||||
def normalize_empty(self, v): return "" if v in (None,"","None") else str(v)
|
||||
def first_present(self, m, *keys, default=""):
|
||||
for k in keys:
|
||||
if isinstance(m, dict) and k in m: return m[k]
|
||||
return default
|
||||
def public_ip_from_server(self, s): return (s or {}).get("_public_ip","")
|
||||
def json_safe(self, o): return o
|
||||
def map_region(self, r): return {"CANADA-1":"ca1","US-1":"us1","NORWAY-1":"no1"}.get(r,"")
|
||||
|
||||
STUB = {"host_health": {}, "gpu_census": {}, "failed_event": {}, "vm": {}, "host": {}}
|
||||
cxbridge.cx = lambda: FakeCx()
|
||||
cxbridge.host_health = lambda region, host: STUB["host_health"]
|
||||
cxbridge.host_gpu_census = lambda region, host: STUB["gpu_census"]
|
||||
cxbridge.failed_openstack_event = lambda region, osid, scan=5: STUB["failed_event"]
|
||||
cxbridge.collect_vm = lambda t, **k: STUB["vm"]
|
||||
cxbridge.collect_host = lambda h, **k: STUB["host"]
|
||||
cxbridge.json_safe = lambda o: o
|
||||
|
||||
HEALTHY = {"ok": True, "nova_state": "up", "nova_status": "enabled", "ovs_alive": True,
|
||||
"ovs_state": "UP", "uptime": "17:54", "aggregates": "agg", "bad_signals": []}
|
||||
SICK = {**HEALTHY, "nova_state": "down", "ovs_state": "DOWN", "ovs_alive": False,
|
||||
"bad_signals": ["Nova state is down", "OVS state is DOWN"]}
|
||||
|
||||
def vm(**over):
|
||||
base = {"ok": True, "exit_code": 0, "mode": "vm", "infrahub_id": "123456",
|
||||
"openstack_id": "9ec7a021-d741-484f-9387-4eaf8879fd77", "name": "test-vm",
|
||||
"region": "ca1", "region_display": "CANADA-1", "ih_status": "ACTIVE", "os_status": "ACTIVE",
|
||||
"task_state": "None", "host": "CA1-ESC8-040", "flavor": "n3-H100x8", "gpu_count": "8",
|
||||
"floating_ip": "69.19.140.110", "created": "2026-07-01 10:00:00 UTC", "ssh_text": "reachable",
|
||||
"ssh_raw": "reachable", "volumes_summary": "None", "openstack_fault": "None",
|
||||
"faults": [], "ih_events": [], "ih_events_all": [], "mismatches": [], "warn_reasons": [],
|
||||
"info_notes": [], "org_value": "8463 - Acme Corp", "owners": ["Lars <lars@simli.com>"],
|
||||
"server": {"status": "ACTIVE"}, "infrahub": {"floating_ip": "69.19.140.110"}}
|
||||
base.update(over); return base
|
||||
|
||||
def alert(name, **labels):
|
||||
return A.from_labels({"alertname": name, **labels})
|
||||
|
||||
def run(name, labels, vmdata=None, hostdata=None, health=None, census=None, fevent=None, prom=None):
|
||||
STUB.update({"vm": vmdata or {}, "host": hostdata or {}, "host_health": health or HEALTHY,
|
||||
"gpu_census": census or {}, "failed_event": fevent or {}})
|
||||
return runbooks.diagnose(alert(name, **labels), prom)
|
||||
|
||||
def check(label, cond, extra=""):
|
||||
print(f" {'PASS' if cond else 'FAIL'} {label}" + (f" <- {extra}" if not cond and extra else ""))
|
||||
return cond
|
||||
|
||||
fails = 0
|
||||
def expect(label, cond, extra=""):
|
||||
global fails
|
||||
if not check(label, cond, extra): fails += 1
|
||||
|
||||
L_ERR = dict(openstack_id="9ec7a021-d741-484f-9387-4eaf8879fd77", region="CANADA-1",
|
||||
instance_name="test-vm", organization="8463 - Acme Corp", status="ERROR")
|
||||
|
||||
print("\n[1] ERROR - creation failed, never reached ACTIVE, insufficient stock")
|
||||
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
|
||||
vm(host="N/A", ih_status="ERROR", os_status="ERROR", server={"status": "ERROR"},
|
||||
openstack_fault="No valid host was found. There are not enough hosts available"))
|
||||
expect("matched no_valid_host fault", "scheduler could not place" in d.verdict.lower(), d.verdict)
|
||||
expect("identified as never-ACTIVE", any("never placed on a host" in f.value for f in d.findings))
|
||||
expect("chose the insufficient-stock template", any(x.template_id == "error_never_active" for x in d.drafts),
|
||||
[x.template_id for x in d.drafts])
|
||||
expect("mentions 7-day outreach window", any("7 days" in x.when for x in d.drafts))
|
||||
expect("escalates to Infrastructure", any(a.owner == runbooks.INFRA for a in d.actions))
|
||||
expect("contacts resolved", d.contacts.get("resolved"))
|
||||
|
||||
print("\n[2] ERROR - was ACTIVE, stale LVM on host")
|
||||
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
|
||||
vm(ih_status="ERROR", os_status="ERROR",
|
||||
faults=[["500", "Build of instance aborted: Failed to remove volume(s): lvremove -f /dev/nova-vg/x_disk", "2026-07-29"]]))
|
||||
expect("matched lvremove fault", "stale LVM" in d.verdict, d.verdict)
|
||||
expect("identified as previously ACTIVE", any("hypervisor is recorded" in f.value for f in d.findings))
|
||||
expect("chose the was-ACTIVE template", any(x.template_id == "error_was_active" for x in d.drafts),
|
||||
[x.template_id for x in d.drafts])
|
||||
expect("assessment flags possible customer data", "customer data" in d.assessment)
|
||||
|
||||
print("\n[3] ERROR - NUMA/PCI fault, host is FULL (proves host fine)")
|
||||
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
|
||||
vm(ih_status="ERROR", os_status="ERROR",
|
||||
openstack_fault="Insufficient compute resources: Requested instance NUMA topology together with requested PCI devices cannot fit the given host NUMA topology; Claim pci failed."),
|
||||
census={"ok": True, "total_gpus": 8, "instances": [{"name":"a"}]*4})
|
||||
expect("ran the GPU census", any("GPUs allocated on host" in f.label for f in d.findings))
|
||||
expect("concluded host is FULL", any("FULL" in f.detail for f in d.findings))
|
||||
expect("marked the capacity check as already done", any(a.status == "done" for a in d.actions))
|
||||
|
||||
print("\n[3b] same fault, host NOT full -> must escalate")
|
||||
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
|
||||
vm(ih_status="ERROR", os_status="ERROR",
|
||||
openstack_fault="Claim pci failed."),
|
||||
census={"ok": True, "total_gpus": 4, "instances": [{"name":"a"}]*2})
|
||||
expect("raises a Jira for Infra", any("Jira" in a.text and a.owner == runbooks.INFRA for a in d.actions))
|
||||
|
||||
print("\n[4] SHUTOFF - billing notice only")
|
||||
d = run("Instance in SHUTOFF state in :flag-ca:CA-1 for greater than 30 min",
|
||||
dict(openstack_id="dc156762-3fa8-481d-89e0-12c2fb55e046", region="CANADA-1",
|
||||
instance_name="energetic-galileo", organization="27358 - Zyra", status="SHUTOFF"),
|
||||
vm(name="energetic-galileo", ih_status="SHUTOFF", os_status="SHUTOFF", server={"status":"SHUTOFF"}))
|
||||
expect("verdict is customer-initiated", "customer-initiated" in d.verdict.lower(), d.verdict)
|
||||
expect("used the shutoff snippet", any(x.template_id == "shutoff" for x in d.drafts))
|
||||
expect("substituted the VM name", any("energetic-galileo" in x.body for x in d.drafts))
|
||||
expect("no Infra escalation", not any(a.owner == runbooks.INFRA for a in d.actions))
|
||||
|
||||
print("\n[5] DELETING - server still in OpenStack, delete request found")
|
||||
d = run("Instance in DELETING state in :flag-ca:CA-1 for greater than 30 min",
|
||||
dict(openstack_id="13a855a4-c563-4350-9a8e-ffa9efa27d9e", region="CANADA-1",
|
||||
instance_name="external-hs-h100", organization="10420 - Inceptions AI", status="DELETING"),
|
||||
vm(ih_status="DELETING", os_status="ACTIVE", server={"status": "ACTIVE"},
|
||||
ih_events_all=[["2026-07-29 21:40:00", "InstanceDeleteRequest", "Delete Instance Request Sent."]]))
|
||||
expect("confirmed customer intent from events", any(a.status == "done" and "intent" in a.text for a in d.actions))
|
||||
expect("tells CX to delete in OpenStack", any("Delete the server in OpenStack" in a.text for a in d.actions))
|
||||
expect("offers both the notice and the ticket-closing reply",
|
||||
sorted(x.template_id for x in d.drafts) == ["deleting", "deleting_resolved"],
|
||||
[x.template_id for x in d.drafts])
|
||||
expect("tells CX to close the Infrahub record too, not just the server",
|
||||
any("InfraInsight" in a.text for a in d.actions), [a.text for a in d.actions])
|
||||
|
||||
print("\n[5c] DELETING - server exists but never reached a host")
|
||||
d = run("Instance in DELETING state in :flag-ca:CA-1 for greater than 30 min",
|
||||
dict(openstack_id="x", region="CANADA-1", status="DELETING"),
|
||||
vm(ih_status="DELETING", os_status="ERROR", host="N/A",
|
||||
server={"status": "ERROR"}, openstack_fault="No valid host was found."))
|
||||
expect("flags that the build never completed", any("build never completed" in f.value for f in d.findings))
|
||||
expect("still requires the InfraInsight close-out", any("InfraInsight" in a.text for a in d.actions))
|
||||
|
||||
print("\n[5b] DELETING - already gone from OpenStack, no delete event")
|
||||
d = run("Instance in DELETING state in :flag-ca:CA-1 for greater than 30 min",
|
||||
dict(openstack_id="x", region="CANADA-1", status="DELETING"),
|
||||
vm(ih_status="DELETING", os_status="N/A", server={}, ih_events_all=[]))
|
||||
expect("notes the server is already gone", any("already gone" in f.value for f in d.findings))
|
||||
expect("routes attribution to DevOps", any(a.owner == runbooks.DEVOPS for a in d.actions))
|
||||
|
||||
print("\n[6] CREATING - never got an OpenStack ID")
|
||||
d = run("Instance in CREATING state in :flag-ca:CA-1 for greater than 30min",
|
||||
dict(openstack_id="None", region="CANADA-1", instance_name="vm-28070520-5d1f2",
|
||||
organization="5574 - Nexgen", status="CREATING"),
|
||||
vm(openstack_id="N/A", ih_status="CREATING", os_status="N/A", server={}, host="N/A"))
|
||||
expect("verdict says never got an OpenStack ID", "never got an OpenStack ID" in d.verdict, d.verdict)
|
||||
expect("used the creating template", any(x.template_id == "creating" for x in d.drafts))
|
||||
expect("instructs deletion", any("Delete the stuck instance" in a.text for a in d.actions))
|
||||
|
||||
print("\n[7] HIBERNATING - sick host")
|
||||
d = run("Instance in HIBERNATING state in :flag-ca:CA-1 for greater than 120min",
|
||||
dict(openstack_id="28171e6f", region="CANADA-1", instance="CA1-ESC812-211",
|
||||
instance_name="apt25-prod", status="HIBERNATING"),
|
||||
vm(ih_status="HIBERNATING", os_status="ACTIVE"), health=SICK)
|
||||
expect("verdict blames the host", "host problem" in d.verdict, d.verdict)
|
||||
expect("escalates to Infra with the bad signals", any(a.owner == runbooks.INFRA and "OVS" in a.text for a in d.actions))
|
||||
expect("still drives the shelve", any("shelve" in a.text.lower() for a in d.actions))
|
||||
|
||||
print("\n[8] Suspected Rogue VM - host with two different mismatches")
|
||||
host_result = {"ok": True, "mode": "host", "host": "CA1-ESC8-068", "region": "ca1", "server_count": 3,
|
||||
"hypervisor": {"state": "up", "status": "enabled"}, "ovs": {"alive": True, "state": "UP"},
|
||||
"instances": [
|
||||
{"idx": 1, "name": "vm-hib-shutoff", "infrahub_id": "1", "openstack_id": "a", "ih_status": "HIBERNATED",
|
||||
"os_status": "SHUTOFF", "host": "CA1-ESC8-068", "mismatches": [{"check": "State", "detail": "IH HIBERNATED vs OS SHUTOFF"}],
|
||||
"warn_reasons": ["mismatch detected"], "org_value": "1 - A", "owners": ["a@x.com"]},
|
||||
{"idx": 2, "name": "vm-hib-active", "infrahub_id": "2", "openstack_id": "b", "ih_status": "HIBERNATED",
|
||||
"os_status": "ACTIVE", "host": "CA1-ESC8-068", "mismatches": [{"check": "State", "detail": "IH HIBERNATED vs OS ACTIVE"}],
|
||||
"warn_reasons": ["mismatch detected"], "org_value": "2 - B", "owners": ["b@x.com"]},
|
||||
{"idx": 3, "name": "tempest-thing", "tempest": True, "ih_status": "N/A", "os_status": "ACTIVE",
|
||||
"mismatches": [{"check": "Infrahub Missing", "detail": "not in Infrahub"}], "warn_reasons": []},
|
||||
]}
|
||||
d = run(":ninja:Suspected Rogue VM", dict(instance="CA1-ESC8-068"), hostdata=host_result)
|
||||
expect("counted 2 of 3 as mismatched", "2 of 3" in d.verdict, d.verdict)
|
||||
expect("ignored the tempest instance", d.evidence.get("ignored_tempest") == 1)
|
||||
expect("HIBERNATED/SHUTOFF -> Windmill stale-image cleanup", any("Windmill" in a.text for a in d.actions))
|
||||
expect("HIBERNATED/ACTIVE -> sync-error comms", any(x.template_id == "sync_state" for x in d.drafts))
|
||||
expect("actions are scoped per instance", any(a.text.startswith("[vm-hib-active]") for a in d.actions))
|
||||
|
||||
print("\n[9] Suspected Rogue VM - clean host")
|
||||
d = run(":ninja:Suspected Rogue VM", dict(instance="CA1-ESC8-068"),
|
||||
hostdata={**host_result, "instances": [{"idx":1,"name":"ok","ih_status":"ACTIVE","os_status":"ACTIVE",
|
||||
"mismatches": [], "warn_reasons": []}]})
|
||||
expect("verdict says no mismatch", "No Infrahub/OpenStack mismatch" in d.verdict, d.verdict)
|
||||
expect("redirects to the InfraInsight host query", any("InfraInsight" in a.text for a in d.actions))
|
||||
|
||||
print("\n[10] Duplicated IPs - one DELETING claimant, one with a wrong Infrahub IP")
|
||||
multi = {"ok": True, "mode": "multi_vm", "instances": [
|
||||
{"name": "old-vm", "infrahub_id": "1", "openstack_id": "a", "ih_status": "DELETING", "os_status": "ACTIVE",
|
||||
"server": {"status": "ACTIVE", "_public_ip": "69.19.137.135"}, "infrahub": {"floating_ip": "69.19.137.135"},
|
||||
"org_value": "1 - A", "owners": ["a@x.com"]},
|
||||
{"name": "new-vm", "infrahub_id": "2", "openstack_id": "b", "ih_status": "ACTIVE", "os_status": "ACTIVE",
|
||||
"server": {"status": "ACTIVE", "_public_ip": "69.19.140.9"}, "infrahub": {"floating_ip": "69.19.137.135"},
|
||||
"org_value": "2 - B", "owners": ["b@x.com"]},
|
||||
]}
|
||||
class FakeProm:
|
||||
def resources_by_floating_ip(self, fip):
|
||||
return [{"instance_name": "preprod-vm", "status": "ACTIVE", "region": "CANADA-1", "environment": "preprod"}]
|
||||
d = run(":awkward:Duplicated IPs", dict(floating_ip="69.19.137.135"), vmdata=multi, prom=FakeProm())
|
||||
expect("found 2 claimants needing correction", "2 of 2" in d.verdict, d.verdict)
|
||||
expect("DELETING claimant -> delete it", any("[old-vm]" in a.text and "stuck DELETING" in a.text for a in d.actions))
|
||||
expect("wrong-IP claimant -> Scenario #2", any("Scenario #2" in f.value for f in d.findings))
|
||||
expect("Scenario #2 comms filled with the real IP", any(
|
||||
x.template_id == "dupip_corrected" and "69.19.140.9" in x.body for x in d.drafts))
|
||||
expect("an unset sign-off name is reported rather than left as a placeholder", any(
|
||||
"AGENT_NAME" in x.unfilled for x in d.drafts if x.template_id == "dupip_corrected"))
|
||||
expect("surfaced the PreProd claimant from Prometheus", any("preprod-vm" in f.label for f in d.findings))
|
||||
expect("asks for a re-check after 5-10 min", any("5-10 minutes" in a.text for a in d.actions))
|
||||
|
||||
print("\n[11] Problem with Total GPUs - customers on host")
|
||||
d = run("Problem with Total GPUs in a System", dict(instance="CA1-ESC8-111", region="CANADA-1", gpu_name="B200-SXM"),
|
||||
census={"ok": True, "total_gpus": 6, "instances": [
|
||||
{"name": "cust-vm", "status": "ACTIVE", "flavor": "n3-B200x6", "gpus": "6", "openstack_id": "z"}]})
|
||||
expect("verdict names the host", "CA1-ESC8-111" in d.verdict, d.verdict)
|
||||
expect("lists the affected instance", any("cust-vm" in f.label for f in d.findings))
|
||||
expect("flags host-maintenance comms", any(a.kind == "comms" for a in d.actions))
|
||||
expect("escalates a Jira to Infra", any("Jira" in a.text and a.owner == runbooks.INFRA for a in d.actions))
|
||||
expect("notes there is no approved template", any("no approved customer template" in n for n in d.notes))
|
||||
expect("drafts nothing", not d.drafts)
|
||||
|
||||
print("\n[12] K8s instance name is flagged")
|
||||
d = run("Instance in ERROR state in :flag-ca:CA-1", {**L_ERR, "instance_name": "hyperstack-minion-lydhjev"},
|
||||
vm(host="N/A", ih_status="ERROR", os_status="ERROR"))
|
||||
expect("noted the likely K8s node", any("Kubernetes" in n for n in d.notes))
|
||||
|
||||
|
||||
print("\n[13] GPU sockets - every physical GPU accounted for")
|
||||
from triagelib.runbooks import _gpu_slots
|
||||
import collections as _c
|
||||
|
||||
R289 = [{"name": "luminous-hubble", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"},
|
||||
{"name": "vm832adbe203242", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"},
|
||||
{"name": "noble-maxwell", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"},
|
||||
{"name": "clever-schrodinger", "gpus": "2", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"}]
|
||||
|
||||
# Real numbers from CA1-ESC812-289: 8 physical, 7 reported in use, 4 VMs on 5 GPUs.
|
||||
s = _gpu_slots({"physical": 8, "in_use_metric": 7, "spare_capacity_artifact": False}, R289)
|
||||
c = _c.Counter(x["kind"] for x in s)
|
||||
expect("8 sockets drawn for an 8-GPU host", len(s) == 8, len(s))
|
||||
expect("5 named + 2 unaccounted + 1 free", (c["vm"], c["unaccounted"], c["free"]) == (5, 2, 1), dict(c))
|
||||
|
||||
s = _gpu_slots({"physical": 8, "in_use_metric": 8, "spare_capacity_artifact": True}, R289)
|
||||
c = _c.Counter(x["kind"] for x in s)
|
||||
expect("artifact host shows spare sockets as free, not unaccounted",
|
||||
(c["vm"], c["unaccounted"], c["free"]) == (5, 0, 3), dict(c))
|
||||
|
||||
s = _gpu_slots({"physical": None, "in_use_metric": 2, "spare_capacity_artifact": False},
|
||||
[{"name": "basilica", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"}])
|
||||
c = _c.Counter(x["kind"] for x in s)
|
||||
expect("unknown socket count degrades gracefully", (c["vm"], c["unaccounted"]) == (1, 1), dict(c))
|
||||
|
||||
s = _gpu_slots({"physical": 8, "in_use_metric": 8, "spare_capacity_artifact": False},
|
||||
[{"name": "ghost", "gpus": "8", "linked": False, "match": False,
|
||||
"ih_status": "not in Infrahub", "os_status": "ACTIVE"}])
|
||||
expect("a VM with no Infrahub record still fills its sockets and is flagged",
|
||||
len(s) == 8 and all(x["kind"] == "vm" and not x["linked"] for x in s))
|
||||
|
||||
s = _gpu_slots({"physical": 8, "in_use_metric": 5, "spare_capacity_artifact": False}, R289)
|
||||
expect("no negative slots when in_use is below what VMs claim", len(s) >= 5 and all(
|
||||
x["kind"] in ("vm", "free", "unaccounted") for x in s), len(s))
|
||||
|
||||
print(f"\n{'ALL CHECKS PASSED' if not fails else str(fails) + ' CHECK(S) FAILED'}")
|
||||
sys.exit(1 if fails else 0)
|
||||
Reference in New Issue
Block a user