Parham Monfared
1262690276
The single-file stdlib server became the limit: no way to track what had been done about an alert, no accounts, and a UI that had to be hand-rolled in template strings. This restructures it into something deployable. Backend (FastAPI) - app/ holds config, database, auth, delivery and the routers; triagelib keeps the triage engine unchanged, so the validated screening and runbook logic is untouched. - Cases persist per alert fingerprint with a status workflow (investigating, customer contacted, escalated to Infra, waiting, remediated, resolved, won't fix, false positive), an assignee, notes and an append-only history. An alert that stops and re-fires lands back on the same case and counts as a reopen. - Suppression rules move from a JSON file into the database. Auth - Signed session cookies over PBKDF2 local accounts, plus an OIDC flow ready for Authentik: users are created on first login and admin follows a group claim. Local login can be switched off entirely once SSO is live. Zendesk and Jira - Delivery is now implemented, behind three gates: the integration must be configured, its feature flag on, and CX_FEATURE_SEND_ENABLED on. A demo instance leaves the last off and cannot mail anyone. Both search before creating, so re-diagnosing an alert updates one ticket rather than opening several, and a rolling daily cap stops a loop mailing everybody. Deployment - Multi-stage Dockerfile builds the bundle and serves it from the API origin. - docker-compose for local and single-host use; Gitea Actions runs the tests, builds the image and renders deploy/k8s with envsubst. Two fixes found while testing: assigning a case returned a null assignee, and add_event could leave an already-loaded history collection stale. Known gap: the engine reaches OpenStack via `docker exec <region>-osc`, which does not work in a pod without the CX-Tools containers alongside it. docs/DEPLOYMENT.md sets out the three ways to close that. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
269 lines
18 KiB
Python
269 lines
18 KiB
Python
"""Runbook decision tests: fixtures shaped like real CX-Tools collector output."""
|
|
import sys, os
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
|
|
from triagelib import alerts as A, cxbridge, runbooks
|
|
|
|
# --- stub the CX-Tools boundary so the decision logic can be tested alone ---
|
|
class FakeCx:
|
|
def status_pair_ok(self, ih, os_, task):
|
|
ih, os_ = (ih or "").upper(), (os_ or "").upper()
|
|
if ih == os_: return True
|
|
return (ih, os_) in {("HIBERNATED","SHELVED_OFFLOADED"), ("REBOOTING","HARD_REBOOT")}
|
|
def mismatch_parts(self, item):
|
|
return str(item.get("check","Mismatch")), str(item.get("detail",""))
|
|
def normalize_empty(self, v): return "" if v in (None,"","None") else str(v)
|
|
def first_present(self, m, *keys, default=""):
|
|
for k in keys:
|
|
if isinstance(m, dict) and k in m: return m[k]
|
|
return default
|
|
def public_ip_from_server(self, s): return (s or {}).get("_public_ip","")
|
|
def json_safe(self, o): return o
|
|
def map_region(self, r): return {"CANADA-1":"ca1","US-1":"us1","NORWAY-1":"no1"}.get(r,"")
|
|
|
|
STUB = {"host_health": {}, "gpu_census": {}, "failed_event": {}, "vm": {}, "host": {}}
|
|
cxbridge.cx = lambda: FakeCx()
|
|
cxbridge.host_health = lambda region, host: STUB["host_health"]
|
|
cxbridge.host_gpu_census = lambda region, host: STUB["gpu_census"]
|
|
cxbridge.failed_openstack_event = lambda region, osid, scan=5: STUB["failed_event"]
|
|
cxbridge.collect_vm = lambda t, **k: STUB["vm"]
|
|
cxbridge.collect_host = lambda h, **k: STUB["host"]
|
|
cxbridge.json_safe = lambda o: o
|
|
|
|
HEALTHY = {"ok": True, "nova_state": "up", "nova_status": "enabled", "ovs_alive": True,
|
|
"ovs_state": "UP", "uptime": "17:54", "aggregates": "agg", "bad_signals": []}
|
|
SICK = {**HEALTHY, "nova_state": "down", "ovs_state": "DOWN", "ovs_alive": False,
|
|
"bad_signals": ["Nova state is down", "OVS state is DOWN"]}
|
|
|
|
def vm(**over):
|
|
base = {"ok": True, "exit_code": 0, "mode": "vm", "infrahub_id": "123456",
|
|
"openstack_id": "9ec7a021-d741-484f-9387-4eaf8879fd77", "name": "test-vm",
|
|
"region": "ca1", "region_display": "CANADA-1", "ih_status": "ACTIVE", "os_status": "ACTIVE",
|
|
"task_state": "None", "host": "CA1-ESC8-040", "flavor": "n3-H100x8", "gpu_count": "8",
|
|
"floating_ip": "69.19.140.110", "created": "2026-07-01 10:00:00 UTC", "ssh_text": "reachable",
|
|
"ssh_raw": "reachable", "volumes_summary": "None", "openstack_fault": "None",
|
|
"faults": [], "ih_events": [], "ih_events_all": [], "mismatches": [], "warn_reasons": [],
|
|
"info_notes": [], "org_value": "8463 - Acme Corp", "owners": ["Lars <lars@simli.com>"],
|
|
"server": {"status": "ACTIVE"}, "infrahub": {"floating_ip": "69.19.140.110"}}
|
|
base.update(over); return base
|
|
|
|
def alert(name, **labels):
|
|
return A.from_labels({"alertname": name, **labels})
|
|
|
|
def run(name, labels, vmdata=None, hostdata=None, health=None, census=None, fevent=None, prom=None):
|
|
STUB.update({"vm": vmdata or {}, "host": hostdata or {}, "host_health": health or HEALTHY,
|
|
"gpu_census": census or {}, "failed_event": fevent or {}})
|
|
return runbooks.diagnose(alert(name, **labels), prom)
|
|
|
|
def check(label, cond, extra=""):
|
|
print(f" {'PASS' if cond else 'FAIL'} {label}" + (f" <- {extra}" if not cond and extra else ""))
|
|
return cond
|
|
|
|
fails = 0
|
|
def expect(label, cond, extra=""):
|
|
global fails
|
|
if not check(label, cond, extra): fails += 1
|
|
|
|
L_ERR = dict(openstack_id="9ec7a021-d741-484f-9387-4eaf8879fd77", region="CANADA-1",
|
|
instance_name="test-vm", organization="8463 - Acme Corp", status="ERROR")
|
|
|
|
print("\n[1] ERROR - creation failed, never reached ACTIVE, insufficient stock")
|
|
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
|
|
vm(host="N/A", ih_status="ERROR", os_status="ERROR", server={"status": "ERROR"},
|
|
openstack_fault="No valid host was found. There are not enough hosts available"))
|
|
expect("matched no_valid_host fault", "scheduler could not place" in d.verdict.lower(), d.verdict)
|
|
expect("identified as never-ACTIVE", any("never placed on a host" in f.value for f in d.findings))
|
|
expect("chose the insufficient-stock template", any(x.template_id == "error_never_active" for x in d.drafts),
|
|
[x.template_id for x in d.drafts])
|
|
expect("mentions 7-day outreach window", any("7 days" in x.when for x in d.drafts))
|
|
expect("escalates to Infrastructure", any(a.owner == runbooks.INFRA for a in d.actions))
|
|
expect("contacts resolved", d.contacts.get("resolved"))
|
|
|
|
print("\n[2] ERROR - was ACTIVE, stale LVM on host")
|
|
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
|
|
vm(ih_status="ERROR", os_status="ERROR",
|
|
faults=[["500", "Build of instance aborted: Failed to remove volume(s): lvremove -f /dev/nova-vg/x_disk", "2026-07-29"]]))
|
|
expect("matched lvremove fault", "stale LVM" in d.verdict, d.verdict)
|
|
expect("identified as previously ACTIVE", any("hypervisor is recorded" in f.value for f in d.findings))
|
|
expect("chose the was-ACTIVE template", any(x.template_id == "error_was_active" for x in d.drafts),
|
|
[x.template_id for x in d.drafts])
|
|
expect("assessment flags possible customer data", "customer data" in d.assessment)
|
|
|
|
print("\n[3] ERROR - NUMA/PCI fault, host is FULL (proves host fine)")
|
|
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
|
|
vm(ih_status="ERROR", os_status="ERROR",
|
|
openstack_fault="Insufficient compute resources: Requested instance NUMA topology together with requested PCI devices cannot fit the given host NUMA topology; Claim pci failed."),
|
|
census={"ok": True, "total_gpus": 8, "instances": [{"name":"a"}]*4})
|
|
expect("ran the GPU census", any("GPUs allocated on host" in f.label for f in d.findings))
|
|
expect("concluded host is FULL", any("FULL" in f.detail for f in d.findings))
|
|
expect("marked the capacity check as already done", any(a.status == "done" for a in d.actions))
|
|
|
|
print("\n[3b] same fault, host NOT full -> must escalate")
|
|
d = run("Instance in ERROR state in :flag-ca:CA-1", L_ERR,
|
|
vm(ih_status="ERROR", os_status="ERROR",
|
|
openstack_fault="Claim pci failed."),
|
|
census={"ok": True, "total_gpus": 4, "instances": [{"name":"a"}]*2})
|
|
expect("raises a Jira for Infra", any("Jira" in a.text and a.owner == runbooks.INFRA for a in d.actions))
|
|
|
|
print("\n[4] SHUTOFF - billing notice only")
|
|
d = run("Instance in SHUTOFF state in :flag-ca:CA-1 for greater than 30 min",
|
|
dict(openstack_id="dc156762-3fa8-481d-89e0-12c2fb55e046", region="CANADA-1",
|
|
instance_name="energetic-galileo", organization="27358 - Zyra", status="SHUTOFF"),
|
|
vm(name="energetic-galileo", ih_status="SHUTOFF", os_status="SHUTOFF", server={"status":"SHUTOFF"}))
|
|
expect("verdict is customer-initiated", "customer-initiated" in d.verdict.lower(), d.verdict)
|
|
expect("used the shutoff snippet", any(x.template_id == "shutoff" for x in d.drafts))
|
|
expect("substituted the VM name", any("energetic-galileo" in x.body for x in d.drafts))
|
|
expect("no Infra escalation", not any(a.owner == runbooks.INFRA for a in d.actions))
|
|
|
|
print("\n[5] DELETING - server still in OpenStack, delete request found")
|
|
d = run("Instance in DELETING state in :flag-ca:CA-1 for greater than 30 min",
|
|
dict(openstack_id="13a855a4-c563-4350-9a8e-ffa9efa27d9e", region="CANADA-1",
|
|
instance_name="external-hs-h100", organization="10420 - Inceptions AI", status="DELETING"),
|
|
vm(ih_status="DELETING", os_status="ACTIVE", server={"status": "ACTIVE"},
|
|
ih_events_all=[["2026-07-29 21:40:00", "InstanceDeleteRequest", "Delete Instance Request Sent."]]))
|
|
expect("confirmed customer intent from events", any(a.status == "done" and "intent" in a.text for a in d.actions))
|
|
expect("tells CX to delete in OpenStack", any("Delete the server in OpenStack" in a.text for a in d.actions))
|
|
expect("offers both the notice and the ticket-closing reply",
|
|
sorted(x.template_id for x in d.drafts) == ["deleting", "deleting_resolved"],
|
|
[x.template_id for x in d.drafts])
|
|
expect("tells CX to close the Infrahub record too, not just the server",
|
|
any("InfraInsight" in a.text for a in d.actions), [a.text for a in d.actions])
|
|
|
|
print("\n[5c] DELETING - server exists but never reached a host")
|
|
d = run("Instance in DELETING state in :flag-ca:CA-1 for greater than 30 min",
|
|
dict(openstack_id="x", region="CANADA-1", status="DELETING"),
|
|
vm(ih_status="DELETING", os_status="ERROR", host="N/A",
|
|
server={"status": "ERROR"}, openstack_fault="No valid host was found."))
|
|
expect("flags that the build never completed", any("build never completed" in f.value for f in d.findings))
|
|
expect("still requires the InfraInsight close-out", any("InfraInsight" in a.text for a in d.actions))
|
|
|
|
print("\n[5b] DELETING - already gone from OpenStack, no delete event")
|
|
d = run("Instance in DELETING state in :flag-ca:CA-1 for greater than 30 min",
|
|
dict(openstack_id="x", region="CANADA-1", status="DELETING"),
|
|
vm(ih_status="DELETING", os_status="N/A", server={}, ih_events_all=[]))
|
|
expect("notes the server is already gone", any("already gone" in f.value for f in d.findings))
|
|
expect("routes attribution to DevOps", any(a.owner == runbooks.DEVOPS for a in d.actions))
|
|
|
|
print("\n[6] CREATING - never got an OpenStack ID")
|
|
d = run("Instance in CREATING state in :flag-ca:CA-1 for greater than 30min",
|
|
dict(openstack_id="None", region="CANADA-1", instance_name="vm-28070520-5d1f2",
|
|
organization="5574 - Nexgen", status="CREATING"),
|
|
vm(openstack_id="N/A", ih_status="CREATING", os_status="N/A", server={}, host="N/A"))
|
|
expect("verdict says never got an OpenStack ID", "never got an OpenStack ID" in d.verdict, d.verdict)
|
|
expect("used the creating template", any(x.template_id == "creating" for x in d.drafts))
|
|
expect("instructs deletion", any("Delete the stuck instance" in a.text for a in d.actions))
|
|
|
|
print("\n[7] HIBERNATING - sick host")
|
|
d = run("Instance in HIBERNATING state in :flag-ca:CA-1 for greater than 120min",
|
|
dict(openstack_id="28171e6f", region="CANADA-1", instance="CA1-ESC812-211",
|
|
instance_name="apt25-prod", status="HIBERNATING"),
|
|
vm(ih_status="HIBERNATING", os_status="ACTIVE"), health=SICK)
|
|
expect("verdict blames the host", "host problem" in d.verdict, d.verdict)
|
|
expect("escalates to Infra with the bad signals", any(a.owner == runbooks.INFRA and "OVS" in a.text for a in d.actions))
|
|
expect("still drives the shelve", any("shelve" in a.text.lower() for a in d.actions))
|
|
|
|
print("\n[8] Suspected Rogue VM - host with two different mismatches")
|
|
host_result = {"ok": True, "mode": "host", "host": "CA1-ESC8-068", "region": "ca1", "server_count": 3,
|
|
"hypervisor": {"state": "up", "status": "enabled"}, "ovs": {"alive": True, "state": "UP"},
|
|
"instances": [
|
|
{"idx": 1, "name": "vm-hib-shutoff", "infrahub_id": "1", "openstack_id": "a", "ih_status": "HIBERNATED",
|
|
"os_status": "SHUTOFF", "host": "CA1-ESC8-068", "mismatches": [{"check": "State", "detail": "IH HIBERNATED vs OS SHUTOFF"}],
|
|
"warn_reasons": ["mismatch detected"], "org_value": "1 - A", "owners": ["a@x.com"]},
|
|
{"idx": 2, "name": "vm-hib-active", "infrahub_id": "2", "openstack_id": "b", "ih_status": "HIBERNATED",
|
|
"os_status": "ACTIVE", "host": "CA1-ESC8-068", "mismatches": [{"check": "State", "detail": "IH HIBERNATED vs OS ACTIVE"}],
|
|
"warn_reasons": ["mismatch detected"], "org_value": "2 - B", "owners": ["b@x.com"]},
|
|
{"idx": 3, "name": "tempest-thing", "tempest": True, "ih_status": "N/A", "os_status": "ACTIVE",
|
|
"mismatches": [{"check": "Infrahub Missing", "detail": "not in Infrahub"}], "warn_reasons": []},
|
|
]}
|
|
d = run(":ninja:Suspected Rogue VM", dict(instance="CA1-ESC8-068"), hostdata=host_result)
|
|
expect("counted 2 of 3 as mismatched", "2 of 3" in d.verdict, d.verdict)
|
|
expect("ignored the tempest instance", d.evidence.get("ignored_tempest") == 1)
|
|
expect("HIBERNATED/SHUTOFF -> Windmill stale-image cleanup", any("Windmill" in a.text for a in d.actions))
|
|
expect("HIBERNATED/ACTIVE -> sync-error comms", any(x.template_id == "sync_state" for x in d.drafts))
|
|
expect("actions are scoped per instance", any(a.text.startswith("[vm-hib-active]") for a in d.actions))
|
|
|
|
print("\n[9] Suspected Rogue VM - clean host")
|
|
d = run(":ninja:Suspected Rogue VM", dict(instance="CA1-ESC8-068"),
|
|
hostdata={**host_result, "instances": [{"idx":1,"name":"ok","ih_status":"ACTIVE","os_status":"ACTIVE",
|
|
"mismatches": [], "warn_reasons": []}]})
|
|
expect("verdict says no mismatch", "No Infrahub/OpenStack mismatch" in d.verdict, d.verdict)
|
|
expect("redirects to the InfraInsight host query", any("InfraInsight" in a.text for a in d.actions))
|
|
|
|
print("\n[10] Duplicated IPs - one DELETING claimant, one with a wrong Infrahub IP")
|
|
multi = {"ok": True, "mode": "multi_vm", "instances": [
|
|
{"name": "old-vm", "infrahub_id": "1", "openstack_id": "a", "ih_status": "DELETING", "os_status": "ACTIVE",
|
|
"server": {"status": "ACTIVE", "_public_ip": "69.19.137.135"}, "infrahub": {"floating_ip": "69.19.137.135"},
|
|
"org_value": "1 - A", "owners": ["a@x.com"]},
|
|
{"name": "new-vm", "infrahub_id": "2", "openstack_id": "b", "ih_status": "ACTIVE", "os_status": "ACTIVE",
|
|
"server": {"status": "ACTIVE", "_public_ip": "69.19.140.9"}, "infrahub": {"floating_ip": "69.19.137.135"},
|
|
"org_value": "2 - B", "owners": ["b@x.com"]},
|
|
]}
|
|
class FakeProm:
|
|
def resources_by_floating_ip(self, fip):
|
|
return [{"instance_name": "preprod-vm", "status": "ACTIVE", "region": "CANADA-1", "environment": "preprod"}]
|
|
d = run(":awkward:Duplicated IPs", dict(floating_ip="69.19.137.135"), vmdata=multi, prom=FakeProm())
|
|
expect("found 2 claimants needing correction", "2 of 2" in d.verdict, d.verdict)
|
|
expect("DELETING claimant -> delete it", any("[old-vm]" in a.text and "stuck DELETING" in a.text for a in d.actions))
|
|
expect("wrong-IP claimant -> Scenario #2", any("Scenario #2" in f.value for f in d.findings))
|
|
expect("Scenario #2 comms filled with the real IP", any(
|
|
x.template_id == "dupip_corrected" and "69.19.140.9" in x.body for x in d.drafts))
|
|
expect("an unset sign-off name is reported rather than left as a placeholder", any(
|
|
"AGENT_NAME" in x.unfilled for x in d.drafts if x.template_id == "dupip_corrected"))
|
|
expect("surfaced the PreProd claimant from Prometheus", any("preprod-vm" in f.label for f in d.findings))
|
|
expect("asks for a re-check after 5-10 min", any("5-10 minutes" in a.text for a in d.actions))
|
|
|
|
print("\n[11] Problem with Total GPUs - customers on host")
|
|
d = run("Problem with Total GPUs in a System", dict(instance="CA1-ESC8-111", region="CANADA-1", gpu_name="B200-SXM"),
|
|
census={"ok": True, "total_gpus": 6, "instances": [
|
|
{"name": "cust-vm", "status": "ACTIVE", "flavor": "n3-B200x6", "gpus": "6", "openstack_id": "z"}]})
|
|
expect("verdict names the host", "CA1-ESC8-111" in d.verdict, d.verdict)
|
|
expect("lists the affected instance", any("cust-vm" in f.label for f in d.findings))
|
|
expect("flags host-maintenance comms", any(a.kind == "comms" for a in d.actions))
|
|
expect("escalates a Jira to Infra", any("Jira" in a.text and a.owner == runbooks.INFRA for a in d.actions))
|
|
expect("notes there is no approved template", any("no approved customer template" in n for n in d.notes))
|
|
expect("drafts nothing", not d.drafts)
|
|
|
|
print("\n[12] K8s instance name is flagged")
|
|
d = run("Instance in ERROR state in :flag-ca:CA-1", {**L_ERR, "instance_name": "hyperstack-minion-lydhjev"},
|
|
vm(host="N/A", ih_status="ERROR", os_status="ERROR"))
|
|
expect("noted the likely K8s node", any("Kubernetes" in n for n in d.notes))
|
|
|
|
|
|
print("\n[13] GPU sockets - every physical GPU accounted for")
|
|
from triagelib.runbooks import _gpu_slots
|
|
import collections as _c
|
|
|
|
R289 = [{"name": "luminous-hubble", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"},
|
|
{"name": "vm832adbe203242", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"},
|
|
{"name": "noble-maxwell", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"},
|
|
{"name": "clever-schrodinger", "gpus": "2", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"}]
|
|
|
|
# Real numbers from CA1-ESC812-289: 8 physical, 7 reported in use, 4 VMs on 5 GPUs.
|
|
s = _gpu_slots({"physical": 8, "in_use_metric": 7, "spare_capacity_artifact": False}, R289)
|
|
c = _c.Counter(x["kind"] for x in s)
|
|
expect("8 sockets drawn for an 8-GPU host", len(s) == 8, len(s))
|
|
expect("5 named + 2 unaccounted + 1 free", (c["vm"], c["unaccounted"], c["free"]) == (5, 2, 1), dict(c))
|
|
|
|
s = _gpu_slots({"physical": 8, "in_use_metric": 8, "spare_capacity_artifact": True}, R289)
|
|
c = _c.Counter(x["kind"] for x in s)
|
|
expect("artifact host shows spare sockets as free, not unaccounted",
|
|
(c["vm"], c["unaccounted"], c["free"]) == (5, 0, 3), dict(c))
|
|
|
|
s = _gpu_slots({"physical": None, "in_use_metric": 2, "spare_capacity_artifact": False},
|
|
[{"name": "basilica", "gpus": "1", "linked": True, "match": True, "ih_status": "ACTIVE", "os_status": "ACTIVE"}])
|
|
c = _c.Counter(x["kind"] for x in s)
|
|
expect("unknown socket count degrades gracefully", (c["vm"], c["unaccounted"]) == (1, 1), dict(c))
|
|
|
|
s = _gpu_slots({"physical": 8, "in_use_metric": 8, "spare_capacity_artifact": False},
|
|
[{"name": "ghost", "gpus": "8", "linked": False, "match": False,
|
|
"ih_status": "not in Infrahub", "os_status": "ACTIVE"}])
|
|
expect("a VM with no Infrahub record still fills its sockets and is flagged",
|
|
len(s) == 8 and all(x["kind"] == "vm" and not x["linked"] for x in s))
|
|
|
|
s = _gpu_slots({"physical": 8, "in_use_metric": 5, "spare_capacity_artifact": False}, R289)
|
|
expect("no negative slots when in_use is below what VMs claim", len(s) >= 5 and all(
|
|
x["kind"] in ("vm", "free", "unaccounted") for x in s), len(s))
|
|
|
|
print(f"\n{'ALL CHECKS PASSED' if not fails else str(fails) + ' CHECK(S) FAILED'}")
|
|
sys.exit(1 if fails else 0)
|