Parham Monfared
1262690276
The single-file stdlib server became the limit: no way to track what had been done about an alert, no accounts, and a UI that had to be hand-rolled in template strings. This restructures it into something deployable. Backend (FastAPI) - app/ holds config, database, auth, delivery and the routers; triagelib keeps the triage engine unchanged, so the validated screening and runbook logic is untouched. - Cases persist per alert fingerprint with a status workflow (investigating, customer contacted, escalated to Infra, waiting, remediated, resolved, won't fix, false positive), an assignee, notes and an append-only history. An alert that stops and re-fires lands back on the same case and counts as a reopen. - Suppression rules move from a JSON file into the database. Auth - Signed session cookies over PBKDF2 local accounts, plus an OIDC flow ready for Authentik: users are created on first login and admin follows a group claim. Local login can be switched off entirely once SSO is live. Zendesk and Jira - Delivery is now implemented, behind three gates: the integration must be configured, its feature flag on, and CX_FEATURE_SEND_ENABLED on. A demo instance leaves the last off and cannot mail anyone. Both search before creating, so re-diagnosing an alert updates one ticket rather than opening several, and a rolling daily cap stops a loop mailing everybody. Deployment - Multi-stage Dockerfile builds the bundle and serves it from the API origin. - docker-compose for local and single-host use; Gitea Actions runs the tests, builds the image and renders deploy/k8s with envsubst. Two fixes found while testing: assigning a case returned a null assignee, and add_event could leave an already-loaded history collection stale. Known gap: the engine reaches OpenStack via `docker exec <region>-osc`, which does not work in a pod without the CX-Tools containers alongside it. docs/DEPLOYMENT.md sets out the three ways to close that. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1498 lines
68 KiB
Python
1498 lines
68 KiB
Python
"""The diagnosis engine: CX runbooks expressed as decisions over CX-Tools data.
|
|
|
|
Each alert kind maps to one Confluence runbook. For every alert this module
|
|
gathers the evidence the runbook asks for (via the read-only CX-Tools
|
|
collectors), reaches the verdict the runbook's decision table implies, and emits
|
|
the remaining steps a human still has to perform. Nothing here mutates state.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Optional
|
|
|
|
from . import comms, cxbridge
|
|
from .alerts import Alert
|
|
|
|
CX = "CX"
|
|
INFRA = "Infrastructure team"
|
|
DEVOPS = "DevOps"
|
|
CUSTOMER = "Customer"
|
|
|
|
TRANSITIONAL = {"BUILD", "CREATING", "DELETING", "HIBERNATING", "RESTORING", "REBOOTING"}
|
|
|
|
|
|
# --- result model -----------------------------------------------------------
|
|
|
|
@dataclass
|
|
class Finding:
|
|
label: str
|
|
value: str
|
|
tone: str = "info" # ok | warn | bad | info
|
|
detail: str = ""
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {"label": self.label, "value": self.value, "tone": self.tone, "detail": self.detail}
|
|
|
|
|
|
@dataclass
|
|
class Action:
|
|
text: str
|
|
owner: str = CX
|
|
kind: str = "check" # check | remediate | escalate | comms | verify
|
|
guide: str = ""
|
|
status: str = "todo" # todo | done | skipped (done = the app verified it)
|
|
detail: str = ""
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"text": self.text, "owner": self.owner, "kind": self.kind,
|
|
"guide": self.guide, "status": self.status, "detail": self.detail,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class Diagnosis:
|
|
alert: Alert
|
|
verdict: str = ""
|
|
assessment: str = ""
|
|
confidence: str = "medium" # high | medium | low
|
|
findings: list[Finding] = field(default_factory=list)
|
|
actions: list[Action] = field(default_factory=list)
|
|
drafts: list[comms.Draft] = field(default_factory=list)
|
|
contacts: dict[str, Any] = field(default_factory=dict)
|
|
evidence: dict[str, Any] = field(default_factory=dict)
|
|
notes: list[str] = field(default_factory=list)
|
|
error: str = ""
|
|
# A small structured description of the problem, so the UI can draw it
|
|
# rather than make the reader parse a table.
|
|
visual: dict[str, Any] = field(default_factory=dict)
|
|
agent_name: str = ""
|
|
|
|
def add(self, label: str, value: Any, tone: str = "info", detail: str = "") -> None:
|
|
text = str(value if value not in (None, "") else "N/A")
|
|
self.findings.append(Finding(label, text, tone, detail))
|
|
|
|
def act(self, text: str, owner: str = CX, kind: str = "check", guide: str = "",
|
|
status: str = "todo", detail: str = "") -> None:
|
|
self.actions.append(Action(text, owner, kind, guide, status, detail))
|
|
|
|
def note(self, text: str) -> None:
|
|
if text and text not in self.notes:
|
|
self.notes.append(text)
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
from . import integrations
|
|
|
|
return {
|
|
"integrations": integrations.build_all(self),
|
|
"alert": self.alert.to_json(),
|
|
"verdict": self.verdict,
|
|
"assessment": self.assessment,
|
|
"confidence": self.confidence,
|
|
"findings": [f.to_json() for f in self.findings],
|
|
"actions": [a.to_json() for a in self.actions],
|
|
"drafts": [d.to_json() for d in self.drafts],
|
|
"contacts": self.contacts,
|
|
"evidence": self.evidence,
|
|
"notes": self.notes,
|
|
"error": self.error,
|
|
"visual": self.visual,
|
|
}
|
|
|
|
|
|
# --- the ERROR-state fault table -------------------------------------------
|
|
# Transcribed from the fault table in the "Instance in ERROR state" runbook.
|
|
|
|
FAULT_TABLE: list[dict[str, Any]] = [
|
|
{
|
|
"id": "conflict_image_pending_upload",
|
|
"pattern": r"Conflict updating instance.*task_state.*image_pending_upload",
|
|
"summary": "Instance was deleted while it was hibernating.",
|
|
"owner": f"{INFRA} / {CX}",
|
|
"guidance": "Known case: the instance was deleted mid-hibernation. It just needs deleting again in OpenStack.",
|
|
},
|
|
{
|
|
"id": "no_valid_host",
|
|
"pattern": r"No valid host was found|not enough hosts available",
|
|
"summary": "The scheduler could not place the instance - stock shortage or a race condition.",
|
|
"owner": INFRA,
|
|
"guidance": "Judge whether there was enough stock at the time; ask a peer if unsure. If there should have been "
|
|
"enough, try reproducing with the same flavor in pre-prod - otherwise it is most likely a race "
|
|
"condition from several near-simultaneous API creates.",
|
|
"comms": "error_never_active",
|
|
"comms_note": "Contact the customer if the instance is still not deleted after a day.",
|
|
},
|
|
{
|
|
"id": "numa_pci",
|
|
"pattern": r"NUMA topology together with requested PCI devices|Claim pci failed",
|
|
"summary": "NUMA/PCI claim failure on the host.",
|
|
"owner": INFRA,
|
|
"guidance": "Check whether the host is full BEFORE escalating. If the summed GPU count across the host's "
|
|
"instances is 8, the host is full and that proves the host itself is fine.",
|
|
"gpu_census": True,
|
|
},
|
|
{
|
|
"id": "lvremove",
|
|
"pattern": r"Failed to remove volume\(s\)|lvremove",
|
|
"summary": "A stale LVM volume on the host blocked the build.",
|
|
"owner": INFRA,
|
|
"guidance": "Raise a Jira for Infrastructure to investigate the host and the instance error.",
|
|
},
|
|
{
|
|
"id": "pci_header",
|
|
"pattern": r"Unknown PCI header type",
|
|
"summary": "Host PCI device is reporting an invalid header - hardware/host fault.",
|
|
"owner": INFRA,
|
|
"guidance": "Raise a Jira for Infrastructure to investigate the host and the instance error.",
|
|
},
|
|
{
|
|
"id": "pci_in_use",
|
|
"pattern": r"PCI device \S+ is in use by driver QEMU",
|
|
"summary": "The GPU is still claimed by another domain on the host.",
|
|
"owner": INFRA,
|
|
"guidance": "Raise a Jira for Infrastructure to investigate the host and the instance error.",
|
|
},
|
|
{
|
|
"id": "client_socket_closed",
|
|
"pattern": r"internal error: client socket is closed",
|
|
"summary": "libvirt client socket closed during the operation.",
|
|
"owner": INFRA,
|
|
"guidance": "Can be transient, but it is associated with host issues - escalate to be safe.",
|
|
},
|
|
]
|
|
|
|
|
|
def match_faults(texts: list[str]) -> list[dict[str, Any]]:
|
|
"""Match collected fault text against the runbook fault table."""
|
|
blob = "\n".join(t for t in texts if t)
|
|
matched: list[dict[str, Any]] = []
|
|
for entry in FAULT_TABLE:
|
|
if re.search(entry["pattern"], blob, re.I | re.S):
|
|
matched.append(entry)
|
|
return matched
|
|
|
|
|
|
# --- shared evidence gathering ---------------------------------------------
|
|
|
|
def _vm_target(alert: Alert) -> tuple[str, Optional[str]]:
|
|
"""Pick the best CX-Tools lookup target for this alert.
|
|
|
|
OpenStack ID is preferred. CREATING alerts routinely carry
|
|
openstack_id="None" because the VM never reached OpenStack, so those fall
|
|
back to the instance name plus the org id parsed from the alert.
|
|
"""
|
|
if alert.openstack_id:
|
|
return alert.openstack_id, None
|
|
if alert.instance_name and alert.org_id:
|
|
return alert.instance_name, alert.org_id
|
|
return alert.instance_name, None
|
|
|
|
|
|
def _tone_for_status_pair(ih_status: str, os_status: str, task_state: str) -> str:
|
|
try:
|
|
ok = cxbridge.cx().status_pair_ok(ih_status, os_status, task_state)
|
|
except Exception:
|
|
ok = ih_status.upper() == os_status.upper()
|
|
return "ok" if ok else "bad"
|
|
|
|
|
|
def _add_vm_findings(d: Diagnosis, vm: dict[str, Any]) -> None:
|
|
ih_status = str(vm.get("ih_status") or "N/A")
|
|
os_status = str(vm.get("os_status") or "N/A")
|
|
task_state = str(vm.get("task_state") or "None")
|
|
host = str(vm.get("host") or "N/A")
|
|
|
|
d.visual = {
|
|
"type": "states",
|
|
"infrahub": ih_status,
|
|
"openstack": os_status,
|
|
"task": task_state,
|
|
"match": _tone_for_status_pair(ih_status, os_status, task_state) == "ok",
|
|
"host": host,
|
|
"name": str(vm.get("name") or ""),
|
|
"flavor": str(vm.get("flavor") or ""),
|
|
"gpus": str(vm.get("gpu_count") or ""),
|
|
"fault": str(vm.get("openstack_fault") or "None"),
|
|
"never_built": host in ("N/A", "Unknown", ""),
|
|
}
|
|
|
|
d.add("Infrahub ID", vm.get("infrahub_id"))
|
|
d.add("OpenStack ID", vm.get("openstack_id"))
|
|
d.add("Name", vm.get("name"))
|
|
d.add("Region", vm.get("region_display") or vm.get("region"))
|
|
d.add("Infrahub status", ih_status, "info")
|
|
d.add("OpenStack status", os_status,
|
|
_tone_for_status_pair(ih_status, os_status, task_state) if "N/A" not in (ih_status, os_status) else "warn")
|
|
d.add("Task state", task_state, "warn" if task_state not in ("None", "N/A") else "info")
|
|
d.add("Hypervisor", host, "warn" if host in ("N/A", "Unknown") else "info")
|
|
d.add("Flavor", vm.get("flavor"))
|
|
d.add("GPU(s)", vm.get("gpu_count"))
|
|
d.add("Floating IP", vm.get("floating_ip"))
|
|
d.add("Created", vm.get("created"))
|
|
d.add("SSH", vm.get("ssh_text"), "ok" if vm.get("ssh_raw") == "reachable" else "info")
|
|
d.add("Volumes", vm.get("volumes_summary"))
|
|
if vm.get("project_name"):
|
|
d.add("OpenStack project", vm.get("project_name"))
|
|
if vm.get("project_environment"):
|
|
d.add("Environment", vm.get("project_environment"), "warn",
|
|
"Found outside production - Infrahub production records will not match.")
|
|
|
|
os_fault = str(vm.get("openstack_fault") or "None")
|
|
d.add("OpenStack fault", os_fault, "bad" if os_fault not in ("None", "N/A") else "ok")
|
|
faults = vm.get("faults") or []
|
|
d.add("InfraInsight faults", f"{len(faults)} recorded" if faults else "None", "bad" if faults else "ok")
|
|
|
|
for item in vm.get("mismatches") or []:
|
|
check, detail = cxbridge.cx().mismatch_parts(item)
|
|
d.add(f"Mismatch: {check}", detail, "bad")
|
|
|
|
for reason in vm.get("warn_reasons") or []:
|
|
d.add("Warning", str(reason), "warn")
|
|
for note in vm.get("info_notes") or []:
|
|
d.note(str(note))
|
|
|
|
|
|
def _fault_texts(vm: dict[str, Any], failed_event: dict[str, Any]) -> list[str]:
|
|
texts: list[str] = []
|
|
os_fault = str(vm.get("openstack_fault") or "")
|
|
if os_fault and os_fault not in ("None", "N/A"):
|
|
texts.append(os_fault)
|
|
for row in vm.get("faults") or []:
|
|
texts.extend(str(x) for x in row)
|
|
server = vm.get("server") if isinstance(vm.get("server"), dict) else {}
|
|
fault = server.get("fault")
|
|
if isinstance(fault, dict):
|
|
texts.append(str(fault.get("message") or ""))
|
|
texts.append(str(fault.get("details") or ""))
|
|
elif fault:
|
|
texts.append(str(fault))
|
|
rows = failed_event.get("rows") or {}
|
|
for key in ("Detail", "Traceback", "Result"):
|
|
if rows.get(key) and rows[key] != "N/A":
|
|
texts.append(str(rows[key]))
|
|
return texts
|
|
|
|
|
|
def _host_health_findings(d: Diagnosis, region: str, host: str) -> dict[str, Any]:
|
|
"""Run the cheap host signals and turn them into findings + a verdict."""
|
|
if not host or host in ("N/A", "Unknown") or not region:
|
|
d.act("Identify the hypervisor, then run Host Health Checks manually.", CX, "check", "Host Health Checks")
|
|
d.note("No hypervisor was recorded on this alert, so host health could not be checked automatically.")
|
|
return {}
|
|
|
|
health = cxbridge.host_health(region, host)
|
|
if not health.get("ok"):
|
|
d.add("Host health", f"lookup failed: {health.get('error', '')}", "warn")
|
|
d.act(f"Check host {host} manually.", CX, "check", "Host Health Checks")
|
|
return health
|
|
|
|
nova_state = str(health.get("nova_state") or "")
|
|
nova_status = str(health.get("nova_status") or "")
|
|
ovs_alive = health.get("ovs_alive")
|
|
ovs_state = str(health.get("ovs_state") or "")
|
|
|
|
bad_signals: list[str] = []
|
|
d.add("Host", host)
|
|
d.add("Nova state", nova_state, "ok" if nova_state.lower() == "up" else "bad")
|
|
if nova_state.lower() != "up":
|
|
bad_signals.append(f"Nova state is {nova_state}")
|
|
d.add("Nova status", nova_status, "ok" if nova_status.lower() == "enabled" else "bad")
|
|
if nova_status.lower() != "enabled":
|
|
bad_signals.append(f"Nova status is {nova_status}")
|
|
if health.get("disabled_reason"):
|
|
d.add("Disabled reason", health["disabled_reason"], "bad")
|
|
bad_signals.append(f"disabled: {health['disabled_reason']}")
|
|
if ovs_alive is not None:
|
|
d.add("OVS alive", str(ovs_alive), "ok" if ovs_alive in (True, "True", "true") else "bad")
|
|
if ovs_alive not in (True, "True", "true"):
|
|
bad_signals.append("OVS agent is not alive")
|
|
if ovs_state:
|
|
d.add("OVS state", ovs_state, "ok" if ovs_state.lower() == "up" else "bad")
|
|
if ovs_state.lower() != "up":
|
|
bad_signals.append(f"OVS state is {ovs_state}")
|
|
d.add("Host uptime", health.get("uptime"))
|
|
d.add("Aggregates", health.get("aggregates"))
|
|
|
|
health["bad_signals"] = bad_signals
|
|
if bad_signals:
|
|
d.act(
|
|
f"Host {host} shows problems ({'; '.join(bad_signals)}) - escalate to the Infrastructure team.",
|
|
INFRA, "escalate", "Host Health Checks",
|
|
)
|
|
else:
|
|
d.act(
|
|
f"Nova and OVS signals on {host} look healthy; complete the remaining Host Health Checks "
|
|
"(disk/dmesg/GPU checks the guide covers and this app does not).",
|
|
CX, "check", "Host Health Checks",
|
|
)
|
|
return health
|
|
|
|
|
|
def _vm_draft(d: Diagnosis, template: str, vm: dict[str, Any], alert: Alert,
|
|
*, floating_ip: str = "", note: Optional[str] = None) -> Optional[comms.Draft]:
|
|
"""Build a customer draft with the VM identity the house style expects."""
|
|
owners = (vm.get("owners") or []) if isinstance(vm, dict) else []
|
|
return comms.draft(
|
|
template,
|
|
instance_name=str(vm.get("name") or alert.instance_name or ""),
|
|
infrahub_id=str(vm.get("infrahub_id") or ""),
|
|
openstack_id=str(vm.get("openstack_id") or alert.openstack_id or ""),
|
|
greeting_name=comms.first_name(owners[0] if owners else ""),
|
|
agent_name=d.agent_name,
|
|
floating_ip=floating_ip,
|
|
note=note,
|
|
)
|
|
|
|
|
|
def _attach_contacts(d: Diagnosis, vm: dict[str, Any]) -> None:
|
|
d.contacts = comms.contacts_from_result(vm)
|
|
if not d.contacts.get("resolved"):
|
|
d.note(
|
|
"No organization owner contacts came back from Infrahub. Look the organization up in the Admin "
|
|
"Portal before sending anything."
|
|
)
|
|
|
|
|
|
def _common_preamble(d: Diagnosis, alert: Alert) -> None:
|
|
if alert.is_kubernetes:
|
|
d.note(
|
|
f"Instance name '{alert.instance_name}' looks like a Kubernetes node. Per the general process, treat it "
|
|
"as part of a K8s cluster - deleting a single node may be handled by the cluster instead."
|
|
)
|
|
if alert.threshold_min and alert.age_minutes is not None and alert.age_minutes < alert.threshold_min:
|
|
d.note(
|
|
f"This alert has only been active {alert.age_minutes} min against a {alert.threshold_min} min threshold; "
|
|
"it may still clear on its own."
|
|
)
|
|
|
|
|
|
# --- per-kind runbooks -----------------------------------------------------
|
|
|
|
def _diagnose_error(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
|
|
"""Instance in ERROR state."""
|
|
host = str(vm.get("host") or "N/A")
|
|
os_status = str(vm.get("os_status") or "N/A")
|
|
ih_status = str(vm.get("ih_status") or "N/A")
|
|
region = str(vm.get("region") or alert.region)
|
|
osid = str(vm.get("openstack_id") or alert.openstack_id)
|
|
|
|
failed_event: dict[str, Any] = {}
|
|
if osid and osid != "N/A" and region:
|
|
failed_event = cxbridge.failed_openstack_event(region, osid)
|
|
if failed_event:
|
|
rows = failed_event.get("rows") or {}
|
|
d.add("Failed OpenStack event", f"{failed_event.get('action', '?')} ({failed_event.get('request_id', '')})", "bad",
|
|
rows.get("Detail", ""))
|
|
d.evidence["failed_event"] = failed_event
|
|
|
|
# "We are more concerned if a VM goes into ERROR and it was previously
|
|
# ACTIVE" - no hypervisor means it never fully created.
|
|
was_active = host not in ("N/A", "Unknown", "")
|
|
d.add("Reached ACTIVE previously", "yes - a hypervisor is recorded" if was_active else "no - never placed on a host",
|
|
"bad" if was_active else "warn",
|
|
"" if was_active else "No hostname means the instance was never created fully and could not have become ACTIVE.")
|
|
|
|
texts = _fault_texts(vm, failed_event)
|
|
matches = match_faults(texts)
|
|
d.evidence["fault_text"] = [t for t in texts if t][:12]
|
|
|
|
for row in vm.get("faults") or []:
|
|
d.add("InfraInsight fault", " | ".join(str(x) for x in row), "bad")
|
|
|
|
if matches:
|
|
names = "; ".join(m["summary"] for m in matches)
|
|
d.verdict = f"ERROR with a known fault: {names}"
|
|
d.confidence = "high"
|
|
for m in matches:
|
|
d.act(m["guidance"], m["owner"], "escalate" if m["owner"] != CX else "remediate")
|
|
if m.get("gpu_census") and host not in ("N/A", "Unknown") and region:
|
|
census = cxbridge.host_gpu_census(region, host)
|
|
d.evidence["gpu_census"] = census
|
|
if census.get("ok"):
|
|
total = census.get("total_gpus")
|
|
full = total is not None and total >= 8
|
|
d.add("GPUs allocated on host", f"{total} (from {len(census.get('instances', []))} instances)",
|
|
"ok" if full else "warn",
|
|
"Host is FULL - this proves the host itself is fine." if full
|
|
else "Host is not full, so the NUMA/PCI failure is not simple capacity. Raise a Jira for Infra.")
|
|
if full:
|
|
d.act("Host is full (8 GPUs allocated) - this proves the host is fine. No Infra escalation needed "
|
|
"for capacity; treat as a race condition.", CX, "verify", status="done")
|
|
else:
|
|
d.act(f"Raise a Jira for the Infrastructure team to investigate host {host} and the instance error.",
|
|
INFRA, "escalate")
|
|
else:
|
|
d.verdict = "ERROR with no fault matching the runbook table"
|
|
d.confidence = "low" if not texts else "medium"
|
|
d.act(
|
|
"Fault is not in the runbook table - escalate to a peer, and add the fault to the runbook table once known.",
|
|
CX, "escalate",
|
|
)
|
|
|
|
if ih_status.upper() == "ERROR" and os_status.upper() not in ("ERROR", "N/A"):
|
|
d.add("Note", f"Infrahub says ERROR but OpenStack says {os_status}", "warn")
|
|
|
|
d.assessment = (
|
|
"The instance had reached ACTIVE, so customer data may be on it - remediate rather than leaving it. "
|
|
if was_active else
|
|
"The instance never became ACTIVE, so there is no customer data to protect; it needs deleting and recreating. "
|
|
)
|
|
if not was_active:
|
|
d.assessment += "Customers usually just delete ERROR VMs and retry."
|
|
|
|
d.act("Record the fault message and event output in the Slack alert thread and any HubSpot ticket.", CX, "verify")
|
|
|
|
created = str(vm.get("created") or "")
|
|
template = "error_was_active" if was_active else "error_never_active"
|
|
note = None
|
|
for m in matches:
|
|
if m.get("comms"):
|
|
template = m["comms"]
|
|
note = m.get("comms_note")
|
|
draft = _vm_draft(d, template, vm, alert, note=note)
|
|
if draft:
|
|
if not was_active:
|
|
draft.when += " Outreach is optional and only if the instance is less than 7 days old."
|
|
if created and created != "N/A":
|
|
draft.when += f" Created: {created}."
|
|
d.drafts.append(draft)
|
|
|
|
|
|
def _diagnose_deleting(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
|
|
"""Instance in DELETING state."""
|
|
ih_status = str(vm.get("ih_status") or "N/A")
|
|
os_status = str(vm.get("os_status") or "N/A")
|
|
server_exists = bool(vm.get("server"))
|
|
|
|
d.verdict = f"Stuck in DELETING for {alert.age_minutes or '?'} min - unlikely to finish on its own"
|
|
d.confidence = "high"
|
|
d.assessment = (
|
|
"The customer's intent is clear (they asked for a delete), so the remediation is to finish the delete. "
|
|
)
|
|
|
|
delete_requests = [
|
|
row for row in (vm.get("ih_events_all") or vm.get("ih_events") or [])
|
|
if "delete" in " ".join(str(x) for x in row).lower()
|
|
]
|
|
d.evidence["delete_requests"] = delete_requests[:5]
|
|
if delete_requests:
|
|
d.add("Delete request found", delete_requests[0][0], "ok", " | ".join(str(x) for x in delete_requests[0][1:]))
|
|
d.act("Delete request confirmed in Infrahub events - customer intent verified.", CX, "verify", status="done")
|
|
else:
|
|
d.add("Delete request found", "none in Infrahub events", "warn")
|
|
d.act(
|
|
"Confirm who requested the deletion via the InfraInsight InstanceDeleteRequest query. If no record "
|
|
"exists, escalate to DevOps - it could be an admin action or an anomaly.",
|
|
DEVOPS, "check",
|
|
)
|
|
d.note(
|
|
"Requester attribution (name/email) comes from the InfraInsight SQL query in the runbook; CX-Tools "
|
|
"exposes the Infrahub event but not the requesting user."
|
|
)
|
|
|
|
never_built = str(vm.get("host") or "N/A") in ("N/A", "Unknown", "")
|
|
|
|
if server_exists:
|
|
d.add("OpenStack server", f"still present ({os_status})", "bad")
|
|
d.act(f"Delete the server in OpenStack ({vm.get('openstack_id')}).", CX, "remediate", "Deleting an Instance")
|
|
if never_built:
|
|
# It reached OpenStack but never landed on a host, so this is not a
|
|
# normal delete: the build failed and the record still needs closing.
|
|
d.add("Reached a host", "no - the build never completed", "warn",
|
|
"The instance failed to build, so the customer's delete could never finish normally.")
|
|
d.assessment += (
|
|
"The instance never made it onto a host, so removing the OpenStack server is only half of it - "
|
|
"the Infrahub record has to be closed out too. "
|
|
)
|
|
else:
|
|
d.add("OpenStack server", "already gone", "ok")
|
|
d.assessment += (
|
|
"The server is already gone from OpenStack, so this is an Infrahub-side record that never closed out. "
|
|
)
|
|
|
|
# Deleting the OpenStack server does not clear the Infrahub record. Whatever
|
|
# broke the delete the first time will still leave it in DELETING, and it is
|
|
# the record that keeps the alert firing and the resource on the books.
|
|
if ih_status.upper() == "DELETING":
|
|
d.act(
|
|
"Mark the instance deleted in InfraInsight so the Infrahub record closes out - removing the OpenStack "
|
|
"server alone leaves it stuck in DELETING and the alert still firing.",
|
|
CX, "remediate", "Infrahub Insights (Infra-Insight)",
|
|
)
|
|
else:
|
|
d.add("Note", f"Infrahub now reports {ih_status}, not DELETING - the alert may already be stale", "warn")
|
|
d.confidence = "medium"
|
|
|
|
d.act("Confirm the record is gone from Admin Portal Production and the alert clears.", CX, "verify")
|
|
|
|
for template in ("deleting", "deleting_resolved"):
|
|
made = _vm_draft(d, template, vm, alert)
|
|
if made:
|
|
d.drafts.append(made)
|
|
d.note(
|
|
"The customer asked for this delete, so contact is optional - but your sent examples show CX does confirm "
|
|
"it. Two drafts are offered: a proactive notice, and a reply that closes an existing ticket."
|
|
)
|
|
|
|
|
|
def _diagnose_shutoff(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
|
|
"""Instance in SHUTOFF state."""
|
|
os_status = str(vm.get("os_status") or "N/A")
|
|
d.verdict = "Customer-initiated SHUTOFF - billing awareness notice required"
|
|
d.confidence = "high"
|
|
d.assessment = (
|
|
"SHUTOFF is a result of customer action, not a platform fault. The only job here is making sure the "
|
|
"customer knows a SHUTOFF VM still accrues full cost."
|
|
)
|
|
if os_status.upper() == "SHUTOFF":
|
|
d.act("Confirmed SHUTOFF in OpenStack.", CX, "verify", status="done")
|
|
else:
|
|
d.add("Note", f"OpenStack reports {os_status}, not SHUTOFF - the alert may be stale", "warn")
|
|
d.confidence = "medium"
|
|
d.act("Create a HubSpot ticket and send the billing awareness note (snippet: #shutoff).", CX, "comms",
|
|
"HubSpot Ticket Creation")
|
|
draft = _vm_draft(d, "shutoff", vm, alert)
|
|
if draft:
|
|
d.drafts.append(draft)
|
|
|
|
|
|
def _diagnose_hibernating(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
|
|
"""Instance in HIBERNATING state."""
|
|
region = str(vm.get("region") or alert.region)
|
|
host = str(vm.get("host") or alert.host or "N/A")
|
|
d.verdict = "Stuck HIBERNATING - almost certainly a host issue"
|
|
d.confidence = "high"
|
|
d.assessment = (
|
|
"The runbook treats stuck HIBERNATING as almost certainly a host problem. Check the host, then drive the "
|
|
"shelve to completion."
|
|
)
|
|
health = _host_health_findings(d, region, host)
|
|
d.evidence["host_health"] = health
|
|
d.act("Complete the shelve for the instance so it lands in SHELVED_OFFLOADED.", CX, "remediate",
|
|
"Shelving an Instance")
|
|
d.act("Confirm the instance ends up HIBERNATED in Infrahub and SHELVED_OFFLOADED in OpenStack.", CX, "verify")
|
|
if health.get("bad_signals"):
|
|
d.verdict = f"Stuck HIBERNATING due to a host problem on {host}"
|
|
d.note("This runbook has no customer comms step of its own.")
|
|
|
|
|
|
def _diagnose_creating(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
|
|
"""Instance in CREATING state."""
|
|
osid = str(vm.get("openstack_id") or "")
|
|
has_osid = bool(osid) and osid != "N/A"
|
|
server_exists = bool(vm.get("server"))
|
|
|
|
d.confidence = "high"
|
|
if not has_osid:
|
|
d.verdict = "Stuck CREATING and never got an OpenStack ID - the VM does not exist in OpenStack"
|
|
d.add("OpenStack ID", "never assigned", "bad",
|
|
"The instance never reached OpenStack, so it cannot be recovered and must be recreated.")
|
|
d.assessment = (
|
|
"The instance never got an OpenStack ID, so nothing was ever built. The customer cannot delete it "
|
|
"themselves in this transitional state - CX has to clear it and tell them to retry."
|
|
)
|
|
elif not server_exists:
|
|
d.verdict = "Stuck CREATING with an OpenStack ID, but no server exists in OpenStack"
|
|
d.assessment = "Infrahub holds an OpenStack ID that OpenStack does not know about; the record needs clearing."
|
|
else:
|
|
d.verdict = f"Stuck CREATING while OpenStack reports {vm.get('os_status')}"
|
|
d.confidence = "medium"
|
|
d.assessment = "The server does exist in OpenStack, so this is a sync failure rather than a failed build."
|
|
|
|
d.act("Confirm the instance and its Infrahub ID in the Admin Portal (Billing -> Manage Organization -> Resources).",
|
|
CX, "check")
|
|
d.act("Delete the stuck instance - it cannot be recovered in this state.", CX, "remediate", "Deleting an Instance")
|
|
d.act("Create a HubSpot ticket and tell the customer they can retry deploying.", CX, "comms",
|
|
"HubSpot Ticket Creation")
|
|
draft = _vm_draft(d, "creating", vm, alert)
|
|
if draft:
|
|
d.drafts.append(draft)
|
|
|
|
|
|
def _diagnose_restoring(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
|
|
"""Instance in RESTORING state."""
|
|
region = str(vm.get("region") or alert.region)
|
|
host = str(vm.get("host") or alert.host or "N/A")
|
|
osid = str(vm.get("openstack_id") or "")
|
|
|
|
d.verdict = "Stuck RESTORING - unshelve did not complete"
|
|
d.confidence = "high"
|
|
d.assessment = (
|
|
"A stuck restore blocks the customer from reaching their data. Larger images can be slow, but past an hour "
|
|
"this needs investigating."
|
|
)
|
|
health = _host_health_findings(d, region, host)
|
|
d.evidence["host_health"] = health
|
|
|
|
failed_event = cxbridge.failed_openstack_event(region, osid) if osid and osid != "N/A" and region else {}
|
|
if failed_event:
|
|
rows = failed_event.get("rows") or {}
|
|
d.add("Failed OpenStack event", f"{failed_event.get('action', '?')} ({failed_event.get('request_id', '')})",
|
|
"bad", rows.get("Detail", ""))
|
|
d.evidence["failed_event"] = failed_event
|
|
d.act("Escalate the failed unshelve event and its fault to the Infrastructure team.", INFRA, "escalate")
|
|
elif not health.get("bad_signals"):
|
|
d.add("Failed OpenStack event", "none found in recent events", "warn")
|
|
d.act("Analyze the unshelve event in OpenStack for fault information, then escalate it to Infrastructure.",
|
|
INFRA, "escalate")
|
|
|
|
d.act("Once resolved, shelve the instance so it returns to its pre-restore state and the customer can retry.",
|
|
CX, "remediate", "Shelving an Instance")
|
|
d.act("Validate the instance is SHELVED_OFFLOADED in OpenStack.", CX, "verify")
|
|
d.act("Set the instance state back to HIBERNATED in InfraInsight.", CX, "remediate",
|
|
"Infrahub Insights (Infra-Insight)")
|
|
d.act("Create a HubSpot ticket telling the customer they can retry restoring.", CX, "comms",
|
|
"HubSpot Ticket Creation")
|
|
draft = _vm_draft(d, "restoring", vm, alert)
|
|
if draft:
|
|
d.drafts.append(draft)
|
|
|
|
|
|
def _diagnose_rebooting(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
|
|
"""Instance in REBOOTING state."""
|
|
region = str(vm.get("region") or alert.region)
|
|
host = str(vm.get("host") or alert.host or "N/A")
|
|
os_status = str(vm.get("os_status") or "N/A")
|
|
task_state = str(vm.get("task_state") or "None")
|
|
|
|
d.verdict = "Stuck REBOOTING - the reboot did not complete"
|
|
d.confidence = "high"
|
|
d.assessment = "Instances get stuck rebooting from host problems; confirm the reboot request, then escalate."
|
|
|
|
health = _host_health_findings(d, region, host)
|
|
d.evidence["host_health"] = health
|
|
|
|
reboot_events = [
|
|
row for row in (vm.get("ih_events_all") or vm.get("ih_events") or [])
|
|
if "reboot" in " ".join(str(x) for x in row).lower()
|
|
]
|
|
d.evidence["reboot_requests"] = reboot_events[:5]
|
|
if reboot_events:
|
|
d.add("InstanceRebootRequest", reboot_events[0][0], "ok", " | ".join(str(x) for x in reboot_events[0][1:]))
|
|
d.act("Reboot request confirmed in Infrahub events.", CX, "verify", status="done")
|
|
else:
|
|
d.add("InstanceRebootRequest", "not found in Infrahub events", "warn")
|
|
d.act("Confirm the instance received a HARD_REBOOT request (event InstanceRebootRequest) in the Admin Portal.",
|
|
CX, "check")
|
|
|
|
expected = os_status.upper() == "HARD_REBOOT" or "REBOOT" in task_state.upper()
|
|
d.add("OpenStack reboot state", f"status={os_status}, task_state={task_state}", "ok" if expected else "warn",
|
|
"" if expected else "The runbook expects status HARD_REBOOT while a reboot is in flight.")
|
|
|
|
failed_event = cxbridge.failed_openstack_event(region, str(vm.get("openstack_id") or "")) \
|
|
if vm.get("openstack_id") not in (None, "", "N/A") and region else {}
|
|
if failed_event:
|
|
rows = failed_event.get("rows") or {}
|
|
d.add("Failed OpenStack event", f"{failed_event.get('action', '?')} ({failed_event.get('request_id', '')})",
|
|
"bad", rows.get("Detail", ""))
|
|
d.evidence["failed_event"] = failed_event
|
|
|
|
d.act("Escalate the instance event to the Infrastructure team for review.", INFRA, "escalate")
|
|
d.act("Once cleared, confirm the instance is ACTIVE in both Infrahub and OpenStack.", CX, "verify")
|
|
d.act("Create a HubSpot ticket telling the customer they can retry rebooting.", CX, "comms",
|
|
"HubSpot Ticket Creation")
|
|
draft = _vm_draft(d, "rebooting", vm, alert)
|
|
if draft:
|
|
d.drafts.append(draft)
|
|
|
|
|
|
def _diagnose_build(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
|
|
"""Instance in BUILD state."""
|
|
region = str(vm.get("region") or alert.region)
|
|
host = str(vm.get("host") or alert.host or "N/A")
|
|
os_status = str(vm.get("os_status") or "N/A")
|
|
age = alert.age_minutes
|
|
|
|
gpus = str(vm.get("gpu_count") or "")
|
|
large = gpus.isdigit() and int(gpus) >= 4
|
|
|
|
if age is not None and age < 60 and large:
|
|
d.verdict = f"BUILD for {age} min on a large flavor - may still be transient"
|
|
d.confidence = "medium"
|
|
d.assessment = (
|
|
"Larger flavors (more GPUs/storage) can take a while to build and often resolve on their own. "
|
|
"The runbook says to investigate host/infrastructure problems once it is stuck past an hour."
|
|
)
|
|
else:
|
|
d.verdict = f"Stuck in BUILD{f' for {age} min' if age is not None else ''} - investigate host/infrastructure"
|
|
d.confidence = "high"
|
|
d.assessment = "Past roughly an hour in BUILD this is a host or infrastructure problem, not slow provisioning."
|
|
|
|
health = _host_health_findings(d, region, host)
|
|
d.evidence["host_health"] = health
|
|
|
|
d.add("OpenStack status", os_status, "ok" if os_status.upper() == "BUILD" else "warn",
|
|
"" if os_status.upper() == "BUILD" else "The runbook expects the instance to be in status BUILD.")
|
|
|
|
failed_event = cxbridge.failed_openstack_event(region, str(vm.get("openstack_id") or "")) \
|
|
if vm.get("openstack_id") not in (None, "", "N/A") and region else {}
|
|
if failed_event:
|
|
rows = failed_event.get("rows") or {}
|
|
d.add("Failed OpenStack event", f"{failed_event.get('action', '?')} ({failed_event.get('request_id', '')})",
|
|
"bad", rows.get("Detail", ""))
|
|
d.evidence["failed_event"] = failed_event
|
|
|
|
d.act("Escalate to the Infrastructure team with the server event and server show output.", INFRA, "escalate")
|
|
d.act(
|
|
"Tell the customer the instance must be deleted and recreated. It will eventually transition to ERROR, "
|
|
"which lets them delete it themselves; otherwise InfraInsight can delete the Infrahub record (Infra may need "
|
|
"to delete it in OpenStack while it is stuck in BUILD).",
|
|
CX, "comms", "HubSpot Ticket Creation",
|
|
)
|
|
draft = _vm_draft(d, "build", vm, alert)
|
|
if draft:
|
|
d.drafts.append(draft)
|
|
|
|
|
|
# --- rogue VM --------------------------------------------------------------
|
|
|
|
MISMATCH_TABLE: list[dict[str, Any]] = [
|
|
{
|
|
"ih": "HIBERNATED", "os": "SHUTOFF",
|
|
"verdict": "Hibernation did not complete - commonly host disk capacity",
|
|
"steps": [
|
|
(CX, "Run Host Health Checks on the hypervisor.", "Host Health Checks"),
|
|
(CX, "Run the Windmill workflow to clean stale images on the host and free space for hibernation.", ""),
|
|
(CX, "Complete the shelve for the instance.", "Shelving an Instance"),
|
|
],
|
|
"comms": None,
|
|
},
|
|
{
|
|
"ih": "HIBERNATED", "os": "ACTIVE",
|
|
"verdict": "Infrahub thinks the VM is hibernated but it is still running and billable",
|
|
"steps": [
|
|
(CX, "Run Host Health Checks on the hypervisor.", "Host Health Checks"),
|
|
(CX, "If the host shows no visible issues, shelve the instance.", "Shelving an Instance"),
|
|
(CX, "Create a HubSpot ticket for the state mismatch.", "HubSpot Ticket Creation"),
|
|
],
|
|
"comms": "sync_state",
|
|
},
|
|
{
|
|
"ih": "DELETING", "os": "*",
|
|
"verdict": "Infrahub is stuck DELETING - follow the DELETING runbook",
|
|
"steps": [(CX, "Follow the Instance in DELETING state runbook.", "Instance in DELETING state")],
|
|
"comms": None,
|
|
},
|
|
{
|
|
"ih": "RESTORING", "os": "*",
|
|
"verdict": "Infrahub is stuck RESTORING - follow the RESTORING runbook",
|
|
"steps": [(CX, "Follow the Instance in RESTORING state runbook.", "Instance in RESTORING state")],
|
|
"comms": None,
|
|
},
|
|
]
|
|
|
|
|
|
def _match_mismatch_table(ih_status: str, os_status: str) -> Optional[dict[str, Any]]:
|
|
ih = (ih_status or "").upper()
|
|
os_ = (os_status or "").upper()
|
|
if ih == "ERROR" or os_ == "ERROR":
|
|
return {
|
|
"ih": ih, "os": os_,
|
|
"verdict": "ERROR on one side - follow the ERROR runbook",
|
|
"steps": [(CX, "Follow the Instance in ERROR state runbook.", "Instance in ERROR state")],
|
|
"comms": None,
|
|
}
|
|
for entry in MISMATCH_TABLE:
|
|
if entry["ih"] == ih and entry["os"] in ("*", os_):
|
|
return entry
|
|
return None
|
|
|
|
|
|
def _gpu_slots(visual: dict[str, Any], roster: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""One entry per physical GPU on the host.
|
|
|
|
A host has a fixed number of GPU sockets. Every one of them is in exactly
|
|
one of three states, and showing all of them is the only way the arithmetic
|
|
reads as complete:
|
|
|
|
* claimed by a named VM - one slot per GPU that VM holds
|
|
* in use but unclaimed - the host says busy, no instance says so
|
|
* free - physically present, nothing using it
|
|
|
|
Drawing only the first two (as the earlier bar did) drops the free sockets
|
|
and leaves a total that does not match the host's GPU count.
|
|
"""
|
|
physical = visual.get("physical")
|
|
in_use = visual.get("in_use_metric")
|
|
artifact = bool(visual.get("spare_capacity_artifact"))
|
|
|
|
slots: list[dict[str, Any]] = []
|
|
for vm in roster:
|
|
try:
|
|
count = int(str(vm.get("gpus") or "0"))
|
|
except ValueError:
|
|
count = 0
|
|
for _ in range(max(0, count)):
|
|
slots.append({
|
|
"kind": "vm",
|
|
"name": vm["name"],
|
|
"linked": vm["linked"],
|
|
"match": vm["match"],
|
|
"ih_status": vm["ih_status"],
|
|
"os_status": vm["os_status"],
|
|
})
|
|
|
|
named = len(slots)
|
|
if in_use is None:
|
|
return slots
|
|
|
|
# Sockets the host counts as busy that no instance accounts for. When
|
|
# In_Use_Gpus is really just the physical count, that difference is spare
|
|
# capacity rather than a missing workload, so label it honestly.
|
|
unaccounted = max(0, int(in_use) - named)
|
|
for _ in range(unaccounted):
|
|
slots.append({"kind": "free" if artifact else "unaccounted"})
|
|
|
|
if physical is not None:
|
|
for _ in range(max(0, int(physical) - max(int(in_use), named))):
|
|
slots.append({"kind": "free"})
|
|
return slots
|
|
|
|
|
|
def _diagnose_rogue_vm(d: Diagnosis, alert: Alert) -> None:
|
|
"""Suspected Rogue VM - host-wide Infrahub/OpenStack reconciliation."""
|
|
host = alert.host or alert.instance_name
|
|
region = alert.region
|
|
|
|
if not host:
|
|
d.error = "This alert carries no hypervisor, so there is nothing to reconcile."
|
|
return
|
|
if not region:
|
|
d.error = f"Could not derive a CX-Tools region from host '{host}'."
|
|
return
|
|
|
|
result = cxbridge.collect_host(host)
|
|
d.evidence["host_result"] = cxbridge.json_safe(result)
|
|
if not result.get("ok"):
|
|
d.error = str(result.get("error") or f"Host collection failed for {host}.")
|
|
return
|
|
|
|
hv = result.get("hypervisor") if isinstance(result.get("hypervisor"), dict) else {}
|
|
ovs = result.get("ovs") if isinstance(result.get("ovs"), dict) else {}
|
|
c = cxbridge.cx()
|
|
|
|
d.add("Host", host)
|
|
d.add("Region", region)
|
|
d.add("Instances on host", result.get("server_count"))
|
|
d.add("Nova state", c.normalize_empty(c.first_present(hv, "state", "State", default="")) or "N/A",
|
|
"ok" if str(c.first_present(hv, "state", "State", default="")).lower() == "up" else "bad")
|
|
d.add("Nova status", c.normalize_empty(c.first_present(hv, "status", "Status", default="")) or "N/A",
|
|
"ok" if str(c.first_present(hv, "status", "Status", default="")).lower() == "enabled" else "bad")
|
|
if result.get("compute_service_disabled_reason"):
|
|
d.add("Disabled reason", result["compute_service_disabled_reason"], "bad")
|
|
if ovs:
|
|
d.add("OVS alive", str(ovs.get("alive")), "ok" if ovs.get("alive") in (True, "True", "true") else "bad")
|
|
d.add("OVS state", str(ovs.get("state") or "N/A"), "ok" if str(ovs.get("state") or "").lower() == "up" else "bad")
|
|
|
|
instances = result.get("instances") or []
|
|
problem_rows: list[dict[str, Any]] = []
|
|
ignored = 0
|
|
|
|
# Per-instance reconciliation for the visual: every VM on the host, and
|
|
# whether Infrahub has a counterpart for it.
|
|
roster: list[dict[str, Any]] = []
|
|
for vm in instances:
|
|
ih_status = str(vm.get("ih_status") or "N/A")
|
|
os_status = str(vm.get("os_status") or "N/A")
|
|
linked = bool(vm.get("infrahub"))
|
|
roster.append({
|
|
"name": str(vm.get("name") or "N/A"),
|
|
"openstack_id": str(vm.get("openstack_id") or "N/A"),
|
|
"infrahub_id": str(vm.get("infrahub_id") or "N/A"),
|
|
"ih_status": ih_status if linked else "not in Infrahub",
|
|
"os_status": os_status,
|
|
"gpus": str(vm.get("gpu_count") or "?"),
|
|
"linked": linked,
|
|
"match": linked and _tone_for_status_pair(ih_status, os_status, str(vm.get("task_state") or "None")) == "ok",
|
|
"tempest": bool(vm.get("tempest")),
|
|
"org": str(vm.get("org_value") or ""),
|
|
})
|
|
|
|
for vm in instances:
|
|
mismatches = vm.get("mismatches") or []
|
|
warn_reasons = vm.get("warn_reasons") or []
|
|
if vm.get("tempest"):
|
|
ignored += 1
|
|
continue
|
|
if not mismatches and not warn_reasons:
|
|
continue
|
|
|
|
ih_status = str(vm.get("ih_status") or "N/A")
|
|
os_status = str(vm.get("os_status") or "N/A")
|
|
checks = [c.mismatch_parts(m)[0] for m in mismatches]
|
|
entry = _match_mismatch_table(ih_status, os_status)
|
|
|
|
row: dict[str, Any] = {
|
|
"idx": vm.get("idx"),
|
|
"name": vm.get("name"),
|
|
"infrahub_id": vm.get("infrahub_id"),
|
|
"openstack_id": vm.get("openstack_id"),
|
|
"ih_status": ih_status,
|
|
"os_status": os_status,
|
|
"floating_ip": vm.get("floating_ip"),
|
|
"environment": vm.get("project_environment") or "",
|
|
"mismatches": [{"check": c.mismatch_parts(m)[0], "detail": c.mismatch_parts(m)[1]} for m in mismatches],
|
|
"warn_reasons": [str(x) for x in warn_reasons],
|
|
"contacts": comms.contacts_from_result(vm),
|
|
"verdict": "",
|
|
"steps": [],
|
|
"comms_template": None,
|
|
}
|
|
|
|
if "Infrahub Missing" in checks:
|
|
row["verdict"] = "Exists in OpenStack with no production Infrahub record"
|
|
row["steps"] = [
|
|
(CX, "Search for the VM in Admin Portal PreProd and Staging with 'Include Deleted' checked.", ""),
|
|
(CX, "If it is DELETED or absent everywhere, delete the server in OpenStack.", "Deleting an Instance"),
|
|
]
|
|
if vm.get("project_environment"):
|
|
row["verdict"] = f"Lives in {vm['project_environment']}, not production"
|
|
row["steps"] = [(CX, f"Confirm in Admin Portal {vm['project_environment']} whether this VM is still needed.", "")]
|
|
elif entry:
|
|
row["verdict"] = entry["verdict"]
|
|
row["steps"] = list(entry["steps"])
|
|
row["comms_template"] = entry.get("comms")
|
|
elif "Openstack Missing" in checks:
|
|
row["verdict"] = "Infrahub holds a record OpenStack does not have"
|
|
row["steps"] = [
|
|
(CX, "Confirm the VM is DELETED in Admin Portal Production.", ""),
|
|
(CX, "If Infrahub still shows it live, correct the record in InfraInsight.", "Infrahub Insights (Infra-Insight)"),
|
|
]
|
|
elif ih_status.upper() == "HIBERNATED" and str(vm.get("host") or "N/A") not in ("N/A", ""):
|
|
row["verdict"] = "Infrahub kept a stale host on a HIBERNATED VM"
|
|
row["steps"] = [(CX, "Use the InfraInsight update-resource tool to remove the host from the VM record.",
|
|
"Infrahub Insights (Infra-Insight)")]
|
|
else:
|
|
row["verdict"] = "Mismatch is not in the runbook table"
|
|
row["steps"] = [
|
|
(CX, "Ping Kheano Martinez or John Priest for a runbook update, and escalate to Infrastructure for next steps.", ""),
|
|
]
|
|
|
|
problem_rows.append(row)
|
|
|
|
d.evidence["instances"] = problem_rows
|
|
d.evidence["ignored_tempest"] = ignored
|
|
d.visual = {**(d.visual or {}), "roster": roster, "slots": _gpu_slots(d.visual or {}, roster)}
|
|
|
|
if ignored:
|
|
d.add("Ignored (tempest/OIE testing)", str(ignored), "info",
|
|
"CX-Tools suppresses tempest-project instances, which the runbook says to ignore.")
|
|
|
|
if not problem_rows:
|
|
gap = int((d.visual or {}).get("gap") or 0)
|
|
artifact = bool((d.visual or {}).get("spare_capacity_artifact"))
|
|
d.confidence = "high"
|
|
|
|
if gap >= 1 and not artifact:
|
|
# Every Infrahub record matches OpenStack, yet the host reports more
|
|
# GPUs in use than the instances account for. Nothing on the CX side
|
|
# explains that - it is a host-level allocation question.
|
|
d.verdict = (
|
|
f"{gap} GPU(s) in use on {host} belong to no instance on either side"
|
|
)
|
|
d.assessment = (
|
|
"Every Infrahub record reconciles with OpenStack, so this is not a stale record or a rogue VM "
|
|
"that CX can correct. The host is reporting GPUs in use beyond what any instance claims, which "
|
|
"points at a leaked allocation on the hypervisor."
|
|
)
|
|
d.act(
|
|
f"Escalate to the Infrastructure team: {host} reports {gap} GPU(s) in use that no OpenStack "
|
|
"instance claims. Ask them to check for allocations left behind by deleted domains.",
|
|
INFRA, "escalate", "Host Health Checks",
|
|
)
|
|
d.act(
|
|
"Confirm from the hypervisor's own PCI/GPU view before escalating, in case the exporter is at fault.",
|
|
CX, "check",
|
|
)
|
|
return
|
|
|
|
d.verdict = f"No Infrahub/OpenStack mismatch found on {host}"
|
|
if artifact:
|
|
d.verdict = f"Not a rogue VM - {gap} spare GPU(s) on {host} reported as a discrepancy"
|
|
d.assessment = (
|
|
"Every instance reconciles, and the 'gap' is the host's unallocated capacity: the rule's "
|
|
"In_Use_Gpus reading equals the physical GPU count, so it subtracts allocated GPUs from total "
|
|
"GPUs. Nothing to do on this host - the alert rule itself needs fixing."
|
|
)
|
|
d.act("No CX action. Raise the rule defect with the alert owner so these stop firing.", CX, "escalate")
|
|
return
|
|
|
|
d.assessment = (
|
|
"Every instance on the host reconciles. Per the runbook, when there is no mismatch the issue likely "
|
|
"exists only on the Infrahub side."
|
|
)
|
|
d.act(
|
|
"Query InfraInsight for all instances on this host and check for HIBERNATED VMs that still carry a host "
|
|
"value; remove the host with the update-resource tool.",
|
|
CX, "check", "SQL Queries - Infra Insight",
|
|
)
|
|
d.note("The alert may already have cleared, or the discrepancy may be Infrahub-only and invisible to OpenStack.")
|
|
return
|
|
|
|
d.verdict = f"{len(problem_rows)} of {len(instances)} instances on {host} do not reconcile"
|
|
d.confidence = "high"
|
|
d.assessment = (
|
|
"Each mismatched instance below is mapped to its remediation from the Suspected Rogue VM table. Work them "
|
|
"individually - they can need different runbooks."
|
|
)
|
|
|
|
for row in problem_rows:
|
|
label = f"#{row['idx']} {row['name']} ({row['ih_status']} / {row['os_status']})"
|
|
d.add(f"Mismatch {label}", row["verdict"], "bad",
|
|
"; ".join(m["detail"] for m in row["mismatches"]))
|
|
for owner, text, guide in row["steps"]:
|
|
d.act(f"[{row['name']}] {text}", owner, "remediate", guide)
|
|
if row["comms_template"]:
|
|
draft = comms.draft(row["comms_template"], instance_name=str(row["name"] or ""),
|
|
infrahub_id=str(row.get("infrahub_id") or ""),
|
|
openstack_id=str(row.get("openstack_id") or ""),
|
|
greeting_name=comms.first_name((row["contacts"].get("owners") or [""])[0]),
|
|
agent_name=d.agent_name)
|
|
if draft:
|
|
draft.label += f" - {row['name']}"
|
|
d.drafts.append(draft)
|
|
d.contacts = row["contacts"] if row["contacts"].get("resolved") else d.contacts
|
|
|
|
d.act("Record the findings and the alert link in the Slack thread and any associated ticket.", CX, "verify")
|
|
d.act(
|
|
"If a customer request (reboot, hibernation, restore) has been incomplete for over 30 minutes, treat it as a "
|
|
"host issue.",
|
|
CX, "check",
|
|
)
|
|
|
|
|
|
# --- duplicated IPs -------------------------------------------------------
|
|
|
|
def _diagnose_duplicate_ip(d: Diagnosis, alert: Alert, prom: Any = None) -> None:
|
|
"""Duplicated IPs."""
|
|
fip = alert.floating_ip
|
|
if not fip:
|
|
d.error = "This alert carries no floating_ip label."
|
|
return
|
|
|
|
d.add("Floating IP", fip)
|
|
summary = alert.annotations.get("summary", "")
|
|
claimed = re.search(r"present on `(\d+)` VMs", summary)
|
|
if claimed:
|
|
d.add("Prometheus reports", f"{claimed.group(1)} VMs holding this IP", "bad")
|
|
|
|
# The runbook's "EASY WAY": Resources{floating_ip="..."} covers every
|
|
# environment, which is how a PreProd/Staging claimant is found.
|
|
prom_rows: list[dict[str, Any]] = []
|
|
if prom is not None:
|
|
try:
|
|
prom_rows = prom.resources_by_floating_ip(fip)
|
|
except Exception as exc:
|
|
d.note(f"Prometheus Resources lookup failed: {exc}")
|
|
d.evidence["prometheus_claimants"] = prom_rows
|
|
for row in prom_rows:
|
|
env = row.get("environment") or row.get("env") or ""
|
|
d.add(
|
|
f"Prometheus claimant: {row.get('instance_name', '?')}",
|
|
" / ".join(x for x in [row.get("status", ""), row.get("region", ""), str(env)] if x),
|
|
"warn",
|
|
)
|
|
|
|
result = cxbridge.collect_vm(fip, region=alert.region)
|
|
d.evidence["vm_result"] = cxbridge.json_safe(result)
|
|
if not result.get("ok"):
|
|
d.error = str(result.get("error") or f"CX-Tools could not resolve floating IP {fip}.")
|
|
return
|
|
|
|
claimants = result.get("instances") if result.get("mode") == "multi_vm" else [result]
|
|
claimants = [c for c in claimants if isinstance(c, dict)]
|
|
d.add("Production claimants found by CX-Tools", str(len(claimants)),
|
|
"bad" if len(claimants) > 1 else "warn")
|
|
|
|
c = cxbridge.cx()
|
|
scenarios: list[dict[str, Any]] = []
|
|
for vm in claimants:
|
|
ih_status = str(vm.get("ih_status") or "N/A")
|
|
server = vm.get("server") if isinstance(vm.get("server"), dict) else {}
|
|
os_fip = c.public_ip_from_server(server) if server else ""
|
|
ih_fip = str((vm.get("infrahub") or {}).get("floating_ip") or "") if isinstance(vm.get("infrahub"), dict) else ""
|
|
|
|
entry: dict[str, Any] = {
|
|
"name": vm.get("name"),
|
|
"infrahub_id": vm.get("infrahub_id"),
|
|
"openstack_id": vm.get("openstack_id"),
|
|
"ih_status": ih_status,
|
|
"os_status": vm.get("os_status"),
|
|
"infrahub_fip": ih_fip or "none",
|
|
"openstack_fip": os_fip or "none",
|
|
"server_present": bool(server),
|
|
"contacts": comms.contacts_from_result(vm),
|
|
}
|
|
|
|
if ih_status.upper() == "DELETING":
|
|
entry["verdict"] = "Stuck DELETING in Infrahub while the IP has moved on"
|
|
entry["steps"] = [(CX, "Delete this instance - it is stuck DELETING per Infrahub.", "Deleting an Instance")]
|
|
entry["comms"] = None
|
|
elif not server:
|
|
entry["verdict"] = "No longer exists in OpenStack, but Infrahub still holds the IP"
|
|
entry["steps"] = [
|
|
(CX, "Remove the stale floating IP in InfraInsight and set Floating IP Status to NO FLOATING IP.",
|
|
"Infrahub Insights (Infra-Insight)"),
|
|
]
|
|
entry["comms"] = "dupip_removed"
|
|
elif not os_fip:
|
|
entry["verdict"] = "Scenario #1 - the VM has no floating IP in OpenStack"
|
|
entry["steps"] = [
|
|
(CX, "Remove the incorrect floating IP in InfraInsight and set Floating IP Status to NO FLOATING IP.",
|
|
"Infrahub Insights (Infra-Insight)"),
|
|
]
|
|
entry["comms"] = "dupip_removed"
|
|
elif ih_fip and os_fip and ih_fip != os_fip:
|
|
entry["verdict"] = f"Scenario #2 - Infrahub says {ih_fip} but OpenStack says {os_fip}"
|
|
entry["steps"] = [
|
|
(CX, f"Update the Infrahub floating IP to {os_fip} in InfraInsight.",
|
|
"Infrahub Insights (Infra-Insight)"),
|
|
]
|
|
entry["comms"] = "dupip_corrected"
|
|
entry["new_fip"] = os_fip
|
|
else:
|
|
entry["verdict"] = "Infrahub and OpenStack agree - this is the rightful owner of the IP"
|
|
entry["steps"] = []
|
|
entry["comms"] = None
|
|
scenarios.append(entry)
|
|
|
|
d.evidence["claimants"] = scenarios
|
|
d.visual = {
|
|
"type": "claimants",
|
|
"ip": fip,
|
|
"items": [{
|
|
"name": str(e["name"]), "ih_status": str(e["ih_status"]), "os_status": str(e["os_status"]),
|
|
"verdict": e["verdict"],
|
|
} for e in scenarios],
|
|
}
|
|
|
|
for entry in scenarios:
|
|
d.add(
|
|
f"Claimant {entry['name']} ({entry['ih_status']} / {entry['os_status']})",
|
|
entry["verdict"],
|
|
"ok" if not entry["steps"] else "bad",
|
|
f"Infrahub FIP={entry['infrahub_fip']}, OpenStack FIP={entry['openstack_fip']}",
|
|
)
|
|
for owner, text, guide in entry["steps"]:
|
|
d.act(f"[{entry['name']}] {text}", owner, "remediate", guide)
|
|
if entry.get("comms"):
|
|
draft = comms.draft(entry["comms"], instance_name=str(entry["name"] or ""),
|
|
infrahub_id=str(entry.get("infrahub_id") or ""),
|
|
openstack_id=str(entry.get("openstack_id") or ""),
|
|
greeting_name=comms.first_name((entry["contacts"].get("owners") or [""])[0]),
|
|
agent_name=d.agent_name,
|
|
floating_ip=str(entry.get("new_fip") or ""))
|
|
if draft:
|
|
draft.label += f" - {entry['name']}"
|
|
d.drafts.append(draft)
|
|
if entry["contacts"].get("resolved"):
|
|
d.contacts = entry["contacts"]
|
|
|
|
actionable = [e for e in scenarios if e["steps"]]
|
|
if actionable:
|
|
d.verdict = f"{len(actionable)} of {len(scenarios)} claimants of {fip} need correcting"
|
|
d.confidence = "high"
|
|
elif len(claimants) <= 1:
|
|
d.verdict = f"Only one production claimant of {fip} - the duplicate is likely outside production"
|
|
d.confidence = "medium"
|
|
d.assessment = (
|
|
"CX-Tools reconciles production Infrahub only. When production shows a single owner, the duplicate is "
|
|
"usually a PreProd or Staging record."
|
|
)
|
|
d.act("Check PreProd and Staging for VMs holding this floating IP (repeat with those API keys).", CX, "check")
|
|
if prom_rows:
|
|
d.note("The Prometheus Resources rows above span every environment - use them to spot the other claimant.")
|
|
else:
|
|
d.verdict = f"{len(scenarios)} claimants of {fip}, none needing a correction"
|
|
d.confidence = "low"
|
|
d.note("Both sides agree for every claimant, so the alert may already be stale. Re-run in 5-10 minutes.")
|
|
|
|
d.assessment = d.assessment or (
|
|
"Each claimant is classified against the Duplicated IPs scenarios. Correct the losing records, then re-run "
|
|
"the query after 5-10 minutes to confirm the alert clears."
|
|
)
|
|
d.act("Re-run the check after ~5-10 minutes to confirm only one VM holds the IP and the alert has cleared.",
|
|
CX, "verify")
|
|
|
|
|
|
# --- total GPUs ------------------------------------------------------------
|
|
|
|
def _diagnose_total_gpus(d: Diagnosis, alert: Alert) -> None:
|
|
"""Problem with Total GPUs in a System."""
|
|
host = alert.host or alert.instance_name
|
|
region = alert.region
|
|
if not host:
|
|
d.error = "This alert carries no hypervisor label."
|
|
return
|
|
|
|
d.verdict = f"Host {host} is reporting missing GPU(s)"
|
|
d.confidence = "high"
|
|
d.assessment = (
|
|
"Monitoring detected a host with fewer GPUs than expected. This hits revenue and customer experience, so the "
|
|
"job is to find out who is on the host and escalate the hardware fault to Infrastructure."
|
|
)
|
|
d.add("Host", host)
|
|
d.add("GPU model", alert.gpu_name or "N/A")
|
|
summary = alert.annotations.get("summary", "")
|
|
count = re.search(r"\*\s*" + re.escape(alert.gpu_name or "") + r":\*\s*`(\d+)`", summary) if alert.gpu_name else None
|
|
if count:
|
|
d.add("GPUs reported present", count.group(1), "bad",
|
|
"Compare against the expected count for this chassis (typically 8).")
|
|
|
|
if region:
|
|
census = cxbridge.host_gpu_census(region, host)
|
|
d.evidence["gpu_census"] = census
|
|
if census.get("ok"):
|
|
customers = [i for i in census.get("instances", [])]
|
|
d.add("Instances on host", str(len(customers)), "warn" if customers else "ok")
|
|
d.add("GPUs allocated to instances", str(census.get("total_gpus")))
|
|
for inst in customers:
|
|
d.add(f"Instance {inst['name']}", f"{inst['status']} - {inst['flavor']} ({inst['gpus']} GPU)", "warn")
|
|
if customers:
|
|
d.act(
|
|
"Customers are on this host - they may need contacting for host maintenance, depending on the "
|
|
"Infrastructure team's assessment.",
|
|
CX, "comms",
|
|
)
|
|
d.act("Run List Instances on a Hypervisor (Windmill) to confirm the customer list.", CX, "check")
|
|
else:
|
|
d.act("No instances on the host - no customer impact to coordinate.", CX, "verify", status="done")
|
|
else:
|
|
d.add("Instance census", f"failed: {census.get('error', '')}", "warn")
|
|
d.act("Run List Instances on a Hypervisor (Windmill) to determine if any customers are on the host.",
|
|
CX, "check")
|
|
else:
|
|
d.act("Run List Instances on a Hypervisor (Windmill) to determine if any customers are on the host.",
|
|
CX, "check")
|
|
|
|
d.act(
|
|
f"Search Jira for an existing OPEN issue for {host}. If one exists, update it with the alert details and ask "
|
|
"whether a new ticket is needed.",
|
|
CX, "check",
|
|
)
|
|
d.act(f"Otherwise raise a Jira for the Infrastructure team with '{host}' in the title, describing the issue and "
|
|
"copying the alert details.", INFRA, "escalate")
|
|
d.note("There is no approved customer template for this alert - host-maintenance comms are coordinated separately.")
|
|
|
|
|
|
# --- single-VM status mismatch ---------------------------------------------
|
|
|
|
def _diagnose_status_mismatch(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
|
|
"""The per-region "Openstack status=X and Infrahub status!=X" rules.
|
|
|
|
Same class of problem as Suspected Rogue VM, scoped to one VM, so it takes
|
|
the same Mismatch Remediation table.
|
|
"""
|
|
ih_status = str(vm.get("ih_status") or "N/A")
|
|
os_status = str(vm.get("os_status") or "N/A")
|
|
entry = _match_mismatch_table(ih_status, os_status)
|
|
|
|
if entry:
|
|
d.verdict = entry["verdict"]
|
|
d.confidence = "high"
|
|
for owner, text, guide in entry["steps"]:
|
|
d.act(text, owner, "remediate", guide)
|
|
if entry.get("comms"):
|
|
draft = _vm_draft(d, entry["comms"], vm, alert)
|
|
if draft:
|
|
d.drafts.append(draft)
|
|
elif not vm.get("mismatches"):
|
|
d.verdict = f"Infrahub and OpenStack now agree ({ih_status} / {os_status})"
|
|
d.confidence = "high"
|
|
d.assessment = "The mismatch has already cleared; the alert should stop on the next evaluation."
|
|
else:
|
|
d.verdict = f"Mismatch ({ih_status} / {os_status}) is not in the runbook table"
|
|
d.confidence = "low"
|
|
d.act("Ping Kheano Martinez or John Priest for a runbook update, and escalate to Infrastructure for next steps.",
|
|
CX, "escalate")
|
|
|
|
d.assessment = d.assessment or (
|
|
"A single-VM state mismatch. The Suspected Rogue VM remediation table covers which side to correct."
|
|
)
|
|
|
|
|
|
# --- dispatch --------------------------------------------------------------
|
|
|
|
_VM_KINDS = {
|
|
"error": _diagnose_error,
|
|
"deleting": _diagnose_deleting,
|
|
"shutoff": _diagnose_shutoff,
|
|
"hibernating": _diagnose_hibernating,
|
|
"creating": _diagnose_creating,
|
|
"restoring": _diagnose_restoring,
|
|
"rebooting": _diagnose_rebooting,
|
|
"build": _diagnose_build,
|
|
"status_mismatch": _diagnose_status_mismatch,
|
|
}
|
|
|
|
|
|
def _screened_out(d: Diagnosis, alert: Alert) -> None:
|
|
"""Report a screened-out alert without spending calls diagnosing it."""
|
|
screen = alert.screen or {}
|
|
d.verdict = f"No action needed - {screen.get('label', 'screened out')}"
|
|
d.confidence = "high"
|
|
d.assessment = str(screen.get("reason") or "")
|
|
d.add("Screening verdict", screen.get("label", ""), "ok")
|
|
if screen.get("current_state"):
|
|
d.add("Current state", screen["current_state"], "ok")
|
|
if screen.get("detail"):
|
|
d.note(str(screen["detail"]))
|
|
d.act(
|
|
"Nothing to do. If the alert is still up in Prometheus it should clear on the next evaluation; "
|
|
"if it does not, the rule may need a silence or a fix.",
|
|
CX, "verify", status="done",
|
|
)
|
|
d.note("Re-run with 'Diagnose anyway' to query Infrahub and OpenStack directly for this alert.")
|
|
|
|
|
|
def _add_screening_findings(d: Diagnosis, alert: Alert) -> None:
|
|
screen = alert.screen or {}
|
|
if not screen:
|
|
return
|
|
tone = {"real": "bad", "unverified": "warn", "chronic": "warn",
|
|
"low_impact": "warn", "pending": "warn", "resolved": "ok"}.get(str(screen.get("verdict")), "info")
|
|
d.add("Screening", f"{screen.get('label', '')} - {screen.get('reason', '')}", tone,
|
|
str(screen.get("detail") or ""))
|
|
|
|
|
|
def artifact_now(d: Diagnosis) -> bool:
|
|
return bool((d.visual or {}).get("spare_capacity_artifact"))
|
|
|
|
|
|
def _add_rogue_gpu_gap(d: Diagnosis, alert: Alert, snap: Any) -> None:
|
|
"""Show the GPU accounting gap the Rogue VM rule actually fires on.
|
|
|
|
The rule is `sum by(instance)(In_Use_Gpus) - sum by(instance)(Resources{
|
|
status=~"ACTIVE|SHUTOFF|PRE_ACTIVE"}) >= 1` - a per-host GPU accounting gap,
|
|
not a status comparison. Two different faults produce that gap and Prometheus
|
|
cannot tell them apart, so both are stated and the host reconciliation below
|
|
is what settles it.
|
|
"""
|
|
host = alert.host or alert.instance_name
|
|
if snap is None or not host:
|
|
return
|
|
delta = getattr(snap, "rogue_delta", {}).get(host)
|
|
if delta is None:
|
|
return
|
|
|
|
total = getattr(snap, "total_gpus", {}).get(host)
|
|
in_use = getattr(snap, "in_use_gpus", {}).get(host)
|
|
rows = getattr(snap, "resources_by_host", {}).get(host, [])
|
|
counted = sum(int(r.get("_gpus", "0") or 0) for r in rows
|
|
if r.get("status", "").upper() in ("ACTIVE", "SHUTOFF", "PRE_ACTIVE"))
|
|
|
|
artifact = in_use is not None and total is not None and in_use == total
|
|
d.visual = {
|
|
"type": "gpu",
|
|
"host": host,
|
|
"physical": int(total) if total is not None else None,
|
|
"in_use_metric": int(in_use) if in_use is not None else None,
|
|
"accounted": counted,
|
|
"gap": int(delta),
|
|
"instances": len(rows),
|
|
# When the rule's "in use" reading is just the physical count, the gap
|
|
# it reports is spare capacity, not a missing VM.
|
|
"spare_capacity_artifact": artifact,
|
|
}
|
|
|
|
if total is not None:
|
|
d.add("GPUs on host (physical)", str(int(total)))
|
|
if in_use is not None:
|
|
d.add("GPUs allocated on host", str(int(in_use)))
|
|
d.add("GPUs Infrahub accounts for", str(counted))
|
|
d.add("Accounting gap", f"{int(delta)} GPU(s)", "bad" if delta >= 1 else "ok",
|
|
"This is the quantity the alert fired on.")
|
|
|
|
if rows:
|
|
by_status: dict[str, int] = {}
|
|
for row in rows:
|
|
by_status[row.get("status", "?")] = by_status.get(row.get("status", "?"), 0) + 1
|
|
d.add("Infrahub VMs on host", ", ".join(f"{v} {k}" for k, v in sorted(by_status.items())))
|
|
else:
|
|
d.add("Infrahub VMs on host", "none recorded", "bad")
|
|
|
|
if delta and delta >= 1 and not artifact_now(d):
|
|
d.add(
|
|
"Why there is no ID to chase", "nothing in OpenStack claims these GPUs", "warn",
|
|
"`server list --host` is the complete set of instances Nova knows on this host, and their flavours "
|
|
"account for fewer GPUs than the host reports in use. There is no instance UUID to look up because no "
|
|
"instance owns them - which is exactly why this is a host-level escalation. Identifying them means "
|
|
"looking at the host itself.",
|
|
)
|
|
d.evidence["host_probe_commands"] = [
|
|
f"{alert.region} server list --all-projects --host {host} -c ID -c Name -c Status -c Flavor",
|
|
f"{alert.region} hypervisor show {host} -f json",
|
|
f"ssh {host} nvidia-smi --query-gpu=index,uuid,pci.bus_id --format=csv",
|
|
f"ssh {host} 'virsh list --all'",
|
|
]
|
|
|
|
if delta and delta >= 1:
|
|
unattributed = getattr(snap, "unattributed_active", 0)
|
|
unattributed_gpus = getattr(snap, "unattributed_active_gpus", 0)
|
|
d.add(
|
|
"Two possible causes", "instances OpenStack has that Infrahub does not, or Infrahub VMs with no host set",
|
|
"warn",
|
|
"Either there are instances running on this host with no Infrahub record (a true rogue VM), or Infrahub "
|
|
"has ACTIVE VMs whose host field is unset, so they are not counted against this host. The per-instance "
|
|
"reconciliation below distinguishes them: a genuine rogue VM shows up as 'Infrahub Missing'.",
|
|
)
|
|
if unattributed:
|
|
d.note(
|
|
f"Platform-wide, Infrahub currently has {unattributed} ACTIVE/SHUTOFF VM(s) ({unattributed_gpus} GPUs) "
|
|
"with no host recorded. That is enough to explain gaps like this one without any rogue VM existing, "
|
|
"so confirm against the per-instance list before escalating."
|
|
)
|
|
|
|
|
|
def diagnose(alert: Alert, prom: Any = None, snap: Any = None, force: bool = False,
|
|
user_settings: Any = None) -> Diagnosis:
|
|
"""Gather evidence for one alert and reach the runbook's verdict."""
|
|
d = Diagnosis(alert=alert)
|
|
d.agent_name = getattr(user_settings, "agent_name", "") or ""
|
|
|
|
if not force and alert.screen and not alert.screen.get("actionable", True):
|
|
_screened_out(d, alert)
|
|
return d
|
|
|
|
_common_preamble(d, alert)
|
|
_add_screening_findings(d, alert)
|
|
|
|
try:
|
|
if alert.kind in ("rogue_vm", "orphan_vm"):
|
|
_add_rogue_gpu_gap(d, alert, snap)
|
|
_diagnose_rogue_vm(d, alert)
|
|
elif alert.kind == "duplicate_ip":
|
|
_diagnose_duplicate_ip(d, alert, prom)
|
|
elif alert.kind == "total_gpus":
|
|
_diagnose_total_gpus(d, alert)
|
|
elif alert.kind in _VM_KINDS:
|
|
target, org_id = _vm_target(alert)
|
|
if not target:
|
|
d.error = "This alert has neither an OpenStack ID nor an instance name to look up."
|
|
return d
|
|
vm = cxbridge.collect_vm(target, region=alert.region, org_id=org_id)
|
|
d.evidence["vm_result"] = cxbridge.json_safe(vm)
|
|
if not vm.get("ok"):
|
|
d.error = str(vm.get("error") or f"CX-Tools returned no usable telemetry for {target}.")
|
|
if alert.kind == "creating":
|
|
# A CREATING VM that never reached OpenStack legitimately has
|
|
# nothing to collect; the runbook conclusion still holds.
|
|
d.error = ""
|
|
d.add("CX-Tools lookup", "no telemetry available", "bad",
|
|
str(vm.get("error") or ""))
|
|
_diagnose_creating(d, alert, vm)
|
|
_attach_contacts(d, vm)
|
|
return d
|
|
_add_vm_findings(d, vm)
|
|
_VM_KINDS[alert.kind](d, alert, vm)
|
|
_attach_contacts(d, vm)
|
|
else:
|
|
d.error = f"Alert kind '{alert.kind}' is not covered by the CX runbooks."
|
|
except cxbridge.BridgeError as exc:
|
|
d.error = str(exc)
|
|
except Exception as exc: # surface, don't crash the request
|
|
d.error = f"Diagnosis failed: {type(exc).__name__}: {exc}"
|
|
|
|
return d
|