cx-ui/triagelib/runbooks.py

"""The diagnosis engine: CX runbooks expressed as decisions over CX-Tools data.

Each alert kind maps to one Confluence runbook. For every alert this module
gathers the evidence the runbook asks for (via the read-only CX-Tools
collectors), reaches the verdict the runbook's decision table implies, and emits
the remaining steps a human still has to perform. Nothing here mutates state.
"""
from __future__ import annotations

import re
from dataclasses import dataclass, field
from typing import Any, Optional

from . import comms, cxbridge
from .alerts import Alert

CX = "CX"
INFRA = "Infrastructure team"
DEVOPS = "DevOps"
CUSTOMER = "Customer"

TRANSITIONAL = {"BUILD", "CREATING", "DELETING", "HIBERNATING", "RESTORING", "REBOOTING"}


# --- result model -----------------------------------------------------------

@dataclass
class Finding:
    label: str
    value: str
    tone: str = "info"          # ok | warn | bad | info
    detail: str = ""

    def to_json(self) -> dict[str, Any]:
        return {"label": self.label, "value": self.value, "tone": self.tone, "detail": self.detail}


@dataclass
class Action:
    text: str
    owner: str = CX
    kind: str = "check"         # check | remediate | escalate | comms | verify
    guide: str = ""
    status: str = "todo"        # todo | done | skipped  (done = the app verified it)
    detail: str = ""

    def to_json(self) -> dict[str, Any]:
        return {
            "text": self.text, "owner": self.owner, "kind": self.kind,
            "guide": self.guide, "status": self.status, "detail": self.detail,
        }


@dataclass
class Diagnosis:
    alert: Alert
    verdict: str = ""
    assessment: str = ""
    confidence: str = "medium"   # high | medium | low
    findings: list[Finding] = field(default_factory=list)
    actions: list[Action] = field(default_factory=list)
    drafts: list[comms.Draft] = field(default_factory=list)
    contacts: dict[str, Any] = field(default_factory=dict)
    evidence: dict[str, Any] = field(default_factory=dict)
    notes: list[str] = field(default_factory=list)
    error: str = ""
    # A small structured description of the problem, so the UI can draw it
    # rather than make the reader parse a table.
    visual: dict[str, Any] = field(default_factory=dict)
    agent_name: str = ""

    def add(self, label: str, value: Any, tone: str = "info", detail: str = "") -> None:
        text = str(value if value not in (None, "") else "N/A")
        self.findings.append(Finding(label, text, tone, detail))

    def act(self, text: str, owner: str = CX, kind: str = "check", guide: str = "",
            status: str = "todo", detail: str = "") -> None:
        self.actions.append(Action(text, owner, kind, guide, status, detail))

    def note(self, text: str) -> None:
        if text and text not in self.notes:
            self.notes.append(text)

    def to_json(self) -> dict[str, Any]:
        from . import integrations

        return {
            "integrations": integrations.build_all(self),
            "alert": self.alert.to_json(),
            "verdict": self.verdict,
            "assessment": self.assessment,
            "confidence": self.confidence,
            "findings": [f.to_json() for f in self.findings],
            "actions": [a.to_json() for a in self.actions],
            "drafts": [d.to_json() for d in self.drafts],
            "contacts": self.contacts,
            "evidence": self.evidence,
            "notes": self.notes,
            "error": self.error,
            "visual": self.visual,
        }


# --- the ERROR-state fault table -------------------------------------------
# Transcribed from the fault table in the "Instance in ERROR state" runbook.

FAULT_TABLE: list[dict[str, Any]] = [
    {
        "id": "conflict_image_pending_upload",
        "pattern": r"Conflict updating instance.*task_state.*image_pending_upload",
        "summary": "Instance was deleted while it was hibernating.",
        "owner": f"{INFRA} / {CX}",
        "guidance": "Known case: the instance was deleted mid-hibernation. It just needs deleting again in OpenStack.",
    },
    {
        "id": "no_valid_host",
        "pattern": r"No valid host was found|not enough hosts available",
        "summary": "The scheduler could not place the instance - stock shortage or a race condition.",
        "owner": INFRA,
        "guidance": "Judge whether there was enough stock at the time; ask a peer if unsure. If there should have been "
                    "enough, try reproducing with the same flavor in pre-prod - otherwise it is most likely a race "
                    "condition from several near-simultaneous API creates.",
        "comms": "error_never_active",
        "comms_note": "Contact the customer if the instance is still not deleted after a day.",
    },
    {
        "id": "numa_pci",
        "pattern": r"NUMA topology together with requested PCI devices|Claim pci failed",
        "summary": "NUMA/PCI claim failure on the host.",
        "owner": INFRA,
        "guidance": "Check whether the host is full BEFORE escalating. If the summed GPU count across the host's "
                    "instances is 8, the host is full and that proves the host itself is fine.",
        "gpu_census": True,
    },
    {
        "id": "lvremove",
        "pattern": r"Failed to remove volume\(s\)|lvremove",
        "summary": "A stale LVM volume on the host blocked the build.",
        "owner": INFRA,
        "guidance": "Raise a Jira for Infrastructure to investigate the host and the instance error.",
    },
    {
        "id": "pci_header",
        "pattern": r"Unknown PCI header type",
        "summary": "Host PCI device is reporting an invalid header - hardware/host fault.",
        "owner": INFRA,
        "guidance": "Raise a Jira for Infrastructure to investigate the host and the instance error.",
    },
    {
        "id": "pci_in_use",
        "pattern": r"PCI device \S+ is in use by driver QEMU",
        "summary": "The GPU is still claimed by another domain on the host.",
        "owner": INFRA,
        "guidance": "Raise a Jira for Infrastructure to investigate the host and the instance error.",
    },
    {
        "id": "client_socket_closed",
        "pattern": r"internal error: client socket is closed",
        "summary": "libvirt client socket closed during the operation.",
        "owner": INFRA,
        "guidance": "Can be transient, but it is associated with host issues - escalate to be safe.",
    },
]


def match_faults(texts: list[str]) -> list[dict[str, Any]]:
    """Match collected fault text against the runbook fault table."""
    blob = "\n".join(t for t in texts if t)
    matched: list[dict[str, Any]] = []
    for entry in FAULT_TABLE:
        if re.search(entry["pattern"], blob, re.I | re.S):
            matched.append(entry)
    return matched


# --- shared evidence gathering ---------------------------------------------

def _vm_target(alert: Alert) -> tuple[str, Optional[str]]:
    """Pick the best CX-Tools lookup target for this alert.

    OpenStack ID is preferred. CREATING alerts routinely carry
    openstack_id="None" because the VM never reached OpenStack, so those fall
    back to the instance name plus the org id parsed from the alert.
    """
    if alert.openstack_id:
        return alert.openstack_id, None
    if alert.instance_name and alert.org_id:
        return alert.instance_name, alert.org_id
    return alert.instance_name, None


def _tone_for_status_pair(ih_status: str, os_status: str, task_state: str) -> str:
    try:
        ok = cxbridge.cx().status_pair_ok(ih_status, os_status, task_state)
    except Exception:
        ok = ih_status.upper() == os_status.upper()
    return "ok" if ok else "bad"


def _add_vm_findings(d: Diagnosis, vm: dict[str, Any]) -> None:
    ih_status = str(vm.get("ih_status") or "N/A")
    os_status = str(vm.get("os_status") or "N/A")
    task_state = str(vm.get("task_state") or "None")
    host = str(vm.get("host") or "N/A")

    d.visual = {
        "type": "states",
        "infrahub": ih_status,
        "openstack": os_status,
        "task": task_state,
        "match": _tone_for_status_pair(ih_status, os_status, task_state) == "ok",
        "host": host,
        "name": str(vm.get("name") or ""),
        "flavor": str(vm.get("flavor") or ""),
        "gpus": str(vm.get("gpu_count") or ""),
        "fault": str(vm.get("openstack_fault") or "None"),
        "never_built": host in ("N/A", "Unknown", ""),
    }

    d.add("Infrahub ID", vm.get("infrahub_id"))
    d.add("OpenStack ID", vm.get("openstack_id"))
    d.add("Name", vm.get("name"))
    d.add("Region", vm.get("region_display") or vm.get("region"))
    d.add("Infrahub status", ih_status, "info")
    d.add("OpenStack status", os_status,
          _tone_for_status_pair(ih_status, os_status, task_state) if "N/A" not in (ih_status, os_status) else "warn")
    d.add("Task state", task_state, "warn" if task_state not in ("None", "N/A") else "info")
    d.add("Hypervisor", host, "warn" if host in ("N/A", "Unknown") else "info")
    d.add("Flavor", vm.get("flavor"))
    d.add("GPU(s)", vm.get("gpu_count"))
    d.add("Floating IP", vm.get("floating_ip"))
    d.add("Created", vm.get("created"))
    d.add("SSH", vm.get("ssh_text"), "ok" if vm.get("ssh_raw") == "reachable" else "info")
    d.add("Volumes", vm.get("volumes_summary"))
    if vm.get("project_name"):
        d.add("OpenStack project", vm.get("project_name"))
    if vm.get("project_environment"):
        d.add("Environment", vm.get("project_environment"), "warn",
              "Found outside production - Infrahub production records will not match.")

    os_fault = str(vm.get("openstack_fault") or "None")
    d.add("OpenStack fault", os_fault, "bad" if os_fault not in ("None", "N/A") else "ok")
    faults = vm.get("faults") or []
    d.add("InfraInsight faults", f"{len(faults)} recorded" if faults else "None", "bad" if faults else "ok")

    for item in vm.get("mismatches") or []:
        check, detail = cxbridge.cx().mismatch_parts(item)
        d.add(f"Mismatch: {check}", detail, "bad")

    for reason in vm.get("warn_reasons") or []:
        d.add("Warning", str(reason), "warn")
    for note in vm.get("info_notes") or []:
        d.note(str(note))


def _fault_texts(vm: dict[str, Any], failed_event: dict[str, Any]) -> list[str]:
    texts: list[str] = []
    os_fault = str(vm.get("openstack_fault") or "")
    if os_fault and os_fault not in ("None", "N/A"):
        texts.append(os_fault)
    for row in vm.get("faults") or []:
        texts.extend(str(x) for x in row)
    server = vm.get("server") if isinstance(vm.get("server"), dict) else {}
    fault = server.get("fault")
    if isinstance(fault, dict):
        texts.append(str(fault.get("message") or ""))
        texts.append(str(fault.get("details") or ""))
    elif fault:
        texts.append(str(fault))
    rows = failed_event.get("rows") or {}
    for key in ("Detail", "Traceback", "Result"):
        if rows.get(key) and rows[key] != "N/A":
            texts.append(str(rows[key]))
    return texts


def _host_health_findings(d: Diagnosis, region: str, host: str) -> dict[str, Any]:
    """Run the cheap host signals and turn them into findings + a verdict."""
    if not host or host in ("N/A", "Unknown") or not region:
        d.act("Identify the hypervisor, then run Host Health Checks manually.", CX, "check", "Host Health Checks")
        d.note("No hypervisor was recorded on this alert, so host health could not be checked automatically.")
        return {}

    health = cxbridge.host_health(region, host)
    if not health.get("ok"):
        d.add("Host health", f"lookup failed: {health.get('error', '')}", "warn")
        d.act(f"Check host {host} manually.", CX, "check", "Host Health Checks")
        return health

    nova_state = str(health.get("nova_state") or "")
    nova_status = str(health.get("nova_status") or "")
    ovs_alive = health.get("ovs_alive")
    ovs_state = str(health.get("ovs_state") or "")

    bad_signals: list[str] = []
    d.add("Host", host)
    d.add("Nova state", nova_state, "ok" if nova_state.lower() == "up" else "bad")
    if nova_state.lower() != "up":
        bad_signals.append(f"Nova state is {nova_state}")
    d.add("Nova status", nova_status, "ok" if nova_status.lower() == "enabled" else "bad")
    if nova_status.lower() != "enabled":
        bad_signals.append(f"Nova status is {nova_status}")
    if health.get("disabled_reason"):
        d.add("Disabled reason", health["disabled_reason"], "bad")
        bad_signals.append(f"disabled: {health['disabled_reason']}")
    if ovs_alive is not None:
        d.add("OVS alive", str(ovs_alive), "ok" if ovs_alive in (True, "True", "true") else "bad")
        if ovs_alive not in (True, "True", "true"):
            bad_signals.append("OVS agent is not alive")
    if ovs_state:
        d.add("OVS state", ovs_state, "ok" if ovs_state.lower() == "up" else "bad")
        if ovs_state.lower() != "up":
            bad_signals.append(f"OVS state is {ovs_state}")
    d.add("Host uptime", health.get("uptime"))
    d.add("Aggregates", health.get("aggregates"))

    health["bad_signals"] = bad_signals
    if bad_signals:
        d.act(
            f"Host {host} shows problems ({'; '.join(bad_signals)}) - escalate to the Infrastructure team.",
            INFRA, "escalate", "Host Health Checks",
        )
    else:
        d.act(
            f"Nova and OVS signals on {host} look healthy; complete the remaining Host Health Checks "
            "(disk/dmesg/GPU checks the guide covers and this app does not).",
            CX, "check", "Host Health Checks",
        )
    return health


def _vm_draft(d: Diagnosis, template: str, vm: dict[str, Any], alert: Alert,
              *, floating_ip: str = "", note: Optional[str] = None) -> Optional[comms.Draft]:
    """Build a customer draft with the VM identity the house style expects."""
    owners = (vm.get("owners") or []) if isinstance(vm, dict) else []
    return comms.draft(
        template,
        instance_name=str(vm.get("name") or alert.instance_name or ""),
        infrahub_id=str(vm.get("infrahub_id") or ""),
        openstack_id=str(vm.get("openstack_id") or alert.openstack_id or ""),
        greeting_name=comms.first_name(owners[0] if owners else ""),
        agent_name=d.agent_name,
        floating_ip=floating_ip,
        note=note,
    )


def _attach_contacts(d: Diagnosis, vm: dict[str, Any]) -> None:
    d.contacts = comms.contacts_from_result(vm)
    if not d.contacts.get("resolved"):
        d.note(
            "No organization owner contacts came back from Infrahub. Look the organization up in the Admin "
            "Portal before sending anything."
        )


def _common_preamble(d: Diagnosis, alert: Alert) -> None:
    if alert.is_kubernetes:
        d.note(
            f"Instance name '{alert.instance_name}' looks like a Kubernetes node. Per the general process, treat it "
            "as part of a K8s cluster - deleting a single node may be handled by the cluster instead."
        )
    if alert.threshold_min and alert.age_minutes is not None and alert.age_minutes < alert.threshold_min:
        d.note(
            f"This alert has only been active {alert.age_minutes} min against a {alert.threshold_min} min threshold; "
            "it may still clear on its own."
        )


# --- per-kind runbooks -----------------------------------------------------

def _diagnose_error(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
    """Instance in ERROR state."""
    host = str(vm.get("host") or "N/A")
    os_status = str(vm.get("os_status") or "N/A")
    ih_status = str(vm.get("ih_status") or "N/A")
    region = str(vm.get("region") or alert.region)
    osid = str(vm.get("openstack_id") or alert.openstack_id)

    failed_event: dict[str, Any] = {}
    if osid and osid != "N/A" and region:
        failed_event = cxbridge.failed_openstack_event(region, osid)
    if failed_event:
        rows = failed_event.get("rows") or {}
        d.add("Failed OpenStack event", f"{failed_event.get('action', '?')} ({failed_event.get('request_id', '')})", "bad",
              rows.get("Detail", ""))
        d.evidence["failed_event"] = failed_event

    # "We are more concerned if a VM goes into ERROR and it was previously
    # ACTIVE" - no hypervisor means it never fully created.
    was_active = host not in ("N/A", "Unknown", "")
    d.add("Reached ACTIVE previously", "yes - a hypervisor is recorded" if was_active else "no - never placed on a host",
          "bad" if was_active else "warn",
          "" if was_active else "No hostname means the instance was never created fully and could not have become ACTIVE.")

    texts = _fault_texts(vm, failed_event)
    matches = match_faults(texts)
    d.evidence["fault_text"] = [t for t in texts if t][:12]

    for row in vm.get("faults") or []:
        d.add("InfraInsight fault", " | ".join(str(x) for x in row), "bad")

    if matches:
        names = "; ".join(m["summary"] for m in matches)
        d.verdict = f"ERROR with a known fault: {names}"
        d.confidence = "high"
        for m in matches:
            d.act(m["guidance"], m["owner"], "escalate" if m["owner"] != CX else "remediate")
            if m.get("gpu_census") and host not in ("N/A", "Unknown") and region:
                census = cxbridge.host_gpu_census(region, host)
                d.evidence["gpu_census"] = census
                if census.get("ok"):
                    total = census.get("total_gpus")
                    full = total is not None and total >= 8
                    d.add("GPUs allocated on host", f"{total} (from {len(census.get('instances', []))} instances)",
                          "ok" if full else "warn",
                          "Host is FULL - this proves the host itself is fine." if full
                          else "Host is not full, so the NUMA/PCI failure is not simple capacity. Raise a Jira for Infra.")
                    if full:
                        d.act("Host is full (8 GPUs allocated) - this proves the host is fine. No Infra escalation needed "
                              "for capacity; treat as a race condition.", CX, "verify", status="done")
                    else:
                        d.act(f"Raise a Jira for the Infrastructure team to investigate host {host} and the instance error.",
                              INFRA, "escalate")
    else:
        d.verdict = "ERROR with no fault matching the runbook table"
        d.confidence = "low" if not texts else "medium"
        d.act(
            "Fault is not in the runbook table - escalate to a peer, and add the fault to the runbook table once known.",
            CX, "escalate",
        )

    if ih_status.upper() == "ERROR" and os_status.upper() not in ("ERROR", "N/A"):
        d.add("Note", f"Infrahub says ERROR but OpenStack says {os_status}", "warn")

    d.assessment = (
        "The instance had reached ACTIVE, so customer data may be on it - remediate rather than leaving it. "
        if was_active else
        "The instance never became ACTIVE, so there is no customer data to protect; it needs deleting and recreating. "
    )
    if not was_active:
        d.assessment += "Customers usually just delete ERROR VMs and retry."

    d.act("Record the fault message and event output in the Slack alert thread and any HubSpot ticket.", CX, "verify")

    created = str(vm.get("created") or "")
    template = "error_was_active" if was_active else "error_never_active"
    note = None
    for m in matches:
        if m.get("comms"):
            template = m["comms"]
            note = m.get("comms_note")
    draft = _vm_draft(d, template, vm, alert, note=note)
    if draft:
        if not was_active:
            draft.when += " Outreach is optional and only if the instance is less than 7 days old."
            if created and created != "N/A":
                draft.when += f" Created: {created}."
        d.drafts.append(draft)


def _diagnose_deleting(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
    """Instance in DELETING state."""
    ih_status = str(vm.get("ih_status") or "N/A")
    os_status = str(vm.get("os_status") or "N/A")
    server_exists = bool(vm.get("server"))

    d.verdict = f"Stuck in DELETING for {alert.age_minutes or '?'} min - unlikely to finish on its own"
    d.confidence = "high"
    d.assessment = (
        "The customer's intent is clear (they asked for a delete), so the remediation is to finish the delete. "
    )

    delete_requests = [
        row for row in (vm.get("ih_events_all") or vm.get("ih_events") or [])
        if "delete" in " ".join(str(x) for x in row).lower()
    ]
    d.evidence["delete_requests"] = delete_requests[:5]
    if delete_requests:
        d.add("Delete request found", delete_requests[0][0], "ok", " | ".join(str(x) for x in delete_requests[0][1:]))
        d.act("Delete request confirmed in Infrahub events - customer intent verified.", CX, "verify", status="done")
    else:
        d.add("Delete request found", "none in Infrahub events", "warn")
        d.act(
            "Confirm who requested the deletion via the InfraInsight InstanceDeleteRequest query. If no record "
            "exists, escalate to DevOps - it could be an admin action or an anomaly.",
            DEVOPS, "check",
        )
        d.note(
            "Requester attribution (name/email) comes from the InfraInsight SQL query in the runbook; CX-Tools "
            "exposes the Infrahub event but not the requesting user."
        )

    never_built = str(vm.get("host") or "N/A") in ("N/A", "Unknown", "")

    if server_exists:
        d.add("OpenStack server", f"still present ({os_status})", "bad")
        d.act(f"Delete the server in OpenStack ({vm.get('openstack_id')}).", CX, "remediate", "Deleting an Instance")
        if never_built:
            # It reached OpenStack but never landed on a host, so this is not a
            # normal delete: the build failed and the record still needs closing.
            d.add("Reached a host", "no - the build never completed", "warn",
                  "The instance failed to build, so the customer's delete could never finish normally.")
            d.assessment += (
                "The instance never made it onto a host, so removing the OpenStack server is only half of it - "
                "the Infrahub record has to be closed out too. "
            )
    else:
        d.add("OpenStack server", "already gone", "ok")
        d.assessment += (
            "The server is already gone from OpenStack, so this is an Infrahub-side record that never closed out. "
        )

    # Deleting the OpenStack server does not clear the Infrahub record. Whatever
    # broke the delete the first time will still leave it in DELETING, and it is
    # the record that keeps the alert firing and the resource on the books.
    if ih_status.upper() == "DELETING":
        d.act(
            "Mark the instance deleted in InfraInsight so the Infrahub record closes out - removing the OpenStack "
            "server alone leaves it stuck in DELETING and the alert still firing.",
            CX, "remediate", "Infrahub Insights (Infra-Insight)",
        )
    else:
        d.add("Note", f"Infrahub now reports {ih_status}, not DELETING - the alert may already be stale", "warn")
        d.confidence = "medium"

    d.act("Confirm the record is gone from Admin Portal Production and the alert clears.", CX, "verify")

    for template in ("deleting", "deleting_resolved"):
        made = _vm_draft(d, template, vm, alert)
        if made:
            d.drafts.append(made)
    d.note(
        "The customer asked for this delete, so contact is optional - but your sent examples show CX does confirm "
        "it. Two drafts are offered: a proactive notice, and a reply that closes an existing ticket."
    )


def _diagnose_shutoff(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
    """Instance in SHUTOFF state."""
    os_status = str(vm.get("os_status") or "N/A")
    d.verdict = "Customer-initiated SHUTOFF - billing awareness notice required"
    d.confidence = "high"
    d.assessment = (
        "SHUTOFF is a result of customer action, not a platform fault. The only job here is making sure the "
        "customer knows a SHUTOFF VM still accrues full cost."
    )
    if os_status.upper() == "SHUTOFF":
        d.act("Confirmed SHUTOFF in OpenStack.", CX, "verify", status="done")
    else:
        d.add("Note", f"OpenStack reports {os_status}, not SHUTOFF - the alert may be stale", "warn")
        d.confidence = "medium"
    d.act("Create a HubSpot ticket and send the billing awareness note (snippet: #shutoff).", CX, "comms",
          "HubSpot Ticket Creation")
    draft = _vm_draft(d, "shutoff", vm, alert)
    if draft:
        d.drafts.append(draft)


def _diagnose_hibernating(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
    """Instance in HIBERNATING state."""
    region = str(vm.get("region") or alert.region)
    host = str(vm.get("host") or alert.host or "N/A")
    d.verdict = "Stuck HIBERNATING - almost certainly a host issue"
    d.confidence = "high"
    d.assessment = (
        "The runbook treats stuck HIBERNATING as almost certainly a host problem. Check the host, then drive the "
        "shelve to completion."
    )
    health = _host_health_findings(d, region, host)
    d.evidence["host_health"] = health
    d.act("Complete the shelve for the instance so it lands in SHELVED_OFFLOADED.", CX, "remediate",
          "Shelving an Instance")
    d.act("Confirm the instance ends up HIBERNATED in Infrahub and SHELVED_OFFLOADED in OpenStack.", CX, "verify")
    if health.get("bad_signals"):
        d.verdict = f"Stuck HIBERNATING due to a host problem on {host}"
    d.note("This runbook has no customer comms step of its own.")


def _diagnose_creating(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
    """Instance in CREATING state."""
    osid = str(vm.get("openstack_id") or "")
    has_osid = bool(osid) and osid != "N/A"
    server_exists = bool(vm.get("server"))

    d.confidence = "high"
    if not has_osid:
        d.verdict = "Stuck CREATING and never got an OpenStack ID - the VM does not exist in OpenStack"
        d.add("OpenStack ID", "never assigned", "bad",
              "The instance never reached OpenStack, so it cannot be recovered and must be recreated.")
        d.assessment = (
            "The instance never got an OpenStack ID, so nothing was ever built. The customer cannot delete it "
            "themselves in this transitional state - CX has to clear it and tell them to retry."
        )
    elif not server_exists:
        d.verdict = "Stuck CREATING with an OpenStack ID, but no server exists in OpenStack"
        d.assessment = "Infrahub holds an OpenStack ID that OpenStack does not know about; the record needs clearing."
    else:
        d.verdict = f"Stuck CREATING while OpenStack reports {vm.get('os_status')}"
        d.confidence = "medium"
        d.assessment = "The server does exist in OpenStack, so this is a sync failure rather than a failed build."

    d.act("Confirm the instance and its Infrahub ID in the Admin Portal (Billing -> Manage Organization -> Resources).",
          CX, "check")
    d.act("Delete the stuck instance - it cannot be recovered in this state.", CX, "remediate", "Deleting an Instance")
    d.act("Create a HubSpot ticket and tell the customer they can retry deploying.", CX, "comms",
          "HubSpot Ticket Creation")
    draft = _vm_draft(d, "creating", vm, alert)
    if draft:
        d.drafts.append(draft)


def _diagnose_restoring(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
    """Instance in RESTORING state."""
    region = str(vm.get("region") or alert.region)
    host = str(vm.get("host") or alert.host or "N/A")
    osid = str(vm.get("openstack_id") or "")

    d.verdict = "Stuck RESTORING - unshelve did not complete"
    d.confidence = "high"
    d.assessment = (
        "A stuck restore blocks the customer from reaching their data. Larger images can be slow, but past an hour "
        "this needs investigating."
    )
    health = _host_health_findings(d, region, host)
    d.evidence["host_health"] = health

    failed_event = cxbridge.failed_openstack_event(region, osid) if osid and osid != "N/A" and region else {}
    if failed_event:
        rows = failed_event.get("rows") or {}
        d.add("Failed OpenStack event", f"{failed_event.get('action', '?')} ({failed_event.get('request_id', '')})",
              "bad", rows.get("Detail", ""))
        d.evidence["failed_event"] = failed_event
        d.act("Escalate the failed unshelve event and its fault to the Infrastructure team.", INFRA, "escalate")
    elif not health.get("bad_signals"):
        d.add("Failed OpenStack event", "none found in recent events", "warn")
        d.act("Analyze the unshelve event in OpenStack for fault information, then escalate it to Infrastructure.",
              INFRA, "escalate")

    d.act("Once resolved, shelve the instance so it returns to its pre-restore state and the customer can retry.",
          CX, "remediate", "Shelving an Instance")
    d.act("Validate the instance is SHELVED_OFFLOADED in OpenStack.", CX, "verify")
    d.act("Set the instance state back to HIBERNATED in InfraInsight.", CX, "remediate",
          "Infrahub Insights (Infra-Insight)")
    d.act("Create a HubSpot ticket telling the customer they can retry restoring.", CX, "comms",
          "HubSpot Ticket Creation")
    draft = _vm_draft(d, "restoring", vm, alert)
    if draft:
        d.drafts.append(draft)


def _diagnose_rebooting(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
    """Instance in REBOOTING state."""
    region = str(vm.get("region") or alert.region)
    host = str(vm.get("host") or alert.host or "N/A")
    os_status = str(vm.get("os_status") or "N/A")
    task_state = str(vm.get("task_state") or "None")

    d.verdict = "Stuck REBOOTING - the reboot did not complete"
    d.confidence = "high"
    d.assessment = "Instances get stuck rebooting from host problems; confirm the reboot request, then escalate."

    health = _host_health_findings(d, region, host)
    d.evidence["host_health"] = health

    reboot_events = [
        row for row in (vm.get("ih_events_all") or vm.get("ih_events") or [])
        if "reboot" in " ".join(str(x) for x in row).lower()
    ]
    d.evidence["reboot_requests"] = reboot_events[:5]
    if reboot_events:
        d.add("InstanceRebootRequest", reboot_events[0][0], "ok", " | ".join(str(x) for x in reboot_events[0][1:]))
        d.act("Reboot request confirmed in Infrahub events.", CX, "verify", status="done")
    else:
        d.add("InstanceRebootRequest", "not found in Infrahub events", "warn")
        d.act("Confirm the instance received a HARD_REBOOT request (event InstanceRebootRequest) in the Admin Portal.",
              CX, "check")

    expected = os_status.upper() == "HARD_REBOOT" or "REBOOT" in task_state.upper()
    d.add("OpenStack reboot state", f"status={os_status}, task_state={task_state}", "ok" if expected else "warn",
          "" if expected else "The runbook expects status HARD_REBOOT while a reboot is in flight.")

    failed_event = cxbridge.failed_openstack_event(region, str(vm.get("openstack_id") or "")) \
        if vm.get("openstack_id") not in (None, "", "N/A") and region else {}
    if failed_event:
        rows = failed_event.get("rows") or {}
        d.add("Failed OpenStack event", f"{failed_event.get('action', '?')} ({failed_event.get('request_id', '')})",
              "bad", rows.get("Detail", ""))
        d.evidence["failed_event"] = failed_event

    d.act("Escalate the instance event to the Infrastructure team for review.", INFRA, "escalate")
    d.act("Once cleared, confirm the instance is ACTIVE in both Infrahub and OpenStack.", CX, "verify")
    d.act("Create a HubSpot ticket telling the customer they can retry rebooting.", CX, "comms",
          "HubSpot Ticket Creation")
    draft = _vm_draft(d, "rebooting", vm, alert)
    if draft:
        d.drafts.append(draft)


def _diagnose_build(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
    """Instance in BUILD state."""
    region = str(vm.get("region") or alert.region)
    host = str(vm.get("host") or alert.host or "N/A")
    os_status = str(vm.get("os_status") or "N/A")
    age = alert.age_minutes

    gpus = str(vm.get("gpu_count") or "")
    large = gpus.isdigit() and int(gpus) >= 4

    if age is not None and age < 60 and large:
        d.verdict = f"BUILD for {age} min on a large flavor - may still be transient"
        d.confidence = "medium"
        d.assessment = (
            "Larger flavors (more GPUs/storage) can take a while to build and often resolve on their own. "
            "The runbook says to investigate host/infrastructure problems once it is stuck past an hour."
        )
    else:
        d.verdict = f"Stuck in BUILD{f' for {age} min' if age is not None else ''} - investigate host/infrastructure"
        d.confidence = "high"
        d.assessment = "Past roughly an hour in BUILD this is a host or infrastructure problem, not slow provisioning."

    health = _host_health_findings(d, region, host)
    d.evidence["host_health"] = health

    d.add("OpenStack status", os_status, "ok" if os_status.upper() == "BUILD" else "warn",
          "" if os_status.upper() == "BUILD" else "The runbook expects the instance to be in status BUILD.")

    failed_event = cxbridge.failed_openstack_event(region, str(vm.get("openstack_id") or "")) \
        if vm.get("openstack_id") not in (None, "", "N/A") and region else {}
    if failed_event:
        rows = failed_event.get("rows") or {}
        d.add("Failed OpenStack event", f"{failed_event.get('action', '?')} ({failed_event.get('request_id', '')})",
              "bad", rows.get("Detail", ""))
        d.evidence["failed_event"] = failed_event

    d.act("Escalate to the Infrastructure team with the server event and server show output.", INFRA, "escalate")
    d.act(
        "Tell the customer the instance must be deleted and recreated. It will eventually transition to ERROR, "
        "which lets them delete it themselves; otherwise InfraInsight can delete the Infrahub record (Infra may need "
        "to delete it in OpenStack while it is stuck in BUILD).",
        CX, "comms", "HubSpot Ticket Creation",
    )
    draft = _vm_draft(d, "build", vm, alert)
    if draft:
        d.drafts.append(draft)


# --- rogue VM --------------------------------------------------------------

MISMATCH_TABLE: list[dict[str, Any]] = [
    {
        "ih": "HIBERNATED", "os": "SHUTOFF",
        "verdict": "Hibernation did not complete - commonly host disk capacity",
        "steps": [
            (CX, "Run Host Health Checks on the hypervisor.", "Host Health Checks"),
            (CX, "Run the Windmill workflow to clean stale images on the host and free space for hibernation.", ""),
            (CX, "Complete the shelve for the instance.", "Shelving an Instance"),
        ],
        "comms": None,
    },
    {
        "ih": "HIBERNATED", "os": "ACTIVE",
        "verdict": "Infrahub thinks the VM is hibernated but it is still running and billable",
        "steps": [
            (CX, "Run Host Health Checks on the hypervisor.", "Host Health Checks"),
            (CX, "If the host shows no visible issues, shelve the instance.", "Shelving an Instance"),
            (CX, "Create a HubSpot ticket for the state mismatch.", "HubSpot Ticket Creation"),
        ],
        "comms": "sync_state",
    },
    {
        "ih": "DELETING", "os": "*",
        "verdict": "Infrahub is stuck DELETING - follow the DELETING runbook",
        "steps": [(CX, "Follow the Instance in DELETING state runbook.", "Instance in DELETING state")],
        "comms": None,
    },
    {
        "ih": "RESTORING", "os": "*",
        "verdict": "Infrahub is stuck RESTORING - follow the RESTORING runbook",
        "steps": [(CX, "Follow the Instance in RESTORING state runbook.", "Instance in RESTORING state")],
        "comms": None,
    },
]


def _match_mismatch_table(ih_status: str, os_status: str) -> Optional[dict[str, Any]]:
    ih = (ih_status or "").upper()
    os_ = (os_status or "").upper()
    if ih == "ERROR" or os_ == "ERROR":
        return {
            "ih": ih, "os": os_,
            "verdict": "ERROR on one side - follow the ERROR runbook",
            "steps": [(CX, "Follow the Instance in ERROR state runbook.", "Instance in ERROR state")],
            "comms": None,
        }
    for entry in MISMATCH_TABLE:
        if entry["ih"] == ih and entry["os"] in ("*", os_):
            return entry
    return None


def _gpu_slots(visual: dict[str, Any], roster: list[dict[str, Any]]) -> list[dict[str, Any]]:
    """One entry per physical GPU on the host.

    A host has a fixed number of GPU sockets. Every one of them is in exactly
    one of three states, and showing all of them is the only way the arithmetic
    reads as complete:

      * claimed by a named VM  - one slot per GPU that VM holds
      * in use but unclaimed   - the host says busy, no instance says so
      * free                   - physically present, nothing using it

    Drawing only the first two (as the earlier bar did) drops the free sockets
    and leaves a total that does not match the host's GPU count.
    """
    physical = visual.get("physical")
    in_use = visual.get("in_use_metric")
    artifact = bool(visual.get("spare_capacity_artifact"))

    slots: list[dict[str, Any]] = []
    for vm in roster:
        try:
            count = int(str(vm.get("gpus") or "0"))
        except ValueError:
            count = 0
        for _ in range(max(0, count)):
            slots.append({
                "kind": "vm",
                "name": vm["name"],
                "linked": vm["linked"],
                "match": vm["match"],
                "ih_status": vm["ih_status"],
                "os_status": vm["os_status"],
            })

    named = len(slots)
    if in_use is None:
        return slots

    # Sockets the host counts as busy that no instance accounts for. When
    # In_Use_Gpus is really just the physical count, that difference is spare
    # capacity rather than a missing workload, so label it honestly.
    unaccounted = max(0, int(in_use) - named)
    for _ in range(unaccounted):
        slots.append({"kind": "free" if artifact else "unaccounted"})

    if physical is not None:
        for _ in range(max(0, int(physical) - max(int(in_use), named))):
            slots.append({"kind": "free"})
    return slots


def _diagnose_rogue_vm(d: Diagnosis, alert: Alert) -> None:
    """Suspected Rogue VM - host-wide Infrahub/OpenStack reconciliation."""
    host = alert.host or alert.instance_name
    region = alert.region

    if not host:
        d.error = "This alert carries no hypervisor, so there is nothing to reconcile."
        return
    if not region:
        d.error = f"Could not derive a CX-Tools region from host '{host}'."
        return

    result = cxbridge.collect_host(host)
    d.evidence["host_result"] = cxbridge.json_safe(result)
    if not result.get("ok"):
        d.error = str(result.get("error") or f"Host collection failed for {host}.")
        return

    hv = result.get("hypervisor") if isinstance(result.get("hypervisor"), dict) else {}
    ovs = result.get("ovs") if isinstance(result.get("ovs"), dict) else {}
    c = cxbridge.cx()

    d.add("Host", host)
    d.add("Region", region)
    d.add("Instances on host", result.get("server_count"))
    d.add("Nova state", c.normalize_empty(c.first_present(hv, "state", "State", default="")) or "N/A",
          "ok" if str(c.first_present(hv, "state", "State", default="")).lower() == "up" else "bad")
    d.add("Nova status", c.normalize_empty(c.first_present(hv, "status", "Status", default="")) or "N/A",
          "ok" if str(c.first_present(hv, "status", "Status", default="")).lower() == "enabled" else "bad")
    if result.get("compute_service_disabled_reason"):
        d.add("Disabled reason", result["compute_service_disabled_reason"], "bad")
    if ovs:
        d.add("OVS alive", str(ovs.get("alive")), "ok" if ovs.get("alive") in (True, "True", "true") else "bad")
        d.add("OVS state", str(ovs.get("state") or "N/A"), "ok" if str(ovs.get("state") or "").lower() == "up" else "bad")

    instances = result.get("instances") or []
    problem_rows: list[dict[str, Any]] = []
    ignored = 0

    # Per-instance reconciliation for the visual: every VM on the host, and
    # whether Infrahub has a counterpart for it.
    roster: list[dict[str, Any]] = []
    for vm in instances:
        ih_status = str(vm.get("ih_status") or "N/A")
        os_status = str(vm.get("os_status") or "N/A")
        linked = bool(vm.get("infrahub"))
        roster.append({
            "name": str(vm.get("name") or "N/A"),
            "openstack_id": str(vm.get("openstack_id") or "N/A"),
            "infrahub_id": str(vm.get("infrahub_id") or "N/A"),
            "ih_status": ih_status if linked else "not in Infrahub",
            "os_status": os_status,
            "gpus": str(vm.get("gpu_count") or "?"),
            "linked": linked,
            "match": linked and _tone_for_status_pair(ih_status, os_status, str(vm.get("task_state") or "None")) == "ok",
            "tempest": bool(vm.get("tempest")),
            "org": str(vm.get("org_value") or ""),
        })

    for vm in instances:
        mismatches = vm.get("mismatches") or []
        warn_reasons = vm.get("warn_reasons") or []
        if vm.get("tempest"):
            ignored += 1
            continue
        if not mismatches and not warn_reasons:
            continue

        ih_status = str(vm.get("ih_status") or "N/A")
        os_status = str(vm.get("os_status") or "N/A")
        checks = [c.mismatch_parts(m)[0] for m in mismatches]
        entry = _match_mismatch_table(ih_status, os_status)

        row: dict[str, Any] = {
            "idx": vm.get("idx"),
            "name": vm.get("name"),
            "infrahub_id": vm.get("infrahub_id"),
            "openstack_id": vm.get("openstack_id"),
            "ih_status": ih_status,
            "os_status": os_status,
            "floating_ip": vm.get("floating_ip"),
            "environment": vm.get("project_environment") or "",
            "mismatches": [{"check": c.mismatch_parts(m)[0], "detail": c.mismatch_parts(m)[1]} for m in mismatches],
            "warn_reasons": [str(x) for x in warn_reasons],
            "contacts": comms.contacts_from_result(vm),
            "verdict": "",
            "steps": [],
            "comms_template": None,
        }

        if "Infrahub Missing" in checks:
            row["verdict"] = "Exists in OpenStack with no production Infrahub record"
            row["steps"] = [
                (CX, "Search for the VM in Admin Portal PreProd and Staging with 'Include Deleted' checked.", ""),
                (CX, "If it is DELETED or absent everywhere, delete the server in OpenStack.", "Deleting an Instance"),
            ]
            if vm.get("project_environment"):
                row["verdict"] = f"Lives in {vm['project_environment']}, not production"
                row["steps"] = [(CX, f"Confirm in Admin Portal {vm['project_environment']} whether this VM is still needed.", "")]
        elif entry:
            row["verdict"] = entry["verdict"]
            row["steps"] = list(entry["steps"])
            row["comms_template"] = entry.get("comms")
        elif "Openstack Missing" in checks:
            row["verdict"] = "Infrahub holds a record OpenStack does not have"
            row["steps"] = [
                (CX, "Confirm the VM is DELETED in Admin Portal Production.", ""),
                (CX, "If Infrahub still shows it live, correct the record in InfraInsight.", "Infrahub Insights (Infra-Insight)"),
            ]
        elif ih_status.upper() == "HIBERNATED" and str(vm.get("host") or "N/A") not in ("N/A", ""):
            row["verdict"] = "Infrahub kept a stale host on a HIBERNATED VM"
            row["steps"] = [(CX, "Use the InfraInsight update-resource tool to remove the host from the VM record.",
                             "Infrahub Insights (Infra-Insight)")]
        else:
            row["verdict"] = "Mismatch is not in the runbook table"
            row["steps"] = [
                (CX, "Ping Kheano Martinez or John Priest for a runbook update, and escalate to Infrastructure for next steps.", ""),
            ]

        problem_rows.append(row)

    d.evidence["instances"] = problem_rows
    d.evidence["ignored_tempest"] = ignored
    d.visual = {**(d.visual or {}), "roster": roster, "slots": _gpu_slots(d.visual or {}, roster)}

    if ignored:
        d.add("Ignored (tempest/OIE testing)", str(ignored), "info",
              "CX-Tools suppresses tempest-project instances, which the runbook says to ignore.")

    if not problem_rows:
        gap = int((d.visual or {}).get("gap") or 0)
        artifact = bool((d.visual or {}).get("spare_capacity_artifact"))
        d.confidence = "high"

        if gap >= 1 and not artifact:
            # Every Infrahub record matches OpenStack, yet the host reports more
            # GPUs in use than the instances account for. Nothing on the CX side
            # explains that - it is a host-level allocation question.
            d.verdict = (
                f"{gap} GPU(s) in use on {host} belong to no instance on either side"
            )
            d.assessment = (
                "Every Infrahub record reconciles with OpenStack, so this is not a stale record or a rogue VM "
                "that CX can correct. The host is reporting GPUs in use beyond what any instance claims, which "
                "points at a leaked allocation on the hypervisor."
            )
            d.act(
                f"Escalate to the Infrastructure team: {host} reports {gap} GPU(s) in use that no OpenStack "
                "instance claims. Ask them to check for allocations left behind by deleted domains.",
                INFRA, "escalate", "Host Health Checks",
            )
            d.act(
                "Confirm from the hypervisor's own PCI/GPU view before escalating, in case the exporter is at fault.",
                CX, "check",
            )
            return

        d.verdict = f"No Infrahub/OpenStack mismatch found on {host}"
        if artifact:
            d.verdict = f"Not a rogue VM - {gap} spare GPU(s) on {host} reported as a discrepancy"
            d.assessment = (
                "Every instance reconciles, and the 'gap' is the host's unallocated capacity: the rule's "
                "In_Use_Gpus reading equals the physical GPU count, so it subtracts allocated GPUs from total "
                "GPUs. Nothing to do on this host - the alert rule itself needs fixing."
            )
            d.act("No CX action. Raise the rule defect with the alert owner so these stop firing.", CX, "escalate")
            return

        d.assessment = (
            "Every instance on the host reconciles. Per the runbook, when there is no mismatch the issue likely "
            "exists only on the Infrahub side."
        )
        d.act(
            "Query InfraInsight for all instances on this host and check for HIBERNATED VMs that still carry a host "
            "value; remove the host with the update-resource tool.",
            CX, "check", "SQL Queries - Infra Insight",
        )
        d.note("The alert may already have cleared, or the discrepancy may be Infrahub-only and invisible to OpenStack.")
        return

    d.verdict = f"{len(problem_rows)} of {len(instances)} instances on {host} do not reconcile"
    d.confidence = "high"
    d.assessment = (
        "Each mismatched instance below is mapped to its remediation from the Suspected Rogue VM table. Work them "
        "individually - they can need different runbooks."
    )

    for row in problem_rows:
        label = f"#{row['idx']} {row['name']} ({row['ih_status']} / {row['os_status']})"
        d.add(f"Mismatch {label}", row["verdict"], "bad",
              "; ".join(m["detail"] for m in row["mismatches"]))
        for owner, text, guide in row["steps"]:
            d.act(f"[{row['name']}] {text}", owner, "remediate", guide)
        if row["comms_template"]:
            draft = comms.draft(row["comms_template"], instance_name=str(row["name"] or ""),
                                infrahub_id=str(row.get("infrahub_id") or ""),
                                openstack_id=str(row.get("openstack_id") or ""),
                                greeting_name=comms.first_name((row["contacts"].get("owners") or [""])[0]),
                                agent_name=d.agent_name)
            if draft:
                draft.label += f" - {row['name']}"
                d.drafts.append(draft)
                d.contacts = row["contacts"] if row["contacts"].get("resolved") else d.contacts

    d.act("Record the findings and the alert link in the Slack thread and any associated ticket.", CX, "verify")
    d.act(
        "If a customer request (reboot, hibernation, restore) has been incomplete for over 30 minutes, treat it as a "
        "host issue.",
        CX, "check",
    )


# --- duplicated IPs -------------------------------------------------------

def _diagnose_duplicate_ip(d: Diagnosis, alert: Alert, prom: Any = None) -> None:
    """Duplicated IPs."""
    fip = alert.floating_ip
    if not fip:
        d.error = "This alert carries no floating_ip label."
        return

    d.add("Floating IP", fip)
    summary = alert.annotations.get("summary", "")
    claimed = re.search(r"present on `(\d+)` VMs", summary)
    if claimed:
        d.add("Prometheus reports", f"{claimed.group(1)} VMs holding this IP", "bad")

    # The runbook's "EASY WAY": Resources{floating_ip="..."} covers every
    # environment, which is how a PreProd/Staging claimant is found.
    prom_rows: list[dict[str, Any]] = []
    if prom is not None:
        try:
            prom_rows = prom.resources_by_floating_ip(fip)
        except Exception as exc:
            d.note(f"Prometheus Resources lookup failed: {exc}")
    d.evidence["prometheus_claimants"] = prom_rows
    for row in prom_rows:
        env = row.get("environment") or row.get("env") or ""
        d.add(
            f"Prometheus claimant: {row.get('instance_name', '?')}",
            " / ".join(x for x in [row.get("status", ""), row.get("region", ""), str(env)] if x),
            "warn",
        )

    result = cxbridge.collect_vm(fip, region=alert.region)
    d.evidence["vm_result"] = cxbridge.json_safe(result)
    if not result.get("ok"):
        d.error = str(result.get("error") or f"CX-Tools could not resolve floating IP {fip}.")
        return

    claimants = result.get("instances") if result.get("mode") == "multi_vm" else [result]
    claimants = [c for c in claimants if isinstance(c, dict)]
    d.add("Production claimants found by CX-Tools", str(len(claimants)),
          "bad" if len(claimants) > 1 else "warn")

    c = cxbridge.cx()
    scenarios: list[dict[str, Any]] = []
    for vm in claimants:
        ih_status = str(vm.get("ih_status") or "N/A")
        server = vm.get("server") if isinstance(vm.get("server"), dict) else {}
        os_fip = c.public_ip_from_server(server) if server else ""
        ih_fip = str((vm.get("infrahub") or {}).get("floating_ip") or "") if isinstance(vm.get("infrahub"), dict) else ""

        entry: dict[str, Any] = {
            "name": vm.get("name"),
            "infrahub_id": vm.get("infrahub_id"),
            "openstack_id": vm.get("openstack_id"),
            "ih_status": ih_status,
            "os_status": vm.get("os_status"),
            "infrahub_fip": ih_fip or "none",
            "openstack_fip": os_fip or "none",
            "server_present": bool(server),
            "contacts": comms.contacts_from_result(vm),
        }

        if ih_status.upper() == "DELETING":
            entry["verdict"] = "Stuck DELETING in Infrahub while the IP has moved on"
            entry["steps"] = [(CX, "Delete this instance - it is stuck DELETING per Infrahub.", "Deleting an Instance")]
            entry["comms"] = None
        elif not server:
            entry["verdict"] = "No longer exists in OpenStack, but Infrahub still holds the IP"
            entry["steps"] = [
                (CX, "Remove the stale floating IP in InfraInsight and set Floating IP Status to NO FLOATING IP.",
                 "Infrahub Insights (Infra-Insight)"),
            ]
            entry["comms"] = "dupip_removed"
        elif not os_fip:
            entry["verdict"] = "Scenario #1 - the VM has no floating IP in OpenStack"
            entry["steps"] = [
                (CX, "Remove the incorrect floating IP in InfraInsight and set Floating IP Status to NO FLOATING IP.",
                 "Infrahub Insights (Infra-Insight)"),
            ]
            entry["comms"] = "dupip_removed"
        elif ih_fip and os_fip and ih_fip != os_fip:
            entry["verdict"] = f"Scenario #2 - Infrahub says {ih_fip} but OpenStack says {os_fip}"
            entry["steps"] = [
                (CX, f"Update the Infrahub floating IP to {os_fip} in InfraInsight.",
                 "Infrahub Insights (Infra-Insight)"),
            ]
            entry["comms"] = "dupip_corrected"
            entry["new_fip"] = os_fip
        else:
            entry["verdict"] = "Infrahub and OpenStack agree - this is the rightful owner of the IP"
            entry["steps"] = []
            entry["comms"] = None
        scenarios.append(entry)

    d.evidence["claimants"] = scenarios
    d.visual = {
        "type": "claimants",
        "ip": fip,
        "items": [{
            "name": str(e["name"]), "ih_status": str(e["ih_status"]), "os_status": str(e["os_status"]),
            "verdict": e["verdict"],
        } for e in scenarios],
    }

    for entry in scenarios:
        d.add(
            f"Claimant {entry['name']} ({entry['ih_status']} / {entry['os_status']})",
            entry["verdict"],
            "ok" if not entry["steps"] else "bad",
            f"Infrahub FIP={entry['infrahub_fip']}, OpenStack FIP={entry['openstack_fip']}",
        )
        for owner, text, guide in entry["steps"]:
            d.act(f"[{entry['name']}] {text}", owner, "remediate", guide)
        if entry.get("comms"):
            draft = comms.draft(entry["comms"], instance_name=str(entry["name"] or ""),
                                infrahub_id=str(entry.get("infrahub_id") or ""),
                                openstack_id=str(entry.get("openstack_id") or ""),
                                greeting_name=comms.first_name((entry["contacts"].get("owners") or [""])[0]),
                                agent_name=d.agent_name,
                                floating_ip=str(entry.get("new_fip") or ""))
            if draft:
                draft.label += f" - {entry['name']}"
                d.drafts.append(draft)
                if entry["contacts"].get("resolved"):
                    d.contacts = entry["contacts"]

    actionable = [e for e in scenarios if e["steps"]]
    if actionable:
        d.verdict = f"{len(actionable)} of {len(scenarios)} claimants of {fip} need correcting"
        d.confidence = "high"
    elif len(claimants) <= 1:
        d.verdict = f"Only one production claimant of {fip} - the duplicate is likely outside production"
        d.confidence = "medium"
        d.assessment = (
            "CX-Tools reconciles production Infrahub only. When production shows a single owner, the duplicate is "
            "usually a PreProd or Staging record."
        )
        d.act("Check PreProd and Staging for VMs holding this floating IP (repeat with those API keys).", CX, "check")
        if prom_rows:
            d.note("The Prometheus Resources rows above span every environment - use them to spot the other claimant.")
    else:
        d.verdict = f"{len(scenarios)} claimants of {fip}, none needing a correction"
        d.confidence = "low"
        d.note("Both sides agree for every claimant, so the alert may already be stale. Re-run in 5-10 minutes.")

    d.assessment = d.assessment or (
        "Each claimant is classified against the Duplicated IPs scenarios. Correct the losing records, then re-run "
        "the query after 5-10 minutes to confirm the alert clears."
    )
    d.act("Re-run the check after ~5-10 minutes to confirm only one VM holds the IP and the alert has cleared.",
          CX, "verify")


# --- total GPUs ------------------------------------------------------------

def _diagnose_total_gpus(d: Diagnosis, alert: Alert) -> None:
    """Problem with Total GPUs in a System."""
    host = alert.host or alert.instance_name
    region = alert.region
    if not host:
        d.error = "This alert carries no hypervisor label."
        return

    d.verdict = f"Host {host} is reporting missing GPU(s)"
    d.confidence = "high"
    d.assessment = (
        "Monitoring detected a host with fewer GPUs than expected. This hits revenue and customer experience, so the "
        "job is to find out who is on the host and escalate the hardware fault to Infrastructure."
    )
    d.add("Host", host)
    d.add("GPU model", alert.gpu_name or "N/A")
    summary = alert.annotations.get("summary", "")
    count = re.search(r"\*\s*" + re.escape(alert.gpu_name or "") + r":\*\s*`(\d+)`", summary) if alert.gpu_name else None
    if count:
        d.add("GPUs reported present", count.group(1), "bad",
              "Compare against the expected count for this chassis (typically 8).")

    if region:
        census = cxbridge.host_gpu_census(region, host)
        d.evidence["gpu_census"] = census
        if census.get("ok"):
            customers = [i for i in census.get("instances", [])]
            d.add("Instances on host", str(len(customers)), "warn" if customers else "ok")
            d.add("GPUs allocated to instances", str(census.get("total_gpus")))
            for inst in customers:
                d.add(f"Instance {inst['name']}", f"{inst['status']} - {inst['flavor']} ({inst['gpus']} GPU)", "warn")
            if customers:
                d.act(
                    "Customers are on this host - they may need contacting for host maintenance, depending on the "
                    "Infrastructure team's assessment.",
                    CX, "comms",
                )
                d.act("Run List Instances on a Hypervisor (Windmill) to confirm the customer list.", CX, "check")
            else:
                d.act("No instances on the host - no customer impact to coordinate.", CX, "verify", status="done")
        else:
            d.add("Instance census", f"failed: {census.get('error', '')}", "warn")
            d.act("Run List Instances on a Hypervisor (Windmill) to determine if any customers are on the host.",
                  CX, "check")
    else:
        d.act("Run List Instances on a Hypervisor (Windmill) to determine if any customers are on the host.",
              CX, "check")

    d.act(
        f"Search Jira for an existing OPEN issue for {host}. If one exists, update it with the alert details and ask "
        "whether a new ticket is needed.",
        CX, "check",
    )
    d.act(f"Otherwise raise a Jira for the Infrastructure team with '{host}' in the title, describing the issue and "
          "copying the alert details.", INFRA, "escalate")
    d.note("There is no approved customer template for this alert - host-maintenance comms are coordinated separately.")


# --- single-VM status mismatch ---------------------------------------------

def _diagnose_status_mismatch(d: Diagnosis, alert: Alert, vm: dict[str, Any]) -> None:
    """The per-region "Openstack status=X and Infrahub status!=X" rules.

    Same class of problem as Suspected Rogue VM, scoped to one VM, so it takes
    the same Mismatch Remediation table.
    """
    ih_status = str(vm.get("ih_status") or "N/A")
    os_status = str(vm.get("os_status") or "N/A")
    entry = _match_mismatch_table(ih_status, os_status)

    if entry:
        d.verdict = entry["verdict"]
        d.confidence = "high"
        for owner, text, guide in entry["steps"]:
            d.act(text, owner, "remediate", guide)
        if entry.get("comms"):
            draft = _vm_draft(d, entry["comms"], vm, alert)
            if draft:
                d.drafts.append(draft)
    elif not vm.get("mismatches"):
        d.verdict = f"Infrahub and OpenStack now agree ({ih_status} / {os_status})"
        d.confidence = "high"
        d.assessment = "The mismatch has already cleared; the alert should stop on the next evaluation."
    else:
        d.verdict = f"Mismatch ({ih_status} / {os_status}) is not in the runbook table"
        d.confidence = "low"
        d.act("Ping Kheano Martinez or John Priest for a runbook update, and escalate to Infrastructure for next steps.",
              CX, "escalate")

    d.assessment = d.assessment or (
        "A single-VM state mismatch. The Suspected Rogue VM remediation table covers which side to correct."
    )


# --- dispatch --------------------------------------------------------------

_VM_KINDS = {
    "error": _diagnose_error,
    "deleting": _diagnose_deleting,
    "shutoff": _diagnose_shutoff,
    "hibernating": _diagnose_hibernating,
    "creating": _diagnose_creating,
    "restoring": _diagnose_restoring,
    "rebooting": _diagnose_rebooting,
    "build": _diagnose_build,
    "status_mismatch": _diagnose_status_mismatch,
}


def _screened_out(d: Diagnosis, alert: Alert) -> None:
    """Report a screened-out alert without spending calls diagnosing it."""
    screen = alert.screen or {}
    d.verdict = f"No action needed - {screen.get('label', 'screened out')}"
    d.confidence = "high"
    d.assessment = str(screen.get("reason") or "")
    d.add("Screening verdict", screen.get("label", ""), "ok")
    if screen.get("current_state"):
        d.add("Current state", screen["current_state"], "ok")
    if screen.get("detail"):
        d.note(str(screen["detail"]))
    d.act(
        "Nothing to do. If the alert is still up in Prometheus it should clear on the next evaluation; "
        "if it does not, the rule may need a silence or a fix.",
        CX, "verify", status="done",
    )
    d.note("Re-run with 'Diagnose anyway' to query Infrahub and OpenStack directly for this alert.")


def _add_screening_findings(d: Diagnosis, alert: Alert) -> None:
    screen = alert.screen or {}
    if not screen:
        return
    tone = {"real": "bad", "unverified": "warn", "chronic": "warn",
            "low_impact": "warn", "pending": "warn", "resolved": "ok"}.get(str(screen.get("verdict")), "info")
    d.add("Screening", f"{screen.get('label', '')} - {screen.get('reason', '')}", tone,
          str(screen.get("detail") or ""))


def artifact_now(d: Diagnosis) -> bool:
    return bool((d.visual or {}).get("spare_capacity_artifact"))


def _add_rogue_gpu_gap(d: Diagnosis, alert: Alert, snap: Any) -> None:
    """Show the GPU accounting gap the Rogue VM rule actually fires on.

    The rule is `sum by(instance)(In_Use_Gpus) - sum by(instance)(Resources{
    status=~"ACTIVE|SHUTOFF|PRE_ACTIVE"}) >= 1` - a per-host GPU accounting gap,
    not a status comparison. Two different faults produce that gap and Prometheus
    cannot tell them apart, so both are stated and the host reconciliation below
    is what settles it.
    """
    host = alert.host or alert.instance_name
    if snap is None or not host:
        return
    delta = getattr(snap, "rogue_delta", {}).get(host)
    if delta is None:
        return

    total = getattr(snap, "total_gpus", {}).get(host)
    in_use = getattr(snap, "in_use_gpus", {}).get(host)
    rows = getattr(snap, "resources_by_host", {}).get(host, [])
    counted = sum(int(r.get("_gpus", "0") or 0) for r in rows
                  if r.get("status", "").upper() in ("ACTIVE", "SHUTOFF", "PRE_ACTIVE"))

    artifact = in_use is not None and total is not None and in_use == total
    d.visual = {
        "type": "gpu",
        "host": host,
        "physical": int(total) if total is not None else None,
        "in_use_metric": int(in_use) if in_use is not None else None,
        "accounted": counted,
        "gap": int(delta),
        "instances": len(rows),
        # When the rule's "in use" reading is just the physical count, the gap
        # it reports is spare capacity, not a missing VM.
        "spare_capacity_artifact": artifact,
    }

    if total is not None:
        d.add("GPUs on host (physical)", str(int(total)))
    if in_use is not None:
        d.add("GPUs allocated on host", str(int(in_use)))
    d.add("GPUs Infrahub accounts for", str(counted))
    d.add("Accounting gap", f"{int(delta)} GPU(s)", "bad" if delta >= 1 else "ok",
          "This is the quantity the alert fired on.")

    if rows:
        by_status: dict[str, int] = {}
        for row in rows:
            by_status[row.get("status", "?")] = by_status.get(row.get("status", "?"), 0) + 1
        d.add("Infrahub VMs on host", ", ".join(f"{v} {k}" for k, v in sorted(by_status.items())))
    else:
        d.add("Infrahub VMs on host", "none recorded", "bad")

    if delta and delta >= 1 and not artifact_now(d):
        d.add(
            "Why there is no ID to chase", "nothing in OpenStack claims these GPUs", "warn",
            "`server list --host` is the complete set of instances Nova knows on this host, and their flavours "
            "account for fewer GPUs than the host reports in use. There is no instance UUID to look up because no "
            "instance owns them - which is exactly why this is a host-level escalation. Identifying them means "
            "looking at the host itself.",
        )
        d.evidence["host_probe_commands"] = [
            f"{alert.region} server list --all-projects --host {host} -c ID -c Name -c Status -c Flavor",
            f"{alert.region} hypervisor show {host} -f json",
            f"ssh {host} nvidia-smi --query-gpu=index,uuid,pci.bus_id --format=csv",
            f"ssh {host} 'virsh list --all'",
        ]

    if delta and delta >= 1:
        unattributed = getattr(snap, "unattributed_active", 0)
        unattributed_gpus = getattr(snap, "unattributed_active_gpus", 0)
        d.add(
            "Two possible causes", "instances OpenStack has that Infrahub does not, or Infrahub VMs with no host set",
            "warn",
            "Either there are instances running on this host with no Infrahub record (a true rogue VM), or Infrahub "
            "has ACTIVE VMs whose host field is unset, so they are not counted against this host. The per-instance "
            "reconciliation below distinguishes them: a genuine rogue VM shows up as 'Infrahub Missing'.",
        )
        if unattributed:
            d.note(
                f"Platform-wide, Infrahub currently has {unattributed} ACTIVE/SHUTOFF VM(s) ({unattributed_gpus} GPUs) "
                "with no host recorded. That is enough to explain gaps like this one without any rogue VM existing, "
                "so confirm against the per-instance list before escalating."
            )


def diagnose(alert: Alert, prom: Any = None, snap: Any = None, force: bool = False,
             user_settings: Any = None) -> Diagnosis:
    """Gather evidence for one alert and reach the runbook's verdict."""
    d = Diagnosis(alert=alert)
    d.agent_name = getattr(user_settings, "agent_name", "") or ""

    if not force and alert.screen and not alert.screen.get("actionable", True):
        _screened_out(d, alert)
        return d

    _common_preamble(d, alert)
    _add_screening_findings(d, alert)

    try:
        if alert.kind in ("rogue_vm", "orphan_vm"):
            _add_rogue_gpu_gap(d, alert, snap)
            _diagnose_rogue_vm(d, alert)
        elif alert.kind == "duplicate_ip":
            _diagnose_duplicate_ip(d, alert, prom)
        elif alert.kind == "total_gpus":
            _diagnose_total_gpus(d, alert)
        elif alert.kind in _VM_KINDS:
            target, org_id = _vm_target(alert)
            if not target:
                d.error = "This alert has neither an OpenStack ID nor an instance name to look up."
                return d
            vm = cxbridge.collect_vm(target, region=alert.region, org_id=org_id)
            d.evidence["vm_result"] = cxbridge.json_safe(vm)
            if not vm.get("ok"):
                d.error = str(vm.get("error") or f"CX-Tools returned no usable telemetry for {target}.")
                if alert.kind == "creating":
                    # A CREATING VM that never reached OpenStack legitimately has
                    # nothing to collect; the runbook conclusion still holds.
                    d.error = ""
                    d.add("CX-Tools lookup", "no telemetry available", "bad",
                          str(vm.get("error") or ""))
                    _diagnose_creating(d, alert, vm)
                    _attach_contacts(d, vm)
                return d
            _add_vm_findings(d, vm)
            _VM_KINDS[alert.kind](d, alert, vm)
            _attach_contacts(d, vm)
        else:
            d.error = f"Alert kind '{alert.kind}' is not covered by the CX runbooks."
    except cxbridge.BridgeError as exc:
        d.error = str(exc)
    except Exception as exc:  # surface, don't crash the request
        d.error = f"Diagnosis failed: {type(exc).__name__}: {exc}"

    return d