cx-ui/backend/triagelib/cxbridge.py

"""Read-only adapter over the CX-Tools (vmc) collectors.

CX-Tools is imported as an unmodified library: this module never writes to the
CX-Tools tree and only calls collectors and query helpers that read. Every
OpenStack subcommand this module can reach is checked against READ_ONLY_VERBS
before it runs, so a bug here cannot mutate a live instance.
"""
from __future__ import annotations

import os
import sys
import threading
from typing import Any, Optional

DEFAULT_CX_TOOLS_PATHS = (
    os.environ.get("CX_TOOLS_PATH", ""),
    os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "CX-Tools"),
    os.path.expanduser("~/scripts/CX-Tools"),
    os.path.expanduser("~/scripts/cx-tooling"),
)

# OpenStack verbs the diagnosis path is allowed to reach. Anything that could
# change state (create/delete/set/unset/shelve/reboot/...) is absent on purpose.
READ_ONLY_VERBS = frozenset({"show", "list"})


class BridgeError(RuntimeError):
    """Raised when CX-Tools cannot be located, imported, or authenticated."""


def locate_cx_tools() -> str:
    for candidate in DEFAULT_CX_TOOLS_PATHS:
        if not candidate:
            continue
        path = os.path.abspath(os.path.expanduser(candidate))
        if os.path.isfile(os.path.join(path, "cxlib", "__init__.py")):
            return path
    raise BridgeError(
        "Could not find the CX-Tools checkout. Set CX_TOOLS_PATH to the directory "
        "that contains cxlib/ and the vmc entry point."
    )


_lock = threading.Lock()
_state: dict[str, Any] = {"path": "", "cx": None, "config": None}


def _import_cxlib(path: str):
    if path not in sys.path:
        sys.path.insert(0, path)
    try:
        import cxlib  # noqa: PLC0415  (import is intentionally deferred)
    except Exception as exc:  # pragma: no cover - depends on the local checkout
        raise BridgeError(f"Failed to import cxlib from {path}: {exc}") from exc
    return cxlib


def bootstrap() -> tuple[Any, Any]:
    """Import cxlib and build the shared Config, loading secrets exactly once.

    Call this from the foreground at startup: constructing Config triggers the
    CX-Tools 1Password loader, which may need an interactive sign-in.
    """
    with _lock:
        if _state["config"] is not None:
            return _state["cx"], _state["config"]
        path = locate_cx_tools()
        cx = _import_cxlib(path)
        config = cx.Config(no_color=True, debug=bool(os.environ.get("CX_DEBUG")))
        if not config.api_key or config.api_key in {"REDACT", "REPLACE_WITH_API_KEY"}:
            raise BridgeError(
                "CX-Tools could not load the Infrahub API key from 1Password. "
                "Run `op signin` in this shell, then restart cx-triage."
            )
        _state.update({"path": path, "cx": cx, "config": config})
        return cx, config


def cx() -> Any:
    return bootstrap()[0]


def config() -> Any:
    return bootstrap()[1]


def cx_tools_path() -> str:
    bootstrap()
    return str(_state["path"])


def quiet_progress() -> Any:
    """A Progress object that renders nothing, for use off the terminal."""
    c = cx()
    return c.Progress(c.C(False), 1, enabled=False)


def _guard_openstack(args: list[str]) -> None:
    verbs = [a for a in args if not str(a).startswith("-")]
    if not any(v in READ_ONLY_VERBS for v in verbs):
        raise BridgeError(f"Refusing to run a non-read-only OpenStack command: {' '.join(args)}")


def os_json(region: str, args: list[str], timeout: int = 90) -> tuple[bool, Any, str]:
    """Run a read-only `openstack ... -f json` command through CX-Tools."""
    _guard_openstack(args)
    return cx().os_json(config(), region, args, timeout=timeout)


# --- collectors -------------------------------------------------------------

def collect_vm(target: str, *, region: str = "", org_id: Optional[str] = None, ssh_timeout: int = 3) -> dict[str, Any]:
    """Full VM reconciliation: the same payload `vmc --json <target>` emits."""
    return cx().collect_vm(
        config(),
        target,
        region_arg=region or "",
        org_id=org_id,
        ssh_timeout=ssh_timeout,
        include_ih_events=True,
        include_volumes=True,
        include_all_ih_events=True,
        progress=quiet_progress(),
    )


def collect_host(host: str, *, ssh_timeout: int = 3) -> dict[str, Any]:
    """Host reconciliation: the same payload `vmc --json --host <host>` emits."""
    return cx().collect_host(
        config(),
        host,
        ssh_timeout=ssh_timeout,
        include_ih_events=True,
        include_volumes=True,
        progress=quiet_progress(),
    )


def collect_vm_contacts(target: str, *, region: str = "", org_id: Optional[str] = None) -> dict[str, Any]:
    return cx().collect_vm_contacts(
        config(),
        target,
        region_arg=region or "",
        org_id=org_id,
        progress=quiet_progress(),
    )


# --- targeted queries used by individual runbooks --------------------------

def openstack_events(region: str, openstack_id: str, limit: Optional[int] = 5) -> list[dict[str, Any]]:
    ok, events, _raw = cx().server_event_list(config(), region, openstack_id)
    if not ok:
        return []
    return events[:limit] if limit else events


def openstack_event_detail(region: str, openstack_id: str, request_id: str) -> dict[str, Any]:
    ok, detail, _raw = cx().server_event_show(config(), region, openstack_id, request_id)
    return detail if ok else {}


def failed_openstack_event(region: str, openstack_id: str, scan: int = 5) -> dict[str, Any]:
    """Return the most recent OpenStack event whose detail reports a failure.

    The state runbooks all say "the most recent failed event is the thing to
    escalate", so this walks recent events newest-first and returns the first
    one whose result is not Success, together with its detail rows.
    """
    c = cx()
    for event in openstack_events(region, openstack_id, limit=scan):
        request_id = c.event_request_id(event)
        if not request_id:
            continue
        detail = openstack_event_detail(region, openstack_id, request_id)
        if not detail:
            continue
        rows = dict((str(k), str(v)) for k, v in c.event_detail_rows(detail))
        result = rows.get("Result", "")
        if result and result.lower() != "success":
            return {"request_id": request_id, "action": rows.get("Action", ""), "rows": rows}
    return {}


def infrahub_events(infrahub_id: str, limit: Optional[int] = None) -> list[list[str]]:
    c = cx()
    ok, data, _raw = c.query_vm_events(config(), str(infrahub_id))
    if not ok:
        return []
    return c.infrahub_event_rows(data, limit)


def host_health(region: str, host: str) -> dict[str, Any]:
    """Hypervisor, Nova service and OVS agent signals for one host.

    This is the cheap subset of `vmc --host` - the runbooks' "Host Health
    Checks" entry point - without collecting every instance on the host.
    """
    c = cx()
    cfg = config()
    ok_hv, hv, raw_hv, hv_name = c.hypervisor_show_host(cfg, region, host)
    if not ok_hv:
        return {"ok": False, "error": raw_hv, "host": host, "region": region}

    state = c.normalize_empty(c.first_present(hv, "state", "State", default=""))
    status = c.normalize_empty(c.first_present(hv, "status", "Status", default=""))

    disabled_reason = ""
    if status.lower() == "disabled":
        for candidate in dict.fromkeys([x for x in (c.normalize_empty(hv_name), host) if x]):
            ok_svc, services, _raw = c.compute_service_list_host(cfg, region, candidate)
            if ok_svc:
                disabled_reason = c.disabled_reason_from_services(services)
                if disabled_reason:
                    break

    ovs: dict[str, Any] = {}
    ok_agents, agents, _raw_agents = c.network_agent_list_host(cfg, region, host)
    if ok_agents:
        ovs = c.ovs_agent_summary(agents)
        if ovs.get("agent_id"):
            ok_show, detail, _raw_show = c.network_agent_show(cfg, region, str(ovs["agent_id"]))
            if ok_show:
                ovs["last_heartbeat_at"] = c.normalize_empty(
                    detail.get("last_heartbeat_at") or detail.get("Last Heartbeat At") or ovs.get("last_heartbeat_at")
                )

    return {
        "ok": True,
        "error": "",
        "host": host,
        "hypervisor_name": hv_name,
        "region": region,
        "nova_state": state or "N/A",
        "nova_status": status or "N/A",
        "disabled_reason": disabled_reason,
        "uptime": c.host_uptime_summary(hv),
        "aggregates": c.host_aggregates_summary(hv),
        "ovs_alive": ovs.get("alive"),
        "ovs_state": ovs.get("state"),
        "ovs_last_heartbeat": ovs.get("last_heartbeat_at") or "",
        "running_vms": c.normalize_empty(c.first_present(hv, "running_vms", "Running VMs", default="")) or "N/A",
        "free_disk_gb": c.normalize_empty(c.first_present(hv, "free_disk_gb", "Free Disk GB", default="")) or "N/A",
        "local_disk_free": c.normalize_empty(c.first_present(hv, "disk_available_least", "Disk Available Least", default="")) or "N/A",
    }


def host_gpu_census(region: str, host: str) -> dict[str, Any]:
    """Sum GPU counts of every instance on a host.

    Implements the ERROR-runbook check for the NUMA/PCI fault: "check if the
    host is full prior to escalation - add the values after the x, if it = 8
    then it is FULL".
    """
    c = cx()
    ok, rows, raw = c.server_list_on_host(config(), region, host)
    if not ok:
        return {"ok": False, "error": raw, "total_gpus": None, "instances": []}
    total = 0
    unknown = 0
    instances: list[dict[str, str]] = []
    for row in rows:
        flavor = c.get_row_field(row, "Flavor", "flavor") or c.flavor_name_from_any(row)
        count = c.gpu_count_from_flavor_name(flavor)
        if count.isdigit():
            total += int(count)
        else:
            unknown += 1
        instances.append({
            "name": c.get_row_field(row, "Name", "name") or "N/A",
            "openstack_id": c.openstack_id_from_row(row) or "N/A",
            "status": c.get_row_field(row, "Status", "status") or "N/A",
            "flavor": flavor or "N/A",
            "gpus": count,
        })
    return {
        "ok": True,
        "error": "",
        "total_gpus": total,
        "unknown_flavors": unknown,
        "instances": instances,
    }


def json_safe(obj: Any) -> Any:
    return cx().json_safe(obj)