"""Read-only adapter over the CX-Tools (vmc) collectors. CX-Tools is imported as an unmodified library: this module never writes to the CX-Tools tree and only calls collectors and query helpers that read. Every OpenStack subcommand this module can reach is checked against READ_ONLY_VERBS before it runs, so a bug here cannot mutate a live instance. """ from __future__ import annotations import os import sys import threading from typing import Any, Optional DEFAULT_CX_TOOLS_PATHS = ( os.environ.get("CX_TOOLS_PATH", ""), os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "CX-Tools"), os.path.expanduser("~/scripts/CX-Tools"), os.path.expanduser("~/scripts/cx-tooling"), ) # OpenStack verbs the diagnosis path is allowed to reach. Anything that could # change state (create/delete/set/unset/shelve/reboot/...) is absent on purpose. READ_ONLY_VERBS = frozenset({"show", "list"}) class BridgeError(RuntimeError): """Raised when CX-Tools cannot be located, imported, or authenticated.""" def locate_cx_tools() -> str: for candidate in DEFAULT_CX_TOOLS_PATHS: if not candidate: continue path = os.path.abspath(os.path.expanduser(candidate)) if os.path.isfile(os.path.join(path, "cxlib", "__init__.py")): return path raise BridgeError( "Could not find the CX-Tools checkout. Set CX_TOOLS_PATH to the directory " "that contains cxlib/ and the vmc entry point." ) _lock = threading.Lock() _state: dict[str, Any] = {"path": "", "cx": None, "config": None} def _import_cxlib(path: str): if path not in sys.path: sys.path.insert(0, path) try: import cxlib # noqa: PLC0415 (import is intentionally deferred) except Exception as exc: # pragma: no cover - depends on the local checkout raise BridgeError(f"Failed to import cxlib from {path}: {exc}") from exc return cxlib def _tokens_from_env() -> dict[str, str]: """API keys supplied directly, bypassing 1Password. CX-Tools reads its keys from 1Password, which needs an interactive session and a desktop app - neither exists in a container. `Config` is a dataclass whose 1Password lookups live in per-field default factories, so passing the values in means those factories never run. CX-Tools itself is unmodified. """ return { "api_key": os.environ.get("CX_INFRAHUB_TOKEN", "").strip(), "insight_api_key": os.environ.get("CX_INFRAINSIGHT_TOKEN", "").strip(), } def _silence_1password(path: str) -> None: """Stop the CX-Tools secret loader from reaching for `op`. `Config.os_cmd` re-reads the merged config on every OpenStack call, which would otherwise retry a sign-in that cannot succeed here. Marking the module as already loaded makes those calls no-ops. """ if path not in sys.path: sys.path.insert(0, path) try: import secrets_1password # noqa: PLC0415 except Exception: return secrets_1password._loaded = True def bootstrap() -> tuple[Any, Any]: """Import cxlib and build the shared Config, loading secrets exactly once. With CX_INFRAHUB_TOKEN / CX_INFRAINSIGHT_TOKEN set, credentials come from the environment. Without them, CX-Tools falls back to 1Password, which is what a laptop run does. """ with _lock: if _state["config"] is not None: return _state["cx"], _state["config"] path = locate_cx_tools() cx = _import_cxlib(path) tokens = _tokens_from_env() if tokens["api_key"]: _silence_1password(path) config = cx.Config(no_color=True, debug=bool(os.environ.get("CX_DEBUG")), **{k: v for k, v in tokens.items() if v}) else: config = cx.Config(no_color=True, debug=bool(os.environ.get("CX_DEBUG"))) if not config.api_key or config.api_key in {"REDACT", "REPLACE_WITH_API_KEY"}: raise BridgeError( "No Infrahub API key. Set CX_INFRAHUB_TOKEN (and CX_INFRAINSIGHT_TOKEN), " "or run `op signin` in this shell for the 1Password path." ) _state.update({"path": path, "cx": cx, "config": config}) return cx, config def cx() -> Any: return bootstrap()[0] def config() -> Any: return bootstrap()[1] def cx_tools_path() -> str: bootstrap() return str(_state["path"]) def quiet_progress() -> Any: """A Progress object that renders nothing, for use off the terminal.""" c = cx() return c.Progress(c.C(False), 1, enabled=False) def _guard_openstack(args: list[str]) -> None: verbs = [a for a in args if not str(a).startswith("-")] if not any(v in READ_ONLY_VERBS for v in verbs): raise BridgeError(f"Refusing to run a non-read-only OpenStack command: {' '.join(args)}") def os_json(region: str, args: list[str], timeout: int = 90) -> tuple[bool, Any, str]: """Run a read-only `openstack ... -f json` command through CX-Tools.""" _guard_openstack(args) return cx().os_json(config(), region, args, timeout=timeout) # --- collectors ------------------------------------------------------------- def collect_vm(target: str, *, region: str = "", org_id: Optional[str] = None, ssh_timeout: int = 3) -> dict[str, Any]: """Full VM reconciliation: the same payload `vmc --json ` emits.""" return cx().collect_vm( config(), target, region_arg=region or "", org_id=org_id, ssh_timeout=ssh_timeout, include_ih_events=True, include_volumes=True, include_all_ih_events=True, progress=quiet_progress(), ) def collect_host(host: str, *, ssh_timeout: int = 3) -> dict[str, Any]: """Host reconciliation: the same payload `vmc --json --host ` emits.""" return cx().collect_host( config(), host, ssh_timeout=ssh_timeout, include_ih_events=True, include_volumes=True, progress=quiet_progress(), ) def collect_vm_contacts(target: str, *, region: str = "", org_id: Optional[str] = None) -> dict[str, Any]: return cx().collect_vm_contacts( config(), target, region_arg=region or "", org_id=org_id, progress=quiet_progress(), ) # --- targeted queries used by individual runbooks -------------------------- def openstack_events(region: str, openstack_id: str, limit: Optional[int] = 5) -> list[dict[str, Any]]: ok, events, _raw = cx().server_event_list(config(), region, openstack_id) if not ok: return [] return events[:limit] if limit else events def openstack_event_detail(region: str, openstack_id: str, request_id: str) -> dict[str, Any]: ok, detail, _raw = cx().server_event_show(config(), region, openstack_id, request_id) return detail if ok else {} def failed_openstack_event(region: str, openstack_id: str, scan: int = 5) -> dict[str, Any]: """Return the most recent OpenStack event whose detail reports a failure. The state runbooks all say "the most recent failed event is the thing to escalate", so this walks recent events newest-first and returns the first one whose result is not Success, together with its detail rows. """ c = cx() for event in openstack_events(region, openstack_id, limit=scan): request_id = c.event_request_id(event) if not request_id: continue detail = openstack_event_detail(region, openstack_id, request_id) if not detail: continue rows = dict((str(k), str(v)) for k, v in c.event_detail_rows(detail)) result = rows.get("Result", "") if result and result.lower() != "success": return {"request_id": request_id, "action": rows.get("Action", ""), "rows": rows} return {} def infrahub_events(infrahub_id: str, limit: Optional[int] = None) -> list[list[str]]: c = cx() ok, data, _raw = c.query_vm_events(config(), str(infrahub_id)) if not ok: return [] return c.infrahub_event_rows(data, limit) def host_health(region: str, host: str) -> dict[str, Any]: """Hypervisor, Nova service and OVS agent signals for one host. This is the cheap subset of `vmc --host` - the runbooks' "Host Health Checks" entry point - without collecting every instance on the host. """ c = cx() cfg = config() ok_hv, hv, raw_hv, hv_name = c.hypervisor_show_host(cfg, region, host) if not ok_hv: return {"ok": False, "error": raw_hv, "host": host, "region": region} state = c.normalize_empty(c.first_present(hv, "state", "State", default="")) status = c.normalize_empty(c.first_present(hv, "status", "Status", default="")) disabled_reason = "" if status.lower() == "disabled": for candidate in dict.fromkeys([x for x in (c.normalize_empty(hv_name), host) if x]): ok_svc, services, _raw = c.compute_service_list_host(cfg, region, candidate) if ok_svc: disabled_reason = c.disabled_reason_from_services(services) if disabled_reason: break ovs: dict[str, Any] = {} ok_agents, agents, _raw_agents = c.network_agent_list_host(cfg, region, host) if ok_agents: ovs = c.ovs_agent_summary(agents) if ovs.get("agent_id"): ok_show, detail, _raw_show = c.network_agent_show(cfg, region, str(ovs["agent_id"])) if ok_show: ovs["last_heartbeat_at"] = c.normalize_empty( detail.get("last_heartbeat_at") or detail.get("Last Heartbeat At") or ovs.get("last_heartbeat_at") ) return { "ok": True, "error": "", "host": host, "hypervisor_name": hv_name, "region": region, "nova_state": state or "N/A", "nova_status": status or "N/A", "disabled_reason": disabled_reason, "uptime": c.host_uptime_summary(hv), "aggregates": c.host_aggregates_summary(hv), "ovs_alive": ovs.get("alive"), "ovs_state": ovs.get("state"), "ovs_last_heartbeat": ovs.get("last_heartbeat_at") or "", "running_vms": c.normalize_empty(c.first_present(hv, "running_vms", "Running VMs", default="")) or "N/A", "free_disk_gb": c.normalize_empty(c.first_present(hv, "free_disk_gb", "Free Disk GB", default="")) or "N/A", "local_disk_free": c.normalize_empty(c.first_present(hv, "disk_available_least", "Disk Available Least", default="")) or "N/A", } def host_gpu_census(region: str, host: str) -> dict[str, Any]: """Sum GPU counts of every instance on a host. Implements the ERROR-runbook check for the NUMA/PCI fault: "check if the host is full prior to escalation - add the values after the x, if it = 8 then it is FULL". """ c = cx() ok, rows, raw = c.server_list_on_host(config(), region, host) if not ok: return {"ok": False, "error": raw, "total_gpus": None, "instances": []} total = 0 unknown = 0 instances: list[dict[str, str]] = [] for row in rows: flavor = c.get_row_field(row, "Flavor", "flavor") or c.flavor_name_from_any(row) count = c.gpu_count_from_flavor_name(flavor) if count.isdigit(): total += int(count) else: unknown += 1 instances.append({ "name": c.get_row_field(row, "Name", "name") or "N/A", "openstack_id": c.openstack_id_from_row(row) or "N/A", "status": c.get_row_field(row, "Status", "status") or "N/A", "flavor": flavor or "N/A", "gpus": count, }) return { "ok": True, "error": "", "total_gpus": total, "unknown_flavors": unknown, "instances": instances, } def json_safe(obj: Any) -> Any: return cx().json_safe(obj)