Split into a FastAPI backend and a React frontend, add case state and SSO
The single-file stdlib server became the limit: no way to track what had been done about an alert, no accounts, and a UI that had to be hand-rolled in template strings. This restructures it into something deployable. Backend (FastAPI) - app/ holds config, database, auth, delivery and the routers; triagelib keeps the triage engine unchanged, so the validated screening and runbook logic is untouched. - Cases persist per alert fingerprint with a status workflow (investigating, customer contacted, escalated to Infra, waiting, remediated, resolved, won't fix, false positive), an assignee, notes and an append-only history. An alert that stops and re-fires lands back on the same case and counts as a reopen. - Suppression rules move from a JSON file into the database. Auth - Signed session cookies over PBKDF2 local accounts, plus an OIDC flow ready for Authentik: users are created on first login and admin follows a group claim. Local login can be switched off entirely once SSO is live. Zendesk and Jira - Delivery is now implemented, behind three gates: the integration must be configured, its feature flag on, and CX_FEATURE_SEND_ENABLED on. A demo instance leaves the last off and cannot mail anyone. Both search before creating, so re-diagnosing an alert updates one ticket rather than opening several, and a rolling daily cap stops a loop mailing everybody. Deployment - Multi-stage Dockerfile builds the bundle and serves it from the API origin. - docker-compose for local and single-host use; Gitea Actions runs the tests, builds the image and renders deploy/k8s with envsubst. Two fixes found while testing: assigning a case returned a null assignee, and add_event could leave an already-loaded history collection stale. Known gap: the engine reaches OpenStack via `docker exec <region>-osc`, which does not work in a pod without the CX-Tools containers alongside it. docs/DEPLOYMENT.md sets out the three ways to close that. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
286
backend/triagelib/cxbridge.py
Normal file
286
backend/triagelib/cxbridge.py
Normal file
@@ -0,0 +1,286 @@
|
||||
"""Read-only adapter over the CX-Tools (vmc) collectors.
|
||||
|
||||
CX-Tools is imported as an unmodified library: this module never writes to the
|
||||
CX-Tools tree and only calls collectors and query helpers that read. Every
|
||||
OpenStack subcommand this module can reach is checked against READ_ONLY_VERBS
|
||||
before it runs, so a bug here cannot mutate a live instance.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from typing import Any, Optional
|
||||
|
||||
DEFAULT_CX_TOOLS_PATHS = (
|
||||
os.environ.get("CX_TOOLS_PATH", ""),
|
||||
os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "CX-Tools"),
|
||||
os.path.expanduser("~/scripts/CX-Tools"),
|
||||
os.path.expanduser("~/scripts/cx-tooling"),
|
||||
)
|
||||
|
||||
# OpenStack verbs the diagnosis path is allowed to reach. Anything that could
|
||||
# change state (create/delete/set/unset/shelve/reboot/...) is absent on purpose.
|
||||
READ_ONLY_VERBS = frozenset({"show", "list"})
|
||||
|
||||
|
||||
class BridgeError(RuntimeError):
|
||||
"""Raised when CX-Tools cannot be located, imported, or authenticated."""
|
||||
|
||||
|
||||
def locate_cx_tools() -> str:
|
||||
for candidate in DEFAULT_CX_TOOLS_PATHS:
|
||||
if not candidate:
|
||||
continue
|
||||
path = os.path.abspath(os.path.expanduser(candidate))
|
||||
if os.path.isfile(os.path.join(path, "cxlib", "__init__.py")):
|
||||
return path
|
||||
raise BridgeError(
|
||||
"Could not find the CX-Tools checkout. Set CX_TOOLS_PATH to the directory "
|
||||
"that contains cxlib/ and the vmc entry point."
|
||||
)
|
||||
|
||||
|
||||
_lock = threading.Lock()
|
||||
_state: dict[str, Any] = {"path": "", "cx": None, "config": None}
|
||||
|
||||
|
||||
def _import_cxlib(path: str):
|
||||
if path not in sys.path:
|
||||
sys.path.insert(0, path)
|
||||
try:
|
||||
import cxlib # noqa: PLC0415 (import is intentionally deferred)
|
||||
except Exception as exc: # pragma: no cover - depends on the local checkout
|
||||
raise BridgeError(f"Failed to import cxlib from {path}: {exc}") from exc
|
||||
return cxlib
|
||||
|
||||
|
||||
def bootstrap() -> tuple[Any, Any]:
|
||||
"""Import cxlib and build the shared Config, loading secrets exactly once.
|
||||
|
||||
Call this from the foreground at startup: constructing Config triggers the
|
||||
CX-Tools 1Password loader, which may need an interactive sign-in.
|
||||
"""
|
||||
with _lock:
|
||||
if _state["config"] is not None:
|
||||
return _state["cx"], _state["config"]
|
||||
path = locate_cx_tools()
|
||||
cx = _import_cxlib(path)
|
||||
config = cx.Config(no_color=True, debug=bool(os.environ.get("CX_DEBUG")))
|
||||
if not config.api_key or config.api_key in {"REDACT", "REPLACE_WITH_API_KEY"}:
|
||||
raise BridgeError(
|
||||
"CX-Tools could not load the Infrahub API key from 1Password. "
|
||||
"Run `op signin` in this shell, then restart cx-triage."
|
||||
)
|
||||
_state.update({"path": path, "cx": cx, "config": config})
|
||||
return cx, config
|
||||
|
||||
|
||||
def cx() -> Any:
|
||||
return bootstrap()[0]
|
||||
|
||||
|
||||
def config() -> Any:
|
||||
return bootstrap()[1]
|
||||
|
||||
|
||||
def cx_tools_path() -> str:
|
||||
bootstrap()
|
||||
return str(_state["path"])
|
||||
|
||||
|
||||
def quiet_progress() -> Any:
|
||||
"""A Progress object that renders nothing, for use off the terminal."""
|
||||
c = cx()
|
||||
return c.Progress(c.C(False), 1, enabled=False)
|
||||
|
||||
|
||||
def _guard_openstack(args: list[str]) -> None:
|
||||
verbs = [a for a in args if not str(a).startswith("-")]
|
||||
if not any(v in READ_ONLY_VERBS for v in verbs):
|
||||
raise BridgeError(f"Refusing to run a non-read-only OpenStack command: {' '.join(args)}")
|
||||
|
||||
|
||||
def os_json(region: str, args: list[str], timeout: int = 90) -> tuple[bool, Any, str]:
|
||||
"""Run a read-only `openstack ... -f json` command through CX-Tools."""
|
||||
_guard_openstack(args)
|
||||
return cx().os_json(config(), region, args, timeout=timeout)
|
||||
|
||||
|
||||
# --- collectors -------------------------------------------------------------
|
||||
|
||||
def collect_vm(target: str, *, region: str = "", org_id: Optional[str] = None, ssh_timeout: int = 3) -> dict[str, Any]:
|
||||
"""Full VM reconciliation: the same payload `vmc --json <target>` emits."""
|
||||
return cx().collect_vm(
|
||||
config(),
|
||||
target,
|
||||
region_arg=region or "",
|
||||
org_id=org_id,
|
||||
ssh_timeout=ssh_timeout,
|
||||
include_ih_events=True,
|
||||
include_volumes=True,
|
||||
include_all_ih_events=True,
|
||||
progress=quiet_progress(),
|
||||
)
|
||||
|
||||
|
||||
def collect_host(host: str, *, ssh_timeout: int = 3) -> dict[str, Any]:
|
||||
"""Host reconciliation: the same payload `vmc --json --host <host>` emits."""
|
||||
return cx().collect_host(
|
||||
config(),
|
||||
host,
|
||||
ssh_timeout=ssh_timeout,
|
||||
include_ih_events=True,
|
||||
include_volumes=True,
|
||||
progress=quiet_progress(),
|
||||
)
|
||||
|
||||
|
||||
def collect_vm_contacts(target: str, *, region: str = "", org_id: Optional[str] = None) -> dict[str, Any]:
|
||||
return cx().collect_vm_contacts(
|
||||
config(),
|
||||
target,
|
||||
region_arg=region or "",
|
||||
org_id=org_id,
|
||||
progress=quiet_progress(),
|
||||
)
|
||||
|
||||
|
||||
# --- targeted queries used by individual runbooks --------------------------
|
||||
|
||||
def openstack_events(region: str, openstack_id: str, limit: Optional[int] = 5) -> list[dict[str, Any]]:
|
||||
ok, events, _raw = cx().server_event_list(config(), region, openstack_id)
|
||||
if not ok:
|
||||
return []
|
||||
return events[:limit] if limit else events
|
||||
|
||||
|
||||
def openstack_event_detail(region: str, openstack_id: str, request_id: str) -> dict[str, Any]:
|
||||
ok, detail, _raw = cx().server_event_show(config(), region, openstack_id, request_id)
|
||||
return detail if ok else {}
|
||||
|
||||
|
||||
def failed_openstack_event(region: str, openstack_id: str, scan: int = 5) -> dict[str, Any]:
|
||||
"""Return the most recent OpenStack event whose detail reports a failure.
|
||||
|
||||
The state runbooks all say "the most recent failed event is the thing to
|
||||
escalate", so this walks recent events newest-first and returns the first
|
||||
one whose result is not Success, together with its detail rows.
|
||||
"""
|
||||
c = cx()
|
||||
for event in openstack_events(region, openstack_id, limit=scan):
|
||||
request_id = c.event_request_id(event)
|
||||
if not request_id:
|
||||
continue
|
||||
detail = openstack_event_detail(region, openstack_id, request_id)
|
||||
if not detail:
|
||||
continue
|
||||
rows = dict((str(k), str(v)) for k, v in c.event_detail_rows(detail))
|
||||
result = rows.get("Result", "")
|
||||
if result and result.lower() != "success":
|
||||
return {"request_id": request_id, "action": rows.get("Action", ""), "rows": rows}
|
||||
return {}
|
||||
|
||||
|
||||
def infrahub_events(infrahub_id: str, limit: Optional[int] = None) -> list[list[str]]:
|
||||
c = cx()
|
||||
ok, data, _raw = c.query_vm_events(config(), str(infrahub_id))
|
||||
if not ok:
|
||||
return []
|
||||
return c.infrahub_event_rows(data, limit)
|
||||
|
||||
|
||||
def host_health(region: str, host: str) -> dict[str, Any]:
|
||||
"""Hypervisor, Nova service and OVS agent signals for one host.
|
||||
|
||||
This is the cheap subset of `vmc --host` - the runbooks' "Host Health
|
||||
Checks" entry point - without collecting every instance on the host.
|
||||
"""
|
||||
c = cx()
|
||||
cfg = config()
|
||||
ok_hv, hv, raw_hv, hv_name = c.hypervisor_show_host(cfg, region, host)
|
||||
if not ok_hv:
|
||||
return {"ok": False, "error": raw_hv, "host": host, "region": region}
|
||||
|
||||
state = c.normalize_empty(c.first_present(hv, "state", "State", default=""))
|
||||
status = c.normalize_empty(c.first_present(hv, "status", "Status", default=""))
|
||||
|
||||
disabled_reason = ""
|
||||
if status.lower() == "disabled":
|
||||
for candidate in dict.fromkeys([x for x in (c.normalize_empty(hv_name), host) if x]):
|
||||
ok_svc, services, _raw = c.compute_service_list_host(cfg, region, candidate)
|
||||
if ok_svc:
|
||||
disabled_reason = c.disabled_reason_from_services(services)
|
||||
if disabled_reason:
|
||||
break
|
||||
|
||||
ovs: dict[str, Any] = {}
|
||||
ok_agents, agents, _raw_agents = c.network_agent_list_host(cfg, region, host)
|
||||
if ok_agents:
|
||||
ovs = c.ovs_agent_summary(agents)
|
||||
if ovs.get("agent_id"):
|
||||
ok_show, detail, _raw_show = c.network_agent_show(cfg, region, str(ovs["agent_id"]))
|
||||
if ok_show:
|
||||
ovs["last_heartbeat_at"] = c.normalize_empty(
|
||||
detail.get("last_heartbeat_at") or detail.get("Last Heartbeat At") or ovs.get("last_heartbeat_at")
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"error": "",
|
||||
"host": host,
|
||||
"hypervisor_name": hv_name,
|
||||
"region": region,
|
||||
"nova_state": state or "N/A",
|
||||
"nova_status": status or "N/A",
|
||||
"disabled_reason": disabled_reason,
|
||||
"uptime": c.host_uptime_summary(hv),
|
||||
"aggregates": c.host_aggregates_summary(hv),
|
||||
"ovs_alive": ovs.get("alive"),
|
||||
"ovs_state": ovs.get("state"),
|
||||
"ovs_last_heartbeat": ovs.get("last_heartbeat_at") or "",
|
||||
"running_vms": c.normalize_empty(c.first_present(hv, "running_vms", "Running VMs", default="")) or "N/A",
|
||||
"free_disk_gb": c.normalize_empty(c.first_present(hv, "free_disk_gb", "Free Disk GB", default="")) or "N/A",
|
||||
"local_disk_free": c.normalize_empty(c.first_present(hv, "disk_available_least", "Disk Available Least", default="")) or "N/A",
|
||||
}
|
||||
|
||||
|
||||
def host_gpu_census(region: str, host: str) -> dict[str, Any]:
|
||||
"""Sum GPU counts of every instance on a host.
|
||||
|
||||
Implements the ERROR-runbook check for the NUMA/PCI fault: "check if the
|
||||
host is full prior to escalation - add the values after the x, if it = 8
|
||||
then it is FULL".
|
||||
"""
|
||||
c = cx()
|
||||
ok, rows, raw = c.server_list_on_host(config(), region, host)
|
||||
if not ok:
|
||||
return {"ok": False, "error": raw, "total_gpus": None, "instances": []}
|
||||
total = 0
|
||||
unknown = 0
|
||||
instances: list[dict[str, str]] = []
|
||||
for row in rows:
|
||||
flavor = c.get_row_field(row, "Flavor", "flavor") or c.flavor_name_from_any(row)
|
||||
count = c.gpu_count_from_flavor_name(flavor)
|
||||
if count.isdigit():
|
||||
total += int(count)
|
||||
else:
|
||||
unknown += 1
|
||||
instances.append({
|
||||
"name": c.get_row_field(row, "Name", "name") or "N/A",
|
||||
"openstack_id": c.openstack_id_from_row(row) or "N/A",
|
||||
"status": c.get_row_field(row, "Status", "status") or "N/A",
|
||||
"flavor": flavor or "N/A",
|
||||
"gpus": count,
|
||||
})
|
||||
return {
|
||||
"ok": True,
|
||||
"error": "",
|
||||
"total_gpus": total,
|
||||
"unknown_flavors": unknown,
|
||||
"instances": instances,
|
||||
}
|
||||
|
||||
|
||||
def json_safe(obj: Any) -> Any:
|
||||
return cx().json_safe(obj)
|
||||
Reference in New Issue
Block a user