Parham Monfared
a039e0b5fd
Read-only triage for the Infrahub error alerts. Pulls the Prometheus alert queue, re-checks each alert's condition against live state to separate real work from noise, diagnoses it using the CX runbooks, and drafts the customer comms with contacts resolved from Infrahub. Findings from validating against production: - "Suspected Rogue VM" fires on spare GPU capacity, not rogue VMs: In_Use_Gpus equals the physical count on 71 of 75 firing hosts, so the rule reduces to "this host has a free GPU". Verified against OpenStack on 10 hosts. - "Exists in Infrahub but does not exist in OpenStack" matches every VM because openstack_nova_server_status returns no series; excluded as a rule defect. - Prometheus activeAt is reset several times a day by dips in the Resources metric, so alert ages are recovered from ALERTS history instead. Takes ~2,650 firing alerts down to ~20 that need a decision. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
438 lines
19 KiB
Python
438 lines
19 KiB
Python
"""Localhost HTTP server: alert queue, background triage jobs, JSON API."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import traceback
|
|
import urllib.parse
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from typing import Any, Optional
|
|
|
|
from . import (VERSION, alerts as alertlib, comms, cxbridge, integrations, linkage,
|
|
runbooks, screening, settings as settings_mod, ui, ui_linkage,
|
|
ui_settings, ui_v2)
|
|
from .prometheus import (
|
|
AlertCache, PrometheusClient, PrometheusError, RuleIndex, StateSnapshot, TrueAgeIndex,
|
|
parse_alert_text,
|
|
)
|
|
|
|
TRIAGE_WORKERS = 3
|
|
JOB_TTL_SECONDS = 30 * 60
|
|
|
|
|
|
class Jobs:
|
|
"""In-memory triage jobs. Diagnosis takes tens of seconds, so the UI polls."""
|
|
|
|
def __init__(self, workers: int = TRIAGE_WORKERS):
|
|
self._pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="triage")
|
|
self._lock = threading.Lock()
|
|
self._jobs: dict[str, dict[str, Any]] = {}
|
|
|
|
def submit(self, alert: alertlib.Alert, prom: PrometheusClient,
|
|
snap: Any = None, force: bool = False, user_settings: Any = None) -> str:
|
|
job_id = uuid.uuid4().hex[:12]
|
|
with self._lock:
|
|
self._reap()
|
|
self._jobs[job_id] = {
|
|
"id": job_id,
|
|
"state": "running",
|
|
"created": time.time(),
|
|
"alert": alert.to_json(),
|
|
"result": None,
|
|
"error": "",
|
|
}
|
|
self._pool.submit(self._run, job_id, alert, prom, snap, force, user_settings)
|
|
return job_id
|
|
|
|
def _run(self, job_id: str, alert: alertlib.Alert, prom: PrometheusClient,
|
|
snap: Any = None, force: bool = False, user_settings: Any = None) -> None:
|
|
started = time.monotonic()
|
|
try:
|
|
diagnosis = runbooks.diagnose(alert, prom, snap, force, user_settings)
|
|
payload = diagnosis.to_json()
|
|
payload["elapsed_seconds"] = round(time.monotonic() - started, 1)
|
|
with self._lock:
|
|
job = self._jobs.get(job_id)
|
|
if job is not None:
|
|
job.update({"state": "done", "result": payload})
|
|
except Exception:
|
|
with self._lock:
|
|
job = self._jobs.get(job_id)
|
|
if job is not None:
|
|
job.update({"state": "error", "error": traceback.format_exc(limit=4)})
|
|
|
|
def get(self, job_id: str) -> Optional[dict[str, Any]]:
|
|
with self._lock:
|
|
job = self._jobs.get(job_id)
|
|
return dict(job) if job else None
|
|
|
|
def _reap(self) -> None:
|
|
cutoff = time.time() - JOB_TTL_SECONDS
|
|
for key in [k for k, v in self._jobs.items() if v["created"] < cutoff]:
|
|
self._jobs.pop(key, None)
|
|
|
|
|
|
class App:
|
|
def __init__(self, prom: PrometheusClient):
|
|
self.prom = prom
|
|
self.cache = AlertCache(prom)
|
|
self.rules = RuleIndex(prom)
|
|
self.snapshot = StateSnapshot(prom)
|
|
self.true_age = TrueAgeIndex(prom)
|
|
self.jobs = Jobs()
|
|
self.scan = linkage.Scan()
|
|
self.settings = settings_mod.Settings()
|
|
|
|
def warm(self, log=print) -> None:
|
|
"""Populate the caches before serving.
|
|
|
|
Recovering true alert ages reads a week of ALERTS history, so doing it
|
|
lazily would make the first page load take ~20 seconds.
|
|
"""
|
|
try:
|
|
self.rules.ensure()
|
|
log(f" rule index: {self.rules.count} alerting rule(s)")
|
|
snap = self.snapshot.get()
|
|
log(f" state snapshot: {len(snap.by_openstack_id)} VMs, {len(snap.total_gpus)} hosts")
|
|
if snap.pipeline_dips:
|
|
log(f" {len(snap.pipeline_dips)} Resources metric dip(s) in the last 24h "
|
|
"- alert ages will be recovered from history")
|
|
ages = self.true_age.get()
|
|
log(f" alert history: {ages.count} alert(s) indexed over {ages.WINDOW_DAYS} days")
|
|
except PrometheusError as exc:
|
|
log(f" WARN: could not warm Prometheus caches: {exc}")
|
|
|
|
# --- endpoints ---------------------------------------------------------
|
|
|
|
def health(self) -> dict[str, Any]:
|
|
checks: list[dict[str, Any]] = []
|
|
|
|
def add(name: str, ok: bool, detail: str) -> None:
|
|
checks.append({"name": name, "ok": ok, "detail": detail})
|
|
|
|
try:
|
|
path = cxbridge.cx_tools_path()
|
|
add("CX-Tools", True, path)
|
|
except cxbridge.BridgeError as exc:
|
|
add("CX-Tools", False, str(exc))
|
|
|
|
try:
|
|
cfg = cxbridge.config()
|
|
add("API credentials", True, "Infrahub and InfraInsight keys loaded from 1Password")
|
|
add("Infrahub endpoint", True, cfg.infrahub_base)
|
|
except cxbridge.BridgeError as exc:
|
|
add("API credentials", False, str(exc))
|
|
|
|
try:
|
|
rc, out, _err = _run(["docker", "ps", "--format", "{{.Names}}"], 10)
|
|
running = {x.strip() for x in out.splitlines() if x.strip()} if rc == 0 else set()
|
|
expected = {"ca1-osc", "ca2-osc", "us1-osc", "no1-osc"}
|
|
missing = sorted(expected - running)
|
|
add("OpenStack CLI containers", not missing,
|
|
"all present" if not missing else f"not running: {', '.join(missing)}")
|
|
except Exception as exc:
|
|
add("OpenStack CLI containers", False, str(exc))
|
|
|
|
try:
|
|
add("Prometheus", True, f"{self.prom.base} ({self.prom.describe_transport()})")
|
|
except PrometheusError as exc:
|
|
add("Prometheus", False, str(exc))
|
|
|
|
return {"version": VERSION, "ok": all(c["ok"] for c in checks), "checks": checks}
|
|
|
|
def alert_queue(self, force: bool = False) -> dict[str, Any]:
|
|
raw, error, age = self.cache.get(force=force)
|
|
snap = self.snapshot.get()
|
|
ages = self.true_age.get()
|
|
parsed = [alertlib.from_prometheus(a, self.rules, ages) for a in raw]
|
|
|
|
excluded = [a for a in parsed if alertlib.is_excluded(a)]
|
|
candidates = [a for a in parsed if not alertlib.is_excluded(a)]
|
|
|
|
cx = [a for a in candidates if a.category == "cx" and alertlib.cx_relevant(a)]
|
|
screening.screen_all(cx, snap, self.settings)
|
|
|
|
# Everything that is not a CX runbook alert: node-exporter in its own
|
|
# section, then the rest of the platform rules. Split by identity, since
|
|
# two distinct alerts can compare equal field-for-field.
|
|
in_cx = {id(a) for a in cx}
|
|
infra = [a for a in candidates if id(a) not in in_cx]
|
|
infra.sort(key=alertlib.sort_key)
|
|
|
|
return {
|
|
"error": error or snap.error or ages.error,
|
|
"cache_age_seconds": round(age, 1),
|
|
"warnings": screening.health_warnings(snap),
|
|
"totals": {
|
|
"prometheus": len(parsed),
|
|
"cx": len(cx),
|
|
"infrastructure": len(infra),
|
|
"excluded": len(excluded),
|
|
},
|
|
"excluded_note": (
|
|
f"{len(excluded)} '{', '.join(sorted({a.alertname for a in excluded}))}' alerts hidden"
|
|
if excluded else ""
|
|
),
|
|
"integrations": {"zendesk": integrations.zendesk_configured(),
|
|
"jira": integrations.jira_configured()},
|
|
"summary": screening.summarize(cx),
|
|
"groups": alertlib.group_alerts(cx),
|
|
"infrastructure": _infra_sections(infra),
|
|
}
|
|
|
|
def triage(self, labels: dict[str, str], annotations: Optional[dict[str, str]] = None,
|
|
state: str = "firing", active_at: Any = None, force: bool = False) -> dict[str, Any]:
|
|
alert = alertlib.from_labels(labels, annotations, state=state, active_at=active_at)
|
|
if alert.kind == "excluded":
|
|
return {"error": f"'{alertlib.clean_alertname(alert.alertname)}' is excluded as a monitoring fault."}
|
|
if alert.kind == "other":
|
|
return {"error": f"'{alertlib.clean_alertname(alert.alertname)}' is not an alert type the CX runbooks cover."}
|
|
meta = self.rules.get(alert.alertname)
|
|
if meta:
|
|
alert.rule_file = str(meta.get("file") or "")
|
|
alert.rule_group = str(meta.get("group") or "")
|
|
alert.for_seconds = int(meta.get("for_seconds") or 0)
|
|
snap = self.snapshot.get()
|
|
alert.true_age_minutes, alert.true_age_capped = self.true_age.get().lookup(alert.labels)
|
|
alert.screen = screening.screen(alert, snap, self.settings)
|
|
job_id = self.jobs.submit(alert, self.prom, snap, force, self.settings)
|
|
return {"job_id": job_id, "alert": alert.to_json()}
|
|
|
|
def triage_by_id(self, alert_id: str, force: bool = False) -> dict[str, Any]:
|
|
raw, error, _age = self.cache.get()
|
|
if error and not raw:
|
|
return {"error": error}
|
|
for item in raw:
|
|
alert = alertlib.from_prometheus(item, self.rules, self.true_age.get())
|
|
if alert.fingerprint() == alert_id:
|
|
return self.triage(alert.labels, alert.annotations, alert.state,
|
|
item.get("activeAt"), force=force)
|
|
return {"error": "That alert is no longer firing. Refresh the queue."}
|
|
|
|
def parse(self, text: str) -> dict[str, Any]:
|
|
found = parse_alert_text(text)
|
|
if not found:
|
|
return {"error": "Could not find any label set in that text. Paste an ALERTS{...} line or a Prometheus graph URL."}
|
|
out = []
|
|
for labels in found:
|
|
alert = alertlib.from_labels(labels)
|
|
out.append({"alert": alert.to_json(), "supported": alertlib.cx_relevant(alert)})
|
|
return {"parsed": out}
|
|
|
|
def settings_payload(self) -> dict[str, Any]:
|
|
return self.settings.to_json()
|
|
|
|
def _current_alerts(self) -> list[Any]:
|
|
raw, _error, _age = self.cache.get()
|
|
return [a for a in (alertlib.from_prometheus(x, self.rules) for x in raw)
|
|
if not alertlib.is_excluded(a) and alertlib.cx_relevant(a)]
|
|
|
|
def settings_action(self, body: dict[str, Any]) -> dict[str, Any]:
|
|
action = str(body.get("action") or "")
|
|
if action == "save_rule":
|
|
rule = self.settings.upsert_rule(body.get("rule") or {})
|
|
return {"ok": True, "rule": rule, "settings": self.settings.to_json()}
|
|
if action == "delete_rule":
|
|
self.settings.delete_rule(str(body.get("id") or ""))
|
|
return {"ok": True, "settings": self.settings.to_json()}
|
|
if action == "toggle_rule":
|
|
self.settings.toggle_rule(str(body.get("id") or ""), bool(body.get("enabled")))
|
|
return {"ok": True, "settings": self.settings.to_json()}
|
|
if action == "general":
|
|
self.settings.set_general(body.get("agent_name"), body.get("chronic_days"))
|
|
return {"ok": True, "settings": self.settings.to_json()}
|
|
if action == "preview":
|
|
hits = settings_mod.preview(self.settings, body.get("rule") or {}, self._current_alerts())
|
|
return {"ok": True, "matches": hits, "count": len(hits)}
|
|
return {"ok": False, "error": f"Unknown action: {action}"}
|
|
|
|
def start_scan(self) -> dict[str, Any]:
|
|
if self.scan.state == "running":
|
|
return {"started": False, "reason": "already running"}
|
|
snap = self.snapshot.get()
|
|
threading.Thread(target=self.scan.run, args=(snap,), daemon=True).start()
|
|
return {"started": True}
|
|
|
|
def send_zendesk(self, body: dict[str, Any]) -> dict[str, Any]:
|
|
"""Deliberately refuses until Zendesk is configured AND enabled.
|
|
|
|
Contacting a customer is the one thing this tool must never do as a
|
|
side effect, so delivery stays behind explicit configuration rather
|
|
than being reachable from the UI by default.
|
|
"""
|
|
if not integrations.zendesk_configured():
|
|
return {"ok": False, "error": "Zendesk is not configured. Set CX_ZENDESK_SUBDOMAIN, "
|
|
"CX_ZENDESK_EMAIL and CX_ZENDESK_TOKEN, then restart."}
|
|
return {"ok": False, "error": "Sending is not enabled in this build. The payload is ready; "
|
|
"wiring delivery is a deliberate, separate step."}
|
|
|
|
def templates(self) -> dict[str, Any]:
|
|
return {"templates": [d.to_json() for d in (comms.draft(k) for k in comms._TEMPLATES) if d]}
|
|
|
|
|
|
SOURCE_LABELS = {
|
|
"node-exporter-rules.yml": "Node exporter (hosts)",
|
|
"ceph-rules.yml": "Ceph",
|
|
"mysql-rules.yml": "MySQL",
|
|
"mysql-performance-rules.yml": "MySQL performance",
|
|
"galera-rules.yml": "Galera",
|
|
"openstack-rules.yml": "OpenStack services",
|
|
"blackbox.yml": "Blackbox / OOB",
|
|
"infrahub-rules.yml": "Infrahub (no CX runbook)",
|
|
}
|
|
|
|
|
|
def _infra_sections(items: list[alertlib.Alert]) -> list[dict[str, Any]]:
|
|
"""Group infrastructure alerts by their rule file, node-exporter first."""
|
|
buckets: dict[str, list[alertlib.Alert]] = {}
|
|
for alert in items:
|
|
buckets.setdefault(alert.rule_file or "unknown", []).append(alert)
|
|
|
|
sections = []
|
|
for source, members in buckets.items():
|
|
by_name: dict[str, int] = {}
|
|
for alert in members:
|
|
name = alertlib.clean_alertname(alert.alertname)
|
|
by_name[name] = by_name.get(name, 0) + 1
|
|
sections.append({
|
|
"source": source,
|
|
"label": SOURCE_LABELS.get(source, source),
|
|
"total": len(members),
|
|
"by_alertname": sorted(({"name": k, "count": v} for k, v in by_name.items()),
|
|
key=lambda x: (-x["count"], x["name"])),
|
|
"alerts": [a.to_json() for a in members],
|
|
})
|
|
sections.sort(key=lambda s: (s["source"] != alertlib.NODE_RULE_FILE, -s["total"]))
|
|
return sections
|
|
|
|
|
|
def _run(cmd: list[str], timeout: int) -> tuple[int, str, str]:
|
|
try:
|
|
p = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout)
|
|
return p.returncode, p.stdout, p.stderr
|
|
except Exception as exc:
|
|
return 1, "", str(exc)
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
server_version = f"cx-triage/{VERSION}"
|
|
app: App
|
|
|
|
def log_message(self, fmt: str, *args: Any) -> None:
|
|
if self.path.startswith("/api/jobs/"):
|
|
return
|
|
print(f" {self.command} {self.path}")
|
|
|
|
# --- helpers ----------------------------------------------------------
|
|
|
|
def _send_json(self, payload: Any, code: int = 200) -> None:
|
|
body = json.dumps(payload, default=str).encode()
|
|
self.send_response(code)
|
|
self.send_header("Content-Type", "application/json; charset=utf-8")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.send_header("Cache-Control", "no-store")
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def _send_html(self, html: str) -> None:
|
|
body = html.encode()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def _body_json(self) -> dict[str, Any]:
|
|
length = int(self.headers.get("Content-Length") or 0)
|
|
if not length:
|
|
return {}
|
|
try:
|
|
return json.loads(self.rfile.read(length).decode("utf-8", errors="replace")) or {}
|
|
except json.JSONDecodeError:
|
|
return {}
|
|
|
|
# --- routes -----------------------------------------------------------
|
|
|
|
def do_GET(self) -> None: # noqa: N802
|
|
parsed = urllib.parse.urlparse(self.path)
|
|
path = parsed.path
|
|
query = urllib.parse.parse_qs(parsed.query)
|
|
|
|
try:
|
|
if path in ("/", "/index.html"):
|
|
self._send_html(ui_v2.PAGE)
|
|
elif path == "/classic":
|
|
self._send_html(ui.PAGE)
|
|
elif path == "/settings":
|
|
self._send_html(ui_settings.PAGE)
|
|
elif path == "/api/settings":
|
|
self._send_json(self.app.settings_payload())
|
|
elif path == "/linkage":
|
|
self._send_html(ui_linkage.PAGE)
|
|
elif path == "/api/linkage":
|
|
self._send_json(self.app.scan.to_json())
|
|
elif path == "/api/health":
|
|
self._send_json(self.app.health())
|
|
elif path == "/api/alerts":
|
|
self._send_json(self.app.alert_queue(force=query.get("force", ["0"])[0] == "1"))
|
|
elif path == "/api/templates":
|
|
self._send_json(self.app.templates())
|
|
elif path.startswith("/api/jobs/"):
|
|
job = self.app.jobs.get(path.rsplit("/", 1)[-1])
|
|
self._send_json(job or {"error": "Unknown job."}, 200 if job else 404)
|
|
else:
|
|
self._send_json({"error": "Not found."}, 404)
|
|
except Exception as exc:
|
|
self._send_json({"error": f"{type(exc).__name__}: {exc}"}, 500)
|
|
|
|
def do_POST(self) -> None: # noqa: N802
|
|
path = urllib.parse.urlparse(self.path).path
|
|
body = self._body_json()
|
|
try:
|
|
force = bool(body.get("force"))
|
|
if path == "/api/triage":
|
|
if body.get("alert_id"):
|
|
self._send_json(self.app.triage_by_id(str(body["alert_id"]), force=force))
|
|
elif isinstance(body.get("labels"), dict):
|
|
self._send_json(self.app.triage(body["labels"], body.get("annotations"), force=force))
|
|
else:
|
|
self._send_json({"error": "Provide alert_id or labels."}, 400)
|
|
elif path == "/api/parse":
|
|
self._send_json(self.app.parse(str(body.get("text") or "")))
|
|
elif path == "/api/actions/zendesk":
|
|
self._send_json(self.app.send_zendesk(body))
|
|
elif path == "/api/settings":
|
|
self._send_json(self.app.settings_action(body))
|
|
elif path == "/api/linkage/scan":
|
|
self._send_json(self.app.start_scan())
|
|
elif path == "/api/linkage/enrich":
|
|
self._send_json(linkage.enrich(str(body.get("region") or ""),
|
|
str(body.get("openstack_id") or "")))
|
|
else:
|
|
self._send_json({"error": "Not found."}, 404)
|
|
except Exception as exc:
|
|
self._send_json({"error": f"{type(exc).__name__}: {exc}"}, 500)
|
|
|
|
|
|
def serve(host: str = "127.0.0.1", port: int = 8765, prometheus_base: Optional[str] = None) -> None:
|
|
from .prometheus import DEFAULT_BASE
|
|
|
|
prom = PrometheusClient(prometheus_base or DEFAULT_BASE)
|
|
app = App(prom)
|
|
print("Warming caches...")
|
|
app.warm()
|
|
Handler.app = app
|
|
httpd = ThreadingHTTPServer((host, port), Handler)
|
|
httpd.daemon_threads = True
|
|
print(f"cx-triage listening on http://{host}:{port}")
|
|
try:
|
|
httpd.serve_forever()
|
|
except KeyboardInterrupt:
|
|
print("\nshutting down")
|
|
finally:
|
|
httpd.server_close()
|