Files
cx-ui/triagelib/settings.py
Parham Monfared a039e0b5fd CX Triage: alert diagnosis over the CX-Tools collectors
Read-only triage for the Infrahub error alerts. Pulls the Prometheus alert
queue, re-checks each alert's condition against live state to separate real
work from noise, diagnoses it using the CX runbooks, and drafts the customer
comms with contacts resolved from Infrahub.

Findings from validating against production:
- "Suspected Rogue VM" fires on spare GPU capacity, not rogue VMs: In_Use_Gpus
  equals the physical count on 71 of 75 firing hosts, so the rule reduces to
  "this host has a free GPU". Verified against OpenStack on 10 hosts.
- "Exists in Infrahub but does not exist in OpenStack" matches every VM because
  openstack_nova_server_status returns no series; excluded as a rule defect.
- Prometheus activeAt is reset several times a day by dips in the Resources
  metric, so alert ages are recovered from ALERTS history instead.

Takes ~2,650 firing alerts down to ~20 that need a decision.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-06 06:48:34 +01:00

246 lines
8.5 KiB
Python

"""User settings: suppression rules and comms identity.
Suppression rules replace hardcoded judgement calls. The internal-organisation
check used to be baked into the screening code, which meant the one person who
knew about it was whoever read the source. Now it ships as an editable default
rule that says what it does and why, and anyone can add their own.
A rule matches when *every* condition it sets matches (AND); within one
condition, any listed value matches (OR). So "type is error AND organisation
contains modal" is one rule with two conditions.
"""
from __future__ import annotations
import json
import os
import re
import threading
import time
import uuid
from typing import Any, Optional
SETTINGS_DIR = os.path.expanduser(os.environ.get("CX_TRIAGE_HOME", "~/.cx-triage"))
SETTINGS_PATH = os.path.join(SETTINGS_DIR, "settings.json")
# Conditions a rule can set. Every one is a substring match, case-insensitive,
# except `kind` and `region` which are exact.
CONDITIONS = {
"kind": "Alert type",
"organization": "Organisation contains",
"instance_name": "VM name contains",
"host": "Host contains",
"region": "Region",
"status": "Status is",
}
DEFAULT_RULES: list[dict[str, Any]] = [
{
"id": "builtin-internal-orgs",
"name": "Internal NexGen organisations",
"enabled": True,
"reason": "Owned by an internal test or platform organisation, not a customer.",
"conditions": {"organization": ["nexgencloud.com"]},
},
{
"id": "builtin-runpod-storage",
"name": "Runpod storage nodes (Luis)",
"enabled": True,
"reason": "Platform-owned storage nodes; SHUTOFF on these is expected and not customer-impacting.",
"conditions": {"kind": ["shutoff"], "instance_name": ["stor-runpod"]},
},
]
DEFAULTS: dict[str, Any] = {
"rules": DEFAULT_RULES,
"agent_name": "",
"chronic_days": 3,
}
def _blank(value: Any) -> bool:
return value is None or str(value).strip() == ""
class Settings:
"""Loaded once, written through on every change."""
def __init__(self, path: str = SETTINGS_PATH):
self.path = path
self._lock = threading.Lock()
self._data: dict[str, Any] = {}
self.load()
# --- persistence -------------------------------------------------------
def load(self) -> None:
data = dict(DEFAULTS)
try:
with open(self.path, encoding="utf-8") as handle:
stored = json.load(handle)
if isinstance(stored, dict):
data.update(stored)
except (OSError, json.JSONDecodeError):
pass
data["rules"] = [r for r in (data.get("rules") or []) if isinstance(r, dict)]
with self._lock:
self._data = data
def save(self) -> None:
with self._lock:
payload = json.dumps(self._data, indent=2, sort_keys=True)
try:
os.makedirs(os.path.dirname(self.path), exist_ok=True)
tmp = f"{self.path}.tmp"
with open(tmp, "w", encoding="utf-8") as handle:
handle.write(payload)
os.replace(tmp, self.path)
except OSError:
pass
# --- accessors ---------------------------------------------------------
@property
def rules(self) -> list[dict[str, Any]]:
with self._lock:
return [dict(r) for r in self._data.get("rules", [])]
@property
def agent_name(self) -> str:
with self._lock:
return str(self._data.get("agent_name") or "")
@property
def chronic_days(self) -> int:
with self._lock:
try:
return max(1, int(self._data.get("chronic_days") or 3))
except (TypeError, ValueError):
return 3
def to_json(self) -> dict[str, Any]:
with self._lock:
return {
"rules": [dict(r) for r in self._data.get("rules", [])],
"agent_name": self._data.get("agent_name") or "",
"chronic_days": self._data.get("chronic_days", 3),
"conditions": CONDITIONS,
"path": self.path,
}
# --- mutations ---------------------------------------------------------
def set_general(self, agent_name: Optional[str] = None, chronic_days: Optional[Any] = None) -> None:
with self._lock:
if agent_name is not None:
self._data["agent_name"] = str(agent_name).strip()
if chronic_days is not None:
try:
self._data["chronic_days"] = max(1, int(chronic_days))
except (TypeError, ValueError):
pass
self.save()
def upsert_rule(self, rule: dict[str, Any]) -> dict[str, Any]:
clean = _normalize_rule(rule)
with self._lock:
rules = self._data.setdefault("rules", [])
for idx, existing in enumerate(rules):
if existing.get("id") == clean["id"]:
rules[idx] = clean
break
else:
rules.append(clean)
self.save()
return clean
def delete_rule(self, rule_id: str) -> None:
with self._lock:
self._data["rules"] = [r for r in self._data.get("rules", []) if r.get("id") != rule_id]
self.save()
def toggle_rule(self, rule_id: str, enabled: bool) -> None:
with self._lock:
for rule in self._data.get("rules", []):
if rule.get("id") == rule_id:
rule["enabled"] = bool(enabled)
self.save()
def _normalize_rule(rule: dict[str, Any]) -> dict[str, Any]:
conditions: dict[str, list[str]] = {}
for field, values in (rule.get("conditions") or {}).items():
if field not in CONDITIONS:
continue
if isinstance(values, str):
values = [v.strip() for v in values.split(",")]
cleaned = [str(v).strip() for v in (values or []) if str(v).strip()]
if cleaned:
conditions[field] = cleaned
return {
"id": str(rule.get("id") or f"rule-{uuid.uuid4().hex[:8]}"),
"name": str(rule.get("name") or "Untitled rule").strip(),
"enabled": bool(rule.get("enabled", True)),
"reason": str(rule.get("reason") or "").strip(),
"conditions": conditions,
"created": rule.get("created") or time.strftime("%Y-%m-%d"),
}
# --- matching ---------------------------------------------------------------
def _alert_field(alert: Any, field: str) -> str:
if field == "kind":
return str(getattr(alert, "kind", ""))
if field == "organization":
return f"{getattr(alert, 'org_id', '')} {getattr(alert, 'org_name', '')}"
if field == "instance_name":
return str(getattr(alert, "instance_name", ""))
if field == "host":
return str(getattr(alert, "host", ""))
if field == "region":
return f"{getattr(alert, 'region', '')} {getattr(alert, 'region_label', '')}"
if field == "status":
return str(getattr(alert, "status", ""))
return ""
def _condition_matches(field: str, values: list[str], alert: Any) -> bool:
actual = _alert_field(alert, field).lower()
if field in ("kind", "status"):
return any(actual == str(v).strip().lower() for v in values)
if field == "region":
return any(str(v).strip().lower() in actual for v in values)
return any(str(v).strip().lower() in actual for v in values)
def rule_matches(rule: dict[str, Any], alert: Any) -> bool:
"""Every condition in the rule must match (AND)."""
conditions = rule.get("conditions") or {}
if not conditions:
return False # an empty rule would swallow the whole queue
return all(_condition_matches(f, v, alert) for f, v in conditions.items())
def first_match(settings: Settings, alert: Any) -> Optional[dict[str, Any]]:
for rule in settings.rules:
if rule.get("enabled") and rule_matches(rule, alert):
return rule
return None
def preview(settings: Settings, rule: dict[str, Any], alerts: list[Any]) -> list[dict[str, Any]]:
"""Which currently-firing alerts a rule would hide - shown before saving."""
clean = _normalize_rule(rule)
hits = []
for alert in alerts:
if rule_matches(clean, alert):
hits.append({
"kind": alert.kind,
"title": alert.title,
"instance_name": alert.instance_name,
"host": alert.host,
"org_name": alert.org_name,
"region": alert.region,
})
return hits