Split into a FastAPI backend and a React frontend, add case state and SSO
The single-file stdlib server became the limit: no way to track what had been done about an alert, no accounts, and a UI that had to be hand-rolled in template strings. This restructures it into something deployable. Backend (FastAPI) - app/ holds config, database, auth, delivery and the routers; triagelib keeps the triage engine unchanged, so the validated screening and runbook logic is untouched. - Cases persist per alert fingerprint with a status workflow (investigating, customer contacted, escalated to Infra, waiting, remediated, resolved, won't fix, false positive), an assignee, notes and an append-only history. An alert that stops and re-fires lands back on the same case and counts as a reopen. - Suppression rules move from a JSON file into the database. Auth - Signed session cookies over PBKDF2 local accounts, plus an OIDC flow ready for Authentik: users are created on first login and admin follows a group claim. Local login can be switched off entirely once SSO is live. Zendesk and Jira - Delivery is now implemented, behind three gates: the integration must be configured, its feature flag on, and CX_FEATURE_SEND_ENABLED on. A demo instance leaves the last off and cannot mail anyone. Both search before creating, so re-diagnosing an alert updates one ticket rather than opening several, and a rolling daily cap stops a loop mailing everybody. Deployment - Multi-stage Dockerfile builds the bundle and serves it from the API origin. - docker-compose for local and single-host use; Gitea Actions runs the tests, builds the image and renders deploy/k8s with envsubst. Two fixes found while testing: assigning a case returned a null assignee, and add_event could leave an already-loaded history collection stale. Known gap: the engine reaches OpenStack via `docker exec <region>-osc`, which does not work in a pod without the CX-Tools containers alongside it. docs/DEPLOYMENT.md sets out the three ways to close that. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2
backend/app/__init__.py
Normal file
2
backend/app/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""CX Triage backend."""
|
||||
VERSION = "0.2.0"
|
||||
253
backend/app/auth.py
Normal file
253
backend/app/auth.py
Normal file
@@ -0,0 +1,253 @@
|
||||
"""Authentication: signed session cookies, local accounts, and OIDC.
|
||||
|
||||
Local accounts exist so the tool runs on a laptop and in a cluster that has not
|
||||
been wired to SSO yet. When CX_OIDC_ENABLED is set, Authentik becomes the source
|
||||
of truth: users are created on first login, and admin rights follow a group
|
||||
claim rather than being set here.
|
||||
|
||||
The session itself is the same either way - a signed, expiring cookie - so the
|
||||
rest of the app never has to care which provider a user came from.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
import httpx
|
||||
from fastapi import Depends, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .config import get_settings
|
||||
from .db import get_db
|
||||
from .models import AuthProvider, User
|
||||
|
||||
SESSION_COOKIE = "cx_session"
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
# --- password hashing -------------------------------------------------------
|
||||
# PBKDF2 from the standard library: no native build step in the image, and
|
||||
# strong enough for a small internal user table.
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
salt = secrets.token_bytes(16)
|
||||
digest = hashlib.pbkdf2_hmac("sha256", password.encode(), salt, 240_000)
|
||||
return f"pbkdf2_sha256$240000${base64.b64encode(salt).decode()}${base64.b64encode(digest).decode()}"
|
||||
|
||||
|
||||
def verify_password(password: str, stored: str) -> bool:
|
||||
try:
|
||||
algo, rounds, salt_b64, digest_b64 = stored.split("$")
|
||||
if algo != "pbkdf2_sha256":
|
||||
return False
|
||||
expected = base64.b64decode(digest_b64)
|
||||
actual = hashlib.pbkdf2_hmac("sha256", password.encode(),
|
||||
base64.b64decode(salt_b64), int(rounds))
|
||||
return hmac.compare_digest(expected, actual)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
# --- session cookie ---------------------------------------------------------
|
||||
|
||||
def _sign(payload: bytes) -> str:
|
||||
mac = hmac.new(settings.secret_key.encode(), payload, hashlib.sha256).digest()
|
||||
return f"{base64.urlsafe_b64encode(payload).decode()}.{base64.urlsafe_b64encode(mac).decode()}"
|
||||
|
||||
|
||||
def issue_session(user_id: int) -> str:
|
||||
payload = json.dumps({
|
||||
"uid": user_id,
|
||||
"exp": int(time.time()) + settings.session_hours * 3600,
|
||||
}).encode()
|
||||
return _sign(payload)
|
||||
|
||||
|
||||
def read_session(token: str) -> Optional[int]:
|
||||
try:
|
||||
body_b64, mac_b64 = token.split(".")
|
||||
payload = base64.urlsafe_b64decode(body_b64)
|
||||
expected = hmac.new(settings.secret_key.encode(), payload, hashlib.sha256).digest()
|
||||
if not hmac.compare_digest(expected, base64.urlsafe_b64decode(mac_b64)):
|
||||
return None
|
||||
data = json.loads(payload)
|
||||
if int(data.get("exp", 0)) < time.time():
|
||||
return None
|
||||
return int(data["uid"])
|
||||
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
# --- dependencies -----------------------------------------------------------
|
||||
|
||||
def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
token = request.cookies.get(SESSION_COOKIE, "")
|
||||
uid = read_session(token) if token else None
|
||||
if not uid:
|
||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Not signed in")
|
||||
user = db.get(User, uid)
|
||||
if not user or not user.is_active:
|
||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Account is not active")
|
||||
return user
|
||||
|
||||
|
||||
def require_admin(user: User = Depends(current_user)) -> User:
|
||||
if not user.is_admin:
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, "Administrator access required")
|
||||
return user
|
||||
|
||||
|
||||
def optional_user(request: Request, db: Session = Depends(get_db)) -> Optional[User]:
|
||||
token = request.cookies.get(SESSION_COOKIE, "")
|
||||
uid = read_session(token) if token else None
|
||||
return db.get(User, uid) if uid else None
|
||||
|
||||
|
||||
# --- local accounts ---------------------------------------------------------
|
||||
|
||||
def authenticate_local(db: Session, email: str, password: str) -> Optional[User]:
|
||||
if not settings.auth_local_enabled:
|
||||
return None
|
||||
user = db.query(User).filter(User.email == email.strip().lower()).one_or_none()
|
||||
if not user or not user.is_active or user.provider != AuthProvider.LOCAL:
|
||||
return None
|
||||
if not user.password_hash or not verify_password(password, user.password_hash):
|
||||
return None
|
||||
return user
|
||||
|
||||
|
||||
def ensure_bootstrap_admin(db: Session) -> Optional[str]:
|
||||
"""Create the first admin so a fresh deployment is reachable.
|
||||
|
||||
Only ever runs when the user table is empty, and only with a password
|
||||
supplied through the environment - it never invents one.
|
||||
"""
|
||||
if db.query(User).count():
|
||||
return None
|
||||
if not settings.bootstrap_admin_password:
|
||||
return ("No users exist and CX_BOOTSTRAP_ADMIN_PASSWORD is unset. "
|
||||
"Set it and restart, or sign in through SSO.")
|
||||
db.add(User(
|
||||
email=settings.bootstrap_admin_email.strip().lower(),
|
||||
name="Bootstrap admin",
|
||||
is_admin=True,
|
||||
provider=AuthProvider.LOCAL,
|
||||
password_hash=hash_password(settings.bootstrap_admin_password),
|
||||
))
|
||||
db.commit()
|
||||
return f"Created bootstrap admin {settings.bootstrap_admin_email}"
|
||||
|
||||
|
||||
# --- OIDC / Authentik -------------------------------------------------------
|
||||
|
||||
class OIDCError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
_discovery_cache: dict[str, Any] = {}
|
||||
|
||||
|
||||
async def oidc_discovery() -> dict[str, Any]:
|
||||
"""Fetch and cache the provider metadata."""
|
||||
if not settings.oidc_issuer:
|
||||
raise OIDCError("CX_OIDC_ISSUER is not set.")
|
||||
if _discovery_cache.get("_issuer") == settings.oidc_issuer:
|
||||
return _discovery_cache
|
||||
url = settings.oidc_issuer.rstrip("/") + "/.well-known/openid-configuration"
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
resp = await client.get(url)
|
||||
if resp.status_code != 200:
|
||||
raise OIDCError(f"OIDC discovery failed ({resp.status_code}) at {url}")
|
||||
data = resp.json()
|
||||
data["_issuer"] = settings.oidc_issuer
|
||||
_discovery_cache.clear()
|
||||
_discovery_cache.update(data)
|
||||
return data
|
||||
|
||||
|
||||
def oidc_state() -> str:
|
||||
"""A signed, short-lived value tying the callback to this browser."""
|
||||
payload = json.dumps({"n": secrets.token_urlsafe(16), "exp": int(time.time()) + 600}).encode()
|
||||
return _sign(payload)
|
||||
|
||||
|
||||
def oidc_state_valid(state: str) -> bool:
|
||||
try:
|
||||
body_b64, mac_b64 = state.split(".")
|
||||
payload = base64.urlsafe_b64decode(body_b64)
|
||||
expected = hmac.new(settings.secret_key.encode(), payload, hashlib.sha256).digest()
|
||||
if not hmac.compare_digest(expected, base64.urlsafe_b64decode(mac_b64)):
|
||||
return False
|
||||
return int(json.loads(payload).get("exp", 0)) >= time.time()
|
||||
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
|
||||
return False
|
||||
|
||||
|
||||
async def oidc_exchange(code: str, redirect_uri: str) -> dict[str, Any]:
|
||||
meta = await oidc_discovery()
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
token_resp = await client.post(meta["token_endpoint"], data={
|
||||
"grant_type": "authorization_code",
|
||||
"code": code,
|
||||
"redirect_uri": redirect_uri,
|
||||
"client_id": settings.oidc_client_id,
|
||||
"client_secret": settings.oidc_client_secret,
|
||||
}, headers={"Accept": "application/json"})
|
||||
if token_resp.status_code != 200:
|
||||
raise OIDCError(f"Token exchange failed ({token_resp.status_code}): {token_resp.text[:200]}")
|
||||
access = token_resp.json().get("access_token")
|
||||
if not access:
|
||||
raise OIDCError("Token response contained no access_token.")
|
||||
|
||||
info_resp = await client.get(meta["userinfo_endpoint"],
|
||||
headers={"Authorization": f"Bearer {access}"})
|
||||
if info_resp.status_code != 200:
|
||||
raise OIDCError(f"userinfo failed ({info_resp.status_code}): {info_resp.text[:200]}")
|
||||
return info_resp.json()
|
||||
|
||||
|
||||
def upsert_oidc_user(db: Session, claims: dict[str, Any]) -> User:
|
||||
"""Find or create the local record for an SSO identity.
|
||||
|
||||
Matching is on `sub` first so a changed email still lands on the same user;
|
||||
an existing local account with the same address is adopted rather than
|
||||
duplicated.
|
||||
"""
|
||||
sub = str(claims.get("sub") or "").strip()
|
||||
email = str(claims.get("email") or "").strip().lower()
|
||||
if not sub and not email:
|
||||
raise OIDCError("SSO returned neither sub nor email.")
|
||||
|
||||
groups = claims.get(settings.oidc_groups_claim) or []
|
||||
if isinstance(groups, str):
|
||||
groups = [groups]
|
||||
is_admin = settings.oidc_admin_group in {str(g) for g in groups}
|
||||
|
||||
user = None
|
||||
if sub:
|
||||
user = db.query(User).filter(User.oidc_sub == sub).one_or_none()
|
||||
if user is None and email:
|
||||
user = db.query(User).filter(User.email == email).one_or_none()
|
||||
|
||||
if user is None:
|
||||
user = User(email=email or f"{sub}@sso.local", provider=AuthProvider.OIDC)
|
||||
db.add(user)
|
||||
|
||||
user.oidc_sub = sub or user.oidc_sub
|
||||
user.provider = AuthProvider.OIDC
|
||||
user.name = str(claims.get("name") or claims.get("preferred_username") or user.name or email)
|
||||
if email:
|
||||
user.email = email
|
||||
user.is_admin = is_admin
|
||||
user.is_active = True
|
||||
user.last_login = dt.datetime.now(dt.timezone.utc)
|
||||
if not user.signoff_name:
|
||||
user.signoff_name = user.name
|
||||
db.commit()
|
||||
return user
|
||||
101
backend/app/config.py
Normal file
101
backend/app/config.py
Normal file
@@ -0,0 +1,101 @@
|
||||
"""Environment-driven configuration.
|
||||
|
||||
Everything deployment-specific comes from the environment so the same image runs
|
||||
locally under compose and in Kubernetes with only a ConfigMap/Secret difference.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from functools import lru_cache
|
||||
|
||||
|
||||
def _bool(name: str, default: bool = False) -> bool:
|
||||
return str(os.environ.get(name, str(default))).strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _int(name: str, default: int) -> int:
|
||||
try:
|
||||
return int(os.environ.get(name, default))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
class Settings:
|
||||
# --- app ---------------------------------------------------------------
|
||||
app_name = os.environ.get("CX_APP_NAME", "CX Triage")
|
||||
base_url = os.environ.get("CX_BASE_URL", "http://localhost:8080")
|
||||
secret_key = os.environ.get("CX_SECRET_KEY", "dev-only-change-me")
|
||||
session_hours = _int("CX_SESSION_HOURS", 12)
|
||||
static_dir = os.environ.get("CX_STATIC_DIR", "/app/static")
|
||||
|
||||
# --- database ----------------------------------------------------------
|
||||
# sqlite for local/compose, postgres in the cluster.
|
||||
database_url = os.environ.get("CX_DATABASE_URL", "sqlite:////data/cx-triage.db")
|
||||
|
||||
# --- data sources ------------------------------------------------------
|
||||
prometheus_base = os.environ.get("CX_PROMETHEUS_BASE", "http://10.11.254.250:9090")
|
||||
prometheus_relay = os.environ.get("CX_PROMETHEUS_RELAY", "")
|
||||
cx_tools_path = os.environ.get("CX_TOOLS_PATH", "")
|
||||
|
||||
# --- auth --------------------------------------------------------------
|
||||
# Local accounts are for development and for a cluster without SSO yet.
|
||||
# When CX_OIDC_ENABLED is on, Authentik becomes the source of truth.
|
||||
auth_local_enabled = _bool("CX_AUTH_LOCAL_ENABLED", True)
|
||||
bootstrap_admin_email = os.environ.get("CX_BOOTSTRAP_ADMIN_EMAIL", "admin@localhost")
|
||||
bootstrap_admin_password = os.environ.get("CX_BOOTSTRAP_ADMIN_PASSWORD", "")
|
||||
|
||||
oidc_enabled = _bool("CX_OIDC_ENABLED", False)
|
||||
oidc_issuer = os.environ.get("CX_OIDC_ISSUER", "") # e.g. https://sso/application/o/cx-triage/
|
||||
oidc_client_id = os.environ.get("CX_OIDC_CLIENT_ID", "")
|
||||
oidc_client_secret = os.environ.get("CX_OIDC_CLIENT_SECRET", "")
|
||||
oidc_scopes = os.environ.get("CX_OIDC_SCOPES", "openid email profile")
|
||||
oidc_admin_group = os.environ.get("CX_OIDC_ADMIN_GROUP", "cx-triage-admins")
|
||||
oidc_groups_claim = os.environ.get("CX_OIDC_GROUPS_CLAIM", "groups")
|
||||
|
||||
# --- feature flags -----------------------------------------------------
|
||||
# Sending must be switched on deliberately; a demo instance cannot email.
|
||||
feature_send_enabled = _bool("CX_FEATURE_SEND_ENABLED", False)
|
||||
feature_zendesk = _bool("CX_FEATURE_ZENDESK", False)
|
||||
feature_jira = _bool("CX_FEATURE_JIRA", False)
|
||||
feature_linkage_scan = _bool("CX_FEATURE_LINKAGE_SCAN", True)
|
||||
send_daily_cap = _int("CX_SEND_DAILY_CAP", 25)
|
||||
|
||||
# --- integrations ------------------------------------------------------
|
||||
zendesk_subdomain = os.environ.get("CX_ZENDESK_SUBDOMAIN", "")
|
||||
zendesk_email = os.environ.get("CX_ZENDESK_EMAIL", "")
|
||||
zendesk_token = os.environ.get("CX_ZENDESK_TOKEN", "")
|
||||
zendesk_default_public = _bool("CX_ZENDESK_PUBLIC_REPLY", True)
|
||||
|
||||
jira_base = os.environ.get("CX_JIRA_BASE", "")
|
||||
jira_email = os.environ.get("CX_JIRA_EMAIL", "")
|
||||
jira_token = os.environ.get("CX_JIRA_TOKEN", "")
|
||||
jira_project = os.environ.get("CX_JIRA_PROJECT", "INFRA")
|
||||
jira_issue_type = os.environ.get("CX_JIRA_ISSUE_TYPE", "Task")
|
||||
|
||||
@property
|
||||
def zendesk_ready(self) -> bool:
|
||||
return bool(self.feature_zendesk and self.zendesk_subdomain
|
||||
and self.zendesk_email and self.zendesk_token)
|
||||
|
||||
@property
|
||||
def jira_ready(self) -> bool:
|
||||
return bool(self.feature_jira and self.jira_base and self.jira_email
|
||||
and self.jira_token and self.jira_project)
|
||||
|
||||
def public_flags(self) -> dict:
|
||||
"""What the frontend is allowed to know - never secrets."""
|
||||
return {
|
||||
"app_name": self.app_name,
|
||||
"oidc_enabled": self.oidc_enabled,
|
||||
"local_login": self.auth_local_enabled,
|
||||
"zendesk_ready": self.zendesk_ready,
|
||||
"jira_ready": self.jira_ready,
|
||||
"send_enabled": self.feature_send_enabled,
|
||||
"linkage_scan": self.feature_linkage_scan,
|
||||
"jira_project": self.jira_project if self.jira_ready else "",
|
||||
}
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
29
backend/app/db.py
Normal file
29
backend/app/db.py
Normal file
@@ -0,0 +1,29 @@
|
||||
"""Database engine and session handling."""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from .config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
_connect_args = {"check_same_thread": False} if settings.database_url.startswith("sqlite") else {}
|
||||
engine = create_engine(settings.database_url, pool_pre_ping=True, connect_args=_connect_args)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
|
||||
|
||||
|
||||
def get_db() -> Iterator[Session]:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
from . import models # noqa: F401 (registers the tables)
|
||||
|
||||
models.Base.metadata.create_all(engine)
|
||||
174
backend/app/delivery.py
Normal file
174
backend/app/delivery.py
Normal file
@@ -0,0 +1,174 @@
|
||||
"""Outbound delivery to Zendesk and Jira.
|
||||
|
||||
Three independent gates have to be open before anything leaves this process:
|
||||
|
||||
1. the integration is configured (subdomain/email/token present)
|
||||
2. its feature flag is on - CX_FEATURE_ZENDESK / CX_FEATURE_JIRA
|
||||
3. sending is globally enabled - CX_FEATURE_SEND_ENABLED
|
||||
|
||||
A demo or staging instance simply leaves the third off, and then no combination
|
||||
of clicks can email a customer. Every send is recorded as a case event before it
|
||||
is attempted, so an audit trail exists even when the call fails.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from typing import Any, Optional
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .config import get_settings
|
||||
from .models import Case, CaseEvent, CaseStatus, User
|
||||
from .services import add_event, set_status
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
class DeliveryError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _guard(kind: str) -> None:
|
||||
if not settings.feature_send_enabled:
|
||||
raise DeliveryError(
|
||||
"Sending is disabled on this instance (CX_FEATURE_SEND_ENABLED is off). "
|
||||
"The payload is ready but nothing will leave the server."
|
||||
)
|
||||
if kind == "zendesk" and not settings.zendesk_ready:
|
||||
raise DeliveryError("Zendesk is not configured. Set CX_FEATURE_ZENDESK plus "
|
||||
"CX_ZENDESK_SUBDOMAIN, CX_ZENDESK_EMAIL and CX_ZENDESK_TOKEN.")
|
||||
if kind == "jira" and not settings.jira_ready:
|
||||
raise DeliveryError("Jira is not configured. Set CX_FEATURE_JIRA plus "
|
||||
"CX_JIRA_BASE, CX_JIRA_EMAIL, CX_JIRA_TOKEN and CX_JIRA_PROJECT.")
|
||||
|
||||
|
||||
def sends_today(db: Session) -> int:
|
||||
since = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
|
||||
return (db.query(CaseEvent)
|
||||
.filter(CaseEvent.action.in_(["zendesk_sent", "jira_created"]))
|
||||
.filter(CaseEvent.created_at >= since).count())
|
||||
|
||||
|
||||
def _check_cap(db: Session) -> None:
|
||||
used = sends_today(db)
|
||||
if used >= settings.send_daily_cap:
|
||||
raise DeliveryError(
|
||||
f"Daily send cap reached ({used}/{settings.send_daily_cap}). Raise CX_SEND_DAILY_CAP "
|
||||
"if this is deliberate - the cap exists so a loop cannot mail every customer."
|
||||
)
|
||||
|
||||
|
||||
# --- Zendesk ----------------------------------------------------------------
|
||||
|
||||
async def send_zendesk(db: Session, case: Case, actor: User, *, to: str, subject: str,
|
||||
body: str, priority: str = "normal",
|
||||
tags: Optional[list[str]] = None, public: Optional[bool] = None) -> dict[str, Any]:
|
||||
_guard("zendesk")
|
||||
_check_cap(db)
|
||||
if not to.strip():
|
||||
raise DeliveryError("No recipient address.")
|
||||
|
||||
base = f"https://{settings.zendesk_subdomain}.zendesk.com/api/v2"
|
||||
auth = (f"{settings.zendesk_email}/token", settings.zendesk_token)
|
||||
external_id = f"cx-triage-{case.fingerprint}"
|
||||
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
# Search first so a re-diagnosed alert comments on the existing ticket
|
||||
# instead of opening a second one for the same customer.
|
||||
found = await client.get(f"{base}/search.json",
|
||||
params={"query": f'type:ticket external_id:"{external_id}"'}, auth=auth)
|
||||
existing = None
|
||||
if found.status_code == 200:
|
||||
results = found.json().get("results") or []
|
||||
existing = results[0] if results else None
|
||||
|
||||
comment = {"body": body, "public": settings.zendesk_default_public if public is None else public}
|
||||
if existing:
|
||||
resp = await client.put(f"{base}/tickets/{existing['id']}.json",
|
||||
json={"ticket": {"comment": comment}}, auth=auth)
|
||||
action = "updated"
|
||||
else:
|
||||
payload = {"ticket": {
|
||||
"subject": subject,
|
||||
"comment": comment,
|
||||
"requester": {"name": to.split("@")[0], "email": to},
|
||||
"priority": priority,
|
||||
"type": "incident",
|
||||
"tags": tags or ["cx-triage", f"alert-{case.kind}"],
|
||||
"external_id": external_id,
|
||||
}}
|
||||
resp = await client.post(f"{base}/tickets.json", json=payload, auth=auth)
|
||||
action = "created"
|
||||
|
||||
if resp.status_code not in (200, 201):
|
||||
add_event(db, case, actor, "zendesk_failed", f"HTTP {resp.status_code}: {resp.text[:300]}")
|
||||
db.commit()
|
||||
raise DeliveryError(f"Zendesk returned {resp.status_code}: {resp.text[:300]}")
|
||||
|
||||
ticket = resp.json().get("ticket") or {}
|
||||
ticket_id = str(ticket.get("id") or (existing or {}).get("id") or "")
|
||||
url = f"https://{settings.zendesk_subdomain}.zendesk.com/agent/tickets/{ticket_id}"
|
||||
|
||||
case.zendesk_ticket_id = ticket_id
|
||||
case.zendesk_ticket_url = url
|
||||
add_event(db, case, actor, "zendesk_sent",
|
||||
f"Ticket {ticket_id} {action} for {to}",
|
||||
{"ticket_id": ticket_id, "to": to, "subject": subject, "action": action})
|
||||
if case.status in (CaseStatus.NEW, CaseStatus.INVESTIGATING):
|
||||
set_status(db, case, CaseStatus.CUSTOMER_CONTACTED, actor, f"Zendesk ticket {ticket_id}")
|
||||
db.commit()
|
||||
return {"ok": True, "ticket_id": ticket_id, "url": url, "action": action}
|
||||
|
||||
|
||||
# --- Jira -------------------------------------------------------------------
|
||||
|
||||
async def create_jira(db: Session, case: Case, actor: User, *, summary: str, description: str,
|
||||
project: str = "", issue_type: str = "",
|
||||
labels: Optional[list[str]] = None) -> dict[str, Any]:
|
||||
_guard("jira")
|
||||
_check_cap(db)
|
||||
|
||||
base = settings.jira_base.rstrip("/")
|
||||
auth = (settings.jira_email, settings.jira_token)
|
||||
label = f"cx-triage-{case.fingerprint}"
|
||||
all_labels = sorted(set((labels or []) + ["cx-triage", label]))
|
||||
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
found = await client.get(f"{base}/rest/api/3/search",
|
||||
params={"jql": f'labels = "{label}"', "maxResults": 1}, auth=auth)
|
||||
if found.status_code == 200 and (found.json().get("issues") or []):
|
||||
issue = found.json()["issues"][0]
|
||||
key = issue["key"]
|
||||
url = f"{base}/browse/{key}"
|
||||
case.jira_issue_key, case.jira_issue_url = key, url
|
||||
add_event(db, case, actor, "jira_exists", f"Issue {key} already exists for this case")
|
||||
db.commit()
|
||||
return {"ok": True, "key": key, "url": url, "action": "existing"}
|
||||
|
||||
payload = {"fields": {
|
||||
"project": {"key": project or settings.jira_project},
|
||||
"summary": summary[:250],
|
||||
"issuetype": {"name": issue_type or settings.jira_issue_type},
|
||||
"labels": all_labels,
|
||||
"description": {
|
||||
"type": "doc", "version": 1,
|
||||
"content": [{"type": "paragraph",
|
||||
"content": [{"type": "text", "text": description[:30000]}]}],
|
||||
},
|
||||
}}
|
||||
resp = await client.post(f"{base}/rest/api/3/issue", json=payload, auth=auth)
|
||||
|
||||
if resp.status_code not in (200, 201):
|
||||
add_event(db, case, actor, "jira_failed", f"HTTP {resp.status_code}: {resp.text[:300]}")
|
||||
db.commit()
|
||||
raise DeliveryError(f"Jira returned {resp.status_code}: {resp.text[:300]}")
|
||||
|
||||
key = resp.json().get("key", "")
|
||||
url = f"{base}/browse/{key}"
|
||||
case.jira_issue_key, case.jira_issue_url = key, url
|
||||
add_event(db, case, actor, "jira_created", f"Issue {key} created", {"key": key, "summary": summary})
|
||||
if case.status in (CaseStatus.NEW, CaseStatus.INVESTIGATING):
|
||||
set_status(db, case, CaseStatus.ESCALATED_INFRA, actor, f"Jira {key}")
|
||||
db.commit()
|
||||
return {"ok": True, "key": key, "url": url, "action": "created"}
|
||||
66
backend/app/main.py
Normal file
66
backend/app/main.py
Normal file
@@ -0,0 +1,66 @@
|
||||
"""FastAPI application entry point."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from . import VERSION
|
||||
from .auth import ensure_bootstrap_admin
|
||||
from .config import get_settings
|
||||
from .db import SessionLocal, init_db
|
||||
from .routers import (actions_router, alerts_router, auth_router, cases_router,
|
||||
linkage_router, settings_router)
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
init_db()
|
||||
db = SessionLocal()
|
||||
try:
|
||||
message = ensure_bootstrap_admin(db)
|
||||
if message:
|
||||
print(f"[auth] {message}")
|
||||
finally:
|
||||
db.close()
|
||||
print(f"[startup] {settings.app_name} {VERSION}")
|
||||
print(f"[startup] prometheus: {settings.prometheus_base}")
|
||||
print(f"[startup] sso: {'on' if settings.oidc_enabled else 'off'} | "
|
||||
f"zendesk: {'ready' if settings.zendesk_ready else 'off'} | "
|
||||
f"jira: {'ready' if settings.jira_ready else 'off'} | "
|
||||
f"sending: {'ENABLED' if settings.feature_send_enabled else 'disabled'}")
|
||||
alerts_router.engine.warm()
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(title=settings.app_name, version=VERSION, lifespan=lifespan)
|
||||
|
||||
for module in (auth_router, alerts_router, cases_router, settings_router,
|
||||
linkage_router, actions_router):
|
||||
app.include_router(module.router)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
def health():
|
||||
return {"status": "ok", "version": VERSION, "config": settings.public_flags()}
|
||||
|
||||
|
||||
# The built React bundle is copied into the image; in development Vite serves it
|
||||
# instead and proxies /api here, so a missing directory is not an error.
|
||||
if os.path.isdir(settings.static_dir):
|
||||
app.mount("/assets", StaticFiles(directory=os.path.join(settings.static_dir, "assets")),
|
||||
name="assets")
|
||||
|
||||
@app.get("/{full_path:path}")
|
||||
def spa(full_path: str):
|
||||
if full_path.startswith("api/"):
|
||||
return JSONResponse({"detail": "Not found"}, status_code=404)
|
||||
index = os.path.join(settings.static_dir, "index.html")
|
||||
if os.path.isfile(index):
|
||||
return FileResponse(index)
|
||||
return JSONResponse({"detail": "Frontend not built"}, status_code=404)
|
||||
208
backend/app/models.py
Normal file
208
backend/app/models.py
Normal file
@@ -0,0 +1,208 @@
|
||||
"""Persistent state: who is working what, and what has been done about it.
|
||||
|
||||
The alert queue itself stays stateless - it is recomputed from Prometheus every
|
||||
minute. What is worth persisting is the human layer on top: which alerts someone
|
||||
has picked up, what was done, and the audit trail behind it. Cases are keyed by
|
||||
the alert fingerprint so an alert that stops and re-fires lands back on the same
|
||||
case rather than losing its history.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import enum
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (JSON, Boolean, DateTime, Enum, ForeignKey, Index, Integer,
|
||||
String, Text, UniqueConstraint)
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
def _now() -> dt.datetime:
|
||||
return dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
|
||||
class CaseStatus(str, enum.Enum):
|
||||
"""Where a piece of work has got to."""
|
||||
|
||||
NEW = "new" # seen, nobody has touched it
|
||||
INVESTIGATING = "investigating" # someone has picked it up
|
||||
CUSTOMER_CONTACTED = "customer_contacted"
|
||||
ESCALATED_INFRA = "escalated_infra"
|
||||
WAITING_CUSTOMER = "waiting_customer"
|
||||
WAITING_INFRA = "waiting_infra"
|
||||
REMEDIATED = "remediated" # fixed, waiting for the alert to clear
|
||||
RESOLVED = "resolved"
|
||||
WONT_FIX = "wont_fix" # deliberate no-action
|
||||
FALSE_POSITIVE = "false_positive" # the alert itself was wrong
|
||||
|
||||
|
||||
OPEN_STATUSES = {
|
||||
CaseStatus.NEW, CaseStatus.INVESTIGATING, CaseStatus.CUSTOMER_CONTACTED,
|
||||
CaseStatus.ESCALATED_INFRA, CaseStatus.WAITING_CUSTOMER, CaseStatus.WAITING_INFRA,
|
||||
CaseStatus.REMEDIATED,
|
||||
}
|
||||
|
||||
|
||||
class AuthProvider(str, enum.Enum):
|
||||
LOCAL = "local"
|
||||
OIDC = "oidc"
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String(200), default="")
|
||||
is_admin: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
provider: Mapped[AuthProvider] = mapped_column(Enum(AuthProvider), default=AuthProvider.LOCAL)
|
||||
# Local accounts only; OIDC users never have one.
|
||||
password_hash: Mapped[str] = mapped_column(String(255), default="")
|
||||
# Stable Authentik subject, so a rename or email change keeps the same user.
|
||||
oidc_sub: Mapped[str] = mapped_column(String(255), default="", index=True)
|
||||
signoff_name: Mapped[str] = mapped_column(String(200), default="")
|
||||
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
last_login: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
"id": self.id, "email": self.email, "name": self.name or self.email,
|
||||
"is_admin": self.is_admin, "provider": self.provider.value,
|
||||
"signoff_name": self.signoff_name or self.name,
|
||||
}
|
||||
|
||||
|
||||
class Case(Base):
|
||||
"""One tracked alert, keyed by its fingerprint."""
|
||||
|
||||
__tablename__ = "cases"
|
||||
__table_args__ = (Index("ix_cases_status_seen", "status", "last_seen_at"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
||||
|
||||
kind: Mapped[str] = mapped_column(String(40), index=True)
|
||||
title: Mapped[str] = mapped_column(String(300), default="")
|
||||
subject: Mapped[str] = mapped_column(String(300), default="") # VM name, host or IP
|
||||
openstack_id: Mapped[str] = mapped_column(String(64), default="", index=True)
|
||||
instance_name: Mapped[str] = mapped_column(String(200), default="", index=True)
|
||||
host: Mapped[str] = mapped_column(String(120), default="", index=True)
|
||||
region: Mapped[str] = mapped_column(String(16), default="")
|
||||
org_id: Mapped[str] = mapped_column(String(32), default="", index=True)
|
||||
org_name: Mapped[str] = mapped_column(String(300), default="")
|
||||
|
||||
status: Mapped[CaseStatus] = mapped_column(Enum(CaseStatus), default=CaseStatus.NEW, index=True)
|
||||
assignee_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True)
|
||||
assignee: Mapped[User | None] = relationship(lazy="joined")
|
||||
|
||||
# Outbound references, so a case shows what already exists elsewhere.
|
||||
zendesk_ticket_id: Mapped[str] = mapped_column(String(40), default="")
|
||||
zendesk_ticket_url: Mapped[str] = mapped_column(String(500), default="")
|
||||
jira_issue_key: Mapped[str] = mapped_column(String(40), default="")
|
||||
jira_issue_url: Mapped[str] = mapped_column(String(500), default="")
|
||||
|
||||
notes: Mapped[str] = mapped_column(Text, default="")
|
||||
snooze_until: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
first_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
last_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
closed_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
# How many separate times this alert has come back after being closed.
|
||||
reopen_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
events: Mapped[list["CaseEvent"]] = relationship(
|
||||
back_populates="case", cascade="all, delete-orphan", order_by="CaseEvent.created_at.desc()")
|
||||
|
||||
@property
|
||||
def is_open(self) -> bool:
|
||||
return self.status in OPEN_STATUSES
|
||||
|
||||
def to_json(self, with_events: bool = False) -> dict[str, Any]:
|
||||
data = {
|
||||
"id": self.id, "fingerprint": self.fingerprint, "kind": self.kind,
|
||||
"title": self.title, "subject": self.subject,
|
||||
"openstack_id": self.openstack_id, "instance_name": self.instance_name,
|
||||
"host": self.host, "region": self.region,
|
||||
"org_id": self.org_id, "org_name": self.org_name,
|
||||
"status": self.status.value, "is_open": self.is_open,
|
||||
"assignee": self.assignee.to_json() if self.assignee else None,
|
||||
"zendesk_ticket_id": self.zendesk_ticket_id,
|
||||
"zendesk_ticket_url": self.zendesk_ticket_url,
|
||||
"jira_issue_key": self.jira_issue_key,
|
||||
"jira_issue_url": self.jira_issue_url,
|
||||
"notes": self.notes,
|
||||
"snooze_until": self.snooze_until.isoformat() if self.snooze_until else None,
|
||||
"first_seen_at": self.first_seen_at.isoformat() if self.first_seen_at else None,
|
||||
"last_seen_at": self.last_seen_at.isoformat() if self.last_seen_at else None,
|
||||
"closed_at": self.closed_at.isoformat() if self.closed_at else None,
|
||||
"reopen_count": self.reopen_count,
|
||||
}
|
||||
if with_events:
|
||||
data["events"] = [e.to_json() for e in self.events]
|
||||
return data
|
||||
|
||||
|
||||
class CaseEvent(Base):
|
||||
"""Append-only history. Nothing here is ever edited or deleted."""
|
||||
|
||||
__tablename__ = "case_events"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
case_id: Mapped[int] = mapped_column(ForeignKey("cases.id"), index=True)
|
||||
case: Mapped[Case] = relationship(back_populates="events")
|
||||
|
||||
actor_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True)
|
||||
actor: Mapped[User | None] = relationship(lazy="joined")
|
||||
actor_label: Mapped[str] = mapped_column(String(200), default="") # survives user deletion
|
||||
|
||||
action: Mapped[str] = mapped_column(String(60)) # status_changed | zendesk_sent | note | ...
|
||||
detail: Mapped[str] = mapped_column(Text, default="")
|
||||
payload: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
"id": self.id, "action": self.action, "detail": self.detail,
|
||||
"actor": self.actor_label or (self.actor.email if self.actor else "system"),
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"payload": self.payload,
|
||||
}
|
||||
|
||||
|
||||
class SuppressionRule(Base):
|
||||
"""Alerts the team has decided not to see, with the reason recorded."""
|
||||
|
||||
__tablename__ = "suppression_rules"
|
||||
__table_args__ = (UniqueConstraint("name", name="uq_rule_name"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(200))
|
||||
reason: Mapped[str] = mapped_column(Text, default="")
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
# {"kind": ["error"], "organization": ["modal"]} - all keys must match.
|
||||
conditions: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
created_by: Mapped[str] = mapped_column(String(200), default="")
|
||||
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
"id": str(self.id), "name": self.name, "reason": self.reason,
|
||||
"enabled": self.enabled, "conditions": self.conditions or {},
|
||||
"created_by": self.created_by,
|
||||
"created": self.created_at.strftime("%Y-%m-%d") if self.created_at else "",
|
||||
}
|
||||
|
||||
|
||||
class AppSetting(Base):
|
||||
"""Small key/value bag for things an admin can change at runtime."""
|
||||
|
||||
__tablename__ = "app_settings"
|
||||
|
||||
key: Mapped[str] = mapped_column(String(80), primary_key=True)
|
||||
value: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||
updated_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
|
||||
0
backend/app/routers/__init__.py
Normal file
0
backend/app/routers/__init__.py
Normal file
77
backend/app/routers/actions_router.py
Normal file
77
backend/app/routers/actions_router.py
Normal file
@@ -0,0 +1,77 @@
|
||||
"""Outbound actions: contact the customer, escalate to Infrastructure."""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..auth import current_user
|
||||
from ..config import get_settings
|
||||
from ..db import get_db
|
||||
from ..delivery import DeliveryError, create_jira, send_zendesk, sends_today
|
||||
from ..models import Case, User
|
||||
|
||||
router = APIRouter(prefix="/api/actions", tags=["actions"])
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
class ZendeskBody(BaseModel):
|
||||
fingerprint: str
|
||||
to: str
|
||||
subject: str
|
||||
body: str
|
||||
priority: str = "normal"
|
||||
tags: list[str] = []
|
||||
public: bool | None = None
|
||||
|
||||
|
||||
class JiraBody(BaseModel):
|
||||
fingerprint: str
|
||||
summary: str
|
||||
description: str
|
||||
project: str = ""
|
||||
issue_type: str = ""
|
||||
labels: list[str] = []
|
||||
|
||||
|
||||
def _case(db: Session, fingerprint: str) -> Case:
|
||||
found = db.query(Case).filter(Case.fingerprint == fingerprint).one_or_none()
|
||||
if not found:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND,
|
||||
"Open the case first - nothing is tracked for that alert yet.")
|
||||
return found
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
def action_status(db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
return {
|
||||
"zendesk_ready": settings.zendesk_ready,
|
||||
"jira_ready": settings.jira_ready,
|
||||
"send_enabled": settings.feature_send_enabled,
|
||||
"sends_today": sends_today(db),
|
||||
"daily_cap": settings.send_daily_cap,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/zendesk")
|
||||
async def zendesk(body: ZendeskBody, db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user)):
|
||||
case = _case(db, body.fingerprint)
|
||||
try:
|
||||
return await send_zendesk(db, case, user, to=body.to, subject=body.subject,
|
||||
body=body.body, priority=body.priority,
|
||||
tags=body.tags or None, public=body.public)
|
||||
except DeliveryError as exc:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/jira")
|
||||
async def jira(body: JiraBody, db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user)):
|
||||
case = _case(db, body.fingerprint)
|
||||
try:
|
||||
return await create_jira(db, case, user, summary=body.summary,
|
||||
description=body.description, project=body.project,
|
||||
issue_type=body.issue_type, labels=body.labels or None)
|
||||
except DeliveryError as exc:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc
|
||||
88
backend/app/routers/alerts_router.py
Normal file
88
backend/app/routers/alerts_router.py
Normal file
@@ -0,0 +1,88 @@
|
||||
"""The alert queue and per-alert diagnosis."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from triagelib import runbooks
|
||||
|
||||
from ..auth import current_user
|
||||
from ..db import SessionLocal, get_db
|
||||
from ..models import User
|
||||
from ..services import Engine, RuleAdapter, get_or_create_case
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["alerts"])
|
||||
engine = Engine()
|
||||
|
||||
_jobs: dict[str, dict[str, Any]] = {}
|
||||
_jobs_lock = threading.Lock()
|
||||
_pool = ThreadPoolExecutor(max_workers=3, thread_name_prefix="triage")
|
||||
JOB_TTL = 30 * 60
|
||||
|
||||
|
||||
class TriageBody(BaseModel):
|
||||
fingerprint: str
|
||||
force: bool = False
|
||||
|
||||
|
||||
@router.get("/alerts")
|
||||
def alert_queue(force: bool = False, db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user)):
|
||||
return engine.queue(db, force=force)
|
||||
|
||||
|
||||
@router.post("/triage")
|
||||
def start_triage(body: TriageBody, db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user)):
|
||||
alert = engine.find_alert(db, body.fingerprint)
|
||||
if alert is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND,
|
||||
"That alert is no longer firing. Refresh the queue.")
|
||||
case = get_or_create_case(db, alert, user)
|
||||
job_id = uuid.uuid4().hex[:12]
|
||||
with _jobs_lock:
|
||||
_reap()
|
||||
_jobs[job_id] = {"id": job_id, "state": "running", "created": time.time(),
|
||||
"result": None, "error": ""}
|
||||
_pool.submit(_run, job_id, alert, body.force)
|
||||
return {"job_id": job_id, "alert": alert.to_json(), "case": case.to_json(with_events=True)}
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}")
|
||||
def job(job_id: str, user: User = Depends(current_user)):
|
||||
with _jobs_lock:
|
||||
found = _jobs.get(job_id)
|
||||
if not found:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "Unknown job")
|
||||
return found
|
||||
|
||||
|
||||
def _run(job_id: str, alert: Any, force: bool) -> None:
|
||||
started = time.monotonic()
|
||||
db = SessionLocal()
|
||||
try:
|
||||
diagnosis = runbooks.diagnose(alert, engine.prom, engine.snapshot.get(), force,
|
||||
RuleAdapter(db))
|
||||
payload = diagnosis.to_json()
|
||||
payload["elapsed_seconds"] = round(time.monotonic() - started, 1)
|
||||
with _jobs_lock:
|
||||
_jobs[job_id].update({"state": "done", "result": payload})
|
||||
except Exception:
|
||||
with _jobs_lock:
|
||||
_jobs[job_id].update({"state": "error", "error": traceback.format_exc(limit=4)})
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _reap() -> None:
|
||||
cutoff = time.time() - JOB_TTL
|
||||
for key in [k for k, v in _jobs.items() if v["created"] < cutoff]:
|
||||
_jobs.pop(key, None)
|
||||
99
backend/app/routers/auth_router.py
Normal file
99
backend/app/routers/auth_router.py
Normal file
@@ -0,0 +1,99 @@
|
||||
"""Sign-in: local accounts and the Authentik OIDC round trip."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
from fastapi.responses import RedirectResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .. import auth as auth_mod
|
||||
from ..config import get_settings
|
||||
from ..db import get_db
|
||||
from ..models import User
|
||||
|
||||
router = APIRouter(prefix="/api/auth", tags=["auth"])
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
class LoginBody(BaseModel):
|
||||
email: str
|
||||
password: str
|
||||
|
||||
|
||||
def _set_cookie(response: Response, user: User) -> None:
|
||||
response.set_cookie(
|
||||
auth_mod.SESSION_COOKIE, auth_mod.issue_session(user.id),
|
||||
max_age=settings.session_hours * 3600, httponly=True, samesite="lax",
|
||||
secure=settings.base_url.startswith("https://"), path="/",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
def me(user: User | None = Depends(auth_mod.optional_user)):
|
||||
return {"user": user.to_json() if user else None, "config": settings.public_flags()}
|
||||
|
||||
|
||||
@router.post("/login")
|
||||
def login(body: LoginBody, response: Response, db: Session = Depends(get_db)):
|
||||
if not settings.auth_local_enabled:
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, "Local login is disabled; use SSO.")
|
||||
user = auth_mod.authenticate_local(db, body.email, body.password)
|
||||
if not user:
|
||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Incorrect email or password")
|
||||
user.last_login = dt.datetime.now(dt.timezone.utc)
|
||||
db.commit()
|
||||
_set_cookie(response, user)
|
||||
return {"user": user.to_json()}
|
||||
|
||||
|
||||
@router.post("/logout")
|
||||
def logout(response: Response):
|
||||
response.delete_cookie(auth_mod.SESSION_COOKIE, path="/")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.get("/oidc/start")
|
||||
async def oidc_start(request: Request):
|
||||
if not settings.oidc_enabled:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "SSO is not enabled")
|
||||
try:
|
||||
meta = await auth_mod.oidc_discovery()
|
||||
except auth_mod.OIDCError as exc:
|
||||
raise HTTPException(status.HTTP_502_BAD_GATEWAY, str(exc)) from exc
|
||||
|
||||
redirect_uri = f"{settings.base_url.rstrip('/')}/api/auth/oidc/callback"
|
||||
state = auth_mod.oidc_state()
|
||||
url = (f"{meta['authorization_endpoint']}?response_type=code"
|
||||
f"&client_id={settings.oidc_client_id}"
|
||||
f"&redirect_uri={redirect_uri}"
|
||||
f"&scope={settings.oidc_scopes.replace(' ', '%20')}"
|
||||
f"&state={state}")
|
||||
response = RedirectResponse(url, status_code=302)
|
||||
response.set_cookie("cx_oidc_state", state, max_age=600, httponly=True, samesite="lax", path="/")
|
||||
return response
|
||||
|
||||
|
||||
@router.get("/oidc/callback")
|
||||
async def oidc_callback(request: Request, code: str = "", state: str = "",
|
||||
db: Session = Depends(get_db)):
|
||||
if not settings.oidc_enabled:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "SSO is not enabled")
|
||||
# The state must match the cookie we set *and* still verify - one guards
|
||||
# against a swapped browser, the other against a forged value.
|
||||
if not code or not state or state != request.cookies.get("cx_oidc_state") \
|
||||
or not auth_mod.oidc_state_valid(state):
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid or expired SSO state")
|
||||
|
||||
redirect_uri = f"{settings.base_url.rstrip('/')}/api/auth/oidc/callback"
|
||||
try:
|
||||
claims = await auth_mod.oidc_exchange(code, redirect_uri)
|
||||
user = auth_mod.upsert_oidc_user(db, claims)
|
||||
except auth_mod.OIDCError as exc:
|
||||
raise HTTPException(status.HTTP_502_BAD_GATEWAY, str(exc)) from exc
|
||||
|
||||
response = RedirectResponse("/", status_code=302)
|
||||
_set_cookie(response, user)
|
||||
response.delete_cookie("cx_oidc_state", path="/")
|
||||
return response
|
||||
99
backend/app/routers/cases_router.py
Normal file
99
backend/app/routers/cases_router.py
Normal file
@@ -0,0 +1,99 @@
|
||||
"""Case state: the human layer over the alert queue."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..auth import current_user
|
||||
from ..db import get_db
|
||||
from ..models import Case, CaseStatus, User
|
||||
from ..services import add_event, set_status
|
||||
|
||||
router = APIRouter(prefix="/api/cases", tags=["cases"])
|
||||
|
||||
|
||||
class StatusBody(BaseModel):
|
||||
status: str
|
||||
note: str = ""
|
||||
|
||||
|
||||
class NoteBody(BaseModel):
|
||||
note: str
|
||||
|
||||
|
||||
class SnoozeBody(BaseModel):
|
||||
hours: int = 24
|
||||
note: str = ""
|
||||
|
||||
|
||||
def _case(db: Session, fingerprint: str) -> Case:
|
||||
found = db.query(Case).filter(Case.fingerprint == fingerprint).one_or_none()
|
||||
if not found:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "No case for that alert yet")
|
||||
return found
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_cases(open_only: bool = True, limit: int = 200, db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user)):
|
||||
query = db.query(Case).order_by(Case.last_seen_at.desc())
|
||||
rows = [c for c in query.limit(max(1, min(limit, 1000))).all()
|
||||
if (c.is_open or not open_only)]
|
||||
return {"cases": [c.to_json() for c in rows], "statuses": [s.value for s in CaseStatus]}
|
||||
|
||||
|
||||
@router.get("/{fingerprint}")
|
||||
def get_case(fingerprint: str, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
return _case(db, fingerprint).to_json(with_events=True)
|
||||
|
||||
|
||||
@router.post("/{fingerprint}/status")
|
||||
def change_status(fingerprint: str, body: StatusBody, db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user)):
|
||||
try:
|
||||
new_status = CaseStatus(body.status)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST,
|
||||
f"Unknown status '{body.status}'") from exc
|
||||
case = set_status(db, _case(db, fingerprint), new_status, user, body.note)
|
||||
return case.to_json(with_events=True)
|
||||
|
||||
|
||||
@router.post("/{fingerprint}/assign")
|
||||
def assign(fingerprint: str, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
case = _case(db, fingerprint)
|
||||
# Set the relationship, not just the id: the response is serialised from
|
||||
# this same object and a bare id leaves `assignee` null in the payload.
|
||||
case.assignee = user
|
||||
if case.status == CaseStatus.NEW:
|
||||
case.status = CaseStatus.INVESTIGATING
|
||||
add_event(db, case, user, "assigned", f"Picked up by {user.email}")
|
||||
db.commit()
|
||||
db.refresh(case)
|
||||
return case.to_json(with_events=True)
|
||||
|
||||
|
||||
@router.post("/{fingerprint}/note")
|
||||
def add_note(fingerprint: str, body: NoteBody, db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user)):
|
||||
if not body.note.strip():
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, "Empty note")
|
||||
case = _case(db, fingerprint)
|
||||
case.notes = (case.notes + "\n" if case.notes else "") + body.note.strip()
|
||||
add_event(db, case, user, "note", body.note.strip())
|
||||
db.commit()
|
||||
return case.to_json(with_events=True)
|
||||
|
||||
|
||||
@router.post("/{fingerprint}/snooze")
|
||||
def snooze(fingerprint: str, body: SnoozeBody, db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user)):
|
||||
case = _case(db, fingerprint)
|
||||
until = dt.datetime.now(dt.timezone.utc) + dt.timedelta(hours=max(1, body.hours))
|
||||
case.snooze_until = until
|
||||
add_event(db, case, user, "snoozed", f"Snoozed for {body.hours}h. {body.note}".strip())
|
||||
db.commit()
|
||||
return case.to_json(with_events=True)
|
||||
42
backend/app/routers/linkage_router.py
Normal file
42
backend/app/routers/linkage_router.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""Linkage scan endpoints."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
|
||||
from triagelib import linkage as linkage_mod
|
||||
|
||||
from ..auth import current_user
|
||||
from ..config import get_settings
|
||||
from ..models import User
|
||||
from ..routers.alerts_router import engine
|
||||
|
||||
router = APIRouter(prefix="/api/linkage", tags=["linkage"])
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
class EnrichBody(BaseModel):
|
||||
region: str
|
||||
openstack_id: str
|
||||
|
||||
|
||||
@router.get("")
|
||||
def scan_state(user: User = Depends(current_user)):
|
||||
return engine.scan.to_json()
|
||||
|
||||
|
||||
@router.post("/scan")
|
||||
def start_scan(user: User = Depends(current_user)):
|
||||
if not settings.feature_linkage_scan:
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, "The linkage scan is disabled on this instance.")
|
||||
if engine.scan.state == "running":
|
||||
return {"started": False, "reason": "already running"}
|
||||
threading.Thread(target=engine.scan.run, args=(engine.snapshot.get(),), daemon=True).start()
|
||||
return {"started": True}
|
||||
|
||||
|
||||
@router.post("/enrich")
|
||||
def enrich(body: EnrichBody, user: User = Depends(current_user)):
|
||||
return linkage_mod.enrich(body.region, body.openstack_id)
|
||||
103
backend/app/routers/settings_router.py
Normal file
103
backend/app/routers/settings_router.py
Normal file
@@ -0,0 +1,103 @@
|
||||
"""Suppression rules and per-instance preferences."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from triagelib import settings as legacy
|
||||
|
||||
from ..auth import current_user, require_admin
|
||||
from ..config import get_settings
|
||||
from ..db import get_db
|
||||
from ..models import AppSetting, SuppressionRule, User
|
||||
from ..routers.alerts_router import engine
|
||||
|
||||
router = APIRouter(prefix="/api/settings", tags=["settings"])
|
||||
app_settings = get_settings()
|
||||
|
||||
|
||||
class RuleBody(BaseModel):
|
||||
id: str | None = None
|
||||
name: str
|
||||
reason: str = ""
|
||||
enabled: bool = True
|
||||
conditions: dict[str, Any] = {}
|
||||
|
||||
|
||||
class GeneralBody(BaseModel):
|
||||
agent_name: str | None = None
|
||||
chronic_days: int | None = None
|
||||
|
||||
|
||||
@router.get("")
|
||||
def read_settings(db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
row = db.get(AppSetting, "general")
|
||||
general = (row.value if row else {}) or {}
|
||||
return {
|
||||
"rules": [r.to_json() for r in db.query(SuppressionRule).order_by(SuppressionRule.id).all()],
|
||||
"conditions": legacy.CONDITIONS,
|
||||
"agent_name": general.get("agent_name") or "",
|
||||
"chronic_days": general.get("chronic_days") or 3,
|
||||
"config": app_settings.public_flags(),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/general")
|
||||
def save_general(body: GeneralBody, db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user)):
|
||||
row = db.get(AppSetting, "general") or AppSetting(key="general", value={})
|
||||
value = dict(row.value or {})
|
||||
if body.agent_name is not None:
|
||||
value["agent_name"] = body.agent_name.strip()
|
||||
if body.chronic_days is not None:
|
||||
value["chronic_days"] = max(1, int(body.chronic_days))
|
||||
row.value = value
|
||||
db.merge(row)
|
||||
db.commit()
|
||||
return read_settings(db, user)
|
||||
|
||||
|
||||
@router.post("/rules")
|
||||
def save_rule(body: RuleBody, db: Session = Depends(get_db), user: User = Depends(require_admin)):
|
||||
conditions = {k: v for k, v in (body.conditions or {}).items() if k in legacy.CONDITIONS and v}
|
||||
if not conditions:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST,
|
||||
"A rule needs at least one condition, otherwise it would hide everything.")
|
||||
rule = db.get(SuppressionRule, int(body.id)) if (body.id or "").isdigit() else None
|
||||
if rule is None:
|
||||
rule = SuppressionRule(created_by=user.email)
|
||||
db.add(rule)
|
||||
rule.name = body.name.strip() or "Untitled rule"
|
||||
rule.reason = body.reason.strip()
|
||||
rule.enabled = body.enabled
|
||||
rule.conditions = conditions
|
||||
db.commit()
|
||||
return read_settings(db, user)
|
||||
|
||||
|
||||
@router.delete("/rules/{rule_id}")
|
||||
def delete_rule(rule_id: int, db: Session = Depends(get_db), user: User = Depends(require_admin)):
|
||||
rule = db.get(SuppressionRule, rule_id)
|
||||
if rule:
|
||||
db.delete(rule)
|
||||
db.commit()
|
||||
return read_settings(db, user)
|
||||
|
||||
|
||||
@router.post("/rules/preview")
|
||||
def preview_rule(body: RuleBody, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
"""Show which firing alerts a rule would hide, before it is saved."""
|
||||
from triagelib import alerts as alertlib
|
||||
|
||||
raw, _error, _age = engine.cache.get()
|
||||
candidates = [a for a in (alertlib.from_prometheus(x, engine.rules) for x in raw)
|
||||
if not alertlib.is_excluded(a) and alertlib.cx_relevant(a)]
|
||||
rule = legacy._normalize_rule({"name": body.name, "conditions": body.conditions})
|
||||
hits = [{
|
||||
"kind": a.kind, "title": a.title, "instance_name": a.instance_name,
|
||||
"host": a.host, "org_name": a.org_name, "region": a.region,
|
||||
} for a in candidates if legacy.rule_matches(rule, a)]
|
||||
return {"count": len(hits), "matches": hits[:60]}
|
||||
200
backend/app/services.py
Normal file
200
backend/app/services.py
Normal file
@@ -0,0 +1,200 @@
|
||||
"""Glue between the triage engine, the database and the outside world."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import threading
|
||||
from typing import Any, Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from triagelib import alerts as alertlib, screening, settings as legacy_settings
|
||||
from triagelib.prometheus import (AlertCache, PrometheusClient, PrometheusError,
|
||||
RuleIndex, StateSnapshot, TrueAgeIndex)
|
||||
from triagelib import linkage as linkage_mod
|
||||
|
||||
from .config import get_settings
|
||||
from .models import AppSetting, Case, CaseEvent, CaseStatus, SuppressionRule, User
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
class RuleAdapter:
|
||||
"""Presents DB-backed suppression rules the way the engine expects."""
|
||||
|
||||
def __init__(self, db: Session):
|
||||
self._rules = [r.to_json() for r in db.query(SuppressionRule).all()]
|
||||
row = db.get(AppSetting, "general")
|
||||
general = (row.value if row else {}) or {}
|
||||
self._agent = str(general.get("agent_name") or "")
|
||||
self._chronic = int(general.get("chronic_days") or 3)
|
||||
|
||||
@property
|
||||
def rules(self) -> list[dict[str, Any]]:
|
||||
return self._rules
|
||||
|
||||
@property
|
||||
def agent_name(self) -> str:
|
||||
return self._agent
|
||||
|
||||
@property
|
||||
def chronic_days(self) -> int:
|
||||
return self._chronic
|
||||
|
||||
|
||||
class Engine:
|
||||
"""Process-wide caches over Prometheus. Cheap to share, expensive to rebuild."""
|
||||
|
||||
def __init__(self):
|
||||
self.prom = PrometheusClient(settings.prometheus_base)
|
||||
self.cache = AlertCache(self.prom)
|
||||
self.rules = RuleIndex(self.prom)
|
||||
self.snapshot = StateSnapshot(self.prom)
|
||||
self.true_age = TrueAgeIndex(self.prom)
|
||||
self.scan = linkage_mod.Scan()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def warm(self, log=print) -> None:
|
||||
try:
|
||||
self.rules.ensure()
|
||||
snap = self.snapshot.get()
|
||||
log(f" state snapshot: {len(snap.by_openstack_id)} VMs, {len(snap.total_gpus)} hosts")
|
||||
ages = self.true_age.get()
|
||||
log(f" alert history: {ages.count} alerts indexed over {ages.WINDOW_DAYS} days")
|
||||
except PrometheusError as exc:
|
||||
log(f" WARN: Prometheus caches not warmed: {exc}")
|
||||
|
||||
def queue(self, db: Session, force: bool = False) -> dict[str, Any]:
|
||||
raw, error, age = self.cache.get(force=force)
|
||||
snap = self.snapshot.get()
|
||||
ages = self.true_age.get()
|
||||
parsed = [alertlib.from_prometheus(a, self.rules, ages) for a in raw]
|
||||
|
||||
excluded = [a for a in parsed if alertlib.is_excluded(a)]
|
||||
candidates = [a for a in parsed if not alertlib.is_excluded(a)]
|
||||
cx = [a for a in candidates if a.category == "cx" and alertlib.cx_relevant(a)]
|
||||
screening.screen_all(cx, snap, RuleAdapter(db))
|
||||
|
||||
cases = {c.fingerprint: c for c in
|
||||
db.query(Case).filter(Case.fingerprint.in_([a.fingerprint() for a in cx])).all()}
|
||||
groups = alertlib.group_alerts(cx)
|
||||
for group in groups:
|
||||
for item in group["alerts"]:
|
||||
case = cases.get(item["id"])
|
||||
item["case"] = case.to_json() if case else None
|
||||
|
||||
in_cx = {id(a) for a in cx}
|
||||
infra = sorted([a for a in candidates if id(a) not in in_cx], key=alertlib.sort_key)
|
||||
|
||||
return {
|
||||
"error": error or snap.error or ages.error,
|
||||
"warnings": screening.health_warnings(snap),
|
||||
"totals": {"prometheus": len(parsed), "cx": len(cx),
|
||||
"infrastructure": len(infra), "excluded": len(excluded)},
|
||||
"excluded_note": (f"{len(excluded)} '{', '.join(sorted({a.alertname for a in excluded}))}' alerts hidden"
|
||||
if excluded else ""),
|
||||
"summary": screening.summarize(cx),
|
||||
"groups": groups,
|
||||
"infrastructure": _infra_sections(infra),
|
||||
"cache_age_seconds": round(age, 1),
|
||||
}
|
||||
|
||||
def find_alert(self, db: Session, fingerprint: str) -> Optional[Any]:
|
||||
raw, _error, _age = self.cache.get()
|
||||
ages = self.true_age.get()
|
||||
for item in raw:
|
||||
alert = alertlib.from_prometheus(item, self.rules, ages)
|
||||
if alert.fingerprint() == fingerprint:
|
||||
alert.screen = screening.screen(alert, self.snapshot.get(), RuleAdapter(db))
|
||||
return alert
|
||||
return None
|
||||
|
||||
|
||||
SOURCE_LABELS = {
|
||||
"node-exporter-rules.yml": "Node exporter (hosts)",
|
||||
"ceph-rules.yml": "Ceph", "mysql-rules.yml": "MySQL",
|
||||
"mysql-performance-rules.yml": "MySQL performance", "galera-rules.yml": "Galera",
|
||||
"openstack-rules.yml": "OpenStack services", "blackbox.yml": "Blackbox / OOB",
|
||||
"infrahub-rules.yml": "Infrahub (no CX runbook)",
|
||||
}
|
||||
|
||||
|
||||
def _infra_sections(items: list[Any]) -> list[dict[str, Any]]:
|
||||
buckets: dict[str, list[Any]] = {}
|
||||
for alert in items:
|
||||
buckets.setdefault(alert.rule_file or "unknown", []).append(alert)
|
||||
sections = []
|
||||
for source, members in buckets.items():
|
||||
by_name: dict[str, int] = {}
|
||||
for alert in members:
|
||||
name = alertlib.clean_alertname(alert.alertname)
|
||||
by_name[name] = by_name.get(name, 0) + 1
|
||||
sections.append({
|
||||
"source": source, "label": SOURCE_LABELS.get(source, source), "total": len(members),
|
||||
"by_alertname": sorted(({"name": k, "count": v} for k, v in by_name.items()),
|
||||
key=lambda x: (-x["count"], x["name"])),
|
||||
})
|
||||
sections.sort(key=lambda s: (s["source"] != alertlib.NODE_RULE_FILE, -s["total"]))
|
||||
return sections
|
||||
|
||||
|
||||
# --- case bookkeeping -------------------------------------------------------
|
||||
|
||||
def get_or_create_case(db: Session, alert: Any, actor: Optional[User] = None) -> Case:
|
||||
case = db.query(Case).filter(Case.fingerprint == alert.fingerprint()).one_or_none()
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
subject = (alert.floating_ip if alert.kind == "duplicate_ip"
|
||||
else alert.host if alert.kind in ("rogue_vm", "total_gpus", "orphan_vm")
|
||||
else alert.instance_name or alert.openstack_id)
|
||||
if case is None:
|
||||
case = Case(
|
||||
fingerprint=alert.fingerprint(), kind=alert.kind, title=alert.title,
|
||||
subject=subject or "", openstack_id=alert.openstack_id,
|
||||
instance_name=alert.instance_name, host=alert.host, region=alert.region,
|
||||
org_id=alert.org_id, org_name=alert.org_name,
|
||||
)
|
||||
db.add(case)
|
||||
db.flush()
|
||||
add_event(db, case, actor, "opened", f"Case opened for {alert.title}")
|
||||
else:
|
||||
# A closed case whose alert has come back is new work again.
|
||||
if not case.is_open and case.closed_at:
|
||||
case.reopen_count += 1
|
||||
case.status = CaseStatus.NEW
|
||||
case.closed_at = None
|
||||
add_event(db, case, None, "reopened",
|
||||
f"Alert fired again after being {case.status.value}")
|
||||
case.last_seen_at = now
|
||||
case.title = alert.title
|
||||
case.subject = subject or case.subject
|
||||
db.commit()
|
||||
return case
|
||||
|
||||
|
||||
def add_event(db: Session, case: Case, actor: Optional[User], action: str,
|
||||
detail: str = "", payload: Optional[dict[str, Any]] = None) -> CaseEvent:
|
||||
event = CaseEvent(
|
||||
actor_id=actor.id if actor else None,
|
||||
actor_label=(actor.email if actor else "system"),
|
||||
action=action, detail=detail, payload=payload,
|
||||
)
|
||||
# Appended through the relationship rather than inserted by id: sessions use
|
||||
# expire_on_commit=False, so a collection already loaded would otherwise stay
|
||||
# stale and the new event would be missing from the response.
|
||||
case.events.append(event)
|
||||
db.add(event)
|
||||
return event
|
||||
|
||||
|
||||
def set_status(db: Session, case: Case, status: CaseStatus, actor: Optional[User],
|
||||
note: str = "") -> Case:
|
||||
previous = case.status
|
||||
case.status = status
|
||||
if status in (CaseStatus.RESOLVED, CaseStatus.WONT_FIX, CaseStatus.FALSE_POSITIVE):
|
||||
case.closed_at = dt.datetime.now(dt.timezone.utc)
|
||||
else:
|
||||
case.closed_at = None
|
||||
add_event(db, case, actor, "status_changed",
|
||||
f"{previous.value} -> {status.value}" + (f": {note}" if note else ""),
|
||||
{"from": previous.value, "to": status.value})
|
||||
db.commit()
|
||||
return case
|
||||
Reference in New Issue
Block a user