Parham Monfared
1262690276
The single-file stdlib server became the limit: no way to track what had been done about an alert, no accounts, and a UI that had to be hand-rolled in template strings. This restructures it into something deployable. Backend (FastAPI) - app/ holds config, database, auth, delivery and the routers; triagelib keeps the triage engine unchanged, so the validated screening and runbook logic is untouched. - Cases persist per alert fingerprint with a status workflow (investigating, customer contacted, escalated to Infra, waiting, remediated, resolved, won't fix, false positive), an assignee, notes and an append-only history. An alert that stops and re-fires lands back on the same case and counts as a reopen. - Suppression rules move from a JSON file into the database. Auth - Signed session cookies over PBKDF2 local accounts, plus an OIDC flow ready for Authentik: users are created on first login and admin follows a group claim. Local login can be switched off entirely once SSO is live. Zendesk and Jira - Delivery is now implemented, behind three gates: the integration must be configured, its feature flag on, and CX_FEATURE_SEND_ENABLED on. A demo instance leaves the last off and cannot mail anyone. Both search before creating, so re-diagnosing an alert updates one ticket rather than opening several, and a rolling daily cap stops a loop mailing everybody. Deployment - Multi-stage Dockerfile builds the bundle and serves it from the API origin. - docker-compose for local and single-host use; Gitea Actions runs the tests, builds the image and renders deploy/k8s with envsubst. Two fixes found while testing: assigning a case returned a null assignee, and add_event could leave an already-loaded history collection stale. Known gap: the engine reaches OpenStack via `docker exec <region>-osc`, which does not work in a pod without the CX-Tools containers alongside it. docs/DEPLOYMENT.md sets out the three ways to close that. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
89 lines
2.8 KiB
Python
89 lines
2.8 KiB
Python
"""The alert queue and per-alert diagnosis."""
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
import time
|
|
import traceback
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from typing import Any
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, status
|
|
from pydantic import BaseModel
|
|
from sqlalchemy.orm import Session
|
|
|
|
from triagelib import runbooks
|
|
|
|
from ..auth import current_user
|
|
from ..db import SessionLocal, get_db
|
|
from ..models import User
|
|
from ..services import Engine, RuleAdapter, get_or_create_case
|
|
|
|
router = APIRouter(prefix="/api", tags=["alerts"])
|
|
engine = Engine()
|
|
|
|
_jobs: dict[str, dict[str, Any]] = {}
|
|
_jobs_lock = threading.Lock()
|
|
_pool = ThreadPoolExecutor(max_workers=3, thread_name_prefix="triage")
|
|
JOB_TTL = 30 * 60
|
|
|
|
|
|
class TriageBody(BaseModel):
|
|
fingerprint: str
|
|
force: bool = False
|
|
|
|
|
|
@router.get("/alerts")
|
|
def alert_queue(force: bool = False, db: Session = Depends(get_db),
|
|
user: User = Depends(current_user)):
|
|
return engine.queue(db, force=force)
|
|
|
|
|
|
@router.post("/triage")
|
|
def start_triage(body: TriageBody, db: Session = Depends(get_db),
|
|
user: User = Depends(current_user)):
|
|
alert = engine.find_alert(db, body.fingerprint)
|
|
if alert is None:
|
|
raise HTTPException(status.HTTP_404_NOT_FOUND,
|
|
"That alert is no longer firing. Refresh the queue.")
|
|
case = get_or_create_case(db, alert, user)
|
|
job_id = uuid.uuid4().hex[:12]
|
|
with _jobs_lock:
|
|
_reap()
|
|
_jobs[job_id] = {"id": job_id, "state": "running", "created": time.time(),
|
|
"result": None, "error": ""}
|
|
_pool.submit(_run, job_id, alert, body.force)
|
|
return {"job_id": job_id, "alert": alert.to_json(), "case": case.to_json(with_events=True)}
|
|
|
|
|
|
@router.get("/jobs/{job_id}")
|
|
def job(job_id: str, user: User = Depends(current_user)):
|
|
with _jobs_lock:
|
|
found = _jobs.get(job_id)
|
|
if not found:
|
|
raise HTTPException(status.HTTP_404_NOT_FOUND, "Unknown job")
|
|
return found
|
|
|
|
|
|
def _run(job_id: str, alert: Any, force: bool) -> None:
|
|
started = time.monotonic()
|
|
db = SessionLocal()
|
|
try:
|
|
diagnosis = runbooks.diagnose(alert, engine.prom, engine.snapshot.get(), force,
|
|
RuleAdapter(db))
|
|
payload = diagnosis.to_json()
|
|
payload["elapsed_seconds"] = round(time.monotonic() - started, 1)
|
|
with _jobs_lock:
|
|
_jobs[job_id].update({"state": "done", "result": payload})
|
|
except Exception:
|
|
with _jobs_lock:
|
|
_jobs[job_id].update({"state": "error", "error": traceback.format_exc(limit=4)})
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _reap() -> None:
|
|
cutoff = time.time() - JOB_TTL
|
|
for key in [k for k, v in _jobs.items() if v["created"] < cutoff]:
|
|
_jobs.pop(key, None)
|