The single-file stdlib server became the limit: no way to track what had been done about an alert, no accounts, and a UI that had to be hand-rolled in template strings. This restructures it into something deployable. Backend (FastAPI) - app/ holds config, database, auth, delivery and the routers; triagelib keeps the triage engine unchanged, so the validated screening and runbook logic is untouched. - Cases persist per alert fingerprint with a status workflow (investigating, customer contacted, escalated to Infra, waiting, remediated, resolved, won't fix, false positive), an assignee, notes and an append-only history. An alert that stops and re-fires lands back on the same case and counts as a reopen. - Suppression rules move from a JSON file into the database. Auth - Signed session cookies over PBKDF2 local accounts, plus an OIDC flow ready for Authentik: users are created on first login and admin follows a group claim. Local login can be switched off entirely once SSO is live. Zendesk and Jira - Delivery is now implemented, behind three gates: the integration must be configured, its feature flag on, and CX_FEATURE_SEND_ENABLED on. A demo instance leaves the last off and cannot mail anyone. Both search before creating, so re-diagnosing an alert updates one ticket rather than opening several, and a rolling daily cap stops a loop mailing everybody. Deployment - Multi-stage Dockerfile builds the bundle and serves it from the API origin. - docker-compose for local and single-host use; Gitea Actions runs the tests, builds the image and renders deploy/k8s with envsubst. Two fixes found while testing: assigning a case returned a null assignee, and add_event could leave an already-loaded history collection stale. Known gap: the engine reaches OpenStack via `docker exec <region>-osc`, which does not work in a pod without the CX-Tools containers alongside it. docs/DEPLOYMENT.md sets out the three ways to close that. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
209 lines
9.4 KiB
Python
209 lines
9.4 KiB
Python
"""Persistent state: who is working what, and what has been done about it.
|
|
|
|
The alert queue itself stays stateless - it is recomputed from Prometheus every
|
|
minute. What is worth persisting is the human layer on top: which alerts someone
|
|
has picked up, what was done, and the audit trail behind it. Cases are keyed by
|
|
the alert fingerprint so an alert that stops and re-fires lands back on the same
|
|
case rather than losing its history.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import enum
|
|
from typing import Any
|
|
|
|
from sqlalchemy import (JSON, Boolean, DateTime, Enum, ForeignKey, Index, Integer,
|
|
String, Text, UniqueConstraint)
|
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
|
|
|
|
|
class Base(DeclarativeBase):
|
|
pass
|
|
|
|
|
|
def _now() -> dt.datetime:
|
|
return dt.datetime.now(dt.timezone.utc)
|
|
|
|
|
|
class CaseStatus(str, enum.Enum):
|
|
"""Where a piece of work has got to."""
|
|
|
|
NEW = "new" # seen, nobody has touched it
|
|
INVESTIGATING = "investigating" # someone has picked it up
|
|
CUSTOMER_CONTACTED = "customer_contacted"
|
|
ESCALATED_INFRA = "escalated_infra"
|
|
WAITING_CUSTOMER = "waiting_customer"
|
|
WAITING_INFRA = "waiting_infra"
|
|
REMEDIATED = "remediated" # fixed, waiting for the alert to clear
|
|
RESOLVED = "resolved"
|
|
WONT_FIX = "wont_fix" # deliberate no-action
|
|
FALSE_POSITIVE = "false_positive" # the alert itself was wrong
|
|
|
|
|
|
OPEN_STATUSES = {
|
|
CaseStatus.NEW, CaseStatus.INVESTIGATING, CaseStatus.CUSTOMER_CONTACTED,
|
|
CaseStatus.ESCALATED_INFRA, CaseStatus.WAITING_CUSTOMER, CaseStatus.WAITING_INFRA,
|
|
CaseStatus.REMEDIATED,
|
|
}
|
|
|
|
|
|
class AuthProvider(str, enum.Enum):
|
|
LOCAL = "local"
|
|
OIDC = "oidc"
|
|
|
|
|
|
class User(Base):
|
|
__tablename__ = "users"
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
|
|
name: Mapped[str] = mapped_column(String(200), default="")
|
|
is_admin: Mapped[bool] = mapped_column(Boolean, default=False)
|
|
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
|
provider: Mapped[AuthProvider] = mapped_column(Enum(AuthProvider), default=AuthProvider.LOCAL)
|
|
# Local accounts only; OIDC users never have one.
|
|
password_hash: Mapped[str] = mapped_column(String(255), default="")
|
|
# Stable Authentik subject, so a rename or email change keeps the same user.
|
|
oidc_sub: Mapped[str] = mapped_column(String(255), default="", index=True)
|
|
signoff_name: Mapped[str] = mapped_column(String(200), default="")
|
|
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
|
last_login: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.id, "email": self.email, "name": self.name or self.email,
|
|
"is_admin": self.is_admin, "provider": self.provider.value,
|
|
"signoff_name": self.signoff_name or self.name,
|
|
}
|
|
|
|
|
|
class Case(Base):
|
|
"""One tracked alert, keyed by its fingerprint."""
|
|
|
|
__tablename__ = "cases"
|
|
__table_args__ = (Index("ix_cases_status_seen", "status", "last_seen_at"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
|
|
|
kind: Mapped[str] = mapped_column(String(40), index=True)
|
|
title: Mapped[str] = mapped_column(String(300), default="")
|
|
subject: Mapped[str] = mapped_column(String(300), default="") # VM name, host or IP
|
|
openstack_id: Mapped[str] = mapped_column(String(64), default="", index=True)
|
|
instance_name: Mapped[str] = mapped_column(String(200), default="", index=True)
|
|
host: Mapped[str] = mapped_column(String(120), default="", index=True)
|
|
region: Mapped[str] = mapped_column(String(16), default="")
|
|
org_id: Mapped[str] = mapped_column(String(32), default="", index=True)
|
|
org_name: Mapped[str] = mapped_column(String(300), default="")
|
|
|
|
status: Mapped[CaseStatus] = mapped_column(Enum(CaseStatus), default=CaseStatus.NEW, index=True)
|
|
assignee_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True)
|
|
assignee: Mapped[User | None] = relationship(lazy="joined")
|
|
|
|
# Outbound references, so a case shows what already exists elsewhere.
|
|
zendesk_ticket_id: Mapped[str] = mapped_column(String(40), default="")
|
|
zendesk_ticket_url: Mapped[str] = mapped_column(String(500), default="")
|
|
jira_issue_key: Mapped[str] = mapped_column(String(40), default="")
|
|
jira_issue_url: Mapped[str] = mapped_column(String(500), default="")
|
|
|
|
notes: Mapped[str] = mapped_column(Text, default="")
|
|
snooze_until: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
|
|
first_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
|
last_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
|
closed_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
# How many separate times this alert has come back after being closed.
|
|
reopen_count: Mapped[int] = mapped_column(Integer, default=0)
|
|
|
|
events: Mapped[list["CaseEvent"]] = relationship(
|
|
back_populates="case", cascade="all, delete-orphan", order_by="CaseEvent.created_at.desc()")
|
|
|
|
@property
|
|
def is_open(self) -> bool:
|
|
return self.status in OPEN_STATUSES
|
|
|
|
def to_json(self, with_events: bool = False) -> dict[str, Any]:
|
|
data = {
|
|
"id": self.id, "fingerprint": self.fingerprint, "kind": self.kind,
|
|
"title": self.title, "subject": self.subject,
|
|
"openstack_id": self.openstack_id, "instance_name": self.instance_name,
|
|
"host": self.host, "region": self.region,
|
|
"org_id": self.org_id, "org_name": self.org_name,
|
|
"status": self.status.value, "is_open": self.is_open,
|
|
"assignee": self.assignee.to_json() if self.assignee else None,
|
|
"zendesk_ticket_id": self.zendesk_ticket_id,
|
|
"zendesk_ticket_url": self.zendesk_ticket_url,
|
|
"jira_issue_key": self.jira_issue_key,
|
|
"jira_issue_url": self.jira_issue_url,
|
|
"notes": self.notes,
|
|
"snooze_until": self.snooze_until.isoformat() if self.snooze_until else None,
|
|
"first_seen_at": self.first_seen_at.isoformat() if self.first_seen_at else None,
|
|
"last_seen_at": self.last_seen_at.isoformat() if self.last_seen_at else None,
|
|
"closed_at": self.closed_at.isoformat() if self.closed_at else None,
|
|
"reopen_count": self.reopen_count,
|
|
}
|
|
if with_events:
|
|
data["events"] = [e.to_json() for e in self.events]
|
|
return data
|
|
|
|
|
|
class CaseEvent(Base):
|
|
"""Append-only history. Nothing here is ever edited or deleted."""
|
|
|
|
__tablename__ = "case_events"
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
case_id: Mapped[int] = mapped_column(ForeignKey("cases.id"), index=True)
|
|
case: Mapped[Case] = relationship(back_populates="events")
|
|
|
|
actor_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True)
|
|
actor: Mapped[User | None] = relationship(lazy="joined")
|
|
actor_label: Mapped[str] = mapped_column(String(200), default="") # survives user deletion
|
|
|
|
action: Mapped[str] = mapped_column(String(60)) # status_changed | zendesk_sent | note | ...
|
|
detail: Mapped[str] = mapped_column(Text, default="")
|
|
payload: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.id, "action": self.action, "detail": self.detail,
|
|
"actor": self.actor_label or (self.actor.email if self.actor else "system"),
|
|
"created_at": self.created_at.isoformat() if self.created_at else None,
|
|
"payload": self.payload,
|
|
}
|
|
|
|
|
|
class SuppressionRule(Base):
|
|
"""Alerts the team has decided not to see, with the reason recorded."""
|
|
|
|
__tablename__ = "suppression_rules"
|
|
__table_args__ = (UniqueConstraint("name", name="uq_rule_name"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
name: Mapped[str] = mapped_column(String(200))
|
|
reason: Mapped[str] = mapped_column(Text, default="")
|
|
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
|
# {"kind": ["error"], "organization": ["modal"]} - all keys must match.
|
|
conditions: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
created_by: Mapped[str] = mapped_column(String(200), default="")
|
|
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"id": str(self.id), "name": self.name, "reason": self.reason,
|
|
"enabled": self.enabled, "conditions": self.conditions or {},
|
|
"created_by": self.created_by,
|
|
"created": self.created_at.strftime("%Y-%m-%d") if self.created_at else "",
|
|
}
|
|
|
|
|
|
class AppSetting(Base):
|
|
"""Small key/value bag for things an admin can change at runtime."""
|
|
|
|
__tablename__ = "app_settings"
|
|
|
|
key: Mapped[str] = mapped_column(String(80), primary_key=True)
|
|
value: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
updated_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
|