"""Persistent state: who is working what, and what has been done about it. The alert queue itself stays stateless - it is recomputed from Prometheus every minute. What is worth persisting is the human layer on top: which alerts someone has picked up, what was done, and the audit trail behind it. Cases are keyed by the alert fingerprint so an alert that stops and re-fires lands back on the same case rather than losing its history. """ from __future__ import annotations import datetime as dt import enum from typing import Any from sqlalchemy import (JSON, Boolean, Date, DateTime, Enum, ForeignKey, Index, Integer, String, Text, UniqueConstraint) from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship class Base(DeclarativeBase): pass def _now() -> dt.datetime: return dt.datetime.now(dt.timezone.utc) def _as_utc(value: dt.datetime | None) -> dt.datetime | None: """Re-attach UTC to a value read back from the database. SQLite has no timezone type, so a datetime stored as aware comes back naive and cannot be compared with `now()`. Postgres round-trips correctly; this keeps both behaving the same. """ if value is None: return None return value if value.tzinfo else value.replace(tzinfo=dt.timezone.utc) class CaseStatus(str, enum.Enum): """Where a piece of work has got to.""" NEW = "new" # seen, nobody has touched it INVESTIGATING = "investigating" # someone has picked it up CUSTOMER_CONTACTED = "customer_contacted" ESCALATED_INFRA = "escalated_infra" WAITING_CUSTOMER = "waiting_customer" WAITING_INFRA = "waiting_infra" REMEDIATED = "remediated" # fixed, waiting for the alert to clear RESOLVED = "resolved" WONT_FIX = "wont_fix" # deliberate no-action FALSE_POSITIVE = "false_positive" # the alert itself was wrong OPEN_STATUSES = { CaseStatus.NEW, CaseStatus.INVESTIGATING, CaseStatus.CUSTOMER_CONTACTED, CaseStatus.ESCALATED_INFRA, CaseStatus.WAITING_CUSTOMER, CaseStatus.WAITING_INFRA, CaseStatus.REMEDIATED, } class AuthProvider(str, enum.Enum): LOCAL = "local" OIDC = "oidc" class User(Base): __tablename__ = "users" id: Mapped[int] = mapped_column(primary_key=True) email: Mapped[str] = mapped_column(String(320), unique=True, index=True) name: Mapped[str] = mapped_column(String(200), default="") is_admin: Mapped[bool] = mapped_column(Boolean, default=False) is_active: Mapped[bool] = mapped_column(Boolean, default=True) provider: Mapped[AuthProvider] = mapped_column(Enum(AuthProvider), default=AuthProvider.LOCAL) # Local accounts only; OIDC users never have one. password_hash: Mapped[str] = mapped_column(String(255), default="") # Stable Authentik subject, so a rename or email change keeps the same user. oidc_sub: Mapped[str] = mapped_column(String(255), default="", index=True) signoff_name: Mapped[str] = mapped_column(String(200), default="") created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now) last_login: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) def to_json(self) -> dict[str, Any]: return { "id": self.id, "email": self.email, "name": self.name or self.email, "is_admin": self.is_admin, "provider": self.provider.value, "signoff_name": self.signoff_name or self.name, } class Case(Base): """One tracked alert, keyed by its fingerprint.""" __tablename__ = "cases" __table_args__ = (Index("ix_cases_status_seen", "status", "last_seen_at"),) id: Mapped[int] = mapped_column(primary_key=True) fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True) kind: Mapped[str] = mapped_column(String(40), index=True) title: Mapped[str] = mapped_column(String(300), default="") subject: Mapped[str] = mapped_column(String(300), default="") # VM name, host or IP openstack_id: Mapped[str] = mapped_column(String(64), default="", index=True) instance_name: Mapped[str] = mapped_column(String(200), default="", index=True) host: Mapped[str] = mapped_column(String(120), default="", index=True) region: Mapped[str] = mapped_column(String(16), default="") org_id: Mapped[str] = mapped_column(String(32), default="", index=True) org_name: Mapped[str] = mapped_column(String(300), default="") status: Mapped[CaseStatus] = mapped_column(Enum(CaseStatus), default=CaseStatus.NEW, index=True) assignee_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True) assignee: Mapped[User | None] = relationship(lazy="joined") # Outbound references, so a case shows what already exists elsewhere. zendesk_ticket_id: Mapped[str] = mapped_column(String(40), default="") zendesk_ticket_url: Mapped[str] = mapped_column(String(500), default="") jira_issue_key: Mapped[str] = mapped_column(String(40), default="") jira_issue_url: Mapped[str] = mapped_column(String(500), default="") notes: Mapped[str] = mapped_column(Text, default="") snooze_until: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) first_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now) last_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now) closed_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) # How many separate times this alert has come back after being closed. reopen_count: Mapped[int] = mapped_column(Integer, default=0) events: Mapped[list["CaseEvent"]] = relationship( back_populates="case", cascade="all, delete-orphan", order_by="CaseEvent.created_at.desc()") @property def is_open(self) -> bool: return self.status in OPEN_STATUSES def to_json(self, with_events: bool = False) -> dict[str, Any]: data = { "id": self.id, "fingerprint": self.fingerprint, "kind": self.kind, "title": self.title, "subject": self.subject, "openstack_id": self.openstack_id, "instance_name": self.instance_name, "host": self.host, "region": self.region, "org_id": self.org_id, "org_name": self.org_name, "status": self.status.value, "is_open": self.is_open, "assignee": self.assignee.to_json() if self.assignee else None, "zendesk_ticket_id": self.zendesk_ticket_id, "zendesk_ticket_url": self.zendesk_ticket_url, "jira_issue_key": self.jira_issue_key, "jira_issue_url": self.jira_issue_url, "notes": self.notes, "snooze_until": self.snooze_until.isoformat() if self.snooze_until else None, "first_seen_at": self.first_seen_at.isoformat() if self.first_seen_at else None, "last_seen_at": self.last_seen_at.isoformat() if self.last_seen_at else None, "closed_at": self.closed_at.isoformat() if self.closed_at else None, "reopen_count": self.reopen_count, } if with_events: data["events"] = [e.to_json() for e in self.events] return data class CaseEvent(Base): """Append-only history. Nothing here is ever edited or deleted.""" __tablename__ = "case_events" id: Mapped[int] = mapped_column(primary_key=True) case_id: Mapped[int] = mapped_column(ForeignKey("cases.id"), index=True) case: Mapped[Case] = relationship(back_populates="events") actor_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True) actor: Mapped[User | None] = relationship(lazy="joined") actor_label: Mapped[str] = mapped_column(String(200), default="") # survives user deletion action: Mapped[str] = mapped_column(String(60)) # status_changed | zendesk_sent | note | ... detail: Mapped[str] = mapped_column(Text, default="") payload: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True) def to_json(self) -> dict[str, Any]: return { "id": self.id, "action": self.action, "detail": self.detail, "actor": self.actor_label or (self.actor.email if self.actor else "system"), "created_at": self.created_at.isoformat() if self.created_at else None, "payload": self.payload, } class SuppressionRule(Base): """Alerts the team has decided not to see, with the reason recorded.""" __tablename__ = "suppression_rules" __table_args__ = (UniqueConstraint("name", name="uq_rule_name"),) id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column(String(200)) reason: Mapped[str] = mapped_column(Text, default="") enabled: Mapped[bool] = mapped_column(Boolean, default=True) # {"kind": ["error"], "organization": ["modal"]} - all keys must match. conditions: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) created_by: Mapped[str] = mapped_column(String(200), default="") created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now) def to_json(self) -> dict[str, Any]: return { "id": str(self.id), "name": self.name, "reason": self.reason, "enabled": self.enabled, "conditions": self.conditions or {}, "created_by": self.created_by, "created": self.created_at.strftime("%Y-%m-%d") if self.created_at else "", } class AppSetting(Base): """Small key/value bag for things an admin can change at runtime.""" __tablename__ = "app_settings" key: Mapped[str] = mapped_column(String(80), primary_key=True) value: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) updated_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now) # --- shift handover --------------------------------------------------------- class ShiftName(str, enum.Enum): APAC = "APAC" EMEA = "EMEA" AMER = "AMER" class HandoverStatus(str, enum.Enum): DRAFT = "draft" HANDED_OVER = "handed_over" class ItemState(str, enum.Enum): """The status line CX writes at the bottom of each key update.""" PENDING_INFRA = "pending_infra" PENDING_CUSTOMER = "pending_customer" PENDING_RUNPOD = "pending_runpod" PENDING_RMA = "pending_rma" PENDING_CX = "pending_cx" IN_PROGRESS = "in_progress" MONITORING = "monitoring" NO_FURTHER_ENGAGEMENT = "no_further_engagement" DONE = "done" class Handover(Base): """One shift handover document.""" __tablename__ = "handovers" __table_args__ = (UniqueConstraint("shift_date", "shift", name="uq_handover_date_shift"),) id: Mapped[int] = mapped_column(primary_key=True) shift_date: Mapped[dt.date] = mapped_column(Date, index=True) shift: Mapped[ShiftName] = mapped_column(Enum(ShiftName), default=ShiftName.APAC) handing_to: Mapped[str] = mapped_column(String(40), default="") # "EMEA" in "APAC -> EMEA" status: Mapped[HandoverStatus] = mapped_column(Enum(HandoverStatus), default=HandoverStatus.DRAFT) team_members: Mapped[str] = mapped_column(String(400), default="") # The tick-boxes at the top of the Confluence page. significant_issues_checked: Mapped[bool] = mapped_column(Boolean, default=False) hubspot_checked: Mapped[bool] = mapped_column(Boolean, default=False) total_open_tickets: Mapped[int | None] = mapped_column(Integer, nullable=True) # [{"name": "Parham", "hs_checked": true, "jira_checked": true}, ...] member_checks: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, default=list) other_comments: Mapped[str] = mapped_column(Text, default="") reviewed_by: Mapped[str] = mapped_column(String(200), default="") reviewed_at_utc: Mapped[str] = mapped_column(String(40), default="") following_shift_checked: Mapped[bool] = mapped_column(Boolean, default=False) created_by_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True) created_by: Mapped[User | None] = relationship(lazy="joined") created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now) updated_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now) items: Mapped[list["HandoverItem"]] = relationship( back_populates="handover", cascade="all, delete-orphan", order_by="HandoverItem.position") def to_json(self, with_items: bool = False) -> dict[str, Any]: data = { "id": self.id, "shift_date": self.shift_date.isoformat() if self.shift_date else None, "shift": self.shift.value, "handing_to": self.handing_to, "status": self.status.value, "title": f"{self.shift.value} {self.shift_date.strftime('%d %B %Y')}" if self.shift_date else "", "team_members": self.team_members, "significant_issues_checked": self.significant_issues_checked, "hubspot_checked": self.hubspot_checked, "total_open_tickets": self.total_open_tickets, "member_checks": self.member_checks or [], "other_comments": self.other_comments, "reviewed_by": self.reviewed_by, "reviewed_at_utc": self.reviewed_at_utc, "following_shift_checked": self.following_shift_checked, "created_by": self.created_by.to_json() if self.created_by else None, "updated_at": self.updated_at.isoformat() if self.updated_at else None, "item_count": len(self.items), } if with_items: data["items"] = [i.to_json() for i in self.items] return data class HandoverItem(Base): """One key update: the Zendesk/Jira refs, the narrative, and where it stands. Carried forward between shifts rather than retyped - `carried_from_id` records where it came from, so an issue keeps one thread across days. """ __tablename__ = "handover_items" id: Mapped[int] = mapped_column(primary_key=True) handover_id: Mapped[int] = mapped_column(ForeignKey("handovers.id"), index=True) handover: Mapped[Handover] = relationship(back_populates="items") position: Mapped[int] = mapped_column(Integer, default=0) title: Mapped[str] = mapped_column(String(400), default="") zendesk_tickets: Mapped[str] = mapped_column(String(300), default="") # "#8495, #8453" jira_key: Mapped[str] = mapped_column(String(60), default="") jira_status: Mapped[str] = mapped_column(String(60), default="") body: Mapped[str] = mapped_column(Text, default="") links: Mapped[list[str] | None] = mapped_column(JSON, default=list) state: Mapped[ItemState] = mapped_column(Enum(ItemState), default=ItemState.IN_PROGRESS) # "Remove at end of shift" in the doc. remove_at_end_of_shift: Mapped[bool] = mapped_column(Boolean, default=False) carried_from_id: Mapped[int | None] = mapped_column(ForeignKey("handover_items.id"), nullable=True) first_raised_on: Mapped[dt.date | None] = mapped_column(Date, nullable=True) created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now) def to_json(self) -> dict[str, Any]: return { "id": self.id, "position": self.position, "title": self.title, "zendesk_tickets": self.zendesk_tickets, "jira_key": self.jira_key, "jira_status": self.jira_status, "body": self.body, "links": self.links or [], "state": self.state.value, "remove_at_end_of_shift": self.remove_at_end_of_shift, "carried_from_id": self.carried_from_id, "first_raised_on": self.first_raised_on.isoformat() if self.first_raised_on else None, } # --- RunPod ----------------------------------------------------------------- class RunpodColour(str, enum.Enum): """The colour key CX uses on the handover doc, kept verbatim.""" RED = "red" # blocked from relisting, recurring issue PURPLE = "purple" # waiting on RunPod YELLOW = "yellow" # waiting on Infrastructure or the DC team BLUE = "blue" # pending removal from the RunPod platform GREEN = "green" # stress testing >24h, GPUs may look reserved WHITE = "white" # actionable by CX class RunpodEventType(str, enum.Enum): UNLISTED = "unlisted" LISTED = "listed" MAINTENANCE_SCHEDULED = "maintenance_scheduled" MAINTENANCE_ENDED = "maintenance_ended" DRAINED = "drained" # unlisted and rented GPUs reached zero NOTE = "note" ZENDESK_TICKET = "zendesk_ticket" JIRA_LINKED = "jira_linked" class RunpodHost(Base): """A RunPod machine, and where CX has got to with it.""" __tablename__ = "runpod_hosts" id: Mapped[int] = mapped_column(primary_key=True) machine_id: Mapped[str] = mapped_column(String(40), unique=True, index=True) name: Mapped[str] = mapped_column(String(200), default="", index=True) listed: Mapped[bool] = mapped_column(Boolean, default=True, index=True) gpu_reserved: Mapped[int] = mapped_column(Integer, default=0) gpu_total: Mapped[int] = mapped_column(Integer, default=0) gpu_type: Mapped[str] = mapped_column(String(80), default="") data_center: Mapped[str] = mapped_column(String(40), default="") uptime_one_week: Mapped[float | None] = mapped_column(nullable=True) maintenance_start: Mapped[str] = mapped_column(String(40), default="") maintenance_end: Mapped[str] = mapped_column(String(40), default="") # CX-side state, mirrored onto the handover doc. colour: Mapped[RunpodColour] = mapped_column(Enum(RunpodColour), default=RunpodColour.WHITE) zendesk_ticket: Mapped[str] = mapped_column(String(40), default="") jira_key: Mapped[str] = mapped_column(String(60), default="") jira_status: Mapped[str] = mapped_column(String(60), default="") last_error: Mapped[str] = mapped_column(Text, default="") # hint from the RunPod email next_steps: Mapped[str] = mapped_column(Text, default="") # How often this machine has been unlisted; the "problem host" ranking. unlist_count: Mapped[int] = mapped_column(Integer, default=0, index=True) historic_count: Mapped[int] = mapped_column(Integer, default=0) unlisted_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) last_listed_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) first_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now) updated_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now) events: Mapped[list["RunpodEvent"]] = relationship( back_populates="host", cascade="all, delete-orphan", order_by="RunpodEvent.occurred_at.desc()") @property def hours_unlisted(self) -> float | None: started = _as_utc(self.unlisted_at) if self.listed or not started: return None return round((_now() - started).total_seconds() / 3600, 1) def to_json(self, with_events: bool = False) -> dict[str, Any]: data = { "id": self.id, "machine_id": self.machine_id, "name": self.name, "listed": self.listed, "gpu_reserved": self.gpu_reserved, "gpu_total": self.gpu_total, "gpu_type": self.gpu_type, "data_center": self.data_center, "uptime_one_week": self.uptime_one_week, "maintenance_start": self.maintenance_start, "maintenance_end": self.maintenance_end, "colour": self.colour.value, "zendesk_ticket": self.zendesk_ticket, "jira_key": self.jira_key, "jira_status": self.jira_status, "last_error": self.last_error, "next_steps": self.next_steps, "unlist_count": self.unlist_count, "historic_count": self.historic_count, "unlisted_at": self.unlisted_at.isoformat() if self.unlisted_at else None, "hours_unlisted": self.hours_unlisted, "event_count": len(self.events), } if with_events: data["events"] = [e.to_json() for e in self.events] return data class RunpodEvent(Base): """Append-only history for a machine: listings, unlistings and who did what.""" __tablename__ = "runpod_events" id: Mapped[int] = mapped_column(primary_key=True) host_id: Mapped[int] = mapped_column(ForeignKey("runpod_hosts.id"), index=True) host: Mapped[RunpodHost] = relationship(back_populates="events") event_type: Mapped[RunpodEventType] = mapped_column(Enum(RunpodEventType), index=True) # "runpod" when RunPod unlisted it automatically, otherwise the operator. actor: Mapped[str] = mapped_column(String(200), default="system") detail: Mapped[str] = mapped_column(Text, default="") error_hint: Mapped[str] = mapped_column(Text, default="") # parsed from the unlisting email zendesk_ticket: Mapped[str] = mapped_column(String(40), default="") jira_key: Mapped[str] = mapped_column(String(60), default="") gpu_reserved: Mapped[int | None] = mapped_column(Integer, nullable=True) payload: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) occurred_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True) def to_json(self) -> dict[str, Any]: return { "id": self.id, "event_type": self.event_type.value, "actor": self.actor, "detail": self.detail, "error_hint": self.error_hint, "zendesk_ticket": self.zendesk_ticket, "jira_key": self.jira_key, "gpu_reserved": self.gpu_reserved, "occurred_at": self.occurred_at.isoformat() if self.occurred_at else None, "payload": self.payload, }