Parham Monfared
8892144e0a
Handover - The Confluence shift doc becomes the landing page: shift metadata, the top-of-page checks, key updates with their Zendesk/Jira refs and status, and the free-text comments. "Hand over shift" closes the shift, opens the next one and carries the live items across, dropping anything done or marked "remove at end of shift" - the retyping this replaces. - The RunPod table on that page is read from live host state instead of being copied in by hand, with the six-colour key preserved. RunPod - GraphQL client keyed on CX_RUNPOD_API_KEY. The old console login is kept as a fallback but cannot run unattended: the account has 2FA, so Clerk verifies the password and then asks for an emailed code and never issues a session. That is the real cause of the "No active session found" failure, and the client now says so instead of failing opaquely. TOTP is supported if the account moves to an authenticator app. - Hosts and their listing history are persisted, so "most problematic hosts" can be ranked and each machine has a timeline of who listed or unlisted it, with the Zendesk comment and the error hint. - The unlisting emails are parsed for the error block (they arrive quoted-printable) and classified into a likely cause and a next step. Zendesk and Jira - Unlisting raises a Zendesk ticket that follows the format of RunPod's own email, keyed on the machine so one machine keeps one thread, posted as an internal note. - Jira is split in two: the Infrahub/OIE instance and the RunPod/RMA one, which may be a different Atlassian site. Blank RunPod values fall back to the defaults rather than failing. Running in a container - CX-Tools reads its keys from 1Password, which needs a desktop app. Config is a dataclass whose lookups live in per-field default factories, so passing CX_INFRAHUB_TOKEN/CX_INFRAINSIGHT_TOKEN in means those factories never run and CX-Tools itself stays unmodified. - CX-Tools reaches OpenStack with `docker exec <region>-osc`, so the image now carries the Docker client (the static binary, not the docker.io package) and compose mounts the host socket with group_add for it. Verified from inside the container: live OpenStack and Infrahub calls both succeed. Also fixes Settings, which read the environment at class-definition time and so ignored anything set afterwards; a fresh Settings() silently returned stale values. Caught by the Jira scoping tests. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
467 lines
22 KiB
Python
467 lines
22 KiB
Python
"""Persistent state: who is working what, and what has been done about it.
|
|
|
|
The alert queue itself stays stateless - it is recomputed from Prometheus every
|
|
minute. What is worth persisting is the human layer on top: which alerts someone
|
|
has picked up, what was done, and the audit trail behind it. Cases are keyed by
|
|
the alert fingerprint so an alert that stops and re-fires lands back on the same
|
|
case rather than losing its history.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import enum
|
|
from typing import Any
|
|
|
|
from sqlalchemy import (JSON, Boolean, Date, DateTime, Enum, ForeignKey, Index, Integer,
|
|
String, Text, UniqueConstraint)
|
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
|
|
|
|
|
class Base(DeclarativeBase):
|
|
pass
|
|
|
|
|
|
def _now() -> dt.datetime:
|
|
return dt.datetime.now(dt.timezone.utc)
|
|
|
|
|
|
def _as_utc(value: dt.datetime | None) -> dt.datetime | None:
|
|
"""Re-attach UTC to a value read back from the database.
|
|
|
|
SQLite has no timezone type, so a datetime stored as aware comes back naive
|
|
and cannot be compared with `now()`. Postgres round-trips correctly; this
|
|
keeps both behaving the same.
|
|
"""
|
|
if value is None:
|
|
return None
|
|
return value if value.tzinfo else value.replace(tzinfo=dt.timezone.utc)
|
|
|
|
|
|
class CaseStatus(str, enum.Enum):
|
|
"""Where a piece of work has got to."""
|
|
|
|
NEW = "new" # seen, nobody has touched it
|
|
INVESTIGATING = "investigating" # someone has picked it up
|
|
CUSTOMER_CONTACTED = "customer_contacted"
|
|
ESCALATED_INFRA = "escalated_infra"
|
|
WAITING_CUSTOMER = "waiting_customer"
|
|
WAITING_INFRA = "waiting_infra"
|
|
REMEDIATED = "remediated" # fixed, waiting for the alert to clear
|
|
RESOLVED = "resolved"
|
|
WONT_FIX = "wont_fix" # deliberate no-action
|
|
FALSE_POSITIVE = "false_positive" # the alert itself was wrong
|
|
|
|
|
|
OPEN_STATUSES = {
|
|
CaseStatus.NEW, CaseStatus.INVESTIGATING, CaseStatus.CUSTOMER_CONTACTED,
|
|
CaseStatus.ESCALATED_INFRA, CaseStatus.WAITING_CUSTOMER, CaseStatus.WAITING_INFRA,
|
|
CaseStatus.REMEDIATED,
|
|
}
|
|
|
|
|
|
class AuthProvider(str, enum.Enum):
|
|
LOCAL = "local"
|
|
OIDC = "oidc"
|
|
|
|
|
|
class User(Base):
|
|
__tablename__ = "users"
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
|
|
name: Mapped[str] = mapped_column(String(200), default="")
|
|
is_admin: Mapped[bool] = mapped_column(Boolean, default=False)
|
|
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
|
provider: Mapped[AuthProvider] = mapped_column(Enum(AuthProvider), default=AuthProvider.LOCAL)
|
|
# Local accounts only; OIDC users never have one.
|
|
password_hash: Mapped[str] = mapped_column(String(255), default="")
|
|
# Stable Authentik subject, so a rename or email change keeps the same user.
|
|
oidc_sub: Mapped[str] = mapped_column(String(255), default="", index=True)
|
|
signoff_name: Mapped[str] = mapped_column(String(200), default="")
|
|
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
|
last_login: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.id, "email": self.email, "name": self.name or self.email,
|
|
"is_admin": self.is_admin, "provider": self.provider.value,
|
|
"signoff_name": self.signoff_name or self.name,
|
|
}
|
|
|
|
|
|
class Case(Base):
|
|
"""One tracked alert, keyed by its fingerprint."""
|
|
|
|
__tablename__ = "cases"
|
|
__table_args__ = (Index("ix_cases_status_seen", "status", "last_seen_at"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
|
|
|
kind: Mapped[str] = mapped_column(String(40), index=True)
|
|
title: Mapped[str] = mapped_column(String(300), default="")
|
|
subject: Mapped[str] = mapped_column(String(300), default="") # VM name, host or IP
|
|
openstack_id: Mapped[str] = mapped_column(String(64), default="", index=True)
|
|
instance_name: Mapped[str] = mapped_column(String(200), default="", index=True)
|
|
host: Mapped[str] = mapped_column(String(120), default="", index=True)
|
|
region: Mapped[str] = mapped_column(String(16), default="")
|
|
org_id: Mapped[str] = mapped_column(String(32), default="", index=True)
|
|
org_name: Mapped[str] = mapped_column(String(300), default="")
|
|
|
|
status: Mapped[CaseStatus] = mapped_column(Enum(CaseStatus), default=CaseStatus.NEW, index=True)
|
|
assignee_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True)
|
|
assignee: Mapped[User | None] = relationship(lazy="joined")
|
|
|
|
# Outbound references, so a case shows what already exists elsewhere.
|
|
zendesk_ticket_id: Mapped[str] = mapped_column(String(40), default="")
|
|
zendesk_ticket_url: Mapped[str] = mapped_column(String(500), default="")
|
|
jira_issue_key: Mapped[str] = mapped_column(String(40), default="")
|
|
jira_issue_url: Mapped[str] = mapped_column(String(500), default="")
|
|
|
|
notes: Mapped[str] = mapped_column(Text, default="")
|
|
snooze_until: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
|
|
first_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
|
last_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
|
closed_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
# How many separate times this alert has come back after being closed.
|
|
reopen_count: Mapped[int] = mapped_column(Integer, default=0)
|
|
|
|
events: Mapped[list["CaseEvent"]] = relationship(
|
|
back_populates="case", cascade="all, delete-orphan", order_by="CaseEvent.created_at.desc()")
|
|
|
|
@property
|
|
def is_open(self) -> bool:
|
|
return self.status in OPEN_STATUSES
|
|
|
|
def to_json(self, with_events: bool = False) -> dict[str, Any]:
|
|
data = {
|
|
"id": self.id, "fingerprint": self.fingerprint, "kind": self.kind,
|
|
"title": self.title, "subject": self.subject,
|
|
"openstack_id": self.openstack_id, "instance_name": self.instance_name,
|
|
"host": self.host, "region": self.region,
|
|
"org_id": self.org_id, "org_name": self.org_name,
|
|
"status": self.status.value, "is_open": self.is_open,
|
|
"assignee": self.assignee.to_json() if self.assignee else None,
|
|
"zendesk_ticket_id": self.zendesk_ticket_id,
|
|
"zendesk_ticket_url": self.zendesk_ticket_url,
|
|
"jira_issue_key": self.jira_issue_key,
|
|
"jira_issue_url": self.jira_issue_url,
|
|
"notes": self.notes,
|
|
"snooze_until": self.snooze_until.isoformat() if self.snooze_until else None,
|
|
"first_seen_at": self.first_seen_at.isoformat() if self.first_seen_at else None,
|
|
"last_seen_at": self.last_seen_at.isoformat() if self.last_seen_at else None,
|
|
"closed_at": self.closed_at.isoformat() if self.closed_at else None,
|
|
"reopen_count": self.reopen_count,
|
|
}
|
|
if with_events:
|
|
data["events"] = [e.to_json() for e in self.events]
|
|
return data
|
|
|
|
|
|
class CaseEvent(Base):
|
|
"""Append-only history. Nothing here is ever edited or deleted."""
|
|
|
|
__tablename__ = "case_events"
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
case_id: Mapped[int] = mapped_column(ForeignKey("cases.id"), index=True)
|
|
case: Mapped[Case] = relationship(back_populates="events")
|
|
|
|
actor_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True)
|
|
actor: Mapped[User | None] = relationship(lazy="joined")
|
|
actor_label: Mapped[str] = mapped_column(String(200), default="") # survives user deletion
|
|
|
|
action: Mapped[str] = mapped_column(String(60)) # status_changed | zendesk_sent | note | ...
|
|
detail: Mapped[str] = mapped_column(Text, default="")
|
|
payload: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.id, "action": self.action, "detail": self.detail,
|
|
"actor": self.actor_label or (self.actor.email if self.actor else "system"),
|
|
"created_at": self.created_at.isoformat() if self.created_at else None,
|
|
"payload": self.payload,
|
|
}
|
|
|
|
|
|
class SuppressionRule(Base):
|
|
"""Alerts the team has decided not to see, with the reason recorded."""
|
|
|
|
__tablename__ = "suppression_rules"
|
|
__table_args__ = (UniqueConstraint("name", name="uq_rule_name"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
name: Mapped[str] = mapped_column(String(200))
|
|
reason: Mapped[str] = mapped_column(Text, default="")
|
|
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
|
# {"kind": ["error"], "organization": ["modal"]} - all keys must match.
|
|
conditions: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
created_by: Mapped[str] = mapped_column(String(200), default="")
|
|
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"id": str(self.id), "name": self.name, "reason": self.reason,
|
|
"enabled": self.enabled, "conditions": self.conditions or {},
|
|
"created_by": self.created_by,
|
|
"created": self.created_at.strftime("%Y-%m-%d") if self.created_at else "",
|
|
}
|
|
|
|
|
|
class AppSetting(Base):
|
|
"""Small key/value bag for things an admin can change at runtime."""
|
|
|
|
__tablename__ = "app_settings"
|
|
|
|
key: Mapped[str] = mapped_column(String(80), primary_key=True)
|
|
value: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
updated_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
|
|
|
|
|
|
# --- shift handover ---------------------------------------------------------
|
|
|
|
class ShiftName(str, enum.Enum):
|
|
APAC = "APAC"
|
|
EMEA = "EMEA"
|
|
AMER = "AMER"
|
|
|
|
|
|
class HandoverStatus(str, enum.Enum):
|
|
DRAFT = "draft"
|
|
HANDED_OVER = "handed_over"
|
|
|
|
|
|
class ItemState(str, enum.Enum):
|
|
"""The status line CX writes at the bottom of each key update."""
|
|
|
|
PENDING_INFRA = "pending_infra"
|
|
PENDING_CUSTOMER = "pending_customer"
|
|
PENDING_RUNPOD = "pending_runpod"
|
|
PENDING_RMA = "pending_rma"
|
|
PENDING_CX = "pending_cx"
|
|
IN_PROGRESS = "in_progress"
|
|
MONITORING = "monitoring"
|
|
NO_FURTHER_ENGAGEMENT = "no_further_engagement"
|
|
DONE = "done"
|
|
|
|
|
|
class Handover(Base):
|
|
"""One shift handover document."""
|
|
|
|
__tablename__ = "handovers"
|
|
__table_args__ = (UniqueConstraint("shift_date", "shift", name="uq_handover_date_shift"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
shift_date: Mapped[dt.date] = mapped_column(Date, index=True)
|
|
shift: Mapped[ShiftName] = mapped_column(Enum(ShiftName), default=ShiftName.APAC)
|
|
handing_to: Mapped[str] = mapped_column(String(40), default="") # "EMEA" in "APAC -> EMEA"
|
|
status: Mapped[HandoverStatus] = mapped_column(Enum(HandoverStatus), default=HandoverStatus.DRAFT)
|
|
|
|
team_members: Mapped[str] = mapped_column(String(400), default="")
|
|
# The tick-boxes at the top of the Confluence page.
|
|
significant_issues_checked: Mapped[bool] = mapped_column(Boolean, default=False)
|
|
hubspot_checked: Mapped[bool] = mapped_column(Boolean, default=False)
|
|
total_open_tickets: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
# [{"name": "Parham", "hs_checked": true, "jira_checked": true}, ...]
|
|
member_checks: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, default=list)
|
|
|
|
other_comments: Mapped[str] = mapped_column(Text, default="")
|
|
reviewed_by: Mapped[str] = mapped_column(String(200), default="")
|
|
reviewed_at_utc: Mapped[str] = mapped_column(String(40), default="")
|
|
following_shift_checked: Mapped[bool] = mapped_column(Boolean, default=False)
|
|
|
|
created_by_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True)
|
|
created_by: Mapped[User | None] = relationship(lazy="joined")
|
|
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
|
updated_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
|
|
|
|
items: Mapped[list["HandoverItem"]] = relationship(
|
|
back_populates="handover", cascade="all, delete-orphan", order_by="HandoverItem.position")
|
|
|
|
def to_json(self, with_items: bool = False) -> dict[str, Any]:
|
|
data = {
|
|
"id": self.id,
|
|
"shift_date": self.shift_date.isoformat() if self.shift_date else None,
|
|
"shift": self.shift.value,
|
|
"handing_to": self.handing_to,
|
|
"status": self.status.value,
|
|
"title": f"{self.shift.value} {self.shift_date.strftime('%d %B %Y')}" if self.shift_date else "",
|
|
"team_members": self.team_members,
|
|
"significant_issues_checked": self.significant_issues_checked,
|
|
"hubspot_checked": self.hubspot_checked,
|
|
"total_open_tickets": self.total_open_tickets,
|
|
"member_checks": self.member_checks or [],
|
|
"other_comments": self.other_comments,
|
|
"reviewed_by": self.reviewed_by,
|
|
"reviewed_at_utc": self.reviewed_at_utc,
|
|
"following_shift_checked": self.following_shift_checked,
|
|
"created_by": self.created_by.to_json() if self.created_by else None,
|
|
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
|
"item_count": len(self.items),
|
|
}
|
|
if with_items:
|
|
data["items"] = [i.to_json() for i in self.items]
|
|
return data
|
|
|
|
|
|
class HandoverItem(Base):
|
|
"""One key update: the Zendesk/Jira refs, the narrative, and where it stands.
|
|
|
|
Carried forward between shifts rather than retyped - `carried_from_id`
|
|
records where it came from, so an issue keeps one thread across days.
|
|
"""
|
|
|
|
__tablename__ = "handover_items"
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
handover_id: Mapped[int] = mapped_column(ForeignKey("handovers.id"), index=True)
|
|
handover: Mapped[Handover] = relationship(back_populates="items")
|
|
|
|
position: Mapped[int] = mapped_column(Integer, default=0)
|
|
title: Mapped[str] = mapped_column(String(400), default="")
|
|
zendesk_tickets: Mapped[str] = mapped_column(String(300), default="") # "#8495, #8453"
|
|
jira_key: Mapped[str] = mapped_column(String(60), default="")
|
|
jira_status: Mapped[str] = mapped_column(String(60), default="")
|
|
body: Mapped[str] = mapped_column(Text, default="")
|
|
links: Mapped[list[str] | None] = mapped_column(JSON, default=list)
|
|
state: Mapped[ItemState] = mapped_column(Enum(ItemState), default=ItemState.IN_PROGRESS)
|
|
# "Remove at end of shift" in the doc.
|
|
remove_at_end_of_shift: Mapped[bool] = mapped_column(Boolean, default=False)
|
|
|
|
carried_from_id: Mapped[int | None] = mapped_column(ForeignKey("handover_items.id"), nullable=True)
|
|
first_raised_on: Mapped[dt.date | None] = mapped_column(Date, nullable=True)
|
|
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.id, "position": self.position, "title": self.title,
|
|
"zendesk_tickets": self.zendesk_tickets, "jira_key": self.jira_key,
|
|
"jira_status": self.jira_status, "body": self.body, "links": self.links or [],
|
|
"state": self.state.value,
|
|
"remove_at_end_of_shift": self.remove_at_end_of_shift,
|
|
"carried_from_id": self.carried_from_id,
|
|
"first_raised_on": self.first_raised_on.isoformat() if self.first_raised_on else None,
|
|
}
|
|
|
|
|
|
# --- RunPod -----------------------------------------------------------------
|
|
|
|
class RunpodColour(str, enum.Enum):
|
|
"""The colour key CX uses on the handover doc, kept verbatim."""
|
|
|
|
RED = "red" # blocked from relisting, recurring issue
|
|
PURPLE = "purple" # waiting on RunPod
|
|
YELLOW = "yellow" # waiting on Infrastructure or the DC team
|
|
BLUE = "blue" # pending removal from the RunPod platform
|
|
GREEN = "green" # stress testing >24h, GPUs may look reserved
|
|
WHITE = "white" # actionable by CX
|
|
|
|
|
|
class RunpodEventType(str, enum.Enum):
|
|
UNLISTED = "unlisted"
|
|
LISTED = "listed"
|
|
MAINTENANCE_SCHEDULED = "maintenance_scheduled"
|
|
MAINTENANCE_ENDED = "maintenance_ended"
|
|
DRAINED = "drained" # unlisted and rented GPUs reached zero
|
|
NOTE = "note"
|
|
ZENDESK_TICKET = "zendesk_ticket"
|
|
JIRA_LINKED = "jira_linked"
|
|
|
|
|
|
class RunpodHost(Base):
|
|
"""A RunPod machine, and where CX has got to with it."""
|
|
|
|
__tablename__ = "runpod_hosts"
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
machine_id: Mapped[str] = mapped_column(String(40), unique=True, index=True)
|
|
name: Mapped[str] = mapped_column(String(200), default="", index=True)
|
|
|
|
listed: Mapped[bool] = mapped_column(Boolean, default=True, index=True)
|
|
gpu_reserved: Mapped[int] = mapped_column(Integer, default=0)
|
|
gpu_total: Mapped[int] = mapped_column(Integer, default=0)
|
|
gpu_type: Mapped[str] = mapped_column(String(80), default="")
|
|
data_center: Mapped[str] = mapped_column(String(40), default="")
|
|
uptime_one_week: Mapped[float | None] = mapped_column(nullable=True)
|
|
maintenance_start: Mapped[str] = mapped_column(String(40), default="")
|
|
maintenance_end: Mapped[str] = mapped_column(String(40), default="")
|
|
|
|
# CX-side state, mirrored onto the handover doc.
|
|
colour: Mapped[RunpodColour] = mapped_column(Enum(RunpodColour), default=RunpodColour.WHITE)
|
|
zendesk_ticket: Mapped[str] = mapped_column(String(40), default="")
|
|
jira_key: Mapped[str] = mapped_column(String(60), default="")
|
|
jira_status: Mapped[str] = mapped_column(String(60), default="")
|
|
last_error: Mapped[str] = mapped_column(Text, default="") # hint from the RunPod email
|
|
next_steps: Mapped[str] = mapped_column(Text, default="")
|
|
|
|
# How often this machine has been unlisted; the "problem host" ranking.
|
|
unlist_count: Mapped[int] = mapped_column(Integer, default=0, index=True)
|
|
historic_count: Mapped[int] = mapped_column(Integer, default=0)
|
|
unlisted_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
last_listed_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
first_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
|
updated_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
|
|
|
|
events: Mapped[list["RunpodEvent"]] = relationship(
|
|
back_populates="host", cascade="all, delete-orphan",
|
|
order_by="RunpodEvent.occurred_at.desc()")
|
|
|
|
@property
|
|
def hours_unlisted(self) -> float | None:
|
|
started = _as_utc(self.unlisted_at)
|
|
if self.listed or not started:
|
|
return None
|
|
return round((_now() - started).total_seconds() / 3600, 1)
|
|
|
|
def to_json(self, with_events: bool = False) -> dict[str, Any]:
|
|
data = {
|
|
"id": self.id, "machine_id": self.machine_id, "name": self.name,
|
|
"listed": self.listed, "gpu_reserved": self.gpu_reserved, "gpu_total": self.gpu_total,
|
|
"gpu_type": self.gpu_type, "data_center": self.data_center,
|
|
"uptime_one_week": self.uptime_one_week,
|
|
"maintenance_start": self.maintenance_start, "maintenance_end": self.maintenance_end,
|
|
"colour": self.colour.value, "zendesk_ticket": self.zendesk_ticket,
|
|
"jira_key": self.jira_key, "jira_status": self.jira_status,
|
|
"last_error": self.last_error, "next_steps": self.next_steps,
|
|
"unlist_count": self.unlist_count, "historic_count": self.historic_count,
|
|
"unlisted_at": self.unlisted_at.isoformat() if self.unlisted_at else None,
|
|
"hours_unlisted": self.hours_unlisted,
|
|
"event_count": len(self.events),
|
|
}
|
|
if with_events:
|
|
data["events"] = [e.to_json() for e in self.events]
|
|
return data
|
|
|
|
|
|
class RunpodEvent(Base):
|
|
"""Append-only history for a machine: listings, unlistings and who did what."""
|
|
|
|
__tablename__ = "runpod_events"
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
host_id: Mapped[int] = mapped_column(ForeignKey("runpod_hosts.id"), index=True)
|
|
host: Mapped[RunpodHost] = relationship(back_populates="events")
|
|
|
|
event_type: Mapped[RunpodEventType] = mapped_column(Enum(RunpodEventType), index=True)
|
|
# "runpod" when RunPod unlisted it automatically, otherwise the operator.
|
|
actor: Mapped[str] = mapped_column(String(200), default="system")
|
|
detail: Mapped[str] = mapped_column(Text, default="")
|
|
error_hint: Mapped[str] = mapped_column(Text, default="") # parsed from the unlisting email
|
|
zendesk_ticket: Mapped[str] = mapped_column(String(40), default="")
|
|
jira_key: Mapped[str] = mapped_column(String(60), default="")
|
|
gpu_reserved: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
payload: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
occurred_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.id, "event_type": self.event_type.value, "actor": self.actor,
|
|
"detail": self.detail, "error_hint": self.error_hint,
|
|
"zendesk_ticket": self.zendesk_ticket, "jira_key": self.jira_key,
|
|
"gpu_reserved": self.gpu_reserved,
|
|
"occurred_at": self.occurred_at.isoformat() if self.occurred_at else None,
|
|
"payload": self.payload,
|
|
}
|