Add shift handover and RunPod, and make CX-Tools work in a container
Handover - The Confluence shift doc becomes the landing page: shift metadata, the top-of-page checks, key updates with their Zendesk/Jira refs and status, and the free-text comments. "Hand over shift" closes the shift, opens the next one and carries the live items across, dropping anything done or marked "remove at end of shift" - the retyping this replaces. - The RunPod table on that page is read from live host state instead of being copied in by hand, with the six-colour key preserved. RunPod - GraphQL client keyed on CX_RUNPOD_API_KEY. The old console login is kept as a fallback but cannot run unattended: the account has 2FA, so Clerk verifies the password and then asks for an emailed code and never issues a session. That is the real cause of the "No active session found" failure, and the client now says so instead of failing opaquely. TOTP is supported if the account moves to an authenticator app. - Hosts and their listing history are persisted, so "most problematic hosts" can be ranked and each machine has a timeline of who listed or unlisted it, with the Zendesk comment and the error hint. - The unlisting emails are parsed for the error block (they arrive quoted-printable) and classified into a likely cause and a next step. Zendesk and Jira - Unlisting raises a Zendesk ticket that follows the format of RunPod's own email, keyed on the machine so one machine keeps one thread, posted as an internal note. - Jira is split in two: the Infrahub/OIE instance and the RunPod/RMA one, which may be a different Atlassian site. Blank RunPod values fall back to the defaults rather than failing. Running in a container - CX-Tools reads its keys from 1Password, which needs a desktop app. Config is a dataclass whose lookups live in per-field default factories, so passing CX_INFRAHUB_TOKEN/CX_INFRAINSIGHT_TOKEN in means those factories never run and CX-Tools itself stays unmodified. - CX-Tools reaches OpenStack with `docker exec <region>-osc`, so the image now carries the Docker client (the static binary, not the docker.io package) and compose mounts the host socket with group_add for it. Verified from inside the container: live OpenStack and Infrahub calls both succeed. Also fixes Settings, which read the environment at class-definition time and so ignored anything set afterwards; a fresh Settings() silently returned stale values. Caught by the Jira scoping tests. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
114
backend/app/runpod/email_parse.py
Normal file
114
backend/app/runpod/email_parse.py
Normal file
@@ -0,0 +1,114 @@
|
||||
"""Parse RunPod's automated unlisting emails.
|
||||
|
||||
Subject: "<host> Unlisted - CRITICAL ERROR"
|
||||
Body: the machine id in brackets, an "Error detected:" block, and an
|
||||
"Impact: N GPU(s) currently rented" line.
|
||||
|
||||
The error block is the useful part - it is the only place the actual reason
|
||||
appears, and it is what CX pastes into the Zendesk ticket and the handover.
|
||||
Written to cope with the HTML mail as well as a plain-text paste.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import quopri
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
SUBJECT_RE = re.compile(r"^(?P<host>[\w.-]+)\s+Unlisted\b", re.I)
|
||||
MACHINE_RE = re.compile(r"machine\s+(?P<host>[\w.-]+)\s*\((?P<machine_id>[a-z0-9]{8,})\)", re.I)
|
||||
IMPACT_RE = re.compile(r"Impact:\s*(?P<gpus>\d+)\s*GPU", re.I)
|
||||
|
||||
ERROR_START = re.compile(r"Error detected:\s*", re.I)
|
||||
ERROR_END = re.compile(r"(Impact:|Common causes|Need to take the machine offline)", re.I)
|
||||
|
||||
# Recognisable failure signatures, mapped to what CX should do about them.
|
||||
SIGNATURES: list[tuple[str, str, str]] = [
|
||||
(r"memory_remap|uncorrectable remapped memory",
|
||||
"GPU memory remapping failure", "Hardware. Likely RMA - raise with the vendor."),
|
||||
(r"xid", "XID error", "Check dmesg and nvidia-smi; usually a burn-in test before relisting."),
|
||||
(r"gpu_cuda_ok.*expected 1, got 0|cuda initialization|fallen off the bus",
|
||||
"GPU not visible to CUDA", "Check nvidia-smi and the PCIe link; reboot then burn-in."),
|
||||
(r"nvidia-smi.*too many failures", "nvidia-smi failing repeatedly",
|
||||
"Host-level GPU fault - stress test, then RMA if it repeats."),
|
||||
(r"docker (service )?(unresponsive|hung)|container stuck",
|
||||
"Docker daemon unresponsive", "Restart docker; check /var/lib/docker is XFS and mounted."),
|
||||
(r"pod sync (failed|errors)", "Pod sync failing",
|
||||
"Check df -h for a hung moosefs mount and network to the moosefs cluster."),
|
||||
(r"portallocator|public port check fail", "Public port check failing",
|
||||
"Verify the publicIp ports in /etc/runpod/config.json are reachable."),
|
||||
(r"disk|filesystem|xfs", "Disk or filesystem error", "Check dmesg for disk failures."),
|
||||
]
|
||||
|
||||
|
||||
# Soft line breaks ("=\n") and "=3D" are the giveaways for a quoted-printable
|
||||
# body, which is how these mails arrive. Left undecoded, the error block comes
|
||||
# out full of "=3D" and split mid-word.
|
||||
_QP_HINT = re.compile(r"=\r?\n|=[0-9A-F]{2}")
|
||||
|
||||
|
||||
def _maybe_decode_qp(raw: str) -> str:
|
||||
if not _QP_HINT.search(raw):
|
||||
return raw
|
||||
try:
|
||||
return quopri.decodestring(raw.encode("utf-8", "replace")).decode("utf-8", "replace")
|
||||
except Exception:
|
||||
return raw
|
||||
|
||||
|
||||
def _text_from_html(raw: str) -> str:
|
||||
text = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", raw, flags=re.S | re.I)
|
||||
text = re.sub(r"<br\s*/?>|</(p|div|tr|li|h[1-6]|table)>", "\n", text, flags=re.I)
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
return html.unescape(text)
|
||||
|
||||
|
||||
def classify(error_text: str) -> dict[str, str]:
|
||||
"""Name the failure and say what it usually means."""
|
||||
blob = (error_text or "").lower()
|
||||
for pattern, label, action in SIGNATURES:
|
||||
if re.search(pattern, blob, re.I):
|
||||
return {"signature": label, "suggested_action": action}
|
||||
return {"signature": "Unrecognised error",
|
||||
"suggested_action": "Read the error text and escalate if it is not obvious."}
|
||||
|
||||
|
||||
def parse(raw: str, subject: str = "") -> dict[str, Any]:
|
||||
"""Pull the machine, the error block and the rented-GPU impact out of an email."""
|
||||
raw = _maybe_decode_qp(raw)
|
||||
text = _text_from_html(raw) if "<" in raw and ">" in raw else raw
|
||||
lines = [ln.strip() for ln in text.splitlines()]
|
||||
body = "\n".join(ln for ln in lines if ln)
|
||||
|
||||
host = ""
|
||||
machine_id = ""
|
||||
match = MACHINE_RE.search(body)
|
||||
if match:
|
||||
host, machine_id = match.group("host"), match.group("machine_id")
|
||||
if not host and subject:
|
||||
subject_match = SUBJECT_RE.match(subject.strip())
|
||||
if subject_match:
|
||||
host = subject_match.group("host")
|
||||
|
||||
error_text = ""
|
||||
start = ERROR_START.search(body)
|
||||
if start:
|
||||
rest = body[start.end():]
|
||||
end = ERROR_END.search(rest)
|
||||
error_text = (rest[:end.start()] if end else rest).strip()
|
||||
# The mail repeats its own body; one copy is enough.
|
||||
error_text = "\n".join(dict.fromkeys(ln for ln in error_text.splitlines() if ln.strip() != "---"))
|
||||
|
||||
gpus = None
|
||||
impact = IMPACT_RE.search(body)
|
||||
if impact:
|
||||
gpus = int(impact.group("gpus"))
|
||||
|
||||
return {
|
||||
"host": host,
|
||||
"machine_id": machine_id,
|
||||
"error_text": error_text.strip(),
|
||||
"gpus_rented": gpus,
|
||||
"is_critical": "critical" in (subject or body).lower(),
|
||||
**classify(error_text),
|
||||
}
|
||||
Reference in New Issue
Block a user