"""Parse RunPod's automated unlisting emails. Subject: " Unlisted - CRITICAL ERROR" Body: the machine id in brackets, an "Error detected:" block, and an "Impact: N GPU(s) currently rented" line. The error block is the useful part - it is the only place the actual reason appears, and it is what CX pastes into the Zendesk ticket and the handover. Written to cope with the HTML mail as well as a plain-text paste. """ from __future__ import annotations import html import quopri import re from typing import Any SUBJECT_RE = re.compile(r"^(?P[\w.-]+)\s+Unlisted\b", re.I) MACHINE_RE = re.compile(r"machine\s+(?P[\w.-]+)\s*\((?P[a-z0-9]{8,})\)", re.I) IMPACT_RE = re.compile(r"Impact:\s*(?P\d+)\s*GPU", re.I) ERROR_START = re.compile(r"Error detected:\s*", re.I) ERROR_END = re.compile(r"(Impact:|Common causes|Need to take the machine offline)", re.I) # Recognisable failure signatures, mapped to what CX should do about them. SIGNATURES: list[tuple[str, str, str]] = [ (r"memory_remap|uncorrectable remapped memory", "GPU memory remapping failure", "Hardware. Likely RMA - raise with the vendor."), (r"xid", "XID error", "Check dmesg and nvidia-smi; usually a burn-in test before relisting."), (r"gpu_cuda_ok.*expected 1, got 0|cuda initialization|fallen off the bus", "GPU not visible to CUDA", "Check nvidia-smi and the PCIe link; reboot then burn-in."), (r"nvidia-smi.*too many failures", "nvidia-smi failing repeatedly", "Host-level GPU fault - stress test, then RMA if it repeats."), (r"docker (service )?(unresponsive|hung)|container stuck", "Docker daemon unresponsive", "Restart docker; check /var/lib/docker is XFS and mounted."), (r"pod sync (failed|errors)", "Pod sync failing", "Check df -h for a hung moosefs mount and network to the moosefs cluster."), (r"portallocator|public port check fail", "Public port check failing", "Verify the publicIp ports in /etc/runpod/config.json are reachable."), (r"disk|filesystem|xfs", "Disk or filesystem error", "Check dmesg for disk failures."), ] # Soft line breaks ("=\n") and "=3D" are the giveaways for a quoted-printable # body, which is how these mails arrive. Left undecoded, the error block comes # out full of "=3D" and split mid-word. _QP_HINT = re.compile(r"=\r?\n|=[0-9A-F]{2}") def _maybe_decode_qp(raw: str) -> str: if not _QP_HINT.search(raw): return raw try: return quopri.decodestring(raw.encode("utf-8", "replace")).decode("utf-8", "replace") except Exception: return raw def _text_from_html(raw: str) -> str: text = re.sub(r"<(script|style)[^>]*>.*?", "", raw, flags=re.S | re.I) text = re.sub(r"|", "\n", text, flags=re.I) text = re.sub(r"<[^>]+>", "", text) return html.unescape(text) def classify(error_text: str) -> dict[str, str]: """Name the failure and say what it usually means.""" blob = (error_text or "").lower() for pattern, label, action in SIGNATURES: if re.search(pattern, blob, re.I): return {"signature": label, "suggested_action": action} return {"signature": "Unrecognised error", "suggested_action": "Read the error text and escalate if it is not obvious."} def parse(raw: str, subject: str = "") -> dict[str, Any]: """Pull the machine, the error block and the rented-GPU impact out of an email.""" raw = _maybe_decode_qp(raw) text = _text_from_html(raw) if "<" in raw and ">" in raw else raw lines = [ln.strip() for ln in text.splitlines()] body = "\n".join(ln for ln in lines if ln) host = "" machine_id = "" match = MACHINE_RE.search(body) if match: host, machine_id = match.group("host"), match.group("machine_id") if not host and subject: subject_match = SUBJECT_RE.match(subject.strip()) if subject_match: host = subject_match.group("host") error_text = "" start = ERROR_START.search(body) if start: rest = body[start.end():] end = ERROR_END.search(rest) error_text = (rest[:end.start()] if end else rest).strip() # The mail repeats its own body; one copy is enough. error_text = "\n".join(dict.fromkeys(ln for ln in error_text.splitlines() if ln.strip() != "---")) gpus = None impact = IMPACT_RE.search(body) if impact: gpus = int(impact.group("gpus")) return { "host": host, "machine_id": machine_id, "error_text": error_text.strip(), "gpus_rented": gpus, "is_critical": "critical" in (subject or body).lower(), **classify(error_text), }