"""Outbound actions for RunPod hosts: Zendesk notification and Jira/RMA escalation. An unlisted machine earns nothing and may be stranding rented workloads, so the team wants a ticket the moment it happens. RunPod's own email already says what broke; this reproduces that content in Zendesk so it lands in the tracking platform even when the unlisting was done through the API and no email was sent. The same three gates as customer comms apply - configured, feature-flagged, and sending enabled globally - so a demo instance cannot raise tickets. """ from __future__ import annotations import datetime as dt from typing import Any, Optional import httpx from sqlalchemy.orm import Session from ..config import get_settings from ..delivery import DeliveryError, _check_cap from ..models import RunpodEventType, RunpodHost, User from .email_parse import classify from .service import record_event settings = get_settings() def unlisting_ticket(host: RunpodHost, error_text: str = "") -> dict[str, str]: """Compose the ticket body, following the shape of RunPod's own email. Keeping the same structure means whoever picks the ticket up reads the familiar thing: what broke, how many GPUs it is costing, and where to look. """ error = (error_text or host.last_error or "").strip() hint = classify(error) rented = f"{host.gpu_reserved} GPU(s) currently rented on this machine." if host.gpu_reserved \ else "No GPUs currently rented - the machine has drained." body = "\n".join([ f"Machine {host.name} ({host.machine_id}) is unlisted and is not accepting new users.", "", "Error detected:", error or "(no error text captured - check the RunPod dashboard or the notification email)", "", f"Impact: {rented}", f"Likely cause: {hint['signature']}", f"Suggested next step: {hint['suggested_action']}", "", "Common causes and where to look:", "- GPU error or failure: check nvidia-smi for GPU health and dmesg for Xid errors.", "- Unresponsive Docker daemon: check whether the docker service is running or hung.", "- Pod sync errors: if df -h hangs, suspect a disk error or hung moosefs mount.", "- Docker overlay storage: confirm the docker filesystem is XFS and /var/lib/docker is mounted.", "- Portallocator port check: verify the publicIp ports in /etc/runpod/config.json are reachable.", "", "If the fix needs a reboot or hardware work, schedule maintenance from the Machines " "Dashboard rather than pulling the machine abruptly - that drains workloads gracefully.", "", f"Unlisted {host.unlist_count} time(s) to date.", "Raised automatically by CX Triage.", ]) return { "subject": f"{host.name} Unlisted - {hint['signature']}", "body": body, "signature": hint["signature"], } async def raise_unlisting_ticket(db: Session, host: RunpodHost, actor: User, *, error_text: str = "", requester: str = "", subject: str = "", body: str = "") -> dict[str, Any]: """Create (or comment on) the Zendesk ticket for an unlisted machine.""" if not settings.feature_send_enabled: raise DeliveryError("Sending is disabled on this instance (CX_FEATURE_SEND_ENABLED is off).") if not settings.zendesk_ready: raise DeliveryError("Zendesk is not configured. Set CX_FEATURE_ZENDESK plus " "CX_ZENDESK_SUBDOMAIN, CX_ZENDESK_EMAIL and CX_ZENDESK_TOKEN.") _check_cap(db) composed = unlisting_ticket(host, error_text) subject = subject or composed["subject"] body = body or composed["body"] base = f"https://{settings.zendesk_subdomain}.zendesk.com/api/v2" auth = (f"{settings.zendesk_email}/token", settings.zendesk_token) # Keyed on the machine, not the incident: one machine, one running thread. external_id = f"cx-triage-runpod-{host.machine_id}" async with httpx.AsyncClient(timeout=30) as client: found = await client.get(f"{base}/search.json", params={"query": f'type:ticket external_id:"{external_id}"'}, auth=auth) existing = None if found.status_code == 200: results = found.json().get("results") or [] # Only reuse a ticket that is still open; a solved one starts a new thread. existing = next((t for t in results if t.get("status") not in ("solved", "closed")), None) # Internal note by default: this is an operations ticket, not a customer reply. comment = {"body": body, "public": False} if existing: resp = await client.put(f"{base}/tickets/{existing['id']}.json", json={"ticket": {"comment": comment}}, auth=auth) action = "updated" else: ticket: dict[str, Any] = { "subject": subject, "comment": comment, "priority": "high" if host.gpu_reserved else "normal", "type": "incident", "tags": ["cx-triage", "runpod", "unlisted", f"dc-{(host.data_center or 'unknown').lower()}"], "external_id": external_id, } if requester: ticket["requester"] = {"name": requester.split("@")[0], "email": requester} resp = await client.post(f"{base}/tickets.json", json={"ticket": ticket}, auth=auth) action = "created" if resp.status_code not in (200, 201): record_event(db, host, RunpodEventType.NOTE, actor=actor.email, detail=f"Zendesk ticket failed: HTTP {resp.status_code}") db.commit() raise DeliveryError(f"Zendesk returned {resp.status_code}: {resp.text[:300]}") ticket_id = str((resp.json().get("ticket") or {}).get("id") or (existing or {}).get("id") or "") url = f"https://{settings.zendesk_subdomain}.zendesk.com/agent/tickets/{ticket_id}" host.zendesk_ticket = ticket_id or host.zendesk_ticket record_event(db, host, RunpodEventType.ZENDESK_TICKET, actor=actor.email, detail=f"Zendesk ticket {ticket_id} {action} - {composed['signature']}", error_hint=error_text or host.last_error, zendesk_ticket=ticket_id) db.commit() return {"ok": True, "ticket_id": ticket_id, "url": url, "action": action, "subject": subject, "signature": composed["signature"]} async def raise_rma_issue(db: Session, host: RunpodHost, actor: User, *, summary: str = "", description: str = "") -> dict[str, Any]: """Open an issue on the RunPod/RMA Jira - a different instance to the OIE one.""" creds = settings.jira_for("runpod") if not settings.feature_send_enabled: raise DeliveryError("Sending is disabled on this instance (CX_FEATURE_SEND_ENABLED is off).") if not (settings.feature_jira and creds["base"] and creds["email"] and creds["token"]): raise DeliveryError( "The RunPod Jira is not configured. Set CX_RUNPOD_JIRA_BASE, CX_RUNPOD_JIRA_EMAIL and " "CX_RUNPOD_JIRA_TOKEN - or leave them blank to reuse the default instance." ) _check_cap(db) hint = classify(host.last_error or "") summary = summary or f"RunPod host {host.name} ({host.machine_id}) - {hint['signature']}" description = description or "\n".join([ f"Machine: {host.name} ({host.machine_id})", f"Data centre: {host.data_center or 'unknown'}", f"GPUs: {host.gpu_reserved}/{host.gpu_total} rented", f"Unlisted {host.unlist_count} time(s) to date.", "", "Last error:", host.last_error or "(none captured)", "", f"Likely cause: {hint['signature']}", f"Suggested next step: {hint['suggested_action']}", "", f"Zendesk: {host.zendesk_ticket or '(none)'}", "Raised from CX Triage.", ]) base = creds["base"].rstrip("/") auth = (creds["email"], creds["token"]) label = f"cx-triage-runpod-{host.machine_id}" async with httpx.AsyncClient(timeout=30) as client: found = await client.get(f"{base}/rest/api/3/search", params={"jql": f'labels = "{label}"', "maxResults": 1}, auth=auth) if found.status_code == 200 and (found.json().get("issues") or []): issue = found.json()["issues"][0] key = issue["key"] host.jira_key = key record_event(db, host, RunpodEventType.JIRA_LINKED, actor=actor.email, detail=f"Issue {key} already exists", jira_key=key) db.commit() return {"ok": True, "key": key, "url": f"{base}/browse/{key}", "action": "existing"} resp = await client.post(f"{base}/rest/api/3/issue", json={"fields": { "project": {"key": creds["project"]}, "summary": summary[:250], "issuetype": {"name": creds["issue_type"]}, "labels": ["cx-triage", "runpod", label], "description": {"type": "doc", "version": 1, "content": [ {"type": "paragraph", "content": [{"type": "text", "text": description[:30000]}]}]}, }}, auth=auth) if resp.status_code not in (200, 201): raise DeliveryError(f"Jira returned {resp.status_code}: {resp.text[:300]}") key = resp.json().get("key", "") host.jira_key = key record_event(db, host, RunpodEventType.JIRA_LINKED, actor=actor.email, detail=f"Issue {key} created on the RunPod Jira", jira_key=key) db.commit() return {"ok": True, "key": key, "url": f"{base}/browse/{key}", "action": "created"}