"""Seed data drawn from the real exports, so a fresh stack is worth looking at. The handover comes from APAC 06 August 2026; the RunPod hosts come from the monitoring script's own JSON. Only enough is transcribed to make the pages representative - it is sample data, not a migration. """ from __future__ import annotations HANDOVER = { "shift_date": "2026-08-06", "shift": "APAC", "handing_to": "EMEA", "team_members": "Parham", "significant_issues_checked": True, "hubspot_checked": True, "total_open_tickets": 214, "member_checks": [ {"name": "Prasad", "hs_checked": False, "jira_checked": False}, {"name": "Parham", "hs_checked": True, "jira_checked": True}, ], "other_comments": ( "Weekend Coverage Annual Leave Sheet is on SharePoint.\n\n" "Confluence Sanity Check: going through current docs to see what needs updating " "and removing.\n\n" "Kodekloud Training: more team time needed on Kubernetes CKNA. Team leads are " "working on a schedule.\n\n" "New runbook WIP to protect customers and infrastructure from suspicious activity - " "locking a VM and, if required, detaching a public IP allocation.\n\n" "For any Windmill flows that require a ticket reference, use the numeric number and " "not the URL for now.\n\n" "Check the Zendesk sync tool - some Hyperstack orgs are not showing in Zendesk even " "though they exist in InfraInsight (e.g. org 29084, 29548)." ), "items": [ { "title": "Request to detach volume from hibernated VM", "zendesk_tickets": "#8495", "state": "no_further_engagement", "remove_at_end_of_shift": True, "body": ("Details confirmed. Ran the Windmill flow but hit an error on API keys. " "Volume has since been detached successfully and the customer has been informed."), "links": ["https://windmill.ngbackend.cloud/run/019fd28c-662b-3b56-199a-6af73bb1993c"], }, { "title": "Baseten node-level failure on b200-worker-frank-quetzal", "zendesk_tickets": "#8453", "state": "pending_infra", "body": ("Aranya reached out on behalf of Baseten about a node-level failure " "(internal IP 10.32.10.2). Triage provided three times; the first two uploads " "showed no issues. Customer still reports the node is wedged. Waiting on the " "third upload for review. Asked Infra whether the nodes just need a reboot."), "links": ["https://nexgen-cloud.slack.com/archives/C0AP4KV1XT5/p1785827125227079"], }, { "title": "[NO1] Investigate inbound traffic spike", "zendesk_tickets": "8238", "jira_key": "OIE-3213", "jira_status": "Complete", "state": "pending_customer", "body": ("CARMA Media Insight (org 3429) reported a transatlantic throughput collapse - " "EU fine, North America down to ~45-141 KB/s. Kheano traced it to edge-level " "congestion in NO1 affecting all tenants, top talker FIP 149.36.0.199 showing " "signs of a flood attack. Infra have confirmed WAN utilisation has normalised."), "links": ["https://nexgen-cloud.slack.com/archives/C049Q9JRGM7/p1785526451626789"], }, { "title": "[EU1-BM2] Reported network latency investigation", "zendesk_tickets": "#8409", "jira_key": "OIE-3185", "jira_status": "In Progress", "state": "pending_infra", "body": "Baseten experienced a latency spike on 31 July and have requested investigation.", "links": ["https://nexgen-cloud.slack.com/archives/C0AP4KV1XT5/p1784855398156009"], }, { "title": "Network storage speed", "zendesk_tickets": "#8207", "jira_key": "OIE-3210", "jira_status": "In Progress", "state": "pending_infra", "body": ("RunPod advised one of their machines is reporting network slowness to their " "storage. Confirmed against another machine in the same region using the same " "storage by creating files of random data."), "links": ["https://nexgen-cloud.slack.com/archives/C049Q9JRGM7/p1785451129222409"], }, { "title": "[EU1-BM2] GPU issues on eu1-bm2-lv1-b200sxm-0(16|55)", "zendesk_tickets": "#7772", "jira_key": "OIE-3177", "jira_status": "Pending", "state": "pending_infra", "body": ("Baseten requested support on 2 nodes with XID errors. One looks transient; " "eu1-bm2-lv1-b200sxm-055 is more concerning with UECC. Raised with Lenovo for " "next steps."), "links": ["https://nexgen-cloud.slack.com/archives/C0AP4KV1XT5/p1784749263687239"], }, { "title": "Shadeform locked VM - billing adjustment", "zendesk_tickets": "#8215", "state": "no_further_engagement", "remove_at_end_of_shift": True, "body": ("DevOps raised a ticket to notify the user their VM was locked (945280 / " "O6QTQyQgqDRN, Shadeform) due to unusual activity. Customer asked for it to be " "deleted; we unlocked and they deleted it. Locked for 5 days 3 hours 21 minutes " "= 7401 minutes = 123.35 hours at $1.6756/hr. Total credit $206.69 to come off " "the next invoice, since the state was never changed to shutoff/locked."), }, { "title": "185.216.20.188 network slowness - CA1-SRV-CPU6", "zendesk_tickets": "#7793", "jira_key": "OIE-3189", "jira_status": "Done", "state": "pending_customer", "body": ("Customer reported intermittent network instability on VM 298953 " "(CA1-SRV-CPU6, 185.216.20.188). The problematic VM 924706 / JqXcTaGOt has been " "siloed by Infra and the routers investigated. Network has been stable for the " "past couple of days."), }, { "title": "VM stuck - CA1-ESC8-068 host not reachable", "zendesk_tickets": "#8078, #8076, #7787, #8077", "jira_key": "OIE-3183", "jira_status": "In Progress", "state": "pending_customer", "body": ("Host maintenance scheduled 06 August 2026 at 00:00 UTC. Communication sent to " "the customer. Four VMs still on the host and reachable: vm-regA (786651), " "dubbix_instance_1 (737376), gianpaolo-dev-2 (602721), triton-backup-2 (213340). " "Follow-up emails sent. Pending customer reply on the maintenance window."), }, ], } # The colour key from the handover doc, so the UI can explain itself. RUNPOD_COLOURS = { "red": "Blocked from being relisted due to recurring issues. Needs in-depth investigation or is tied to a system issue.", "purple": "Pending RunPod. We are waiting on RunPod for something.", "yellow": "Unactionable by CX. Pending investigation or remediation from Infrastructure or the DC team.", "blue": "Unactionable by CX. Pending removal from the RunPod platform.", "green": "Stress testing for >24 hours. GPUs may appear reserved.", "white": "Actionable by CX.", } # The unlisted table from the same handover, with the states CX had recorded. RUNPOD_BOARD = [ {"name": "no1-os1-5090-016-contract-001", "machine_id": "hedh664udd3g", "zendesk_ticket": "8405", "colour": "green", "last_error": "std burn in failed on 03 Aug", "next_steps": "24 Hr default burn-in test in progress. 05 Aug 2026 15:06 UTC (16:06 BST)"}, {"name": "no1-os1-5090-099-contract-001", "machine_id": "9bpfxa1o9our", "zendesk_ticket": "6279", "colour": "red", "jira_key": "RMA-103", "jira_status": "Waiting", "last_error": "nvidia-smi: too many failures. Raised for RMA", "next_steps": "24 HR burn-in started 05 Aug 2026 18:38 UTC. Luis asked for multiple 24hr stress tests with full logs in Jira."}, {"name": "no1-os1-5090-027-contract-001", "machine_id": "seed-5090-027", "zendesk_ticket": "#8385", "colour": "green", "last_error": "nvidia-smi many failures. First burn-in failed so running again", "next_steps": "24HR burn-in running, started 05 Aug 2026 20:15 UTC (21:15 BST)"}, {"name": "no1-os1-5090-050", "machine_id": "gpc1q3vboxfo", "zendesk_ticket": "#8497", "colour": "green", "last_error": "dcgm-xid-check: potential XID issue detected\ngpu health check failed: metric gpu_cuda_ok: expected 1, got 0", "next_steps": "24HR burn-in running, started 05 Aug 2026 17:28 UTC (18:28 BST)"}, {"name": "no1-os1-4090-009", "machine_id": "f2vwayvyxorv", "zendesk_ticket": "#7342", "colour": "yellow", "last_error": "nvidia-smi: too many failures. 2/8 GPU in use", "next_steps": "Maintenance scheduled 4 Aug 2026 19:00 UTC. Host has connectivity issues - may need escalating."}, {"name": "no1-os1-4090-020", "machine_id": "s7kl180cusjr", "zendesk_ticket": "#7359", "colour": "yellow", "last_error": "pod sync failed 16 times. 2/8 GPU in use", "next_steps": "Maintenance scheduled 04 Aug 2026 13:02 UTC for 1 day"}, {"name": "ca1-esc8-106", "machine_id": "x0gn8v2rthk4", "zendesk_ticket": "419195749569", "colour": "red", "jira_status": "Under Test", "last_error": ("dcgm-xid-check: potential XID issue detected\n" "gpu health check failed: error indicator present: " "gpu_failed{reason=memory_remap,uuid=GPU-d29f591c-63e9-5f1d-add1-ed0e2dad3660}"), "next_steps": "Repeated offender - does not stay listed for more than 48 hours. Booked maintenance 17 Jun 2026 14:52 BST."}, {"name": "ca1-esc8-121", "machine_id": "seed-esc8-121", "zendesk_ticket": "419270908096", "colour": "yellow", "jira_status": "Maintenance Scheduled", "last_error": ("dcgm-xid-check: potential XID issue detected\n" "gpu health check failed: metric gpu_cuda_ok: expected 1, got 0"), "next_steps": "3/8 in use. Maintenance 18 Jun 2026 19:00 UTC (20:00 BST)"}, {"name": "no1-os1-4090-013", "machine_id": "seed-4090-013", "zendesk_ticket": "418292796638", "colour": "blue", "jira_key": "RMA-60", "jira_status": "Pending RMA", "last_error": "container stuck: docker service unresponsive", "next_steps": "Reboot failed, received timeout error. Pending RMA."}, ]