Parham Monfared
8892144e0a
Handover - The Confluence shift doc becomes the landing page: shift metadata, the top-of-page checks, key updates with their Zendesk/Jira refs and status, and the free-text comments. "Hand over shift" closes the shift, opens the next one and carries the live items across, dropping anything done or marked "remove at end of shift" - the retyping this replaces. - The RunPod table on that page is read from live host state instead of being copied in by hand, with the six-colour key preserved. RunPod - GraphQL client keyed on CX_RUNPOD_API_KEY. The old console login is kept as a fallback but cannot run unattended: the account has 2FA, so Clerk verifies the password and then asks for an emailed code and never issues a session. That is the real cause of the "No active session found" failure, and the client now says so instead of failing opaquely. TOTP is supported if the account moves to an authenticator app. - Hosts and their listing history are persisted, so "most problematic hosts" can be ranked and each machine has a timeline of who listed or unlisted it, with the Zendesk comment and the error hint. - The unlisting emails are parsed for the error block (they arrive quoted-printable) and classified into a likely cause and a next step. Zendesk and Jira - Unlisting raises a Zendesk ticket that follows the format of RunPod's own email, keyed on the machine so one machine keeps one thread, posted as an internal note. - Jira is split in two: the Infrahub/OIE instance and the RunPod/RMA one, which may be a different Atlassian site. Blank RunPod values fall back to the defaults rather than failing. Running in a container - CX-Tools reads its keys from 1Password, which needs a desktop app. Config is a dataclass whose lookups live in per-field default factories, so passing CX_INFRAHUB_TOKEN/CX_INFRAINSIGHT_TOKEN in means those factories never run and CX-Tools itself stays unmodified. - CX-Tools reaches OpenStack with `docker exec <region>-osc`, so the image now carries the Docker client (the static binary, not the docker.io package) and compose mounts the host socket with group_add for it. Verified from inside the container: live OpenStack and Infrahub calls both succeed. Also fixes Settings, which read the environment at class-definition time and so ignored anything set afterwards; a fresh Settings() silently returned stale values. Caught by the Jira scoping tests. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
105 lines
4.9 KiB
Python
105 lines
4.9 KiB
Python
"""RunPod client modes, email parsing, ticket composition and Jira scoping."""
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
|
|
os.environ.setdefault("CX_DATABASE_URL", f"sqlite:///{tempfile.mkdtemp()}/rp.db")
|
|
|
|
FAILS = []
|
|
|
|
|
|
def expect(label, cond, got=""):
|
|
print((" PASS " if cond else " FAIL ") + label + ("" if cond else f" <- {got}"))
|
|
if not cond:
|
|
FAILS.append(label)
|
|
|
|
|
|
print("\nCLIENT MODES")
|
|
from app.runpod.client import RunPodClient, RunPodError
|
|
|
|
expect("api key wins over console credentials",
|
|
RunPodClient(api_key="k", email="e", password="p").mode == "api_key")
|
|
expect("falls back to console login", RunPodClient(email="e", password="p").mode == "console_login")
|
|
expect("reports unconfigured", RunPodClient().mode == "unconfigured")
|
|
try:
|
|
RunPodClient()._console_jwt()
|
|
expect("unconfigured client refuses to call out", False)
|
|
except RunPodError as exc:
|
|
expect("unconfigured client refuses to call out", "not configured" in str(exc).lower())
|
|
|
|
code = RunPodClient(totp_secret="JBSWY3DPEHPK3PXP")._totp_code()
|
|
expect("TOTP generator returns a 6-digit code", code.isdigit() and len(code) == 6, code)
|
|
|
|
print("\nEMAIL PARSING")
|
|
from app.runpod.email_parse import classify, parse
|
|
|
|
SAMPLE = """Your machine ca1-esc8-106 (x0gn8v2rthk4) hit a critical error and was automatically unlisted.
|
|
Error detected:
|
|
dcgm-xid-check: potential XID issue detected
|
|
gpu health check failed: metric gpu_cuda_ok: expected 1, got 0
|
|
error indicator present: gpu_failed{reason=memory_remap,uuid=GPU-d29f}
|
|
Impact: 8 GPU(s) currently rented on this machine.
|
|
Common causes and where to look:
|
|
"""
|
|
r = parse(SAMPLE, subject="ca1-esc8-106 Unlisted - CRITICAL ERROR")
|
|
expect("host extracted", r["host"] == "ca1-esc8-106", r["host"])
|
|
expect("machine id extracted", r["machine_id"] == "x0gn8v2rthk4", r["machine_id"])
|
|
expect("rented GPUs extracted", r["gpus_rented"] == 8, r["gpus_rented"])
|
|
expect("marked critical", r["is_critical"])
|
|
expect("error block captured, boilerplate excluded",
|
|
"dcgm-xid-check" in r["error_text"] and "Common causes" not in r["error_text"])
|
|
expect("memory remap recognised as hardware",
|
|
r["signature"] == "GPU memory remapping failure", r["signature"])
|
|
|
|
expect("quoted-printable is decoded",
|
|
"=3D" not in parse("Error detected:\ngpu_failed{a=3Db}\nImpact: 1 GPU(s)")["error_text"])
|
|
expect("subject-only still yields a host",
|
|
parse("no machine line here", subject="no1-os1-5090-050 Unlisted")["host"] == "no1-os1-5090-050")
|
|
for text, want in [("nvidia-smi: too many failures", "nvidia-smi failing repeatedly"),
|
|
("pod sync failed 16 times", "Pod sync failing"),
|
|
("container stuck: docker service unresponsive", "Docker daemon unresponsive"),
|
|
("total gibberish", "Unrecognised error")]:
|
|
expect(f"classify: {text[:34]}", classify(text)["signature"] == want, classify(text)["signature"])
|
|
|
|
print("\nTICKET COMPOSITION")
|
|
from app.models import RunpodHost
|
|
from app.runpod.delivery import unlisting_ticket
|
|
|
|
host = RunpodHost(machine_id="x0gn8v2rthk4", name="ca1-esc8-106", gpu_reserved=8, gpu_total=8,
|
|
unlist_count=7, data_center="CA1",
|
|
last_error="dcgm-xid-check: potential XID issue detected\nreason=memory_remap")
|
|
t = unlisting_ticket(host)
|
|
expect("subject names the host and the fault",
|
|
"ca1-esc8-106" in t["subject"] and "memory" in t["subject"].lower(), t["subject"])
|
|
expect("body carries the error text", "dcgm-xid-check" in t["body"])
|
|
expect("body states the rented impact", "8 GPU(s) currently rented" in t["body"])
|
|
expect("body carries the repeat count", "7 time(s)" in t["body"])
|
|
|
|
drained = RunpodHost(machine_id="m", name="h", gpu_reserved=0, gpu_total=8, unlist_count=1)
|
|
expect("a drained machine says so", "drained" in unlisting_ticket(drained)["body"])
|
|
|
|
print("\nJIRA SCOPING")
|
|
from app.config import Settings
|
|
|
|
os.environ.update({"CX_JIRA_BASE": "https://oie.atlassian.net", "CX_JIRA_EMAIL": "oie@x",
|
|
"CX_JIRA_TOKEN": "t1", "CX_JIRA_PROJECT": "OIE"})
|
|
s = Settings()
|
|
expect("default scope uses the OIE instance", s.jira_for("default")["base"] == "https://oie.atlassian.net")
|
|
expect("runpod inherits when unset", s.jira_for("runpod")["base"] == "https://oie.atlassian.net")
|
|
expect("runpod keeps its own project", s.jira_for("runpod")["project"] == "RMA")
|
|
|
|
os.environ.update({"CX_RUNPOD_JIRA_BASE": "https://rma.atlassian.net",
|
|
"CX_RUNPOD_JIRA_EMAIL": "rma@x", "CX_RUNPOD_JIRA_TOKEN": "t2"})
|
|
s = Settings()
|
|
expect("runpod uses its own instance when given one",
|
|
s.jira_for("runpod")["base"] == "https://rma.atlassian.net")
|
|
expect("the two scopes stay separate",
|
|
s.jira_for("runpod")["token"] != s.jira_for("default")["token"])
|
|
expect("default is untouched by the runpod values",
|
|
s.jira_for("default")["base"] == "https://oie.atlassian.net")
|
|
|
|
print("\n" + ("ALL CHECKS PASSED" if not FAILS else f"{len(FAILS)} FAILED: {FAILS}"))
|
|
sys.exit(1 if FAILS else 0)
|