Add shift handover and RunPod, and make CX-Tools work in a container

Handover
- The Confluence shift doc becomes the landing page: shift metadata, the
  top-of-page checks, key updates with their Zendesk/Jira refs and status, and
  the free-text comments. "Hand over shift" closes the shift, opens the next
  one and carries the live items across, dropping anything done or marked
  "remove at end of shift" - the retyping this replaces.
- The RunPod table on that page is read from live host state instead of being
  copied in by hand, with the six-colour key preserved.

RunPod
- GraphQL client keyed on CX_RUNPOD_API_KEY. The old console login is kept as a
  fallback but cannot run unattended: the account has 2FA, so Clerk verifies the
  password and then asks for an emailed code and never issues a session. That is
  the real cause of the "No active session found" failure, and the client now
  says so instead of failing opaquely. TOTP is supported if the account moves to
  an authenticator app.
- Hosts and their listing history are persisted, so "most problematic hosts" can
  be ranked and each machine has a timeline of who listed or unlisted it, with
  the Zendesk comment and the error hint.
- The unlisting emails are parsed for the error block (they arrive
  quoted-printable) and classified into a likely cause and a next step.

Zendesk and Jira
- Unlisting raises a Zendesk ticket that follows the format of RunPod's own
  email, keyed on the machine so one machine keeps one thread, posted as an
  internal note.
- Jira is split in two: the Infrahub/OIE instance and the RunPod/RMA one, which
  may be a different Atlassian site. Blank RunPod values fall back to the
  defaults rather than failing.

Running in a container
- CX-Tools reads its keys from 1Password, which needs a desktop app. Config is a
  dataclass whose lookups live in per-field default factories, so passing
  CX_INFRAHUB_TOKEN/CX_INFRAINSIGHT_TOKEN in means those factories never run and
  CX-Tools itself stays unmodified.
- CX-Tools reaches OpenStack with `docker exec <region>-osc`, so the image now
  carries the Docker client (the static binary, not the docker.io package) and
  compose mounts the host socket with group_add for it. Verified from inside the
  container: live OpenStack and Infrahub calls both succeed.

Also fixes Settings, which read the environment at class-definition time and so
ignored anything set afterwards; a fresh Settings() silently returned stale
values. Caught by the Jira scoping tests.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/app/routers/handover_router.py

@@ -0,0 +1,222 @@
+"""Shift handover: the document CX fills in at the end of every shift."""
+from __future__ import annotations
+
+import datetime as dt
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from pydantic import BaseModel
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from ..auth import current_user
+from ..db import get_db
+from ..models import (Handover, HandoverItem, HandoverStatus, ItemState, RunpodColour,
+                      RunpodHost, ShiftName, User)
+
+router = APIRouter(prefix="/api/handover", tags=["handover"])
+
+
+class HandoverBody(BaseModel):
+    shift_date: str | None = None
+    shift: str = "APAC"
+    handing_to: str = ""
+    team_members: str = ""
+    significant_issues_checked: bool = False
+    hubspot_checked: bool = False
+    total_open_tickets: int | None = None
+    member_checks: list[dict] = []
+    other_comments: str = ""
+    reviewed_by: str = ""
+    reviewed_at_utc: str = ""
+    following_shift_checked: bool = False
+
+
+class ItemBody(BaseModel):
+    title: str = ""
+    zendesk_tickets: str = ""
+    jira_key: str = ""
+    jira_status: str = ""
+    body: str = ""
+    links: list[str] = []
+    state: str = "in_progress"
+    remove_at_end_of_shift: bool = False
+    position: int | None = None
+
+
+def _get(db: Session, handover_id: int) -> Handover:
+    found = db.get(Handover, handover_id)
+    if not found:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "No such handover")
+    return found
+
+
+@router.get("")
+def list_handovers(limit: int = 30, db: Session = Depends(get_db),
+                   user: User = Depends(current_user)):
+    rows = db.scalars(select(Handover).order_by(Handover.shift_date.desc(), Handover.id.desc())
+                      .limit(max(1, min(limit, 200)))).all()
+    return {
+        "handovers": [h.to_json() for h in rows],
+        "shifts": [s.value for s in ShiftName],
+        "states": [s.value for s in ItemState],
+    }
+
+
+@router.get("/current")
+def current(db: Session = Depends(get_db), user: User = Depends(current_user)):
+    """The newest draft, or the newest handover of any kind."""
+    row = db.scalars(select(Handover).where(Handover.status == HandoverStatus.DRAFT)
+                     .order_by(Handover.shift_date.desc(), Handover.id.desc()).limit(1)).first()
+    if row is None:
+        row = db.scalars(select(Handover).order_by(Handover.shift_date.desc(),
+                                                   Handover.id.desc()).limit(1)).first()
+    if row is None:
+        return {"handover": None}
+    return {"handover": row.to_json(with_items=True), "runpod": _runpod_section(db)}
+
+
+@router.get("/{handover_id}")
+def get_one(handover_id: int, db: Session = Depends(get_db), user: User = Depends(current_user)):
+    return {"handover": _get(db, handover_id).to_json(with_items=True), "runpod": _runpod_section(db)}
+
+
+def _runpod_section(db: Session) -> list[dict]:
+    """The RunPod table on the handover, straight from live host state.
+
+    This is the part that used to be retyped by hand every shift.
+    """
+    rows = db.scalars(select(RunpodHost).where(RunpodHost.listed.is_(False))
+                      .order_by(RunpodHost.unlist_count.desc(), RunpodHost.name)).all()
+    return [{
+        "machine_id": h.machine_id, "name": h.name, "colour": h.colour.value,
+        "zendesk_ticket": h.zendesk_ticket, "jira_key": h.jira_key, "jira_status": h.jira_status,
+        "gpu_reserved": h.gpu_reserved, "gpu_total": h.gpu_total,
+        "last_error": h.last_error, "next_steps": h.next_steps,
+        "hours_unlisted": h.hours_unlisted, "unlist_count": h.unlist_count,
+    } for h in rows]
+
+
+@router.post("")
+def create(body: HandoverBody, db: Session = Depends(get_db), user: User = Depends(current_user)):
+    shift_date = dt.date.fromisoformat(body.shift_date) if body.shift_date else dt.date.today()
+    try:
+        shift = ShiftName(body.shift)
+    except ValueError as exc:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Unknown shift '{body.shift}'") from exc
+
+    existing = db.scalars(select(Handover).where(Handover.shift_date == shift_date,
+                                                 Handover.shift == shift)).first()
+    if existing:
+        raise HTTPException(status.HTTP_409_CONFLICT,
+                            f"A {shift.value} handover already exists for {shift_date}")
+
+    row = Handover(shift_date=shift_date, shift=shift, handing_to=body.handing_to,
+                   team_members=body.team_members, created_by_id=user.id)
+    db.add(row)
+    db.commit()
+    return row.to_json(with_items=True)
+
+
+@router.post("/{handover_id}")
+def update(handover_id: int, body: HandoverBody, db: Session = Depends(get_db),
+           user: User = Depends(current_user)):
+    row = _get(db, handover_id)
+    for field in ("handing_to", "team_members", "significant_issues_checked", "hubspot_checked",
+                  "total_open_tickets", "member_checks", "other_comments", "reviewed_by",
+                  "reviewed_at_utc", "following_shift_checked"):
+        setattr(row, field, getattr(body, field))
+    if body.shift_date:
+        row.shift_date = dt.date.fromisoformat(body.shift_date)
+    db.commit()
+    return row.to_json(with_items=True)
+
+
+@router.post("/{handover_id}/hand-over")
+def hand_over(handover_id: int, db: Session = Depends(get_db), user: User = Depends(current_user)):
+    """Close the shift and start the next one, carrying the live items forward.
+
+    Items flagged "remove at end of shift", and anything already done, are left
+    behind - which is the manual step this replaces.
+    """
+    row = _get(db, handover_id)
+    row.status = HandoverStatus.HANDED_OVER
+
+    order = [ShiftName.APAC, ShiftName.EMEA, ShiftName.AMER]
+    next_shift = order[(order.index(row.shift) + 1) % len(order)]
+    next_date = row.shift_date + dt.timedelta(days=1) if next_shift == ShiftName.APAC else row.shift_date
+
+    following = db.scalars(select(Handover).where(Handover.shift_date == next_date,
+                                                  Handover.shift == next_shift)).first()
+    if following is None:
+        following = Handover(shift_date=next_date, shift=next_shift,
+                             handing_to=order[(order.index(next_shift) + 1) % len(order)].value,
+                             created_by_id=user.id)
+        db.add(following)
+        db.flush()
+
+    carried = 0
+    closed = {ItemState.DONE, ItemState.NO_FURTHER_ENGAGEMENT}
+    for item in row.items:
+        if item.remove_at_end_of_shift or item.state in closed:
+            continue
+        following.items.append(HandoverItem(
+            position=item.position, title=item.title, zendesk_tickets=item.zendesk_tickets,
+            jira_key=item.jira_key, jira_status=item.jira_status, body=item.body,
+            links=list(item.links or []), state=item.state,
+            carried_from_id=item.id,
+            first_raised_on=item.first_raised_on or row.shift_date,
+        ))
+        carried += 1
+
+    db.commit()
+    return {"closed": row.to_json(), "next": following.to_json(with_items=True), "carried": carried}
+
+
+@router.post("/{handover_id}/items")
+def add_item(handover_id: int, body: ItemBody, db: Session = Depends(get_db),
+             user: User = Depends(current_user)):
+    row = _get(db, handover_id)
+    try:
+        state = ItemState(body.state)
+    except ValueError as exc:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Unknown state '{body.state}'") from exc
+    item = HandoverItem(
+        position=body.position if body.position is not None else len(row.items),
+        title=body.title, zendesk_tickets=body.zendesk_tickets, jira_key=body.jira_key,
+        jira_status=body.jira_status, body=body.body, links=body.links, state=state,
+        remove_at_end_of_shift=body.remove_at_end_of_shift, first_raised_on=row.shift_date,
+    )
+    row.items.append(item)
+    db.commit()
+    return row.to_json(with_items=True)
+
+
+@router.post("/{handover_id}/items/{item_id}")
+def update_item(handover_id: int, item_id: int, body: ItemBody, db: Session = Depends(get_db),
+                user: User = Depends(current_user)):
+    row = _get(db, handover_id)
+    item = next((i for i in row.items if i.id == item_id), None)
+    if item is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "No such item on this handover")
+    try:
+        item.state = ItemState(body.state)
+    except ValueError as exc:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Unknown state '{body.state}'") from exc
+    for field in ("title", "zendesk_tickets", "jira_key", "jira_status", "body", "links",
+                  "remove_at_end_of_shift"):
+        setattr(item, field, getattr(body, field))
+    if body.position is not None:
+        item.position = body.position
+    db.commit()
+    return row.to_json(with_items=True)
+
+
+@router.delete("/{handover_id}/items/{item_id}")
+def delete_item(handover_id: int, item_id: int, db: Session = Depends(get_db),
+                user: User = Depends(current_user)):
+    row = _get(db, handover_id)
+    item = next((i for i in row.items if i.id == item_id), None)
+    if item is not None:
+        db.delete(item)
+        db.commit()
+    return _get(db, handover_id).to_json(with_items=True)

backend/app/routers/runpod_router.py

@@ -0,0 +1,262 @@
+"""RunPod machines: current state, per-host history, and problem ranking."""
+from __future__ import annotations
+
+import datetime as dt
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from pydantic import BaseModel
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from ..auth import current_user
+from ..config import get_settings
+from ..db import get_db
+from ..models import RunpodColour, RunpodEvent, RunpodEventType, RunpodHost, User
+from ..runpod.client import RunPodClient, RunPodError
+from ..runpod.email_parse import parse as parse_email
+from ..runpod.delivery import raise_rma_issue, raise_unlisting_ticket, unlisting_ticket
+from ..runpod.service import record_event, sync_machines
+from ..delivery import DeliveryError
+
+router = APIRouter(prefix="/api/runpod", tags=["runpod"])
+settings = get_settings()
+
+
+def _client() -> RunPodClient:
+    return RunPodClient(
+        api_key=settings.runpod_api_key, email=settings.runpod_email,
+        password=settings.runpod_password, team_id=settings.runpod_team_id,
+        totp_secret=settings.runpod_totp_secret,
+    )
+
+
+class HostPatch(BaseModel):
+    colour: str | None = None
+    zendesk_ticket: str | None = None
+    jira_key: str | None = None
+    jira_status: str | None = None
+    next_steps: str | None = None
+    last_error: str | None = None
+
+
+class NoteBody(BaseModel):
+    detail: str
+    event_type: str = "note"
+    zendesk_ticket: str = ""
+    jira_key: str = ""
+
+
+class EmailBody(BaseModel):
+    raw: str
+    subject: str = ""
+
+
+class TicketBody(BaseModel):
+    error_text: str = ""
+    requester: str = ""
+    subject: str = ""
+    body: str = ""
+
+
+class RmaBody(BaseModel):
+    summary: str = ""
+    description: str = ""
+
+
+@router.get("/status")
+def runpod_status(db: Session = Depends(get_db), user: User = Depends(current_user)):
+    hosts = db.scalars(select(RunpodHost)).all()
+    unlisted = [h for h in hosts if not h.listed]
+    return {
+        "configured": settings.runpod_ready,
+        "mode": _client().mode,
+        "write_enabled": settings.feature_runpod_write,
+        "totals": {
+            "machines": len(hosts),
+            "listed": len(hosts) - len(unlisted),
+            "unlisted": len(unlisted),
+            "gpus_rented": sum(h.gpu_reserved for h in hosts),
+            "gpus_total": sum(h.gpu_total for h in hosts),
+        },
+        "colours": [c.value for c in RunpodColour],
+        "event_types": [e.value for e in RunpodEventType],
+    }
+
+
+@router.get("/hosts")
+def hosts(unlisted_only: bool = False, db: Session = Depends(get_db),
+          user: User = Depends(current_user)):
+    query = select(RunpodHost)
+    if unlisted_only:
+        query = query.where(RunpodHost.listed.is_(False))
+    rows = db.scalars(query.order_by(RunpodHost.listed, RunpodHost.name)).all()
+    return {"hosts": [h.to_json() for h in rows]}
+
+
+@router.get("/hosts/{machine_id}")
+def host_detail(machine_id: str, db: Session = Depends(get_db), user: User = Depends(current_user)):
+    row = db.scalars(select(RunpodHost).where(RunpodHost.machine_id == machine_id)).first()
+    if row is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "No such machine")
+    return row.to_json(with_events=True)
+
+
+@router.get("/problem-hosts")
+def problem_hosts(limit: int = 40, db: Session = Depends(get_db),
+                  user: User = Depends(current_user)):
+    """Machines ranked by how often they have been unlisted.
+
+    The repeat offenders are the ones worth an RMA conversation rather than
+    another burn-in, which is what the old script's historical table showed.
+    """
+    unlist_counts = (
+        select(RunpodEvent.host_id, func.count(RunpodEvent.id).label("n"),
+               func.max(RunpodEvent.occurred_at).label("last"))
+        .where(RunpodEvent.event_type == RunpodEventType.UNLISTED)
+        .group_by(RunpodEvent.host_id).subquery()
+    )
+    rows = db.execute(
+        select(RunpodHost, unlist_counts.c.n, unlist_counts.c.last)
+        .join(unlist_counts, unlist_counts.c.host_id == RunpodHost.id, isouter=True)
+        .order_by(func.coalesce(unlist_counts.c.n, 0).desc(), RunpodHost.historic_count.desc())
+        .limit(max(1, min(limit, 200)))
+    ).all()
+
+    out = []
+    for host, count, last in rows:
+        total = int(count or 0) or host.historic_count
+        if not total:
+            continue
+        payload = host.to_json()
+        payload["unlist_events"] = int(count or 0)
+        payload["effective_count"] = total
+        payload["last_unlisted"] = last.isoformat() if last else (
+            host.unlisted_at.isoformat() if host.unlisted_at else None)
+        out.append(payload)
+    return {"hosts": out}
+
+
+@router.post("/sync")
+def sync(db: Session = Depends(get_db), user: User = Depends(current_user)):
+    if not settings.runpod_ready:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST,
+                            "RunPod is not configured. Set CX_RUNPOD_API_KEY.")
+    try:
+        result = sync_machines(db, _client(), actor=user.email)
+    except RunPodError as exc:
+        raise HTTPException(status.HTTP_502_BAD_GATEWAY, str(exc)) from exc
+    return result
+
+
+@router.post("/hosts/{machine_id}")
+def patch_host(machine_id: str, body: HostPatch, db: Session = Depends(get_db),
+               user: User = Depends(current_user)):
+    row = db.scalars(select(RunpodHost).where(RunpodHost.machine_id == machine_id)).first()
+    if row is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "No such machine")
+    changes = []
+    if body.colour is not None:
+        try:
+            row.colour = RunpodColour(body.colour)
+        except ValueError as exc:
+            raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Unknown colour '{body.colour}'") from exc
+        changes.append(f"colour={body.colour}")
+    for field in ("zendesk_ticket", "jira_key", "jira_status", "next_steps", "last_error"):
+        value = getattr(body, field)
+        if value is not None:
+            setattr(row, field, value)
+            changes.append(field)
+    if changes:
+        record_event(db, row, RunpodEventType.NOTE, actor=user.email,
+                     detail="Updated " + ", ".join(changes))
+    db.commit()
+    return row.to_json(with_events=True)
+
+
+@router.post("/hosts/{machine_id}/events")
+def add_event(machine_id: str, body: NoteBody, db: Session = Depends(get_db),
+              user: User = Depends(current_user)):
+    row = db.scalars(select(RunpodHost).where(RunpodHost.machine_id == machine_id)).first()
+    if row is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "No such machine")
+    try:
+        event_type = RunpodEventType(body.event_type)
+    except ValueError as exc:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Unknown event type") from exc
+    record_event(db, row, event_type, actor=user.email, detail=body.detail,
+                 zendesk_ticket=body.zendesk_ticket, jira_key=body.jira_key)
+    db.commit()
+    return row.to_json(with_events=True)
+
+
+@router.get("/hosts/{machine_id}/ticket-preview")
+def ticket_preview(machine_id: str, db: Session = Depends(get_db),
+                   user: User = Depends(current_user)):
+    """The Zendesk ticket that would be raised, without raising it."""
+    row = db.scalars(select(RunpodHost).where(RunpodHost.machine_id == machine_id)).first()
+    if row is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "No such machine")
+    return {
+        **unlisting_ticket(row),
+        "zendesk_ready": settings.zendesk_ready,
+        "send_enabled": settings.feature_send_enabled,
+        "runpod_jira_ready": settings.runpod_jira_ready,
+        "existing_ticket": row.zendesk_ticket,
+        "existing_jira": row.jira_key,
+    }
+
+
+@router.post("/hosts/{machine_id}/zendesk")
+async def zendesk_ticket(machine_id: str, body: TicketBody, db: Session = Depends(get_db),
+                         user: User = Depends(current_user)):
+    row = db.scalars(select(RunpodHost).where(RunpodHost.machine_id == machine_id)).first()
+    if row is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "No such machine")
+    try:
+        return await raise_unlisting_ticket(db, row, user, error_text=body.error_text,
+                                            requester=body.requester, subject=body.subject,
+                                            body=body.body)
+    except DeliveryError as exc:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc
+
+
+@router.post("/hosts/{machine_id}/rma")
+async def rma_issue(machine_id: str, body: RmaBody, db: Session = Depends(get_db),
+                    user: User = Depends(current_user)):
+    row = db.scalars(select(RunpodHost).where(RunpodHost.machine_id == machine_id)).first()
+    if row is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "No such machine")
+    try:
+        return await raise_rma_issue(db, row, user, summary=body.summary,
+                                     description=body.description)
+    except DeliveryError as exc:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc
+
+
+@router.post("/ingest-email")
+def ingest_email(body: EmailBody, db: Session = Depends(get_db), user: User = Depends(current_user)):
+    """Take a RunPod unlisting email and attach its error hint to the machine.
+
+    The email is the only place the actual failure reason appears, so it is
+    worth capturing even when the unlisting itself came through the API.
+    """
+    parsed = parse_email(body.raw, body.subject)
+    if not parsed["machine_id"] and not parsed["host"]:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST,
+                            "Could not find a machine in that email.")
+    query = select(RunpodHost)
+    row = db.scalars(query.where(RunpodHost.machine_id == parsed["machine_id"])).first() \
+        if parsed["machine_id"] else None
+    if row is None and parsed["host"]:
+        row = db.scalars(query.where(RunpodHost.name == parsed["host"])).first()
+    if row is None:
+        return {"parsed": parsed, "matched": False,
+                "detail": "Parsed the email, but no machine in the database matches."}
+
+    row.last_error = parsed["error_text"] or row.last_error
+    record_event(db, row, RunpodEventType.UNLISTED, actor="runpod",
+                 detail=f"{parsed['signature']} — {parsed['suggested_action']}",
+                 error_hint=parsed["error_text"], gpu_reserved=parsed["gpus_rented"],
+                 payload={"source": "email", "critical": parsed["is_critical"]})
+    db.commit()
+    return {"parsed": parsed, "matched": True, "host": row.to_json(with_events=True)}