CX Triage: alert diagnosis over the CX-Tools collectors
Read-only triage for the Infrahub error alerts. Pulls the Prometheus alert queue, re-checks each alert's condition against live state to separate real work from noise, diagnoses it using the CX runbooks, and drafts the customer comms with contacts resolved from Infrahub. Findings from validating against production: - "Suspected Rogue VM" fires on spare GPU capacity, not rogue VMs: In_Use_Gpus equals the physical count on 71 of 75 firing hosts, so the rule reduces to "this host has a free GPU". Verified against OpenStack on 10 hosts. - "Exists in Infrahub but does not exist in OpenStack" matches every VM because openstack_nova_server_status returns no series; excluded as a rule defect. - Prometheus activeAt is reset several times a day by dips in the Resources metric, so alert ages are recovered from ALERTS history instead. Takes ~2,650 firing alerts down to ~20 that need a decision. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
196
triagelib/integrations.py
Normal file
196
triagelib/integrations.py
Normal file
@@ -0,0 +1,196 @@
|
||||
"""Outbound action payloads: Zendesk tickets and Jira issues.
|
||||
|
||||
This module *builds* payloads and never sends them. Delivery is a separate,
|
||||
explicitly configured step - see `outbox.py` - so that a diagnosis can never
|
||||
contact a customer as a side effect of being viewed.
|
||||
|
||||
Every payload carries the evidence that justified it, so the ticket a customer
|
||||
or the Infrastructure team receives is self-contained.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
|
||||
# Set these to enable the Send buttons. Absent = preview only.
|
||||
ZENDESK_SUBDOMAIN = os.environ.get("CX_ZENDESK_SUBDOMAIN", "")
|
||||
ZENDESK_EMAIL = os.environ.get("CX_ZENDESK_EMAIL", "")
|
||||
ZENDESK_TOKEN = os.environ.get("CX_ZENDESK_TOKEN", "")
|
||||
JIRA_BASE = os.environ.get("CX_JIRA_BASE", "https://nexgencloud.atlassian.net")
|
||||
JIRA_EMAIL = os.environ.get("CX_JIRA_EMAIL", "")
|
||||
JIRA_TOKEN = os.environ.get("CX_JIRA_TOKEN", "")
|
||||
JIRA_PROJECT = os.environ.get("CX_JIRA_PROJECT", "INFRA")
|
||||
|
||||
PRIORITY_BY_VERDICT = {"overdue": "high", "real": "normal", "unverified": "low"}
|
||||
|
||||
|
||||
def zendesk_configured() -> bool:
|
||||
return bool(ZENDESK_SUBDOMAIN and ZENDESK_EMAIL and ZENDESK_TOKEN)
|
||||
|
||||
|
||||
def jira_configured() -> bool:
|
||||
return bool(JIRA_BASE and JIRA_EMAIL and JIRA_TOKEN)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Action:
|
||||
"""One proposed outbound action, ready to send once a human confirms."""
|
||||
|
||||
id: str
|
||||
kind: str # zendesk | jira | manual
|
||||
label: str
|
||||
summary: str # one line: what this does
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
recipients: list[str] = field(default_factory=list)
|
||||
enabled: bool = False # is the integration configured?
|
||||
blocked_reason: str = ""
|
||||
requires_confirmation: bool = True
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
"id": self.id, "kind": self.kind, "label": self.label, "summary": self.summary,
|
||||
"payload": self.payload, "recipients": self.recipients, "enabled": self.enabled,
|
||||
"blocked_reason": self.blocked_reason, "requires_confirmation": self.requires_confirmation,
|
||||
}
|
||||
|
||||
|
||||
def _parse_owner(owner: str) -> tuple[str, str]:
|
||||
"""'Name <email@x>' -> ('Name', 'email@x')."""
|
||||
text = str(owner or "").strip()
|
||||
if "<" in text and ">" in text:
|
||||
name = text.split("<", 1)[0].strip()
|
||||
email = text.split("<", 1)[1].split(">", 1)[0].strip()
|
||||
return name, email
|
||||
return ("", text) if "@" in text else (text, "")
|
||||
|
||||
|
||||
def _evidence_block(diagnosis: Any) -> str:
|
||||
lines = [f"Alert: {diagnosis.alert.title}", f"Verdict: {diagnosis.verdict}", ""]
|
||||
for finding in diagnosis.findings[:16]:
|
||||
lines.append(f"- {finding.label}: {finding.value}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_zendesk(diagnosis: Any) -> Optional[Action]:
|
||||
"""A customer ticket, only when the runbook actually calls for contact."""
|
||||
if not diagnosis.drafts:
|
||||
return None
|
||||
draft = diagnosis.drafts[0]
|
||||
alert = diagnosis.alert
|
||||
contacts = diagnosis.contacts or {}
|
||||
owners = contacts.get("owners") or []
|
||||
if not owners:
|
||||
return Action(
|
||||
id="zendesk", kind="zendesk", label="Contact customer (Zendesk)",
|
||||
summary="No owner contact resolved from Infrahub - look the organization up first.",
|
||||
enabled=False, blocked_reason="No customer contact could be resolved.",
|
||||
)
|
||||
|
||||
name, email = _parse_owner(owners[0])
|
||||
verdict = (alert.screen or {}).get("verdict", "real")
|
||||
payload = {
|
||||
"ticket": {
|
||||
"subject": draft.subject,
|
||||
"comment": {"body": draft.body, "public": True},
|
||||
"requester": {"name": name or email, "email": email},
|
||||
"priority": PRIORITY_BY_VERDICT.get(verdict, "normal"),
|
||||
"type": "incident",
|
||||
"tags": ["cx-triage", f"alert-{alert.kind}", f"region-{alert.region or 'unknown'}"],
|
||||
"external_id": f"cx-triage-{alert.fingerprint()}",
|
||||
"custom_fields_note": {
|
||||
"instance_name": alert.instance_name,
|
||||
"openstack_id": alert.openstack_id,
|
||||
"organization": alert.org_name,
|
||||
"infrahub_org_id": alert.org_id,
|
||||
},
|
||||
},
|
||||
"_template": draft.template_id,
|
||||
"_when": draft.when,
|
||||
"_unfilled": draft.unfilled,
|
||||
}
|
||||
blocked = ""
|
||||
if draft.unfilled:
|
||||
blocked = f"Template still has placeholders: {', '.join(draft.unfilled)}"
|
||||
return Action(
|
||||
id="zendesk", kind="zendesk",
|
||||
label="Contact customer (Zendesk)",
|
||||
summary=f"Public reply to {name or email} - {draft.label}",
|
||||
payload=payload,
|
||||
recipients=[o for o in owners],
|
||||
enabled=zendesk_configured() and not blocked,
|
||||
blocked_reason=blocked or ("" if zendesk_configured() else "Zendesk is not configured."),
|
||||
)
|
||||
|
||||
|
||||
def build_jira(diagnosis: Any) -> Optional[Action]:
|
||||
"""An Infrastructure escalation, only when a step is owned by Infra."""
|
||||
infra_steps = [a for a in diagnosis.actions if a.owner != "CX" and a.kind == "escalate"]
|
||||
if not infra_steps:
|
||||
return None
|
||||
alert = diagnosis.alert
|
||||
subject = alert.host or alert.instance_name or alert.floating_ip or "unknown"
|
||||
|
||||
description = "\n".join([
|
||||
_evidence_block(diagnosis),
|
||||
"",
|
||||
"Requested of Infrastructure:",
|
||||
*[f"- {s.text}" for s in infra_steps],
|
||||
"",
|
||||
f"Raised from CX Triage. Alert has held for {alert.effective_age_text}.",
|
||||
])
|
||||
payload = {
|
||||
"fields": {
|
||||
"project": {"key": JIRA_PROJECT},
|
||||
"summary": f"{subject}: {diagnosis.verdict}"[:250],
|
||||
"description": description,
|
||||
"issuetype": {"name": "Task"},
|
||||
"labels": ["cx-triage", f"alert-{alert.kind}", f"region-{alert.region or 'unknown'}"],
|
||||
}
|
||||
}
|
||||
return Action(
|
||||
id="jira", kind="jira",
|
||||
label="Escalate to Infrastructure (Jira)",
|
||||
summary=f"Create a {JIRA_PROJECT} issue for {subject}",
|
||||
payload=payload,
|
||||
enabled=jira_configured(),
|
||||
blocked_reason="" if jira_configured() else "Jira is not configured.",
|
||||
)
|
||||
|
||||
|
||||
def build_manual(diagnosis: Any) -> list[Action]:
|
||||
"""Steps a human must perform; surfaced as copyable commands, not buttons."""
|
||||
out: list[Action] = []
|
||||
alert = diagnosis.alert
|
||||
osid = alert.openstack_id
|
||||
region = alert.region
|
||||
for step in diagnosis.actions:
|
||||
if step.kind != "remediate" or step.status == "done":
|
||||
continue
|
||||
command = ""
|
||||
low = step.text.lower()
|
||||
if "delete the server in openstack" in low and osid and region:
|
||||
command = f"{region} server delete {osid}"
|
||||
elif "shelve" in low and osid and region:
|
||||
command = f"{region} server shelve {osid}"
|
||||
out.append(Action(
|
||||
id=f"manual-{len(out)}", kind="manual", label=step.text,
|
||||
summary=step.guide or "", payload={"command": command} if command else {},
|
||||
enabled=False, blocked_reason="Perform manually - this tool is read-only.",
|
||||
requires_confirmation=False,
|
||||
))
|
||||
return out
|
||||
|
||||
|
||||
def build_all(diagnosis: Any) -> dict[str, Any]:
|
||||
actions: list[Action] = []
|
||||
for builder in (build_zendesk, build_jira):
|
||||
action = builder(diagnosis)
|
||||
if action:
|
||||
actions.append(action)
|
||||
actions.extend(build_manual(diagnosis))
|
||||
return {
|
||||
"actions": [a.to_json() for a in actions],
|
||||
"zendesk_configured": zendesk_configured(),
|
||||
"jira_configured": jira_configured(),
|
||||
}
|
||||
Reference in New Issue
Block a user