CX Triage: alert diagnosis over the CX-Tools collectors

Read-only triage for the Infrahub error alerts. Pulls the Prometheus alert queue, re-checks each alert's condition against live state to separate real work from noise, diagnoses it using the CX runbooks, and drafts the customer comms with contacts resolved from Infrahub. Findings from validating against production: - "Suspected Rogue VM" fires on spare GPU capacity, not rogue VMs: In_Use_Gpus equals the physical count on 71 of 75 firing hosts, so the rule reduces to "this host has a free GPU". Verified against OpenStack on 10 hosts. - "Exists in Infrahub but does not exist in OpenStack" matches every VM because openstack_nova_server_status returns no series; excluded as a rule defect. - Prometheus activeAt is reset several times a day by dips in the Resources metric, so alert ages are recovered from ALERTS history instead. Takes ~2,650 firing alerts down to ~20 that need a decision. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-06 06:48:34 +01:00
commit a039e0b5fd
23 changed files with 7192 additions and 0 deletions
--- a/triagelib/ui_settings.py
+++ b/triagelib/ui_settings.py
@@ -0,0 +1,221 @@
+"""Settings page: suppression rules and comms identity."""
+from __future__ import annotations
+
+PAGE = r"""<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>CX Triage — Settings</title>
+<style>
+  :root{--bg:#0d1017;--panel:#141821;--panel2:#1b202b;--line:#262d3a;--line2:#333b4a;
+    --fg:#e8ebf2;--dim:#8a93a5;--faint:#5d6675;--accent:#4c8dff;
+    --ok:#35c46a;--warn:#e0a336;--bad:#f2545b;--violet:#a97bf0;
+    --mono:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}
+  @media(prefers-color-scheme:light){:root{--bg:#f4f6f9;--panel:#fff;--panel2:#f1f4f8;--line:#e0e5ec;
+    --line2:#cfd6e0;--fg:#151a22;--dim:#5b6473;--faint:#8e97a5;--accent:#1f6feb;
+    --ok:#12864a;--warn:#96650a;--bad:#cf2530;--violet:#7a44d6}}
+  *{box-sizing:border-box}
+  body{margin:0;background:var(--bg);color:var(--fg);
+    font:14px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif}
+  header{display:flex;align-items:center;gap:14px;padding:9px 18px;background:var(--panel);
+    border-bottom:1px solid var(--line);position:sticky;top:0;z-index:10;flex-wrap:wrap}
+  .brand{font-weight:680;font-size:14px}
+  .brand em{font-style:normal;color:var(--faint);font-weight:400;font-size:12.5px}
+  a.navlink{font-size:12.5px;color:var(--accent);text-decoration:none;border:1px solid var(--line2);
+    padding:4px 10px;border-radius:7px}
+  .spacer{margin-left:auto}
+  button{font:inherit;font-size:13px;padding:7px 13px;border-radius:8px;border:1px solid var(--line2);
+    background:var(--panel2);color:var(--fg);cursor:pointer}
+  button:hover:not(:disabled){border-color:var(--accent)}
+  button.pri{background:var(--accent);border-color:var(--accent);color:#fff;font-weight:600}
+  button.sm{padding:3px 9px;font-size:11.5px}
+  button.del{color:var(--bad);border-color:color-mix(in srgb,var(--bad) 45%,transparent)}
+  main{padding:22px 26px 80px;max-width:1080px}
+  h2{font-size:13px;text-transform:uppercase;letter-spacing:.6px;color:var(--dim);margin:26px 0 10px;font-weight:600}
+  h2:first-child{margin-top:0}
+  .lede{color:var(--dim);max-width:82ch;margin-bottom:16px}
+  .card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:16px 18px;margin-bottom:12px}
+  .rule{background:var(--panel);border:1px solid var(--line);border-radius:10px;margin-bottom:10px;overflow:hidden}
+  .rule.off{opacity:.55}
+  .rh{display:flex;align-items:center;gap:10px;padding:12px 16px;flex-wrap:wrap}
+  .rh .nm{font-weight:640;font-size:14px}
+  .rh .why{color:var(--dim);font-size:12.5px;width:100%;margin-top:-4px}
+  .conds{display:flex;gap:6px;flex-wrap:wrap;padding:0 16px 12px}
+  .cond{font-size:11.5px;font-family:var(--mono);padding:3px 9px;border-radius:6px;
+    background:var(--panel2);border:1px solid var(--line2)}
+  .cond b{color:var(--accent);font-weight:600}
+  .andor{font-size:10.5px;color:var(--faint);align-self:center;text-transform:uppercase;letter-spacing:.5px}
+  .sw{position:relative;width:34px;height:19px;flex:none}
+  .sw input{opacity:0;width:0;height:0}
+  .sw span{position:absolute;inset:0;background:var(--line2);border-radius:99px;transition:.15s;cursor:pointer}
+  .sw span::before{content:"";position:absolute;width:15px;height:15px;left:2px;top:2px;background:#fff;
+    border-radius:50%;transition:.15s}
+  .sw input:checked + span{background:var(--ok)}
+  .sw input:checked + span::before{transform:translateX(15px)}
+  .grid{display:grid;grid-template-columns:170px 1fr;gap:10px 12px;align-items:center}
+  label.f{font-size:11px;text-transform:uppercase;letter-spacing:.5px;color:var(--faint)}
+  input,select,textarea{font:inherit;font-size:13px;padding:7px 10px;border-radius:7px;
+    border:1px solid var(--line2);background:var(--bg);color:var(--fg);width:100%}
+  .condrow{display:grid;grid-template-columns:190px 1fr 34px;gap:8px;margin-bottom:7px}
+  .hint{font-size:11.5px;color:var(--faint);margin-top:4px}
+  .pv{margin-top:12px;border:1px solid var(--line2);border-radius:8px;padding:11px 13px;background:var(--panel2);font-size:12.5px}
+  .pv ul{margin:6px 0 0;padding-left:18px;max-height:180px;overflow-y:auto}
+  .t-ok{color:var(--ok)}.t-bad{color:var(--bad)}.t-warn{color:var(--warn)}
+  .empty{color:var(--faint);padding:26px;text-align:center}
+  .saved{color:var(--ok);font-size:12.5px}
+</style>
+</head>
+<body>
+<header>
+  <div class="brand">CX Triage <em>— settings</em></div>
+  <a class="navlink" href="/">&larr; Alert queue</a>
+  <a class="navlink" href="/linkage">Linkage scan</a>
+  <div class="spacer"></div>
+  <span class="saved" id="saved"></span>
+</header>
+
+<main>
+  <h2>Suppression rules</h2>
+  <p class="lede">Hide alerts you already know about. A rule fires when <b>every</b> condition it sets matches,
+  so you can combine them &mdash; for example type <code>error</code> <em>and</em> organisation containing
+  <code>modal</code>. Suppressed alerts are not deleted: they stay reachable under the
+  <b>hidden by a rule</b> filter on the queue.</p>
+
+  <div id="rules"></div>
+  <button class="pri" id="addRule">Add a rule</button>
+
+  <div id="editor"></div>
+
+  <h2>Comms identity</h2>
+  <div class="card">
+    <div class="grid">
+      <label class="f">Sign-off name</label>
+      <div><input id="agent" placeholder="e.g. Mohammad Affan">
+        <div class="hint">Appended after &ldquo;Kind Regards&rdquo; in customer emails.</div></div>
+      <label class="f">Chronic after (days)</label>
+      <div><input id="chronic" type="number" min="1" max="90" style="max-width:110px">
+        <div class="hint">An alert whose condition has held longer than this is treated as chronic rather than new
+          work &mdash; except for types with a runbook time commitment, which become <b>overdue</b> instead.</div></div>
+    </div>
+    <div style="margin-top:14px"><button class="pri" id="saveGeneral">Save</button></div>
+  </div>
+
+  <h2>Where this is stored</h2>
+  <div class="card"><span class="hint" id="path"></span></div>
+</main>
+
+<script>
+const $=s=>document.querySelector(s);
+const esc=s=>String(s==null?"":s).replace(/[&<>"']/g,c=>({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c]));
+const api=(p,o)=>fetch(p,o).then(r=>r.json());
+let CFG=null, EDIT=null;
+
+function flash(msg){$("#saved").textContent=msg;setTimeout(()=>$("#saved").textContent="",2200);}
+
+async function load(){ CFG=await api("/api/settings"); render(); }
+
+function render(){
+  $("#agent").value=CFG.agent_name||"";
+  $("#chronic").value=CFG.chronic_days||3;
+  $("#path").textContent=CFG.path;
+  const rs=CFG.rules||[];
+  $("#rules").innerHTML = rs.length ? rs.map(r=>`
+    <div class="rule ${r.enabled?"":"off"}">
+      <div class="rh">
+        <label class="sw"><input type="checkbox" ${r.enabled?"checked":""} onchange="toggle('${esc(r.id)}',this.checked)"><span></span></label>
+        <span class="nm">${esc(r.name)}</span>
+        <span class="spacer" style="margin-left:auto"></span>
+        <button class="sm" onclick='edit(${JSON.stringify(JSON.stringify(r))})'>Edit</button>
+        <button class="sm del" onclick="del('${esc(r.id)}','${esc(r.name)}')">Delete</button>
+        ${r.reason?`<span class="why">${esc(r.reason)}</span>`:""}
+      </div>
+      <div class="conds">${Object.entries(r.conditions||{}).map(([k,v],i)=>
+        `${i?'<span class="andor">and</span>':""}<span class="cond"><b>${esc(CFG.conditions[k]||k)}</b> ${esc(v.join(" or "))}</span>`
+      ).join("")||'<span class="cond t-bad">no conditions — inactive</span>'}</div>
+    </div>`).join("") : '<div class="card"><div class="empty">No rules yet.</div></div>';
+}
+
+function blank(){return {id:"",name:"",enabled:true,reason:"",conditions:{}};}
+
+function edit(json){ EDIT=typeof json==="string"?JSON.parse(json):json; drawEditor(); }
+$("#addRule").onclick=()=>{EDIT=blank();drawEditor();};
+
+function drawEditor(){
+  if(!EDIT){$("#editor").innerHTML="";return;}
+  const conds=Object.entries(EDIT.conditions||{});
+  if(!conds.length) conds.push(["kind",[]]);
+  $("#editor").innerHTML=`<div class="card" style="border-color:var(--accent)">
+    <div class="grid">
+      <label class="f">Rule name</label><input id="rName" value="${esc(EDIT.name)}" placeholder="e.g. Modal ERROR noise">
+      <label class="f">Why (shown on the alert)</label><input id="rWhy" value="${esc(EDIT.reason)}" placeholder="e.g. Known batch churn, customer already aware">
+    </div>
+    <div style="margin-top:14px">
+      <label class="f">Conditions &mdash; all must match</label>
+      <div id="condList" style="margin-top:7px">${conds.map((c,i)=>condRow(c[0],c[1],i)).join("")}</div>
+      <button class="sm" onclick="addCond()">+ Add condition</button>
+    </div>
+    <div id="pv"></div>
+    <div style="margin-top:14px;display:flex;gap:9px;flex-wrap:wrap">
+      <button class="pri" onclick="saveRule()">Save rule</button>
+      <button onclick="previewRule()">Preview what this hides</button>
+      <button onclick="EDIT=null;drawEditor()">Cancel</button>
+    </div></div>`;
+}
+
+function condRow(field,values,i){
+  return `<div class="condrow" data-i="${i}">
+    <select class="cf">${Object.entries(CFG.conditions).map(([k,v])=>
+      `<option value="${esc(k)}" ${k===field?"selected":""}>${esc(v)}</option>`).join("")}</select>
+    <input class="cv" value="${esc((values||[]).join(", "))}" placeholder="comma-separated; any one matches">
+    <button class="sm del" onclick="this.parentElement.remove()">&times;</button></div>`;
+}
+function addCond(){ $("#condList").insertAdjacentHTML("beforeend", condRow("organization",[],Date.now())); }
+
+function collect(){
+  const conditions={};
+  document.querySelectorAll("#condList .condrow").forEach(row=>{
+    const f=row.querySelector(".cf").value;
+    const v=row.querySelector(".cv").value.split(",").map(x=>x.trim()).filter(Boolean);
+    if(v.length) conditions[f]=(conditions[f]||[]).concat(v);
+  });
+  return {id:EDIT.id,name:$("#rName").value||"Untitled rule",reason:$("#rWhy").value,
+          enabled:EDIT.enabled!==false,conditions};
+}
+
+async function previewRule(){
+  const r=await api("/api/settings",{method:"POST",headers:{"Content-Type":"application/json"},
+    body:JSON.stringify({action:"preview",rule:collect()})});
+  $("#pv").innerHTML=`<div class="pv">
+    <b class="${r.count?"t-warn":"t-ok"}">${r.count}</b> currently-firing alert(s) would be hidden.
+    ${r.count?`<ul>${r.matches.slice(0,40).map(m=>
+      `<li>${esc(m.title)} &mdash; ${esc(m.instance_name||m.host||"?")} <span style="color:var(--faint)">${esc(m.org_name||"")}</span></li>`
+    ).join("")}</ul>`:""}</div>`;
+}
+
+async function saveRule(){
+  const r=await api("/api/settings",{method:"POST",headers:{"Content-Type":"application/json"},
+    body:JSON.stringify({action:"save_rule",rule:collect()})});
+  if(r.ok){CFG=r.settings;EDIT=null;drawEditor();render();flash("Rule saved");}
+}
+async function toggle(id,on){
+  const r=await api("/api/settings",{method:"POST",headers:{"Content-Type":"application/json"},
+    body:JSON.stringify({action:"toggle_rule",id,enabled:on})});
+  if(r.ok){CFG=r.settings;render();flash(on?"Rule enabled":"Rule disabled");}
+}
+async function del(id,name){
+  if(!confirm(`Delete the rule “${name}”? Alerts it was hiding will come back.`)) return;
+  const r=await api("/api/settings",{method:"POST",headers:{"Content-Type":"application/json"},
+    body:JSON.stringify({action:"delete_rule",id})});
+  if(r.ok){CFG=r.settings;render();flash("Rule deleted");}
+}
+$("#saveGeneral").onclick=async()=>{
+  const r=await api("/api/settings",{method:"POST",headers:{"Content-Type":"application/json"},
+    body:JSON.stringify({action:"general",agent_name:$("#agent").value,chronic_days:$("#chronic").value})});
+  if(r.ok){CFG=r.settings;flash("Saved");}
+};
+load();
+</script>
+</body>
+</html>
+"""