Parham Monfared
a039e0b5fd
Read-only triage for the Infrahub error alerts. Pulls the Prometheus alert queue, re-checks each alert's condition against live state to separate real work from noise, diagnoses it using the CX runbooks, and drafts the customer comms with contacts resolved from Infrahub. Findings from validating against production: - "Suspected Rogue VM" fires on spare GPU capacity, not rogue VMs: In_Use_Gpus equals the physical count on 71 of 75 firing hosts, so the rule reduces to "this host has a free GPU". Verified against OpenStack on 10 hosts. - "Exists in Infrahub but does not exist in OpenStack" matches every VM because openstack_nova_server_status returns no series; excluded as a rule defect. - Prometheus activeAt is reset several times a day by dips in the Resources metric, so alert ages are recovered from ALERTS history instead. Takes ~2,650 firing alerts down to ~20 that need a decision. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
182 lines
10 KiB
Python
182 lines
10 KiB
Python
"""Linkage scan page: Infrahub records and OpenStack servers that lost each other."""
|
|
from __future__ import annotations
|
|
|
|
PAGE = r"""<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>CX Triage — Linkage</title>
|
|
<style>
|
|
:root{--bg:#0d1017;--panel:#141821;--panel2:#1b202b;--line:#262d3a;--line2:#333b4a;
|
|
--fg:#e8ebf2;--dim:#8a93a5;--faint:#5d6675;--accent:#4c8dff;
|
|
--ok:#35c46a;--warn:#e0a336;--bad:#f2545b;--violet:#a97bf0;
|
|
--mono:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}
|
|
@media(prefers-color-scheme:light){:root{--bg:#f4f6f9;--panel:#fff;--panel2:#f1f4f8;--line:#e0e5ec;
|
|
--line2:#cfd6e0;--fg:#151a22;--dim:#5b6473;--faint:#8e97a5;--accent:#1f6feb;
|
|
--ok:#12864a;--warn:#96650a;--bad:#cf2530;--violet:#7a44d6}}
|
|
*{box-sizing:border-box}
|
|
body{margin:0;background:var(--bg);color:var(--fg);
|
|
font:14px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif}
|
|
header{display:flex;align-items:center;gap:14px;padding:9px 18px;background:var(--panel);
|
|
border-bottom:1px solid var(--line);position:sticky;top:0;z-index:10;flex-wrap:wrap}
|
|
.brand{font-weight:680;font-size:14px}
|
|
.brand em{font-style:normal;color:var(--faint);font-weight:400;font-size:12.5px}
|
|
a.navlink{font-size:12.5px;color:var(--accent);text-decoration:none;border:1px solid var(--line2);
|
|
padding:4px 10px;border-radius:7px}
|
|
.spacer{margin-left:auto}
|
|
button{font:inherit;font-size:13px;padding:7px 13px;border-radius:8px;border:1px solid var(--line2);
|
|
background:var(--panel2);color:var(--fg);cursor:pointer}
|
|
button.pri{background:var(--accent);border-color:var(--accent);color:#fff;font-weight:600}
|
|
button:disabled{opacity:.45;cursor:not-allowed}
|
|
button.sm{padding:3px 8px;font-size:11.5px}
|
|
main{padding:22px 26px 70px;max-width:1500px}
|
|
.lede{color:var(--dim);max-width:88ch;margin-bottom:18px}
|
|
.cards{display:flex;gap:12px;flex-wrap:wrap;margin-bottom:20px}
|
|
.stat{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:13px 17px;min-width:150px}
|
|
.stat .n{font-size:23px;font-weight:680;font-family:var(--mono)}
|
|
.stat .l{font-size:11px;text-transform:uppercase;letter-spacing:.5px;color:var(--faint);margin-top:2px}
|
|
.stat.hot .n{color:var(--bad)}
|
|
.stat.warn .n{color:var(--warn)}
|
|
h2{font-size:13px;text-transform:uppercase;letter-spacing:.6px;color:var(--dim);
|
|
margin:26px 0 10px;font-weight:600}
|
|
.tbl{background:var(--panel);border:1px solid var(--line);border-radius:10px;overflow:hidden}
|
|
.tr{display:grid;grid-template-columns:1.5fr 110px 1.4fr 110px 96px 90px;gap:10px;padding:9px 14px;
|
|
border-top:1px solid var(--line);align-items:center;font-size:12.5px}
|
|
.tr:first-child{border-top:none;background:var(--panel2);font-size:10.5px;text-transform:uppercase;
|
|
letter-spacing:.5px;color:var(--faint)}
|
|
.tr .nm{font-weight:600;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
|
.tr .mono{font-family:var(--mono);font-size:11.5px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
|
.tag{font-size:10px;padding:1px 7px;border-radius:99px;border:1px solid var(--line2);color:var(--dim)}
|
|
.tag.high{color:var(--bad);border-color:color-mix(in srgb,var(--bad) 50%,transparent)}
|
|
.tag.medium{color:var(--warn);border-color:color-mix(in srgb,var(--warn) 50%,transparent)}
|
|
.tag.none{color:var(--faint)}
|
|
.arrow{color:var(--ok);text-align:center}
|
|
.sub{color:var(--faint);font-size:11px}
|
|
.empty{color:var(--faint);padding:40px;text-align:center}
|
|
.spin{width:15px;height:15px;border:2px solid var(--line2);border-top-color:var(--accent);
|
|
border-radius:50%;display:inline-block;animation:sp .7s linear infinite;vertical-align:-3px}
|
|
@keyframes sp{to{transform:rotate(360deg)}}
|
|
.note{border:1px solid color-mix(in srgb,var(--warn) 45%,transparent);
|
|
background:color-mix(in srgb,var(--warn) 9%,transparent);border-radius:8px;padding:11px 14px;
|
|
font-size:12.5px;margin-bottom:16px}
|
|
.t-bad{color:var(--bad)} .t-ok{color:var(--ok)} .t-warn{color:var(--warn)}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<header>
|
|
<div class="brand">CX Triage <em>— linkage scan</em></div>
|
|
<a class="navlink" href="/">← Alert queue</a>
|
|
<div class="spacer"></div>
|
|
<span class="sub" id="meta"></span>
|
|
<button class="pri" id="run">Run scan</button>
|
|
</header>
|
|
|
|
<main>
|
|
<p class="lede">A VM in ERROR is not always a failed build. If the server was created but the link back to
|
|
Infrahub was never written, Infrahub shows ERROR or CREATING with no usable <code>openstack_id</code> while a
|
|
perfectly good server of the same name is running. This scans both sides in bulk and pairs them up by name —
|
|
and finds the reverse too: OpenStack servers that no Infrahub record claims, which is what
|
|
<em>Suspected Orphan VM</em> was meant to catch before its input metric went empty.</p>
|
|
|
|
<div id="body"><div class="empty">Run a scan to begin. It lists every server in all four regions, so it takes a few minutes.</div></div>
|
|
</main>
|
|
|
|
<script>
|
|
const $=s=>document.querySelector(s);
|
|
const esc=s=>String(s==null?"":s).replace(/[&<>"']/g,c=>({"&":"&","<":"<",">":">",'"':""","'":"'"}[c]));
|
|
const api=(p,o)=>fetch(p,o).then(r=>r.json());
|
|
let POLL=null;
|
|
|
|
async function tick(){
|
|
const s=await api("/api/linkage");
|
|
$("#meta").textContent = s.state==="running" ? "scanning… "+(s.progress||"")
|
|
: s.finished ? "last scan "+(s.age_seconds>90?Math.round(s.age_seconds/60)+"m":s.age_seconds+"s")+" ago" : "";
|
|
$("#run").disabled = s.state==="running";
|
|
if(s.state==="running"){
|
|
$("#body").innerHTML=`<div class="empty"><span class="spin"></span><br><br>${esc(s.progress||"scanning")}…<br>
|
|
<span class="sub">Listing every server across four regions — a few minutes.</span></div>`;
|
|
return;
|
|
}
|
|
if(POLL){clearInterval(POLL);POLL=null;}
|
|
if(s.state==="error"){$("#body").innerHTML=`<div class="empty t-bad">${esc(s.error)}</div>`;return;}
|
|
if(s.state==="done") render(s.result);
|
|
}
|
|
|
|
function render(r){
|
|
const links=r.broken_links||[], orphans=r.orphans||[];
|
|
const high=links.filter(l=>l.confidence==="high");
|
|
const out=[];
|
|
|
|
out.push(`<div class="cards">
|
|
<div class="stat ${high.length?"hot":""}"><div class="n">${high.length}</div><div class="l">likely linkage failures</div></div>
|
|
<div class="stat warn"><div class="n">${links.length}</div><div class="l">Infrahub records with a broken link</div></div>
|
|
<div class="stat ${r.orphan_total?"warn":""}"><div class="n">${r.orphan_total}</div><div class="l">OpenStack servers no record claims</div></div>
|
|
<div class="stat"><div class="n">${r.openstack_servers}</div><div class="l">servers scanned</div></div>
|
|
<div class="stat"><div class="n">${r.infrahub_records}</div><div class="l">Infrahub records</div></div>
|
|
</div>`);
|
|
|
|
if(Object.keys(r.region_failures||{}).length)
|
|
out.push(`<div class="note">Some regions could not be listed: ${
|
|
Object.entries(r.region_failures).map(([k,v])=>`<b>${esc(k)}</b> (${esc(v)})`).join(", ")}.
|
|
Results below exclude those regions entirely (${r.skipped_unscanned_regions||0} record(s) skipped), so nothing
|
|
here is a false positive from a failed listing — but the scan is not complete.</div>`);
|
|
|
|
out.push(`<h2>Infrahub records whose OpenStack server is missing or unlinked</h2>`);
|
|
if(!links.length) out.push('<div class="tbl"><div class="empty">None — every record resolves.</div></div>');
|
|
else out.push(`<div class="tbl">
|
|
<div class="tr"><span>Infrahub instance</span><span>IH status</span><span>Matching OpenStack server</span>
|
|
<span>OS status</span><span>Confidence</span><span></span></div>
|
|
${links.slice(0,300).map((l,i)=>`<div class="tr">
|
|
<span class="nm">${esc(l.instance_name)}<span class="sub"><br>${esc(l.organization)} · ${esc(l.region)}</span></span>
|
|
<span class="mono t-bad">${esc(l.infrahub_status)}</span>
|
|
<span>${l.candidate
|
|
? `<span class="mono">${esc(l.candidate.id)}</span><span class="sub"><br>${esc(l.candidate.name)} · ${esc(l.candidate.region)}${l.candidate_claimed_by_other?' · <span class="t-warn">claimed by another record</span>':""}</span>`
|
|
: `<span class="sub">${esc(l.reason)}</span>`}</span>
|
|
<span class="mono">${esc(l.candidate?l.candidate.status:"—")}</span>
|
|
<span><span class="tag ${esc(l.confidence)}">${esc(l.confidence)}</span></span>
|
|
<span>${l.candidate?`<button class="sm" onclick="enrich(${i},'${esc(l.candidate.region)}','${esc(l.candidate.id)}')">Details</button>`:""}</span>
|
|
</div><div class="tr" id="ex${i}" style="display:none;grid-template-columns:1fr"></div>`).join("")}
|
|
</div>`);
|
|
if(links.length>300) out.push(`<div class="sub" style="margin-top:8px">Showing the first 300 of ${links.length}.</div>`);
|
|
|
|
out.push(`<h2>OpenStack servers with no Infrahub record</h2>`);
|
|
if(!orphans.length) out.push('<div class="tbl"><div class="empty">None.</div></div>');
|
|
else out.push(`<div class="tbl">
|
|
<div class="tr" style="grid-template-columns:1.4fr 1.4fr 110px 1fr 120px"><span>Server</span><span>OpenStack ID</span>
|
|
<span>Status</span><span>Host</span><span>Name in Infrahub?</span></div>
|
|
${orphans.slice(0,200).map(o=>`<div class="tr" style="grid-template-columns:1.4fr 1.4fr 110px 1fr 120px">
|
|
<span class="nm">${esc(o.name||"(unnamed)")}<span class="sub"><br>${esc(o.region)}</span></span>
|
|
<span class="mono">${esc(o.id)}</span>
|
|
<span class="mono">${esc(o.status)}</span>
|
|
<span class="mono">${esc(o.host||"—")}</span>
|
|
<span class="${o.name_known_to_infrahub?"t-warn":"t-bad"}">${o.name_known_to_infrahub?"name exists":"unknown"}</span>
|
|
</div>`).join("")}
|
|
</div>`);
|
|
if(r.orphan_total>orphans.length) out.push(`<div class="sub" style="margin-top:8px">Showing ${orphans.length} of ${r.orphan_total}.</div>`);
|
|
|
|
$("#body").innerHTML=out.join("");
|
|
}
|
|
|
|
async function enrich(i,region,osid){
|
|
const row=$("#ex"+i);
|
|
row.style.display="block"; row.innerHTML='<span class="spin"></span> loading…';
|
|
const d=await api("/api/linkage/enrich",{method:"POST",headers:{"Content-Type":"application/json"},
|
|
body:JSON.stringify({region,openstack_id:osid})});
|
|
row.innerHTML = d.ok
|
|
? `<span class="mono">created ${esc(d.created)} · launched ${esc(d.launched||"—")} · status ${esc(d.status)} · host ${esc(d.host||"—")}</span>
|
|
<span class="sub"><br>fault: ${esc(d.fault)}</span>`
|
|
: `<span class="t-bad">${esc(d.error)}</span>`;
|
|
}
|
|
|
|
$("#run").onclick=async()=>{
|
|
await api("/api/linkage/scan",{method:"POST",headers:{"Content-Type":"application/json"},body:"{}"});
|
|
if(POLL) clearInterval(POLL);
|
|
POLL=setInterval(tick,2500); tick();
|
|
};
|
|
tick();
|
|
</script>
|
|
</body>
|
|
</html>
|
|
"""
|