Split into a FastAPI backend and a React frontend, add case state and SSO
Some checks failed
build-and-deploy / test (push) Has been cancelled
build-and-deploy / image (push) Has been cancelled
build-and-deploy / deploy (push) Has been cancelled

The single-file stdlib server became the limit: no way to track what had been
done about an alert, no accounts, and a UI that had to be hand-rolled in
template strings. This restructures it into something deployable.

Backend (FastAPI)
- app/ holds config, database, auth, delivery and the routers; triagelib keeps
  the triage engine unchanged, so the validated screening and runbook logic is
  untouched.
- Cases persist per alert fingerprint with a status workflow (investigating,
  customer contacted, escalated to Infra, waiting, remediated, resolved, won't
  fix, false positive), an assignee, notes and an append-only history. An alert
  that stops and re-fires lands back on the same case and counts as a reopen.
- Suppression rules move from a JSON file into the database.

Auth
- Signed session cookies over PBKDF2 local accounts, plus an OIDC flow ready for
  Authentik: users are created on first login and admin follows a group claim.
  Local login can be switched off entirely once SSO is live.

Zendesk and Jira
- Delivery is now implemented, behind three gates: the integration must be
  configured, its feature flag on, and CX_FEATURE_SEND_ENABLED on. A demo
  instance leaves the last off and cannot mail anyone. Both search before
  creating, so re-diagnosing an alert updates one ticket rather than opening
  several, and a rolling daily cap stops a loop mailing everybody.

Deployment
- Multi-stage Dockerfile builds the bundle and serves it from the API origin.
- docker-compose for local and single-host use; Gitea Actions runs the tests,
  builds the image and renders deploy/k8s with envsubst.

Two fixes found while testing: assigning a case returned a null assignee, and
add_event could leave an already-loaded history collection stale.

Known gap: the engine reaches OpenStack via `docker exec <region>-osc`, which
does not work in a pod without the CX-Tools containers alongside it.
docs/DEPLOYMENT.md sets out the three ways to close that.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-06 07:11:28 +01:00
parent a039e0b5fd
commit 1262690276
68 changed files with 3839 additions and 2223 deletions

67
.env.example Normal file
View File

@@ -0,0 +1,67 @@
# ---------------------------------------------------------------------------
# Copy to .env and edit. Everything here is read at startup.
# ---------------------------------------------------------------------------
# --- app -------------------------------------------------------------------
CX_APP_NAME=CX Triage
CX_PORT=8080
# Must be the externally reachable URL: the SSO redirect is built from it.
CX_BASE_URL=http://localhost:8080
# openssl rand -hex 32
CX_SECRET_KEY=change-me
CX_SESSION_HOURS=12
# --- database --------------------------------------------------------------
# compose provides postgres; sqlite is fine for a single user.
# CX_DATABASE_URL=sqlite:////data/cx-triage.db
CX_DATABASE_URL=postgresql+psycopg://cx:cx@db:5432/cxtriage
# --- data sources ----------------------------------------------------------
CX_PROMETHEUS_BASE=http://10.11.254.250:9090
# Container to relay Prometheus queries through when the host has no route to
# the internal network. Leave blank to auto-detect (ca1-osc, us1-osc, ...).
CX_PROMETHEUS_RELAY=
CX_TOOLS_PATH=/opt/cx-tools
# --- authentication --------------------------------------------------------
CX_AUTH_LOCAL_ENABLED=true
# Only used to create the very first account, when the user table is empty.
CX_BOOTSTRAP_ADMIN_EMAIL=admin@nexgencloud.com
CX_BOOTSTRAP_ADMIN_PASSWORD=
# --- Authentik SSO ---------------------------------------------------------
# Redirect URI to register in Authentik:
# ${CX_BASE_URL}/api/auth/oidc/callback
CX_OIDC_ENABLED=false
CX_OIDC_ISSUER=https://sso.nexgencloud.com/application/o/cx-triage/
CX_OIDC_CLIENT_ID=
CX_OIDC_CLIENT_SECRET=
CX_OIDC_SCOPES=openid email profile
# Members of this group become administrators.
CX_OIDC_ADMIN_GROUP=cx-triage-admins
CX_OIDC_GROUPS_CLAIM=groups
# --- feature flags ---------------------------------------------------------
# The master switch. With this off nothing can reach a customer, whatever else
# is configured. Leave it off on demo and staging instances.
CX_FEATURE_SEND_ENABLED=false
CX_FEATURE_ZENDESK=false
CX_FEATURE_JIRA=false
CX_FEATURE_LINKAGE_SCAN=true
# Refuses to send more than this in a rolling 24h, so a loop cannot mail everyone.
CX_SEND_DAILY_CAP=25
# --- Zendesk ---------------------------------------------------------------
# Admin Center -> Apps and integrations -> APIs -> Zendesk API -> Add API token
CX_ZENDESK_SUBDOMAIN=
CX_ZENDESK_EMAIL=
CX_ZENDESK_TOKEN=
CX_ZENDESK_PUBLIC_REPLY=true
# --- Jira ------------------------------------------------------------------
# https://id.atlassian.com/manage-profile/security/api-tokens
CX_JIRA_BASE=https://nexgencloud.atlassian.net
CX_JIRA_EMAIL=
CX_JIRA_TOKEN=
CX_JIRA_PROJECT=INFRA
CX_JIRA_ISSUE_TYPE=Task

128
.gitea/workflows/ci.yaml Normal file
View File

@@ -0,0 +1,128 @@
name: build-and-deploy
on:
push:
branches: [main]
tags: ["v*"]
pull_request:
branches: [main]
workflow_dispatch:
env:
REGISTRY: ${{ vars.REGISTRY || 'git.ngbackend.cloud' }}
IMAGE: ${{ vars.IMAGE_NAME || 'parham.monfared/cx-ui' }}
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with: { python-version: "3.12" }
- name: Install backend dependencies
run: pip install -r backend/requirements.txt
- name: Runbook and screening tests
# These are pure-logic tests: no Prometheus, no CX-Tools, no network.
run: |
cd backend
python tests/test_runbooks.py
python tests/test_screening.py
python tests/test_api.py
- uses: actions/setup-node@v4
with: { node-version: "22" }
- name: Build frontend
run: |
cd frontend
npm ci --no-audit --no-fund || npm install --no-audit --no-fund
npm run typecheck
npm run build
image:
needs: test
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: docker/setup-buildx-action@v3
- uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ secrets.REGISTRY_USERNAME }}
password: ${{ secrets.REGISTRY_TOKEN }}
- uses: docker/metadata-action@v5
id: meta
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE }}
tags: |
type=ref,event=branch
type=semver,pattern={{version}}
type=sha,prefix=,format=short
- uses: docker/build-push-action@v6
with:
context: .
file: backend/Dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
deploy:
needs: image
if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
runs-on: ubuntu-latest
environment: ${{ vars.DEPLOY_ENVIRONMENT || 'production' }}
steps:
- uses: actions/checkout@v4
- name: Write kubeconfig
run: |
mkdir -p "$HOME/.kube"
echo "${{ secrets.KUBECONFIG }}" | base64 -d > "$HOME/.kube/config"
chmod 600 "$HOME/.kube/config"
- name: Sync secrets
# Applied imperatively so nothing sensitive is ever committed. Every
# value comes from the repository/environment secret store.
run: |
kubectl -n "${{ vars.K8S_NAMESPACE || 'cx-triage' }}" create secret generic cx-triage \
--from-literal=CX_SECRET_KEY='${{ secrets.CX_SECRET_KEY }}' \
--from-literal=CX_DATABASE_URL='${{ secrets.CX_DATABASE_URL }}' \
--from-literal=CX_OIDC_CLIENT_ID='${{ secrets.CX_OIDC_CLIENT_ID }}' \
--from-literal=CX_OIDC_CLIENT_SECRET='${{ secrets.CX_OIDC_CLIENT_SECRET }}' \
--from-literal=CX_ZENDESK_SUBDOMAIN='${{ secrets.CX_ZENDESK_SUBDOMAIN }}' \
--from-literal=CX_ZENDESK_EMAIL='${{ secrets.CX_ZENDESK_EMAIL }}' \
--from-literal=CX_ZENDESK_TOKEN='${{ secrets.CX_ZENDESK_TOKEN }}' \
--from-literal=CX_JIRA_BASE='${{ vars.CX_JIRA_BASE }}' \
--from-literal=CX_JIRA_EMAIL='${{ secrets.CX_JIRA_EMAIL }}' \
--from-literal=CX_JIRA_TOKEN='${{ secrets.CX_JIRA_TOKEN }}' \
--from-literal=CX_BOOTSTRAP_ADMIN_PASSWORD='${{ secrets.CX_BOOTSTRAP_ADMIN_PASSWORD }}' \
--dry-run=client -o yaml | kubectl apply -f -
- name: Render and apply manifests
env:
IMAGE_REF: ${{ env.REGISTRY }}/${{ env.IMAGE }}:${{ github.sha }}
NAMESPACE: ${{ vars.K8S_NAMESPACE || 'cx-triage' }}
DOMAIN: ${{ vars.CX_DOMAIN }}
BASE_URL: https://${{ vars.CX_DOMAIN }}
OIDC_ENABLED: ${{ vars.CX_OIDC_ENABLED || 'true' }}
OIDC_ISSUER: ${{ vars.CX_OIDC_ISSUER }}
OIDC_ADMIN_GROUP: ${{ vars.CX_OIDC_ADMIN_GROUP || 'cx-triage-admins' }}
PROMETHEUS_BASE: ${{ vars.CX_PROMETHEUS_BASE }}
FEATURE_SEND_ENABLED: ${{ vars.CX_FEATURE_SEND_ENABLED || 'false' }}
FEATURE_ZENDESK: ${{ vars.CX_FEATURE_ZENDESK || 'false' }}
FEATURE_JIRA: ${{ vars.CX_FEATURE_JIRA || 'false' }}
FEATURE_LINKAGE_SCAN: ${{ vars.CX_FEATURE_LINKAGE_SCAN || 'true' }}
SEND_DAILY_CAP: ${{ vars.CX_SEND_DAILY_CAP || '25' }}
JIRA_PROJECT: ${{ vars.CX_JIRA_PROJECT || 'INFRA' }}
INGRESS_CLASS: ${{ vars.INGRESS_CLASS || 'nginx' }}
TLS_ISSUER: ${{ vars.TLS_ISSUER || 'letsencrypt-prod' }}
REPLICAS: ${{ vars.REPLICAS || '1' }}
run: |
for f in deploy/k8s/*.yaml; do
envsubst < "$f"
done > /tmp/rendered.yaml
kubectl apply -f /tmp/rendered.yaml
kubectl -n "$NAMESPACE" rollout status deployment/cx-triage --timeout=5m

View File

@@ -1,89 +0,0 @@
# Wiring Zendesk and Jira
"Not wired" meant: the app **builds** the full API payload and shows it to you,
but the code path that would POST it to Zendesk deliberately returns an error.
Nothing can reach a customer today, even by accident. Here is what is needed to
change that.
---
## What I need from you
### Zendesk
| Thing | Where it comes from | Example |
|---|---|---|
| Subdomain | your Zendesk URL `https://<subdomain>.zendesk.com` | `nexgencloud` |
| API email | the agent account tickets are created as | `cx-bot@nexgencloud.com` |
| API token | Zendesk **Admin Center → Apps and integrations → APIs → Zendesk API → Add API token** | 40-char string |
Plus four decisions:
1. **Which agent account should own these tickets?** A dedicated `cx-triage` agent
is better than a person's account — the audit trail stays clear.
2. **Public reply or internal note on first send?** The runbook wording is written
for the customer, so public — but confirm.
3. **Requester matching.** Infrahub gives us the owner's name and email. If they
are not already a Zendesk user, should we create them, or fail and ask a human?
4. **Do you want a tag convention** beyond the `cx-triage`, `alert-<kind>`,
`region-<x>` tags I currently set?
### Jira
| Thing | Where it comes from | Example |
|---|---|---|
| Base URL | your Atlassian site | `https://nexgencloud.atlassian.net` |
| API email | Atlassian account | `cx-bot@nexgencloud.com` |
| API token | <https://id.atlassian.com/manage-profile/security/api-tokens> | token string |
| Project key | the Infrastructure project | `INFRA`? |
| Issue type | must exist in that project | `Task`? `Bug`? |
Confirm the project key and issue type — I guessed `INFRA` / `Task`.
### Where to put the credentials
**Don't paste them to me.** Two options:
- **Preferred — 1Password.** Create two items in the `Employee` vault, e.g.
`Zendesk API (CX Triage)` and `Jira API (CX Triage)`, each with the token in the
`password` field and the email in a `username` field. I extend the existing
CX-Tools secrets loader to read them, exactly like the Infrahub key. Nothing
touches disk.
- **Quick and dirty** — environment variables in your shell before launching:
`CX_ZENDESK_SUBDOMAIN`, `CX_ZENDESK_EMAIL`, `CX_ZENDESK_TOKEN`,
`CX_JIRA_BASE`, `CX_JIRA_EMAIL`, `CX_JIRA_TOKEN`, `CX_JIRA_PROJECT`.
Fine for a trial, worse for a shared tool.
---
## What I build once I have that
1. **Delivery** — replace the refusal in `App.send_zendesk` with a real POST, and
add the same for Jira.
2. **Search before create**`GET /api/v2/search?query=external_id:cx-triage-<fp>`
so re-diagnosing an alert comments on the existing ticket rather than opening a
second one. Same for Jira via a `cx-triage-<fp>` label.
3. **A send gate** — the server refuses to send unless started with `--allow-send`.
A demo instance then physically cannot email a customer, no matter what is
clicked. Default stays off.
4. **Audit** — every send appended to `vmc-audit.log` with who, what, and the
resulting ticket URL, and the URL shown back on the case.
5. **Rate limiting** — Zendesk allows 700 req/min; a simple per-minute cap plus a
refusal to send more than N tickets in one session, so a bad loop cannot mail
a hundred customers.
## Guard rails that stay regardless
- Every send needs a human click plus a confirm naming the recipient.
- No auto-send, ever — a verdict never triggers an email on its own.
- No bulk send in v1. The 13 overdue ERROR alerts are tempting, but one wrong
template across 13 customers is a bad first outing.
- The message stays editable before sending.
- Deleting, shelving and InfraInsight edits remain copy-a-command. The read-only
guarantee is what makes this safe against production.
## Suggested first run
Point it at a **Zendesk sandbox** first, or send the first real ticket to your own
address by editing the To field. Once one round-trip looks right in Zendesk, turn
it on for real.

263
README.md
View File

@@ -1,238 +1,71 @@
# CX Triage # CX Triage
A small local webapp that takes the Infrahub error alerts out of Prometheus, Turns the Infrahub error-alert firehose into a short list of things that
diagnoses each one using the **unmodified** CX-Tools (`vmc`) collectors, tells you actually need doing then helps you do them.
what the runbook says to do next, and — when the next step is contacting the
customer — shows the approved wording alongside the customer's contact details.
**It is read-only.** It queries Infrahub, OpenStack, InfraInsight and Prometheus. Python/FastAPI backend, React frontend, PostgreSQL for case state.
It never changes platform state, never deletes or shelves anything, and never
sends a message. Every action it identifies is presented for a human to perform.
## Separating noise from real work ## What it does
Thousands of alerts fire; only a handful are work. Before anything is shown, each 1. **Pulls** the alert queue from Prometheus.
alert's condition is **re-checked against current state**, and the verdict is 2. **Screens** every alert by re-checking its condition against live state, so
displayed with its reason: noise and already-resolved alerts drop out.
3. **Diagnoses** what is left using the CX runbooks, reconciling Infrahub
against OpenStack through the CX-Tools collectors.
4. **Drafts** the customer email with contacts resolved from Infrahub, and the
Jira escalation with the evidence attached.
5. **Tracks** each case — who owns it, what was done, what was sent.
| Verdict | Meaning | In the queue? | On live data this takes roughly **2,650 firing alerts down to ~20** that need a
|---|---|---| decision.
| **needs action** | The condition still holds | yes |
| **needs action (unverified)** | Couldn't be re-checked — never hidden on a guess | yes |
| **already resolved** | Infrahub has moved on / the IP is no longer duplicated / the GPU gap has closed | hidden |
| **not yet firing** | Prometheus still has it pending | hidden |
| **chronic** | Still true, but firing over 3 days — already ticketed, not new work | hidden |
| **low impact** | Still true, but owned by an internal org or a platform-owned node | hidden |
Screening only ever demotes an alert on **positive evidence**; anything it can't ## Findings that shaped it
settle stays in the queue. Hidden alerts are one checkbox away, and any of them
can be force-diagnosed with **Diagnose anyway**.
### Alert ages are recovered, not taken from Prometheus Validated against production, not assumed:
Prometheus' own `activeAt` is unreliable here. The Infrahub `Resources` metric - **`Suspected Rogue VM` is measuring spare capacity.** `In_Use_Gpus` equals the
drops most of its series for ~5 minutes several times a day (4 dips in the last physical GPU count on 71 of 75 firing hosts, so the rule reduces to "this host
24h observed; one took it from ~4,370 series to 1,359). Every alert alive during has a free GPU". Checked against OpenStack on 10 hosts: Infrahub and OpenStack
a dip resolves and re-fires, so `activeAt` resets on all of them at once — which agreed exactly on all of them. Those alerts are flagged as a rule defect.
is why the Prometheus UI shows dozens of unrelated alerts with the *same* age. - **`Exists in Infrahub but does not exist in OpenStack` matches every VM**,
because `openstack_nova_server_status` returns no series. Excluded outright.
The same gap means `Suspected Orphan VM` cannot fire at all.
- **Prometheus alert ages are unreliable here.** The `Resources` metric drops
most of its series several times a day; every alert alive at the time resolves
and re-fires, resetting `activeAt`. Ages are recovered from `ALERTS` history
instead.
So the app walks the `ALERTS` series backwards over 7 days instead, bridging gaps ## Read-only by design
under 45 minutes, and reports how long each condition has **actually** held. In
practice this is the difference between "40 alerts all 7h old" and "11 that are
genuinely new, 33 that have been true for days". Both numbers are shown: the
recovered duration, with Prometheus' value in a tooltip when they disagree.
The app detects these dips and warns about them, since they also mean any alert The app queries and advises. It never deletes, shelves, or edits a VM — those
with a long `for:` may never reach firing state. stay copy-a-command. The only thing it can send is a Zendesk ticket or a Jira
issue, behind three gates and a confirm step. See
[docs/INTEGRATIONS.md](docs/INTEGRATIONS.md).
The re-check is cheap on purpose: it reads the same Prometheus series the rules ## Run it
are built from — `Resources`, `In_Use_Gpus`, `Total_Gpus` — in one bulk snapshot
for the entire queue, rather than an Infrahub and OpenStack call per alert. Only
alerts you actually open cost a CX-Tools query.
On live data this takes **~2,670 firing alerts down to ~11** that need a decision.
Caches are warmed at startup (~20s, mostly the 7-day history read), so page loads
are instant afterwards. They refresh on a 30s/60s/5min cadence.
**Excluded outright:** `Exists in Infrahub but does not exist in OpenStack`. It is
built as `Resources unless on(openstack_id) openstack_nova_server_status`, and
that second metric is currently returning **zero series** — so nothing gets
excluded by the `unless` and every Infrahub VM alerts. It is a broken exporter,
not a queue of work. The app detects this class of failure and shows a banner,
because the same gap also means `Suspected Orphan VM` cannot fire at all.
## Two tabs
- **CX runbooks** — the alerts below, grouped into collapsible sections in
working order (rogue VMs, duplicate IPs, total GPUs, hibernating, creating,
shutoff, deleting, error), each showing how long it has been firing, **newest
first** so long-running alerts sink to the bottom.
- **Infrastructure** — everything else, so it stays out of the triage queue:
node-exporter host alerts in their own section, then Ceph, MySQL, Galera,
OpenStack services, blackbox. Listed and counted, not diagnosed. Routing is
keyed off the **rule file**, not the alert name, because two different rule
files both use the group name "Imported Rules".
## What it covers
One runbook per alert type, from *Infrahub Errors Remediation*:
| Alert | Priority | What the app works out for you |
|---|---|---|
| Instance in ERROR state | LOWHIGH | Matches the fault against the runbook fault table; decides whether the VM was ever ACTIVE (which changes both the urgency and the comms template); for the NUMA/PCI fault it sums GPUs on the host to check whether the host is full before you escalate |
| Instance in DELETING state | LOW | Confirms the delete request in Infrahub events, and whether the OpenStack server is still there or already gone |
| Instance in SHUTOFF state | LOW | Confirms SHUTOFF and drafts the billing-awareness note |
| Instance in HIBERNATING state | HIGH | Runs the host signals (Nova state/status, disabled reason, OVS liveness) and escalates when they're bad |
| Instance in CREATING state | MEDIUM | Determines whether the VM ever got an OpenStack ID |
| Instance in RESTORING state | HIGH | Host signals plus the most recent *failed* OpenStack event to escalate |
| Instance in REBOOTING state | HIGH | Confirms `InstanceRebootRequest` and the expected `HARD_REBOOT` state |
| Instance in BUILD state | MEDIUM | Distinguishes "large flavor, still transient" from "stuck, escalate" |
| Suspected Rogue VM | HIGH | Quantifies the per-host GPU accounting gap the rule actually fires on, then reconciles every instance on the host and maps each mismatch to its row in the Mismatch Remediation table; tempest instances are ignored |
| Duplicated IPs | HIGH | Classifies each claimant of the IP as Scenario #1 / #2 / rightful owner, and pulls the cross-environment claimant list from `Resources{floating_ip=...}` |
| Problem with Total GPUs | HIGH | Lists which customers are on the affected host |
| Suspected Orphan VM | HIGH | Same host reconciliation as Rogue VM (cannot currently fire — see the exporter note above) |
| Openstack status=X / Infrahub status!=X | HIGH | Single-VM mismatch, taken through the same remediation table |
### A note on Suspected Rogue VM
The rule is not a status comparison — it is
`sum by(instance)(In_Use_Gpus) - sum by(instance)(Resources{status=~"ACTIVE|SHUTOFF|PRE_ACTIVE"}) >= 1`,
a **per-host GPU accounting gap**. Two different faults produce that gap:
1. instances running on the host that Infrahub has no record of (a true rogue VM), or
2. Infrahub VMs that are ACTIVE but have **no host recorded**, so they are never
counted against the host that is actually running them.
Prometheus cannot tell these apart, so the app states both and lets the
per-instance host reconciliation settle it — a genuine rogue VM shows up as
`Infrahub Missing`. It also reports how many unattributed ACTIVE VMs exist
platform-wide, because that number alone can be large enough to explain the gaps
without any rogue VM existing.
## Requirements
Whatever `vmc` already needs, plus nothing:
- The CX-Tools checkout (`cxlib/` + `vmc`), unmodified
- Python 3 (standard library only — no pip install)
- Docker with the `ca1-osc` / `ca2-osc` / `us1-osc` / `no1-osc` containers running
- A signed-in 1Password CLI session
## Run
```bash ```bash
op signin cp .env.example .env
docker compose up --build
``` ```
```bash More: [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) ·
./cx-triage [docs/LINKAGE.md](docs/LINKAGE.md) · [docs/PLAN.md](docs/PLAN.md)
```
It opens <http://127.0.0.1:8765>. Bound to localhost only.
```bash
./cx-triage --check
```
Runs preflight (CX-Tools located, credentials loaded, containers up, Prometheus
reachable) and exits.
Useful flags: `--port`, `--prometheus <url>`, `--no-open`.
If CX-Tools isn't found automatically, point at it:
```bash
CX_TOOLS_PATH=~/scripts/CX-Tools ./cx-triage
```
## How it reaches things
- **CX-Tools** is imported as a library. `cxbridge.py` calls `collect_vm`,
`collect_host` and the query helpers — the same code paths as `vmc --json`
and guards every OpenStack subcommand against a read-only allowlist, so a bug
here cannot mutate an instance.
- **Prometheus** at `10.11.254.250:9090` is on the internal network, which the
laptop has no route to (10.11.* leaves via the default gateway). So queries are
relayed `docker exec ca1-osc curl ...` — the same trick CX-Tools uses for
OpenStack. A direct HTTP transport is tried first, so this still works from a
host that does have a route. Override the relay with `CX_PROMETHEUS_RELAY`.
## Working an alert
1. Pick an alert from the queue. Or paste an `ALERTS{...}` line or a Prometheus
graph URL into the box.
2. **What the platforms say** — the reconciled Infrahub/OpenStack/InfraInsight
facts, with mismatches called out in red.
3. **Next steps** — the remaining runbook steps, each tagged with its owner (CX /
Infrastructure team / DevOps). Steps the app has already verified are ticked
off, so you can see what's left rather than re-deriving it.
4. **Suggested customer comms** — only when the runbook calls for it. Verbatim
approved wording with the instance name (and floating IP) substituted, above
the organization and owner contacts CX-Tools resolved. Copy it and send it
from HubSpot.
5. **Evidence** — the raw alert labels and the raw CX-Tools output, for pasting
into a Slack thread or a Jira ticket.
## Caveats
These are real limits, not bugs:
- **Production Infrahub only.** CX-Tools queries production. When a Duplicated
IPs or Rogue VM alert points at a PreProd/Staging record, the app says so and
tells you to check the other environments — it can't query them. The
Prometheus `Resources` series does span environments, which is why the
Duplicated IPs view uses it.
- **No InfraInsight SQL.** The DELETING runbook identifies the requesting *user*
via a SQL query. The app confirms the delete request from Infrahub events but
cannot name the requester, and says so when it matters.
- **Host Health Checks is partial.** The app reports the host signals CX-Tools
exposes (Nova state/status, disabled reason, OVS liveness/heartbeat, uptime,
aggregates). The rest of that guide — disk, dmesg, GPU checks — is still
manual, and the app says which part it did.
- **Fault table coverage.** ERROR faults outside the runbook's table produce an
explicit "not in the table, escalate to a peer" verdict rather than a guess.
- **Chronic/low-impact thresholds are judgement calls**, not runbook rules:
3 days for chronic (`CHRONIC_DAYS` in `screening.py`), and "internal" means an
`@nexgencloud.com` owner. Adjust to taste.
- **Recovered ages are bounded by a 7-day window** (`TrueAgeIndex.WINDOW_DAYS`).
Anything older shows as `7d+`.
- **"Chronic" does not mean "ignore".** It means the condition has been true for
days, so it is not *new* work. Several ERROR alerts are 67 days old; if those
have not actually been ticketed, they are a backlog, not noise.
## Layout ## Layout
```text
cx-triage entry point (preflight, then serve)
triagelib/
cxbridge.py read-only adapter over cxlib
prometheus.py alert source, rule index, bulk state snapshot, relay
alerts.py normalization, classification, exclusions, grouping
screening.py noise-vs-real verdicts
runbooks.py the decision engine
comms.py customer comms templates, verbatim from Confluence
server.py HTTP API + background triage jobs
ui.py the single-page UI
tests/test_runbooks.py runbook decisions against fixture payloads
tests/test_screening.py screening, exclusion, tab routing, ordering
``` ```
backend/
app/ FastAPI: config, db, models, auth, delivery, routers
triagelib/ the triage engine (screening, runbooks, comms, linkage)
tests/ pure-logic tests — no network, no CX-Tools
frontend/ React + TypeScript + Vite
deploy/k8s/ manifests, rendered by the pipeline
.gitea/workflows/ test → build → deploy
```
## Tests
```bash ```bash
python3 tests/test_runbooks.py && python3 tests/test_screening.py cd backend && python tests/test_runbooks.py && python tests/test_screening.py
``` ```
## Updating a runbook
The decision logic is meant to be edited by whoever owns the runbook:
- Fault table → `FAULT_TABLE` in `runbooks.py`
- Rogue VM mismatch table → `MISMATCH_TABLE` in `runbooks.py`
- Priority / ETTR → `KIND_META` in `alerts.py`
- Queue order → `FOCUS_ORDER` in `alerts.py`
- Alerts to suppress → `EXCLUDED_ALERTNAMES` in `alerts.py`
- Noise rules → `screening.py` (`CHRONIC_DAYS`, `_KIND_SCREENS`)
- Customer wording → `_TEMPLATES` in `comms.py`
If a wording change lands in Confluence, change it in `comms.py` and nowhere
else.

35
backend/Dockerfile Normal file
View File

@@ -0,0 +1,35 @@
# ---- frontend ---------------------------------------------------------------
FROM node:22-alpine AS ui
WORKDIR /ui
COPY frontend/package.json frontend/package-lock.json* ./
RUN npm ci --no-audit --no-fund 2>/dev/null || npm install --no-audit --no-fund
COPY frontend/ .
RUN npm run build
# ---- backend ----------------------------------------------------------------
FROM python:3.12-slim
ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1
# curl is what the triage engine shells out to for the Infrahub API; the docker
# CLI is only needed when Prometheus/OpenStack are reachable through the
# CX-Tools containers rather than directly (see docs/DEPLOYMENT.md).
RUN apt-get update \
&& apt-get install -y --no-install-recommends curl ca-certificates docker.io \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY backend/requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY backend/app ./app
COPY backend/triagelib ./triagelib
COPY --from=ui /ui/dist ./static
RUN useradd --uid 10001 --create-home cx && mkdir -p /data && chown -R cx /data /app
USER cx
ENV CX_STATIC_DIR=/app/static CX_DATABASE_URL=sqlite:////data/cx-triage.db
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=5s --start-period=40s \
CMD python -c "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/api/health').read()"
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"]

2
backend/app/__init__.py Normal file
View File

@@ -0,0 +1,2 @@
"""CX Triage backend."""
VERSION = "0.2.0"

253
backend/app/auth.py Normal file
View File

@@ -0,0 +1,253 @@
"""Authentication: signed session cookies, local accounts, and OIDC.
Local accounts exist so the tool runs on a laptop and in a cluster that has not
been wired to SSO yet. When CX_OIDC_ENABLED is set, Authentik becomes the source
of truth: users are created on first login, and admin rights follow a group
claim rather than being set here.
The session itself is the same either way - a signed, expiring cookie - so the
rest of the app never has to care which provider a user came from.
"""
from __future__ import annotations
import base64
import datetime as dt
import hashlib
import hmac
import json
import secrets
import time
from typing import Any, Optional
import httpx
from fastapi import Depends, HTTPException, Request, status
from sqlalchemy.orm import Session
from .config import get_settings
from .db import get_db
from .models import AuthProvider, User
SESSION_COOKIE = "cx_session"
settings = get_settings()
# --- password hashing -------------------------------------------------------
# PBKDF2 from the standard library: no native build step in the image, and
# strong enough for a small internal user table.
def hash_password(password: str) -> str:
salt = secrets.token_bytes(16)
digest = hashlib.pbkdf2_hmac("sha256", password.encode(), salt, 240_000)
return f"pbkdf2_sha256$240000${base64.b64encode(salt).decode()}${base64.b64encode(digest).decode()}"
def verify_password(password: str, stored: str) -> bool:
try:
algo, rounds, salt_b64, digest_b64 = stored.split("$")
if algo != "pbkdf2_sha256":
return False
expected = base64.b64decode(digest_b64)
actual = hashlib.pbkdf2_hmac("sha256", password.encode(),
base64.b64decode(salt_b64), int(rounds))
return hmac.compare_digest(expected, actual)
except (ValueError, TypeError):
return False
# --- session cookie ---------------------------------------------------------
def _sign(payload: bytes) -> str:
mac = hmac.new(settings.secret_key.encode(), payload, hashlib.sha256).digest()
return f"{base64.urlsafe_b64encode(payload).decode()}.{base64.urlsafe_b64encode(mac).decode()}"
def issue_session(user_id: int) -> str:
payload = json.dumps({
"uid": user_id,
"exp": int(time.time()) + settings.session_hours * 3600,
}).encode()
return _sign(payload)
def read_session(token: str) -> Optional[int]:
try:
body_b64, mac_b64 = token.split(".")
payload = base64.urlsafe_b64decode(body_b64)
expected = hmac.new(settings.secret_key.encode(), payload, hashlib.sha256).digest()
if not hmac.compare_digest(expected, base64.urlsafe_b64decode(mac_b64)):
return None
data = json.loads(payload)
if int(data.get("exp", 0)) < time.time():
return None
return int(data["uid"])
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
return None
# --- dependencies -----------------------------------------------------------
def current_user(request: Request, db: Session = Depends(get_db)) -> User:
token = request.cookies.get(SESSION_COOKIE, "")
uid = read_session(token) if token else None
if not uid:
raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Not signed in")
user = db.get(User, uid)
if not user or not user.is_active:
raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Account is not active")
return user
def require_admin(user: User = Depends(current_user)) -> User:
if not user.is_admin:
raise HTTPException(status.HTTP_403_FORBIDDEN, "Administrator access required")
return user
def optional_user(request: Request, db: Session = Depends(get_db)) -> Optional[User]:
token = request.cookies.get(SESSION_COOKIE, "")
uid = read_session(token) if token else None
return db.get(User, uid) if uid else None
# --- local accounts ---------------------------------------------------------
def authenticate_local(db: Session, email: str, password: str) -> Optional[User]:
if not settings.auth_local_enabled:
return None
user = db.query(User).filter(User.email == email.strip().lower()).one_or_none()
if not user or not user.is_active or user.provider != AuthProvider.LOCAL:
return None
if not user.password_hash or not verify_password(password, user.password_hash):
return None
return user
def ensure_bootstrap_admin(db: Session) -> Optional[str]:
"""Create the first admin so a fresh deployment is reachable.
Only ever runs when the user table is empty, and only with a password
supplied through the environment - it never invents one.
"""
if db.query(User).count():
return None
if not settings.bootstrap_admin_password:
return ("No users exist and CX_BOOTSTRAP_ADMIN_PASSWORD is unset. "
"Set it and restart, or sign in through SSO.")
db.add(User(
email=settings.bootstrap_admin_email.strip().lower(),
name="Bootstrap admin",
is_admin=True,
provider=AuthProvider.LOCAL,
password_hash=hash_password(settings.bootstrap_admin_password),
))
db.commit()
return f"Created bootstrap admin {settings.bootstrap_admin_email}"
# --- OIDC / Authentik -------------------------------------------------------
class OIDCError(RuntimeError):
pass
_discovery_cache: dict[str, Any] = {}
async def oidc_discovery() -> dict[str, Any]:
"""Fetch and cache the provider metadata."""
if not settings.oidc_issuer:
raise OIDCError("CX_OIDC_ISSUER is not set.")
if _discovery_cache.get("_issuer") == settings.oidc_issuer:
return _discovery_cache
url = settings.oidc_issuer.rstrip("/") + "/.well-known/openid-configuration"
async with httpx.AsyncClient(timeout=10) as client:
resp = await client.get(url)
if resp.status_code != 200:
raise OIDCError(f"OIDC discovery failed ({resp.status_code}) at {url}")
data = resp.json()
data["_issuer"] = settings.oidc_issuer
_discovery_cache.clear()
_discovery_cache.update(data)
return data
def oidc_state() -> str:
"""A signed, short-lived value tying the callback to this browser."""
payload = json.dumps({"n": secrets.token_urlsafe(16), "exp": int(time.time()) + 600}).encode()
return _sign(payload)
def oidc_state_valid(state: str) -> bool:
try:
body_b64, mac_b64 = state.split(".")
payload = base64.urlsafe_b64decode(body_b64)
expected = hmac.new(settings.secret_key.encode(), payload, hashlib.sha256).digest()
if not hmac.compare_digest(expected, base64.urlsafe_b64decode(mac_b64)):
return False
return int(json.loads(payload).get("exp", 0)) >= time.time()
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
return False
async def oidc_exchange(code: str, redirect_uri: str) -> dict[str, Any]:
meta = await oidc_discovery()
async with httpx.AsyncClient(timeout=15) as client:
token_resp = await client.post(meta["token_endpoint"], data={
"grant_type": "authorization_code",
"code": code,
"redirect_uri": redirect_uri,
"client_id": settings.oidc_client_id,
"client_secret": settings.oidc_client_secret,
}, headers={"Accept": "application/json"})
if token_resp.status_code != 200:
raise OIDCError(f"Token exchange failed ({token_resp.status_code}): {token_resp.text[:200]}")
access = token_resp.json().get("access_token")
if not access:
raise OIDCError("Token response contained no access_token.")
info_resp = await client.get(meta["userinfo_endpoint"],
headers={"Authorization": f"Bearer {access}"})
if info_resp.status_code != 200:
raise OIDCError(f"userinfo failed ({info_resp.status_code}): {info_resp.text[:200]}")
return info_resp.json()
def upsert_oidc_user(db: Session, claims: dict[str, Any]) -> User:
"""Find or create the local record for an SSO identity.
Matching is on `sub` first so a changed email still lands on the same user;
an existing local account with the same address is adopted rather than
duplicated.
"""
sub = str(claims.get("sub") or "").strip()
email = str(claims.get("email") or "").strip().lower()
if not sub and not email:
raise OIDCError("SSO returned neither sub nor email.")
groups = claims.get(settings.oidc_groups_claim) or []
if isinstance(groups, str):
groups = [groups]
is_admin = settings.oidc_admin_group in {str(g) for g in groups}
user = None
if sub:
user = db.query(User).filter(User.oidc_sub == sub).one_or_none()
if user is None and email:
user = db.query(User).filter(User.email == email).one_or_none()
if user is None:
user = User(email=email or f"{sub}@sso.local", provider=AuthProvider.OIDC)
db.add(user)
user.oidc_sub = sub or user.oidc_sub
user.provider = AuthProvider.OIDC
user.name = str(claims.get("name") or claims.get("preferred_username") or user.name or email)
if email:
user.email = email
user.is_admin = is_admin
user.is_active = True
user.last_login = dt.datetime.now(dt.timezone.utc)
if not user.signoff_name:
user.signoff_name = user.name
db.commit()
return user

101
backend/app/config.py Normal file
View File

@@ -0,0 +1,101 @@
"""Environment-driven configuration.
Everything deployment-specific comes from the environment so the same image runs
locally under compose and in Kubernetes with only a ConfigMap/Secret difference.
"""
from __future__ import annotations
import os
from functools import lru_cache
def _bool(name: str, default: bool = False) -> bool:
return str(os.environ.get(name, str(default))).strip().lower() in {"1", "true", "yes", "on"}
def _int(name: str, default: int) -> int:
try:
return int(os.environ.get(name, default))
except (TypeError, ValueError):
return default
class Settings:
# --- app ---------------------------------------------------------------
app_name = os.environ.get("CX_APP_NAME", "CX Triage")
base_url = os.environ.get("CX_BASE_URL", "http://localhost:8080")
secret_key = os.environ.get("CX_SECRET_KEY", "dev-only-change-me")
session_hours = _int("CX_SESSION_HOURS", 12)
static_dir = os.environ.get("CX_STATIC_DIR", "/app/static")
# --- database ----------------------------------------------------------
# sqlite for local/compose, postgres in the cluster.
database_url = os.environ.get("CX_DATABASE_URL", "sqlite:////data/cx-triage.db")
# --- data sources ------------------------------------------------------
prometheus_base = os.environ.get("CX_PROMETHEUS_BASE", "http://10.11.254.250:9090")
prometheus_relay = os.environ.get("CX_PROMETHEUS_RELAY", "")
cx_tools_path = os.environ.get("CX_TOOLS_PATH", "")
# --- auth --------------------------------------------------------------
# Local accounts are for development and for a cluster without SSO yet.
# When CX_OIDC_ENABLED is on, Authentik becomes the source of truth.
auth_local_enabled = _bool("CX_AUTH_LOCAL_ENABLED", True)
bootstrap_admin_email = os.environ.get("CX_BOOTSTRAP_ADMIN_EMAIL", "admin@localhost")
bootstrap_admin_password = os.environ.get("CX_BOOTSTRAP_ADMIN_PASSWORD", "")
oidc_enabled = _bool("CX_OIDC_ENABLED", False)
oidc_issuer = os.environ.get("CX_OIDC_ISSUER", "") # e.g. https://sso/application/o/cx-triage/
oidc_client_id = os.environ.get("CX_OIDC_CLIENT_ID", "")
oidc_client_secret = os.environ.get("CX_OIDC_CLIENT_SECRET", "")
oidc_scopes = os.environ.get("CX_OIDC_SCOPES", "openid email profile")
oidc_admin_group = os.environ.get("CX_OIDC_ADMIN_GROUP", "cx-triage-admins")
oidc_groups_claim = os.environ.get("CX_OIDC_GROUPS_CLAIM", "groups")
# --- feature flags -----------------------------------------------------
# Sending must be switched on deliberately; a demo instance cannot email.
feature_send_enabled = _bool("CX_FEATURE_SEND_ENABLED", False)
feature_zendesk = _bool("CX_FEATURE_ZENDESK", False)
feature_jira = _bool("CX_FEATURE_JIRA", False)
feature_linkage_scan = _bool("CX_FEATURE_LINKAGE_SCAN", True)
send_daily_cap = _int("CX_SEND_DAILY_CAP", 25)
# --- integrations ------------------------------------------------------
zendesk_subdomain = os.environ.get("CX_ZENDESK_SUBDOMAIN", "")
zendesk_email = os.environ.get("CX_ZENDESK_EMAIL", "")
zendesk_token = os.environ.get("CX_ZENDESK_TOKEN", "")
zendesk_default_public = _bool("CX_ZENDESK_PUBLIC_REPLY", True)
jira_base = os.environ.get("CX_JIRA_BASE", "")
jira_email = os.environ.get("CX_JIRA_EMAIL", "")
jira_token = os.environ.get("CX_JIRA_TOKEN", "")
jira_project = os.environ.get("CX_JIRA_PROJECT", "INFRA")
jira_issue_type = os.environ.get("CX_JIRA_ISSUE_TYPE", "Task")
@property
def zendesk_ready(self) -> bool:
return bool(self.feature_zendesk and self.zendesk_subdomain
and self.zendesk_email and self.zendesk_token)
@property
def jira_ready(self) -> bool:
return bool(self.feature_jira and self.jira_base and self.jira_email
and self.jira_token and self.jira_project)
def public_flags(self) -> dict:
"""What the frontend is allowed to know - never secrets."""
return {
"app_name": self.app_name,
"oidc_enabled": self.oidc_enabled,
"local_login": self.auth_local_enabled,
"zendesk_ready": self.zendesk_ready,
"jira_ready": self.jira_ready,
"send_enabled": self.feature_send_enabled,
"linkage_scan": self.feature_linkage_scan,
"jira_project": self.jira_project if self.jira_ready else "",
}
@lru_cache
def get_settings() -> Settings:
return Settings()

29
backend/app/db.py Normal file
View File

@@ -0,0 +1,29 @@
"""Database engine and session handling."""
from __future__ import annotations
from collections.abc import Iterator
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
from .config import get_settings
settings = get_settings()
_connect_args = {"check_same_thread": False} if settings.database_url.startswith("sqlite") else {}
engine = create_engine(settings.database_url, pool_pre_ping=True, connect_args=_connect_args)
SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
def get_db() -> Iterator[Session]:
db = SessionLocal()
try:
yield db
finally:
db.close()
def init_db() -> None:
from . import models # noqa: F401 (registers the tables)
models.Base.metadata.create_all(engine)

174
backend/app/delivery.py Normal file
View File

@@ -0,0 +1,174 @@
"""Outbound delivery to Zendesk and Jira.
Three independent gates have to be open before anything leaves this process:
1. the integration is configured (subdomain/email/token present)
2. its feature flag is on - CX_FEATURE_ZENDESK / CX_FEATURE_JIRA
3. sending is globally enabled - CX_FEATURE_SEND_ENABLED
A demo or staging instance simply leaves the third off, and then no combination
of clicks can email a customer. Every send is recorded as a case event before it
is attempted, so an audit trail exists even when the call fails.
"""
from __future__ import annotations
import datetime as dt
from typing import Any, Optional
import httpx
from sqlalchemy.orm import Session
from .config import get_settings
from .models import Case, CaseEvent, CaseStatus, User
from .services import add_event, set_status
settings = get_settings()
class DeliveryError(RuntimeError):
pass
def _guard(kind: str) -> None:
if not settings.feature_send_enabled:
raise DeliveryError(
"Sending is disabled on this instance (CX_FEATURE_SEND_ENABLED is off). "
"The payload is ready but nothing will leave the server."
)
if kind == "zendesk" and not settings.zendesk_ready:
raise DeliveryError("Zendesk is not configured. Set CX_FEATURE_ZENDESK plus "
"CX_ZENDESK_SUBDOMAIN, CX_ZENDESK_EMAIL and CX_ZENDESK_TOKEN.")
if kind == "jira" and not settings.jira_ready:
raise DeliveryError("Jira is not configured. Set CX_FEATURE_JIRA plus "
"CX_JIRA_BASE, CX_JIRA_EMAIL, CX_JIRA_TOKEN and CX_JIRA_PROJECT.")
def sends_today(db: Session) -> int:
since = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
return (db.query(CaseEvent)
.filter(CaseEvent.action.in_(["zendesk_sent", "jira_created"]))
.filter(CaseEvent.created_at >= since).count())
def _check_cap(db: Session) -> None:
used = sends_today(db)
if used >= settings.send_daily_cap:
raise DeliveryError(
f"Daily send cap reached ({used}/{settings.send_daily_cap}). Raise CX_SEND_DAILY_CAP "
"if this is deliberate - the cap exists so a loop cannot mail every customer."
)
# --- Zendesk ----------------------------------------------------------------
async def send_zendesk(db: Session, case: Case, actor: User, *, to: str, subject: str,
body: str, priority: str = "normal",
tags: Optional[list[str]] = None, public: Optional[bool] = None) -> dict[str, Any]:
_guard("zendesk")
_check_cap(db)
if not to.strip():
raise DeliveryError("No recipient address.")
base = f"https://{settings.zendesk_subdomain}.zendesk.com/api/v2"
auth = (f"{settings.zendesk_email}/token", settings.zendesk_token)
external_id = f"cx-triage-{case.fingerprint}"
async with httpx.AsyncClient(timeout=30) as client:
# Search first so a re-diagnosed alert comments on the existing ticket
# instead of opening a second one for the same customer.
found = await client.get(f"{base}/search.json",
params={"query": f'type:ticket external_id:"{external_id}"'}, auth=auth)
existing = None
if found.status_code == 200:
results = found.json().get("results") or []
existing = results[0] if results else None
comment = {"body": body, "public": settings.zendesk_default_public if public is None else public}
if existing:
resp = await client.put(f"{base}/tickets/{existing['id']}.json",
json={"ticket": {"comment": comment}}, auth=auth)
action = "updated"
else:
payload = {"ticket": {
"subject": subject,
"comment": comment,
"requester": {"name": to.split("@")[0], "email": to},
"priority": priority,
"type": "incident",
"tags": tags or ["cx-triage", f"alert-{case.kind}"],
"external_id": external_id,
}}
resp = await client.post(f"{base}/tickets.json", json=payload, auth=auth)
action = "created"
if resp.status_code not in (200, 201):
add_event(db, case, actor, "zendesk_failed", f"HTTP {resp.status_code}: {resp.text[:300]}")
db.commit()
raise DeliveryError(f"Zendesk returned {resp.status_code}: {resp.text[:300]}")
ticket = resp.json().get("ticket") or {}
ticket_id = str(ticket.get("id") or (existing or {}).get("id") or "")
url = f"https://{settings.zendesk_subdomain}.zendesk.com/agent/tickets/{ticket_id}"
case.zendesk_ticket_id = ticket_id
case.zendesk_ticket_url = url
add_event(db, case, actor, "zendesk_sent",
f"Ticket {ticket_id} {action} for {to}",
{"ticket_id": ticket_id, "to": to, "subject": subject, "action": action})
if case.status in (CaseStatus.NEW, CaseStatus.INVESTIGATING):
set_status(db, case, CaseStatus.CUSTOMER_CONTACTED, actor, f"Zendesk ticket {ticket_id}")
db.commit()
return {"ok": True, "ticket_id": ticket_id, "url": url, "action": action}
# --- Jira -------------------------------------------------------------------
async def create_jira(db: Session, case: Case, actor: User, *, summary: str, description: str,
project: str = "", issue_type: str = "",
labels: Optional[list[str]] = None) -> dict[str, Any]:
_guard("jira")
_check_cap(db)
base = settings.jira_base.rstrip("/")
auth = (settings.jira_email, settings.jira_token)
label = f"cx-triage-{case.fingerprint}"
all_labels = sorted(set((labels or []) + ["cx-triage", label]))
async with httpx.AsyncClient(timeout=30) as client:
found = await client.get(f"{base}/rest/api/3/search",
params={"jql": f'labels = "{label}"', "maxResults": 1}, auth=auth)
if found.status_code == 200 and (found.json().get("issues") or []):
issue = found.json()["issues"][0]
key = issue["key"]
url = f"{base}/browse/{key}"
case.jira_issue_key, case.jira_issue_url = key, url
add_event(db, case, actor, "jira_exists", f"Issue {key} already exists for this case")
db.commit()
return {"ok": True, "key": key, "url": url, "action": "existing"}
payload = {"fields": {
"project": {"key": project or settings.jira_project},
"summary": summary[:250],
"issuetype": {"name": issue_type or settings.jira_issue_type},
"labels": all_labels,
"description": {
"type": "doc", "version": 1,
"content": [{"type": "paragraph",
"content": [{"type": "text", "text": description[:30000]}]}],
},
}}
resp = await client.post(f"{base}/rest/api/3/issue", json=payload, auth=auth)
if resp.status_code not in (200, 201):
add_event(db, case, actor, "jira_failed", f"HTTP {resp.status_code}: {resp.text[:300]}")
db.commit()
raise DeliveryError(f"Jira returned {resp.status_code}: {resp.text[:300]}")
key = resp.json().get("key", "")
url = f"{base}/browse/{key}"
case.jira_issue_key, case.jira_issue_url = key, url
add_event(db, case, actor, "jira_created", f"Issue {key} created", {"key": key, "summary": summary})
if case.status in (CaseStatus.NEW, CaseStatus.INVESTIGATING):
set_status(db, case, CaseStatus.ESCALATED_INFRA, actor, f"Jira {key}")
db.commit()
return {"ok": True, "key": key, "url": url, "action": "created"}

66
backend/app/main.py Normal file
View File

@@ -0,0 +1,66 @@
"""FastAPI application entry point."""
from __future__ import annotations
import os
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from . import VERSION
from .auth import ensure_bootstrap_admin
from .config import get_settings
from .db import SessionLocal, init_db
from .routers import (actions_router, alerts_router, auth_router, cases_router,
linkage_router, settings_router)
settings = get_settings()
@asynccontextmanager
async def lifespan(app: FastAPI):
init_db()
db = SessionLocal()
try:
message = ensure_bootstrap_admin(db)
if message:
print(f"[auth] {message}")
finally:
db.close()
print(f"[startup] {settings.app_name} {VERSION}")
print(f"[startup] prometheus: {settings.prometheus_base}")
print(f"[startup] sso: {'on' if settings.oidc_enabled else 'off'} | "
f"zendesk: {'ready' if settings.zendesk_ready else 'off'} | "
f"jira: {'ready' if settings.jira_ready else 'off'} | "
f"sending: {'ENABLED' if settings.feature_send_enabled else 'disabled'}")
alerts_router.engine.warm()
yield
app = FastAPI(title=settings.app_name, version=VERSION, lifespan=lifespan)
for module in (auth_router, alerts_router, cases_router, settings_router,
linkage_router, actions_router):
app.include_router(module.router)
@app.get("/api/health")
def health():
return {"status": "ok", "version": VERSION, "config": settings.public_flags()}
# The built React bundle is copied into the image; in development Vite serves it
# instead and proxies /api here, so a missing directory is not an error.
if os.path.isdir(settings.static_dir):
app.mount("/assets", StaticFiles(directory=os.path.join(settings.static_dir, "assets")),
name="assets")
@app.get("/{full_path:path}")
def spa(full_path: str):
if full_path.startswith("api/"):
return JSONResponse({"detail": "Not found"}, status_code=404)
index = os.path.join(settings.static_dir, "index.html")
if os.path.isfile(index):
return FileResponse(index)
return JSONResponse({"detail": "Frontend not built"}, status_code=404)

208
backend/app/models.py Normal file
View File

@@ -0,0 +1,208 @@
"""Persistent state: who is working what, and what has been done about it.
The alert queue itself stays stateless - it is recomputed from Prometheus every
minute. What is worth persisting is the human layer on top: which alerts someone
has picked up, what was done, and the audit trail behind it. Cases are keyed by
the alert fingerprint so an alert that stops and re-fires lands back on the same
case rather than losing its history.
"""
from __future__ import annotations
import datetime as dt
import enum
from typing import Any
from sqlalchemy import (JSON, Boolean, DateTime, Enum, ForeignKey, Index, Integer,
String, Text, UniqueConstraint)
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
class Base(DeclarativeBase):
pass
def _now() -> dt.datetime:
return dt.datetime.now(dt.timezone.utc)
class CaseStatus(str, enum.Enum):
"""Where a piece of work has got to."""
NEW = "new" # seen, nobody has touched it
INVESTIGATING = "investigating" # someone has picked it up
CUSTOMER_CONTACTED = "customer_contacted"
ESCALATED_INFRA = "escalated_infra"
WAITING_CUSTOMER = "waiting_customer"
WAITING_INFRA = "waiting_infra"
REMEDIATED = "remediated" # fixed, waiting for the alert to clear
RESOLVED = "resolved"
WONT_FIX = "wont_fix" # deliberate no-action
FALSE_POSITIVE = "false_positive" # the alert itself was wrong
OPEN_STATUSES = {
CaseStatus.NEW, CaseStatus.INVESTIGATING, CaseStatus.CUSTOMER_CONTACTED,
CaseStatus.ESCALATED_INFRA, CaseStatus.WAITING_CUSTOMER, CaseStatus.WAITING_INFRA,
CaseStatus.REMEDIATED,
}
class AuthProvider(str, enum.Enum):
LOCAL = "local"
OIDC = "oidc"
class User(Base):
__tablename__ = "users"
id: Mapped[int] = mapped_column(primary_key=True)
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
name: Mapped[str] = mapped_column(String(200), default="")
is_admin: Mapped[bool] = mapped_column(Boolean, default=False)
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
provider: Mapped[AuthProvider] = mapped_column(Enum(AuthProvider), default=AuthProvider.LOCAL)
# Local accounts only; OIDC users never have one.
password_hash: Mapped[str] = mapped_column(String(255), default="")
# Stable Authentik subject, so a rename or email change keeps the same user.
oidc_sub: Mapped[str] = mapped_column(String(255), default="", index=True)
signoff_name: Mapped[str] = mapped_column(String(200), default="")
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
last_login: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
def to_json(self) -> dict[str, Any]:
return {
"id": self.id, "email": self.email, "name": self.name or self.email,
"is_admin": self.is_admin, "provider": self.provider.value,
"signoff_name": self.signoff_name or self.name,
}
class Case(Base):
"""One tracked alert, keyed by its fingerprint."""
__tablename__ = "cases"
__table_args__ = (Index("ix_cases_status_seen", "status", "last_seen_at"),)
id: Mapped[int] = mapped_column(primary_key=True)
fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True)
kind: Mapped[str] = mapped_column(String(40), index=True)
title: Mapped[str] = mapped_column(String(300), default="")
subject: Mapped[str] = mapped_column(String(300), default="") # VM name, host or IP
openstack_id: Mapped[str] = mapped_column(String(64), default="", index=True)
instance_name: Mapped[str] = mapped_column(String(200), default="", index=True)
host: Mapped[str] = mapped_column(String(120), default="", index=True)
region: Mapped[str] = mapped_column(String(16), default="")
org_id: Mapped[str] = mapped_column(String(32), default="", index=True)
org_name: Mapped[str] = mapped_column(String(300), default="")
status: Mapped[CaseStatus] = mapped_column(Enum(CaseStatus), default=CaseStatus.NEW, index=True)
assignee_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True)
assignee: Mapped[User | None] = relationship(lazy="joined")
# Outbound references, so a case shows what already exists elsewhere.
zendesk_ticket_id: Mapped[str] = mapped_column(String(40), default="")
zendesk_ticket_url: Mapped[str] = mapped_column(String(500), default="")
jira_issue_key: Mapped[str] = mapped_column(String(40), default="")
jira_issue_url: Mapped[str] = mapped_column(String(500), default="")
notes: Mapped[str] = mapped_column(Text, default="")
snooze_until: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
first_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
last_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
closed_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
# How many separate times this alert has come back after being closed.
reopen_count: Mapped[int] = mapped_column(Integer, default=0)
events: Mapped[list["CaseEvent"]] = relationship(
back_populates="case", cascade="all, delete-orphan", order_by="CaseEvent.created_at.desc()")
@property
def is_open(self) -> bool:
return self.status in OPEN_STATUSES
def to_json(self, with_events: bool = False) -> dict[str, Any]:
data = {
"id": self.id, "fingerprint": self.fingerprint, "kind": self.kind,
"title": self.title, "subject": self.subject,
"openstack_id": self.openstack_id, "instance_name": self.instance_name,
"host": self.host, "region": self.region,
"org_id": self.org_id, "org_name": self.org_name,
"status": self.status.value, "is_open": self.is_open,
"assignee": self.assignee.to_json() if self.assignee else None,
"zendesk_ticket_id": self.zendesk_ticket_id,
"zendesk_ticket_url": self.zendesk_ticket_url,
"jira_issue_key": self.jira_issue_key,
"jira_issue_url": self.jira_issue_url,
"notes": self.notes,
"snooze_until": self.snooze_until.isoformat() if self.snooze_until else None,
"first_seen_at": self.first_seen_at.isoformat() if self.first_seen_at else None,
"last_seen_at": self.last_seen_at.isoformat() if self.last_seen_at else None,
"closed_at": self.closed_at.isoformat() if self.closed_at else None,
"reopen_count": self.reopen_count,
}
if with_events:
data["events"] = [e.to_json() for e in self.events]
return data
class CaseEvent(Base):
"""Append-only history. Nothing here is ever edited or deleted."""
__tablename__ = "case_events"
id: Mapped[int] = mapped_column(primary_key=True)
case_id: Mapped[int] = mapped_column(ForeignKey("cases.id"), index=True)
case: Mapped[Case] = relationship(back_populates="events")
actor_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True)
actor: Mapped[User | None] = relationship(lazy="joined")
actor_label: Mapped[str] = mapped_column(String(200), default="") # survives user deletion
action: Mapped[str] = mapped_column(String(60)) # status_changed | zendesk_sent | note | ...
detail: Mapped[str] = mapped_column(Text, default="")
payload: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
def to_json(self) -> dict[str, Any]:
return {
"id": self.id, "action": self.action, "detail": self.detail,
"actor": self.actor_label or (self.actor.email if self.actor else "system"),
"created_at": self.created_at.isoformat() if self.created_at else None,
"payload": self.payload,
}
class SuppressionRule(Base):
"""Alerts the team has decided not to see, with the reason recorded."""
__tablename__ = "suppression_rules"
__table_args__ = (UniqueConstraint("name", name="uq_rule_name"),)
id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(200))
reason: Mapped[str] = mapped_column(Text, default="")
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
# {"kind": ["error"], "organization": ["modal"]} - all keys must match.
conditions: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
created_by: Mapped[str] = mapped_column(String(200), default="")
created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now)
def to_json(self) -> dict[str, Any]:
return {
"id": str(self.id), "name": self.name, "reason": self.reason,
"enabled": self.enabled, "conditions": self.conditions or {},
"created_by": self.created_by,
"created": self.created_at.strftime("%Y-%m-%d") if self.created_at else "",
}
class AppSetting(Base):
"""Small key/value bag for things an admin can change at runtime."""
__tablename__ = "app_settings"
key: Mapped[str] = mapped_column(String(80), primary_key=True)
value: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
updated_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)

View File

View File

@@ -0,0 +1,77 @@
"""Outbound actions: contact the customer, escalate to Infrastructure."""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from ..auth import current_user
from ..config import get_settings
from ..db import get_db
from ..delivery import DeliveryError, create_jira, send_zendesk, sends_today
from ..models import Case, User
router = APIRouter(prefix="/api/actions", tags=["actions"])
settings = get_settings()
class ZendeskBody(BaseModel):
fingerprint: str
to: str
subject: str
body: str
priority: str = "normal"
tags: list[str] = []
public: bool | None = None
class JiraBody(BaseModel):
fingerprint: str
summary: str
description: str
project: str = ""
issue_type: str = ""
labels: list[str] = []
def _case(db: Session, fingerprint: str) -> Case:
found = db.query(Case).filter(Case.fingerprint == fingerprint).one_or_none()
if not found:
raise HTTPException(status.HTTP_404_NOT_FOUND,
"Open the case first - nothing is tracked for that alert yet.")
return found
@router.get("/status")
def action_status(db: Session = Depends(get_db), user: User = Depends(current_user)):
return {
"zendesk_ready": settings.zendesk_ready,
"jira_ready": settings.jira_ready,
"send_enabled": settings.feature_send_enabled,
"sends_today": sends_today(db),
"daily_cap": settings.send_daily_cap,
}
@router.post("/zendesk")
async def zendesk(body: ZendeskBody, db: Session = Depends(get_db),
user: User = Depends(current_user)):
case = _case(db, body.fingerprint)
try:
return await send_zendesk(db, case, user, to=body.to, subject=body.subject,
body=body.body, priority=body.priority,
tags=body.tags or None, public=body.public)
except DeliveryError as exc:
raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc
@router.post("/jira")
async def jira(body: JiraBody, db: Session = Depends(get_db),
user: User = Depends(current_user)):
case = _case(db, body.fingerprint)
try:
return await create_jira(db, case, user, summary=body.summary,
description=body.description, project=body.project,
issue_type=body.issue_type, labels=body.labels or None)
except DeliveryError as exc:
raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc

View File

@@ -0,0 +1,88 @@
"""The alert queue and per-alert diagnosis."""
from __future__ import annotations
import threading
import time
import traceback
import uuid
from concurrent.futures import ThreadPoolExecutor
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from triagelib import runbooks
from ..auth import current_user
from ..db import SessionLocal, get_db
from ..models import User
from ..services import Engine, RuleAdapter, get_or_create_case
router = APIRouter(prefix="/api", tags=["alerts"])
engine = Engine()
_jobs: dict[str, dict[str, Any]] = {}
_jobs_lock = threading.Lock()
_pool = ThreadPoolExecutor(max_workers=3, thread_name_prefix="triage")
JOB_TTL = 30 * 60
class TriageBody(BaseModel):
fingerprint: str
force: bool = False
@router.get("/alerts")
def alert_queue(force: bool = False, db: Session = Depends(get_db),
user: User = Depends(current_user)):
return engine.queue(db, force=force)
@router.post("/triage")
def start_triage(body: TriageBody, db: Session = Depends(get_db),
user: User = Depends(current_user)):
alert = engine.find_alert(db, body.fingerprint)
if alert is None:
raise HTTPException(status.HTTP_404_NOT_FOUND,
"That alert is no longer firing. Refresh the queue.")
case = get_or_create_case(db, alert, user)
job_id = uuid.uuid4().hex[:12]
with _jobs_lock:
_reap()
_jobs[job_id] = {"id": job_id, "state": "running", "created": time.time(),
"result": None, "error": ""}
_pool.submit(_run, job_id, alert, body.force)
return {"job_id": job_id, "alert": alert.to_json(), "case": case.to_json(with_events=True)}
@router.get("/jobs/{job_id}")
def job(job_id: str, user: User = Depends(current_user)):
with _jobs_lock:
found = _jobs.get(job_id)
if not found:
raise HTTPException(status.HTTP_404_NOT_FOUND, "Unknown job")
return found
def _run(job_id: str, alert: Any, force: bool) -> None:
started = time.monotonic()
db = SessionLocal()
try:
diagnosis = runbooks.diagnose(alert, engine.prom, engine.snapshot.get(), force,
RuleAdapter(db))
payload = diagnosis.to_json()
payload["elapsed_seconds"] = round(time.monotonic() - started, 1)
with _jobs_lock:
_jobs[job_id].update({"state": "done", "result": payload})
except Exception:
with _jobs_lock:
_jobs[job_id].update({"state": "error", "error": traceback.format_exc(limit=4)})
finally:
db.close()
def _reap() -> None:
cutoff = time.time() - JOB_TTL
for key in [k for k, v in _jobs.items() if v["created"] < cutoff]:
_jobs.pop(key, None)

View File

@@ -0,0 +1,99 @@
"""Sign-in: local accounts and the Authentik OIDC round trip."""
from __future__ import annotations
import datetime as dt
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from fastapi.responses import RedirectResponse
from pydantic import BaseModel
from sqlalchemy.orm import Session
from .. import auth as auth_mod
from ..config import get_settings
from ..db import get_db
from ..models import User
router = APIRouter(prefix="/api/auth", tags=["auth"])
settings = get_settings()
class LoginBody(BaseModel):
email: str
password: str
def _set_cookie(response: Response, user: User) -> None:
response.set_cookie(
auth_mod.SESSION_COOKIE, auth_mod.issue_session(user.id),
max_age=settings.session_hours * 3600, httponly=True, samesite="lax",
secure=settings.base_url.startswith("https://"), path="/",
)
@router.get("/me")
def me(user: User | None = Depends(auth_mod.optional_user)):
return {"user": user.to_json() if user else None, "config": settings.public_flags()}
@router.post("/login")
def login(body: LoginBody, response: Response, db: Session = Depends(get_db)):
if not settings.auth_local_enabled:
raise HTTPException(status.HTTP_403_FORBIDDEN, "Local login is disabled; use SSO.")
user = auth_mod.authenticate_local(db, body.email, body.password)
if not user:
raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Incorrect email or password")
user.last_login = dt.datetime.now(dt.timezone.utc)
db.commit()
_set_cookie(response, user)
return {"user": user.to_json()}
@router.post("/logout")
def logout(response: Response):
response.delete_cookie(auth_mod.SESSION_COOKIE, path="/")
return {"ok": True}
@router.get("/oidc/start")
async def oidc_start(request: Request):
if not settings.oidc_enabled:
raise HTTPException(status.HTTP_404_NOT_FOUND, "SSO is not enabled")
try:
meta = await auth_mod.oidc_discovery()
except auth_mod.OIDCError as exc:
raise HTTPException(status.HTTP_502_BAD_GATEWAY, str(exc)) from exc
redirect_uri = f"{settings.base_url.rstrip('/')}/api/auth/oidc/callback"
state = auth_mod.oidc_state()
url = (f"{meta['authorization_endpoint']}?response_type=code"
f"&client_id={settings.oidc_client_id}"
f"&redirect_uri={redirect_uri}"
f"&scope={settings.oidc_scopes.replace(' ', '%20')}"
f"&state={state}")
response = RedirectResponse(url, status_code=302)
response.set_cookie("cx_oidc_state", state, max_age=600, httponly=True, samesite="lax", path="/")
return response
@router.get("/oidc/callback")
async def oidc_callback(request: Request, code: str = "", state: str = "",
db: Session = Depends(get_db)):
if not settings.oidc_enabled:
raise HTTPException(status.HTTP_404_NOT_FOUND, "SSO is not enabled")
# The state must match the cookie we set *and* still verify - one guards
# against a swapped browser, the other against a forged value.
if not code or not state or state != request.cookies.get("cx_oidc_state") \
or not auth_mod.oidc_state_valid(state):
raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid or expired SSO state")
redirect_uri = f"{settings.base_url.rstrip('/')}/api/auth/oidc/callback"
try:
claims = await auth_mod.oidc_exchange(code, redirect_uri)
user = auth_mod.upsert_oidc_user(db, claims)
except auth_mod.OIDCError as exc:
raise HTTPException(status.HTTP_502_BAD_GATEWAY, str(exc)) from exc
response = RedirectResponse("/", status_code=302)
_set_cookie(response, user)
response.delete_cookie("cx_oidc_state", path="/")
return response

View File

@@ -0,0 +1,99 @@
"""Case state: the human layer over the alert queue."""
from __future__ import annotations
import datetime as dt
from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from ..auth import current_user
from ..db import get_db
from ..models import Case, CaseStatus, User
from ..services import add_event, set_status
router = APIRouter(prefix="/api/cases", tags=["cases"])
class StatusBody(BaseModel):
status: str
note: str = ""
class NoteBody(BaseModel):
note: str
class SnoozeBody(BaseModel):
hours: int = 24
note: str = ""
def _case(db: Session, fingerprint: str) -> Case:
found = db.query(Case).filter(Case.fingerprint == fingerprint).one_or_none()
if not found:
raise HTTPException(status.HTTP_404_NOT_FOUND, "No case for that alert yet")
return found
@router.get("")
def list_cases(open_only: bool = True, limit: int = 200, db: Session = Depends(get_db),
user: User = Depends(current_user)):
query = db.query(Case).order_by(Case.last_seen_at.desc())
rows = [c for c in query.limit(max(1, min(limit, 1000))).all()
if (c.is_open or not open_only)]
return {"cases": [c.to_json() for c in rows], "statuses": [s.value for s in CaseStatus]}
@router.get("/{fingerprint}")
def get_case(fingerprint: str, db: Session = Depends(get_db), user: User = Depends(current_user)):
return _case(db, fingerprint).to_json(with_events=True)
@router.post("/{fingerprint}/status")
def change_status(fingerprint: str, body: StatusBody, db: Session = Depends(get_db),
user: User = Depends(current_user)):
try:
new_status = CaseStatus(body.status)
except ValueError as exc:
raise HTTPException(status.HTTP_400_BAD_REQUEST,
f"Unknown status '{body.status}'") from exc
case = set_status(db, _case(db, fingerprint), new_status, user, body.note)
return case.to_json(with_events=True)
@router.post("/{fingerprint}/assign")
def assign(fingerprint: str, db: Session = Depends(get_db), user: User = Depends(current_user)):
case = _case(db, fingerprint)
# Set the relationship, not just the id: the response is serialised from
# this same object and a bare id leaves `assignee` null in the payload.
case.assignee = user
if case.status == CaseStatus.NEW:
case.status = CaseStatus.INVESTIGATING
add_event(db, case, user, "assigned", f"Picked up by {user.email}")
db.commit()
db.refresh(case)
return case.to_json(with_events=True)
@router.post("/{fingerprint}/note")
def add_note(fingerprint: str, body: NoteBody, db: Session = Depends(get_db),
user: User = Depends(current_user)):
if not body.note.strip():
raise HTTPException(status.HTTP_400_BAD_REQUEST, "Empty note")
case = _case(db, fingerprint)
case.notes = (case.notes + "\n" if case.notes else "") + body.note.strip()
add_event(db, case, user, "note", body.note.strip())
db.commit()
return case.to_json(with_events=True)
@router.post("/{fingerprint}/snooze")
def snooze(fingerprint: str, body: SnoozeBody, db: Session = Depends(get_db),
user: User = Depends(current_user)):
case = _case(db, fingerprint)
until = dt.datetime.now(dt.timezone.utc) + dt.timedelta(hours=max(1, body.hours))
case.snooze_until = until
add_event(db, case, user, "snoozed", f"Snoozed for {body.hours}h. {body.note}".strip())
db.commit()
return case.to_json(with_events=True)

View File

@@ -0,0 +1,42 @@
"""Linkage scan endpoints."""
from __future__ import annotations
import threading
from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel
from triagelib import linkage as linkage_mod
from ..auth import current_user
from ..config import get_settings
from ..models import User
from ..routers.alerts_router import engine
router = APIRouter(prefix="/api/linkage", tags=["linkage"])
settings = get_settings()
class EnrichBody(BaseModel):
region: str
openstack_id: str
@router.get("")
def scan_state(user: User = Depends(current_user)):
return engine.scan.to_json()
@router.post("/scan")
def start_scan(user: User = Depends(current_user)):
if not settings.feature_linkage_scan:
raise HTTPException(status.HTTP_403_FORBIDDEN, "The linkage scan is disabled on this instance.")
if engine.scan.state == "running":
return {"started": False, "reason": "already running"}
threading.Thread(target=engine.scan.run, args=(engine.snapshot.get(),), daemon=True).start()
return {"started": True}
@router.post("/enrich")
def enrich(body: EnrichBody, user: User = Depends(current_user)):
return linkage_mod.enrich(body.region, body.openstack_id)

View File

@@ -0,0 +1,103 @@
"""Suppression rules and per-instance preferences."""
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from triagelib import settings as legacy
from ..auth import current_user, require_admin
from ..config import get_settings
from ..db import get_db
from ..models import AppSetting, SuppressionRule, User
from ..routers.alerts_router import engine
router = APIRouter(prefix="/api/settings", tags=["settings"])
app_settings = get_settings()
class RuleBody(BaseModel):
id: str | None = None
name: str
reason: str = ""
enabled: bool = True
conditions: dict[str, Any] = {}
class GeneralBody(BaseModel):
agent_name: str | None = None
chronic_days: int | None = None
@router.get("")
def read_settings(db: Session = Depends(get_db), user: User = Depends(current_user)):
row = db.get(AppSetting, "general")
general = (row.value if row else {}) or {}
return {
"rules": [r.to_json() for r in db.query(SuppressionRule).order_by(SuppressionRule.id).all()],
"conditions": legacy.CONDITIONS,
"agent_name": general.get("agent_name") or "",
"chronic_days": general.get("chronic_days") or 3,
"config": app_settings.public_flags(),
}
@router.post("/general")
def save_general(body: GeneralBody, db: Session = Depends(get_db),
user: User = Depends(current_user)):
row = db.get(AppSetting, "general") or AppSetting(key="general", value={})
value = dict(row.value or {})
if body.agent_name is not None:
value["agent_name"] = body.agent_name.strip()
if body.chronic_days is not None:
value["chronic_days"] = max(1, int(body.chronic_days))
row.value = value
db.merge(row)
db.commit()
return read_settings(db, user)
@router.post("/rules")
def save_rule(body: RuleBody, db: Session = Depends(get_db), user: User = Depends(require_admin)):
conditions = {k: v for k, v in (body.conditions or {}).items() if k in legacy.CONDITIONS and v}
if not conditions:
raise HTTPException(status.HTTP_400_BAD_REQUEST,
"A rule needs at least one condition, otherwise it would hide everything.")
rule = db.get(SuppressionRule, int(body.id)) if (body.id or "").isdigit() else None
if rule is None:
rule = SuppressionRule(created_by=user.email)
db.add(rule)
rule.name = body.name.strip() or "Untitled rule"
rule.reason = body.reason.strip()
rule.enabled = body.enabled
rule.conditions = conditions
db.commit()
return read_settings(db, user)
@router.delete("/rules/{rule_id}")
def delete_rule(rule_id: int, db: Session = Depends(get_db), user: User = Depends(require_admin)):
rule = db.get(SuppressionRule, rule_id)
if rule:
db.delete(rule)
db.commit()
return read_settings(db, user)
@router.post("/rules/preview")
def preview_rule(body: RuleBody, db: Session = Depends(get_db), user: User = Depends(current_user)):
"""Show which firing alerts a rule would hide, before it is saved."""
from triagelib import alerts as alertlib
raw, _error, _age = engine.cache.get()
candidates = [a for a in (alertlib.from_prometheus(x, engine.rules) for x in raw)
if not alertlib.is_excluded(a) and alertlib.cx_relevant(a)]
rule = legacy._normalize_rule({"name": body.name, "conditions": body.conditions})
hits = [{
"kind": a.kind, "title": a.title, "instance_name": a.instance_name,
"host": a.host, "org_name": a.org_name, "region": a.region,
} for a in candidates if legacy.rule_matches(rule, a)]
return {"count": len(hits), "matches": hits[:60]}

200
backend/app/services.py Normal file
View File

@@ -0,0 +1,200 @@
"""Glue between the triage engine, the database and the outside world."""
from __future__ import annotations
import datetime as dt
import threading
from typing import Any, Optional
from sqlalchemy.orm import Session
from triagelib import alerts as alertlib, screening, settings as legacy_settings
from triagelib.prometheus import (AlertCache, PrometheusClient, PrometheusError,
RuleIndex, StateSnapshot, TrueAgeIndex)
from triagelib import linkage as linkage_mod
from .config import get_settings
from .models import AppSetting, Case, CaseEvent, CaseStatus, SuppressionRule, User
settings = get_settings()
class RuleAdapter:
"""Presents DB-backed suppression rules the way the engine expects."""
def __init__(self, db: Session):
self._rules = [r.to_json() for r in db.query(SuppressionRule).all()]
row = db.get(AppSetting, "general")
general = (row.value if row else {}) or {}
self._agent = str(general.get("agent_name") or "")
self._chronic = int(general.get("chronic_days") or 3)
@property
def rules(self) -> list[dict[str, Any]]:
return self._rules
@property
def agent_name(self) -> str:
return self._agent
@property
def chronic_days(self) -> int:
return self._chronic
class Engine:
"""Process-wide caches over Prometheus. Cheap to share, expensive to rebuild."""
def __init__(self):
self.prom = PrometheusClient(settings.prometheus_base)
self.cache = AlertCache(self.prom)
self.rules = RuleIndex(self.prom)
self.snapshot = StateSnapshot(self.prom)
self.true_age = TrueAgeIndex(self.prom)
self.scan = linkage_mod.Scan()
self._lock = threading.Lock()
def warm(self, log=print) -> None:
try:
self.rules.ensure()
snap = self.snapshot.get()
log(f" state snapshot: {len(snap.by_openstack_id)} VMs, {len(snap.total_gpus)} hosts")
ages = self.true_age.get()
log(f" alert history: {ages.count} alerts indexed over {ages.WINDOW_DAYS} days")
except PrometheusError as exc:
log(f" WARN: Prometheus caches not warmed: {exc}")
def queue(self, db: Session, force: bool = False) -> dict[str, Any]:
raw, error, age = self.cache.get(force=force)
snap = self.snapshot.get()
ages = self.true_age.get()
parsed = [alertlib.from_prometheus(a, self.rules, ages) for a in raw]
excluded = [a for a in parsed if alertlib.is_excluded(a)]
candidates = [a for a in parsed if not alertlib.is_excluded(a)]
cx = [a for a in candidates if a.category == "cx" and alertlib.cx_relevant(a)]
screening.screen_all(cx, snap, RuleAdapter(db))
cases = {c.fingerprint: c for c in
db.query(Case).filter(Case.fingerprint.in_([a.fingerprint() for a in cx])).all()}
groups = alertlib.group_alerts(cx)
for group in groups:
for item in group["alerts"]:
case = cases.get(item["id"])
item["case"] = case.to_json() if case else None
in_cx = {id(a) for a in cx}
infra = sorted([a for a in candidates if id(a) not in in_cx], key=alertlib.sort_key)
return {
"error": error or snap.error or ages.error,
"warnings": screening.health_warnings(snap),
"totals": {"prometheus": len(parsed), "cx": len(cx),
"infrastructure": len(infra), "excluded": len(excluded)},
"excluded_note": (f"{len(excluded)} '{', '.join(sorted({a.alertname for a in excluded}))}' alerts hidden"
if excluded else ""),
"summary": screening.summarize(cx),
"groups": groups,
"infrastructure": _infra_sections(infra),
"cache_age_seconds": round(age, 1),
}
def find_alert(self, db: Session, fingerprint: str) -> Optional[Any]:
raw, _error, _age = self.cache.get()
ages = self.true_age.get()
for item in raw:
alert = alertlib.from_prometheus(item, self.rules, ages)
if alert.fingerprint() == fingerprint:
alert.screen = screening.screen(alert, self.snapshot.get(), RuleAdapter(db))
return alert
return None
SOURCE_LABELS = {
"node-exporter-rules.yml": "Node exporter (hosts)",
"ceph-rules.yml": "Ceph", "mysql-rules.yml": "MySQL",
"mysql-performance-rules.yml": "MySQL performance", "galera-rules.yml": "Galera",
"openstack-rules.yml": "OpenStack services", "blackbox.yml": "Blackbox / OOB",
"infrahub-rules.yml": "Infrahub (no CX runbook)",
}
def _infra_sections(items: list[Any]) -> list[dict[str, Any]]:
buckets: dict[str, list[Any]] = {}
for alert in items:
buckets.setdefault(alert.rule_file or "unknown", []).append(alert)
sections = []
for source, members in buckets.items():
by_name: dict[str, int] = {}
for alert in members:
name = alertlib.clean_alertname(alert.alertname)
by_name[name] = by_name.get(name, 0) + 1
sections.append({
"source": source, "label": SOURCE_LABELS.get(source, source), "total": len(members),
"by_alertname": sorted(({"name": k, "count": v} for k, v in by_name.items()),
key=lambda x: (-x["count"], x["name"])),
})
sections.sort(key=lambda s: (s["source"] != alertlib.NODE_RULE_FILE, -s["total"]))
return sections
# --- case bookkeeping -------------------------------------------------------
def get_or_create_case(db: Session, alert: Any, actor: Optional[User] = None) -> Case:
case = db.query(Case).filter(Case.fingerprint == alert.fingerprint()).one_or_none()
now = dt.datetime.now(dt.timezone.utc)
subject = (alert.floating_ip if alert.kind == "duplicate_ip"
else alert.host if alert.kind in ("rogue_vm", "total_gpus", "orphan_vm")
else alert.instance_name or alert.openstack_id)
if case is None:
case = Case(
fingerprint=alert.fingerprint(), kind=alert.kind, title=alert.title,
subject=subject or "", openstack_id=alert.openstack_id,
instance_name=alert.instance_name, host=alert.host, region=alert.region,
org_id=alert.org_id, org_name=alert.org_name,
)
db.add(case)
db.flush()
add_event(db, case, actor, "opened", f"Case opened for {alert.title}")
else:
# A closed case whose alert has come back is new work again.
if not case.is_open and case.closed_at:
case.reopen_count += 1
case.status = CaseStatus.NEW
case.closed_at = None
add_event(db, case, None, "reopened",
f"Alert fired again after being {case.status.value}")
case.last_seen_at = now
case.title = alert.title
case.subject = subject or case.subject
db.commit()
return case
def add_event(db: Session, case: Case, actor: Optional[User], action: str,
detail: str = "", payload: Optional[dict[str, Any]] = None) -> CaseEvent:
event = CaseEvent(
actor_id=actor.id if actor else None,
actor_label=(actor.email if actor else "system"),
action=action, detail=detail, payload=payload,
)
# Appended through the relationship rather than inserted by id: sessions use
# expire_on_commit=False, so a collection already loaded would otherwise stay
# stale and the new event would be missing from the response.
case.events.append(event)
db.add(event)
return event
def set_status(db: Session, case: Case, status: CaseStatus, actor: Optional[User],
note: str = "") -> Case:
previous = case.status
case.status = status
if status in (CaseStatus.RESOLVED, CaseStatus.WONT_FIX, CaseStatus.FALSE_POSITIVE):
case.closed_at = dt.datetime.now(dt.timezone.utc)
else:
case.closed_at = None
add_event(db, case, actor, "status_changed",
f"{previous.value} -> {status.value}" + (f": {note}" if note else ""),
{"from": previous.value, "to": status.value})
db.commit()
return case

6
backend/requirements.txt Normal file
View File

@@ -0,0 +1,6 @@
fastapi==0.141.1
uvicorn[standard]==0.34.0
sqlalchemy==2.0.51
httpx==0.28.1
pydantic==2.10.6
psycopg[binary]==3.2.4

124
backend/tests/test_api.py Normal file
View File

@@ -0,0 +1,124 @@
"""API-level tests: auth gates, case lifecycle, suppression rules, send gating.
Runs against an in-memory database with CX-Tools stubbed out, so it needs no
credentials. Prometheus is only touched for cache warming, which is tolerant of
being unreachable.
"""
import os
import sys
import tempfile
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
os.environ.update({
"CX_DATABASE_URL": f"sqlite:///{tempfile.mkdtemp()}/test.db",
"CX_BOOTSTRAP_ADMIN_EMAIL": "admin@localhost",
"CX_BOOTSTRAP_ADMIN_PASSWORD": "test-password",
"CX_SECRET_KEY": "test-secret",
"CX_FEATURE_SEND_ENABLED": "false",
})
from triagelib import cxbridge # noqa: E402
cxbridge.bootstrap = lambda: (_ for _ in ()).throw(cxbridge.BridgeError("stubbed"))
from fastapi.testclient import TestClient # noqa: E402
from app import auth as auth_mod # noqa: E402
from app.main import app # noqa: E402
FAILS = []
def expect(label, cond, got=""):
print((" PASS " if cond else " FAIL ") + label + ("" if cond else f" <- {got}"))
if not cond:
FAILS.append(label)
print("\nPASSWORDS AND SESSIONS")
h = auth_mod.hash_password("hunter2")
expect("correct password verifies", auth_mod.verify_password("hunter2", h))
expect("wrong password rejected", not auth_mod.verify_password("hunter3", h))
expect("hash is salted (two hashes differ)", auth_mod.hash_password("x") != auth_mod.hash_password("x"))
token = auth_mod.issue_session(7)
expect("session round-trips", auth_mod.read_session(token) == 7)
expect("tampered session rejected", auth_mod.read_session(token[:-4] + "aaaa") is None)
expect("garbage session rejected", auth_mod.read_session("not-a-token") is None)
expect("oidc state verifies", auth_mod.oidc_state_valid(auth_mod.oidc_state()))
expect("forged oidc state rejected", not auth_mod.oidc_state_valid("aaa.bbb"))
with TestClient(app) as client:
print("\nAUTH GATES")
expect("health is public", client.get("/api/health").status_code == 200)
expect("alerts need a session", client.get("/api/alerts").status_code == 401)
expect("cases need a session", client.get("/api/cases").status_code == 401)
expect("wrong password is 401", client.post(
"/api/auth/login", json={"email": "admin@localhost", "password": "no"}).status_code == 401)
login = client.post("/api/auth/login", json={"email": "admin@localhost", "password": "test-password"})
expect("login succeeds", login.status_code == 200, login.text[:120])
expect("bootstrap user is admin", login.json()["user"]["is_admin"])
expect("session works after login", client.get("/api/cases").status_code == 200)
print("\nCONFIG EXPOSURE")
cfg = client.get("/api/auth/me").json()["config"]
expect("send disabled by default", cfg["send_enabled"] is False, cfg)
expect("no secret leaks into public config",
not any("token" in k.lower() or "secret" in k.lower() for k in cfg), list(cfg))
print("\nSUPPRESSION RULES")
rule = {"name": "Modal ERROR churn", "reason": "known batch churn",
"conditions": {"kind": ["error"], "organization": ["modal"]}}
expect("admin can save a rule", client.post("/api/settings/rules", json=rule).status_code == 200)
expect("rule is persisted", len(client.get("/api/settings").json()["rules"]) == 1)
expect("a rule with no conditions is refused", client.post(
"/api/settings/rules", json={"name": "catch all", "conditions": {}}).status_code == 400)
expect("unknown condition fields are dropped", client.post(
"/api/settings/rules", json={"name": "bogus", "conditions": {"nope": ["x"]}}).status_code == 400)
print("\nSEND GATING")
status = client.get("/api/actions/status").json()
expect("send reported as disabled", status["send_enabled"] is False)
expect("zendesk reported as not ready", status["zendesk_ready"] is False)
blocked = client.post("/api/actions/zendesk", json={
"fingerprint": "does-not-exist", "to": "a@b.c", "subject": "s", "body": "b"})
expect("sending on an untracked alert is refused", blocked.status_code == 404, blocked.text[:120])
print("\nCASE LIFECYCLE")
from app.db import SessionLocal
from app.models import Case, CaseStatus
from app.services import add_event, set_status
db = SessionLocal()
case = Case(fingerprint="test-fp", kind="error", title="Instance in ERROR state", subject="vm-1")
db.add(case)
db.commit()
expect("new case starts open", case.is_open and case.status == CaseStatus.NEW)
set_status(db, case, CaseStatus.ESCALATED_INFRA, None, "INFRA-1")
expect("escalated is still open", case.is_open)
expect("status change is recorded", any(e.action == "status_changed" for e in case.events))
set_status(db, case, CaseStatus.RESOLVED, None)
expect("resolved closes the case", not case.is_open and case.closed_at is not None)
add_event(db, case, None, "note", "manual note")
db.commit()
expect("history is append-only and ordered newest first",
case.events[0].action in ("note", "status_changed"), [e.action for e in case.events])
payload = case.to_json(with_events=True)
expect("serialises for the API", payload["status"] == "resolved" and len(payload["events"]) >= 3)
db.close()
expect("bad status is rejected", client.post(
"/api/cases/test-fp/status", json={"status": "banana"}).status_code == 400)
expect("unknown case is 404", client.get("/api/cases/nope").status_code == 404)
print("\nLOGOUT")
client.post("/api/auth/logout")
expect("session is cleared", client.get("/api/cases").status_code == 401)
print("\n" + ("ALL CHECKS PASSED" if not FAILS else f"{len(FAILS)} CHECK(S) FAILED: {FAILS}"))
sys.exit(1 if FAILS else 0)

View File

@@ -1,79 +0,0 @@
#!/usr/bin/env python3
"""CX Triage - diagnose Infrahub error alerts using the CX-Tools collectors.
Read-only: this tool queries Infrahub, OpenStack, InfraInsight and Prometheus,
and suggests what to do. It never changes platform state and never sends
customer comms.
"""
from __future__ import annotations
import argparse
import os
import sys
import webbrowser
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from triagelib import VERSION, cxbridge # noqa: E402
from triagelib.prometheus import DEFAULT_BASE, PrometheusClient, PrometheusError # noqa: E402
def preflight(prometheus_base: str) -> bool:
ok = True
print("Locating CX-Tools...")
try:
path = cxbridge.locate_cx_tools()
print(f" found: {path}")
except cxbridge.BridgeError as exc:
print(f" FAIL: {exc}", file=sys.stderr)
return False
# Done in the foreground so any 1Password prompt reaches the terminal
# rather than a background HTTP worker.
print("Loading API credentials from 1Password (CX-Tools loader)...")
try:
cxbridge.bootstrap()
print(" credentials loaded")
except cxbridge.BridgeError as exc:
print(f" FAIL: {exc}", file=sys.stderr)
ok = False
print(f"Checking Prometheus at {prometheus_base}...")
try:
print(f" reachable ({PrometheusClient(prometheus_base).describe_transport()})")
except PrometheusError as exc:
print(f" WARN: {exc}", file=sys.stderr)
print(" The app will still start; paste alerts manually until this is fixed.", file=sys.stderr)
return ok
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--host", default="127.0.0.1", help="bind address (default: localhost only)")
parser.add_argument("--port", type=int, default=8765, help="port (default: 8765)")
parser.add_argument("--prometheus", default=DEFAULT_BASE, help=f"Prometheus base URL (default: {DEFAULT_BASE})")
parser.add_argument("--no-open", action="store_true", help="do not open a browser on start")
parser.add_argument("--check", action="store_true", help="run preflight checks and exit")
parser.add_argument("--version", action="version", version=VERSION)
args = parser.parse_args()
if not preflight(args.prometheus):
return 1
if args.check:
print("preflight OK")
return 0
url = f"http://{args.host}:{args.port}"
if not args.no_open:
webbrowser.open(url)
from triagelib.server import serve
serve(host=args.host, port=args.port, prometheus_base=args.prometheus)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: ${NAMESPACE}

30
deploy/k8s/10-config.yaml Normal file
View File

@@ -0,0 +1,30 @@
# Non-secret configuration. Secrets live in the `cx-triage` Secret, created by
# the pipeline from the repository secret store.
apiVersion: v1
kind: ConfigMap
metadata:
name: cx-triage-config
namespace: ${NAMESPACE}
data:
CX_APP_NAME: "CX Triage"
CX_BASE_URL: "${BASE_URL}"
CX_SESSION_HOURS: "12"
CX_STATIC_DIR: "/app/static"
CX_PROMETHEUS_BASE: "${PROMETHEUS_BASE}"
CX_AUTH_LOCAL_ENABLED: "false"
CX_OIDC_ENABLED: "${OIDC_ENABLED}"
CX_OIDC_ISSUER: "${OIDC_ISSUER}"
CX_OIDC_SCOPES: "openid email profile"
CX_OIDC_ADMIN_GROUP: "${OIDC_ADMIN_GROUP}"
CX_OIDC_GROUPS_CLAIM: "groups"
CX_FEATURE_SEND_ENABLED: "${FEATURE_SEND_ENABLED}"
CX_FEATURE_ZENDESK: "${FEATURE_ZENDESK}"
CX_FEATURE_JIRA: "${FEATURE_JIRA}"
CX_FEATURE_LINKAGE_SCAN: "${FEATURE_LINKAGE_SCAN}"
CX_SEND_DAILY_CAP: "${SEND_DAILY_CAP}"
CX_JIRA_PROJECT: "${JIRA_PROJECT}"
CX_JIRA_ISSUE_TYPE: "Task"

View File

@@ -0,0 +1,51 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: cx-triage
namespace: ${NAMESPACE}
labels: { app: cx-triage }
spec:
replicas: ${REPLICAS}
strategy: { type: RollingUpdate }
selector:
matchLabels: { app: cx-triage }
template:
metadata:
labels: { app: cx-triage }
spec:
securityContext:
runAsNonRoot: true
runAsUser: 10001
fsGroup: 10001
containers:
- name: app
image: ${IMAGE_REF}
imagePullPolicy: IfNotPresent
ports: [{ containerPort: 8080, name: http }]
envFrom:
- configMapRef: { name: cx-triage-config }
- secretRef: { name: cx-triage }
readinessProbe:
httpGet: { path: /api/health, port: http }
initialDelaySeconds: 10
periodSeconds: 10
livenessProbe:
httpGet: { path: /api/health, port: http }
# Startup warms a week of alert history, so allow a slow first start.
initialDelaySeconds: 60
periodSeconds: 30
resources:
requests: { cpu: 100m, memory: 256Mi }
limits: { cpu: "1", memory: 1Gi }
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: { drop: ["ALL"] }
volumeMounts:
- { name: tmp, mountPath: /tmp }
- { name: data, mountPath: /data }
volumes:
- name: tmp
emptyDir: {}
- name: data
emptyDir: {} # state lives in Postgres; this is scratch only

View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Service
metadata:
name: cx-triage
namespace: ${NAMESPACE}
spec:
selector: { app: cx-triage }
ports:
- { name: http, port: 80, targetPort: http }

View File

@@ -0,0 +1,23 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: cx-triage
namespace: ${NAMESPACE}
annotations:
cert-manager.io/cluster-issuer: "${TLS_ISSUER}"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
spec:
ingressClassName: ${INGRESS_CLASS}
tls:
- hosts: ["${DOMAIN}"]
secretName: cx-triage-tls
rules:
- host: ${DOMAIN}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: cx-triage
port: { name: http }

47
docker-compose.yml Normal file
View File

@@ -0,0 +1,47 @@
# Local development and single-host deployment.
#
# cp .env.example .env # then edit
# docker compose up --build
#
# Reaching Prometheus and OpenStack still depends on the CX-Tools containers
# being up on the same host - see docs/DEPLOYMENT.md.
services:
app:
build:
context: .
dockerfile: backend/Dockerfile
image: cx-triage:local
ports:
- "${CX_PORT:-8080}:8080"
env_file: [.env]
environment:
CX_DATABASE_URL: ${CX_DATABASE_URL:-postgresql+psycopg://cx:cx@db:5432/cxtriage}
CX_STATIC_DIR: /app/static
volumes:
# The engine shells out to `docker exec <region>-osc ...`, so it needs the
# host's Docker socket. Mount read-only and drop it if you point the app
# at Prometheus/OpenStack directly instead.
- /var/run/docker.sock:/var/run/docker.sock:ro
- cx-data:/data
depends_on:
db: { condition: service_healthy }
restart: unless-stopped
db:
image: postgres:16-alpine
environment:
POSTGRES_USER: cx
POSTGRES_PASSWORD: cx
POSTGRES_DB: cxtriage
volumes:
- cx-db:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U cx -d cxtriage"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
volumes:
cx-data:
cx-db:

87
docs/DEPLOYMENT.md Normal file
View File

@@ -0,0 +1,87 @@
# Deployment
## Local
```bash
cp .env.example .env # set CX_SECRET_KEY and CX_BOOTSTRAP_ADMIN_PASSWORD
docker compose up --build
```
<http://localhost:8080>, sign in with the bootstrap admin.
Without Docker:
```bash
cd backend && pip install -r requirements.txt
uvicorn app.main:app --reload --port 8080 # terminal 1
cd frontend && npm install && npm run dev # terminal 2 -> :5173, proxies /api
```
## Kubernetes via Gitea Actions
`.gitea/workflows/ci.yaml` runs tests → builds the image → renders
`deploy/k8s/*.yaml` with `envsubst` → applies them.
### Repository **variables** (not secret)
| Variable | Example | Meaning |
|---|---|---|
| `CX_DOMAIN` | `cx-triage.ngbackend.cloud` | Ingress host; `CX_BASE_URL` derives from it |
| `K8S_NAMESPACE` | `cx-triage` | Target namespace |
| `REGISTRY` | `git.ngbackend.cloud` | Image registry |
| `IMAGE_NAME` | `parham.monfared/cx-ui` | Image repository |
| `CX_PROMETHEUS_BASE` | `http://10.11.254.250:9090` | Alert source |
| `CX_OIDC_ENABLED` | `true` | Authentik on |
| `CX_OIDC_ISSUER` | `https://sso…/application/o/cx-triage/` | Discovery base |
| `CX_OIDC_ADMIN_GROUP` | `cx-triage-admins` | Group granting admin |
| `CX_FEATURE_SEND_ENABLED` | `false` | **Master send switch** |
| `CX_FEATURE_ZENDESK` / `CX_FEATURE_JIRA` | `false` | Per-integration flags |
| `CX_FEATURE_LINKAGE_SCAN` | `true` | Expensive scan on/off |
| `CX_SEND_DAILY_CAP` | `25` | Rolling 24h send limit |
| `CX_JIRA_PROJECT` / `CX_JIRA_BASE` | `INFRA` | Jira target |
| `INGRESS_CLASS` / `TLS_ISSUER` | `nginx` / `letsencrypt-prod` | Ingress wiring |
| `REPLICAS` | `1` | See the caveat below |
### Repository **secrets**
`KUBECONFIG` (base64), `REGISTRY_USERNAME`, `REGISTRY_TOKEN`, `CX_SECRET_KEY`,
`CX_DATABASE_URL`, `CX_OIDC_CLIENT_ID`, `CX_OIDC_CLIENT_SECRET`,
`CX_ZENDESK_SUBDOMAIN`, `CX_ZENDESK_EMAIL`, `CX_ZENDESK_TOKEN`,
`CX_JIRA_EMAIL`, `CX_JIRA_TOKEN`, `CX_BOOTSTRAP_ADMIN_PASSWORD`.
## Authentik
1. **Applications → Providers → Create → OAuth2/OpenID Provider**
- Client type: **Confidential**
- Redirect URI: `https://<CX_DOMAIN>/api/auth/oidc/callback`
- Scopes: `openid`, `email`, `profile`
2. Create the Application and bind the provider.
3. Copy the client ID/secret into the Gitea secrets above.
4. Create a group `cx-triage-admins`; its members get admin rights.
5. Set `CX_OIDC_ISSUER` to the provider's OpenID configuration base URL — the
app appends `/.well-known/openid-configuration`.
Users are created on first login. `CX_AUTH_LOCAL_ENABLED=false` in the cluster
config turns off password login entirely once SSO works.
## Two things to plan for
**Reaching OpenStack.** The diagnosis engine shells out to
`docker exec <region>-osc openstack …`, and the internal Prometheus is only
reachable from inside those containers. That works on a laptop with CX-Tools
running; it does **not** work in a pod by default. Options, cheapest first:
1. Run the app on a host that already has the CX-Tools containers, mounting the
Docker socket (what `docker-compose.yml` does).
2. Run the `*-osc` containers as sidecars in the pod.
3. Replace `cxbridge.os_json` with direct authenticated OpenStack API calls and
give the pod a network route. Cleanest, most work.
Until one of those is in place, a cluster deployment can read Prometheus (if
routable) but per-alert diagnosis will fail. That is a real gap, not an
oversight.
**`REPLICAS` should stay at 1** for now. The Prometheus caches, the background
triage jobs and the linkage scan are per-process, so a second replica would
duplicate the work and serve inconsistent job IDs. Moving jobs into the database
or a queue is what unlocks scaling out.

133
docs/INTEGRATIONS.md Normal file
View File

@@ -0,0 +1,133 @@
# Enabling Zendesk and Jira
Nothing leaves this app until **three separate gates** are open. Until then the
UI builds the full payload, shows it to you, and the Send button stays disabled
with the reason written on it.
```
1. the integration is configured CX_ZENDESK_* / CX_JIRA_*
2. its feature flag is on CX_FEATURE_ZENDESK / CX_FEATURE_JIRA
3. sending is enabled globally CX_FEATURE_SEND_ENABLED
```
Gate 3 is the important one. A demo or staging instance simply leaves it off,
and then no combination of clicks can email a customer.
Check where you stand at any time on **Settings → Integrations**, or:
```bash
curl -s localhost:8080/api/health | python3 -m json.tool
```
---
## Zendesk
### 1. Create an API token
Zendesk **Admin Center → Apps and integrations → APIs → Zendesk API**, turn on
*Token access*, then **Add API token**. Copy it — Zendesk shows it once.
### 2. Decide which agent owns the tickets
Use a dedicated agent (e.g. `cx-triage@…`) rather than a person's account, so
the audit trail stays clear when someone leaves.
### 3. Set the variables
```bash
CX_FEATURE_ZENDESK=true
CX_ZENDESK_SUBDOMAIN=nexgencloud # from https://<this>.zendesk.com
CX_ZENDESK_EMAIL=cx-triage@nexgencloud.com
CX_ZENDESK_TOKEN=<the token>
CX_ZENDESK_PUBLIC_REPLY=true # false posts an internal note instead
CX_FEATURE_SEND_ENABLED=true # the master switch
```
Restart. The Send button becomes live.
### What happens on send
1. Searches for `external_id:cx-triage-<fingerprint>`.
2. If a ticket exists → adds a comment. If not → creates one, with the requester
set from the Infrahub owner, priority from the screening verdict, and tags
`cx-triage`, `alert-<kind>`.
3. Records a `zendesk_sent` event on the case and moves it to
**Customer contacted**.
So re-diagnosing the same alert updates one ticket instead of opening five.
---
## Jira
### 1. Create an API token
<https://id.atlassian.com/manage-profile/security/api-tokens> → *Create API
token*.
### 2. Confirm the project and issue type
The defaults are `INFRA` / `Task`. If your Infrastructure project uses something
else, set it — a wrong `issuetype` is the usual cause of a 400 from Jira.
### 3. Set the variables
```bash
CX_FEATURE_JIRA=true
CX_JIRA_BASE=https://nexgencloud.atlassian.net
CX_JIRA_EMAIL=cx-triage@nexgencloud.com
CX_JIRA_TOKEN=<the token>
CX_JIRA_PROJECT=INFRA
CX_JIRA_ISSUE_TYPE=Task
CX_FEATURE_SEND_ENABLED=true
```
Issues are labelled `cx-triage-<fingerprint>` and searched for before creating,
so the same alert never opens two tickets.
---
## Where the credentials go
**Never commit them.**
| Where | How |
|---|---|
| Local | `.env` (git-ignored) — copy from `.env.example` |
| Kubernetes | The `cx-triage` Secret, written by the pipeline from Gitea secrets |
| Gitea | Repository → Settings → Actions → Secrets |
The deploy job creates the Secret imperatively from the secret store, so no
credential is ever in a manifest in git.
---
## Safety rails that stay on
- Every send needs a click **and** a confirm naming the recipient.
- No auto-send: a verdict never triggers an email by itself.
- `CX_SEND_DAILY_CAP` (default 25) refuses further sends in a rolling 24 hours,
so a loop cannot mail every customer.
- The body is editable before sending.
- Every attempt is written to the case history — including failures.
- Deleting, shelving and InfraInsight edits stay copy-a-command. The read-only
guarantee is what makes this safe against production.
## First run
Point at a **Zendesk sandbox**, or send the first ticket to your own address by
editing the To field. Once one round trip looks right, switch it on for real.
## When it fails
| Symptom | Cause |
|---|---|
| Button disabled, "sending is switched off" | `CX_FEATURE_SEND_ENABLED` is false |
| Button disabled, "not configured" | A `CX_ZENDESK_*` / `CX_JIRA_*` value is missing |
| `401` from Zendesk | The email must be the **agent** address, and token access must be enabled |
| `400` from Jira | Usually `issuetype` or `project` does not exist |
| "Daily send cap reached" | Raise `CX_SEND_DAILY_CAP` if deliberate |
| "Open the case first" | The alert has no case yet — open it in the queue once |

2
frontend/.dockerignore Normal file
View File

@@ -0,0 +1,2 @@
node_modules
dist

10
frontend/Dockerfile Normal file
View File

@@ -0,0 +1,10 @@
# Builds the static bundle. The backend image copies the result out of here.
FROM node:22-alpine AS build
WORKDIR /app
COPY package.json package-lock.json* ./
RUN npm ci --no-audit --no-fund 2>/dev/null || npm install --no-audit --no-fund
COPY . .
RUN npm run build
FROM busybox:1.36
COPY --from=build /app/dist /dist

12
frontend/index.html Normal file
View File

@@ -0,0 +1,12 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>CX Triage</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>

24
frontend/package.json Normal file
View File

@@ -0,0 +1,24 @@
{
"name": "cx-triage-frontend",
"private": true,
"version": "0.2.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "tsc -b && vite build",
"preview": "vite preview",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"react": "^18.3.1",
"react-dom": "^18.3.1",
"react-router-dom": "^6.28.0"
},
"devDependencies": {
"@types/react": "^18.3.12",
"@types/react-dom": "^18.3.1",
"@vitejs/plugin-react": "^4.3.4",
"typescript": "^5.7.2",
"vite": "^6.0.5"
}
}

61
frontend/src/App.tsx Normal file
View File

@@ -0,0 +1,61 @@
import { useCallback, useEffect, useState } from "react";
import { NavLink, Navigate, Route, Routes } from "react-router-dom";
import { api, ApiError } from "./lib/api";
import type { AppConfig, User } from "./types";
import Login from "./pages/Login";
import Queue from "./pages/Queue";
import Linkage from "./pages/Linkage";
import SettingsPage from "./pages/Settings";
export default function App() {
const [user, setUser] = useState<User | null>(null);
const [config, setConfig] = useState<AppConfig | null>(null);
const [ready, setReady] = useState(false);
const refresh = useCallback(async () => {
try {
const me = await api.get<{ user: User | null; config: AppConfig }>("/api/auth/me");
setUser(me.user);
setConfig(me.config);
} catch (err) {
if (!(err instanceof ApiError)) console.error(err);
setUser(null);
} finally {
setReady(true);
}
}, []);
useEffect(() => { void refresh(); }, [refresh]);
if (!ready) return <div className="empty"><span className="spin" /></div>;
if (!user) return <Login config={config} onSignedIn={refresh} />;
const logout = async () => { await api.post("/api/auth/logout"); await refresh(); };
return (
<>
<header className="top">
<div className="brand">
{config?.app_name ?? "CX Triage"} <em>&mdash; alert triage</em>
</div>
<NavLink to="/" end className={({ isActive }) => `navlink${isActive ? " on" : ""}`}>Queue</NavLink>
{config?.linkage_scan && (
<NavLink to="/linkage" className={({ isActive }) => `navlink${isActive ? " on" : ""}`}>Linkage</NavLink>
)}
<NavLink to="/settings" className={({ isActive }) => `navlink${isActive ? " on" : ""}`}>Settings</NavLink>
<span className="spacer" />
<span className="who">
{user.name || user.email}
{!config?.send_enabled && <span className="badge" style={{ marginLeft: 8 }}>sending off</span>}
</span>
<button className="sm" onClick={logout}>Sign out</button>
</header>
<Routes>
<Route path="/" element={<Queue config={config!} />} />
<Route path="/linkage" element={<Linkage />} />
<Route path="/settings" element={<SettingsPage user={user} config={config!} />} />
<Route path="*" element={<Navigate to="/" replace />} />
</Routes>
</>
);
}

View File

@@ -0,0 +1,139 @@
import { useState } from "react";
import { api } from "../lib/api";
import type { Diagnosis, IntegrationAction } from "../types";
interface Props {
kind: "zendesk" | "jira";
action: IntegrationAction;
diagnosis: Diagnosis;
sendEnabled: boolean;
onClose: () => void;
onDone: () => void;
}
/**
* Compose-and-confirm. The Send button stays disabled unless the integration is
* configured *and* sending is enabled for this instance, and the confirm step
* names the recipient so nobody mails the wrong customer by muscle memory.
*/
export default function ActionDrawer({ kind, action, diagnosis, sendEnabled, onClose, onDone }: Props) {
const zendesk = kind === "zendesk";
const ticket = action.payload?.ticket ?? {};
const fields = action.payload?.fields ?? {};
const [to, setTo] = useState<string>(ticket?.requester?.email ?? "");
const [subject, setSubject] = useState<string>(ticket?.subject ?? "");
const [body, setBody] = useState<string>(ticket?.comment?.body ?? "");
const [priority, setPriority] = useState<string>(ticket?.priority ?? "normal");
const [tags, setTags] = useState<string>((ticket?.tags ?? []).join(", "));
const [summary, setSummary] = useState<string>(fields?.summary ?? "");
const [description, setDescription] = useState<string>(fields?.description ?? "");
const [project, setProject] = useState<string>(fields?.project?.key ?? "");
const [confirming, setConfirming] = useState(false);
const [busy, setBusy] = useState(false);
const [result, setResult] = useState<{ ok: boolean; text: string; url?: string } | null>(null);
const blocked = !action.enabled || !sendEnabled;
const blockedWhy = !sendEnabled
? "Sending is switched off on this instance (CX_FEATURE_SEND_ENABLED). Nothing will be sent."
: action.blocked_reason;
const send = async () => {
setBusy(true);
try {
const fingerprint = diagnosis.alert.id;
const res = zendesk
? await api.post<{ ticket_id: string; url: string; action: string }>("/api/actions/zendesk", {
fingerprint, to, subject, body, priority,
tags: tags.split(",").map((t) => t.trim()).filter(Boolean),
})
: await api.post<{ key: string; url: string; action: string }>("/api/actions/jira", {
fingerprint, summary, description, project,
labels: fields?.labels ?? [],
});
const label = zendesk ? `Ticket ${(res as any).ticket_id}` : `Issue ${(res as any).key}`;
setResult({ ok: true, text: `${label} ${(res as any).action}`, url: (res as any).url });
onDone();
} catch (err) {
setResult({ ok: false, text: err instanceof Error ? err.message : "Send failed" });
} finally {
setBusy(false);
}
};
return (
<>
<div className="scrim" onClick={onClose} />
<aside className="drawer">
<div className="dh">
<h2>{zendesk ? "Contact customer via Zendesk" : "Escalate to Infrastructure"}</h2>
<button className="sm" onClick={onClose}>Close</button>
</div>
<div className="db">
{blocked && <div className="warnbox"><b>Preview only.</b> {blockedWhy}</div>}
{zendesk && action.payload?._when && (
<div className="warnbox">When to send: {action.payload._when}</div>
)}
{zendesk ? (
<>
<div className="fld"><label>To</label>
<input value={to} onChange={(e) => setTo(e.target.value)} /></div>
<div className="fld"><label>Subject</label>
<input value={subject} onChange={(e) => setSubject(e.target.value)} /></div>
<div className="fld"><label>Message (approved wording edit if needed)</label>
<textarea value={body} onChange={(e) => setBody(e.target.value)} /></div>
<div className="fld"><label>Priority</label>
<select value={priority} onChange={(e) => setPriority(e.target.value)}>
{["low", "normal", "high", "urgent"].map((p) => <option key={p}>{p}</option>)}
</select></div>
<div className="fld"><label>Tags</label>
<input value={tags} onChange={(e) => setTags(e.target.value)} /></div>
</>
) : (
<>
<div className="fld"><label>Project</label>
<input value={project} onChange={(e) => setProject(e.target.value)} /></div>
<div className="fld"><label>Summary</label>
<input value={summary} onChange={(e) => setSummary(e.target.value)} /></div>
<div className="fld"><label>Description</label>
<textarea value={description} onChange={(e) => setDescription(e.target.value)} /></div>
</>
)}
</div>
<div className="df">
{result ? (
<>
<span className={result.ok ? "t-ok" : "t-bad"}>{result.text}</span>
{result.url && <a className="navlink" href={result.url} target="_blank" rel="noreferrer">Open</a>}
<button onClick={onClose}>Close</button>
</>
) : confirming ? (
<>
<span style={{ fontSize: 13 }}>
{zendesk ? <>Send a reply to <b>{to}</b>?</> : <>Create an issue in <b>{project}</b>?</>}
</span>
<button className="pri" disabled={busy} onClick={send}>
{busy ? "Sending…" : "Yes, send"}
</button>
<button onClick={() => setConfirming(false)}>Back</button>
</>
) : (
<>
<button className="pri" disabled={blocked} onClick={() => setConfirming(true)}>
{zendesk ? "Send to customer" : "Create issue"}
</button>
<button onClick={onClose}>Cancel</button>
<span className="hint" style={{ margin: 0 }}>
{blocked ? blockedWhy : "You will be asked to confirm."}
</span>
</>
)}
</div>
</aside>
</>
);
}

View File

@@ -0,0 +1,83 @@
import { useState } from "react";
import { api } from "../lib/api";
import type { CaseRef } from "../types";
const STATUS_LABELS: Record<string, string> = {
new: "New",
investigating: "Investigating",
customer_contacted: "Customer contacted",
escalated_infra: "Escalated to Infra",
waiting_customer: "Waiting on customer",
waiting_infra: "Waiting on Infra",
remediated: "Remediated",
resolved: "Resolved",
wont_fix: "Won't fix",
false_positive: "False positive",
};
/** Status, ownership, notes and the audit trail for one tracked alert. */
export default function CasePanel({ kase, onChange }: {
kase: CaseRef; onChange: (next: CaseRef) => void;
}) {
const [note, setNote] = useState("");
const [busy, setBusy] = useState(false);
const call = async (fn: () => Promise<CaseRef>) => {
setBusy(true);
try { onChange(await fn()); } finally { setBusy(false); }
};
return (
<div className="card">
<h3 className="sec">Case</h3>
<div className="row">
<select value={kase.status} disabled={busy}
onChange={(e) => call(() => api.post<CaseRef>(
`/api/cases/${kase.fingerprint}/status`, { status: e.target.value }))}>
{Object.entries(STATUS_LABELS).map(([v, l]) => <option key={v} value={v}>{l}</option>)}
</select>
<button className="sm" disabled={busy}
onClick={() => call(() => api.post<CaseRef>(`/api/cases/${kase.fingerprint}/assign`))}>
{kase.assignee ? `Assigned to ${kase.assignee.name}` : "Assign to me"}
</button>
<button className="sm" disabled={busy}
onClick={() => call(() => api.post<CaseRef>(
`/api/cases/${kase.fingerprint}/snooze`, { hours: 24 }))}>Snooze 24h</button>
{kase.reopen_count > 0 && <span className="badge">reopened {kase.reopen_count}×</span>}
{kase.zendesk_ticket_url && (
<a className="navlink" href={kase.zendesk_ticket_url} target="_blank" rel="noreferrer">
Zendesk #{kase.zendesk_ticket_id}</a>
)}
{kase.jira_issue_url && (
<a className="navlink" href={kase.jira_issue_url} target="_blank" rel="noreferrer">
{kase.jira_issue_key}</a>
)}
</div>
<div className="row mt">
<input placeholder="Add a note…" value={note} onChange={(e) => setNote(e.target.value)}
style={{ flex: 1 }}
onKeyDown={(e) => {
if (e.key === "Enter" && note.trim()) {
call(() => api.post<CaseRef>(`/api/cases/${kase.fingerprint}/note`, { note }))
.then(() => setNote(""));
}
}} />
</div>
{(kase.events?.length ?? 0) > 0 && (
<details className="fold" style={{ marginTop: 12, border: "none" }}>
<summary>History ({kase.events!.length})</summary>
<ul className="timeline">
{kase.events!.map((e) => (
<li key={e.id}>
<b>{e.action.replace(/_/g, " ")}</b> {e.detail}
<div className="ts">{new Date(e.created_at).toLocaleString()} · {e.actor}</div>
</li>
))}
</ul>
</details>
)}
</div>
);
}

View File

@@ -0,0 +1,135 @@
import type { Visual } from "../types";
/** Infrahub vs OpenStack, side by side. */
function States({ v }: { v: Visual }) {
const bad = !v.match;
return (
<>
<div className="states">
<div className={`sbox ${bad ? "bad" : "ok"}`}>
<div className="lbl">Infrahub says</div><div className="val">{v.infrahub}</div>
</div>
<div className={`eqlink ${bad ? "bad" : ""}`}>{bad ? "≠" : "="}</div>
<div className={`sbox ${bad ? "bad" : "ok"}`}>
<div className="lbl">OpenStack says</div><div className="val">{v.openstack}</div>
</div>
</div>
<div className="legend">
{v.task && v.task !== "None" && <span>task state <b>{v.task}</b></span>}
<span>host <b>{v.never_built ? "never placed" : v.host}</b></span>
{v.flavor && <span>flavor <b>{v.flavor}</b></span>}
{v.fault && v.fault !== "None" && <span className="t-bad">fault present</span>}
</div>
</>
);
}
/**
* One tile per physical GPU. Drawing only the used ones drops the free sockets
* and leaves a total that does not match the host's GPU count.
*/
function Sockets({ v }: { v: Visual }) {
const slots = v.slots ?? [];
const named = slots.filter((s) => s.kind === "vm").length;
const unclaimed = slots.filter((s) => s.kind === "unaccounted").length;
const free = slots.filter((s) => s.kind === "free").length;
return (
<>
<div className="slots">
{slots.map((s, i) => {
if (s.kind === "vm") {
const cls = s.linked ? (s.match ? "" : "bad") : "unlinked";
return (
<div key={i} className={`slot vm ${cls}`}
title={`${s.name} — Infrahub ${s.ih_status} / OpenStack ${s.os_status}`}>
<span className="idx">GPU {i + 1}</span>
<span className="sn">{s.name}</span>
<span className="ss">
{s.linked ? (s.match ? "in sync" : "state mismatch") : "not in Infrahub"}
</span>
</div>
);
}
if (s.kind === "unaccounted") {
return (
<div key={i} className="slot unaccounted"
title="The host reports this GPU in use, but no instance claims it">
<span className="sn">unaccounted</span>
</div>
);
}
return (
<div key={i} className="slot free" title="Physically present, nothing using it">
<span className="sn">free</span>
</div>
);
})}
</div>
<div className="legend">
<span>{v.physical != null ? `${v.physical} GPU sockets on this host` : "GPU count unknown"}</span>
<span><i style={{ background: "color-mix(in srgb,var(--ok) 60%,transparent)" }} />
{named} held by {v.instances} VM(s)</span>
{unclaimed > 0 && (
<span className="t-bad">
<i style={{ background: "color-mix(in srgb,var(--bad) 55%,transparent)" }} />
{unclaimed} in use but unclaimed
</span>
)}
{free > 0 && <span><i style={{ border: "1px dashed var(--line2)" }} />{free} free</span>}
{v.in_use_metric != null && <span>host reports {v.in_use_metric} in use</span>}
</div>
</>
);
}
/** Every VM on the host, and whether Infrahub has a counterpart for it. */
export function Roster({ v }: { v: Visual }) {
const rows = v.roster ?? [];
if (!rows.length) return null;
const unlinked = rows.filter((r) => !r.linked).length;
const mismatched = rows.filter((r) => r.linked && !r.match).length;
return (
<>
<div className="roster">
<div className="rrow">
<span>VM on this host</span><span>Infrahub</span><span /><span>OpenStack</span><span className="rg">GPU</span>
</div>
{rows.map((r) => (
<div key={r.openstack_id || r.name}
className={`rrow ${r.linked ? (r.match ? "" : "bad") : "unlinked"}`}>
<span className="rn">{r.name}{r.tempest && <span className="badge"> tempest</span>}</span>
<span className="rs">{r.ih_status}</span>
<span className="eq">{r.linked ? (r.match ? "=" : "≠") : "✗"}</span>
<span className="rs">{r.os_status}</span>
<span className="rg">{r.gpus}</span>
</div>
))}
</div>
<div className="legend">
<span>{rows.length} VM(s) on host</span>
<span className={unlinked ? "t-bad" : ""}>{unlinked} with no Infrahub record</span>
<span className={mismatched ? "t-bad" : ""}>{mismatched} state mismatch(es)</span>
</div>
</>
);
}
export default function Visuals({ v }: { v: Visual }) {
if (!v || !v.type) return v?.roster ? <Roster v={v} /> : null;
if (v.type === "states") return <States v={v} />;
if (v.type === "gpu") return <><Sockets v={v} /><Roster v={v} /></>;
if (v.type === "claimants") {
return (
<div style={{ marginTop: 14, display: "flex", flexDirection: "column", gap: 7 }}>
{(v.items ?? []).map((c, i) => (
<div key={i} className="sbox" style={{ display: "flex", gap: 10, alignItems: "center" }}>
<b style={{ flex: 1 }}>{c.name}</b>
<span className="badge">{c.ih_status} / {c.os_status}</span>
<span style={{ color: "var(--dim)", fontSize: 12 }}>{c.verdict}</span>
</div>
))}
</div>
);
}
return null;
}

37
frontend/src/lib/api.ts Normal file
View File

@@ -0,0 +1,37 @@
const json = { "Content-Type": "application/json" };
async function handle<T>(res: Response): Promise<T> {
if (res.status === 401) throw new ApiError("Not signed in", 401);
if (!res.ok) {
let detail = res.statusText;
try { detail = (await res.json()).detail ?? detail; } catch { /* keep statusText */ }
throw new ApiError(String(detail), res.status);
}
return res.json() as Promise<T>;
}
export class ApiError extends Error {
constructor(message: string, public status: number) { super(message); }
}
export const api = {
get: <T>(path: string) => fetch(path, { credentials: "same-origin" }).then(handle<T>),
post: <T>(path: string, body?: unknown) =>
fetch(path, {
method: "POST", headers: json, credentials: "same-origin",
body: body === undefined ? undefined : JSON.stringify(body),
}).then(handle<T>),
del: <T>(path: string) =>
fetch(path, { method: "DELETE", credentials: "same-origin" }).then(handle<T>),
};
/** Poll a background triage job until it finishes. */
export async function pollJob<T>(jobId: string, signal: AbortSignal): Promise<T> {
for (;;) {
if (signal.aborted) throw new DOMException("aborted", "AbortError");
const job = await api.get<{ state: string; result: T; error: string }>(`/api/jobs/${jobId}`);
if (job.state === "done") return job.result;
if (job.state === "error") throw new Error(job.error);
await new Promise((r) => setTimeout(r, 900));
}
}

13
frontend/src/main.tsx Normal file
View File

@@ -0,0 +1,13 @@
import React from "react";
import ReactDOM from "react-dom/client";
import { BrowserRouter } from "react-router-dom";
import App from "./App";
import "./styles.css";
ReactDOM.createRoot(document.getElementById("root")!).render(
<React.StrictMode>
<BrowserRouter>
<App />
</BrowserRouter>
</React.StrictMode>,
);

View File

@@ -0,0 +1,134 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { api } from "../lib/api";
interface Candidate { id: string; name: string; status: string; region: string }
interface BrokenLink {
instance_name: string; infrahub_status: string; organization: string; region: string;
reason: string; candidate: Candidate | null; candidate_claimed_by_other: boolean; confidence: string;
}
interface Orphan { id: string; name: string; status: string; region: string; host: string; name_known_to_infrahub: boolean }
interface Result {
scanned_regions: Record<string, number>; region_failures: Record<string, string>;
openstack_servers: number; infrahub_records: number; broken_links: BrokenLink[];
likely_linkage_failures: number; orphans: Orphan[]; orphan_total: number;
skipped_unscanned_regions?: number;
}
interface State { state: string; error: string; progress: string; age_seconds: number | null; result: Result }
export default function LinkagePage() {
const [scan, setScan] = useState<State | null>(null);
const [detail, setDetail] = useState<Record<number, string>>({});
const timer = useRef<number | null>(null);
const tick = useCallback(async () => {
const s = await api.get<State>("/api/linkage");
setScan(s);
if (s.state !== "running" && timer.current) { clearInterval(timer.current); timer.current = null; }
}, []);
useEffect(() => { void tick(); return () => { if (timer.current) clearInterval(timer.current); }; }, [tick]);
const start = async () => {
await api.post("/api/linkage/scan");
if (timer.current) clearInterval(timer.current);
timer.current = window.setInterval(() => void tick(), 2500);
void tick();
};
const r = scan?.result;
return (
<main style={{ padding: "22px 26px 80px", maxWidth: 1500 }}>
<div className="row">
<h3 className="sec" style={{ flex: 1 }}>Linkage scan</h3>
<span className="hint" style={{ margin: 0 }}>
{scan?.state === "running" ? `scanning… ${scan.progress}`
: scan?.age_seconds != null ? `last scan ${Math.round(scan.age_seconds / 60)}m ago` : ""}
</span>
<button className="pri" disabled={scan?.state === "running"} onClick={start}>Run scan</button>
</div>
<p style={{ color: "var(--dim)", maxWidth: "90ch" }}>
A VM in ERROR is not always a failed build. If the server was created but the link back to Infrahub was
never written, Infrahub shows ERROR or CREATING with no usable <code>openstack_id</code> while a perfectly
good server of the same name is running. This scans both sides in bulk and pairs them up by name and
finds the reverse too: OpenStack servers that no Infrahub record claims.
</p>
{scan?.state === "running" && (
<div className="empty"><span className="spin" /><br /><br />{scan.progress}<br />
<span className="hint">Listing every server across all regions a few minutes.</span></div>
)}
{scan?.state === "error" && <div className="warnbox t-bad">{scan.error}</div>}
{scan?.state === "done" && r && (
<>
<div className="row mt">
{[["likely linkage failures", r.likely_linkage_failures],
["records with a broken link", r.broken_links.length],
["OpenStack servers no record claims", r.orphan_total],
["servers scanned", r.openstack_servers],
["Infrahub records", r.infrahub_records]].map(([label, n]) => (
<div className="card" key={label as string} style={{ minWidth: 160, marginBottom: 0 }}>
<div style={{ fontSize: 23, fontWeight: 680, fontFamily: "var(--mono)" }}>{n as number}</div>
<div className="hint" style={{ margin: 0 }}>{label as string}</div>
</div>
))}
</div>
{Object.keys(r.region_failures).length > 0 && (
<div className="warnbox mt">
Some regions could not be listed:{" "}
{Object.entries(r.region_failures).map(([k, v]) => <span key={k}><b>{k}</b> ({v}) </span>)}.
Results exclude those regions entirely ({r.skipped_unscanned_regions ?? 0} record(s) skipped),
so nothing here is a false positive from a failed listing but the scan is not complete.
</div>
)}
<h3 className="sec" style={{ marginTop: 24 }}>Infrahub records whose OpenStack server is missing</h3>
<div className="card" style={{ padding: 0 }}>
{r.broken_links.length === 0 ? <div className="empty">None every record resolves.</div>
: r.broken_links.slice(0, 300).map((l, i) => (
<div key={i} style={{ padding: "9px 14px", borderTop: i ? "1px solid var(--line)" : undefined }}>
<div className="row">
<b style={{ flex: 1 }}>{l.instance_name}</b>
<span className="t-bad" style={{ fontFamily: "var(--mono)", fontSize: 12 }}>{l.infrahub_status}</span>
<span className={`badge ${l.confidence === "high" ? "overdue" : ""}`}>{l.confidence}</span>
{l.candidate && (
<button className="sm" onClick={async () => {
const d = await api.post<any>("/api/linkage/enrich",
{ region: l.candidate!.region, openstack_id: l.candidate!.id });
setDetail({ ...detail, [i]: d.ok
? `created ${d.created} · status ${d.status} · host ${d.host || "—"} · fault: ${d.fault}`
: d.error });
}}>Details</button>
)}
</div>
<div className="hint" style={{ margin: 0 }}>
{l.organization} · {l.region} · {l.reason}
{l.candidate && <> OpenStack <code>{l.candidate.id}</code> {l.candidate.name} {l.candidate.status}
{l.candidate_claimed_by_other && <span className="t-warn"> · claimed by another record</span>}</>}
</div>
{detail[i] && <div className="hint" style={{ fontFamily: "var(--mono)" }}>{detail[i]}</div>}
</div>
))}
</div>
<h3 className="sec" style={{ marginTop: 24 }}>OpenStack servers with no Infrahub record</h3>
<div className="card" style={{ padding: 0 }}>
{r.orphans.length === 0 ? <div className="empty">None.</div>
: r.orphans.slice(0, 200).map((o, i) => (
<div key={o.id} className="row" style={{ padding: "7px 14px", borderTop: i ? "1px solid var(--line)" : undefined }}>
<b style={{ flex: 1 }}>{o.name || "(unnamed)"}</b>
<code style={{ fontSize: 11.5 }}>{o.id}</code>
<span style={{ fontFamily: "var(--mono)", fontSize: 12 }}>{o.status}</span>
<span className="hint" style={{ margin: 0 }}>{o.host || "—"}</span>
<span className={o.name_known_to_infrahub ? "t-warn" : "t-bad"}>
{o.name_known_to_infrahub ? "name exists" : "unknown"}</span>
</div>
))}
</div>
</>
)}
</main>
);
}

View File

@@ -0,0 +1,57 @@
import { useState } from "react";
import { api } from "../lib/api";
import type { AppConfig } from "../types";
export default function Login({ config, onSignedIn }: {
config: AppConfig | null; onSignedIn: () => void;
}) {
const [email, setEmail] = useState("");
const [password, setPassword] = useState("");
const [error, setError] = useState("");
const [busy, setBusy] = useState(false);
const submit = async (e: React.FormEvent) => {
e.preventDefault();
setBusy(true); setError("");
try {
await api.post("/api/auth/login", { email, password });
onSignedIn();
} catch (err) {
setError(err instanceof Error ? err.message : "Sign-in failed");
} finally {
setBusy(false);
}
};
return (
<div className="card login">
<h2 style={{ marginTop: 0 }}>{config?.app_name ?? "CX Triage"}</h2>
{config?.oidc_enabled && (
<>
<a className="navlink" style={{ display: "block", textAlign: "center", padding: 10 }}
href="/api/auth/oidc/start">Sign in with SSO</a>
{config.local_login && <div className="hint" style={{ textAlign: "center" }}>or use a local account</div>}
</>
)}
{config?.local_login !== false && (
<form onSubmit={submit} style={{ marginTop: 14 }}>
<div className="fld">
<label>Email</label>
<input value={email} onChange={(e) => setEmail(e.target.value)} autoComplete="username" />
</div>
<div className="fld">
<label>Password</label>
<input type="password" value={password} onChange={(e) => setPassword(e.target.value)}
autoComplete="current-password" />
</div>
{error && <div className="warnbox t-bad">{error}</div>}
<button className="pri" style={{ width: "100%" }} disabled={busy}>
{busy ? "Signing in…" : "Sign in"}
</button>
</form>
)}
</div>
);
}

View File

@@ -0,0 +1,286 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { api, pollJob } from "../lib/api";
import type { AppConfig, CaseRef, Diagnosis, Queue as QueueData } from "../types";
import Visuals from "../components/Visuals";
import ActionDrawer from "../components/ActionDrawer";
import CasePanel from "../components/CasePanel";
const VERDICT_ORDER = ["overdue", "real", "unverified", "chronic", "low_impact",
"pending", "resolved", "rule_defect", "suppressed"];
const VERDICT_HELP: Record<string, string> = {
overdue: "Still true, and past the point where the runbook says to contact the customer.",
real: "Still true right now — re-checked against live Infrahub/OpenStack state.",
unverified: "Could not be re-checked, so it stays in the queue rather than being hidden on a guess.",
chronic: "Still true, but it has been for days — known rather than new work.",
low_impact: "Still true, but internally owned.",
pending: "Prometheus has not committed to this alert yet; it may clear on its own.",
resolved: "The condition no longer holds.",
rule_defect: "The alert rule itself is wrong, so the alert is not evidence of a problem.",
suppressed: "Hidden by a rule you configured in Settings.",
};
// Alerts Prometheus has not committed to are never shown unless asked for by name.
const HIDDEN_STATES = ["pending"];
function subjectOf(a: { kind: string; floating_ip: string; host: string; instance_name: string; openstack_id: string }) {
if (a.kind === "duplicate_ip") return a.floating_ip;
if (["rogue_vm", "total_gpus", "orphan_vm"].includes(a.kind)) return a.host;
return a.instance_name || a.openstack_id || "unknown";
}
export default function QueuePage({ config }: { config: AppConfig }) {
const [data, setData] = useState<QueueData | null>(null);
const [selected, setSelected] = useState<string | null>(null);
const [diagnosis, setDiagnosis] = useState<Diagnosis | null>(null);
const [kase, setKase] = useState<CaseRef | null>(null);
const [loadingCase, setLoadingCase] = useState(false);
const [verdictFilter, setVerdictFilter] = useState("actionable");
const [kindFilter, setKindFilter] = useState("all");
const [shut, setShut] = useState<Record<string, boolean>>({});
const [drawer, setDrawer] = useState<"zendesk" | "jira" | null>(null);
const abortRef = useRef<AbortController | null>(null);
const load = useCallback(async (force = false) => {
try { setData(await api.get<QueueData>(`/api/alerts${force ? "?force=1" : ""}`)); }
catch (err) { console.error(err); }
}, []);
useEffect(() => {
void load();
const t = setInterval(() => void load(), 60_000);
return () => clearInterval(t);
}, [load]);
// Each request supersedes the last: an abandoned diagnosis must never land on
// screen after the user has moved on to another case.
const openCase = useCallback(async (fingerprint: string, force = false) => {
abortRef.current?.abort();
const controller = new AbortController();
abortRef.current = controller;
setSelected(fingerprint);
setDiagnosis(null);
setKase(null);
setLoadingCase(true);
try {
const started = await api.post<{ job_id: string; case: CaseRef }>("/api/triage", { fingerprint, force });
if (controller.signal.aborted) return;
setKase(started.case);
const result = await pollJob<Diagnosis>(started.job_id, controller.signal);
if (!controller.signal.aborted) setDiagnosis(result);
} catch (err) {
if ((err as Error)?.name !== "AbortError") console.error(err);
} finally {
if (!controller.signal.aborted) setLoadingCase(false);
}
}, []);
const allAlerts = useMemo(() => (data?.groups ?? []).flatMap((g) => g.alerts), [data]);
const actionable = useMemo(
() => allAlerts.filter((a) => a.screen.actionable && !HIDDEN_STATES.includes(a.state)).length,
[allAlerts]);
const visible = useCallback((a: (typeof allAlerts)[number]) => {
if (kindFilter !== "all" && a.kind !== kindFilter) return false;
if (HIDDEN_STATES.includes(a.state) && verdictFilter !== a.state) return false;
if (verdictFilter === "all") return true;
if (verdictFilter === "actionable") return a.screen.actionable;
return a.screen.verdict === verdictFilter;
}, [kindFilter, verdictFilter]);
if (!data) return <div className="empty"><span className="spin" /></div>;
const counts: Record<string, number> = {};
allAlerts.forEach((a) => { counts[a.screen.verdict] = (counts[a.screen.verdict] ?? 0) + 1; });
const byKind: Record<string, number> = {};
allAlerts.forEach((a) => { byKind[a.kind] = (byKind[a.kind] ?? 0) + 1; });
const zAction = diagnosis?.integrations?.actions?.find((x) => x.kind === "zendesk");
const jAction = diagnosis?.integrations?.actions?.find((x) => x.kind === "jira");
const manual = diagnosis?.integrations?.actions?.filter((x) => x.kind === "manual") ?? [];
return (
<>
<div className="filters">
<div className="frow">
<span className="flab">Status</span>
<div className="chips">
<span className={`chip${verdictFilter === "actionable" ? " on" : ""}`}
onClick={() => setVerdictFilter("actionable")}>To action <b>{actionable}</b></span>
{VERDICT_ORDER.filter((v) => counts[v]).map((v) => (
<span key={v} className={`chip ${v}${verdictFilter === v ? " on" : ""}`}
title={VERDICT_HELP[v]} onClick={() => setVerdictFilter(v)}>
{data.summary.labels[v] ?? v} <b>{counts[v]}</b>
</span>
))}
<span className={`chip${verdictFilter === "all" ? " on" : ""}`}
title="Everything except alerts Prometheus has not committed to yet"
onClick={() => setVerdictFilter("all")}>All firing</span>
</div>
</div>
<div className="frow">
<span className="flab">Type</span>
<div className="chips">
<span className={`chip${kindFilter === "all" ? " on" : ""}`}
onClick={() => setKindFilter("all")}>All types</span>
{data.groups.map((g) => (
<span key={g.kind} className={`chip${kindFilter === g.kind ? " on" : ""}`}
onClick={() => setKindFilter(g.kind)}>
{g.title.replace(/^Instance in /, "").replace(/ state$/, "")} <b>{byKind[g.kind] ?? 0}</b>
</span>
))}
</div>
</div>
</div>
{data.warnings.map((w, i) => <div key={i} className="warnbox" style={{ margin: "10px 18px" }}>{w}</div>)}
<div className="split">
<aside className="rail">
{data.groups.map((g) => {
const rows = g.alerts.filter(visible);
if (!rows.length) return null;
const hot = rows.filter((a) => a.screen.actionable).length;
const closed = shut[g.kind];
return (
<div key={g.kind}>
<div className="grp-h" onClick={() => setShut({ ...shut, [g.kind]: !closed })}>
<span>{closed ? "▸" : "▾"}</span>{g.title}
<span className={`n${hot ? " hot" : ""}`} title={`${rows.length} shown of ${g.total} firing`}>
{rows.length}{rows.length < g.total && <span style={{ opacity: .6 }}> of {g.total}</span>}
</span>
</div>
{!closed && rows.map((a) => (
<div key={a.id} className={`case${selected === a.id ? " on" : ""}${a.screen.actionable ? "" : " off"}`}
onClick={() => void openCase(a.id)}>
<span className={`bar ${a.screen.verdict}`} title={a.screen.label} />
<span className="mid">
<span className="nm">{subjectOf(a)}</span>
<span className="sub">{[a.org_name, a.region_label || a.region].filter(Boolean).join(" · ")}</span>
{a.case && a.case.status !== "new" && (
<span className="sub"><span className="badge">{a.case.status.replace(/_/g, " ")}</span></span>
)}
</span>
<span className="ag">{a.effective_age_text}</span>
</div>
))}
</div>
);
})}
</aside>
<section className="stage">
{!selected && <div className="empty">Select a case.</div>}
{selected && loadingCase && !diagnosis && (
<div className="empty"><span className="spin" /><br /><br />
Checking Infrahub, OpenStack and InfraInsight</div>
)}
{diagnosis && (
<>
<div className="card hero">
<div className="crumb">
<span className={`badge ${diagnosis.alert.screen.verdict}`}
title={VERDICT_HELP[diagnosis.alert.screen.verdict]}>
{diagnosis.alert.screen.label}</span>
<span>{diagnosis.alert.title}</span><span>·</span>
<span>{diagnosis.alert.region_label || diagnosis.alert.region}</span><span>·</span>
<span>held {diagnosis.alert.effective_age_text}</span><span>·</span>
<span>{diagnosis.alert.state} in prometheus</span>
</div>
<div className="vd">{diagnosis.verdict || diagnosis.error}</div>
{diagnosis.assessment && <div className="vsub">{diagnosis.assessment}</div>}
<Visuals v={diagnosis.visual} />
</div>
{kase && <CasePanel kase={kase} onChange={setKase} />}
<div className="card">
<h3 className="sec">Do this</h3>
{!diagnosis.alert.screen.actionable && (
<div className="hint">
Screened out ({diagnosis.alert.screen.label}). {diagnosis.alert.screen.reason}
</div>
)}
{zAction && (
<div className="row mt">
<span style={{ color: "var(--dim)" }}>Customer</span>
<b style={{ fontFamily: "var(--mono)", fontSize: 12.5 }}>
{zAction.recipients[0] ?? "unresolved"}</b>
{diagnosis.alert.org_name && <span className="badge">{diagnosis.alert.org_name}</span>}
</div>
)}
<div className="row mt">
{zAction && <button className="pri" onClick={() => setDrawer("zendesk")}>
Contact customer via Zendesk</button>}
{jAction && <button onClick={() => setDrawer("jira")}>
Escalate to Infrastructure (Jira)</button>}
{!diagnosis.alert.screen.actionable && (
<button className="sm" onClick={() => void openCase(diagnosis.alert.id, true)}>
Re-run full diagnosis</button>
)}
{!zAction && !jAction && !manual.length && (
<span className="hint" style={{ margin: 0 }}>
No outbound action for this case the runbook keeps it internal.</span>
)}
</div>
{manual.map((m, i) => (
<div className="cmd" key={i}>
<code>{m.payload?.command ?? m.label}</code>
{m.payload?.command && (
<button className="sm"
onClick={() => navigator.clipboard.writeText(m.payload.command)}>Copy</button>
)}
</div>
))}
{manual.length > 0 && (
<div className="hint">Run these yourself CX Triage never mutates the platform.</div>
)}
</div>
{diagnosis.notes.length > 0 && (
<details className="fold"><summary>Caveats ({diagnosis.notes.length})</summary>
<div className="foldb"><ul style={{ margin: 0, paddingLeft: 18, color: "var(--warn)" }}>
{diagnosis.notes.map((n, i) => <li key={i}>{n}</li>)}</ul></div>
</details>
)}
<details className="fold"><summary>Why what the platforms say</summary>
<div className="foldb"><table><tbody>
{diagnosis.findings.map((f, i) => (
<tr key={i}><td className="k">{f.label}</td>
<td className={`v t-${f.tone}`}>{f.value}
{f.detail && <span className="det">{f.detail}</span>}</td></tr>
))}
</tbody></table></div>
</details>
<details className="fold"><summary>Runbook steps ({diagnosis.actions.length})</summary>
<div className="foldb"><ol style={{ margin: 0, paddingLeft: 18, fontSize: 13 }}>
{diagnosis.actions.map((a, i) => (
<li key={i} style={{ marginBottom: 7 }}>{a.text}
<span className="badge" style={{ marginLeft: 6 }}>{a.owner}</span>
{a.guide && <span className="det">Guide: {a.guide}</span>}</li>
))}
</ol></div>
</details>
<details className="fold"><summary>Raw evidence</summary>
<div className="foldb"><pre>{JSON.stringify(
{ screen: diagnosis.alert.screen, labels: diagnosis.alert.labels,
evidence: diagnosis.evidence }, null, 2)}</pre></div>
</details>
</>
)}
</section>
</div>
{drawer && diagnosis && (
<ActionDrawer kind={drawer} diagnosis={diagnosis} sendEnabled={config.send_enabled}
action={(drawer === "zendesk" ? zAction : jAction)!}
onClose={() => setDrawer(null)}
onDone={() => { void load(true); if (selected) void openCase(selected); }} />
)}
</>
);
}

View File

@@ -0,0 +1,172 @@
import { useCallback, useEffect, useState } from "react";
import { api } from "../lib/api";
import type { AppConfig, User } from "../types";
interface Rule {
id: string; name: string; reason: string; enabled: boolean;
conditions: Record<string, string[]>; created_by?: string;
}
interface Config {
rules: Rule[]; conditions: Record<string, string>;
agent_name: string; chronic_days: number; config: AppConfig;
}
const blank = (): Rule => ({ id: "", name: "", reason: "", enabled: true, conditions: {} });
export default function SettingsPage({ user, config }: { user: User; config: AppConfig }) {
const [cfg, setCfg] = useState<Config | null>(null);
const [editing, setEditing] = useState<Rule | null>(null);
const [rows, setRows] = useState<{ field: string; values: string }[]>([]);
const [preview, setPreview] = useState<{ count: number; matches: any[] } | null>(null);
const [saved, setSaved] = useState("");
const load = useCallback(async () => setCfg(await api.get<Config>("/api/settings")), []);
useEffect(() => { void load(); }, [load]);
const flash = (m: string) => { setSaved(m); setTimeout(() => setSaved(""), 2200); };
const startEdit = (rule: Rule) => {
setEditing(rule);
const entries = Object.entries(rule.conditions ?? {});
setRows(entries.length ? entries.map(([f, v]) => ({ field: f, values: v.join(", ") }))
: [{ field: "kind", values: "" }]);
setPreview(null);
};
const collect = (): Rule => ({
...editing!,
conditions: rows.reduce<Record<string, string[]>>((acc, r) => {
const vals = r.values.split(",").map((v) => v.trim()).filter(Boolean);
if (vals.length) acc[r.field] = [...(acc[r.field] ?? []), ...vals];
return acc;
}, {}),
});
if (!cfg) return <div className="empty"><span className="spin" /></div>;
return (
<main style={{ padding: "22px 26px 80px", maxWidth: 1040 }}>
<h3 className="sec">Suppression rules {saved && <span className="t-ok">· {saved}</span>}</h3>
<p style={{ color: "var(--dim)", maxWidth: "82ch" }}>
Hide alerts you already know about. A rule fires when <b>every</b> condition it sets matches, so you
can combine them for example type <code>error</code> <i>and</i> organisation containing <code>modal</code>.
Suppressed alerts are not deleted: they stay reachable under the <b>hidden by a rule</b> filter.
</p>
{cfg.rules.map((r) => (
<div className="card" key={r.id} style={{ opacity: r.enabled ? 1 : 0.55 }}>
<div className="row">
<input type="checkbox" checked={r.enabled} style={{ width: "auto" }}
disabled={!user.is_admin}
onChange={async (e) => {
await api.post("/api/settings/rules", { ...r, enabled: e.target.checked });
await load(); flash(e.target.checked ? "Rule enabled" : "Rule disabled");
}} />
<b style={{ flex: 1 }}>{r.name}</b>
{user.is_admin && <>
<button className="sm" onClick={() => startEdit(r)}>Edit</button>
<button className="sm del" onClick={async () => {
if (!confirm(`Delete “${r.name}”? Alerts it was hiding will come back.`)) return;
await api.del(`/api/settings/rules/${r.id}`); await load(); flash("Rule deleted");
}}>Delete</button>
</>}
</div>
{r.reason && <div style={{ color: "var(--dim)", fontSize: 12.5 }}>{r.reason}</div>}
<div className="row mt">
{Object.entries(r.conditions).map(([k, v], i) => (
<span key={k}>
{i > 0 && <span style={{ color: "var(--faint)", fontSize: 10.5 }}> AND </span>}
<span className="badge"><b>{cfg.conditions[k] ?? k}</b> {v.join(" or ")}</span>
</span>
))}
</div>
</div>
))}
{user.is_admin && !editing && (
<button className="pri" onClick={() => startEdit(blank())}>Add a rule</button>
)}
{editing && (
<div className="card" style={{ borderColor: "var(--accent)" }}>
<div className="fld"><label>Rule name</label>
<input value={editing.name} onChange={(e) => setEditing({ ...editing, name: e.target.value })} /></div>
<div className="fld"><label>Why (shown on the alert)</label>
<input value={editing.reason} onChange={(e) => setEditing({ ...editing, reason: e.target.value })} /></div>
<label style={{ fontSize: 11, textTransform: "uppercase", color: "var(--faint)" }}>
Conditions all must match</label>
{rows.map((r, i) => (
<div className="row mt" key={i}>
<select value={r.field} style={{ maxWidth: 200 }}
onChange={(e) => setRows(rows.map((x, j) => j === i ? { ...x, field: e.target.value } : x))}>
{Object.entries(cfg.conditions).map(([k, l]) => <option key={k} value={k}>{l}</option>)}
</select>
<input value={r.values} placeholder="comma-separated; any one matches" style={{ flex: 1 }}
onChange={(e) => setRows(rows.map((x, j) => j === i ? { ...x, values: e.target.value } : x))} />
<button className="sm del" onClick={() => setRows(rows.filter((_, j) => j !== i))}>×</button>
</div>
))}
<button className="sm mt" onClick={() => setRows([...rows, { field: "organization", values: "" }])}>
+ Add condition</button>
{preview && (
<div className="warnbox mt">
<b className={preview.count ? "t-warn" : "t-ok"}>{preview.count}</b> currently-firing alert(s) would be hidden.
<ul style={{ maxHeight: 170, overflowY: "auto", marginTop: 6 }}>
{preview.matches.slice(0, 40).map((m, i) => (
<li key={i}>{m.title} {m.instance_name || m.host} <span style={{ color: "var(--faint)" }}>{m.org_name}</span></li>
))}
</ul>
</div>
)}
<div className="row mt">
<button className="pri" onClick={async () => {
try {
await api.post("/api/settings/rules", collect());
setEditing(null); await load(); flash("Rule saved");
} catch (err) { alert(err instanceof Error ? err.message : "Save failed"); }
}}>Save rule</button>
<button onClick={async () => setPreview(
await api.post("/api/settings/rules/preview", collect()))}>Preview what this hides</button>
<button onClick={() => setEditing(null)}>Cancel</button>
</div>
</div>
)}
<h3 className="sec" style={{ marginTop: 28 }}>Comms identity</h3>
<div className="card">
<div className="fld"><label>Sign-off name</label>
<input defaultValue={cfg.agent_name} placeholder="e.g. Mohammad Affan"
onBlur={async (e) => { await api.post("/api/settings/general", { agent_name: e.target.value }); flash("Saved"); }} />
<div className="hint">Appended after Kind Regards in customer emails.</div></div>
<div className="fld"><label>Chronic after (days)</label>
<input type="number" min={1} max={90} defaultValue={cfg.chronic_days} style={{ maxWidth: 120 }}
onBlur={async (e) => { await api.post("/api/settings/general", { chronic_days: Number(e.target.value) }); flash("Saved"); }} />
<div className="hint">Types with a runbook time commitment become <b>overdue</b> instead.</div></div>
</div>
<h3 className="sec" style={{ marginTop: 28 }}>Integrations</h3>
<div className="card">
<table><tbody>
<tr><td className="k">Zendesk</td><td className="v">
{config.zendesk_ready ? <span className="t-ok">configured</span>
: <span className="t-warn">not configured</span>}</td></tr>
<tr><td className="k">Jira</td><td className="v">
{config.jira_ready ? <span className="t-ok">configured ({config.jira_project})</span>
: <span className="t-warn">not configured</span>}</td></tr>
<tr><td className="k">Sending</td><td className="v">
{config.send_enabled ? <span className="t-ok">enabled</span>
: <span className="t-warn">disabled nothing can leave this instance</span>}</td></tr>
<tr><td className="k">SSO</td><td className="v">
{config.oidc_enabled ? "Authentik" : "local accounts only"}</td></tr>
</tbody></table>
<div className="hint">
These come from the environment see <code>docs/INTEGRATIONS.md</code>. They are deliberately not
editable here so a UI bug cannot switch on customer email.
</div>
</div>
</main>
);
}

164
frontend/src/styles.css Normal file
View File

@@ -0,0 +1,164 @@
:root {
--bg:#0d1017; --panel:#141821; --panel2:#1b202b; --line:#262d3a; --line2:#333b4a;
--fg:#e8ebf2; --dim:#8a93a5; --faint:#5d6675; --accent:#4c8dff;
--ok:#35c46a; --warn:#e0a336; --bad:#f2545b; --info:#59a0f5; --violet:#a97bf0;
--mono:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace; --r:10px;
}
@media (prefers-color-scheme:light){:root{
--bg:#f4f6f9;--panel:#fff;--panel2:#f1f4f8;--line:#e0e5ec;--line2:#cfd6e0;
--fg:#151a22;--dim:#5b6473;--faint:#8e97a5;--accent:#1f6feb;
--ok:#12864a;--warn:#96650a;--bad:#cf2530;--info:#0969da;--violet:#7a44d6;}}
*{box-sizing:border-box}
body{margin:0;background:var(--bg);color:var(--fg);
font:14px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif}
a{color:var(--accent)}
button{font:inherit;font-size:13px;padding:7px 13px;border-radius:8px;border:1px solid var(--line2);
background:var(--panel2);color:var(--fg);cursor:pointer;transition:.12s}
button:hover:not(:disabled){border-color:var(--accent)}
button:disabled{opacity:.45;cursor:not-allowed}
button.pri{background:var(--accent);border-color:var(--accent);color:#fff;font-weight:600}
button.sm{padding:4px 9px;font-size:12px}
button.del{color:var(--bad);border-color:color-mix(in srgb,var(--bad) 45%,transparent)}
input,select,textarea{font:inherit;font-size:13px;padding:7px 10px;border-radius:7px;
border:1px solid var(--line2);background:var(--bg);color:var(--fg);width:100%}
textarea{min-height:220px;resize:vertical;line-height:1.55}
header.top{display:flex;align-items:center;gap:14px;padding:9px 18px;background:var(--panel);
border-bottom:1px solid var(--line);position:sticky;top:0;z-index:20;flex-wrap:wrap}
.brand{font-weight:680;font-size:14px}
.brand em{font-style:normal;color:var(--faint);font-weight:400;font-size:12.5px}
.navlink{font-size:12.5px;text-decoration:none;border:1px solid var(--line2);
padding:4px 10px;border-radius:7px;color:var(--accent)}
.navlink.on{background:var(--accent);border-color:var(--accent);color:#fff}
.spacer{margin-left:auto}
.who{font-size:12px;color:var(--dim)}
.filters{background:var(--panel);border-bottom:1px solid var(--line);padding:7px 18px;
display:flex;flex-direction:column;gap:5px;position:sticky;top:51px;z-index:19}
.frow{display:flex;align-items:center;gap:9px}
.flab{font-size:10.5px;text-transform:uppercase;letter-spacing:.5px;color:var(--faint);width:44px;flex:none}
.chips{display:flex;gap:6px;flex-wrap:wrap}
.chip{font-size:11.5px;padding:3px 10px;border-radius:99px;border:1px solid var(--line2);
background:var(--panel2);color:var(--dim);cursor:pointer;white-space:nowrap;user-select:none}
.chip.on{background:var(--accent);border-color:var(--accent);color:#fff}
.chip.overdue:not(.on){color:var(--bad);border-color:color-mix(in srgb,var(--bad) 45%,transparent)}
.split{display:grid;grid-template-columns:320px 1fr;height:calc(100vh - 118px)}
@media(max-width:940px){.split{grid-template-columns:1fr}}
.rail{border-right:1px solid var(--line);background:var(--panel);overflow-y:auto}
.stage{overflow-y:auto;padding:22px 26px 60px}
.grp-h{display:flex;align-items:center;gap:8px;padding:8px 14px;cursor:pointer;background:var(--panel2);
user-select:none;font-size:12px;letter-spacing:.3px;text-transform:uppercase;color:var(--dim);
border-bottom:1px solid var(--line)}
.grp-h:hover{color:var(--fg)}
.grp-h .n{margin-left:auto;font-size:11px;padding:1px 7px;border-radius:99px;background:var(--bg);
color:var(--dim);text-transform:none}
.grp-h .n.hot{background:var(--bad);color:#fff}
.case{padding:9px 14px;border-bottom:1px solid var(--line);cursor:pointer;display:flex;gap:9px;align-items:flex-start}
.case:hover{background:var(--panel2)}
.case.on{background:color-mix(in srgb,var(--accent) 15%,transparent);box-shadow:inset 3px 0 var(--accent)}
.case.off{opacity:.55}
.case .bar{width:3px;align-self:stretch;border-radius:2px;background:var(--faint);flex:none}
.bar.overdue{background:var(--bad)} .bar.real{background:var(--warn)} .bar.unverified{background:var(--info)}
.bar.rule_defect{background:var(--violet)} .bar.resolved{background:var(--ok)}
.bar.pending{background:var(--info);opacity:.5}
.case .mid{min-width:0;flex:1}
.case .nm{font-weight:600;font-size:13px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.case .sub{color:var(--dim);font-size:11.5px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.case .ag{font-family:var(--mono);font-size:11px;color:var(--faint);flex:none}
.card{background:var(--panel);border:1px solid var(--line);border-radius:var(--r);padding:16px 18px;margin-bottom:14px}
.hero{padding:20px 22px}
.crumb{font-size:11.5px;color:var(--faint);letter-spacing:.3px;text-transform:uppercase;margin-bottom:9px;
display:flex;gap:8px;align-items:center;flex-wrap:wrap}
.vd{font-size:20px;font-weight:660;line-height:1.3}
.vsub{color:var(--dim);margin-top:7px;font-size:13.5px;max-width:80ch}
h3.sec{margin:0 0 4px;font-size:12px;text-transform:uppercase;letter-spacing:.6px;color:var(--dim)}
.badge{font-size:11px;padding:2px 9px;border-radius:99px;border:1px solid var(--line2);color:var(--dim)}
.badge.overdue{background:var(--bad);border-color:var(--bad);color:#fff;font-weight:600}
.badge.real{color:var(--warn);border-color:color-mix(in srgb,var(--warn) 50%,transparent)}
.badge.rule_defect{color:var(--violet);border-color:color-mix(in srgb,var(--violet) 50%,transparent)}
.badge.resolved,.badge.ok{color:var(--ok);border-color:color-mix(in srgb,var(--ok) 45%,transparent)}
.row{display:flex;gap:10px;flex-wrap:wrap;align-items:center}
.mt{margin-top:12px}
.states{display:flex;align-items:center;gap:14px;flex-wrap:wrap;margin-top:16px}
.sbox{flex:1;min-width:150px;background:var(--panel2);border:1px solid var(--line);border-radius:9px;padding:11px 14px}
.sbox .lbl{font-size:10.5px;text-transform:uppercase;letter-spacing:.5px;color:var(--faint)}
.sbox .val{font-family:var(--mono);font-size:16px;font-weight:600;margin-top:3px}
.sbox.bad{border-color:color-mix(in srgb,var(--bad) 45%,transparent)} .sbox.bad .val{color:var(--bad)}
.sbox.ok .val{color:var(--ok)}
.eqlink{font-size:20px;color:var(--faint)} .eqlink.bad{color:var(--bad)}
.slots{display:flex;gap:4px;margin-top:14px;flex-wrap:wrap}
.slot{flex:1 1 78px;min-width:70px;height:52px;border-radius:7px;border:1px solid var(--line2);
display:flex;flex-direction:column;justify-content:center;padding:5px 7px;overflow:hidden}
.slot .sn{font-size:10.5px;font-weight:600;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.slot .ss{font-size:9px;text-transform:uppercase;letter-spacing:.4px;opacity:.75;margin-top:1px}
.slot .idx{font-size:9px;color:var(--faint)}
.slot.vm{background:color-mix(in srgb,var(--ok) 22%,transparent);border-color:color-mix(in srgb,var(--ok) 55%,transparent)}
.slot.vm.bad{background:color-mix(in srgb,var(--bad) 20%,transparent);border-color:color-mix(in srgb,var(--bad) 55%,transparent)}
.slot.vm.unlinked{background:color-mix(in srgb,var(--bad) 28%,transparent);border-color:var(--bad)}
.slot.unaccounted{background:color-mix(in srgb,var(--bad) 16%,transparent);border-style:dashed;
border-color:color-mix(in srgb,var(--bad) 50%,transparent);align-items:center;justify-content:center;color:var(--bad)}
.slot.free{border-style:dashed;color:var(--faint);align-items:center;justify-content:center}
.legend{display:flex;gap:16px;margin-top:8px;font-size:12px;color:var(--dim);flex-wrap:wrap}
.legend i{width:9px;height:9px;border-radius:2px;display:inline-block;margin-right:5px}
.roster{margin-top:10px;border:1px solid var(--line);border-radius:9px;overflow:hidden}
.rrow{display:grid;grid-template-columns:1fr 118px 26px 118px 52px;gap:8px;align-items:center;
padding:7px 12px;border-top:1px solid var(--line);font-size:12.5px}
.rrow:first-child{border-top:none;background:var(--panel2);font-size:10.5px;text-transform:uppercase;
letter-spacing:.5px;color:var(--faint)}
.rrow .rn{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-weight:600}
.rrow .rs{font-family:var(--mono);font-size:12px}
.rrow .eq{text-align:center;color:var(--ok)}
.rrow.bad .eq,.rrow.bad .rs{color:var(--bad)}
.rrow.unlinked{background:color-mix(in srgb,var(--bad) 9%,transparent)}
.rrow .rg{text-align:right;color:var(--faint);font-family:var(--mono);font-size:11.5px}
details.fold{background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin-bottom:10px}
details.fold>summary{cursor:pointer;padding:11px 18px;font-size:12px;text-transform:uppercase;
letter-spacing:.5px;color:var(--dim);list-style:none}
details.fold>summary::-webkit-details-marker{display:none}
details.fold>summary::before{content:"▸ ";font-size:10px;color:var(--faint)}
details.fold[open]>summary::before{content:"▾ "}
.foldb{padding:2px 18px 16px}
table{width:100%;border-collapse:collapse}
td{padding:5px 6px;border-bottom:1px solid var(--line);vertical-align:top;font-size:12.5px}
tr:last-child td{border-bottom:none}
td.k{color:var(--dim);width:180px;white-space:nowrap}
td.v{font-family:var(--mono);word-break:break-word}
.t-ok{color:var(--ok)} .t-warn{color:var(--warn)} .t-bad{color:var(--bad)}
.det{display:block;color:var(--faint);font-family:inherit;font-size:11.5px;margin-top:2px}
pre{background:var(--bg);border:1px solid var(--line);padding:10px;border-radius:7px;
overflow-x:auto;font-size:11.5px;margin:0;white-space:pre-wrap}
.cmd{display:flex;gap:8px;align-items:center;background:var(--bg);border:1px solid var(--line);
border-radius:7px;padding:7px 10px;margin-top:8px;font-family:var(--mono);font-size:12.5px}
.cmd code{flex:1;min-width:0;overflow-x:auto;white-space:nowrap}
.hint{font-size:12px;color:var(--faint);margin-top:9px}
.warnbox{border:1px solid color-mix(in srgb,var(--warn) 50%,transparent);
background:color-mix(in srgb,var(--warn) 11%,transparent);border-radius:8px;padding:10px 12px;
font-size:12.5px;margin-bottom:14px}
.empty{color:var(--faint);text-align:center;padding:60px 20px}
.spin{width:15px;height:15px;border:2px solid var(--line2);border-top-color:var(--accent);
border-radius:50%;display:inline-block;animation:sp .7s linear infinite;vertical-align:-3px}
@keyframes sp{to{transform:rotate(360deg)}}
.scrim{position:fixed;inset:0;background:rgba(0,0,0,.55);z-index:40}
.drawer{position:fixed;top:0;right:0;height:100%;width:min(640px,95vw);background:var(--panel);
border-left:1px solid var(--line);z-index:50;display:flex;flex-direction:column}
.dh{padding:15px 20px;border-bottom:1px solid var(--line);display:flex;align-items:center;gap:10px}
.dh h2{margin:0;font-size:15px;flex:1}
.db{padding:18px 20px;overflow-y:auto;flex:1}
.df{padding:14px 20px;border-top:1px solid var(--line);display:flex;gap:10px;align-items:center;flex-wrap:wrap}
.fld{margin-bottom:14px}
.fld label{display:block;font-size:11px;text-transform:uppercase;letter-spacing:.5px;color:var(--faint);margin-bottom:5px}
.timeline{list-style:none;margin:0;padding:0}
.timeline li{padding:8px 0 8px 16px;border-left:2px solid var(--line);position:relative;font-size:12.5px}
.timeline li::before{content:"";position:absolute;left:-5px;top:13px;width:8px;height:8px;
border-radius:50%;background:var(--line2)}
.timeline .ts{color:var(--faint);font-size:11px}
.login{max-width:380px;margin:12vh auto;padding:26px}

139
frontend/src/types.ts Normal file
View File

@@ -0,0 +1,139 @@
export type Verdict =
| "overdue" | "real" | "unverified" | "chronic"
| "low_impact" | "pending" | "resolved" | "rule_defect" | "suppressed";
export interface Screen {
verdict: Verdict;
label: string;
reason: string;
actionable: boolean;
current_state?: string;
detail?: string;
}
export interface CaseRef {
id: number;
fingerprint: string;
status: string;
is_open: boolean;
assignee: { id: number; email: string; name: string } | null;
zendesk_ticket_id: string;
zendesk_ticket_url: string;
jira_issue_key: string;
jira_issue_url: string;
notes: string;
reopen_count: number;
events?: CaseEvent[];
}
export interface CaseEvent {
id: number;
action: string;
detail: string;
actor: string;
created_at: string;
payload?: Record<string, unknown> | null;
}
export interface Alert {
id: string;
kind: string;
title: string;
state: string;
priority: string;
ettr: string;
effective_age_text: string;
age_text: string;
age_is_reset: boolean;
openstack_id: string;
instance_name: string;
host: string;
region: string;
region_label: string;
status: string;
floating_ip: string;
org_id: string;
org_name: string;
is_kubernetes: boolean;
labels: Record<string, string>;
annotations: Record<string, string>;
screen: Screen;
case: CaseRef | null;
}
export interface Group {
kind: string;
title: string;
total: number;
actionable: number;
noise: number;
alerts: Alert[];
}
export interface Queue {
error: string;
warnings: string[];
totals: { prometheus: number; cx: number; infrastructure: number; excluded: number };
excluded_note: string;
summary: { counts: Record<string, number>; actionable: number; screened_out: number; labels: Record<string, string> };
groups: Group[];
infrastructure: { source: string; label: string; total: number; by_alertname: { name: string; count: number }[] }[];
}
export interface Finding { label: string; value: string; tone: string; detail: string }
export interface ActionStep { text: string; owner: string; kind: string; guide: string; status: string; detail: string }
export interface Draft {
template_id: string; label: string; subject: string; body: string;
channel: string; when: string; unfilled: string[]; source: string;
}
export interface GpuSlot {
kind: "vm" | "unaccounted" | "free";
name?: string; linked?: boolean; match?: boolean;
ih_status?: string; os_status?: string;
}
export interface RosterRow {
name: string; openstack_id: string; infrahub_id: string;
ih_status: string; os_status: string; gpus: string;
linked: boolean; match: boolean; tempest: boolean; org: string;
}
export interface Visual {
type?: "states" | "gpu" | "claimants";
infrahub?: string; openstack?: string; task?: string; match?: boolean;
host?: string; name?: string; flavor?: string; gpus?: string; fault?: string; never_built?: boolean;
physical?: number | null; in_use_metric?: number | null; accounted?: number;
gap?: number; instances?: number; spare_capacity_artifact?: boolean;
slots?: GpuSlot[]; roster?: RosterRow[];
ip?: string; items?: { name: string; ih_status: string; os_status: string; verdict: string }[];
}
export interface IntegrationAction {
id: string; kind: "zendesk" | "jira" | "manual"; label: string; summary: string;
payload: any; recipients: string[]; enabled: boolean; blocked_reason: string;
}
export interface Diagnosis {
alert: Alert;
verdict: string;
assessment: string;
confidence: string;
findings: Finding[];
actions: ActionStep[];
drafts: Draft[];
contacts: { organization: string; owners: string[]; resolved: boolean };
evidence: Record<string, unknown>;
notes: string[];
error: string;
visual: Visual;
integrations: { actions: IntegrationAction[]; zendesk_configured: boolean; jira_configured: boolean };
elapsed_seconds?: number;
}
export interface User { id: number; email: string; name: string; is_admin: boolean; provider: string; signoff_name: string }
export interface AppConfig {
app_name: string; oidc_enabled: boolean; local_login: boolean;
zendesk_ready: boolean; jira_ready: boolean; send_enabled: boolean;
linkage_scan: boolean; jira_project: string;
}

19
frontend/tsconfig.json Normal file
View File

@@ -0,0 +1,19 @@
{
"compilerOptions": {
"target": "ES2022",
"lib": ["ES2022", "DOM", "DOM.Iterable"],
"module": "ESNext",
"moduleResolution": "bundler",
"jsx": "react-jsx",
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noFallthroughCasesInSwitch": true,
"skipLibCheck": true,
"isolatedModules": true,
"resolveJsonModule": true,
"allowImportingTsExtensions": false,
"noEmit": true
},
"include": ["src"]
}

19
frontend/vite.config.ts Normal file
View File

@@ -0,0 +1,19 @@
import { defineConfig } from "vite";
import react from "@vitejs/plugin-react";
// In development Vite serves the app and proxies the API to the Python backend,
// so the two can be worked on independently. In production the built bundle is
// copied into the backend image and served from the same origin.
export default defineConfig({
plugins: [react()],
server: {
port: 5173,
proxy: {
"/api": {
target: process.env.VITE_API_TARGET ?? "http://localhost:8080",
changeOrigin: true,
},
},
},
build: { outDir: "dist", sourcemap: false },
});

View File

@@ -1,437 +0,0 @@
"""Localhost HTTP server: alert queue, background triage jobs, JSON API."""
from __future__ import annotations
import json
import subprocess
import threading
import time
import traceback
import urllib.parse
import uuid
from concurrent.futures import ThreadPoolExecutor
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Any, Optional
from . import (VERSION, alerts as alertlib, comms, cxbridge, integrations, linkage,
runbooks, screening, settings as settings_mod, ui, ui_linkage,
ui_settings, ui_v2)
from .prometheus import (
AlertCache, PrometheusClient, PrometheusError, RuleIndex, StateSnapshot, TrueAgeIndex,
parse_alert_text,
)
TRIAGE_WORKERS = 3
JOB_TTL_SECONDS = 30 * 60
class Jobs:
"""In-memory triage jobs. Diagnosis takes tens of seconds, so the UI polls."""
def __init__(self, workers: int = TRIAGE_WORKERS):
self._pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="triage")
self._lock = threading.Lock()
self._jobs: dict[str, dict[str, Any]] = {}
def submit(self, alert: alertlib.Alert, prom: PrometheusClient,
snap: Any = None, force: bool = False, user_settings: Any = None) -> str:
job_id = uuid.uuid4().hex[:12]
with self._lock:
self._reap()
self._jobs[job_id] = {
"id": job_id,
"state": "running",
"created": time.time(),
"alert": alert.to_json(),
"result": None,
"error": "",
}
self._pool.submit(self._run, job_id, alert, prom, snap, force, user_settings)
return job_id
def _run(self, job_id: str, alert: alertlib.Alert, prom: PrometheusClient,
snap: Any = None, force: bool = False, user_settings: Any = None) -> None:
started = time.monotonic()
try:
diagnosis = runbooks.diagnose(alert, prom, snap, force, user_settings)
payload = diagnosis.to_json()
payload["elapsed_seconds"] = round(time.monotonic() - started, 1)
with self._lock:
job = self._jobs.get(job_id)
if job is not None:
job.update({"state": "done", "result": payload})
except Exception:
with self._lock:
job = self._jobs.get(job_id)
if job is not None:
job.update({"state": "error", "error": traceback.format_exc(limit=4)})
def get(self, job_id: str) -> Optional[dict[str, Any]]:
with self._lock:
job = self._jobs.get(job_id)
return dict(job) if job else None
def _reap(self) -> None:
cutoff = time.time() - JOB_TTL_SECONDS
for key in [k for k, v in self._jobs.items() if v["created"] < cutoff]:
self._jobs.pop(key, None)
class App:
def __init__(self, prom: PrometheusClient):
self.prom = prom
self.cache = AlertCache(prom)
self.rules = RuleIndex(prom)
self.snapshot = StateSnapshot(prom)
self.true_age = TrueAgeIndex(prom)
self.jobs = Jobs()
self.scan = linkage.Scan()
self.settings = settings_mod.Settings()
def warm(self, log=print) -> None:
"""Populate the caches before serving.
Recovering true alert ages reads a week of ALERTS history, so doing it
lazily would make the first page load take ~20 seconds.
"""
try:
self.rules.ensure()
log(f" rule index: {self.rules.count} alerting rule(s)")
snap = self.snapshot.get()
log(f" state snapshot: {len(snap.by_openstack_id)} VMs, {len(snap.total_gpus)} hosts")
if snap.pipeline_dips:
log(f" {len(snap.pipeline_dips)} Resources metric dip(s) in the last 24h "
"- alert ages will be recovered from history")
ages = self.true_age.get()
log(f" alert history: {ages.count} alert(s) indexed over {ages.WINDOW_DAYS} days")
except PrometheusError as exc:
log(f" WARN: could not warm Prometheus caches: {exc}")
# --- endpoints ---------------------------------------------------------
def health(self) -> dict[str, Any]:
checks: list[dict[str, Any]] = []
def add(name: str, ok: bool, detail: str) -> None:
checks.append({"name": name, "ok": ok, "detail": detail})
try:
path = cxbridge.cx_tools_path()
add("CX-Tools", True, path)
except cxbridge.BridgeError as exc:
add("CX-Tools", False, str(exc))
try:
cfg = cxbridge.config()
add("API credentials", True, "Infrahub and InfraInsight keys loaded from 1Password")
add("Infrahub endpoint", True, cfg.infrahub_base)
except cxbridge.BridgeError as exc:
add("API credentials", False, str(exc))
try:
rc, out, _err = _run(["docker", "ps", "--format", "{{.Names}}"], 10)
running = {x.strip() for x in out.splitlines() if x.strip()} if rc == 0 else set()
expected = {"ca1-osc", "ca2-osc", "us1-osc", "no1-osc"}
missing = sorted(expected - running)
add("OpenStack CLI containers", not missing,
"all present" if not missing else f"not running: {', '.join(missing)}")
except Exception as exc:
add("OpenStack CLI containers", False, str(exc))
try:
add("Prometheus", True, f"{self.prom.base} ({self.prom.describe_transport()})")
except PrometheusError as exc:
add("Prometheus", False, str(exc))
return {"version": VERSION, "ok": all(c["ok"] for c in checks), "checks": checks}
def alert_queue(self, force: bool = False) -> dict[str, Any]:
raw, error, age = self.cache.get(force=force)
snap = self.snapshot.get()
ages = self.true_age.get()
parsed = [alertlib.from_prometheus(a, self.rules, ages) for a in raw]
excluded = [a for a in parsed if alertlib.is_excluded(a)]
candidates = [a for a in parsed if not alertlib.is_excluded(a)]
cx = [a for a in candidates if a.category == "cx" and alertlib.cx_relevant(a)]
screening.screen_all(cx, snap, self.settings)
# Everything that is not a CX runbook alert: node-exporter in its own
# section, then the rest of the platform rules. Split by identity, since
# two distinct alerts can compare equal field-for-field.
in_cx = {id(a) for a in cx}
infra = [a for a in candidates if id(a) not in in_cx]
infra.sort(key=alertlib.sort_key)
return {
"error": error or snap.error or ages.error,
"cache_age_seconds": round(age, 1),
"warnings": screening.health_warnings(snap),
"totals": {
"prometheus": len(parsed),
"cx": len(cx),
"infrastructure": len(infra),
"excluded": len(excluded),
},
"excluded_note": (
f"{len(excluded)} '{', '.join(sorted({a.alertname for a in excluded}))}' alerts hidden"
if excluded else ""
),
"integrations": {"zendesk": integrations.zendesk_configured(),
"jira": integrations.jira_configured()},
"summary": screening.summarize(cx),
"groups": alertlib.group_alerts(cx),
"infrastructure": _infra_sections(infra),
}
def triage(self, labels: dict[str, str], annotations: Optional[dict[str, str]] = None,
state: str = "firing", active_at: Any = None, force: bool = False) -> dict[str, Any]:
alert = alertlib.from_labels(labels, annotations, state=state, active_at=active_at)
if alert.kind == "excluded":
return {"error": f"'{alertlib.clean_alertname(alert.alertname)}' is excluded as a monitoring fault."}
if alert.kind == "other":
return {"error": f"'{alertlib.clean_alertname(alert.alertname)}' is not an alert type the CX runbooks cover."}
meta = self.rules.get(alert.alertname)
if meta:
alert.rule_file = str(meta.get("file") or "")
alert.rule_group = str(meta.get("group") or "")
alert.for_seconds = int(meta.get("for_seconds") or 0)
snap = self.snapshot.get()
alert.true_age_minutes, alert.true_age_capped = self.true_age.get().lookup(alert.labels)
alert.screen = screening.screen(alert, snap, self.settings)
job_id = self.jobs.submit(alert, self.prom, snap, force, self.settings)
return {"job_id": job_id, "alert": alert.to_json()}
def triage_by_id(self, alert_id: str, force: bool = False) -> dict[str, Any]:
raw, error, _age = self.cache.get()
if error and not raw:
return {"error": error}
for item in raw:
alert = alertlib.from_prometheus(item, self.rules, self.true_age.get())
if alert.fingerprint() == alert_id:
return self.triage(alert.labels, alert.annotations, alert.state,
item.get("activeAt"), force=force)
return {"error": "That alert is no longer firing. Refresh the queue."}
def parse(self, text: str) -> dict[str, Any]:
found = parse_alert_text(text)
if not found:
return {"error": "Could not find any label set in that text. Paste an ALERTS{...} line or a Prometheus graph URL."}
out = []
for labels in found:
alert = alertlib.from_labels(labels)
out.append({"alert": alert.to_json(), "supported": alertlib.cx_relevant(alert)})
return {"parsed": out}
def settings_payload(self) -> dict[str, Any]:
return self.settings.to_json()
def _current_alerts(self) -> list[Any]:
raw, _error, _age = self.cache.get()
return [a for a in (alertlib.from_prometheus(x, self.rules) for x in raw)
if not alertlib.is_excluded(a) and alertlib.cx_relevant(a)]
def settings_action(self, body: dict[str, Any]) -> dict[str, Any]:
action = str(body.get("action") or "")
if action == "save_rule":
rule = self.settings.upsert_rule(body.get("rule") or {})
return {"ok": True, "rule": rule, "settings": self.settings.to_json()}
if action == "delete_rule":
self.settings.delete_rule(str(body.get("id") or ""))
return {"ok": True, "settings": self.settings.to_json()}
if action == "toggle_rule":
self.settings.toggle_rule(str(body.get("id") or ""), bool(body.get("enabled")))
return {"ok": True, "settings": self.settings.to_json()}
if action == "general":
self.settings.set_general(body.get("agent_name"), body.get("chronic_days"))
return {"ok": True, "settings": self.settings.to_json()}
if action == "preview":
hits = settings_mod.preview(self.settings, body.get("rule") or {}, self._current_alerts())
return {"ok": True, "matches": hits, "count": len(hits)}
return {"ok": False, "error": f"Unknown action: {action}"}
def start_scan(self) -> dict[str, Any]:
if self.scan.state == "running":
return {"started": False, "reason": "already running"}
snap = self.snapshot.get()
threading.Thread(target=self.scan.run, args=(snap,), daemon=True).start()
return {"started": True}
def send_zendesk(self, body: dict[str, Any]) -> dict[str, Any]:
"""Deliberately refuses until Zendesk is configured AND enabled.
Contacting a customer is the one thing this tool must never do as a
side effect, so delivery stays behind explicit configuration rather
than being reachable from the UI by default.
"""
if not integrations.zendesk_configured():
return {"ok": False, "error": "Zendesk is not configured. Set CX_ZENDESK_SUBDOMAIN, "
"CX_ZENDESK_EMAIL and CX_ZENDESK_TOKEN, then restart."}
return {"ok": False, "error": "Sending is not enabled in this build. The payload is ready; "
"wiring delivery is a deliberate, separate step."}
def templates(self) -> dict[str, Any]:
return {"templates": [d.to_json() for d in (comms.draft(k) for k in comms._TEMPLATES) if d]}
SOURCE_LABELS = {
"node-exporter-rules.yml": "Node exporter (hosts)",
"ceph-rules.yml": "Ceph",
"mysql-rules.yml": "MySQL",
"mysql-performance-rules.yml": "MySQL performance",
"galera-rules.yml": "Galera",
"openstack-rules.yml": "OpenStack services",
"blackbox.yml": "Blackbox / OOB",
"infrahub-rules.yml": "Infrahub (no CX runbook)",
}
def _infra_sections(items: list[alertlib.Alert]) -> list[dict[str, Any]]:
"""Group infrastructure alerts by their rule file, node-exporter first."""
buckets: dict[str, list[alertlib.Alert]] = {}
for alert in items:
buckets.setdefault(alert.rule_file or "unknown", []).append(alert)
sections = []
for source, members in buckets.items():
by_name: dict[str, int] = {}
for alert in members:
name = alertlib.clean_alertname(alert.alertname)
by_name[name] = by_name.get(name, 0) + 1
sections.append({
"source": source,
"label": SOURCE_LABELS.get(source, source),
"total": len(members),
"by_alertname": sorted(({"name": k, "count": v} for k, v in by_name.items()),
key=lambda x: (-x["count"], x["name"])),
"alerts": [a.to_json() for a in members],
})
sections.sort(key=lambda s: (s["source"] != alertlib.NODE_RULE_FILE, -s["total"]))
return sections
def _run(cmd: list[str], timeout: int) -> tuple[int, str, str]:
try:
p = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout)
return p.returncode, p.stdout, p.stderr
except Exception as exc:
return 1, "", str(exc)
class Handler(BaseHTTPRequestHandler):
server_version = f"cx-triage/{VERSION}"
app: App
def log_message(self, fmt: str, *args: Any) -> None:
if self.path.startswith("/api/jobs/"):
return
print(f" {self.command} {self.path}")
# --- helpers ----------------------------------------------------------
def _send_json(self, payload: Any, code: int = 200) -> None:
body = json.dumps(payload, default=str).encode()
self.send_response(code)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
def _send_html(self, html: str) -> None:
body = html.encode()
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _body_json(self) -> dict[str, Any]:
length = int(self.headers.get("Content-Length") or 0)
if not length:
return {}
try:
return json.loads(self.rfile.read(length).decode("utf-8", errors="replace")) or {}
except json.JSONDecodeError:
return {}
# --- routes -----------------------------------------------------------
def do_GET(self) -> None: # noqa: N802
parsed = urllib.parse.urlparse(self.path)
path = parsed.path
query = urllib.parse.parse_qs(parsed.query)
try:
if path in ("/", "/index.html"):
self._send_html(ui_v2.PAGE)
elif path == "/classic":
self._send_html(ui.PAGE)
elif path == "/settings":
self._send_html(ui_settings.PAGE)
elif path == "/api/settings":
self._send_json(self.app.settings_payload())
elif path == "/linkage":
self._send_html(ui_linkage.PAGE)
elif path == "/api/linkage":
self._send_json(self.app.scan.to_json())
elif path == "/api/health":
self._send_json(self.app.health())
elif path == "/api/alerts":
self._send_json(self.app.alert_queue(force=query.get("force", ["0"])[0] == "1"))
elif path == "/api/templates":
self._send_json(self.app.templates())
elif path.startswith("/api/jobs/"):
job = self.app.jobs.get(path.rsplit("/", 1)[-1])
self._send_json(job or {"error": "Unknown job."}, 200 if job else 404)
else:
self._send_json({"error": "Not found."}, 404)
except Exception as exc:
self._send_json({"error": f"{type(exc).__name__}: {exc}"}, 500)
def do_POST(self) -> None: # noqa: N802
path = urllib.parse.urlparse(self.path).path
body = self._body_json()
try:
force = bool(body.get("force"))
if path == "/api/triage":
if body.get("alert_id"):
self._send_json(self.app.triage_by_id(str(body["alert_id"]), force=force))
elif isinstance(body.get("labels"), dict):
self._send_json(self.app.triage(body["labels"], body.get("annotations"), force=force))
else:
self._send_json({"error": "Provide alert_id or labels."}, 400)
elif path == "/api/parse":
self._send_json(self.app.parse(str(body.get("text") or "")))
elif path == "/api/actions/zendesk":
self._send_json(self.app.send_zendesk(body))
elif path == "/api/settings":
self._send_json(self.app.settings_action(body))
elif path == "/api/linkage/scan":
self._send_json(self.app.start_scan())
elif path == "/api/linkage/enrich":
self._send_json(linkage.enrich(str(body.get("region") or ""),
str(body.get("openstack_id") or "")))
else:
self._send_json({"error": "Not found."}, 404)
except Exception as exc:
self._send_json({"error": f"{type(exc).__name__}: {exc}"}, 500)
def serve(host: str = "127.0.0.1", port: int = 8765, prometheus_base: Optional[str] = None) -> None:
from .prometheus import DEFAULT_BASE
prom = PrometheusClient(prometheus_base or DEFAULT_BASE)
app = App(prom)
print("Warming caches...")
app.warm()
Handler.app = app
httpd = ThreadingHTTPServer((host, port), Handler)
httpd.daemon_threads = True
print(f"cx-triage listening on http://{host}:{port}")
try:
httpd.serve_forever()
except KeyboardInterrupt:
print("\nshutting down")
finally:
httpd.server_close()

View File

@@ -1,429 +0,0 @@
"""The single-page UI, served inline so the app needs no assets or CDN."""
from __future__ import annotations
PAGE = r"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>CX Triage</title>
<style>
:root {
--bg: #0f1115; --panel: #171a21; --panel2: #1d2129; --line: #2a2f3a;
--fg: #e6e9ef; --muted: #8b93a3; --accent: #5b9cf8;
--ok: #3fb950; --warn: #d29922; --bad: #f85149; --info: #58a6ff;
--mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
}
@media (prefers-color-scheme: light) {
:root {
--bg: #f6f7f9; --panel: #ffffff; --panel2: #f0f2f5; --line: #dfe3e8;
--fg: #1a1d23; --muted: #5c6472; --accent: #1f6feb;
--ok: #1a7f37; --warn: #9a6700; --bad: #cf222e; --info: #0969da;
}
}
* { box-sizing: border-box; }
body {
margin: 0; background: var(--bg); color: var(--fg);
font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
}
header {
display: flex; align-items: center; gap: 14px; flex-wrap: wrap;
padding: 10px 20px; border-bottom: 1px solid var(--line); background: var(--panel);
position: sticky; top: 0; z-index: 10;
}
h1 { font-size: 15px; margin: 0; font-weight: 650; }
h1 span { color: var(--muted); font-weight: 400; }
.pills { display: flex; gap: 6px; flex-wrap: wrap; margin-left: auto; }
.pill { font-size: 11px; padding: 3px 9px; border-radius: 999px; border: 1px solid var(--line);
background: var(--panel2); color: var(--muted); white-space: nowrap; }
.pill.ok { color: var(--ok); border-color: color-mix(in srgb, var(--ok) 40%, transparent); }
.pill.bad { color: var(--bad); border-color: color-mix(in srgb, var(--bad) 40%, transparent); }
button { font: inherit; font-size: 13px; padding: 5px 12px; border-radius: 6px;
border: 1px solid var(--line); background: var(--panel2); color: var(--fg); cursor: pointer; }
button:hover { border-color: var(--accent); }
button.primary { background: var(--accent); border-color: var(--accent); color: #fff; }
main { display: grid; grid-template-columns: minmax(340px, 32%) 1fr; min-height: calc(100vh - 52px); }
@media (max-width: 900px) { main { grid-template-columns: 1fr; } }
#queue { border-right: 1px solid var(--line); background: var(--panel);
overflow-y: auto; max-height: calc(100vh - 52px); }
#detail { padding: 20px 24px; overflow-y: auto; max-height: calc(100vh - 52px); }
.tabs { display: flex; border-bottom: 1px solid var(--line); }
.tab { flex: 1; padding: 10px 8px; text-align: center; cursor: pointer; font-size: 13px;
color: var(--muted); border-bottom: 2px solid transparent; }
.tab.on { color: var(--fg); border-bottom-color: var(--accent); font-weight: 600; }
.tab b { font-weight: 650; }
.qhead { padding: 10px 14px; border-bottom: 1px solid var(--line);
display: flex; flex-direction: column; gap: 8px; }
.qhead input, .qhead select, textarea {
font: inherit; width: 100%; padding: 6px 9px; border-radius: 6px;
border: 1px solid var(--line); background: var(--bg); color: var(--fg);
}
textarea { font-family: var(--mono); font-size: 12px; min-height: 62px; resize: vertical; }
.row { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--muted); }
.banner { margin: 10px 14px; padding: 9px 11px; border-radius: 7px; font-size: 12px;
border: 1px solid color-mix(in srgb, var(--warn) 45%, transparent);
background: color-mix(in srgb, var(--warn) 10%, transparent); color: var(--fg); }
.group { border-bottom: 1px solid var(--line); }
.ghead { padding: 9px 14px; display: flex; align-items: center; gap: 8px;
cursor: pointer; user-select: none; background: var(--panel2); }
.ghead:hover { background: color-mix(in srgb, var(--accent) 8%, var(--panel2)); }
.caret { color: var(--muted); font-size: 10px; width: 10px; transition: transform .12s; }
.group.closed .caret { transform: rotate(-90deg); }
.group.closed .glist { display: none; }
.gtitle { font-weight: 620; font-size: 13px; flex: 1; }
.gcount { font-size: 11px; padding: 1px 7px; border-radius: 999px; background: var(--bg); color: var(--muted); }
.gcount.live { color: #fff; background: var(--bad); }
.alert { padding: 9px 14px 9px 26px; border-top: 1px solid var(--line); cursor: pointer; }
.alert:hover { background: var(--panel2); }
.alert.sel { background: color-mix(in srgb, var(--accent) 14%, transparent); box-shadow: inset 3px 0 var(--accent); }
.alert.muted { opacity: .55; }
.alert .t { font-weight: 600; font-size: 13px; word-break: break-all; }
.alert .m { color: var(--muted); font-size: 12px; }
.alert .why { font-size: 11.5px; margin-top: 3px; }
.age { font-family: var(--mono); font-size: 11.5px; color: var(--muted); }
.tag { font-size: 10px; padding: 1px 6px; border-radius: 4px; border: 1px solid var(--line); color: var(--muted); }
.v-real { color: var(--bad); border-color: color-mix(in srgb, var(--bad) 45%, transparent); }
.v-unverified { color: var(--warn); border-color: color-mix(in srgb, var(--warn) 45%, transparent); }
.v-chronic, .v-low_impact, .v-pending { color: var(--muted); }
.v-resolved { color: var(--ok); border-color: color-mix(in srgb, var(--ok) 40%, transparent); }
.card { background: var(--panel); border: 1px solid var(--line); border-radius: 10px;
padding: 16px 18px; margin-bottom: 16px; }
.card h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .6px; color: var(--muted);
margin: 0 0 12px; font-weight: 600; }
.verdict { font-size: 17px; font-weight: 650; line-height: 1.35; margin-bottom: 8px; }
.assessment { color: var(--muted); }
.meta { display: flex; gap: 8px; flex-wrap: wrap; margin-top: 12px; align-items: center; }
table { width: 100%; border-collapse: collapse; }
td { padding: 5px 8px; border-bottom: 1px solid var(--line); vertical-align: top; font-size: 13px; }
tr:last-child td { border-bottom: none; }
td.k { color: var(--muted); width: 190px; white-space: nowrap; }
td.v { font-family: var(--mono); font-size: 12.5px; word-break: break-word; }
.tone-ok { color: var(--ok); } .tone-warn { color: var(--warn); }
.tone-bad { color: var(--bad); } .tone-info { color: var(--fg); }
.det { display: block; color: var(--muted); font-family: inherit; font-size: 12px; margin-top: 3px; }
ol.actions { margin: 0; padding-left: 0; list-style: none; counter-reset: a; }
ol.actions li { counter-increment: a; padding: 9px 0 9px 30px; border-bottom: 1px solid var(--line); position: relative; }
ol.actions li:last-child { border-bottom: none; }
ol.actions li::before {
content: counter(a); position: absolute; left: 0; top: 9px; width: 20px; height: 20px;
border-radius: 50%; border: 1px solid var(--line); font-size: 11px; color: var(--muted);
display: grid; place-items: center;
}
ol.actions li.done::before { content: "\2713"; color: var(--ok); border-color: var(--ok); }
.owner { font-size: 11px; padding: 1px 6px; border-radius: 4px; background: var(--panel2);
color: var(--muted); margin-left: 6px; }
.owner.infra { color: var(--warn); }
.kind { font-size: 10px; text-transform: uppercase; letter-spacing: .5px; color: var(--muted); margin-left: 6px; }
.guide { display: block; font-size: 12px; color: var(--muted); margin-top: 2px; }
.draft { border: 1px solid var(--line); border-radius: 8px; margin-bottom: 12px; overflow: hidden; }
.draft .dh { padding: 9px 12px; background: var(--panel2); display: flex; align-items: center; gap: 10px; flex-wrap: wrap; }
.draft .dl { font-weight: 600; font-size: 13px; flex: 1; min-width: 180px; }
.draft .dw { padding: 8px 12px; color: var(--warn); font-size: 12px; border-bottom: 1px solid var(--line); }
.draft pre { margin: 0; padding: 12px; white-space: pre-wrap; font-family: var(--mono); font-size: 12.5px; }
.contact { font-family: var(--mono); font-size: 12.5px; }
.notes li { color: var(--warn); margin-bottom: 5px; font-size: 13px; }
.empty { color: var(--muted); padding: 40px 20px; text-align: center; }
.spinner { width: 14px; height: 14px; border: 2px solid var(--line); border-top-color: var(--accent);
border-radius: 50%; display: inline-block; animation: spin .7s linear infinite; vertical-align: -2px; }
@keyframes spin { to { transform: rotate(360deg); } }
details.raw summary { cursor: pointer; color: var(--muted); font-size: 12px; }
details.raw pre { background: var(--panel2); padding: 10px; border-radius: 6px; overflow-x: auto; font-size: 11.5px; }
.ro { font-size: 11px; color: var(--muted); border: 1px dashed var(--line); padding: 2px 8px; border-radius: 999px; }
.infsec { border-bottom: 1px solid var(--line); }
.infsec .ghead { background: var(--panel2); }
.infrow { padding: 6px 14px 6px 26px; border-top: 1px solid var(--line); display: flex; gap: 8px; font-size: 12.5px; }
.infrow .n { flex: 1; }
</style>
</head>
<body>
<header>
<h1>CX Triage <span>alert diagnosis over CX-Tools</span></h1>
<span class="ro">read-only &middot; suggests, never acts</span>
<div class="pills" id="pills"></div>
<button id="refresh">Refresh</button>
</header>
<main>
<section id="queue">
<div class="tabs">
<div class="tab on" data-tab="cx" id="tabCx">CX runbooks</div>
<div class="tab" data-tab="infra" id="tabInfra">Infrastructure</div>
</div>
<div id="warnings"></div>
<div class="qhead">
<input id="search" type="search" placeholder="Filter by name, host, IP, org, ID...">
<label class="row"><input type="checkbox" id="showNoise" style="width:auto">
Show screened-out (chronic / pending / resolved) <span id="noiseCount"></span></label>
<div class="row" id="excludedNote"></div>
<details>
<summary style="cursor:pointer;color:var(--muted);font-size:12px">Paste an alert manually</summary>
<textarea id="paste" placeholder='Paste an ALERTS{...} line or a Prometheus graph URL'></textarea>
<button id="parseBtn" style="margin-top:6px">Diagnose pasted alert</button>
</details>
</div>
<div id="list"></div>
</section>
<section id="detail"><div class="empty">Pick an alert to diagnose.</div></section>
</main>
<script>
const $ = (s) => document.querySelector(s);
const esc = (s) => String(s == null ? "" : s).replace(/[&<>"']/g, c => (
{"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c]));
let STATE = { data: null, tab: "cx", selected: null, polling: null, closed: {}, lastBody: null };
const api = (p, o) => fetch(p, o).then(r => r.json());
async function loadHealth() {
const h = await api("/api/health");
$("#pills").innerHTML = (h.checks || []).map(c =>
`<span class="pill ${c.ok ? "ok" : "bad"}" title="${esc(c.detail)}">${esc(c.name)}</span>`).join("");
}
async function loadAlerts(force) {
if (!STATE.data) $("#list").innerHTML = '<div class="empty"><span class="spinner"></span> Loading alerts…</div>';
const d = await api("/api/alerts" + (force ? "?force=1" : ""));
STATE.data = d;
if (d.error) $("#list").innerHTML = `<div class="empty tone-bad">${esc(d.error)}</div>`;
const t = d.totals || {};
const s = d.summary || {};
$("#tabCx").innerHTML = `CX runbooks <b>${s.actionable || 0}</b>`;
$("#tabInfra").innerHTML = `Infrastructure <b>${t.infrastructure || 0}</b>`;
$("#noiseCount").innerHTML = `<span class="tag">${s.screened_out || 0} screened out</span>`;
$("#excludedNote").textContent = d.excluded_note || "";
$("#warnings").innerHTML = (d.warnings || []).map(w => `<div class="banner">${esc(w)}</div>`).join("");
render();
}
function matches(a, q) {
if (!q) return true;
return [a.instance_name, a.host, a.floating_ip, a.openstack_id, a.org_name, a.org_id, a.title, a.alertname]
.join(" ").toLowerCase().includes(q);
}
function render() {
document.querySelectorAll(".tab").forEach(t => t.classList.toggle("on", t.dataset.tab === STATE.tab));
STATE.tab === "cx" ? renderCx() : renderInfra();
}
function renderCx() {
const d = STATE.data || {};
const q = $("#search").value.trim().toLowerCase();
const showNoise = $("#showNoise").checked;
const out = [];
for (const g of d.groups || []) {
let rows = (g.alerts || []).filter(a => matches(a, q));
if (!showNoise) rows = rows.filter(a => a.screen && a.screen.actionable);
if (!rows.length) continue;
const live = rows.filter(a => a.screen && a.screen.actionable).length;
const closed = STATE.closed[g.kind] === true;
out.push(`<div class="group ${closed ? "closed" : ""}" data-kind="${esc(g.kind)}">
<div class="ghead">
<span class="caret">&#9660;</span>
<span class="gtitle">${esc(g.title)}</span>
<span class="gcount ${live ? "live" : ""}">${live ? live + " to action" : "0 to action"}</span>
${g.noise ? `<span class="gcount" title="screened out: chronic, pending, low impact or already resolved">${g.noise} not new</span>` : ""}
</div>
<div class="glist">${rows.map(a => alertRow(a)).join("")}</div>
</div>`);
}
$("#list").innerHTML = out.length ? out.join("") :
`<div class="empty">${(d.summary || {}).actionable === 0
? "Nothing new needs action. " + ((d.summary || {}).screened_out || 0) + " alert(s) screened out \u2014 tick the box above to see them."
: "No matching alerts."}</div>`;
document.querySelectorAll(".group .ghead").forEach(el => el.onclick = () => {
const kind = el.parentElement.dataset.kind;
STATE.closed[kind] = !STATE.closed[kind];
render();
});
document.querySelectorAll(".alert").forEach(el =>
el.onclick = () => triage({ alert_id: el.dataset.id }, el.dataset.id));
}
function alertRow(a) {
const subject = a.kind === "duplicate_ip" ? a.floating_ip
: (a.kind === "rogue_vm" || a.kind === "total_gpus" || a.kind === "orphan_vm") ? a.host
: (a.instance_name || a.openstack_id || "unknown");
const bits = [a.region_label || a.region, a.status, a.org_name].filter(Boolean);
const sc = a.screen || {};
return `<div class="alert ${STATE.selected === a.id ? "sel" : ""} ${sc.actionable ? "" : "muted"}" data-id="${esc(a.id)}">
<div style="display:flex;gap:8px;align-items:baseline">
<div class="t" style="flex:1">${esc(subject)}</div>
<div class="age" title="${a.age_is_reset ? "condition has held this long; Prometheus says only " + esc(a.age_text) + " because a pipeline dip reset it" : "how long the condition has held"}">${
esc(a.effective_age_text || "")}${a.age_is_reset ? "*" : ""}</div>
</div>
<div class="m">${esc(bits.join(" · "))}</div>
<div class="why"><span class="tag v-${esc(sc.verdict || "unverified")}">${esc(sc.label || "")}</span>
<span style="color:var(--muted)"> ${esc(sc.reason || "")}</span></div>
</div>`;
}
function renderInfra() {
const d = STATE.data || {};
const q = $("#search").value.trim().toLowerCase();
const out = [`<div class="banner">These are not CX runbook alerts &mdash; they are host and platform
rules, shown here so they stay out of the triage queue.</div>`];
for (const s of d.infrastructure || []) {
const names = (s.by_alertname || []).filter(n => !q || n.name.toLowerCase().includes(q));
if (!names.length) continue;
const closed = STATE.closed["inf:" + s.source] === true;
out.push(`<div class="infsec group ${closed ? "closed" : ""}" data-kind="inf:${esc(s.source)}">
<div class="ghead">
<span class="caret">&#9660;</span>
<span class="gtitle">${esc(s.label)}</span>
<span class="gcount">${s.total} firing</span>
</div>
<div class="glist">${names.map(n =>
`<div class="infrow"><span class="n">${esc(n.name)}</span><span class="tag">${n.count}</span></div>`
).join("")}</div>
</div>`);
}
$("#list").innerHTML = out.join("");
document.querySelectorAll(".infsec .ghead").forEach(el => el.onclick = () => {
const k = el.parentElement.dataset.kind;
STATE.closed[k] = !STATE.closed[k];
render();
});
}
async function triage(body, id) {
STATE.selected = id || null;
STATE.lastBody = body;
render();
if (STATE.polling) { clearInterval(STATE.polling); STATE.polling = null; }
$("#detail").innerHTML = '<div class="empty"><span class="spinner"></span> Checking Infrahub, OpenStack and InfraInsight…<br><small>Host-wide checks can take up to a minute.</small></div>';
const started = await api("/api/triage", {
method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) });
if (started.error) { $("#detail").innerHTML = `<div class="empty tone-bad">${esc(started.error)}</div>`; return; }
STATE.polling = setInterval(async () => {
const job = await api("/api/jobs/" + started.job_id);
if (job.state === "running") return;
clearInterval(STATE.polling); STATE.polling = null;
if (job.state === "error") {
$("#detail").innerHTML = `<div class="card"><h2>Triage failed</h2><pre>${esc(job.error)}</pre></div>`;
return;
}
renderDiagnosis(job.result);
}, 900);
}
function diagnoseAnyway() {
triage({ ...(STATE.lastBody || {}), force: true }, STATE.selected);
}
function renderDiagnosis(d) {
const a = d.alert || {};
const sc = a.screen || {};
const out = [];
out.push(`<div class="card">
<h2>${esc(a.title)} &middot; ${esc(a.region_label || a.region || "region unknown")}</h2>
<div class="verdict">${esc(d.verdict || d.error || "No verdict")}</div>
${d.assessment ? `<div class="assessment">${esc(d.assessment)}</div>` : ""}
<div class="meta">
<span class="tag v-${esc(sc.verdict || "unverified")}">${esc(sc.label || "")}</span>
<span class="tag">priority ${esc(a.priority)}</span>
<span class="tag">confidence ${esc(d.confidence)}</span>
<span class="tag">ETTR ${esc(a.ettr)}</span>
<span class="tag">held ${esc(a.effective_age_text || "?")}</span>
${a.age_is_reset ? `<span class="tag" title="a metric-pipeline dip reset Prometheus' activeAt">prometheus says ${esc(a.age_text)}</span>` : ""}
${d.elapsed_seconds ? `<span class="tag">diagnosed in ${d.elapsed_seconds}s</span>` : ""}
${a.is_kubernetes ? '<span class="tag">likely K8s node</span>' : ""}
${!sc.actionable ? '<button onclick="diagnoseAnyway()">Diagnose anyway</button>' : ""}
</div>
</div>`);
if (d.error) out.push(`<div class="card"><h2>Lookup problem</h2><div class="tone-bad">${esc(d.error)}</div></div>`);
if ((d.notes || []).length)
out.push(`<div class="card"><h2>Caveats</h2><ul class="notes">${
d.notes.map(n => `<li>${esc(n)}</li>`).join("")}</ul></div>`);
if ((d.findings || []).length)
out.push(`<div class="card"><h2>What the platforms say</h2><table>${
d.findings.map(f => `<tr><td class="k">${esc(f.label)}</td><td class="v tone-${esc(f.tone)}">${
esc(f.value)}${f.detail ? `<span class="det">${esc(f.detail)}</span>` : ""}</td></tr>`).join("")
}</table></div>`);
if ((d.actions || []).length)
out.push(`<div class="card"><h2>Next steps</h2><ol class="actions">${
d.actions.map(x => `<li class="${x.status === "done" ? "done" : ""}">${esc(x.text)}
<span class="owner ${x.owner !== "CX" ? "infra" : ""}">${esc(x.owner)}</span>
<span class="kind">${esc(x.kind)}</span>
${x.guide ? `<span class="guide">Guide: ${esc(x.guide)}</span>` : ""}
${x.detail ? `<span class="guide">${esc(x.detail)}</span>` : ""}</li>`).join("")
}</ol></div>`);
const c = d.contacts || {};
if ((d.drafts || []).length)
out.push(`<div class="card">
<h2>Suggested customer comms</h2>
<div style="margin-bottom:12px">
${c.organization ? `<div class="contact">Org: ${esc(c.organization)}</div>` : ""}
${(c.owners || []).length ? c.owners.map(o => `<div class="contact">${esc(o)}</div>`).join("")
: '<div class="tone-warn">No owner contacts resolved — look the org up in the Admin Portal.</div>'}
</div>
${d.drafts.map((x, i) => `<div class="draft">
<div class="dh"><span class="dl">${esc(x.label)}</span>
<span class="tag">${esc(x.channel)}</span>
<button onclick="copyDraft(${i})" id="cp${i}">Copy</button></div>
${x.when ? `<div class="dw">When: ${esc(x.when)}</div>` : ""}
${x.unfilled.length ? `<div class="dw">Still to fill in: ${esc(x.unfilled.join(", "))}</div>` : ""}
<pre>${esc(x.body)}</pre></div>`).join("")}
<div style="color:var(--muted);font-size:12px">Send from HubSpot. This app never contacts anyone.</div>
</div>`);
out.push(`<div class="card"><h2>Evidence</h2>
<details class="raw"><summary>Alert labels, annotations and screening</summary><pre>${
esc(JSON.stringify({ rule: { file: a.rule_file, group: a.rule_group, for_seconds: a.for_seconds },
screen: a.screen, labels: a.labels, annotations: a.annotations }, null, 2))}</pre></details>
<details class="raw" style="margin-top:8px"><summary>Raw CX-Tools output</summary><pre>${
esc(JSON.stringify(d.evidence, null, 2))}</pre></details></div>`);
$("#detail").innerHTML = out.join("");
window.__drafts = d.drafts || [];
}
function copyDraft(i) {
navigator.clipboard.writeText((window.__drafts[i] || {}).body || "").then(() => {
const b = $("#cp" + i), old = b.textContent;
b.textContent = "Copied"; setTimeout(() => b.textContent = old, 1200);
});
}
$("#parseBtn").onclick = async () => {
const text = $("#paste").value.trim();
if (!text) return;
const res = await api("/api/parse", {
method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text }) });
if (res.error) { $("#detail").innerHTML = `<div class="empty tone-bad">${esc(res.error)}</div>`; return; }
const first = (res.parsed || [])[0];
if (!first) return;
if (!first.supported) {
$("#detail").innerHTML = `<div class="empty tone-warn">Parsed "${esc(first.alert.alertname)}" but no CX runbook covers it.</div>`;
return;
}
triage({ labels: first.alert.labels, annotations: first.alert.annotations }, null);
};
document.querySelectorAll(".tab").forEach(t => t.onclick = () => { STATE.tab = t.dataset.tab; render(); });
$("#refresh").onclick = () => { loadHealth(); loadAlerts(true); };
$("#search").oninput = render;
$("#showNoise").onchange = render;
loadHealth();
loadAlerts(false);
setInterval(() => loadAlerts(false), 60000);
</script>
</body>
</html>
"""

View File

@@ -1,181 +0,0 @@
"""Linkage scan page: Infrahub records and OpenStack servers that lost each other."""
from __future__ import annotations
PAGE = r"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>CX Triage — Linkage</title>
<style>
:root{--bg:#0d1017;--panel:#141821;--panel2:#1b202b;--line:#262d3a;--line2:#333b4a;
--fg:#e8ebf2;--dim:#8a93a5;--faint:#5d6675;--accent:#4c8dff;
--ok:#35c46a;--warn:#e0a336;--bad:#f2545b;--violet:#a97bf0;
--mono:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}
@media(prefers-color-scheme:light){:root{--bg:#f4f6f9;--panel:#fff;--panel2:#f1f4f8;--line:#e0e5ec;
--line2:#cfd6e0;--fg:#151a22;--dim:#5b6473;--faint:#8e97a5;--accent:#1f6feb;
--ok:#12864a;--warn:#96650a;--bad:#cf2530;--violet:#7a44d6}}
*{box-sizing:border-box}
body{margin:0;background:var(--bg);color:var(--fg);
font:14px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif}
header{display:flex;align-items:center;gap:14px;padding:9px 18px;background:var(--panel);
border-bottom:1px solid var(--line);position:sticky;top:0;z-index:10;flex-wrap:wrap}
.brand{font-weight:680;font-size:14px}
.brand em{font-style:normal;color:var(--faint);font-weight:400;font-size:12.5px}
a.navlink{font-size:12.5px;color:var(--accent);text-decoration:none;border:1px solid var(--line2);
padding:4px 10px;border-radius:7px}
.spacer{margin-left:auto}
button{font:inherit;font-size:13px;padding:7px 13px;border-radius:8px;border:1px solid var(--line2);
background:var(--panel2);color:var(--fg);cursor:pointer}
button.pri{background:var(--accent);border-color:var(--accent);color:#fff;font-weight:600}
button:disabled{opacity:.45;cursor:not-allowed}
button.sm{padding:3px 8px;font-size:11.5px}
main{padding:22px 26px 70px;max-width:1500px}
.lede{color:var(--dim);max-width:88ch;margin-bottom:18px}
.cards{display:flex;gap:12px;flex-wrap:wrap;margin-bottom:20px}
.stat{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:13px 17px;min-width:150px}
.stat .n{font-size:23px;font-weight:680;font-family:var(--mono)}
.stat .l{font-size:11px;text-transform:uppercase;letter-spacing:.5px;color:var(--faint);margin-top:2px}
.stat.hot .n{color:var(--bad)}
.stat.warn .n{color:var(--warn)}
h2{font-size:13px;text-transform:uppercase;letter-spacing:.6px;color:var(--dim);
margin:26px 0 10px;font-weight:600}
.tbl{background:var(--panel);border:1px solid var(--line);border-radius:10px;overflow:hidden}
.tr{display:grid;grid-template-columns:1.5fr 110px 1.4fr 110px 96px 90px;gap:10px;padding:9px 14px;
border-top:1px solid var(--line);align-items:center;font-size:12.5px}
.tr:first-child{border-top:none;background:var(--panel2);font-size:10.5px;text-transform:uppercase;
letter-spacing:.5px;color:var(--faint)}
.tr .nm{font-weight:600;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.tr .mono{font-family:var(--mono);font-size:11.5px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.tag{font-size:10px;padding:1px 7px;border-radius:99px;border:1px solid var(--line2);color:var(--dim)}
.tag.high{color:var(--bad);border-color:color-mix(in srgb,var(--bad) 50%,transparent)}
.tag.medium{color:var(--warn);border-color:color-mix(in srgb,var(--warn) 50%,transparent)}
.tag.none{color:var(--faint)}
.arrow{color:var(--ok);text-align:center}
.sub{color:var(--faint);font-size:11px}
.empty{color:var(--faint);padding:40px;text-align:center}
.spin{width:15px;height:15px;border:2px solid var(--line2);border-top-color:var(--accent);
border-radius:50%;display:inline-block;animation:sp .7s linear infinite;vertical-align:-3px}
@keyframes sp{to{transform:rotate(360deg)}}
.note{border:1px solid color-mix(in srgb,var(--warn) 45%,transparent);
background:color-mix(in srgb,var(--warn) 9%,transparent);border-radius:8px;padding:11px 14px;
font-size:12.5px;margin-bottom:16px}
.t-bad{color:var(--bad)} .t-ok{color:var(--ok)} .t-warn{color:var(--warn)}
</style>
</head>
<body>
<header>
<div class="brand">CX Triage <em>— linkage scan</em></div>
<a class="navlink" href="/">&larr; Alert queue</a>
<div class="spacer"></div>
<span class="sub" id="meta"></span>
<button class="pri" id="run">Run scan</button>
</header>
<main>
<p class="lede">A VM in ERROR is not always a failed build. If the server was created but the link back to
Infrahub was never written, Infrahub shows ERROR or CREATING with no usable <code>openstack_id</code> while a
perfectly good server of the same name is running. This scans both sides in bulk and pairs them up by name —
and finds the reverse too: OpenStack servers that no Infrahub record claims, which is what
<em>Suspected Orphan VM</em> was meant to catch before its input metric went empty.</p>
<div id="body"><div class="empty">Run a scan to begin. It lists every server in all four regions, so it takes a few minutes.</div></div>
</main>
<script>
const $=s=>document.querySelector(s);
const esc=s=>String(s==null?"":s).replace(/[&<>"']/g,c=>({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c]));
const api=(p,o)=>fetch(p,o).then(r=>r.json());
let POLL=null;
async function tick(){
const s=await api("/api/linkage");
$("#meta").textContent = s.state==="running" ? "scanning… "+(s.progress||"")
: s.finished ? "last scan "+(s.age_seconds>90?Math.round(s.age_seconds/60)+"m":s.age_seconds+"s")+" ago" : "";
$("#run").disabled = s.state==="running";
if(s.state==="running"){
$("#body").innerHTML=`<div class="empty"><span class="spin"></span><br><br>${esc(s.progress||"scanning")}…<br>
<span class="sub">Listing every server across four regions — a few minutes.</span></div>`;
return;
}
if(POLL){clearInterval(POLL);POLL=null;}
if(s.state==="error"){$("#body").innerHTML=`<div class="empty t-bad">${esc(s.error)}</div>`;return;}
if(s.state==="done") render(s.result);
}
function render(r){
const links=r.broken_links||[], orphans=r.orphans||[];
const high=links.filter(l=>l.confidence==="high");
const out=[];
out.push(`<div class="cards">
<div class="stat ${high.length?"hot":""}"><div class="n">${high.length}</div><div class="l">likely linkage failures</div></div>
<div class="stat warn"><div class="n">${links.length}</div><div class="l">Infrahub records with a broken link</div></div>
<div class="stat ${r.orphan_total?"warn":""}"><div class="n">${r.orphan_total}</div><div class="l">OpenStack servers no record claims</div></div>
<div class="stat"><div class="n">${r.openstack_servers}</div><div class="l">servers scanned</div></div>
<div class="stat"><div class="n">${r.infrahub_records}</div><div class="l">Infrahub records</div></div>
</div>`);
if(Object.keys(r.region_failures||{}).length)
out.push(`<div class="note">Some regions could not be listed: ${
Object.entries(r.region_failures).map(([k,v])=>`<b>${esc(k)}</b> (${esc(v)})`).join(", ")}.
Results below exclude those regions entirely (${r.skipped_unscanned_regions||0} record(s) skipped), so nothing
here is a false positive from a failed listing — but the scan is not complete.</div>`);
out.push(`<h2>Infrahub records whose OpenStack server is missing or unlinked</h2>`);
if(!links.length) out.push('<div class="tbl"><div class="empty">None — every record resolves.</div></div>');
else out.push(`<div class="tbl">
<div class="tr"><span>Infrahub instance</span><span>IH status</span><span>Matching OpenStack server</span>
<span>OS status</span><span>Confidence</span><span></span></div>
${links.slice(0,300).map((l,i)=>`<div class="tr">
<span class="nm">${esc(l.instance_name)}<span class="sub"><br>${esc(l.organization)} · ${esc(l.region)}</span></span>
<span class="mono t-bad">${esc(l.infrahub_status)}</span>
<span>${l.candidate
? `<span class="mono">${esc(l.candidate.id)}</span><span class="sub"><br>${esc(l.candidate.name)} · ${esc(l.candidate.region)}${l.candidate_claimed_by_other?' · <span class="t-warn">claimed by another record</span>':""}</span>`
: `<span class="sub">${esc(l.reason)}</span>`}</span>
<span class="mono">${esc(l.candidate?l.candidate.status:"")}</span>
<span><span class="tag ${esc(l.confidence)}">${esc(l.confidence)}</span></span>
<span>${l.candidate?`<button class="sm" onclick="enrich(${i},'${esc(l.candidate.region)}','${esc(l.candidate.id)}')">Details</button>`:""}</span>
</div><div class="tr" id="ex${i}" style="display:none;grid-template-columns:1fr"></div>`).join("")}
</div>`);
if(links.length>300) out.push(`<div class="sub" style="margin-top:8px">Showing the first 300 of ${links.length}.</div>`);
out.push(`<h2>OpenStack servers with no Infrahub record</h2>`);
if(!orphans.length) out.push('<div class="tbl"><div class="empty">None.</div></div>');
else out.push(`<div class="tbl">
<div class="tr" style="grid-template-columns:1.4fr 1.4fr 110px 1fr 120px"><span>Server</span><span>OpenStack ID</span>
<span>Status</span><span>Host</span><span>Name in Infrahub?</span></div>
${orphans.slice(0,200).map(o=>`<div class="tr" style="grid-template-columns:1.4fr 1.4fr 110px 1fr 120px">
<span class="nm">${esc(o.name||"(unnamed)")}<span class="sub"><br>${esc(o.region)}</span></span>
<span class="mono">${esc(o.id)}</span>
<span class="mono">${esc(o.status)}</span>
<span class="mono">${esc(o.host||"")}</span>
<span class="${o.name_known_to_infrahub?"t-warn":"t-bad"}">${o.name_known_to_infrahub?"name exists":"unknown"}</span>
</div>`).join("")}
</div>`);
if(r.orphan_total>orphans.length) out.push(`<div class="sub" style="margin-top:8px">Showing ${orphans.length} of ${r.orphan_total}.</div>`);
$("#body").innerHTML=out.join("");
}
async function enrich(i,region,osid){
const row=$("#ex"+i);
row.style.display="block"; row.innerHTML='<span class="spin"></span> loading…';
const d=await api("/api/linkage/enrich",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({region,openstack_id:osid})});
row.innerHTML = d.ok
? `<span class="mono">created ${esc(d.created)} · launched ${esc(d.launched||"")} · status ${esc(d.status)} · host ${esc(d.host||"")}</span>
<span class="sub"><br>fault: ${esc(d.fault)}</span>`
: `<span class="t-bad">${esc(d.error)}</span>`;
}
$("#run").onclick=async()=>{
await api("/api/linkage/scan",{method:"POST",headers:{"Content-Type":"application/json"},body:"{}"});
if(POLL) clearInterval(POLL);
POLL=setInterval(tick,2500); tick();
};
tick();
</script>
</body>
</html>
"""

View File

@@ -1,221 +0,0 @@
"""Settings page: suppression rules and comms identity."""
from __future__ import annotations
PAGE = r"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>CX Triage — Settings</title>
<style>
:root{--bg:#0d1017;--panel:#141821;--panel2:#1b202b;--line:#262d3a;--line2:#333b4a;
--fg:#e8ebf2;--dim:#8a93a5;--faint:#5d6675;--accent:#4c8dff;
--ok:#35c46a;--warn:#e0a336;--bad:#f2545b;--violet:#a97bf0;
--mono:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}
@media(prefers-color-scheme:light){:root{--bg:#f4f6f9;--panel:#fff;--panel2:#f1f4f8;--line:#e0e5ec;
--line2:#cfd6e0;--fg:#151a22;--dim:#5b6473;--faint:#8e97a5;--accent:#1f6feb;
--ok:#12864a;--warn:#96650a;--bad:#cf2530;--violet:#7a44d6}}
*{box-sizing:border-box}
body{margin:0;background:var(--bg);color:var(--fg);
font:14px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif}
header{display:flex;align-items:center;gap:14px;padding:9px 18px;background:var(--panel);
border-bottom:1px solid var(--line);position:sticky;top:0;z-index:10;flex-wrap:wrap}
.brand{font-weight:680;font-size:14px}
.brand em{font-style:normal;color:var(--faint);font-weight:400;font-size:12.5px}
a.navlink{font-size:12.5px;color:var(--accent);text-decoration:none;border:1px solid var(--line2);
padding:4px 10px;border-radius:7px}
.spacer{margin-left:auto}
button{font:inherit;font-size:13px;padding:7px 13px;border-radius:8px;border:1px solid var(--line2);
background:var(--panel2);color:var(--fg);cursor:pointer}
button:hover:not(:disabled){border-color:var(--accent)}
button.pri{background:var(--accent);border-color:var(--accent);color:#fff;font-weight:600}
button.sm{padding:3px 9px;font-size:11.5px}
button.del{color:var(--bad);border-color:color-mix(in srgb,var(--bad) 45%,transparent)}
main{padding:22px 26px 80px;max-width:1080px}
h2{font-size:13px;text-transform:uppercase;letter-spacing:.6px;color:var(--dim);margin:26px 0 10px;font-weight:600}
h2:first-child{margin-top:0}
.lede{color:var(--dim);max-width:82ch;margin-bottom:16px}
.card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:16px 18px;margin-bottom:12px}
.rule{background:var(--panel);border:1px solid var(--line);border-radius:10px;margin-bottom:10px;overflow:hidden}
.rule.off{opacity:.55}
.rh{display:flex;align-items:center;gap:10px;padding:12px 16px;flex-wrap:wrap}
.rh .nm{font-weight:640;font-size:14px}
.rh .why{color:var(--dim);font-size:12.5px;width:100%;margin-top:-4px}
.conds{display:flex;gap:6px;flex-wrap:wrap;padding:0 16px 12px}
.cond{font-size:11.5px;font-family:var(--mono);padding:3px 9px;border-radius:6px;
background:var(--panel2);border:1px solid var(--line2)}
.cond b{color:var(--accent);font-weight:600}
.andor{font-size:10.5px;color:var(--faint);align-self:center;text-transform:uppercase;letter-spacing:.5px}
.sw{position:relative;width:34px;height:19px;flex:none}
.sw input{opacity:0;width:0;height:0}
.sw span{position:absolute;inset:0;background:var(--line2);border-radius:99px;transition:.15s;cursor:pointer}
.sw span::before{content:"";position:absolute;width:15px;height:15px;left:2px;top:2px;background:#fff;
border-radius:50%;transition:.15s}
.sw input:checked + span{background:var(--ok)}
.sw input:checked + span::before{transform:translateX(15px)}
.grid{display:grid;grid-template-columns:170px 1fr;gap:10px 12px;align-items:center}
label.f{font-size:11px;text-transform:uppercase;letter-spacing:.5px;color:var(--faint)}
input,select,textarea{font:inherit;font-size:13px;padding:7px 10px;border-radius:7px;
border:1px solid var(--line2);background:var(--bg);color:var(--fg);width:100%}
.condrow{display:grid;grid-template-columns:190px 1fr 34px;gap:8px;margin-bottom:7px}
.hint{font-size:11.5px;color:var(--faint);margin-top:4px}
.pv{margin-top:12px;border:1px solid var(--line2);border-radius:8px;padding:11px 13px;background:var(--panel2);font-size:12.5px}
.pv ul{margin:6px 0 0;padding-left:18px;max-height:180px;overflow-y:auto}
.t-ok{color:var(--ok)}.t-bad{color:var(--bad)}.t-warn{color:var(--warn)}
.empty{color:var(--faint);padding:26px;text-align:center}
.saved{color:var(--ok);font-size:12.5px}
</style>
</head>
<body>
<header>
<div class="brand">CX Triage <em>— settings</em></div>
<a class="navlink" href="/">&larr; Alert queue</a>
<a class="navlink" href="/linkage">Linkage scan</a>
<div class="spacer"></div>
<span class="saved" id="saved"></span>
</header>
<main>
<h2>Suppression rules</h2>
<p class="lede">Hide alerts you already know about. A rule fires when <b>every</b> condition it sets matches,
so you can combine them &mdash; for example type <code>error</code> <em>and</em> organisation containing
<code>modal</code>. Suppressed alerts are not deleted: they stay reachable under the
<b>hidden by a rule</b> filter on the queue.</p>
<div id="rules"></div>
<button class="pri" id="addRule">Add a rule</button>
<div id="editor"></div>
<h2>Comms identity</h2>
<div class="card">
<div class="grid">
<label class="f">Sign-off name</label>
<div><input id="agent" placeholder="e.g. Mohammad Affan">
<div class="hint">Appended after &ldquo;Kind Regards&rdquo; in customer emails.</div></div>
<label class="f">Chronic after (days)</label>
<div><input id="chronic" type="number" min="1" max="90" style="max-width:110px">
<div class="hint">An alert whose condition has held longer than this is treated as chronic rather than new
work &mdash; except for types with a runbook time commitment, which become <b>overdue</b> instead.</div></div>
</div>
<div style="margin-top:14px"><button class="pri" id="saveGeneral">Save</button></div>
</div>
<h2>Where this is stored</h2>
<div class="card"><span class="hint" id="path"></span></div>
</main>
<script>
const $=s=>document.querySelector(s);
const esc=s=>String(s==null?"":s).replace(/[&<>"']/g,c=>({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c]));
const api=(p,o)=>fetch(p,o).then(r=>r.json());
let CFG=null, EDIT=null;
function flash(msg){$("#saved").textContent=msg;setTimeout(()=>$("#saved").textContent="",2200);}
async function load(){ CFG=await api("/api/settings"); render(); }
function render(){
$("#agent").value=CFG.agent_name||"";
$("#chronic").value=CFG.chronic_days||3;
$("#path").textContent=CFG.path;
const rs=CFG.rules||[];
$("#rules").innerHTML = rs.length ? rs.map(r=>`
<div class="rule ${r.enabled?"":"off"}">
<div class="rh">
<label class="sw"><input type="checkbox" ${r.enabled?"checked":""} onchange="toggle('${esc(r.id)}',this.checked)"><span></span></label>
<span class="nm">${esc(r.name)}</span>
<span class="spacer" style="margin-left:auto"></span>
<button class="sm" onclick='edit(${JSON.stringify(JSON.stringify(r))})'>Edit</button>
<button class="sm del" onclick="del('${esc(r.id)}','${esc(r.name)}')">Delete</button>
${r.reason?`<span class="why">${esc(r.reason)}</span>`:""}
</div>
<div class="conds">${Object.entries(r.conditions||{}).map(([k,v],i)=>
`${i?'<span class="andor">and</span>':""}<span class="cond"><b>${esc(CFG.conditions[k]||k)}</b> ${esc(v.join(" or "))}</span>`
).join("")||'<span class="cond t-bad">no conditions — inactive</span>'}</div>
</div>`).join("") : '<div class="card"><div class="empty">No rules yet.</div></div>';
}
function blank(){return {id:"",name:"",enabled:true,reason:"",conditions:{}};}
function edit(json){ EDIT=typeof json==="string"?JSON.parse(json):json; drawEditor(); }
$("#addRule").onclick=()=>{EDIT=blank();drawEditor();};
function drawEditor(){
if(!EDIT){$("#editor").innerHTML="";return;}
const conds=Object.entries(EDIT.conditions||{});
if(!conds.length) conds.push(["kind",[]]);
$("#editor").innerHTML=`<div class="card" style="border-color:var(--accent)">
<div class="grid">
<label class="f">Rule name</label><input id="rName" value="${esc(EDIT.name)}" placeholder="e.g. Modal ERROR noise">
<label class="f">Why (shown on the alert)</label><input id="rWhy" value="${esc(EDIT.reason)}" placeholder="e.g. Known batch churn, customer already aware">
</div>
<div style="margin-top:14px">
<label class="f">Conditions &mdash; all must match</label>
<div id="condList" style="margin-top:7px">${conds.map((c,i)=>condRow(c[0],c[1],i)).join("")}</div>
<button class="sm" onclick="addCond()">+ Add condition</button>
</div>
<div id="pv"></div>
<div style="margin-top:14px;display:flex;gap:9px;flex-wrap:wrap">
<button class="pri" onclick="saveRule()">Save rule</button>
<button onclick="previewRule()">Preview what this hides</button>
<button onclick="EDIT=null;drawEditor()">Cancel</button>
</div></div>`;
}
function condRow(field,values,i){
return `<div class="condrow" data-i="${i}">
<select class="cf">${Object.entries(CFG.conditions).map(([k,v])=>
`<option value="${esc(k)}" ${k===field?"selected":""}>${esc(v)}</option>`).join("")}</select>
<input class="cv" value="${esc((values||[]).join(", "))}" placeholder="comma-separated; any one matches">
<button class="sm del" onclick="this.parentElement.remove()">&times;</button></div>`;
}
function addCond(){ $("#condList").insertAdjacentHTML("beforeend", condRow("organization",[],Date.now())); }
function collect(){
const conditions={};
document.querySelectorAll("#condList .condrow").forEach(row=>{
const f=row.querySelector(".cf").value;
const v=row.querySelector(".cv").value.split(",").map(x=>x.trim()).filter(Boolean);
if(v.length) conditions[f]=(conditions[f]||[]).concat(v);
});
return {id:EDIT.id,name:$("#rName").value||"Untitled rule",reason:$("#rWhy").value,
enabled:EDIT.enabled!==false,conditions};
}
async function previewRule(){
const r=await api("/api/settings",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({action:"preview",rule:collect()})});
$("#pv").innerHTML=`<div class="pv">
<b class="${r.count?"t-warn":"t-ok"}">${r.count}</b> currently-firing alert(s) would be hidden.
${r.count?`<ul>${r.matches.slice(0,40).map(m=>
`<li>${esc(m.title)} &mdash; ${esc(m.instance_name||m.host||"?")} <span style="color:var(--faint)">${esc(m.org_name||"")}</span></li>`
).join("")}</ul>`:""}</div>`;
}
async function saveRule(){
const r=await api("/api/settings",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({action:"save_rule",rule:collect()})});
if(r.ok){CFG=r.settings;EDIT=null;drawEditor();render();flash("Rule saved");}
}
async function toggle(id,on){
const r=await api("/api/settings",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({action:"toggle_rule",id,enabled:on})});
if(r.ok){CFG=r.settings;render();flash(on?"Rule enabled":"Rule disabled");}
}
async function del(id,name){
if(!confirm(`Delete the rule “${name}”? Alerts it was hiding will come back.`)) return;
const r=await api("/api/settings",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({action:"delete_rule",id})});
if(r.ok){CFG=r.settings;render();flash("Rule deleted");}
}
$("#saveGeneral").onclick=async()=>{
const r=await api("/api/settings",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({action:"general",agent_name:$("#agent").value,chronic_days:$("#chronic").value})});
if(r.ok){CFG=r.settings;flash("Saved");}
};
load();
</script>
</body>
</html>
"""

View File

@@ -1,572 +0,0 @@
"""Action-oriented UI.
Design intent: the reader already knows the runbooks. Each case shows the state
of the world as a picture, one line of verdict, and the buttons that actually
move it forward. Everything else is collapsed.
"""
from __future__ import annotations
PAGE = r"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>CX Triage</title>
<style>
:root {
--bg:#0d1017; --panel:#141821; --panel2:#1b202b; --line:#262d3a; --line2:#333b4a;
--fg:#e8ebf2; --dim:#8a93a5; --faint:#5d6675; --accent:#4c8dff;
--ok:#35c46a; --warn:#e0a336; --bad:#f2545b; --info:#59a0f5; --violet:#a97bf0;
--mono:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;
--r:10px;
}
@media (prefers-color-scheme:light){:root{
--bg:#f4f6f9;--panel:#fff;--panel2:#f1f4f8;--line:#e0e5ec;--line2:#cfd6e0;
--fg:#151a22;--dim:#5b6473;--faint:#8e97a5;--accent:#1f6feb;
--ok:#12864a;--warn:#96650a;--bad:#cf2530;--info:#0969da;--violet:#7a44d6;}}
*{box-sizing:border-box}
body{margin:0;background:var(--bg);color:var(--fg);
font:14px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif;}
button{font:inherit;font-size:13px;padding:7px 13px;border-radius:8px;border:1px solid var(--line2);
background:var(--panel2);color:var(--fg);cursor:pointer;transition:.12s}
button:hover:not(:disabled){border-color:var(--accent)}
button:disabled{opacity:.45;cursor:not-allowed}
button.pri{background:var(--accent);border-color:var(--accent);color:#fff;font-weight:600}
button.pri:hover:not(:disabled){filter:brightness(1.1)}
button.danger{border-color:color-mix(in srgb,var(--bad) 50%,transparent);color:var(--bad)}
button.sm{padding:4px 9px;font-size:12px}
header{display:flex;align-items:center;gap:14px;padding:9px 18px;background:var(--panel);
border-bottom:1px solid var(--line);position:sticky;top:0;z-index:20;flex-wrap:wrap}
.brand{font-weight:680;font-size:14px;letter-spacing:.2px}
.brand em{font-style:normal;color:var(--faint);font-weight:400;font-size:12.5px}
.navlink{font-size:12.5px;color:var(--accent);text-decoration:none;border:1px solid var(--line2);padding:4px 10px;border-radius:7px}
.navlink:hover{border-color:var(--accent)}
.chips{display:flex;gap:6px;flex-wrap:wrap;margin-left:6px}
.chip{font-size:11.5px;padding:3px 10px;border-radius:99px;border:1px solid var(--line2);
background:var(--panel2);color:var(--dim);cursor:pointer;white-space:nowrap;user-select:none}
.chip.on{background:var(--accent);border-color:var(--accent);color:#fff}
.chip b{font-weight:700}
.chip.overdue:not(.on){color:var(--bad);border-color:color-mix(in srgb,var(--bad) 45%,transparent)}
.spacer{margin-left:auto}
.dotstat{width:7px;height:7px;border-radius:50%;display:inline-block;margin-right:5px}
.filters{background:var(--panel);border-bottom:1px solid var(--line);padding:7px 18px;
display:flex;flex-direction:column;gap:5px;position:sticky;top:51px;z-index:19}
.frow{display:flex;align-items:center;gap:9px}
.flab{font-size:10.5px;text-transform:uppercase;letter-spacing:.5px;color:var(--faint);width:44px;flex:none}
main{display:grid;grid-template-columns:320px 1fr;height:calc(100vh - 118px)}
@media(max-width:940px){main{grid-template-columns:1fr}}
#rail{border-right:1px solid var(--line);background:var(--panel);overflow-y:auto}
.case .bar.chronic,.case .bar.low_impact{background:var(--faint)}
.case .bar.rule_defect{background:var(--violet)}
.case .bar.pending{background:var(--info);opacity:.5}
.case .bar.resolved{background:var(--ok)}
.statepill{font-size:9.5px;text-transform:uppercase;letter-spacing:.4px;padding:0 5px;
border-radius:3px;border:1px solid var(--line2);color:var(--faint);margin-left:6px}
.statepill.pending{color:var(--info);border-color:color-mix(in srgb,var(--info) 45%,transparent)}
#stage{overflow-y:auto;padding:22px 26px 60px}
.grp{border-bottom:1px solid var(--line)}
.grp-h{display:flex;align-items:center;gap:8px;padding:8px 14px;cursor:pointer;
background:var(--panel2);user-select:none;font-size:12px;letter-spacing:.3px;text-transform:uppercase;color:var(--dim)}
.grp-h:hover{color:var(--fg)}
.car{font-size:9px;width:9px;transition:.12s}
.grp.shut .car{transform:rotate(-90deg)}
.grp.shut .grp-b{display:none}
.grp-h .n{margin-left:auto;font-size:11px;padding:1px 7px;border-radius:99px;background:var(--bg);color:var(--dim);text-transform:none}
.grp-h .n.hot{background:var(--bad);color:#fff}
.case{padding:9px 14px;border-top:1px solid var(--line);cursor:pointer;display:flex;gap:9px;align-items:flex-start}
.case:hover{background:var(--panel2)}
.case.on{background:color-mix(in srgb,var(--accent) 15%,transparent);box-shadow:inset 3px 0 var(--accent)}
.case.off{opacity:.5}
.case .bar{width:3px;align-self:stretch;border-radius:2px;background:var(--faint);flex:none}
.case .bar.overdue{background:var(--bad)} .case .bar.real{background:var(--warn)}
.case .bar.unverified{background:var(--info)}
.case .mid{min-width:0;flex:1}
.case .nm{font-weight:600;font-size:13px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.case .sub{color:var(--dim);font-size:11.5px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.case .ag{font-family:var(--mono);font-size:11px;color:var(--faint);flex:none}
.hero{background:var(--panel);border:1px solid var(--line);border-radius:var(--r);padding:20px 22px;margin-bottom:14px}
.crumb{font-size:11.5px;color:var(--faint);letter-spacing:.3px;text-transform:uppercase;margin-bottom:9px;
display:flex;gap:8px;align-items:center;flex-wrap:wrap}
.vd{font-size:20px;font-weight:660;line-height:1.3;letter-spacing:-.2px}
.vsub{color:var(--dim);margin-top:7px;font-size:13.5px;max-width:80ch}
.badge{font-size:11px;padding:2px 9px;border-radius:99px;border:1px solid var(--line2);color:var(--dim);
text-transform:none;letter-spacing:0}
.badge.overdue{background:var(--bad);border-color:var(--bad);color:#fff;font-weight:600}
.badge.real{color:var(--warn);border-color:color-mix(in srgb,var(--warn) 50%,transparent)}
.badge.rule_defect{color:var(--violet);border-color:color-mix(in srgb,var(--violet) 50%,transparent)}
.badge.resolved{color:var(--ok);border-color:color-mix(in srgb,var(--ok) 45%,transparent)}
/* ---- visuals ---- */
.viz{margin:18px 0 4px}
.states{display:flex;align-items:center;gap:14px;flex-wrap:wrap}
.sbox{flex:1;min-width:150px;background:var(--panel2);border:1px solid var(--line);border-radius:9px;padding:11px 14px}
.sbox .lbl{font-size:10.5px;text-transform:uppercase;letter-spacing:.5px;color:var(--faint)}
.sbox .val{font-family:var(--mono);font-size:16px;font-weight:600;margin-top:3px}
.sbox.bad{border-color:color-mix(in srgb,var(--bad) 45%,transparent)}
.sbox.bad .val{color:var(--bad)}
.sbox.ok .val{color:var(--ok)}
.link{font-size:20px;color:var(--faint);flex:none}
.link.bad{color:var(--bad)}
.slots{display:flex;gap:4px;margin-top:8px;flex-wrap:wrap}
.slot{flex:1 1 78px;min-width:70px;height:52px;border-radius:7px;border:1px solid var(--line2);
display:flex;flex-direction:column;justify-content:center;padding:5px 7px;overflow:hidden}
.slot .sn{font-size:10.5px;font-weight:600;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.slot .ss{font-size:9px;text-transform:uppercase;letter-spacing:.4px;opacity:.75;margin-top:1px}
.slot.vm{background:color-mix(in srgb,var(--ok) 22%,transparent);
border-color:color-mix(in srgb,var(--ok) 55%,transparent)}
.slot.vm.bad{background:color-mix(in srgb,var(--bad) 20%,transparent);
border-color:color-mix(in srgb,var(--bad) 55%,transparent)}
.slot.vm.unlinked{background:color-mix(in srgb,var(--bad) 28%,transparent);
border-color:var(--bad)}
.slot.unaccounted{background:color-mix(in srgb,var(--bad) 16%,transparent);
border-color:color-mix(in srgb,var(--bad) 50%,transparent);border-style:dashed;
align-items:center;justify-content:center;color:var(--bad)}
.slot.free{border-style:dashed;border-color:var(--line2);color:var(--faint);
align-items:center;justify-content:center;background:transparent}
.slotnum{font-size:9px;color:var(--faint);margin-bottom:1px}
.gpubar{display:flex;height:34px;border-radius:8px;overflow:hidden;border:1px solid var(--line2);margin-top:6px}
.gseg{display:flex;align-items:center;justify-content:center;font-size:11.5px;font-weight:600;
font-family:var(--mono);color:#fff;min-width:0;overflow:hidden;white-space:nowrap;padding:0 4px}
.gseg.alloc{background:color-mix(in srgb,var(--ok) 78%,#000)}
.gseg.gap{background:color-mix(in srgb,var(--warn) 72%,#000)}
.gseg.gapbad{background:color-mix(in srgb,var(--bad) 72%,#000)}
.glegend{display:flex;gap:16px;margin-top:8px;font-size:12px;color:var(--dim);flex-wrap:wrap}
.glegend i{width:9px;height:9px;border-radius:2px;display:inline-block;margin-right:5px}
.roster{margin-top:8px;border:1px solid var(--line);border-radius:9px;overflow:hidden}
.rrow{display:grid;grid-template-columns:1fr 118px 26px 118px 52px;gap:8px;align-items:center;
padding:7px 12px;border-top:1px solid var(--line);font-size:12.5px}
.rrow:first-child{border-top:none;background:var(--panel2);font-size:10.5px;text-transform:uppercase;
letter-spacing:.5px;color:var(--faint)}
.rrow .rn{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-weight:600}
.rrow .rs{font-family:var(--mono);font-size:12px}
.rrow .eq{text-align:center;font-size:14px;color:var(--ok)}
.rrow.bad .eq{color:var(--bad)} .rrow.bad .rs{color:var(--bad)}
.rrow.unlinked{background:color-mix(in srgb,var(--bad) 9%,transparent)}
.rrow.unlinked .rs.ih{color:var(--bad);font-style:italic}
.rrow .rg{text-align:right;color:var(--faint);font-family:var(--mono);font-size:11.5px}
.claims{display:flex;flex-direction:column;gap:7px;margin-top:6px}
.claim{display:flex;gap:10px;align-items:center;background:var(--panel2);border:1px solid var(--line);
border-radius:8px;padding:8px 12px;font-size:13px}
.claim .cn{font-weight:600;flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
/* ---- actions ---- */
.acts{background:var(--panel);border:1px solid var(--line);border-radius:var(--r);padding:16px 18px;margin-bottom:14px}
.acts h3{margin:0 0 4px;font-size:12px;text-transform:uppercase;letter-spacing:.6px;color:var(--dim)}
.actrow{display:flex;gap:10px;flex-wrap:wrap;margin-top:12px}
.who{display:flex;align-items:center;gap:8px;background:var(--panel2);border:1px solid var(--line);
border-radius:8px;padding:8px 12px;margin-top:11px;font-size:13px;flex-wrap:wrap}
.who .em{font-family:var(--mono);font-size:12.5px}
.hintline{font-size:12px;color:var(--faint);margin-top:9px}
.cmd{display:flex;gap:8px;align-items:center;background:var(--bg);border:1px solid var(--line);
border-radius:7px;padding:7px 10px;margin-top:8px;font-family:var(--mono);font-size:12.5px}
.cmd code{flex:1;min-width:0;overflow-x:auto;white-space:nowrap}
details.fold{background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin-bottom:10px}
details.fold>summary{cursor:pointer;padding:11px 18px;font-size:12px;text-transform:uppercase;
letter-spacing:.5px;color:var(--dim);user-select:none;list-style:none;display:flex;align-items:center;gap:8px}
details.fold>summary::-webkit-details-marker{display:none}
details.fold>summary::before{content:"";font-size:10px;color:var(--faint)}
details.fold[open]>summary::before{content:""}
details.fold>summary:hover{color:var(--fg)}
.foldb{padding:2px 18px 16px}
table{width:100%;border-collapse:collapse}
td{padding:5px 6px;border-bottom:1px solid var(--line);vertical-align:top;font-size:12.5px}
tr:last-child td{border-bottom:none}
td.k{color:var(--dim);width:180px;white-space:nowrap}
td.v{font-family:var(--mono);word-break:break-word}
.t-ok{color:var(--ok)}.t-warn{color:var(--warn)}.t-bad{color:var(--bad)}
.det{display:block;color:var(--faint);font-family:inherit;font-size:11.5px;margin-top:2px}
ol.steps{margin:0;padding-left:18px;font-size:13px}
ol.steps li{margin-bottom:7px}
ol.steps .ow{font-size:10.5px;padding:1px 6px;border-radius:4px;background:var(--panel2);color:var(--dim);margin-left:5px}
ol.steps .ow.i{color:var(--warn)}
pre{background:var(--bg);border:1px solid var(--line);padding:10px;border-radius:7px;overflow-x:auto;
font-size:11.5px;margin:0}
/* ---- drawer ---- */
#scrim{position:fixed;inset:0;background:rgba(0,0,0,.55);opacity:0;pointer-events:none;transition:.16s;z-index:40}
#scrim.on{opacity:1;pointer-events:auto}
#drawer{position:fixed;top:0;right:0;height:100%;width:min(620px,94vw);background:var(--panel);
border-left:1px solid var(--line);z-index:50;transform:translateX(100%);transition:.18s;
display:flex;flex-direction:column}
#drawer.on{transform:none}
.dh{padding:15px 20px;border-bottom:1px solid var(--line);display:flex;align-items:center;gap:10px}
.dh h2{margin:0;font-size:15px;font-weight:650;flex:1}
.db{padding:18px 20px;overflow-y:auto;flex:1}
.df{padding:14px 20px;border-top:1px solid var(--line);display:flex;gap:10px;align-items:center;flex-wrap:wrap}
.fld{margin-bottom:14px}
.fld label{display:block;font-size:11px;text-transform:uppercase;letter-spacing:.5px;color:var(--faint);margin-bottom:5px}
.fld input,.fld textarea,.fld select{width:100%;font:inherit;font-size:13px;padding:8px 10px;
border-radius:7px;border:1px solid var(--line2);background:var(--bg);color:var(--fg)}
.fld textarea{min-height:230px;resize:vertical;line-height:1.55}
.warnbox{border:1px solid color-mix(in srgb,var(--warn) 50%,transparent);
background:color-mix(in srgb,var(--warn) 11%,transparent);border-radius:8px;padding:10px 12px;
font-size:12.5px;margin-bottom:14px}
.empty{color:var(--faint);text-align:center;padding:70px 20px}
.spin{width:15px;height:15px;border:2px solid var(--line2);border-top-color:var(--accent);
border-radius:50%;display:inline-block;animation:sp .7s linear infinite;vertical-align:-3px}
@keyframes sp{to{transform:rotate(360deg)}}
.banner{margin:0 0 12px;padding:10px 13px;border-radius:8px;font-size:12.5px;
border:1px solid color-mix(in srgb,var(--warn) 45%,transparent);
background:color-mix(in srgb,var(--warn) 9%,transparent)}
</style>
</head>
<body>
<header>
<div class="brand">CX Triage <em id="qsum"></em></div>
<a class="navlink" href="/linkage">Linkage scan</a>
<a class="navlink" href="/settings">Settings</a>
<div class="spacer"></div>
<span class="badge" id="conn"></span>
<button id="refresh" class="sm">Refresh</button>
</header>
<div class="filters">
<div class="frow"><span class="flab">Status</span><div class="chips" id="chips"></div></div>
<div class="frow"><span class="flab">Type</span><div class="chips" id="kinds"></div></div>
</div>
<main>
<aside id="rail"><div class="empty"><span class="spin"></span></div></aside>
<section id="stage"><div class="empty">Select a case.</div></section>
</main>
<div id="scrim"></div>
<aside id="drawer">
<div class="dh"><h2 id="dTitle">Action</h2><button class="sm" onclick="closeDrawer()">Close</button></div>
<div class="db" id="dBody"></div>
<div class="df" id="dFoot"></div>
</aside>
<script>
const $=s=>document.querySelector(s);
const esc=s=>String(s==null?"":s).replace(/[&<>"']/g,c=>({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c]));
const api=(p,o)=>fetch(p,o).then(r=>r.json());
let S={data:null,sel:null,poll:null,gen:0,shut:{},filter:"actionable",kind:"all",diag:null};
// Prometheus states we never surface: an alert that has not committed yet
// is not work, and showing it invites acting on something that may clear.
const HIDDEN_STATES=["pending"];
const VERDICT_HELP={
overdue:"The condition still holds and has passed the point where the runbook says to contact the customer.",
real:"The condition still holds right now - re-checked against live Infrahub/OpenStack state.",
unverified:"Could not be re-checked, so it is kept in the queue rather than hidden on a guess.",
chronic:"Still true, but it has been true for days - already known rather than new work.",
low_impact:"Still true, but the owner is internal or the VM is platform-owned.",
pending:"Prometheus has not committed to this alert yet; it may still clear on its own.",
resolved:"The condition no longer holds - Infrahub/OpenStack have moved on since it fired.",
rule_defect:"The alert rule itself is wrong, so the alert is not evidence of a problem. Opening the case explains which part of the expression misfires.",
suppressed:"Hidden by a rule you configured in Settings.",
};
const VERDICT_ORDER=["overdue","real","unverified","chronic","low_impact","pending","resolved","rule_defect"];
async function load(force){
const d=await api("/api/alerts"+(force?"?force=1":""));
S.data=d; drawChips(); drawRail();
const ic=d.integrations||{};
$("#conn").innerHTML=`Zendesk ${ic.zendesk?"&#9679; connected":"&#9675; not connected"} &nbsp;|&nbsp; Jira ${ic.jira?"&#9679; connected":"&#9675; not connected"}`;
}
function allCases(){return (S.data?.groups||[]).flatMap(g=>g.alerts);}
function drawChips(){
const all=allCases();
const counts={};
all.forEach(a=>{const v=a.screen?.verdict||"unverified";counts[v]=(counts[v]||0)+1;});
const act=all.filter(a=>a.screen?.actionable&&!HIDDEN_STATES.includes(a.state)).length;
const chips=[`<span class="chip ${S.filter==="actionable"?"on":""}" data-f="actionable">To action <b>${act}</b></span>`];
VERDICT_ORDER.forEach(v=>{ if(!counts[v]) return;
chips.push(`<span class="chip ${v} ${S.filter===v?"on":""}" data-f="${v}">${esc(S.data.summary.labels[v]||v)} <b>${counts[v]}</b></span>`);});
chips.push(`<span class="chip ${S.filter==="all"?"on":""}" data-f="all" title="Everything except alerts Prometheus has not committed to yet">All firing</span>`);
$("#chips").innerHTML=chips.join("");
document.querySelectorAll("#chips .chip").forEach(c=>c.onclick=()=>{S.filter=c.dataset.f;drawChips();drawRail();});
const byKind={};
all.forEach(a=>{byKind[a.kind]=(byKind[a.kind]||0)+1;});
const kinds=[`<span class="chip ${S.kind==="all"?"on":""}" data-k="all">All types</span>`];
(S.data.groups||[]).forEach(g=>{
kinds.push(`<span class="chip ${S.kind===g.kind?"on":""}" data-k="${esc(g.kind)}">${esc(g.title.replace(/^Instance in /,"").replace(/ state$/,""))} <b>${byKind[g.kind]||0}</b></span>`);});
$("#kinds").innerHTML=kinds.join("");
document.querySelectorAll("#kinds .chip").forEach(c=>c.onclick=()=>{S.kind=c.dataset.k;drawChips();drawRail();});
const t=S.data.totals||{};
$("#qsum").textContent=`${act} to action \u00b7 ${t.cx||0} CX alerts \u00b7 ${t.prometheus||0} firing in Prometheus`;
}
function visible(a){
if(S.kind!=="all" && a.kind!==S.kind) return false;
const st=a.state||"firing";
// Pending alerts are only reachable by asking for them by name.
if(HIDDEN_STATES.includes(st) && S.filter!==st) return false;
if(S.filter==="all") return true;
if(S.filter==="actionable") return !!a.screen?.actionable;
return a.screen?.verdict===S.filter;
}
function drawRail(){
const out=[];
for(const g of S.data?.groups||[]){
const rows=(g.alerts||[]).filter(visible);
if(!rows.length) continue;
const hot=rows.filter(a=>a.screen?.actionable).length;
const shut=S.shut[g.kind]===true;
out.push(`<div class="grp ${shut?"shut":""}" data-k="${esc(g.kind)}">
<div class="grp-h"><span class="car">&#9660;</span>${esc(g.title)}
<span class="n ${hot?"hot":""}" title="${rows.length} shown of ${g.total} firing">${
rows.length}${rows.length<g.total?` <span style="opacity:.6">of ${g.total}</span>`:""}</span></div>
<div class="grp-b">${rows.map(row).join("")}</div></div>`);
}
$("#rail").innerHTML=out.length?out.join(""):'<div class="empty">Nothing here.</div>';
document.querySelectorAll(".grp-h").forEach(h=>h.onclick=()=>{
const k=h.parentElement.dataset.k;S.shut[k]=!S.shut[k];drawRail();});
document.querySelectorAll(".case").forEach(c=>c.onclick=()=>open(c.dataset.id));
}
function subjectOf(a){
if(a.kind==="duplicate_ip") return a.floating_ip;
if(["rogue_vm","total_gpus","orphan_vm"].includes(a.kind)) return a.host;
return a.instance_name||a.openstack_id||"unknown";
}
function row(a){
const v=a.screen?.verdict||"unverified";
return `<div class="case ${S.sel===a.id?"on":""} ${a.screen?.actionable?"":"off"}" data-id="${esc(a.id)}">
<span class="bar ${esc(v)}" title="${esc(S.data.summary.labels[v]||v)}"></span>
<span class="mid"><span class="nm">${esc(subjectOf(a))}${a.state==="pending"?'<span class="statepill pending">pending</span>':""}</span>
<span class="sub">${esc([a.org_name,a.region_label||a.region].filter(Boolean).join(" · "))}</span></span>
<span class="ag">${esc(a.effective_age_text||"")}</span></div>`;
}
async function open(id, force){
// Every request gets a generation number. Clicking a second case while the
// first is still polling used to leave the first timer running, and its
// result would later overwrite the stage - collapsing whatever the reader
// had expanded, and sometimes showing the wrong case. Stale generations now
// stop themselves.
const gen = ++S.gen;
S.sel=id; drawRail();
if(S.poll){clearInterval(S.poll);S.poll=null;}
$("#stage").innerHTML='<div class="empty"><span class="spin"></span><br><br>Checking Infrahub, OpenStack and InfraInsight…</div>';
const st=await api("/api/triage",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({alert_id:id,force:!!force})});
if(gen!==S.gen) return;
if(st.error){$("#stage").innerHTML=`<div class="empty t-bad">${esc(st.error)}</div>`;return;}
const timer=setInterval(async()=>{
if(gen!==S.gen){clearInterval(timer);return;}
const j=await api("/api/jobs/"+st.job_id);
if(gen!==S.gen){clearInterval(timer);return;}
if(j.state==="running") return;
clearInterval(timer); if(S.poll===timer) S.poll=null;
if(j.state==="error"){$("#stage").innerHTML=`<pre>${esc(j.error)}</pre>`;return;}
S.diag=j.result; drawCase(j.result);
},900);
S.poll=timer;
}
function rosterHTML(v){
const rows=v.roster||[];
if(!rows.length) return "";
const unlinked=rows.filter(r=>!r.linked).length, mismatched=rows.filter(r=>r.linked&&!r.match).length;
return `<div class="roster">
<div class="rrow"><span>VM on this host</span><span>Infrahub</span><span></span><span>OpenStack</span><span class="rg">GPU</span></div>
${rows.map(r=>`<div class="rrow ${r.linked?(r.match?"":"bad"):"unlinked"}">
<span class="rn">${esc(r.name)}${r.tempest?' <span class="badge">tempest</span>':""}</span>
<span class="rs ih">${esc(r.ih_status)}</span>
<span class="eq">${r.linked?(r.match?"=":"\u2260"):"\u2717"}</span>
<span class="rs">${esc(r.os_status)}</span>
<span class="rg">${esc(r.gpus)}</span></div>`).join("")}
</div>
<div class="glegend"><span>${rows.length} VM(s) on host</span>
<span class="${unlinked?"t-bad":""}">${unlinked} with no Infrahub record</span>
<span class="${mismatched?"t-bad":""}">${mismatched} state mismatch(es)</span></div>`;
}
function vizHTML(d){
const v=d.visual||{};
if(v.type==="states"){
const bad=!v.match;
return `<div class="viz"><div class="states">
<div class="sbox ${bad?"bad":"ok"}"><div class="lbl">Infrahub says</div><div class="val">${esc(v.infrahub)}</div></div>
<div class="link ${bad?"bad":""}">${bad?"&#8800;":"&#61;"}</div>
<div class="sbox ${bad?"bad":"ok"}"><div class="lbl">OpenStack says</div><div class="val">${esc(v.openstack)}</div></div>
</div>
<div class="glegend">
${v.task&&v.task!=="None"?`<span>task state <b>${esc(v.task)}</b></span>`:""}
<span>host <b>${esc(v.never_built?"never placed":v.host)}</b></span>
${v.flavor?`<span>flavor <b>${esc(v.flavor)}</b></span>`:""}
${v.fault&&v.fault!=="None"?`<span class="t-bad">fault present</span>`:""}
</div></div>`;
}
if(v.type==="gpu"){
const slots=v.slots||[];
const named=slots.filter(x=>x.kind==="vm").length;
const un=slots.filter(x=>x.kind==="unaccounted").length;
const free=slots.filter(x=>x.kind==="free").length;
return `<div class="viz">
<div class="slots">${slots.map((x,i)=>{
if(x.kind==="vm") return `<div class="slot vm ${x.linked?(x.match?"":"bad"):"unlinked"}" title="${esc(x.name)} — Infrahub ${esc(x.ih_status)} / OpenStack ${esc(x.os_status)}">
<span class="slotnum">GPU ${i+1}</span>
<span class="sn">${esc(x.name)}</span>
<span class="ss">${x.linked?(x.match?"in sync":"state mismatch"):"not in Infrahub"}</span></div>`;
if(x.kind==="unaccounted") return `<div class="slot unaccounted" title="The host reports this GPU in use, but no instance claims it">
<span class="sn">unaccounted</span></div>`;
return `<div class="slot free" title="Physically present, nothing using it"><span class="sn">free</span></div>`;
}).join("")}</div>
<div class="glegend">
<span>${v.physical!=null?v.physical+" GPU sockets on this host":"GPU count unknown"}</span>
<span><i style="background:color-mix(in srgb,var(--ok) 60%,transparent)"></i>${named} held by ${v.instances} VM(s)</span>
${un?`<span class="t-bad"><i style="background:color-mix(in srgb,var(--bad) 55%,transparent)"></i>${un} in use but unclaimed</span>`:""}
${free?`<span><i style="border:1px dashed var(--line2)"></i>${free} free</span>`:""}
${v.in_use_metric!=null?`<span class="sub">host reports ${v.in_use_metric} in use</span>`:""}
</div>
${rosterHTML(v)}</div>`;
}
if(v.roster && v.type!=="gpu"){ return `<div class="viz">${rosterHTML(v)}</div>`; }
if(v.type==="claimants"){
return `<div class="viz"><div class="claims">${(v.items||[]).map(c=>`
<div class="claim"><span class="cn">${esc(c.name)}</span>
<span class="badge">${esc(c.ih_status)} / ${esc(c.os_status)}</span>
<span style="color:var(--dim);font-size:12px">${esc(c.verdict||"")}</span></div>`).join("")}</div></div>`;
}
return "";
}
function drawCase(d){
const a=d.alert||{}, sc=a.screen||{}, ig=d.integrations||{};
const acts=(ig.actions||[]);
const zd=acts.find(x=>x.kind==="zendesk"), jr=acts.find(x=>x.kind==="jira");
const manual=acts.filter(x=>x.kind==="manual");
const out=[];
out.push(`<div class="hero">
<div class="crumb">
<span class="badge ${esc(sc.verdict||"")}" title="${esc(VERDICT_HELP[sc.verdict]||"")}">${esc(sc.label||"")}</span>
<span>${esc(a.title)}</span><span>&middot;</span><span>${esc(a.region_label||a.region||"")}</span>
<span>&middot;</span><span>held ${esc(a.effective_age_text||"?")}</span>
<span>&middot;</span><span class="${a.state==="pending"?"t-warn":""}">${esc(a.state||"firing")} in prometheus</span>
${a.age_is_reset?`<span title="Prometheus activeAt was reset by a metric-pipeline dip">&middot; prometheus says ${esc(a.age_text)}</span>`:""}
</div>
<div class="vd">${esc(d.verdict||d.error||"No verdict")}</div>
${d.assessment?`<div class="vsub">${esc(d.assessment)}</div>`:""}
${vizHTML(d)}
</div>`);
// The point of the screen: what to do now.
const hasAny = zd||jr||manual.length;
out.push(`<div class="acts">
<h3>Do this</h3>
${sc.actionable?"":`<div class="hintline">This case is screened out (${esc(sc.label)}). ${esc(sc.reason||"")}</div>`}
${zd?`<div class="who">
<span style="color:var(--dim)">Customer</span>
<span class="em">${esc((zd.recipients||[])[0]||"unresolved")}</span>
${a.org_name?`<span class="badge">${esc(a.org_name)}</span>`:""}
</div>`:""}
<div class="actrow">
${zd?`<button class="pri" onclick="openZendesk()">Contact customer via Zendesk</button>`:""}
${jr?`<button onclick="openJira()">Escalate to Infrastructure (Jira)</button>`:""}
${!sc.actionable?`<button class="sm" onclick="open('${esc(a.id)}',true)">Re-run full diagnosis</button>`:""}
${!hasAny?`<span class="hintline">No outbound action for this case — the runbook keeps it internal.</span>`:""}
</div>
${manual.map(m=>`<div class="cmd"><code>${esc(m.payload.command||m.label)}</code>
${m.payload.command?`<button class="sm" onclick="cp(this,'${esc(m.payload.command)}')">Copy</button>`:""}</div>`).join("")}
${manual.length?`<div class="hintline">Run these yourself — CX Triage is read-only and never mutates the platform.</div>`:""}
</div>`);
if((d.notes||[]).length)
out.push(`<details class="fold"><summary>Caveats (${d.notes.length})</summary><div class="foldb">
<ul style="margin:0;padding-left:18px;font-size:13px;color:var(--warn)">
${d.notes.map(n=>`<li>${esc(n)}</li>`).join("")}</ul></div></details>`);
out.push(`<details class="fold"><summary>Why — what the platforms say</summary><div class="foldb"><table>
${(d.findings||[]).map(f=>`<tr><td class="k">${esc(f.label)}</td><td class="v t-${esc(f.tone)}">${esc(f.value)}
${f.detail?`<span class="det">${esc(f.detail)}</span>`:""}</td></tr>`).join("")}
</table></div></details>`);
out.push(`<details class="fold"><summary>Runbook steps (${(d.actions||[]).length})</summary><div class="foldb">
<ol class="steps">${(d.actions||[]).map(x=>`<li>${esc(x.text)}
<span class="ow ${x.owner!=="CX"?"i":""}">${esc(x.owner)}</span>
${x.guide?`<span class="det">Guide: ${esc(x.guide)}</span>`:""}</li>`).join("")}</ol></div></details>`);
out.push(`<details class="fold"><summary>Raw evidence</summary><div class="foldb">
<pre>${esc(JSON.stringify({screen:a.screen,labels:a.labels,evidence:d.evidence},null,2))}</pre></div></details>`);
$("#stage").innerHTML=out.join("");
}
function cp(btn,text){navigator.clipboard.writeText(text).then(()=>{const o=btn.textContent;btn.textContent="Copied";setTimeout(()=>btn.textContent=o,1100);});}
/* ---------- drawers ---------- */
function showDrawer(){$("#scrim").classList.add("on");$("#drawer").classList.add("on");}
function closeDrawer(){$("#scrim").classList.remove("on");$("#drawer").classList.remove("on");}
$("#scrim").onclick=closeDrawer;
function openZendesk(){
const ig=S.diag.integrations||{}, z=(ig.actions||[]).find(x=>x.kind==="zendesk");
if(!z) return;
const t=z.payload.ticket||{};
$("#dTitle").textContent="Contact customer via Zendesk";
$("#dBody").innerHTML=`
${z.enabled?"":`<div class="warnbox"><b>Preview only.</b> ${esc(z.blocked_reason)} Nothing will be sent.</div>`}
${z.payload._when?`<div class="warnbox">When to send: ${esc(z.payload._when)}</div>`:""}
<div class="fld"><label>To</label><input id="zTo" value="${esc((t.requester||{}).email||"")}"></div>
<div class="fld"><label>Subject</label><input id="zSub" value="${esc(t.subject||"")}"></div>
<div class="fld"><label>Message (approved runbook wording — edit if needed)</label>
<textarea id="zBody">${esc((t.comment||{}).body||"")}</textarea></div>
<div class="fld"><label>Priority</label><select id="zPri">
${["low","normal","high","urgent"].map(p=>`<option ${p===t.priority?"selected":""}>${p}</option>`).join("")}
</select></div>
<div class="fld"><label>Tags</label><input id="zTags" value="${esc((t.tags||[]).join(", "))}"></div>`;
$("#dFoot").innerHTML=`
<button class="pri" id="zSend" ${z.enabled?"":"disabled"}>Send to customer</button>
<button onclick="closeDrawer()">Cancel</button>
<span class="hintline" style="margin:0">${z.enabled?"You will be asked to confirm.":"Configure Zendesk to enable sending."}</span>`;
if(z.enabled) $("#zSend").onclick=confirmSend;
showDrawer();
}
function confirmSend(){
const to=$("#zTo").value;
$("#dFoot").innerHTML=`<span style="font-size:13px">Send a public reply to <b>${esc(to)}</b>?</span>
<button class="pri" id="zYes">Yes, send</button><button onclick="openZendesk()">Back</button>`;
$("#zYes").onclick=async()=>{
$("#zYes").disabled=true;$("#zYes").textContent="Sending…";
const r=await api("/api/actions/zendesk",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({alert_id:S.sel,to,subject:$("#zSub").value,body:$("#zBody").value,
priority:$("#zPri").value,tags:$("#zTags").value.split(",").map(s=>s.trim()).filter(Boolean)})});
$("#dFoot").innerHTML=r.ok
? `<span class="t-ok">Sent — ticket #${esc(r.ticket_id)}</span><button onclick="closeDrawer()">Close</button>`
: `<span class="t-bad">${esc(r.error||"Send failed")}</span><button onclick="openZendesk()">Back</button>`;
};
}
function openJira(){
const ig=S.diag.integrations||{}, j=(ig.actions||[]).find(x=>x.kind==="jira");
if(!j) return;
const f=j.payload.fields||{};
$("#dTitle").textContent="Escalate to Infrastructure";
$("#dBody").innerHTML=`
${j.enabled?"":`<div class="warnbox"><b>Preview only.</b> ${esc(j.blocked_reason)}</div>`}
<div class="fld"><label>Project</label><input id="jProj" value="${esc((f.project||{}).key||"")}"></div>
<div class="fld"><label>Summary</label><input id="jSum" value="${esc(f.summary||"")}"></div>
<div class="fld"><label>Description</label><textarea id="jDesc">${esc(f.description||"")}</textarea></div>
<div class="fld"><label>Labels</label><input id="jLab" value="${esc((f.labels||[]).join(", "))}"></div>`;
$("#dFoot").innerHTML=`<button class="pri" ${j.enabled?"":"disabled"}>Create issue</button>
<button onclick="closeDrawer()">Cancel</button>
<span class="hintline" style="margin:0">${j.enabled?"":"Configure Jira to enable."}</span>`;
showDrawer();
}
$("#refresh").onclick=()=>load(true);
load(false);
setInterval(()=>load(false),60000);
</script>
</body>
</html>
"""