From 12626902760af9bca4b7d971db757bbca3a96fc3 Mon Sep 17 00:00:00 2001 From: Parham Monfared Date: Thu, 6 Aug 2026 07:11:28 +0100 Subject: [PATCH] Split into a FastAPI backend and a React frontend, add case state and SSO The single-file stdlib server became the limit: no way to track what had been done about an alert, no accounts, and a UI that had to be hand-rolled in template strings. This restructures it into something deployable. Backend (FastAPI) - app/ holds config, database, auth, delivery and the routers; triagelib keeps the triage engine unchanged, so the validated screening and runbook logic is untouched. - Cases persist per alert fingerprint with a status workflow (investigating, customer contacted, escalated to Infra, waiting, remediated, resolved, won't fix, false positive), an assignee, notes and an append-only history. An alert that stops and re-fires lands back on the same case and counts as a reopen. - Suppression rules move from a JSON file into the database. Auth - Signed session cookies over PBKDF2 local accounts, plus an OIDC flow ready for Authentik: users are created on first login and admin follows a group claim. Local login can be switched off entirely once SSO is live. Zendesk and Jira - Delivery is now implemented, behind three gates: the integration must be configured, its feature flag on, and CX_FEATURE_SEND_ENABLED on. A demo instance leaves the last off and cannot mail anyone. Both search before creating, so re-diagnosing an alert updates one ticket rather than opening several, and a rolling daily cap stops a loop mailing everybody. Deployment - Multi-stage Dockerfile builds the bundle and serves it from the API origin. - docker-compose for local and single-host use; Gitea Actions runs the tests, builds the image and renders deploy/k8s with envsubst. Two fixes found while testing: assigning a case returned a null assignee, and add_event could leave an already-loaded history collection stale. Known gap: the engine reaches OpenStack via `docker exec -osc`, which does not work in a pod without the CX-Tools containers alongside it. docs/DEPLOYMENT.md sets out the three ways to close that. Co-Authored-By: Claude Opus 5 --- .env.example | 67 ++ .gitea/workflows/ci.yaml | 128 ++++ INTEGRATIONS.md | 89 --- README.md | 263 ++------ backend/Dockerfile | 35 ++ backend/app/__init__.py | 2 + backend/app/auth.py | 253 ++++++++ backend/app/config.py | 101 ++++ backend/app/db.py | 29 + backend/app/delivery.py | 174 ++++++ backend/app/main.py | 66 ++ backend/app/models.py | 208 +++++++ backend/app/routers/__init__.py | 0 backend/app/routers/actions_router.py | 77 +++ backend/app/routers/alerts_router.py | 88 +++ backend/app/routers/auth_router.py | 99 +++ backend/app/routers/cases_router.py | 99 +++ backend/app/routers/linkage_router.py | 42 ++ backend/app/routers/settings_router.py | 103 ++++ backend/app/services.py | 200 ++++++ backend/requirements.txt | 6 + backend/tests/test_api.py | 124 ++++ {tests => backend/tests}/test_runbooks.py | 0 {tests => backend/tests}/test_screening.py | 0 {triagelib => backend/triagelib}/__init__.py | 0 {triagelib => backend/triagelib}/alerts.py | 0 {triagelib => backend/triagelib}/comms.py | 0 {triagelib => backend/triagelib}/cxbridge.py | 0 .../triagelib}/integrations.py | 0 {triagelib => backend/triagelib}/linkage.py | 0 .../triagelib}/prometheus.py | 0 {triagelib => backend/triagelib}/runbooks.py | 0 {triagelib => backend/triagelib}/screening.py | 0 {triagelib => backend/triagelib}/settings.py | 0 cx-triage | 79 --- deploy/k8s/00-namespace.yaml | 4 + deploy/k8s/10-config.yaml | 30 + deploy/k8s/20-deployment.yaml | 51 ++ deploy/k8s/30-service.yaml | 9 + deploy/k8s/40-ingress.yaml | 23 + docker-compose.yml | 47 ++ docs/DEPLOYMENT.md | 87 +++ docs/INTEGRATIONS.md | 133 ++++ LINKAGE.md => docs/LINKAGE.md | 0 PLAN.md => docs/PLAN.md | 0 frontend/.dockerignore | 2 + frontend/Dockerfile | 10 + frontend/index.html | 12 + frontend/package.json | 24 + frontend/src/App.tsx | 61 ++ frontend/src/components/ActionDrawer.tsx | 139 +++++ frontend/src/components/CasePanel.tsx | 83 +++ frontend/src/components/Visuals.tsx | 135 +++++ frontend/src/lib/api.ts | 37 ++ frontend/src/main.tsx | 13 + frontend/src/pages/Linkage.tsx | 134 ++++ frontend/src/pages/Login.tsx | 57 ++ frontend/src/pages/Queue.tsx | 286 +++++++++ frontend/src/pages/Settings.tsx | 172 ++++++ frontend/src/styles.css | 164 +++++ frontend/src/types.ts | 139 +++++ frontend/tsconfig.json | 19 + frontend/vite.config.ts | 19 + triagelib/server.py | 437 ------------- triagelib/ui.py | 429 ------------- triagelib/ui_linkage.py | 181 ------ triagelib/ui_settings.py | 221 ------- triagelib/ui_v2.py | 572 ------------------ 68 files changed, 3839 insertions(+), 2223 deletions(-) create mode 100644 .env.example create mode 100644 .gitea/workflows/ci.yaml delete mode 100644 INTEGRATIONS.md create mode 100644 backend/Dockerfile create mode 100644 backend/app/__init__.py create mode 100644 backend/app/auth.py create mode 100644 backend/app/config.py create mode 100644 backend/app/db.py create mode 100644 backend/app/delivery.py create mode 100644 backend/app/main.py create mode 100644 backend/app/models.py create mode 100644 backend/app/routers/__init__.py create mode 100644 backend/app/routers/actions_router.py create mode 100644 backend/app/routers/alerts_router.py create mode 100644 backend/app/routers/auth_router.py create mode 100644 backend/app/routers/cases_router.py create mode 100644 backend/app/routers/linkage_router.py create mode 100644 backend/app/routers/settings_router.py create mode 100644 backend/app/services.py create mode 100644 backend/requirements.txt create mode 100644 backend/tests/test_api.py rename {tests => backend/tests}/test_runbooks.py (100%) rename {tests => backend/tests}/test_screening.py (100%) rename {triagelib => backend/triagelib}/__init__.py (100%) rename {triagelib => backend/triagelib}/alerts.py (100%) rename {triagelib => backend/triagelib}/comms.py (100%) rename {triagelib => backend/triagelib}/cxbridge.py (100%) rename {triagelib => backend/triagelib}/integrations.py (100%) rename {triagelib => backend/triagelib}/linkage.py (100%) rename {triagelib => backend/triagelib}/prometheus.py (100%) rename {triagelib => backend/triagelib}/runbooks.py (100%) rename {triagelib => backend/triagelib}/screening.py (100%) rename {triagelib => backend/triagelib}/settings.py (100%) delete mode 100755 cx-triage create mode 100644 deploy/k8s/00-namespace.yaml create mode 100644 deploy/k8s/10-config.yaml create mode 100644 deploy/k8s/20-deployment.yaml create mode 100644 deploy/k8s/30-service.yaml create mode 100644 deploy/k8s/40-ingress.yaml create mode 100644 docker-compose.yml create mode 100644 docs/DEPLOYMENT.md create mode 100644 docs/INTEGRATIONS.md rename LINKAGE.md => docs/LINKAGE.md (100%) rename PLAN.md => docs/PLAN.md (100%) create mode 100644 frontend/.dockerignore create mode 100644 frontend/Dockerfile create mode 100644 frontend/index.html create mode 100644 frontend/package.json create mode 100644 frontend/src/App.tsx create mode 100644 frontend/src/components/ActionDrawer.tsx create mode 100644 frontend/src/components/CasePanel.tsx create mode 100644 frontend/src/components/Visuals.tsx create mode 100644 frontend/src/lib/api.ts create mode 100644 frontend/src/main.tsx create mode 100644 frontend/src/pages/Linkage.tsx create mode 100644 frontend/src/pages/Login.tsx create mode 100644 frontend/src/pages/Queue.tsx create mode 100644 frontend/src/pages/Settings.tsx create mode 100644 frontend/src/styles.css create mode 100644 frontend/src/types.ts create mode 100644 frontend/tsconfig.json create mode 100644 frontend/vite.config.ts delete mode 100644 triagelib/server.py delete mode 100644 triagelib/ui.py delete mode 100644 triagelib/ui_linkage.py delete mode 100644 triagelib/ui_settings.py delete mode 100644 triagelib/ui_v2.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..362919b --- /dev/null +++ b/.env.example @@ -0,0 +1,67 @@ +# --------------------------------------------------------------------------- +# Copy to .env and edit. Everything here is read at startup. +# --------------------------------------------------------------------------- + +# --- app ------------------------------------------------------------------- +CX_APP_NAME=CX Triage +CX_PORT=8080 +# Must be the externally reachable URL: the SSO redirect is built from it. +CX_BASE_URL=http://localhost:8080 +# openssl rand -hex 32 +CX_SECRET_KEY=change-me +CX_SESSION_HOURS=12 + +# --- database -------------------------------------------------------------- +# compose provides postgres; sqlite is fine for a single user. +# CX_DATABASE_URL=sqlite:////data/cx-triage.db +CX_DATABASE_URL=postgresql+psycopg://cx:cx@db:5432/cxtriage + +# --- data sources ---------------------------------------------------------- +CX_PROMETHEUS_BASE=http://10.11.254.250:9090 +# Container to relay Prometheus queries through when the host has no route to +# the internal network. Leave blank to auto-detect (ca1-osc, us1-osc, ...). +CX_PROMETHEUS_RELAY= +CX_TOOLS_PATH=/opt/cx-tools + +# --- authentication -------------------------------------------------------- +CX_AUTH_LOCAL_ENABLED=true +# Only used to create the very first account, when the user table is empty. +CX_BOOTSTRAP_ADMIN_EMAIL=admin@nexgencloud.com +CX_BOOTSTRAP_ADMIN_PASSWORD= + +# --- Authentik SSO --------------------------------------------------------- +# Redirect URI to register in Authentik: +# ${CX_BASE_URL}/api/auth/oidc/callback +CX_OIDC_ENABLED=false +CX_OIDC_ISSUER=https://sso.nexgencloud.com/application/o/cx-triage/ +CX_OIDC_CLIENT_ID= +CX_OIDC_CLIENT_SECRET= +CX_OIDC_SCOPES=openid email profile +# Members of this group become administrators. +CX_OIDC_ADMIN_GROUP=cx-triage-admins +CX_OIDC_GROUPS_CLAIM=groups + +# --- feature flags --------------------------------------------------------- +# The master switch. With this off nothing can reach a customer, whatever else +# is configured. Leave it off on demo and staging instances. +CX_FEATURE_SEND_ENABLED=false +CX_FEATURE_ZENDESK=false +CX_FEATURE_JIRA=false +CX_FEATURE_LINKAGE_SCAN=true +# Refuses to send more than this in a rolling 24h, so a loop cannot mail everyone. +CX_SEND_DAILY_CAP=25 + +# --- Zendesk --------------------------------------------------------------- +# Admin Center -> Apps and integrations -> APIs -> Zendesk API -> Add API token +CX_ZENDESK_SUBDOMAIN= +CX_ZENDESK_EMAIL= +CX_ZENDESK_TOKEN= +CX_ZENDESK_PUBLIC_REPLY=true + +# --- Jira ------------------------------------------------------------------ +# https://id.atlassian.com/manage-profile/security/api-tokens +CX_JIRA_BASE=https://nexgencloud.atlassian.net +CX_JIRA_EMAIL= +CX_JIRA_TOKEN= +CX_JIRA_PROJECT=INFRA +CX_JIRA_ISSUE_TYPE=Task diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml new file mode 100644 index 0000000..5be3a5a --- /dev/null +++ b/.gitea/workflows/ci.yaml @@ -0,0 +1,128 @@ +name: build-and-deploy + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + branches: [main] + workflow_dispatch: + +env: + REGISTRY: ${{ vars.REGISTRY || 'git.ngbackend.cloud' }} + IMAGE: ${{ vars.IMAGE_NAME || 'parham.monfared/cx-ui' }} + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: { python-version: "3.12" } + - name: Install backend dependencies + run: pip install -r backend/requirements.txt + - name: Runbook and screening tests + # These are pure-logic tests: no Prometheus, no CX-Tools, no network. + run: | + cd backend + python tests/test_runbooks.py + python tests/test_screening.py + python tests/test_api.py + + - uses: actions/setup-node@v4 + with: { node-version: "22" } + - name: Build frontend + run: | + cd frontend + npm ci --no-audit --no-fund || npm install --no-audit --no-fund + npm run typecheck + npm run build + + image: + needs: test + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + - uses: docker/metadata-action@v5 + id: meta + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=sha,prefix=,format=short + - uses: docker/build-push-action@v6 + with: + context: . + file: backend/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + deploy: + needs: image + if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + environment: ${{ vars.DEPLOY_ENVIRONMENT || 'production' }} + steps: + - uses: actions/checkout@v4 + + - name: Write kubeconfig + run: | + mkdir -p "$HOME/.kube" + echo "${{ secrets.KUBECONFIG }}" | base64 -d > "$HOME/.kube/config" + chmod 600 "$HOME/.kube/config" + + - name: Sync secrets + # Applied imperatively so nothing sensitive is ever committed. Every + # value comes from the repository/environment secret store. + run: | + kubectl -n "${{ vars.K8S_NAMESPACE || 'cx-triage' }}" create secret generic cx-triage \ + --from-literal=CX_SECRET_KEY='${{ secrets.CX_SECRET_KEY }}' \ + --from-literal=CX_DATABASE_URL='${{ secrets.CX_DATABASE_URL }}' \ + --from-literal=CX_OIDC_CLIENT_ID='${{ secrets.CX_OIDC_CLIENT_ID }}' \ + --from-literal=CX_OIDC_CLIENT_SECRET='${{ secrets.CX_OIDC_CLIENT_SECRET }}' \ + --from-literal=CX_ZENDESK_SUBDOMAIN='${{ secrets.CX_ZENDESK_SUBDOMAIN }}' \ + --from-literal=CX_ZENDESK_EMAIL='${{ secrets.CX_ZENDESK_EMAIL }}' \ + --from-literal=CX_ZENDESK_TOKEN='${{ secrets.CX_ZENDESK_TOKEN }}' \ + --from-literal=CX_JIRA_BASE='${{ vars.CX_JIRA_BASE }}' \ + --from-literal=CX_JIRA_EMAIL='${{ secrets.CX_JIRA_EMAIL }}' \ + --from-literal=CX_JIRA_TOKEN='${{ secrets.CX_JIRA_TOKEN }}' \ + --from-literal=CX_BOOTSTRAP_ADMIN_PASSWORD='${{ secrets.CX_BOOTSTRAP_ADMIN_PASSWORD }}' \ + --dry-run=client -o yaml | kubectl apply -f - + + - name: Render and apply manifests + env: + IMAGE_REF: ${{ env.REGISTRY }}/${{ env.IMAGE }}:${{ github.sha }} + NAMESPACE: ${{ vars.K8S_NAMESPACE || 'cx-triage' }} + DOMAIN: ${{ vars.CX_DOMAIN }} + BASE_URL: https://${{ vars.CX_DOMAIN }} + OIDC_ENABLED: ${{ vars.CX_OIDC_ENABLED || 'true' }} + OIDC_ISSUER: ${{ vars.CX_OIDC_ISSUER }} + OIDC_ADMIN_GROUP: ${{ vars.CX_OIDC_ADMIN_GROUP || 'cx-triage-admins' }} + PROMETHEUS_BASE: ${{ vars.CX_PROMETHEUS_BASE }} + FEATURE_SEND_ENABLED: ${{ vars.CX_FEATURE_SEND_ENABLED || 'false' }} + FEATURE_ZENDESK: ${{ vars.CX_FEATURE_ZENDESK || 'false' }} + FEATURE_JIRA: ${{ vars.CX_FEATURE_JIRA || 'false' }} + FEATURE_LINKAGE_SCAN: ${{ vars.CX_FEATURE_LINKAGE_SCAN || 'true' }} + SEND_DAILY_CAP: ${{ vars.CX_SEND_DAILY_CAP || '25' }} + JIRA_PROJECT: ${{ vars.CX_JIRA_PROJECT || 'INFRA' }} + INGRESS_CLASS: ${{ vars.INGRESS_CLASS || 'nginx' }} + TLS_ISSUER: ${{ vars.TLS_ISSUER || 'letsencrypt-prod' }} + REPLICAS: ${{ vars.REPLICAS || '1' }} + run: | + for f in deploy/k8s/*.yaml; do + envsubst < "$f" + done > /tmp/rendered.yaml + kubectl apply -f /tmp/rendered.yaml + kubectl -n "$NAMESPACE" rollout status deployment/cx-triage --timeout=5m diff --git a/INTEGRATIONS.md b/INTEGRATIONS.md deleted file mode 100644 index a03f02a..0000000 --- a/INTEGRATIONS.md +++ /dev/null @@ -1,89 +0,0 @@ -# Wiring Zendesk and Jira - -"Not wired" meant: the app **builds** the full API payload and shows it to you, -but the code path that would POST it to Zendesk deliberately returns an error. -Nothing can reach a customer today, even by accident. Here is what is needed to -change that. - ---- - -## What I need from you - -### Zendesk - -| Thing | Where it comes from | Example | -|---|---|---| -| Subdomain | your Zendesk URL `https://.zendesk.com` | `nexgencloud` | -| API email | the agent account tickets are created as | `cx-bot@nexgencloud.com` | -| API token | Zendesk **Admin Center → Apps and integrations → APIs → Zendesk API → Add API token** | 40-char string | - -Plus four decisions: - -1. **Which agent account should own these tickets?** A dedicated `cx-triage` agent - is better than a person's account — the audit trail stays clear. -2. **Public reply or internal note on first send?** The runbook wording is written - for the customer, so public — but confirm. -3. **Requester matching.** Infrahub gives us the owner's name and email. If they - are not already a Zendesk user, should we create them, or fail and ask a human? -4. **Do you want a tag convention** beyond the `cx-triage`, `alert-`, - `region-` tags I currently set? - -### Jira - -| Thing | Where it comes from | Example | -|---|---|---| -| Base URL | your Atlassian site | `https://nexgencloud.atlassian.net` | -| API email | Atlassian account | `cx-bot@nexgencloud.com` | -| API token | | token string | -| Project key | the Infrastructure project | `INFRA`? | -| Issue type | must exist in that project | `Task`? `Bug`? | - -Confirm the project key and issue type — I guessed `INFRA` / `Task`. - -### Where to put the credentials - -**Don't paste them to me.** Two options: - -- **Preferred — 1Password.** Create two items in the `Employee` vault, e.g. - `Zendesk API (CX Triage)` and `Jira API (CX Triage)`, each with the token in the - `password` field and the email in a `username` field. I extend the existing - CX-Tools secrets loader to read them, exactly like the Infrahub key. Nothing - touches disk. -- **Quick and dirty** — environment variables in your shell before launching: - `CX_ZENDESK_SUBDOMAIN`, `CX_ZENDESK_EMAIL`, `CX_ZENDESK_TOKEN`, - `CX_JIRA_BASE`, `CX_JIRA_EMAIL`, `CX_JIRA_TOKEN`, `CX_JIRA_PROJECT`. - Fine for a trial, worse for a shared tool. - ---- - -## What I build once I have that - -1. **Delivery** — replace the refusal in `App.send_zendesk` with a real POST, and - add the same for Jira. -2. **Search before create** — `GET /api/v2/search?query=external_id:cx-triage-` - so re-diagnosing an alert comments on the existing ticket rather than opening a - second one. Same for Jira via a `cx-triage-` label. -3. **A send gate** — the server refuses to send unless started with `--allow-send`. - A demo instance then physically cannot email a customer, no matter what is - clicked. Default stays off. -4. **Audit** — every send appended to `vmc-audit.log` with who, what, and the - resulting ticket URL, and the URL shown back on the case. -5. **Rate limiting** — Zendesk allows 700 req/min; a simple per-minute cap plus a - refusal to send more than N tickets in one session, so a bad loop cannot mail - a hundred customers. - -## Guard rails that stay regardless - -- Every send needs a human click plus a confirm naming the recipient. -- No auto-send, ever — a verdict never triggers an email on its own. -- No bulk send in v1. The 13 overdue ERROR alerts are tempting, but one wrong - template across 13 customers is a bad first outing. -- The message stays editable before sending. -- Deleting, shelving and InfraInsight edits remain copy-a-command. The read-only - guarantee is what makes this safe against production. - -## Suggested first run - -Point it at a **Zendesk sandbox** first, or send the first real ticket to your own -address by editing the To field. Once one round-trip looks right in Zendesk, turn -it on for real. diff --git a/README.md b/README.md index 23c4f3c..f363f64 100644 --- a/README.md +++ b/README.md @@ -1,238 +1,71 @@ # CX Triage -A small local webapp that takes the Infrahub error alerts out of Prometheus, -diagnoses each one using the **unmodified** CX-Tools (`vmc`) collectors, tells you -what the runbook says to do next, and — when the next step is contacting the -customer — shows the approved wording alongside the customer's contact details. +Turns the Infrahub error-alert firehose into a short list of things that +actually need doing — then helps you do them. -**It is read-only.** It queries Infrahub, OpenStack, InfraInsight and Prometheus. -It never changes platform state, never deletes or shelves anything, and never -sends a message. Every action it identifies is presented for a human to perform. +Python/FastAPI backend, React frontend, PostgreSQL for case state. -## Separating noise from real work +## What it does -Thousands of alerts fire; only a handful are work. Before anything is shown, each -alert's condition is **re-checked against current state**, and the verdict is -displayed with its reason: +1. **Pulls** the alert queue from Prometheus. +2. **Screens** every alert by re-checking its condition against live state, so + noise and already-resolved alerts drop out. +3. **Diagnoses** what is left using the CX runbooks, reconciling Infrahub + against OpenStack through the CX-Tools collectors. +4. **Drafts** the customer email with contacts resolved from Infrahub, and the + Jira escalation with the evidence attached. +5. **Tracks** each case — who owns it, what was done, what was sent. -| Verdict | Meaning | In the queue? | -|---|---|---| -| **needs action** | The condition still holds | yes | -| **needs action (unverified)** | Couldn't be re-checked — never hidden on a guess | yes | -| **already resolved** | Infrahub has moved on / the IP is no longer duplicated / the GPU gap has closed | hidden | -| **not yet firing** | Prometheus still has it pending | hidden | -| **chronic** | Still true, but firing over 3 days — already ticketed, not new work | hidden | -| **low impact** | Still true, but owned by an internal org or a platform-owned node | hidden | +On live data this takes roughly **2,650 firing alerts down to ~20** that need a +decision. -Screening only ever demotes an alert on **positive evidence**; anything it can't -settle stays in the queue. Hidden alerts are one checkbox away, and any of them -can be force-diagnosed with **Diagnose anyway**. +## Findings that shaped it -### Alert ages are recovered, not taken from Prometheus +Validated against production, not assumed: -Prometheus' own `activeAt` is unreliable here. The Infrahub `Resources` metric -drops most of its series for ~5 minutes several times a day (4 dips in the last -24h observed; one took it from ~4,370 series to 1,359). Every alert alive during -a dip resolves and re-fires, so `activeAt` resets on all of them at once — which -is why the Prometheus UI shows dozens of unrelated alerts with the *same* age. +- **`Suspected Rogue VM` is measuring spare capacity.** `In_Use_Gpus` equals the + physical GPU count on 71 of 75 firing hosts, so the rule reduces to "this host + has a free GPU". Checked against OpenStack on 10 hosts: Infrahub and OpenStack + agreed exactly on all of them. Those alerts are flagged as a rule defect. +- **`Exists in Infrahub but does not exist in OpenStack` matches every VM**, + because `openstack_nova_server_status` returns no series. Excluded outright. + The same gap means `Suspected Orphan VM` cannot fire at all. +- **Prometheus alert ages are unreliable here.** The `Resources` metric drops + most of its series several times a day; every alert alive at the time resolves + and re-fires, resetting `activeAt`. Ages are recovered from `ALERTS` history + instead. -So the app walks the `ALERTS` series backwards over 7 days instead, bridging gaps -under 45 minutes, and reports how long each condition has **actually** held. In -practice this is the difference between "40 alerts all 7h old" and "11 that are -genuinely new, 33 that have been true for days". Both numbers are shown: the -recovered duration, with Prometheus' value in a tooltip when they disagree. +## Read-only by design -The app detects these dips and warns about them, since they also mean any alert -with a long `for:` may never reach firing state. +The app queries and advises. It never deletes, shelves, or edits a VM — those +stay copy-a-command. The only thing it can send is a Zendesk ticket or a Jira +issue, behind three gates and a confirm step. See +[docs/INTEGRATIONS.md](docs/INTEGRATIONS.md). -The re-check is cheap on purpose: it reads the same Prometheus series the rules -are built from — `Resources`, `In_Use_Gpus`, `Total_Gpus` — in one bulk snapshot -for the entire queue, rather than an Infrahub and OpenStack call per alert. Only -alerts you actually open cost a CX-Tools query. - -On live data this takes **~2,670 firing alerts down to ~11** that need a decision. - -Caches are warmed at startup (~20s, mostly the 7-day history read), so page loads -are instant afterwards. They refresh on a 30s/60s/5min cadence. - -**Excluded outright:** `Exists in Infrahub but does not exist in OpenStack`. It is -built as `Resources unless on(openstack_id) openstack_nova_server_status`, and -that second metric is currently returning **zero series** — so nothing gets -excluded by the `unless` and every Infrahub VM alerts. It is a broken exporter, -not a queue of work. The app detects this class of failure and shows a banner, -because the same gap also means `Suspected Orphan VM` cannot fire at all. - -## Two tabs - -- **CX runbooks** — the alerts below, grouped into collapsible sections in - working order (rogue VMs, duplicate IPs, total GPUs, hibernating, creating, - shutoff, deleting, error), each showing how long it has been firing, **newest - first** so long-running alerts sink to the bottom. -- **Infrastructure** — everything else, so it stays out of the triage queue: - node-exporter host alerts in their own section, then Ceph, MySQL, Galera, - OpenStack services, blackbox. Listed and counted, not diagnosed. Routing is - keyed off the **rule file**, not the alert name, because two different rule - files both use the group name "Imported Rules". - -## What it covers - -One runbook per alert type, from *Infrahub Errors Remediation*: - -| Alert | Priority | What the app works out for you | -|---|---|---| -| Instance in ERROR state | LOW–HIGH | Matches the fault against the runbook fault table; decides whether the VM was ever ACTIVE (which changes both the urgency and the comms template); for the NUMA/PCI fault it sums GPUs on the host to check whether the host is full before you escalate | -| Instance in DELETING state | LOW | Confirms the delete request in Infrahub events, and whether the OpenStack server is still there or already gone | -| Instance in SHUTOFF state | LOW | Confirms SHUTOFF and drafts the billing-awareness note | -| Instance in HIBERNATING state | HIGH | Runs the host signals (Nova state/status, disabled reason, OVS liveness) and escalates when they're bad | -| Instance in CREATING state | MEDIUM | Determines whether the VM ever got an OpenStack ID | -| Instance in RESTORING state | HIGH | Host signals plus the most recent *failed* OpenStack event to escalate | -| Instance in REBOOTING state | HIGH | Confirms `InstanceRebootRequest` and the expected `HARD_REBOOT` state | -| Instance in BUILD state | MEDIUM | Distinguishes "large flavor, still transient" from "stuck, escalate" | -| Suspected Rogue VM | HIGH | Quantifies the per-host GPU accounting gap the rule actually fires on, then reconciles every instance on the host and maps each mismatch to its row in the Mismatch Remediation table; tempest instances are ignored | -| Duplicated IPs | HIGH | Classifies each claimant of the IP as Scenario #1 / #2 / rightful owner, and pulls the cross-environment claimant list from `Resources{floating_ip=...}` | -| Problem with Total GPUs | HIGH | Lists which customers are on the affected host | -| Suspected Orphan VM | HIGH | Same host reconciliation as Rogue VM (cannot currently fire — see the exporter note above) | -| Openstack status=X / Infrahub status!=X | HIGH | Single-VM mismatch, taken through the same remediation table | - -### A note on Suspected Rogue VM - -The rule is not a status comparison — it is -`sum by(instance)(In_Use_Gpus) - sum by(instance)(Resources{status=~"ACTIVE|SHUTOFF|PRE_ACTIVE"}) >= 1`, -a **per-host GPU accounting gap**. Two different faults produce that gap: - -1. instances running on the host that Infrahub has no record of (a true rogue VM), or -2. Infrahub VMs that are ACTIVE but have **no host recorded**, so they are never - counted against the host that is actually running them. - -Prometheus cannot tell these apart, so the app states both and lets the -per-instance host reconciliation settle it — a genuine rogue VM shows up as -`Infrahub Missing`. It also reports how many unattributed ACTIVE VMs exist -platform-wide, because that number alone can be large enough to explain the gaps -without any rogue VM existing. - -## Requirements - -Whatever `vmc` already needs, plus nothing: - -- The CX-Tools checkout (`cxlib/` + `vmc`), unmodified -- Python 3 (standard library only — no pip install) -- Docker with the `ca1-osc` / `ca2-osc` / `us1-osc` / `no1-osc` containers running -- A signed-in 1Password CLI session - -## Run +## Run it ```bash -op signin +cp .env.example .env +docker compose up --build ``` -```bash -./cx-triage -``` - -It opens . Bound to localhost only. - -```bash -./cx-triage --check -``` - -Runs preflight (CX-Tools located, credentials loaded, containers up, Prometheus -reachable) and exits. - -Useful flags: `--port`, `--prometheus `, `--no-open`. - -If CX-Tools isn't found automatically, point at it: - -```bash -CX_TOOLS_PATH=~/scripts/CX-Tools ./cx-triage -``` - -## How it reaches things - -- **CX-Tools** is imported as a library. `cxbridge.py` calls `collect_vm`, - `collect_host` and the query helpers — the same code paths as `vmc --json` — - and guards every OpenStack subcommand against a read-only allowlist, so a bug - here cannot mutate an instance. -- **Prometheus** at `10.11.254.250:9090` is on the internal network, which the - laptop has no route to (10.11.* leaves via the default gateway). So queries are - relayed `docker exec ca1-osc curl ...` — the same trick CX-Tools uses for - OpenStack. A direct HTTP transport is tried first, so this still works from a - host that does have a route. Override the relay with `CX_PROMETHEUS_RELAY`. - -## Working an alert - -1. Pick an alert from the queue. Or paste an `ALERTS{...}` line or a Prometheus - graph URL into the box. -2. **What the platforms say** — the reconciled Infrahub/OpenStack/InfraInsight - facts, with mismatches called out in red. -3. **Next steps** — the remaining runbook steps, each tagged with its owner (CX / - Infrastructure team / DevOps). Steps the app has already verified are ticked - off, so you can see what's left rather than re-deriving it. -4. **Suggested customer comms** — only when the runbook calls for it. Verbatim - approved wording with the instance name (and floating IP) substituted, above - the organization and owner contacts CX-Tools resolved. Copy it and send it - from HubSpot. -5. **Evidence** — the raw alert labels and the raw CX-Tools output, for pasting - into a Slack thread or a Jira ticket. - -## Caveats - -These are real limits, not bugs: - -- **Production Infrahub only.** CX-Tools queries production. When a Duplicated - IPs or Rogue VM alert points at a PreProd/Staging record, the app says so and - tells you to check the other environments — it can't query them. The - Prometheus `Resources` series does span environments, which is why the - Duplicated IPs view uses it. -- **No InfraInsight SQL.** The DELETING runbook identifies the requesting *user* - via a SQL query. The app confirms the delete request from Infrahub events but - cannot name the requester, and says so when it matters. -- **Host Health Checks is partial.** The app reports the host signals CX-Tools - exposes (Nova state/status, disabled reason, OVS liveness/heartbeat, uptime, - aggregates). The rest of that guide — disk, dmesg, GPU checks — is still - manual, and the app says which part it did. -- **Fault table coverage.** ERROR faults outside the runbook's table produce an - explicit "not in the table, escalate to a peer" verdict rather than a guess. -- **Chronic/low-impact thresholds are judgement calls**, not runbook rules: - 3 days for chronic (`CHRONIC_DAYS` in `screening.py`), and "internal" means an - `@nexgencloud.com` owner. Adjust to taste. -- **Recovered ages are bounded by a 7-day window** (`TrueAgeIndex.WINDOW_DAYS`). - Anything older shows as `7d+`. -- **"Chronic" does not mean "ignore".** It means the condition has been true for - days, so it is not *new* work. Several ERROR alerts are 6–7 days old; if those - have not actually been ticketed, they are a backlog, not noise. +More: [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) · +[docs/LINKAGE.md](docs/LINKAGE.md) · [docs/PLAN.md](docs/PLAN.md) ## Layout -```text -cx-triage entry point (preflight, then serve) -triagelib/ - cxbridge.py read-only adapter over cxlib - prometheus.py alert source, rule index, bulk state snapshot, relay - alerts.py normalization, classification, exclusions, grouping - screening.py noise-vs-real verdicts - runbooks.py the decision engine - comms.py customer comms templates, verbatim from Confluence - server.py HTTP API + background triage jobs - ui.py the single-page UI -tests/test_runbooks.py runbook decisions against fixture payloads -tests/test_screening.py screening, exclusion, tab routing, ordering ``` +backend/ + app/ FastAPI: config, db, models, auth, delivery, routers + triagelib/ the triage engine (screening, runbooks, comms, linkage) + tests/ pure-logic tests — no network, no CX-Tools +frontend/ React + TypeScript + Vite +deploy/k8s/ manifests, rendered by the pipeline +.gitea/workflows/ test → build → deploy +``` + +## Tests ```bash -python3 tests/test_runbooks.py && python3 tests/test_screening.py +cd backend && python tests/test_runbooks.py && python tests/test_screening.py ``` - -## Updating a runbook - -The decision logic is meant to be edited by whoever owns the runbook: - -- Fault table → `FAULT_TABLE` in `runbooks.py` -- Rogue VM mismatch table → `MISMATCH_TABLE` in `runbooks.py` -- Priority / ETTR → `KIND_META` in `alerts.py` -- Queue order → `FOCUS_ORDER` in `alerts.py` -- Alerts to suppress → `EXCLUDED_ALERTNAMES` in `alerts.py` -- Noise rules → `screening.py` (`CHRONIC_DAYS`, `_KIND_SCREENS`) -- Customer wording → `_TEMPLATES` in `comms.py` - -If a wording change lands in Confluence, change it in `comms.py` and nowhere -else. diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..8a951c5 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,35 @@ +# ---- frontend --------------------------------------------------------------- +FROM node:22-alpine AS ui +WORKDIR /ui +COPY frontend/package.json frontend/package-lock.json* ./ +RUN npm ci --no-audit --no-fund 2>/dev/null || npm install --no-audit --no-fund +COPY frontend/ . +RUN npm run build + +# ---- backend ---------------------------------------------------------------- +FROM python:3.12-slim +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 + +# curl is what the triage engine shells out to for the Infrahub API; the docker +# CLI is only needed when Prometheus/OpenStack are reachable through the +# CX-Tools containers rather than directly (see docs/DEPLOYMENT.md). +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates docker.io \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY backend/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY backend/app ./app +COPY backend/triagelib ./triagelib +COPY --from=ui /ui/dist ./static + +RUN useradd --uid 10001 --create-home cx && mkdir -p /data && chown -R cx /data /app +USER cx + +ENV CX_STATIC_DIR=/app/static CX_DATABASE_URL=sqlite:////data/cx-triage.db +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=40s \ + CMD python -c "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/api/health').read()" +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..19aaa43 --- /dev/null +++ b/backend/app/__init__.py @@ -0,0 +1,2 @@ +"""CX Triage backend.""" +VERSION = "0.2.0" diff --git a/backend/app/auth.py b/backend/app/auth.py new file mode 100644 index 0000000..c3d2963 --- /dev/null +++ b/backend/app/auth.py @@ -0,0 +1,253 @@ +"""Authentication: signed session cookies, local accounts, and OIDC. + +Local accounts exist so the tool runs on a laptop and in a cluster that has not +been wired to SSO yet. When CX_OIDC_ENABLED is set, Authentik becomes the source +of truth: users are created on first login, and admin rights follow a group +claim rather than being set here. + +The session itself is the same either way - a signed, expiring cookie - so the +rest of the app never has to care which provider a user came from. +""" +from __future__ import annotations + +import base64 +import datetime as dt +import hashlib +import hmac +import json +import secrets +import time +from typing import Any, Optional + +import httpx +from fastapi import Depends, HTTPException, Request, status +from sqlalchemy.orm import Session + +from .config import get_settings +from .db import get_db +from .models import AuthProvider, User + +SESSION_COOKIE = "cx_session" +settings = get_settings() + + +# --- password hashing ------------------------------------------------------- +# PBKDF2 from the standard library: no native build step in the image, and +# strong enough for a small internal user table. + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + digest = hashlib.pbkdf2_hmac("sha256", password.encode(), salt, 240_000) + return f"pbkdf2_sha256$240000${base64.b64encode(salt).decode()}${base64.b64encode(digest).decode()}" + + +def verify_password(password: str, stored: str) -> bool: + try: + algo, rounds, salt_b64, digest_b64 = stored.split("$") + if algo != "pbkdf2_sha256": + return False + expected = base64.b64decode(digest_b64) + actual = hashlib.pbkdf2_hmac("sha256", password.encode(), + base64.b64decode(salt_b64), int(rounds)) + return hmac.compare_digest(expected, actual) + except (ValueError, TypeError): + return False + + +# --- session cookie --------------------------------------------------------- + +def _sign(payload: bytes) -> str: + mac = hmac.new(settings.secret_key.encode(), payload, hashlib.sha256).digest() + return f"{base64.urlsafe_b64encode(payload).decode()}.{base64.urlsafe_b64encode(mac).decode()}" + + +def issue_session(user_id: int) -> str: + payload = json.dumps({ + "uid": user_id, + "exp": int(time.time()) + settings.session_hours * 3600, + }).encode() + return _sign(payload) + + +def read_session(token: str) -> Optional[int]: + try: + body_b64, mac_b64 = token.split(".") + payload = base64.urlsafe_b64decode(body_b64) + expected = hmac.new(settings.secret_key.encode(), payload, hashlib.sha256).digest() + if not hmac.compare_digest(expected, base64.urlsafe_b64decode(mac_b64)): + return None + data = json.loads(payload) + if int(data.get("exp", 0)) < time.time(): + return None + return int(data["uid"]) + except (ValueError, TypeError, KeyError, json.JSONDecodeError): + return None + + +# --- dependencies ----------------------------------------------------------- + +def current_user(request: Request, db: Session = Depends(get_db)) -> User: + token = request.cookies.get(SESSION_COOKIE, "") + uid = read_session(token) if token else None + if not uid: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Not signed in") + user = db.get(User, uid) + if not user or not user.is_active: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Account is not active") + return user + + +def require_admin(user: User = Depends(current_user)) -> User: + if not user.is_admin: + raise HTTPException(status.HTTP_403_FORBIDDEN, "Administrator access required") + return user + + +def optional_user(request: Request, db: Session = Depends(get_db)) -> Optional[User]: + token = request.cookies.get(SESSION_COOKIE, "") + uid = read_session(token) if token else None + return db.get(User, uid) if uid else None + + +# --- local accounts --------------------------------------------------------- + +def authenticate_local(db: Session, email: str, password: str) -> Optional[User]: + if not settings.auth_local_enabled: + return None + user = db.query(User).filter(User.email == email.strip().lower()).one_or_none() + if not user or not user.is_active or user.provider != AuthProvider.LOCAL: + return None + if not user.password_hash or not verify_password(password, user.password_hash): + return None + return user + + +def ensure_bootstrap_admin(db: Session) -> Optional[str]: + """Create the first admin so a fresh deployment is reachable. + + Only ever runs when the user table is empty, and only with a password + supplied through the environment - it never invents one. + """ + if db.query(User).count(): + return None + if not settings.bootstrap_admin_password: + return ("No users exist and CX_BOOTSTRAP_ADMIN_PASSWORD is unset. " + "Set it and restart, or sign in through SSO.") + db.add(User( + email=settings.bootstrap_admin_email.strip().lower(), + name="Bootstrap admin", + is_admin=True, + provider=AuthProvider.LOCAL, + password_hash=hash_password(settings.bootstrap_admin_password), + )) + db.commit() + return f"Created bootstrap admin {settings.bootstrap_admin_email}" + + +# --- OIDC / Authentik ------------------------------------------------------- + +class OIDCError(RuntimeError): + pass + + +_discovery_cache: dict[str, Any] = {} + + +async def oidc_discovery() -> dict[str, Any]: + """Fetch and cache the provider metadata.""" + if not settings.oidc_issuer: + raise OIDCError("CX_OIDC_ISSUER is not set.") + if _discovery_cache.get("_issuer") == settings.oidc_issuer: + return _discovery_cache + url = settings.oidc_issuer.rstrip("/") + "/.well-known/openid-configuration" + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.get(url) + if resp.status_code != 200: + raise OIDCError(f"OIDC discovery failed ({resp.status_code}) at {url}") + data = resp.json() + data["_issuer"] = settings.oidc_issuer + _discovery_cache.clear() + _discovery_cache.update(data) + return data + + +def oidc_state() -> str: + """A signed, short-lived value tying the callback to this browser.""" + payload = json.dumps({"n": secrets.token_urlsafe(16), "exp": int(time.time()) + 600}).encode() + return _sign(payload) + + +def oidc_state_valid(state: str) -> bool: + try: + body_b64, mac_b64 = state.split(".") + payload = base64.urlsafe_b64decode(body_b64) + expected = hmac.new(settings.secret_key.encode(), payload, hashlib.sha256).digest() + if not hmac.compare_digest(expected, base64.urlsafe_b64decode(mac_b64)): + return False + return int(json.loads(payload).get("exp", 0)) >= time.time() + except (ValueError, TypeError, KeyError, json.JSONDecodeError): + return False + + +async def oidc_exchange(code: str, redirect_uri: str) -> dict[str, Any]: + meta = await oidc_discovery() + async with httpx.AsyncClient(timeout=15) as client: + token_resp = await client.post(meta["token_endpoint"], data={ + "grant_type": "authorization_code", + "code": code, + "redirect_uri": redirect_uri, + "client_id": settings.oidc_client_id, + "client_secret": settings.oidc_client_secret, + }, headers={"Accept": "application/json"}) + if token_resp.status_code != 200: + raise OIDCError(f"Token exchange failed ({token_resp.status_code}): {token_resp.text[:200]}") + access = token_resp.json().get("access_token") + if not access: + raise OIDCError("Token response contained no access_token.") + + info_resp = await client.get(meta["userinfo_endpoint"], + headers={"Authorization": f"Bearer {access}"}) + if info_resp.status_code != 200: + raise OIDCError(f"userinfo failed ({info_resp.status_code}): {info_resp.text[:200]}") + return info_resp.json() + + +def upsert_oidc_user(db: Session, claims: dict[str, Any]) -> User: + """Find or create the local record for an SSO identity. + + Matching is on `sub` first so a changed email still lands on the same user; + an existing local account with the same address is adopted rather than + duplicated. + """ + sub = str(claims.get("sub") or "").strip() + email = str(claims.get("email") or "").strip().lower() + if not sub and not email: + raise OIDCError("SSO returned neither sub nor email.") + + groups = claims.get(settings.oidc_groups_claim) or [] + if isinstance(groups, str): + groups = [groups] + is_admin = settings.oidc_admin_group in {str(g) for g in groups} + + user = None + if sub: + user = db.query(User).filter(User.oidc_sub == sub).one_or_none() + if user is None and email: + user = db.query(User).filter(User.email == email).one_or_none() + + if user is None: + user = User(email=email or f"{sub}@sso.local", provider=AuthProvider.OIDC) + db.add(user) + + user.oidc_sub = sub or user.oidc_sub + user.provider = AuthProvider.OIDC + user.name = str(claims.get("name") or claims.get("preferred_username") or user.name or email) + if email: + user.email = email + user.is_admin = is_admin + user.is_active = True + user.last_login = dt.datetime.now(dt.timezone.utc) + if not user.signoff_name: + user.signoff_name = user.name + db.commit() + return user diff --git a/backend/app/config.py b/backend/app/config.py new file mode 100644 index 0000000..ca82cb1 --- /dev/null +++ b/backend/app/config.py @@ -0,0 +1,101 @@ +"""Environment-driven configuration. + +Everything deployment-specific comes from the environment so the same image runs +locally under compose and in Kubernetes with only a ConfigMap/Secret difference. +""" +from __future__ import annotations + +import os +from functools import lru_cache + + +def _bool(name: str, default: bool = False) -> bool: + return str(os.environ.get(name, str(default))).strip().lower() in {"1", "true", "yes", "on"} + + +def _int(name: str, default: int) -> int: + try: + return int(os.environ.get(name, default)) + except (TypeError, ValueError): + return default + + +class Settings: + # --- app --------------------------------------------------------------- + app_name = os.environ.get("CX_APP_NAME", "CX Triage") + base_url = os.environ.get("CX_BASE_URL", "http://localhost:8080") + secret_key = os.environ.get("CX_SECRET_KEY", "dev-only-change-me") + session_hours = _int("CX_SESSION_HOURS", 12) + static_dir = os.environ.get("CX_STATIC_DIR", "/app/static") + + # --- database ---------------------------------------------------------- + # sqlite for local/compose, postgres in the cluster. + database_url = os.environ.get("CX_DATABASE_URL", "sqlite:////data/cx-triage.db") + + # --- data sources ------------------------------------------------------ + prometheus_base = os.environ.get("CX_PROMETHEUS_BASE", "http://10.11.254.250:9090") + prometheus_relay = os.environ.get("CX_PROMETHEUS_RELAY", "") + cx_tools_path = os.environ.get("CX_TOOLS_PATH", "") + + # --- auth -------------------------------------------------------------- + # Local accounts are for development and for a cluster without SSO yet. + # When CX_OIDC_ENABLED is on, Authentik becomes the source of truth. + auth_local_enabled = _bool("CX_AUTH_LOCAL_ENABLED", True) + bootstrap_admin_email = os.environ.get("CX_BOOTSTRAP_ADMIN_EMAIL", "admin@localhost") + bootstrap_admin_password = os.environ.get("CX_BOOTSTRAP_ADMIN_PASSWORD", "") + + oidc_enabled = _bool("CX_OIDC_ENABLED", False) + oidc_issuer = os.environ.get("CX_OIDC_ISSUER", "") # e.g. https://sso/application/o/cx-triage/ + oidc_client_id = os.environ.get("CX_OIDC_CLIENT_ID", "") + oidc_client_secret = os.environ.get("CX_OIDC_CLIENT_SECRET", "") + oidc_scopes = os.environ.get("CX_OIDC_SCOPES", "openid email profile") + oidc_admin_group = os.environ.get("CX_OIDC_ADMIN_GROUP", "cx-triage-admins") + oidc_groups_claim = os.environ.get("CX_OIDC_GROUPS_CLAIM", "groups") + + # --- feature flags ----------------------------------------------------- + # Sending must be switched on deliberately; a demo instance cannot email. + feature_send_enabled = _bool("CX_FEATURE_SEND_ENABLED", False) + feature_zendesk = _bool("CX_FEATURE_ZENDESK", False) + feature_jira = _bool("CX_FEATURE_JIRA", False) + feature_linkage_scan = _bool("CX_FEATURE_LINKAGE_SCAN", True) + send_daily_cap = _int("CX_SEND_DAILY_CAP", 25) + + # --- integrations ------------------------------------------------------ + zendesk_subdomain = os.environ.get("CX_ZENDESK_SUBDOMAIN", "") + zendesk_email = os.environ.get("CX_ZENDESK_EMAIL", "") + zendesk_token = os.environ.get("CX_ZENDESK_TOKEN", "") + zendesk_default_public = _bool("CX_ZENDESK_PUBLIC_REPLY", True) + + jira_base = os.environ.get("CX_JIRA_BASE", "") + jira_email = os.environ.get("CX_JIRA_EMAIL", "") + jira_token = os.environ.get("CX_JIRA_TOKEN", "") + jira_project = os.environ.get("CX_JIRA_PROJECT", "INFRA") + jira_issue_type = os.environ.get("CX_JIRA_ISSUE_TYPE", "Task") + + @property + def zendesk_ready(self) -> bool: + return bool(self.feature_zendesk and self.zendesk_subdomain + and self.zendesk_email and self.zendesk_token) + + @property + def jira_ready(self) -> bool: + return bool(self.feature_jira and self.jira_base and self.jira_email + and self.jira_token and self.jira_project) + + def public_flags(self) -> dict: + """What the frontend is allowed to know - never secrets.""" + return { + "app_name": self.app_name, + "oidc_enabled": self.oidc_enabled, + "local_login": self.auth_local_enabled, + "zendesk_ready": self.zendesk_ready, + "jira_ready": self.jira_ready, + "send_enabled": self.feature_send_enabled, + "linkage_scan": self.feature_linkage_scan, + "jira_project": self.jira_project if self.jira_ready else "", + } + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/backend/app/db.py b/backend/app/db.py new file mode 100644 index 0000000..042f6bb --- /dev/null +++ b/backend/app/db.py @@ -0,0 +1,29 @@ +"""Database engine and session handling.""" +from __future__ import annotations + +from collections.abc import Iterator + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker + +from .config import get_settings + +settings = get_settings() + +_connect_args = {"check_same_thread": False} if settings.database_url.startswith("sqlite") else {} +engine = create_engine(settings.database_url, pool_pre_ping=True, connect_args=_connect_args) +SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False) + + +def get_db() -> Iterator[Session]: + db = SessionLocal() + try: + yield db + finally: + db.close() + + +def init_db() -> None: + from . import models # noqa: F401 (registers the tables) + + models.Base.metadata.create_all(engine) diff --git a/backend/app/delivery.py b/backend/app/delivery.py new file mode 100644 index 0000000..ecfab81 --- /dev/null +++ b/backend/app/delivery.py @@ -0,0 +1,174 @@ +"""Outbound delivery to Zendesk and Jira. + +Three independent gates have to be open before anything leaves this process: + + 1. the integration is configured (subdomain/email/token present) + 2. its feature flag is on - CX_FEATURE_ZENDESK / CX_FEATURE_JIRA + 3. sending is globally enabled - CX_FEATURE_SEND_ENABLED + +A demo or staging instance simply leaves the third off, and then no combination +of clicks can email a customer. Every send is recorded as a case event before it +is attempted, so an audit trail exists even when the call fails. +""" +from __future__ import annotations + +import datetime as dt +from typing import Any, Optional + +import httpx +from sqlalchemy.orm import Session + +from .config import get_settings +from .models import Case, CaseEvent, CaseStatus, User +from .services import add_event, set_status + +settings = get_settings() + + +class DeliveryError(RuntimeError): + pass + + +def _guard(kind: str) -> None: + if not settings.feature_send_enabled: + raise DeliveryError( + "Sending is disabled on this instance (CX_FEATURE_SEND_ENABLED is off). " + "The payload is ready but nothing will leave the server." + ) + if kind == "zendesk" and not settings.zendesk_ready: + raise DeliveryError("Zendesk is not configured. Set CX_FEATURE_ZENDESK plus " + "CX_ZENDESK_SUBDOMAIN, CX_ZENDESK_EMAIL and CX_ZENDESK_TOKEN.") + if kind == "jira" and not settings.jira_ready: + raise DeliveryError("Jira is not configured. Set CX_FEATURE_JIRA plus " + "CX_JIRA_BASE, CX_JIRA_EMAIL, CX_JIRA_TOKEN and CX_JIRA_PROJECT.") + + +def sends_today(db: Session) -> int: + since = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1) + return (db.query(CaseEvent) + .filter(CaseEvent.action.in_(["zendesk_sent", "jira_created"])) + .filter(CaseEvent.created_at >= since).count()) + + +def _check_cap(db: Session) -> None: + used = sends_today(db) + if used >= settings.send_daily_cap: + raise DeliveryError( + f"Daily send cap reached ({used}/{settings.send_daily_cap}). Raise CX_SEND_DAILY_CAP " + "if this is deliberate - the cap exists so a loop cannot mail every customer." + ) + + +# --- Zendesk ---------------------------------------------------------------- + +async def send_zendesk(db: Session, case: Case, actor: User, *, to: str, subject: str, + body: str, priority: str = "normal", + tags: Optional[list[str]] = None, public: Optional[bool] = None) -> dict[str, Any]: + _guard("zendesk") + _check_cap(db) + if not to.strip(): + raise DeliveryError("No recipient address.") + + base = f"https://{settings.zendesk_subdomain}.zendesk.com/api/v2" + auth = (f"{settings.zendesk_email}/token", settings.zendesk_token) + external_id = f"cx-triage-{case.fingerprint}" + + async with httpx.AsyncClient(timeout=30) as client: + # Search first so a re-diagnosed alert comments on the existing ticket + # instead of opening a second one for the same customer. + found = await client.get(f"{base}/search.json", + params={"query": f'type:ticket external_id:"{external_id}"'}, auth=auth) + existing = None + if found.status_code == 200: + results = found.json().get("results") or [] + existing = results[0] if results else None + + comment = {"body": body, "public": settings.zendesk_default_public if public is None else public} + if existing: + resp = await client.put(f"{base}/tickets/{existing['id']}.json", + json={"ticket": {"comment": comment}}, auth=auth) + action = "updated" + else: + payload = {"ticket": { + "subject": subject, + "comment": comment, + "requester": {"name": to.split("@")[0], "email": to}, + "priority": priority, + "type": "incident", + "tags": tags or ["cx-triage", f"alert-{case.kind}"], + "external_id": external_id, + }} + resp = await client.post(f"{base}/tickets.json", json=payload, auth=auth) + action = "created" + + if resp.status_code not in (200, 201): + add_event(db, case, actor, "zendesk_failed", f"HTTP {resp.status_code}: {resp.text[:300]}") + db.commit() + raise DeliveryError(f"Zendesk returned {resp.status_code}: {resp.text[:300]}") + + ticket = resp.json().get("ticket") or {} + ticket_id = str(ticket.get("id") or (existing or {}).get("id") or "") + url = f"https://{settings.zendesk_subdomain}.zendesk.com/agent/tickets/{ticket_id}" + + case.zendesk_ticket_id = ticket_id + case.zendesk_ticket_url = url + add_event(db, case, actor, "zendesk_sent", + f"Ticket {ticket_id} {action} for {to}", + {"ticket_id": ticket_id, "to": to, "subject": subject, "action": action}) + if case.status in (CaseStatus.NEW, CaseStatus.INVESTIGATING): + set_status(db, case, CaseStatus.CUSTOMER_CONTACTED, actor, f"Zendesk ticket {ticket_id}") + db.commit() + return {"ok": True, "ticket_id": ticket_id, "url": url, "action": action} + + +# --- Jira ------------------------------------------------------------------- + +async def create_jira(db: Session, case: Case, actor: User, *, summary: str, description: str, + project: str = "", issue_type: str = "", + labels: Optional[list[str]] = None) -> dict[str, Any]: + _guard("jira") + _check_cap(db) + + base = settings.jira_base.rstrip("/") + auth = (settings.jira_email, settings.jira_token) + label = f"cx-triage-{case.fingerprint}" + all_labels = sorted(set((labels or []) + ["cx-triage", label])) + + async with httpx.AsyncClient(timeout=30) as client: + found = await client.get(f"{base}/rest/api/3/search", + params={"jql": f'labels = "{label}"', "maxResults": 1}, auth=auth) + if found.status_code == 200 and (found.json().get("issues") or []): + issue = found.json()["issues"][0] + key = issue["key"] + url = f"{base}/browse/{key}" + case.jira_issue_key, case.jira_issue_url = key, url + add_event(db, case, actor, "jira_exists", f"Issue {key} already exists for this case") + db.commit() + return {"ok": True, "key": key, "url": url, "action": "existing"} + + payload = {"fields": { + "project": {"key": project or settings.jira_project}, + "summary": summary[:250], + "issuetype": {"name": issue_type or settings.jira_issue_type}, + "labels": all_labels, + "description": { + "type": "doc", "version": 1, + "content": [{"type": "paragraph", + "content": [{"type": "text", "text": description[:30000]}]}], + }, + }} + resp = await client.post(f"{base}/rest/api/3/issue", json=payload, auth=auth) + + if resp.status_code not in (200, 201): + add_event(db, case, actor, "jira_failed", f"HTTP {resp.status_code}: {resp.text[:300]}") + db.commit() + raise DeliveryError(f"Jira returned {resp.status_code}: {resp.text[:300]}") + + key = resp.json().get("key", "") + url = f"{base}/browse/{key}" + case.jira_issue_key, case.jira_issue_url = key, url + add_event(db, case, actor, "jira_created", f"Issue {key} created", {"key": key, "summary": summary}) + if case.status in (CaseStatus.NEW, CaseStatus.INVESTIGATING): + set_status(db, case, CaseStatus.ESCALATED_INFRA, actor, f"Jira {key}") + db.commit() + return {"ok": True, "key": key, "url": url, "action": "created"} diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000..5688677 --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,66 @@ +"""FastAPI application entry point.""" +from __future__ import annotations + +import os +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from fastapi.responses import FileResponse, JSONResponse +from fastapi.staticfiles import StaticFiles + +from . import VERSION +from .auth import ensure_bootstrap_admin +from .config import get_settings +from .db import SessionLocal, init_db +from .routers import (actions_router, alerts_router, auth_router, cases_router, + linkage_router, settings_router) + +settings = get_settings() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + init_db() + db = SessionLocal() + try: + message = ensure_bootstrap_admin(db) + if message: + print(f"[auth] {message}") + finally: + db.close() + print(f"[startup] {settings.app_name} {VERSION}") + print(f"[startup] prometheus: {settings.prometheus_base}") + print(f"[startup] sso: {'on' if settings.oidc_enabled else 'off'} | " + f"zendesk: {'ready' if settings.zendesk_ready else 'off'} | " + f"jira: {'ready' if settings.jira_ready else 'off'} | " + f"sending: {'ENABLED' if settings.feature_send_enabled else 'disabled'}") + alerts_router.engine.warm() + yield + + +app = FastAPI(title=settings.app_name, version=VERSION, lifespan=lifespan) + +for module in (auth_router, alerts_router, cases_router, settings_router, + linkage_router, actions_router): + app.include_router(module.router) + + +@app.get("/api/health") +def health(): + return {"status": "ok", "version": VERSION, "config": settings.public_flags()} + + +# The built React bundle is copied into the image; in development Vite serves it +# instead and proxies /api here, so a missing directory is not an error. +if os.path.isdir(settings.static_dir): + app.mount("/assets", StaticFiles(directory=os.path.join(settings.static_dir, "assets")), + name="assets") + + @app.get("/{full_path:path}") + def spa(full_path: str): + if full_path.startswith("api/"): + return JSONResponse({"detail": "Not found"}, status_code=404) + index = os.path.join(settings.static_dir, "index.html") + if os.path.isfile(index): + return FileResponse(index) + return JSONResponse({"detail": "Frontend not built"}, status_code=404) diff --git a/backend/app/models.py b/backend/app/models.py new file mode 100644 index 0000000..6ef347e --- /dev/null +++ b/backend/app/models.py @@ -0,0 +1,208 @@ +"""Persistent state: who is working what, and what has been done about it. + +The alert queue itself stays stateless - it is recomputed from Prometheus every +minute. What is worth persisting is the human layer on top: which alerts someone +has picked up, what was done, and the audit trail behind it. Cases are keyed by +the alert fingerprint so an alert that stops and re-fires lands back on the same +case rather than losing its history. +""" +from __future__ import annotations + +import datetime as dt +import enum +from typing import Any + +from sqlalchemy import (JSON, Boolean, DateTime, Enum, ForeignKey, Index, Integer, + String, Text, UniqueConstraint) +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + + +class Base(DeclarativeBase): + pass + + +def _now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +class CaseStatus(str, enum.Enum): + """Where a piece of work has got to.""" + + NEW = "new" # seen, nobody has touched it + INVESTIGATING = "investigating" # someone has picked it up + CUSTOMER_CONTACTED = "customer_contacted" + ESCALATED_INFRA = "escalated_infra" + WAITING_CUSTOMER = "waiting_customer" + WAITING_INFRA = "waiting_infra" + REMEDIATED = "remediated" # fixed, waiting for the alert to clear + RESOLVED = "resolved" + WONT_FIX = "wont_fix" # deliberate no-action + FALSE_POSITIVE = "false_positive" # the alert itself was wrong + + +OPEN_STATUSES = { + CaseStatus.NEW, CaseStatus.INVESTIGATING, CaseStatus.CUSTOMER_CONTACTED, + CaseStatus.ESCALATED_INFRA, CaseStatus.WAITING_CUSTOMER, CaseStatus.WAITING_INFRA, + CaseStatus.REMEDIATED, +} + + +class AuthProvider(str, enum.Enum): + LOCAL = "local" + OIDC = "oidc" + + +class User(Base): + __tablename__ = "users" + + id: Mapped[int] = mapped_column(primary_key=True) + email: Mapped[str] = mapped_column(String(320), unique=True, index=True) + name: Mapped[str] = mapped_column(String(200), default="") + is_admin: Mapped[bool] = mapped_column(Boolean, default=False) + is_active: Mapped[bool] = mapped_column(Boolean, default=True) + provider: Mapped[AuthProvider] = mapped_column(Enum(AuthProvider), default=AuthProvider.LOCAL) + # Local accounts only; OIDC users never have one. + password_hash: Mapped[str] = mapped_column(String(255), default="") + # Stable Authentik subject, so a rename or email change keeps the same user. + oidc_sub: Mapped[str] = mapped_column(String(255), default="", index=True) + signoff_name: Mapped[str] = mapped_column(String(200), default="") + created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now) + last_login: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + def to_json(self) -> dict[str, Any]: + return { + "id": self.id, "email": self.email, "name": self.name or self.email, + "is_admin": self.is_admin, "provider": self.provider.value, + "signoff_name": self.signoff_name or self.name, + } + + +class Case(Base): + """One tracked alert, keyed by its fingerprint.""" + + __tablename__ = "cases" + __table_args__ = (Index("ix_cases_status_seen", "status", "last_seen_at"),) + + id: Mapped[int] = mapped_column(primary_key=True) + fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True) + + kind: Mapped[str] = mapped_column(String(40), index=True) + title: Mapped[str] = mapped_column(String(300), default="") + subject: Mapped[str] = mapped_column(String(300), default="") # VM name, host or IP + openstack_id: Mapped[str] = mapped_column(String(64), default="", index=True) + instance_name: Mapped[str] = mapped_column(String(200), default="", index=True) + host: Mapped[str] = mapped_column(String(120), default="", index=True) + region: Mapped[str] = mapped_column(String(16), default="") + org_id: Mapped[str] = mapped_column(String(32), default="", index=True) + org_name: Mapped[str] = mapped_column(String(300), default="") + + status: Mapped[CaseStatus] = mapped_column(Enum(CaseStatus), default=CaseStatus.NEW, index=True) + assignee_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True) + assignee: Mapped[User | None] = relationship(lazy="joined") + + # Outbound references, so a case shows what already exists elsewhere. + zendesk_ticket_id: Mapped[str] = mapped_column(String(40), default="") + zendesk_ticket_url: Mapped[str] = mapped_column(String(500), default="") + jira_issue_key: Mapped[str] = mapped_column(String(40), default="") + jira_issue_url: Mapped[str] = mapped_column(String(500), default="") + + notes: Mapped[str] = mapped_column(Text, default="") + snooze_until: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + first_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now) + last_seen_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now) + closed_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + # How many separate times this alert has come back after being closed. + reopen_count: Mapped[int] = mapped_column(Integer, default=0) + + events: Mapped[list["CaseEvent"]] = relationship( + back_populates="case", cascade="all, delete-orphan", order_by="CaseEvent.created_at.desc()") + + @property + def is_open(self) -> bool: + return self.status in OPEN_STATUSES + + def to_json(self, with_events: bool = False) -> dict[str, Any]: + data = { + "id": self.id, "fingerprint": self.fingerprint, "kind": self.kind, + "title": self.title, "subject": self.subject, + "openstack_id": self.openstack_id, "instance_name": self.instance_name, + "host": self.host, "region": self.region, + "org_id": self.org_id, "org_name": self.org_name, + "status": self.status.value, "is_open": self.is_open, + "assignee": self.assignee.to_json() if self.assignee else None, + "zendesk_ticket_id": self.zendesk_ticket_id, + "zendesk_ticket_url": self.zendesk_ticket_url, + "jira_issue_key": self.jira_issue_key, + "jira_issue_url": self.jira_issue_url, + "notes": self.notes, + "snooze_until": self.snooze_until.isoformat() if self.snooze_until else None, + "first_seen_at": self.first_seen_at.isoformat() if self.first_seen_at else None, + "last_seen_at": self.last_seen_at.isoformat() if self.last_seen_at else None, + "closed_at": self.closed_at.isoformat() if self.closed_at else None, + "reopen_count": self.reopen_count, + } + if with_events: + data["events"] = [e.to_json() for e in self.events] + return data + + +class CaseEvent(Base): + """Append-only history. Nothing here is ever edited or deleted.""" + + __tablename__ = "case_events" + + id: Mapped[int] = mapped_column(primary_key=True) + case_id: Mapped[int] = mapped_column(ForeignKey("cases.id"), index=True) + case: Mapped[Case] = relationship(back_populates="events") + + actor_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True) + actor: Mapped[User | None] = relationship(lazy="joined") + actor_label: Mapped[str] = mapped_column(String(200), default="") # survives user deletion + + action: Mapped[str] = mapped_column(String(60)) # status_changed | zendesk_sent | note | ... + detail: Mapped[str] = mapped_column(Text, default="") + payload: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True) + + def to_json(self) -> dict[str, Any]: + return { + "id": self.id, "action": self.action, "detail": self.detail, + "actor": self.actor_label or (self.actor.email if self.actor else "system"), + "created_at": self.created_at.isoformat() if self.created_at else None, + "payload": self.payload, + } + + +class SuppressionRule(Base): + """Alerts the team has decided not to see, with the reason recorded.""" + + __tablename__ = "suppression_rules" + __table_args__ = (UniqueConstraint("name", name="uq_rule_name"),) + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(200)) + reason: Mapped[str] = mapped_column(Text, default="") + enabled: Mapped[bool] = mapped_column(Boolean, default=True) + # {"kind": ["error"], "organization": ["modal"]} - all keys must match. + conditions: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + created_by: Mapped[str] = mapped_column(String(200), default="") + created_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now) + + def to_json(self) -> dict[str, Any]: + return { + "id": str(self.id), "name": self.name, "reason": self.reason, + "enabled": self.enabled, "conditions": self.conditions or {}, + "created_by": self.created_by, + "created": self.created_at.strftime("%Y-%m-%d") if self.created_at else "", + } + + +class AppSetting(Base): + """Small key/value bag for things an admin can change at runtime.""" + + __tablename__ = "app_settings" + + key: Mapped[str] = mapped_column(String(80), primary_key=True) + value: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) + updated_at: Mapped[dt.datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now) diff --git a/backend/app/routers/__init__.py b/backend/app/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/routers/actions_router.py b/backend/app/routers/actions_router.py new file mode 100644 index 0000000..e311d46 --- /dev/null +++ b/backend/app/routers/actions_router.py @@ -0,0 +1,77 @@ +"""Outbound actions: contact the customer, escalate to Infrastructure.""" +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from ..auth import current_user +from ..config import get_settings +from ..db import get_db +from ..delivery import DeliveryError, create_jira, send_zendesk, sends_today +from ..models import Case, User + +router = APIRouter(prefix="/api/actions", tags=["actions"]) +settings = get_settings() + + +class ZendeskBody(BaseModel): + fingerprint: str + to: str + subject: str + body: str + priority: str = "normal" + tags: list[str] = [] + public: bool | None = None + + +class JiraBody(BaseModel): + fingerprint: str + summary: str + description: str + project: str = "" + issue_type: str = "" + labels: list[str] = [] + + +def _case(db: Session, fingerprint: str) -> Case: + found = db.query(Case).filter(Case.fingerprint == fingerprint).one_or_none() + if not found: + raise HTTPException(status.HTTP_404_NOT_FOUND, + "Open the case first - nothing is tracked for that alert yet.") + return found + + +@router.get("/status") +def action_status(db: Session = Depends(get_db), user: User = Depends(current_user)): + return { + "zendesk_ready": settings.zendesk_ready, + "jira_ready": settings.jira_ready, + "send_enabled": settings.feature_send_enabled, + "sends_today": sends_today(db), + "daily_cap": settings.send_daily_cap, + } + + +@router.post("/zendesk") +async def zendesk(body: ZendeskBody, db: Session = Depends(get_db), + user: User = Depends(current_user)): + case = _case(db, body.fingerprint) + try: + return await send_zendesk(db, case, user, to=body.to, subject=body.subject, + body=body.body, priority=body.priority, + tags=body.tags or None, public=body.public) + except DeliveryError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc + + +@router.post("/jira") +async def jira(body: JiraBody, db: Session = Depends(get_db), + user: User = Depends(current_user)): + case = _case(db, body.fingerprint) + try: + return await create_jira(db, case, user, summary=body.summary, + description=body.description, project=body.project, + issue_type=body.issue_type, labels=body.labels or None) + except DeliveryError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc diff --git a/backend/app/routers/alerts_router.py b/backend/app/routers/alerts_router.py new file mode 100644 index 0000000..c6a5734 --- /dev/null +++ b/backend/app/routers/alerts_router.py @@ -0,0 +1,88 @@ +"""The alert queue and per-alert diagnosis.""" +from __future__ import annotations + +import threading +import time +import traceback +import uuid +from concurrent.futures import ThreadPoolExecutor +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from triagelib import runbooks + +from ..auth import current_user +from ..db import SessionLocal, get_db +from ..models import User +from ..services import Engine, RuleAdapter, get_or_create_case + +router = APIRouter(prefix="/api", tags=["alerts"]) +engine = Engine() + +_jobs: dict[str, dict[str, Any]] = {} +_jobs_lock = threading.Lock() +_pool = ThreadPoolExecutor(max_workers=3, thread_name_prefix="triage") +JOB_TTL = 30 * 60 + + +class TriageBody(BaseModel): + fingerprint: str + force: bool = False + + +@router.get("/alerts") +def alert_queue(force: bool = False, db: Session = Depends(get_db), + user: User = Depends(current_user)): + return engine.queue(db, force=force) + + +@router.post("/triage") +def start_triage(body: TriageBody, db: Session = Depends(get_db), + user: User = Depends(current_user)): + alert = engine.find_alert(db, body.fingerprint) + if alert is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, + "That alert is no longer firing. Refresh the queue.") + case = get_or_create_case(db, alert, user) + job_id = uuid.uuid4().hex[:12] + with _jobs_lock: + _reap() + _jobs[job_id] = {"id": job_id, "state": "running", "created": time.time(), + "result": None, "error": ""} + _pool.submit(_run, job_id, alert, body.force) + return {"job_id": job_id, "alert": alert.to_json(), "case": case.to_json(with_events=True)} + + +@router.get("/jobs/{job_id}") +def job(job_id: str, user: User = Depends(current_user)): + with _jobs_lock: + found = _jobs.get(job_id) + if not found: + raise HTTPException(status.HTTP_404_NOT_FOUND, "Unknown job") + return found + + +def _run(job_id: str, alert: Any, force: bool) -> None: + started = time.monotonic() + db = SessionLocal() + try: + diagnosis = runbooks.diagnose(alert, engine.prom, engine.snapshot.get(), force, + RuleAdapter(db)) + payload = diagnosis.to_json() + payload["elapsed_seconds"] = round(time.monotonic() - started, 1) + with _jobs_lock: + _jobs[job_id].update({"state": "done", "result": payload}) + except Exception: + with _jobs_lock: + _jobs[job_id].update({"state": "error", "error": traceback.format_exc(limit=4)}) + finally: + db.close() + + +def _reap() -> None: + cutoff = time.time() - JOB_TTL + for key in [k for k, v in _jobs.items() if v["created"] < cutoff]: + _jobs.pop(key, None) diff --git a/backend/app/routers/auth_router.py b/backend/app/routers/auth_router.py new file mode 100644 index 0000000..906e895 --- /dev/null +++ b/backend/app/routers/auth_router.py @@ -0,0 +1,99 @@ +"""Sign-in: local accounts and the Authentik OIDC round trip.""" +from __future__ import annotations + +import datetime as dt + +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status +from fastapi.responses import RedirectResponse +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from .. import auth as auth_mod +from ..config import get_settings +from ..db import get_db +from ..models import User + +router = APIRouter(prefix="/api/auth", tags=["auth"]) +settings = get_settings() + + +class LoginBody(BaseModel): + email: str + password: str + + +def _set_cookie(response: Response, user: User) -> None: + response.set_cookie( + auth_mod.SESSION_COOKIE, auth_mod.issue_session(user.id), + max_age=settings.session_hours * 3600, httponly=True, samesite="lax", + secure=settings.base_url.startswith("https://"), path="/", + ) + + +@router.get("/me") +def me(user: User | None = Depends(auth_mod.optional_user)): + return {"user": user.to_json() if user else None, "config": settings.public_flags()} + + +@router.post("/login") +def login(body: LoginBody, response: Response, db: Session = Depends(get_db)): + if not settings.auth_local_enabled: + raise HTTPException(status.HTTP_403_FORBIDDEN, "Local login is disabled; use SSO.") + user = auth_mod.authenticate_local(db, body.email, body.password) + if not user: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Incorrect email or password") + user.last_login = dt.datetime.now(dt.timezone.utc) + db.commit() + _set_cookie(response, user) + return {"user": user.to_json()} + + +@router.post("/logout") +def logout(response: Response): + response.delete_cookie(auth_mod.SESSION_COOKIE, path="/") + return {"ok": True} + + +@router.get("/oidc/start") +async def oidc_start(request: Request): + if not settings.oidc_enabled: + raise HTTPException(status.HTTP_404_NOT_FOUND, "SSO is not enabled") + try: + meta = await auth_mod.oidc_discovery() + except auth_mod.OIDCError as exc: + raise HTTPException(status.HTTP_502_BAD_GATEWAY, str(exc)) from exc + + redirect_uri = f"{settings.base_url.rstrip('/')}/api/auth/oidc/callback" + state = auth_mod.oidc_state() + url = (f"{meta['authorization_endpoint']}?response_type=code" + f"&client_id={settings.oidc_client_id}" + f"&redirect_uri={redirect_uri}" + f"&scope={settings.oidc_scopes.replace(' ', '%20')}" + f"&state={state}") + response = RedirectResponse(url, status_code=302) + response.set_cookie("cx_oidc_state", state, max_age=600, httponly=True, samesite="lax", path="/") + return response + + +@router.get("/oidc/callback") +async def oidc_callback(request: Request, code: str = "", state: str = "", + db: Session = Depends(get_db)): + if not settings.oidc_enabled: + raise HTTPException(status.HTTP_404_NOT_FOUND, "SSO is not enabled") + # The state must match the cookie we set *and* still verify - one guards + # against a swapped browser, the other against a forged value. + if not code or not state or state != request.cookies.get("cx_oidc_state") \ + or not auth_mod.oidc_state_valid(state): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid or expired SSO state") + + redirect_uri = f"{settings.base_url.rstrip('/')}/api/auth/oidc/callback" + try: + claims = await auth_mod.oidc_exchange(code, redirect_uri) + user = auth_mod.upsert_oidc_user(db, claims) + except auth_mod.OIDCError as exc: + raise HTTPException(status.HTTP_502_BAD_GATEWAY, str(exc)) from exc + + response = RedirectResponse("/", status_code=302) + _set_cookie(response, user) + response.delete_cookie("cx_oidc_state", path="/") + return response diff --git a/backend/app/routers/cases_router.py b/backend/app/routers/cases_router.py new file mode 100644 index 0000000..6222201 --- /dev/null +++ b/backend/app/routers/cases_router.py @@ -0,0 +1,99 @@ +"""Case state: the human layer over the alert queue.""" +from __future__ import annotations + +import datetime as dt + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from ..auth import current_user +from ..db import get_db +from ..models import Case, CaseStatus, User +from ..services import add_event, set_status + +router = APIRouter(prefix="/api/cases", tags=["cases"]) + + +class StatusBody(BaseModel): + status: str + note: str = "" + + +class NoteBody(BaseModel): + note: str + + +class SnoozeBody(BaseModel): + hours: int = 24 + note: str = "" + + +def _case(db: Session, fingerprint: str) -> Case: + found = db.query(Case).filter(Case.fingerprint == fingerprint).one_or_none() + if not found: + raise HTTPException(status.HTTP_404_NOT_FOUND, "No case for that alert yet") + return found + + +@router.get("") +def list_cases(open_only: bool = True, limit: int = 200, db: Session = Depends(get_db), + user: User = Depends(current_user)): + query = db.query(Case).order_by(Case.last_seen_at.desc()) + rows = [c for c in query.limit(max(1, min(limit, 1000))).all() + if (c.is_open or not open_only)] + return {"cases": [c.to_json() for c in rows], "statuses": [s.value for s in CaseStatus]} + + +@router.get("/{fingerprint}") +def get_case(fingerprint: str, db: Session = Depends(get_db), user: User = Depends(current_user)): + return _case(db, fingerprint).to_json(with_events=True) + + +@router.post("/{fingerprint}/status") +def change_status(fingerprint: str, body: StatusBody, db: Session = Depends(get_db), + user: User = Depends(current_user)): + try: + new_status = CaseStatus(body.status) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, + f"Unknown status '{body.status}'") from exc + case = set_status(db, _case(db, fingerprint), new_status, user, body.note) + return case.to_json(with_events=True) + + +@router.post("/{fingerprint}/assign") +def assign(fingerprint: str, db: Session = Depends(get_db), user: User = Depends(current_user)): + case = _case(db, fingerprint) + # Set the relationship, not just the id: the response is serialised from + # this same object and a bare id leaves `assignee` null in the payload. + case.assignee = user + if case.status == CaseStatus.NEW: + case.status = CaseStatus.INVESTIGATING + add_event(db, case, user, "assigned", f"Picked up by {user.email}") + db.commit() + db.refresh(case) + return case.to_json(with_events=True) + + +@router.post("/{fingerprint}/note") +def add_note(fingerprint: str, body: NoteBody, db: Session = Depends(get_db), + user: User = Depends(current_user)): + if not body.note.strip(): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Empty note") + case = _case(db, fingerprint) + case.notes = (case.notes + "\n" if case.notes else "") + body.note.strip() + add_event(db, case, user, "note", body.note.strip()) + db.commit() + return case.to_json(with_events=True) + + +@router.post("/{fingerprint}/snooze") +def snooze(fingerprint: str, body: SnoozeBody, db: Session = Depends(get_db), + user: User = Depends(current_user)): + case = _case(db, fingerprint) + until = dt.datetime.now(dt.timezone.utc) + dt.timedelta(hours=max(1, body.hours)) + case.snooze_until = until + add_event(db, case, user, "snoozed", f"Snoozed for {body.hours}h. {body.note}".strip()) + db.commit() + return case.to_json(with_events=True) diff --git a/backend/app/routers/linkage_router.py b/backend/app/routers/linkage_router.py new file mode 100644 index 0000000..4969ec5 --- /dev/null +++ b/backend/app/routers/linkage_router.py @@ -0,0 +1,42 @@ +"""Linkage scan endpoints.""" +from __future__ import annotations + +import threading + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel + +from triagelib import linkage as linkage_mod + +from ..auth import current_user +from ..config import get_settings +from ..models import User +from ..routers.alerts_router import engine + +router = APIRouter(prefix="/api/linkage", tags=["linkage"]) +settings = get_settings() + + +class EnrichBody(BaseModel): + region: str + openstack_id: str + + +@router.get("") +def scan_state(user: User = Depends(current_user)): + return engine.scan.to_json() + + +@router.post("/scan") +def start_scan(user: User = Depends(current_user)): + if not settings.feature_linkage_scan: + raise HTTPException(status.HTTP_403_FORBIDDEN, "The linkage scan is disabled on this instance.") + if engine.scan.state == "running": + return {"started": False, "reason": "already running"} + threading.Thread(target=engine.scan.run, args=(engine.snapshot.get(),), daemon=True).start() + return {"started": True} + + +@router.post("/enrich") +def enrich(body: EnrichBody, user: User = Depends(current_user)): + return linkage_mod.enrich(body.region, body.openstack_id) diff --git a/backend/app/routers/settings_router.py b/backend/app/routers/settings_router.py new file mode 100644 index 0000000..ca79f6b --- /dev/null +++ b/backend/app/routers/settings_router.py @@ -0,0 +1,103 @@ +"""Suppression rules and per-instance preferences.""" +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from triagelib import settings as legacy + +from ..auth import current_user, require_admin +from ..config import get_settings +from ..db import get_db +from ..models import AppSetting, SuppressionRule, User +from ..routers.alerts_router import engine + +router = APIRouter(prefix="/api/settings", tags=["settings"]) +app_settings = get_settings() + + +class RuleBody(BaseModel): + id: str | None = None + name: str + reason: str = "" + enabled: bool = True + conditions: dict[str, Any] = {} + + +class GeneralBody(BaseModel): + agent_name: str | None = None + chronic_days: int | None = None + + +@router.get("") +def read_settings(db: Session = Depends(get_db), user: User = Depends(current_user)): + row = db.get(AppSetting, "general") + general = (row.value if row else {}) or {} + return { + "rules": [r.to_json() for r in db.query(SuppressionRule).order_by(SuppressionRule.id).all()], + "conditions": legacy.CONDITIONS, + "agent_name": general.get("agent_name") or "", + "chronic_days": general.get("chronic_days") or 3, + "config": app_settings.public_flags(), + } + + +@router.post("/general") +def save_general(body: GeneralBody, db: Session = Depends(get_db), + user: User = Depends(current_user)): + row = db.get(AppSetting, "general") or AppSetting(key="general", value={}) + value = dict(row.value or {}) + if body.agent_name is not None: + value["agent_name"] = body.agent_name.strip() + if body.chronic_days is not None: + value["chronic_days"] = max(1, int(body.chronic_days)) + row.value = value + db.merge(row) + db.commit() + return read_settings(db, user) + + +@router.post("/rules") +def save_rule(body: RuleBody, db: Session = Depends(get_db), user: User = Depends(require_admin)): + conditions = {k: v for k, v in (body.conditions or {}).items() if k in legacy.CONDITIONS and v} + if not conditions: + raise HTTPException(status.HTTP_400_BAD_REQUEST, + "A rule needs at least one condition, otherwise it would hide everything.") + rule = db.get(SuppressionRule, int(body.id)) if (body.id or "").isdigit() else None + if rule is None: + rule = SuppressionRule(created_by=user.email) + db.add(rule) + rule.name = body.name.strip() or "Untitled rule" + rule.reason = body.reason.strip() + rule.enabled = body.enabled + rule.conditions = conditions + db.commit() + return read_settings(db, user) + + +@router.delete("/rules/{rule_id}") +def delete_rule(rule_id: int, db: Session = Depends(get_db), user: User = Depends(require_admin)): + rule = db.get(SuppressionRule, rule_id) + if rule: + db.delete(rule) + db.commit() + return read_settings(db, user) + + +@router.post("/rules/preview") +def preview_rule(body: RuleBody, db: Session = Depends(get_db), user: User = Depends(current_user)): + """Show which firing alerts a rule would hide, before it is saved.""" + from triagelib import alerts as alertlib + + raw, _error, _age = engine.cache.get() + candidates = [a for a in (alertlib.from_prometheus(x, engine.rules) for x in raw) + if not alertlib.is_excluded(a) and alertlib.cx_relevant(a)] + rule = legacy._normalize_rule({"name": body.name, "conditions": body.conditions}) + hits = [{ + "kind": a.kind, "title": a.title, "instance_name": a.instance_name, + "host": a.host, "org_name": a.org_name, "region": a.region, + } for a in candidates if legacy.rule_matches(rule, a)] + return {"count": len(hits), "matches": hits[:60]} diff --git a/backend/app/services.py b/backend/app/services.py new file mode 100644 index 0000000..e31fb71 --- /dev/null +++ b/backend/app/services.py @@ -0,0 +1,200 @@ +"""Glue between the triage engine, the database and the outside world.""" +from __future__ import annotations + +import datetime as dt +import threading +from typing import Any, Optional + +from sqlalchemy.orm import Session + +from triagelib import alerts as alertlib, screening, settings as legacy_settings +from triagelib.prometheus import (AlertCache, PrometheusClient, PrometheusError, + RuleIndex, StateSnapshot, TrueAgeIndex) +from triagelib import linkage as linkage_mod + +from .config import get_settings +from .models import AppSetting, Case, CaseEvent, CaseStatus, SuppressionRule, User + +settings = get_settings() + + +class RuleAdapter: + """Presents DB-backed suppression rules the way the engine expects.""" + + def __init__(self, db: Session): + self._rules = [r.to_json() for r in db.query(SuppressionRule).all()] + row = db.get(AppSetting, "general") + general = (row.value if row else {}) or {} + self._agent = str(general.get("agent_name") or "") + self._chronic = int(general.get("chronic_days") or 3) + + @property + def rules(self) -> list[dict[str, Any]]: + return self._rules + + @property + def agent_name(self) -> str: + return self._agent + + @property + def chronic_days(self) -> int: + return self._chronic + + +class Engine: + """Process-wide caches over Prometheus. Cheap to share, expensive to rebuild.""" + + def __init__(self): + self.prom = PrometheusClient(settings.prometheus_base) + self.cache = AlertCache(self.prom) + self.rules = RuleIndex(self.prom) + self.snapshot = StateSnapshot(self.prom) + self.true_age = TrueAgeIndex(self.prom) + self.scan = linkage_mod.Scan() + self._lock = threading.Lock() + + def warm(self, log=print) -> None: + try: + self.rules.ensure() + snap = self.snapshot.get() + log(f" state snapshot: {len(snap.by_openstack_id)} VMs, {len(snap.total_gpus)} hosts") + ages = self.true_age.get() + log(f" alert history: {ages.count} alerts indexed over {ages.WINDOW_DAYS} days") + except PrometheusError as exc: + log(f" WARN: Prometheus caches not warmed: {exc}") + + def queue(self, db: Session, force: bool = False) -> dict[str, Any]: + raw, error, age = self.cache.get(force=force) + snap = self.snapshot.get() + ages = self.true_age.get() + parsed = [alertlib.from_prometheus(a, self.rules, ages) for a in raw] + + excluded = [a for a in parsed if alertlib.is_excluded(a)] + candidates = [a for a in parsed if not alertlib.is_excluded(a)] + cx = [a for a in candidates if a.category == "cx" and alertlib.cx_relevant(a)] + screening.screen_all(cx, snap, RuleAdapter(db)) + + cases = {c.fingerprint: c for c in + db.query(Case).filter(Case.fingerprint.in_([a.fingerprint() for a in cx])).all()} + groups = alertlib.group_alerts(cx) + for group in groups: + for item in group["alerts"]: + case = cases.get(item["id"]) + item["case"] = case.to_json() if case else None + + in_cx = {id(a) for a in cx} + infra = sorted([a for a in candidates if id(a) not in in_cx], key=alertlib.sort_key) + + return { + "error": error or snap.error or ages.error, + "warnings": screening.health_warnings(snap), + "totals": {"prometheus": len(parsed), "cx": len(cx), + "infrastructure": len(infra), "excluded": len(excluded)}, + "excluded_note": (f"{len(excluded)} '{', '.join(sorted({a.alertname for a in excluded}))}' alerts hidden" + if excluded else ""), + "summary": screening.summarize(cx), + "groups": groups, + "infrastructure": _infra_sections(infra), + "cache_age_seconds": round(age, 1), + } + + def find_alert(self, db: Session, fingerprint: str) -> Optional[Any]: + raw, _error, _age = self.cache.get() + ages = self.true_age.get() + for item in raw: + alert = alertlib.from_prometheus(item, self.rules, ages) + if alert.fingerprint() == fingerprint: + alert.screen = screening.screen(alert, self.snapshot.get(), RuleAdapter(db)) + return alert + return None + + +SOURCE_LABELS = { + "node-exporter-rules.yml": "Node exporter (hosts)", + "ceph-rules.yml": "Ceph", "mysql-rules.yml": "MySQL", + "mysql-performance-rules.yml": "MySQL performance", "galera-rules.yml": "Galera", + "openstack-rules.yml": "OpenStack services", "blackbox.yml": "Blackbox / OOB", + "infrahub-rules.yml": "Infrahub (no CX runbook)", +} + + +def _infra_sections(items: list[Any]) -> list[dict[str, Any]]: + buckets: dict[str, list[Any]] = {} + for alert in items: + buckets.setdefault(alert.rule_file or "unknown", []).append(alert) + sections = [] + for source, members in buckets.items(): + by_name: dict[str, int] = {} + for alert in members: + name = alertlib.clean_alertname(alert.alertname) + by_name[name] = by_name.get(name, 0) + 1 + sections.append({ + "source": source, "label": SOURCE_LABELS.get(source, source), "total": len(members), + "by_alertname": sorted(({"name": k, "count": v} for k, v in by_name.items()), + key=lambda x: (-x["count"], x["name"])), + }) + sections.sort(key=lambda s: (s["source"] != alertlib.NODE_RULE_FILE, -s["total"])) + return sections + + +# --- case bookkeeping ------------------------------------------------------- + +def get_or_create_case(db: Session, alert: Any, actor: Optional[User] = None) -> Case: + case = db.query(Case).filter(Case.fingerprint == alert.fingerprint()).one_or_none() + now = dt.datetime.now(dt.timezone.utc) + subject = (alert.floating_ip if alert.kind == "duplicate_ip" + else alert.host if alert.kind in ("rogue_vm", "total_gpus", "orphan_vm") + else alert.instance_name or alert.openstack_id) + if case is None: + case = Case( + fingerprint=alert.fingerprint(), kind=alert.kind, title=alert.title, + subject=subject or "", openstack_id=alert.openstack_id, + instance_name=alert.instance_name, host=alert.host, region=alert.region, + org_id=alert.org_id, org_name=alert.org_name, + ) + db.add(case) + db.flush() + add_event(db, case, actor, "opened", f"Case opened for {alert.title}") + else: + # A closed case whose alert has come back is new work again. + if not case.is_open and case.closed_at: + case.reopen_count += 1 + case.status = CaseStatus.NEW + case.closed_at = None + add_event(db, case, None, "reopened", + f"Alert fired again after being {case.status.value}") + case.last_seen_at = now + case.title = alert.title + case.subject = subject or case.subject + db.commit() + return case + + +def add_event(db: Session, case: Case, actor: Optional[User], action: str, + detail: str = "", payload: Optional[dict[str, Any]] = None) -> CaseEvent: + event = CaseEvent( + actor_id=actor.id if actor else None, + actor_label=(actor.email if actor else "system"), + action=action, detail=detail, payload=payload, + ) + # Appended through the relationship rather than inserted by id: sessions use + # expire_on_commit=False, so a collection already loaded would otherwise stay + # stale and the new event would be missing from the response. + case.events.append(event) + db.add(event) + return event + + +def set_status(db: Session, case: Case, status: CaseStatus, actor: Optional[User], + note: str = "") -> Case: + previous = case.status + case.status = status + if status in (CaseStatus.RESOLVED, CaseStatus.WONT_FIX, CaseStatus.FALSE_POSITIVE): + case.closed_at = dt.datetime.now(dt.timezone.utc) + else: + case.closed_at = None + add_event(db, case, actor, "status_changed", + f"{previous.value} -> {status.value}" + (f": {note}" if note else ""), + {"from": previous.value, "to": status.value}) + db.commit() + return case diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..7cc78a7 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.141.1 +uvicorn[standard]==0.34.0 +sqlalchemy==2.0.51 +httpx==0.28.1 +pydantic==2.10.6 +psycopg[binary]==3.2.4 diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py new file mode 100644 index 0000000..9e173fb --- /dev/null +++ b/backend/tests/test_api.py @@ -0,0 +1,124 @@ +"""API-level tests: auth gates, case lifecycle, suppression rules, send gating. + +Runs against an in-memory database with CX-Tools stubbed out, so it needs no +credentials. Prometheus is only touched for cache warming, which is tolerant of +being unreachable. +""" +import os +import sys +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +os.environ.update({ + "CX_DATABASE_URL": f"sqlite:///{tempfile.mkdtemp()}/test.db", + "CX_BOOTSTRAP_ADMIN_EMAIL": "admin@localhost", + "CX_BOOTSTRAP_ADMIN_PASSWORD": "test-password", + "CX_SECRET_KEY": "test-secret", + "CX_FEATURE_SEND_ENABLED": "false", +}) + +from triagelib import cxbridge # noqa: E402 + +cxbridge.bootstrap = lambda: (_ for _ in ()).throw(cxbridge.BridgeError("stubbed")) + +from fastapi.testclient import TestClient # noqa: E402 + +from app import auth as auth_mod # noqa: E402 +from app.main import app # noqa: E402 + +FAILS = [] + + +def expect(label, cond, got=""): + print((" PASS " if cond else " FAIL ") + label + ("" if cond else f" <- {got}")) + if not cond: + FAILS.append(label) + + +print("\nPASSWORDS AND SESSIONS") +h = auth_mod.hash_password("hunter2") +expect("correct password verifies", auth_mod.verify_password("hunter2", h)) +expect("wrong password rejected", not auth_mod.verify_password("hunter3", h)) +expect("hash is salted (two hashes differ)", auth_mod.hash_password("x") != auth_mod.hash_password("x")) +token = auth_mod.issue_session(7) +expect("session round-trips", auth_mod.read_session(token) == 7) +expect("tampered session rejected", auth_mod.read_session(token[:-4] + "aaaa") is None) +expect("garbage session rejected", auth_mod.read_session("not-a-token") is None) +expect("oidc state verifies", auth_mod.oidc_state_valid(auth_mod.oidc_state())) +expect("forged oidc state rejected", not auth_mod.oidc_state_valid("aaa.bbb")) + +with TestClient(app) as client: + print("\nAUTH GATES") + expect("health is public", client.get("/api/health").status_code == 200) + expect("alerts need a session", client.get("/api/alerts").status_code == 401) + expect("cases need a session", client.get("/api/cases").status_code == 401) + expect("wrong password is 401", client.post( + "/api/auth/login", json={"email": "admin@localhost", "password": "no"}).status_code == 401) + + login = client.post("/api/auth/login", json={"email": "admin@localhost", "password": "test-password"}) + expect("login succeeds", login.status_code == 200, login.text[:120]) + expect("bootstrap user is admin", login.json()["user"]["is_admin"]) + expect("session works after login", client.get("/api/cases").status_code == 200) + + print("\nCONFIG EXPOSURE") + cfg = client.get("/api/auth/me").json()["config"] + expect("send disabled by default", cfg["send_enabled"] is False, cfg) + expect("no secret leaks into public config", + not any("token" in k.lower() or "secret" in k.lower() for k in cfg), list(cfg)) + + print("\nSUPPRESSION RULES") + rule = {"name": "Modal ERROR churn", "reason": "known batch churn", + "conditions": {"kind": ["error"], "organization": ["modal"]}} + expect("admin can save a rule", client.post("/api/settings/rules", json=rule).status_code == 200) + expect("rule is persisted", len(client.get("/api/settings").json()["rules"]) == 1) + expect("a rule with no conditions is refused", client.post( + "/api/settings/rules", json={"name": "catch all", "conditions": {}}).status_code == 400) + expect("unknown condition fields are dropped", client.post( + "/api/settings/rules", json={"name": "bogus", "conditions": {"nope": ["x"]}}).status_code == 400) + + print("\nSEND GATING") + status = client.get("/api/actions/status").json() + expect("send reported as disabled", status["send_enabled"] is False) + expect("zendesk reported as not ready", status["zendesk_ready"] is False) + blocked = client.post("/api/actions/zendesk", json={ + "fingerprint": "does-not-exist", "to": "a@b.c", "subject": "s", "body": "b"}) + expect("sending on an untracked alert is refused", blocked.status_code == 404, blocked.text[:120]) + + print("\nCASE LIFECYCLE") + from app.db import SessionLocal + from app.models import Case, CaseStatus + from app.services import add_event, set_status + + db = SessionLocal() + case = Case(fingerprint="test-fp", kind="error", title="Instance in ERROR state", subject="vm-1") + db.add(case) + db.commit() + expect("new case starts open", case.is_open and case.status == CaseStatus.NEW) + + set_status(db, case, CaseStatus.ESCALATED_INFRA, None, "INFRA-1") + expect("escalated is still open", case.is_open) + expect("status change is recorded", any(e.action == "status_changed" for e in case.events)) + + set_status(db, case, CaseStatus.RESOLVED, None) + expect("resolved closes the case", not case.is_open and case.closed_at is not None) + + add_event(db, case, None, "note", "manual note") + db.commit() + expect("history is append-only and ordered newest first", + case.events[0].action in ("note", "status_changed"), [e.action for e in case.events]) + + payload = case.to_json(with_events=True) + expect("serialises for the API", payload["status"] == "resolved" and len(payload["events"]) >= 3) + db.close() + + expect("bad status is rejected", client.post( + "/api/cases/test-fp/status", json={"status": "banana"}).status_code == 400) + expect("unknown case is 404", client.get("/api/cases/nope").status_code == 404) + + print("\nLOGOUT") + client.post("/api/auth/logout") + expect("session is cleared", client.get("/api/cases").status_code == 401) + +print("\n" + ("ALL CHECKS PASSED" if not FAILS else f"{len(FAILS)} CHECK(S) FAILED: {FAILS}")) +sys.exit(1 if FAILS else 0) diff --git a/tests/test_runbooks.py b/backend/tests/test_runbooks.py similarity index 100% rename from tests/test_runbooks.py rename to backend/tests/test_runbooks.py diff --git a/tests/test_screening.py b/backend/tests/test_screening.py similarity index 100% rename from tests/test_screening.py rename to backend/tests/test_screening.py diff --git a/triagelib/__init__.py b/backend/triagelib/__init__.py similarity index 100% rename from triagelib/__init__.py rename to backend/triagelib/__init__.py diff --git a/triagelib/alerts.py b/backend/triagelib/alerts.py similarity index 100% rename from triagelib/alerts.py rename to backend/triagelib/alerts.py diff --git a/triagelib/comms.py b/backend/triagelib/comms.py similarity index 100% rename from triagelib/comms.py rename to backend/triagelib/comms.py diff --git a/triagelib/cxbridge.py b/backend/triagelib/cxbridge.py similarity index 100% rename from triagelib/cxbridge.py rename to backend/triagelib/cxbridge.py diff --git a/triagelib/integrations.py b/backend/triagelib/integrations.py similarity index 100% rename from triagelib/integrations.py rename to backend/triagelib/integrations.py diff --git a/triagelib/linkage.py b/backend/triagelib/linkage.py similarity index 100% rename from triagelib/linkage.py rename to backend/triagelib/linkage.py diff --git a/triagelib/prometheus.py b/backend/triagelib/prometheus.py similarity index 100% rename from triagelib/prometheus.py rename to backend/triagelib/prometheus.py diff --git a/triagelib/runbooks.py b/backend/triagelib/runbooks.py similarity index 100% rename from triagelib/runbooks.py rename to backend/triagelib/runbooks.py diff --git a/triagelib/screening.py b/backend/triagelib/screening.py similarity index 100% rename from triagelib/screening.py rename to backend/triagelib/screening.py diff --git a/triagelib/settings.py b/backend/triagelib/settings.py similarity index 100% rename from triagelib/settings.py rename to backend/triagelib/settings.py diff --git a/cx-triage b/cx-triage deleted file mode 100755 index 64627c1..0000000 --- a/cx-triage +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 -"""CX Triage - diagnose Infrahub error alerts using the CX-Tools collectors. - -Read-only: this tool queries Infrahub, OpenStack, InfraInsight and Prometheus, -and suggests what to do. It never changes platform state and never sends -customer comms. -""" -from __future__ import annotations - -import argparse -import os -import sys -import webbrowser - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from triagelib import VERSION, cxbridge # noqa: E402 -from triagelib.prometheus import DEFAULT_BASE, PrometheusClient, PrometheusError # noqa: E402 - - -def preflight(prometheus_base: str) -> bool: - ok = True - - print("Locating CX-Tools...") - try: - path = cxbridge.locate_cx_tools() - print(f" found: {path}") - except cxbridge.BridgeError as exc: - print(f" FAIL: {exc}", file=sys.stderr) - return False - - # Done in the foreground so any 1Password prompt reaches the terminal - # rather than a background HTTP worker. - print("Loading API credentials from 1Password (CX-Tools loader)...") - try: - cxbridge.bootstrap() - print(" credentials loaded") - except cxbridge.BridgeError as exc: - print(f" FAIL: {exc}", file=sys.stderr) - ok = False - - print(f"Checking Prometheus at {prometheus_base}...") - try: - print(f" reachable ({PrometheusClient(prometheus_base).describe_transport()})") - except PrometheusError as exc: - print(f" WARN: {exc}", file=sys.stderr) - print(" The app will still start; paste alerts manually until this is fixed.", file=sys.stderr) - - return ok - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--host", default="127.0.0.1", help="bind address (default: localhost only)") - parser.add_argument("--port", type=int, default=8765, help="port (default: 8765)") - parser.add_argument("--prometheus", default=DEFAULT_BASE, help=f"Prometheus base URL (default: {DEFAULT_BASE})") - parser.add_argument("--no-open", action="store_true", help="do not open a browser on start") - parser.add_argument("--check", action="store_true", help="run preflight checks and exit") - parser.add_argument("--version", action="version", version=VERSION) - args = parser.parse_args() - - if not preflight(args.prometheus): - return 1 - if args.check: - print("preflight OK") - return 0 - - url = f"http://{args.host}:{args.port}" - if not args.no_open: - webbrowser.open(url) - - from triagelib.server import serve - - serve(host=args.host, port=args.port, prometheus_base=args.prometheus) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/deploy/k8s/00-namespace.yaml b/deploy/k8s/00-namespace.yaml new file mode 100644 index 0000000..b8e69c7 --- /dev/null +++ b/deploy/k8s/00-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ${NAMESPACE} diff --git a/deploy/k8s/10-config.yaml b/deploy/k8s/10-config.yaml new file mode 100644 index 0000000..96ad657 --- /dev/null +++ b/deploy/k8s/10-config.yaml @@ -0,0 +1,30 @@ +# Non-secret configuration. Secrets live in the `cx-triage` Secret, created by +# the pipeline from the repository secret store. +apiVersion: v1 +kind: ConfigMap +metadata: + name: cx-triage-config + namespace: ${NAMESPACE} +data: + CX_APP_NAME: "CX Triage" + CX_BASE_URL: "${BASE_URL}" + CX_SESSION_HOURS: "12" + CX_STATIC_DIR: "/app/static" + + CX_PROMETHEUS_BASE: "${PROMETHEUS_BASE}" + + CX_AUTH_LOCAL_ENABLED: "false" + CX_OIDC_ENABLED: "${OIDC_ENABLED}" + CX_OIDC_ISSUER: "${OIDC_ISSUER}" + CX_OIDC_SCOPES: "openid email profile" + CX_OIDC_ADMIN_GROUP: "${OIDC_ADMIN_GROUP}" + CX_OIDC_GROUPS_CLAIM: "groups" + + CX_FEATURE_SEND_ENABLED: "${FEATURE_SEND_ENABLED}" + CX_FEATURE_ZENDESK: "${FEATURE_ZENDESK}" + CX_FEATURE_JIRA: "${FEATURE_JIRA}" + CX_FEATURE_LINKAGE_SCAN: "${FEATURE_LINKAGE_SCAN}" + CX_SEND_DAILY_CAP: "${SEND_DAILY_CAP}" + + CX_JIRA_PROJECT: "${JIRA_PROJECT}" + CX_JIRA_ISSUE_TYPE: "Task" diff --git a/deploy/k8s/20-deployment.yaml b/deploy/k8s/20-deployment.yaml new file mode 100644 index 0000000..7a9ee13 --- /dev/null +++ b/deploy/k8s/20-deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cx-triage + namespace: ${NAMESPACE} + labels: { app: cx-triage } +spec: + replicas: ${REPLICAS} + strategy: { type: RollingUpdate } + selector: + matchLabels: { app: cx-triage } + template: + metadata: + labels: { app: cx-triage } + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + containers: + - name: app + image: ${IMAGE_REF} + imagePullPolicy: IfNotPresent + ports: [{ containerPort: 8080, name: http }] + envFrom: + - configMapRef: { name: cx-triage-config } + - secretRef: { name: cx-triage } + readinessProbe: + httpGet: { path: /api/health, port: http } + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /api/health, port: http } + # Startup warms a week of alert history, so allow a slow first start. + initialDelaySeconds: 60 + periodSeconds: 30 + resources: + requests: { cpu: 100m, memory: 256Mi } + limits: { cpu: "1", memory: 1Gi } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } + volumeMounts: + - { name: tmp, mountPath: /tmp } + - { name: data, mountPath: /data } + volumes: + - name: tmp + emptyDir: {} + - name: data + emptyDir: {} # state lives in Postgres; this is scratch only diff --git a/deploy/k8s/30-service.yaml b/deploy/k8s/30-service.yaml new file mode 100644 index 0000000..257b3f6 --- /dev/null +++ b/deploy/k8s/30-service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: cx-triage + namespace: ${NAMESPACE} +spec: + selector: { app: cx-triage } + ports: + - { name: http, port: 80, targetPort: http } diff --git a/deploy/k8s/40-ingress.yaml b/deploy/k8s/40-ingress.yaml new file mode 100644 index 0000000..4a7dd61 --- /dev/null +++ b/deploy/k8s/40-ingress.yaml @@ -0,0 +1,23 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: cx-triage + namespace: ${NAMESPACE} + annotations: + cert-manager.io/cluster-issuer: "${TLS_ISSUER}" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" +spec: + ingressClassName: ${INGRESS_CLASS} + tls: + - hosts: ["${DOMAIN}"] + secretName: cx-triage-tls + rules: + - host: ${DOMAIN} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: cx-triage + port: { name: http } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6588c3a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,47 @@ +# Local development and single-host deployment. +# +# cp .env.example .env # then edit +# docker compose up --build +# +# Reaching Prometheus and OpenStack still depends on the CX-Tools containers +# being up on the same host - see docs/DEPLOYMENT.md. +services: + app: + build: + context: . + dockerfile: backend/Dockerfile + image: cx-triage:local + ports: + - "${CX_PORT:-8080}:8080" + env_file: [.env] + environment: + CX_DATABASE_URL: ${CX_DATABASE_URL:-postgresql+psycopg://cx:cx@db:5432/cxtriage} + CX_STATIC_DIR: /app/static + volumes: + # The engine shells out to `docker exec -osc ...`, so it needs the + # host's Docker socket. Mount read-only and drop it if you point the app + # at Prometheus/OpenStack directly instead. + - /var/run/docker.sock:/var/run/docker.sock:ro + - cx-data:/data + depends_on: + db: { condition: service_healthy } + restart: unless-stopped + + db: + image: postgres:16-alpine + environment: + POSTGRES_USER: cx + POSTGRES_PASSWORD: cx + POSTGRES_DB: cxtriage + volumes: + - cx-db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U cx -d cxtriage"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + +volumes: + cx-data: + cx-db: diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..1cac75d --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,87 @@ +# Deployment + +## Local + +```bash +cp .env.example .env # set CX_SECRET_KEY and CX_BOOTSTRAP_ADMIN_PASSWORD +docker compose up --build +``` + +, sign in with the bootstrap admin. + +Without Docker: + +```bash +cd backend && pip install -r requirements.txt +uvicorn app.main:app --reload --port 8080 # terminal 1 +cd frontend && npm install && npm run dev # terminal 2 -> :5173, proxies /api +``` + +## Kubernetes via Gitea Actions + +`.gitea/workflows/ci.yaml` runs tests → builds the image → renders +`deploy/k8s/*.yaml` with `envsubst` → applies them. + +### Repository **variables** (not secret) + +| Variable | Example | Meaning | +|---|---|---| +| `CX_DOMAIN` | `cx-triage.ngbackend.cloud` | Ingress host; `CX_BASE_URL` derives from it | +| `K8S_NAMESPACE` | `cx-triage` | Target namespace | +| `REGISTRY` | `git.ngbackend.cloud` | Image registry | +| `IMAGE_NAME` | `parham.monfared/cx-ui` | Image repository | +| `CX_PROMETHEUS_BASE` | `http://10.11.254.250:9090` | Alert source | +| `CX_OIDC_ENABLED` | `true` | Authentik on | +| `CX_OIDC_ISSUER` | `https://sso…/application/o/cx-triage/` | Discovery base | +| `CX_OIDC_ADMIN_GROUP` | `cx-triage-admins` | Group granting admin | +| `CX_FEATURE_SEND_ENABLED` | `false` | **Master send switch** | +| `CX_FEATURE_ZENDESK` / `CX_FEATURE_JIRA` | `false` | Per-integration flags | +| `CX_FEATURE_LINKAGE_SCAN` | `true` | Expensive scan on/off | +| `CX_SEND_DAILY_CAP` | `25` | Rolling 24h send limit | +| `CX_JIRA_PROJECT` / `CX_JIRA_BASE` | `INFRA` | Jira target | +| `INGRESS_CLASS` / `TLS_ISSUER` | `nginx` / `letsencrypt-prod` | Ingress wiring | +| `REPLICAS` | `1` | See the caveat below | + +### Repository **secrets** + +`KUBECONFIG` (base64), `REGISTRY_USERNAME`, `REGISTRY_TOKEN`, `CX_SECRET_KEY`, +`CX_DATABASE_URL`, `CX_OIDC_CLIENT_ID`, `CX_OIDC_CLIENT_SECRET`, +`CX_ZENDESK_SUBDOMAIN`, `CX_ZENDESK_EMAIL`, `CX_ZENDESK_TOKEN`, +`CX_JIRA_EMAIL`, `CX_JIRA_TOKEN`, `CX_BOOTSTRAP_ADMIN_PASSWORD`. + +## Authentik + +1. **Applications → Providers → Create → OAuth2/OpenID Provider** + - Client type: **Confidential** + - Redirect URI: `https:///api/auth/oidc/callback` + - Scopes: `openid`, `email`, `profile` +2. Create the Application and bind the provider. +3. Copy the client ID/secret into the Gitea secrets above. +4. Create a group `cx-triage-admins`; its members get admin rights. +5. Set `CX_OIDC_ISSUER` to the provider's OpenID configuration base URL — the + app appends `/.well-known/openid-configuration`. + +Users are created on first login. `CX_AUTH_LOCAL_ENABLED=false` in the cluster +config turns off password login entirely once SSO works. + +## Two things to plan for + +**Reaching OpenStack.** The diagnosis engine shells out to +`docker exec -osc openstack …`, and the internal Prometheus is only +reachable from inside those containers. That works on a laptop with CX-Tools +running; it does **not** work in a pod by default. Options, cheapest first: + +1. Run the app on a host that already has the CX-Tools containers, mounting the + Docker socket (what `docker-compose.yml` does). +2. Run the `*-osc` containers as sidecars in the pod. +3. Replace `cxbridge.os_json` with direct authenticated OpenStack API calls and + give the pod a network route. Cleanest, most work. + +Until one of those is in place, a cluster deployment can read Prometheus (if +routable) but per-alert diagnosis will fail. That is a real gap, not an +oversight. + +**`REPLICAS` should stay at 1** for now. The Prometheus caches, the background +triage jobs and the linkage scan are per-process, so a second replica would +duplicate the work and serve inconsistent job IDs. Moving jobs into the database +or a queue is what unlocks scaling out. diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md new file mode 100644 index 0000000..d3c1a4e --- /dev/null +++ b/docs/INTEGRATIONS.md @@ -0,0 +1,133 @@ +# Enabling Zendesk and Jira + +Nothing leaves this app until **three separate gates** are open. Until then the +UI builds the full payload, shows it to you, and the Send button stays disabled +with the reason written on it. + +``` + 1. the integration is configured CX_ZENDESK_* / CX_JIRA_* + 2. its feature flag is on CX_FEATURE_ZENDESK / CX_FEATURE_JIRA + 3. sending is enabled globally CX_FEATURE_SEND_ENABLED +``` + +Gate 3 is the important one. A demo or staging instance simply leaves it off, +and then no combination of clicks can email a customer. + +Check where you stand at any time on **Settings → Integrations**, or: + +```bash +curl -s localhost:8080/api/health | python3 -m json.tool +``` + +--- + +## Zendesk + +### 1. Create an API token + +Zendesk **Admin Center → Apps and integrations → APIs → Zendesk API**, turn on +*Token access*, then **Add API token**. Copy it — Zendesk shows it once. + +### 2. Decide which agent owns the tickets + +Use a dedicated agent (e.g. `cx-triage@…`) rather than a person's account, so +the audit trail stays clear when someone leaves. + +### 3. Set the variables + +```bash +CX_FEATURE_ZENDESK=true +CX_ZENDESK_SUBDOMAIN=nexgencloud # from https://.zendesk.com +CX_ZENDESK_EMAIL=cx-triage@nexgencloud.com +CX_ZENDESK_TOKEN= +CX_ZENDESK_PUBLIC_REPLY=true # false posts an internal note instead + +CX_FEATURE_SEND_ENABLED=true # the master switch +``` + +Restart. The Send button becomes live. + +### What happens on send + +1. Searches for `external_id:cx-triage-`. +2. If a ticket exists → adds a comment. If not → creates one, with the requester + set from the Infrahub owner, priority from the screening verdict, and tags + `cx-triage`, `alert-`. +3. Records a `zendesk_sent` event on the case and moves it to + **Customer contacted**. + +So re-diagnosing the same alert updates one ticket instead of opening five. + +--- + +## Jira + +### 1. Create an API token + + → *Create API +token*. + +### 2. Confirm the project and issue type + +The defaults are `INFRA` / `Task`. If your Infrastructure project uses something +else, set it — a wrong `issuetype` is the usual cause of a 400 from Jira. + +### 3. Set the variables + +```bash +CX_FEATURE_JIRA=true +CX_JIRA_BASE=https://nexgencloud.atlassian.net +CX_JIRA_EMAIL=cx-triage@nexgencloud.com +CX_JIRA_TOKEN= +CX_JIRA_PROJECT=INFRA +CX_JIRA_ISSUE_TYPE=Task + +CX_FEATURE_SEND_ENABLED=true +``` + +Issues are labelled `cx-triage-` and searched for before creating, +so the same alert never opens two tickets. + +--- + +## Where the credentials go + +**Never commit them.** + +| Where | How | +|---|---| +| Local | `.env` (git-ignored) — copy from `.env.example` | +| Kubernetes | The `cx-triage` Secret, written by the pipeline from Gitea secrets | +| Gitea | Repository → Settings → Actions → Secrets | + +The deploy job creates the Secret imperatively from the secret store, so no +credential is ever in a manifest in git. + +--- + +## Safety rails that stay on + +- Every send needs a click **and** a confirm naming the recipient. +- No auto-send: a verdict never triggers an email by itself. +- `CX_SEND_DAILY_CAP` (default 25) refuses further sends in a rolling 24 hours, + so a loop cannot mail every customer. +- The body is editable before sending. +- Every attempt is written to the case history — including failures. +- Deleting, shelving and InfraInsight edits stay copy-a-command. The read-only + guarantee is what makes this safe against production. + +## First run + +Point at a **Zendesk sandbox**, or send the first ticket to your own address by +editing the To field. Once one round trip looks right, switch it on for real. + +## When it fails + +| Symptom | Cause | +|---|---| +| Button disabled, "sending is switched off" | `CX_FEATURE_SEND_ENABLED` is false | +| Button disabled, "not configured" | A `CX_ZENDESK_*` / `CX_JIRA_*` value is missing | +| `401` from Zendesk | The email must be the **agent** address, and token access must be enabled | +| `400` from Jira | Usually `issuetype` or `project` does not exist | +| "Daily send cap reached" | Raise `CX_SEND_DAILY_CAP` if deliberate | +| "Open the case first" | The alert has no case yet — open it in the queue once | diff --git a/LINKAGE.md b/docs/LINKAGE.md similarity index 100% rename from LINKAGE.md rename to docs/LINKAGE.md diff --git a/PLAN.md b/docs/PLAN.md similarity index 100% rename from PLAN.md rename to docs/PLAN.md diff --git a/frontend/.dockerignore b/frontend/.dockerignore new file mode 100644 index 0000000..f06235c --- /dev/null +++ b/frontend/.dockerignore @@ -0,0 +1,2 @@ +node_modules +dist diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 0000000..fcce898 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,10 @@ +# Builds the static bundle. The backend image copies the result out of here. +FROM node:22-alpine AS build +WORKDIR /app +COPY package.json package-lock.json* ./ +RUN npm ci --no-audit --no-fund 2>/dev/null || npm install --no-audit --no-fund +COPY . . +RUN npm run build + +FROM busybox:1.36 +COPY --from=build /app/dist /dist diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..0fb7861 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,12 @@ + + + + + + CX Triage + + +
+ + + diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..3059d86 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,24 @@ +{ + "name": "cx-triage-frontend", + "private": true, + "version": "0.2.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "preview": "vite preview", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^6.28.0" + }, + "devDependencies": { + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^4.3.4", + "typescript": "^5.7.2", + "vite": "^6.0.5" + } +} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 0000000..78180a6 --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,61 @@ +import { useCallback, useEffect, useState } from "react"; +import { NavLink, Navigate, Route, Routes } from "react-router-dom"; +import { api, ApiError } from "./lib/api"; +import type { AppConfig, User } from "./types"; +import Login from "./pages/Login"; +import Queue from "./pages/Queue"; +import Linkage from "./pages/Linkage"; +import SettingsPage from "./pages/Settings"; + +export default function App() { + const [user, setUser] = useState(null); + const [config, setConfig] = useState(null); + const [ready, setReady] = useState(false); + + const refresh = useCallback(async () => { + try { + const me = await api.get<{ user: User | null; config: AppConfig }>("/api/auth/me"); + setUser(me.user); + setConfig(me.config); + } catch (err) { + if (!(err instanceof ApiError)) console.error(err); + setUser(null); + } finally { + setReady(true); + } + }, []); + + useEffect(() => { void refresh(); }, [refresh]); + + if (!ready) return
; + if (!user) return ; + + const logout = async () => { await api.post("/api/auth/logout"); await refresh(); }; + + return ( + <> +
+
+ {config?.app_name ?? "CX Triage"} — alert triage +
+ `navlink${isActive ? " on" : ""}`}>Queue + {config?.linkage_scan && ( + `navlink${isActive ? " on" : ""}`}>Linkage + )} + `navlink${isActive ? " on" : ""}`}>Settings + + + {user.name || user.email} + {!config?.send_enabled && sending off} + + +
+ + } /> + } /> + } /> + } /> + + + ); +} diff --git a/frontend/src/components/ActionDrawer.tsx b/frontend/src/components/ActionDrawer.tsx new file mode 100644 index 0000000..7ef0d96 --- /dev/null +++ b/frontend/src/components/ActionDrawer.tsx @@ -0,0 +1,139 @@ +import { useState } from "react"; +import { api } from "../lib/api"; +import type { Diagnosis, IntegrationAction } from "../types"; + +interface Props { + kind: "zendesk" | "jira"; + action: IntegrationAction; + diagnosis: Diagnosis; + sendEnabled: boolean; + onClose: () => void; + onDone: () => void; +} + +/** + * Compose-and-confirm. The Send button stays disabled unless the integration is + * configured *and* sending is enabled for this instance, and the confirm step + * names the recipient so nobody mails the wrong customer by muscle memory. + */ +export default function ActionDrawer({ kind, action, diagnosis, sendEnabled, onClose, onDone }: Props) { + const zendesk = kind === "zendesk"; + const ticket = action.payload?.ticket ?? {}; + const fields = action.payload?.fields ?? {}; + + const [to, setTo] = useState(ticket?.requester?.email ?? ""); + const [subject, setSubject] = useState(ticket?.subject ?? ""); + const [body, setBody] = useState(ticket?.comment?.body ?? ""); + const [priority, setPriority] = useState(ticket?.priority ?? "normal"); + const [tags, setTags] = useState((ticket?.tags ?? []).join(", ")); + const [summary, setSummary] = useState(fields?.summary ?? ""); + const [description, setDescription] = useState(fields?.description ?? ""); + const [project, setProject] = useState(fields?.project?.key ?? ""); + + const [confirming, setConfirming] = useState(false); + const [busy, setBusy] = useState(false); + const [result, setResult] = useState<{ ok: boolean; text: string; url?: string } | null>(null); + + const blocked = !action.enabled || !sendEnabled; + const blockedWhy = !sendEnabled + ? "Sending is switched off on this instance (CX_FEATURE_SEND_ENABLED). Nothing will be sent." + : action.blocked_reason; + + const send = async () => { + setBusy(true); + try { + const fingerprint = diagnosis.alert.id; + const res = zendesk + ? await api.post<{ ticket_id: string; url: string; action: string }>("/api/actions/zendesk", { + fingerprint, to, subject, body, priority, + tags: tags.split(",").map((t) => t.trim()).filter(Boolean), + }) + : await api.post<{ key: string; url: string; action: string }>("/api/actions/jira", { + fingerprint, summary, description, project, + labels: fields?.labels ?? [], + }); + const label = zendesk ? `Ticket ${(res as any).ticket_id}` : `Issue ${(res as any).key}`; + setResult({ ok: true, text: `${label} ${(res as any).action}`, url: (res as any).url }); + onDone(); + } catch (err) { + setResult({ ok: false, text: err instanceof Error ? err.message : "Send failed" }); + } finally { + setBusy(false); + } + }; + + return ( + <> +
+