Parham Monfared
a039e0b5fd
Read-only triage for the Infrahub error alerts. Pulls the Prometheus alert queue, re-checks each alert's condition against live state to separate real work from noise, diagnoses it using the CX runbooks, and drafts the customer comms with contacts resolved from Infrahub. Findings from validating against production: - "Suspected Rogue VM" fires on spare GPU capacity, not rogue VMs: In_Use_Gpus equals the physical count on 71 of 75 firing hosts, so the rule reduces to "this host has a free GPU". Verified against OpenStack on 10 hosts. - "Exists in Infrahub but does not exist in OpenStack" matches every VM because openstack_nova_server_status returns no series; excluded as a rule defect. - Prometheus activeAt is reset several times a day by dips in the Resources metric, so alert ages are recovered from ALERTS history instead. Takes ~2,650 firing alerts down to ~20 that need a decision. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
80 lines
2.7 KiB
Python
Executable File
80 lines
2.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""CX Triage - diagnose Infrahub error alerts using the CX-Tools collectors.
|
|
|
|
Read-only: this tool queries Infrahub, OpenStack, InfraInsight and Prometheus,
|
|
and suggests what to do. It never changes platform state and never sends
|
|
customer comms.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import webbrowser
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
from triagelib import VERSION, cxbridge # noqa: E402
|
|
from triagelib.prometheus import DEFAULT_BASE, PrometheusClient, PrometheusError # noqa: E402
|
|
|
|
|
|
def preflight(prometheus_base: str) -> bool:
|
|
ok = True
|
|
|
|
print("Locating CX-Tools...")
|
|
try:
|
|
path = cxbridge.locate_cx_tools()
|
|
print(f" found: {path}")
|
|
except cxbridge.BridgeError as exc:
|
|
print(f" FAIL: {exc}", file=sys.stderr)
|
|
return False
|
|
|
|
# Done in the foreground so any 1Password prompt reaches the terminal
|
|
# rather than a background HTTP worker.
|
|
print("Loading API credentials from 1Password (CX-Tools loader)...")
|
|
try:
|
|
cxbridge.bootstrap()
|
|
print(" credentials loaded")
|
|
except cxbridge.BridgeError as exc:
|
|
print(f" FAIL: {exc}", file=sys.stderr)
|
|
ok = False
|
|
|
|
print(f"Checking Prometheus at {prometheus_base}...")
|
|
try:
|
|
print(f" reachable ({PrometheusClient(prometheus_base).describe_transport()})")
|
|
except PrometheusError as exc:
|
|
print(f" WARN: {exc}", file=sys.stderr)
|
|
print(" The app will still start; paste alerts manually until this is fixed.", file=sys.stderr)
|
|
|
|
return ok
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument("--host", default="127.0.0.1", help="bind address (default: localhost only)")
|
|
parser.add_argument("--port", type=int, default=8765, help="port (default: 8765)")
|
|
parser.add_argument("--prometheus", default=DEFAULT_BASE, help=f"Prometheus base URL (default: {DEFAULT_BASE})")
|
|
parser.add_argument("--no-open", action="store_true", help="do not open a browser on start")
|
|
parser.add_argument("--check", action="store_true", help="run preflight checks and exit")
|
|
parser.add_argument("--version", action="version", version=VERSION)
|
|
args = parser.parse_args()
|
|
|
|
if not preflight(args.prometheus):
|
|
return 1
|
|
if args.check:
|
|
print("preflight OK")
|
|
return 0
|
|
|
|
url = f"http://{args.host}:{args.port}"
|
|
if not args.no_open:
|
|
webbrowser.open(url)
|
|
|
|
from triagelib.server import serve
|
|
|
|
serve(host=args.host, port=args.port, prometheus_base=args.prometheus)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|