CX Triage: alert diagnosis over the CX-Tools collectors
Read-only triage for the Infrahub error alerts. Pulls the Prometheus alert queue, re-checks each alert's condition against live state to separate real work from noise, diagnoses it using the CX runbooks, and drafts the customer comms with contacts resolved from Infrahub. Findings from validating against production: - "Suspected Rogue VM" fires on spare GPU capacity, not rogue VMs: In_Use_Gpus equals the physical count on 71 of 75 firing hosts, so the rule reduces to "this host has a free GPU". Verified against OpenStack on 10 hosts. - "Exists in Infrahub but does not exist in OpenStack" matches every VM because openstack_nova_server_status returns no series; excluded as a rule defect. - Prometheus activeAt is reset several times a day by dips in the Resources metric, so alert ages are recovered from ALERTS history instead. Takes ~2,650 firing alerts down to ~20 that need a decision. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
383
triagelib/comms.py
Normal file
383
triagelib/comms.py
Normal file
@@ -0,0 +1,383 @@
|
||||
"""Customer comms templates, transcribed from the CX runbooks.
|
||||
|
||||
Wording is kept verbatim from Confluence so what CX sends stays consistent with
|
||||
the approved snippets; only the named placeholders are substituted. Nothing here
|
||||
sends anything - the app renders the draft for a human to review and send from
|
||||
HubSpot.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
|
||||
INSTANCE_PLACEHOLDER = "INFRAHUB_INSTANCE_NAME"
|
||||
FIP_PLACEHOLDER = "NEW_INFRAHUB_FLOATING_IP"
|
||||
ID_PLACEHOLDER = "INFRAHUB_ID"
|
||||
OSID_PLACEHOLDER = "OPENSTACK_ID"
|
||||
NAME_PLACEHOLDER = "GREETING_NAME"
|
||||
AGENT_PLACEHOLDER = "AGENT_NAME"
|
||||
|
||||
|
||||
def first_name(owner: str) -> str:
|
||||
"""'Bojan Jovanovic <bojan@polycam.ai>' -> 'Bojan'.
|
||||
|
||||
Falls back to an empty greeting rather than guessing: the examples show
|
||||
'Hello,' is acceptable, but 'Hello shettyatulya@gmail.com,' is not.
|
||||
"""
|
||||
text = str(owner or "").split("<", 1)[0].strip()
|
||||
if not text or "@" in text:
|
||||
return ""
|
||||
first = text.split()[0]
|
||||
return first if first[:1].isalpha() else ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Draft:
|
||||
template_id: str
|
||||
label: str
|
||||
subject: str
|
||||
body: str
|
||||
channel: str = "HubSpot ticket"
|
||||
when: str = ""
|
||||
unfilled: list[str] = field(default_factory=list)
|
||||
source: str = ""
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
"template_id": self.template_id,
|
||||
"label": self.label,
|
||||
"subject": self.subject,
|
||||
"body": self.body,
|
||||
"channel": self.channel,
|
||||
"when": self.when,
|
||||
"unfilled": self.unfilled,
|
||||
"source": self.source,
|
||||
}
|
||||
|
||||
|
||||
# Wording follows the house style CX actually sends: first-name greeting, the VM
|
||||
# named with its Infrahub ID, an explicit "you will not be charged" line, the
|
||||
# billing-states link, and a personal sign-off. Placeholders are substituted;
|
||||
# everything else is left alone so what goes out stays consistent.
|
||||
BILLING_DOC = ("Here is our documentation on VM states and their cost:\n"
|
||||
"Which virtual machine states incur billing costs?")
|
||||
STOCK_DOC = ("You can use our Stock API to check availability at the time of deploying a VM here - "
|
||||
"Stock Availability")
|
||||
|
||||
_TEMPLATES: dict[str, dict[str, str]] = {
|
||||
"error_never_active": {
|
||||
"label": "ERROR - never deployed (transient stock issue)",
|
||||
"subject": "VM in Error state",
|
||||
"when": "The instance never reached a host, so nothing was built. Recommend delete and retry.",
|
||||
"source": "Instance in ERROR state",
|
||||
"body": f"""Hello GREETING_NAME,
|
||||
|
||||
We hope you are well.
|
||||
|
||||
We are emailing you in regards to VM: INFRAHUB_INSTANCE_NAME (INFRAHUB_ID)
|
||||
|
||||
Due to a transient stock issue this VM never fully deployed and is currently showing in Error.
|
||||
|
||||
Our recommendation would be to delete the VM and try recreating a new one. Please be aware that whilst the VM is in an Error state you will not be charged for its usage.
|
||||
|
||||
{BILLING_DOC}
|
||||
|
||||
{STOCK_DOC}
|
||||
|
||||
Just so you are aware, if the VM is not deleted after 14 calendar days we will proceed with deleting the VM on your behalf.
|
||||
|
||||
Kind Regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
"error_was_active": {
|
||||
"label": "ERROR - VM had been running, escalated",
|
||||
"subject": "VM in Error state",
|
||||
"when": "The instance had reached ACTIVE, so customer data may be involved. Escalate first, then send.",
|
||||
"source": "Instance in ERROR state",
|
||||
"body": f"""Hello GREETING_NAME,
|
||||
|
||||
We hope you are well.
|
||||
|
||||
We are emailing you in regards to VM: INFRAHUB_INSTANCE_NAME (INFRAHUB_ID)
|
||||
|
||||
We can see this VM has gone into an Error state. We have escalated this to the appropriate team on your behalf and will come back to you as soon as we have more information.
|
||||
|
||||
Please be assured that whilst a VM is in an Error state you will not be charged for its usage.
|
||||
|
||||
{BILLING_DOC}
|
||||
|
||||
Kind Regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
"creating": {
|
||||
"label": "CREATING - stuck on deploy, VM deleted for the customer",
|
||||
"subject": "VM stuck in creating state",
|
||||
"when": "Send after the stuck instance has been deleted.",
|
||||
"source": "Instance in CREATING state",
|
||||
"body": f"""Hello GREETING_NAME,
|
||||
|
||||
We hope you are well.
|
||||
|
||||
We are emailing you in regards to VM INFRAHUB_INSTANCE_NAME (INFRAHUB_ID).
|
||||
|
||||
We can see you tried to deploy this VM but due to a transient error the VM got stuck in a creating state.
|
||||
|
||||
Unfortunately as the VM was not able to fully deploy, the safest option was to delete the VM which we have actioned for you.
|
||||
|
||||
Please be assured that whilst a VM is in a creating state you will not be charged for it's usage.
|
||||
|
||||
{BILLING_DOC}
|
||||
|
||||
If you have any queries please let us know.
|
||||
|
||||
Kind Regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
"deleting": {
|
||||
"label": "DELETING - stuck delete finalised for the customer",
|
||||
"subject": "VM stuck in deleting state",
|
||||
"when": "Send once the delete has actually been finalised.",
|
||||
"source": "Instance in DELETING state",
|
||||
"body": f"""Hello GREETING_NAME,
|
||||
|
||||
We hope you are well.
|
||||
|
||||
We are emailing you in regards to VM: INFRAHUB_INSTANCE_NAME (INFRAHUB_ID)
|
||||
|
||||
We can see one of your users requested the deletion and that due to a transient error the VM got stuck in a deleting state. I wanted to make sure you are aware that we have gone and finalised the deletion for you.
|
||||
|
||||
Please be assured that whilst a VM is in a deleting state you will not be charged for it's usage.
|
||||
|
||||
{BILLING_DOC}
|
||||
|
||||
Kind Regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
"deleting_resolved": {
|
||||
"label": "DELETING - confirming resolution and closing the ticket",
|
||||
"subject": "VM deleted - closing this ticket",
|
||||
"when": "Use when replying on an existing ticket that can now be closed.",
|
||||
"source": "Instance in DELETING state",
|
||||
"body": """Hello GREETING_NAME,
|
||||
|
||||
Upon reviewing this ticket, we found that the VM below, which was previously stuck in a DELETING state, has now been deleted:
|
||||
|
||||
* OPENSTACK_ID (INFRAHUB_ID)
|
||||
|
||||
As the VM has been deleted, we are marking the issue as resolved and closing this ticket.
|
||||
|
||||
If you require further assistance, please feel free to contact us at support@hyperstack.cloud or open a new Live Chat via the Hyperstack Console.
|
||||
|
||||
Have a great rest of your day and thank you for using Hyperstack.
|
||||
|
||||
Kind regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
"build": {
|
||||
"label": "BUILD - failed to build, escalated",
|
||||
"subject": "VM stuck in build state",
|
||||
"when": "Send once escalated to Infrastructure. The instance must be recreated.",
|
||||
"source": "Instance in BUILD state",
|
||||
"body": f"""Hello GREETING_NAME,
|
||||
|
||||
We hope you are well.
|
||||
|
||||
We are emailing you in regards to VM INFRAHUB_INSTANCE_NAME (INFRAHUB_ID).
|
||||
|
||||
We can see you tried to deploy this VM but due to a transient error it did not finish building. We have escalated this on your behalf and ask that you retry creating the instance at your convenience.
|
||||
|
||||
Please be assured that whilst a VM is in this state you will not be charged for it's usage.
|
||||
|
||||
{BILLING_DOC}
|
||||
|
||||
If you have any queries please let us know.
|
||||
|
||||
Kind Regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
"rebooting": {
|
||||
"label": "REBOOTING - reboot failed, now resolved",
|
||||
"subject": "VM stuck rebooting",
|
||||
"when": "Send only once the instance is confirmed ACTIVE in both Infrahub and OpenStack.",
|
||||
"source": "Instance in REBOOTING state",
|
||||
"body": f"""Hello GREETING_NAME,
|
||||
|
||||
We hope you are well.
|
||||
|
||||
We are emailing you in regards to VM INFRAHUB_INSTANCE_NAME (INFRAHUB_ID).
|
||||
|
||||
We can see a reboot was requested but due to a transient error the VM got stuck. This has now been resolved and you may retry rebooting at your convenience.
|
||||
|
||||
{BILLING_DOC}
|
||||
|
||||
If you have any queries please let us know.
|
||||
|
||||
Kind Regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
"restoring": {
|
||||
"label": "RESTORING - restore failed, now resolved",
|
||||
"subject": "VM stuck restoring",
|
||||
"when": "Send after the instance is back to SHELVED_OFFLOADED in OpenStack and HIBERNATED in Infrahub.",
|
||||
"source": "Instance in RESTORING state",
|
||||
"body": f"""Hello GREETING_NAME,
|
||||
|
||||
We hope you are well.
|
||||
|
||||
We are emailing you in regards to VM INFRAHUB_INSTANCE_NAME (INFRAHUB_ID).
|
||||
|
||||
We can see you tried to restore this VM but due to a transient error it got stuck. This has now been resolved and you may retry restoring at your convenience.
|
||||
|
||||
{BILLING_DOC}
|
||||
|
||||
If you have any queries please let us know.
|
||||
|
||||
Kind Regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
"shutoff": {
|
||||
"label": "SHUTOFF - billing awareness notice",
|
||||
"subject": "VM in SHUT-OFF state is still accruing costs",
|
||||
"when": "Send as-is. A HubSpot snippet also exists: type #shutoff.",
|
||||
"source": "Instance in SHUTOFF state",
|
||||
"body": f"""Hello GREETING_NAME,
|
||||
|
||||
We hope you are well.
|
||||
|
||||
We are emailing you in regards to VM: INFRAHUB_INSTANCE_NAME (INFRAHUB_ID)
|
||||
|
||||
We can see that this is in a SHUT-OFF state and wanted to make sure you are aware that in this state the VM is still accruing full costs.
|
||||
|
||||
{BILLING_DOC}
|
||||
|
||||
Kind Regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
"dupip_removed": {
|
||||
"label": "Duplicated IP - incorrect IP removed, customer must attach a new one",
|
||||
"subject": "Instance assigned an incorrect public IP",
|
||||
"when": "The VM has no floating IP in OpenStack and the stale IP was removed in InfraInsight.",
|
||||
"source": "Duplicated IPs",
|
||||
"body": """Hello GREETING_NAME,
|
||||
|
||||
We hope you are well.
|
||||
|
||||
We are emailing you in regards to VM: INFRAHUB_INSTANCE_NAME (INFRAHUB_ID)
|
||||
|
||||
Due to a transient synchronisation issue this instance was showing an incorrect public IP. This has now been corrected. To restore external connectivity the instance will need a new public IP attached, which you can do at your convenience using either the Hyperstack UI or API.
|
||||
|
||||
We apologise for any inconvenience this may have caused.
|
||||
|
||||
Kind Regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
"dupip_corrected": {
|
||||
"label": "Duplicated IP - Infrahub corrected to match OpenStack",
|
||||
"subject": "Instance assigned an incorrect public IP",
|
||||
"when": "The VM does have a floating IP in OpenStack and Infrahub was corrected to match.",
|
||||
"source": "Duplicated IPs",
|
||||
"body": """Hello GREETING_NAME,
|
||||
|
||||
We hope you are well.
|
||||
|
||||
We are emailing you in regards to VM: INFRAHUB_INSTANCE_NAME (INFRAHUB_ID)
|
||||
|
||||
Due to a transient synchronisation issue this instance was showing an incorrect public IP. This has now been resolved and your instance is reachable at NEW_INFRAHUB_FLOATING_IP.
|
||||
|
||||
We apologise for any inconvenience this may have caused.
|
||||
|
||||
Kind Regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
"sync_state": {
|
||||
"label": "Rogue VM - instance was in the incorrect state",
|
||||
"subject": "VM was showing an incorrect state",
|
||||
"when": "Send after the state mismatch has been remediated.",
|
||||
"source": "Suspected Rogue VM",
|
||||
"body": """Hello GREETING_NAME,
|
||||
|
||||
We hope you are well.
|
||||
|
||||
We are emailing you in regards to VM: INFRAHUB_INSTANCE_NAME (INFRAHUB_ID)
|
||||
|
||||
Due to a transient sync error this instance was showing an incorrect state. This has since been resolved, and we ask that you retry any operations that failed as a result.
|
||||
|
||||
We apologise for any delay this may have caused.
|
||||
|
||||
Kind Regards,
|
||||
|
||||
AGENT_NAME""",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def draft(template_id: str, *, instance_name: str = "", floating_ip: str = "",
|
||||
infrahub_id: str = "", openstack_id: str = "", greeting_name: str = "",
|
||||
agent_name: str = "", note: Optional[str] = None) -> Optional[Draft]:
|
||||
spec = _TEMPLATES.get(template_id)
|
||||
if not spec:
|
||||
return None
|
||||
body = spec["body"]
|
||||
unfilled: list[str] = []
|
||||
|
||||
# "Hello Bojan," when we know the name, "Hello," when we do not - never
|
||||
# "Hello <placeholder>,".
|
||||
body = body.replace(f"Hello {NAME_PLACEHOLDER},", f"Hello {greeting_name}," if greeting_name else "Hello,")
|
||||
|
||||
substitutions = {
|
||||
INSTANCE_PLACEHOLDER: instance_name,
|
||||
ID_PLACEHOLDER: infrahub_id,
|
||||
OSID_PLACEHOLDER: openstack_id,
|
||||
FIP_PLACEHOLDER: floating_ip,
|
||||
AGENT_PLACEHOLDER: agent_name,
|
||||
}
|
||||
for placeholder, value in substitutions.items():
|
||||
if placeholder not in body:
|
||||
continue
|
||||
if value and value not in ("N/A", "None"):
|
||||
body = body.replace(placeholder, str(value))
|
||||
else:
|
||||
unfilled.append(placeholder)
|
||||
|
||||
# "(INFRAHUB_ID)" with nothing to put in it reads worse than no bracket.
|
||||
if ID_PLACEHOLDER in unfilled:
|
||||
body = body.replace(f" ({ID_PLACEHOLDER})", "").replace(f"({ID_PLACEHOLDER})", "")
|
||||
unfilled.remove(ID_PLACEHOLDER)
|
||||
|
||||
when = spec["when"]
|
||||
if note:
|
||||
when = f"{when} {note}".strip()
|
||||
|
||||
return Draft(
|
||||
template_id=template_id,
|
||||
label=spec["label"],
|
||||
subject=spec["subject"],
|
||||
body=body,
|
||||
when=when,
|
||||
unfilled=unfilled,
|
||||
source=spec["source"],
|
||||
)
|
||||
|
||||
|
||||
def contacts_from_result(result: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Pull the organization and owner contacts CX-Tools resolved for a VM."""
|
||||
if not isinstance(result, dict):
|
||||
return {"organization": "", "owners": [], "resolved": False}
|
||||
org = str(result.get("org_value") or "").strip()
|
||||
owners = [str(x) for x in (result.get("owners") or []) if str(x).strip()]
|
||||
return {
|
||||
"organization": org if org and org != "N/A" else "",
|
||||
"owners": owners,
|
||||
"resolved": bool(owners),
|
||||
}
|
||||
Reference in New Issue
Block a user