[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6CDRETS8QJKHC668PD3JD7V":3,"skill-same:01M6CDRETS8QJKHC668PD3JD7V":82},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":15,"categories":16,"keywords":17,"publishedAt":49,"linkOnly":50,"markdownContent":51,"markdownBody":52,"frontMatter":53,"refName":29,"commitSha":62,"skillDirectory":63,"forksCount":64,"repositoryHtmlUrl":65,"repositoryHomepageUrl":6,"ownerHtmlUrl":66,"ownerBlogUrl":6,"ownerDescription":6,"updatedAt":67,"install":68,"resources":81},"01M6CDRETS8QJKHC668PD3JD7V","ai-sre-incident-response",null,"Build AI-focused SRE incident response practices for LLM outages, degraded quality, runaway cost events, and safety regressions.","boisenoise\u002Fskills-collections","boisenoise","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F193180?v=4","01M6CDBDQK6BDN55ZMA6B6PKBM","01M6CDBDVSJ3JPTBYN26FZ7TZK",12,"MIT","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-ai-sre-incident-response\u002FSKILL.md",[],[18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48],"incident response","ai sre","incident","ai sre incident response","sre","environment","2026","relevant","events","scripts","critical","main","community","cost","agent-skills","only","devops","github","quality","access","helm","devops-security","Safety","templates","kubectl","ci","https","runners","license","git","security","2026-11-10 10:12:30",false,"---\nname: ai-sre-incident-response\ndescription: Build AI-focused SRE incident response practices for LLM outages, degraded\n  quality, runaway cost events, and safety regressions.\ncategory: devops\nrisk: critical\nsource: https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills\nsource_repo: BagelHole\u002FDevOps-Security-Agent-Skills\nsource_type: community\ndate_added: '2026-09-20'\nlicense: MIT\nlicense_source: https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills\u002Fblob\u002Fmain\u002FLICENSE\ncompatibility: Requires the relevant platform CLIs (kubectl, helm, terraform, git,\n  CI runners) and authorized access to the target environment. Docs-only; helper scripts\n  and templates not bundled.\nmetadata:\n  author: devops-skills\n  version: '1.0'\n---\n\n# AI SRE Incident Response\n\nApply SRE rigor to AI systems where incidents include quality regressions, unsafe outputs, and budget explosions.\n\n## When to Use This Skill\n\n- An LLM endpoint begins returning degraded or hallucinated answers\n- Token spend spikes beyond budget thresholds\n- A model provider goes down and traffic must fail over\n- Safety guardrails fire at abnormal rates\n- A new model deployment causes latency or accuracy regression\n\n## Prerequisites\n\n- Prometheus and Alertmanager deployed with scrape targets for AI services\n- Grafana dashboards for golden signals (latency, error rate, cost, quality)\n- On-call rotation configured in PagerDuty, Opsgenie, or equivalent\n- Runbook repository accessible to responders\n- Rollback mechanism for model and prompt versions (GitOps or feature flags)\n\n## AI Incident Classes\n\n- **Availability incident**: model\u002Fprovider unavailable, timeout storm.\n- **Quality incident**: answer accuracy or tool success drops below SLO.\n- **Safety incident**: harmful or policy-violating outputs increase.\n- **Cost incident**: unexpected token or provider spend spike.\n\n## Severity Framework\n\n| Severity | Criteria | Response Time | Notification |\n|----------|----------|---------------|--------------|\n| SEV1 | User-facing outage, compliance risk, data leak | 5 min | Page on-call + incident commander |\n| SEV2 | Major degradation in key flows | 15 min | Page on-call |\n| SEV3 | Limited impact or internal-only issue | 1 hour | Slack alert |\n| SEV4 | Cosmetic or low-priority regression | Next business day | Ticket |\n\n## Golden Signals for AI Services\n\n- Request success rate\n- Latency (queue + generation + tool execution)\n- Hallucination\u002Fgroundedness proxy metrics\n- Cost per minute and per tenant\n- Guardrail violation rate\n\n## Prometheus Alert Rules\n\nThe thresholds below are illustrative. Calibrate them to the declared SLO, metric units, tenant budgets and workload; a groundedness proxy is not a verified quality measurement. Counter rates are per second, so the cost-per-minute alert explicitly multiplies by 60.\n\n```yaml\n# prometheus-ai-alerts.yaml\ngroups:\n  - name: ai-service-alerts\n    rules:\n      - alert: ModelEndpointDown\n        expr: up{job=\"llm-inference\"} == 0\n        for: 2m\n        labels:\n          severity: sev1\n        annotations:\n          summary: \"LLM inference endpoint {{ $labels.instance }} is down\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fmodel-outage\"\n\n      - alert: HighHallucinationRate\n        expr: |\n          rate(llm_hallucination_detected_total[10m])\n          \u002F rate(llm_requests_total[10m]) > 0.15\n        for: 5m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Hallucination rate above 15% for {{ $labels.model }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fquality-regression\"\n\n      - alert: TokenCostExplosion\n        expr: |\n          60 * sum(rate(llm_token_cost_dollars[5m])) by (tenant)\n          > 0.50\n        for: 3m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Token spend exceeds $0.50\u002Fmin for tenant {{ $labels.tenant }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fcost-spike\"\n\n      - alert: LatencyP95Exceeded\n        expr: |\n          histogram_quantile(0.95,\n            rate(llm_request_duration_seconds_bucket[5m])\n          ) > 5\n        for: 5m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"LLM p95 latency exceeds 5s for {{ $labels.service }}\"\n\n      - alert: GuardrailViolationSpike\n        expr: |\n          rate(llm_guardrail_violations_total[10m])\n          \u002F rate(llm_requests_total[10m]) > 0.05\n        for: 5m\n        labels:\n          severity: sev1\n        annotations:\n          summary: \"Guardrail violations above 5% for {{ $labels.model }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fsafety-incident\"\n\n      - alert: ModelQualityDrop\n        expr: |\n          llm_eval_score{metric=\"groundedness\"} \u003C 0.70\n        for: 10m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Groundedness score dropped below 0.70 for {{ $labels.model }}\"\n\n      - alert: ProviderErrorRateHigh\n        expr: |\n          rate(llm_provider_errors_total[5m])\n          \u002F rate(llm_provider_requests_total[5m]) > 0.10\n        for: 3m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Provider {{ $labels.provider }} error rate above 10%\"\n```\n\n## Response Playbooks\n\n### Model Outage Runbook\n\n```text\nTRIGGER: ModelEndpointDown fires for > 2 minutes\nRESPONDER: On-call AI platform engineer\n\n1. Acknowledge alert in PagerDuty.\n2. Check provider status page (e.g., status.openai.com).\n3. Verify network connectivity:\n     curl -s -o \u002Fdev\u002Fnull -w \"%{http_code}\" https:\u002F\u002Fapi.provider.com\u002Fhealth\n4. If provider is down:\n     a. Enable fallback model route in gateway config.\n     b. kubectl set env deployment\u002Fllm-gateway FALLBACK_ENABLED=true\n     c. Verify fallback traffic is flowing via Grafana dashboard.\n5. If self-hosted model is down:\n     a. Check pod status: kubectl get pods -l app=llm-inference -n ai\n     b. Check GPU health: kubectl logs -l app=llm-inference --tail=50\n     c. Restart if OOM: kubectl rollout restart deployment\u002Fllm-inference -n ai\n6. Request the existing deployment-freeze mechanism for the exact affected scope.\n   A Kubernetes annotation by itself does not enforce a freeze; verify that the configured controller\u002Fpipeline actually honors it.\n7. Communicate ETA in #incident-channel.\n8. When resolved, unfreeze and run smoke tests.\n```\n\n### Quality Regression Runbook (Hallucination Spike)\n\n```text\nTRIGGER: HighHallucinationRate or ModelQualityDrop fires\nRESPONDER: On-call AI engineer + ML lead\n\n1. Acknowledge alert. Open incident ticket.\n2. Identify scope:\n     - Which model version? Check deployment metadata.\n     - Which routes\u002Ftenants affected? Filter by labels in Grafana.\n3. Check recent changes:\n     - Model version promotion in last 24h?\n     - Prompt template changes in last 24h?\n     - Retrieval index rebuild in last 24h?\n4. If recent model change:\n     kubectl rollout undo deployment\u002Fllm-inference -n ai\n5. If recent prompt change:\n     Prepare a reviewed revert on a topic branch, verify it in staging, and use the protected deployment path; do not push a production rollback blindly.\n6. If authorized, increase redacted trace sampling for the affected route only after checking personal-data exposure and the remaining storage\u002Fcost budget:\n     kubectl set env deployment\u002Fllm-gateway TRACE_SAMPLE_RATE=1.0\n7. Run the existing version-bound evaluation suite against an authorized isolated copy of the model\u002Fprompt configuration using synthetic or redacted cases.\n   No run_evals.py helper is bundled; never describe a production call as offline.\n8. Confirm metrics return to baseline before closing.\n```\n\n### Token Cost Explosion Runbook\n\n```text\nTRIGGER: TokenCostExplosion fires\nRESPONDER: On-call platform engineer\n\n1. Identify top consumers:\n     Query: topk(10, sum(rate(llm_token_cost_dollars[15m])) by (tenant, model, route))\n2. Check for runaway loops:\n     - Agent retry storms (exponential token growth per request)\n     - Missing max_tokens caps on new routes\n     - Cache bypass due to config change\n3. Apply immediate caps:\n     kubectl patch configmap llm-quotas -n ai --patch '\n       data:\n         max_tokens_per_request: \"4096\"\n         rpm_limit: \"60\"\n     '\n4. Review existing cache isolation before any enablement: tenant\u002Fuser identity, permissions, invalidation and private-content retention must be enforced.\n   Do not turn on a generic shared semantic cache during an incident.\n5. If authorized, route only to an already approved fallback meeting the declared quality, privacy and cost constraints. Review the exact change before updating gateway configuration.\n6. Notify affected tenants of temporary limits.\n7. Open postmortem with cost attribution analysis.\n```\n\n## Escalation Procedures\n\n```text\nLevel 1 (0-15 min):  On-call AI platform engineer\nLevel 2 (15-30 min): AI platform team lead + affected product owner\nLevel 3 (30-60 min): Engineering director + security (if safety incident)\nLevel 4 (60+ min):   VP Engineering + legal (if compliance\u002Fdata incident)\n\nSafety incidents always start at Level 2 minimum.\nProvider-side incidents: open support ticket immediately at Level 1.\n```\n\n## Detection Queries (PromQL)\n\n```promql\n# Request success rate by model\n1 - (\n  sum(rate(llm_requests_total{status=\"error\"}[5m])) by (model)\n  \u002F sum(rate(llm_requests_total[5m])) by (model)\n)\n\n# Cost per successful answer\nsum(rate(llm_token_cost_dollars[5m])) by (route)\n\u002F sum(rate(llm_requests_total{status=\"success\"}[5m])) by (route)\n\n# Hallucination rate trend (1h window, 5m steps)\nrate(llm_hallucination_detected_total[1h])\n\u002F rate(llm_requests_total[1h])\n\n# Latency breakdown by stage\nhistogram_quantile(0.95, rate(llm_retrieval_duration_seconds_bucket[5m]))\nhistogram_quantile(0.95, rate(llm_generation_duration_seconds_bucket[5m]))\nhistogram_quantile(0.95, rate(llm_tool_execution_duration_seconds_bucket[5m]))\n\n# Tenant cost leaderboard\ntopk(10, sum(rate(llm_token_cost_dollars[1h])) by (tenant))\n```\n\n## Postmortem Requirements\n\n- Timeline with detector and responder timestamps\n- Blast radius by tenant and feature\n- Missed signals and alert tuning actions\n- Concrete hardening tasks with owners and due dates\n- Cost impact (dollars, tokens, affected requests)\n- Customer communication log\n\n## Postmortem Template\n\n```markdown\n## Incident Summary\n- **Severity**: SEVx\n- **Duration**: start_time - end_time (Xh Ym)\n- **Detection**: How was it detected? (alert \u002F customer report \u002F manual)\n- **Impact**: X tenants, Y requests, $Z cost\n\n## Timeline\n| Time (UTC) | Event |\n|------------|-------|\n| HH:MM | Alert fired |\n| HH:MM | Responder acknowledged |\n| HH:MM | Root cause identified |\n| HH:MM | Mitigation applied |\n| HH:MM | Incident resolved |\n\n## Root Cause\n[Description]\n\n## Action Items\n| Action | Owner | Due Date | Status |\n|--------|-------|----------|--------|\n| Tune alert threshold | @engineer | YYYY-MM-DD | Open |\n| Add fallback route | @platform | YYYY-MM-DD | Open |\n```\n\n## Chaos Engineering for AI Systems\n\nTest drills only in a disposable isolated harness or separately authorized staging environment. Use synthetic inputs and a predeclared fault schedule, bounded duration, costs, recovery and notification policy; never deploy a known-bad model or block a real provider in production to create a drill. Regularly test incident readiness:\n\n- **Provider failover drill**: block provider API at network level, verify fallback activates within SLO.\n- **Model rollback drill**: deploy known-bad model version, verify automated quality gate catches it.\n- **Cost cap drill**: simulate runaway token usage, verify quotas trigger before budget threshold.\n- **Cache failure drill**: disable semantic cache, verify system degrades gracefully.\n\n## Troubleshooting\n\n| Symptom | Check | Fix |\n|---------|-------|-----|\n| All requests timing out | Provider status page, DNS resolution | Enable fallback provider |\n| Gradual quality decline | Recent model\u002Fprompt deployments | Roll back to last known good |\n| Sudden cost spike | Per-tenant token usage dashboard | Apply emergency token caps |\n| Guardrail violations spike | Model version, prompt injection logs | Enable stricter input filtering |\n| Intermittent 503 errors | Pod restarts, GPU OOM events | Increase memory limits or reduce batch size |\n\n## Related Skills\n\n- incident-response (`incident-response`) - Standard incident process and evidence\n- alerting-oncall (`alerting-oncall`) - Paging and escalation policy\n- llm-cost-optimization (`llm-cost-optimization`) - Spend controls and efficiency patterns\n- agent-observability (`agent-observability`) - Instrument requests, traces, and costs\n- rag-observability-evals (`rag-observability-evals`) - RAG quality monitoring\n\n## Limitations\n\n- Guidance executes against real environments: confirm target, blast radius, and rollback plan before applying anything.\n- Never deploy to production without explicit approval. Docs-only import: upstream scripts and templates not bundled.\n\n### Example\n\n```bash\ngit status && git diff --stat\nkubectl diff -f manifest.yaml\n```\n\n> Adapted from [BagelHole\u002FDevOps-Security-Agent-Skills](https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills) (MIT); frontmatter, When to Use\u002FLimitations, and safety boundaries added for upstream compliance. Docs-only import: helper scripts and templates not bundled.\n","# AI SRE Incident Response\n\nApply SRE rigor to AI systems where incidents include quality regressions, unsafe outputs, and budget explosions.\n\n## When to Use This Skill\n\n- An LLM endpoint begins returning degraded or hallucinated answers\n- Token spend spikes beyond budget thresholds\n- A model provider goes down and traffic must fail over\n- Safety guardrails fire at abnormal rates\n- A new model deployment causes latency or accuracy regression\n\n## Prerequisites\n\n- Prometheus and Alertmanager deployed with scrape targets for AI services\n- Grafana dashboards for golden signals (latency, error rate, cost, quality)\n- On-call rotation configured in PagerDuty, Opsgenie, or equivalent\n- Runbook repository accessible to responders\n- Rollback mechanism for model and prompt versions (GitOps or feature flags)\n\n## AI Incident Classes\n\n- **Availability incident**: model\u002Fprovider unavailable, timeout storm.\n- **Quality incident**: answer accuracy or tool success drops below SLO.\n- **Safety incident**: harmful or policy-violating outputs increase.\n- **Cost incident**: unexpected token or provider spend spike.\n\n## Severity Framework\n\n| Severity | Criteria | Response Time | Notification |\n|----------|----------|---------------|--------------|\n| SEV1 | User-facing outage, compliance risk, data leak | 5 min | Page on-call + incident commander |\n| SEV2 | Major degradation in key flows | 15 min | Page on-call |\n| SEV3 | Limited impact or internal-only issue | 1 hour | Slack alert |\n| SEV4 | Cosmetic or low-priority regression | Next business day | Ticket |\n\n## Golden Signals for AI Services\n\n- Request success rate\n- Latency (queue + generation + tool execution)\n- Hallucination\u002Fgroundedness proxy metrics\n- Cost per minute and per tenant\n- Guardrail violation rate\n\n## Prometheus Alert Rules\n\nThe thresholds below are illustrative. Calibrate them to the declared SLO, metric units, tenant budgets and workload; a groundedness proxy is not a verified quality measurement. Counter rates are per second, so the cost-per-minute alert explicitly multiplies by 60.\n\n```yaml\n# prometheus-ai-alerts.yaml\ngroups:\n  - name: ai-service-alerts\n    rules:\n      - alert: ModelEndpointDown\n        expr: up{job=\"llm-inference\"} == 0\n        for: 2m\n        labels:\n          severity: sev1\n        annotations:\n          summary: \"LLM inference endpoint {{ $labels.instance }} is down\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fmodel-outage\"\n\n      - alert: HighHallucinationRate\n        expr: |\n          rate(llm_hallucination_detected_total[10m])\n          \u002F rate(llm_requests_total[10m]) > 0.15\n        for: 5m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Hallucination rate above 15% for {{ $labels.model }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fquality-regression\"\n\n      - alert: TokenCostExplosion\n        expr: |\n          60 * sum(rate(llm_token_cost_dollars[5m])) by (tenant)\n          > 0.50\n        for: 3m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Token spend exceeds $0.50\u002Fmin for tenant {{ $labels.tenant }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fcost-spike\"\n\n      - alert: LatencyP95Exceeded\n        expr: |\n          histogram_quantile(0.95,\n            rate(llm_request_duration_seconds_bucket[5m])\n          ) > 5\n        for: 5m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"LLM p95 latency exceeds 5s for {{ $labels.service }}\"\n\n      - alert: GuardrailViolationSpike\n        expr: |\n          rate(llm_guardrail_violations_total[10m])\n          \u002F rate(llm_requests_total[10m]) > 0.05\n        for: 5m\n        labels:\n          severity: sev1\n        annotations:\n          summary: \"Guardrail violations above 5% for {{ $labels.model }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fsafety-incident\"\n\n      - alert: ModelQualityDrop\n        expr: |\n          llm_eval_score{metric=\"groundedness\"} \u003C 0.70\n        for: 10m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Groundedness score dropped below 0.70 for {{ $labels.model }}\"\n\n      - alert: ProviderErrorRateHigh\n        expr: |\n          rate(llm_provider_errors_total[5m])\n          \u002F rate(llm_provider_requests_total[5m]) > 0.10\n        for: 3m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Provider {{ $labels.provider }} error rate above 10%\"\n```\n\n## Response Playbooks\n\n### Model Outage Runbook\n\n```text\nTRIGGER: ModelEndpointDown fires for > 2 minutes\nRESPONDER: On-call AI platform engineer\n\n1. Acknowledge alert in PagerDuty.\n2. Check provider status page (e.g., status.openai.com).\n3. Verify network connectivity:\n     curl -s -o \u002Fdev\u002Fnull -w \"%{http_code}\" https:\u002F\u002Fapi.provider.com\u002Fhealth\n4. If provider is down:\n     a. Enable fallback model route in gateway config.\n     b. kubectl set env deployment\u002Fllm-gateway FALLBACK_ENABLED=true\n     c. Verify fallback traffic is flowing via Grafana dashboard.\n5. If self-hosted model is down:\n     a. Check pod status: kubectl get pods -l app=llm-inference -n ai\n     b. Check GPU health: kubectl logs -l app=llm-inference --tail=50\n     c. Restart if OOM: kubectl rollout restart deployment\u002Fllm-inference -n ai\n6. Request the existing deployment-freeze mechanism for the exact affected scope.\n   A Kubernetes annotation by itself does not enforce a freeze; verify that the configured controller\u002Fpipeline actually honors it.\n7. Communicate ETA in #incident-channel.\n8. When resolved, unfreeze and run smoke tests.\n```\n\n### Quality Regression Runbook (Hallucination Spike)\n\n```text\nTRIGGER: HighHallucinationRate or ModelQualityDrop fires\nRESPONDER: On-call AI engineer + ML lead\n\n1. Acknowledge alert. Open incident ticket.\n2. Identify scope:\n     - Which model version? Check deployment metadata.\n     - Which routes\u002Ftenants affected? Filter by labels in Grafana.\n3. Check recent changes:\n     - Model version promotion in last 24h?\n     - Prompt template changes in last 24h?\n     - Retrieval index rebuild in last 24h?\n4. If recent model change:\n     kubectl rollout undo deployment\u002Fllm-inference -n ai\n5. If recent prompt change:\n     Prepare a reviewed revert on a topic branch, verify it in staging, and use the protected deployment path; do not push a production rollback blindly.\n6. If authorized, increase redacted trace sampling for the affected route only after checking personal-data exposure and the remaining storage\u002Fcost budget:\n     kubectl set env deployment\u002Fllm-gateway TRACE_SAMPLE_RATE=1.0\n7. Run the existing version-bound evaluation suite against an authorized isolated copy of the model\u002Fprompt configuration using synthetic or redacted cases.\n   No run_evals.py helper is bundled; never describe a production call as offline.\n8. Confirm metrics return to baseline before closing.\n```\n\n### Token Cost Explosion Runbook\n\n```text\nTRIGGER: TokenCostExplosion fires\nRESPONDER: On-call platform engineer\n\n1. Identify top consumers:\n     Query: topk(10, sum(rate(llm_token_cost_dollars[15m])) by (tenant, model, route))\n2. Check for runaway loops:\n     - Agent retry storms (exponential token growth per request)\n     - Missing max_tokens caps on new routes\n     - Cache bypass due to config change\n3. Apply immediate caps:\n     kubectl patch configmap llm-quotas -n ai --patch '\n       data:\n         max_tokens_per_request: \"4096\"\n         rpm_limit: \"60\"\n     '\n4. Review existing cache isolation before any enablement: tenant\u002Fuser identity, permissions, invalidation and private-content retention must be enforced.\n   Do not turn on a generic shared semantic cache during an incident.\n5. If authorized, route only to an already approved fallback meeting the declared quality, privacy and cost constraints. Review the exact change before updating gateway configuration.\n6. Notify affected tenants of temporary limits.\n7. Open postmortem with cost attribution analysis.\n```\n\n## Escalation Procedures\n\n```text\nLevel 1 (0-15 min):  On-call AI platform engineer\nLevel 2 (15-30 min): AI platform team lead + affected product owner\nLevel 3 (30-60 min): Engineering director + security (if safety incident)\nLevel 4 (60+ min):   VP Engineering + legal (if compliance\u002Fdata incident)\n\nSafety incidents always start at Level 2 minimum.\nProvider-side incidents: open support ticket immediately at Level 1.\n```\n\n## Detection Queries (PromQL)\n\n```promql\n# Request success rate by model\n1 - (\n  sum(rate(llm_requests_total{status=\"error\"}[5m])) by (model)\n  \u002F sum(rate(llm_requests_total[5m])) by (model)\n)\n\n# Cost per successful answer\nsum(rate(llm_token_cost_dollars[5m])) by (route)\n\u002F sum(rate(llm_requests_total{status=\"success\"}[5m])) by (route)\n\n# Hallucination rate trend (1h window, 5m steps)\nrate(llm_hallucination_detected_total[1h])\n\u002F rate(llm_requests_total[1h])\n\n# Latency breakdown by stage\nhistogram_quantile(0.95, rate(llm_retrieval_duration_seconds_bucket[5m]))\nhistogram_quantile(0.95, rate(llm_generation_duration_seconds_bucket[5m]))\nhistogram_quantile(0.95, rate(llm_tool_execution_duration_seconds_bucket[5m]))\n\n# Tenant cost leaderboard\ntopk(10, sum(rate(llm_token_cost_dollars[1h])) by (tenant))\n```\n\n## Postmortem Requirements\n\n- Timeline with detector and responder timestamps\n- Blast radius by tenant and feature\n- Missed signals and alert tuning actions\n- Concrete hardening tasks with owners and due dates\n- Cost impact (dollars, tokens, affected requests)\n- Customer communication log\n\n## Postmortem Template\n\n```markdown\n## Incident Summary\n- **Severity**: SEVx\n- **Duration**: start_time - end_time (Xh Ym)\n- **Detection**: How was it detected? (alert \u002F customer report \u002F manual)\n- **Impact**: X tenants, Y requests, $Z cost\n\n## Timeline\n| Time (UTC) | Event |\n|------------|-------|\n| HH:MM | Alert fired |\n| HH:MM | Responder acknowledged |\n| HH:MM | Root cause identified |\n| HH:MM | Mitigation applied |\n| HH:MM | Incident resolved |\n\n## Root Cause\n[Description]\n\n## Action Items\n| Action | Owner | Due Date | Status |\n|--------|-------|----------|--------|\n| Tune alert threshold | @engineer | YYYY-MM-DD | Open |\n| Add fallback route | @platform | YYYY-MM-DD | Open |\n```\n\n## Chaos Engineering for AI Systems\n\nTest drills only in a disposable isolated harness or separately authorized staging environment. Use synthetic inputs and a predeclared fault schedule, bounded duration, costs, recovery and notification policy; never deploy a known-bad model or block a real provider in production to create a drill. Regularly test incident readiness:\n\n- **Provider failover drill**: block provider API at network level, verify fallback activates within SLO.\n- **Model rollback drill**: deploy known-bad model version, verify automated quality gate catches it.\n- **Cost cap drill**: simulate runaway token usage, verify quotas trigger before budget threshold.\n- **Cache failure drill**: disable semantic cache, verify system degrades gracefully.\n\n## Troubleshooting\n\n| Symptom | Check | Fix |\n|---------|-------|-----|\n| All requests timing out | Provider status page, DNS resolution | Enable fallback provider |\n| Gradual quality decline | Recent model\u002Fprompt deployments | Roll back to last known good |\n| Sudden cost spike | Per-tenant token usage dashboard | Apply emergency token caps |\n| Guardrail violations spike | Model version, prompt injection logs | Enable stricter input filtering |\n| Intermittent 503 errors | Pod restarts, GPU OOM events | Increase memory limits or reduce batch size |\n\n## Related Skills\n\n- incident-response (`incident-response`) - Standard incident process and evidence\n- alerting-oncall (`alerting-oncall`) - Paging and escalation policy\n- llm-cost-optimization (`llm-cost-optimization`) - Spend controls and efficiency patterns\n- agent-observability (`agent-observability`) - Instrument requests, traces, and costs\n- rag-observability-evals (`rag-observability-evals`) - RAG quality monitoring\n\n## Limitations\n\n- Guidance executes against real environments: confirm target, blast radius, and rollback plan before applying anything.\n- Never deploy to production without explicit approval. Docs-only import: upstream scripts and templates not bundled.\n\n### Example\n\n```bash\ngit status && git diff --stat\nkubectl diff -f manifest.yaml\n```\n\n> Adapted from [BagelHole\u002FDevOps-Security-Agent-Skills](https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills) (MIT); frontmatter, When to Use\u002FLimitations, and safety boundaries added for upstream compliance. Docs-only import: helper scripts and templates not bundled.\n",{"name":5,"risk":28,"source":54,"license":14,"category":34,"metadata":55,"date_added":58,"description":7,"source_repo":59,"source_type":30,"compatibility":60,"license_source":61},"https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills",{"author":56,"version":57},"devops-skills","1.0","2026-09-20","BagelHole\u002FDevOps-Security-Agent-Skills","Requires the relevant platform CLIs (kubectl, helm, terraform, git, CI runners) and authorized access to the target environment. Docs-only; helper scripts and templates not bundled.","https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills\u002Fblob\u002Fmain\u002FLICENSE","e8c4cb5cbafb600fab08ffdf4b23359c9f608d9d","skills\u002Fantigravity-ai-sre-incident-response",3,"https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections","https:\u002F\u002Fgithub.com\u002Fboisenoise","2026-11-10 14:20:15",{"source":8,"skillName":5,"commands":69,"prompt":79,"zipAvailable":80},[70,73,76],{"tool":71,"command":72},"npx","npx skills add boisenoise\u002Fskills-collections --skill ai-sre-incident-response",{"tool":74,"command":75},"bunx","bunx skills add boisenoise\u002Fskills-collections --skill ai-sre-incident-response",{"tool":77,"command":78},"pnpm","pnpm dlx skills add boisenoise\u002Fskills-collections --skill ai-sre-incident-response","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-ai-sre-incident-response\u002FSKILL.md",true,[],[83,109,135,160,179,202],{"skillNo":84,"name":85,"nameZh":6,"description":86,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":87,"licenseFileUrl":88,"sourceHtmlUrl":89,"categories":90,"keywords":91,"publishedAt":49},"01M6WNH92BHVFXHGPDZCPJ0AG0","web-access","Use Treg for public web or platform data only when existing search, browser and connected tools cannot cover the task, with explicit cost and data boundaries.","Apache-2.0","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-web-access\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-web-access\u002FSKILL.md",[],[92,37,93,94,95,96,97,98,99,100,101,33,102,103,104,28,105,106,107,108,24],"web","web access","official","development","browser","search","data","Tools","read","fetch","youtube","free tools","platform","free","cover","normal","when",{"skillNo":110,"name":111,"nameZh":6,"description":112,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":87,"licenseFileUrl":113,"sourceHtmlUrl":114,"categories":115,"keywords":121,"publishedAt":49},"01M6WNF9XAYAB2CZDDJBAKNM93","treg","Use the Treg CLI or MCP catalog to discover, price and call authorized external tools with team-scoped credentials and a user-approved budget.","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-treg\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-treg\u002FSKILL.md",[116],{"categoryNo":117,"categoryName":118,"taxonomyType":119,"taxonomyTypeDesc":120},"01JBSEEDCAT000000000000023","Skill与智能体开发","SCENARIO","使用场景",[111,122,123,35,94,95,46,99,124,125,126,127,128,129,130,28,131,132,133,134,24],"MCP","cli","Budget","credentials","team","catalog","use","call","discover","external tools","with","scoped","blob",{"skillNo":136,"name":137,"nameZh":6,"description":138,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":87,"licenseFileUrl":139,"sourceHtmlUrl":140,"categories":141,"keywords":147,"publishedAt":49},"01M6WN93MEAZ4E41DYAHJRPHQT","recruit","Turn a job description into an evidence-linked candidate shortlist through Treg, with job-related ranking and optional authorized business contact enrichment.","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-recruit\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-recruit\u002FSKILL.md",[142],{"categoryNo":143,"categoryName":144,"taxonomyType":145,"taxonomyTypeDesc":146},"01JBSEEDCAT000000000000038","人力资源","PROFESSION","适用职业",[137,35,94,46,44,148,149,150,151,152,28,153,154,155,132,156,157,158,159,134,111,24],"email","enrichment","business","evidence","phone","ranking","post","job description","for","person","candidate","reason",{"skillNo":161,"name":162,"nameZh":6,"description":163,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":87,"licenseFileUrl":164,"sourceHtmlUrl":165,"categories":166,"keywords":167,"publishedAt":49},"01M6WN8R5RXSMHM9B4D9FAFX42","rea-growth","Analyze a company’s public growth and content through Treg, distinguish observations from hypotheses, and produce an evidence-linked playbook within an approved budget.","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-rea-growth\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-rea-growth\u002FSKILL.md",[],[168,169,170,35,94,46,171,172,44,173,124,174,175,176,151,28,177,178,134,111,24],"growth","rea","rea growth","produce","content","playbook","marketing","analyze","public","observations","company",{"skillNo":180,"name":181,"nameZh":6,"description":182,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":87,"licenseFileUrl":183,"sourceHtmlUrl":184,"categories":185,"keywords":186,"publishedAt":49},"01M6WN2EVE3RQRGWAKBT4S7YT2","make-ugc","Create original AI UGC-style product videos through Treg: research hooks, approve a licensed presenter and voice, generate clips, add captions and report costs.","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-make-ugc\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-make-ugc\u002FSKILL.md",[],[187,188,35,189,94,190,46,191,192,193,194,195,28,196,197,198,199,200,201,134],"make","make ugc","research","create","Video","Captions","ads","short","hooks","head","style","niche","trending","talking-head","approve",{"skillNo":203,"name":204,"nameZh":6,"description":205,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":87,"licenseFileUrl":206,"sourceHtmlUrl":207,"categories":208,"keywords":209,"publishedAt":49},"01M6WN10ARAA63Y8K1YNGXJ2RM","lead-signals","Find and qualify public buying signals through Treg, rank fitting business accounts and optionally enrich authorized business contact information.","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-lead-signals\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fboisenoise\u002Fskills-collections\u002Fblob\u002Fmain\u002Fskills\u002Fantigravity-lead-signals\u002FSKILL.md",[],[210,211,212,35,94,46,213,214,150,215,216,28,217,218,219,220,221,222,223,134],"signals","lead","lead signals","how","find","ready","as","how to","qualify","people","information","enrich","changes","funding"]