kubeintellect 2.0.2__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kubeintellect-2.2.0/.gitignore +157 -0
- kubeintellect-2.2.0/PKG-INFO +55 -0
- kubeintellect-2.2.0/README.md +10 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/__init__.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/__main__.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/__init__.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/hitl.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/nodes/__init__.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/nodes/context_fetcher.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/nodes/coordinator.py +66 -8
- kubeintellect-2.2.0/app/agent/nodes/memory_loader.py +88 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/nodes/subagent.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/__init__.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/command_hardcoded_failure.yaml +1 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/containercreating_stuck.yaml +10 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/crashloopbackoff.yaml +14 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/createcontainerconfigerror.yaml +11 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/evicted.yaml +10 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/imagepullbackoff.yaml +11 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/init_container_failing.yaml +12 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/job_backoff_limit.yaml +8 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/loader.py +5 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/node_not_ready.yaml +11 -0
- kubeintellect-2.2.0/app/agent/playbooks/oomkilled.yaml +37 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/pending_resources.yaml +8 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/pending_scheduling.yaml +9 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/quota_exceeded.yaml +8 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/readiness_probe_failing.yaml +9 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/service_no_endpoints.yaml +10 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/service_unreachable.yaml +1 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/terminating_stuck.yaml +13 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/webhook_admission_rejected.yaml +9 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/state.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/workflow.py +42 -8
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/api/__init__.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/api/middleware.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/api/v1/__init__.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/api/v1/auth.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/api/v1/endpoints/__init__.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/api/v1/endpoints/auth.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/api/v1/endpoints/chat_completions.py +0 -0
- kubeintellect-2.2.0/app/api/v1/endpoints/detectors.py +97 -0
- kubeintellect-2.2.0/app/api/v1/endpoints/digest.py +19 -0
- kubeintellect-2.2.0/app/api/v1/endpoints/episodes.py +50 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/api/v1/endpoints/events.py +0 -0
- kubeintellect-2.2.0/app/api/v1/endpoints/findings.py +25 -0
- kubeintellect-2.2.0/app/api/v1/endpoints/health.py +24 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/api/v1/endpoints/namespaces.py +0 -0
- kubeintellect-2.2.0/app/api/v1/endpoints/postmortem.py +22 -0
- kubeintellect-2.2.0/app/api/v1/endpoints/preferences.py +70 -0
- kubeintellect-2.2.0/app/api/v1/endpoints/v5_status.py +39 -0
- kubeintellect-2.2.0/app/api/v1/router.py +30 -0
- kubeintellect-2.2.0/app/autonomy/__init__.py +7 -0
- kubeintellect-2.2.0/app/autonomy/blast_radius.py +52 -0
- kubeintellect-2.2.0/app/autonomy/budget.py +103 -0
- kubeintellect-2.2.0/app/autonomy/failure_domain.py +62 -0
- kubeintellect-2.2.0/app/autonomy/ladder.py +66 -0
- kubeintellect-2.2.0/app/autonomy/promotion_engine.py +99 -0
- kubeintellect-2.2.0/app/autonomy/promotion_fixtures.py +98 -0
- kubeintellect-2.2.0/app/autonomy/promotion_source.py +48 -0
- kubeintellect-2.2.0/app/autonomy/promotion_stats.py +246 -0
- kubeintellect-2.2.0/app/autonomy/spend_source.py +43 -0
- kubeintellect-2.2.0/app/autonomy/staged_propagation.py +54 -0
- kubeintellect-2.2.0/app/autonomy/watchtower.py +176 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/cli.py +9 -7
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/cluster_id.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/core/__init__.py +0 -0
- kubeintellect-2.2.0/app/core/config.py +599 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/core/llm.py +26 -10
- kubeintellect-2.2.0/app/core/version.py +68 -0
- kubeintellect-2.2.0/app/cortex/__init__.py +7 -0
- kubeintellect-2.2.0/app/cortex/briefs.py +140 -0
- kubeintellect-2.2.0/app/cortex/change_rca.py +80 -0
- kubeintellect-2.2.0/app/cortex/graph.py +499 -0
- kubeintellect-2.2.0/app/cortex/harness/__init__.py +32 -0
- kubeintellect-2.2.0/app/cortex/harness/runner.py +192 -0
- kubeintellect-2.2.0/app/cortex/harness/subagent.py +84 -0
- kubeintellect-2.2.0/app/cortex/model_router.py +51 -0
- kubeintellect-2.2.0/app/cortex/models.py +74 -0
- kubeintellect-2.2.0/app/cortex/responsiveness.py +120 -0
- kubeintellect-2.2.0/app/cortex/skills.py +46 -0
- kubeintellect-2.2.0/app/cortex/verify.py +146 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/db/__init__.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/db/audit.py +0 -0
- kubeintellect-2.2.0/app/db/flight_recorder.py +261 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/db/memory_store.py +32 -5
- kubeintellect-2.2.0/app/db/otel_spans.py +141 -0
- kubeintellect-2.2.0/app/db/schema.sql +456 -0
- kubeintellect-2.2.0/app/detectors/__init__.py +1 -0
- kubeintellect-2.2.0/app/detectors/agentic_gpu.py +67 -0
- kubeintellect-2.2.0/app/detectors/agentic_gpu_collector.py +72 -0
- kubeintellect-2.2.0/app/detectors/authoring.py +124 -0
- kubeintellect-2.2.0/app/detectors/detector_promotion.py +48 -0
- kubeintellect-2.2.0/app/detectors/engine.py +456 -0
- kubeintellect-2.2.0/app/detectors/models.py +158 -0
- kubeintellect-2.2.0/app/detectors/review.py +82 -0
- kubeintellect-2.2.0/app/detectors/rightsizing.py +82 -0
- kubeintellect-2.2.0/app/detectors/service.py +101 -0
- kubeintellect-2.2.0/app/digest/__init__.py +1 -0
- kubeintellect-2.2.0/app/digest/builder.py +159 -0
- kubeintellect-2.2.0/app/digest/postmortem.py +232 -0
- kubeintellect-2.2.0/app/eval/__init__.py +1 -0
- kubeintellect-2.2.0/app/eval/opsmembench.py +123 -0
- kubeintellect-2.2.0/app/eval/security_outcome.py +61 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/main.py +23 -0
- kubeintellect-2.2.0/app/memory/__init__.py +8 -0
- kubeintellect-2.2.0/app/memory/change_ledger.py +97 -0
- kubeintellect-2.2.0/app/memory/consolidation.py +224 -0
- kubeintellect-2.2.0/app/memory/episodes.py +406 -0
- kubeintellect-2.2.0/app/memory/file_plane.py +128 -0
- kubeintellect-2.2.0/app/memory/fleet_exchange.py +57 -0
- kubeintellect-2.2.0/app/memory/fleet_signal_store.py +40 -0
- kubeintellect-2.2.0/app/memory/fleet_signals.py +62 -0
- kubeintellect-2.2.0/app/memory/fleet_store_pg.py +54 -0
- kubeintellect-2.2.0/app/memory/kg.py +740 -0
- kubeintellect-2.2.0/app/memory/preferences.py +254 -0
- kubeintellect-2.2.0/app/memory/promotion.py +148 -0
- kubeintellect-2.2.0/app/memory/prospective.py +191 -0
- kubeintellect-2.2.0/app/memory/security.py +280 -0
- kubeintellect-2.2.0/app/memory/service.py +98 -0
- kubeintellect-2.2.0/app/memory/summaries.py +164 -0
- kubeintellect-2.2.0/app/memory/writeback.py +73 -0
- kubeintellect-2.2.0/app/sensorium/__init__.py +6 -0
- kubeintellect-2.2.0/app/sensorium/change_watchdog.py +96 -0
- kubeintellect-2.2.0/app/sensorium/k8s_watcher.py +182 -0
- kubeintellect-2.2.0/app/sensorium/observations.py +58 -0
- kubeintellect-2.2.0/app/sensorium/pre_capture.py +53 -0
- kubeintellect-2.2.0/app/sensorium/predictive_fusion.py +54 -0
- kubeintellect-2.2.0/app/sensorium/watchdog_dispatch.py +74 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/streaming/__init__.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/streaming/emitter.py +39 -82
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/tools/__init__.py +0 -0
- kubeintellect-2.2.0/app/tools/aci/__init__.py +26 -0
- kubeintellect-2.2.0/app/tools/aci/bounds.py +112 -0
- kubeintellect-2.2.0/app/tools/aci/fix_pr.py +56 -0
- kubeintellect-2.2.0/app/tools/aci/gitops.py +59 -0
- kubeintellect-2.2.0/app/tools/aci/models.py +96 -0
- kubeintellect-2.2.0/app/tools/aci/mutating.py +141 -0
- kubeintellect-2.2.0/app/tools/aci/postcondition.py +55 -0
- kubeintellect-2.2.0/app/tools/aci/read_verbs.py +212 -0
- kubeintellect-2.2.0/app/tools/aci/repair.py +60 -0
- kubeintellect-2.2.0/app/tools/aci/sandbox.py +59 -0
- kubeintellect-2.2.0/app/tools/aci/transactional.py +78 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/tools/helm_tool.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/tools/kubectl_errors.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/tools/kubectl_tool.py +94 -1
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/tools/loki_tool.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/tools/prometheus_tool.py +71 -50
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/tools/registry.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/utils/__init__.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/utils/logger.py +0 -1
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/utils/redact.py +0 -0
- {kubeintellect-2.0.2 → kubeintellect-2.2.0}/pyproject.toml +18 -42
- kubeintellect-2.0.2/.env.example +0 -244
- kubeintellect-2.0.2/PKG-INFO +0 -366
- kubeintellect-2.0.2/README.md +0 -322
- kubeintellect-2.0.2/app/agent/nodes/memory_loader.py +0 -31
- kubeintellect-2.0.2/app/agent/playbooks/oomkilled.yaml +0 -15
- kubeintellect-2.0.2/app/api/v1/endpoints/health.py +0 -15
- kubeintellect-2.0.2/app/api/v1/router.py +0 -16
- kubeintellect-2.0.2/app/core/config.py +0 -270
- kubeintellect-2.0.2/app/db/schema.sql +0 -124
- kubeintellect-2.0.2/overrides/main.html +0 -6
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
2
|
+
# .gitignore — KubeIntellect
|
|
3
|
+
# Universal build/junk/secret rules honoured by every git operating in this tree.
|
|
4
|
+
# NOTE: private research materials (design/, papers/, v5/, .claude/, …) are NOT
|
|
5
|
+
# listed here on purpose — they are versioned in the private git and kept out of
|
|
6
|
+
# the public remote by the repo-local allowlist in .git/info/exclude.
|
|
7
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
8
|
+
|
|
9
|
+
# ── Secrets ──────────────────────────────────────────────────────────────────
|
|
10
|
+
.env
|
|
11
|
+
.env.*
|
|
12
|
+
*.local
|
|
13
|
+
*.pem
|
|
14
|
+
*.key
|
|
15
|
+
*.pub
|
|
16
|
+
*.token
|
|
17
|
+
.envrc
|
|
18
|
+
.direnv/
|
|
19
|
+
credentials.json
|
|
20
|
+
secrets.yaml
|
|
21
|
+
secrets.yml
|
|
22
|
+
# Generated Kubernetes/Helm secret overlays (contain real credentials)
|
|
23
|
+
charts/kubeintellect/values-kind-secrets.yaml
|
|
24
|
+
values-prod.yaml
|
|
25
|
+
values-production.yaml
|
|
26
|
+
values-vm.yaml
|
|
27
|
+
values-aks.yaml
|
|
28
|
+
values-eks.yaml
|
|
29
|
+
values-gke.yaml
|
|
30
|
+
charts/*/charts/
|
|
31
|
+
|
|
32
|
+
# ── Python ───────────────────────────────────────────────────────────────────
|
|
33
|
+
__pycache__/
|
|
34
|
+
*.py[cod]
|
|
35
|
+
*$py.class
|
|
36
|
+
*.so
|
|
37
|
+
*.egg
|
|
38
|
+
*.egg-info/
|
|
39
|
+
.eggs/
|
|
40
|
+
develop-eggs/
|
|
41
|
+
.installed.cfg
|
|
42
|
+
MANIFEST
|
|
43
|
+
dist/
|
|
44
|
+
build/
|
|
45
|
+
wheels/
|
|
46
|
+
pip-wheel-metadata/
|
|
47
|
+
share/python-wheels/
|
|
48
|
+
pip-log.txt
|
|
49
|
+
pip-delete-this-directory.txt
|
|
50
|
+
.pybuilder/
|
|
51
|
+
*.manifest
|
|
52
|
+
*.spec
|
|
53
|
+
|
|
54
|
+
# ── Node / Next.js (v4/packages/kube-q/web — the browser demo front-end) ─────
|
|
55
|
+
node_modules/
|
|
56
|
+
.next/
|
|
57
|
+
out/
|
|
58
|
+
.turbo/
|
|
59
|
+
*.tsbuildinfo
|
|
60
|
+
npm-debug.log*
|
|
61
|
+
yarn-error.log*
|
|
62
|
+
|
|
63
|
+
# ── Virtual environments ─────────────────────────────────────────────────────
|
|
64
|
+
.venv/
|
|
65
|
+
venv/
|
|
66
|
+
env/
|
|
67
|
+
ENV/
|
|
68
|
+
|
|
69
|
+
# ── Test / coverage / type-checkers / linters ────────────────────────────────
|
|
70
|
+
.pytest_cache/
|
|
71
|
+
.tox/
|
|
72
|
+
.nox/
|
|
73
|
+
.coverage
|
|
74
|
+
.coverage.*
|
|
75
|
+
coverage.xml
|
|
76
|
+
nosetests.xml
|
|
77
|
+
*.cover
|
|
78
|
+
htmlcov/
|
|
79
|
+
.hypothesis/
|
|
80
|
+
.mypy_cache/
|
|
81
|
+
.dmypy.json
|
|
82
|
+
dmypy.json
|
|
83
|
+
.pyre/
|
|
84
|
+
.pytype/
|
|
85
|
+
.ruff_cache/
|
|
86
|
+
|
|
87
|
+
# ── Jupyter ──────────────────────────────────────────────────────────────────
|
|
88
|
+
.ipynb_checkpoints/
|
|
89
|
+
|
|
90
|
+
# ── Editors / IDEs ───────────────────────────────────────────────────────────
|
|
91
|
+
.vscode/
|
|
92
|
+
.idea/
|
|
93
|
+
*.iml
|
|
94
|
+
*.code-workspace
|
|
95
|
+
*.swp
|
|
96
|
+
*.swo
|
|
97
|
+
*~
|
|
98
|
+
|
|
99
|
+
# ── OS ───────────────────────────────────────────────────────────────────────
|
|
100
|
+
.DS_Store
|
|
101
|
+
._*
|
|
102
|
+
.Spotlight-V100
|
|
103
|
+
.Trashes
|
|
104
|
+
Thumbs.db
|
|
105
|
+
desktop.ini
|
|
106
|
+
|
|
107
|
+
# ── Logs / temp / runtime ────────────────────────────────────────────────────
|
|
108
|
+
*.log
|
|
109
|
+
*.tmp
|
|
110
|
+
*.temp
|
|
111
|
+
*.bak
|
|
112
|
+
*.orig
|
|
113
|
+
.cache/
|
|
114
|
+
logs/
|
|
115
|
+
*.pid
|
|
116
|
+
.server.log
|
|
117
|
+
.server.pid
|
|
118
|
+
.server.*.pid
|
|
119
|
+
.server.*.log
|
|
120
|
+
|
|
121
|
+
# ── MCP / Playwright tool artifacts ──────────────────────────────────────────
|
|
122
|
+
.playwright-mcp/
|
|
123
|
+
.mcp-*/
|
|
124
|
+
*.playwright.yml
|
|
125
|
+
*.playwright.png
|
|
126
|
+
*.cast
|
|
127
|
+
|
|
128
|
+
# ── Docs build output ────────────────────────────────────────────────────────
|
|
129
|
+
site/
|
|
130
|
+
_site/
|
|
131
|
+
.docusaurus/
|
|
132
|
+
|
|
133
|
+
# ── LaTeX build byproducts (sources kept) ────────────────────────────────────
|
|
134
|
+
*.aux
|
|
135
|
+
*.bbl
|
|
136
|
+
*.blg
|
|
137
|
+
*.fls
|
|
138
|
+
*.fdb_latexmk
|
|
139
|
+
*.out
|
|
140
|
+
*.spl
|
|
141
|
+
*.synctex.gz
|
|
142
|
+
*.toc
|
|
143
|
+
*.lof
|
|
144
|
+
*.lot
|
|
145
|
+
|
|
146
|
+
# ── Terraform / IaC ──────────────────────────────────────────────────────────
|
|
147
|
+
.terraform/
|
|
148
|
+
terraform.tfstate
|
|
149
|
+
terraform.tfstate.backup
|
|
150
|
+
*.tfstate
|
|
151
|
+
*.tfstate.*
|
|
152
|
+
|
|
153
|
+
# ── Snapcraft build byproducts (snap/snapcraft.yaml is tracked) ──────────────
|
|
154
|
+
*.snap
|
|
155
|
+
/parts/
|
|
156
|
+
/stage/
|
|
157
|
+
/prime/
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kubeintellect
|
|
3
|
+
Version: 2.2.0
|
|
4
|
+
Summary: AI-powered Kubernetes management — local server and CLI
|
|
5
|
+
Project-URL: Homepage, https://github.com/mskazemi/kubeintellect
|
|
6
|
+
Project-URL: Documentation, https://github.com/mskazemi/kubeintellect#readme
|
|
7
|
+
Project-URL: Issues, https://github.com/mskazemi/kubeintellect/issues
|
|
8
|
+
License: AGPL-3.0-or-later
|
|
9
|
+
Keywords: ai,devops,k8s,kubectl,kubernetes,llm
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: System Administrators
|
|
12
|
+
Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: System :: Systems Administration
|
|
16
|
+
Requires-Python: >=3.12
|
|
17
|
+
Requires-Dist: asyncpg>=0.30
|
|
18
|
+
Requires-Dist: fastapi>=0.115
|
|
19
|
+
Requires-Dist: httpx>=0.28
|
|
20
|
+
Requires-Dist: ki-protocol>=1.0.0
|
|
21
|
+
Requires-Dist: kube-q>=1.5.0
|
|
22
|
+
Requires-Dist: langchain-openai>=1.0
|
|
23
|
+
Requires-Dist: langchain>=1.0
|
|
24
|
+
Requires-Dist: langfuse>=4.5
|
|
25
|
+
Requires-Dist: langgraph-checkpoint-postgres>=3.0
|
|
26
|
+
Requires-Dist: langgraph-checkpoint-sqlite>=3.0
|
|
27
|
+
Requires-Dist: langgraph>=1.0
|
|
28
|
+
Requires-Dist: langsmith>=0.8
|
|
29
|
+
Requires-Dist: psycopg[binary,pool]>=3.3.4
|
|
30
|
+
Requires-Dist: pydantic-settings>=2.6
|
|
31
|
+
Requires-Dist: pydantic>=2.10
|
|
32
|
+
Requires-Dist: python-dotenv>=1.0
|
|
33
|
+
Requires-Dist: pyyaml>=6.0
|
|
34
|
+
Requires-Dist: uvicorn[standard]>=0.52.1
|
|
35
|
+
Provides-Extra: all
|
|
36
|
+
Requires-Dist: langfuse>=4.5; extra == 'all'
|
|
37
|
+
Requires-Dist: opentelemetry-api>=1.0; extra == 'all'
|
|
38
|
+
Requires-Dist: prometheus-fastapi-instrumentator>=7.1.0; extra == 'all'
|
|
39
|
+
Provides-Extra: metrics
|
|
40
|
+
Requires-Dist: prometheus-fastapi-instrumentator>=7.1.0; extra == 'metrics'
|
|
41
|
+
Provides-Extra: tracing
|
|
42
|
+
Requires-Dist: langfuse>=4.5; extra == 'tracing'
|
|
43
|
+
Requires-Dist: opentelemetry-api>=1.0; extra == 'tracing'
|
|
44
|
+
Description-Content-Type: text/markdown
|
|
45
|
+
|
|
46
|
+
# kubeintellect (server)
|
|
47
|
+
|
|
48
|
+
The KubeIntellect server — FastAPI app, agent graph, tools, memory, and the
|
|
49
|
+
`kubeintellect` CLI entry point. This is the `kubeintellect` distribution on
|
|
50
|
+
PyPI; it depends on its sibling workspace packages `kube-q` (the `kq` client)
|
|
51
|
+
and `ki-protocol` (the shared SSE wire protocol), so installing it gives the
|
|
52
|
+
full stack.
|
|
53
|
+
|
|
54
|
+
The Python module name remains `app` (V2 heritage); see the repo root README
|
|
55
|
+
for project documentation.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# kubeintellect (server)
|
|
2
|
+
|
|
3
|
+
The KubeIntellect server — FastAPI app, agent graph, tools, memory, and the
|
|
4
|
+
`kubeintellect` CLI entry point. This is the `kubeintellect` distribution on
|
|
5
|
+
PyPI; it depends on its sibling workspace packages `kube-q` (the `kq` client)
|
|
6
|
+
and `ki-protocol` (the shared SSE wire protocol), so installing it gives the
|
|
7
|
+
full stack.
|
|
8
|
+
|
|
9
|
+
The Python module name remains `app` (V2 heritage); see the repo root README
|
|
10
|
+
for project documentation.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -3,8 +3,15 @@ from __future__ import annotations
|
|
|
3
3
|
|
|
4
4
|
import re
|
|
5
5
|
import time
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
from langchain_core.messages import (
|
|
9
|
+
AIMessage,
|
|
10
|
+
BaseMessage,
|
|
11
|
+
HumanMessage,
|
|
12
|
+
SystemMessage,
|
|
13
|
+
ToolMessage,
|
|
14
|
+
)
|
|
8
15
|
from langchain_core.runnables import RunnableConfig
|
|
9
16
|
from langgraph.errors import GraphInterrupt
|
|
10
17
|
|
|
@@ -246,8 +253,11 @@ def _extract_plan(messages: list[BaseMessage]) -> tuple[list[PlanStep], list[Bas
|
|
|
246
253
|
|
|
247
254
|
# ── Coordinator system prompt ─────────────────────────────────────────────────
|
|
248
255
|
|
|
249
|
-
|
|
250
|
-
|
|
256
|
+
# Raw string on purpose. The prompt quotes jsonpath separators verbatim — `{"\n"}`,
|
|
257
|
+
# `{"\t"}` and a lone backslash. Interpreted, those became a real newline and tab, which
|
|
258
|
+
# split the example kubectl commands mid-line before the model ever saw them, and the
|
|
259
|
+
# lone `\` raised a SyntaxWarning that Python will eventually make a SyntaxError.
|
|
260
|
+
_COORDINATOR_SYSTEM = r"""You are KubeIntellect, an expert Kubernetes operations AI.
|
|
251
261
|
|
|
252
262
|
You have access to four tools:
|
|
253
263
|
- run_kubectl: run any kubectl command against the cluster
|
|
@@ -633,7 +643,13 @@ def _playbooks_block(state: AgentState) -> str:
|
|
|
633
643
|
return "\n".join(sections)
|
|
634
644
|
|
|
635
645
|
|
|
636
|
-
|
|
646
|
+
# `Optional[RunnableConfig]`, not `RunnableConfig | None`: this module uses
|
|
647
|
+
# `from __future__ import annotations`, so LangGraph sees the annotation as a string and
|
|
648
|
+
# matches it against a fixed list ("RunnableConfig", "Optional[RunnableConfig]", …) in
|
|
649
|
+
# langgraph._internal._runnable.KWARGS_CONFIG_KEYS. The PEP-604 spelling is not on that
|
|
650
|
+
# list, so it would silently stop the run config being injected — taking user_role and
|
|
651
|
+
# hitl_bypass with it. Covered by tests/test_workflow_config_injection.py.
|
|
652
|
+
async def coordinator(state: AgentState, config: Optional[RunnableConfig] = None) -> dict: # noqa: UP045
|
|
637
653
|
"""
|
|
638
654
|
Coordinator node. Always returns a plain state-update dict — never Send objects.
|
|
639
655
|
|
|
@@ -667,7 +683,7 @@ async def coordinator(state: AgentState, config: RunnableConfig = None) -> dict:
|
|
|
667
683
|
last_user_msg = ""
|
|
668
684
|
for m in reversed(state.get("messages", [])):
|
|
669
685
|
if hasattr(m, "type") and m.type == "human":
|
|
670
|
-
last_user_msg = m.content[:120]
|
|
686
|
+
last_user_msg = m.content[:120] if isinstance(m.content, str) else ""
|
|
671
687
|
break
|
|
672
688
|
logger.debug(f"coordinator: invoking LLM user={user_id} session={session_id} msg={last_user_msg!r}")
|
|
673
689
|
|
|
@@ -728,7 +744,7 @@ async def coordinator(state: AgentState, config: RunnableConfig = None) -> dict:
|
|
|
728
744
|
return result
|
|
729
745
|
|
|
730
746
|
|
|
731
|
-
async def _direct_answer(state: AgentState, config: RunnableConfig = None) -> dict:
|
|
747
|
+
async def _direct_answer(state: AgentState, config: Optional[RunnableConfig] = None) -> dict: # noqa: UP045
|
|
732
748
|
"""Run coordinator LLM with tools for simple queries."""
|
|
733
749
|
from langgraph.prebuilt import create_react_agent
|
|
734
750
|
|
|
@@ -874,6 +890,14 @@ def _extract_mutation_pairs(messages: list[BaseMessage]) -> list[dict]:
|
|
|
874
890
|
return pairs
|
|
875
891
|
|
|
876
892
|
|
|
893
|
+
def _last_user_text(state: AgentState) -> str:
|
|
894
|
+
"""Most recent human message content (for episode trigger_detail)."""
|
|
895
|
+
for message in reversed(state.get("messages", [])):
|
|
896
|
+
if isinstance(message, HumanMessage) and isinstance(message.content, str):
|
|
897
|
+
return message.content
|
|
898
|
+
return ""
|
|
899
|
+
|
|
900
|
+
|
|
877
901
|
def _outcome_key(state: AgentState, namespace: str | None) -> str:
|
|
878
902
|
"""Build a stable structured pattern key (R2)."""
|
|
879
903
|
cluster_id = state.get("cluster_id") or "unknown"
|
|
@@ -1065,6 +1089,23 @@ def _maybe_record_direct_outcome(state: AgentState, messages: list[BaseMessage])
|
|
|
1065
1089
|
playbooks_matched=list(state.get("matched_playbooks") or []),
|
|
1066
1090
|
created_by_role=state.get("user_role"),
|
|
1067
1091
|
))
|
|
1092
|
+
# V4 hippocampus: the same outcome becomes an L1 episode.
|
|
1093
|
+
from app.memory.episodes import write_episode
|
|
1094
|
+
_asyncio.create_task(write_episode(
|
|
1095
|
+
cluster_id=cluster_id,
|
|
1096
|
+
namespace=namespace,
|
|
1097
|
+
trigger_kind="user_query",
|
|
1098
|
+
trigger_detail=_last_user_text(state)[:300],
|
|
1099
|
+
summary=f"Direct fix in ns={namespace}: {key[:200]}",
|
|
1100
|
+
root_cause=key[:240],
|
|
1101
|
+
actions=pairs,
|
|
1102
|
+
outcome=feedback,
|
|
1103
|
+
verified=verified,
|
|
1104
|
+
confidence=confidence,
|
|
1105
|
+
playbooks=list(state.get("matched_playbooks") or []),
|
|
1106
|
+
created_by_role=state.get("user_role"),
|
|
1107
|
+
request_id=state.get("session_id"),
|
|
1108
|
+
))
|
|
1068
1109
|
logger.info(
|
|
1069
1110
|
f"rca_outcome_written session={state.get('session_id', '-')} "
|
|
1070
1111
|
f"cluster={cluster_id} confidence={confidence:.2f}",
|
|
@@ -1122,7 +1163,9 @@ Synthesize these into a single root-cause analysis. Respond with ONLY a JSON obj
|
|
|
1122
1163
|
[SystemMessage(content=_COORDINATOR_SYSTEM), HumanMessage(content=synthesis_prompt)]
|
|
1123
1164
|
)
|
|
1124
1165
|
|
|
1125
|
-
|
|
1166
|
+
# `.text` flattens a content-block list (some providers never return a bare
|
|
1167
|
+
# string) into the concatenated text; `.content.strip()` would raise on one.
|
|
1168
|
+
raw = response.text.strip()
|
|
1126
1169
|
if raw.startswith("```"):
|
|
1127
1170
|
raw = raw.split("```")[1]
|
|
1128
1171
|
if raw.startswith("json"):
|
|
@@ -1167,6 +1210,21 @@ Synthesize these into a single root-cause analysis. Respond with ONLY a JSON obj
|
|
|
1167
1210
|
recommended_fix=rca.recommended_fix,
|
|
1168
1211
|
outcome_feedback=None,
|
|
1169
1212
|
))
|
|
1213
|
+
from app.cluster_id import get_cluster_id as _gcid
|
|
1214
|
+
from app.memory.episodes import write_episode
|
|
1215
|
+
_asyncio.create_task(write_episode(
|
|
1216
|
+
cluster_id=state.get("cluster_id") or _gcid(),
|
|
1217
|
+
trigger_kind="user_query",
|
|
1218
|
+
trigger_detail=_last_user_text(state)[:300],
|
|
1219
|
+
summary=summary[:1200],
|
|
1220
|
+
root_cause=rca.root_cause,
|
|
1221
|
+
outcome="report_only",
|
|
1222
|
+
verified=None,
|
|
1223
|
+
confidence=rca.confidence,
|
|
1224
|
+
playbooks=list(state.get("matched_playbooks") or []),
|
|
1225
|
+
created_by_role=state.get("user_role"),
|
|
1226
|
+
request_id=state.get("session_id"),
|
|
1227
|
+
))
|
|
1170
1228
|
logger.info(
|
|
1171
1229
|
f"rca_outcome_written session={state.get('session_id', '-')} "
|
|
1172
1230
|
f"confidence={rca.confidence:.2f}",
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""memory_loader node — async DB reads pinned into coordinator SystemMessage."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
|
|
6
|
+
from app.agent.state import AgentState
|
|
7
|
+
from app.db.memory_store import load_memory_context
|
|
8
|
+
from app.streaming.emitter import StatusEvent, emit
|
|
9
|
+
from app.utils.logger import get_logger
|
|
10
|
+
|
|
11
|
+
logger = get_logger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
async def _hierarchy_context(state: AgentState) -> str:
|
|
15
|
+
"""V4 hippocampus injection: similar episodes (L1) + recent KG changes (L2).
|
|
16
|
+
|
|
17
|
+
Budget-bounded and latency-tracked (exit gate: < 200 ms p95). Returns ''
|
|
18
|
+
when the hierarchy is inactive or has nothing relevant.
|
|
19
|
+
"""
|
|
20
|
+
from app.core.config import settings
|
|
21
|
+
from app.memory import episodes, kg
|
|
22
|
+
from app.memory.service import memory_active
|
|
23
|
+
|
|
24
|
+
if not memory_active():
|
|
25
|
+
return ""
|
|
26
|
+
started = time.perf_counter()
|
|
27
|
+
try:
|
|
28
|
+
from app.cluster_id import get_cluster_id
|
|
29
|
+
cluster_id = get_cluster_id()
|
|
30
|
+
except Exception:
|
|
31
|
+
cluster_id = "unknown"
|
|
32
|
+
|
|
33
|
+
query_text = ""
|
|
34
|
+
for message in reversed(state.get("messages", [])):
|
|
35
|
+
content = getattr(message, "content", None)
|
|
36
|
+
if isinstance(content, str) and content.strip():
|
|
37
|
+
query_text = content
|
|
38
|
+
break
|
|
39
|
+
|
|
40
|
+
recalled = await episodes.recall_episodes(query_text, cluster_id, k=3)
|
|
41
|
+
parts = []
|
|
42
|
+
block = episodes.render_recall_block(recalled)
|
|
43
|
+
if block:
|
|
44
|
+
parts.append(block)
|
|
45
|
+
|
|
46
|
+
# Memory V5 P8 (spec R7): zoom out — inject matching theme summaries so a query gets
|
|
47
|
+
# cross-incident context ("this whole theme keeps failing"), not just the top-k episodes.
|
|
48
|
+
themes = []
|
|
49
|
+
if settings.MEMORY_SUMMARY_TREE:
|
|
50
|
+
from app.memory import summaries
|
|
51
|
+
themes = await summaries.recall_theme_summaries(query_text, cluster_id, k=2)
|
|
52
|
+
theme_block = summaries.render_summaries_block(themes)
|
|
53
|
+
if theme_block:
|
|
54
|
+
parts.append(theme_block)
|
|
55
|
+
|
|
56
|
+
changes = await kg.recent_changes_block(cluster_id, minutes=15, limit=12)
|
|
57
|
+
if changes:
|
|
58
|
+
parts.append("## Recent cluster changes (last 15m)\n" + changes)
|
|
59
|
+
|
|
60
|
+
elapsed_ms = (time.perf_counter() - started) * 1000
|
|
61
|
+
logger.info(
|
|
62
|
+
f"memory_hierarchy_injected ms={elapsed_ms:.1f} "
|
|
63
|
+
f"episodes={len(recalled)} themes={len(themes)}"
|
|
64
|
+
)
|
|
65
|
+
return "\n\n".join(parts)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
async def memory_loader(state: AgentState) -> dict:
|
|
69
|
+
"""Load user prefs, failure hints, past RCA, runbooks into memory_context."""
|
|
70
|
+
session_id = state["session_id"]
|
|
71
|
+
user_id = state["user_id"]
|
|
72
|
+
|
|
73
|
+
await emit(session_id, StatusEvent(
|
|
74
|
+
phase="loading",
|
|
75
|
+
message="Loading conversation context…",
|
|
76
|
+
session_id=session_id,
|
|
77
|
+
))
|
|
78
|
+
|
|
79
|
+
logger.debug(f"memory_loader: loading context for user={user_id} session={session_id}")
|
|
80
|
+
|
|
81
|
+
context = await load_memory_context(user_id=user_id, session_id=session_id)
|
|
82
|
+
hierarchy = await _hierarchy_context(state)
|
|
83
|
+
if hierarchy:
|
|
84
|
+
context = f"{context}\n\n{hierarchy}" if context else hierarchy
|
|
85
|
+
|
|
86
|
+
# findings=None triggers _findings_reducer's reset path, clearing any
|
|
87
|
+
# stale findings that accumulated during a previous RCA in this session.
|
|
88
|
+
return {"memory_context": context, "findings": None}
|
|
File without changes
|
|
File without changes
|
{kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/containercreating_stuck.yaml
RENAMED
|
@@ -1,4 +1,14 @@
|
|
|
1
1
|
name: ContainerCreatingStuck
|
|
2
|
+
detect:
|
|
3
|
+
watch_predicates:
|
|
4
|
+
- kind: Pod
|
|
5
|
+
status_regex: "^ContainerCreating$"
|
|
6
|
+
- kind: Event
|
|
7
|
+
reason_regex: "^(FailedMount|FailedAttachVolume)$"
|
|
8
|
+
message_regex: "MountVolume|FailedMount|attachdetach|VolumeBinding|unbound.*PersistentVolumeClaim"
|
|
9
|
+
promql:
|
|
10
|
+
- 'min_over_time(kube_pod_container_status_waiting_reason{reason="ContainerCreating"}[5m]) == 1'
|
|
11
|
+
debounce_seconds: 300
|
|
2
12
|
triggers:
|
|
3
13
|
- pod_status_regex: "ContainerCreating"
|
|
4
14
|
- event_message_regex: "FailedMount|MountVolume|attach|VolumeBinding"
|
|
@@ -1,4 +1,18 @@
|
|
|
1
1
|
name: CrashLoopBackOff
|
|
2
|
+
detect:
|
|
3
|
+
watch_predicates:
|
|
4
|
+
- kind: Pod
|
|
5
|
+
status_regex: "^CrashLoopBackOff$" # anchored: init-container variant
|
|
6
|
+
# ("Init:CrashLoopBackOff") is owned
|
|
7
|
+
# by InitContainerFailing
|
|
8
|
+
- kind: Event
|
|
9
|
+
reason_regex: "^BackOff$"
|
|
10
|
+
message_regex: "Back-off restarting failed container"
|
|
11
|
+
involved_kind: Pod
|
|
12
|
+
promql:
|
|
13
|
+
- 'kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"} == 1'
|
|
14
|
+
- 'increase(kube_pod_container_status_restarts_total[10m]) > 3'
|
|
15
|
+
debounce_seconds: 60
|
|
2
16
|
triggers:
|
|
3
17
|
- pod_status_regex: "CrashLoopBackOff"
|
|
4
18
|
- event_reason_regex: "BackOff"
|
{kubeintellect-2.0.2 → kubeintellect-2.2.0}/app/agent/playbooks/createcontainerconfigerror.yaml
RENAMED
|
@@ -1,4 +1,15 @@
|
|
|
1
1
|
name: CreateContainerConfigError
|
|
2
|
+
detect:
|
|
3
|
+
watch_predicates:
|
|
4
|
+
- kind: Pod
|
|
5
|
+
status_regex: "CreateContainerConfigError|CreateContainerError"
|
|
6
|
+
- kind: Event
|
|
7
|
+
reason_regex: "^Failed$"
|
|
8
|
+
message_regex: "couldn't find key|not found in Secret|not found in ConfigMap|secret .* not found|configmap .* not found"
|
|
9
|
+
involved_kind: Pod
|
|
10
|
+
promql:
|
|
11
|
+
- 'kube_pod_container_status_waiting_reason{reason=~"CreateContainerConfigError|CreateContainerError"} == 1'
|
|
12
|
+
debounce_seconds: 60
|
|
2
13
|
triggers:
|
|
3
14
|
- pod_status_regex: "CreateContainerConfigError|CreateContainerError"
|
|
4
15
|
- event_reason_regex: "Failed"
|
|
@@ -1,4 +1,14 @@
|
|
|
1
1
|
name: Evicted
|
|
2
|
+
detect:
|
|
3
|
+
watch_predicates:
|
|
4
|
+
- kind: Pod
|
|
5
|
+
status_regex: "^Evicted$"
|
|
6
|
+
- kind: Event
|
|
7
|
+
reason_regex: "^Evicted$"
|
|
8
|
+
message_regex: "evicted|disk.*pressure|memory.*pressure|node.*pressure|low.*inodes"
|
|
9
|
+
promql:
|
|
10
|
+
- 'kube_pod_status_reason{reason="Evicted"} == 1'
|
|
11
|
+
debounce_seconds: 30
|
|
2
12
|
triggers:
|
|
3
13
|
- pod_status_regex: "Evicted"
|
|
4
14
|
- event_reason_regex: "Evicted|Evicting"
|
|
@@ -1,4 +1,15 @@
|
|
|
1
1
|
name: ImagePullBackOff
|
|
2
|
+
detect:
|
|
3
|
+
watch_predicates:
|
|
4
|
+
- kind: Pod
|
|
5
|
+
status_regex: "ImagePullBackOff|ErrImagePull|InvalidImageName"
|
|
6
|
+
- kind: Event
|
|
7
|
+
reason_regex: "^(Failed|ErrImagePull|BackOff)$"
|
|
8
|
+
message_regex: "Failed to pull image|Back-off pulling image"
|
|
9
|
+
involved_kind: Pod
|
|
10
|
+
promql:
|
|
11
|
+
- 'kube_pod_container_status_waiting_reason{reason=~"ErrImagePull|ImagePullBackOff|InvalidImageName"} == 1'
|
|
12
|
+
debounce_seconds: 90
|
|
2
13
|
triggers:
|
|
3
14
|
- pod_status_regex: "ImagePullBackOff|ErrImagePull"
|
|
4
15
|
- event_reason_regex: "Failed|ErrImagePull"
|
|
@@ -1,4 +1,16 @@
|
|
|
1
1
|
name: InitContainerFailing
|
|
2
|
+
detect:
|
|
3
|
+
watch_predicates:
|
|
4
|
+
- kind: Pod
|
|
5
|
+
status_regex: "^Init:(CrashLoopBackOff|Error)$" # immediate-fire arm
|
|
6
|
+
- kind: Pod
|
|
7
|
+
status_regex: "^Init:[0-9]+/[0-9]+$" # stuck-progress arm — needs debounce
|
|
8
|
+
- kind: Event
|
|
9
|
+
reason_regex: "^BackOff$"
|
|
10
|
+
message_regex: "back-off restarting failed init|init container"
|
|
11
|
+
promql:
|
|
12
|
+
- 'increase(kube_pod_init_container_status_restarts_total[10m]) > 3'
|
|
13
|
+
debounce_seconds: 180
|
|
2
14
|
triggers:
|
|
3
15
|
- pod_status_regex: "Init:[0-9]+/[0-9]+"
|
|
4
16
|
- event_reason_regex: "BackOff"
|
|
@@ -1,4 +1,12 @@
|
|
|
1
1
|
name: JobBackoffLimitExceeded
|
|
2
|
+
detect:
|
|
3
|
+
watch_predicates:
|
|
4
|
+
- kind: Event
|
|
5
|
+
reason_regex: "^BackoffLimitExceeded$"
|
|
6
|
+
involved_kind: Job
|
|
7
|
+
promql:
|
|
8
|
+
- 'kube_job_failed{condition="true"} == 1'
|
|
9
|
+
debounce_seconds: 0
|
|
2
10
|
triggers:
|
|
3
11
|
- event_reason_regex: "BackoffLimitExceeded"
|
|
4
12
|
- event_message_regex: "backoff limit.*exceeded|job.*failed.*backoff|reached.*backoff limit"
|
|
@@ -24,6 +24,7 @@ from typing import Iterable
|
|
|
24
24
|
|
|
25
25
|
import yaml
|
|
26
26
|
|
|
27
|
+
from app.detectors.models import DetectBlock, parse_detect_block
|
|
27
28
|
from app.utils.logger import get_logger
|
|
28
29
|
|
|
29
30
|
logger = get_logger(__name__)
|
|
@@ -45,6 +46,9 @@ class Playbook:
|
|
|
45
46
|
investigation_steps: tuple[str, ...] = field(default_factory=tuple)
|
|
46
47
|
expected_evidence: tuple[str, ...] = field(default_factory=tuple)
|
|
47
48
|
recommended_fix_template: str = ""
|
|
49
|
+
# Schema v2 (ADR-006): compiled zero-token detection predicate.
|
|
50
|
+
# None = LLM/triage-detected only.
|
|
51
|
+
detect: DetectBlock | None = None
|
|
48
52
|
|
|
49
53
|
|
|
50
54
|
def _compile_trigger(raw: dict) -> Trigger:
|
|
@@ -78,6 +82,7 @@ def _load_one(path: Path) -> Playbook:
|
|
|
78
82
|
investigation_steps=steps,
|
|
79
83
|
expected_evidence=evidence,
|
|
80
84
|
recommended_fix_template=fix,
|
|
85
|
+
detect=parse_detect_block(name, data.get("detect")),
|
|
81
86
|
)
|
|
82
87
|
|
|
83
88
|
|
|
@@ -1,4 +1,15 @@
|
|
|
1
1
|
name: NodeNotReady
|
|
2
|
+
detect:
|
|
3
|
+
watch_predicates:
|
|
4
|
+
- kind: Node
|
|
5
|
+
status_regex: "NotReady|Unknown" # matches "NotReady" and
|
|
6
|
+
# "NotReady,SchedulingDisabled"
|
|
7
|
+
- kind: Event
|
|
8
|
+
reason_regex: "^(NodeNotReady|NodeHasDiskPressure|NodeHasMemoryPressure|KubeletNotReady)$"
|
|
9
|
+
promql:
|
|
10
|
+
- 'kube_node_status_condition{condition="Ready",status="true"} == 0'
|
|
11
|
+
- 'kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure",condition!="Ready",status="true"} == 1'
|
|
12
|
+
debounce_seconds: 300
|
|
2
13
|
triggers:
|
|
3
14
|
- event_reason_regex: "NodeNotReady|NodeHasDiskPressure|NodeHasMemoryPressure|KubeletNotReady"
|
|
4
15
|
- event_message_regex: "node.*not ready|kubelet.*not posting|node.*unreachable|node.*condition.*unknown"
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
name: OOMKilled
|
|
2
|
+
detect:
|
|
3
|
+
watch_predicates:
|
|
4
|
+
- kind: Pod
|
|
5
|
+
status_regex: "^OOMKilled$" # visible only briefly in STATUS
|
|
6
|
+
- kind: Event
|
|
7
|
+
reason_regex: "^OOMKilling$" # node-scoped kubelet event
|
|
8
|
+
promql:
|
|
9
|
+
- >-
|
|
10
|
+
(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"} == 1)
|
|
11
|
+
and on(namespace, pod, container)
|
|
12
|
+
(increase(kube_pod_container_status_restarts_total[15m]) > 0)
|
|
13
|
+
trend_predicates: # ADR-010 — fire BEFORE the kill
|
|
14
|
+
- metric: >-
|
|
15
|
+
sum by (namespace, pod) (container_memory_working_set_bytes{container!=""})
|
|
16
|
+
/ sum by (namespace, pod) (kube_pod_container_resource_limits{resource="memory"})
|
|
17
|
+
threshold: 1.0 # ratio of working-set to limit
|
|
18
|
+
window_minutes: 30
|
|
19
|
+
projection_horizon_minutes: 60
|
|
20
|
+
fire_if_eta_within_minutes: 30
|
|
21
|
+
direction: rising
|
|
22
|
+
min_r2: 0.6
|
|
23
|
+
debounce_seconds: 60
|
|
24
|
+
triggers:
|
|
25
|
+
- pod_status_regex: "OOMKilled"
|
|
26
|
+
- event_reason_regex: "OOMKilling"
|
|
27
|
+
investigation_steps:
|
|
28
|
+
- "kubectl describe pod <pod> -n <ns> — confirm Last State.Reason=OOMKilled and check resources.limits.memory."
|
|
29
|
+
- "query_prometheus container_memory_working_set_bytes for the pod over the last 30 minutes — find the spike."
|
|
30
|
+
- "kubectl logs <pod> -n <ns> --previous --tail=50 — look for memory-pressure messages just before the kill."
|
|
31
|
+
expected_evidence:
|
|
32
|
+
- "Last State.Reason=OOMKilled"
|
|
33
|
+
- "Memory usage approached or exceeded limits.memory just before termination"
|
|
34
|
+
- "Application logs show heavy allocation right before death"
|
|
35
|
+
recommended_fix_template: |
|
|
36
|
+
Container was killed for exceeding its memory limit. Increase resources.limits.memory,
|
|
37
|
+
add a memory-aware GC tuning, or fix the underlying leak. Suggested fix: <ACTION>.
|