kubesift 2.0.1__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kubesift-2.1.0/.github/workflows/ai-eval.yml +56 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/PKG-INFO +4 -2
- {kubesift-2.0.1 → kubesift-2.1.0}/README.md +3 -1
- {kubesift-2.0.1 → kubesift-2.1.0}/deploy/helm/kubesift/Chart.yaml +2 -2
- {kubesift-2.0.1 → kubesift-2.1.0}/docs/configuration.md +9 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/docs/releasing.md +2 -0
- kubesift-2.1.0/scripts/eval_ai.py +124 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/__init__.py +1 -1
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/analysis/engine.py +24 -10
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/cli.py +12 -9
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/cluster/client.py +78 -3
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/cluster/snapshot.py +2 -1
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/server/app.py +25 -5
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/server/static/index.html +17 -2
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/store.py +4 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/test_analysis.py +52 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/test_client.py +45 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/test_interfaces.py +35 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/.dockerignore +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/.gitattributes +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/.github/workflows/ci.yml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/.github/workflows/pages.yml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/.github/workflows/release.yml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/.gitignore +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/.pre-commit-hooks.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/CONTRIBUTING.md +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/Dockerfile +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/LICENSE +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/SECURITY.md +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/action.yml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/deploy/helm/kubesift/templates/NOTES.txt +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/deploy/helm/kubesift/templates/_helpers.tpl +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/deploy/helm/kubesift/templates/deployment.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/deploy/helm/kubesift/templates/rbac.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/deploy/helm/kubesift/templates/secret.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/deploy/helm/kubesift/templates/service.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/deploy/helm/kubesift/values.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/docs/architecture.md +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/docs/checks.md +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/docs/mcp.md +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/docs/preflight.md +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/docs/production.md +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/docs/slack.md +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/pyproject.toml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/scripts/e2e.sh +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/scripts/e2e_check.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/scripts/gen_check_docs.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/analysis/__init__.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/analysis/changes.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/analysis/correlate.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/analysis/describe.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/analysis/llm.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/__init__.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/base.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/config_refs.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/events.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/hygiene.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/lifecycle.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/network.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/nodes.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/operations.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/pods.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/storage.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/checks/workloads.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/cluster/__init__.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/config.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/cost.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/01-crashloop-missing-env.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/02-imagepull-wrong-tag.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/03-oomkilled-low-memory.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/04-service-selector-mismatch.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/05-missing-probe-crashloop.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/06-pending-insufficient-resources.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/07-missing-secret.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/08-pvc-missing-storageclass.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/09-netpol-blocks-dns.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/10a-rollout-good.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/10b-rollout-bad.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drill_scenarios/__init__.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/drills.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/knowledge.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/leader.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/lint.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/mcp_server.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/models.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/notify.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/postmortem.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/preflight.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/redact.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/remediate.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/report.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/runbooks.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/server/__init__.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/slack.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/timetravel.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/upgrade.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/src/kubesift/util.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/01-crashloop-missing-env.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/02-imagepull-wrong-tag.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/03-oomkilled-low-memory.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/04-service-selector-mismatch.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/05-missing-probe-crashloop.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/06-pending-insufficient-resources.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/07-missing-secret.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/08-pvc-missing-storageclass.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/09-netpol-blocks-dns.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/10a-rollout-good.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/10b-rollout-bad.yaml +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/apply-scenario.sh +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/cleanup.sh +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/test-scenarios/expected.json +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/conftest.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/fixtures.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/test_checks.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/test_features.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/test_ha.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/test_hardening.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/test_lint_redact.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/test_llm.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/test_more_checks.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/tests/test_production.py +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/website/README.md +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/website/favicon.svg +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/website/index.html +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/website/main.js +0 -0
- {kubesift-2.0.1 → kubesift-2.1.0}/website/styles.css +0 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
name: AI eval
|
|
2
|
+
|
|
3
|
+
# Grades a real LLM on the demo broken cluster (scripts/eval_ai.py). Run it by hand from the Actions tab, for
|
|
4
|
+
# example before a release or after changing the prompt. It needs the provider's API key as a repository secret
|
|
5
|
+
# (ANTHROPIC_API_KEY, OPENAI_API_KEY or OPENROUTER_API_KEY); without one it skips instead of failing.
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
inputs:
|
|
10
|
+
provider:
|
|
11
|
+
description: LLM provider
|
|
12
|
+
type: choice
|
|
13
|
+
options: [anthropic, openai, openrouter]
|
|
14
|
+
default: anthropic
|
|
15
|
+
model:
|
|
16
|
+
description: Model name (empty = the provider's default)
|
|
17
|
+
required: false
|
|
18
|
+
default: ""
|
|
19
|
+
|
|
20
|
+
permissions:
|
|
21
|
+
contents: read
|
|
22
|
+
|
|
23
|
+
jobs:
|
|
24
|
+
eval:
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
timeout-minutes: 30
|
|
27
|
+
env:
|
|
28
|
+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
29
|
+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
30
|
+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
|
31
|
+
PROVIDER: ${{ inputs.provider }}
|
|
32
|
+
MODEL: ${{ inputs.model }}
|
|
33
|
+
steps:
|
|
34
|
+
- uses: actions/checkout@v4
|
|
35
|
+
- uses: actions/setup-python@v5
|
|
36
|
+
with:
|
|
37
|
+
python-version: "3.12"
|
|
38
|
+
- run: pip install -e ".[anthropic]"
|
|
39
|
+
- name: Grade the model
|
|
40
|
+
shell: bash # pipefail, so a failing grade is not hidden by tee
|
|
41
|
+
run: |
|
|
42
|
+
key_var="$(echo "$PROVIDER" | tr '[:lower:]' '[:upper:]')_API_KEY"
|
|
43
|
+
if [ -z "${!key_var}" ]; then
|
|
44
|
+
echo "::notice::No $key_var secret; add it under Settings > Secrets and variables > Actions to run the AI eval."
|
|
45
|
+
exit 0
|
|
46
|
+
fi
|
|
47
|
+
args=(--provider "$PROVIDER" --json ai-eval.json)
|
|
48
|
+
if [ -n "$MODEL" ]; then args+=(--model "$MODEL"); fi
|
|
49
|
+
python scripts/eval_ai.py "${args[@]}" | tee ai-eval.txt
|
|
50
|
+
{ echo '```'; cat ai-eval.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
|
|
51
|
+
- uses: actions/upload-artifact@v4
|
|
52
|
+
if: always()
|
|
53
|
+
with:
|
|
54
|
+
name: ai-eval
|
|
55
|
+
path: ai-eval.*
|
|
56
|
+
if-no-files-found: ignore
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: kubesift
|
|
3
|
-
Version: 2.0
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Kubernetes troubleshooting that explains itself: evidence-based diagnosis, change correlation, cost and upgrade checks. CLI, kubectl plugin, MCP server and in-cluster service.
|
|
5
5
|
Project-URL: Homepage, https://github.com/OpeyemiAdeniji/KubeSift
|
|
6
6
|
Project-URL: Issues, https://github.com/OpeyemiAdeniji/KubeSift/issues
|
|
@@ -169,7 +169,8 @@ and `lint` takes `-o sarif` (GitHub code scanning). `--fail-on high` makes CI fa
|
|
|
169
169
|
- Tokens, passwords, API keys, connection-string credentials and emails are redacted from logs before they are
|
|
170
170
|
stored or sent. `send_logs_to_llm: false` keeps logs out of prompts entirely.
|
|
171
171
|
- LLM answers must cite real findings; uncited claims are capped at 40% confidence. Any LLM failure falls back
|
|
172
|
-
to the rule-based diagnosis.
|
|
172
|
+
to the rule-based diagnosis. `python scripts/eval_ai.py --provider <name>` scores a model on the demo cluster
|
|
173
|
+
(does it find the real causes, does it cite real evidence) before you rely on it.
|
|
173
174
|
- Read-only by default. Fixes need an explicit confirmation (CLI) or `allow_writes` (server/MCP), always run a
|
|
174
175
|
server-side dry run first and are written to an audit log.
|
|
175
176
|
|
|
@@ -189,4 +190,5 @@ and `lint` takes `-o sarif` (GitHub code scanning). `--fail-on high` makes CI fa
|
|
|
189
190
|
pip install -e ".[dev]"
|
|
190
191
|
pytest # unit tests on a synthetic broken cluster
|
|
191
192
|
./scripts/e2e.sh # breaks a kind cluster on purpose and checks every failure is explained
|
|
193
|
+
python scripts/eval_ai.py --provider anthropic # grades an LLM's explanations (needs its API key)
|
|
192
194
|
```
|
|
@@ -117,7 +117,8 @@ and `lint` takes `-o sarif` (GitHub code scanning). `--fail-on high` makes CI fa
|
|
|
117
117
|
- Tokens, passwords, API keys, connection-string credentials and emails are redacted from logs before they are
|
|
118
118
|
stored or sent. `send_logs_to_llm: false` keeps logs out of prompts entirely.
|
|
119
119
|
- LLM answers must cite real findings; uncited claims are capped at 40% confidence. Any LLM failure falls back
|
|
120
|
-
to the rule-based diagnosis.
|
|
120
|
+
to the rule-based diagnosis. `python scripts/eval_ai.py --provider <name>` scores a model on the demo cluster
|
|
121
|
+
(does it find the real causes, does it cite real evidence) before you rely on it.
|
|
121
122
|
- Read-only by default. Fixes need an explicit confirmation (CLI) or `allow_writes` (server/MCP), always run a
|
|
122
123
|
server-side dry run first and are written to an audit log.
|
|
123
124
|
|
|
@@ -137,4 +138,5 @@ and `lint` takes `-o sarif` (GitHub code scanning). `--fail-on high` makes CI fa
|
|
|
137
138
|
pip install -e ".[dev]"
|
|
138
139
|
pytest # unit tests on a synthetic broken cluster
|
|
139
140
|
./scripts/e2e.sh # breaks a kind cluster on purpose and checks every failure is explained
|
|
141
|
+
python scripts/eval_ai.py --provider anthropic # grades an LLM's explanations (needs its API key)
|
|
140
142
|
```
|
|
@@ -2,7 +2,7 @@ apiVersion: v2
|
|
|
2
2
|
name: kubesift
|
|
3
3
|
description: Evidence-based Kubernetes troubleshooting service (dashboard, API, Alertmanager receiver, background scans)
|
|
4
4
|
type: application
|
|
5
|
-
version: 2.0
|
|
6
|
-
appVersion: "2.0
|
|
5
|
+
version: 2.1.0
|
|
6
|
+
appVersion: "2.1.0"
|
|
7
7
|
home: https://github.com/OpeyemiAdeniji/KubeSift
|
|
8
8
|
keywords: [kubernetes, troubleshooting, sre, observability]
|
|
@@ -27,6 +27,11 @@ cost_gb_hour: 0.0042
|
|
|
27
27
|
Inside a pod, KubeSift uses the service account automatically. It needs **read** access only; see
|
|
28
28
|
`deploy/helm/kubesift/templates/rbac.yaml` for the exact rules. Missing permissions are reported, not fatal.
|
|
29
29
|
|
|
30
|
+
Before reading anything KubeSift checks that the cluster answers and accepts the login, and stops with an error
|
|
31
|
+
(exit code 1) when it doesn't, instead of producing an empty report that looks healthy. The message names the
|
|
32
|
+
context and the cause: unreachable API server, login rejected (expired, revoked or deleted account), or a
|
|
33
|
+
failing login helper such as `gke-gcloud-auth-plugin`. `kubesift doctor` runs the same check.
|
|
34
|
+
|
|
30
35
|
## LLM
|
|
31
36
|
|
|
32
37
|
KubeSift never needs an LLM: every finding comes from deterministic checks. An LLM only ranks and explains them.
|
|
@@ -98,3 +103,7 @@ Without Prometheus, metrics-server's current usage is used (one sample; treat as
|
|
|
98
103
|
| `slack_webhook_url`, `teams_webhook_url`, `webhook_url` | none | Notifications for new problems and Alertmanager investigations |
|
|
99
104
|
| `slack_signing_secret` | none | Enables the `/sift` Slack command and fix buttons (see [slack.md](slack.md)) |
|
|
100
105
|
| `slack_allowed_users` | none | Slack user IDs allowed to apply fixes (comma-separated); empty means nobody; also needs `allow_writes` |
|
|
106
|
+
|
|
107
|
+
The dashboard shows the rule-based report at once and fills in the LLM explanation when it arrives, so a slow
|
|
108
|
+
local model never blocks it. API clients get the same with `POST /api/v1/diagnose {"wait_for_llm": false}`: the
|
|
109
|
+
response has `llm_pending: true` and an `id`; poll `GET /api/v1/history/{id}` until `llm_pending` is false.
|
|
@@ -31,6 +31,8 @@ profile (**Packages**) and set their visibility to **Public**, so people can pul
|
|
|
31
31
|
1. Update the version in `src/kubesift/__init__.py` and `deploy/helm/kubesift/Chart.yaml`
|
|
32
32
|
(`version` and `appVersion`).
|
|
33
33
|
2. Merge to `main`, then tag. The workflow refuses a tag that doesn't match the package version.
|
|
34
|
+
Optionally run the **AI eval** workflow (Actions tab) first if the prompt or LLM code changed; it needs a
|
|
35
|
+
provider API key as a repository secret.
|
|
34
36
|
|
|
35
37
|
```bash
|
|
36
38
|
git tag v2.0.0
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Score an LLM on KubeSift's demo broken cluster: does it find the real causes and cite real evidence?
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
python scripts/eval_ai.py --provider anthropic # needs ANTHROPIC_API_KEY
|
|
5
|
+
python scripts/eval_ai.py --provider ollama --model llama3.1 # local; slow without a GPU
|
|
6
|
+
python scripts/eval_ai.py --provider openrouter --model openai/gpt-5 --json results.json
|
|
7
|
+
|
|
8
|
+
What is measured (the deterministic checks already found every fault; this grades the explanation):
|
|
9
|
+
coverage share of the five root problems that some root cause cites as evidence
|
|
10
|
+
cited share of root causes that cite at least one real finding (the rest are capped at 40% confidence)
|
|
11
|
+
questions three "why is X failing?" questions answered with the right evidence
|
|
12
|
+
|
|
13
|
+
Exit code 1 when coverage or cited is below its threshold, so it can gate a CI job.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import json
|
|
20
|
+
import sys
|
|
21
|
+
import time
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tests"))
|
|
26
|
+
|
|
27
|
+
from fixtures import broken_cluster # noqa: E402
|
|
28
|
+
|
|
29
|
+
from kubesift.analysis import engine # noqa: E402
|
|
30
|
+
from kubesift.config import load_settings # noqa: E402
|
|
31
|
+
|
|
32
|
+
# The five faults an on-call engineer must not miss in the demo cluster, keyed by the check that proves each.
|
|
33
|
+
ROOT_PROBLEMS = {
|
|
34
|
+
"pod-crashloop": "checkout crash loops after the 1.5 rollout removed DATABASE_URL",
|
|
35
|
+
"pod-oomkilled": "api is OOMKilled at a 64Mi limit",
|
|
36
|
+
"webhook-unavailable": "policy webhook points at a missing Service and blocks changes",
|
|
37
|
+
"node-not-ready": "node-2 is NotReady",
|
|
38
|
+
"config-ref-missing": "reports needs a ConfigMap that does not exist",
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
QUESTIONS = [
|
|
42
|
+
("why is checkout failing?", "pod-crashloop", "DATABASE_URL"),
|
|
43
|
+
("why does the api deployment keep restarting?", "pod-oomkilled", "memory"),
|
|
44
|
+
("why does the web service have no endpoints?", "service-no-endpoints", "selector"),
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def evaluate(settings: Any) -> dict[str, Any]:
|
|
49
|
+
rules = engine.investigate(settings, broken_cluster(), use_llm=False)
|
|
50
|
+
by_check: dict[str, set[str]] = {}
|
|
51
|
+
for incident in rules.incidents:
|
|
52
|
+
ids = {f.fingerprint for f in incident.findings}
|
|
53
|
+
for f in incident.findings:
|
|
54
|
+
by_check.setdefault(f.check, set()).update(ids)
|
|
55
|
+
|
|
56
|
+
started = time.monotonic()
|
|
57
|
+
report = engine.investigate(settings, broken_cluster(), use_llm=True)
|
|
58
|
+
diagnose_s = time.monotonic() - started
|
|
59
|
+
diagnosis = report.diagnosis
|
|
60
|
+
if diagnosis is None or diagnosis.generated_by == "rules":
|
|
61
|
+
return {"error": "the LLM did not answer: " + "; ".join(report.warnings[-2:]), "diagnose_seconds": diagnose_s}
|
|
62
|
+
|
|
63
|
+
cited_ids = {e for rc in diagnosis.root_causes for e in rc.evidence}
|
|
64
|
+
found = {check: bool(by_check.get(check, set()) & cited_ids) for check in ROOT_PROBLEMS}
|
|
65
|
+
causes = [{"title": rc.title, "confidence": rc.confidence, "cites": len(rc.evidence)} for rc in diagnosis.root_causes]
|
|
66
|
+
cited_share = sum(1 for rc in diagnosis.root_causes if rc.evidence) / max(1, len(diagnosis.root_causes))
|
|
67
|
+
|
|
68
|
+
answers = []
|
|
69
|
+
for question, check, keyword in QUESTIONS:
|
|
70
|
+
t = time.monotonic()
|
|
71
|
+
_, answer = engine.ask(settings, broken_cluster(), question, use_llm=True)
|
|
72
|
+
llm = not str(answer.get("generated_by", "rules")).startswith("rules")
|
|
73
|
+
answers.append({
|
|
74
|
+
"question": question, "llm": llm, "seconds": round(time.monotonic() - t, 1),
|
|
75
|
+
"right_evidence": bool(by_check.get(check, set()) & set(answer.get("evidence") or [])),
|
|
76
|
+
"mentions": keyword.lower() in str(answer.get("answer", "")).lower(),
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
"model": diagnosis.generated_by,
|
|
81
|
+
"diagnose_seconds": round(diagnose_s, 1),
|
|
82
|
+
"coverage": sum(found.values()) / len(found),
|
|
83
|
+
"found": found,
|
|
84
|
+
"cited": round(cited_share, 2),
|
|
85
|
+
"root_causes": causes,
|
|
86
|
+
"questions": answers,
|
|
87
|
+
"questions_right": sum(1 for a in answers if a["right_evidence"]),
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def main() -> int:
|
|
92
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
93
|
+
ap.add_argument("--provider", required=True, help="anthropic | openai | openrouter | ollama")
|
|
94
|
+
ap.add_argument("--model", help="model name (default: the provider's default)")
|
|
95
|
+
ap.add_argument("--json", help="also write the results to this file")
|
|
96
|
+
ap.add_argument("--min-coverage", type=float, default=0.6)
|
|
97
|
+
ap.add_argument("--min-cited", type=float, default=0.8)
|
|
98
|
+
args = ap.parse_args()
|
|
99
|
+
|
|
100
|
+
settings = load_settings(llm_provider=args.provider, llm_model=args.model)
|
|
101
|
+
result = evaluate(settings)
|
|
102
|
+
if args.json:
|
|
103
|
+
Path(args.json).write_text(json.dumps(result, indent=2), encoding="utf-8")
|
|
104
|
+
if "error" in result:
|
|
105
|
+
print(f"FAIL {result['error']}")
|
|
106
|
+
return 1
|
|
107
|
+
|
|
108
|
+
print(f"model {result['model']} ({result['diagnose_seconds']}s for the diagnosis)")
|
|
109
|
+
print(f"coverage {result['coverage']:.0%} of the root problems cited (min {args.min_coverage:.0%})")
|
|
110
|
+
for check, ok in result["found"].items():
|
|
111
|
+
print(f" {'+' if ok else '-'} {ROOT_PROBLEMS[check]}")
|
|
112
|
+
print(f"cited {result['cited']:.0%} of root causes cite real evidence (min {args.min_cited:.0%})")
|
|
113
|
+
for rc in result["root_causes"]:
|
|
114
|
+
print(f" [{rc['confidence']:>3}%] {rc['title'][:90]} ({rc['cites']} citation(s))")
|
|
115
|
+
print(f"questions {result['questions_right']}/{len(result['questions'])} answered with the right evidence")
|
|
116
|
+
for a in result["questions"]:
|
|
117
|
+
print(f" {'+' if a['right_evidence'] else '-'} {a['question']} ({a['seconds']}s{'' if a['llm'] else ', rules fallback'})")
|
|
118
|
+
ok = result["coverage"] >= args.min_coverage and result["cited"] >= args.min_cited
|
|
119
|
+
print("PASS" if ok else "FAIL")
|
|
120
|
+
return 0 if ok else 1
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
if __name__ == "__main__":
|
|
124
|
+
sys.exit(main())
|
|
@@ -28,7 +28,8 @@ SYSTEM_PROMPT = """You are KubeSift, a senior Kubernetes SRE helping an on-call
|
|
|
28
28
|
You receive findings that deterministic checks proved from live cluster data, recent changes, and resource summaries.
|
|
29
29
|
|
|
30
30
|
Rules:
|
|
31
|
-
- Base every conclusion on the provided
|
|
31
|
+
- Base every conclusion on the provided findings. Each finding has an `id` (12 hex characters) and `facts`.
|
|
32
|
+
List the `id` of every finding a conclusion rests on in `evidence_ids`, copied exactly, e.g. "3f2a9c1b7d0e".
|
|
32
33
|
- Never invent resource names, versions, or log lines. If the data does not show the cause, say what is missing and how to get it.
|
|
33
34
|
- Separate the root cause from its symptoms: a bad rollout, a config edit or a broken node often explains several findings.
|
|
34
35
|
- When a recent change lines up with the start of a problem, say so and give the rollback command if one is provided.
|
|
@@ -47,12 +48,13 @@ DIAGNOSIS_SCHEMA: dict[str, Any] = {
|
|
|
47
48
|
"items": {
|
|
48
49
|
"type": "object",
|
|
49
50
|
"additionalProperties": False,
|
|
50
|
-
"required": ["title", "explanation", "confidence", "
|
|
51
|
+
"required": ["title", "explanation", "confidence", "evidence_ids", "fix_steps", "commands"],
|
|
51
52
|
"properties": {
|
|
52
53
|
"title": {"type": "string"},
|
|
53
54
|
"explanation": {"type": "string"},
|
|
54
55
|
"confidence": {"type": "integer"},
|
|
55
|
-
"
|
|
56
|
+
"evidence_ids": {"type": "array", "items": {"type": "string"},
|
|
57
|
+
"description": "The `id` values of the findings this cause rests on."},
|
|
56
58
|
"fix_steps": {"type": "array", "items": {"type": "string"}},
|
|
57
59
|
"commands": {"type": "array", "items": {"type": "string"}},
|
|
58
60
|
},
|
|
@@ -65,11 +67,12 @@ DIAGNOSIS_SCHEMA: dict[str, Any] = {
|
|
|
65
67
|
ANSWER_SCHEMA: dict[str, Any] = {
|
|
66
68
|
"type": "object",
|
|
67
69
|
"additionalProperties": False,
|
|
68
|
-
"required": ["answer", "confidence", "
|
|
70
|
+
"required": ["answer", "confidence", "evidence_ids", "commands", "follow_up"],
|
|
69
71
|
"properties": {
|
|
70
72
|
"answer": {"type": "string"},
|
|
71
73
|
"confidence": {"type": "integer"},
|
|
72
|
-
"
|
|
74
|
+
"evidence_ids": {"type": "array", "items": {"type": "string"},
|
|
75
|
+
"description": "The `id` values of the findings the answer rests on."},
|
|
73
76
|
"commands": {"type": "array", "items": {"type": "string"}},
|
|
74
77
|
"follow_up": {"type": "array", "items": {"type": "string"}},
|
|
75
78
|
},
|
|
@@ -149,9 +152,9 @@ def rules_diagnosis(report: Report) -> Diagnosis:
|
|
|
149
152
|
def _finding_payload(f: Finding, logs: bool = True) -> dict[str, Any]:
|
|
150
153
|
evidence = [e for e in f.evidence if logs or not e.startswith("log ")]
|
|
151
154
|
return {
|
|
152
|
-
"
|
|
155
|
+
"id": f.fingerprint, "check": f.check, "severity": f.severity.value, "resource": str(f.resource),
|
|
153
156
|
"owner": str(f.owner) if f.owner else None, "node": f.node, "message": f.message,
|
|
154
|
-
"
|
|
157
|
+
"facts": evidence[:8], "suggested_fix": f.remediation, "commands": f.commands[:3],
|
|
155
158
|
"started": f.since.isoformat() if f.since else None,
|
|
156
159
|
"related_changes": [c.summary + (f" [rollback: {c.rollback_command}]" if c.rollback_command else "")
|
|
157
160
|
for c in f.related_changes],
|
|
@@ -164,7 +167,7 @@ def _context_json(report: Report, extra: dict[str, Any] | None = None, logs: boo
|
|
|
164
167
|
"cluster": report.cluster, "namespace": report.namespace or "all", "kubernetes": report.server_version,
|
|
165
168
|
"incidents": [{"title": i.title, "layer": i.layer, "severity": i.severity.value,
|
|
166
169
|
"findings": [_finding_payload(f, logs) for f in i.findings[:6]]} for i in report.incidents[:12]],
|
|
167
|
-
"risks": [{"
|
|
170
|
+
"risks": [{"id": f.fingerprint, "check": f.check, "resource": str(f.resource), "message": f.message}
|
|
168
171
|
for f in report.findings if f.kind == "risk"][:25],
|
|
169
172
|
"recent_changes": [{"time": c.time.isoformat(), "summary": c.summary, "details": c.details[:5],
|
|
170
173
|
"rollback": c.rollback_command} for c in report.changes[:15]],
|
|
@@ -201,11 +204,22 @@ def explain(settings: Settings, report: Report, use_llm: bool = True) -> Diagnos
|
|
|
201
204
|
return base
|
|
202
205
|
|
|
203
206
|
|
|
207
|
+
def _cited(item: dict[str, Any], known: set[str]) -> list[str]:
|
|
208
|
+
"""Finding ids the model cited. Accepts an id wrapped in extra text ("finding 3f2a9c1b7d0e")."""
|
|
209
|
+
cited: list[str] = []
|
|
210
|
+
for raw in item.get("evidence_ids") or item.get("evidence") or []:
|
|
211
|
+
text = str(raw)
|
|
212
|
+
for fp in [text] if text in known else [k for k in known if k in text]:
|
|
213
|
+
if fp not in cited:
|
|
214
|
+
cited.append(fp)
|
|
215
|
+
return cited
|
|
216
|
+
|
|
217
|
+
|
|
204
218
|
def _validated(data: dict[str, Any], report: Report, by: str) -> Diagnosis:
|
|
205
219
|
known = {f.fingerprint for f in report.findings}
|
|
206
220
|
causes = []
|
|
207
221
|
for rc in data.get("root_causes") or []:
|
|
208
|
-
evidence =
|
|
222
|
+
evidence = _cited(rc, known)
|
|
209
223
|
conf = max(0, min(100, int(rc.get("confidence", 0))))
|
|
210
224
|
if not evidence:
|
|
211
225
|
conf = min(conf, 40) # unsupported claims cannot be confident
|
|
@@ -301,7 +315,7 @@ def ask(settings: Settings, snap: Snapshot, question: str, use_llm: bool = True,
|
|
|
301
315
|
answer = {
|
|
302
316
|
"answer": str(data.get("answer", "")),
|
|
303
317
|
"confidence": max(0, min(100, int(data.get("confidence", 0)))),
|
|
304
|
-
"evidence":
|
|
318
|
+
"evidence": _cited(data, known),
|
|
305
319
|
"commands": [str(c) for c in data.get("commands") or []][:6],
|
|
306
320
|
"follow_up": [str(c) for c in data.get("follow_up") or []][:5],
|
|
307
321
|
"generated_by": f"{provider.name}:{settings.resolved_model()}",
|
|
@@ -100,8 +100,10 @@ def _snapshot(namespace: str | None, from_file: str | None = None, with_logs: bo
|
|
|
100
100
|
try:
|
|
101
101
|
with err.status("Reading cluster state…"):
|
|
102
102
|
return snapshot_for(state.settings, namespace, with_logs=with_logs)
|
|
103
|
-
except Exception as exc: # kubeconfig missing, cluster unreachable
|
|
104
|
-
|
|
103
|
+
except Exception as exc: # kubeconfig missing, cluster unreachable, login rejected
|
|
104
|
+
from kubesift.cluster.client import ClusterAccessError
|
|
105
|
+
|
|
106
|
+
_fail(str(exc) if isinstance(exc, ClusterAccessError) else f"cannot connect to the cluster: {exc}")
|
|
105
107
|
|
|
106
108
|
|
|
107
109
|
def _store():
|
|
@@ -535,15 +537,16 @@ def doctor() -> None:
|
|
|
535
537
|
mark = {True: "[green]✓[/]", False: "[red]✗[/]", None: "[yellow]![/]"}[good]
|
|
536
538
|
console.print(f"{mark} {text}")
|
|
537
539
|
|
|
540
|
+
connected = False
|
|
538
541
|
try:
|
|
539
542
|
client = KubeClient(kubeconfig=s.kubeconfig, context=s.context, timeout=s.request_timeout)
|
|
540
|
-
version = client.
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
except Exception as exc:
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
if
|
|
543
|
+
version = client.check_connection()
|
|
544
|
+
connected = True
|
|
545
|
+
line(True, f"cluster {client.context_name}: Kubernetes {version or '(version hidden)'}")
|
|
546
|
+
except Exception as exc: # unreachable, login rejected, bad kubeconfig: still check the rest
|
|
547
|
+
ok = False
|
|
548
|
+
line(False, str(exc))
|
|
549
|
+
if connected:
|
|
547
550
|
missing = []
|
|
548
551
|
for key, _, prefix, plural, _, optional in KINDS:
|
|
549
552
|
if optional and key != "secrets":
|
|
@@ -8,6 +8,7 @@ snapshots and manifest files.
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import json
|
|
11
|
+
import logging
|
|
11
12
|
import os
|
|
12
13
|
from typing import Any
|
|
13
14
|
|
|
@@ -19,6 +20,25 @@ class KubeError(Exception):
|
|
|
19
20
|
self.message = message
|
|
20
21
|
|
|
21
22
|
|
|
23
|
+
class ClusterAccessError(Exception):
|
|
24
|
+
"""The cluster cannot be used at all: unreachable, or the login was rejected."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
_LOGIN_HINT = ("log in again (gcloud auth login / az login / aws sso login, or refresh the kubeconfig), "
|
|
28
|
+
"or pick another context: kubectl config get-contexts, then --context NAME")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class _LoginFailures(logging.Handler):
|
|
32
|
+
def __init__(self) -> None:
|
|
33
|
+
super().__init__(logging.ERROR)
|
|
34
|
+
self.messages: list[str] = []
|
|
35
|
+
|
|
36
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
37
|
+
msg = record.getMessage()
|
|
38
|
+
if msg.startswith("exec:"):
|
|
39
|
+
self.messages.append(msg.removeprefix("exec:").strip())
|
|
40
|
+
|
|
41
|
+
|
|
22
42
|
class KubeClient:
|
|
23
43
|
def __init__(self, kubeconfig: str | None = None, context: str | None = None, timeout: int = 20):
|
|
24
44
|
from kubernetes import client, config
|
|
@@ -30,14 +50,56 @@ class KubeClient:
|
|
|
30
50
|
self.context_name = "in-cluster"
|
|
31
51
|
else:
|
|
32
52
|
cfg = client.Configuration()
|
|
33
|
-
|
|
53
|
+
failures = _LoginFailures()
|
|
54
|
+
logging.getLogger().addHandler(failures) # the client only *logs* a failing exec login helper
|
|
55
|
+
try:
|
|
56
|
+
config.load_kube_config(config_file=kubeconfig, context=context, client_configuration=cfg)
|
|
57
|
+
except Exception as exc: # missing file, unknown context
|
|
58
|
+
raise ClusterAccessError(f"cannot load kubeconfig{f' context {context}' if context else ''}: {exc}") from exc
|
|
59
|
+
finally:
|
|
60
|
+
logging.getLogger().removeHandler(failures)
|
|
34
61
|
if context:
|
|
35
62
|
self.context_name = context
|
|
36
63
|
else:
|
|
37
64
|
_, active = config.list_kube_config_contexts(config_file=kubeconfig)
|
|
38
65
|
self.context_name = (active or {}).get("name", "default")
|
|
66
|
+
if failures.messages:
|
|
67
|
+
raise ClusterAccessError(f"the login command for context '{self.context_name}' failed: "
|
|
68
|
+
f"{failures.messages[-1][:300]}. Fix: {_LOGIN_HINT}")
|
|
69
|
+
cfg.retries = 1 # one retry for a network blip; urllib3's default (3) multiplies every timeout
|
|
70
|
+
self.server = cfg.host
|
|
39
71
|
self.api = client.ApiClient(cfg)
|
|
40
72
|
|
|
73
|
+
def check_connection(self, namespace: str | None = None) -> str | None:
|
|
74
|
+
"""Fail fast, with a useful message, when the cluster cannot be used at all.
|
|
75
|
+
|
|
76
|
+
Without this an unreachable cluster or a dead login made every list call fail separately, which was
|
|
77
|
+
slow and ended in an empty report that looked like a healthy cluster.
|
|
78
|
+
"""
|
|
79
|
+
where = f"context '{self.context_name}' ({self.server})"
|
|
80
|
+
try:
|
|
81
|
+
version = (self.request("GET", "/version", timeout=(5, 15)) or {}).get("gitVersion")
|
|
82
|
+
except KubeError as exc:
|
|
83
|
+
if exc.status != 401:
|
|
84
|
+
raise ClusterAccessError(f"the API server for {where} answered HTTP {exc.status}: {exc.message}") from exc
|
|
85
|
+
version = None # /version needs a login on this cluster; the next call tells us if ours works
|
|
86
|
+
except Exception as exc:
|
|
87
|
+
raise ClusterAccessError(
|
|
88
|
+
f"cannot reach the API server for {where}: {_short(exc)}. Check your network or VPN and that the "
|
|
89
|
+
"cluster still exists, or pick another context: kubectl config get-contexts, then --context NAME"
|
|
90
|
+
) from exc
|
|
91
|
+
probe = f"/api/v1/namespaces/{namespace}" if namespace else "/api/v1/namespaces"
|
|
92
|
+
try:
|
|
93
|
+
self.request("GET", probe, query={"limit": 1} if not namespace else None, timeout=(5, 15))
|
|
94
|
+
except KubeError as exc:
|
|
95
|
+
if exc.status == 401:
|
|
96
|
+
raise ClusterAccessError(f"the cluster rejected the login for {where} (expired, revoked or deleted "
|
|
97
|
+
f"account): {_LOGIN_HINT}") from exc
|
|
98
|
+
# 403/404: logged in but limited; the checks report what they could not read
|
|
99
|
+
except Exception as exc:
|
|
100
|
+
raise ClusterAccessError(f"cannot reach the API server for {where}: {_short(exc)}") from exc
|
|
101
|
+
return version
|
|
102
|
+
|
|
41
103
|
def request(
|
|
42
104
|
self,
|
|
43
105
|
method: str,
|
|
@@ -46,6 +108,7 @@ class KubeClient:
|
|
|
46
108
|
body: Any = None,
|
|
47
109
|
content_type: str | None = None,
|
|
48
110
|
raw: bool = False,
|
|
111
|
+
timeout: Any = None,
|
|
49
112
|
) -> Any:
|
|
50
113
|
from kubernetes.client.exceptions import ApiException
|
|
51
114
|
|
|
@@ -59,14 +122,14 @@ class KubeClient:
|
|
|
59
122
|
method=method, resource_path=path, query_params=query_params,
|
|
60
123
|
header_params=headers, body=body, auth_settings=["BearerToken"],
|
|
61
124
|
)
|
|
62
|
-
resp = self.api.call_api(*params, _request_timeout=self.timeout)
|
|
125
|
+
resp = self.api.call_api(*params, _request_timeout=timeout or self.timeout)
|
|
63
126
|
data = resp.read()
|
|
64
127
|
status = resp.status
|
|
65
128
|
else: # kubernetes <= 36
|
|
66
129
|
resp = self.api.call_api(
|
|
67
130
|
path, method, query_params=query_params, header_params=headers, body=body,
|
|
68
131
|
auth_settings=["BearerToken"], _preload_content=False, _return_http_data_only=True,
|
|
69
|
-
_request_timeout=self.timeout,
|
|
132
|
+
_request_timeout=timeout or self.timeout,
|
|
70
133
|
)
|
|
71
134
|
data = resp.data
|
|
72
135
|
status = resp.status
|
|
@@ -110,6 +173,18 @@ class KubeClient:
|
|
|
110
173
|
return None
|
|
111
174
|
|
|
112
175
|
|
|
176
|
+
def _short(exc: Exception) -> str:
|
|
177
|
+
"""The useful tail of a urllib3 error ('Connection refused', 'timed out', 'Name or service not known')."""
|
|
178
|
+
text = str(exc)
|
|
179
|
+
if "actively refused" in text: # Windows wording
|
|
180
|
+
return "connection refused"
|
|
181
|
+
for marker in ("Connection refused", "timed out", "Name or service not known",
|
|
182
|
+
"getaddrinfo failed", "No route to host", "certificate verify failed", "Network is unreachable"):
|
|
183
|
+
if marker in text:
|
|
184
|
+
return marker
|
|
185
|
+
return text[:200]
|
|
186
|
+
|
|
187
|
+
|
|
113
188
|
def _message(body: Any) -> str:
|
|
114
189
|
if isinstance(body, bytes):
|
|
115
190
|
body = body.decode("utf-8", "replace")
|
|
@@ -235,7 +235,8 @@ def collect(
|
|
|
235
235
|
|
|
236
236
|
check_name(namespace, "namespace")
|
|
237
237
|
snap = Snapshot(context=client.context_name, namespace=namespace, live=True)
|
|
238
|
-
|
|
238
|
+
probe = getattr(client, "check_connection", None)
|
|
239
|
+
snap.server_version = probe(namespace) if probe else client.server_version()
|
|
239
240
|
|
|
240
241
|
def fetch(entry: tuple[str, str, str, str, bool, bool]) -> None:
|
|
241
242
|
key, kind, prefix, plural, namespaced, optional = entry
|
|
@@ -21,6 +21,7 @@ from pydantic import BaseModel, Field
|
|
|
21
21
|
from kubesift import __version__, notify
|
|
22
22
|
from kubesift.analysis.changes import detect_changes
|
|
23
23
|
from kubesift.analysis.engine import ask, explain_resource, investigate, snapshot_for
|
|
24
|
+
from kubesift.analysis.engine import explain as llm_explain
|
|
24
25
|
from kubesift.cluster.client import KubeClient, list_contexts
|
|
25
26
|
from kubesift.config import Settings
|
|
26
27
|
from kubesift.leader import AlwaysLeader, LeaderElector, pod_namespace
|
|
@@ -37,6 +38,9 @@ class DiagnoseRequest(BaseModel):
|
|
|
37
38
|
context: str | None = None
|
|
38
39
|
llm: bool = True
|
|
39
40
|
since: str = "6h"
|
|
41
|
+
# false: return the rule-based report at once and add the LLM explanation in the background
|
|
42
|
+
# (poll GET /api/v1/history/{id} until llm_pending is false). Useful with slow local models.
|
|
43
|
+
wait_for_llm: bool = True
|
|
40
44
|
|
|
41
45
|
|
|
42
46
|
class AskRequest(BaseModel):
|
|
@@ -157,13 +161,29 @@ def create_app(settings: Settings) -> FastAPI:
|
|
|
157
161
|
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
|
158
162
|
return {"contexts": items, "current": current}
|
|
159
163
|
|
|
164
|
+
pending: set[str] = set()
|
|
165
|
+
|
|
166
|
+
def explain_later(rid: str, report: Report) -> None:
|
|
167
|
+
try:
|
|
168
|
+
with _LLM_SLOTS:
|
|
169
|
+
report.diagnosis = llm_explain(settings, report, use_llm=True)
|
|
170
|
+
store.update_report(rid, report)
|
|
171
|
+
except Exception:
|
|
172
|
+
log.exception("background LLM explanation failed for report %s", rid)
|
|
173
|
+
finally:
|
|
174
|
+
pending.discard(rid)
|
|
175
|
+
|
|
160
176
|
@app.post("/api/v1/diagnose", dependencies=[Depends(auth)])
|
|
161
177
|
def diagnose(req: DiagnoseRequest) -> dict[str, Any]:
|
|
162
|
-
|
|
178
|
+
later = req.llm and not req.wait_for_llm and settings.resolved_provider() != "none"
|
|
179
|
+
report = investigate(settings, snap(req.namespace, req.context), use_llm=req.llm and not later,
|
|
163
180
|
window=parse_duration(req.since), store=store)
|
|
164
181
|
rid = store.save_report(report, source="api", retention_days=settings.history_days)
|
|
165
182
|
state["last_report"] = report
|
|
166
|
-
|
|
183
|
+
if later and report.incidents:
|
|
184
|
+
pending.add(rid)
|
|
185
|
+
threading.Thread(target=explain_later, args=(rid, report), daemon=True, name=f"llm-{rid}").start()
|
|
186
|
+
return {"id": rid, "llm_pending": rid in pending, **report.model_dump(mode="json")}
|
|
167
187
|
|
|
168
188
|
@app.post("/api/v1/ask", dependencies=[Depends(auth)])
|
|
169
189
|
def ask_question(req: AskRequest) -> dict[str, Any]:
|
|
@@ -210,7 +230,7 @@ def create_app(settings: Settings) -> FastAPI:
|
|
|
210
230
|
report = store.get_report(rid)
|
|
211
231
|
if report is None:
|
|
212
232
|
raise HTTPException(status_code=404, detail="not found")
|
|
213
|
-
return report.model_dump(mode="json")
|
|
233
|
+
return {"id": rid, "llm_pending": rid in pending, **report.model_dump(mode="json")}
|
|
214
234
|
|
|
215
235
|
@app.post("/api/v1/fix", dependencies=[Depends(auth)])
|
|
216
236
|
def fix(req: FixRequest) -> dict[str, Any]:
|
|
@@ -369,6 +389,7 @@ def create_app(settings: Settings) -> FastAPI:
|
|
|
369
389
|
return app
|
|
370
390
|
|
|
371
391
|
|
|
392
|
+
_LLM_SLOTS = threading.BoundedSemaphore(2) # at most two background LLM explanations at a time
|
|
372
393
|
_ALERT_SLOTS = threading.BoundedSemaphore(2) # at most two alert investigations at a time
|
|
373
394
|
|
|
374
395
|
|
|
@@ -408,8 +429,7 @@ def _scan_loop(settings: Settings, store: Store, state: dict[str, Any]) -> None:
|
|
|
408
429
|
urgent = [f for f in report.findings if f.kind == "problem" and f.severity.rank >= threshold]
|
|
409
430
|
fresh = store.new_findings(report.cluster, urgent)
|
|
410
431
|
if fresh:
|
|
411
|
-
|
|
412
|
-
report.diagnosis = explain(settings, report, use_llm=True)
|
|
432
|
+
report.diagnosis = llm_explain(settings, report, use_llm=True)
|
|
413
433
|
store.save_report(report, source="scan", retention_days=settings.history_days)
|
|
414
434
|
notify.send(settings, report, fresh)
|
|
415
435
|
except Exception as exc:
|