kubesift 2.0.0__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kubesift-2.1.0/.github/workflows/ai-eval.yml +56 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/PKG-INFO +4 -2
- {kubesift-2.0.0 → kubesift-2.1.0}/README.md +3 -1
- {kubesift-2.0.0 → kubesift-2.1.0}/deploy/helm/kubesift/Chart.yaml +2 -2
- {kubesift-2.0.0 → kubesift-2.1.0}/deploy/helm/kubesift/templates/deployment.yaml +4 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/deploy/helm/kubesift/values.yaml +2 -1
- {kubesift-2.0.0 → kubesift-2.1.0}/docs/configuration.md +19 -1
- {kubesift-2.0.0 → kubesift-2.1.0}/docs/releasing.md +2 -0
- kubesift-2.1.0/scripts/eval_ai.py +124 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/__init__.py +1 -1
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/analysis/engine.py +24 -10
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/analysis/llm.py +104 -6
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/cli.py +58 -22
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/cluster/client.py +78 -3
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/cluster/snapshot.py +2 -1
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/config.py +9 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/server/app.py +25 -5
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/server/static/index.html +17 -2
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/store.py +4 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/test_analysis.py +52 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/test_client.py +45 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/test_interfaces.py +35 -0
- kubesift-2.1.0/tests/test_llm.py +192 -0
- kubesift-2.0.0/tests/test_llm.py +0 -103
- {kubesift-2.0.0 → kubesift-2.1.0}/.dockerignore +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/.gitattributes +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/.github/workflows/ci.yml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/.github/workflows/pages.yml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/.github/workflows/release.yml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/.gitignore +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/.pre-commit-hooks.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/CONTRIBUTING.md +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/Dockerfile +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/LICENSE +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/SECURITY.md +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/action.yml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/deploy/helm/kubesift/templates/NOTES.txt +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/deploy/helm/kubesift/templates/_helpers.tpl +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/deploy/helm/kubesift/templates/rbac.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/deploy/helm/kubesift/templates/secret.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/deploy/helm/kubesift/templates/service.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/docs/architecture.md +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/docs/checks.md +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/docs/mcp.md +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/docs/preflight.md +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/docs/production.md +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/docs/slack.md +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/pyproject.toml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/scripts/e2e.sh +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/scripts/e2e_check.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/scripts/gen_check_docs.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/analysis/__init__.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/analysis/changes.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/analysis/correlate.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/analysis/describe.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/__init__.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/base.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/config_refs.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/events.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/hygiene.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/lifecycle.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/network.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/nodes.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/operations.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/pods.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/storage.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/checks/workloads.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/cluster/__init__.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/cost.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/01-crashloop-missing-env.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/02-imagepull-wrong-tag.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/03-oomkilled-low-memory.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/04-service-selector-mismatch.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/05-missing-probe-crashloop.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/06-pending-insufficient-resources.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/07-missing-secret.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/08-pvc-missing-storageclass.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/09-netpol-blocks-dns.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/10a-rollout-good.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/10b-rollout-bad.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drill_scenarios/__init__.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/drills.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/knowledge.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/leader.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/lint.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/mcp_server.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/models.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/notify.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/postmortem.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/preflight.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/redact.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/remediate.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/report.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/runbooks.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/server/__init__.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/slack.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/timetravel.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/upgrade.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/src/kubesift/util.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/01-crashloop-missing-env.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/02-imagepull-wrong-tag.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/03-oomkilled-low-memory.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/04-service-selector-mismatch.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/05-missing-probe-crashloop.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/06-pending-insufficient-resources.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/07-missing-secret.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/08-pvc-missing-storageclass.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/09-netpol-blocks-dns.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/10a-rollout-good.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/10b-rollout-bad.yaml +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/apply-scenario.sh +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/cleanup.sh +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/test-scenarios/expected.json +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/conftest.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/fixtures.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/test_checks.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/test_features.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/test_ha.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/test_hardening.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/test_lint_redact.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/test_more_checks.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/tests/test_production.py +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/website/README.md +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/website/favicon.svg +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/website/index.html +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/website/main.js +0 -0
- {kubesift-2.0.0 → kubesift-2.1.0}/website/styles.css +0 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
name: AI eval
|
|
2
|
+
|
|
3
|
+
# Grades a real LLM on the demo broken cluster (scripts/eval_ai.py). Run it by hand from the Actions tab, for
|
|
4
|
+
# example before a release or after changing the prompt. It needs the provider's API key as a repository secret
|
|
5
|
+
# (ANTHROPIC_API_KEY, OPENAI_API_KEY or OPENROUTER_API_KEY); without one it skips instead of failing.
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
inputs:
|
|
10
|
+
provider:
|
|
11
|
+
description: LLM provider
|
|
12
|
+
type: choice
|
|
13
|
+
options: [anthropic, openai, openrouter]
|
|
14
|
+
default: anthropic
|
|
15
|
+
model:
|
|
16
|
+
description: Model name (empty = the provider's default)
|
|
17
|
+
required: false
|
|
18
|
+
default: ""
|
|
19
|
+
|
|
20
|
+
permissions:
|
|
21
|
+
contents: read
|
|
22
|
+
|
|
23
|
+
jobs:
|
|
24
|
+
eval:
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
timeout-minutes: 30
|
|
27
|
+
env:
|
|
28
|
+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
29
|
+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
30
|
+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
|
31
|
+
PROVIDER: ${{ inputs.provider }}
|
|
32
|
+
MODEL: ${{ inputs.model }}
|
|
33
|
+
steps:
|
|
34
|
+
- uses: actions/checkout@v4
|
|
35
|
+
- uses: actions/setup-python@v5
|
|
36
|
+
with:
|
|
37
|
+
python-version: "3.12"
|
|
38
|
+
- run: pip install -e ".[anthropic]"
|
|
39
|
+
- name: Grade the model
|
|
40
|
+
shell: bash # pipefail, so a failing grade is not hidden by tee
|
|
41
|
+
run: |
|
|
42
|
+
key_var="$(echo "$PROVIDER" | tr '[:lower:]' '[:upper:]')_API_KEY"
|
|
43
|
+
if [ -z "${!key_var}" ]; then
|
|
44
|
+
echo "::notice::No $key_var secret; add it under Settings > Secrets and variables > Actions to run the AI eval."
|
|
45
|
+
exit 0
|
|
46
|
+
fi
|
|
47
|
+
args=(--provider "$PROVIDER" --json ai-eval.json)
|
|
48
|
+
if [ -n "$MODEL" ]; then args+=(--model "$MODEL"); fi
|
|
49
|
+
python scripts/eval_ai.py "${args[@]}" | tee ai-eval.txt
|
|
50
|
+
{ echo '```'; cat ai-eval.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
|
|
51
|
+
- uses: actions/upload-artifact@v4
|
|
52
|
+
if: always()
|
|
53
|
+
with:
|
|
54
|
+
name: ai-eval
|
|
55
|
+
path: ai-eval.*
|
|
56
|
+
if-no-files-found: ignore
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: kubesift
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Kubernetes troubleshooting that explains itself: evidence-based diagnosis, change correlation, cost and upgrade checks. CLI, kubectl plugin, MCP server and in-cluster service.
|
|
5
5
|
Project-URL: Homepage, https://github.com/OpeyemiAdeniji/KubeSift
|
|
6
6
|
Project-URL: Issues, https://github.com/OpeyemiAdeniji/KubeSift/issues
|
|
@@ -169,7 +169,8 @@ and `lint` takes `-o sarif` (GitHub code scanning). `--fail-on high` makes CI fa
|
|
|
169
169
|
- Tokens, passwords, API keys, connection-string credentials and emails are redacted from logs before they are
|
|
170
170
|
stored or sent. `send_logs_to_llm: false` keeps logs out of prompts entirely.
|
|
171
171
|
- LLM answers must cite real findings; uncited claims are capped at 40% confidence. Any LLM failure falls back
|
|
172
|
-
to the rule-based diagnosis.
|
|
172
|
+
to the rule-based diagnosis. `python scripts/eval_ai.py --provider <name>` scores a model on the demo cluster
|
|
173
|
+
(does it find the real causes, does it cite real evidence) before you rely on it.
|
|
173
174
|
- Read-only by default. Fixes need an explicit confirmation (CLI) or `allow_writes` (server/MCP), always run a
|
|
174
175
|
server-side dry run first and are written to an audit log.
|
|
175
176
|
|
|
@@ -189,4 +190,5 @@ and `lint` takes `-o sarif` (GitHub code scanning). `--fail-on high` makes CI fa
|
|
|
189
190
|
pip install -e ".[dev]"
|
|
190
191
|
pytest # unit tests on a synthetic broken cluster
|
|
191
192
|
./scripts/e2e.sh # breaks a kind cluster on purpose and checks every failure is explained
|
|
193
|
+
python scripts/eval_ai.py --provider anthropic # grades an LLM's explanations (needs its API key)
|
|
192
194
|
```
|
|
@@ -117,7 +117,8 @@ and `lint` takes `-o sarif` (GitHub code scanning). `--fail-on high` makes CI fa
|
|
|
117
117
|
- Tokens, passwords, API keys, connection-string credentials and emails are redacted from logs before they are
|
|
118
118
|
stored or sent. `send_logs_to_llm: false` keeps logs out of prompts entirely.
|
|
119
119
|
- LLM answers must cite real findings; uncited claims are capped at 40% confidence. Any LLM failure falls back
|
|
120
|
-
to the rule-based diagnosis.
|
|
120
|
+
to the rule-based diagnosis. `python scripts/eval_ai.py --provider <name>` scores a model on the demo cluster
|
|
121
|
+
(does it find the real causes, does it cite real evidence) before you rely on it.
|
|
121
122
|
- Read-only by default. Fixes need an explicit confirmation (CLI) or `allow_writes` (server/MCP), always run a
|
|
122
123
|
server-side dry run first and are written to an audit log.
|
|
123
124
|
|
|
@@ -137,4 +138,5 @@ and `lint` takes `-o sarif` (GitHub code scanning). `--fail-on high` makes CI fa
|
|
|
137
138
|
pip install -e ".[dev]"
|
|
138
139
|
pytest # unit tests on a synthetic broken cluster
|
|
139
140
|
./scripts/e2e.sh # breaks a kind cluster on purpose and checks every failure is explained
|
|
141
|
+
python scripts/eval_ai.py --provider anthropic # grades an LLM's explanations (needs its API key)
|
|
140
142
|
```
|
|
@@ -2,7 +2,7 @@ apiVersion: v2
|
|
|
2
2
|
name: kubesift
|
|
3
3
|
description: Evidence-based Kubernetes troubleshooting service (dashboard, API, Alertmanager receiver, background scans)
|
|
4
4
|
type: application
|
|
5
|
-
version: 2.
|
|
6
|
-
appVersion: "2.
|
|
5
|
+
version: 2.1.0
|
|
6
|
+
appVersion: "2.1.0"
|
|
7
7
|
home: https://github.com/OpeyemiAdeniji/KubeSift
|
|
8
8
|
keywords: [kubernetes, troubleshooting, sre, observability]
|
|
@@ -60,6 +60,10 @@ spec:
|
|
|
60
60
|
- name: KUBESIFT_LLM_BASE_URL
|
|
61
61
|
value: {{ . | quote }}
|
|
62
62
|
{{- end }}
|
|
63
|
+
{{- with .Values.llm.timeout }}
|
|
64
|
+
- name: KUBESIFT_LLM_TIMEOUT
|
|
65
|
+
value: {{ . | quote }}
|
|
66
|
+
{{- end }}
|
|
63
67
|
- name: KUBESIFT_SEND_LOGS_TO_LLM
|
|
64
68
|
value: {{ .Values.llm.sendLogs | quote }}
|
|
65
69
|
{{- with .Values.prometheus.url }}
|
|
@@ -23,7 +23,8 @@ existingSecret: ""
|
|
|
23
23
|
llm:
|
|
24
24
|
provider: none # anthropic | openai | openrouter | ollama | none
|
|
25
25
|
model: ""
|
|
26
|
-
baseUrl: "" # e.g. http://ollama.ollama:11434
|
|
26
|
+
baseUrl: "" # e.g. http://ollama.ollama:11434 for Ollama
|
|
27
|
+
timeout: 0 # seconds to wait for one answer; 0 = default (180, Ollama 900)
|
|
27
28
|
anthropicApiKey: ""
|
|
28
29
|
openaiApiKey: ""
|
|
29
30
|
openrouterApiKey: ""
|
|
@@ -27,6 +27,11 @@ cost_gb_hour: 0.0042
|
|
|
27
27
|
Inside a pod, KubeSift uses the service account automatically. It needs **read** access only; see
|
|
28
28
|
`deploy/helm/kubesift/templates/rbac.yaml` for the exact rules. Missing permissions are reported, not fatal.
|
|
29
29
|
|
|
30
|
+
Before reading anything KubeSift checks that the cluster answers and accepts the login, and stops with an error
|
|
31
|
+
(exit code 1) when it doesn't, instead of producing an empty report that looks healthy. The message names the
|
|
32
|
+
context and the cause: unreachable API server, login rejected (expired, revoked or deleted account), or a
|
|
33
|
+
failing login helper such as `gke-gcloud-auth-plugin`. `kubesift doctor` runs the same check.
|
|
34
|
+
|
|
30
35
|
## LLM
|
|
31
36
|
|
|
32
37
|
KubeSift never needs an LLM: every finding comes from deterministic checks. An LLM only ranks and explains them.
|
|
@@ -35,9 +40,11 @@ KubeSift never needs an LLM: every finding comes from deterministic checks. An L
|
|
|
35
40
|
|---|---|---|
|
|
36
41
|
| `llm_provider` | `auto` | `auto` picks the first of `ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY` that is set, else `none` |
|
|
37
42
|
| `llm_model` | per provider | Anthropic: `claude-opus-5-5` |
|
|
38
|
-
| `llm_base_url` | per provider | Any OpenAI-compatible endpoint (vLLM, LiteLLM, Azure gateway) |
|
|
43
|
+
| `llm_base_url` | per provider | Any OpenAI-compatible endpoint (vLLM, LiteLLM, Azure gateway), or your Ollama server |
|
|
39
44
|
| `llm_api_key` | from provider env var | |
|
|
40
45
|
| `llm_effort` | `medium` | Anthropic effort: `low`, `medium`, `high` |
|
|
46
|
+
| `llm_timeout` | `180` (`900` for Ollama) | Seconds to wait for one answer. A slow answer is not retried |
|
|
47
|
+
| `llm_context` | sized to the prompt | Ollama only: context window in tokens (8192–32768 by default) |
|
|
41
48
|
| `send_logs_to_llm` | `true` | `false` keeps log lines out of prompts |
|
|
42
49
|
|
|
43
50
|
The Anthropic provider sends requests with server-side refusal fallback enabled (`fallbacks: "default"`), so a
|
|
@@ -51,6 +58,13 @@ addresses in log lines are redacted before anything is stored or sent. To keep e
|
|
|
51
58
|
export KUBESIFT_LLM_PROVIDER=ollama KUBESIFT_LLM_MODEL=llama3.1
|
|
52
59
|
```
|
|
53
60
|
|
|
61
|
+
KubeSift talks to Ollama's native API: it turns off hidden reasoning ("thinking") so answers come faster, sizes the
|
|
62
|
+
context window to the prompt (Ollama's default window is small and silently drops the start of a long prompt), and
|
|
63
|
+
constrains the output to the expected JSON. A diagnosis prompt is a few thousand tokens. On a machine with a GPU a
|
|
64
|
+
7–8B model answers in seconds; on a laptop CPU it can take many minutes, so raise `llm_timeout`, pick a smaller
|
|
65
|
+
model, use one of Ollama's cloud models, or run with `--no-llm`. If the model does not answer in time KubeSift
|
|
66
|
+
shows the rule-based diagnosis instead.
|
|
67
|
+
|
|
54
68
|
## Usage data and cost
|
|
55
69
|
|
|
56
70
|
| Name | Default | Notes |
|
|
@@ -89,3 +103,7 @@ Without Prometheus, metrics-server's current usage is used (one sample; treat as
|
|
|
89
103
|
| `slack_webhook_url`, `teams_webhook_url`, `webhook_url` | none | Notifications for new problems and Alertmanager investigations |
|
|
90
104
|
| `slack_signing_secret` | none | Enables the `/sift` Slack command and fix buttons (see [slack.md](slack.md)) |
|
|
91
105
|
| `slack_allowed_users` | none | Slack user IDs allowed to apply fixes (comma-separated); empty means nobody; also needs `allow_writes` |
|
|
106
|
+
|
|
107
|
+
The dashboard shows the rule-based report at once and fills in the LLM explanation when it arrives, so a slow
|
|
108
|
+
local model never blocks it. API clients get the same with `POST /api/v1/diagnose {"wait_for_llm": false}`: the
|
|
109
|
+
response has `llm_pending: true` and an `id`; poll `GET /api/v1/history/{id}` until `llm_pending` is false.
|
|
@@ -31,6 +31,8 @@ profile (**Packages**) and set their visibility to **Public**, so people can pul
|
|
|
31
31
|
1. Update the version in `src/kubesift/__init__.py` and `deploy/helm/kubesift/Chart.yaml`
|
|
32
32
|
(`version` and `appVersion`).
|
|
33
33
|
2. Merge to `main`, then tag. The workflow refuses a tag that doesn't match the package version.
|
|
34
|
+
Optionally run the **AI eval** workflow (Actions tab) first if the prompt or LLM code changed; it needs a
|
|
35
|
+
provider API key as a repository secret.
|
|
34
36
|
|
|
35
37
|
```bash
|
|
36
38
|
git tag v2.0.0
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Score an LLM on KubeSift's demo broken cluster: does it find the real causes and cite real evidence?
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
python scripts/eval_ai.py --provider anthropic # needs ANTHROPIC_API_KEY
|
|
5
|
+
python scripts/eval_ai.py --provider ollama --model llama3.1 # local; slow without a GPU
|
|
6
|
+
python scripts/eval_ai.py --provider openrouter --model openai/gpt-5 --json results.json
|
|
7
|
+
|
|
8
|
+
What is measured (the deterministic checks already found every fault; this grades the explanation):
|
|
9
|
+
coverage share of the five root problems that some root cause cites as evidence
|
|
10
|
+
cited share of root causes that cite at least one real finding (the rest are capped at 40% confidence)
|
|
11
|
+
questions three "why is X failing?" questions answered with the right evidence
|
|
12
|
+
|
|
13
|
+
Exit code 1 when coverage or cited is below its threshold, so it can gate a CI job.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import json
|
|
20
|
+
import sys
|
|
21
|
+
import time
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tests"))
|
|
26
|
+
|
|
27
|
+
from fixtures import broken_cluster # noqa: E402
|
|
28
|
+
|
|
29
|
+
from kubesift.analysis import engine # noqa: E402
|
|
30
|
+
from kubesift.config import load_settings # noqa: E402
|
|
31
|
+
|
|
32
|
+
# The five faults an on-call engineer must not miss in the demo cluster, keyed by the check that proves each.
|
|
33
|
+
ROOT_PROBLEMS = {
|
|
34
|
+
"pod-crashloop": "checkout crash loops after the 1.5 rollout removed DATABASE_URL",
|
|
35
|
+
"pod-oomkilled": "api is OOMKilled at a 64Mi limit",
|
|
36
|
+
"webhook-unavailable": "policy webhook points at a missing Service and blocks changes",
|
|
37
|
+
"node-not-ready": "node-2 is NotReady",
|
|
38
|
+
"config-ref-missing": "reports needs a ConfigMap that does not exist",
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
QUESTIONS = [
|
|
42
|
+
("why is checkout failing?", "pod-crashloop", "DATABASE_URL"),
|
|
43
|
+
("why does the api deployment keep restarting?", "pod-oomkilled", "memory"),
|
|
44
|
+
("why does the web service have no endpoints?", "service-no-endpoints", "selector"),
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def evaluate(settings: Any) -> dict[str, Any]:
|
|
49
|
+
rules = engine.investigate(settings, broken_cluster(), use_llm=False)
|
|
50
|
+
by_check: dict[str, set[str]] = {}
|
|
51
|
+
for incident in rules.incidents:
|
|
52
|
+
ids = {f.fingerprint for f in incident.findings}
|
|
53
|
+
for f in incident.findings:
|
|
54
|
+
by_check.setdefault(f.check, set()).update(ids)
|
|
55
|
+
|
|
56
|
+
started = time.monotonic()
|
|
57
|
+
report = engine.investigate(settings, broken_cluster(), use_llm=True)
|
|
58
|
+
diagnose_s = time.monotonic() - started
|
|
59
|
+
diagnosis = report.diagnosis
|
|
60
|
+
if diagnosis is None or diagnosis.generated_by == "rules":
|
|
61
|
+
return {"error": "the LLM did not answer: " + "; ".join(report.warnings[-2:]), "diagnose_seconds": diagnose_s}
|
|
62
|
+
|
|
63
|
+
cited_ids = {e for rc in diagnosis.root_causes for e in rc.evidence}
|
|
64
|
+
found = {check: bool(by_check.get(check, set()) & cited_ids) for check in ROOT_PROBLEMS}
|
|
65
|
+
causes = [{"title": rc.title, "confidence": rc.confidence, "cites": len(rc.evidence)} for rc in diagnosis.root_causes]
|
|
66
|
+
cited_share = sum(1 for rc in diagnosis.root_causes if rc.evidence) / max(1, len(diagnosis.root_causes))
|
|
67
|
+
|
|
68
|
+
answers = []
|
|
69
|
+
for question, check, keyword in QUESTIONS:
|
|
70
|
+
t = time.monotonic()
|
|
71
|
+
_, answer = engine.ask(settings, broken_cluster(), question, use_llm=True)
|
|
72
|
+
llm = not str(answer.get("generated_by", "rules")).startswith("rules")
|
|
73
|
+
answers.append({
|
|
74
|
+
"question": question, "llm": llm, "seconds": round(time.monotonic() - t, 1),
|
|
75
|
+
"right_evidence": bool(by_check.get(check, set()) & set(answer.get("evidence") or [])),
|
|
76
|
+
"mentions": keyword.lower() in str(answer.get("answer", "")).lower(),
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
"model": diagnosis.generated_by,
|
|
81
|
+
"diagnose_seconds": round(diagnose_s, 1),
|
|
82
|
+
"coverage": sum(found.values()) / len(found),
|
|
83
|
+
"found": found,
|
|
84
|
+
"cited": round(cited_share, 2),
|
|
85
|
+
"root_causes": causes,
|
|
86
|
+
"questions": answers,
|
|
87
|
+
"questions_right": sum(1 for a in answers if a["right_evidence"]),
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def main() -> int:
|
|
92
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
93
|
+
ap.add_argument("--provider", required=True, help="anthropic | openai | openrouter | ollama")
|
|
94
|
+
ap.add_argument("--model", help="model name (default: the provider's default)")
|
|
95
|
+
ap.add_argument("--json", help="also write the results to this file")
|
|
96
|
+
ap.add_argument("--min-coverage", type=float, default=0.6)
|
|
97
|
+
ap.add_argument("--min-cited", type=float, default=0.8)
|
|
98
|
+
args = ap.parse_args()
|
|
99
|
+
|
|
100
|
+
settings = load_settings(llm_provider=args.provider, llm_model=args.model)
|
|
101
|
+
result = evaluate(settings)
|
|
102
|
+
if args.json:
|
|
103
|
+
Path(args.json).write_text(json.dumps(result, indent=2), encoding="utf-8")
|
|
104
|
+
if "error" in result:
|
|
105
|
+
print(f"FAIL {result['error']}")
|
|
106
|
+
return 1
|
|
107
|
+
|
|
108
|
+
print(f"model {result['model']} ({result['diagnose_seconds']}s for the diagnosis)")
|
|
109
|
+
print(f"coverage {result['coverage']:.0%} of the root problems cited (min {args.min_coverage:.0%})")
|
|
110
|
+
for check, ok in result["found"].items():
|
|
111
|
+
print(f" {'+' if ok else '-'} {ROOT_PROBLEMS[check]}")
|
|
112
|
+
print(f"cited {result['cited']:.0%} of root causes cite real evidence (min {args.min_cited:.0%})")
|
|
113
|
+
for rc in result["root_causes"]:
|
|
114
|
+
print(f" [{rc['confidence']:>3}%] {rc['title'][:90]} ({rc['cites']} citation(s))")
|
|
115
|
+
print(f"questions {result['questions_right']}/{len(result['questions'])} answered with the right evidence")
|
|
116
|
+
for a in result["questions"]:
|
|
117
|
+
print(f" {'+' if a['right_evidence'] else '-'} {a['question']} ({a['seconds']}s{'' if a['llm'] else ', rules fallback'})")
|
|
118
|
+
ok = result["coverage"] >= args.min_coverage and result["cited"] >= args.min_cited
|
|
119
|
+
print("PASS" if ok else "FAIL")
|
|
120
|
+
return 0 if ok else 1
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
if __name__ == "__main__":
|
|
124
|
+
sys.exit(main())
|
|
@@ -28,7 +28,8 @@ SYSTEM_PROMPT = """You are KubeSift, a senior Kubernetes SRE helping an on-call
|
|
|
28
28
|
You receive findings that deterministic checks proved from live cluster data, recent changes, and resource summaries.
|
|
29
29
|
|
|
30
30
|
Rules:
|
|
31
|
-
- Base every conclusion on the provided
|
|
31
|
+
- Base every conclusion on the provided findings. Each finding has an `id` (12 hex characters) and `facts`.
|
|
32
|
+
List the `id` of every finding a conclusion rests on in `evidence_ids`, copied exactly, e.g. "3f2a9c1b7d0e".
|
|
32
33
|
- Never invent resource names, versions, or log lines. If the data does not show the cause, say what is missing and how to get it.
|
|
33
34
|
- Separate the root cause from its symptoms: a bad rollout, a config edit or a broken node often explains several findings.
|
|
34
35
|
- When a recent change lines up with the start of a problem, say so and give the rollback command if one is provided.
|
|
@@ -47,12 +48,13 @@ DIAGNOSIS_SCHEMA: dict[str, Any] = {
|
|
|
47
48
|
"items": {
|
|
48
49
|
"type": "object",
|
|
49
50
|
"additionalProperties": False,
|
|
50
|
-
"required": ["title", "explanation", "confidence", "
|
|
51
|
+
"required": ["title", "explanation", "confidence", "evidence_ids", "fix_steps", "commands"],
|
|
51
52
|
"properties": {
|
|
52
53
|
"title": {"type": "string"},
|
|
53
54
|
"explanation": {"type": "string"},
|
|
54
55
|
"confidence": {"type": "integer"},
|
|
55
|
-
"
|
|
56
|
+
"evidence_ids": {"type": "array", "items": {"type": "string"},
|
|
57
|
+
"description": "The `id` values of the findings this cause rests on."},
|
|
56
58
|
"fix_steps": {"type": "array", "items": {"type": "string"}},
|
|
57
59
|
"commands": {"type": "array", "items": {"type": "string"}},
|
|
58
60
|
},
|
|
@@ -65,11 +67,12 @@ DIAGNOSIS_SCHEMA: dict[str, Any] = {
|
|
|
65
67
|
ANSWER_SCHEMA: dict[str, Any] = {
|
|
66
68
|
"type": "object",
|
|
67
69
|
"additionalProperties": False,
|
|
68
|
-
"required": ["answer", "confidence", "
|
|
70
|
+
"required": ["answer", "confidence", "evidence_ids", "commands", "follow_up"],
|
|
69
71
|
"properties": {
|
|
70
72
|
"answer": {"type": "string"},
|
|
71
73
|
"confidence": {"type": "integer"},
|
|
72
|
-
"
|
|
74
|
+
"evidence_ids": {"type": "array", "items": {"type": "string"},
|
|
75
|
+
"description": "The `id` values of the findings the answer rests on."},
|
|
73
76
|
"commands": {"type": "array", "items": {"type": "string"}},
|
|
74
77
|
"follow_up": {"type": "array", "items": {"type": "string"}},
|
|
75
78
|
},
|
|
@@ -149,9 +152,9 @@ def rules_diagnosis(report: Report) -> Diagnosis:
|
|
|
149
152
|
def _finding_payload(f: Finding, logs: bool = True) -> dict[str, Any]:
|
|
150
153
|
evidence = [e for e in f.evidence if logs or not e.startswith("log ")]
|
|
151
154
|
return {
|
|
152
|
-
"
|
|
155
|
+
"id": f.fingerprint, "check": f.check, "severity": f.severity.value, "resource": str(f.resource),
|
|
153
156
|
"owner": str(f.owner) if f.owner else None, "node": f.node, "message": f.message,
|
|
154
|
-
"
|
|
157
|
+
"facts": evidence[:8], "suggested_fix": f.remediation, "commands": f.commands[:3],
|
|
155
158
|
"started": f.since.isoformat() if f.since else None,
|
|
156
159
|
"related_changes": [c.summary + (f" [rollback: {c.rollback_command}]" if c.rollback_command else "")
|
|
157
160
|
for c in f.related_changes],
|
|
@@ -164,7 +167,7 @@ def _context_json(report: Report, extra: dict[str, Any] | None = None, logs: boo
|
|
|
164
167
|
"cluster": report.cluster, "namespace": report.namespace or "all", "kubernetes": report.server_version,
|
|
165
168
|
"incidents": [{"title": i.title, "layer": i.layer, "severity": i.severity.value,
|
|
166
169
|
"findings": [_finding_payload(f, logs) for f in i.findings[:6]]} for i in report.incidents[:12]],
|
|
167
|
-
"risks": [{"
|
|
170
|
+
"risks": [{"id": f.fingerprint, "check": f.check, "resource": str(f.resource), "message": f.message}
|
|
168
171
|
for f in report.findings if f.kind == "risk"][:25],
|
|
169
172
|
"recent_changes": [{"time": c.time.isoformat(), "summary": c.summary, "details": c.details[:5],
|
|
170
173
|
"rollback": c.rollback_command} for c in report.changes[:15]],
|
|
@@ -201,11 +204,22 @@ def explain(settings: Settings, report: Report, use_llm: bool = True) -> Diagnos
|
|
|
201
204
|
return base
|
|
202
205
|
|
|
203
206
|
|
|
207
|
+
def _cited(item: dict[str, Any], known: set[str]) -> list[str]:
|
|
208
|
+
"""Finding ids the model cited. Accepts an id wrapped in extra text ("finding 3f2a9c1b7d0e")."""
|
|
209
|
+
cited: list[str] = []
|
|
210
|
+
for raw in item.get("evidence_ids") or item.get("evidence") or []:
|
|
211
|
+
text = str(raw)
|
|
212
|
+
for fp in [text] if text in known else [k for k in known if k in text]:
|
|
213
|
+
if fp not in cited:
|
|
214
|
+
cited.append(fp)
|
|
215
|
+
return cited
|
|
216
|
+
|
|
217
|
+
|
|
204
218
|
def _validated(data: dict[str, Any], report: Report, by: str) -> Diagnosis:
|
|
205
219
|
known = {f.fingerprint for f in report.findings}
|
|
206
220
|
causes = []
|
|
207
221
|
for rc in data.get("root_causes") or []:
|
|
208
|
-
evidence =
|
|
222
|
+
evidence = _cited(rc, known)
|
|
209
223
|
conf = max(0, min(100, int(rc.get("confidence", 0))))
|
|
210
224
|
if not evidence:
|
|
211
225
|
conf = min(conf, 40) # unsupported claims cannot be confident
|
|
@@ -301,7 +315,7 @@ def ask(settings: Settings, snap: Snapshot, question: str, use_llm: bool = True,
|
|
|
301
315
|
answer = {
|
|
302
316
|
"answer": str(data.get("answer", "")),
|
|
303
317
|
"confidence": max(0, min(100, int(data.get("confidence", 0)))),
|
|
304
|
-
"evidence":
|
|
318
|
+
"evidence": _cited(data, known),
|
|
305
319
|
"commands": [str(c) for c in data.get("commands") or []][:6],
|
|
306
320
|
"follow_up": [str(c) for c in data.get("follow_up") or []][:5],
|
|
307
321
|
"generated_by": f"{provider.name}:{settings.resolved_model()}",
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
Providers:
|
|
4
4
|
anthropic Claude through the official Anthropic SDK (pip install 'kubesift[anthropic]')
|
|
5
|
-
openai any OpenAI-compatible endpoint: OpenAI, OpenRouter,
|
|
5
|
+
openai any OpenAI-compatible endpoint: OpenAI, OpenRouter, vLLM, LiteLLM, Azure gateways
|
|
6
|
+
openrouter OpenRouter (OpenAI-compatible)
|
|
7
|
+
ollama local models through Ollama's native API
|
|
6
8
|
none rules only
|
|
7
9
|
"""
|
|
8
10
|
|
|
@@ -40,7 +42,8 @@ class AnthropicProvider(Provider):
|
|
|
40
42
|
except ImportError as exc:
|
|
41
43
|
raise LLMError("Install the Anthropic SDK: pip install 'kubesift[anthropic]'") from exc
|
|
42
44
|
self._anthropic = anthropic
|
|
43
|
-
|
|
45
|
+
self.timeout = settings.resolved_llm_timeout()
|
|
46
|
+
kwargs: dict[str, Any] = {"timeout": self.timeout}
|
|
44
47
|
if settings.llm_api_key:
|
|
45
48
|
kwargs["api_key"] = settings.llm_api_key
|
|
46
49
|
if settings.llm_base_url:
|
|
@@ -68,8 +71,12 @@ class AnthropicProvider(Provider):
|
|
|
68
71
|
response = self.client.messages.create(**request)
|
|
69
72
|
except a.APIStatusError as exc:
|
|
70
73
|
raise LLMError(f"Anthropic API error {exc.status_code}: {exc.message}") from exc
|
|
74
|
+
except a.APITimeoutError as exc:
|
|
75
|
+
raise LLMError(_timeout_message(self.model, self.timeout)) from exc
|
|
71
76
|
except a.APIConnectionError as exc:
|
|
72
77
|
raise LLMError(f"Cannot reach the Anthropic API: {exc}") from exc
|
|
78
|
+
except a.APITimeoutError as exc:
|
|
79
|
+
raise LLMError(_timeout_message(self.model, self.timeout)) from exc
|
|
73
80
|
except a.AuthenticationError as exc:
|
|
74
81
|
raise LLMError("Anthropic rejected the API key (set ANTHROPIC_API_KEY).") from exc
|
|
75
82
|
except a.RateLimitError as exc:
|
|
@@ -84,6 +91,15 @@ class AnthropicProvider(Provider):
|
|
|
84
91
|
return parse_json(text)
|
|
85
92
|
|
|
86
93
|
|
|
94
|
+
def _timeout_message(model: str, seconds: float) -> str:
|
|
95
|
+
return (f"{model} did not answer within {seconds:.0f}s. Local models on a CPU can take many minutes: "
|
|
96
|
+
"raise llm_timeout (KUBESIFT_LLM_TIMEOUT), use a smaller or hosted model, or run with --no-llm.")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _with_schema(system: str, schema: dict[str, Any]) -> str:
|
|
100
|
+
return system + "\n\nReply with one JSON object matching this schema:\n" + json.dumps(schema)
|
|
101
|
+
|
|
102
|
+
|
|
87
103
|
class OpenAICompatibleProvider(Provider):
|
|
88
104
|
name = "openai"
|
|
89
105
|
|
|
@@ -92,6 +108,7 @@ class OpenAICompatibleProvider(Provider):
|
|
|
92
108
|
self.model = settings.resolved_model()
|
|
93
109
|
self.api_key = settings.resolved_api_key()
|
|
94
110
|
self.provider = settings.resolved_provider()
|
|
111
|
+
self.timeout = settings.resolved_llm_timeout()
|
|
95
112
|
if self.provider != "ollama" and not self.api_key and "localhost" not in self.base_url:
|
|
96
113
|
raise LLMError(f"No API key for {self.provider}; set {self.provider.upper()}_API_KEY or KUBESIFT_LLM_API_KEY.")
|
|
97
114
|
|
|
@@ -104,16 +121,20 @@ class OpenAICompatibleProvider(Provider):
|
|
|
104
121
|
body: dict[str, Any] = {
|
|
105
122
|
"model": self.model,
|
|
106
123
|
"messages": [
|
|
107
|
-
{"role": "system", "content": system
|
|
108
|
-
+ json.dumps(schema)},
|
|
124
|
+
{"role": "system", "content": _with_schema(system, schema)},
|
|
109
125
|
{"role": "user", "content": user},
|
|
110
126
|
],
|
|
111
127
|
"response_format": {"type": "json_object"},
|
|
112
128
|
}
|
|
113
|
-
with httpx.Client(timeout=
|
|
129
|
+
with httpx.Client(timeout=httpx.Timeout(self.timeout, connect=15.0)) as http:
|
|
114
130
|
for attempt in range(3):
|
|
115
131
|
try:
|
|
116
132
|
r = http.post(f"{self.base_url}/chat/completions", headers=headers, json=body)
|
|
133
|
+
except httpx.TimeoutException as exc:
|
|
134
|
+
if isinstance(exc, httpx.ConnectTimeout) and attempt < 2:
|
|
135
|
+
continue
|
|
136
|
+
# a slow answer is not retried: the next attempt would redo the same slow work
|
|
137
|
+
raise LLMError(_timeout_message(self.model, self.timeout)) from exc
|
|
117
138
|
except httpx.HTTPError as exc:
|
|
118
139
|
if attempt == 2:
|
|
119
140
|
raise LLMError(f"Cannot reach {self.base_url}: {exc}") from exc
|
|
@@ -130,6 +151,81 @@ class OpenAICompatibleProvider(Provider):
|
|
|
130
151
|
raise LLMError("LLM request failed after retries")
|
|
131
152
|
|
|
132
153
|
|
|
154
|
+
class OllamaProvider(Provider):
|
|
155
|
+
"""Ollama's native API: turns off hidden reasoning, sizes the context window to the prompt (Ollama's
|
|
156
|
+
default window is small and silently drops the start of longer prompts) and constrains output to the schema."""
|
|
157
|
+
|
|
158
|
+
name = "ollama"
|
|
159
|
+
MAX_OUTPUT = 4096
|
|
160
|
+
|
|
161
|
+
def __init__(self, settings: Settings):
|
|
162
|
+
base = (settings.resolved_base_url() or "http://localhost:11434").rstrip("/")
|
|
163
|
+
self.base_url = base[:-3] if base.endswith("/v1") else base
|
|
164
|
+
self.model = settings.resolved_model()
|
|
165
|
+
self.timeout = settings.resolved_llm_timeout()
|
|
166
|
+
self.context = settings.llm_context
|
|
167
|
+
self.api_key = settings.resolved_api_key()
|
|
168
|
+
|
|
169
|
+
def status(self) -> tuple[bool, str]:
|
|
170
|
+
"""Is the server up and the model pulled? Used by `kubesift doctor`."""
|
|
171
|
+
headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
|
|
172
|
+
try:
|
|
173
|
+
r = httpx.get(f"{self.base_url}/api/tags", headers=headers, timeout=5.0)
|
|
174
|
+
r.raise_for_status()
|
|
175
|
+
names = {m.get("name") for m in r.json().get("models") or []}
|
|
176
|
+
except (httpx.HTTPError, ValueError) as exc:
|
|
177
|
+
return False, f"cannot reach Ollama at {self.base_url} ({exc.__class__.__name__}); is it running? (ollama serve)"
|
|
178
|
+
wanted = self.model if ":" in self.model else f"{self.model}:latest"
|
|
179
|
+
if wanted not in names and self.model not in names:
|
|
180
|
+
return False, f"Ollama is running but has no model {self.model!r}; run: ollama pull {self.model}"
|
|
181
|
+
return True, f"Ollama at {self.base_url} has {self.model}"
|
|
182
|
+
|
|
183
|
+
def context_size(self, text_chars: int) -> int:
|
|
184
|
+
if self.context:
|
|
185
|
+
return self.context
|
|
186
|
+
needed = text_chars // 3 + self.MAX_OUTPUT # ~3 characters per token for JSON-heavy text
|
|
187
|
+
return min(max(8192, -(-needed // 2048) * 2048), 32768)
|
|
188
|
+
|
|
189
|
+
def complete_json(self, system: str, user: str, schema: dict[str, Any]) -> dict[str, Any]:
|
|
190
|
+
system = _with_schema(system, schema)
|
|
191
|
+
body: dict[str, Any] = {
|
|
192
|
+
"model": self.model,
|
|
193
|
+
"stream": False,
|
|
194
|
+
"think": False,
|
|
195
|
+
"format": schema,
|
|
196
|
+
"options": {"num_ctx": self.context_size(len(system) + len(user)), "num_predict": self.MAX_OUTPUT,
|
|
197
|
+
"temperature": 0.2},
|
|
198
|
+
"messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
|
|
199
|
+
}
|
|
200
|
+
url = f"{self.base_url}/api/chat"
|
|
201
|
+
headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
|
|
202
|
+
with httpx.Client(timeout=httpx.Timeout(self.timeout, connect=10.0)) as http:
|
|
203
|
+
for _ in range(3):
|
|
204
|
+
try:
|
|
205
|
+
r = http.post(url, json=body, headers=headers)
|
|
206
|
+
except httpx.ConnectError as exc:
|
|
207
|
+
raise LLMError(f"Cannot reach Ollama at {self.base_url}; is it running? (ollama serve)") from exc
|
|
208
|
+
except httpx.TimeoutException as exc:
|
|
209
|
+
raise LLMError(_timeout_message(self.model, self.timeout)) from exc
|
|
210
|
+
except httpx.HTTPError as exc:
|
|
211
|
+
raise LLMError(f"Ollama request failed: {exc}") from exc
|
|
212
|
+
if r.status_code == 404 and "not found" in r.text.lower():
|
|
213
|
+
raise LLMError(f"Ollama has no model {self.model!r}; run: ollama pull {self.model}")
|
|
214
|
+
if r.status_code == 400 and "think" in body and "think" in r.text.lower():
|
|
215
|
+
body.pop("think") # older Ollama or a model without a thinking switch
|
|
216
|
+
continue
|
|
217
|
+
if r.status_code == 400 and isinstance(body.get("format"), dict):
|
|
218
|
+
body["format"] = "json" # older Ollama without schema-constrained output
|
|
219
|
+
continue
|
|
220
|
+
if r.status_code >= 400:
|
|
221
|
+
raise LLMError(f"Ollama returned HTTP {r.status_code}: {r.text[:300]}")
|
|
222
|
+
data = r.json()
|
|
223
|
+
if data.get("done_reason") == "length":
|
|
224
|
+
raise LLMError("The model ran out of output tokens before finishing its answer.")
|
|
225
|
+
return parse_json((data.get("message") or {}).get("content") or "")
|
|
226
|
+
raise LLMError("Ollama rejected the request.")
|
|
227
|
+
|
|
228
|
+
|
|
133
229
|
def parse_json(text: str) -> dict[str, Any]:
|
|
134
230
|
text = re.sub(r"```(?:json)?", "", text).strip()
|
|
135
231
|
start, end = text.find("{"), text.rfind("}")
|
|
@@ -147,6 +243,8 @@ def get_provider(settings: Settings) -> Provider | None:
|
|
|
147
243
|
return None
|
|
148
244
|
if name == "anthropic":
|
|
149
245
|
return AnthropicProvider(settings)
|
|
150
|
-
if name
|
|
246
|
+
if name == "ollama":
|
|
247
|
+
return OllamaProvider(settings)
|
|
248
|
+
if name in ("openai", "openrouter"):
|
|
151
249
|
return OpenAICompatibleProvider(settings)
|
|
152
250
|
raise LLMError(f"Unknown LLM provider {name!r}; use anthropic, openai, openrouter, ollama or none.")
|