vmware-debug 1.12.2__tar.gz → 1.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/.claude-plugin/plugin.json +1 -1
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/.mcp.json +1 -1
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/PKG-INFO +1 -1
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/RELEASE_NOTES.md +25 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/openclaw.plugin.json +1 -1
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/package.json +1 -1
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/pyproject.toml +1 -1
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/server.json +2 -2
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/skills/vmware-debug/SKILL.md +5 -2
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/skills/vmware-debug/references/setup-guide.md +2 -2
- vmware_debug-1.13.0/tests/test_case_grading_per_hypothesis.py +131 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/uv.lock +1 -1
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/__init__.py +1 -1
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/mcp_server/server.py +9 -3
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/api.py +1 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/grading.py +95 -24
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/.clawhubignore +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/.gitignore +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/README-CN.md +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/README.md +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/SECURITY.md +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/docs/plans/2026-08-28-investigation-layer-design.md +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/skills/vmware-debug/references/agent-guardrails.md +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/skills/vmware-debug/references/capabilities.md +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/skills/vmware-debug/references/cli-reference.md +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/skills/vmware-debug/references/event-envelope.md +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/skills/vmware-debug/references/routing.md +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/__init__.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/conftest.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/__init__.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/__init__.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/_family.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/_scores.json +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/_scoring.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/_skill.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/conftest.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/test_entity_reachability.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/test_error_actionability.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/test_tool_description_quality.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/test_tool_manifest_budget.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/__init__.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/_vsphere_event_vocabulary.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_audit_isolation.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_capability_grader.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_case_id_non_ascii.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_case_plan_max_steps.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_classification_against_vsphere_vocabulary.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_debug_regressions.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_environment_resolver_isolation.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_every_tool_parameter_is_documented.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_evidence_payload_contract.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_host_lifecycle_classification.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_investigation_metrics.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_real_alert_titles_classify.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_result_envelope.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_skill_name_aliases.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_spike_resolution.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_tool_annotations.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_case_conclusion.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_case_evidence.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_case_grading.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_case_hypotheses.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_case_ids.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_case_ledger_concurrency.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_case_plan.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_case_readiness.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_case_store.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_case_timeline_close.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_case_tools.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_knowledge.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_mcp_audit.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_safe_error_passthrough.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/test_timeline.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/cli.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/envelope.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/mcp/__init__.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/mcp/tools.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/mcp_server/__init__.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/__init__.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/__init__.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/conclusion.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/evidence.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/hypotheses.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/ids.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/knowledge.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/model.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/payloads.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/plan.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/readiness.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/sources.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/store.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/cases/timeline.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/skill_names.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/ops/timeline.py +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/rules/evidence_sources.yaml +0 -0
- {vmware_debug-1.12.2 → vmware_debug-1.13.0}/vmware_debug/rules/grading_rules.yaml +0 -0
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://anthropic.com/claude-code/plugin.schema.json",
|
|
3
3
|
"name": "vmware-debug",
|
|
4
4
|
"displayName": "VMware Debug",
|
|
5
|
-
"version": "1.
|
|
5
|
+
"version": "1.13.0",
|
|
6
6
|
"description": "VMware incident timeline, root-cause routing, local case ledger; no vSphere access. 14 MCP tools.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Wei Zhou",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: vmware-debug
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.13.0
|
|
4
4
|
Summary: VMware diagnostic brain — offline incident triage, log/event correlation, and root-cause routing across the VMware skill family
|
|
5
5
|
Project-URL: Homepage, https://github.com/vmware-skills/VMware-Debug
|
|
6
6
|
Project-URL: Repository, https://github.com/vmware-skills/VMware-Debug
|
|
@@ -1,3 +1,28 @@
|
|
|
1
|
+
## v1.13.0 — exclusion is per hypothesis
|
|
2
|
+
|
|
3
|
+
Found in a real investigation of an Aria "vCenter app health is affected" alert on 2026-09-15, then corrected
|
|
4
|
+
after an independent review the same day.
|
|
5
|
+
|
|
6
|
+
**Ruling out one hypothesis no longer excludes the case.** Two sources ruled out H2 (host memory reclaim) while the
|
|
7
|
+
leading H1 stayed open behind three gaps, and `case_grade` graded the whole case Excluded — which reads as "the
|
|
8
|
+
answer was ruled out". Now the case is Excluded only when every registered hypothesis is ruled out; otherwise it
|
|
9
|
+
is graded for what remains.
|
|
10
|
+
|
|
11
|
+
* **Only evidence that rules nothing out counts for the hypotheses still open.** Evidence records what it
|
|
12
|
+
falsifies, never what it supports. The first version of this change counted the observations that ruled H2 out
|
|
13
|
+
as corroboration for H1, and a hypothesis added with no evidence at all inherited Probable. Independent sources
|
|
14
|
+
and decisive items for the open hypotheses are now counted only from items with an empty `falsifies`.
|
|
15
|
+
* A gap whose `blocks` lists only ruled-out hypotheses no longer holds the grade back; the reasons list it under
|
|
16
|
+
"No longer blocking" instead of "Blocking gap(s)".
|
|
17
|
+
* **Behaviour change:** `case_grade` returns `hypotheses`, every registered hypothesis as `{id, status}` with
|
|
18
|
+
status `open` or `excluded`. A case directory without a readable `hypotheses.md` is graded as if none were
|
|
19
|
+
registered, with a reason, instead of failing.
|
|
20
|
+
* Re-running the same investigation after the change: H2 and H3 excluded, H1 open, **Probable** from two sources
|
|
21
|
+
that rule nothing out, held below Confirmed by four open gaps.
|
|
22
|
+
|
|
23
|
+
Known and unchanged: excluding a hypothesis still needs two independent sources across the whole case, not two
|
|
24
|
+
that falsify that hypothesis; `case_hypotheses` marks a hypothesis refuted on a single falsifier.
|
|
25
|
+
|
|
1
26
|
## v1.12.2 — every tool call is audited, failures included
|
|
2
27
|
|
|
3
28
|
Until now no vmware-debug MCP tool wrote to `~/.vmware/audit.db`. The server said that was on
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "vmware-debug"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.13.0"
|
|
8
8
|
description = "VMware diagnostic brain — offline incident triage, log/event correlation, and root-cause routing across the VMware skill family"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -7,12 +7,12 @@
|
|
|
7
7
|
"url": "https://github.com/vmware-skills/VMware-Debug",
|
|
8
8
|
"source": "github"
|
|
9
9
|
},
|
|
10
|
-
"version": "1.
|
|
10
|
+
"version": "1.13.0",
|
|
11
11
|
"packages": [
|
|
12
12
|
{
|
|
13
13
|
"registryType": "pypi",
|
|
14
14
|
"identifier": "vmware-debug",
|
|
15
|
-
"version": "1.
|
|
15
|
+
"version": "1.13.0",
|
|
16
16
|
"runtimeHint": "uvx",
|
|
17
17
|
"packageArguments": [
|
|
18
18
|
{
|
|
@@ -51,7 +51,7 @@ write tools touch only the local case ledger.
|
|
|
51
51
|
## Quick Install
|
|
52
52
|
|
|
53
53
|
```bash
|
|
54
|
-
uv tool install vmware-debug==1.
|
|
54
|
+
uv tool install vmware-debug==1.13.0
|
|
55
55
|
vmware-debug categories # see what it can diagnose
|
|
56
56
|
```
|
|
57
57
|
|
|
@@ -175,7 +175,10 @@ ledger — submit the missing evidence, or record the gap that is blocking it.
|
|
|
175
175
|
- **Confirmed** — that, plus a decisive item: a direct hardware diagnostic, a
|
|
176
176
|
version-checked knowledge-base entry, or a vendor SR; and no gap left open
|
|
177
177
|
- **Excluded** — an observation that actually rules it out. "We looked and found
|
|
178
|
-
nothing" is a gap, not an exclusion
|
|
178
|
+
nothing" is a gap, not an exclusion. Exclusion is per hypothesis: the case is
|
|
179
|
+
Excluded only when every registered hypothesis is ruled out. Otherwise
|
|
180
|
+
`case_grade` grades what remains, counting only evidence that rules nothing
|
|
181
|
+
out, and lists each hypothesis as `open` or `excluded` in `hypotheses`
|
|
179
182
|
|
|
180
183
|
`case_plan` is not a checklist: submit evidence and the next plan is shorter,
|
|
181
184
|
lose a source and it routes around it. Its `unavailable` half is the important
|
|
@@ -6,7 +6,7 @@ is a pure, offline correlation engine. There is no `config.yaml` and no `.env`.
|
|
|
6
6
|
## Install
|
|
7
7
|
|
|
8
8
|
```bash
|
|
9
|
-
uv tool install vmware-debug==1.
|
|
9
|
+
uv tool install vmware-debug==1.13.0
|
|
10
10
|
vmware-debug categories # verify it runs
|
|
11
11
|
```
|
|
12
12
|
|
|
@@ -15,7 +15,7 @@ vmware-debug categories # verify it runs
|
|
|
15
15
|
```json
|
|
16
16
|
{
|
|
17
17
|
"command": "uvx",
|
|
18
|
-
"args": ["--from", "vmware-debug==1.
|
|
18
|
+
"args": ["--from", "vmware-debug==1.13.0", "vmware-debug-mcp"]
|
|
19
19
|
}
|
|
20
20
|
```
|
|
21
21
|
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Excluded is a verdict about a hypothesis, not about the whole case.
|
|
2
|
+
|
|
3
|
+
2026-09-15, a real investigation of a vCenter app-health alert: two sources
|
|
4
|
+
ruled out H2 (host memory reclaim) while the leading H1 stayed open behind three
|
|
5
|
+
gaps. ``case_grade`` graded the whole case Excluded, which reads as "the answer
|
|
6
|
+
was ruled out". The investigator had to explain in prose that only H2 was.
|
|
7
|
+
|
|
8
|
+
The first version of the fix over-corrected, and an independent review caught
|
|
9
|
+
it: the observations that ruled H2 out were then counted as corroboration for
|
|
10
|
+
H1, and a hypothesis added with no evidence at all inherited Probable. Evidence
|
|
11
|
+
records only what it falsifies, never what it supports, so an item that rules a
|
|
12
|
+
hypothesis out must not count for the ones left open.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import pytest
|
|
18
|
+
|
|
19
|
+
from vmware_debug.ops.cases.api import grade
|
|
20
|
+
from vmware_debug.ops.cases.evidence import Evidence, Gap, record_evidence, record_gap
|
|
21
|
+
from vmware_debug.ops.cases.grading import grade_case
|
|
22
|
+
from vmware_debug.ops.cases.hypotheses import add_hypothesis
|
|
23
|
+
from vmware_debug.ops.cases.model import Scope
|
|
24
|
+
from vmware_debug.ops.cases.store import create_case
|
|
25
|
+
|
|
26
|
+
AT = "2026-09-15T13:30:00Z"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@pytest.fixture(autouse=True)
|
|
30
|
+
def isolated_home(tmp_path, monkeypatch):
|
|
31
|
+
monkeypatch.setenv("OPS_HOME", str(tmp_path / "vmware"))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@pytest.fixture
|
|
35
|
+
def two_hypotheses():
|
|
36
|
+
cid = create_case(
|
|
37
|
+
Scope(
|
|
38
|
+
summary="vcenter app health",
|
|
39
|
+
determined_by="aria alert",
|
|
40
|
+
product_versions={"vsphere": "8.0.3"},
|
|
41
|
+
),
|
|
42
|
+
at=AT,
|
|
43
|
+
).case_id
|
|
44
|
+
add_hypothesis(cid, "appliance memory pressure") # H1
|
|
45
|
+
add_hypothesis(cid, "host reclaims appliance memory") # H2
|
|
46
|
+
return cid
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _evidence(skill: str, falsifies: tuple[str, ...] = ()) -> Evidence:
|
|
50
|
+
return Evidence(
|
|
51
|
+
source_skill=skill,
|
|
52
|
+
source_tool="t",
|
|
53
|
+
query={},
|
|
54
|
+
fetched_at="2026-09-15T13:31:00Z",
|
|
55
|
+
summary="s",
|
|
56
|
+
falsifies=falsifies,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _gap(blocks: tuple[str, ...], could_falsify: bool = True) -> Gap:
|
|
61
|
+
return Gap(
|
|
62
|
+
what="swap history",
|
|
63
|
+
why="not collected",
|
|
64
|
+
blocks=blocks,
|
|
65
|
+
could_falsify=could_falsify,
|
|
66
|
+
how_to_close="vami",
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_ruling_out_one_hypothesis_does_not_exclude_the_case(two_hypotheses):
|
|
71
|
+
record_evidence(two_hypotheses, _evidence("vmware-aria", falsifies=("H2",)))
|
|
72
|
+
record_evidence(two_hypotheses, _evidence("vmware-monitor", falsifies=("H2",)))
|
|
73
|
+
r = grade_case(two_hypotheses)
|
|
74
|
+
assert r.grade == "candidate"
|
|
75
|
+
assert r.hypotheses == (("H1", "open"), ("H2", "excluded"))
|
|
76
|
+
assert any("Still open: H1" in reason for reason in r.reasons)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_the_case_is_excluded_when_every_hypothesis_is_ruled_out(two_hypotheses):
|
|
80
|
+
record_evidence(two_hypotheses, _evidence("vmware-aria", falsifies=("H1", "H2")))
|
|
81
|
+
record_evidence(two_hypotheses, _evidence("vmware-monitor"))
|
|
82
|
+
assert grade_case(two_hypotheses).grade == "excluded"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_evidence_that_rules_one_out_is_not_corroboration_for_the_rest(two_hypotheses):
|
|
86
|
+
"""aria rules H2 out; monitor says something unrelated. Nothing is about H1."""
|
|
87
|
+
record_evidence(two_hypotheses, _evidence("vmware-aria", falsifies=("H2",)))
|
|
88
|
+
record_evidence(two_hypotheses, _evidence("vmware-monitor"))
|
|
89
|
+
assert grade_case(two_hypotheses).grade == "candidate"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_a_new_hypothesis_does_not_inherit_probable(two_hypotheses):
|
|
93
|
+
record_evidence(two_hypotheses, _evidence("vmware-aria", falsifies=("H1", "H2")))
|
|
94
|
+
record_evidence(two_hypotheses, _evidence("vmware-monitor", falsifies=("H1", "H2")))
|
|
95
|
+
assert grade_case(two_hypotheses).grade == "excluded"
|
|
96
|
+
add_hypothesis(two_hypotheses, "something else") # H3, no evidence at all
|
|
97
|
+
r = grade_case(two_hypotheses)
|
|
98
|
+
assert r.grade == "candidate"
|
|
99
|
+
assert ("H3", "open") in r.hypotheses
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_a_gap_that_blocks_only_a_ruled_out_hypothesis_holds_nothing_back(two_hypotheses):
|
|
103
|
+
"""Otherwise a question nobody needs answered any more pins H1 at Candidate."""
|
|
104
|
+
record_evidence(two_hypotheses, _evidence("vmware-aria", falsifies=("H2",)))
|
|
105
|
+
record_evidence(two_hypotheses, _evidence("vmware-monitor", falsifies=("H2",)))
|
|
106
|
+
record_evidence(two_hypotheses, _evidence("vmware-monitor"))
|
|
107
|
+
record_evidence(two_hypotheses, _evidence("vmware-log-insight"))
|
|
108
|
+
record_gap(two_hypotheses, _gap(blocks=("H2",)))
|
|
109
|
+
r = grade_case(two_hypotheses)
|
|
110
|
+
assert r.grade == "probable"
|
|
111
|
+
assert not any(reason.startswith("Blocking gap(s)") for reason in r.reasons)
|
|
112
|
+
assert any("No longer blocking" in reason for reason in r.reasons)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_a_gap_that_still_blocks_an_open_hypothesis_still_holds(two_hypotheses):
|
|
116
|
+
record_evidence(two_hypotheses, _evidence("vmware-aria", falsifies=("H2",)))
|
|
117
|
+
record_evidence(two_hypotheses, _evidence("vmware-monitor", falsifies=("H2",)))
|
|
118
|
+
record_evidence(two_hypotheses, _evidence("vmware-monitor"))
|
|
119
|
+
record_evidence(two_hypotheses, _evidence("vmware-log-insight"))
|
|
120
|
+
record_gap(two_hypotheses, _gap(blocks=("H1", "H2")))
|
|
121
|
+
assert grade_case(two_hypotheses).grade == "candidate"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def test_the_tool_result_lists_each_hypothesis(two_hypotheses):
|
|
125
|
+
record_evidence(two_hypotheses, _evidence("vmware-aria", falsifies=("H2",)))
|
|
126
|
+
record_evidence(two_hypotheses, _evidence("vmware-monitor"))
|
|
127
|
+
out = grade(two_hypotheses, at=AT)
|
|
128
|
+
assert out["hypotheses"] == [
|
|
129
|
+
{"id": "H1", "status": "open"},
|
|
130
|
+
{"id": "H2", "status": "excluded"},
|
|
131
|
+
]
|
|
@@ -572,11 +572,17 @@ def build_server() -> FastMCP:
|
|
|
572
572
|
decisive item: a direct hardware diagnostic, a version-checked
|
|
573
573
|
knowledge-base entry, or a vendor SR, and no gap left open); Excluded
|
|
574
574
|
(an observation that actually rules the hypothesis out — "we looked and
|
|
575
|
-
found nothing" is a gap, not an exclusion).
|
|
575
|
+
found nothing" is a gap, not an exclusion). Exclusion is per
|
|
576
|
+
hypothesis: ruling out H2 does not exclude the case while H1 is still
|
|
577
|
+
open. The case is Excluded only when every registered hypothesis is
|
|
578
|
+
ruled out; otherwise the grade is for what remains, and gaps that block
|
|
579
|
+
only ruled-out hypotheses no longer hold it back.
|
|
576
580
|
|
|
577
581
|
RETURNS: {grade, previous, direction, reasons, ceiling, ceiling_reasons,
|
|
578
|
-
rules_source, rules_origin}. `
|
|
579
|
-
|
|
582
|
+
rules_source, rules_origin, hypotheses}. `hypotheses` lists every
|
|
583
|
+
registered hypothesis as {id, status: open|excluded}. `direction` is
|
|
584
|
+
initial/up/down/unchanged — grades may go DOWN, and the history records
|
|
585
|
+
it when they do.
|
|
580
586
|
|
|
581
587
|
GOTCHAS: on a stock install `ceiling` is "probable", because Confirmed
|
|
582
588
|
needs a decisive source and there is neither a hardware-diagnostic
|
|
@@ -223,6 +223,7 @@ def grade(case_id: str, at: str | None = None) -> dict[str, Any]:
|
|
|
223
223
|
"ceiling_reasons": list(result.ceiling_reasons),
|
|
224
224
|
"rules_source": result.rules_source,
|
|
225
225
|
"rules_origin": result.rules_origin,
|
|
226
|
+
"hypotheses": [{"id": h, "status": status} for h, status in result.hypotheses],
|
|
226
227
|
}
|
|
227
228
|
|
|
228
229
|
|
|
@@ -24,7 +24,8 @@ from typing import Any
|
|
|
24
24
|
from vmware_policy.paths import ops_path
|
|
25
25
|
|
|
26
26
|
from vmware_debug.ops.cases.evidence import load_evidence, load_gaps
|
|
27
|
-
from vmware_debug.ops.cases.
|
|
27
|
+
from vmware_debug.ops.cases.hypotheses import load_hypotheses
|
|
28
|
+
from vmware_debug.ops.cases.store import CaseError, load_case
|
|
28
29
|
from vmware_debug.ops.skill_names import group_by_skill
|
|
29
30
|
|
|
30
31
|
PACKAGED_RULES = Path(__file__).resolve().parents[2] / "rules" / "grading_rules.yaml"
|
|
@@ -41,6 +42,24 @@ class GradeResult:
|
|
|
41
42
|
ceiling_reasons: tuple[str, ...]
|
|
42
43
|
rules_source: str
|
|
43
44
|
rules_origin: str
|
|
45
|
+
hypotheses: tuple[tuple[str, str], ...] = ()
|
|
46
|
+
"""``(hypothesis_id, "open" | "excluded")`` for every registered hypothesis,
|
|
47
|
+
in registration order. The case is Excluded only when none is left open."""
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _registered_hypotheses(case_id: str, reasons: list[str]) -> tuple[str, ...]:
|
|
51
|
+
"""Hypothesis ids from the ledger; an unreadable ledger grades as none registered.
|
|
52
|
+
|
|
53
|
+
Grading worked on a case directory without ``hypotheses.md`` before
|
|
54
|
+
exclusion became per hypothesis. Refusing to grade it now would turn a
|
|
55
|
+
missing bookkeeping file into no conclusion at all, so the gap is said
|
|
56
|
+
instead.
|
|
57
|
+
"""
|
|
58
|
+
try:
|
|
59
|
+
return tuple(h.hypothesis_id for h in load_hypotheses(case_id))
|
|
60
|
+
except CaseError as exc:
|
|
61
|
+
reasons.append(f"Hypothesis ledger unreadable, graded as if none were registered: {exc}")
|
|
62
|
+
return ()
|
|
44
63
|
|
|
45
64
|
|
|
46
65
|
def site_rules_path() -> Path:
|
|
@@ -169,7 +188,6 @@ def grade_case(case_id: str) -> GradeResult:
|
|
|
169
188
|
by_skill = group_by_skill([e.source_skill for e in evidence])
|
|
170
189
|
sources = set(by_skill)
|
|
171
190
|
blocking = tuple(g for g in gaps if g.blocks)
|
|
172
|
-
falsifiable = tuple(g for g in blocking if g.could_falsify)
|
|
173
191
|
reasons: list[str] = []
|
|
174
192
|
|
|
175
193
|
if evidence:
|
|
@@ -187,22 +205,87 @@ def grade_case(case_id: str) -> GradeResult:
|
|
|
187
205
|
)
|
|
188
206
|
else:
|
|
189
207
|
reasons.append("No evidence recorded yet.")
|
|
190
|
-
|
|
208
|
+
|
|
209
|
+
# Exclusion is per hypothesis. A falsifying observation rules out the
|
|
210
|
+
# hypotheses it names, not the case: on 2026-09-15 two sources ruled out H2
|
|
211
|
+
# while the leading H1 was still open behind three gaps, and the whole case
|
|
212
|
+
# was graded Excluded — which reads as "the answer was ruled out".
|
|
213
|
+
excluded_ids: frozenset[str] = frozenset()
|
|
214
|
+
if _check_excluded(grades.get("excluded") or {}, evidence, sources, reasons):
|
|
215
|
+
excluded_ids = frozenset(h for e in evidence for h in e.falsifies)
|
|
216
|
+
registered = _registered_hypotheses(case_id, reasons)
|
|
217
|
+
still_open = tuple(h for h in registered if h not in excluded_ids)
|
|
218
|
+
statuses = tuple((h, "excluded" if h in excluded_ids else "open") for h in registered)
|
|
219
|
+
# A gap that blocks only ruled-out hypotheses no longer holds the case back.
|
|
220
|
+
live_blocking = tuple(g for g in blocking if any(h not in excluded_ids for h in g.blocks))
|
|
221
|
+
retired = tuple(g for g in blocking if g not in live_blocking)
|
|
222
|
+
falsifiable = tuple(g for g in live_blocking if g.could_falsify)
|
|
223
|
+
if live_blocking:
|
|
191
224
|
reasons.append(
|
|
192
225
|
"Blocking gap(s): "
|
|
193
226
|
+ "; ".join(
|
|
194
227
|
f"{g.gap_id} {g.what} (blocks {', '.join(g.blocks)}"
|
|
195
228
|
+ (", could overturn it)" if g.could_falsify else ")")
|
|
196
|
-
for g in
|
|
229
|
+
for g in live_blocking
|
|
197
230
|
)
|
|
198
231
|
+ "."
|
|
199
232
|
)
|
|
233
|
+
if retired:
|
|
234
|
+
reasons.append(
|
|
235
|
+
"No longer blocking, because every hypothesis they block is ruled out: "
|
|
236
|
+
+ ", ".join(g.gap_id for g in retired)
|
|
237
|
+
+ "."
|
|
238
|
+
)
|
|
200
239
|
|
|
201
|
-
|
|
202
|
-
if excluded:
|
|
240
|
+
if excluded_ids and not still_open:
|
|
203
241
|
grade = "excluded"
|
|
204
|
-
|
|
205
|
-
|
|
242
|
+
else:
|
|
243
|
+
open_evidence = evidence
|
|
244
|
+
open_sources = sources
|
|
245
|
+
if excluded_ids:
|
|
246
|
+
# Evidence carries no link to the hypothesis it supports, only to the
|
|
247
|
+
# ones it falsifies. An item that ruled one hypothesis out says
|
|
248
|
+
# nothing for the others, so it must not buy them corroboration —
|
|
249
|
+
# otherwise the observations that excluded H2 promote H1 to
|
|
250
|
+
# Probable, and a brand-new hypothesis with no evidence at all
|
|
251
|
+
# inherits the same grade (independent review, 2026-09-15).
|
|
252
|
+
open_evidence = tuple(e for e in evidence if not e.falsifies)
|
|
253
|
+
open_sources = set(group_by_skill([e.source_skill for e in open_evidence]))
|
|
254
|
+
reasons.append(
|
|
255
|
+
f"Ruled out: {', '.join(sorted(excluded_ids))}. Still open: "
|
|
256
|
+
f"{', '.join(still_open)}. The grade is for what remains, counting "
|
|
257
|
+
f"only evidence that rules nothing out: {len(open_sources)} "
|
|
258
|
+
f"independent source(s)."
|
|
259
|
+
)
|
|
260
|
+
grade = _grade_open(
|
|
261
|
+
grades, open_evidence, open_sources, live_blocking, falsifiable, reasons, scope
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
ceiling, ceiling_reasons = _ceiling(rules)
|
|
265
|
+
return GradeResult(
|
|
266
|
+
case_id=case_id,
|
|
267
|
+
grade=grade,
|
|
268
|
+
reasons=tuple(reasons),
|
|
269
|
+
ceiling=ceiling,
|
|
270
|
+
ceiling_reasons=ceiling_reasons,
|
|
271
|
+
rules_source=source,
|
|
272
|
+
rules_origin=origin,
|
|
273
|
+
hypotheses=statuses,
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _grade_open(
|
|
278
|
+
grades: dict[str, Any],
|
|
279
|
+
evidence: tuple,
|
|
280
|
+
sources: set[str],
|
|
281
|
+
blocking: tuple,
|
|
282
|
+
falsifiable: tuple,
|
|
283
|
+
reasons: list[str],
|
|
284
|
+
scope: Any,
|
|
285
|
+
) -> str:
|
|
286
|
+
"""Candidate / Probable / Confirmed for the hypotheses that are still open."""
|
|
287
|
+
if _meets_probable(grades.get("probable") or {}, sources, falsifiable):
|
|
288
|
+
return (
|
|
206
289
|
"confirmed"
|
|
207
290
|
if _meets_confirmed(
|
|
208
291
|
grades.get("confirmed") or {},
|
|
@@ -214,7 +297,7 @@ def grade_case(case_id: str) -> GradeResult:
|
|
|
214
297
|
)
|
|
215
298
|
else "probable"
|
|
216
299
|
)
|
|
217
|
-
|
|
300
|
+
if _meets_confirmed(
|
|
218
301
|
grades.get("confirmed") or {},
|
|
219
302
|
evidence,
|
|
220
303
|
blocking,
|
|
@@ -225,21 +308,9 @@ def grade_case(case_id: str) -> GradeResult:
|
|
|
225
308
|
# Only reachable when the rules file relaxes `confirmed.requires`.
|
|
226
309
|
# Mutation-testing found that key was never read: changing it looked
|
|
227
310
|
# like it worked and did nothing, which is worse than not offering it.
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
reasons.append(_why_not_probable(grades.get("probable") or {}, sources, falsifiable))
|
|
232
|
-
|
|
233
|
-
ceiling, ceiling_reasons = _ceiling(rules)
|
|
234
|
-
return GradeResult(
|
|
235
|
-
case_id=case_id,
|
|
236
|
-
grade=grade,
|
|
237
|
-
reasons=tuple(reasons),
|
|
238
|
-
ceiling=ceiling,
|
|
239
|
-
ceiling_reasons=ceiling_reasons,
|
|
240
|
-
rules_source=source,
|
|
241
|
-
rules_origin=origin,
|
|
242
|
-
)
|
|
311
|
+
return "confirmed"
|
|
312
|
+
reasons.append(_why_not_probable(grades.get("probable") or {}, sources, falsifiable))
|
|
313
|
+
return "candidate"
|
|
243
314
|
|
|
244
315
|
|
|
245
316
|
def _meets_probable(rule: dict[str, Any], sources: set[str], falsifiable: tuple) -> bool:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/docs/plans/2026-08-28-investigation-layer-design.md
RENAMED
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/skills/vmware-debug/references/agent-guardrails.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/skills/vmware-debug/references/event-envelope.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/test_entity_reachability.py
RENAMED
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/test_error_actionability.py
RENAMED
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/test_tool_description_quality.py
RENAMED
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/capability/test_tool_manifest_budget.py
RENAMED
|
File without changes
|
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/_vsphere_event_vocabulary.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_case_plan_max_steps.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_evidence_payload_contract.py
RENAMED
|
File without changes
|
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_investigation_metrics.py
RENAMED
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_real_alert_titles_classify.py
RENAMED
|
File without changes
|
|
File without changes
|
{vmware_debug-1.12.2 → vmware_debug-1.13.0}/tests/eval/regression/test_skill_name_aliases.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|