vmware-debug 1.10.0__tar.gz → 1.11.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/.claude-plugin/plugin.json +1 -1
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/.mcp.json +1 -1
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/PKG-INFO +2 -2
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/RELEASE_NOTES.md +67 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/pyproject.toml +2 -2
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/server.json +11 -2
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/skills/vmware-debug/SKILL.md +4 -2
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/skills/vmware-debug/references/capabilities.md +8 -1
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/skills/vmware-debug/references/event-envelope.md +1 -1
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/skills/vmware-debug/references/routing.md +1 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/capability/_scores.json +35 -57
- vmware_debug-1.11.1/tests/eval/regression/test_case_id_non_ascii.py +86 -0
- vmware_debug-1.11.1/tests/eval/regression/test_every_tool_parameter_is_documented.py +132 -0
- vmware_debug-1.11.1/tests/eval/regression/test_evidence_payload_contract.py +124 -0
- vmware_debug-1.11.1/tests/eval/regression/test_host_lifecycle_classification.py +244 -0
- vmware_debug-1.11.1/tests/eval/regression/test_skill_name_aliases.py +124 -0
- vmware_debug-1.11.1/tests/eval/regression/test_spike_resolution.py +171 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_timeline.py +5 -1
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/uv.lock +5 -5
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/__init__.py +1 -1
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/mcp_server/server.py +239 -80
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/api.py +14 -1
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/grading.py +16 -2
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/ids.py +25 -7
- vmware_debug-1.11.1/vmware_debug/ops/cases/payloads.py +110 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/plan.py +48 -13
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/readiness.py +43 -15
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/store.py +8 -4
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/timeline.py +26 -29
- vmware_debug-1.11.1/vmware_debug/ops/skill_names.py +92 -0
- vmware_debug-1.11.1/vmware_debug/ops/timeline.py +568 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/rules/evidence_sources.yaml +9 -0
- vmware_debug-1.10.0/vmware_debug/ops/timeline.py +0 -326
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/.gitignore +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/README-CN.md +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/README.md +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/SECURITY.md +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/docs/plans/2026-08-28-investigation-layer-design.md +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/skills/vmware-debug/references/agent-guardrails.md +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/skills/vmware-debug/references/cli-reference.md +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/skills/vmware-debug/references/setup-guide.md +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/__init__.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/capability/__init__.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/capability/_family.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/capability/_scoring.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/capability/_skill.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/capability/conftest.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/capability/test_entity_reachability.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/capability/test_error_actionability.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/capability/test_tool_description_quality.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/capability/test_tool_manifest_budget.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/regression/__init__.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/regression/test_capability_grader.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/regression/test_debug_regressions.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/regression/test_investigation_metrics.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/regression/test_result_envelope.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/eval/regression/test_tool_annotations.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_case_conclusion.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_case_evidence.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_case_grading.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_case_hypotheses.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_case_ids.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_case_plan.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_case_readiness.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_case_store.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_case_timeline_close.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_case_tools.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_knowledge.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/tests/test_safe_error_passthrough.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/cli.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/envelope.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/mcp/__init__.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/mcp/tools.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/mcp_server/__init__.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/__init__.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/__init__.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/conclusion.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/evidence.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/hypotheses.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/knowledge.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/model.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/ops/cases/sources.py +0 -0
- {vmware_debug-1.10.0 → vmware_debug-1.11.1}/vmware_debug/rules/grading_rules.yaml +0 -0
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://anthropic.com/claude-code/plugin.schema.json",
|
|
3
3
|
"name": "vmware-debug",
|
|
4
4
|
"displayName": "VMware Debug",
|
|
5
|
-
"version": "1.
|
|
5
|
+
"version": "1.11.1",
|
|
6
6
|
"description": "Read-only VMware incident correlation: timeline, spikes, root-cause routing. 2 MCP tools.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Wei Zhou",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: vmware-debug
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.11.1
|
|
4
4
|
Summary: VMware diagnostic brain — read-only incident triage, log/event correlation, and root-cause routing across the VMware skill family
|
|
5
5
|
Project-URL: Homepage, https://github.com/vmware-skills/VMware-Debug
|
|
6
6
|
Project-URL: Repository, https://github.com/vmware-skills/VMware-Debug
|
|
@@ -18,7 +18,7 @@ Requires-Dist: mcp[cli]<2.0,>=1.10
|
|
|
18
18
|
Requires-Dist: pyyaml<7.0,>=6.0
|
|
19
19
|
Requires-Dist: rich<15.0,>=13.0
|
|
20
20
|
Requires-Dist: typer<1.0,>=0.12
|
|
21
|
-
Requires-Dist: vmware-policy<2.0,>=1.
|
|
21
|
+
Requires-Dist: vmware-policy<2.0,>=1.10.0
|
|
22
22
|
Description-Content-Type: text/markdown
|
|
23
23
|
|
|
24
24
|
<!-- mcp-name: io.github.vmware-skills/vmware-debug -->
|
|
@@ -1,3 +1,70 @@
|
|
|
1
|
+
## v1.11.1 — the schema an agent reads now carries the descriptions
|
|
2
|
+
|
|
3
|
+
Parameter descriptions reach the JSON schema for the first time. An MCP client
|
|
4
|
+
sees the schema, not the docstring, and this repo's coverage of `description`
|
|
5
|
+
and `additionalProperties` was 0% — while nearly every parameter was already
|
|
6
|
+
described in an `Args:` block no client ever receives.
|
|
7
|
+
|
|
8
|
+
Measured on a real VCF 9.1 estate, the gap produced a silent failure with no
|
|
9
|
+
error at any stage: a parameter name guessed wrong is discarded and the tool
|
|
10
|
+
returns the full unfiltered result; a value guessed wrong (`power_state=
|
|
11
|
+
"running"`) returns 0 rows where there were 11.
|
|
12
|
+
|
|
13
|
+
vmware-policy 1.10.0's `describe_tool_parameters` copies what is already
|
|
14
|
+
written, so the docstring is now load-bearing and the two cannot drift apart. It
|
|
15
|
+
removes the `Args:` block from the description once copied — both travel in
|
|
16
|
+
every `tools/list` response, so leaving it bills the same sentences twice
|
|
17
|
+
against the manifest's token budget. `additionalProperties` is closed: an open
|
|
18
|
+
schema is room for a model to invent arguments that are then silently
|
|
19
|
+
discarded, which is the other half of the same failure.
|
|
20
|
+
|
|
21
|
+
**The `vmware-policy` floor moves to >=1.10.0.** Older releases have no
|
|
22
|
+
`describe_tool_parameters`, and resolving one gives an ImportError at server
|
|
23
|
+
start rather than a missing feature.
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
## v1.11.0 — the correlation engine, on a real incident
|
|
27
|
+
|
|
28
|
+
A tester fed the investigation layer a genuine failure: on 08-03 at 05:49, four
|
|
29
|
+
ESXi hosts went `EnteringMaintenanceMode` → `EnteredMaintenanceMode` →
|
|
30
|
+
`HostShutdown` → `HostConnectionLost` → `HostSyncFailed` ×24 within 32 seconds,
|
|
31
|
+
and never came back. 3,818 real vCenter events over 40 days, pulled per this
|
|
32
|
+
skill's own envelope documentation. The case-management layer held up. The
|
|
33
|
+
correlation engine — the thing this skill is named for — did not.
|
|
34
|
+
|
|
35
|
+
**80% of events were `uncategorized`, and the headline conclusion was "cannot
|
|
36
|
+
classify, widen your search window".** The classification table's
|
|
37
|
+
`power_lifecycle` keywords were all VM-centric (`power on`/`power off`, `vmx`,
|
|
38
|
+
`ovf`, `clone`, `snapshot`); nothing covered host maintenance mode or host
|
|
39
|
+
shutdown, and `"Shut down of esxi05"` does not match `"power off"`. There is now
|
|
40
|
+
a `host_lifecycle` category — a separate category rather than more keywords,
|
|
41
|
+
because it is a separate investigation, routed to monitor + logs + harden (a
|
|
42
|
+
host that left service on cue was usually told to). `classification_coverage()`
|
|
43
|
+
reports what went unread in every result, and above ~25% it reaches
|
|
44
|
+
`next_checks`. The "widen the window" remedy is gone: the window was already 40
|
|
45
|
+
days, and widening makes the incident *less* findable.
|
|
46
|
+
|
|
47
|
+
**The only spike detected was the day the hosts came back up.**
|
|
48
|
+
`_auto_bin_seconds` was `span/30`, so a 29-day window gave 23.2-hour buckets;
|
|
49
|
+
the incident hour was the second-busiest in the dataset and absent from the
|
|
50
|
+
spike list, while `--bin-seconds 3600` found it at z=8.25. Binning now chooses
|
|
51
|
+
by event *density*, from a floor of 4 events per bin that is derived rather than
|
|
52
|
+
tuned: spikes flag at mean+2σ, and a count only exceeds twice its mean once the
|
|
53
|
+
mean reaches 4. The chosen width, how it was chosen, and the bin count are
|
|
54
|
+
reported.
|
|
55
|
+
|
|
56
|
+
**The same incident described five ways routed five different ways** — decided
|
|
57
|
+
by an incidental noun, with the most accurate description routing nowhere.
|
|
58
|
+
Categories are now ranked by matched-keyword length, so a long phrase beats a
|
|
59
|
+
stray word, and the losing categories are named.
|
|
60
|
+
|
|
61
|
+
Also fixed: `case_timeline`'s payload contract was undocumented and its MCP
|
|
62
|
+
parameter type refused the array shape the docstring implied; pure-Chinese
|
|
63
|
+
summaries collapsed to one `case_id` per second, with a collision message that
|
|
64
|
+
was false; and skill names were compared as raw strings, so `monitor` and
|
|
65
|
+
`vmware-monitor` counted as two independent evidence sources — buying *Probable*
|
|
66
|
+
out of orthography, in the layer built to prevent exactly that.
|
|
67
|
+
|
|
1
68
|
## v1.10.0 — the knowledge layer is implemented, not just described
|
|
2
69
|
|
|
3
70
|
v1.9.0 shipped the investigation layer with `applies_to` documented as the thing
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "vmware-debug"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.11.1"
|
|
8
8
|
description = "VMware diagnostic brain — read-only incident triage, log/event correlation, and root-cause routing across the VMware skill family"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -21,7 +21,7 @@ dependencies = [
|
|
|
21
21
|
"typer>=0.12,<1.0",
|
|
22
22
|
"rich>=13.0,<15.0",
|
|
23
23
|
"mcp[cli]>=1.10,<2.0",
|
|
24
|
-
"vmware-policy>=1.
|
|
24
|
+
"vmware-policy>=1.10.0,<2.0",
|
|
25
25
|
# Imported directly by ops/cases/grading.py for the rules file. It has
|
|
26
26
|
# always arrived transitively via vmware-policy; a package should
|
|
27
27
|
# declare what it imports rather than rely on a dependency's dependency.
|
|
@@ -7,12 +7,21 @@
|
|
|
7
7
|
"url": "https://github.com/vmware-skills/VMware-Debug",
|
|
8
8
|
"source": "github"
|
|
9
9
|
},
|
|
10
|
-
"version": "1.
|
|
10
|
+
"version": "1.11.1",
|
|
11
11
|
"packages": [
|
|
12
12
|
{
|
|
13
13
|
"registryType": "pypi",
|
|
14
14
|
"identifier": "vmware-debug",
|
|
15
|
-
"version": "1.
|
|
15
|
+
"version": "1.11.1",
|
|
16
|
+
"runtimeHint": "uvx",
|
|
17
|
+
"packageArguments": [
|
|
18
|
+
{
|
|
19
|
+
"type": "positional",
|
|
20
|
+
"valueHint": "subcommand",
|
|
21
|
+
"value": "mcp",
|
|
22
|
+
"description": "Starts the stdio MCP server. Without it the console script runs the human CLI and exits."
|
|
23
|
+
}
|
|
24
|
+
],
|
|
16
25
|
"transport": {
|
|
17
26
|
"type": "stdio"
|
|
18
27
|
}
|
|
@@ -256,8 +256,10 @@ vmware-debug mcp # start stdio MCP server (proxy-
|
|
|
256
256
|
## Troubleshooting
|
|
257
257
|
|
|
258
258
|
- **`incident_timeline` raises "event[N] could not be normalised"** — event N is missing a timestamp or has an unparseable one. Every event needs `ts` (ISO-8601, epoch seconds, or millis).
|
|
259
|
-
- **
|
|
260
|
-
- **No spikes detected on an obvious burst** — you
|
|
259
|
+
- **Most events come back "uncategorized"** — read `classification` in the result: it says how many, what share, and quotes the texts that matched nothing. Do **not** widen the window; a wider window adds baseline, not signal. The spikes still tell you *when*, which is answerable without a category. If the samples name a subsystem the taxonomy does not know, add a signature (see `references/routing.md`).
|
|
260
|
+
- **No spikes detected on an obvious burst** — check `binning` for the resolution you were given. The width is chosen from event density so that bins average ≥4 events; a burst shorter than one bin can still be flattened by it. Pass `bin_seconds` to narrow. Below three bins there is no baseline at all and nothing is reported.
|
|
261
|
+
- **`case_timeline` says zero events after you submitted results** — check `payload_events` in each `case_submit_evidence` reply. Events are read from a bare list, or from `items`/`events`/`rows`; a *summary* of a tool's result carries none. `note` names which items carried nothing and what keys they held instead.
|
|
262
|
+
- **`case_readiness` says a skill you have is not installed** — both spellings are accepted (`monitor` and `vmware-monitor`). A name it did not recognise comes back in `unrecognised_skills` rather than being read as missing.
|
|
261
263
|
- **It won't execute the fix** — by design. Route to vmware-aiops or vmware-pilot.
|
|
262
264
|
|
|
263
265
|
## Audit & Safety
|
|
@@ -27,9 +27,16 @@ fixed in-process constant, so `total` is a real count and `limit` is `null`.
|
|
|
27
27
|
|
|
28
28
|
## Symptom categories
|
|
29
29
|
|
|
30
|
-
`storage`, `network`, `compute`, `ha_drs`, `
|
|
30
|
+
`storage`, `network`, `compute`, `ha_drs`, `host_lifecycle`, `power_lifecycle`,
|
|
31
|
+
`auth`, `platform`.
|
|
31
32
|
See `references/routing.md` for keyword signatures and the skill each routes to.
|
|
32
33
|
|
|
34
|
+
`host_lifecycle` is a host changing its own availability state — maintenance
|
|
35
|
+
mode, shutdown, reboot, standby, connection loss, sync failure.
|
|
36
|
+
`power_lifecycle` is the VM-level equivalent. They are separate because they are
|
|
37
|
+
separate investigations: the second is a task question for vmware-aiops, the
|
|
38
|
+
first is a cluster, DPM, vLCM or drift question.
|
|
39
|
+
|
|
33
40
|
## Design properties
|
|
34
41
|
|
|
35
42
|
- **Zero cross-skill runtime deps** — correlation is pure functions over plain
|
{vmware_debug-1.10.0 → vmware_debug-1.11.1}/skills/vmware-debug/references/event-envelope.md
RENAMED
|
@@ -21,7 +21,7 @@ runtime dependency** on the other packages (no version lockstep, no heavy instal
|
|
|
21
21
|
| Field | Type | Notes |
|
|
22
22
|
|---|---|---|
|
|
23
23
|
| `ts` | string \| number | ISO-8601, epoch **seconds**, or epoch **millis** (auto-detected). Required. |
|
|
24
|
-
| `source` | string | `monitor` \| `aria` \| `loginsight` \| `nsx` \| `nsx-security` \| `storage` \| ... |
|
|
24
|
+
| `source` | string | `monitor` \| `aria` \| `loginsight` \| `nsx` \| `nsx-security` \| `storage` \| ... The catalogue in `rules/evidence_sources.yaml` spells the same skills `vmware-monitor`, `vmware-aria`, `vmware-log-insight`. Both spellings are understood wherever a skill is named — `case_readiness(available_skills=...)`, and the grader's count of independent sources, which treats two spellings of one skill as one source. |
|
|
25
25
|
| `severity` | string | Free text; normalised to `critical`/`error`/`warning`/`info`/`unknown`. |
|
|
26
26
|
| `entity` | string | The object the event is about (VM/host/datastore). May be empty. |
|
|
27
27
|
| `text` | string | Human-readable message — this is what the symptom classifier matches on. |
|
|
@@ -10,6 +10,7 @@ Keep this in sync with `_CATEGORY_SIGNATURES` in `vmware_debug/ops/timeline.py`
|
|
|
10
10
|
| **network** | vmotion, uplink, link down, mtu, firewall, dfw, segment, tier-0, bgp | vmware-nsx, vmware-nsx-security (traceflow, DFW hits) | pilot |
|
|
11
11
|
| **compute** | cpu ready, memory, balloon, swap, contention, numa | vmware-aria (metrics + anomalies) | pilot (rightsizing) |
|
|
12
12
|
| **ha_drs** | ha, high availability, drs, failover, admission control, isolation | vmware-monitor, vmware-aiops (cluster) | pilot |
|
|
13
|
+
| **host_lifecycle** | maintenance mode, shut down of, host reboot, standby mode, lost connection to, cannot synchronize, not responding | vmware-monitor (host/cluster state), vmware-harden (drift — a host that left service on cue was told to), vmware-log-insight (vpxd/hostd) | pilot |
|
|
13
14
|
| **power_lifecycle** | power on/off, failed to start, boot, vmx, ovf, clone, snapshot | vmware-aiops (task status, snapshot tree), vmware-monitor | aiops |
|
|
14
15
|
| **auth** | login, authentication, denied, 401, 403, token, certificate, tls | config/.env, target cert + time sync | config fix |
|
|
15
16
|
| **platform** | vpxd, hostd, service restart, crash, 503, not responding, disconnected | vmware-monitor (connection/service), vmware-log-insight (vpxd/hostd) | pilot |
|
|
@@ -94,7 +94,7 @@
|
|
|
94
94
|
"names_artifact",
|
|
95
95
|
"states_remedy"
|
|
96
96
|
],
|
|
97
|
-
"where": "grading.py:
|
|
97
|
+
"where": "grading.py:57"
|
|
98
98
|
},
|
|
99
99
|
{
|
|
100
100
|
"message": "grading_rules.yaml at {} is not valid YAML: {}. Fix it rather than deleting it \u2014 falling back to the",
|
|
@@ -102,7 +102,7 @@
|
|
|
102
102
|
"names_artifact",
|
|
103
103
|
"states_remedy"
|
|
104
104
|
],
|
|
105
|
-
"where": "grading.py:
|
|
105
|
+
"where": "grading.py:59"
|
|
106
106
|
},
|
|
107
107
|
{
|
|
108
108
|
"message": "A hypothesis needs a statement \u2014 what you think happened, in one line. An unnamed hypothesis cannot ",
|
|
@@ -135,7 +135,7 @@
|
|
|
135
135
|
"names_artifact",
|
|
136
136
|
"states_remedy"
|
|
137
137
|
],
|
|
138
|
-
"where": "plan.py:
|
|
138
|
+
"where": "plan.py:84"
|
|
139
139
|
},
|
|
140
140
|
{
|
|
141
141
|
"message": "evidence_sources.yaml at {} is missing 'classes' or 'routing'. An empty catalogue would make every r",
|
|
@@ -167,7 +167,7 @@
|
|
|
167
167
|
"names_artifact",
|
|
168
168
|
"states_remedy"
|
|
169
169
|
],
|
|
170
|
-
"where": "timeline.py:
|
|
170
|
+
"where": "timeline.py:137"
|
|
171
171
|
},
|
|
172
172
|
{
|
|
173
173
|
"message": "Cannot read case.json for {}: {}",
|
|
@@ -175,7 +175,7 @@
|
|
|
175
175
|
"names_artifact",
|
|
176
176
|
"states_remedy"
|
|
177
177
|
],
|
|
178
|
-
"where": "timeline.py:
|
|
178
|
+
"where": "timeline.py:151"
|
|
179
179
|
}
|
|
180
180
|
],
|
|
181
181
|
"per_dimension_pct": {
|
|
@@ -193,58 +193,47 @@
|
|
|
193
193
|
"manifest_context_headroom": {
|
|
194
194
|
"detail": {
|
|
195
195
|
"heaviest_tools": {
|
|
196
|
-
"case_open":
|
|
197
|
-
"case_plan":
|
|
198
|
-
"case_record_gap":
|
|
199
|
-
"case_submit_evidence":
|
|
200
|
-
"incident_timeline":
|
|
196
|
+
"case_open": 478,
|
|
197
|
+
"case_plan": 444,
|
|
198
|
+
"case_record_gap": 380,
|
|
199
|
+
"case_submit_evidence": 902,
|
|
200
|
+
"incident_timeline": 454
|
|
201
201
|
},
|
|
202
|
-
"manifest_tokens":
|
|
203
|
-
"mean_tokens_per_tool":
|
|
202
|
+
"manifest_tokens": 4812,
|
|
203
|
+
"mean_tokens_per_tool": 343.7,
|
|
204
204
|
"reference_context": 16384,
|
|
205
205
|
"tool_count": 14
|
|
206
206
|
},
|
|
207
207
|
"maximum": 16384,
|
|
208
|
-
"pct":
|
|
208
|
+
"pct": 70.6,
|
|
209
209
|
"unit": "tokens",
|
|
210
|
-
"value":
|
|
210
|
+
"value": 11572
|
|
211
211
|
},
|
|
212
212
|
"parameter_documentation_coverage": {
|
|
213
213
|
"detail": {
|
|
214
|
-
"undocumented_by_tool": {
|
|
215
|
-
"case_grade": [
|
|
216
|
-
"case_id"
|
|
217
|
-
],
|
|
218
|
-
"case_submit_evidence": [
|
|
219
|
-
"fetched_at",
|
|
220
|
-
"summary"
|
|
221
|
-
],
|
|
222
|
-
"case_timeline": [
|
|
223
|
-
"bin_seconds",
|
|
224
|
-
"case_id",
|
|
225
|
-
"top_n",
|
|
226
|
-
"z_threshold"
|
|
227
|
-
]
|
|
228
|
-
}
|
|
214
|
+
"undocumented_by_tool": {}
|
|
229
215
|
},
|
|
230
216
|
"maximum": 45,
|
|
231
|
-
"pct":
|
|
217
|
+
"pct": 100.0,
|
|
232
218
|
"unit": "parameters",
|
|
233
|
-
"value":
|
|
219
|
+
"value": 45
|
|
234
220
|
},
|
|
235
221
|
"per_tool_token_discipline": {
|
|
236
222
|
"detail": {
|
|
237
223
|
"ceiling": 350,
|
|
238
224
|
"over_ceiling": {
|
|
239
|
-
"case_open":
|
|
240
|
-
"case_plan":
|
|
241
|
-
"
|
|
225
|
+
"case_open": 478,
|
|
226
|
+
"case_plan": 444,
|
|
227
|
+
"case_record_gap": 380,
|
|
228
|
+
"case_submit_evidence": 902,
|
|
229
|
+
"case_timeline": 374,
|
|
230
|
+
"incident_timeline": 454
|
|
242
231
|
}
|
|
243
232
|
},
|
|
244
233
|
"maximum": 14,
|
|
245
|
-
"pct":
|
|
234
|
+
"pct": 57.1,
|
|
246
235
|
"unit": "tools",
|
|
247
|
-
"value":
|
|
236
|
+
"value": 8
|
|
248
237
|
},
|
|
249
238
|
"read_only_manifest_saving": {
|
|
250
239
|
"detail": {
|
|
@@ -281,7 +270,7 @@
|
|
|
281
270
|
"evidence.py:179",
|
|
282
271
|
"evidence.py:211",
|
|
283
272
|
"evidence.py:248",
|
|
284
|
-
"grading.py:
|
|
273
|
+
"grading.py:67",
|
|
285
274
|
"hypotheses.py:47",
|
|
286
275
|
"hypotheses.py:70"
|
|
287
276
|
]
|
|
@@ -301,48 +290,37 @@
|
|
|
301
290
|
"tool_description_quality": {
|
|
302
291
|
"detail": {
|
|
303
292
|
"per_dimension_pct": {
|
|
304
|
-
"args":
|
|
305
|
-
"gotcha":
|
|
293
|
+
"args": 100.0,
|
|
294
|
+
"gotcha": 100.0,
|
|
306
295
|
"marker": 100.0,
|
|
307
|
-
"next_hop":
|
|
296
|
+
"next_hop": 78.6,
|
|
308
297
|
"what": 100.0,
|
|
309
298
|
"when": 92.9
|
|
310
299
|
},
|
|
311
300
|
"tools_graded": 14,
|
|
312
301
|
"weakest_tools": {
|
|
313
|
-
"case_close": [
|
|
314
|
-
|
|
315
|
-
],
|
|
316
|
-
"
|
|
317
|
-
"gotcha"
|
|
318
|
-
],
|
|
319
|
-
"case_grade": [
|
|
320
|
-
"args",
|
|
321
|
-
"next_hop"
|
|
322
|
-
],
|
|
302
|
+
"case_close": [],
|
|
303
|
+
"case_get": [],
|
|
304
|
+
"case_grade": [],
|
|
305
|
+
"case_hypotheses": [],
|
|
323
306
|
"case_knowledge": [
|
|
324
307
|
"next_hop"
|
|
325
308
|
],
|
|
326
309
|
"case_list": [
|
|
327
|
-
"gotcha",
|
|
328
310
|
"next_hop"
|
|
329
311
|
],
|
|
330
312
|
"case_readiness": [
|
|
331
313
|
"next_hop"
|
|
332
314
|
],
|
|
333
315
|
"case_record_gap": [
|
|
334
|
-
"next_hop",
|
|
335
316
|
"when"
|
|
336
|
-
],
|
|
337
|
-
"case_submit_evidence": [
|
|
338
|
-
"args"
|
|
339
317
|
]
|
|
340
318
|
}
|
|
341
319
|
},
|
|
342
320
|
"maximum": 84,
|
|
343
|
-
"pct":
|
|
321
|
+
"pct": 95.2,
|
|
344
322
|
"unit": "points",
|
|
345
|
-
"value":
|
|
323
|
+
"value": 80
|
|
346
324
|
},
|
|
347
325
|
"tool_failure_payload_quality": {
|
|
348
326
|
"detail": {
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""A case opened in Chinese got no identity of its own.
|
|
2
|
+
|
|
3
|
+
``slugify`` reduces on ``[^a-z0-9]+``, so a summary written entirely in Chinese
|
|
4
|
+
leaves nothing behind and the id falls back to the literal word ``case``:
|
|
5
|
+
``20260830-061014-case``. Two completely different investigations opened in the
|
|
6
|
+
same second then collide — and the collision message said they were "the same
|
|
7
|
+
summary", which was false, and sent the reporter looking for a duplicate that
|
|
8
|
+
did not exist.
|
|
9
|
+
|
|
10
|
+
The id has to stay inside its character set: it is a directory name, and
|
|
11
|
+
``validate_case_id`` is the only thing between a model-chosen string and
|
|
12
|
+
``open(root / case_id)``. So the fix is not to widen the alphabet. It is that an
|
|
13
|
+
id must distinguish summaries that differ, and that the collision message must
|
|
14
|
+
describe what actually collided.
|
|
15
|
+
|
|
16
|
+
The control is readability: an ASCII summary must still produce the readable
|
|
17
|
+
slug the listing was designed around, not a digest for everybody.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import pytest
|
|
23
|
+
|
|
24
|
+
from vmware_debug.ops.cases.ids import new_case_id, validate_case_id
|
|
25
|
+
from vmware_debug.ops.cases.model import Scope
|
|
26
|
+
from vmware_debug.ops.cases.store import CaseExists, create_case
|
|
27
|
+
|
|
28
|
+
AT = "2026-08-30T06:10:14Z"
|
|
29
|
+
|
|
30
|
+
#: Both must be free of Latin characters. An earlier draft wrote the first one
|
|
31
|
+
#: as "四台 ESXi 主机…", and "esxi" survived slugify — so the two ids differed
|
|
32
|
+
#: whatever the fallback did, and the test passed against the defect it was
|
|
33
|
+
#: written to catch.
|
|
34
|
+
ZH_A = "四台主机同时进入维护模式后关机"
|
|
35
|
+
ZH_B = "存储集群延迟升高导致虚拟机无响应"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@pytest.fixture(autouse=True)
|
|
39
|
+
def isolated_home(tmp_path, monkeypatch):
|
|
40
|
+
monkeypatch.setenv("OPS_HOME", str(tmp_path / "vmware"))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class TestANonAsciiSummaryStillGetsAnIdentity:
|
|
44
|
+
def test_two_different_chinese_summaries_do_not_collide(self):
|
|
45
|
+
assert new_case_id(ZH_A, AT) != new_case_id(ZH_B, AT)
|
|
46
|
+
|
|
47
|
+
def test_the_id_is_still_a_safe_directory_name(self):
|
|
48
|
+
validate_case_id(new_case_id(ZH_A, AT))
|
|
49
|
+
|
|
50
|
+
def test_the_same_summary_still_yields_the_same_id(self):
|
|
51
|
+
"""Ids are replayable: a case folder re-opened from its scope has to
|
|
52
|
+
reconstruct the id it already has."""
|
|
53
|
+
assert new_case_id(ZH_A, AT) == new_case_id(ZH_A, AT)
|
|
54
|
+
|
|
55
|
+
def test_both_cases_can_actually_be_opened_in_the_same_second(self):
|
|
56
|
+
create_case(Scope(summary=ZH_A, determined_by="user report"), at=AT)
|
|
57
|
+
create_case(Scope(summary=ZH_B, determined_by="user report"), at=AT)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class TestTheCollisionMessageIsTrue:
|
|
61
|
+
def test_it_does_not_claim_the_summaries_were_identical(self):
|
|
62
|
+
"""Two summaries that differ only outside the id character set still
|
|
63
|
+
collide, and telling the reporter they submitted the same summary sends
|
|
64
|
+
them looking for something that is not there."""
|
|
65
|
+
create_case(Scope(summary="vsan latency!", determined_by="a"), at=AT)
|
|
66
|
+
with pytest.raises(CaseExists) as exc:
|
|
67
|
+
create_case(Scope(summary="vsan latency?", determined_by="a"), at=AT)
|
|
68
|
+
assert "same summary" not in str(exc.value)
|
|
69
|
+
|
|
70
|
+
def test_it_says_what_to_do_about_it(self):
|
|
71
|
+
create_case(Scope(summary="vsan latency", determined_by="a"), at=AT)
|
|
72
|
+
with pytest.raises(CaseExists) as exc:
|
|
73
|
+
create_case(Scope(summary="vsan latency", determined_by="a"), at=AT)
|
|
74
|
+
assert "case_get" in str(exc.value)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class TestControlReadabilityIsNotSacrificed:
|
|
78
|
+
def test_an_ascii_summary_still_gets_its_readable_slug(self):
|
|
79
|
+
"""A digest for every case would pass every test above and destroy the
|
|
80
|
+
one property the slug exists for — a directory listing you can read."""
|
|
81
|
+
assert new_case_id("vSAN datastore latency on cluster-01", AT).endswith(
|
|
82
|
+
"-vsan-datastore-latency-on-cluster-01"
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
def test_a_mixed_summary_keeps_the_part_that_survives(self):
|
|
86
|
+
assert "esxi05" in new_case_id("主机 esxi05 关机", AT)
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Regression: every MCP tool parameter carries a Google-style ``Args:`` entry.
|
|
2
|
+
|
|
3
|
+
Why this exists
|
|
4
|
+
---------------
|
|
5
|
+
The JSON schema is the only part of a tool an agent can actually read. Prose in
|
|
6
|
+
the docstring body explains the tool; it says nothing about a *parameter*. A
|
|
7
|
+
measurement across the family's 327 tools found ``description`` coverage on
|
|
8
|
+
parameter schemas at 0% — every word of guidance was in prose the schema never
|
|
9
|
+
carried — and every one of this skill's 45 parameters was undescribed — including
|
|
10
|
+
``case_plan.category`` and ``case_submit_evidence.source_skill``, both of
|
|
11
|
+
which have a closed set of legal values that appeared nowhere in the schema.
|
|
12
|
+
|
|
13
|
+
The failure that produces is silent twice over. Guess a parameter name wrong
|
|
14
|
+
and the extra key is discarded, so a filtered query quietly returns the
|
|
15
|
+
unfiltered set. Guess a *value* wrong — ``aggregation="count()"`` where the
|
|
16
|
+
appliance wants ``COUNT`` — and you get an error or an empty series where there
|
|
17
|
+
was data. Neither looks like a mistake from the outside.
|
|
18
|
+
|
|
19
|
+
The family is landing a shared helper that copies each parameter's ``Args:``
|
|
20
|
+
entry into the schema at registration time. That helper can only copy text that
|
|
21
|
+
exists, so this test guards the input side: it fails the moment a tool grows a
|
|
22
|
+
parameter nobody described.
|
|
23
|
+
|
|
24
|
+
Not a style check
|
|
25
|
+
-----------------
|
|
26
|
+
It also refuses a description that only restates the name (``target: The
|
|
27
|
+
target.``). That is worse than an absent one: it fills the slot, so the gap
|
|
28
|
+
stops being visible while the agent still learns nothing.
|
|
29
|
+
|
|
30
|
+
形态 #1 — the check must not pass vacuously. If the registry hands back no
|
|
31
|
+
tools, or no tool has parameters, that is this test being broken rather than
|
|
32
|
+
this skill being clean, and it fails saying so.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import inspect
|
|
38
|
+
import re
|
|
39
|
+
|
|
40
|
+
from vmware_debug.mcp_server.server import build_server
|
|
41
|
+
|
|
42
|
+
#: Words that carry no information about a parameter beyond its own name.
|
|
43
|
+
#: A description built only from these has filled the slot without saying
|
|
44
|
+
#: anything — the shape this test exists to reject.
|
|
45
|
+
_FILLER = {"a", "an", "the", "of", "for", "to", "this", "s", "id", "name", "optional"}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def parse_args_block(doc: str | None) -> dict[str, str]:
|
|
49
|
+
"""Return {parameter: description} from a Google-style ``Args:`` block.
|
|
50
|
+
|
|
51
|
+
Deliberately the same parsing the family's schema-injection helper does:
|
|
52
|
+
a check that reads docstrings differently from the code that consumes them
|
|
53
|
+
would go green on text the helper cannot use.
|
|
54
|
+
"""
|
|
55
|
+
if not doc:
|
|
56
|
+
return {}
|
|
57
|
+
block = re.search(
|
|
58
|
+
r"\n\s*Args:\s*\n(.*?)(?:\n\s*(?:Returns|Raises|Yields|Examples?|Notes?):|\Z)",
|
|
59
|
+
doc,
|
|
60
|
+
re.DOTALL,
|
|
61
|
+
)
|
|
62
|
+
if not block:
|
|
63
|
+
return {}
|
|
64
|
+
out: dict[str, str] = {}
|
|
65
|
+
current: str | None = None
|
|
66
|
+
for line in block.group(1).splitlines():
|
|
67
|
+
if not line.strip():
|
|
68
|
+
continue
|
|
69
|
+
entry = re.match(r"\s{4,}(\*{0,2}\w+)\s*(?:\([^)]*\))?\s*:\s*(.*)", line)
|
|
70
|
+
if entry:
|
|
71
|
+
current = entry.group(1).lstrip("*")
|
|
72
|
+
out[current] = entry.group(2).strip()
|
|
73
|
+
elif current:
|
|
74
|
+
out[current] += " " + line.strip()
|
|
75
|
+
return out
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _says_something(param: str, text: str) -> bool:
|
|
79
|
+
"""True when the description adds anything beyond the parameter's own name.
|
|
80
|
+
|
|
81
|
+
Deliberately the weakest rule that still catches ``target: The target.`` —
|
|
82
|
+
one word that is neither part of the name nor a connective. A stricter
|
|
83
|
+
length or word-count rule would start failing short descriptions that are
|
|
84
|
+
complete ("Server IP address."), and a check that fires on correct text is
|
|
85
|
+
a check people learn to route around.
|
|
86
|
+
"""
|
|
87
|
+
words = {w for w in re.findall(r"[a-z0-9]+", text.lower()) if w}
|
|
88
|
+
words -= _FILLER
|
|
89
|
+
words -= set(re.findall(r"[a-z0-9]+", param.lower()))
|
|
90
|
+
return bool(words)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_every_tool_parameter_has_an_args_entry():
|
|
94
|
+
"""No registered tool parameter may be missing from its ``Args:`` block."""
|
|
95
|
+
tools = build_server()._tool_manager._tools
|
|
96
|
+
assert tools, "no MCP tools registered — this check would pass vacuously"
|
|
97
|
+
|
|
98
|
+
checked = 0
|
|
99
|
+
missing: list[str] = []
|
|
100
|
+
for name, tool in tools.items():
|
|
101
|
+
documented = parse_args_block(inspect.getdoc(tool.fn))
|
|
102
|
+
for param in (tool.parameters or {}).get("properties", {}):
|
|
103
|
+
checked += 1
|
|
104
|
+
if param not in documented:
|
|
105
|
+
missing.append(f"{name}.{param}")
|
|
106
|
+
|
|
107
|
+
assert checked, "no tool parameters found — this check would pass vacuously"
|
|
108
|
+
assert not missing, (
|
|
109
|
+
f"{len(missing)} of {checked} tool parameters have no Args: entry, so the "
|
|
110
|
+
f"schema an agent reads describes them not at all: {sorted(missing)}"
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_no_parameter_description_merely_restates_its_name():
|
|
115
|
+
"""A description that only echoes the name hides the gap instead of closing it."""
|
|
116
|
+
tools = build_server()._tool_manager._tools
|
|
117
|
+
assert tools, "no MCP tools registered — this check would pass vacuously"
|
|
118
|
+
|
|
119
|
+
checked = 0
|
|
120
|
+
empty: list[str] = []
|
|
121
|
+
for name, tool in tools.items():
|
|
122
|
+
documented = parse_args_block(inspect.getdoc(tool.fn))
|
|
123
|
+
for param in (tool.parameters or {}).get("properties", {}):
|
|
124
|
+
text = documented.get(param)
|
|
125
|
+
if text is None:
|
|
126
|
+
continue # the test above owns this failure
|
|
127
|
+
checked += 1
|
|
128
|
+
if not _says_something(param, text):
|
|
129
|
+
empty.append(f"{name}.{param}: {text!r}")
|
|
130
|
+
|
|
131
|
+
assert checked, "no documented parameters found — this check would pass vacuously"
|
|
132
|
+
assert not empty, "parameter descriptions that say nothing beyond the name: " + str(empty)
|