vmware-debug 1.8.2__tar.gz → 1.8.4__tar.gz

{vmware_debug-1.8.2 → vmware_debug-1.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vmware-debug
-Version: 1.8.2
+Version: 1.8.4
 Summary: VMware diagnostic brain — read-only incident triage, log/event correlation, and root-cause routing across the VMware skill family
 Author-email: Wei Zhou <wei-wz.zhou@broadcom.com>
 License-Expression: MIT
@@ -13,7 +13,7 @@ Requires-Python: >=3.10
 Requires-Dist: mcp[cli]<2.0,>=1.10
 Requires-Dist: rich<15.0,>=13.0
 Requires-Dist: typer<1.0,>=0.12
-Requires-Dist: vmware-policy<2.0,>=1.8.0
+Requires-Dist: vmware-policy<2.0,>=1.8.4
 Description-Content-Type: text/markdown
 
 <!-- mcp-name: io.github.zw008/vmware-debug -->

{vmware_debug-1.8.2 → vmware_debug-1.8.4}/RELEASE_NOTES.md

@@ -1,3 +1,78 @@
+## v1.8.4 (2026-07-20) — errors that teach, and tool descriptions a small model can route from
+
+A capability eval was rolled out across the family and asked two open questions:
+when a call fails, is the model told enough to fix it, and can it pick the right
+tool from the description alone? Both answers were worse than anyone thought, and
+in several places the reason was that the measurement was looking somewhere other
+than where the model reads.
+
+### Fixed — teaching messages were being discarded on the way to the agent
+
+`_safe_error` reduces unrecognised exceptions to `"<Class>: operation failed."`
+so raw API text, credentials in URLs and internal paths cannot reach an agent.
+Its allowlist held only the builtin validation errors — so this skill's **own**
+domain exceptions, the ones that exist precisely to carry a corrected next step,
+had their messages replaced by their class names.
+
+The effect was invisible from the CLI, which prints those messages in full.
+
+The worst case was shared by nine skills: `config.py` raises exactly one
+`OSError`, the missing-password error, whose entire remedy is the environment
+variable name it names. An agent hitting an unconfigured target received
+`OSError: operation failed.` and had nothing to act on. That is the family's most
+common first-run failure, and it landed one release after the documented variable
+names were corrected — so the message that would have unstuck the operator was
+the one being thrown away.
+
+The rule is now the property it always meant: **every exception this skill raises
+on purpose passes through**, and only genuinely unplanned ones are reduced.
+`RuntimeError` stays reduced — it is the generic catch-all and in several skills
+carries raw upstream text.
+
+### Fixed — error messages now carry the correction
+
+Every message that reported a failure without saying how to recover was
+rewritten: it names the offending value, gives an imperative remedy, and names
+something concrete to act on — a tool that exists, a real CLI command, a config
+file, an environment variable. Recovery becomes an instruction-following problem
+rather than an inference one, which is what a weak model can still do.
+
+Three classes of defect surfaced while doing it:
+
+- **Remedies that were never delivered.** `_safe_error` truncates with no
+  ellipsis, so a message longer than the cap loses its closing sentence
+  silently. One message had been shipping at 396 characters against a 300-char
+  cap — its remedy had never once reached an agent. Messages now lead with the
+  remedy so a long interpolated value truncates the expendable detail instead.
+- **Commands that do not exist.** One skill's error hints named a `doctor`
+  subcommand it does not have.
+- **Tools that do not exist.** A tool description pointed at two sibling-skill
+  tools that had been renamed, and another named a tool that had moved to a
+  different skill entirely.
+
+### Improved — tool descriptions state when to use them and what to call next
+
+The description is the API for a small model: an unstated routing rule is a
+routing rule that does not exist, and a tool with no stated next hop is one the
+model stops at. Descriptions now say when to prefer this tool over a sibling,
+what shape comes back, the caveat that bites, and which tool to call after.
+
+**Manifest size did not grow.** Descriptions load into every session, so the
+routing clauses were paid for by cutting duplicated reference material —
+repeated boilerplate, examples that restated the parameter list, and prose
+copies of the pagination contract.
+
+### Note
+
+Every tool and CLI command named anywhere in this release was verified against
+the live MCP registry and the live command tree, not against documentation.
+
+## v1.8.3 (2026-07-20) — credentials resolve as a pair; documented env vars now exist
+
+### Changed — version alignment
+
+No functional change in this skill. The family release adds an env-var override for the per-target username in the credential-bearing skills; this package has no per-target credentials.
+
 ## v1.8.2 (2026-07-20) — the MCP server moves into the package namespace
 
 ### Fixed — co-installing two skills broke all but the last one

{vmware_debug-1.8.2 → vmware_debug-1.8.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vmware-debug"
-version = "1.8.2"
+version = "1.8.4"
 description = "VMware diagnostic brain — read-only incident triage, log/event correlation, and root-cause routing across the VMware skill family"
 readme = "README.md"
 license = "MIT"
@@ -21,7 +21,7 @@ dependencies = [
     "typer>=0.12,<1.0",
     "rich>=13.0,<15.0",
     "mcp[cli]>=1.10,<2.0",
-    "vmware-policy>=1.8.0,<2.0",
+    "vmware-policy>=1.8.4,<2.0",
 ]
 
 [project.scripts]
@@ -31,6 +31,12 @@ vmware-debug-mcp = "vmware_debug.mcp_server.server:main"
 [tool.hatch.build.targets.wheel]
 packages = ["vmware_debug"]
 
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "capability: Capability evals — scored trends, not pass/fail gates. Excluded from the default run (they measure quality, so <100% is expected and must not read as a broken build). Run them with: pytest -m capability",
+]
+addopts = "-m 'not capability'"
 [dependency-groups]
 dev = [
     "pytest>=8.0,<10.0",

{vmware_debug-1.8.2 → vmware_debug-1.8.4}/server.json

@@ -7,12 +7,12 @@
     "url": "https://github.com/zw008/VMware-Debug",
     "source": "github"
   },
-  "version": "1.8.2",
+  "version": "1.8.4",
   "packages": [
     {
       "registryType": "pypi",
       "identifier": "vmware-debug",
-      "version": "1.8.2",
+      "version": "1.8.4",
       "transport": {
         "type": "stdio"
       }

vmware_debug-1.8.4/tests/eval/capability/_scores.json

@@ -0,0 +1,184 @@
+{
+  "_comment": "Capability eval scores. Regenerate with: pytest -m capability. These are tracked trends, not pass/fail gates \u2014 see _scoring.py.",
+  "scores": {
+    "entity_reachability_full": {
+      "detail": {
+        "broken_chains": [],
+        "entry_points": [
+          "incident_timeline",
+          "list_symptom_categories"
+        ],
+        "reachable_on_surface": 0,
+        "reachable_only_via_description": 0,
+        "required_entity_params": 0,
+        "required_params_classified": "0/1",
+        "tool_count": 2
+      },
+      "maximum": 1,
+      "pct": 100.0,
+      "unit": "required entity params",
+      "value": 1
+    },
+    "entity_reachability_read_only": {
+      "detail": {
+        "broken_chains": [],
+        "entry_points": [
+          "incident_timeline",
+          "list_symptom_categories"
+        ],
+        "reachable_on_surface": 0,
+        "reachable_only_via_description": 0,
+        "required_entity_params": 0,
+        "required_params_classified": "0/1",
+        "tool_count": 2
+      },
+      "maximum": 1,
+      "pct": 100.0,
+      "unit": "required entity params",
+      "value": 1
+    },
+    "entry_point_availability": {
+      "detail": {
+        "full_entry_points": 2,
+        "read_only_entry_points": 2
+      },
+      "maximum": 2,
+      "pct": 100.0,
+      "unit": "entry points",
+      "value": 2
+    },
+    "error_actionability": {
+      "detail": {
+        "composed_hint_sites": [],
+        "dead_end_count": 0,
+        "dead_end_errors": [],
+        "per_dimension_pct": {
+          "names_artifact": 100.0,
+          "names_input": 100.0,
+          "states_remedy": 100.0
+        },
+        "raise_sites": 7
+      },
+      "maximum": 21,
+      "pct": 100.0,
+      "unit": "points",
+      "value": 21
+    },
+    "manifest_context_headroom": {
+      "detail": {
+        "heaviest_tools": {
+          "incident_timeline": 329,
+          "list_symptom_categories": 121
+        },
+        "manifest_tokens": 450,
+        "mean_tokens_per_tool": 225.0,
+        "reference_context": 16384,
+        "tool_count": 2
+      },
+      "maximum": 16384,
+      "pct": 97.3,
+      "unit": "tokens",
+      "value": 15934
+    },
+    "parameter_documentation_coverage": {
+      "detail": {
+        "undocumented_by_tool": {}
+      },
+      "maximum": 4,
+      "pct": 100.0,
+      "unit": "parameters",
+      "value": 4
+    },
+    "per_tool_token_discipline": {
+      "detail": {
+        "ceiling": 350,
+        "over_ceiling": {}
+      },
+      "maximum": 2,
+      "pct": 100.0,
+      "unit": "tools",
+      "value": 2
+    },
+    "read_only_manifest_saving": {
+      "detail": {
+        "full_manifest_tokens": 450,
+        "full_tool_count": 2,
+        "gated_manifest_tokens": 450,
+        "gated_tool_count": 2,
+        "withheld_tools": []
+      },
+      "maximum": 450,
+      "pct": 0.0,
+      "unit": "tokens",
+      "value": 0
+    },
+    "read_write_marker_coverage": {
+      "detail": {
+        "unmarked": []
+      },
+      "maximum": 2,
+      "pct": 100.0,
+      "unit": "points",
+      "value": 2
+    },
+    "remedy_survives_truncation": {
+      "detail": {
+        "budget_chars": 500,
+        "over_budget": [],
+        "remedy_after_interpolation": [
+          "envelope.py:135",
+          "envelope.py:95",
+          "envelope.py:160",
+          "envelope.py:107",
+          "envelope.py:200",
+          "timeline.py:119"
+        ]
+      },
+      "maximum": 7,
+      "pct": 100.0,
+      "unit": "messages",
+      "value": 7
+    },
+    "teaching_error_rate": {
+      "detail": {},
+      "maximum": 7,
+      "pct": 100.0,
+      "unit": "messages",
+      "value": 7
+    },
+    "tool_description_quality": {
+      "detail": {
+        "per_dimension_pct": {
+          "args": 100.0,
+          "gotcha": 100.0,
+          "marker": 100.0,
+          "next_hop": 100.0,
+          "what": 100.0,
+          "when": 100.0
+        },
+        "tools_graded": 2,
+        "weakest_tools": {
+          "incident_timeline": [],
+          "list_symptom_categories": []
+        }
+      },
+      "maximum": 12,
+      "pct": 100.0,
+      "unit": "points",
+      "value": 12
+    },
+    "tool_failure_payload_quality": {
+      "detail": {
+        "bare_string_error_lines": [],
+        "carries_hint": 1,
+        "dict_shaped": 1,
+        "error_return_sites": 1,
+        "hint_names_artifact": 1
+      },
+      "maximum": 3,
+      "pct": 100.0,
+      "unit": "checks",
+      "value": 3
+    }
+  }
+}

vmware_debug-1.8.4/tests/eval/capability/_scoring.py

@@ -0,0 +1,221 @@
+"""Scoring primitives shared by this skill's capability evals.
+
+Why this exists
+---------------
+A regression eval answers a yes/no question ("does bug #31 still bite?") and
+must sit at 100%. A capability eval answers a *how well* question ("can a small
+model actually drive this tool surface?") and is expected to sit below 100%
+forever — the number is the product, not the pass/fail.
+
+So every capability eval here does two things:
+
+1. **records a score** into ``_scores.json`` next to this file, so the next
+   release can diff against it rather than re-deriving a feeling; and
+2. **asserts a floor**, deliberately set well under the current score. The floor
+   is a ratchet against collapse, not a quality bar. A test going red here means
+   something fell off a cliff, not that the surface is imperfect.
+
+Do not raise a floor to match a score. The floor's job is to stay boring.
+
+Token estimation
+----------------
+``estimate_tokens`` is a BPE approximation (word/punct segmentation × 0.75), not
+a real tokenizer — none of the family venvs carry ``tiktoken`` and a capability
+eval must not add a dependency to be runnable. It lands within roughly ±15% of
+cl100k on this kind of English-plus-JSON text, which is ample: every budget here
+is a *trend* measurement compared against the same estimator in the previous
+release, and against thresholds chosen with the error bar already in mind.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field, replace
+from pathlib import Path
+from typing import Any
+
+SCORES_PATH = Path(__file__).with_name("_scores.json")
+
+_WORDISH = re.compile(r"\w+|[^\w\s]")
+
+
+def estimate_tokens(text: str) -> int:
+    """Approximate BPE token count for ``text``. See module docstring for error bar."""
+    if not text:
+        return 0
+    return int(len(_WORDISH.findall(text)) * 0.75)
+
+
+@dataclass(frozen=True)
+class Score:
+    """One recorded capability measurement.
+
+    ``value``/``maximum`` are the raw numbers; ``pct`` is what release-to-release
+    comparison actually reads. ``detail`` carries the per-item breakdown so a
+    regression in the aggregate can be traced to the item that caused it without
+    re-running anything.
+    """
+
+    name: str
+    value: float
+    maximum: float
+    unit: str = "points"
+    detail: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def pct(self) -> float:
+        if self.maximum == 0:
+            return 0.0
+        return round(100.0 * self.value / self.maximum, 1)
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "value": round(self.value, 2),
+            "maximum": round(self.maximum, 2),
+            "unit": self.unit,
+            "pct": self.pct,
+            "detail": self.detail,
+        }
+
+
+@dataclass
+class ScoreBoard:
+    """Session-scoped collector.
+
+    The dataclass itself is mutable by necessity — pytest hands results in one
+    test at a time — but ``add`` never mutates a :class:`Score`, and ``records``
+    is replaced rather than appended in place, so no caller can observe a
+    half-updated board.
+    """
+
+    records: tuple[Score, ...] = ()
+
+    def add(self, score: Score) -> Score:
+        self.records = (*self.records, score)
+        return score
+
+    def as_dict(self) -> dict[str, Any]:
+        return {s.name: s.as_dict() for s in sorted(self.records, key=lambda s: s.name)}
+
+    def write(self, path: Path = SCORES_PATH) -> None:
+        if not self.records:
+            return
+        payload = {
+            "_comment": (
+                "Capability eval scores. Regenerate with: pytest -m capability. "
+                "These are tracked trends, not pass/fail gates — see _scoring.py."
+            ),
+            "scores": self.as_dict(),
+        }
+        path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+
+
+def previous_scores(path: Path = SCORES_PATH) -> dict[str, Any]:
+    """Load the last recorded run, or ``{}`` on a first run / unreadable file."""
+    try:
+        return json.loads(path.read_text()).get("scores", {})
+    except (OSError, ValueError):
+        return {}
+
+
+# ---------------------------------------------------------------------------
+# Text probes reused by several evals
+# ---------------------------------------------------------------------------
+
+#: Phrases that signal a description told the agent *when* to reach for this
+#: tool rather than a sibling. Routing is the single hardest thing for a small
+#: model to infer, because inferring it requires holding the whole tool list.
+WHEN_MARKERS = (
+    "use this",
+    "use it",
+    "use for",
+    "use when",
+    "instead of",
+    " instead",
+    "prefer ",
+    "before ",
+    "first",
+    "start here",
+    "rather than",
+    "for detail",
+    "drill into",
+    "follow up",
+    "then ",
+    # "Use after a storage array presents new LUNs" is a complete when-clause.
+    # Without these the rubric scored it zero, and the only way to earn the
+    # point was to reword it to "Use this when ..." — identical meaning, no
+    # information added. A rubric that pays for phrasing buys churn.
+    "use after",
+    "after ",
+    "once ",
+    "whenever ",
+)
+
+#: Phrases that signal the description stated what comes back.
+WHAT_MARKERS = ("returns", "return ", "yields", "reports", "->", "→")
+
+#: Phrases that signal a caveat — the class of information a strong model infers
+#: from experience and a weak model simply never learns.
+GOTCHA_MARKERS = (
+    "note",
+    "only",
+    "requires",
+    "does not",
+    "do not",
+    "cannot",
+    "never",
+    "always",
+    "may ",
+    "must ",
+    "caution",
+    "warning",
+    "n/a",
+    "point-in-time",
+    "no trending",
+    "not supported",
+    "unavailable",
+    "beware",
+    "careful",
+    "irreversible",
+    "cannot be undone",
+    "double",
+    "confirm",
+    "dry-run",
+    "dry run",
+    "skip",
+    "fall back",
+    "fallback",
+    "if the",
+    "when the",
+    "unless",
+    "except",
+)
+
+
+def has_any(text: str, markers: tuple[str, ...]) -> bool:
+    """Does ``text`` contain any marker, ignoring how the source happens to wrap?
+
+    Whitespace is collapsed first. Markers carry trailing spaces (``"before "``,
+    ``"then "``), so a docstring that wrapped at exactly that word scored zero
+    for content it plainly contained -- ``"...check the rule count before\\n
+    deleting"`` missed ``"before "`` on a line break. Two of one skill's
+    forty-two apparent gaps were this, which means the rubric was reporting
+    formatting as absence and inviting a rewrite that changes nothing.
+    """
+    low = " ".join(text.lower().split())
+    return any(m in low for m in markers)
+
+
+def documented_args(description: str, schema: dict[str, Any]) -> tuple[int, int]:
+    """Return ``(documented, total)`` schema properties named in ``description``.
+
+    An undocumented parameter is one a model must guess the meaning of from its
+    name alone. That is survivable for ``target`` and not survivable for
+    ``top_n`` or ``folder_filter``.
+    """
+    props = tuple((schema or {}).get("properties", {}))
+    if not props:
+        return (0, 0)
+    low = description.lower()
+    return (sum(1 for p in props if p.lower() in low), len(props))

vmware_debug-1.8.4/tests/eval/capability/_skill.py

@@ -0,0 +1,62 @@
+"""The only repo-specific facts in this capability suite.
+
+Every ``test_*.py`` file in this directory is identical across the family repos;
+they differ only through this module. Keeping the difference in one small file is
+what makes a rubric change portable — edit the eval once, copy it, and the scores
+stay comparable between skills.
+"""
+
+from __future__ import annotations
+
+#: Import path of the Python package under test.
+PACKAGE = "vmware_debug"
+
+#: Module holding the FastMCP server.
+SERVER_MODULE = "vmware_debug.mcp_server.server"
+
+#: CLI entry point name, used when scoring whether an error names something
+#: concrete for the operator to run.
+CLI_NAME = "vmware-debug"
+
+#: Companion skills this one legitimately routes to. A required entity name that
+#: this surface cannot produce is not a dead end *if* the description says which
+#: sibling skill produces it — that is a documented hand-off rather than a gap.
+COMPANION_SKILLS = (
+    "vmware-aiops",
+    "vmware-monitor",
+    "vmware-storage",
+    "vmware-vks",
+    "vmware-nsx",
+    "vmware-nsx-security",
+    "vmware-aria",
+    "vmware-avi",
+    "vmware-harden",
+    "vmware-pilot",
+)
+
+#: Entity tokens this skill's tools name, mapped to the words its listing tools
+#: use. Authored from the registry's own required parameters, not from the
+#: domain in the abstract. Drives ``test_entity_reachability``: a stem that is
+#: not here is invisible to that eval, which is why the suite asserts coverage.
+#: Deliberately empty: neither tool takes a discoverable identifier —
+#: ``incident_timeline`` takes an event payload and
+#: ``list_symptom_categories`` takes nothing. The suite asserts that an
+#: empty result comes from an empty map rather than from a map that
+#: simply failed to fit, so this emptiness has to be stated.
+ENTITY_WORDS = {
+}
+
+#: Skill-specific parameters that end in an entity suffix but are supplied by the
+#: operator rather than discovered from an API. Universal exclusions (``target``,
+#: paths, filters) live in the eval itself.
+NOT_AN_ENTITY: frozenset[str] = frozenset()
+
+def get_server(module):
+    """Return the FastMCP instance ``SERVER_MODULE`` exposes.
+
+    This skill builds its server in a factory rather than at import time, so the
+    read-only gate is applied per call. Declared here rather than probed with a
+    try/except chain — a fallback would let a server that stops exposing what
+    this file says silently resolve to the other shape.
+    """
+    return module.build_server()

vmware_debug-1.8.4/tests/eval/capability/conftest.py

@@ -0,0 +1,80 @@
+"""Session plumbing for the capability evals.
+
+The ``board`` fixture is session-scoped so that one ``pytest -m capability`` run
+produces one ``_scores.json`` holding every measurement taken. Running a single
+capability file rewrites the file with only that file's scores — intentional, so
+a partial run is never mistaken for a full one; regenerate with the full
+``-m capability`` selection before committing a score change.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import importlib
+import os
+import sys
+from typing import Any
+
+import pytest
+
+from ._scoring import ScoreBoard
+from ._skill import SERVER_MODULE, get_server
+
+#: Prefix of the modules the read-only gate affects at import time.
+_SERVER_PREFIX = SERVER_MODULE.split(".")[0] + ".mcp_server"
+
+_READ_ONLY_ENV = "VMWARE_READ_ONLY"
+
+
+def load_tools(read_only: bool = False) -> tuple[Any, ...]:
+    """Import the MCP server fresh and return the tools it registers.
+
+    Re-imports rather than reusing the loaded module because the read-only gate
+    runs at import time. The original module objects are restored afterwards —
+    deleting them would leave other test files monkeypatching a module nobody
+    imports any more, and their patches would silently stop applying.
+    """
+    saved = {n: m for n, m in sys.modules.items() if n.startswith(_SERVER_PREFIX)}
+    prior = os.environ.get(_READ_ONLY_ENV)
+    try:
+        if read_only:
+            os.environ[_READ_ONLY_ENV] = "true"
+        else:
+            os.environ.pop(_READ_ONLY_ENV, None)
+        for name in list(saved):
+            del sys.modules[name]
+        mod = importlib.import_module(SERVER_MODULE)
+        return tuple(asyncio.run(get_server(mod).list_tools()))
+    finally:
+        if prior is None:
+            os.environ.pop(_READ_ONLY_ENV, None)
+        else:
+            os.environ[_READ_ONLY_ENV] = prior
+        for name in [n for n in sys.modules if n.startswith(_SERVER_PREFIX)]:
+            del sys.modules[name]
+        sys.modules.update(saved)
+
+
+@pytest.fixture(scope="session")
+def board() -> Any:
+    b = ScoreBoard()
+    yield b
+    b.write()
+
+
+@pytest.fixture(scope="session")
+def tools() -> tuple[Any, ...]:
+    """Every tool the real FastMCP registry exposes, as the agent would see it.
+
+    Read from ``mcp.list_tools()`` rather than from the source, because the
+    registry is the only thing that reflects decorators, gating, and the schema
+    FastMCP actually derives from each signature — which is what lands in the
+    model's context, not the docstring as written.
+    """
+    return load_tools(read_only=False)
+
+
+@pytest.fixture(scope="session")
+def gated_tools() -> tuple[Any, ...]:
+    """The surface an operator gets under ``VMWARE_READ_ONLY=true``."""
+    return load_tools(read_only=True)