boost-skill-cli 1.0.347__tar.gz → 1.0.348__tar.gz

{boost_skill_cli-1.0.347 → boost_skill_cli-1.0.348}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.347
+Version: 1.0.348
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.347 → boost_skill_cli-1.0.348}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.347'
-__version_tuple__ = version_tuple = (1, 0, 347)
+__version__ = version = '1.0.348'
+__version_tuple__ = version_tuple = (1, 0, 348)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.347 → boost_skill_cli-1.0.348}/boost_cli/core/chat.py

@@ -52,6 +52,18 @@ MIN_FAITHFULNESS = 0.60
 # conversation instead of from what retrieval actually returned.
 HISTORY_TURNS = 4
 
+# A catalogue-name shape: lowercase, hyphenated, e.g. `code-reviewer`.
+_NAME = r"[a-z][a-z0-9]*(?:-[a-z0-9]+)+"
+_TOKEN = re.compile(r"\b%s\b" % _NAME)
+
+# Emphasis around a name-shaped token: `x`, **x**, *x*.
+_EMPHASISED = re.compile(r"`(%(n)s)`|\*\*(%(n)s)\*\*|\*(%(n)s)\*" % {"n": _NAME}, re.IGNORECASE)
+
+# Verbs that introduce a recommendation. What follows one is a claim about what
+# to install, which is the phrasing a reader acts on.
+_RECOMMENDS = re.compile(
+    r"\b(?:use|using|install|try|run|pick|choose|recommends?)\b", re.IGNORECASE)
+
 _SYSTEM = (
     "You help a developer pick an AI coding-agent skill from a catalogue. "
     "Answer ONLY from the numbered skills provided. Never invent a skill name. "
@@ -132,10 +144,36 @@ def retrieve(question: str, history: Sequence[Turn] = (),
     return [h["entry"] for h in hits[:k]], engine
 
 
+# Catalogue descriptions are untrusted upstream text and some are enormous:
+# measured over a real 71,655-entry catalogue, 22.7% exceed 300 characters and
+# the longest is 5,771. One entry rendered as a screenful of embedded
+# `<example>` blocks — a correct answer nobody could read.
+DESC_CHARS = 200
+
+
+def _one_line(text: str) -> str:
+    """Collapse a description to a single line of plain text.
+
+    Handles real newlines *and* the literal two-character ``\\n`` that 635
+    entries carry from double-escaped upstream frontmatter — those reach a
+    terminal as visible backslash-n noise rather than as line breaks.
+    """
+    return " ".join(text.replace("\\n", " ").replace("\\t", " ").split())
+
+
 def _describe(entry: dict) -> str:
-    """One catalogue entry as a line of prompt/answer text."""
+    """One catalogue entry as a line of prompt/answer text.
+
+    Truncation is marked with an ellipsis rather than silent: the reader needs
+    to know there is more to the description before deciding, and `boost info
+    <name>` is where the full text lives.
+    """
+    desc = _one_line(entry.get("description") or "")
+    if len(desc) > DESC_CHARS:
+        # Cut on a word boundary so the tail is not a severed token.
+        desc = desc[:DESC_CHARS].rsplit(" ", 1)[0] + " …"
     return "%s (%s) — %s" % (entry.get("name", "?"), entry.get("tap", "?"),
-                             (entry.get("description") or "").strip() or "no description")
+                             desc or "no description")
 
 
 def source_text(entries: Sequence[dict]) -> str:
@@ -143,22 +181,74 @@ def source_text(entries: Sequence[dict]) -> str:
     return "\n".join("%d. %s" % (i, _describe(e)) for i, e in enumerate(entries, 1))
 
 
-def ungrounded_names(reply: str, entries: Sequence[dict]) -> List[str]:
-    """Skill-shaped tokens in ``reply`` that name nothing in ``entries``.
+def grounding_text(entries: Sequence[dict]) -> str:
+    """The text a reply is checked against — untruncated, unlike the prompt.
+
+    These must be different strings. The prompt is trimmed to keep an answer
+    readable, but checking a reply against the *trimmed* text would score a
+    faithful sentence as unfaithful purely because its evidence fell past
+    :data:`DESC_CHARS` — the truncation would manufacture the very failure the
+    check exists to detect.
+    """
+    return "\n".join(
+        "%s %s" % (e.get("name", ""), _one_line(e.get("description") or ""))
+        for e in entries)
+
+
+def claimed_names(reply: str) -> set:
+    """Name-shaped tokens the reply presents *as skills to install*.
+
+    Not every hyphenated word is a claim. Descriptions in this catalogue are
+    thick with ordinary compounds — "read-only", "multi-agent", "single-pass",
+    "security-focused" — and treating each as a possible fabricated skill name
+    rejected essentially every real answer (reported from live use). The harm
+    this guards against is specific: a reader acting on a recommendation and
+    hunting for a package that does not exist. So a token counts as claimed when
+    it is emphasised, or when it sits in the clause a recommending verb
+    introduces — "try code-reviewer or docker-compose-expert" claims both.
+
+    Scanning the whole clause rather than just the next word is what catches the
+    second item in a list, which is exactly where an invention tends to hide.
+    Prose that merely *describes* ("a read-only multi-agent review") claims
+    nothing and is left to :func:`faithfulness.score`, the second gate.
+    """
+    out = {next(g for g in m.groups() if g).lower()
+           for m in _EMPHASISED.finditer(reply)}
+    for verb in _RECOMMENDS.finditer(reply):
+        # Stop at the sentence end: a later sentence is a new statement, not
+        # part of this recommendation.
+        clause = re.split(r"[.;!?\n]", reply[verb.end():], maxsplit=1)[0]
+        out |= {t.lower() for t in _TOKEN.findall(clause)}
+    return out
 
-    Catalogue names are lowercase hyphenated words (``code-reviewer``,
-    ``pdf``), which is a distinctive enough shape to spot in prose. Anything
-    matching that shape and not present in the candidate set is a name the model
-    supplied itself — the exact failure that sends a user hunting for a skill
-    that does not exist.
 
-    Deliberately narrow: it only flags *hyphenated* tokens. A single ordinary
-    word cannot be told apart from prose, so this under-reports rather than
-    firing on innocent sentences, and the faithfulness score covers the rest.
+def ungrounded_names(reply: str, entries: Sequence[dict]) -> List[str]:
+    """Skill-shaped tokens in ``reply`` that appear nowhere in ``entries``.
+
+    Catalogue names are lowercase hyphenated words (``code-reviewer``), which is
+    a distinctive enough shape to spot in prose. A token of that shape that
+    appears in neither the retrieved names nor the retrieved *text* is one the
+    model supplied itself — the failure that sends a user hunting for a skill
+    which does not exist, or toward a typosquat (see ``core/typosquat.py``).
+
+    THE SOURCES GROUND THE REPLY, NOT JUST THE NAMES. An earlier version checked
+    only against the name list, so ordinary hyphenated English quoted straight
+    out of a description — "read-only", "multi-agent", "pre-deployment" — scored
+    as fabricated. In this catalogue those compounds are everywhere, so a
+    faithful reply was rejected almost every time and the AI path silently
+    degraded to extractive for nearly every real query. Reported from live use.
+
+    Deliberately narrow in the other direction too: it only flags *hyphenated*
+    tokens, because a single ordinary word cannot be told apart from prose. This
+    under-reports rather than firing on innocent sentences, and
+    :func:`faithfulness.score` covers the rest.
     """
-    known = {str(e.get("name", "")).lower() for e in entries}
-    candidates = set(re.findall(r"\b[a-z][a-z0-9]*(?:-[a-z0-9]+)+\b", reply.lower()))
-    return sorted(c for c in candidates if c not in known)
+    # Whole tokens, not a substring scan. `x in "…"` would treat an invented
+    # `code-review` as grounded because `code-reviewer` contains it — precisely
+    # the near-miss shape a typosquat has.
+    grounded = set(_TOKEN.findall(grounding_text(entries).lower()))
+    grounded |= {str(e.get("name", "")).lower() for e in entries}
+    return sorted(c for c in claimed_names(reply) if c not in grounded)
 
 
 def _extractive(question: str, entries: Sequence[dict]) -> str:
@@ -208,7 +298,7 @@ def answer(question: str, history: Sequence[Turn] = (),
         # A manufactured name is unrecoverable — there is no honest way to show
         # prose that points at a skill which does not exist.
         return Reply(extractive, entries.copy(), engine, "extractive", False)
-    if faithfulness.score(reply, source) < MIN_FAITHFULNESS:
+    if faithfulness.score(reply, grounding_text(entries)) < MIN_FAITHFULNESS:
         return Reply(extractive, entries.copy(), engine, "extractive", False)
     return Reply(reply, entries.copy(), engine, "ai", True)
 

{boost_skill_cli-1.0.347 → boost_skill_cli-1.0.348}/boost_skill_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.347
+Version: 1.0.348
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.347 → boost_skill_cli-1.0.348}/tests/unit/test_chat.py

@@ -190,3 +190,84 @@ class TestSourceText:
         # A blank line reads as "no such skill"; the placeholder keeps the
         # numbering honest and tells the model there is nothing to summarise.
         assert "no description" in chat.source_text([_entry("bare")])
+
+
+class TestGroundingDoesNotFlagOrdinaryEnglish:
+    """A hyphenated word is not a skill name.
+
+    Reported from real use: every AI reply was rejected with "named something
+    outside the retrieved skills". The catalogue's own descriptions are full of
+    hyphenated compounds — "read-only", "multi-agent", "pre-deployment",
+    "risk-adaptive", "test-driven" — and the first version of
+    :func:`ungrounded_names` flagged any hyphenated token that was not itself a
+    catalogue name. So a reply that faithfully quoted its sources was scored as
+    fabricating, and the AI path degraded to extractive for essentially every
+    query in this domain.
+
+    The fix is that the *sources* ground the reply, not just the name list:
+    a term that appears in the retrieved text came from the retrieved text.
+    """
+
+    ENTRIES = ({"name": "code-reviewer", "tap": "t",
+                "description": "Parallel read-only multi-agent review of a git diff, "
+                               "risk-adaptive and pre-deployment focused."},)
+
+    def test_compounds_quoted_from_the_sources_are_grounded(self):
+        reply = ("The read-only multi-agent code-reviewer fits — it is "
+                 "risk-adaptive and pre-deployment focused.")
+        assert chat.ungrounded_names(reply, self.ENTRIES) == []
+
+    def test_a_genuinely_invented_name_is_still_caught(self):
+        # The whole point of the check: boost is a package manager, so a
+        # plausible-but-absent name sends the user hunting for a skill that
+        # does not exist, or to a typosquat.
+        reply = "Use docker-compose-expert for that."
+        assert "docker-compose-expert" in chat.ungrounded_names(reply, self.ENTRIES)
+
+    def test_a_near_miss_is_not_grounded_by_a_longer_real_name(self):
+        # A substring scan would treat "code-review" as grounded because
+        # "code-reviewer" contains it — exactly the near-miss shape a
+        # typosquat has, so matching is on whole tokens.
+        reply = "Install code-review for that."
+        assert "code-review" in chat.ungrounded_names(reply, self.ENTRIES)
+
+    def test_grounding_is_case_insensitive(self):
+        reply = "Read-Only review is what Code-Reviewer does."
+        assert chat.ungrounded_names(reply, self.ENTRIES) == []
+
+    def test_a_name_never_needs_the_description_to_be_grounded(self):
+        entries = [{"name": "tdd-workflow", "tap": "t", "description": ""}]
+        assert chat.ungrounded_names("Try tdd-workflow.", entries) == []
+
+
+class TestLongDescriptionsAreReadable:
+    """Catalogue descriptions are untrusted text and some are enormous.
+
+    Measured over a real 71,655-entry catalogue: 22.7% of descriptions exceed
+    300 characters, the longest is 5,771, and 635 contain literal ``\\n``
+    escape sequences. One such entry rendered as a screenful of embedded
+    ``<example>`` blocks, which is what a user actually saw — the answer was
+    correct and unreadable.
+    """
+
+    def test_a_long_description_is_truncated(self):
+        entry = {"name": "x", "tap": "t", "description": "word " * 400}
+        line = chat._describe(entry)
+        assert len(line) < 300, len(line)
+
+    def test_truncation_is_marked_rather_than_silent(self):
+        entry = {"name": "x", "tap": "t", "description": "word " * 400}
+        assert chat._describe(entry).rstrip().endswith("…")
+
+    def test_a_short_description_is_left_alone(self):
+        entry = {"name": "x", "tap": "t", "description": "Short and useful."}
+        assert chat._describe(entry).endswith("Short and useful.")
+
+    def test_embedded_newlines_collapse_to_one_line(self):
+        # Both real newlines and the literal two-character \n seen in 635
+        # entries, which reach the terminal as visible backslash-n noise.
+        entry = {"name": "x", "tap": "t",
+                 "description": "first\nsecond\\nthird\r\nfourth"}
+        line = chat._describe(entry)
+        assert "\n" not in line and "\\n" not in line
+        assert "first second third fourth" in line