boost-skill-cli 1.0.10__tar.gz → 1.0.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/PKG-INFO +1 -1
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/_version.py +2 -2
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/cli.py +2 -1
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/configuration.py +17 -5
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/discovery.py +41 -3
- boost_skill_cli-1.0.12/boost_cli/core/rag.py +394 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_skill_cli.egg-info/PKG-INFO +1 -1
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_skill_cli.egg-info/SOURCES.txt +2 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/index.html +4 -3
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_configuration.py +22 -3
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_discovery.py +38 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_everyday_loop.py +1 -1
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/smoke.sh +1 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_errors_and_cli_table.py +6 -6
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_frontmatter.py +103 -0
- boost_skill_cli-1.0.12/tests/unit/test_rag.py +824 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/.gitignore +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/CLAUDE.md +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/CONTRIBUTING.md +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/LICENSE +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/MANIFEST.in +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/Makefile +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/README.md +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/SECURITY.md +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/__init__.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/__main__.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/__init__.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/info.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/intelligence.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/pkg.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/quality.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/taps.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/team.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/__init__.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/agents.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/ai.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/catalog.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/config.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/frontmatter.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/gitutil.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/journal.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/lockfile.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/logs.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/output.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/paths.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/policy.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/registry.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/store.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/util.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/data/registries.json +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/errors.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_skill_cli.egg-info/dependency_links.txt +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_skill_cli.egg-info/entry_points.txt +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_skill_cli.egg-info/top_level.txt +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/.claude/settings.local.json +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/DEBUGGING.md +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/demo.gif +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/demo.tape +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/rag-architecture.md +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/pyproject.toml +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/scripts/build_registries.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/scripts/mutation_gate.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/setup.cfg +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/conftest.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_info.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_intelligence.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_pkg.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_quality.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_taps.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_team.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/make_fixture.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_agents.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_ai.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_catalog.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_config.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_gitutil.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_journal.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_lockfile.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_logs.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_mutation_hardening.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_output.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_paths.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_policy.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_registry.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_store.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_util.py +0 -0
- {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_version.py +0 -0
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '1.0.
|
|
22
|
-
__version_tuple__ = version_tuple = (1, 0,
|
|
21
|
+
__version__ = version = '1.0.12'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 0, 12)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -60,8 +60,9 @@ COMMANDS = [
|
|
|
60
60
|
("unpin", "pkg", "pkg", "Allow a pinned skill to update again"),
|
|
61
61
|
("snapshot", "pkg", "pkg", "Save & restore whole skill environments"),
|
|
62
62
|
("export", "pkg", "pkg", "Package skills as shareable zip/tar archives"),
|
|
63
|
-
# Discovery & Search (
|
|
63
|
+
# Discovery & Search (9)
|
|
64
64
|
("search", "find", "discovery", "Search skills across tap registries (AI-ranked)"),
|
|
65
|
+
("reindex", "find", "discovery", "Build/refresh the full-content search index"),
|
|
65
66
|
("discover", "find", "discovery", "Browse & search the GitHub-wide skill discovery index"),
|
|
66
67
|
("recommend", "find", "discovery", "Suggest skills based on your project's tech stack"),
|
|
67
68
|
("browse", "find", "discovery", "Interactive full-screen TUI with fuzzy search"),
|
|
@@ -20,7 +20,7 @@ from pathlib import Path
|
|
|
20
20
|
|
|
21
21
|
from .. import __version__
|
|
22
22
|
from ..core import agents, catalog, config, frontmatter, gitutil, journal
|
|
23
|
-
from ..core import lockfile, paths, policy, registry, store, util
|
|
23
|
+
from ..core import lockfile, paths, policy, rag, registry, store, util
|
|
24
24
|
from ..core import output as out
|
|
25
25
|
from ..errors import BoostError
|
|
26
26
|
|
|
@@ -895,11 +895,23 @@ _MCP_TOOLS = [
|
|
|
895
895
|
def _mcp_tool(tool: str, args: dict):
|
|
896
896
|
"""Run one MCP tool -> (text, is_error). (None, _) for unknown tools."""
|
|
897
897
|
if tool == "boost_search":
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
898
|
+
query = str(args.get("query", ""))
|
|
899
|
+
rag_result = rag.search(query, limit=10)
|
|
900
|
+
if rag_result is not None: # full-content index is built
|
|
901
|
+
hits, _ranker = rag_result
|
|
902
|
+
if not hits:
|
|
903
|
+
return "no skills match %r" % query, False
|
|
904
|
+
return "\n".join(
|
|
905
|
+
"%s — %s (%s)" % (h["entry"]["name"],
|
|
906
|
+
h["entry"].get("description", ""),
|
|
907
|
+
h["entry"]["tap"])
|
|
908
|
+
for h in hits), False
|
|
909
|
+
# no index yet -> keep today's frontmatter search so nothing regresses
|
|
910
|
+
scored = catalog.search(query)[:10]
|
|
911
|
+
if not scored:
|
|
912
|
+
return "no skills match %r" % query, False
|
|
901
913
|
return "\n".join("%s — %s (%s)" % (e["name"], e["description"], e["tap"])
|
|
902
|
-
for e, _score in
|
|
914
|
+
for e, _score in scored), False
|
|
903
915
|
if tool == "boost_list":
|
|
904
916
|
skills = lockfile.installed()
|
|
905
917
|
if not skills:
|
|
@@ -15,7 +15,7 @@ import sys
|
|
|
15
15
|
import time
|
|
16
16
|
from pathlib import Path
|
|
17
17
|
|
|
18
|
-
from ..core import agents, ai, catalog, gitutil, journal, lockfile, paths, registry, store, util
|
|
18
|
+
from ..core import agents, ai, catalog, gitutil, journal, lockfile, paths, rag, registry, store, util
|
|
19
19
|
from ..core import output as out
|
|
20
20
|
from ..errors import BoostError
|
|
21
21
|
|
|
@@ -184,7 +184,14 @@ def cmd_search(argv):
|
|
|
184
184
|
if not registry.list_taps():
|
|
185
185
|
raise BoostError("no taps configured — nothing to search",
|
|
186
186
|
hint="add the recommended registries with `boost tap --defaults`")
|
|
187
|
-
|
|
187
|
+
# Prefer the full-content (RAG) index when it exists; otherwise fall back
|
|
188
|
+
# to the frontmatter search so nothing regresses before `boost reindex`.
|
|
189
|
+
use_rag = rag.ready()
|
|
190
|
+
if use_rag:
|
|
191
|
+
scored = [(h["entry"], h["score"])
|
|
192
|
+
for h in rag.retrieve(query, k=max(60, args.limit * 4))]
|
|
193
|
+
else:
|
|
194
|
+
scored = catalog.search(query)
|
|
188
195
|
if args.as_json:
|
|
189
196
|
print(json.dumps([dict(e, score=s) for e, s in scored[:args.limit]]))
|
|
190
197
|
return 0
|
|
@@ -193,7 +200,7 @@ def cmd_search(argv):
|
|
|
193
200
|
out.info(out.c("try `boost discover %s` to search all of GitHub" % query,
|
|
194
201
|
out.DIM))
|
|
195
202
|
return 0
|
|
196
|
-
ranker = "heuristic relevance"
|
|
203
|
+
ranker = "full-content BM25" if use_rag else "heuristic relevance"
|
|
197
204
|
if args.smart:
|
|
198
205
|
if ai.available():
|
|
199
206
|
reranked = _ai_rank(query, scored)
|
|
@@ -214,6 +221,37 @@ def cmd_search(argv):
|
|
|
214
221
|
return 0
|
|
215
222
|
|
|
216
223
|
|
|
224
|
+
def cmd_reindex(argv):
|
|
225
|
+
"""Build/refresh the full-content (RAG) search index over tapped items."""
|
|
226
|
+
p = argparse.ArgumentParser(
|
|
227
|
+
prog="boost reindex",
|
|
228
|
+
description="Build or refresh the full-content search index")
|
|
229
|
+
p.add_argument("--force", action="store_true",
|
|
230
|
+
help="reindex every tap, ignoring cached commits")
|
|
231
|
+
p.add_argument("--json", action="store_true", dest="as_json",
|
|
232
|
+
help="machine-readable output")
|
|
233
|
+
args = p.parse_args(argv)
|
|
234
|
+
if not registry.list_taps():
|
|
235
|
+
raise BoostError("no taps configured — nothing to index",
|
|
236
|
+
hint="add the recommended registries with `boost tap --defaults`")
|
|
237
|
+
stats = rag.build(force=args.force)
|
|
238
|
+
if args.as_json:
|
|
239
|
+
print(json.dumps(stats))
|
|
240
|
+
return 0
|
|
241
|
+
out.ok("indexed %d passages across %d items from %d tap%s"
|
|
242
|
+
% (stats["docs"], stats["entries"], stats["taps"],
|
|
243
|
+
"" if stats["taps"] == 1 else "s"))
|
|
244
|
+
if stats["reused"]:
|
|
245
|
+
out.info(out.c("reused %d unchanged tap%s; reindexed: %s"
|
|
246
|
+
% (len(stats["reused"]),
|
|
247
|
+
"" if len(stats["reused"]) == 1 else "s",
|
|
248
|
+
", ".join(stats["reindexed"]) or "none"),
|
|
249
|
+
out.DIM))
|
|
250
|
+
journal.log("reindex", "%d passages" % stats["docs"],
|
|
251
|
+
entries=stats["entries"])
|
|
252
|
+
return 0
|
|
253
|
+
|
|
254
|
+
|
|
217
255
|
def cmd_index(argv):
|
|
218
256
|
"""Build ~/.boost/cache/discovery.json via `gh api` code search."""
|
|
219
257
|
p = argparse.ArgumentParser(
|
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
"""Full-content retrieval over tapped skills/rules/workflows (RAG — Phase 1).
|
|
2
|
+
|
|
3
|
+
Pure stdlib. Where `catalog.search` scores only an item's name + one-line
|
|
4
|
+
description + frontmatter, this indexes the *whole* body of every item and
|
|
5
|
+
ranks it with BM25, then optionally reranks the shortlist through the
|
|
6
|
+
`core.ai` bridge. See docs/rag-architecture.md.
|
|
7
|
+
|
|
8
|
+
The index is persisted at ``~/.boost/cache/rag_index.json`` and refreshed
|
|
9
|
+
per-tap, keyed on the git commit each tap cache was built from, so the
|
|
10
|
+
expensive step — reading and chunking thousands of files — happens once and
|
|
11
|
+
again only when a tap actually changes.
|
|
12
|
+
|
|
13
|
+
Everything degrades: no index -> callers fall back to ``catalog.search``;
|
|
14
|
+
no AI -> the BM25 order is returned as-is.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import math
|
|
20
|
+
import re
|
|
21
|
+
from collections import defaultdict
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Dict, List, Optional, Tuple
|
|
24
|
+
|
|
25
|
+
try: # TypedDict lives in typing on 3.9+, kept optional for safety
|
|
26
|
+
from typing import TypedDict
|
|
27
|
+
except ImportError: # pragma: no cover - 3.9+ always has it
|
|
28
|
+
TypedDict = None # type: ignore
|
|
29
|
+
|
|
30
|
+
from . import ai, catalog, frontmatter, gitutil, paths, registry
|
|
31
|
+
|
|
32
|
+
INDEX_VERSION = 1
|
|
33
|
+
ENGINE = "bm25"
|
|
34
|
+
|
|
35
|
+
# Chunking defaults (documented in docs/rag-architecture.md §4).
|
|
36
|
+
CHUNK_CHARS = 1000
|
|
37
|
+
OVERLAP = 150
|
|
38
|
+
MAX_CHUNKS = 40
|
|
39
|
+
|
|
40
|
+
# BM25 parameters (§6a). Exposed in the persisted `params` so they are
|
|
41
|
+
# tunable without a format change.
|
|
42
|
+
K1 = 1.2
|
|
43
|
+
B = 0.75
|
|
44
|
+
|
|
45
|
+
# A deliberately tiny stopword list — full-content indexing already dilutes
|
|
46
|
+
# common words, so we only drop the most frequent glue.
|
|
47
|
+
_STOPWORDS = frozenset({
|
|
48
|
+
"the", "a", "an", "and", "or", "of", "to", "for", "in", "on", "with",
|
|
49
|
+
"is", "are", "be", "this", "that", "it", "as", "at", "by", "from",
|
|
50
|
+
"your", "you", "can", "will", "if", "when", "how",
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class Hit(TypedDict): # type: ignore[misc]
|
|
55
|
+
entry: dict # the live catalog entry (name, tap, kind, skill_md, ...)
|
|
56
|
+
score: float # BM25 relevance (higher = better)
|
|
57
|
+
snippet: str # best-matching passage, for display + rerank context
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# --------------------------------------------------------------- tokenizing
|
|
61
|
+
|
|
62
|
+
def tokenize(text: str) -> List[str]:
|
|
63
|
+
"""Lowercase, split on non-alphanumerics, drop stopwords and 1-char noise."""
|
|
64
|
+
return [t for t in re.split(r"[^a-z0-9]+", text.lower())
|
|
65
|
+
if len(t) >= 2 and t not in _STOPWORDS]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def chunk(text: str, size: int = CHUNK_CHARS, overlap: int = OVERLAP,
|
|
69
|
+
cap: int = MAX_CHUNKS) -> List[str]:
|
|
70
|
+
"""Split a body into passage-sized chunks on paragraph boundaries.
|
|
71
|
+
|
|
72
|
+
Paragraphs are packed up to ``size`` chars; an oversized paragraph is
|
|
73
|
+
hard-split with ``overlap`` carried across the cut so a match spanning a
|
|
74
|
+
boundary is not lost. At most ``cap`` chunks are returned.
|
|
75
|
+
"""
|
|
76
|
+
text = text.strip()
|
|
77
|
+
if not text:
|
|
78
|
+
return []
|
|
79
|
+
paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
|
|
80
|
+
chunks: List[str] = []
|
|
81
|
+
cur = ""
|
|
82
|
+
for p in paras:
|
|
83
|
+
while len(p) > size:
|
|
84
|
+
chunks.append(p[:size])
|
|
85
|
+
if len(chunks) >= cap:
|
|
86
|
+
return chunks
|
|
87
|
+
p = p[size - overlap:]
|
|
88
|
+
if not cur:
|
|
89
|
+
cur = p
|
|
90
|
+
elif len(cur) + 1 + len(p) <= size:
|
|
91
|
+
cur += "\n" + p
|
|
92
|
+
else:
|
|
93
|
+
chunks.append(cur)
|
|
94
|
+
if len(chunks) >= cap:
|
|
95
|
+
return chunks
|
|
96
|
+
tail = cur[-overlap:] if overlap else ""
|
|
97
|
+
cur = (tail + "\n" + p) if tail else p
|
|
98
|
+
if cur and len(chunks) < cap:
|
|
99
|
+
chunks.append(cur)
|
|
100
|
+
return chunks
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# ------------------------------------------------------------ body reading
|
|
104
|
+
|
|
105
|
+
def _tap_paths() -> Dict[str, Path]:
|
|
106
|
+
return {t.name: t.path for t in registry.list_taps()}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _tap_commits() -> Dict[str, str]:
|
|
110
|
+
"""Map safe tap name -> the commit its cache was built from."""
|
|
111
|
+
commits: Dict[str, str] = {}
|
|
112
|
+
for t in registry.list_taps():
|
|
113
|
+
commit = ""
|
|
114
|
+
try:
|
|
115
|
+
data = json.loads(t.cache_file.read_text())
|
|
116
|
+
commit = str(data.get("commit") or "")
|
|
117
|
+
except (OSError, json.JSONDecodeError):
|
|
118
|
+
pass
|
|
119
|
+
if not commit and t.is_cloned:
|
|
120
|
+
commit = gitutil.head_commit(t.path) or ""
|
|
121
|
+
commits[t.safe_name] = commit
|
|
122
|
+
return commits
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def read_body(entry: dict, tap_paths: Optional[Dict[str, Path]] = None) -> str:
|
|
126
|
+
"""Return an item's searchable text: name + description + prose body.
|
|
127
|
+
|
|
128
|
+
The frontmatter is stripped (reusing ``frontmatter.parse``); the name and
|
|
129
|
+
description are prepended so short items keep keyword parity with the old
|
|
130
|
+
search. Missing files degrade to just the catalog metadata.
|
|
131
|
+
"""
|
|
132
|
+
paths_map = _tap_paths() if tap_paths is None else tap_paths
|
|
133
|
+
header = "%s\n%s" % (entry.get("name", ""), entry.get("description", ""))
|
|
134
|
+
base = paths_map.get(entry.get("tap", ""))
|
|
135
|
+
if base is None:
|
|
136
|
+
base = paths.repos_dir() / str(entry.get("tap", "")).replace("/", "__")
|
|
137
|
+
rel = entry.get("skill_md")
|
|
138
|
+
if not rel:
|
|
139
|
+
return header.strip()
|
|
140
|
+
try:
|
|
141
|
+
text = (base / rel).read_text(encoding="utf-8", errors="replace")
|
|
142
|
+
except OSError:
|
|
143
|
+
return header.strip()
|
|
144
|
+
_meta, body = frontmatter.parse(text)
|
|
145
|
+
return ("%s\n%s" % (header, body)).strip()
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
# ----------------------------------------------------------------- build
|
|
149
|
+
|
|
150
|
+
def index_path() -> Path:
|
|
151
|
+
return paths.cache_dir() / "rag_index.json"
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _make_docs(entries: List[dict], tap_paths: Dict[str, Path]) -> List[dict]:
|
|
155
|
+
"""Chunk + tokenize every entry into scored documents (one per chunk)."""
|
|
156
|
+
docs: List[dict] = []
|
|
157
|
+
for e in entries:
|
|
158
|
+
body = read_body(e, tap_paths)
|
|
159
|
+
pieces = chunk(body) or [body]
|
|
160
|
+
for ci, piece in enumerate(pieces):
|
|
161
|
+
tf: Dict[str, int] = defaultdict(int)
|
|
162
|
+
for tok in tokenize(piece):
|
|
163
|
+
tf[tok] += 1
|
|
164
|
+
if not tf:
|
|
165
|
+
continue
|
|
166
|
+
docs.append({
|
|
167
|
+
"n": e["name"], "t": e["tap"], "k": e.get("kind", "skill"),
|
|
168
|
+
"c": ci, "l": sum(tf.values()),
|
|
169
|
+
"snip": piece[:200].strip(),
|
|
170
|
+
"tf": dict(tf),
|
|
171
|
+
})
|
|
172
|
+
return docs
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _postings_to_doc_tf(raw: dict) -> Dict[int, Dict[str, int]]:
|
|
176
|
+
"""Invert persisted postings back to per-doc term frequencies."""
|
|
177
|
+
doc_tf: Dict[int, Dict[str, int]] = defaultdict(dict)
|
|
178
|
+
for term, plist in raw.get("postings", {}).items():
|
|
179
|
+
for doc_id, tf in plist:
|
|
180
|
+
doc_tf[doc_id][term] = tf
|
|
181
|
+
return doc_tf
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _kept_docs(raw: dict, keep_safe: set) -> List[dict]:
|
|
185
|
+
"""Recover cached docs (with term freqs) for taps that did not change."""
|
|
186
|
+
doc_tf = _postings_to_doc_tf(raw)
|
|
187
|
+
kept: List[dict] = []
|
|
188
|
+
for doc_id, meta in enumerate(raw.get("docs", [])):
|
|
189
|
+
if meta["t"].replace("/", "__") in keep_safe:
|
|
190
|
+
kept.append({**meta, "tf": doc_tf.get(doc_id, {})})
|
|
191
|
+
return kept
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def build(entries: Optional[List[dict]] = None, force: bool = False) -> dict:
|
|
195
|
+
"""(Re)build the full-content index, reusing unchanged taps.
|
|
196
|
+
|
|
197
|
+
Returns stats: ``{"entries", "docs", "taps", "reindexed", "reused"}``.
|
|
198
|
+
"""
|
|
199
|
+
entries = catalog.all_entries() if entries is None else entries
|
|
200
|
+
tap_paths = _tap_paths()
|
|
201
|
+
commits = _tap_commits()
|
|
202
|
+
|
|
203
|
+
old = _load_raw()
|
|
204
|
+
reused_safe: set = set()
|
|
205
|
+
if old is not None and not force:
|
|
206
|
+
old_commits = old.get("commits", {})
|
|
207
|
+
for safe, commit in commits.items():
|
|
208
|
+
if commit and old_commits.get(safe) == commit:
|
|
209
|
+
reused_safe.add(safe)
|
|
210
|
+
|
|
211
|
+
fresh = [e for e in entries
|
|
212
|
+
if e["tap"].replace("/", "__") not in reused_safe]
|
|
213
|
+
docs = _make_docs(fresh, tap_paths)
|
|
214
|
+
if old is not None and reused_safe:
|
|
215
|
+
docs = _kept_docs(old, reused_safe) + docs
|
|
216
|
+
|
|
217
|
+
_save(docs, commits)
|
|
218
|
+
reindexed = sorted({e["tap"] for e in fresh})
|
|
219
|
+
return {
|
|
220
|
+
"entries": len({(e["name"], e["tap"]) for e in entries}),
|
|
221
|
+
"docs": len(docs),
|
|
222
|
+
"taps": len(commits),
|
|
223
|
+
"reindexed": reindexed,
|
|
224
|
+
"reused": sorted(reused_safe),
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _save(docs: List[dict], commits: Dict[str, str]) -> None:
|
|
229
|
+
postings: Dict[str, List[List[int]]] = defaultdict(list)
|
|
230
|
+
meta_docs: List[dict] = []
|
|
231
|
+
total_len = 0
|
|
232
|
+
for doc_id, d in enumerate(docs):
|
|
233
|
+
for term, tf in d["tf"].items():
|
|
234
|
+
postings[term].append([doc_id, tf])
|
|
235
|
+
total_len += d["l"]
|
|
236
|
+
meta_docs.append({"n": d["n"], "t": d["t"], "k": d["k"],
|
|
237
|
+
"c": d["c"], "l": d["l"], "snip": d["snip"]})
|
|
238
|
+
payload = {
|
|
239
|
+
"version": INDEX_VERSION,
|
|
240
|
+
"engine": ENGINE,
|
|
241
|
+
"generated": _now(),
|
|
242
|
+
"commits": commits,
|
|
243
|
+
"params": {"chunk_chars": CHUNK_CHARS, "overlap": OVERLAP,
|
|
244
|
+
"k1": K1, "b": B},
|
|
245
|
+
"stats": {"docs": len(docs),
|
|
246
|
+
"avg_len": (total_len / len(docs)) if docs else 0.0},
|
|
247
|
+
"docs": meta_docs,
|
|
248
|
+
"postings": {t: p for t, p in postings.items()},
|
|
249
|
+
}
|
|
250
|
+
paths.ensure_dirs()
|
|
251
|
+
p = index_path()
|
|
252
|
+
p.write_text(json.dumps(payload))
|
|
253
|
+
# Drop the mtime-keyed cache so a reindex is visible to an immediately
|
|
254
|
+
# following query even when the filesystem mtime granularity is coarse.
|
|
255
|
+
_CACHE.pop(str(p), None)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _now() -> str:
|
|
259
|
+
from . import util
|
|
260
|
+
return util.now_iso()
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
# ----------------------------------------------------------------- query
|
|
264
|
+
|
|
265
|
+
_CACHE: Dict[str, object] = {}
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _load_raw() -> Optional[dict]:
|
|
269
|
+
p = index_path()
|
|
270
|
+
key = str(p)
|
|
271
|
+
try:
|
|
272
|
+
mtime = p.stat().st_mtime
|
|
273
|
+
except OSError:
|
|
274
|
+
_CACHE.pop(key, None)
|
|
275
|
+
return None
|
|
276
|
+
cached = _CACHE.get(key)
|
|
277
|
+
if isinstance(cached, tuple) and cached[0] == mtime:
|
|
278
|
+
return cached[1] # type: ignore[return-value]
|
|
279
|
+
try:
|
|
280
|
+
data = json.loads(p.read_text())
|
|
281
|
+
except (OSError, json.JSONDecodeError):
|
|
282
|
+
return None
|
|
283
|
+
if not isinstance(data, dict) or data.get("version") != INDEX_VERSION:
|
|
284
|
+
return None
|
|
285
|
+
_CACHE[key] = (mtime, data)
|
|
286
|
+
return data
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def ready() -> bool:
|
|
290
|
+
"""True when a usable index exists on disk."""
|
|
291
|
+
raw = _load_raw()
|
|
292
|
+
return bool(raw and raw.get("docs"))
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _bm25(query_terms: List[str], raw: dict) -> Dict[int, float]:
|
|
296
|
+
docs = raw["docs"]
|
|
297
|
+
n = len(docs)
|
|
298
|
+
avg = raw.get("stats", {}).get("avg_len") or 1.0
|
|
299
|
+
postings = raw["postings"]
|
|
300
|
+
scores: Dict[int, float] = defaultdict(float)
|
|
301
|
+
for term in set(query_terms):
|
|
302
|
+
plist = postings.get(term)
|
|
303
|
+
if not plist:
|
|
304
|
+
continue
|
|
305
|
+
df = len(plist)
|
|
306
|
+
idf = math.log(1 + (n - df + 0.5) / (df + 0.5))
|
|
307
|
+
for doc_id, tf in plist:
|
|
308
|
+
dl = docs[doc_id]["l"] or 1
|
|
309
|
+
denom = tf + K1 * (1 - B + B * dl / avg)
|
|
310
|
+
scores[doc_id] += idf * (tf * (K1 + 1)) / denom
|
|
311
|
+
return scores
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def retrieve(query: str, k: int = 60, kind: Optional[str] = None,
|
|
315
|
+
entries: Optional[List[dict]] = None) -> List[Hit]:
|
|
316
|
+
"""Top-k full-content hits for ``query``. ``[]`` if the index is empty."""
|
|
317
|
+
raw = _load_raw()
|
|
318
|
+
if not raw or not raw.get("docs"):
|
|
319
|
+
return []
|
|
320
|
+
terms = tokenize(query)
|
|
321
|
+
if not terms:
|
|
322
|
+
return []
|
|
323
|
+
entries = catalog.all_entries() if entries is None else entries
|
|
324
|
+
live = {(e["name"], e["tap"]): e for e in entries}
|
|
325
|
+
docs = raw["docs"]
|
|
326
|
+
best: Dict[Tuple[str, str], Tuple[float, str]] = {}
|
|
327
|
+
for doc_id, score in _bm25(terms, raw).items():
|
|
328
|
+
d = docs[doc_id]
|
|
329
|
+
if kind is not None and d["k"] != kind:
|
|
330
|
+
continue
|
|
331
|
+
key = (d["n"], d["t"])
|
|
332
|
+
if key not in live:
|
|
333
|
+
continue
|
|
334
|
+
prev = best.get(key)
|
|
335
|
+
if prev is None or score > prev[0]:
|
|
336
|
+
best[key] = (score, d["snip"])
|
|
337
|
+
ranked = sorted(best.items(), key=lambda kv: (-kv[1][0], kv[0][0]))
|
|
338
|
+
hits: List[Hit] = []
|
|
339
|
+
for (name, tap), (score, snip) in ranked[:k]:
|
|
340
|
+
hits.append({"entry": live[(name, tap)], "score": score,
|
|
341
|
+
"snippet": snip}) # type: ignore[typeddict-item]
|
|
342
|
+
return hits
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def rerank(query: str, hits: List[Hit], limit: int = 10) -> Tuple[List[Hit], str]:
|
|
346
|
+
"""Reorder the shortlist with the LLM bridge; degrade to BM25 order."""
|
|
347
|
+
top = hits[:max(limit, 15)]
|
|
348
|
+
if not ai.available():
|
|
349
|
+
return hits[:limit], "BM25 full-content"
|
|
350
|
+
listing = "\n".join(
|
|
351
|
+
"- %s [%s]: %s" % (h["entry"]["name"], h["entry"].get("kind", "skill"),
|
|
352
|
+
(h["snippet"] or h["entry"].get("description", ""))[:180])
|
|
353
|
+
for h in top)
|
|
354
|
+
reply = ai.ask(
|
|
355
|
+
"Task: %s\n\nCandidate skills (name, kind, best matching text):\n%s\n\n"
|
|
356
|
+
"Order the skill names best-match-first for the task. "
|
|
357
|
+
"Reply with ONLY a JSON array of names." % (query, listing),
|
|
358
|
+
system="You rank AI coding skills by how well they fit a task.",
|
|
359
|
+
max_tokens=400)
|
|
360
|
+
order = _json_array(reply)
|
|
361
|
+
if not order:
|
|
362
|
+
return hits[:limit], "BM25 full-content"
|
|
363
|
+
by_name = {h["entry"]["name"]: h for h in hits}
|
|
364
|
+
picked = [by_name[str(n)] for n in order if str(n) in by_name]
|
|
365
|
+
rest = [h for h in hits if h["entry"]["name"]
|
|
366
|
+
not in {str(n) for n in order}]
|
|
367
|
+
return (picked + rest)[:limit], "Claude relevance"
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def search(query: str, limit: int = 10, kind: Optional[str] = None,
|
|
371
|
+
smart: bool = True,
|
|
372
|
+
entries: Optional[List[dict]] = None) -> Optional[Tuple[List[Hit], str]]:
|
|
373
|
+
"""Two-stage RAG search: BM25 retrieve -> optional LLM rerank.
|
|
374
|
+
|
|
375
|
+
Returns ``(hits, ranker_label)``, or ``None`` when no index exists yet so
|
|
376
|
+
the caller can fall back to ``catalog.search``.
|
|
377
|
+
"""
|
|
378
|
+
if not ready():
|
|
379
|
+
return None
|
|
380
|
+
hits = retrieve(query, k=max(60, limit * 4), kind=kind, entries=entries)
|
|
381
|
+
if smart:
|
|
382
|
+
return rerank(query, hits, limit)
|
|
383
|
+
return hits[:limit], "BM25 full-content"
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _json_array(text):
|
|
387
|
+
m = re.search(r"\[.*\]", text or "", re.S)
|
|
388
|
+
if not m:
|
|
389
|
+
return None
|
|
390
|
+
try:
|
|
391
|
+
data = json.loads(m.group(0))
|
|
392
|
+
except json.JSONDecodeError:
|
|
393
|
+
return None
|
|
394
|
+
return data if isinstance(data, list) else None
|
|
@@ -36,6 +36,7 @@ boost_cli/core/logs.py
|
|
|
36
36
|
boost_cli/core/output.py
|
|
37
37
|
boost_cli/core/paths.py
|
|
38
38
|
boost_cli/core/policy.py
|
|
39
|
+
boost_cli/core/rag.py
|
|
39
40
|
boost_cli/core/registry.py
|
|
40
41
|
boost_cli/core/store.py
|
|
41
42
|
boost_cli/core/util.py
|
|
@@ -79,6 +80,7 @@ tests/unit/test_mutation_hardening.py
|
|
|
79
80
|
tests/unit/test_output.py
|
|
80
81
|
tests/unit/test_paths.py
|
|
81
82
|
tests/unit/test_policy.py
|
|
83
|
+
tests/unit/test_rag.py
|
|
82
84
|
tests/unit/test_registry.py
|
|
83
85
|
tests/unit/test_store.py
|
|
84
86
|
tests/unit/test_util.py
|
|
@@ -420,11 +420,11 @@
|
|
|
420
420
|
</p>
|
|
421
421
|
<div class="cta-row">
|
|
422
422
|
<a class="btn btn-grad" href="#workflow">See how it works →</a>
|
|
423
|
-
<a class="btn btn-ghost" href="#commands">Explore all
|
|
423
|
+
<a class="btn btn-ghost" href="#commands">Explore all 73 commands</a>
|
|
424
424
|
</div>
|
|
425
425
|
<div class="stats">
|
|
426
426
|
<div class="stat reveal"><b>8,400+</b><span>Discoverable skills</span></div>
|
|
427
|
-
<div class="stat reveal"><b>
|
|
427
|
+
<div class="stat reveal"><b>73</b><span>Commands · 8 groups</span></div>
|
|
428
428
|
<div class="stat reveal"><b>2</b><span>Pre-tapped registries</span></div>
|
|
429
429
|
<div class="stat reveal"><b>3</b><span>AI agents supported</span></div>
|
|
430
430
|
</div>
|
|
@@ -589,7 +589,7 @@ facilitate structured divergent thinking:
|
|
|
589
589
|
<section id="commands">
|
|
590
590
|
<div class="wrap">
|
|
591
591
|
<span class="eyebrow">What It Can Do</span>
|
|
592
|
-
<h2 class="sec-title">
|
|
592
|
+
<h2 class="sec-title">73 commands, 8 groups</h2>
|
|
593
593
|
<p class="sec-sub">From basic install/update to AI-powered authoring, governance,
|
|
594
594
|
and team rollouts. Filter live to find what you need.</p>
|
|
595
595
|
|
|
@@ -788,6 +788,7 @@ const COMMANDS = [
|
|
|
788
788
|
["snapshot","pkg","Save & restore whole skill environments"],
|
|
789
789
|
["export","pkg","Package skills as shareable zip/tar archives"],
|
|
790
790
|
["search","find","Search skills across tap registries (AI-ranked)"],
|
|
791
|
+
["reindex","find","Build/refresh the full-content search index"],
|
|
791
792
|
["discover","find","Browse & search the GitHub-wide skill discovery index"],
|
|
792
793
|
["recommend","find","Suggest skills based on your project's tech stack"],
|
|
793
794
|
["browse","find","Interactive full-screen TUI with fuzzy search"],
|
{boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_configuration.py
RENAMED
|
@@ -343,7 +343,7 @@ class TestCompletions:
|
|
|
343
343
|
if l.startswith('complete -W "'))
|
|
344
344
|
names = set(line.split('"')[1].split())
|
|
345
345
|
assert names == {n for n, _g, _m, _s in COMMANDS}
|
|
346
|
-
assert len(names) ==
|
|
346
|
+
assert len(names) == 73
|
|
347
347
|
assert "# install: boost completions bash >> ~/.bashrc" in r.out
|
|
348
348
|
|
|
349
349
|
def test_zsh_compdef_with_summaries(self, boost, sandbox):
|
|
@@ -351,14 +351,14 @@ class TestCompletions:
|
|
|
351
351
|
assert r.out.splitlines()[0] == "#compdef boost"
|
|
352
352
|
assert "'install:Install a skill from a tap registry'" in r.out
|
|
353
353
|
entries = [l for l in r.out.splitlines() if l.startswith(" '")]
|
|
354
|
-
assert len(entries) ==
|
|
354
|
+
assert len(entries) == 73
|
|
355
355
|
assert "_describe -t commands 'boost command' _boost_commands" in r.out
|
|
356
356
|
|
|
357
357
|
def test_fish_complete_lines(self, boost, sandbox):
|
|
358
358
|
r = boost("completions", "fish")
|
|
359
359
|
lines = [l for l in r.out.splitlines()
|
|
360
360
|
if l.startswith("complete -c boost -n __fish_use_subcommand -a ")]
|
|
361
|
-
assert len(lines) ==
|
|
361
|
+
assert len(lines) == 73
|
|
362
362
|
assert ("complete -c boost -n __fish_use_subcommand -a install "
|
|
363
363
|
"-d 'Install a skill from a tap registry'") in lines
|
|
364
364
|
|
|
@@ -684,6 +684,25 @@ class TestMcp:
|
|
|
684
684
|
lock = json.loads(paths.lockfile_path().read_text())
|
|
685
685
|
assert "brainstorming" in lock["skills"] # id 8 really installed
|
|
686
686
|
|
|
687
|
+
def test_boost_search_prefers_full_content_index(self, boost, tapped):
|
|
688
|
+
from boost_cli.commands import configuration
|
|
689
|
+
from boost_cli.core import rag
|
|
690
|
+
boost("reindex")
|
|
691
|
+
assert rag.ready() is True
|
|
692
|
+
text, is_err = configuration._mcp_tool(
|
|
693
|
+
"boost_search", {"query": "brainstorming"})
|
|
694
|
+
assert is_err is False
|
|
695
|
+
assert "brainstorming" in text
|
|
696
|
+
assert "(fixture-tap)" in text
|
|
697
|
+
|
|
698
|
+
def test_boost_search_indexed_no_match(self, boost, tapped):
|
|
699
|
+
from boost_cli.commands import configuration
|
|
700
|
+
boost("reindex")
|
|
701
|
+
text, is_err = configuration._mcp_tool(
|
|
702
|
+
"boost_search", {"query": "zzzznothing"})
|
|
703
|
+
assert is_err is False
|
|
704
|
+
assert "no skills match 'zzzznothing'" in text
|
|
705
|
+
|
|
687
706
|
def test_register_without_claude_prints_manual_command(self, boost, sandbox,
|
|
688
707
|
monkeypatch):
|
|
689
708
|
monkeypatch.setattr("boost_cli.commands.configuration.shutil.which",
|