boost-skill-cli 1.0.10__tar.gz → 1.0.12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/PKG-INFO +1 -1
  2. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/_version.py +2 -2
  3. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/cli.py +2 -1
  4. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/configuration.py +17 -5
  5. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/discovery.py +41 -3
  6. boost_skill_cli-1.0.12/boost_cli/core/rag.py +394 -0
  7. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_skill_cli.egg-info/PKG-INFO +1 -1
  8. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_skill_cli.egg-info/SOURCES.txt +2 -0
  9. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/index.html +4 -3
  10. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_configuration.py +22 -3
  11. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_discovery.py +38 -0
  12. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_everyday_loop.py +1 -1
  13. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/smoke.sh +1 -0
  14. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_errors_and_cli_table.py +6 -6
  15. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_frontmatter.py +103 -0
  16. boost_skill_cli-1.0.12/tests/unit/test_rag.py +824 -0
  17. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/.gitignore +0 -0
  18. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/CLAUDE.md +0 -0
  19. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/CONTRIBUTING.md +0 -0
  20. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/LICENSE +0 -0
  21. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/MANIFEST.in +0 -0
  22. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/Makefile +0 -0
  23. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/README.md +0 -0
  24. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/SECURITY.md +0 -0
  25. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost +0 -0
  26. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/__init__.py +0 -0
  27. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/__main__.py +0 -0
  28. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/__init__.py +0 -0
  29. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/info.py +0 -0
  30. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/intelligence.py +0 -0
  31. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/pkg.py +0 -0
  32. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/quality.py +0 -0
  33. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/taps.py +0 -0
  34. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/commands/team.py +0 -0
  35. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/__init__.py +0 -0
  36. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/agents.py +0 -0
  37. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/ai.py +0 -0
  38. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/catalog.py +0 -0
  39. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/config.py +0 -0
  40. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/frontmatter.py +0 -0
  41. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/gitutil.py +0 -0
  42. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/journal.py +0 -0
  43. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/lockfile.py +0 -0
  44. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/logs.py +0 -0
  45. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/output.py +0 -0
  46. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/paths.py +0 -0
  47. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/policy.py +0 -0
  48. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/registry.py +0 -0
  49. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/store.py +0 -0
  50. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/core/util.py +0 -0
  51. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/data/registries.json +0 -0
  52. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_cli/errors.py +0 -0
  53. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_skill_cli.egg-info/dependency_links.txt +0 -0
  54. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_skill_cli.egg-info/entry_points.txt +0 -0
  55. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/boost_skill_cli.egg-info/top_level.txt +0 -0
  56. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/.claude/settings.local.json +0 -0
  57. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/DEBUGGING.md +0 -0
  58. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/demo.gif +0 -0
  59. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/demo.tape +0 -0
  60. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/docs/rag-architecture.md +0 -0
  61. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/pyproject.toml +0 -0
  62. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/scripts/build_registries.py +0 -0
  63. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/scripts/mutation_gate.py +0 -0
  64. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/setup.cfg +0 -0
  65. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/conftest.py +0 -0
  66. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_info.py +0 -0
  67. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_intelligence.py +0 -0
  68. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_pkg.py +0 -0
  69. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_quality.py +0 -0
  70. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_taps.py +0 -0
  71. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/functional/test_cli_team.py +0 -0
  72. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/make_fixture.py +0 -0
  73. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_agents.py +0 -0
  74. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_ai.py +0 -0
  75. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_catalog.py +0 -0
  76. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_config.py +0 -0
  77. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_gitutil.py +0 -0
  78. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_journal.py +0 -0
  79. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_lockfile.py +0 -0
  80. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_logs.py +0 -0
  81. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_mutation_hardening.py +0 -0
  82. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_output.py +0 -0
  83. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_paths.py +0 -0
  84. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_policy.py +0 -0
  85. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_registry.py +0 -0
  86. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_store.py +0 -0
  87. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_util.py +0 -0
  88. {boost_skill_cli-1.0.10 → boost_skill_cli-1.0.12}/tests/unit/test_version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: boost-skill-cli
3
- Version: 1.0.10
3
+ Version: 1.0.12
4
4
  Summary: boost — Homebrew for AI coding skills
5
5
  Author: Jonathan Reyes
6
6
  License: GNU GENERAL PUBLIC LICENSE
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '1.0.10'
22
- __version_tuple__ = version_tuple = (1, 0, 10)
21
+ __version__ = version = '1.0.12'
22
+ __version_tuple__ = version_tuple = (1, 0, 12)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -60,8 +60,9 @@ COMMANDS = [
60
60
  ("unpin", "pkg", "pkg", "Allow a pinned skill to update again"),
61
61
  ("snapshot", "pkg", "pkg", "Save & restore whole skill environments"),
62
62
  ("export", "pkg", "pkg", "Package skills as shareable zip/tar archives"),
63
- # Discovery & Search (8)
63
+ # Discovery & Search (9)
64
64
  ("search", "find", "discovery", "Search skills across tap registries (AI-ranked)"),
65
+ ("reindex", "find", "discovery", "Build/refresh the full-content search index"),
65
66
  ("discover", "find", "discovery", "Browse & search the GitHub-wide skill discovery index"),
66
67
  ("recommend", "find", "discovery", "Suggest skills based on your project's tech stack"),
67
68
  ("browse", "find", "discovery", "Interactive full-screen TUI with fuzzy search"),
@@ -20,7 +20,7 @@ from pathlib import Path
20
20
 
21
21
  from .. import __version__
22
22
  from ..core import agents, catalog, config, frontmatter, gitutil, journal
23
- from ..core import lockfile, paths, policy, registry, store, util
23
+ from ..core import lockfile, paths, policy, rag, registry, store, util
24
24
  from ..core import output as out
25
25
  from ..errors import BoostError
26
26
 
@@ -895,11 +895,23 @@ _MCP_TOOLS = [
895
895
  def _mcp_tool(tool: str, args: dict):
896
896
  """Run one MCP tool -> (text, is_error). (None, _) for unknown tools."""
897
897
  if tool == "boost_search":
898
- hits = catalog.search(str(args.get("query", "")))[:10]
899
- if not hits:
900
- return "no skills match %r" % args.get("query", ""), False
898
+ query = str(args.get("query", ""))
899
+ rag_result = rag.search(query, limit=10)
900
+ if rag_result is not None: # full-content index is built
901
+ hits, _ranker = rag_result
902
+ if not hits:
903
+ return "no skills match %r" % query, False
904
+ return "\n".join(
905
+ "%s — %s (%s)" % (h["entry"]["name"],
906
+ h["entry"].get("description", ""),
907
+ h["entry"]["tap"])
908
+ for h in hits), False
909
+ # no index yet -> keep today's frontmatter search so nothing regresses
910
+ scored = catalog.search(query)[:10]
911
+ if not scored:
912
+ return "no skills match %r" % query, False
901
913
  return "\n".join("%s — %s (%s)" % (e["name"], e["description"], e["tap"])
902
- for e, _score in hits), False
914
+ for e, _score in scored), False
903
915
  if tool == "boost_list":
904
916
  skills = lockfile.installed()
905
917
  if not skills:
@@ -15,7 +15,7 @@ import sys
15
15
  import time
16
16
  from pathlib import Path
17
17
 
18
- from ..core import agents, ai, catalog, gitutil, journal, lockfile, paths, registry, store, util
18
+ from ..core import agents, ai, catalog, gitutil, journal, lockfile, paths, rag, registry, store, util
19
19
  from ..core import output as out
20
20
  from ..errors import BoostError
21
21
 
@@ -184,7 +184,14 @@ def cmd_search(argv):
184
184
  if not registry.list_taps():
185
185
  raise BoostError("no taps configured — nothing to search",
186
186
  hint="add the recommended registries with `boost tap --defaults`")
187
- scored = catalog.search(query)
187
+ # Prefer the full-content (RAG) index when it exists; otherwise fall back
188
+ # to the frontmatter search so nothing regresses before `boost reindex`.
189
+ use_rag = rag.ready()
190
+ if use_rag:
191
+ scored = [(h["entry"], h["score"])
192
+ for h in rag.retrieve(query, k=max(60, args.limit * 4))]
193
+ else:
194
+ scored = catalog.search(query)
188
195
  if args.as_json:
189
196
  print(json.dumps([dict(e, score=s) for e, s in scored[:args.limit]]))
190
197
  return 0
@@ -193,7 +200,7 @@ def cmd_search(argv):
193
200
  out.info(out.c("try `boost discover %s` to search all of GitHub" % query,
194
201
  out.DIM))
195
202
  return 0
196
- ranker = "heuristic relevance"
203
+ ranker = "full-content BM25" if use_rag else "heuristic relevance"
197
204
  if args.smart:
198
205
  if ai.available():
199
206
  reranked = _ai_rank(query, scored)
@@ -214,6 +221,37 @@ def cmd_search(argv):
214
221
  return 0
215
222
 
216
223
 
224
+ def cmd_reindex(argv):
225
+ """Build/refresh the full-content (RAG) search index over tapped items."""
226
+ p = argparse.ArgumentParser(
227
+ prog="boost reindex",
228
+ description="Build or refresh the full-content search index")
229
+ p.add_argument("--force", action="store_true",
230
+ help="reindex every tap, ignoring cached commits")
231
+ p.add_argument("--json", action="store_true", dest="as_json",
232
+ help="machine-readable output")
233
+ args = p.parse_args(argv)
234
+ if not registry.list_taps():
235
+ raise BoostError("no taps configured — nothing to index",
236
+ hint="add the recommended registries with `boost tap --defaults`")
237
+ stats = rag.build(force=args.force)
238
+ if args.as_json:
239
+ print(json.dumps(stats))
240
+ return 0
241
+ out.ok("indexed %d passages across %d items from %d tap%s"
242
+ % (stats["docs"], stats["entries"], stats["taps"],
243
+ "" if stats["taps"] == 1 else "s"))
244
+ if stats["reused"]:
245
+ out.info(out.c("reused %d unchanged tap%s; reindexed: %s"
246
+ % (len(stats["reused"]),
247
+ "" if len(stats["reused"]) == 1 else "s",
248
+ ", ".join(stats["reindexed"]) or "none"),
249
+ out.DIM))
250
+ journal.log("reindex", "%d passages" % stats["docs"],
251
+ entries=stats["entries"])
252
+ return 0
253
+
254
+
217
255
  def cmd_index(argv):
218
256
  """Build ~/.boost/cache/discovery.json via `gh api` code search."""
219
257
  p = argparse.ArgumentParser(
@@ -0,0 +1,394 @@
1
+ """Full-content retrieval over tapped skills/rules/workflows (RAG — Phase 1).
2
+
3
+ Pure stdlib. Where `catalog.search` scores only an item's name + one-line
4
+ description + frontmatter, this indexes the *whole* body of every item and
5
+ ranks it with BM25, then optionally reranks the shortlist through the
6
+ `core.ai` bridge. See docs/rag-architecture.md.
7
+
8
+ The index is persisted at ``~/.boost/cache/rag_index.json`` and refreshed
9
+ per-tap, keyed on the git commit each tap cache was built from, so the
10
+ expensive step — reading and chunking thousands of files — happens once and
11
+ again only when a tap actually changes.
12
+
13
+ Everything degrades: no index -> callers fall back to ``catalog.search``;
14
+ no AI -> the BM25 order is returned as-is.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ import math
20
+ import re
21
+ from collections import defaultdict
22
+ from pathlib import Path
23
+ from typing import Dict, List, Optional, Tuple
24
+
25
+ try: # TypedDict lives in typing on 3.9+, kept optional for safety
26
+ from typing import TypedDict
27
+ except ImportError: # pragma: no cover - 3.9+ always has it
28
+ TypedDict = None # type: ignore
29
+
30
+ from . import ai, catalog, frontmatter, gitutil, paths, registry
31
+
32
+ INDEX_VERSION = 1
33
+ ENGINE = "bm25"
34
+
35
+ # Chunking defaults (documented in docs/rag-architecture.md §4).
36
+ CHUNK_CHARS = 1000
37
+ OVERLAP = 150
38
+ MAX_CHUNKS = 40
39
+
40
+ # BM25 parameters (§6a). Exposed in the persisted `params` so they are
41
+ # tunable without a format change.
42
+ K1 = 1.2
43
+ B = 0.75
44
+
45
+ # A deliberately tiny stopword list — full-content indexing already dilutes
46
+ # common words, so we only drop the most frequent glue.
47
+ _STOPWORDS = frozenset({
48
+ "the", "a", "an", "and", "or", "of", "to", "for", "in", "on", "with",
49
+ "is", "are", "be", "this", "that", "it", "as", "at", "by", "from",
50
+ "your", "you", "can", "will", "if", "when", "how",
51
+ })
52
+
53
+
54
+ class Hit(TypedDict): # type: ignore[misc]
55
+ entry: dict # the live catalog entry (name, tap, kind, skill_md, ...)
56
+ score: float # BM25 relevance (higher = better)
57
+ snippet: str # best-matching passage, for display + rerank context
58
+
59
+
60
+ # --------------------------------------------------------------- tokenizing
61
+
62
+ def tokenize(text: str) -> List[str]:
63
+ """Lowercase, split on non-alphanumerics, drop stopwords and 1-char noise."""
64
+ return [t for t in re.split(r"[^a-z0-9]+", text.lower())
65
+ if len(t) >= 2 and t not in _STOPWORDS]
66
+
67
+
68
+ def chunk(text: str, size: int = CHUNK_CHARS, overlap: int = OVERLAP,
69
+ cap: int = MAX_CHUNKS) -> List[str]:
70
+ """Split a body into passage-sized chunks on paragraph boundaries.
71
+
72
+ Paragraphs are packed up to ``size`` chars; an oversized paragraph is
73
+ hard-split with ``overlap`` carried across the cut so a match spanning a
74
+ boundary is not lost. At most ``cap`` chunks are returned.
75
+ """
76
+ text = text.strip()
77
+ if not text:
78
+ return []
79
+ paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
80
+ chunks: List[str] = []
81
+ cur = ""
82
+ for p in paras:
83
+ while len(p) > size:
84
+ chunks.append(p[:size])
85
+ if len(chunks) >= cap:
86
+ return chunks
87
+ p = p[size - overlap:]
88
+ if not cur:
89
+ cur = p
90
+ elif len(cur) + 1 + len(p) <= size:
91
+ cur += "\n" + p
92
+ else:
93
+ chunks.append(cur)
94
+ if len(chunks) >= cap:
95
+ return chunks
96
+ tail = cur[-overlap:] if overlap else ""
97
+ cur = (tail + "\n" + p) if tail else p
98
+ if cur and len(chunks) < cap:
99
+ chunks.append(cur)
100
+ return chunks
101
+
102
+
103
+ # ------------------------------------------------------------ body reading
104
+
105
+ def _tap_paths() -> Dict[str, Path]:
106
+ return {t.name: t.path for t in registry.list_taps()}
107
+
108
+
109
+ def _tap_commits() -> Dict[str, str]:
110
+ """Map safe tap name -> the commit its cache was built from."""
111
+ commits: Dict[str, str] = {}
112
+ for t in registry.list_taps():
113
+ commit = ""
114
+ try:
115
+ data = json.loads(t.cache_file.read_text())
116
+ commit = str(data.get("commit") or "")
117
+ except (OSError, json.JSONDecodeError):
118
+ pass
119
+ if not commit and t.is_cloned:
120
+ commit = gitutil.head_commit(t.path) or ""
121
+ commits[t.safe_name] = commit
122
+ return commits
123
+
124
+
125
+ def read_body(entry: dict, tap_paths: Optional[Dict[str, Path]] = None) -> str:
126
+ """Return an item's searchable text: name + description + prose body.
127
+
128
+ The frontmatter is stripped (reusing ``frontmatter.parse``); the name and
129
+ description are prepended so short items keep keyword parity with the old
130
+ search. Missing files degrade to just the catalog metadata.
131
+ """
132
+ paths_map = _tap_paths() if tap_paths is None else tap_paths
133
+ header = "%s\n%s" % (entry.get("name", ""), entry.get("description", ""))
134
+ base = paths_map.get(entry.get("tap", ""))
135
+ if base is None:
136
+ base = paths.repos_dir() / str(entry.get("tap", "")).replace("/", "__")
137
+ rel = entry.get("skill_md")
138
+ if not rel:
139
+ return header.strip()
140
+ try:
141
+ text = (base / rel).read_text(encoding="utf-8", errors="replace")
142
+ except OSError:
143
+ return header.strip()
144
+ _meta, body = frontmatter.parse(text)
145
+ return ("%s\n%s" % (header, body)).strip()
146
+
147
+
148
+ # ----------------------------------------------------------------- build
149
+
150
+ def index_path() -> Path:
151
+ return paths.cache_dir() / "rag_index.json"
152
+
153
+
154
+ def _make_docs(entries: List[dict], tap_paths: Dict[str, Path]) -> List[dict]:
155
+ """Chunk + tokenize every entry into scored documents (one per chunk)."""
156
+ docs: List[dict] = []
157
+ for e in entries:
158
+ body = read_body(e, tap_paths)
159
+ pieces = chunk(body) or [body]
160
+ for ci, piece in enumerate(pieces):
161
+ tf: Dict[str, int] = defaultdict(int)
162
+ for tok in tokenize(piece):
163
+ tf[tok] += 1
164
+ if not tf:
165
+ continue
166
+ docs.append({
167
+ "n": e["name"], "t": e["tap"], "k": e.get("kind", "skill"),
168
+ "c": ci, "l": sum(tf.values()),
169
+ "snip": piece[:200].strip(),
170
+ "tf": dict(tf),
171
+ })
172
+ return docs
173
+
174
+
175
+ def _postings_to_doc_tf(raw: dict) -> Dict[int, Dict[str, int]]:
176
+ """Invert persisted postings back to per-doc term frequencies."""
177
+ doc_tf: Dict[int, Dict[str, int]] = defaultdict(dict)
178
+ for term, plist in raw.get("postings", {}).items():
179
+ for doc_id, tf in plist:
180
+ doc_tf[doc_id][term] = tf
181
+ return doc_tf
182
+
183
+
184
+ def _kept_docs(raw: dict, keep_safe: set) -> List[dict]:
185
+ """Recover cached docs (with term freqs) for taps that did not change."""
186
+ doc_tf = _postings_to_doc_tf(raw)
187
+ kept: List[dict] = []
188
+ for doc_id, meta in enumerate(raw.get("docs", [])):
189
+ if meta["t"].replace("/", "__") in keep_safe:
190
+ kept.append({**meta, "tf": doc_tf.get(doc_id, {})})
191
+ return kept
192
+
193
+
194
+ def build(entries: Optional[List[dict]] = None, force: bool = False) -> dict:
195
+ """(Re)build the full-content index, reusing unchanged taps.
196
+
197
+ Returns stats: ``{"entries", "docs", "taps", "reindexed", "reused"}``.
198
+ """
199
+ entries = catalog.all_entries() if entries is None else entries
200
+ tap_paths = _tap_paths()
201
+ commits = _tap_commits()
202
+
203
+ old = _load_raw()
204
+ reused_safe: set = set()
205
+ if old is not None and not force:
206
+ old_commits = old.get("commits", {})
207
+ for safe, commit in commits.items():
208
+ if commit and old_commits.get(safe) == commit:
209
+ reused_safe.add(safe)
210
+
211
+ fresh = [e for e in entries
212
+ if e["tap"].replace("/", "__") not in reused_safe]
213
+ docs = _make_docs(fresh, tap_paths)
214
+ if old is not None and reused_safe:
215
+ docs = _kept_docs(old, reused_safe) + docs
216
+
217
+ _save(docs, commits)
218
+ reindexed = sorted({e["tap"] for e in fresh})
219
+ return {
220
+ "entries": len({(e["name"], e["tap"]) for e in entries}),
221
+ "docs": len(docs),
222
+ "taps": len(commits),
223
+ "reindexed": reindexed,
224
+ "reused": sorted(reused_safe),
225
+ }
226
+
227
+
228
+ def _save(docs: List[dict], commits: Dict[str, str]) -> None:
229
+ postings: Dict[str, List[List[int]]] = defaultdict(list)
230
+ meta_docs: List[dict] = []
231
+ total_len = 0
232
+ for doc_id, d in enumerate(docs):
233
+ for term, tf in d["tf"].items():
234
+ postings[term].append([doc_id, tf])
235
+ total_len += d["l"]
236
+ meta_docs.append({"n": d["n"], "t": d["t"], "k": d["k"],
237
+ "c": d["c"], "l": d["l"], "snip": d["snip"]})
238
+ payload = {
239
+ "version": INDEX_VERSION,
240
+ "engine": ENGINE,
241
+ "generated": _now(),
242
+ "commits": commits,
243
+ "params": {"chunk_chars": CHUNK_CHARS, "overlap": OVERLAP,
244
+ "k1": K1, "b": B},
245
+ "stats": {"docs": len(docs),
246
+ "avg_len": (total_len / len(docs)) if docs else 0.0},
247
+ "docs": meta_docs,
248
+ "postings": {t: p for t, p in postings.items()},
249
+ }
250
+ paths.ensure_dirs()
251
+ p = index_path()
252
+ p.write_text(json.dumps(payload))
253
+ # Drop the mtime-keyed cache so a reindex is visible to an immediately
254
+ # following query even when the filesystem mtime granularity is coarse.
255
+ _CACHE.pop(str(p), None)
256
+
257
+
258
+ def _now() -> str:
259
+ from . import util
260
+ return util.now_iso()
261
+
262
+
263
+ # ----------------------------------------------------------------- query
264
+
265
+ _CACHE: Dict[str, object] = {}
266
+
267
+
268
+ def _load_raw() -> Optional[dict]:
269
+ p = index_path()
270
+ key = str(p)
271
+ try:
272
+ mtime = p.stat().st_mtime
273
+ except OSError:
274
+ _CACHE.pop(key, None)
275
+ return None
276
+ cached = _CACHE.get(key)
277
+ if isinstance(cached, tuple) and cached[0] == mtime:
278
+ return cached[1] # type: ignore[return-value]
279
+ try:
280
+ data = json.loads(p.read_text())
281
+ except (OSError, json.JSONDecodeError):
282
+ return None
283
+ if not isinstance(data, dict) or data.get("version") != INDEX_VERSION:
284
+ return None
285
+ _CACHE[key] = (mtime, data)
286
+ return data
287
+
288
+
289
+ def ready() -> bool:
290
+ """True when a usable index exists on disk."""
291
+ raw = _load_raw()
292
+ return bool(raw and raw.get("docs"))
293
+
294
+
295
+ def _bm25(query_terms: List[str], raw: dict) -> Dict[int, float]:
296
+ docs = raw["docs"]
297
+ n = len(docs)
298
+ avg = raw.get("stats", {}).get("avg_len") or 1.0
299
+ postings = raw["postings"]
300
+ scores: Dict[int, float] = defaultdict(float)
301
+ for term in set(query_terms):
302
+ plist = postings.get(term)
303
+ if not plist:
304
+ continue
305
+ df = len(plist)
306
+ idf = math.log(1 + (n - df + 0.5) / (df + 0.5))
307
+ for doc_id, tf in plist:
308
+ dl = docs[doc_id]["l"] or 1
309
+ denom = tf + K1 * (1 - B + B * dl / avg)
310
+ scores[doc_id] += idf * (tf * (K1 + 1)) / denom
311
+ return scores
312
+
313
+
314
+ def retrieve(query: str, k: int = 60, kind: Optional[str] = None,
315
+ entries: Optional[List[dict]] = None) -> List[Hit]:
316
+ """Top-k full-content hits for ``query``. ``[]`` if the index is empty."""
317
+ raw = _load_raw()
318
+ if not raw or not raw.get("docs"):
319
+ return []
320
+ terms = tokenize(query)
321
+ if not terms:
322
+ return []
323
+ entries = catalog.all_entries() if entries is None else entries
324
+ live = {(e["name"], e["tap"]): e for e in entries}
325
+ docs = raw["docs"]
326
+ best: Dict[Tuple[str, str], Tuple[float, str]] = {}
327
+ for doc_id, score in _bm25(terms, raw).items():
328
+ d = docs[doc_id]
329
+ if kind is not None and d["k"] != kind:
330
+ continue
331
+ key = (d["n"], d["t"])
332
+ if key not in live:
333
+ continue
334
+ prev = best.get(key)
335
+ if prev is None or score > prev[0]:
336
+ best[key] = (score, d["snip"])
337
+ ranked = sorted(best.items(), key=lambda kv: (-kv[1][0], kv[0][0]))
338
+ hits: List[Hit] = []
339
+ for (name, tap), (score, snip) in ranked[:k]:
340
+ hits.append({"entry": live[(name, tap)], "score": score,
341
+ "snippet": snip}) # type: ignore[typeddict-item]
342
+ return hits
343
+
344
+
345
+ def rerank(query: str, hits: List[Hit], limit: int = 10) -> Tuple[List[Hit], str]:
346
+ """Reorder the shortlist with the LLM bridge; degrade to BM25 order."""
347
+ top = hits[:max(limit, 15)]
348
+ if not ai.available():
349
+ return hits[:limit], "BM25 full-content"
350
+ listing = "\n".join(
351
+ "- %s [%s]: %s" % (h["entry"]["name"], h["entry"].get("kind", "skill"),
352
+ (h["snippet"] or h["entry"].get("description", ""))[:180])
353
+ for h in top)
354
+ reply = ai.ask(
355
+ "Task: %s\n\nCandidate skills (name, kind, best matching text):\n%s\n\n"
356
+ "Order the skill names best-match-first for the task. "
357
+ "Reply with ONLY a JSON array of names." % (query, listing),
358
+ system="You rank AI coding skills by how well they fit a task.",
359
+ max_tokens=400)
360
+ order = _json_array(reply)
361
+ if not order:
362
+ return hits[:limit], "BM25 full-content"
363
+ by_name = {h["entry"]["name"]: h for h in hits}
364
+ picked = [by_name[str(n)] for n in order if str(n) in by_name]
365
+ rest = [h for h in hits if h["entry"]["name"]
366
+ not in {str(n) for n in order}]
367
+ return (picked + rest)[:limit], "Claude relevance"
368
+
369
+
370
+ def search(query: str, limit: int = 10, kind: Optional[str] = None,
371
+ smart: bool = True,
372
+ entries: Optional[List[dict]] = None) -> Optional[Tuple[List[Hit], str]]:
373
+ """Two-stage RAG search: BM25 retrieve -> optional LLM rerank.
374
+
375
+ Returns ``(hits, ranker_label)``, or ``None`` when no index exists yet so
376
+ the caller can fall back to ``catalog.search``.
377
+ """
378
+ if not ready():
379
+ return None
380
+ hits = retrieve(query, k=max(60, limit * 4), kind=kind, entries=entries)
381
+ if smart:
382
+ return rerank(query, hits, limit)
383
+ return hits[:limit], "BM25 full-content"
384
+
385
+
386
+ def _json_array(text):
387
+ m = re.search(r"\[.*\]", text or "", re.S)
388
+ if not m:
389
+ return None
390
+ try:
391
+ data = json.loads(m.group(0))
392
+ except json.JSONDecodeError:
393
+ return None
394
+ return data if isinstance(data, list) else None
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: boost-skill-cli
3
- Version: 1.0.10
3
+ Version: 1.0.12
4
4
  Summary: boost — Homebrew for AI coding skills
5
5
  Author: Jonathan Reyes
6
6
  License: GNU GENERAL PUBLIC LICENSE
@@ -36,6 +36,7 @@ boost_cli/core/logs.py
36
36
  boost_cli/core/output.py
37
37
  boost_cli/core/paths.py
38
38
  boost_cli/core/policy.py
39
+ boost_cli/core/rag.py
39
40
  boost_cli/core/registry.py
40
41
  boost_cli/core/store.py
41
42
  boost_cli/core/util.py
@@ -79,6 +80,7 @@ tests/unit/test_mutation_hardening.py
79
80
  tests/unit/test_output.py
80
81
  tests/unit/test_paths.py
81
82
  tests/unit/test_policy.py
83
+ tests/unit/test_rag.py
82
84
  tests/unit/test_registry.py
83
85
  tests/unit/test_store.py
84
86
  tests/unit/test_util.py
@@ -420,11 +420,11 @@
420
420
  </p>
421
421
  <div class="cta-row">
422
422
  <a class="btn btn-grad" href="#workflow">See how it works →</a>
423
- <a class="btn btn-ghost" href="#commands">Explore all 72 commands</a>
423
+ <a class="btn btn-ghost" href="#commands">Explore all 73 commands</a>
424
424
  </div>
425
425
  <div class="stats">
426
426
  <div class="stat reveal"><b>8,400+</b><span>Discoverable skills</span></div>
427
- <div class="stat reveal"><b>72</b><span>Commands · 8 groups</span></div>
427
+ <div class="stat reveal"><b>73</b><span>Commands · 8 groups</span></div>
428
428
  <div class="stat reveal"><b>2</b><span>Pre-tapped registries</span></div>
429
429
  <div class="stat reveal"><b>3</b><span>AI agents supported</span></div>
430
430
  </div>
@@ -589,7 +589,7 @@ facilitate structured divergent thinking:
589
589
  <section id="commands">
590
590
  <div class="wrap">
591
591
  <span class="eyebrow">What It Can Do</span>
592
- <h2 class="sec-title">72 commands, 8 groups</h2>
592
+ <h2 class="sec-title">73 commands, 8 groups</h2>
593
593
  <p class="sec-sub">From basic install/update to AI-powered authoring, governance,
594
594
  and team rollouts. Filter live to find what you need.</p>
595
595
 
@@ -788,6 +788,7 @@ const COMMANDS = [
788
788
  ["snapshot","pkg","Save & restore whole skill environments"],
789
789
  ["export","pkg","Package skills as shareable zip/tar archives"],
790
790
  ["search","find","Search skills across tap registries (AI-ranked)"],
791
+ ["reindex","find","Build/refresh the full-content search index"],
791
792
  ["discover","find","Browse & search the GitHub-wide skill discovery index"],
792
793
  ["recommend","find","Suggest skills based on your project's tech stack"],
793
794
  ["browse","find","Interactive full-screen TUI with fuzzy search"],
@@ -343,7 +343,7 @@ class TestCompletions:
343
343
  if l.startswith('complete -W "'))
344
344
  names = set(line.split('"')[1].split())
345
345
  assert names == {n for n, _g, _m, _s in COMMANDS}
346
- assert len(names) == 72
346
+ assert len(names) == 73
347
347
  assert "# install: boost completions bash >> ~/.bashrc" in r.out
348
348
 
349
349
  def test_zsh_compdef_with_summaries(self, boost, sandbox):
@@ -351,14 +351,14 @@ class TestCompletions:
351
351
  assert r.out.splitlines()[0] == "#compdef boost"
352
352
  assert "'install:Install a skill from a tap registry'" in r.out
353
353
  entries = [l for l in r.out.splitlines() if l.startswith(" '")]
354
- assert len(entries) == 72
354
+ assert len(entries) == 73
355
355
  assert "_describe -t commands 'boost command' _boost_commands" in r.out
356
356
 
357
357
  def test_fish_complete_lines(self, boost, sandbox):
358
358
  r = boost("completions", "fish")
359
359
  lines = [l for l in r.out.splitlines()
360
360
  if l.startswith("complete -c boost -n __fish_use_subcommand -a ")]
361
- assert len(lines) == 72
361
+ assert len(lines) == 73
362
362
  assert ("complete -c boost -n __fish_use_subcommand -a install "
363
363
  "-d 'Install a skill from a tap registry'") in lines
364
364
 
@@ -684,6 +684,25 @@ class TestMcp:
684
684
  lock = json.loads(paths.lockfile_path().read_text())
685
685
  assert "brainstorming" in lock["skills"] # id 8 really installed
686
686
 
687
+ def test_boost_search_prefers_full_content_index(self, boost, tapped):
688
+ from boost_cli.commands import configuration
689
+ from boost_cli.core import rag
690
+ boost("reindex")
691
+ assert rag.ready() is True
692
+ text, is_err = configuration._mcp_tool(
693
+ "boost_search", {"query": "brainstorming"})
694
+ assert is_err is False
695
+ assert "brainstorming" in text
696
+ assert "(fixture-tap)" in text
697
+
698
+ def test_boost_search_indexed_no_match(self, boost, tapped):
699
+ from boost_cli.commands import configuration
700
+ boost("reindex")
701
+ text, is_err = configuration._mcp_tool(
702
+ "boost_search", {"query": "zzzznothing"})
703
+ assert is_err is False
704
+ assert "no skills match 'zzzznothing'" in text
705
+
687
706
  def test_register_without_claude_prints_manual_command(self, boost, sandbox,
688
707
  monkeypatch):
689
708
  monkeypatch.setattr("boost_cli.commands.configuration.shutil.which",