boost-skill-cli 1.0.18__tar.gz → 1.0.20__tar.gz

{boost_skill_cli-1.0.18 → boost_skill_cli-1.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.18
+Version: 1.0.20
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE
@@ -700,6 +700,8 @@ Classifier: Topic :: Software Development :: Build Tools
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Provides-Extra: rag
+Requires-Dist: sqlite-vec>=0.1.6; extra == "rag"
 Dynamic: license-file
 
 # boost — a package manager for AI coding skills
@@ -747,6 +749,19 @@ boost --version
 boost tap --defaults          # pull in the 5 starter registries
 ```
 
+The default install is **zero-dependency** (pure stdlib). `boost search` and the
+`boost_search` MCP tool rank results with a built-in full-content BM25 engine
+(run `boost reindex` once to build the index). For optional **dense semantic
+search**, install the `[rag]` extra and set an embeddings key — retrieval then
+embeds every skill and ranks by vector similarity, falling back to BM25 whenever
+the extra or key is absent:
+
+```bash
+pip install "boost-skill-cli[rag]"            # adds the sqlite-vec vector store
+export VOYAGE_API_KEY=...                      # or OPENAI_API_KEY
+boost reindex --dense                          # embed chunks into the vector store
+```
+
 Want the whole ecosystem instead of the starter set? boost ships a **curated
 registry catalog** — 100+ classified GitHub registries of skills, Cursor/Windsurf
 **rules**, and Claude Code **workflows** (slash commands & subagents),

{boost_skill_cli-1.0.18 → boost_skill_cli-1.0.20}/README.md

@@ -43,6 +43,19 @@ boost --version
 boost tap --defaults          # pull in the 5 starter registries
 ```
 
+The default install is **zero-dependency** (pure stdlib). `boost search` and the
+`boost_search` MCP tool rank results with a built-in full-content BM25 engine
+(run `boost reindex` once to build the index). For optional **dense semantic
+search**, install the `[rag]` extra and set an embeddings key — retrieval then
+embeds every skill and ranks by vector similarity, falling back to BM25 whenever
+the extra or key is absent:
+
+```bash
+pip install "boost-skill-cli[rag]"            # adds the sqlite-vec vector store
+export VOYAGE_API_KEY=...                      # or OPENAI_API_KEY
+boost reindex --dense                          # embed chunks into the vector store
+```
+
 Want the whole ecosystem instead of the starter set? boost ships a **curated
 registry catalog** — 100+ classified GitHub registries of skills, Cursor/Windsurf
 **rules**, and Claude Code **workflows** (slash commands & subagents),

{boost_skill_cli-1.0.18 → boost_skill_cli-1.0.20}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.18'
-__version_tuple__ = version_tuple = (1, 0, 18)
+__version__ = version = '1.0.20'
+__version_tuple__ = version_tuple = (1, 0, 20)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.18 → boost_skill_cli-1.0.20}/boost_cli/cli.py

@@ -144,17 +144,23 @@ def print_version() -> None:
     print("%s %s" % (PRODUCT, __version__))
 
 
+# Per-group accent hue — mirrors the roadmap's colored track-icons, cycling the
+# Aurora palette so each command group carries its own brand color.
+_GROUP_HUES = ("cyan", "violet", "pink", "green", "yellow")
+
+
 def print_help() -> None:
-    print(out.c(PRODUCT, out.BOLD) + " — Homebrew for AI coding skills"
+    print(out.gradient(PRODUCT) + " — Homebrew for AI coding skills"
           + out.c("  (%s · v%s)" % (TAGLINE, __version__), out.DIM))
     print()
     print(out.c("Usage:", out.BOLD) + "  boost <command> [args]   "
           + out.c("boost help <command> for details", out.DIM))
     width = max(len(n) for n, _, _, _ in COMMANDS)
-    for gkey, (_icon, title, desc) in GROUPS.items():
+    for idx, (gkey, (_icon, title, desc)) in enumerate(GROUPS.items()):
         cmds = [(n, s) for n, g, _m, s in COMMANDS if g == gkey]
+        bullet = out.aurora("●", _GROUP_HUES[idx % len(_GROUP_HUES)])
         print()
-        print(out.c("%s" % title, out.BOLD, out.YELLOW)
+        print(bullet + " " + out.c(title, out.BOLD)
               + out.c("  — %s" % desc, out.DIM))
         for n, s in cmds:
             print("  " + out.c(n.ljust(width + 2), out.CYAN) + s)

{boost_skill_cli-1.0.18 → boost_skill_cli-1.0.20}/boost_cli/commands/discovery.py

@@ -15,7 +15,7 @@ import sys
 import time
 from pathlib import Path
 
-from ..core import agents, ai, catalog, gitutil, journal, lockfile, paths, rag, registry, store, util
+from ..core import agents, ai, catalog, dense, embed, gitutil, journal, lockfile, paths, rag, registry, store, util
 from ..core import output as out
 from ..errors import BoostError
 
@@ -228,6 +228,9 @@ def cmd_reindex(argv):
         description="Build or refresh the full-content search index")
     p.add_argument("--force", action="store_true",
                    help="reindex every tap, ignoring cached commits")
+    p.add_argument("--dense", action="store_true",
+                   help="also embed chunks into the opt-in dense vector store "
+                        "(needs the `rag` extra and an embeddings API key)")
     p.add_argument("--json", action="store_true", dest="as_json",
                    help="machine-readable output")
     args = p.parse_args(argv)
@@ -235,8 +238,10 @@ def cmd_reindex(argv):
         raise BoostError("no taps configured — nothing to index",
                         hint="add the recommended registries with `boost tap --defaults`")
     stats = rag.build(force=args.force)
+    dense_stats = _reindex_dense(args.force) if args.dense else None
     if args.as_json:
-        print(json.dumps(stats))
+        print(json.dumps({"bm25": stats, "dense": dense_stats}
+                         if args.dense else stats))
         return 0
     out.ok("indexed %d passages across %d items from %d tap%s"
            % (stats["docs"], stats["entries"], stats["taps"],
@@ -247,11 +252,24 @@ def cmd_reindex(argv):
                           "" if len(stats["reused"]) == 1 else "s",
                           ", ".join(stats["reindexed"]) or "none"),
                        out.DIM))
+    if args.dense:
+        if dense_stats is None:
+            out.warn("dense index skipped — %s" % embed.fallback_note())
+        else:
+            out.ok("embedded %d passages (%s) into the dense vector store"
+                   % (dense_stats["chunks"], dense_stats["provider"]))
     journal.log("reindex", "%d passages" % stats["docs"],
                 entries=stats["entries"])
     return 0
 
 
+def _reindex_dense(force):
+    """Build the dense vector store; returns stats or None when unavailable."""
+    if not dense.have_backend() or not embed.available():
+        return None
+    return dense.build(force=force)
+
+
 def cmd_index(argv):
     """Build ~/.boost/cache/discovery.json via `gh api` code search."""
     p = argparse.ArgumentParser(

boost_skill_cli-1.0.20/boost_cli/core/dense.py

@@ -0,0 +1,267 @@
+"""Optional dense vector backend for RAG Phase 2 (the ``[rag]`` extra).
+
+Embeds every chunk (via :mod:`embed`) into a ``sqlite-vec`` virtual table and
+answers queries by cosine KNN. It needs *both* the ``[rag]`` extra (``sqlite-vec``)
+*and* an embeddings provider; when either is missing, or the persisted vectors
+were built with a different provider/model, :func:`ready` is ``False`` and
+callers fall back to the always-on BM25 engine in :mod:`rag`.
+
+The vector store lives beside the BM25 index at ``~/.boost/cache/rag_vectors.sqlite``
+and is refreshed per-tap on the git commit each tap was built from — embeddings
+are the expensive step, so an unchanged tap is never re-embedded.
+"""
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from . import catalog, embed, paths
+from .rag import Hit, chunk, read_body
+
+INDEX_VERSION = 1
+_BATCH = 128            # texts per embedding request
+_POOL = 8              # chunk over-fetch factor for KNN before per-entry reduce
+
+
+def _load():
+    """The sqlite_vec module, or None when the [rag] extra isn't installed."""
+    try:
+        import sqlite_vec  # type: ignore
+    except ImportError:
+        return None
+    return sqlite_vec
+
+
+def have_backend() -> bool:
+    return _load() is not None
+
+
+def db_path() -> Path:
+    return paths.cache_dir() / "rag_vectors.sqlite"
+
+
+def _connect() -> Optional[sqlite3.Connection]:
+    """Open the vector DB with the sqlite-vec extension loaded, or None."""
+    mod = _load()
+    if mod is None:
+        return None
+    paths.ensure_dirs()
+    con = sqlite3.connect(str(db_path()))
+    try:
+        con.enable_load_extension(True)
+        mod.load(con)
+        con.enable_load_extension(False)
+    except (AttributeError, sqlite3.OperationalError):
+        con.close()
+        return None
+    return con
+
+
+def _ensure_schema(con: sqlite3.Connection, dim: int) -> None:
+    con.execute(
+        "CREATE TABLE IF NOT EXISTS chunks (id INTEGER PRIMARY KEY AUTOINCREMENT,"
+        " name TEXT, tap TEXT, kind TEXT, cix INTEGER, snip TEXT)")
+    con.execute("CREATE INDEX IF NOT EXISTS chunks_tap ON chunks(tap)")
+    con.execute("CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT)")
+    con.execute(
+        "CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING "
+        "vec0(embedding float[%d] distance_metric=cosine)" % dim)
+
+
+def _read_meta(con: sqlite3.Connection) -> Dict[str, object]:
+    try:
+        rows = con.execute("SELECT k, v FROM meta").fetchall()
+    except sqlite3.OperationalError:
+        return {}
+    out: Dict[str, object] = {}
+    for k, v in rows:
+        try:
+            out[k] = json.loads(v)
+        except (json.JSONDecodeError, TypeError):
+            out[k] = v
+    return out
+
+
+def _write_meta(con: sqlite3.Connection, meta: Dict[str, object]) -> None:
+    con.executemany("INSERT OR REPLACE INTO meta (k, v) VALUES (?, ?)",
+                    [(k, json.dumps(v)) for k, v in meta.items()])
+
+
+def _wipe(con: sqlite3.Connection) -> None:
+    con.execute("DROP TABLE IF EXISTS chunks")
+    con.execute("DROP TABLE IF EXISTS vec_chunks")
+
+
+def _chunk_texts(entry: dict, tap_paths: Optional[Dict[str, Path]]
+                 ) -> List[str]:
+    body = read_body(entry, tap_paths)
+    return chunk(body) or [body]
+
+
+def ready() -> bool:
+    """True when a usable, provider-matched vector index exists on disk."""
+    if not have_backend() or not embed.available() or not db_path().exists():
+        return False
+    con = _connect()
+    if con is None:
+        return False
+    try:
+        meta = _read_meta(con)
+        if meta.get("version") != INDEX_VERSION:
+            return False
+        if meta.get("provider") != embed.provider():
+            return False
+        if meta.get("dim") != embed.dimension():
+            return False
+        try:
+            rows = con.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
+        except sqlite3.OperationalError:
+            return False
+        return rows > 0
+    finally:
+        con.close()
+
+
+def build(entries: Optional[List[dict]] = None,
+          force: bool = False) -> Optional[dict]:
+    """(Re)embed and store chunk vectors, reusing unchanged taps.
+
+    Returns stats, or ``None`` if the backend or an embeddings provider is
+    unavailable (nothing is written in that case).
+    """
+    from .rag import _tap_commits, _tap_paths
+    if not have_backend() or not embed.available():
+        return None
+    entries = catalog.all_entries() if entries is None else entries
+    tap_paths = _tap_paths()
+    commits = _tap_commits()
+    dim = embed.dimension()
+    prov = embed.provider()
+    mdl = embed.model()
+    con = _connect()
+    if con is None or dim is None:
+        return None
+    try:
+        meta = _read_meta(con)
+        same_backend = (meta.get("provider") == prov and meta.get("dim") == dim)
+        if force or not same_backend:
+            _wipe(con)
+        _ensure_schema(con, dim)
+
+        reused: set = set()
+        if same_backend and not force:
+            old_commits = meta.get("commits", {})
+            if isinstance(old_commits, dict):
+                for safe, commit in commits.items():
+                    if commit and old_commits.get(safe) == commit:
+                        reused.add(safe)
+
+        fresh = [e for e in entries
+                 if e["tap"].replace("/", "__") not in reused]
+        changed_taps = sorted({e["tap"] for e in fresh})
+        _delete_taps(con, changed_taps)
+
+        added = _embed_and_store(con, fresh, tap_paths)
+        _write_meta(con, {"version": INDEX_VERSION, "provider": prov,
+                          "model": mdl, "dim": dim, "commits": commits})
+        con.commit()
+        total = con.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
+        return {
+            "entries": len({(e["name"], e["tap"]) for e in entries}),
+            "chunks": total,
+            "added": added,
+            "taps": len(commits),
+            "reindexed": changed_taps,
+            "reused": sorted(reused),
+            "provider": prov,
+            "model": mdl,
+        }
+    finally:
+        con.close()
+
+
+def _delete_taps(con: sqlite3.Connection, taps: List[str]) -> None:
+    for tap in taps:
+        ids = [r[0] for r in
+               con.execute("SELECT id FROM chunks WHERE tap = ?", (tap,))]
+        for i in ids:
+            con.execute("DELETE FROM vec_chunks WHERE rowid = ?", (i,))
+        con.execute("DELETE FROM chunks WHERE tap = ?", (tap,))
+
+
+def _embed_and_store(con: sqlite3.Connection, entries: List[dict],
+                     tap_paths: Optional[Dict[str, Path]]) -> int:
+    mod = _load()
+    rows: List[Tuple[dict, int, str]] = []   # (entry, chunk_ix, text)
+    for e in entries:
+        for ci, text in enumerate(_chunk_texts(e, tap_paths)):
+            rows.append((e, ci, text))
+    added = 0
+    for start in range(0, len(rows), _BATCH):
+        batch = rows[start:start + _BATCH]
+        vecs = embed.embed([t for _e, _c, t in batch], input_type="document")
+        if not vecs or len(vecs) != len(batch):
+            continue
+        for (e, ci, text), vec in zip(batch, vecs):
+            cur = con.execute(
+                "INSERT INTO chunks (name, tap, kind, cix, snip) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (e["name"], e["tap"], e.get("kind", "skill"), ci,
+                 text[:200].strip()))
+            con.execute("INSERT INTO vec_chunks (rowid, embedding) VALUES (?, ?)",
+                        (cur.lastrowid, mod.serialize_float32(vec)))
+            added += 1
+    return added
+
+
+def retrieve(query: str, k: int = 60, kind: Optional[str] = None,
+             entries: Optional[List[dict]] = None) -> Optional[List[Hit]]:
+    """Top-k dense hits for ``query``, or None to signal 'fall back to BM25'."""
+    if not ready():
+        return None
+    qv = embed.embed([query], input_type="query")
+    if not qv:
+        return None
+    mod = _load()
+    con = _connect()
+    if con is None:
+        return None
+    try:
+        pool = max(k * _POOL, 200)
+        knn = con.execute(
+            "SELECT rowid, distance FROM vec_chunks "
+            "WHERE embedding MATCH ? ORDER BY distance LIMIT ?",
+            (mod.serialize_float32(qv[0]), pool)).fetchall()
+        if not knn:
+            return []
+        by_id = {r[0]: (r[1], r[2], r[3], r[4]) for r in con.execute(
+            "SELECT id, name, tap, kind, snip FROM chunks WHERE id IN (%s)"
+            % ",".join("?" * len(knn)), [rid for rid, _d in knn])}
+    finally:
+        con.close()
+
+    entries = catalog.all_entries() if entries is None else entries
+    live = {(e["name"], e["tap"]): e for e in entries}
+    best: Dict[Tuple[str, str], Tuple[float, str]] = {}
+    for rid, dist in knn:
+        meta = by_id.get(rid)
+        if meta is None:
+            continue
+        name, tap, ckind, snip = meta
+        if kind is not None and ckind != kind:
+            continue
+        key = (name, tap)
+        if key not in live:
+            continue
+        score = 1.0 - dist                       # cosine distance -> similarity
+        prev = best.get(key)
+        if prev is None or score > prev[0]:
+            best[key] = (score, snip)
+    ranked = sorted(best.items(), key=lambda kv: (-kv[1][0], kv[0][0]))
+    hits: List[Hit] = []
+    for (name, tap), (score, snip) in ranked[:k]:
+        hits.append({"entry": live[(name, tap)], "score": score,
+                     "snippet": snip})  # type: ignore[typeddict-item]
+    return hits

boost_skill_cli-1.0.20/boost_cli/core/embed.py

@@ -0,0 +1,122 @@
+"""Optional dense-embedding bridge for RAG Phase 2.
+
+Strategy mirrors ``core.ai``: prefer Voyage AI when ``VOYAGE_API_KEY`` is set,
+else OpenAI when ``OPENAI_API_KEY`` is set, else return ``None`` so every caller
+degrades to the always-on BM25 engine. The embeddings themselves come from an
+HTTP API over ``urllib`` — pure stdlib, no client library. Anthropic exposes no
+embeddings endpoint, hence Voyage (its recommended partner) and OpenAI.
+
+``BOOST_NO_EMBED=1`` is a hard kill-switch, matching ``BOOST_NO_AI``.
+"""
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.request
+from typing import List, Optional
+
+VOYAGE_URL = "https://api.voyageai.com/v1/embeddings"
+OPENAI_URL = "https://api.openai.com/v1/embeddings"
+
+# model -> output dimension. Stored in the vector index so a provider/model
+# switch is detected and forces a clean rebuild instead of mixing spaces.
+VOYAGE_MODEL = "voyage-3"
+OPENAI_MODEL = "text-embedding-3-small"
+_DIMS = {VOYAGE_MODEL: 1024, OPENAI_MODEL: 1536}
+
+
+def enabled() -> bool:
+    return not os.environ.get("BOOST_NO_EMBED")
+
+
+def provider() -> Optional[str]:
+    """The active provider name, preferring Voyage, or None when unconfigured."""
+    if not enabled():
+        return None
+    if os.environ.get("VOYAGE_API_KEY"):
+        return "voyage"
+    if os.environ.get("OPENAI_API_KEY"):
+        return "openai"
+    return None
+
+
+def available() -> bool:
+    return provider() is not None
+
+
+def model() -> Optional[str]:
+    p = provider()
+    if p == "voyage":
+        return VOYAGE_MODEL
+    if p == "openai":
+        return OPENAI_MODEL
+    return None
+
+
+def dimension() -> Optional[int]:
+    m = model()
+    return _DIMS.get(m) if m is not None else None
+
+
+def fallback_note() -> str:
+    return ("dense search needs the `rag` extra and VOYAGE_API_KEY or "
+            "OPENAI_API_KEY — using the BM25 full-content engine")
+
+
+def embed(texts: List[str], input_type: Optional[str] = None,
+          timeout: int = 60) -> Optional[List[List[float]]]:
+    """Embed a batch of texts.
+
+    Returns one vector per input, ``[]`` for an empty batch, or ``None`` when no
+    provider is configured or the call fails (so callers degrade cleanly).
+    ``input_type`` ("query"/"document") is honored by Voyage and ignored by
+    OpenAI.
+    """
+    p = provider()
+    if p is None:
+        return None
+    if not texts:
+        return []
+    if p == "voyage":
+        body = {"input": list(texts), "model": VOYAGE_MODEL}
+        if input_type:
+            body["input_type"] = input_type
+        return _vectors(_post(VOYAGE_URL, os.environ["VOYAGE_API_KEY"],
+                              body, timeout), len(texts))
+    body = {"input": list(texts), "model": OPENAI_MODEL}
+    return _vectors(_post(OPENAI_URL, os.environ["OPENAI_API_KEY"],
+                          body, timeout), len(texts))
+
+
+def _post(url: str, key: str, payload: dict, timeout: int) -> Optional[dict]:
+    data = json.dumps(payload).encode("utf-8")
+    req = urllib.request.Request(url, data=data, headers={
+        "Authorization": "Bearer %s" % key,
+        "Content-Type": "application/json",
+    })
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read())
+    except (urllib.error.URLError, OSError, ValueError):
+        return None
+
+
+def _vectors(obj: Optional[dict], n: int) -> Optional[List[List[float]]]:
+    """Pull ``data[].embedding`` out of an OpenAI-shaped response.
+
+    Both Voyage and OpenAI return ``{"data": [{"embedding": [...]}, ...]}``.
+    Returns None unless exactly ``n`` numeric vectors come back.
+    """
+    if not isinstance(obj, dict):
+        return None
+    data = obj.get("data")
+    if not isinstance(data, list) or len(data) != n:
+        return None
+    out: List[List[float]] = []
+    for item in data:
+        vec = item.get("embedding") if isinstance(item, dict) else None
+        if not isinstance(vec, list) or not vec:
+            return None
+        out.append([float(x) for x in vec])
+    return out