boost-skill-cli 1.0.307__tar.gz → 1.0.309__tar.gz

{boost_skill_cli-1.0.307 → boost_skill_cli-1.0.309}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.307
+Version: 1.0.309
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE
@@ -702,6 +702,8 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Provides-Extra: rag
 Requires-Dist: sqlite-vec>=0.1.6; extra == "rag"
+Requires-Dist: onnxruntime>=1.17; extra == "rag"
+Requires-Dist: tokenizers>=0.15; extra == "rag"
 Provides-Extra: eval
 Requires-Dist: ranx>=0.3.20; extra == "eval"
 Requires-Dist: ragas<0.3,>=0.2; extra == "eval"

{boost_skill_cli-1.0.307 → boost_skill_cli-1.0.309}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.307'
-__version_tuple__ = version_tuple = (1, 0, 307)
+__version__ = version = '1.0.309'
+__version_tuple__ = version_tuple = (1, 0, 309)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.307 → boost_skill_cli-1.0.309}/boost_cli/commands/quality.py

@@ -365,9 +365,25 @@ def cmd_doctor(argv):
     if not rotation:
         bad("journal is overdue for rotation — run `boost heal`")
 
+    # Which search engine will actually answer a query. Dense retrieval needs
+    # three things to line up and every one of them fails silently, so doctor
+    # is where the answer belongs — `search` only ever reports the engine that
+    # already ran, never that a configured one never got the chance.
+    _report_search_engine(bad)
+
     lp = logs.log_path()
     if lp.exists():
-        out.ok("diagnostic log at %s" % _tilde(lp))
+        # Existence is not health: a log the process cannot open makes every
+        # invocation print a PermissionError traceback from the logging module
+        # while this check happily reported a ✓ for the same file. Diagnostics
+        # are the first thing consulted when something else breaks, so a log
+        # that silently accepts nothing has to read as a fault.
+        if os.access(str(lp), os.W_OK):
+            out.ok("diagnostic log at %s" % _tilde(lp))
+        else:
+            bad("diagnostic log %s is not writable — every invocation is "
+                "failing to record; fix its permissions (chmod u+w)"
+                % _tilde(lp))
     crashes = sorted(paths.logs_dir().glob("crash-*.log")) \
         if paths.logs_dir().is_dir() else []
     if crashes:
@@ -391,6 +407,64 @@ def cmd_doctor(argv):
     return 1 if issues else 0
 
 
+# Why dense retrieval isn't serving, keyed by `dense.status()["reason"]`. Each
+# names the ONE next action; the reason order in `status()` guarantees only the
+# first missing link is ever reported, so these never chain.
+_DENSE_FIX = {
+    "no-backend": "install the extra: `pip install 'boost-skill-cli[rag]'`",
+    # Names the keyless remedy first: since the [rag] extra carries a local
+    # embedding model, an API key is the quality ceiling, not the entry fee.
+    # This reason now means "no key AND no local backend", which in practice is
+    # a partial install or BOOST_NO_EMBED.
+    "no-key": ("reinstall the extra: `pip install 'boost-skill-cli[rag]'` "
+               "(or set VOYAGE_API_KEY / OPENAI_API_KEY for a larger model)"),
+    "no-store": "build it: `boost reindex --dense`",
+    "version-changed": "rebuild it: `boost reindex --dense --force`",
+    "provider-changed": "rebuild it: `boost reindex --dense --force`",
+    "model-changed": "rebuild it: `boost reindex --dense --force`",
+    "dim-changed": "rebuild it: `boost reindex --dense --force`",
+    "empty": "rebuild it: `boost reindex --dense --force`",
+}
+
+
+def _report_search_engine(bad) -> None:
+    """Report the engine `boost search` will use, and why it isn't the best one.
+
+    Only a *degraded* dense tier counts against doctor's exit code: BM25 is the
+    documented default and most users never opt in, so an unconfigured tier is
+    healthy. Vectors already on disk that have stopped serving are not.
+    """
+    # Local import: the dense/embedding engines are opt-in and stay out of
+    # startup for every other command (scripts/import_budget.py enforces it).
+    from ..core import dense
+    st = dense.status()
+
+    if st["ready"]:
+        out.ok("semantic search active — %s %s (%d-d), %d chunk%s across %d tap%s"
+               % (st["provider"], st["model"], st["dim"] or 0,
+                  st["chunks"], _s(st["chunks"]), st["taps"], _s(st["taps"])))
+        return
+
+    fix = _DENSE_FIX.get(str(st["reason"]), "see `boost reindex --dense`")
+    if st["degraded"]:
+        # The store was built and is now dead weight: say what it holds, what
+        # changed, and that search has silently been on BM25 the whole time.
+        built = st["built_model"] or st["built_provider"] or "an older build"
+        detail = "built with %s" % built
+        if st["reason"] == "model-changed":
+            detail += ", live key is %s" % st["model"]
+        elif st["reason"] == "provider-changed":
+            detail += ", live key is %s" % st["provider"]
+        elif st["reason"] == "empty":
+            detail += " but holds no vectors"
+        bad("semantic search silently off — %d-chunk vector store %s; "
+            "searches are using BM25. %s" % (st["chunks"], detail, fix))
+        return
+
+    out.info("semantic search not configured — using the full-content BM25 "
+             "engine (%s)" % fix)
+
+
 def _print_skipped(skipped: List[dict]) -> None:
     """Note the rule/workflow entries `lint` passed over (they have no SKILL.md)."""
     if not skipped:

{boost_skill_cli-1.0.307 → boost_skill_cli-1.0.309}/boost_cli/core/dense.py

@@ -102,6 +102,106 @@ def _chunk_texts(entry: dict, tap_paths: Optional[Dict[str, Path]]
     return chunk(body) or [body]
 
 
+def _recorded_meta() -> dict:
+    """What an existing store says it was built with, read WITHOUT sqlite-vec.
+
+    Deliberately not routed through :func:`_connect`, which loads the extension
+    and returns ``None`` without it: the most useful diagnostic line ("your
+    store was built with voyage-3") would then vanish in exactly the case that
+    needs it — a user who dropped the ``[rag]`` extra and wants to know why
+    dense went quiet. ``meta`` is a plain table, so plain sqlite3 can read it.
+    """
+    p = db_path()
+    if not p.exists():
+        return {}
+    try:
+        con = sqlite3.connect(str(p))
+    except sqlite3.Error:
+        return {}
+    try:
+        meta = _read_meta(con)
+        try:
+            meta["_chunks"] = con.execute(
+                "SELECT COUNT(*) FROM chunks").fetchone()[0]
+        except sqlite3.DatabaseError:
+            meta["_chunks"] = 0
+        return meta
+    except sqlite3.DatabaseError:
+        # A truncated or non-sqlite file: report "no store", never raise into
+        # a health check whose whole job is to survive a broken install.
+        return {}
+    finally:
+        con.close()
+
+
+def status() -> dict:
+    """Why dense retrieval is, or is not, serving queries.
+
+    Pure inspection — never builds, never embeds, never needs the extra. Three
+    independent things must line up (the ``[rag]`` extra, an embeddings key, a
+    built store whose space matches the live one) and every one of them fails
+    *silently* today, because :func:`rag.retrieve_any` floors to BM25 and
+    returns. ``reason`` names which link is missing so a caller can say so.
+
+    ``degraded`` is the load-bearing distinction: a user who never configured
+    dense search is *healthy* (BM25 is the documented default), while a user
+    who did all three steps and is still on BM25 has a real problem no other
+    surface reports. Only the latter should move an exit code.
+    """
+    have_be = have_backend()
+    prov, mdl, dim = embed.provider(), embed.model(), embed.dimension()
+    meta = _recorded_meta()
+    b_prov = meta.get("provider")
+    b_mdl = meta.get("model")
+    b_dim = meta.get("dim")
+    b_ver = meta.get("version")
+    commits = meta.get("commits")
+    chunks = int(meta.get("_chunks") or 0)
+    store_exists = bool(meta)
+
+    # Order matters: report the *first* missing link, so the message names the
+    # next action rather than a downstream symptom of the same gap.
+    if not have_be:
+        reason: Optional[str] = "no-backend"
+    elif prov is None:
+        reason = "no-key"
+    elif not store_exists:
+        reason = "no-store"
+    elif b_ver != INDEX_VERSION:
+        reason = "version-changed"
+    elif b_prov != prov:
+        reason = "provider-changed"
+    elif b_mdl != mdl:
+        reason = "model-changed"
+    elif b_dim != dim:
+        reason = "dim-changed"
+    elif chunks <= 0:
+        reason = "empty"
+    else:
+        reason = None
+
+    # Unconfigured is not degraded; a built-but-unusable store is. Vectors on
+    # disk are the proof of intent — the user paid to embed them, so anything
+    # that stops them serving (a dropped extra, an unset key, a changed model)
+    # is a real fault. Without a store there is nothing to have regressed:
+    # "no-store" is an unfinished setup, and it is the one reason that implies
+    # store_exists is False, so this single clause covers every case.
+    degraded = store_exists and reason is not None
+
+    return {
+        "backend": have_be,
+        "provider": prov, "model": mdl, "dim": dim,
+        "store_exists": store_exists,
+        "built_provider": b_prov, "built_model": b_mdl,
+        "built_dim": b_dim, "built_version": b_ver,
+        "chunks": chunks,
+        "taps": len(commits) if isinstance(commits, dict) else 0,
+        "ready": reason is None,
+        "reason": reason,
+        "degraded": degraded,
+    }
+
+
 def ready() -> bool:
     """True when a usable, provider-matched vector index exists on disk."""
     if not have_backend() or not embed.available() or not db_path().exists():

boost_skill_cli-1.0.309/boost_cli/core/embed.py

@@ -0,0 +1,270 @@
+"""Optional dense-embedding bridge for RAG Phase 2.
+
+Strategy mirrors ``core.ai``: prefer Voyage AI when ``VOYAGE_API_KEY`` is set,
+else OpenAI when ``OPENAI_API_KEY`` is set, else a **local** ONNX model when the
+``[rag]`` extra is installed, else return ``None`` so every caller degrades to
+the always-on BM25 engine. The API embeddings come over ``urllib`` — pure
+stdlib, no client library. Anthropic exposes no embeddings endpoint, hence
+Voyage (its recommended partner) and OpenAI.
+
+Why a local provider exists
+---------------------------
+Without one, dense search needed an account and a key, so in practice every
+keyless user got BM25 — and BM25 answers "my app is slow" with bioinformatics
+packages, because nothing in that query shares vocabulary with the docs. The
+vector *store* was never the gated part: vectors already live on the user's own
+machine in sqlite-vec, keyed on each tap's commit. The only API-bound step is
+turning text into vectors.
+
+That also rules out the obvious shortcut of shipping a prebuilt index instead:
+answering a query means embedding *the query*, so a downloaded index is useless
+without something local to embed with. A local model is the prerequisite, not
+an alternative.
+
+The chain order is load-bearing. Local is tried **last**, so a user who has
+configured Voyage keeps voyage-4 rather than being silently downgraded to a
+384-dim local model. And because ``dense.py`` rebuilds whenever ``model`` or
+``dim`` changes, switching providers can never mix two embedding spaces in one
+index.
+
+``BOOST_NO_EMBED=1`` is a hard kill-switch for all three, matching
+``BOOST_NO_AI``.
+"""
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import urllib.error
+import urllib.request
+from typing import Any, List, Optional
+
+from . import nethttp
+
+VOYAGE_URL = "https://api.voyageai.com/v1/embeddings"
+OPENAI_URL = "https://api.openai.com/v1/embeddings"
+
+# model -> output dimension. Stored in the vector index so a provider/model
+# switch is detected and forces a clean rebuild instead of mixing spaces.
+# voyage-4 over voyage-3: same 1024-d default and same $/token, but the
+# voyage-4 generation carries a 200M-token complimentary allowance where
+# voyage-3 has none — so a full index build of a large corpus is free.
+VOYAGE_MODEL = "voyage-4"
+OPENAI_MODEL = "text-embedding-3-small"
+
+# The keyless path. Small on purpose: skill docs are short and keyword-dense, so
+# a 384-dim model closes most of the gap to a paid API and runs on CPU. Named
+# with its full hub id because that is what ends up recorded in the index
+# metadata — a bare "bge-small-en-v1.5" would collide with any other vendor's
+# rebuild of it, and a rebuild produces different vectors. See core.localembed
+# for the pinned revision and the (133 MB, not 30 MB) download it implies.
+LOCAL_MODEL = "BAAI/bge-small-en-v1.5"
+LOCAL_DIM = 384
+
+_DIMS = {VOYAGE_MODEL: 1024, OPENAI_MODEL: 1536, LOCAL_MODEL: LOCAL_DIM}
+
+# Cached backend class and model instance. Loading an ONNX model is expensive
+# enough that doing it per query would make the local path unusable, and the
+# import itself is retried-on-miss rather than repeated: `provider()` runs on
+# effectively every retrieval call.
+_BACKEND_MISSING = object()
+# Deliberately untyped: the backend is core.localembed, imported lazily, so
+# naming its type here would defeat the point of the lazy import.
+_backend_cache: Any = None
+_model_instance: Any = None
+
+
+def enabled() -> bool:
+    """True unless the ``BOOST_NO_EMBED`` kill-switch is set."""
+    return not os.environ.get("BOOST_NO_EMBED")
+
+
+def _load_backend():
+    """The local embedding class, or None when the ``[rag]`` extra is absent.
+
+    ``localembed`` keeps its own imports lazy for the import budget; this is the
+    seam the tests stub, so the unit suite passes with the extra uninstalled.
+    """
+    from . import localembed
+    return localembed if localembed.available() else None
+
+
+def _backend() -> Any:
+    """``_load_backend`` memoised, including the negative result."""
+    global _backend_cache
+    if _backend_cache is None:
+        _backend_cache = _load_backend() or _BACKEND_MISSING
+    return None if _backend_cache is _BACKEND_MISSING else _backend_cache
+
+
+def reset_local_cache() -> None:
+    """Drop the cached backend and model. For tests, and for a provider switch."""
+    global _backend_cache, _model_instance
+    _backend_cache = None
+    _model_instance = None
+    # Nothing to reset when the extra is absent.
+    with contextlib.suppress(Exception):
+        from . import localembed
+        localembed.reset()
+
+
+def local_available() -> bool:
+    """True when the local embedding backend can be imported."""
+    return _backend() is not None
+
+
+def provider() -> Optional[str]:
+    """The active provider, preferring a configured key, or None when nothing works.
+
+    Order matters: local comes **last** so configuring Voyage keeps voyage-4.
+    A fallback that preempted a paid key would silently downgrade every existing
+    keyed install to a smaller model.
+    """
+    if not enabled():
+        return None
+    if os.environ.get("VOYAGE_API_KEY"):
+        return "voyage"
+    if os.environ.get("OPENAI_API_KEY"):
+        return "openai"
+    if local_available():
+        return "local"
+    return None
+
+
+def available() -> bool:
+    """True when embeddings are enabled and a provider key is present."""
+    return provider() is not None
+
+
+def model() -> Optional[str]:
+    """The active provider's model name, or None when unconfigured."""
+    p = provider()
+    if p == "voyage":
+        return VOYAGE_MODEL
+    if p == "openai":
+        return OPENAI_MODEL
+    if p == "local":
+        return LOCAL_MODEL
+    return None
+
+
+def dimension() -> Optional[int]:
+    """Output vector dimension of the active model, or None when
+
+    unconfigured. Stored in the index so a model switch forces a rebuild.
+    """
+    m = model()
+    return _DIMS.get(m) if m is not None else None
+
+
+def fallback_note() -> str:
+    """The one-line hint shown when dense search degrades to BM25.
+
+    No longer asks for a key: with a local model in the ``[rag]`` extra the
+    extra alone is enough, and telling a user they need an API account when
+    they do not is worse than saying nothing.
+    """
+    if not enabled():
+        return "dense search is off (BOOST_NO_EMBED) — using the BM25 engine"
+    return ("dense search needs the `[rag]` extra — `pip install "
+            "boost-skill-cli[rag]`; using the BM25 full-content engine")
+
+
+def embed(texts: List[str], input_type: Optional[str] = None,
+          timeout: int = 60) -> Optional[List[List[float]]]:
+    """Embed a batch of texts.
+
+    Returns one vector per input, ``[]`` for an empty batch, or ``None`` when no
+    provider is configured or the call fails (so callers degrade cleanly).
+    ``input_type`` ("query"/"document") is honored by Voyage and ignored by
+    OpenAI.
+    """
+    p = provider()
+    if p is None:
+        return None
+    if not texts:
+        return []
+    if p == "local":
+        return _embed_local(texts)
+    if p == "voyage":
+        body = {"input": texts.copy(), "model": VOYAGE_MODEL}
+        if input_type:
+            body["input_type"] = input_type
+        return _vectors(_post(VOYAGE_URL, os.environ["VOYAGE_API_KEY"],
+                              body, timeout), len(texts))
+    body = {"input": texts.copy(), "model": OPENAI_MODEL}
+    return _vectors(_post(OPENAI_URL, os.environ["OPENAI_API_KEY"],
+                          body, timeout), len(texts))
+
+
+def _local_model():
+    """The loaded local model, or None if it cannot be built.
+
+    First use downloads the pinned weights and caches them under
+    ``~/.boost/cache/models``, so this can fail on a machine with no network.
+    That is a degrade, not an error: the caller floors to BM25 exactly as it
+    does when no key is set.
+    """
+    return _backend()
+
+
+def _embed_local(texts: List[str]) -> Optional[List[List[float]]]:
+    """Embed locally. None on any failure, so the caller degrades to BM25.
+
+    Validated as strictly as the API path: a short batch would misalign vectors
+    against their chunks — every hit past the gap pointing at the wrong skill —
+    and a wrong-width row would either error inside sqlite-vec or quietly
+    corrupt the space, so both are rejected rather than stored.
+    """
+    backend = _local_model()
+    if backend is None:
+        return None
+    try:
+        rows = backend.encode(texts.copy())
+    except Exception:      # inference failure degrades, never raises
+        return None
+    if rows is None or len(rows) != len(texts):
+        return None
+    out: List[List[float]] = []
+    for row in rows:
+        try:
+            vec = [float(x) for x in row]
+        except (TypeError, ValueError):
+            return None
+        if len(vec) != LOCAL_DIM:
+            return None
+        out.append(vec)
+    return out
+
+
+def _post(url: str, key: str, payload: dict, timeout: int) -> Optional[dict]:
+    data = json.dumps(payload).encode("utf-8")
+    req = urllib.request.Request(url, data=data, headers={  # noqa: S310  url is a hardcoded VOYAGE_URL/OPENAI_URL constant
+        "Authorization": "Bearer %s" % key,
+        "Content-Type": "application/json",
+    })
+    try:
+        with nethttp.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read())
+    except (urllib.error.URLError, OSError, ValueError):
+        return None
+
+
+def _vectors(obj: Optional[dict], n: int) -> Optional[List[List[float]]]:
+    """Pull ``data[].embedding`` out of an OpenAI-shaped response.
+
+    Both Voyage and OpenAI return ``{"data": [{"embedding": [...]}, ...]}``.
+    Returns None unless exactly ``n`` numeric vectors come back.
+    """
+    if not isinstance(obj, dict):
+        return None
+    data = obj.get("data")
+    if not isinstance(data, list) or len(data) != n:
+        return None
+    out: List[List[float]] = []
+    for item in data:
+        vec = item.get("embedding") if isinstance(item, dict) else None
+        if not isinstance(vec, list) or not vec:
+            return None
+        out.append([float(x) for x in vec])
+    return out