cli-tools-kit 0.6.0__py3-none-any.whl

cli_tools_kit/taxonomy/corpus.py

@@ -0,0 +1,268 @@
+"""Build one text document per installable tool.
+
+A tool's document is its CLAUDE.md plus its README.md. When it has neither,
+the tool is asked for its --advertise metadata and the name/desc from that is
+used instead.
+
+:func:`corpus_fingerprint` hashes those same inputs without running anything,
+so a caller can tell whether a stored grouping is still current.
+"""
+
+import hashlib
+import json
+import os
+import subprocess
+import sys
+from typing import Dict, List, Tuple
+
+MAX_CHARS = 6000
+BLURB_MAX_CHARS = 700
+BLURB_PARAGRAPHS = 2
+BLURB_HEADINGS = 8
+DOC_FILES = ("CLAUDE.md", "README.md")
+ADVERTISE_TIMEOUT = 5
+SKIP_PREFIXES = ("_", ".")
+SKIP_NAMES = {"dev"}
+
+
+def tool_dirs(root: str) -> List[str]:
+    """Names of the top-level directories that are installable tools."""
+    found = []
+    for name in sorted(os.listdir(root)):
+        if name.startswith(SKIP_PREFIXES) or name in SKIP_NAMES:
+            continue
+        path = os.path.join(root, name)
+        if not os.path.isdir(path):
+            continue
+        if os.path.isfile(os.path.join(path, "main.py")) and os.path.isfile(
+            os.path.join(path, "requirements.txt")
+        ):
+            found.append(name)
+    return found
+
+
+def _advertise_entries(tool_path: str) -> List[dict]:
+    """Run the tool's --advertise and return its metadata entries."""
+    main_py = os.path.join(tool_path, "main.py")
+    try:
+        proc = subprocess.run(
+            [sys.executable, main_py, "--advertise"],
+            capture_output=True,
+            text=True,
+            timeout=ADVERTISE_TIMEOUT,
+            cwd=tool_path,
+        )
+    except (subprocess.SubprocessError, OSError):
+        return []
+
+    for line in reversed((proc.stdout or "").strip().splitlines()):
+        line = line.strip()
+        if not line.startswith(("[", "{")):
+            continue
+        try:
+            data = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        entries = data if isinstance(data, list) else [data]
+        return [e for e in entries if isinstance(e, dict)]
+    return []
+
+
+def _advertise_text(tool_path: str) -> str:
+    """The tool's advertised names, descriptions and taxonomy words."""
+    parts = []
+    for entry in _advertise_entries(tool_path):
+        for key in ("name", "desc", "capability", "domain"):
+            value = entry.get(key)
+            if value:
+                parts.append(str(value))
+    return "\n".join(parts)
+
+
+def tool_capabilities(root: str) -> Dict[str, str]:
+    """Map each tool to its advertised capability word, "" when it has none.
+
+    The capability is hand-written, validated against CAPABILITY_VOCAB, and
+    needs no network to read — which is what makes it usable as the last-resort
+    grouping on a host with no API key.
+    """
+    caps = {}
+    for name in tool_dirs(root):
+        capability = ""
+        for entry in _advertise_entries(os.path.join(root, name)):
+            value = str(entry.get("capability") or "").strip()
+            if value:
+                capability = value
+                break
+        caps[name] = capability
+    return caps
+
+
+def tool_documents(root: str) -> Dict[str, str]:
+    """Map each tool directory name to its document text."""
+    docs = {}
+    for name in tool_dirs(root):
+        path = os.path.join(root, name)
+        chunks = [name.replace("_", " ")]
+        for doc_name in DOC_FILES:
+            doc_path = os.path.join(path, doc_name)
+            if os.path.isfile(doc_path):
+                try:
+                    with open(doc_path, encoding="utf-8", errors="replace") as fh:
+                        chunks.append(fh.read())
+                except OSError:
+                    pass
+        if len(chunks) == 1:
+            advertised = _advertise_text(path)
+            if advertised:
+                chunks.append(advertised)
+        docs[name] = "\n\n".join(chunks)[:MAX_CHARS]
+    return docs
+
+
+def _file_digest(path: str) -> str:
+    """sha256 of a file's bytes, "" when it cannot be read."""
+    try:
+        with open(path, "rb") as fh:
+            return hashlib.sha256(fh.read()).hexdigest()
+    except OSError:
+        return ""
+
+
+def corpus_fingerprint(root: str) -> str:
+    """Hash the inputs :func:`tool_documents` reads, without running them.
+
+    Covers the set of tool directories and the contents of every CLAUDE.md and
+    README.md. A tool with neither is represented by its main.py instead, since
+    that is what the --advertise fallback reads. Content hashes rather than
+    mtimes: a Syncthing checkout restamps mtimes fleet-wide without changing a
+    byte.
+    """
+    digest = hashlib.sha256()
+    for name in tool_dirs(root):
+        path = os.path.join(root, name)
+        digest.update(name.encode())
+        documented = False
+        for doc_name in DOC_FILES:
+            doc_path = os.path.join(path, doc_name)
+            if os.path.isfile(doc_path):
+                documented = True
+                digest.update(doc_name.encode())
+                digest.update(_file_digest(doc_path).encode())
+        if not documented:
+            digest.update(b"main.py")
+            digest.update(_file_digest(os.path.join(path, "main.py")).encode())
+    return digest.hexdigest()
+
+
+# --- condensed blurbs -----------------------------------------------------
+#
+# tool_documents() feeds an embedder, which tolerates bulk. The LLM grouping
+# wants the opposite: a short, comparable description of what each tool is for.
+# A tool's CLAUDE.md is mostly operational prose — commit policy, install
+# mechanics, fleet paths — that says nothing about its subject, and the six
+# tools with the longest docs would otherwise drown out the fourteen with none.
+
+_SKIP_LINE_PREFIXES = ("|", ">", "<!--", "<", "!", "```", "---", "===", "*Note")
+_DOC_STOP_HEADINGS = {
+    "installation", "install", "usage", "requirements", "dependencies",
+    "license", "licence", "testing", "tests", "development", "changelog",
+    "troubleshooting", "configuration", "config", "files", "layout",
+    "architecture overview", "environment variables", "cli protocol",
+}
+
+
+def _clean_markdown(text: str) -> Tuple[List[str], List[str]]:
+    """Split a markdown doc into (prose paragraphs, heading names).
+
+    Code fences, tables, block quotes, badges and raw HTML are dropped: they
+    carry syntax, not subject.
+    """
+    paragraphs: List[str] = []
+    headings: List[str] = []
+    buffer: List[str] = []
+    in_fence = False
+
+    def flush():
+        if buffer:
+            para = " ".join(buffer).strip()
+            # A paragraph of mostly punctuation or paths is not a description.
+            if len(para) >= 40 and sum(c.isalpha() for c in para) > len(para) / 2:
+                paragraphs.append(para)
+            buffer.clear()
+
+    for raw in text.splitlines():
+        line = raw.strip()
+        if line.startswith("```"):
+            in_fence = not in_fence
+            flush()
+            continue
+        if in_fence:
+            continue
+        if not line:
+            flush()
+            continue
+        if line.startswith("#"):
+            flush()
+            name = line.lstrip("#").strip().strip("*_`")
+            if name and name.lower() not in _DOC_STOP_HEADINGS:
+                headings.append(name)
+            continue
+        if line.startswith(_SKIP_LINE_PREFIXES):
+            flush()
+            continue
+        buffer.append(line.lstrip("-*+ ").strip())
+    flush()
+    return paragraphs, headings
+
+
+def _advertise_fields(tool_path: str) -> List[str]:
+    """Advertised name/desc/capability/domain lines, deduplicated in order."""
+    text = _advertise_text(tool_path)
+    seen = set()
+    fields = []
+    for line in text.splitlines():
+        line = line.strip()
+        if line and line.lower() not in seen:
+            seen.add(line.lower())
+            fields.append(line)
+    return fields
+
+
+def tool_blurb(root: str, name: str) -> str:
+    """A short, comparable description of one tool, for the LLM grouping.
+
+    Built from, in order: the advertised name/desc/capability/domain, the first
+    couple of real prose paragraphs of CLAUDE.md or README.md, and that doc's
+    section headings (which name features in very few tokens). Capped at
+    BLURB_MAX_CHARS so a heavily documented tool cannot outweigh a bare one.
+    """
+    path = os.path.join(root, name)
+    parts = [f"directory: {name}"]
+
+    fields = _advertise_fields(path)
+    if fields:
+        parts.append("advertises: " + " | ".join(fields))
+
+    for doc_name in DOC_FILES:
+        doc_path = os.path.join(path, doc_name)
+        if not os.path.isfile(doc_path):
+            continue
+        try:
+            with open(doc_path, encoding="utf-8", errors="replace") as fh:
+                text = fh.read(MAX_CHARS * 2)
+        except OSError:
+            continue
+        paragraphs, headings = _clean_markdown(text)
+        if paragraphs:
+            parts.append(" ".join(paragraphs[:BLURB_PARAGRAPHS]))
+        if headings:
+            parts.append("sections: " + "; ".join(headings[:BLURB_HEADINGS]))
+        break
+
+    return "\n".join(parts)[:BLURB_MAX_CHARS]
+
+
+def tool_blurbs(root: str) -> Dict[str, str]:
+    """One condensed blurb per tool."""
+    return {name: tool_blurb(root, name) for name in tool_dirs(root)}

cli_tools_kit/taxonomy/embedder.py

@@ -0,0 +1,170 @@
+"""Text embeddings with a local backend first and Gemini as fallback.
+
+Two backends behind one function. The local OpenAI-compatible
+``/v1/embeddings`` endpoint (LM Studio on this fleet) is tried first because it
+is free and offline; Gemini is used when it is not reachable. If neither works
+the caller gets :class:`EmbeddingUnavailable` rather than a silent empty list.
+"""
+
+import json
+import os
+import urllib.error
+import urllib.request
+from typing import List
+
+_DEFAULT_HOST = "http://localhost:11434"
+_DEFAULT_MODEL = "text-embedding-nomic-embed-text-v1.5"
+_TIMEOUT = 60.0
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = os.path.dirname(os.path.dirname(_HERE))
+
+
+class EmbeddingUnavailable(RuntimeError):
+    """No embedding backend could produce vectors."""
+
+
+def _load_env() -> None:
+    """Load the repo-root .env so GEMINI_API_KEY is present."""
+    try:
+        from dotenv import load_dotenv
+    except ImportError:
+        return
+    load_dotenv(os.path.join(ROOT_DIR, ".env"))
+
+
+def local_host() -> str:
+    """Base URL of the local OpenAI-compatible server."""
+    host = (
+        os.getenv("TOOLS_EMBED_HOST", "")
+        or os.getenv("LMSTUDIO_HOST", "")
+        or os.getenv("OLLAMA_HOST", "")
+        or _DEFAULT_HOST
+    )
+    if not host.startswith("http"):
+        host = f"http://{host}"
+    return host.rstrip("/")
+
+
+def local_model() -> str:
+    """Name of the local embedding model."""
+    return os.getenv("TOOLS_EMBED_MODEL", "") or _DEFAULT_MODEL
+
+
+def _embed_local(texts: List[str]) -> List[List[float]]:
+    """Call the local /v1/embeddings endpoint. Returns [] on any failure."""
+    payload = json.dumps({"model": local_model(), "input": texts}).encode()
+    req = urllib.request.Request(
+        f"{local_host()}/v1/embeddings",
+        data=payload,
+        headers={"Content-Type": "application/json"},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=_TIMEOUT) as resp:
+            body = json.loads(resp.read().decode())
+    except (urllib.error.URLError, OSError, json.JSONDecodeError, ValueError):
+        return []
+
+    rows = body.get("data")
+    if not isinstance(rows, list) or len(rows) != len(texts):
+        return []
+    rows = sorted(rows, key=lambda r: r.get("index", 0))
+    vectors = []
+    for row in rows:
+        vec = row.get("embedding")
+        if not isinstance(vec, list) or not vec:
+            return []
+        vectors.append([float(x) for x in vec])
+    return vectors
+
+
+def _embed_gemini_client(texts: List[str]) -> List[List[float]]:
+    """Try the shared GeminiClient's own embed(). Returns [] if unusable."""
+    try:
+        from _shared.gemini import GeminiClient
+    except ImportError:
+        return []
+    try:
+        client = GeminiClient()
+        result = client.embed(texts)
+    except Exception:
+        return []
+    if not isinstance(result, list) or len(result) != len(texts):
+        return []
+    if not all(isinstance(v, list) and v for v in result):
+        return []
+    return [[float(x) for x in v] for v in result]
+
+
+def _embed_gemini_rest(texts: List[str]) -> List[List[float]]:
+    """Direct REST call to the Gemini embedding endpoint. [] on failure."""
+    api_key = os.getenv("GEMINI_API_KEY", "")
+    if not api_key:
+        return []
+    model = "models/gemini-embedding-001"
+    url = (
+        f"https://generativelanguage.googleapis.com/v1beta/{model}"
+        f":batchEmbedContents?key={api_key}"
+    )
+    payload = json.dumps(
+        {
+            "requests": [
+                {"model": model, "content": {"parts": [{"text": t}]}}
+                for t in texts
+            ]
+        }
+    ).encode()
+    req = urllib.request.Request(
+        url, data=payload, headers={"Content-Type": "application/json"}
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=_TIMEOUT) as resp:
+            body = json.loads(resp.read().decode())
+    except (urllib.error.URLError, OSError, json.JSONDecodeError, ValueError):
+        return []
+
+    rows = body.get("embeddings")
+    if not isinstance(rows, list) or len(rows) != len(texts):
+        return []
+    vectors = []
+    for row in rows:
+        vec = (row or {}).get("values")
+        if not isinstance(vec, list) or not vec:
+            return []
+        vectors.append([float(x) for x in vec])
+    return vectors
+
+
+def embed_texts(texts: List[str]) -> List[List[float]]:
+    """Embed a list of texts, local backend first, Gemini second.
+
+    Raises EmbeddingUnavailable if no backend answers.
+    """
+    texts = [t if t else " " for t in texts]
+    if not texts:
+        return []
+
+    vectors = _embed_local(texts)
+    if vectors:
+        return vectors
+
+    _load_env()
+    for backend in (_embed_gemini_client, _embed_gemini_rest):
+        vectors = backend(texts)
+        if vectors:
+            return vectors
+
+    raise EmbeddingUnavailable(
+        "No embedding backend answered: neither the local "
+        f"/v1/embeddings server at {local_host()} nor Gemini."
+    )
+
+
+def backend_name() -> str:
+    """Name of the backend a fresh embed_texts call would reach first."""
+    try:
+        if _embed_local([" "]):
+            return f"local:{local_model()}"
+    except Exception:
+        pass
+    return "gemini"

cli_tools_kit/taxonomy/groups.py

@@ -0,0 +1,108 @@
+"""Read the stored tool grouping back.
+
+``data/tool_groups.json`` is written by regroup.py, or by
+:func:`ensure_groups` when the installer starts and the file no longer matches
+the tools on disk. It is a build artifact: it is regenerated on demand and
+tracked in git, so every host reads the same groups. Entries in "overrides"
+beat the computed assignment.
+"""
+
+import json
+import os
+from typing import Dict
+
+GROUPS_FILE = os.path.join("data", "tool_groups.json")
+UNGROUPED = "Ungrouped"
+
+
+def groups_path(root: str) -> str:
+    """Absolute path of the groups file for a repo root."""
+    return os.path.join(root, GROUPS_FILE)
+
+
+def read_groups_file(root: str) -> dict:
+    """Return the raw JSON, or {} when it is missing or unreadable."""
+    path = groups_path(root)
+    try:
+        with open(path, encoding="utf-8") as fh:
+            data = json.load(fh)
+    except (OSError, json.JSONDecodeError):
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def load_groups(root: str) -> Dict[str, str]:
+    """Map each tool directory name to its group label.
+
+    Tools absent from the file are not in the result; callers treat a missing
+    tool as UNGROUPED.
+    """
+    data = read_groups_file(root)
+    mapping: Dict[str, str] = {}
+
+    labels = data.get("labels")
+    if isinstance(labels, dict):
+        for label, members in labels.items():
+            if not isinstance(members, list):
+                continue
+            for tool in members:
+                if isinstance(tool, str):
+                    mapping[tool] = str(label)
+
+    overrides = data.get("overrides")
+    if isinstance(overrides, dict):
+        for tool, label in overrides.items():
+            if isinstance(tool, str) and isinstance(label, str):
+                mapping[tool] = label
+
+    return mapping
+
+
+def group_of(root: str, tool: str) -> str:
+    """Group label for one tool, UNGROUPED when it has none."""
+    return load_groups(root).get(tool, UNGROUPED)
+
+
+def is_stale(root: str, k: int) -> bool:
+    """True when the stored grouping no longer matches the corpus on disk.
+
+    Stale means: no labels yet, a different k, or a corpus fingerprint that has
+    moved — a tool added or removed, or a CLAUDE.md/README.md edited.
+    """
+    from .corpus import corpus_fingerprint
+
+    data = read_groups_file(root)
+    if not data.get("labels"):
+        return True
+    if data.get("k") != k:
+        return True
+    return data.get("fingerprint") != corpus_fingerprint(root)
+
+
+# The installer calls ensure_groups on its way to opening a window, so the LLM
+# tier gets a wall-clock cap. Gemini answers well inside it; a local model on a
+# busy host may not, and then the capability tier takes over — instantly, and
+# without disturbing bands that are already there.
+STARTUP_BUDGET_SECONDS = 60.0
+
+
+def ensure_groups(
+    root: str, k: int = None, budget: float = STARTUP_BUDGET_SECONDS
+) -> Dict[str, str]:
+    """Groups for the installer, recomputed first if the corpus has changed.
+
+    Falls back to whatever is stored when the rebuild cannot run at all (read-
+    only tree, no tools): a stale grouping beats no grouping, and the GUI must
+    still open.
+    """
+    from .build import DEFAULT_K
+
+    k = DEFAULT_K if k is None else k
+    try:
+        if is_stale(root, k):
+            from .build import build_groups
+
+            build_groups(root, k=k, budget=budget)
+    except Exception:
+        pass
+    return load_groups(root)