cli-tools-kit 0.6.0__py3-none-any.whl

cli_tools_kit/taxonomy/__init__.py

@@ -0,0 +1,40 @@
+"""Group a tool tree into named bands by what the tools are for.
+
+The installer bands its rows by *something*. ``group_by="capability"`` uses the
+one word each tool advertises, which needs no computation. This package is the
+other option: it reads what a tree already documents about itself and produces
+a small set of named categories, which is what you want once a tree has more
+tools than a person can scan.
+
+Three tiers, tried in order, so the feature degrades instead of failing:
+
+1. **LLM** — name six categories, file every tool into them, then review the
+   result and let the model move misfits and rename a category to match what it
+   actually holds. Uses Gemini when ``GEMINI_API_KEY`` is reachable, otherwise a
+   local LM Studio / Ollama server. This is the tier that produces good names.
+2. **capability** — band the advertised capability words into a fixed set of
+   six. No network, no model, instant, and the names are stable.
+3. **embed + k-means** — embed the documents and cluster them, naming each
+   cluster after its most distinctive token. Names are poor ("git", "clipboard")
+   but the tree still opens.
+
+:func:`~cli_tools_kit.taxonomy.groups.ensure_groups` is the entry point a
+wrapper's discoverer calls. It fingerprints the tree's documents and rebuilds
+only when they changed, so the common case costs a couple of milliseconds.
+
+    from cli_tools_kit.taxonomy import ensure_groups
+    groups = ensure_groups(root)      # {tool_name: band label}
+"""
+
+from .embedder import EmbeddingUnavailable, embed_texts
+from .groups import ensure_groups, load_groups
+from .build import build_groups, DEFAULT_K
+
+__all__ = [
+    "EmbeddingUnavailable",
+    "embed_texts",
+    "ensure_groups",
+    "load_groups",
+    "build_groups",
+    "DEFAULT_K",
+]

cli_tools_kit/taxonomy/build.py

@@ -0,0 +1,171 @@
+"""Compute the tool grouping and write data/tool_groups.json.
+
+The one place that decides the bands and writes the file. regroup.py calls it
+to rebuild on demand; groups.ensure_groups calls it when the installer starts
+and the stored grouping no longer matches the corpus.
+
+Three ways to get there, tried in order:
+
+1. **gemini** — reads a condensed blurb per tool and returns named categories
+   with their members. The normal path: naming a category is a language task.
+2. **capability**, when a grouping is already stored — keep those bands and
+   their names, and file only the tools missing from them by the capability
+   word they advertise. No network. Preferred over re-embedding because it
+   preserves names a model chose over names tf-idf would invent.
+3. **embed** — no stored grouping to extend, so cluster the full documents and
+   name each cluster after its most distinctive token. Needs an embedding
+   backend; if that is missing too, fall back to capability words alone.
+
+Only the first needs an API key, and only steps 1 and 3 need a network. A
+distributed checkout ships with data/tool_groups.json already written, so it
+never reaches any of them until its owner adds a tool of their own.
+"""
+
+import datetime
+import json
+import os
+import tempfile
+from typing import Dict, List, Optional, Tuple
+
+from .capability import capability_groups
+from .cluster import cluster_documents
+from .corpus import (
+    corpus_fingerprint,
+    tool_blurbs,
+    tool_capabilities,
+    tool_documents,
+)
+from .embedder import EmbeddingUnavailable, backend_name, embed_texts
+from .groups import groups_path, read_groups_file
+from .llm_groups import (
+    LLMGroupingUnavailable,
+    active_label,
+    llm_groups,
+)
+
+DEFAULT_K = 6
+DEFAULT_SEED = 0
+
+
+def build_groups(
+    root: str,
+    k: int = DEFAULT_K,
+    seed: int = DEFAULT_SEED,
+    write: bool = True,
+    method: str = "auto",
+    budget: Optional[float] = None,
+    rename: bool = False,
+) -> dict:
+    """Decide the grouping and (unless write=False) store it.
+
+    ``budget`` caps the LLM tier in seconds — the installer passes one, a
+    deliberate ``regroup.py`` run does not. ``rename`` throws the stored
+    category names away and asks for six fresh ones.
+
+    ``method`` is "auto" (the chain described above), or one tier by name:
+    "llm", "embed" or "capability". Raises ValueError when the tree holds no
+    tools, and EmbeddingUnavailable / LLMGroupingUnavailable when a single tier
+    was asked for and could not run. Existing overrides are carried over
+    untouched.
+    """
+    from .corpus import tool_dirs
+
+    names = tool_dirs(root)
+    if not names:
+        raise ValueError("no tools found")
+    previous = read_groups_file(root)
+    stored = previous.get("labels") or {}
+    categories = None if rename else previous.get("categories")
+    if not isinstance(categories, list) or len(categories) != k:
+        categories = None
+
+    labels: Dict[str, List[str]] = {}
+    used = ""
+    if method in ("auto", "llm"):
+        try:
+            labels = llm_groups(
+                tool_blurbs(root), k, budget=budget, categories=categories
+            )
+            used = active_label() or "llm"
+        except LLMGroupingUnavailable:
+            if method == "llm":
+                raise
+
+    if not labels and method in ("auto", "capability"):
+        if stored or method == "capability":
+            labels = capability_groups(tool_capabilities(root), stored or None)
+            used = "capability" + (":extended" if stored else "")
+
+    if not labels and method in ("auto", "embed"):
+        try:
+            labels = _embed_groups(root, names, k, seed)
+            used = f"embed-kmeans:{backend_name()}"
+        except EmbeddingUnavailable:
+            if method == "embed":
+                raise
+
+    if not labels:
+        labels = capability_groups(tool_capabilities(root), stored or None)
+        used = "capability"
+
+    payload = {
+        "generated": datetime.date.today().isoformat(),
+        "k": k,
+        "seed": seed,
+        "method": used,
+        "categories": _carry_scopes(labels, categories),
+        "fingerprint": corpus_fingerprint(root),
+        "labels": labels,
+        "overrides": read_groups_file(root).get("overrides") or {},
+    }
+    if write:
+        write_groups_file(root, payload)
+    return payload
+
+
+def _carry_scopes(
+    labels: Dict[str, List[str]], categories: Optional[List[dict]]
+) -> List[dict]:
+    """The band names to reuse next time, keeping the scope of each survivor.
+
+    A band the review step renamed keeps its members but loses its scope; that
+    is fine, the scope only steers the next assignment.
+    """
+    scopes = {c.get("name"): c.get("scope", "") for c in (categories or [])}
+    return [{"name": name, "scope": scopes.get(name, "")} for name in labels]
+
+
+def _embed_groups(
+    root: str, names: List[str], k: int, seed: int
+) -> Dict[str, List[str]]:
+    """Offline fallback: embed the full documents and k-means them."""
+    docs = tool_documents(root)
+    ordered = sorted(docs)
+    vectors = embed_texts([docs[n] for n in ordered])
+    groups: List[Tuple[str, List[str]]] = cluster_documents(
+        ordered, vectors, docs, k, seed=seed
+    )
+    return {label: members for label, members in groups}
+
+
+def write_groups_file(root: str, payload: dict) -> str:
+    """Write the payload atomically so a concurrent reader sees whole JSON."""
+    path = groups_path(root)
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            json.dump(payload, fh, indent=2, ensure_ascii=False)
+            fh.write("\n")
+        os.replace(tmp, path)
+    except BaseException:
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
+        raise
+    return path
+
+
+__all__ = ["DEFAULT_K", "DEFAULT_SEED", "build_groups", "write_groups_file",
+           "EmbeddingUnavailable", "LLMGroupingUnavailable"]

cli_tools_kit/taxonomy/capability.py

@@ -0,0 +1,116 @@
+"""Group tools by the capability word they advertise. No network, no model.
+
+The last tier of the grouping chain, and the only one that works on a host with
+no API key and no local embedding server — which is every host but this fleet's,
+since the key is read from the repo's own .env. A distributed checkout gets its
+bands from the committed data/tool_groups.json; this is what happens when
+someone then adds a tool of their own.
+
+Two modes. With an existing grouping it is *incremental*: the stored bands and
+their names are kept exactly as they are, and only tools missing from them are
+filed, next to whichever band already holds their capability. Without one it
+builds the bands from DEFAULT_BANDS below.
+"""
+
+from typing import Dict, List, Optional, Sequence, Tuple
+
+# Every word in validate_structure.CAPABILITY_VOCAB has a home here. A tool
+# whose capability is missing or outside the vocabulary lands in the smallest
+# band, which is the honest answer: nothing about it says where it belongs.
+DEFAULT_BANDS: Dict[str, Tuple[str, ...]] = {
+    "AI & Chat": ("agent", "llm-chat", "chat"),
+    "Audio & Speech": ("tts", "stt", "audio-visualizer"),
+    "Vision & Creation": ("image-gen", "image-edit", "ocr", "vision-search"),
+    "Media & Publishing": (
+        "scrape", "summarize", "download", "media-launcher", "publish",
+        "pdf-extract",
+    ),
+    "Developer Tools": (
+        "commit-gen", "dep-check", "forge", "repo-hygiene", "finetune",
+    ),
+    "System & Context": ("smarthome", "vnc-display", "context-picker"),
+}
+
+_BAND_OF_CAPABILITY = {
+    capability: band
+    for band, capabilities in DEFAULT_BANDS.items()
+    for capability in capabilities
+}
+
+
+def _smallest(labels: Dict[str, List[str]]) -> str:
+    """Name of the least populated band, ties broken alphabetically."""
+    return min(labels, key=lambda band: (len(labels[band]), band))
+
+
+def capability_groups(
+    capabilities: Dict[str, str],
+    existing: Optional[Dict[str, Sequence[str]]] = None,
+) -> Dict[str, List[str]]:
+    """Band name -> members, decided by each tool's capability word.
+
+    ``existing`` is a stored grouping to extend rather than replace: its band
+    names survive untouched, tools that no longer exist are dropped, and only
+    the newcomers are placed.
+    """
+    if not capabilities:
+        return {}
+
+    if existing:
+        labels = {
+            band: [tool for tool in members if tool in capabilities]
+            for band, members in existing.items()
+        }
+        labels = {band: members for band, members in labels.items() if members}
+    else:
+        labels = {}
+
+    if not labels:
+        for tool, capability in capabilities.items():
+            band = _BAND_OF_CAPABILITY.get(capability)
+            if band:
+                labels.setdefault(band, []).append(tool)
+        if not labels:
+            labels = {"AI & Chat": []}
+
+    # Which band already holds each capability, and each family of related
+    # capabilities, by weight of numbers.
+    home: Dict[str, Dict[str, int]] = {}
+    family: Dict[str, Dict[str, int]] = {}
+
+    def _record(capability: str, band: str) -> None:
+        if not capability:
+            return
+        home.setdefault(capability, {})
+        home[capability][band] = home[capability].get(band, 0) + 1
+        default = _BAND_OF_CAPABILITY.get(capability)
+        if default:
+            family.setdefault(default, {})
+            family[default][band] = family[default].get(band, 0) + 1
+
+    for band, members in labels.items():
+        for tool in members:
+            _record(capabilities.get(tool, ""), band)
+
+    def _pick(counts: Dict[str, int]) -> str:
+        return max(counts, key=lambda band: (counts[band], band))
+
+    for tool in sorted(capabilities):
+        if any(tool in members for members in labels.values()):
+            continue
+        capability = capabilities[tool]
+        default = _BAND_OF_CAPABILITY.get(capability)
+        if home.get(capability):
+            # A band already holds this exact capability.
+            band = _pick(home[capability])
+        elif default and family.get(default):
+            # No exact match, but a band holds its siblings: stt joins tts.
+            band = _pick(family[default])
+        elif default in labels:
+            band = default
+        else:
+            band = _smallest(labels)
+        labels[band].append(tool)
+        _record(capability, band)
+
+    return {band: sorted(members) for band, members in labels.items() if members}

cli_tools_kit/taxonomy/cluster.py

@@ -0,0 +1,261 @@
+"""Deterministic k-means over normalised vectors, plus group labelling.
+
+Pure python, no numpy, so the result is identical on every host. Vectors are
+L2-normalised, which makes squared euclidean distance a stand-in for cosine
+distance.
+"""
+
+import math
+import random
+import re
+from typing import Dict, List, Sequence, Tuple
+
+STOPWORDS = {
+    "a", "about", "after", "all", "also", "an", "and", "any", "are", "as", "at",
+    "back", "be", "because", "been", "before", "being", "below", "between",
+    "both", "but", "by", "can", "claude", "code", "com", "d", "de", "did", "do",
+    "does", "doing", "done", "each", "etc", "even", "every", "few", "file",
+    "files", "for", "from", "get", "gets", "github", "had", "has", "have", "he",
+    "her", "here", "how", "http", "https", "i", "if", "in", "install", "into",
+    "is", "it", "its", "just", "like", "made", "main", "make", "makes", "many",
+    "md", "more", "most", "must", "my", "need", "needs", "new", "no", "not",
+    "now", "of", "off", "on", "once", "one", "only", "or", "other", "our",
+    "out", "over", "own", "path", "py", "python", "readme", "repo", "run",
+    "runs", "same", "script", "see", "set", "she", "should", "since", "so",
+    "some", "still", "such", "than", "that", "the", "their", "them", "then",
+    "there", "these", "they", "this", "those", "through", "to", "too", "tool",
+    "tools", "two", "under", "until", "up", "us", "use", "used", "uses",
+    "using", "very", "was", "way", "we", "were", "what", "when", "where",
+    "python3", "json", "name", "names", "default", "defaults", "example",
+    "examples", "args", "arg", "note", "notes", "directory", "dir", "output",
+    "which", "while", "who", "why", "will", "with", "would", "you", "your",
+}
+
+_TOKEN_RE = re.compile(r"[a-z0-9]+")
+
+
+def normalize(vectors: Sequence[Sequence[float]]) -> List[List[float]]:
+    """Scale every vector to unit length."""
+    out = []
+    for vec in vectors:
+        norm = math.sqrt(sum(x * x for x in vec))
+        if norm == 0.0:
+            out.append([0.0] * len(vec))
+        else:
+            out.append([x / norm for x in vec])
+    return out
+
+
+def _sqdist(a: Sequence[float], b: Sequence[float]) -> float:
+    return sum((x - y) * (x - y) for x, y in zip(a, b))
+
+
+def _kmeans_plusplus(vectors: List[List[float]], k: int, rng: random.Random) -> List[List[float]]:
+    """Pick k starting centroids, spread out, using the given rng."""
+    first = rng.randrange(len(vectors))
+    centroids = [list(vectors[first])]
+    while len(centroids) < k:
+        dists = [min(_sqdist(v, c) for c in centroids) for v in vectors]
+        total = sum(dists)
+        if total <= 0.0:
+            centroids.append(list(vectors[rng.randrange(len(vectors))]))
+            continue
+        target = rng.random() * total
+        running = 0.0
+        chosen = len(vectors) - 1
+        for i, d in enumerate(dists):
+            running += d
+            if running >= target:
+                chosen = i
+                break
+        centroids.append(list(vectors[chosen]))
+    return centroids
+
+
+def _assign(vectors: List[List[float]], centroids: List[List[float]]) -> List[int]:
+    """Assign each vector to its nearest centroid."""
+    labels = []
+    for vec in vectors:
+        best, best_d = 0, None
+        for ci, cen in enumerate(centroids):
+            d = _sqdist(vec, cen)
+            if best_d is None or d < best_d:
+                best, best_d = ci, d
+        labels.append(best)
+    return labels
+
+
+def _assign_balanced(
+    vectors: List[List[float]], centroids: List[List[float]], cap: int
+) -> List[int]:
+    """Assign vectors nearest-first, never letting a cluster exceed cap."""
+    pairs = []
+    for vi, vec in enumerate(vectors):
+        for ci, cen in enumerate(centroids):
+            pairs.append((_sqdist(vec, cen), vi, ci))
+    pairs.sort()
+
+    labels = [-1] * len(vectors)
+    sizes = [0] * len(centroids)
+    placed = 0
+    for _, vi, ci in pairs:
+        if labels[vi] != -1 or sizes[ci] >= cap:
+            continue
+        labels[vi] = ci
+        sizes[ci] += 1
+        placed += 1
+        if placed == len(vectors):
+            break
+    # Anything left over (all its clusters full) goes to the emptiest cluster.
+    for vi, lab in enumerate(labels):
+        if lab == -1:
+            ci = min(range(len(centroids)), key=lambda c: (sizes[c], c))
+            labels[vi] = ci
+            sizes[ci] += 1
+    return labels
+
+
+def _recentre(
+    vectors: List[List[float]], labels: List[int], centroids: List[List[float]]
+) -> List[List[float]]:
+    """Move each centroid to the mean of its members, normalised."""
+    dim = len(vectors[0])
+    sums = [[0.0] * dim for _ in centroids]
+    counts = [0] * len(centroids)
+    for vec, lab in zip(vectors, labels):
+        counts[lab] += 1
+        row = sums[lab]
+        for i, x in enumerate(vec):
+            row[i] += x
+    new = []
+    for ci, count in enumerate(counts):
+        if count == 0:
+            new.append(list(centroids[ci]))
+        else:
+            new.append([x / count for x in sums[ci]])
+    return normalize(new)
+
+
+def kmeans(
+    vectors: Sequence[Sequence[float]],
+    k: int,
+    seed: int = 0,
+    iters: int = 50,
+    balanced: bool = True,
+) -> List[int]:
+    """Cluster vectors into k groups and return one cluster index per vector.
+
+    With balanced=True no cluster grows past ceil(N/k)+1 members.
+    """
+    points = normalize(vectors)
+    n = len(points)
+    if n == 0:
+        return []
+    k = max(1, min(k, n))
+    cap = math.ceil(n / k) + 1
+
+    rng = random.Random(seed)
+    centroids = _kmeans_plusplus(points, k, rng)
+
+    labels = []
+    for _ in range(iters):
+        if balanced:
+            new_labels = _assign_balanced(points, centroids, cap)
+        else:
+            new_labels = _assign(points, centroids)
+        if new_labels == labels:
+            break
+        labels = new_labels
+        centroids = _recentre(points, labels, centroids)
+    return labels
+
+
+def tokenize(text: str) -> List[str]:
+    """Lowercase alphanumeric tokens, stopwords and 1-2 char words dropped."""
+    return [
+        t
+        for t in _TOKEN_RE.findall(text.lower())
+        if len(t) > 2 and t not in STOPWORDS and not t.isdigit()
+    ]
+
+
+def label_group(
+    group_docs: Sequence[str], other_docs: Sequence[str], top: int = 1
+) -> str:
+    """Name a group by its most distinctive tokens, joined with ' · '.
+
+    ``top`` is how many tokens the name may carry; one by default.
+    """
+    if not group_docs:
+        return "Ungrouped"
+
+    def doc_freq(docs):
+        counts = {}
+        for doc in docs:
+            for token in set(tokenize(doc)):
+                counts[token] = counts.get(token, 0) + 1
+        return counts
+
+    def term_freq(docs):
+        counts = {}
+        for doc in docs:
+            for token in tokenize(doc):
+                counts[token] = counts.get(token, 0) + 1
+        return counts
+
+    inside_df = doc_freq(group_docs)
+    inside_tf = term_freq(group_docs)
+    outside_df = doc_freq(other_docs)
+    n_in = len(group_docs)
+    n_out = max(1, len(other_docs))
+
+    # A token found in only one document of a multi-document group names that
+    # document, not the group, so require two documents whenever we can.
+    min_docs = 2 if n_in > 2 else 1
+
+    scored = []
+    for token, docs_with in inside_df.items():
+        if docs_with < min_docs:
+            continue
+        tf = inside_tf[token] / n_in
+        spread = docs_with / n_in
+        idf = math.log((n_out + 1) / (outside_df.get(token, 0) + 1)) + 1.0
+        scored.append((-(tf * spread * idf), token))
+    scored.sort()
+    picked = [token for _, token in scored[:top]]
+    return " · ".join(picked) if picked else "Ungrouped"
+
+
+def cluster_documents(
+    names: Sequence[str],
+    vectors: Sequence[Sequence[float]],
+    docs: Dict[str, str],
+    k: int,
+    seed: int = 0,
+    balanced: bool = True,
+    top: int = 1,
+) -> List[Tuple[str, List[str]]]:
+    """Cluster the tools and return (label, members) per group."""
+    labels = kmeans(vectors, k, seed=seed, balanced=balanced)
+    buckets: Dict[int, List[str]] = {}
+    for name, lab in zip(names, labels):
+        buckets.setdefault(lab, []).append(name)
+
+    groups = []
+    for lab in sorted(buckets):
+        members = sorted(buckets[lab])
+        inside = [docs.get(m, "") for m in members]
+        outside = [docs.get(n, "") for n in names if n not in set(members)]
+        groups.append((label_group(inside, outside, top=top), members))
+
+    # Two clusters can land on the same tokens; keep the labels unique.
+    seen = {}
+    unique = []
+    for label, members in groups:
+        if label in seen:
+            seen[label] += 1
+            label = f"{label} ({seen[label]})"
+        else:
+            seen[label] = 1
+        unique.append((label, members))
+    return unique