code-puppy-core-plugins 0.0.66__tar.gz → 0.0.67__tar.gz

{code_puppy_core_plugins-0.0.66 → code_puppy_core_plugins-0.0.67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: code-puppy-core-plugins
-Version: 0.0.66
+Version: 0.0.67
 Summary: Official plugin bundle for Code Puppy
 Project-URL: Repository, https://github.com/mpfaffenberger/code_puppy_core_plugins
 Project-URL: Homepage, https://github.com/mpfaffenberger/code_puppy_core_plugins
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: <3.15,>=3.11
 Requires-Dist: httpx<1,>=0.24.1
-Requires-Dist: pydantic-ai-slim[anthropic,mcp,openai]<3,>=2.47.0
+Requires-Dist: pydantic-ai-slim[anthropic,mcp,openai,typesafe]<3,>=2.51.0
 Description-Content-Type: text/markdown
 
 # Code Puppy Core Plugins

code_puppy_core_plugins-0.0.67/code_puppy_core_plugins/jev_grep/__init__.py

@@ -0,0 +1 @@
+"""Semantic code search powered by TypeSafe's Jev decision model."""

code_puppy_core_plugins-0.0.67/code_puppy_core_plugins/jev_grep/chunks.py

@@ -0,0 +1,271 @@
+"""Discovery and chunking: turn a directory into judgeable source snippets.
+
+A port of jevgrep's client-side chunker. Python files are split along the
+AST (functions and methods stay whole; declarations longer than
+``SPLIT_LINES`` are split into the blocks inside them so each snippet is one
+behaviour). Every other file falls back to overlapping line windows. Line
+coverage is exact: every non-blank line lands in at least one snippet.
+"""
+
+from __future__ import annotations
+
+import ast
+import os
+import subprocess
+from collections import Counter
+from dataclasses import dataclass, field
+
+from code_puppy.tools.ripgrep import find_ripgrep
+
+MAX_FILES = 20_000
+MAX_CHUNKS = 20_000
+MAX_FILE_BYTES = 1024 * 1024
+MAX_TOTAL_BYTES = 32 * 1024 * 1024
+MAX_CHUNK_CHARS = 12_000
+WHOLE_DECLARATION_LINES = 160
+SPLIT_LINES = 24
+WINDOW, OVERLAP = 60, 10
+
+
+class LineTooLong(ValueError):
+    """A single line is too long to judge (minified or generated source)."""
+
+
+@dataclass(frozen=True)
+class Chunk:
+    path: str
+    line: int
+    end_line: int
+    text: str
+    symbol: str | None = None
+
+
+@dataclass
+class Discovery:
+    chunks: list[Chunk] = field(default_factory=list)
+    files: int = 0
+    skipped: list[tuple[str, str]] = field(default_factory=list)
+    parsers: Counter = field(default_factory=Counter)
+
+
+@dataclass
+class _Range:
+    start: int
+    end: int
+    symbol: str | None
+
+
+def windows(
+    lines: list[str],
+    path: str,
+    first_line: int = 1,
+    size: int = WINDOW,
+    overlap: int = OVERLAP,
+    symbol: str | None = None,
+) -> list[Chunk]:
+    """Fixed-size line windows; giant windows are halved, never clipped."""
+    out: list[Chunk] = []
+    step = size - overlap
+    for i in range(0, max(len(lines), 1), step):
+        part = lines[i : i + size]
+        text = "\n".join(part)
+        if len(text) > MAX_CHUNK_CHARS:
+            if size == 1:
+                raise LineTooLong(
+                    f"line exceeds {MAX_CHUNK_CHARS} characters: {path}:{first_line + i}"
+                )
+            out.extend(
+                windows(part, path, first_line + i, max(1, size // 2), 0, symbol)
+            )
+        elif text.strip():
+            out.append(
+                Chunk(
+                    path, first_line + i, first_line + i + len(part) - 1, text, symbol
+                )
+            )
+        if i + size >= len(lines):
+            break
+    return out
+
+
+def _begin(node: ast.AST) -> int:
+    decorators = getattr(node, "decorator_list", [])
+    return min([node.lineno, *(d.lineno for d in decorators)])
+
+
+def _partition(node: ast.AST, start: int, end: int, symbol, depth: int = 0):
+    """Split a long body into its blocks, covering every line of [start, end]."""
+    body = getattr(node, "body", None)
+    if not isinstance(body, list) or end - start < SPLIT_LINES:
+        return None
+    stmts = [s for s in body if hasattr(s, "end_lineno")]
+    if not stmts:
+        return None
+    if len(stmts) == 1:
+        return (
+            _partition(stmts[0], start, end, symbol, depth + 1) if depth < 3 else None
+        )
+
+    out: list[_Range] = []
+    cursor = start
+    group: list[ast.stmt] = []
+
+    def flush(last: bool) -> None:
+        nonlocal cursor
+        if not group:
+            return
+        stop = end if last else min(end, group[-1].end_lineno)
+        if stop >= cursor:
+            inner = None
+            if len(group) == 1 and depth < 3:
+                inner = _partition(group[0], cursor, stop, symbol, depth + 1)
+            out.extend(inner or [_Range(cursor, stop, symbol)])
+            cursor = stop + 1
+        group.clear()
+
+    for stmt in stmts:
+        span = stmt.end_lineno - _begin(stmt) + 1
+        if group and (span >= 6 or stmt.end_lineno - cursor + 1 > 20):
+            flush(False)
+        group.append(stmt)
+        if span >= 6 and stmt is not stmts[-1]:
+            flush(False)
+    flush(True)
+    return out if len(out) > 1 else None
+
+
+def _python_ranges(text: str) -> list[_Range]:
+    tree = ast.parse(text)
+    ranges: list[_Range] = []
+
+    def emit(node: ast.AST, start: int, symbol: str | None) -> None:
+        ranges.extend(
+            _partition(node, start, node.end_lineno, symbol)
+            or [_Range(start, node.end_lineno, symbol)]
+        )
+
+    for node in tree.body:
+        if isinstance(node, ast.ClassDef):
+            ranges.append(_Range(_begin(node), node.body[0].lineno - 1, node.name))
+            for member in node.body:
+                name = getattr(member, "name", "body")
+                emit(member, _begin(member), f"{node.name}.{name}")
+        else:
+            emit(node, _begin(node), getattr(node, "name", None))
+    return _merge_short_neighbours(ranges)
+
+
+def _merge_short_neighbours(ranges: list[_Range]) -> list[_Range]:
+    """Fold runs of imports/one-liners into one target (each costs a judgment)."""
+
+    def mergeable(r: _Range) -> bool:
+        return r.end - r.start <= 1 and "." not in (r.symbol or "")
+
+    merged: list[_Range] = []
+    for r in ranges:
+        last = merged[-1] if merged else None
+        if (
+            last
+            and mergeable(last)
+            and mergeable(r)
+            and r.start <= last.end + 1
+            and r.end - last.start < 12
+            and len(f"{last.symbol} {r.symbol}") < 60
+        ):
+            names = dict.fromkeys(s for s in (last.symbol, r.symbol) if s)
+            last.end, last.symbol = r.end, ", ".join(names) or None
+        else:
+            merged.append(_Range(r.start, r.end, r.symbol))
+    return merged
+
+
+def _chunks_from_ranges(
+    lines: list[str], path: str, ranges: list[_Range]
+) -> list[Chunk]:
+    out: list[Chunk] = []
+    next_line = 1
+    for r in ranges:
+        start, end = min(r.start, next_line), min(r.end, len(lines))
+        if end < start:
+            continue
+        part = lines[start - 1 : end]
+        whole = (
+            end - start < WHOLE_DECLARATION_LINES
+            and len("\n".join(part)) <= MAX_CHUNK_CHARS
+        )
+        size, overlap = (len(part), 0) if whole else (WINDOW, OVERLAP)
+        out.extend(windows(part, path, start, size, overlap, r.symbol))
+        next_line = end + 1
+    if next_line <= len(lines):
+        out.extend(windows(lines[next_line - 1 :], path, next_line))
+    return out
+
+
+def source_chunks(text: str, path: str) -> tuple[list[Chunk], str]:
+    """Chunk one file's text. Returns (chunks, parser_name)."""
+    lines = text.splitlines()
+    if path.endswith(".py"):
+        try:
+            ranges = _python_ranges(text)
+        except (SyntaxError, ValueError, RecursionError):
+            ranges = []
+        if ranges:
+            return _chunks_from_ranges(lines, path, ranges), "python"
+    return windows(lines, path), "overlapping-lines"
+
+
+def list_files(directory: str, glob: str | None = None) -> list[str]:
+    """Files ripgrep would search: honours .gitignore, skips hidden files."""
+    rg = find_ripgrep()
+    if rg is None:
+        raise RuntimeError("ripgrep (rg) is required for semantic_grep.")
+    args = [rg, "--files", "-0", *(["-g", glob] if glob else []), "--", directory]
+    proc = subprocess.run(args, capture_output=True, timeout=15, check=False)
+    if proc.returncode not in (0, 1):  # 1 == no files
+        raise RuntimeError(
+            f"ripgrep discovery failed: {proc.stderr.decode(errors='replace')[:300]}"
+        )
+    files = sorted({f for f in proc.stdout.decode(errors="replace").split("\0") if f})
+    if len(files) > MAX_FILES:
+        raise RuntimeError(
+            f"Search exceeds {MAX_FILES} files. Narrow the directory or glob."
+        )
+    return files
+
+
+def discover(directory: str, glob: str | None = None) -> Discovery:
+    """Read and chunk every eligible file under ``directory``."""
+    found = Discovery()
+    files = list_files(directory, glob)
+    found.files = len(files)
+    total_bytes = 0
+    for path in files:
+        if os.path.getsize(path) > MAX_FILE_BYTES:
+            found.skipped.append((path, "larger than 1 MiB"))
+            continue
+        with open(path, "rb") as fh:
+            raw = fh.read()
+        total_bytes += len(raw)
+        if total_bytes > MAX_TOTAL_BYTES:
+            raise RuntimeError(
+                "Search exceeds 32 MiB of source. Narrow the directory or glob."
+            )
+        if b"\0" in raw:
+            found.skipped.append((path, "binary"))
+            continue
+        try:
+            text = raw.decode("utf-8")
+            chunks, parser = source_chunks(text, path)
+        except UnicodeDecodeError:
+            found.skipped.append((path, "not UTF-8"))
+            continue
+        except LineTooLong as exc:
+            found.skipped.append((path, str(exc)))
+            continue
+        found.parsers[parser] += 1
+        found.chunks.extend(chunks)
+        if len(found.chunks) > MAX_CHUNKS:
+            raise RuntimeError(
+                f"Search exceeds {MAX_CHUNKS} snippets. Narrow the directory or glob."
+            )
+    return found

code_puppy_core_plugins-0.0.67/code_puppy_core_plugins/jev_grep/config.py

@@ -0,0 +1,38 @@
+"""Settings for jev_grep. Set with `/set <key> <value>` or the environment."""
+
+from __future__ import annotations
+
+import os
+
+from code_puppy.config import get_api_key, get_value
+
+# TypeSafe's official name first, then the Jev-branded alias. Config lookup
+# is case-insensitive, so `/set jev_api_key ...` matches JEV_API_KEY.
+API_KEY_NAMES = ("TYPESAFE_API_KEY", "JEV_API_KEY")
+API_KEY_NAME = API_KEY_NAMES[0]
+DEFAULT_MODEL = "jev-latest"
+DEFAULT_THRESHOLD = 0.5
+
+
+def get_typesafe_api_key() -> str:
+    """First key found: environment beats config, official name beats alias."""
+    for name in API_KEY_NAMES:
+        if key := os.environ.get(name):
+            return key
+    for name in API_KEY_NAMES:
+        if key := get_api_key(name):
+            return key
+    return ""
+
+
+def get_jev_model_name() -> str:
+    """Pin a version (e.g. ``jev-1.13.0``) once a threshold is tuned against it."""
+    return get_value("jev_grep_model") or DEFAULT_MODEL
+
+
+def get_threshold() -> float:
+    try:
+        value = float(get_value("jev_grep_threshold") or DEFAULT_THRESHOLD)
+    except ValueError:
+        return DEFAULT_THRESHOLD
+    return value if 0 <= value <= 1 else DEFAULT_THRESHOLD

code_puppy_core_plugins-0.0.67/code_puppy_core_plugins/jev_grep/judge.py

@@ -0,0 +1,102 @@
+"""Relevance judgments from Jev, TypeSafe's decision model, via Pydantic AI.
+
+This replaces jevgrep's hosted ``/api/v1/grep`` backend. jevgrep documents its
+score as the product of two independent judgments -- the snippet acts on the
+requested *entity*, and it performs the requested *operation* -- so that is
+exactly what we ask. A bounded ``float`` field on a decision model is answered
+with the raw probability of yes, so there is no confidence-margin math to undo.
+
+Every snippet is judged in its own run (its own Jev state), so neighbouring
+snippets never lend each other evidence.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Sequence
+
+from pydantic import BaseModel, Field
+from pydantic_ai import Agent
+from pydantic_ai.models import Model
+
+from .chunks import Chunk
+
+DEFAULT_CONCURRENCY = 8
+REQUEST_TIMEOUT_SECONDS = 30
+
+
+class Relevance(BaseModel):
+    """Decide whether this source code is what a developer searching the codebase wants to find."""
+
+    entity: float = Field(
+        ge=0,
+        le=1,
+        description=(
+            "Does this code act on the thing the search is about "
+            "(the data, object, resource or concept it names)?"
+        ),
+    )
+    operation: float = Field(
+        ge=0,
+        le=1,
+        description=(
+            "Does this code itself perform the action or behaviour the search "
+            "describes, rather than only mentioning it, importing it, or calling "
+            "something with a similar name?"
+        ),
+    )
+
+    @property
+    def score(self) -> float:
+        return self.entity * self.operation
+
+
+def build_jev_model(model_name: str, api_key: str) -> Model:
+    """A TypeSafe Jev model. Imported lazily: the SDK is an optional extra."""
+    from pydantic_ai.models.typesafe import TypeSafeModel
+    from pydantic_ai.providers.typesafe import TypeSafeProvider
+
+    return TypeSafeModel(model_name, provider=TypeSafeProvider(api_key=api_key))
+
+
+def _state(chunk: Chunk) -> str:
+    """What Jev judges: where the snippet lives, then the snippet itself."""
+    where = f"{chunk.path}:{chunk.line}-{chunk.end_line}"
+    header = f"{where} ({chunk.symbol})" if chunk.symbol else where
+    return f"{header}\n\n{chunk.text}"
+
+
+def build_agent(model: Model, query: str) -> Agent[None, Relevance]:
+    return Agent(
+        model,
+        output_type=Relevance,
+        instructions=(
+            "The text is a snippet of source code from a repository. "
+            f"A developer is searching the codebase for: {query}"
+        ),
+        model_settings={"timeout": REQUEST_TIMEOUT_SECONDS},
+    )
+
+
+async def judge(
+    model: Model,
+    query: str,
+    chunks: Sequence[Chunk],
+    concurrency: int = DEFAULT_CONCURRENCY,
+) -> list[float]:
+    """Score each chunk in [0, 1], order preserved.
+
+    Any failed judgment fails the whole search (siblings are cancelled):
+    a partial scan reported as a complete one is worse than an error.
+    """
+    agent = build_agent(model, query)
+    gate = asyncio.Semaphore(concurrency)
+
+    async def one(chunk: Chunk) -> float:
+        async with gate:
+            result = await agent.run(_state(chunk))
+        return result.output.score
+
+    async with asyncio.TaskGroup() as group:
+        tasks = [group.create_task(one(chunk)) for chunk in chunks]
+    return [task.result() for task in tasks]

code_puppy_core_plugins-0.0.67/code_puppy_core_plugins/jev_grep/locales/en-US.json

@@ -0,0 +1,3 @@
+{
+  "jev_grep.summary": "semantic_grep '{query}' in {directory}: {matches} matches ({evaluated}/{snippets} snippets judged)"
+}

code_puppy_core_plugins-0.0.67/code_puppy_core_plugins/jev_grep/register_callbacks.py

@@ -0,0 +1,25 @@
+"""jev_grep: semantic code search (a jevgrep reimplementation) on TypeSafe's Jev.
+
+Registers a ``semantic_grep`` tool alongside the regular ``grep``. It is only
+advertised to agents once ``TYPESAFE_API_KEY`` is configured, so agents never
+see a tool that can only fail.
+"""
+
+from code_puppy.callbacks import register_callback
+
+from .config import get_typesafe_api_key
+from .tool import register_semantic_grep
+
+TOOL_NAME = "semantic_grep"
+
+
+def _register_tools():
+    return [{"name": TOOL_NAME, "register_func": register_semantic_grep}]
+
+
+def _advertise_when_configured(agent_name=None):
+    return [TOOL_NAME] if get_typesafe_api_key() else []
+
+
+register_callback("register_tools", _register_tools)
+register_callback("register_agent_tools", _advertise_when_configured)

code_puppy_core_plugins-0.0.67/code_puppy_core_plugins/jev_grep/retrieve.py

@@ -0,0 +1,90 @@
+"""Local BM25 shortlist (port of jevgrep's retrieve.mjs).
+
+Cheap lexical ranking decides which snippets are worth a Jev judgment. It is
+a shortlist, not a verdict: code with unrelated wording can be missed, which
+is why coverage is reported back to the caller.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from collections.abc import Sequence
+
+from .chunks import Chunk
+
+_STOP = frozenset(
+    "a an the is are was were to of in on for from with where how does do this "
+    "that it and or code function files file find which".split()
+)
+
+_SYNONYM_GROUPS = (
+    "auth authentication authenticate login signin session",
+    "retry retries backoff attempt",
+    "cache cached caching memo memoize",
+    "delete remove cleanup unlink",
+    "permission permissions authorization authorize access allowed",
+    "expire expired expires expiration expiry timeout ttl",
+    "save persist persistence storage store write",
+    "error errors failure failed exception throw catch",
+    "payment payments billing subscription invoice charge",
+    "cancel abort cancellation",
+    "parallel concurrent concurrency simultaneous",
+    "duplicate duplicates deduplicate dedup unique",
+)
+_VOCABULARY = {
+    term: group.split() for group in _SYNONYM_GROUPS for term in group.split()
+}
+
+_CAMEL = re.compile(r"([a-z\d])([A-Z])")
+_WORD = re.compile(r"[^\W_]+")
+_SYNONYM_WEIGHT = 0.3
+_META_BOOST = 3
+_K1, _B = 1.2, 0.75
+
+
+def terms(text: str) -> list[str]:
+    """Lower-cased identifier-aware tokens: ``refreshToken`` -> refresh, token."""
+    words = _WORD.findall(_CAMEL.sub(r"\1 \2", text).lower())
+    return [w for w in words if len(w) > 1 and w not in _STOP]
+
+
+def _query_weights(query: str) -> dict[str, float]:
+    weights = dict.fromkeys(terms(query), 1.0)
+    for term in list(weights):
+        for synonym in _VOCABULARY.get(term, ()):
+            weights.setdefault(synonym, _SYNONYM_WEIGHT)
+    return weights
+
+
+def rank(query: str, chunks: Sequence[Chunk]) -> list[Chunk]:
+    """Chunks ordered by BM25 relevance to ``query`` (stable on ties)."""
+    weights = _query_weights(query)
+    docs: list[tuple[Counter, int]] = []
+    for chunk in chunks:
+        body, meta = terms(chunk.text), terms(f"{chunk.path} {chunk.symbol or ''}")
+        freq = Counter(body)
+        for word in meta:
+            freq[word] += _META_BOOST
+        docs.append((freq, len(body) + len(meta)))
+
+    n = len(docs)
+    average = (sum(length for _, length in docs) / n if n else 0) or 1
+    df = {t: sum(1 for freq, _ in docs if t in freq) for t in weights}
+
+    def score(doc: tuple[Counter, int]) -> float:
+        freq, length = doc
+        total = 0.0
+        for word, weight in weights.items():
+            tf = freq.get(word, 0)
+            if not tf:
+                continue
+            idf = math.log(1 + (n - df[word] + 0.5) / (df[word] + 0.5))
+            norm = _K1 * (1 - _B + _B * length / average)
+            total += weight * idf * tf * (_K1 + 1) / (tf + norm)
+        return total
+
+    scored = [(score(doc), chunk) for doc, chunk in zip(docs, chunks)]
+    scored.sort(key=lambda sc: (-sc[0], sc[1].path, sc[1].line))
+    return [chunk for _, chunk in scored]