code-puppy-core-plugins 0.0.66__tar.gz → 0.0.68__tar.gz

{code_puppy_core_plugins-0.0.66 → code_puppy_core_plugins-0.0.68}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: code-puppy-core-plugins
-Version: 0.0.66
+Version: 0.0.68
 Summary: Official plugin bundle for Code Puppy
 Project-URL: Repository, https://github.com/mpfaffenberger/code_puppy_core_plugins
 Project-URL: Homepage, https://github.com/mpfaffenberger/code_puppy_core_plugins
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: <3.15,>=3.11
 Requires-Dist: httpx<1,>=0.24.1
-Requires-Dist: pydantic-ai-slim[anthropic,mcp,openai]<3,>=2.47.0
+Requires-Dist: pydantic-ai-slim[anthropic,mcp,openai,typesafe]<3,>=2.51.0
 Description-Content-Type: text/markdown
 
 # Code Puppy Core Plugins

{code_puppy_core_plugins-0.0.66 → code_puppy_core_plugins-0.0.68}/code_puppy_core_plugins/claude_code_oauth/model_provider.py

@@ -98,8 +98,9 @@ def create_claude_code_model(model_name: str, model_config: Dict, config: Dict)
     if verify is None:
         verify = get_cert_bundle_path()
 
-    # No HTTP/2 for OAuth: the UnprefixingStream tool-name rewrite breaks under
-    # HTTP/2's compression handling, causing zlib decompression errors.
+    # No HTTP/2 for OAuth: kept as a precaution from core cfff821b4, which fixed
+    # zlib decompression errors caused by rewriting response streams to strip
+    # the ``cp_`` prefix. Responses are no longer rewritten -- see below.
     kwargs = {}
     # Plugin releases land before core. Old transports keep their legacy path.
     import inspect
@@ -116,7 +117,11 @@ def create_claude_code_model(model_name: str, model_config: Dict, config: Dict)
         timeout=180,
         http2=False,
         # Claude Code OAuth requires the ``cp_`` tool-name prefix; the wire
-        # format Anthropic's CLI uses won't accept un-prefixed tools.
+        # format Anthropic's CLI uses won't accept un-prefixed tools. Only
+        # requests are rewritten: returned ``cp_*`` calls are mapped back to
+        # real tool names at dispatch by core's pydantic_patches
+        # (``_normalize_claude_code_tool_name``), so this model needs
+        # ``apply_all_patches()`` -- cli_runner applies it at import.
         apply_claude_code_prefix=True,
         oauth_reauthentication_callback=lambda: _reauthenticate_after_expired_oauth(
             model_name

code_puppy_core_plugins-0.0.68/code_puppy_core_plugins/jev_grep/__init__.py

@@ -0,0 +1 @@
+"""Semantic code search powered by TypeSafe's Jev decision model."""

code_puppy_core_plugins-0.0.68/code_puppy_core_plugins/jev_grep/chunks.py

@@ -0,0 +1,269 @@
+"""Discovery and chunking: turn a directory into judgeable source snippets.
+
+Python files are split along the
+AST (functions and methods stay whole; declarations longer than
+``SPLIT_LINES`` are split into the blocks inside them so each snippet is one
+behaviour). Every other file falls back to overlapping line windows. Line
+coverage is exact: every non-blank line lands in at least one snippet.
+"""
+
+from __future__ import annotations
+
+import ast
+import os
+import subprocess
+from collections import Counter
+from dataclasses import dataclass, field
+
+from code_puppy.tools.ripgrep import find_ripgrep
+
+# Discovery is bounded by files and bytes only. Snippet count is not capped:
+# only the ranked shortlist is judged, and local BM25 over the ~24k snippets
+# of a repo root takes well under a second.
+MAX_FILES = 20_000
+MAX_FILE_BYTES = 1024 * 1024
+MAX_TOTAL_BYTES = 32 * 1024 * 1024
+MAX_CHUNK_CHARS = 12_000
+WHOLE_DECLARATION_LINES = 160
+SPLIT_LINES = 24
+WINDOW, OVERLAP = 60, 10
+
+
+class LineTooLong(ValueError):
+    """A single line is too long to judge (minified or generated source)."""
+
+
+@dataclass(frozen=True)
+class Chunk:
+    path: str
+    line: int
+    end_line: int
+    text: str
+    symbol: str | None = None
+
+
+@dataclass
+class Discovery:
+    chunks: list[Chunk] = field(default_factory=list)
+    files: int = 0
+    skipped: list[tuple[str, str]] = field(default_factory=list)
+    parsers: Counter = field(default_factory=Counter)
+
+
+@dataclass
+class _Range:
+    start: int
+    end: int
+    symbol: str | None
+
+
+def windows(
+    lines: list[str],
+    path: str,
+    first_line: int = 1,
+    size: int = WINDOW,
+    overlap: int = OVERLAP,
+    symbol: str | None = None,
+) -> list[Chunk]:
+    """Fixed-size line windows; giant windows are halved, never clipped."""
+    out: list[Chunk] = []
+    step = size - overlap
+    for i in range(0, max(len(lines), 1), step):
+        part = lines[i : i + size]
+        text = "\n".join(part)
+        if len(text) > MAX_CHUNK_CHARS:
+            if size == 1:
+                raise LineTooLong(
+                    f"line exceeds {MAX_CHUNK_CHARS} characters: {path}:{first_line + i}"
+                )
+            out.extend(
+                windows(part, path, first_line + i, max(1, size // 2), 0, symbol)
+            )
+        elif text.strip():
+            out.append(
+                Chunk(
+                    path, first_line + i, first_line + i + len(part) - 1, text, symbol
+                )
+            )
+        if i + size >= len(lines):
+            break
+    return out
+
+
+def _begin(node: ast.AST) -> int:
+    decorators = getattr(node, "decorator_list", [])
+    return min([node.lineno, *(d.lineno for d in decorators)])
+
+
+def _partition(node: ast.AST, start: int, end: int, symbol, depth: int = 0):
+    """Split a long body into its blocks, covering every line of [start, end]."""
+    body = getattr(node, "body", None)
+    if not isinstance(body, list) or end - start < SPLIT_LINES:
+        return None
+    stmts = [s for s in body if hasattr(s, "end_lineno")]
+    if not stmts:
+        return None
+    if len(stmts) == 1:
+        return (
+            _partition(stmts[0], start, end, symbol, depth + 1) if depth < 3 else None
+        )
+
+    out: list[_Range] = []
+    cursor = start
+    group: list[ast.stmt] = []
+
+    def flush(last: bool) -> None:
+        nonlocal cursor
+        if not group:
+            return
+        stop = end if last else min(end, group[-1].end_lineno)
+        if stop >= cursor:
+            inner = None
+            if len(group) == 1 and depth < 3:
+                inner = _partition(group[0], cursor, stop, symbol, depth + 1)
+            out.extend(inner or [_Range(cursor, stop, symbol)])
+            cursor = stop + 1
+        group.clear()
+
+    for stmt in stmts:
+        span = stmt.end_lineno - _begin(stmt) + 1
+        if group and (span >= 6 or stmt.end_lineno - cursor + 1 > 20):
+            flush(False)
+        group.append(stmt)
+        if span >= 6 and stmt is not stmts[-1]:
+            flush(False)
+    flush(True)
+    return out if len(out) > 1 else None
+
+
+def _python_ranges(text: str) -> list[_Range]:
+    tree = ast.parse(text)
+    ranges: list[_Range] = []
+
+    def emit(node: ast.AST, start: int, symbol: str | None) -> None:
+        ranges.extend(
+            _partition(node, start, node.end_lineno, symbol)
+            or [_Range(start, node.end_lineno, symbol)]
+        )
+
+    for node in tree.body:
+        if isinstance(node, ast.ClassDef):
+            ranges.append(_Range(_begin(node), node.body[0].lineno - 1, node.name))
+            for member in node.body:
+                name = getattr(member, "name", "body")
+                emit(member, _begin(member), f"{node.name}.{name}")
+        else:
+            emit(node, _begin(node), getattr(node, "name", None))
+    return _merge_short_neighbours(ranges)
+
+
+def _merge_short_neighbours(ranges: list[_Range]) -> list[_Range]:
+    """Fold runs of imports/one-liners into one target (each costs a judgment)."""
+
+    def mergeable(r: _Range) -> bool:
+        return r.end - r.start <= 1 and "." not in (r.symbol or "")
+
+    merged: list[_Range] = []
+    for r in ranges:
+        last = merged[-1] if merged else None
+        if (
+            last
+            and mergeable(last)
+            and mergeable(r)
+            and r.start <= last.end + 1
+            and r.end - last.start < 12
+            and len(f"{last.symbol} {r.symbol}") < 60
+        ):
+            names = dict.fromkeys(s for s in (last.symbol, r.symbol) if s)
+            last.end, last.symbol = r.end, ", ".join(names) or None
+        else:
+            merged.append(_Range(r.start, r.end, r.symbol))
+    return merged
+
+
+def _chunks_from_ranges(
+    lines: list[str], path: str, ranges: list[_Range]
+) -> list[Chunk]:
+    out: list[Chunk] = []
+    next_line = 1
+    for r in ranges:
+        start, end = min(r.start, next_line), min(r.end, len(lines))
+        if end < start:
+            continue
+        part = lines[start - 1 : end]
+        whole = (
+            end - start < WHOLE_DECLARATION_LINES
+            and len("\n".join(part)) <= MAX_CHUNK_CHARS
+        )
+        size, overlap = (len(part), 0) if whole else (WINDOW, OVERLAP)
+        out.extend(windows(part, path, start, size, overlap, r.symbol))
+        next_line = end + 1
+    if next_line <= len(lines):
+        out.extend(windows(lines[next_line - 1 :], path, next_line))
+    return out
+
+
+def source_chunks(text: str, path: str) -> tuple[list[Chunk], str]:
+    """Chunk one file's text. Returns (chunks, parser_name)."""
+    lines = text.splitlines()
+    if path.endswith(".py"):
+        try:
+            ranges = _python_ranges(text)
+        except (SyntaxError, ValueError, RecursionError):
+            ranges = []
+        if ranges:
+            return _chunks_from_ranges(lines, path, ranges), "python"
+    return windows(lines, path), "overlapping-lines"
+
+
+def list_files(directory: str, glob: str | None = None) -> list[str]:
+    """Files ripgrep would search: honours .gitignore, skips hidden files."""
+    rg = find_ripgrep()
+    if rg is None:
+        raise RuntimeError("ripgrep (rg) is required for smart_grep.")
+    args = [rg, "--files", "-0", *(["-g", glob] if glob else []), "--", directory]
+    proc = subprocess.run(args, capture_output=True, timeout=15, check=False)
+    if proc.returncode not in (0, 1):  # 1 == no files
+        raise RuntimeError(
+            f"ripgrep discovery failed: {proc.stderr.decode(errors='replace')[:300]}"
+        )
+    files = sorted({f for f in proc.stdout.decode(errors="replace").split("\0") if f})
+    if len(files) > MAX_FILES:
+        raise RuntimeError(
+            f"Search exceeds {MAX_FILES} files. Narrow the directory or glob."
+        )
+    return files
+
+
+def discover(directory: str, glob: str | None = None) -> Discovery:
+    """Read and chunk every eligible file under ``directory``."""
+    found = Discovery()
+    files = list_files(directory, glob)
+    found.files = len(files)
+    total_bytes = 0
+    for path in files:
+        if os.path.getsize(path) > MAX_FILE_BYTES:
+            found.skipped.append((path, "larger than 1 MiB"))
+            continue
+        with open(path, "rb") as fh:
+            raw = fh.read()
+        total_bytes += len(raw)
+        if total_bytes > MAX_TOTAL_BYTES:
+            raise RuntimeError(
+                "Search exceeds 32 MiB of source. Narrow the directory or glob."
+            )
+        if b"\0" in raw:
+            found.skipped.append((path, "binary"))
+            continue
+        try:
+            text = raw.decode("utf-8")
+            chunks, parser = source_chunks(text, path)
+        except UnicodeDecodeError:
+            found.skipped.append((path, "not UTF-8"))
+            continue
+        except LineTooLong as exc:
+            found.skipped.append((path, str(exc)))
+            continue
+        found.parsers[parser] += 1
+        found.chunks.extend(chunks)
+    return found

code_puppy_core_plugins-0.0.68/code_puppy_core_plugins/jev_grep/config.py

@@ -0,0 +1,55 @@
+"""Settings for smart_grep. Set with `/set <key> <value>` or the environment.
+
+/set smart_grep on          enable (off by default)
+/set jev_api_key <key>      credentials (or TYPESAFE_API_KEY)
+/set smart_grep_model ...   pin a Jev version, e.g. jev-1.13.0
+/set smart_grep_threshold   relevance cut-off, 0-1 (default 0.5)
+"""
+
+from __future__ import annotations
+
+import os
+
+from code_puppy.config import get_api_key, get_truthy_bool_value, get_value
+
+ENABLED_KEY = "smart_grep"
+# TypeSafe's official name first, then the Jev-branded alias. Config lookup
+# is case-insensitive, so `/set jev_api_key ...` matches JEV_API_KEY.
+API_KEY_NAMES = ("TYPESAFE_API_KEY", "JEV_API_KEY")
+API_KEY_NAME = API_KEY_NAMES[0]
+DEFAULT_MODEL = "jev-latest"
+DEFAULT_THRESHOLD = 0.5
+
+
+def get_typesafe_api_key() -> str:
+    """First key found: environment beats config, official name beats alias."""
+    for name in API_KEY_NAMES:
+        if key := os.environ.get(name):
+            return key
+    for name in API_KEY_NAMES:
+        if key := get_api_key(name):
+            return key
+    return ""
+
+
+def is_enabled() -> bool:
+    """Opt-in: smart_grep sends selected source to an external service."""
+    return get_truthy_bool_value(ENABLED_KEY, False)
+
+
+def is_available() -> bool:
+    """The single gate for tool exposure, prompting and execution."""
+    return is_enabled() and bool(get_typesafe_api_key())
+
+
+def get_jev_model_name() -> str:
+    """Pin a version (e.g. ``jev-1.13.0``) once a threshold is tuned against it."""
+    return get_value("smart_grep_model") or DEFAULT_MODEL
+
+
+def get_threshold() -> float:
+    try:
+        value = float(get_value("smart_grep_threshold") or DEFAULT_THRESHOLD)
+    except ValueError:
+        return DEFAULT_THRESHOLD
+    return value if 0 <= value <= 1 else DEFAULT_THRESHOLD

code_puppy_core_plugins-0.0.68/code_puppy_core_plugins/jev_grep/judge.py

@@ -0,0 +1,102 @@
+"""Relevance judgments from Jev, TypeSafe's decision model, via Pydantic AI.
+
+Relevance is scored as the product of two independent judgments -- the snippet
+acts on the requested *entity*, and it performs the requested *operation* -- so
+mentioning a concept is not enough; the code must do the thing. A bounded
+``float`` field on a decision model is answered with the raw probability of
+yes, so there is no confidence-margin math to undo.
+
+Every snippet is judged in its own run (its own Jev state), so neighbouring
+snippets never lend each other evidence.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Sequence
+
+from pydantic import BaseModel, Field
+from pydantic_ai import Agent
+from pydantic_ai.models import Model
+
+from .chunks import Chunk
+
+DEFAULT_CONCURRENCY = 8
+REQUEST_TIMEOUT_SECONDS = 30
+
+
+class Relevance(BaseModel):
+    """Decide whether this source code is what a developer searching the codebase wants to find."""
+
+    entity: float = Field(
+        ge=0,
+        le=1,
+        description=(
+            "Does this code act on the thing the search is about "
+            "(the data, object, resource or concept it names)?"
+        ),
+    )
+    operation: float = Field(
+        ge=0,
+        le=1,
+        description=(
+            "Does this code itself perform the action or behaviour the search "
+            "describes, rather than only mentioning it, importing it, or calling "
+            "something with a similar name?"
+        ),
+    )
+
+    @property
+    def score(self) -> float:
+        return self.entity * self.operation
+
+
+def build_jev_model(model_name: str, api_key: str) -> Model:
+    """A TypeSafe Jev model. Imported lazily: the SDK is an optional extra."""
+    from pydantic_ai.models.typesafe import TypeSafeModel
+    from pydantic_ai.providers.typesafe import TypeSafeProvider
+
+    return TypeSafeModel(model_name, provider=TypeSafeProvider(api_key=api_key))
+
+
+def _state(chunk: Chunk) -> str:
+    """What Jev judges: where the snippet lives, then the snippet itself."""
+    where = f"{chunk.path}:{chunk.line}-{chunk.end_line}"
+    header = f"{where} ({chunk.symbol})" if chunk.symbol else where
+    return f"{header}\n\n{chunk.text}"
+
+
+def build_agent(model: Model, query: str) -> Agent[None, Relevance]:
+    return Agent(
+        model,
+        output_type=Relevance,
+        instructions=(
+            "The text is a snippet of source code from a repository. "
+            f"A developer is searching the codebase for: {query}"
+        ),
+        model_settings={"timeout": REQUEST_TIMEOUT_SECONDS},
+    )
+
+
+async def judge(
+    model: Model,
+    query: str,
+    chunks: Sequence[Chunk],
+    concurrency: int = DEFAULT_CONCURRENCY,
+) -> list[float]:
+    """Score each chunk in [0, 1], order preserved.
+
+    Any failed judgment fails the whole search (siblings are cancelled):
+    a partial scan reported as a complete one is worse than an error.
+    """
+    agent = build_agent(model, query)
+    gate = asyncio.Semaphore(concurrency)
+
+    async def one(chunk: Chunk) -> float:
+        async with gate:
+            result = await agent.run(_state(chunk))
+        return result.output.score
+
+    async with asyncio.TaskGroup() as group:
+        tasks = [group.create_task(one(chunk)) for chunk in chunks]
+    return [task.result() for task in tasks]

code_puppy_core_plugins-0.0.68/code_puppy_core_plugins/jev_grep/locales/en-US.json

@@ -0,0 +1,3 @@
+{
+  "smart_grep.summary": "smart_grep '{query}' in {directory}: {matches} matches ({evaluated}/{snippets} snippets judged)"
+}

code_puppy_core_plugins-0.0.68/code_puppy_core_plugins/jev_grep/register_callbacks.py

@@ -0,0 +1,54 @@
+"""smart_grep: semantic code search judged by TypeSafe's Jev decision model.
+
+Registers a ``smart_grep`` tool alongside the regular ``grep``. It is opt-in:
+agents only see the tool, and the smart_grep-first discovery policy is only
+added to the prompt, once ``/set smart_grep on`` is set AND a Jev/TypeSafe
+key is configured. ``/set`` reloads the agent, so toggling applies at once.
+"""
+
+from code_puppy.callbacks import register_callback
+
+from .config import is_available
+from .tool import TOOL_NAME, register_smart_grep
+
+
+def _register_tools():
+    return [{"name": TOOL_NAME, "register_func": register_smart_grep}]
+
+
+def _advertise_when_configured(agent_name=None):
+    return [TOOL_NAME] if is_available() else []
+
+
+def _discovery_instructions() -> str | None:
+    """Same live gate as tool advertisement; the key itself is never included."""
+    if not is_available():
+        return None
+    return """## Code discovery: smart_grep first
+When smart_grep is available, use it as your first tool for gathering code
+context about an unfamiliar implementation. Describe the behavior you need to
+find in plain English, scoped to the relevant repository or directory. Do this
+instead of guessing symbol names, running exploratory grep chains, or listing
+and reading many files just to locate the implementation.
+
+Use the returned source excerpts and line ranges to decide which files need a
+targeted read. Do not automatically read every match or repeat searches when
+the excerpts already answer the question. Read the relevant existing code before
+editing; semantic search is discovery, not a substitute for understanding it.
+
+Use regular grep for exact symbols, regexes, and exhaustive references. Read a
+known file directly when its location is already established. Use list_files
+when you actually need directory contents, not as a ritual before every search.
+
+A lexical shortlist is not exhaustive and no matches do not prove absence.
+If results are weak, rephrase or narrow the query, or use exact grep as needed.
+If smart_grep is unavailable or errors, fall back to grep and targeted reads;
+do not block the task or repeatedly retry a missing key or failing provider.
+Respect requests to keep code local: smart_grep sends selected source, paths,
+and the query to TypeSafe, so use local grep/reads for local-only work.
+"""
+
+
+register_callback("load_prompt", _discovery_instructions)
+register_callback("register_tools", _register_tools)
+register_callback("register_agent_tools", _advertise_when_configured)

code_puppy_core_plugins-0.0.68/code_puppy_core_plugins/jev_grep/retrieve.py

@@ -0,0 +1,90 @@
+"""Local BM25 shortlist: identifier-aware terms, path/symbol boost, synonyms.
+
+Cheap lexical ranking decides which snippets are worth a Jev judgment. It is
+a shortlist, not a verdict: code with unrelated wording can be missed, which
+is why coverage is reported back to the caller.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from collections.abc import Sequence
+
+from .chunks import Chunk
+
+_STOP = frozenset(
+    "a an the is are was were to of in on for from with where how does do this "
+    "that it and or code function files file find which".split()
+)
+
+_SYNONYM_GROUPS = (
+    "auth authentication authenticate login signin session",
+    "retry retries backoff attempt",
+    "cache cached caching memo memoize",
+    "delete remove cleanup unlink",
+    "permission permissions authorization authorize access allowed",
+    "expire expired expires expiration expiry timeout ttl",
+    "save persist persistence storage store write",
+    "error errors failure failed exception throw catch",
+    "payment payments billing subscription invoice charge",
+    "cancel abort cancellation",
+    "parallel concurrent concurrency simultaneous",
+    "duplicate duplicates deduplicate dedup unique",
+)
+_VOCABULARY = {
+    term: group.split() for group in _SYNONYM_GROUPS for term in group.split()
+}
+
+_CAMEL = re.compile(r"([a-z\d])([A-Z])")
+_WORD = re.compile(r"[^\W_]+")
+_SYNONYM_WEIGHT = 0.3
+_META_BOOST = 3
+_K1, _B = 1.2, 0.75
+
+
+def terms(text: str) -> list[str]:
+    """Lower-cased identifier-aware tokens: ``refreshToken`` -> refresh, token."""
+    words = _WORD.findall(_CAMEL.sub(r"\1 \2", text).lower())
+    return [w for w in words if len(w) > 1 and w not in _STOP]
+
+
+def _query_weights(query: str) -> dict[str, float]:
+    weights = dict.fromkeys(terms(query), 1.0)
+    for term in list(weights):
+        for synonym in _VOCABULARY.get(term, ()):
+            weights.setdefault(synonym, _SYNONYM_WEIGHT)
+    return weights
+
+
+def rank(query: str, chunks: Sequence[Chunk]) -> list[Chunk]:
+    """Chunks ordered by BM25 relevance to ``query`` (stable on ties)."""
+    weights = _query_weights(query)
+    docs: list[tuple[Counter, int]] = []
+    for chunk in chunks:
+        body, meta = terms(chunk.text), terms(f"{chunk.path} {chunk.symbol or ''}")
+        freq = Counter(body)
+        for word in meta:
+            freq[word] += _META_BOOST
+        docs.append((freq, len(body) + len(meta)))
+
+    n = len(docs)
+    average = (sum(length for _, length in docs) / n if n else 0) or 1
+    df = {t: sum(1 for freq, _ in docs if t in freq) for t in weights}
+
+    def score(doc: tuple[Counter, int]) -> float:
+        freq, length = doc
+        total = 0.0
+        for word, weight in weights.items():
+            tf = freq.get(word, 0)
+            if not tf:
+                continue
+            idf = math.log(1 + (n - df[word] + 0.5) / (df[word] + 0.5))
+            norm = _K1 * (1 - _B + _B * length / average)
+            total += weight * idf * tf * (_K1 + 1) / (tf + norm)
+        return total
+
+    scored = [(score(doc), chunk) for doc, chunk in zip(docs, chunks)]
+    scored.sort(key=lambda sc: (-sc[0], sc[1].path, sc[1].line))
+    return [chunk for _, chunk in scored]