rag-your-code 0.4.1__py3-none-any.whl

ragyourcode/descriptions.py

@@ -0,0 +1,248 @@
+"""Agent-authored unit descriptions.
+
+`annotate.py` says it in its own first line: it describes a unit *without an
+LLM*. It humanises the identifier, lists parameter and callee names, and
+appends the docstring verbatim -- so it introduces no vocabulary that was not
+already in the source. That is precisely why retrieval cannot reach a concept
+the author never wrote down: the embedder is a feature hash, so cosine measures
+token overlap, and a query sharing no token with a unit scores exactly zero
+against it.
+
+The agent already consuming this index can write those words. This module
+stores what it writes, keyed by unit id **and a digest of the unit's source**,
+so a description can never be applied to code it does not describe. When the
+source moves, the entry is retained but not used, and the unit reappears in the
+pending queue; incremental indexing means only the units in changed files do.
+
+Stored at the repository root as ``rag-your-code.descriptions.json``, beside
+``rag-your-code.toml`` and for the same two reasons: it is authored rather than
+generated, and ``.rag-your-code/`` is both ignored by Git and the directory
+people delete to clear the cache.
+
+What this is, and is not: it moves the semantic work from query time to index
+time. Matching stays lexical. A description saying `retry` still cannot answer
+a query saying `resend` unless the description also says `resend`, which is why
+the guidance handed to the agent asks for the words a reader would search by
+rather than a restatement of the code.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from .models import CodeUnit
+
+STORE_FILENAME = "rag-your-code.descriptions.json"
+SCHEMA = 1
+
+
+def source_key(unit: CodeUnit) -> str:
+    """Digest of the exact text a description was written about."""
+    return hashlib.sha256(unit.source.encode("utf-8")).hexdigest()[:16]
+
+
+def guidance(languages: tuple[str, ...] | list[str], max_chars: int) -> str:
+    """The instruction handed to the agent with every pending batch.
+
+    It is returned by the protocol rather than living only in SKILL.md so that
+    an agent reaching this index through any host receives the same brief, and
+    so the brief cannot drift away from the ``describe.*`` settings that shape
+    it.
+    """
+    names = {"en": "English", "zh": "Chinese"}
+    written = ", ".join(names.get(code, code) for code in languages)
+    return (
+        f"Write one description per unit, in {written}, at most {max_chars} characters total. "
+        "Retrieval over these descriptions is lexical: a query matches a unit only when they "
+        "share words. So write the words a person would search by -- what the unit is for in "
+        "domain terms, the operation it performs, the failure it handles, and the obvious "
+        "synonyms for each. Do not restate the signature, do not name the parameters, and do "
+        "not describe behaviour the source does not show; the source is included so you can "
+        "check. If a unit is trivial, say so briefly rather than padding it."
+    )
+
+
+@dataclass(slots=True)
+class DescriptionStore:
+    """Authored descriptions, addressed by unit id."""
+
+    path: Path
+    entries: dict[str, dict]
+
+    @property
+    def fingerprint(self) -> str:
+        """Digest of the authored text, so an index can tell when it changed.
+
+        Importing descriptions touches no source file, so nothing the index
+        tracks moves and it would go on serving the previous text until someone
+        happened to rebuild. This is the same failure the configuration
+        fingerprint exists to prevent, in a second authored input.
+        """
+        # An empty store is the empty string rather than the digest of no
+        # bytes, so that it equals what `index_descriptions_fingerprint` reads
+        # out of an index written before this field existed. Otherwise every
+        # index predating 0.4.0 reports itself permanently stale over
+        # descriptions that neither it nor the repository has.
+        if not self.entries:
+            return ""
+        digest = hashlib.sha256()
+        for uid, entry in sorted(self.entries.items()):
+            digest.update(uid.encode("utf-8"))
+            digest.update(str(entry.get("hash", "")).encode("utf-8"))
+            digest.update(str(entry.get("text", "")).encode("utf-8"))
+        return digest.hexdigest()
+
+    def _relocations(self) -> dict[tuple[str, str], str | None]:
+        """Stored text addressed by file and code digest rather than by unit id.
+
+        A unit id embeds the line the declaration starts on, so inserting a
+        comment or an import near the top of a file changes the id of
+        everything below it while changing none of their code. Keyed only by
+        id, every description in that file would be orphaned by an edit that
+        did not touch a single one of the things they describe -- which is
+        what adding a seven-line comment to config.py did to nineteen of them.
+
+        The digest already answers "is this the same code?"; this uses it to
+        answer "where did that code go?" as well. Two units in one file with
+        byte-identical source and different stored text are ambiguous, and map
+        to nothing rather than to a guess.
+        """
+        table: dict[tuple[str, str], str | None] = {}
+        for entry in self.entries.values():
+            path, digest = entry.get("path"), entry.get("hash")
+            if not isinstance(path, str) or not isinstance(digest, str):
+                continue
+            text = entry.get("text")
+            key = (path, digest)
+            if key in table and table[key] != text:
+                table[key] = None
+            else:
+                table.setdefault(key, text)
+        return table
+
+    def applicable(self, units: list[CodeUnit]) -> dict[str, str]:
+        """Descriptions whose digest still matches the unit they describe."""
+        relocated = self._relocations()
+        usable: dict[str, str] = {}
+        for unit in units:
+            digest = source_key(unit)
+            entry = self.entries.get(unit.id)
+            text = entry.get("text") if entry and entry.get("hash") == digest else None
+            if text is None:
+                text = relocated.get((unit.path, digest))
+            if isinstance(text, str) and text.strip():
+                usable[unit.id] = text
+        return usable
+
+    def classify(self, units: list[CodeUnit]) -> dict[str, list[CodeUnit]]:
+        """Split units into described / superseded / missing.
+
+        ``superseded`` is the interesting one: something was written about this
+        declaration and the code has since changed, so it is deliberately not
+        applied. It is distinguished from ``missing`` by name rather than by
+        id, for the same reason applicability is: an id changes when the lines
+        above it do.
+        """
+        usable = self.applicable(units)
+        written = {(entry.get("path"), entry.get("name")) for entry in self.entries.values()}
+        described: list[CodeUnit] = []
+        superseded: list[CodeUnit] = []
+        missing: list[CodeUnit] = []
+        for unit in units:
+            if unit.id in usable:
+                described.append(unit)
+            elif self.entries.get(unit.id) or (unit.path, unit.qualified_name) in written:
+                superseded.append(unit)
+            else:
+                missing.append(unit)
+        return {"described": described, "superseded": superseded, "missing": missing}
+
+    def pending(self, units: list[CodeUnit], limit: int) -> list[CodeUnit]:
+        """Units still needing a description, missing ones before stale ones.
+
+        A unit with no description at all is worth more than a refresh of one
+        that exists, so a budget-limited agent spends its first batches where
+        retrieval is currently blind.
+        """
+        groups = self.classify(units)
+        return (groups["missing"] + groups["superseded"])[: max(0, limit)]
+
+    def put(self, unit: CodeUnit, text: str) -> None:
+        self.entries[unit.id] = {
+            "hash": source_key(unit),
+            "path": unit.path,
+            "name": unit.qualified_name,
+            "text": text,
+        }
+
+    def save(self, units: list[CodeUnit] | None = None) -> None:
+        """Write the store, dropping entries for units that no longer exist.
+
+        Pruning needs the full unit list to be safe, so it only happens when a
+        caller supplies one; a partial list would silently discard the
+        descriptions of everything it omitted.
+
+        An entry is kept when its id is still live *or* its code is, since a
+        declaration that merely moved down the file has a new id and the same
+        digest. Pruning on ids alone would have deleted exactly the entries
+        the relocation lookup exists to rescue.
+        """
+        if units is not None:
+            live_ids = {unit.id for unit in units}
+            live_code = {(unit.path, source_key(unit)) for unit in units}
+            self.entries = {
+                uid: entry
+                for uid, entry in self.entries.items()
+                if uid in live_ids or (entry.get("path"), entry.get("hash")) in live_code
+            }
+        payload = {
+            "schema": SCHEMA,
+            "descriptions": dict(sorted(self.entries.items())),
+        }
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        temp = self.path.with_name(f"{self.path.name}.tmp")
+        with temp.open("w", encoding="utf-8", newline="\n") as stream:
+            json.dump(payload, stream, ensure_ascii=False, indent=2, sort_keys=False)
+            stream.write("\n")
+        temp.replace(self.path)
+
+
+def index_descriptions_fingerprint(payload: dict) -> str:
+    """The descriptions digest an index was published with.
+
+    An index written before 0.4.0 has no such key, which means the same thing
+    as an empty store: no authored description was applied.
+    """
+    stored = payload.get("descriptions_fingerprint")
+    return stored if isinstance(stored, str) else ""
+
+
+def store_path(root: Path) -> Path:
+    return root / STORE_FILENAME
+
+
+def load(root: Path) -> DescriptionStore:
+    """Read the store, treating an unusable file as empty rather than fatal.
+
+    This file is committed, so it will meet merge conflicts and hand-editing. A
+    malformed one costs retrieval quality, which is recoverable by describing
+    again; refusing to search until it is repaired would not be.
+    """
+    path = store_path(root)
+    entries: dict[str, dict] = {}
+    if path.is_file():
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+            stored = payload.get("descriptions") if isinstance(payload, dict) else None
+            if isinstance(stored, dict):
+                entries = {
+                    str(uid): entry
+                    for uid, entry in stored.items()
+                    if isinstance(entry, dict) and isinstance(entry.get("text"), str)
+                }
+        except (OSError, ValueError):
+            entries = {}
+    return DescriptionStore(path, entries)

ragyourcode/embeddings.py

@@ -0,0 +1,60 @@
+"""Dependency-free deterministic embeddings.
+
+The default hasher is deliberately local and reproducible. It is useful for
+small/medium repositories and can later be replaced by an API or sentence
+transformer without changing the index schema.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import re
+from collections.abc import Sequence
+
+TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|[\u4e00-\u9fff]+|\d+")
+CJK_RE = re.compile(r"^[\u4e00-\u9fff]+$")
+DEFAULT_DIMENSIONS = 384
+EMBEDDING_PROVIDER = "signed-feature-hash"
+EMBEDDING_VERSION = 1
+
+
+def embedding_metadata(dimensions: int = DEFAULT_DIMENSIONS) -> dict[str, object]:
+    return {"provider": EMBEDDING_PROVIDER, "version": EMBEDDING_VERSION, "dimensions": dimensions}
+
+
+def tokenize(text: str) -> list[str]:
+    tokens: list[str] = []
+    for raw in TOKEN_RE.findall(text):
+        token = raw.lower()
+        tokens.append(token)
+        if CJK_RE.fullmatch(token) and len(token) > 1:
+            tokens.extend(token[index : index + 2] for index in range(len(token) - 1))
+    return tokens
+
+
+def _bucket(token: str, dimensions: int) -> int:
+    digest = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest()
+    return int.from_bytes(digest, "big") % dimensions
+
+
+def embed(text: str, dimensions: int = DEFAULT_DIMENSIONS) -> list[float]:
+    """Return a normalized signed feature-hash vector."""
+    if dimensions < 32:
+        raise ValueError("dimensions must be at least 32")
+    tokens = tokenize(text)
+    vector = [0.0] * dimensions
+    for token in tokens:
+        index = _bucket(token, dimensions)
+        sign = 1.0 if _bucket("sign:" + token, dimensions) % 2 else -1.0
+        vector[index] += sign
+    norm = math.sqrt(sum(value * value for value in vector))
+    if norm:
+        vector = [value / norm for value in vector]
+    return vector
+
+
+def cosine(left: Sequence[float], right: Sequence[float]) -> float:
+    if not left or not right or len(left) != len(right):
+        return 0.0
+    return sum(a * b for a, b in zip(left, right))

ragyourcode/graph.py

@@ -0,0 +1,198 @@
+"""Explainable symbol graph used by graph-aware retrieval.
+
+The graph is intentionally derived from the parser's stable ``CodeUnit`` IDs.
+Edges are conservative: unresolved calls are omitted rather than guessed. This
+keeps graph expansion useful for navigation without fabricating relationships.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict, deque
+from dataclasses import dataclass
+from typing import Iterable
+
+from .models import CodeUnit, SearchResult
+from .search import DEFAULT_VECTOR_WEIGHT, SearchIndex, search
+
+
+@dataclass(frozen=True, slots=True)
+class CodeEdge:
+    source: str
+    target: str
+    kind: str
+    label: str = ""
+
+    def to_dict(self) -> dict[str, str]:
+        return {"source": self.source, "target": self.target, "kind": self.kind, "label": self.label}
+
+
+class CodeGraph:
+    """Directed code relationship graph with bounded neighborhood traversal."""
+
+    def __init__(self, units: Iterable[CodeUnit], edges: Iterable[CodeEdge] = ()):
+        self.units = {unit.id: unit for unit in units}
+        self.edges = sorted(set(edges), key=lambda edge: (edge.source, edge.kind, edge.target))
+        self._out: dict[str, list[CodeEdge]] = defaultdict(list)
+        self._in: dict[str, list[CodeEdge]] = defaultdict(list)
+        for edge in self.edges:
+            if edge.source in self.units and edge.target in self.units:
+                self._out[edge.source].append(edge)
+                self._in[edge.target].append(edge)
+
+    def neighbors(self, unit_id: str, hops: int = 1, direction: str = "both") -> list[tuple[CodeUnit, list[str]]]:
+        if unit_id not in self.units or hops <= 0:
+            return []
+        directions = {"out"} if direction == "out" else {"in"} if direction == "in" else {"out", "in"}
+        queue: deque[tuple[str, int, list[str]]] = deque([(unit_id, 0, [unit_id])])
+        seen = {unit_id}
+        found: list[tuple[CodeUnit, list[str]]] = []
+        while queue:
+            current, distance, path = queue.popleft()
+            if distance >= hops:
+                continue
+            edges: list[CodeEdge] = []
+            if "out" in directions:
+                edges.extend(self._out.get(current, []))
+            if "in" in directions:
+                edges.extend(self._in.get(current, []))
+            for edge in edges:
+                target = edge.target if edge.source == current else edge.source
+                if target in seen:
+                    continue
+                seen.add(target)
+                edge_text = f"{edge.kind}:{edge.source}->{edge.target}"
+                next_path = path + [edge_text, target]
+                found.append((self.units[target], next_path))
+                queue.append((target, distance + 1, next_path))
+        return found
+
+    def to_dict(self) -> dict[str, object]:
+        return {"edges": [edge.to_dict() for edge in self.edges]}
+
+
+def _name_indexes(units: Iterable[CodeUnit]):
+    by_name: dict[str, list[str]] = defaultdict(list)
+    by_module: dict[str, list[str]] = defaultdict(list)
+    for unit in units:
+        by_name[unit.name].append(unit.id)
+        if unit.qualified_name != unit.name:
+            by_name[unit.qualified_name].append(unit.id)
+        module = unit.path.rsplit("/", 1)[-1].rsplit(".", 1)[0]
+        by_module[module].append(unit.id)
+    return by_name, by_module
+
+
+def _call_targets(call: str, by_name: dict[str, list[str]], local_modules: set[str], unit: CodeUnit, unit_by_id: dict[str, CodeUnit]) -> list[str]:
+    """Resolve a recorded call to local unit ids, guessing only where allowed.
+
+    An exact match on the text the parser recorded always wins. Falling back to
+    the last dotted segment is a guess, and an unrestricted guess is how
+    `os.path.join` acquired a `calls` edge to an unrelated local `join` -- while
+    this module's own docstring promises that unresolved calls are omitted
+    rather than guessed. The fallback now requires the head of the dotted path
+    to be attributable to this repository: `self`/`cls`, or a module some file
+    here actually defines. `os`, `json`, `requests` and every other foreign
+    prefix resolve to nothing, as documented.
+    """
+    exact = by_name.get(call)
+    if exact:
+        return list(exact)
+    if "." not in call:
+        return []
+    head = call.split(".", 1)[0]
+    leaf = call.rsplit(".", 1)[-1]
+    if head in {"self", "cls"}:
+        # A receiver call names a sibling defined alongside this unit.
+        return [target for target in by_name.get(leaf, []) if unit_by_id[target].path == unit.path]
+    if head in local_modules:
+        return list(by_name.get(leaf, []))
+    return []
+
+
+def build_graph(units: Iterable[CodeUnit], max_edges_per_unit: int = 64) -> CodeGraph:
+    units = list(units)
+    unit_by_id = {unit.id: unit for unit in units}
+    by_name, by_module = _name_indexes(units)
+    local_modules = set(by_module)
+    edges: set[CodeEdge] = set()
+    for unit in units:
+        budget = max(1, max_edges_per_unit)
+        if unit.parent:
+            parent_ids = by_name.get(unit.parent, [])
+            same_file_parents = [target for target in parent_ids if unit_by_id[target].path == unit.path]
+            resolved_parents = same_file_parents if len(same_file_parents) == 1 else parent_ids if len(parent_ids) == 1 else []
+            for parent_id in resolved_parents:
+                edges.add(CodeEdge(parent_id, unit.id, "contains", unit.qualified_name))
+                budget -= 1
+        for call in unit.calls:
+            if budget <= 0:
+                break
+            targets = _call_targets(call, by_name, local_modules, unit, unit_by_id)
+            same_file = [target for target in targets if target != unit.id and unit_by_id[target].path == unit.path]
+            resolved = same_file if len(same_file) == 1 else targets if len(targets) == 1 else []
+            for target in resolved:
+                if target != unit.id:
+                    edges.add(CodeEdge(unit.id, target, "calls", call))
+                    budget -= 1
+        for imported in unit.imports:
+            if budget <= 0:
+                break
+            module = imported.rsplit(".", 1)[-1]
+            module_targets = by_module.get(module, [])
+            # A module import is a coarse relationship. Only materialize it
+            # when the target module has a small, unambiguous surface.
+            module_paths = {unit_by_id[target].path for target in module_targets}
+            resolved_module = module_targets if len(module_targets) <= 3 and len(module_paths) == 1 else []
+            for target in resolved_module:
+                if target != unit.id:
+                    edges.add(CodeEdge(unit.id, target, "imports", imported))
+                    budget -= 1
+                    if budget <= 0:
+                        break
+    return CodeGraph(units, edges)
+
+
+def graph_from_dict(units: Iterable[CodeUnit], data: dict[str, object] | None) -> CodeGraph:
+    raw_edges = (data or {}).get("edges", [])
+    edges: list[CodeEdge] = []
+    for edge in raw_edges if isinstance(raw_edges, list) else []:
+        if not isinstance(edge, dict):
+            continue
+        try:
+            edges.append(CodeEdge(**edge))
+        except (TypeError, ValueError):
+            continue
+    return CodeGraph(units, edges)
+
+
+def graph_search(
+    units: list[CodeUnit],
+    query: str,
+    limit: int = 8,
+    hops: int = 1,
+    graph: CodeGraph | None = None,
+    search_index: SearchIndex | None = None,
+    vector_weight: float = DEFAULT_VECTOR_WEIGHT,
+) -> list[SearchResult]:
+    """Search seeds and add bounded graph neighbors with explicit evidence."""
+    if limit <= 0:
+        return []
+    hops = min(3, max(0, hops))
+    graph = graph or build_graph(units)
+    seeds = search(units, query, max(limit * 2, 8), search_index=search_index, vector_weight=vector_weight)
+    ranked: dict[str, SearchResult] = {seed.unit.id: seed for seed in seeds}
+    if hops <= 0:
+        return seeds[:limit]
+    for seed in seeds:
+        for neighbor, path in graph.neighbors(seed.unit.id, hops=hops, direction="both"):
+            weights = {"calls": 0.7, "contains": 0.5, "imports": 0.3}
+            propagated = seed.score
+            for edge_text in path[1::2]:
+                propagated *= weights.get(edge_text.split(":", 1)[0], 0.4)
+            current = ranked.get(neighbor.id)
+            evidence = ["graph:" + " -> ".join(path)]
+            if current is None or propagated > current.score:
+                ranked[neighbor.id] = SearchResult(neighbor, propagated, [], evidence)
+            elif evidence[0] not in current.evidence:
+                current.evidence.extend(evidence)
+    return sorted(ranked.values(), key=lambda result: (-result.score, result.unit.id))[:limit]