SourceIndex 0.1.0__py3-none-any.whl

sourceindex/build/linerange/treesitter.py

@@ -0,0 +1,397 @@
+"""Tree-sitter line-range fixer for non-Python languages.
+
+Mirrors what ``fix_python_line_ranges`` (in the sibling ``python_ast``
+module) does for Python via the stdlib ``ast`` module: replaces LLM-emitted
+(Lstart-Lend) tuples with parser-truth ranges so the tier-2 details file
+points the agent's Read at the correct lines.
+
+Each grammar is shipped as its own per-language wheel
+(``tree-sitter-go``, ``tree-sitter-rust``, …) — a few hundred KB each — and
+declared as default deps in ``pyproject.toml``. The bulk
+``tree-sitter-language-pack`` (~180MB) was dropped because it bundled
+50+ grammars when we only use 9.
+
+Languages added to ``LANG_CFG`` later that aren't yet shipped get an
+optional dynamic-install path: set ``SOURCEINDEX_AUTO_INSTALL_GRAMMARS=1``
+and a missing grammar will be ``pip install``-ed on first use, then
+imported. Off by default so ``sourceindex`` never makes surprise network
+calls.
+
+Design notes:
+- One generic walker, parameterized by per-language definition node types
+  + name-extraction heuristics. ~50 LOC of language-specific config.
+- LLM tier-2 entries are matched by qualified name with progressive suffix
+  fallback (e.g. ``mypkg.Foo.bar`` → ``Foo.bar`` → ``bar``) and well-known
+  trailing-suffix normalization (``.get``/``.set``/``.<init>``/``.constructor``/
+  ``.closure``/``.lambda``) so package prefixes the LLM invents and accessor
+  syntax don't sink the match rate.
+- When tree-sitter finds no match for a name, we leave the LLM's range as-is
+  rather than dropping the entry. The agent still gets the description and
+  rough location even when the parser can't anchor it exactly.
+"""
+from __future__ import annotations
+
+import importlib
+import os
+import re
+import subprocess
+import sys
+from typing import Dict, List, Optional, Tuple
+
+from ...lib.log import get_logger
+
+_log = get_logger(__name__)
+
+# (sourceindex language key) ->
+#   (tree-sitter grammar name, definition node types,
+#    python module attr returning the Language handle)
+# The attr is usually ``language``; typescript's wheel exposes
+# ``language_typescript`` and ``language_tsx`` instead.
+LANG_CFG: Dict[str, Tuple[str, Tuple[str, ...], str]] = {
+    "go":     ("go",         ("function_declaration","method_declaration","type_declaration","type_spec","var_declaration","const_declaration"), "language"),
+    "typescript": ("typescript", ("function_declaration","method_definition","class_declaration","interface_declaration","type_alias_declaration","lexical_declaration","function_signature","method_signature","abstract_method_signature","public_field_definition","enum_declaration"), "language_typescript"),
+    "javascript": ("javascript", ("function_declaration","method_definition","class_declaration","lexical_declaration","field_definition"), "language"),
+    "rust":   ("rust",       ("function_item","impl_item","struct_item","trait_item","enum_item","mod_item","static_item","const_item","type_item","macro_definition"), "language"),
+    "cpp":    ("cpp",        ("function_definition","class_specifier","struct_specifier","namespace_definition","field_declaration","template_declaration","declaration","enum_specifier"), "language"),
+    "c":      ("c",          ("function_definition","struct_specifier","enum_specifier","declaration"), "language"),
+    "java":   ("java",       ("class_declaration","method_declaration","constructor_declaration","field_declaration","interface_declaration","enum_declaration","record_declaration"), "language"),
+    "ruby":   ("ruby",       ("class","module","method","singleton_method"), "language"),
+    "kotlin": ("kotlin",     ("class_declaration","function_declaration","property_declaration","object_declaration","secondary_constructor","companion_object"), "language"),
+}
+
+_IDENT_TYPES = {
+    "identifier","type_identifier","field_identifier","scoped_identifier",
+    "property_identifier","simple_identifier","constant",
+}
+
+_CONTAINER_TYPES = {
+    "class_declaration","class_specifier","struct_specifier","namespace_definition",
+    "impl_item","trait_item","mod_item","object_declaration","companion_object",
+    "interface_declaration","class","module","enum_declaration","enum_specifier",
+    "record_declaration",
+}
+
+_TRAILING_SUFFIXES = (".get",".set",".<init>",".init",".constructor",".ctor",".closure",".lambda",".function_")
+
+_ENTRY_RE = re.compile(
+    r"^(?P<lead>\s*)(?P<name>[^()]+?)\s*\(\s*L?(?P<s>\d+)\s*-\s*L?(?P<e>\d+)\s*\)\s*:(?P<rest>.*)$"
+)
+_ENTRY_SINGLE_RE = re.compile(
+    r"^(?P<lead>\s*)(?P<name>[^()]+?)\s*\(\s*L?(?P<line>\d+)\s*\)\s*:(?P<rest>.*)$"
+)
+
+
+_AUTO_INSTALL_ENV = "SOURCEINDEX_AUTO_INSTALL_GRAMMARS"
+
+# Per-grammar caches: parser objects (hot path) and warned-once set (so a
+# missing package doesn't spam stderr once per file in a multi-thousand-file
+# repo). Keyed by sourceindex language key, not grammar name, since LANG_CFG
+# is the source of truth for both.
+_PARSER_CACHE: Dict[str, object] = {}
+_WARNED_GRAMMARS: set[str] = set()
+
+
+def _warn_missing_grammar(language_key: str, grammar: str, exc: BaseException) -> None:
+    if language_key in _WARNED_GRAMMARS:
+        return
+    _WARNED_GRAMMARS.add(language_key)
+    pkg = f"tree-sitter-{grammar}"
+    _log.warning(
+        f"grammar package '{pkg}' not available for {language_key} "
+        f"({type(exc).__name__}); tier-2 line ranges for this language will use "
+        f"LLM output as-is (less accurate). Install with `pip install {pkg}` "
+        f"or set {_AUTO_INSTALL_ENV}=1 to auto-install on first use.",
+        extra={
+            "event": "tree_sitter.missing_grammar",
+            "language": language_key,
+            "package": pkg,
+            "error_type": type(exc).__name__,
+        },
+    )
+
+
+def _try_pip_install(pkg: str) -> bool:
+    """Best-effort `pip install` of a single package. Returns True on success.
+
+    Gated by the caller — never invoked unless ``SOURCEINDEX_AUTO_INSTALL_GRAMMARS=1``
+    so default usage makes no network calls.
+    """
+    _log.info(
+        f"{_AUTO_INSTALL_ENV}=1: installing {pkg}…",
+        extra={"event": "tree_sitter.install_attempt", "package": pkg},
+    )
+    try:
+        subprocess.run(
+            [sys.executable, "-m", "pip", "install", "--quiet", pkg],
+            check=True,
+        )
+        return True
+    except (subprocess.CalledProcessError, FileNotFoundError) as e:
+        _log.error(
+            f"auto-install of {pkg} failed: {e}",
+            extra={
+                "event": "tree_sitter.install_failed",
+                "package": pkg,
+                "error_type": type(e).__name__,
+                "error": str(e),
+            },
+        )
+        return False
+
+
+def _get_parser(language_key: str) -> Optional[object]:
+    """Resolve and cache a tree-sitter Parser for a sourceindex language key.
+
+    Returns ``None`` (and warns once) if the per-language wheel is missing
+    and either auto-install is disabled or the install attempt failed.
+    """
+    cached = _PARSER_CACHE.get(language_key)
+    if cached is not None:
+        return cached
+    if language_key not in LANG_CFG:
+        return None
+    grammar, _def_types, lang_attr = LANG_CFG[language_key]
+    module_name = f"tree_sitter_{grammar}"
+
+    def _import() -> Optional[object]:
+        try:
+            return importlib.import_module(module_name)
+        except ImportError:
+            return None
+
+    mod = _import()
+    if mod is None and os.environ.get(_AUTO_INSTALL_ENV, "").lower() in ("1", "true", "yes"):
+        if _try_pip_install(f"tree-sitter-{grammar}"):
+            mod = _import()
+    if mod is None:
+        _warn_missing_grammar(language_key, grammar, ImportError(module_name))
+        return None
+    try:
+        from tree_sitter import Language, Parser  # core lib is a hard dep
+        lang_fn = getattr(mod, lang_attr)
+        parser = Parser(Language(lang_fn()))
+    except Exception as e:
+        _warn_missing_grammar(language_key, grammar, e)
+        return None
+    _PARSER_CACHE[language_key] = parser
+    return parser
+
+
+def _normalize_name(qn: str) -> str:
+    """Normalize separators to ``.`` and strip accessor/ctor/closure suffixes.
+
+    Also drops whitespace-prefixed language keywords the LLM sometimes folds
+    into the qualified name (``func Foo``, ``pub fn bar``, ``def quux``).
+    """
+    qn = qn.replace("::", ".").replace("#", ".").strip()
+    # Keep only the last whitespace-separated token: the actual identifier path
+    if " " in qn:
+        qn = qn.split()[-1]
+    while qn.endswith(_TRAILING_SUFFIXES):
+        for suf in _TRAILING_SUFFIXES:
+            if qn.endswith(suf):
+                qn = qn[: -len(suf)]
+                break
+    while ".." in qn:
+        qn = qn.replace("..", ".")
+    return qn.strip(".")
+
+
+def _bare_name(qn: str) -> str:
+    qn = _normalize_name(qn)
+    return qn.rsplit(".", 1)[-1]
+
+
+def _walk_defs(parser, src: bytes, def_types: Tuple[str, ...]):
+    """Yield ``(qualified_name, start_line, end_line)`` for every definition node."""
+    tree = parser.parse(src)
+
+    def text(n) -> str:
+        return src[n.start_byte:n.end_byte].decode("utf-8", errors="replace")
+
+    def first_child_of_types(n, types):
+        for c in n.children:
+            if c.type in types:
+                return c
+        return None
+
+    def deep_first_identifier(n):
+        stack = [n]
+        while stack:
+            cur = stack.pop()
+            if cur.type in _IDENT_TYPES:
+                return cur
+            stack.extend(reversed(cur.children))
+        return None
+
+    def name_of(node) -> Optional[str]:
+        n = node.child_by_field_name("name")
+        if n is not None:
+            if n.type in _IDENT_TYPES:
+                return text(n)
+            inner = deep_first_identifier(n)
+            return text(inner) if inner else None
+        t = node.type
+        if t == "lexical_declaration":          # TS/JS: const x = ...
+            vd = first_child_of_types(node, {"variable_declarator"})
+            if vd:
+                nm = vd.child_by_field_name("name")
+                return text(nm) if nm else None
+        if t == "property_declaration":          # Kotlin
+            vd = first_child_of_types(node, {"variable_declaration"})
+            if vd:
+                inner = deep_first_identifier(vd)
+                return text(inner) if inner else None
+        if t in ("field_declaration", "declaration", "function_definition"):  # C/C++
+            decl = node.child_by_field_name("declarator")
+            if decl is not None:
+                inner = deep_first_identifier(decl)
+                return text(inner) if inner else None
+        if t in ("singleton_method", "method"):
+            inner = deep_first_identifier(node)
+            return text(inner) if inner else None
+        inner = deep_first_identifier(node)
+        return text(inner) if inner else None
+
+    def walk(node, prefix: str = ""):
+        new_prefix = prefix
+        if node.type in def_types:
+            nm = name_of(node)
+            if nm:
+                full = f"{prefix}.{nm}" if prefix else nm
+                yield (full, node.start_point[0] + 1, node.end_point[0] + 1)
+                if node.type in _CONTAINER_TYPES:
+                    new_prefix = full
+        for c in node.children:
+            yield from walk(c, new_prefix)
+
+    yield from walk(tree.root_node)
+
+
+def _build_truth(parser, src: bytes, def_types: Tuple[str, ...]) -> Dict[str, List[Tuple[int, int]]]:
+    """Return {qualified_name: [(start_line, end_line), ...]} for every definition.
+
+    Also populates a bare-name fallback (last segment) so LLM entries that
+    drop package/class prefixes still match. ``_lookup`` prefers qualified.
+    """
+    truth: Dict[str, List[Tuple[int, int]]] = {}
+    for full, s, e in _walk_defs(parser, src, def_types):
+        truth.setdefault(full, []).append((s, e))
+        bare = full.rsplit(".", 1)[-1]
+        if bare != full:
+            truth.setdefault(bare, []).append((s, e))
+    return truth
+
+
+def collect_skeleton(language_key: str, source: str) -> List[Tuple[str, int, int]]:
+    """Return AST-derived definition list for prompt pre-processing.
+
+    Returns sorted ``[(qualified_name, start_line, end_line), ...]`` suitable
+    for embedding in the LLM build prompt as authoritative structure. Empty
+    list signals fall-through (language unsupported, package missing, parse
+    failure) so callers can use the original free-form prompt.
+    """
+    if language_key not in LANG_CFG:
+        return []
+    parser = _get_parser(language_key)
+    if parser is None:
+        return []
+    _grammar, def_types, _attr = LANG_CFG[language_key]
+    try:
+        defs = list(_walk_defs(parser, source.encode("utf-8"), def_types))
+    except Exception as e:
+        _log.warning(
+            f"tree-sitter skeleton failed for {language_key}: {e}",
+            extra={
+                "event": "tree_sitter.parse_failed",
+                "phase": "skeleton",
+                "language": language_key,
+                "error_type": type(e).__name__,
+                "error": str(e),
+            },
+        )
+        return []
+    # Dedup by (name, start, end) — same qualified name can appear twice via
+    # cfg-conditional impls in Rust; we want both.
+    seen = set()
+    out = []
+    for entry in defs:
+        if entry in seen:
+            continue
+        seen.add(entry)
+        out.append(entry)
+    out.sort(key=lambda x: x[1])
+    return out
+
+
+def _lookup(truth: Dict[str, List[Tuple[int, int]]], qn: str) -> List[Tuple[int, int]]:
+    """Match by qualified name; if missing, peel leading segments off; finally bare."""
+    norm = _normalize_name(qn)
+    if not norm:
+        return []
+    if norm in truth:
+        return truth[norm]
+    parts = norm.split(".")
+    for i in range(1, len(parts)):
+        suffix = ".".join(parts[i:])
+        if suffix in truth:
+            return truth[suffix]
+    bare = parts[-1]
+    return truth.get(bare, [])
+
+
+def fix_line_ranges(language_key: str, source: str, functions_text: str) -> str:
+    """Replace LLM-emitted line ranges with tree-sitter truth where possible.
+
+    Returns ``functions_text`` unchanged on:
+      - language not in ``LANG_CFG`` (no grammar configured),
+      - per-language tree-sitter wheel not installed (one-time warning),
+      - parse failure or any unexpected error (best-effort, never raise).
+    """
+    if language_key not in LANG_CFG:
+        return functions_text
+    if not functions_text.strip():
+        return functions_text
+    parser = _get_parser(language_key)
+    if parser is None:
+        return functions_text
+
+    _grammar, def_types, _attr = LANG_CFG[language_key]
+    try:
+        truth = _build_truth(parser, source.encode("utf-8"), def_types)
+    except Exception as e:  # parser failure, grammar missing, etc.
+        _log.warning(
+            f"tree-sitter parse failed for {language_key}: {e}",
+            extra={
+                "event": "tree_sitter.parse_failed",
+                "phase": "fix_line_ranges",
+                "language": language_key,
+                "error_type": type(e).__name__,
+                "error": str(e),
+            },
+        )
+        return functions_text
+
+    out_lines: List[str] = []
+    for line in functions_text.splitlines():
+        m = _ENTRY_RE.match(line)
+        single = False
+        if not m:
+            m = _ENTRY_SINGLE_RE.match(line)
+            single = bool(m)
+        if not m:
+            out_lines.append(line)
+            continue
+        name = m.group("name").strip()
+        reported_start = int(m.group("line") if single else m.group("s"))
+        cands = _lookup(truth, name)
+        if not cands:
+            out_lines.append(line)
+            continue
+        best = min(cands, key=lambda c: abs(c[0] - reported_start))
+        if single:
+            line = f"{m.group('lead')}{name} (L{best[0]}):{m.group('rest')}"
+        else:
+            line = f"{m.group('lead')}{name} (L{best[0]}-L{best[1]}):{m.group('rest')}"
+        out_lines.append(line)
+    return "\n".join(out_lines)

sourceindex/build/prompts.py

@@ -0,0 +1,76 @@
+"""LLM prompts used by the build pipeline.
+
+INDEX_PROMPT_TEMPLATE — full tier-1 + tier-2 indexing for languages with
+                         build_tier2=True (the default).
+ONELINER_PROMPT_TEMPLATE — lighter prompt for misc/text/config/docs files
+                           that only need a tier-1 summary.
+"""
+
+from ..lib.languages import Language
+
+
+INDEX_PROMPT_TEMPLATE = """You are indexing a {language_name} source file for a codebase pseudo-index.
+
+File: {relative_path}
+Content:
+{file_content}
+
+Produce two sections in this exact format:
+
+ONELINER: <one-line description under 80 characters>
+
+FUNCTIONS:
+<one entry per line, exactly: QualifiedName (Lstart-Lend): one-line purpose>
+
+COMPLETENESS RULE — list EVERY definition in this file at every nesting level. For {language_name}, that includes every class/struct/trait/impl/namespace, every method inside every class (including dunder methods like __init__, __repr__, __eq__, __hash__, __new__), every free function, every property, every nested helper function. Methods MUST appear as separate lines IN ADDITION to the class line. A class with 10 methods produces 11 entries (1 for the class + 10 for the methods), not 1. Never omit an entry just because its behavior is hard to describe — write a brief honest line and move on.
+
+DESCRIPTION RULES — different rubrics for different entry kinds:
+
+* **CLASS / type entries** (classes, structs, traits, impls, namespaces) — describe the class's PURPOSE and ROLE, not behavior. Answer "what is this class for, what's its main interface, who uses it?" One sentence is enough. Class entries are NEVER optional, even when their methods are listed individually right after.
+
+* **METHOD / function entries** — describe WHAT THE CODE DOES (its implementation behavior). For meaty methods, name concrete mechanisms: literal SQL fragments (LIKE, COALESCE, IN), normalization forms (NFC), encoding tricks (BYTELOWER, BLOB, escape sequences), algorithm names, returned exception types, edge-case branches. For genuinely trivial methods (plain attribute store, pass-through, inherited dunder, default __repr__), a 4-8 word HONEST description is fine — brief but truthful beats omission.
+
+* **Distinguish siblings** — two classes' __init__s must each say what they uniquely do. Two classes' col_clause methods must each name the actual SQL they emit, not "returns SQL clause".
+
+GOOD class entries (purpose-level, role-oriented):
+  Query (L75-L115): Abstract base for SQL queries; subclasses provide clause and match.
+  RegexpQuery (L325-L362): FieldQuery using compiled regex; raises on invalid patterns.
+  AndQuery (L521-L527): CollectionQuery joining sub-queries with SQL AND and Python all().
+  TrueQuery (L641-L648): Always-match query — returns ('1', ()) and matches every model.
+
+GOOD method entries (concrete mechanism):
+  Bar.col_clause (L40-L45): Builds `field LIKE ? ESCAPE '\\\\'` with backslash-escaped wildcards.
+  PathQuery.match (L100-L110): Compares model.path BLOB against case-folded directory prefix.
+  RegexpQuery.__init__ (L327-L338): Compiles pattern, raises InvalidQueryArgumentValueError on re.error.
+
+GOOD trivial-method entries (brief but honest — DON'T omit):
+  Foo.__repr__ (L48-L49): Default repr from dataclass.
+  ParsingError.__init__ (L46-L47): Inherits Exception.__init__.
+  Query.__hash__ (L106-L110): Hashes by exact type to match __eq__ semantics.
+
+BAD (avoid these patterns):
+  Bar.col_clause (L40-L45): Returns the SQL clause for the field.   ← vague restatement
+  PathQuery.match (L100-L110): Checks whether a path matches.       ← restates the name
+  Query.__init__ (L77-L78): Initializes the Query.                  ← says nothing
+
+Use the natural qualification for {language_name} (e.g. ClassName.method, Namespace::function, impl_target::method).
+Each purpose must be under 100 characters. Use the literal line numbers from the file.
+
+Output ONLY the two sections. No markdown headers, no commentary."""
+
+# Lightweight prompt for files with no code structure (docs/config/text).
+ONELINER_PROMPT_TEMPLATE = """You are summarizing a {language_name} file for a codebase pseudo-index.
+
+File: {relative_path}
+Content:
+{file_content}
+
+Respond in this exact format:
+
+ONELINER: <one-line description under 80 characters>
+
+Output ONLY that single line. No markdown, no commentary."""
+
+
+def prompt_for(language: Language) -> str:
+    return INDEX_PROMPT_TEMPLATE if language.build_tier2 else ONELINER_PROMPT_TEMPLATE