codrspot-processor-mcp 0.1.0__py3-none-any.whl

codepreproc_client/layer2_tooling/condensation/chunking/ast_chunker.py

@@ -0,0 +1,313 @@
+"""AST-first chunking with safe fallback."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import xxhash
+
+from codepreproc_client.layer2_tooling.condensation.chunking.languages import detect_language, get_chunk_query, get_language, get_parser
+from codepreproc_client.layer1_business.git_utils import normalize_content
+from contracts.dtos import ChunkingConfig, CodeChunk, ProjectConfig
+
+_SYMBOL_RE = re.compile(r"\b([A-Za-z_][A-Za-z0-9_\.]{2,})\b")
+
+
+class AstChunker:
+    """Chunk files using tree-sitter when possible, otherwise fallback to line blocks."""
+
+    def __init__(self, chunking: ChunkingConfig) -> None:
+        self.chunking = chunking
+
+    def chunk_file(self, file_path: Path, project: ProjectConfig, git_sha: str) -> list[CodeChunk]:
+        """Chunk a source file."""
+
+        try:
+            if file_path.stat().st_size > self.chunking.max_file_size_kb * 1024:
+                return []
+            raw = file_path.read_bytes()
+        except OSError:
+            return []
+
+        if b"\x00" in raw:
+            return []
+
+        try:
+            text = normalize_content(raw.decode("utf-8"))
+        except UnicodeDecodeError:
+            return []
+
+        relative_path = str(file_path.resolve().relative_to(project.root.resolve()))
+        language = detect_language(file_path) or "text"
+        parser = get_parser(language)
+        query = get_chunk_query(language)
+        if parser is not None and query is not None:
+            chunks = self._chunk_with_tree_sitter(text, relative_path, language, project, git_sha, parser, query)
+            if chunks:
+                return chunks
+        return self._chunk_fallback(text, relative_path, language, project, git_sha)
+
+    def _chunk_with_tree_sitter(
+        self,
+        text: str,
+        relative_path: str,
+        language: str,
+        project: ProjectConfig,
+        git_sha: str,
+        parser: object,
+        query: object,
+    ) -> list[CodeChunk]:
+        """Best-effort tree-sitter chunking."""
+
+        try:
+            from tree_sitter import QueryCursor
+
+            tree = parser.parse(text.encode("utf-8"))
+            cursor = QueryCursor(query)
+            raw_matches = list(cursor.matches(getattr(tree, "root_node")))
+        except Exception:
+            return []
+
+        # matches() yields (pattern_index, {capture_name: Node | [Node]}) —
+        # each dict entry is a single matched pattern, so chunk and symbol are
+        # guaranteed to correspond to each other.
+        matches: list[tuple[object, str | None]] = []
+        for _pat_idx, cap in raw_matches:
+            chunk = cap.get("chunk")
+            sym = cap.get("symbol")
+            if chunk is None:
+                continue
+            if isinstance(chunk, list):
+                chunk = chunk[0] if chunk else None
+            if isinstance(sym, list):
+                sym = sym[0] if sym else None
+            if chunk is None:
+                continue
+            try:
+                symbol_text: str | None = sym.text.decode("utf-8") if sym is not None else None
+            except Exception:
+                symbol_text = None
+            matches.append((chunk, symbol_text))
+
+        results: list[CodeChunk] = []
+        for node, symbol in matches:
+            try:
+                body = normalize_content(node.text.decode("utf-8"))
+                start_line = node.start_point.row + 1
+            except Exception:
+                continue
+            kind = self._kind_from_node_type(getattr(node, "type", "symbol"))
+            results.extend(
+                self._build_chunks(
+                    text=body,
+                    relative_path=relative_path,
+                    language=language,
+                    project_id=project.project_id,
+                    symbol=symbol or f"{Path(relative_path).stem}:{start_line}",
+                    kind=kind,
+                    start_line=start_line,
+                    git_sha=git_sha,
+                )
+            )
+        return results
+
+    def _chunk_fallback(
+        self,
+        text: str,
+        relative_path: str,
+        language: str,
+        project: ProjectConfig,
+        git_sha: str,
+    ) -> list[CodeChunk]:
+        """Split a file into bounded blocks."""
+
+        lines = text.splitlines()
+        blocks: list[list[str]] = []
+        current: list[str] = []
+        current_len = 0
+        for line in lines:
+            line_len = len(line) + 1
+            should_break = current and (
+                current_len + line_len > self.chunking.max_chunk_chars
+                or (not line.strip() and current_len >= self.chunking.max_chunk_chars // 2)
+            )
+            if should_break:
+                blocks.append(current)
+                current = []
+                current_len = 0
+            current.append(line)
+            current_len += line_len
+        if current:
+            blocks.append(current)
+
+        chunks: list[CodeChunk] = []
+        cursor_line = 1
+        for index, block in enumerate(blocks, start=1):
+            body = "\n".join(block).strip("\n")
+            if not body.strip():
+                cursor_line += len(block)
+                continue
+            symbol = self._derive_symbol(relative_path, body, index)
+            chunks.extend(
+                self._build_chunks(
+                    text=body,
+                    relative_path=relative_path,
+                    language=language,
+                    project_id=project.project_id,
+                    symbol=symbol,
+                    kind="module",
+                    start_line=cursor_line,
+                    git_sha=git_sha,
+                )
+            )
+            cursor_line += len(block)
+        return chunks
+
+    def _build_chunks(
+        self,
+        text: str,
+        relative_path: str,
+        language: str,
+        project_id: str,
+        symbol: str,
+        kind: str,
+        start_line: int,
+        git_sha: str,
+    ) -> list[CodeChunk]:
+        """Build one or more CodeChunk instances for bounded text."""
+
+        normalized = normalize_content(text)
+        if len(normalized) <= self.chunking.max_chunk_chars:
+            return [self._make_chunk(normalized, relative_path, language, project_id, symbol, kind, start_line, git_sha, 0)]
+
+        chunks: list[CodeChunk] = []
+        lines = normalized.splitlines()
+        current: list[str] = []
+        current_len = 0
+        block_start = start_line
+        sub_index = 0
+        for line in lines:
+            line_len = len(line) + 1
+            if current and current_len + line_len > self.chunking.max_chunk_chars:
+                body = "\n".join(current)
+                chunks.append(
+                    self._make_chunk(body, relative_path, language, project_id, f"{symbol}#{sub_index}", kind, block_start, git_sha, len(current) - 1)
+                )
+                sub_index += 1
+                block_start += len(current)
+                current = []
+                current_len = 0
+            current.append(line)
+            current_len += line_len
+        if current:
+            body = "\n".join(current)
+            chunks.append(
+                self._make_chunk(body, relative_path, language, project_id, f"{symbol}#{sub_index}", kind, block_start, git_sha, len(current) - 1)
+            )
+        return chunks
+
+    def _make_chunk(
+        self,
+        body: str,
+        relative_path: str,
+        language: str,
+        project_id: str,
+        symbol: str,
+        kind: str,
+        start_line: int,
+        git_sha: str,
+        relative_end_offset: int,
+    ) -> CodeChunk:
+        """Create a chunk model."""
+
+        signature = next((line.strip() for line in body.splitlines() if line.strip()), None)
+        content_hash = xxhash.xxh64_hexdigest(f"{relative_path}\n{symbol}\n{body}")
+        deps = sorted({token for token in _SYMBOL_RE.findall(body) if token != symbol})[:32]
+        return CodeChunk(
+            chunk_id=content_hash,
+            project_id=project_id,
+            file_path=relative_path,
+            language=language,
+            symbol=symbol,
+            kind=kind,
+            start_line=start_line,
+            end_line=start_line + relative_end_offset,
+            body=body,
+            signature=signature,
+            deps=deps,
+            git_sha=git_sha,
+            content_hash=content_hash,
+        )
+
+    @staticmethod
+    def _kind_from_node_type(node_type: str) -> str:
+        _EXACT: dict[str, str] = {
+            # JavaScript / TypeScript
+            "class_declaration": "class",
+            "abstract_class_declaration": "class",
+            "interface_declaration": "class",
+            "enum_declaration": "class",
+            "type_alias_declaration": "class",
+            "function_declaration": "function",
+            "arrow_function": "function",
+            "function_expression": "function",
+            "generator_function_declaration": "function",
+            "lexical_declaration": "function",  # const foo = () => {}
+            "method_definition": "method",
+            # Python
+            "function_definition": "function",
+            "class_definition": "class",
+            # Go
+            "method_declaration": "method",
+            "type_declaration": "class",
+            # Rust
+            "function_item": "function",
+            "struct_item": "class",
+            "impl_item": "class",
+            # C#
+            "record_declaration": "class",
+            "constructor_declaration": "method",
+            "namespace_declaration": "module",
+            "file_scoped_namespace_declaration": "module",
+            # C
+            "struct_specifier": "class",
+            "enum_specifier": "class",
+            "type_definition": "class",
+            "preproc_function_def": "function",
+        }
+        if node_type in _EXACT:
+            return _EXACT[node_type]
+        # Fallback: substring signals
+        if "class" in node_type or "struct" in node_type or "interface" in node_type or "enum" in node_type:
+            return "class"
+        if "method" in node_type:
+            return "method"
+        if "function" in node_type:
+            return "function"
+        return "module"
+
+    @staticmethod
+    def _derive_symbol(relative_path: str, body: str, index: int) -> str:
+        # Python / Go / Rust / generic: def foo, class Foo, func Foo, fn foo
+        m = re.search(
+            r"^\s*(?:export\s+(?:default\s+)?)?(?:async\s+)?(?:def|class|func|fn|function\*?)\s+([A-Za-z_$][A-Za-z0-9_$]*)",
+            body, flags=re.MULTILINE,
+        )
+        if m:
+            return m.group(1)
+        # JS/TS: const/let/var foo = ... (arrow fn, function expr, class expr)
+        m = re.search(
+            r"^\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*[=:]",
+            body, flags=re.MULTILINE,
+        )
+        if m:
+            return m.group(1)
+        # TypeScript-specific declarations: interface, type, enum, abstract class
+        m = re.search(
+            r"^\s*(?:export\s+)?(?:interface|type|enum|abstract\s+class)\s+([A-Za-z_$][A-Za-z0-9_$]*)",
+            body, flags=re.MULTILINE,
+        )
+        if m:
+            return m.group(1)
+        return f"{Path(relative_path).stem}:block_{index}"

codepreproc_client/layer2_tooling/condensation/chunking/languages.py

@@ -0,0 +1,238 @@
+"""Language detection and tree-sitter query helpers."""
+
+from __future__ import annotations
+
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+LANGUAGE_EXTENSIONS: dict[str, tuple[str, ...]] = {
+    "python": (".py",),
+    "typescript": (".ts",),
+    "tsx": (".tsx",),
+    "javascript": (".js", ".mjs", ".cjs", ".jsx"),
+    "go": (".go",),
+    "rust": (".rs",),
+    "c": (".c", ".h"),
+    "csharp": (".cs",),
+    "java": (".java",),
+    "php": (".php", ".phtml"),
+    "yaml": (".yaml", ".yml"),
+    "markdown": (".md",),
+}
+
+# Arrow-function-in-const pattern reused across JS/TS grammars.
+_JS_ARROW_CONST = """
+    (lexical_declaration
+      (variable_declarator name: (identifier) @symbol value: (arrow_function))
+    ) @chunk
+    (lexical_declaration
+      (variable_declarator name: (identifier) @symbol value: (function_expression))
+    ) @chunk
+"""
+
+QUERY_STRINGS: dict[str, str] = {
+    "python": """
+        (function_definition name: (identifier) @symbol) @chunk
+        (class_definition name: (identifier) @symbol) @chunk
+    """,
+    "typescript": """
+        (function_declaration name: (identifier) @symbol) @chunk
+        (class_declaration name: (type_identifier) @symbol) @chunk
+        (method_definition name: (property_identifier) @symbol) @chunk
+        (interface_declaration name: (type_identifier) @symbol) @chunk
+        (enum_declaration name: (identifier) @symbol) @chunk
+        (type_alias_declaration name: (type_identifier) @symbol) @chunk
+    """ + _JS_ARROW_CONST,
+    "tsx": """
+        (function_declaration name: (identifier) @symbol) @chunk
+        (class_declaration name: (type_identifier) @symbol) @chunk
+        (method_definition name: (property_identifier) @symbol) @chunk
+        (interface_declaration name: (type_identifier) @symbol) @chunk
+        (enum_declaration name: (identifier) @symbol) @chunk
+    """ + _JS_ARROW_CONST,
+    "javascript": """
+        (function_declaration name: (identifier) @symbol) @chunk
+        (class_declaration name: (identifier) @symbol) @chunk
+        (method_definition name: (property_identifier) @symbol) @chunk
+    """ + _JS_ARROW_CONST,
+    "go": """
+        (function_declaration name: (identifier) @symbol) @chunk
+        (method_declaration name: (field_identifier) @symbol) @chunk
+        (type_declaration (type_spec name: (type_identifier) @symbol)) @chunk
+    """,
+    "rust": """
+        (function_item name: (identifier) @symbol) @chunk
+        (struct_item name: (type_identifier) @symbol) @chunk
+        (impl_item type: (type_identifier) @symbol) @chunk
+    """,
+    "csharp": """
+        (class_declaration name: (identifier) @symbol) @chunk
+        (interface_declaration name: (identifier) @symbol) @chunk
+        (method_declaration name: (identifier) @symbol) @chunk
+        (constructor_declaration name: (identifier) @symbol) @chunk
+        (struct_declaration name: (identifier) @symbol) @chunk
+        (enum_declaration name: (identifier) @symbol) @chunk
+        (record_declaration name: (identifier) @symbol) @chunk
+        (namespace_declaration name: (qualified_name) @symbol) @chunk
+        (namespace_declaration name: (identifier) @symbol) @chunk
+    """,
+    "java": """
+        (class_declaration name: (identifier) @symbol) @chunk
+        (interface_declaration name: (identifier) @symbol) @chunk
+        (enum_declaration name: (identifier) @symbol) @chunk
+        (method_declaration name: (identifier) @symbol) @chunk
+        (constructor_declaration name: (identifier) @symbol) @chunk
+    """,
+    "php": """
+        (function_definition name: (name) @symbol) @chunk
+        (class_declaration name: (name) @symbol) @chunk
+        (method_declaration name: (name) @symbol) @chunk
+        (interface_declaration name: (name) @symbol) @chunk
+        (trait_declaration name: (name) @symbol) @chunk
+        (enum_declaration name: (name) @symbol) @chunk
+    """,
+    "c": """
+        (function_definition
+          declarator: (function_declarator
+            declarator: (identifier) @symbol)) @chunk
+        (function_definition
+          declarator: (pointer_declarator
+            declarator: (function_declarator
+              declarator: (identifier) @symbol))) @chunk
+        (struct_specifier
+          name: (type_identifier) @symbol
+          body: (_)) @chunk
+        (enum_specifier
+          name: (type_identifier) @symbol
+          body: (_)) @chunk
+        (type_definition
+          declarator: (type_identifier) @symbol) @chunk
+        (preproc_function_def
+          name: (identifier) @symbol) @chunk
+    """,
+    "yaml": "",
+    "markdown": "",
+}
+
+
+def detect_language(file_path: Path) -> str | None:
+    """Return a supported language name from extension."""
+
+    suffix = file_path.suffix.lower()
+    for language, extensions in LANGUAGE_EXTENSIONS.items():
+        if suffix in extensions:
+            return language
+    return None
+
+
+@lru_cache(maxsize=None)
+def get_parser(language: str) -> Any | None:
+    """Return a tree-sitter parser when available."""
+
+    if language not in QUERY_STRINGS or not QUERY_STRINGS[language]:
+        return None
+    try:
+        from tree_sitter_language_pack import get_parser as ts_get_parser
+        return ts_get_parser(language)
+    except Exception:
+        pass
+    try:
+        from tree_sitter_languages import get_parser as ts_get_parser
+        return ts_get_parser(language)
+    except Exception:
+        pass
+    # Fallback: build a parser from the DLL-loaded language when the
+    # tree_sitter_languages package API is incompatible with the installed
+    # tree-sitter binding version.
+    lang = _get_language_via_dll(language)
+    if lang is None:
+        return None
+    try:
+        from tree_sitter import Parser
+        return Parser(lang)
+    except Exception:
+        return None
+
+
+@lru_cache(maxsize=None)
+def _get_language_via_dll(language: str) -> Any | None:
+    """Load a tree-sitter Language directly from tree_sitter_languages' bundled library.
+
+    Needed when tree_sitter_languages was compiled against an older tree-sitter ABI
+    (pre-0.22) but the installed tree-sitter Python binding is 0.22+, which changed
+    the Language() constructor. We bypass tree_sitter_languages.get_language() and
+    call the underlying C symbol ourselves.
+    """
+    try:
+        import ctypes
+        import warnings
+        from pathlib import Path as _Path
+        import tree_sitter_languages as _tsl
+        pkg_dir = _Path(_tsl.__file__).parent
+        # Find the bundled native library (name varies by platform).
+        for candidate in ("languages.dll", "languages.so", "languages.dylib"):
+            lib_path = pkg_dir / candidate
+            if lib_path.exists():
+                break
+        else:
+            # Try .pyd / .so with version suffix (Linux wheels)
+            matches = list(pkg_dir.glob("languages*.so")) + list(pkg_dir.glob("languages*.pyd"))
+            if not matches:
+                return None
+            lib_path = matches[0]
+
+        lib = ctypes.CDLL(str(lib_path))
+        fn_name = f"tree_sitter_{language.replace('-', '_')}"
+        fn = getattr(lib, fn_name, None)
+        if fn is None:
+            return None
+        fn.restype = ctypes.c_void_p
+        ptr = fn()
+        if not ptr:
+            return None
+        from tree_sitter import Language
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            return Language(ptr)
+    except Exception:
+        return None
+
+
+@lru_cache(maxsize=None)
+def get_language(language: str) -> Any | None:
+    """Return the tree-sitter Language object for query compilation."""
+
+    try:
+        from tree_sitter_language_pack import get_language as ts_get_language
+        return ts_get_language(language)
+    except Exception:
+        pass
+    try:
+        from tree_sitter_languages import get_language as ts_get_language
+        return ts_get_language(language)
+    except Exception:
+        pass
+    return _get_language_via_dll(language)
+
+
+@lru_cache(maxsize=None)
+def get_chunk_query(language: str) -> Any | None:
+    """Return a compiled tree-sitter Query when available."""
+
+    query_str = QUERY_STRINGS.get(language, "")
+    if not query_str:
+        return None
+    lang = get_language(language)
+    if lang is None:
+        return None
+    try:
+        from tree_sitter import Query
+        return Query(lang, query_str)
+    except Exception:
+        pass
+    # Fallback for older tree-sitter where lang.query() still works
+    try:
+        return lang.query(query_str)
+    except Exception:
+        return None

codepreproc_client/layer2_tooling/condensation/embed_worker.py

@@ -0,0 +1,128 @@
+"""Embedding subprocess worker.
+
+Launched as ``python -u -m codepreproc_client.layer2_tooling.condensation.embed_worker``.
+Communicates with the parent via stdin/stdout JSON lines.
+
+Protocol
+--------
+Parent → worker (stdin):
+    First line: {"model_name": "...", "device": "..."}
+    Subsequent lines: {"type": "encode", "request_id": "...", "texts": [...], "batch_size": N}
+                   or {"type": "shutdown"}
+
+Worker → parent (stdout):
+    {"type": "worker_ready"}
+    {"type": "worker_failed", "error": "..."}
+    {"type": "batch_done", "request_id": "...", "vectors": [[...]], "start": N, "end": N, "total_texts": N}
+    {"type": "batch_resized", "request_id": "...", "batch_size": N, "reason": "oom"}
+    {"type": "completed", "request_id": "...", "total_texts": N}
+    {"type": "failed", "request_id": "...", "error": "..."}
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+
+import numpy as np
+
+
+def _send(event: dict) -> None:
+    sys.stdout.write(json.dumps(event) + "\n")
+    sys.stdout.flush()
+
+
+def _coerce_batch_vectors(vectors, *, expected_rows: int) -> list[list[float]]:
+    array = np.asarray(vectors, dtype=np.float32)
+    if expected_rows == 0:
+        return []
+    if array.ndim == 1:
+        if expected_rows != 1:
+            raise ValueError(f"embedding_batch_shape_invalid: got {array.shape} for {expected_rows} texts")
+        array = array.reshape(1, -1)
+    if array.ndim != 2:
+        raise ValueError(f"embedding_batch_shape_invalid: got {array.shape} for {expected_rows} texts")
+    if array.shape[0] != expected_rows:
+        raise ValueError(f"embedding_batch_row_mismatch: got {array.shape[0]} rows for {expected_rows} texts")
+    return array.tolist()
+
+
+def main() -> None:
+    config_line = sys.stdin.readline()
+    if not config_line:
+        return
+    try:
+        config = json.loads(config_line)
+    except json.JSONDecodeError as exc:
+        _send({"type": "worker_failed", "error": f"bad config line: {exc}"})
+        return
+
+    model_name = config["model_name"]
+    device = config["device"]
+
+    try:
+        from sentence_transformers import SentenceTransformer
+
+        model = SentenceTransformer(model_name, device=device)
+        _send({"type": "worker_ready"})
+    except Exception as exc:
+        _send({"type": "worker_failed", "error": str(exc)})
+        return
+
+    for raw_line in sys.stdin:
+        line = raw_line.strip()
+        if not line:
+            continue
+        try:
+            command = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+
+        if command["type"] == "shutdown":
+            return
+        if command["type"] != "encode":
+            continue
+
+        request_id = command["request_id"]
+        texts = command["texts"]
+        current_batch = max(1, int(command["batch_size"]))
+        offset = 0
+        try:
+            while offset < len(texts):
+                upper = min(len(texts), offset + current_batch)
+                batch = texts[offset:upper]
+                try:
+                    vectors = model.encode(
+                        batch,
+                        batch_size=current_batch,
+                        show_progress_bar=False,
+                        convert_to_numpy=True,
+                        normalize_embeddings=True,
+                    )
+                except RuntimeError as exc:
+                    if "out of memory" not in str(exc).lower() or current_batch == 1:
+                        raise
+                    current_batch = max(1, current_batch // 2)
+                    _send({
+                        "type": "batch_resized",
+                        "request_id": request_id,
+                        "batch_size": current_batch,
+                        "reason": "oom",
+                    })
+                    continue
+                _send({
+                    "type": "batch_done",
+                    "request_id": request_id,
+                    "vectors": _coerce_batch_vectors(vectors, expected_rows=len(batch)),
+                    "start": offset,
+                    "end": upper,
+                    "total_texts": len(texts),
+                })
+                offset = upper
+            _send({"type": "completed", "request_id": request_id, "total_texts": len(texts)})
+        except Exception as exc:
+            _send({"type": "failed", "request_id": request_id, "error": str(exc)})
+
+
+if __name__ == "__main__":
+    main()