codetool-explore 0.5.0__py3-none-win_amd64.whl

codetool_explore/python_backend/matcher.py

@@ -0,0 +1,79 @@
+"""Content and path matcher construction."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import TypeAlias
+
+from .literal import LiteralMatcher
+from .regex_search import RegexLineMatcher
+
+SearchMatcher: TypeAlias = LiteralMatcher | RegexLineMatcher
+
+
+def build_content_matcher(
+    pattern: str,
+    *,
+    regex: bool,
+    case_sensitive: bool,
+) -> SearchMatcher:
+    """Build the matcher used for content search."""
+
+    if regex:
+        flags = 0 if case_sensitive else re.IGNORECASE
+        return RegexLineMatcher(re.compile(pattern, flags))
+    needle = pattern.encode("utf-8", errors="surrogatepass")
+    return LiteralMatcher(needle=needle, case_sensitive=case_sensitive)
+
+
+@dataclass(frozen=True)
+class PathMatcher:
+    """Prepared path matcher state."""
+
+    pattern: str
+    regex: bool
+    case_sensitive: bool
+    compiled: re.Pattern[str] | None = None
+
+    @classmethod
+    def build(
+        cls,
+        pattern: str,
+        *,
+        regex: bool,
+        case_sensitive: bool,
+    ) -> "PathMatcher":
+        """Build a path matcher, compiling regexes when requested."""
+
+        compiled = None
+        if regex:
+            flags = 0 if case_sensitive else re.IGNORECASE
+            compiled = re.compile(pattern, flags)
+        return cls(
+            pattern=pattern,
+            regex=regex,
+            case_sensitive=case_sensitive,
+            compiled=compiled,
+        )
+
+    def is_match(self, rel_path: str, path_scope: str) -> bool:
+        """Return whether ``rel_path`` matches this matcher."""
+
+        text = path_match_subject(rel_path, path_scope)
+        if self.regex:
+            assert self.compiled is not None
+            return self.compiled.search(text) is not None
+        if not self.pattern:
+            return True
+        if self.case_sensitive:
+            return self.pattern in text
+        return self.pattern.casefold() in text.casefold()
+
+
+def path_match_subject(rel_path: str, path_scope: str) -> str:
+    """Return the relative path component selected by ``path_scope``."""
+
+    if path_scope == "basename":
+        return rel_path.rsplit("/", 1)[-1]
+    return rel_path

codetool_explore/python_backend/models.py

@@ -0,0 +1,49 @@
+"""Data models shared by the Python backend modules."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+class BinaryFileError(ValueError):
+    """Raised internally when a candidate file looks binary."""
+
+
+@dataclass(frozen=True)
+class CandidateFile:
+    """A candidate file found by the scanner."""
+
+    path: str
+    rel_path: str
+    size: int
+
+
+@dataclass(frozen=True)
+class IgnorePatterns:
+    """Ignore patterns split by the path they should match against."""
+
+    common: tuple[str, ...] = ()
+    root: tuple[str, ...] = ()
+
+
+@dataclass
+class SearchCounters:
+    """Mutable counters collected while searching candidates."""
+
+    scanned_files: int = 0
+    scanned_bytes: int = 0
+    path_match_count: int = 0
+    content_match_count: int = 0
+    skipped_binary: int = 0
+    skipped_huge: int = 0
+    skipped_errors: int = 0
+
+    @property
+    def skipped(self) -> dict[str, int]:
+        """Return skipped counters in the public result shape."""
+
+        return {
+            "binary": self.skipped_binary,
+            "huge": self.skipped_huge,
+            "errors": self.skipped_errors,
+        }

codetool_explore/python_backend/output.py

@@ -0,0 +1,82 @@
+"""Result-shaping helpers for the Python backend."""
+
+from __future__ import annotations
+
+
+def path_file_match(
+    rel_path: str,
+    *,
+    mode: str,
+    match_kind: str = "path",
+) -> dict[str, object]:
+    """Build a file-level row for a path-only match."""
+
+    match: dict[str, object] = {
+        "path": rel_path,
+        "kind": "file",
+        "match_kind": match_kind,
+    }
+    if mode == "count":
+        match["count"] = 1
+    return match
+
+
+def record_path_only_match(
+    file_matches: list[dict[str, object]],
+    snippet_matches: list[dict[str, object]],
+    rel_path: str,
+    *,
+    mode: str,
+    collect_snippets: bool,
+) -> None:
+    """Append a path-only match to file/snippet collections."""
+
+    match = path_file_match(rel_path, mode=mode)
+    file_matches.append(match)
+    if collect_snippets:
+        snippet_matches.append(dict(match))
+
+
+def mark_snippets_for_content_or_path_target(
+    snippets: list[dict[str, object]],
+    *,
+    path_matches: bool,
+) -> None:
+    """Mark content snippets with combined content/path target metadata."""
+
+    match_kind = "content_and_path" if path_matches else "content"
+    for snippet in snippets:
+        snippet["kind"] = "file"
+        snippet["match_kind"] = match_kind
+
+
+def base_result(
+    *,
+    pattern: str,
+    root: str | list[str],
+    regex: bool,
+    target: str,
+    path_scope: str,
+    mode: str,
+    requested_case: str,
+    effective_case: str,
+    limit: int,
+    cursor: str | int | None,
+) -> dict[str, object]:
+    """Return the stable metadata prefix for a backend result."""
+
+    result: dict[str, object] = {
+        "pattern": pattern,
+        "root": root,
+        "regex": bool(regex),
+        "target": target,
+        "mode": mode,
+        "case": requested_case,
+        "effective_case": effective_case,
+        "limit": limit,
+        "cursor": None if cursor is None else str(cursor),
+        "backend": "python",
+    }
+    if target != "content":
+        result["path_scope"] = path_scope
+    return result

codetool_explore/python_backend/regex_search.py

@@ -0,0 +1,63 @@
+"""Regex content matching for the Python backend."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from .constants import MAX_SNIPPETS_PER_FILE
+from .models import CandidateFile
+from .text import context_for_lines, crop, decode_line
+
+
+@dataclass(frozen=True)
+class RegexLineMatcher:
+    """Prepared regex line matcher."""
+
+    compiled: re.Pattern[str]
+
+
+def search_regex_file(
+    candidate: CandidateFile,
+    data: bytes,
+    matcher: RegexLineMatcher,
+    *,
+    context_lines: int,
+    collect_snippets: bool,
+) -> tuple[dict[str, object] | None, list[dict[str, object]]]:
+    """Search one file's decoded lines with a compiled regex."""
+
+    lines = data.splitlines()
+
+    count = 0
+    first_line: int | None = None
+    snippets: list[dict[str, object]] = []
+
+    for index, raw_line in enumerate(lines):
+        line = decode_line(raw_line)
+        matches = list(matcher.compiled.finditer(line))
+        if not matches:
+            continue
+        count += len(matches)
+        line_number = index + 1
+        if first_line is None:
+            first_line = line_number
+        if collect_snippets and len(snippets) < MAX_SNIPPETS_PER_FILE:
+            snippet: dict[str, object] = {
+                "path": candidate.rel_path,
+                "line": line_number,
+                "snippet": crop(line),
+            }
+            context = context_for_lines(lines, index, context_lines)
+            if context:
+                snippet["context"] = context
+            snippets.append(snippet)
+
+    if count == 0 or first_line is None:
+        return None, []
+
+    return {
+        "path": candidate.rel_path,
+        "count": count,
+        "first_line": first_line,
+    }, snippets

codetool_explore/python_backend/search.py

@@ -0,0 +1,327 @@
+"""Top-level pure-Python backend search orchestration."""
+
+from __future__ import annotations
+
+import os
+import re
+from collections.abc import Iterable
+
+from ..cursor import normalize_limit, page_items
+from ..errors import ExploreArgumentError, ExplorePatternError
+from ..ignore import normalize_patterns
+from ..ranking import file_sort_key, snippet_sort_key
+from ..roots import RootInput, normalize_search_roots
+from .case import resolve_case
+from .config import normalize_mode, normalize_path_scope, normalize_target
+from .constants import max_file_bytes
+from .file_search import search_file
+from .matcher import PathMatcher, build_content_matcher
+from .models import BinaryFileError, CandidateFile, SearchCounters
+from .output import (
+    base_result,
+    mark_snippets_for_content_or_path_target,
+    record_path_only_match,
+)
+from .walker import iter_candidate_files
+
+
+def search_python(
+    pattern: str,
+    root: RootInput = ".",
+    *,
+    regex: bool = False,
+    target: str = "content",
+    path_scope: str = "path",
+    glob: str | Iterable[str] | None = None,
+    exclude: str | Iterable[str] | None = None,
+    case: str = "smart",
+    mode: str = "files",
+    context_lines: int = 0,
+    limit: int = 50,
+    cursor: str | int | None = None,
+) -> dict[str, object]:
+    """Search file contents, paths, or content/path union using Python."""
+
+    if not isinstance(pattern, str):
+        raise ExploreArgumentError("pattern must be a string")
+    normalised_target = normalize_target(target)
+    if pattern == "" and normalised_target in {"content", "content_or_path"}:
+        raise ExplorePatternError("pattern must not be empty")
+
+    root_set = normalize_search_roots(root)
+
+    normalised_mode = normalize_mode(mode)
+    if normalised_mode == "snippets" and normalised_target == "path":
+        raise ExploreArgumentError(
+            "mode='snippets' is not supported for target='path'; "
+            "use target='content' or target='content_or_path'"
+        )
+    safe_limit = normalize_limit(limit)
+    safe_context_lines = max(0, int(context_lines or 0))
+    requested_case = str(case or "smart").lower()
+    effective_case, case_sensitive = resolve_case(requested_case, pattern)
+    normalised_path_scope = normalize_path_scope(path_scope)
+    glob_patterns = normalize_patterns(glob)
+    exclude_patterns = normalize_patterns(exclude)
+
+    base = base_result(
+        pattern=pattern,
+        root=root_set.display,
+        regex=regex,
+        target=normalised_target,
+        path_scope=normalised_path_scope,
+        mode=normalised_mode,
+        requested_case=requested_case,
+        effective_case=effective_case,
+        limit=safe_limit,
+        cursor=cursor,
+    )
+
+    content_enabled = normalised_target in {"content", "content_or_path"}
+    path_enabled = normalised_target in {"path", "content_or_path"}
+    try:
+        content_matcher = (
+            build_content_matcher(
+                pattern,
+                regex=regex,
+                case_sensitive=case_sensitive,
+            )
+            if content_enabled
+            else None
+        )
+        path_matcher = (
+            PathMatcher.build(
+                pattern,
+                regex=regex,
+                case_sensitive=case_sensitive,
+            )
+            if path_enabled
+            else None
+        )
+    except re.error as exc:
+        raise ExplorePatternError(f"invalid regex: {exc}") from exc
+
+    collect_snippets = normalised_mode == "snippets"
+    file_matches: list[dict[str, object]] = []
+    snippet_matches: list[dict[str, object]] = []
+    counters = SearchCounters()
+    seen_candidate_paths: set[str] = set()
+
+    for search_root in root_set.roots:
+        candidates = iter_candidate_files(
+            search_root.abs_path,
+            rel_base=root_set.rel_base,
+            glob_patterns=glob_patterns,
+            exclude_patterns=exclude_patterns,
+        )
+        for candidate in candidates:
+            candidate_key = os.path.normcase(os.path.abspath(candidate.path))
+            if candidate_key in seen_candidate_paths:
+                continue
+            seen_candidate_paths.add(candidate_key)
+            _search_candidate(
+                candidate,
+                content_matcher=content_matcher,
+                path_matcher=path_matcher,
+                path_scope=normalised_path_scope,
+                target=normalised_target,
+                mode=normalised_mode,
+                context_lines=safe_context_lines,
+                collect_snippets=collect_snippets,
+                counters=counters,
+                file_matches=file_matches,
+                snippet_matches=snippet_matches,
+            )
+
+    file_matches.sort(key=lambda item: file_sort_key(item, pattern))
+    snippet_matches.sort(key=lambda item: snippet_sort_key(item, pattern))
+
+    total_files = len(file_matches)
+    content_occurrences = sum(
+        int(match["count"])
+        for match in file_matches
+        if match.get("match_kind") != "path" and "count" in match
+    )
+    path_only_occurrences = sum(
+        int(match.get("count", 1))
+        for match in file_matches
+        if match.get("match_kind") == "path"
+    )
+    total_occurrences = (
+        content_occurrences
+        if normalised_target == "content"
+        else content_occurrences + path_only_occurrences
+    )
+
+    if normalised_mode == "snippets":
+        all_matches = snippet_matches
+        total_matches = len(snippet_matches)
+    else:
+        all_matches = file_matches
+        total_matches = total_files
+
+    page, truncated, next_cursor, offset = page_items(
+        all_matches, limit=safe_limit, cursor=cursor
+    )
+
+    result = {
+        **base,
+        "matches": page,
+        "returned": len(page),
+        "total_files": total_files,
+        "total_matches": total_matches,
+        "count": total_occurrences,
+        "content_count": content_occurrences,
+        "content_files": counters.content_match_count,
+        "path_matches": counters.path_match_count,
+        "truncated": truncated,
+        "next_cursor": next_cursor,
+        "offset": offset,
+        "scanned_files": counters.scanned_files,
+        "scanned_bytes": counters.scanned_bytes,
+        "skipped": counters.skipped,
+    }
+    if glob_patterns:
+        result["glob"] = list(glob_patterns)
+    if exclude_patterns:
+        result["exclude"] = list(exclude_patterns)
+    return result
+
+
+def _search_candidate(
+    candidate: CandidateFile,
+    *,
+    content_matcher: object | None,
+    path_matcher: PathMatcher | None,
+    path_scope: str,
+    target: str,
+    mode: str,
+    context_lines: int,
+    collect_snippets: bool,
+    counters: SearchCounters,
+    file_matches: list[dict[str, object]],
+    snippet_matches: list[dict[str, object]],
+) -> None:
+    path_matches = (
+        path_matcher.is_match(candidate.rel_path, path_scope)
+        if path_matcher is not None
+        else False
+    )
+
+    if target == "path":
+        counters.scanned_files += 1
+        if path_matches:
+            record_path_only_match(
+                file_matches,
+                snippet_matches,
+                candidate.rel_path,
+                mode=mode,
+                collect_snippets=False,
+            )
+            counters.path_match_count += 1
+        return
+
+    if candidate.size > max_file_bytes():
+        if target == "content_or_path" and path_matches:
+            _record_content_or_path_target_path_fallback(
+                file_matches,
+                snippet_matches,
+                candidate.rel_path,
+                mode=mode,
+                collect_snippets=collect_snippets,
+                counters=counters,
+            )
+        counters.skipped_huge += 1
+        return
+
+    counters.scanned_files += 1
+    counters.scanned_bytes += candidate.size
+    try:
+        assert content_matcher is not None
+        file_match, snippets = search_file(
+            candidate,
+            content_matcher,
+            context_lines=context_lines,
+            collect_snippets=collect_snippets,
+        )
+    except BinaryFileError:
+        if target == "content_or_path" and path_matches:
+            _record_content_or_path_target_path_fallback(
+                file_matches,
+                snippet_matches,
+                candidate.rel_path,
+                mode=mode,
+                collect_snippets=collect_snippets,
+                counters=counters,
+            )
+        counters.skipped_binary += 1
+        return
+    except OverflowError:
+        if target == "content_or_path" and path_matches:
+            _record_content_or_path_target_path_fallback(
+                file_matches,
+                snippet_matches,
+                candidate.rel_path,
+                mode=mode,
+                collect_snippets=collect_snippets,
+                counters=counters,
+            )
+        counters.skipped_huge += 1
+        return
+    except (OSError, UnicodeError, re.error):
+        if target == "content_or_path" and path_matches:
+            _record_content_or_path_target_path_fallback(
+                file_matches,
+                snippet_matches,
+                candidate.rel_path,
+                mode=mode,
+                collect_snippets=collect_snippets,
+                counters=counters,
+            )
+        counters.skipped_errors += 1
+        return
+
+    if file_match is None:
+        if target == "content_or_path" and path_matches:
+            _record_content_or_path_target_path_fallback(
+                file_matches,
+                snippet_matches,
+                candidate.rel_path,
+                mode=mode,
+                collect_snippets=collect_snippets,
+                counters=counters,
+            )
+        return
+
+    if target == "content_or_path":
+        file_match["kind"] = "file"
+        file_match["match_kind"] = "content_and_path" if path_matches else "content"
+        if collect_snippets:
+            mark_snippets_for_content_or_path_target(
+                snippets,
+                path_matches=path_matches,
+            )
+        if path_matches:
+            counters.path_match_count += 1
+    counters.content_match_count += 1
+    file_matches.append(file_match)
+    snippet_matches.extend(snippets)
+
+
+def _record_content_or_path_target_path_fallback(
+    file_matches: list[dict[str, object]],
+    snippet_matches: list[dict[str, object]],
+    rel_path: str,
+    *,
+    mode: str,
+    collect_snippets: bool,
+    counters: SearchCounters,
+) -> None:
+    record_path_only_match(
+        file_matches,
+        snippet_matches,
+        rel_path,
+        mode=mode,
+        collect_snippets=collect_snippets,
+    )
+    counters.path_match_count += 1

codetool_explore/python_backend/text.py

@@ -0,0 +1,39 @@
+"""Text decoding, cropping, and snippet context helpers."""
+
+from __future__ import annotations
+
+from .constants import MAX_SNIPPET_CHARS
+
+
+def decode_line(line: bytes) -> str:
+    """Decode one raw line for display."""
+
+    return line.decode("utf-8", errors="replace").rstrip("\r")
+
+
+def crop(text: str, *, max_chars: int = MAX_SNIPPET_CHARS) -> str:
+    """Compact and crop snippet text."""
+
+    text = text.replace("\t", " ").strip()
+    if len(text) <= max_chars:
+        return text
+    if max_chars <= 1:
+        return text[:max_chars]
+    return text[: max_chars - 1].rstrip() + "…"
+
+
+def context_for_lines(
+    lines: list[bytes],
+    line_index: int,
+    context_lines: int,
+) -> list[dict[str, object]]:
+    """Return cropped context rows around ``line_index``."""
+
+    if context_lines <= 0:
+        return []
+    start = max(0, line_index - context_lines)
+    end = min(len(lines), line_index + context_lines + 1)
+    return [
+        {"line": index + 1, "text": crop(decode_line(lines[index]))}
+        for index in range(start, end)
+    ]