codetool-explore 0.5.0__py3-none-win_arm64.whl

codetool_explore/ignore.py

@@ -0,0 +1,222 @@
+"""Ignore, glob, and path-normalisation helpers.
+
+The implementation deliberately avoids external dependencies. It is not a full
+Git wildmatch clone; it provides fast common ignores plus simple shell-style
+patterns that are good enough for coding-agent search.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import os
+from collections.abc import Iterable
+
+COMMON_IGNORED_DIRS: frozenset[str] = frozenset(
+    {
+        ".git",
+        ".hg",
+        ".svn",
+        ".tox",
+        ".nox",
+        ".venv",
+        "venv",
+        "env",
+        "__pycache__",
+        ".mypy_cache",
+        ".pytest_cache",
+        ".ruff_cache",
+        ".cache",
+        "node_modules",
+        "bower_components",
+        "dist",
+        "build",
+        "target",
+        ".next",
+        ".nuxt",
+        ".idea",
+        ".vscode",
+    }
+)
+
+COMMON_IGNORED_FILES: frozenset[str] = frozenset(
+    {
+        ".DS_Store",
+        "Thumbs.db",
+    }
+)
+
+IGNORE_FILES: tuple[str, ...] = (".pbiignore", ".gitignore")
+
+
+def normalize_relpath(path: str) -> str:
+    """Return a compact relative path using forward slashes."""
+
+    rel = os.fspath(path).replace("\\", "/").replace(os.sep, "/")
+    if os.altsep:
+        rel = rel.replace(os.altsep, "/")
+    while rel.startswith("./"):
+        rel = rel[2:]
+    return "" if rel == "." else rel
+
+
+def relative_path(path: str, root: str) -> str:
+    """Return ``path`` relative to ``root`` with stable slash separators."""
+
+    try:
+        return normalize_relpath(os.path.relpath(path, root))
+    except ValueError:
+        # Different drives on Windows; fall back to a normalised absolute path.
+        return normalize_relpath(os.path.abspath(path))
+
+
+def normalize_patterns(patterns: str | Iterable[str] | None) -> tuple[str, ...]:
+    """Normalise a public glob/exclude value into a tuple of patterns."""
+
+    if patterns is None:
+        return ()
+    if isinstance(patterns, str):
+        raw_patterns = (patterns,)
+    else:
+        raw_patterns = tuple(patterns)
+    normalised: list[str] = []
+    for pattern in raw_patterns:
+        if pattern is None:
+            continue
+        text = normalize_relpath(str(pattern).strip())
+        if not text:
+            continue
+        normalised.append(text)
+    return tuple(normalised)
+
+
+def load_ignore_patterns(root: str) -> tuple[str, ...]:
+    """Load simple root-level patterns from ``.pbiignore`` and ``.gitignore``.
+
+    Supported syntax is intentionally small: comments and blank lines are
+    ignored, leading ``!`` negation is ignored, and trailing slash means the
+    directory and everything under it.
+    """
+
+    patterns: list[str] = []
+    for filename in IGNORE_FILES:
+        path = os.path.join(root, filename)
+        try:
+            with open(path, encoding="utf-8", errors="replace") as handle:
+                lines = handle.readlines()
+        except OSError:
+            continue
+        for line in lines:
+            text = line.strip()
+            if not text or text.startswith("#") or text.startswith("!"):
+                continue
+            text = normalize_relpath(text)
+            if text.endswith("/"):
+                text = text.rstrip("/")
+                patterns.extend((text, f"{text}/**"))
+            else:
+                patterns.append(text)
+    return tuple(patterns)
+
+
+def _path_segments(rel_path: str) -> tuple[str, ...]:
+    return tuple(
+        segment for segment in normalize_relpath(rel_path).split("/") if segment
+    )
+
+
+def is_common_ignored_dir(name: str) -> bool:
+    """Return true when ``name`` is a directory we should hard-prune."""
+
+    return name in COMMON_IGNORED_DIRS
+
+
+def is_common_ignored_file(name: str) -> bool:
+    """Return true when ``name`` is a common unhelpful file."""
+
+    return name in COMMON_IGNORED_FILES
+
+
+def path_matches_pattern(rel_path: str, pattern: str) -> bool:
+    """Match a normalised relative path against a simple shell-style pattern."""
+
+    rel_path = normalize_relpath(rel_path)
+    pattern = normalize_relpath(pattern)
+    if not pattern:
+        return False
+
+    basename = rel_path.rsplit("/", 1)[-1]
+
+    if fnmatch.fnmatchcase(rel_path, pattern) or fnmatch.fnmatchcase(basename, pattern):
+        return True
+
+    # Treat bare directory names as "any segment named X".
+    if "/" not in pattern and pattern in _path_segments(rel_path):
+        return True
+
+    # Treat "dir" and "dir/**" as directory-prefix patterns.
+    prefix = pattern[:-3] if pattern.endswith("/**") else pattern
+    if prefix and (rel_path == prefix or rel_path.startswith(prefix.rstrip("/") + "/")):
+        return True
+
+    return False
+
+
+def path_matches_any(rel_path: str, patterns: Iterable[str]) -> bool:
+    """Return true if ``rel_path`` matches any pattern."""
+
+    return any(path_matches_pattern(rel_path, pattern) for pattern in patterns)
+
+
+def pattern_targets_path_or_descendant(path: str, pattern: str) -> bool:
+    """Return true when a pattern is anchored at ``path`` or its descendants."""
+
+    path = normalize_relpath(path).strip("/")
+    pattern = normalize_relpath(pattern).strip("/")
+    if not path or not pattern:
+        return False
+    return path_matches_pattern(path, pattern) or pattern.startswith(f"{path}/")
+
+
+def matches_glob(rel_path: str, glob_patterns: Iterable[str]) -> bool:
+    """Return true if the path is accepted by the optional glob filters."""
+
+    patterns = tuple(glob_patterns)
+    if not patterns:
+        return True
+    return path_matches_any(rel_path, patterns)
+
+
+def should_ignore_path(
+    rel_path: str,
+    *,
+    is_dir: bool,
+    exclude_patterns: Iterable[str] = (),
+    ignore_patterns: Iterable[str] = (),
+    root_ignore_patterns: Iterable[str] = (),
+    common_rel_path: str | None = None,
+) -> bool:
+    """Return true when a file/directory should be skipped."""
+
+    rel_path = normalize_relpath(rel_path)
+    common_path = normalize_relpath(
+        common_rel_path if common_rel_path is not None else rel_path
+    )
+    common_name = common_path.rsplit("/", 1)[-1]
+    common_segments = _path_segments(common_path)
+
+    if is_dir:
+        if common_name in COMMON_IGNORED_DIRS or any(
+            segment in COMMON_IGNORED_DIRS for segment in common_segments
+        ):
+            return True
+    elif common_name in COMMON_IGNORED_FILES:
+        return True
+
+    all_patterns = tuple(exclude_patterns) + tuple(ignore_patterns)
+    if all_patterns and path_matches_any(rel_path, all_patterns):
+        return True
+
+    if root_ignore_patterns and path_matches_any(common_path, root_ignore_patterns):
+        return True
+
+    return False

codetool_explore/python_backend/__init__.py

@@ -0,0 +1,154 @@
+"""Pure-Python stdlib backend for workspace search.
+
+The package mirrors the Rust helper's ``rust/src`` organization so equivalent
+backend responsibilities live in equivalent module names:
+
+* ``constants`` and ``models`` define shared backend data;
+* ``case`` and ``config`` validate public search options;
+* ``walker`` and ``ignore_rules`` enumerate candidate files;
+* ``matcher``, ``literal``, ``regex_search``, ``file_search``, and ``text``
+  perform path/content matching; and
+* ``search`` coordinates ranking, pagination, and result assembly.
+"""
+
+from __future__ import annotations
+
+import re
+
+from .case import resolve_case
+from .config import normalize_mode, normalize_path_scope, normalize_target
+from .constants import (
+    BINARY_CHECK_BYTES,
+    MAX_FILE_BYTES,
+    MAX_SNIPPETS_PER_FILE,
+    MAX_SNIPPET_CHARS,
+    VALID_MODES,
+    VALID_PATH_SCOPES,
+    VALID_TARGETS,
+)
+from .models import BinaryFileError, CandidateFile, IgnorePatterns
+from .file_search import read_text_candidate
+from .ignore_rules import ignore_patterns_for_root
+from .literal import LiteralMatcher, count_non_overlapping, search_literal_file
+from .matcher import PathMatcher, path_match_subject
+from .output import (
+    base_result,
+    mark_snippets_for_content_or_path_target,
+    path_file_match,
+    record_path_only_match,
+)
+from .regex_search import RegexLineMatcher, search_regex_file
+from .search import search_python
+from .text import context_for_lines, crop, decode_line
+from .walker import iter_candidate_files
+
+__all__ = [
+    "BINARY_CHECK_BYTES",
+    "MAX_FILE_BYTES",
+    "MAX_SNIPPETS_PER_FILE",
+    "MAX_SNIPPET_CHARS",
+    "VALID_MODES",
+    "VALID_PATH_SCOPES",
+    "VALID_TARGETS",
+    "BinaryFileError",
+    "CandidateFile",
+    "IgnorePatterns",
+    "iter_candidate_files",
+    "normalize_mode",
+    "normalize_path_scope",
+    "normalize_target",
+    "resolve_case",
+    "search_python",
+]
+
+
+def _read_text_candidate(path: str, size: int) -> bytes:
+    """Compatibility wrapper for the former single-file module helper."""
+
+    return read_text_candidate(CandidateFile(path=path, rel_path=path, size=size))
+
+
+def _search_literal_file(
+    path: str,
+    rel_path: str,
+    size: int,
+    *,
+    pattern: str,
+    needle: bytes,
+    case_sensitive: bool,
+    context_lines: int,
+    collect_snippets: bool,
+) -> tuple[dict[str, object] | None, list[dict[str, object]]]:
+    """Compatibility wrapper for the former literal search helper."""
+
+    del pattern
+    candidate = CandidateFile(path=path, rel_path=rel_path, size=size)
+    return search_literal_file(
+        candidate,
+        read_text_candidate(candidate),
+        LiteralMatcher(needle=needle, case_sensitive=case_sensitive),
+        context_lines=context_lines,
+        collect_snippets=collect_snippets,
+    )
+
+
+def _search_regex_file(
+    path: str,
+    rel_path: str,
+    size: int,
+    *,
+    compiled: re.Pattern[str],
+    context_lines: int,
+    collect_snippets: bool,
+) -> tuple[dict[str, object] | None, list[dict[str, object]]]:
+    """Compatibility wrapper for the former regex search helper."""
+
+    candidate = CandidateFile(path=path, rel_path=rel_path, size=size)
+    return search_regex_file(
+        candidate,
+        read_text_candidate(candidate),
+        RegexLineMatcher(compiled=compiled),
+        context_lines=context_lines,
+        collect_snippets=collect_snippets,
+    )
+
+
+def _path_matches_literal(
+    rel_path: str,
+    *,
+    pattern: str,
+    case_sensitive: bool,
+    path_scope: str,
+) -> bool:
+    """Compatibility wrapper for the former path-literal helper."""
+
+    return PathMatcher.build(
+        pattern,
+        regex=False,
+        case_sensitive=case_sensitive,
+    ).is_match(rel_path, path_scope)
+
+
+def _path_matches_regex(
+    rel_path: str,
+    *,
+    compiled: re.Pattern[str],
+    path_scope: str,
+) -> bool:
+    """Compatibility wrapper for the former path-regex helper."""
+
+    return compiled.search(path_match_subject(rel_path, path_scope)) is not None
+
+
+_base_result = base_result
+_context_for_lines = context_for_lines
+_count_non_overlapping = count_non_overlapping
+_crop = crop
+_decode_line = decode_line
+_ignore_patterns_for_root = ignore_patterns_for_root
+_mark_snippets_for_content_or_path_target = (
+    mark_snippets_for_content_or_path_target
+)
+_path_file_match = path_file_match
+_path_match_subject = path_match_subject
+_record_path_only_match = record_path_only_match

codetool_explore/python_backend/case.py

@@ -0,0 +1,19 @@
+"""Case-sensitivity handling for the Python backend."""
+
+from __future__ import annotations
+
+from ..errors import ExploreArgumentError
+
+
+def resolve_case(case: str, pattern: str) -> tuple[str, bool]:
+    """Return ``(effective_case, case_sensitive)`` for a search pattern."""
+
+    normalised = str(case or "smart").lower()
+    if normalised == "smart":
+        case_sensitive = any(char.isupper() for char in pattern)
+        return "sensitive" if case_sensitive else "insensitive", case_sensitive
+    if normalised in {"sensitive", "case-sensitive", "exact"}:
+        return "sensitive", True
+    if normalised in {"insensitive", "ignore", "ignorecase", "case-insensitive", "i"}:
+        return "insensitive", False
+    raise ExploreArgumentError("case must be one of: smart, sensitive, insensitive")

codetool_explore/python_backend/config.py

@@ -0,0 +1,35 @@
+"""Public-option normalization for the Python backend."""
+
+from __future__ import annotations
+
+from ..errors import ExploreArgumentError
+from .constants import VALID_MODES, VALID_PATH_SCOPES, VALID_TARGETS
+
+
+def normalize_mode(mode: str) -> str:
+    """Validate and normalise a public output mode."""
+
+    normalised = str(mode or "files").lower()
+    if normalised not in VALID_MODES:
+        raise ExploreArgumentError("mode must be one of: files, snippets, count")
+    return normalised
+
+
+def normalize_target(target: str) -> str:
+    """Validate and normalise the public search target."""
+
+    normalised = str(target or "content").lower()
+    if normalised not in VALID_TARGETS:
+        raise ExploreArgumentError(
+            "target must be one of: content, path, content_or_path"
+        )
+    return normalised
+
+
+def normalize_path_scope(path_scope: str) -> str:
+    """Validate and normalise the path field matched by path search."""
+
+    normalised = str(path_scope or "path").lower()
+    if normalised not in VALID_PATH_SCOPES:
+        raise ExploreArgumentError("path_scope must be one of: path, basename")
+    return normalised

codetool_explore/python_backend/constants.py

@@ -0,0 +1,39 @@
+"""Constants used by the Python backend."""
+
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+DEFAULT_MAX_FILE_BYTES = 5 * 1024 * 1024
+DEFAULT_BINARY_CHECK_BYTES = 8 * 1024
+
+MAX_FILE_BYTES = DEFAULT_MAX_FILE_BYTES
+BINARY_CHECK_BYTES = DEFAULT_BINARY_CHECK_BYTES
+MAX_SNIPPETS_PER_FILE = 3
+MAX_SNIPPET_CHARS = 180
+VALID_MODES = frozenset({"files", "snippets", "count"})
+VALID_TARGETS = frozenset({"content", "path", "content_or_path"})
+VALID_PATH_SCOPES = frozenset({"path", "basename"})
+
+
+def _runtime_constant(name: str, default: int) -> int:
+    """Read a tunable constant, preserving package-level monkeypatch support."""
+
+    package = sys.modules.get("codetool_explore.python_backend")
+    package_value: Any = getattr(package, name, default) if package else default
+    module_value: Any = globals().get(name, default)
+    value = package_value if package_value != default else module_value
+    return int(value)
+
+
+def max_file_bytes() -> int:
+    """Return the active maximum content file size."""
+
+    return _runtime_constant("MAX_FILE_BYTES", DEFAULT_MAX_FILE_BYTES)
+
+
+def binary_check_bytes() -> int:
+    """Return the active binary-probe byte count."""
+
+    return _runtime_constant("BINARY_CHECK_BYTES", DEFAULT_BINARY_CHECK_BYTES)

codetool_explore/python_backend/file_search.py

@@ -0,0 +1,51 @@
+"""Read text candidates and dispatch content searchers."""
+
+from __future__ import annotations
+
+from .constants import binary_check_bytes, max_file_bytes
+from .literal import LiteralMatcher, search_literal_file
+from .matcher import SearchMatcher
+from .models import BinaryFileError, CandidateFile
+from .regex_search import RegexLineMatcher, search_regex_file
+
+
+def read_text_candidate(candidate: CandidateFile) -> bytes:
+    """Read a candidate file after applying huge-file and binary guards."""
+
+    if candidate.size > max_file_bytes():
+        raise OverflowError("huge file")
+    with open(candidate.path, "rb") as handle:
+        first = handle.read(binary_check_bytes())
+        if b"\x00" in first:
+            raise BinaryFileError("binary file")
+        rest = handle.read()
+    return first + rest
+
+
+def search_file(
+    candidate: CandidateFile,
+    matcher: SearchMatcher,
+    *,
+    context_lines: int,
+    collect_snippets: bool,
+) -> tuple[dict[str, object] | None, list[dict[str, object]]]:
+    """Search one already-filtered candidate file."""
+
+    data = read_text_candidate(candidate)
+    if isinstance(matcher, LiteralMatcher):
+        return search_literal_file(
+            candidate,
+            data,
+            matcher,
+            context_lines=context_lines,
+            collect_snippets=collect_snippets,
+        )
+    if isinstance(matcher, RegexLineMatcher):
+        return search_regex_file(
+            candidate,
+            data,
+            matcher,
+            context_lines=context_lines,
+            collect_snippets=collect_snippets,
+        )
+    raise TypeError(f"unsupported matcher: {type(matcher).__name__}")

codetool_explore/python_backend/ignore_rules.py

@@ -0,0 +1,40 @@
+"""Ignore-pattern loading for Python backend candidate walking."""
+
+from __future__ import annotations
+
+import os
+
+from ..ignore import (
+    load_ignore_patterns,
+    pattern_targets_path_or_descendant,
+    relative_path,
+)
+from .models import IgnorePatterns
+
+
+def ignore_patterns_for_root(
+    root_abs: str,
+    *,
+    rel_base_abs: str | None,
+    is_file: bool,
+) -> IgnorePatterns:
+    """Return common-base and root-local ignore patterns for a search root."""
+
+    filter_root = os.path.dirname(root_abs) if is_file else root_abs
+    if rel_base_abs is None:
+        return IgnorePatterns(root=load_ignore_patterns(filter_root))
+
+    root_prefix = relative_path(filter_root, rel_base_abs)
+    patterns: list[str] = []
+    if os.path.isdir(rel_base_abs):
+        patterns.extend(
+            pattern
+            for pattern in load_ignore_patterns(rel_base_abs)
+            if not pattern_targets_path_or_descendant(root_prefix, pattern)
+        )
+
+    root_patterns = load_ignore_patterns(filter_root)
+    return IgnorePatterns(
+        common=tuple(dict.fromkeys(patterns)),
+        root=tuple(dict.fromkeys(root_patterns)),
+    )

codetool_explore/python_backend/literal.py

@@ -0,0 +1,79 @@
+"""Literal content matching for the Python backend."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .constants import MAX_SNIPPETS_PER_FILE
+from .models import CandidateFile
+from .text import context_for_lines, crop, decode_line
+
+
+@dataclass(frozen=True)
+class LiteralMatcher:
+    """Prepared literal matcher state."""
+
+    needle: bytes
+    case_sensitive: bool
+
+
+def count_non_overlapping(haystack: bytes, needle: bytes) -> int:
+    """Count non-overlapping byte literal matches."""
+
+    count = 0
+    start = 0
+    step = max(1, len(needle))
+    while True:
+        index = haystack.find(needle, start)
+        if index < 0:
+            return count
+        count += 1
+        start = index + step
+
+
+def search_literal_file(
+    candidate: CandidateFile,
+    data: bytes,
+    matcher: LiteralMatcher,
+    *,
+    context_lines: int,
+    collect_snippets: bool,
+) -> tuple[dict[str, object] | None, list[dict[str, object]]]:
+    """Search one file's bytes for a literal needle."""
+
+    compare_needle = matcher.needle if matcher.case_sensitive else matcher.needle.lower()
+    lines = data.splitlines()
+
+    count = 0
+    first_line: int | None = None
+    snippets: list[dict[str, object]] = []
+
+    for index, line in enumerate(lines):
+        compare_line = line if matcher.case_sensitive else line.lower()
+        line_count = count_non_overlapping(compare_line, compare_needle)
+        if line_count == 0:
+            continue
+        count += line_count
+        line_number = index + 1
+        if first_line is None:
+            first_line = line_number
+        if collect_snippets and len(snippets) < MAX_SNIPPETS_PER_FILE:
+            snippet: dict[str, object] = {
+                "path": candidate.rel_path,
+                "line": line_number,
+                "snippet": crop(decode_line(line)),
+            }
+            context = context_for_lines(lines, index, context_lines)
+            if context:
+                snippet["context"] = context
+            snippets.append(snippet)
+
+    if count == 0 or first_line is None:
+        return None, []
+
+    file_match: dict[str, object] = {
+        "path": candidate.rel_path,
+        "count": count,
+        "first_line": first_line,
+    }
+    return file_match, snippets