code-muse 0.0.1__py3-none-any.whl

code_muse/fs_scan_cache/scan_cache_core.pyx

@@ -0,0 +1,203 @@
+# cython: language_level=3
+"""ScanCache core — thread-safe LRU cache for filesystem scan results."""
+
+import logging
+import threading
+import time
+from collections import OrderedDict
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+from code_muse.fs_scan_cache.ttl_policy import is_fresh
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class GlobMatch:
+    """A single filesystem entry returned by a scan."""
+
+    path: str
+    file_type: str  # "file" | "dir" | "symlink"
+    mtime: float
+    size: int
+
+
+@dataclass
+class ScanEntry:
+    """Internal cached scan result with timestamp."""
+
+    entries: list[GlobMatch]
+    created_at: float
+
+
+@dataclass
+class CacheStats:
+    """Hit/miss/eviction statistics."""
+
+    hits: int = 0
+    misses: int = 0
+    evictions: int = 0
+    size: int = 0
+
+
+class ScanCache:
+    """Thread-safe LRU cache for filesystem scan results.
+
+    Keyed by a hashable partition tuple (typically ``(root, hidden, gitignore,
+    node_modules)``).  Max 16 entries, LRU eviction.  TTL-based freshness with
+    separate fast-recheck for empty results.
+    """
+
+    def __init__(self, max_entries: int = 16) -> None:
+        if max_entries < 1:
+            raise ValueError("max_entries must be positive")
+        self.max_entries = max_entries
+        # FREE-THREADED: Generic scan cache — may be accessed from sync or async code.
+        # Keep threading.Lock; migrate to asyncio.Lock only if all callers are async.
+        self._lock = threading.Lock()
+        self._cache: OrderedDict[tuple, ScanEntry] = OrderedDict()
+        self._stats = CacheStats()
+
+    def get_or_scan(
+        self,
+        key: tuple,
+        scanner_fn: Callable[[], list[GlobMatch]],
+    ) -> tuple[list[GlobMatch], float]:
+        """Return cached entries if fresh, otherwise call *scanner_fn*.
+
+        The scanner function is invoked **outside** the lock to avoid holding
+        it during I/O.  A double-check is performed after re-acquiring the lock.
+
+        Returns:
+            ``(entries, cache_age_ms)`` — *cache_age_ms* is ``0.0`` for a
+            fresh insertion, otherwise the monotonic age of the cached entry.
+        """
+        cdef double now = time.monotonic()
+        cdef double age_ms
+        cdef double created
+        cdef object entry
+        cdef list scanned
+        cdef int evict_count
+        cdef object new_entry
+
+        with self._lock:
+            if key in self._cache:
+                entry = self._cache[key]
+                if is_fresh(entry, now):
+                    self._cache.move_to_end(key)
+                    age_ms = (now - entry.created_at) * 1000.0
+                    self._stats.hits += 1
+                    self._stats.size = len(self._cache)
+                    return (entry.entries, age_ms)
+                # Stale — remove now so we don't return stale data later
+                del self._cache[key]
+
+        # Slow path: scan outside the lock
+        scanned = scanner_fn()
+
+        with self._lock:
+            # Double-check: another thread may have populated while we scanned
+            if key in self._cache:
+                entry = self._cache[key]
+                if is_fresh(entry, now):
+                    self._cache.move_to_end(key)
+                    age_ms = (now - entry.created_at) * 1000.0
+                    self._stats.hits += 1
+                    self._stats.size = len(self._cache)
+                    return (entry.entries, age_ms)
+                del self._cache[key]
+
+            # Evict oldest if at capacity
+            evict_count = 0
+            while len(self._cache) >= self.max_entries:
+                self._cache.popitem(last=False)
+                evict_count += 1
+            if evict_count:
+                self._stats.evictions += evict_count
+
+            created = time.monotonic()
+            new_entry = ScanEntry(entries=scanned, created_at=created)
+            self._cache[key] = new_entry
+            self._stats.misses += 1
+            self._stats.size = len(self._cache)
+            return (new_entry.entries, 0.0)
+
+    def invalidate(self, root: str | None = None) -> None:
+        """Remove cache entries affected by *root*.
+
+        If *root* is ``None``, clear the entire cache.  Otherwise remove any
+        entry whose cached root is an ancestor of *root* or vice versa.
+        """
+        cdef list keys_to_remove
+        cdef tuple key
+        cdef object target
+        cdef object cached_root
+        cdef bint cached_is_ancestor
+        cdef bint target_is_ancestor
+
+        with self._lock:
+            if root is None:
+                self._cache.clear()
+                self._stats.size = 0
+                logger.debug("ScanCache cleared entirely")
+                return
+
+            target = Path(root).resolve()
+            keys_to_remove = []
+            for key in list(self._cache.keys()):
+                cached_root = Path(key[0]).resolve()
+                # Ancestor check in both directions
+                try:
+                    cached_is_ancestor = target.is_relative_to(cached_root)
+                except AttributeError:
+                    # Python < 3.12 fallback
+                    try:
+                        cached_is_ancestor = (
+                            target == cached_root or cached_root in target.parents
+                        )
+                    except ValueError:
+                        cached_is_ancestor = False
+
+                try:
+                    target_is_ancestor = cached_root.is_relative_to(target)
+                except AttributeError:
+                    try:
+                        target_is_ancestor = (
+                            cached_root == target or target in cached_root.parents
+                        )
+                    except ValueError:
+                        target_is_ancestor = False
+
+                if cached_is_ancestor or target_is_ancestor:
+                    keys_to_remove.append(key)
+
+            for key in keys_to_remove:
+                del self._cache[key]
+                logger.debug(f"ScanCache invalidated key {key}")
+
+            self._stats.size = len(self._cache)
+
+    def invalidate_for_path(self, path: str) -> None:
+        """Invalidate cache entries related to *path*.
+
+        Convenience wrapper around :meth:`invalidate` that accepts a file
+        or directory path directly.
+        """
+        self.invalidate(path)
+
+    def clear(self) -> None:
+        """Remove all entries."""
+        self.invalidate(None)
+
+    @property
+    def stats(self) -> CacheStats:
+        """Return a snapshot of cache statistics."""
+        with self._lock:
+            return CacheStats(
+                hits=self._stats.hits,
+                misses=self._stats.misses,
+                evictions=self._stats.evictions,
+                size=self._stats.size,
+            )

code_muse/fs_scan_cache/tool_integration.py

@@ -0,0 +1,309 @@
+"""Cached wrappers around glob, grep, and find operations."""
+
+import fnmatch
+import os
+import re
+from pathlib import Path
+
+from code_muse.fs_scan_cache.scan_cache_core import GlobMatch, ScanCache
+
+
+def _resolve_root(root: str) -> Path:
+    return Path(root).expanduser().resolve()
+
+
+def _is_hidden(path: Path) -> bool:
+    """Return True if any component of *path* starts with '.'."""
+    return any(
+        part.startswith(".") for part in path.parts if part not in (".", "..", os.sep)
+    )
+
+
+def _should_skip(
+    path: Path,
+    *,
+    include_hidden: bool,
+    use_gitignore: bool,
+    skip_node_modules: bool,
+) -> bool:
+    """Return True if *path* should be excluded from scan results."""
+    if not include_hidden and _is_hidden(path):
+        return True
+    if skip_node_modules and "node_modules" in path.parts:
+        return True
+    if use_gitignore:
+        # Lightweight approximation: skip `.git` directory
+        if ".git" in path.parts:
+            return True
+    return False
+
+
+def _stat_entry(p: Path) -> tuple[str, float, int]:
+    """Return (file_type, mtime, size) for *p*, falling back gracefully."""
+    try:
+        st = p.stat(follow_symlinks=False)
+        if p.is_symlink():
+            return "symlink", st.st_mtime, 0
+        if p.is_dir():
+            return "dir", st.st_mtime, 0
+        return "file", st.st_mtime, st.st_size
+    except OSError, ValueError:
+        return "file", 0.0, 0
+
+
+def _glob_scanner(
+    pattern: str,
+    root: str,
+    *,
+    hidden: bool,
+    gitignore: bool,
+    node_modules: bool,
+) -> list[GlobMatch]:
+    """Perform a filesystem glob and return filtered GlobMatch entries."""
+    base = _resolve_root(root)
+    results: list[GlobMatch] = []
+
+    if "**" in pattern:
+        # Recursive glob via rglob
+        raw_pattern = pattern.replace("**/", "").replace("**", "")
+        # If pattern is just '**', match everything
+        if raw_pattern in {"", "*"}:
+            iterator = base.rglob("*")
+        else:
+            iterator = base.rglob(raw_pattern)
+    else:
+        iterator = base.glob(pattern)
+
+    for p in iterator:
+        try:
+            relative = p.relative_to(base)
+        except ValueError:
+            relative = p
+        if _should_skip(
+            relative,
+            include_hidden=hidden,
+            use_gitignore=gitignore,
+            skip_node_modules=node_modules,
+        ):
+            continue
+        file_type, mtime, size = _stat_entry(p)
+        results.append(
+            GlobMatch(
+                path=str(p),
+                file_type=file_type,
+                mtime=mtime,
+                size=size,
+            )
+        )
+    return results
+
+
+def _grep_scanner(
+    pattern: str,
+    root: str,
+    *,
+    hidden: bool,
+    gitignore: bool,
+    node_modules: bool,
+) -> list[GlobMatch]:
+    """Recursively search file contents with regex and return matching files."""
+    base = _resolve_root(root)
+    results: list[GlobMatch] = []
+    compiled = re.compile(pattern)
+    max_matches = 50
+
+    for p in base.rglob("*"):
+        try:
+            relative = p.relative_to(base)
+        except ValueError:
+            relative = p
+        if _should_skip(
+            relative,
+            include_hidden=hidden,
+            use_gitignore=gitignore,
+            skip_node_modules=node_modules,
+        ):
+            continue
+        if not p.is_file() or p.is_symlink():
+            continue
+        try:
+            # Skip binary / very large files
+            size = p.stat().st_size
+            if size > 5 * 1024 * 1024:
+                continue
+            text = p.read_text(encoding="utf-8", errors="replace")
+        except OSError, UnicodeDecodeError:
+            continue
+        if compiled.search(text):
+            file_type, mtime, fsize = _stat_entry(p)
+            results.append(
+                GlobMatch(
+                    path=str(p),
+                    file_type=file_type,
+                    mtime=mtime,
+                    size=fsize,
+                )
+            )
+            if len(results) >= max_matches:
+                break
+    return results
+
+
+def _find_scanner(
+    name: str,
+    root: str,
+    *,
+    hidden: bool,
+    gitignore: bool,
+    node_modules: bool,
+) -> list[GlobMatch]:
+    """Recursively find files/directories whose basename matches *name*."""
+    base = _resolve_root(root)
+    results: list[GlobMatch] = []
+
+    for p in base.rglob("*"):
+        try:
+            relative = p.relative_to(base)
+        except ValueError:
+            relative = p
+        if _should_skip(
+            relative,
+            include_hidden=hidden,
+            use_gitignore=gitignore,
+            skip_node_modules=node_modules,
+        ):
+            continue
+        if not fnmatch.fnmatch(p.name, name):
+            continue
+        file_type, mtime, size = _stat_entry(p)
+        results.append(
+            GlobMatch(
+                path=str(p),
+                file_type=file_type,
+                mtime=mtime,
+                size=size,
+            )
+        )
+    return results
+
+
+# Module-level default cache used by the cached_* helpers.
+_default_cache: ScanCache | None = None
+
+
+def _get_default_cache() -> ScanCache:
+    global _default_cache
+    if _default_cache is None:
+        _default_cache = ScanCache()
+    return _default_cache
+
+
+def cached_glob(
+    pattern: str,
+    root: str = ".",
+    hidden: bool = False,
+    gitignore: bool = True,
+    node_modules: bool = True,
+    cache: bool = False,
+) -> tuple[list[GlobMatch], float | None]:
+    """Glob with optional scan caching.
+
+    Returns:
+        ``(entries, cache_age_ms)`` — *cache_age_ms* is ``None`` when
+        *cache* is ``False``.
+    """
+    if not cache:
+        return (
+            _glob_scanner(
+                pattern,
+                root,
+                hidden=hidden,
+                gitignore=gitignore,
+                node_modules=node_modules,
+            ),
+            None,
+        )
+
+    key = (root, hidden, gitignore, node_modules, pattern)
+    sc = _get_default_cache()
+    entries, age = sc.get_or_scan(
+        key,
+        lambda: _glob_scanner(
+            pattern, root, hidden=hidden, gitignore=gitignore, node_modules=node_modules
+        ),
+    )
+    return (entries, age)
+
+
+def cached_grep(
+    pattern: str,
+    root: str = ".",
+    hidden: bool = False,
+    gitignore: bool = True,
+    node_modules: bool = True,
+    cache: bool = False,
+) -> tuple[list[GlobMatch], float | None]:
+    """Grep with optional scan caching.
+
+    Returns:
+        ``(entries, cache_age_ms)`` — *cache_age_ms* is ``None`` when
+        *cache* is ``False``.
+    """
+    if not cache:
+        return (
+            _grep_scanner(
+                pattern,
+                root,
+                hidden=hidden,
+                gitignore=gitignore,
+                node_modules=node_modules,
+            ),
+            None,
+        )
+
+    key = (root, hidden, gitignore, node_modules, pattern)
+    sc = _get_default_cache()
+    entries, age = sc.get_or_scan(
+        key,
+        lambda: _grep_scanner(
+            pattern, root, hidden=hidden, gitignore=gitignore, node_modules=node_modules
+        ),
+    )
+    return (entries, age)
+
+
+def cached_find(
+    name: str,
+    root: str = ".",
+    hidden: bool = False,
+    gitignore: bool = True,
+    node_modules: bool = True,
+    cache: bool = False,
+) -> tuple[list[GlobMatch], float | None]:
+    """Filename search with optional scan caching.
+
+    Returns:
+        ``(entries, cache_age_ms)`` — *cache_age_ms* is ``None`` when
+        *cache* is ``False``.
+    """
+    if not cache:
+        return (
+            _find_scanner(
+                name,
+                root,
+                hidden=hidden,
+                gitignore=gitignore,
+                node_modules=node_modules,
+            ),
+            None,
+        )
+
+    key = (root, hidden, gitignore, node_modules, name)
+    sc = _get_default_cache()
+    entries, age = sc.get_or_scan(
+        key,
+        lambda: _find_scanner(
+            name, root, hidden=hidden, gitignore=gitignore, node_modules=node_modules
+        ),
+    )
+    return (entries, age)

code_muse/fs_scan_cache/ttl_policy.py

@@ -0,0 +1,44 @@
+"""TTL policy for ScanCache entries."""
+
+import os
+import typing
+
+if typing.TYPE_CHECKING:
+    from code_muse.fs_scan_cache.scan_cache_core import ScanEntry
+
+
+def env_uint(name: str, default: int) -> int:
+    """Read an unsigned integer from an environment variable.
+
+    Returns *default* if the variable is missing, empty, or not a valid
+    non-negative integer.
+    """
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    try:
+        value = int(raw)
+    except ValueError:
+        return default
+    return value if value >= 0 else default
+
+
+CACHE_TTL_MS: int = env_uint("FS_SCAN_CACHE_TTL_MS", 1000)
+EMPTY_RECHECK_MS: int = env_uint("FS_SCAN_EMPTY_RECHECK_MS", 200)
+
+
+def is_fresh(entry: ScanEntry, now: float) -> bool:
+    """Return ``True`` if *entry* has not exceeded its TTL.
+
+    Rules:
+    * If ``CACHE_TTL_MS == 0``: always stale (cache bypass).
+    * If *entry* has no items: age must be < ``EMPTY_RECHECK_MS``.
+    * If *entry* has items: age must be < ``CACHE_TTL_MS``.
+    """
+    if CACHE_TTL_MS == 0:
+        return False
+
+    age_ms = (now - entry.created_at) * 1000.0
+    if not entry.entries:
+        return age_ms < EMPTY_RECHECK_MS
+    return age_ms < CACHE_TTL_MS