cecli-dev 0.100.3__tar.gz → 0.100.4rc2__tar.gz

{cecli_dev-0.100.3 → cecli_dev-0.100.4rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cecli-dev
-Version: 0.100.3
+Version: 0.100.4rc2
 Summary: we also can't pronounce cecli
 Project-URL: Homepage, https://github.com/dwash96/cecli
 Project-URL: Documentation, https://github.com/dwash96/cecli
@@ -53,6 +53,8 @@ Requires-Dist: truststore
 Requires-Dist: xxhash>=3.6.0
 Requires-Dist: py-cymbal>=0.1.24
 Requires-Dist: cryptography>=42.0.0
+Requires-Dist: marisa-trie>=1.0
+Requires-Dist: rapidfuzz>=3.0
 Requires-Dist: rustworkx>=0.15.0
 Requires-Dist: scipy>=1.15.3
 Requires-Dist: importlib-metadata>=7.2.1

{cecli_dev-0.100.3 → cecli_dev-0.100.4rc2}/cecli/__init__.py

@@ -1,6 +1,6 @@
 from packaging import version
 
-__version__ = "0.100.2.dev"
+__version__ = "0.100.4.dev"
 safe_version = __version__
 
 try:

{cecli_dev-0.100.3 → cecli_dev-0.100.4rc2}/cecli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.100.3'
-__version_tuple__ = version_tuple = (0, 100, 3)
+__version__ = version = '0.100.4rc2'
+__version_tuple__ = version_tuple = (0, 100, 4, 'rc2')
 
-__commit_id__ = commit_id = 'ge53c2cd27'
+__commit_id__ = commit_id = 'gc1c8e3e88'

{cecli_dev-0.100.3 → cecli_dev-0.100.4rc2}/cecli/coders/base_coder.py

@@ -42,6 +42,7 @@ from cecli.commands import Commands, SwitchCoderSignal
 from cecli.exceptions import LiteLLMExceptions
 from cecli.helpers import command_parser, coroutines, nested, responses
 from cecli.helpers.conversation import ConversationService, MessageTag
+from cecli.helpers.file_system import FileSystemService
 from cecli.helpers.io_proxy import IOProxy
 from cecli.helpers.observations.service import ObservationService
 from cecli.helpers.profiler import TokenProfiler
@@ -547,7 +548,6 @@ class Coder(metaclass=UsageMeta):
 
         self.data_cache = {
             "repo": {"last_key": "", "read_only_count": None},
-            "relative_files": None,
         }
 
         self.repo = repo
@@ -594,6 +594,12 @@ class Coder(metaclass=UsageMeta):
         if not self.repo:
             self.root = utils.find_common_root(self.abs_fnames)
 
+        # Initialize the FileSystemService singleton for all agents
+        FileSystemService.get_instance(
+            root=self.root if hasattr(self, "root") else ".",
+            repo=self.repo if hasattr(self, "repo") else None,
+        )
+
         if read_only_fnames:
             self.abs_read_only_fnames = set()
             for fname in read_only_fnames:
@@ -3226,7 +3232,7 @@ class Coder(metaclass=UsageMeta):
         group = ConfirmGroup(new_mentions)
         for rel_fname in sorted(new_mentions):
             message = "Add file to the chat?"
-            if self.args.tui:
+            if self.args and self.args.tui:
                 message = f"Add file to the chat? ({rel_fname})"
 
             if await self.io.confirm_ask(
@@ -3909,37 +3915,16 @@ class Coder(metaclass=UsageMeta):
             return
 
     def get_all_relative_files(self):
-        if self.repo_map and self.repo:
-            try:
-                staged_files_hash = hash(
-                    str([item.a_path for item in self.repo.repo.index.diff("HEAD")])
-                )
-                if (
-                    staged_files_hash == self.data_cache["repo"]["last_key"]
-                    and self.data_cache["relative_files"]
-                ):
-                    return self.data_cache["relative_files"]
-            except ANY_GIT_ERROR as err:
-                # Handle git errors gracefully - fall back to getting tracked files
-                if self.verbose:
-                    self.io.tool_warning(f"Git error while checking staged files: {err}")
-                # Continue to get tracked files normally
-
-        if self.repo:
-            if hasattr(self.repo, "workspace_path") and self.repo.workspace_path:
-                files = self.repo.get_workspace_files()
-            elif not self.repo.cecli_ignore_file or not self.repo.cecli_ignore_file.is_file():
-                files = self.repo.get_tracked_files()
-            else:
-                files = self.repo.get_non_ignored_files_from_root()
-        else:
-            files = self.get_inchat_relative_files()
-        # This is quite slow in large repos
-        # files = [fname for fname in files if self.is_file_safe(fname)]
-
-        self.data_cache["relative_files"] = sorted(set(files))
-
-        return self.data_cache["relative_files"]
+        """Get all files known to the file service singleton."""
+        fs = FileSystemService.get_instance()
+        if fs.trie:
+            # Auto-rebuild if the repository state has changed
+            # (e.g., new commits, staged files, or HEAD change)
+            if fs.needs_rebuild():
+                fs.rebuild()
+            files = fs.list_all()
+            return files
+        return self.get_inchat_relative_files()
 
     def get_all_abs_files(self):
         files = self.get_all_relative_files()

{cecli_dev-0.100.3 → cecli_dev-0.100.4rc2}/cecli/commands/utils/helpers.py

@@ -2,6 +2,8 @@ import os
 from pathlib import Path
 from typing import List
 
+from cecli.helpers.file_system import FileSystemService
+
 
 class CommandError(Exception):
     """Custom exception for command-specific errors."""
@@ -136,6 +138,9 @@ def get_file_completions(
     This function provides unified file completion logic that can be used by
     multiple commands (add, read-only, read-only-stub, etc.).
 
+    Uses FileSystemService when available for efficient trie/trigram lookups,
+    with fallback to legacy behavior.
+
     Args:
         coder: Coder instance
         args: Command arguments to complete
@@ -153,6 +158,10 @@ def get_file_completions(
 
     root = Path(coder.root) if hasattr(coder, "root") else Path.cwd()
 
+    # Try using FileSystemService for efficient lookups
+    fs = FileSystemService.get_instance()
+    fs_available = fs.trie is not None or fs.ngram is not None
+
     if completion_type == "glob":
         # Handle glob pattern completion
         if not args.strip():
@@ -166,6 +175,7 @@ def get_file_completions(
 
     elif completion_type == "directory":
         # Handle directory-based prefix matching (like read-only commands)
+        # Fallback: iterate filesystem directory
         if "/" in args:
             # Has directory component
             dir_part, file_part = args.rsplit("/", 1)
@@ -199,12 +209,19 @@ def get_file_completions(
 
     else:  # "all" completion type
         # Get all available files
-        if filter_in_chat:
-            files = set(coder.get_all_relative_files())
-            files = files - set(coder.get_inchat_relative_files())
-            completions = list(files)
+        if fs_available and fs.trie:
+            all_files = fs.list_all()
+            if filter_in_chat:
+                inchat = set(coder.get_inchat_relative_files())
+                all_files = [f for f in all_files if f not in inchat]
+            completions = all_files
         else:
-            completions = coder.get_all_relative_files()
+            if filter_in_chat:
+                files = set(coder.get_all_relative_files())
+                files = files - set(coder.get_inchat_relative_files())
+                completions = list(files)
+            else:
+                completions = coder.get_all_relative_files()
 
     # Quote filenames with spaces
     completions = [quote_filename(fn) for fn in completions]

cecli_dev-0.100.4rc2/cecli/helpers/file_system/__init__.py

@@ -0,0 +1,3 @@
+from .service import FileSystemService
+
+__all__ = ["FileSystemService"]

cecli_dev-0.100.4rc2/cecli/helpers/file_system/builders.py

@@ -0,0 +1,172 @@
+"""
+Construction strategies for FileSystemService.
+
+- GitBuilder: uses git ls-files (respects .gitignore)
+- ScandirBuilder: uses os.walk with ignore filtering
+"""
+
+import hashlib
+import os
+import subprocess
+from pathlib import Path
+
+from .ignore import FileIgnoreFilter
+
+
+class GitBuilder:
+    """Build file list from git repository."""
+
+    @staticmethod
+    def collect(root: str) -> list[str]:
+        """
+        Collect all tracked files relative to root.
+
+        Uses ``git ls-files -z`` for null-delimited, encoding-safe output.
+        Respects .gitignore naturally (git's built-in behavior).
+
+        Args:
+            root: Root directory of the git repository
+
+        Returns:
+            Sorted list of root-relative paths
+        """
+        try:
+            result = subprocess.check_output(
+                ["git", "ls-files", "-z"],
+                cwd=root,
+                text=True,
+            )
+            paths = [p for p in result.split("\0") if p]
+            return sorted(paths)
+        except subprocess.CalledProcessError:
+            return []
+        except FileNotFoundError:
+            return []
+        return sorted(paths)
+
+    @staticmethod
+    def staged_only(root: str) -> list[str]:
+        """
+        Get only staged files (currently staged but not committed).
+
+        Args:
+            root: Root directory of the git repository
+
+        Returns:
+            List of staged file paths relative to root
+        """
+        result = subprocess.check_output(
+            ["git", "diff", "--cached", "--name-only", "-z"],
+            cwd=root,
+            text=True,
+        )
+        return [p for p in result.split("\0") if p]
+
+    @staticmethod
+    def get_head_sha(root: str) -> str:
+        """
+        Get current HEAD SHA for cache invalidation.
+
+        Args:
+            root: Root directory of the git repository
+
+        Returns:
+            HEAD commit SHA string, or empty string on failure
+        """
+        try:
+            return subprocess.check_output(
+                ["git", "rev-parse", "HEAD"],
+                cwd=root,
+                text=True,
+                stderr=subprocess.DEVNULL,
+            ).strip()
+        except subprocess.CalledProcessError:
+            return ""
+
+    @staticmethod
+    def get_cache_key(root: str) -> str:
+        """
+        Generate a cache key combining HEAD SHA and staged file paths.
+
+        Combines the current HEAD commit hash with the list of staged
+        (but not yet committed) files so that new files appear in the
+        index even before they are committed.
+
+        Args:
+            root: Root directory of the git repository
+
+        Returns:
+            SHA-256 hex digest as cache key
+        """
+        sha = GitBuilder.get_head_sha(root)
+        staged = GitBuilder.staged_only(root)
+        combined = sha + "|" + "".join(sorted(staged))
+        return hashlib.sha256(combined.encode()).hexdigest()
+
+
+class ScandirBuilder:
+    """Build file list by walking filesystem."""
+
+    @staticmethod
+    def collect(
+        root: str,
+        ignore_filter: FileIgnoreFilter | None = None,
+    ) -> list[str]:
+        """
+        Walk filesystem collecting files relative to root.
+
+        Prunes ignored directories during walk (never descends into them),
+        avoiding wasted I/O on subtrees like ``node_modules/``.
+
+        Args:
+            root: Root directory to scan
+            ignore_filter: Optional ignore rules for filtering
+
+        Returns:
+            Sorted list of root-relative paths (POSIX-style)
+        """
+        paths = []
+        root_path = Path(root).resolve()
+
+        for dirpath, dirnames, filenames in os.walk(root_path):
+            rel_dir = os.path.relpath(dirpath, root)
+
+            # Prune ignored directories in-place (os.walk obeys this)
+            if ignore_filter:
+                dirnames[:] = [
+                    d
+                    for d in dirnames
+                    if not ignore_filter.is_dir_ignored(os.path.join(rel_dir, d))
+                ]
+
+            for filename in filenames:
+                rel_path = os.path.join(rel_dir, filename) if rel_dir != "." else filename
+                if ignore_filter and ignore_filter.is_file_ignored(rel_path):
+                    continue
+                # Normalize to POSIX-style paths
+                paths.append(rel_path.replace(os.sep, "/"))
+
+        return sorted(paths)
+
+    @staticmethod
+    def get_cache_key(root: str) -> str:
+        """
+        Generate a cache key from directory mtime.
+
+        Hashes the mtimes of all immediate children under root to detect
+        filesystem changes. A more robust approach would hash mtimes of
+        all nested entries, but this provides a good trade-off for speed.
+
+        Args:
+            root: Root directory to scan
+
+        Returns:
+            SHA-256 hex digest as cache key
+        """
+        mtimes = []
+        for entry in os.scandir(root):
+            try:
+                mtimes.append(str(entry.stat().st_mtime))
+            except OSError:
+                continue
+        return hashlib.sha256("|".join(sorted(mtimes)).encode()).hexdigest()

cecli_dev-0.100.4rc2/cecli/helpers/file_system/ignore.py

@@ -0,0 +1,80 @@
+"""
+File ignore rules for non-git mode.
+
+In git mode, .gitignore is handled by git ls-files natively.
+In scandir mode, we need our own ignore logic.
+"""
+
+from pathlib import Path
+
+import pathspec
+
+# Directories to always ignore (common noise sources)
+DEFAULT_IGNORE_DIRS = frozenset(
+    {
+        "node_modules",
+        ".git",
+        "__pycache__",
+        ".venv",
+        "venv",
+        "env",
+        ".tox",
+        ".eggs",
+        "eggs",
+        ".mypy_cache",
+        ".pytest_cache",
+        ".ruff_cache",
+        ".hypothesis",
+        "build",
+        "dist",
+        ".next",
+        "target",
+        ".gradle",
+    }
+)
+
+# File patterns to always ignore
+DEFAULT_IGNORE_PATTERNS = frozenset(
+    {
+        "*.pyc",
+        "*.pyo",
+        "*.pyd",
+        "*.so",
+        "*.dylib",
+        "*.dll",
+        ".DS_Store",
+        "Thumbs.db",
+        "*.egg-info/",
+    }
+)
+
+
+class FileIgnoreFilter:
+    """
+    Combined ignore filter that checks:
+      1. .cecli.ignore file (if present)
+      2. Common default patterns
+    """
+
+    def __init__(self, cecli_ignore_path: str | None = None):
+        self.cecli_ignore_spec = None
+        if cecli_ignore_path and Path(cecli_ignore_path).is_file():
+            with open(cecli_ignore_path) as f:
+                self.cecli_ignore_spec = pathspec.PathSpec.from_lines(
+                    pathspec.patterns.GitWildMatchPattern, f
+                )
+
+    def is_dir_ignored(self, rel_dir: str) -> bool:
+        """Check if a directory should be ignored during filesystem walk."""
+        dirname = Path(rel_dir).name
+        if dirname in DEFAULT_IGNORE_DIRS:
+            return True
+        if self.cecli_ignore_spec and self.cecli_ignore_spec.match_file(rel_dir):
+            return True
+        return False
+
+    def is_file_ignored(self, rel_path: str) -> bool:
+        """Check if a file should be ignored."""
+        if self.cecli_ignore_spec and self.cecli_ignore_spec.match_file(rel_path):
+            return True
+        return False

cecli_dev-0.100.4rc2/cecli/helpers/file_system/ngram_index.py

@@ -0,0 +1,157 @@
+"""
+RapidFuzz-based fuzzy file path matching engine.
+
+All searches run against the trie index's path list (via list_all())
+to avoid duplicating the full file list in memory. Falls back to an
+internally stored list when no trie reference is available.
+"""
+
+from rapidfuzz import fuzz, process
+
+# Threshold presets for different use cases (0-1 scale, converted to 0-100 internally)
+THRESHOLD_MENTION_DETECTION: float = 0.5
+THRESHOLD_FUZZY_SEARCH: float = 0.3
+THRESHOLD_BASENAME_SEARCH: float = 0.6
+
+
+class TrigramIndex:
+    """
+    RapidFuzz-based fuzzy file path matcher.
+
+    Searches against a trie index's path list to avoid duplicating
+    the full file list in memory. Falls back to an internally stored
+    list when no trie reference is available (e.g., after serialization).
+    """
+
+    def __init__(
+        self,
+        items: list[str] | None = None,
+        trie_index=None,
+    ):
+        """
+        Initialize the trigram index.
+
+        Args:
+            items: Initial list of file paths to index (fallback)
+            trie_index: A TrieIndex whose list_all() provides the path list
+        """
+        self._items = items
+        self._trie_index = trie_index
+
+    def _get_items(self) -> list[str]:
+        """Resolve the path list from the trie or fall back to stored items."""
+        if self._trie_index is not None:
+            return self._trie_index.list_all()
+        return self._items or []
+
+    def build(self, items: list[str]) -> None:
+        """
+        Build or rebuild the index from a list of items.
+
+        Stores the full list for rapidfuzz to search against.
+        Detaches any trie reference.
+
+        Args:
+            items: Full list of file paths to index
+        """
+        self._items = items
+        self._trie_index = None
+
+    def search(
+        self,
+        query: str,
+        threshold: float = THRESHOLD_FUZZY_SEARCH,
+        max_results: int = 20,
+    ) -> list[str]:
+        """
+        Fuzzy search for items matching a query string.
+
+        Uses rapidfuzz.process.extract() with WRatio scorer for
+        fast C++-backed fuzzy matching against the trie's path list.
+
+        Args:
+            query: Search string
+            threshold: Minimum similarity score (0.0-1.0)
+            max_results: Maximum number of results to return
+
+        Returns:
+            List of matching items, sorted by relevance
+        """
+        items = self._get_items()
+
+        if not items or not query:
+            return []
+        score_cutoff = int(threshold * 100)
+        results = process.extract(
+            query,
+            items,
+            scorer=fuzz.WRatio,
+            limit=max_results,
+            score_cutoff=score_cutoff,
+        )
+        return [match for match, score, _ in results]
+
+    def search_with_scores(
+        self,
+        query: str,
+        threshold: float = THRESHOLD_FUZZY_SEARCH,
+        max_results: int = 20,
+    ) -> list[tuple[str, float]]:
+        """
+        Fuzzy search returning items with their similarity scores.
+
+        Args:
+            query: Search string
+            threshold: Minimum similarity score (0.0-1.0)
+            max_results: Maximum number of results to return
+
+        Returns:
+            List of (item, score) tuples, sorted by score descending
+        """
+        items = self._get_items()
+        if not items or not query:
+            return []
+        score_cutoff = int(threshold * 100)
+        results = process.extract(
+            query,
+            items,
+            scorer=fuzz.WRatio,
+            limit=max_results,
+            score_cutoff=score_cutoff,
+        )
+        return [(match, score / 100.0) for match, score, _ in results]
+
+    def count(self) -> int:
+        """Return the number of items in the index."""
+        return len(self._get_items())
+
+    def clear(self) -> None:
+        """Clear the index."""
+        self._items = None
+        self._trie_index = None
+
+    @property
+    def ngram(self):
+        """Backward-compatibility shim — always returns None."""
+        return True
+
+    @property
+    def configuration(self) -> dict:
+        """Return the current configuration parameters."""
+        return {
+            "engine": "rapidfuzz",
+            "scorer": "fuzz.WRatio",
+        }
+
+    # --- Pickle support ---
+
+    def __getstate__(self):
+        """Resolve items before serialization so the trie ref is not pickled."""
+        state = self.__dict__.copy()
+        state["_items"] = self._get_items()
+        state["_trie_index"] = None
+        return state
+
+    def __setstate__(self, state):
+        """Restore from pickled state."""
+        self.__dict__.update(state)