deepagents 0.1.4__py3-none-any.whl → 0.1.5rc1__py3-none-any.whl

deepagents/backends/filesystem.py

@@ -0,0 +1,452 @@
+"""FilesystemBackend: Read and write files directly from the filesystem.
+
+Security and search upgrades:
+- Secure path resolution with root containment when in virtual_mode (sandboxed to cwd)
+- Prevent symlink-following on file I/O using O_NOFOLLOW when available
+- Ripgrep-powered grep with JSON parsing, plus Python fallback with regex
+  and optional glob include filtering, while preserving virtual path behavior
+"""
+
+import os
+import re
+import json
+import subprocess
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from langchain.tools import ToolRuntime
+
+from .utils import (
+    check_empty_content,
+    format_content_with_line_numbers,
+    perform_string_replacement,
+    truncate_if_too_long,
+)
+import wcmatch.glob as wcglob
+from deepagents.backends.utils import FileInfo, GrepMatch
+from deepagents.backends.protocol import WriteResult, EditResult
+
+
+
+class FilesystemBackend:
+    """Backend that reads and writes files directly from the filesystem.
+
+    Files are accessed using their actual filesystem paths. Relative paths are
+    resolved relative to the current working directory. Content is read/written
+    as plain text, and metadata (timestamps) are derived from filesystem stats.
+    """
+
+    def __init__(
+        self, 
+        root_dir: Optional[str | Path] = None,
+        virtual_mode: bool = False,
+        max_file_size_mb: int = 10,
+    ) -> None:
+        """Initialize filesystem backend.
+        
+        Args:
+            root_dir: Optional root directory for file operations. If provided,
+                     all file paths will be resolved relative to this directory.
+                     If not provided, uses the current working directory.
+        """
+        self.cwd = Path(root_dir) if root_dir else Path.cwd()
+        self.virtual_mode = virtual_mode
+        self.max_file_size_bytes = max_file_size_mb * 1024 * 1024
+
+    def _resolve_path(self, key: str) -> Path:
+        """Resolve a file path with security checks.
+
+        When virtual_mode=True, treat incoming paths as virtual absolute paths under
+        self.cwd, disallow traversal (.., ~) and ensure resolved path stays within root.
+        When virtual_mode=False, preserve legacy behavior: absolute paths are allowed
+        as-is; relative paths resolve under cwd.
+
+        Args:
+            key: File path (absolute, relative, or virtual when virtual_mode=True)
+
+        Returns:
+            Resolved absolute Path object
+        """
+        if self.virtual_mode:
+            vpath = key if key.startswith("/") else "/" + key
+            if ".." in vpath or vpath.startswith("~"):
+                raise ValueError("Path traversal not allowed")
+            full = (self.cwd / vpath.lstrip("/")).resolve()
+            try:
+                full.relative_to(self.cwd)
+            except ValueError:
+                raise ValueError(f"Path outside root directory: {key}") from None
+            return full
+
+        path = Path(key)
+        if path.is_absolute():
+            return path
+        return (self.cwd / path).resolve()
+
+    def ls_info(self, path: str) -> list[FileInfo]:
+        """List files from filesystem.
+
+        Args:
+            path: Absolute directory path to list files from.
+        
+        Returns:
+            List of FileInfo-like dicts.
+        """
+        dir_path = self._resolve_path(path)
+        if not dir_path.exists() or not dir_path.is_dir():
+            return []
+
+        results: list[FileInfo] = []
+
+        # Convert cwd to string for comparison
+        cwd_str = str(self.cwd)
+        if not cwd_str.endswith("/"):
+            cwd_str += "/"
+
+        # Walk the directory tree
+        try:
+            for path in dir_path.rglob("*"):
+                try:
+                    is_file = path.is_file()
+                except OSError:
+                    continue
+                if is_file:
+                    abs_path = str(path)
+                    if not self.virtual_mode:
+                        try:
+                            st = path.stat()
+                            results.append({
+                                "path": abs_path,
+                                "is_dir": False,
+                                "size": int(st.st_size),
+                                "modified_at": datetime.fromtimestamp(st.st_mtime).isoformat(),
+                            })
+                        except OSError:
+                            results.append({"path": abs_path, "is_dir": False})
+                        continue
+                    # Strip the cwd prefix if present
+                    if abs_path.startswith(cwd_str):
+                        relative_path = abs_path[len(cwd_str):]
+                    elif abs_path.startswith(str(self.cwd)):
+                        # Handle case where cwd doesn't end with /
+                        relative_path = abs_path[len(str(self.cwd)):].lstrip("/")
+                    else:
+                        # Path is outside cwd, return as-is or skip
+                        relative_path = abs_path
+
+                    virt_path = "/" + relative_path
+                    try:
+                        st = path.stat()
+                        results.append({
+                            "path": virt_path,
+                            "is_dir": False,
+                            "size": int(st.st_size),
+                            "modified_at": datetime.fromtimestamp(st.st_mtime).isoformat(),
+                        })
+                    except OSError:
+                        results.append({"path": virt_path, "is_dir": False})
+        except (OSError, PermissionError):
+            pass
+
+        # Keep deterministic order by path
+        results.sort(key=lambda x: x.get("path", ""))
+        return results
+
+    # Removed legacy ls() convenience to keep lean surface
+    
+    def read(
+        self, 
+        file_path: str,
+        offset: int = 0,
+        limit: int = 2000,
+    ) -> str:
+        """Read file content with line numbers.
+        
+        Args:
+            file_path: Absolute or relative file path
+            offset: Line offset to start reading from (0-indexed)
+            limit: Maximum number of lines to readReturns:
+            Formatted file content with line numbers, or error message.
+        """
+        resolved_path = self._resolve_path(file_path)
+        
+        if not resolved_path.exists() or not resolved_path.is_file():
+            return f"Error: File '{file_path}' not found"
+        
+        try:
+            # Open with O_NOFOLLOW where available to avoid symlink traversal
+            try:
+                fd = os.open(resolved_path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+                with os.fdopen(fd, "r", encoding="utf-8") as f:
+                    content = f.read()
+            except OSError:
+                # Fallback to normal open if O_NOFOLLOW unsupported or fails
+                with open(resolved_path, "r", encoding="utf-8") as f:
+                    content = f.read()
+            
+            empty_msg = check_empty_content(content)
+            if empty_msg:
+                return empty_msg
+            
+            lines = content.splitlines()
+            start_idx = offset
+            end_idx = min(start_idx + limit, len(lines))
+            
+            if start_idx >= len(lines):
+                return f"Error: Line offset {offset} exceeds file length ({len(lines)} lines)"
+            
+            selected_lines = lines[start_idx:end_idx]
+            return format_content_with_line_numbers(selected_lines, start_line=start_idx + 1)
+        except (OSError, UnicodeDecodeError) as e:
+            return f"Error reading file '{file_path}': {e}"
+    
+    def write(
+        self, 
+        file_path: str,
+        content: str,
+    ) -> WriteResult:
+        """Create a new file with content.
+        Returns WriteResult. External storage sets files_update=None.
+        """
+        resolved_path = self._resolve_path(file_path)
+        
+        if resolved_path.exists():
+            return WriteResult(error=f"Cannot write to {file_path} because it already exists. Read and then make an edit, or write to a new path.")
+        
+        try:
+            # Create parent directories if needed
+            resolved_path.parent.mkdir(parents=True, exist_ok=True)
+
+            # Prefer O_NOFOLLOW to avoid writing through symlinks
+            flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+            if hasattr(os, "O_NOFOLLOW"):
+                flags |= os.O_NOFOLLOW
+            fd = os.open(resolved_path, flags, 0o644)
+            with os.fdopen(fd, "w", encoding="utf-8") as f:
+                f.write(content)
+            
+            return WriteResult(path=file_path, files_update=None)
+        except (OSError, UnicodeEncodeError) as e:
+            return WriteResult(error=f"Error writing file '{file_path}': {e}")
+    
+    def edit(
+        self, 
+        file_path: str,
+        old_string: str,
+        new_string: str,
+        replace_all: bool = False,
+    ) -> EditResult:
+        """Edit a file by replacing string occurrences.
+        Returns EditResult. External storage sets files_update=None.
+        """
+        resolved_path = self._resolve_path(file_path)
+        
+        if not resolved_path.exists() or not resolved_path.is_file():
+            return EditResult(error=f"Error: File '{file_path}' not found")
+        
+        try:
+            # Read securely
+            try:
+                fd = os.open(resolved_path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+                with os.fdopen(fd, "r", encoding="utf-8") as f:
+                    content = f.read()
+            except OSError:
+                with open(resolved_path, "r", encoding="utf-8") as f:
+                    content = f.read()
+            
+            result = perform_string_replacement(content, old_string, new_string, replace_all)
+            
+            if isinstance(result, str):
+                return EditResult(error=result)
+            
+            new_content, occurrences = result
+            
+            # Write securely
+            flags = os.O_WRONLY | os.O_TRUNC
+            if hasattr(os, "O_NOFOLLOW"):
+                flags |= os.O_NOFOLLOW
+            fd = os.open(resolved_path, flags)
+            with os.fdopen(fd, "w", encoding="utf-8") as f:
+                f.write(new_content)
+            
+            return EditResult(path=file_path, files_update=None, occurrences=int(occurrences))
+        except (OSError, UnicodeDecodeError, UnicodeEncodeError) as e:
+            return EditResult(error=f"Error editing file '{file_path}': {e}")
+    
+    # Removed legacy grep() convenience to keep lean surface
+
+    def grep_raw(
+        self,
+        pattern: str,
+        path: Optional[str] = None,
+        glob: Optional[str] = None,
+    ) -> list[GrepMatch] | str:
+        # Validate regex
+        try:
+            re.compile(pattern)
+        except re.error as e:
+            return f"Invalid regex pattern: {e}"
+
+        # Resolve base path
+        try:
+            base_full = self._resolve_path(path or ".")
+        except ValueError:
+            return []
+
+        if not base_full.exists():
+            return []
+
+        # Try ripgrep first
+        results = self._ripgrep_search(pattern, base_full, glob)
+        if results is None:
+            results = self._python_search(pattern, base_full, glob)
+
+        matches: list[GrepMatch] = []
+        for fpath, items in results.items():
+            for line_num, line_text in items:
+                matches.append({"path": fpath, "line": int(line_num), "text": line_text})
+        return matches
+
+    def _ripgrep_search(
+        self, pattern: str, base_full: Path, include_glob: Optional[str]
+    ) -> Optional[dict[str, list[tuple[int, str]]]]:
+        cmd = ["rg", "--json"]
+        if include_glob:
+            cmd.extend(["--glob", include_glob])
+        cmd.extend(["--", pattern, str(base_full)])
+
+        try:
+            proc = subprocess.run(  # noqa: S603
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=30,
+                check=False,
+            )
+        except (subprocess.TimeoutExpired, FileNotFoundError):
+            return None
+
+        results: dict[str, list[tuple[int, str]]] = {}
+        for line in proc.stdout.splitlines():
+            try:
+                data = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if data.get("type") != "match":
+                continue
+            pdata = data.get("data", {})
+            ftext = pdata.get("path", {}).get("text")
+            if not ftext:
+                continue
+            p = Path(ftext)
+            if self.virtual_mode:
+                try:
+                    virt = "/" + str(p.resolve().relative_to(self.cwd))
+                except Exception:
+                    continue
+            else:
+                virt = str(p)
+            ln = pdata.get("line_number")
+            lt = pdata.get("lines", {}).get("text", "").rstrip("\n")
+            if ln is None:
+                continue
+            results.setdefault(virt, []).append((int(ln), lt))
+
+        return results
+
+    def _python_search(
+        self, pattern: str, base_full: Path, include_glob: Optional[str]
+    ) -> dict[str, list[tuple[int, str]]]:
+        try:
+            regex = re.compile(pattern)
+        except re.error:
+            return {}
+
+        results: dict[str, list[tuple[int, str]]] = {}
+        root = base_full if base_full.is_dir() else base_full.parent
+
+        for fp in root.rglob("*"):
+            if not fp.is_file():
+                continue
+            if include_glob and not wcglob.globmatch(fp.name, include_glob, flags=wcglob.BRACE):
+                continue
+            try:
+                if fp.stat().st_size > self.max_file_size_bytes:
+                    continue
+            except OSError:
+                continue
+            try:
+                content = fp.read_text()
+            except (UnicodeDecodeError, PermissionError, OSError):
+                continue
+            for line_num, line in enumerate(content.splitlines(), 1):
+                if regex.search(line):
+                    if self.virtual_mode:
+                        try:
+                            virt_path = "/" + str(fp.resolve().relative_to(self.cwd))
+                        except Exception:
+                            continue
+                    else:
+                        virt_path = str(fp)
+                    results.setdefault(virt_path, []).append((line_num, line))
+
+        return results
+    
+    def glob_info(self, pattern: str, path: str = "/") -> list[FileInfo]:
+        if pattern.startswith("/"):
+            pattern = pattern.lstrip("/")
+
+        search_path = self.cwd if path == "/" else self._resolve_path(path)
+        if not search_path.exists() or not search_path.is_dir():
+            return []
+
+        results: list[FileInfo] = []
+        try:
+            # Use recursive globbing to match files in subdirectories as tests expect
+            for matched_path in search_path.rglob(pattern):
+                try:
+                    is_file = matched_path.is_file()
+                except OSError:
+                    continue
+                if not is_file:
+                    continue
+                abs_path = str(matched_path)
+                if not self.virtual_mode:
+                    try:
+                        st = matched_path.stat()
+                        results.append({
+                            "path": abs_path,
+                            "is_dir": False,
+                            "size": int(st.st_size),
+                            "modified_at": datetime.fromtimestamp(st.st_mtime).isoformat(),
+                        })
+                    except OSError:
+                        results.append({"path": abs_path, "is_dir": False})
+                else:
+                    cwd_str = str(self.cwd)
+                    if not cwd_str.endswith("/"):
+                        cwd_str += "/"
+                    if abs_path.startswith(cwd_str):
+                        relative_path = abs_path[len(cwd_str):]
+                    elif abs_path.startswith(str(self.cwd)):
+                        relative_path = abs_path[len(str(self.cwd)):].lstrip("/")
+                    else:
+                        relative_path = abs_path
+                    virt = "/" + relative_path
+                    try:
+                        st = matched_path.stat()
+                        results.append({
+                            "path": virt,
+                            "is_dir": False,
+                            "size": int(st.st_size),
+                            "modified_at": datetime.fromtimestamp(st.st_mtime).isoformat(),
+                        })
+                    except OSError:
+                        results.append({"path": virt, "is_dir": False})
+        except (OSError, ValueError):
+            pass
+
+        results.sort(key=lambda x: x.get("path", ""))
+        return results

deepagents/backends/protocol.py

@@ -0,0 +1,122 @@
+"""Protocol definition for pluggable memory backends.
+
+This module defines the BackendProtocol that all backend implementations
+must follow. Backends can store files in different locations (state, filesystem,
+database, etc.) and provide a uniform interface for file operations.
+"""
+
+from typing import TYPE_CHECKING, Optional, Protocol, runtime_checkable, Callable, TypeAlias, Any
+from langchain.tools import ToolRuntime
+from deepagents.backends.utils import FileInfo, GrepMatch
+
+from dataclasses import dataclass
+
+
+@dataclass
+class WriteResult:
+    """Result from backend write operations.
+    Attributes:
+        error: Error message on failure, None on success.
+        path: Absolute path of written file, None on failure.
+        files_update: State update dict for checkpoint backends, None for external storage.
+            Checkpoint backends populate this with {file_path: file_data} for LangGraph state.
+            External backends set None (already persisted to disk/S3/database/etc).
+    Examples:
+        >>> # Checkpoint storage
+        >>> WriteResult(path="/f.txt", files_update={"/f.txt": {...}})
+        >>> # External storage
+        >>> WriteResult(path="/f.txt", files_update=None)
+        >>> # Error
+        >>> WriteResult(error="File exists")
+    """
+
+    error: str | None = None
+    path: str | None = None
+    files_update: dict[str, Any] | None = None
+
+
+@dataclass
+class EditResult:
+    """Result from backend edit operations.
+    Attributes:
+        error: Error message on failure, None on success.
+        path: Absolute path of edited file, None on failure.
+        files_update: State update dict for checkpoint backends, None for external storage.
+            Checkpoint backends populate this with {file_path: file_data} for LangGraph state.
+            External backends set None (already persisted to disk/S3/database/etc).
+        occurrences: Number of replacements made, None on failure.
+    Examples:
+        >>> # Checkpoint storage
+        >>> EditResult(path="/f.txt", files_update={"/f.txt": {...}}, occurrences=1)
+        >>> # External storage
+        >>> EditResult(path="/f.txt", files_update=None, occurrences=2)
+        >>> # Error
+        >>> EditResult(error="File not found")
+    """
+
+    error: str | None = None
+    path: str | None = None
+    files_update: dict[str, Any] | None = None
+    occurrences: int | None = None
+
+@runtime_checkable
+class BackendProtocol(Protocol):
+    """Protocol for pluggable memory backends (single, unified).
+
+    Backends can store files in different locations (state, filesystem, database, etc.)
+    and provide a uniform interface for file operations.
+
+    All file data is represented as dicts with the following structure:
+    {
+        "content": list[str],      # Lines of text content
+        "created_at": str,         # ISO format timestamp
+        "modified_at": str,        # ISO format timestamp
+    }
+    """
+
+    def ls_info(self, path: str) -> list["FileInfo"]:
+        """Structured listing with file metadata."""
+        ...
+
+    def read(
+        self,
+        file_path: str,
+        offset: int = 0,
+        limit: int = 2000,
+    ) -> str:
+        """Read file content with line numbers or an error string."""
+        ...
+
+    def grep_raw(
+        self,
+        pattern: str,
+        path: Optional[str] = None,
+        glob: Optional[str] = None,
+    ) -> list["GrepMatch"] | str:
+        """Structured search results or error string for invalid input."""
+        ...
+
+    def glob_info(self, pattern: str, path: str = "/") -> list["FileInfo"]:
+        """Structured glob matching returning FileInfo dicts."""
+        ...
+
+    def write(
+            self,
+            file_path: str,
+            content: str,
+    ) -> WriteResult:
+        """Create a new file. Returns WriteResult; error populated on failure."""
+        ...
+
+    def edit(
+            self,
+            file_path: str,
+            old_string: str,
+            new_string: str,
+            replace_all: bool = False,
+    ) -> EditResult:
+        """Edit a file by replacing string occurrences. Returns EditResult."""
+        ...
+
+
+BackendFactory: TypeAlias = Callable[[ToolRuntime], BackendProtocol]