alita-sdk 0.3.465__py3-none-any.whl → 0.3.486__py3-none-any.whl

alita_sdk/cli/tools/filesystem.py

@@ -3,14 +3,56 @@ Filesystem tools for CLI agents.
 
 Provides comprehensive file system operations restricted to specific directories.
 Inspired by MCP filesystem server implementation.
+
+Also provides a FilesystemApiWrapper for integration with the inventory ingestion
+pipeline, enabling local document loading and chunking.
 """
 
+import base64
+import fnmatch
+import hashlib
+import logging
 import os
 from pathlib import Path
-from typing import Optional, List, Dict, Any
+from typing import Optional, List, Dict, Any, Generator, ClassVar
 from datetime import datetime
-from langchain_core.tools import BaseTool
-from pydantic import BaseModel, Field
+from langchain_core.tools import BaseTool, ToolException
+from langchain_core.documents import Document
+from pydantic import BaseModel, Field, model_validator
+
+logger = logging.getLogger(__name__)
+
+
+# Maximum recommended content size for single write operations (in characters)
+MAX_RECOMMENDED_CONTENT_SIZE = 5000  # ~5KB, roughly 1,200-1,500 tokens
+
+# Helpful error message for truncated content
+CONTENT_TRUNCATED_ERROR = """
+⚠️  CONTENT FIELD MISSING - OUTPUT TRUNCATED
+
+Your tool call was cut off because the content was too large for the context window.
+The JSON was truncated, leaving the 'content' field incomplete or missing.
+
+🔧 HOW TO FIX THIS:
+
+1. **Use incremental writes** - Don't write large files in one call:
+   - First: filesystem_write_file(path, "# Header\\nimport x\\n\\n")
+   - Then: filesystem_append_file(path, "def func1():\\n    ...\\n\\n")
+   - Then: filesystem_append_file(path, "def func2():\\n    ...\\n\\n")
+
+2. **Keep each chunk small** - Under 2000 characters per call
+
+3. **Structure first, details later**:
+   - Write skeleton/structure first
+   - Add implementations section by section
+
+4. **For documentation/reports**:
+   - Write one section at a time
+   - Use append_file for each new section
+
+❌ DON'T: Try to write the entire file content again
+✅ DO: Break it into 3-5 smaller append_file calls
+"""
 
 
 class ReadFileInput(BaseModel):
@@ -47,7 +89,38 @@ class ReadMultipleFilesInput(BaseModel):
 class WriteFileInput(BaseModel):
     """Input for writing to a file."""
     path: str = Field(description="Relative path to the file to write")
-    content: str = Field(description="Content to write to the file")
+    content: Optional[str] = Field(
+        default=None,
+        description="Content to write to the file. REQUIRED - this field cannot be empty or omitted."
+    )
+    
+    @model_validator(mode='after')
+    def validate_content_required(self):
+        """Provide helpful error message when content is missing or truncated."""
+        if self.content is None:
+            raise ToolException(CONTENT_TRUNCATED_ERROR)
+        if len(self.content) > MAX_RECOMMENDED_CONTENT_SIZE:
+            logger.warning(
+                f"Content is very large ({len(self.content)} chars). Consider using append_file "
+                "for incremental writes to avoid truncation issues."
+            )
+        return self
+
+
+class AppendFileInput(BaseModel):
+    """Input for appending to a file."""
+    path: str = Field(description="Relative path to the file to append to")
+    content: Optional[str] = Field(
+        default=None,
+        description="Content to append to the end of the file. REQUIRED - this field cannot be empty or omitted."
+    )
+    
+    @model_validator(mode='after')
+    def validate_content_required(self):
+        """Provide helpful error message when content is missing or truncated."""
+        if self.content is None:
+            raise ToolException(CONTENT_TRUNCATED_ERROR)
+        return self
 
 
 class EditFileInput(BaseModel):
@@ -67,13 +140,15 @@ class ListDirectoryInput(BaseModel):
 class DirectoryTreeInput(BaseModel):
     """Input for getting a directory tree."""
     path: str = Field(default=".", description="Relative path to the directory")
-    max_depth: Optional[int] = Field(None, description="Maximum depth to traverse (None for unlimited)")
+    max_depth: Optional[int] = Field(default=3, description="Maximum depth to traverse. Default is 3 to prevent excessive output. Use None for unlimited (caution: may exceed context limits).")
+    max_items: Optional[int] = Field(default=200, description="Maximum number of files/directories to include. Default is 200 to prevent context window overflow. Use None for unlimited (caution: large directories may exceed context limits).")
 
 
 class SearchFilesInput(BaseModel):
     """Input for searching files."""
     path: str = Field(default=".", description="Relative path to search from")
     pattern: str = Field(description="Glob pattern to match (e.g., '*.py', '**/*.txt')")
+    max_results: Optional[int] = Field(default=100, description="Maximum number of results to return. Default is 100 to prevent context overflow. Use None for unlimited.")
 
 
 class DeleteFileInput(BaseModel):
@@ -104,29 +179,58 @@ class EmptyInput(BaseModel):
 
 class FileSystemTool(BaseTool):
     """Base class for filesystem tools with directory restriction."""
-    base_directory: str
+    base_directory: str  # Primary directory (for backward compatibility)
+    allowed_directories: List[str] = []  # Additional allowed directories
+    
+    def _get_all_allowed_directories(self) -> List[Path]:
+        """Get all allowed directories as resolved Paths."""
+        dirs = [Path(self.base_directory).resolve()]
+        for d in self.allowed_directories:
+            resolved = Path(d).resolve()
+            if resolved not in dirs:
+                dirs.append(resolved)
+        return dirs
     
     def _resolve_path(self, relative_path: str) -> Path:
         """
-        Resolve and validate a path within the base directory.
+        Resolve and validate a path within any of the allowed directories.
         
-        Security: Ensures resolved path is within allowed directory.
+        Security: Ensures resolved path is within one of the allowed directories.
         """
-        base = Path(self.base_directory).resolve()
+        allowed_dirs = self._get_all_allowed_directories()
         
-        # Handle both relative and absolute paths
+        # Handle absolute paths - check if within any allowed directory
         if Path(relative_path).is_absolute():
             target = Path(relative_path).resolve()
-        else:
-            target = (base / relative_path).resolve()
+            for base in allowed_dirs:
+                try:
+                    target.relative_to(base)
+                    return target
+                except ValueError:
+                    continue
+            raise ValueError(f"Access denied: path '{relative_path}' is outside allowed directories")
         
-        # Security check: ensure the resolved path is within base directory
-        try:
-            target.relative_to(base)
-        except ValueError:
-            raise ValueError(f"Access denied: path '{relative_path}' is outside allowed directory")
+        # For relative paths, try to resolve against each allowed directory
+        # First check primary base_directory
+        primary_base = allowed_dirs[0]
+        target = (primary_base / relative_path).resolve()
+        
+        # Check if target is within any allowed directory
+        for base in allowed_dirs:
+            try:
+                target.relative_to(base)
+                return target
+            except ValueError:
+                continue
+        
+        # If relative path doesn't work from primary, try finding the file in other directories
+        for base in allowed_dirs[1:]:
+            candidate = (base / relative_path).resolve()
+            if candidate.exists():
+                return candidate
         
-        return target
+        # Default to primary base directory resolution
+        raise ValueError(f"Access denied: path '{relative_path}' is outside allowed directories")
     
     def _format_size(self, size: int) -> str:
         """Format file size in human-readable format."""
@@ -147,6 +251,11 @@ class ReadFileTool(FileSystemTool):
         "Only works within allowed directories."
     )
     args_schema: type[BaseModel] = ReadFileInput
+    truncation_suggestions: ClassVar[List[str]] = [
+        "Use head=100 to read only the first 100 lines",
+        "Use tail=100 to read only the last 100 lines",
+        "Use filesystem_read_file_chunk with start_line and end_line for specific sections",
+    ]
     
     def _run(self, path: str, head: Optional[int] = None, tail: Optional[int] = None) -> str:
         """Read a file with optional head/tail."""
@@ -196,6 +305,10 @@ class ReadFileChunkTool(FileSystemTool):
         "Only works within allowed directories."
     )
     args_schema: type[BaseModel] = ReadFileChunkInput
+    truncation_suggestions: ClassVar[List[str]] = [
+        "Reduce the line range (end_line - start_line) to read fewer lines at once",
+        "Read smaller chunks sequentially if you need to process the entire file",
+    ]
     
     def _run(self, path: str, start_line: int = 1, end_line: Optional[int] = None) -> str:
         """Read a chunk of a file by line range."""
@@ -246,6 +359,10 @@ class ReadMultipleFilesTool(FileSystemTool):
         "Only works within allowed directories."
     )
     args_schema: type[BaseModel] = ReadMultipleFilesInput
+    truncation_suggestions: ClassVar[List[str]] = [
+        "Read fewer files at once - split into multiple smaller batches",
+        "Use filesystem_read_file with head parameter on individual large files instead",
+    ]
     
     def _run(self, paths: List[str]) -> str:
         """Read multiple files."""
@@ -291,6 +408,43 @@ class WriteFileTool(FileSystemTool):
             return f"Error writing to file '{path}': {str(e)}"
 
 
+class AppendFileTool(FileSystemTool):
+    """Append content to the end of a file."""
+    name: str = "filesystem_append_file"
+    description: str = (
+        "Append content to the end of an existing file. Creates the file if it doesn't exist. "
+        "Use this for incremental file creation - write initial structure with write_file, "
+        "then add sections progressively with append_file. This is safer than rewriting "
+        "entire files and prevents context overflow. Only works within allowed directories."
+    )
+    args_schema: type[BaseModel] = AppendFileInput
+    
+    def _run(self, path: str, content: str) -> str:
+        """Append to a file."""
+        try:
+            target = self._resolve_path(path)
+            
+            # Create parent directories if they don't exist
+            target.parent.mkdir(parents=True, exist_ok=True)
+            
+            # Check current file size if it exists
+            existed = target.exists()
+            original_size = target.stat().st_size if existed else 0
+            
+            with open(target, 'a', encoding='utf-8') as f:
+                f.write(content)
+            
+            appended_size = len(content.encode('utf-8'))
+            new_size = original_size + appended_size
+            
+            if existed:
+                return f"Successfully appended {self._format_size(appended_size)} to '{path}' (total: {self._format_size(new_size)})"
+            else:
+                return f"Created '{path}' and wrote {self._format_size(appended_size)}"
+        except Exception as e:
+            return f"Error appending to file '{path}': {str(e)}"
+
+
 class EditFileTool(FileSystemTool):
     """Edit file with precise text replacement."""
     name: str = "filesystem_edit_file"
@@ -443,6 +597,10 @@ class ListDirectoryTool(FileSystemTool):
         "Only works within allowed directories."
     )
     args_schema: type[BaseModel] = ListDirectoryInput
+    truncation_suggestions: ClassVar[List[str]] = [
+        "List a specific subdirectory instead of the root directory",
+        "Consider using filesystem_directory_tree with max_depth=1 for hierarchical overview",
+    ]
     
     def _run(self, path: str = ".", include_sizes: bool = False, sort_by: str = "name") -> str:
         """List directory contents."""
@@ -494,13 +652,23 @@ class ListDirectoryTool(FileSystemTool):
             
             result = "\n".join(lines)
             
+            # Add header showing the listing context
+            if path in (".", "", "./"):
+                header = "Contents of working directory (./):\n\n"
+            else:
+                header = f"Contents of {path}/:\n\n"
+            result = header + result
+            
             if include_sizes:
                 summary = f"\n\nTotal: {total_files} files, {total_dirs} directories"
                 if total_files > 0:
                     summary += f"\nCombined size: {self._format_size(total_size)}"
                 result += summary
             
-            return result if result else "Directory is empty"
+            # Add note about how to access files
+            result += "\n\nNote: Access files using paths shown above (e.g., 'agents/file.md' for items in agents/ directory)"
+            
+            return result if lines else "Directory is empty"
         except Exception as e:
             return f"Error listing directory '{path}': {str(e)}"
 
@@ -511,25 +679,51 @@ class DirectoryTreeTool(FileSystemTool):
     description: str = (
         "Get a recursive tree view of files and directories. "
         "Shows the complete structure in an easy-to-read tree format. "
-        "Use max_depth to limit recursion depth. "
+        "IMPORTANT: For large directories, use max_depth (default: 3) and max_items (default: 200) "
+        "to prevent context window overflow. Increase these only if needed for smaller directories. "
         "Only works within allowed directories."
     )
     args_schema: type[BaseModel] = DirectoryTreeInput
+    truncation_suggestions: ClassVar[List[str]] = [
+        "Use max_depth=2 to limit directory traversal depth",
+        "Use max_items=50 to limit total items returned",
+        "Target a specific subdirectory instead of the root",
+    ]
+    
+    # Track item count during tree building
+    _item_count: int = 0
+    _max_items: Optional[int] = None
+    _truncated: bool = False
     
     def _build_tree(self, directory: Path, prefix: str = "", depth: int = 0, max_depth: Optional[int] = None) -> List[str]:
-        """Recursively build directory tree."""
+        """Recursively build directory tree with item limit."""
+        # Check depth limit
         if max_depth is not None and depth >= max_depth:
             return []
         
+        # Check item limit
+        if self._max_items is not None and self._item_count >= self._max_items:
+            if not self._truncated:
+                self._truncated = True
+            return []
+        
         lines = []
         try:
             entries = sorted(directory.iterdir(), key=lambda x: (not x.is_dir(), x.name.lower()))
             
             for i, entry in enumerate(entries):
+                # Check item limit before adding each entry
+                if self._max_items is not None and self._item_count >= self._max_items:
+                    if not self._truncated:
+                        self._truncated = True
+                    break
+                
                 is_last = i == len(entries) - 1
                 current_prefix = "└── " if is_last else "├── "
                 next_prefix = "    " if is_last else "│   "
                 
+                self._item_count += 1
+                
                 if entry.is_dir():
                     lines.append(f"{prefix}{current_prefix}📁 {entry.name}/")
                     lines.extend(self._build_tree(entry, prefix + next_prefix, depth + 1, max_depth))
@@ -541,8 +735,8 @@ class DirectoryTreeTool(FileSystemTool):
         
         return lines
     
-    def _run(self, path: str = ".", max_depth: Optional[int] = None) -> str:
-        """Get directory tree."""
+    def _run(self, path: str = ".", max_depth: Optional[int] = 3, max_items: Optional[int] = 200) -> str:
+        """Get directory tree with size limits to prevent context overflow."""
         try:
             target = self._resolve_path(path)
             
@@ -552,9 +746,31 @@ class DirectoryTreeTool(FileSystemTool):
             if not target.is_dir():
                 return f"Error: '{path}' is not a directory"
             
-            lines = [f"📁 {target.name or path}/"]
+            # Reset counters for this run
+            self._item_count = 0
+            self._max_items = max_items
+            self._truncated = False
+            
+            # Show relative path from base directory, use '.' for root
+            # This prevents confusion - files should be accessed relative to working directory
+            if path in (".", "", "./"):
+                display_root = "."  # Root of working directory
+            else:
+                display_root = path.rstrip('/')
+            
+            lines = [f"📁 {display_root}/"]
             lines.extend(self._build_tree(target, "", 0, max_depth))
             
+            # Add truncation warning if limit was reached
+            if self._truncated:
+                lines.append("")
+                lines.append(f"⚠️  OUTPUT TRUNCATED: Showing {self._item_count} of more items (max_items={max_items}, max_depth={max_depth})")
+                lines.append(f"   To see more: increase max_items or max_depth, or use filesystem_list_directory on specific subdirectories")
+            
+            # Add note about file paths
+            lines.append("")
+            lines.append("Note: Use paths relative to working directory (e.g., 'agents/file.md', not including the root directory name)")
+            
             return "\n".join(lines)
         except Exception as e:
             return f"Error building directory tree for '{path}': {str(e)}"
@@ -566,13 +782,18 @@ class SearchFilesTool(FileSystemTool):
     description: str = (
         "Recursively search for files and directories matching a glob pattern. "
         "Use patterns like '*.py' for Python files in current dir, or '**/*.py' for all Python files recursively. "
-        "Returns full paths to all matching items. "
+        "Returns paths to matching items (default limit: 100 results to prevent context overflow). "
         "Only searches within allowed directories."
     )
     args_schema: type[BaseModel] = SearchFilesInput
+    truncation_suggestions: ClassVar[List[str]] = [
+        "Use max_results=50 to limit number of results",
+        "Use a more specific glob pattern (e.g., 'src/**/*.py' instead of '**/*.py')",
+        "Search in a specific subdirectory instead of the root",
+    ]
     
-    def _run(self, path: str = ".", pattern: str = "*") -> str:
-        """Search for files."""
+    def _run(self, path: str = ".", pattern: str = "*", max_results: Optional[int] = 100) -> str:
+        """Search for files with result limit."""
         try:
             target = self._resolve_path(path)
             
@@ -583,19 +804,25 @@ class SearchFilesTool(FileSystemTool):
                 return f"Error: '{path}' is not a directory"
             
             # Use glob to find matching files
-            if '**' in pattern:
-                matches = list(target.glob(pattern))
-            else:
-                matches = list(target.glob(pattern))
+            all_matches = list(target.glob(pattern))
+            total_count = len(all_matches)
             
-            if not matches:
+            if not all_matches:
                 return f"No files matching '{pattern}' found in '{path}'"
             
+            # Apply limit
+            truncated = False
+            if max_results is not None and total_count > max_results:
+                matches = sorted(all_matches)[:max_results]
+                truncated = True
+            else:
+                matches = sorted(all_matches)
+            
             # Format results
             base = Path(self.base_directory).resolve()
             results = []
             
-            for match in sorted(matches):
+            for match in matches:
                 rel_path = match.relative_to(base)
                 if match.is_dir():
                     results.append(f"📁 {rel_path}/")
@@ -603,8 +830,14 @@ class SearchFilesTool(FileSystemTool):
                     size = self._format_size(match.stat().st_size)
                     results.append(f"📄 {rel_path} ({size})")
             
-            header = f"Found {len(matches)} matches for '{pattern}':\n\n"
-            return header + "\n".join(results)
+            header = f"Found {total_count} matches for '{pattern}':\n\n"
+            output = header + "\n".join(results)
+            
+            if truncated:
+                output += f"\n\n⚠️  OUTPUT TRUNCATED: Showing {max_results} of {total_count} results (max_results={max_results})"
+                output += "\n   To see more: increase max_results or use a more specific pattern"
+            
+            return output
         except Exception as e:
             return f"Error searching files in '{path}': {str(e)}"
 
@@ -753,7 +986,524 @@ class ListAllowedDirectoriesTool(FileSystemTool):
     
     def _run(self) -> str:
         """List allowed directories."""
-        return f"Allowed directory:\n{self.base_directory}\n\nAll subdirectories within this path are accessible."
+        dirs = self._get_all_allowed_directories()
+        if len(dirs) == 1:
+            return f"Allowed directory:\n{dirs[0]}\n\nAll subdirectories within this path are accessible."
+        else:
+            dir_list = "\n".join(f"  - {d}" for d in dirs)
+            return f"Allowed directories:\n{dir_list}\n\nAll subdirectories within these paths are accessible."
+
+
+# ========== Filesystem API Wrapper for Inventory Ingestion ==========
+
+class FilesystemApiWrapper:
+    """
+    API Wrapper for filesystem operations compatible with inventory ingestion pipeline.
+    
+    Supports both text and non-text files:
+    - Text files: .py, .md, .txt, .json, .yaml, etc.
+    - Documents: .pdf, .docx, .pptx, .xlsx, .xls (converted to markdown)
+    - Images: .png, .jpg, .gif, .webp (base64 encoded or described via LLM)
+    
+    Usage:
+        # Create wrapper for a directory
+        wrapper = FilesystemApiWrapper(base_directory="/path/to/docs")
+        
+        # Load documents (uses inherited loader())
+        for doc in wrapper.loader(whitelist=["*.md", "*.pdf"]):
+            print(doc.page_content[:100])
+        
+        # For image description, provide an LLM
+        wrapper = FilesystemApiWrapper(base_directory="/path/to/docs", llm=my_llm)
+        for doc in wrapper.loader(whitelist=["*.png"]):
+            print(doc.page_content)  # LLM-generated description
+        
+        # Use with inventory ingestion
+        pipeline = IngestionPipeline(llm=llm, graph_path="./graph.json")
+        pipeline.register_toolkit("local_docs", wrapper)
+        result = pipeline.run(source="local_docs", whitelist=["*.md", "*.pdf"])
+    """
+    
+    # Filesystem-specific settings
+    base_directory: str = ""
+    recursive: bool = True
+    follow_symlinks: bool = False
+    llm: Any = None  # Optional LLM for image processing
+    
+    # File type categories
+    BINARY_EXTENSIONS = {'.pdf', '.docx', '.doc', '.pptx', '.ppt', '.xlsx', '.xls'}
+    IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg'}
+    
+    def __init__(
+        self,
+        base_directory: str,
+        recursive: bool = True,
+        follow_symlinks: bool = False,
+        llm: Any = None,
+        **kwargs
+    ):
+        """
+        Initialize filesystem wrapper.
+        
+        Args:
+            base_directory: Root directory for file operations
+            recursive: If True, search subdirectories recursively
+            follow_symlinks: If True, follow symbolic links
+            llm: Optional LLM for image description (if not provided, images are base64 encoded)
+            **kwargs: Additional arguments (ignored, for compatibility)
+        """
+        self.base_directory = str(Path(base_directory).resolve())
+        self.recursive = recursive
+        self.follow_symlinks = follow_symlinks
+        self.llm = llm
+        
+        # For compatibility with BaseCodeToolApiWrapper.loader()
+        self.active_branch = None
+        
+        # Validate directory
+        if not Path(self.base_directory).exists():
+            raise ValueError(f"Directory does not exist: {self.base_directory}")
+        if not Path(self.base_directory).is_dir():
+            raise ValueError(f"Path is not a directory: {self.base_directory}")
+        
+        # Optional RunnableConfig for CLI/standalone usage
+        self._runnable_config = None
+    
+    def set_runnable_config(self, config: Optional[Dict[str, Any]]) -> None:
+        """
+        Set the RunnableConfig for dispatching custom events.
+        
+        This is required when running outside of a LangChain agent context
+        (e.g., from CLI). Without a config containing a run_id, 
+        dispatch_custom_event will fail with "Unable to dispatch an adhoc event 
+        without a parent run id".
+        
+        Args:
+            config: A RunnableConfig dict with at least {'run_id': uuid}
+        """
+        self._runnable_config = config
+    
+    def _log_tool_event(self, message: str, tool_name: str = None, config: Optional[Dict[str, Any]] = None):
+        """Log progress events (mirrors BaseToolApiWrapper).
+        
+        Args:
+            message: The message to log
+            tool_name: Name of the tool (defaults to 'filesystem')
+            config: Optional RunnableConfig. If not provided, uses self._runnable_config.
+                   Required when running outside a LangChain agent context.
+        """
+        logger.info(f"[{tool_name or 'filesystem'}] {message}")
+        try:
+            from langchain_core.callbacks import dispatch_custom_event
+            
+            # Use provided config, fall back to instance config
+            effective_config = config or getattr(self, '_runnable_config', None)
+            
+            dispatch_custom_event(
+                name="thinking_step",
+                data={
+                    "message": message,
+                    "tool_name": tool_name or "filesystem",
+                    "toolkit": "FilesystemApiWrapper",
+                },
+                config=effective_config,
+            )
+        except Exception:
+            pass
+    
+    def _get_files(self, path: str = "", branch: str = None) -> List[str]:
+        """
+        Get list of files in the directory.
+        
+        Implements BaseCodeToolApiWrapper._get_files() for filesystem.
+        
+        Args:
+            path: Subdirectory path (relative to base_directory)
+            branch: Ignored for filesystem (compatibility with git-based toolkits)
+            
+        Returns:
+            List of file paths relative to base_directory
+        """
+        base = Path(self.base_directory)
+        search_path = base / path if path else base
+        
+        if not search_path.exists():
+            return []
+        
+        files = []
+        
+        if self.recursive:
+            for root, dirs, filenames in os.walk(search_path, followlinks=self.follow_symlinks):
+                # Skip hidden directories
+                dirs[:] = [d for d in dirs if not d.startswith('.')]
+                
+                for filename in filenames:
+                    if filename.startswith('.'):
+                        continue
+                    
+                    full_path = Path(root) / filename
+                    try:
+                        rel_path = str(full_path.relative_to(base))
+                        files.append(rel_path)
+                    except ValueError:
+                        continue
+        else:
+            for entry in search_path.iterdir():
+                if entry.is_file() and not entry.name.startswith('.'):
+                    try:
+                        rel_path = str(entry.relative_to(base))
+                        files.append(rel_path)
+                    except ValueError:
+                        continue
+        
+        return sorted(files)
+    
+    def _is_binary_file(self, file_path: str) -> bool:
+        """Check if file is a binary document (PDF, DOCX, etc.)."""
+        ext = Path(file_path).suffix.lower()
+        return ext in self.BINARY_EXTENSIONS
+    
+    def _is_image_file(self, file_path: str) -> bool:
+        """Check if file is an image."""
+        ext = Path(file_path).suffix.lower()
+        return ext in self.IMAGE_EXTENSIONS
+    
+    def _read_binary_file(self, file_path: str) -> Optional[str]:
+        """
+        Read binary file (PDF, DOCX, PPTX, Excel) and convert to text/markdown.
+        
+        Uses the SDK's content_parser for document conversion.
+        
+        Args:
+            file_path: Path relative to base_directory
+            
+        Returns:
+            Converted text content, or None if conversion fails
+        """
+        full_path = Path(self.base_directory) / file_path
+        
+        try:
+            from alita_sdk.tools.utils.content_parser import parse_file_content
+            
+            result = parse_file_content(
+                file_path=str(full_path),
+                is_capture_image=bool(self.llm),  # Capture images if LLM available
+                llm=self.llm
+            )
+            
+            if isinstance(result, Exception):
+                logger.warning(f"Failed to parse {file_path}: {result}")
+                return None
+            
+            return result
+            
+        except ImportError:
+            logger.warning("content_parser not available, skipping binary file")
+            return None
+        except Exception as e:
+            logger.warning(f"Error parsing {file_path}: {e}")
+            return None
+    
+    def _read_image_file(self, file_path: str) -> Optional[str]:
+        """
+        Read image file and convert to text representation.
+        
+        If LLM is available, uses it to describe the image.
+        Otherwise, returns base64-encoded data URI.
+        
+        Args:
+            file_path: Path relative to base_directory
+            
+        Returns:
+            Image description or base64 data URI
+        """
+        full_path = Path(self.base_directory) / file_path
+        
+        if not full_path.exists():
+            return None
+        
+        ext = full_path.suffix.lower()
+        
+        try:
+            # Read image bytes
+            image_bytes = full_path.read_bytes()
+            
+            if self.llm:
+                # Use content_parser with LLM for image description
+                try:
+                    from alita_sdk.tools.utils.content_parser import parse_file_content
+                    
+                    result = parse_file_content(
+                        file_path=str(full_path),
+                        is_capture_image=True,
+                        llm=self.llm
+                    )
+                    
+                    if isinstance(result, Exception):
+                        logger.warning(f"Failed to describe image {file_path}: {result}")
+                    else:
+                        return f"[Image: {Path(file_path).name}]\n\n{result}"
+                        
+                except ImportError:
+                    pass
+            
+            # Fallback: return base64 data URI
+            mime_types = {
+                '.png': 'image/png',
+                '.jpg': 'image/jpeg',
+                '.jpeg': 'image/jpeg',
+                '.gif': 'image/gif',
+                '.webp': 'image/webp',
+                '.bmp': 'image/bmp',
+                '.svg': 'image/svg+xml',
+            }
+            mime_type = mime_types.get(ext, 'application/octet-stream')
+            b64_data = base64.b64encode(image_bytes).decode('utf-8')
+            
+            return f"[Image: {Path(file_path).name}]\ndata:{mime_type};base64,{b64_data}"
+            
+        except Exception as e:
+            logger.warning(f"Error reading image {file_path}: {e}")
+            return None
+    
+    def _read_file(
+        self,
+        file_path: str,
+        branch: str = None,
+        offset: Optional[int] = None,
+        limit: Optional[int] = None,
+        head: Optional[int] = None,
+        tail: Optional[int] = None,
+    ) -> Optional[str]:
+        """
+        Read file content, handling text, binary documents, and images.
+        
+        Supports:
+        - Text files: Read directly with encoding detection
+        - Binary documents (PDF, DOCX, PPTX, Excel): Convert to markdown
+        - Images: Return LLM description or base64 data URI
+        
+        Args:
+            file_path: Path relative to base_directory
+            branch: Ignored for filesystem (compatibility with git-based toolkits)
+            offset: Start line number (1-indexed). If None, start from beginning.
+            limit: Maximum number of lines to read. If None, read to end.
+            head: Read only first N lines (alternative to offset/limit)
+            tail: Read only last N lines (alternative to offset/limit)
+            
+        Returns:
+            File content as string, or None if unreadable
+        """
+        full_path = Path(self.base_directory) / file_path
+        
+        # Security check - prevent path traversal
+        try:
+            full_path.resolve().relative_to(Path(self.base_directory).resolve())
+        except ValueError:
+            logger.warning(f"Access denied: {file_path} is outside base directory")
+            return None
+        
+        if not full_path.exists() or not full_path.is_file():
+            return None
+        
+        # Route to appropriate reader based on file type
+        # Note: offset/limit only apply to text files
+        if self._is_binary_file(file_path):
+            return self._read_binary_file(file_path)
+        
+        if self._is_image_file(file_path):
+            return self._read_image_file(file_path)
+        
+        # Default: read as text with encoding detection
+        encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252']
+        
+        for encoding in encodings:
+            try:
+                content = full_path.read_text(encoding=encoding)
+                
+                # Apply line filtering if specified
+                if offset is not None or limit is not None or head is not None or tail is not None:
+                    lines = content.splitlines(keepends=True)
+                    
+                    if head is not None:
+                        # Read first N lines
+                        lines = lines[:head]
+                    elif tail is not None:
+                        # Read last N lines
+                        lines = lines[-tail:] if tail > 0 else []
+                    else:
+                        # Use offset/limit
+                        start_idx = (offset - 1) if offset and offset > 0 else 0
+                        if limit is not None:
+                            end_idx = start_idx + limit
+                            lines = lines[start_idx:end_idx]
+                        else:
+                            lines = lines[start_idx:]
+                    
+                    content = ''.join(lines)
+                
+                return content
+                
+            except UnicodeDecodeError:
+                continue
+            except Exception as e:
+                logger.warning(f"Failed to read {file_path}: {e}")
+                return None
+        
+        logger.warning(f"Could not decode {file_path} with any known encoding")
+        return None
+    
+    def read_file(
+        self,
+        file_path: str,
+        offset: Optional[int] = None,
+        limit: Optional[int] = None,
+        head: Optional[int] = None,
+        tail: Optional[int] = None,
+    ) -> Optional[str]:
+        """
+        Public method to read file content with optional line range.
+        
+        Args:
+            file_path: Path relative to base_directory
+            offset: Start line number (1-indexed)
+            limit: Maximum number of lines to read
+            head: Read only first N lines
+            tail: Read only last N lines
+            
+        Returns:
+            File content as string
+        """
+        return self._read_file(file_path, offset=offset, limit=limit, head=head, tail=tail)
+    
+    def loader(
+        self,
+        branch: Optional[str] = None,
+        whitelist: Optional[List[str]] = None,
+        blacklist: Optional[List[str]] = None,
+        chunked: bool = True,
+    ) -> Generator[Document, None, None]:
+        """
+        Load documents from the filesystem.
+        
+        Mirrors BaseCodeToolApiWrapper.loader() interface for compatibility.
+        
+        Args:
+            branch: Ignored (kept for API compatibility with git-based loaders)
+            whitelist: File patterns to include (e.g., ['*.py', 'src/**/*.js'])
+            blacklist: File patterns to exclude (e.g., ['*test*', 'node_modules/**'])
+            chunked: If True, applies universal chunker based on file type
+        
+        Yields:
+            Document objects with page_content and metadata
+        """
+        import glob as glob_module
+        
+        base = Path(self.base_directory)
+        
+        def is_blacklisted(file_path: str) -> bool:
+            if not blacklist:
+                return False
+            return (
+                any(fnmatch.fnmatch(file_path, p) for p in blacklist) or
+                any(fnmatch.fnmatch(Path(file_path).name, p) for p in blacklist)
+            )
+        
+        # Optimization: Use glob directly when whitelist has path patterns
+        # This avoids scanning 100K+ files in node_modules etc.
+        def get_files_via_glob() -> Generator[str, None, None]:
+            """Use glob patterns directly - much faster than scanning all files."""
+            seen = set()
+            for pattern in whitelist:
+                # Handle glob patterns
+                full_pattern = str(base / pattern)
+                for match in glob_module.glob(full_pattern, recursive=True):
+                    match_path = Path(match)
+                    if match_path.is_file():
+                        try:
+                            rel_path = str(match_path.relative_to(base))
+                            if rel_path not in seen and not is_blacklisted(rel_path):
+                                seen.add(rel_path)
+                                yield rel_path
+                        except ValueError:
+                            continue
+        
+        def get_files_via_scan() -> Generator[str, None, None]:
+            """Fall back to scanning all files when no whitelist or simple extension patterns."""
+            _files = self._get_files()
+            self._log_tool_event(f"Found {len(_files)} files in {self.base_directory}", "loader")
+            
+            def is_whitelisted(file_path: str) -> bool:
+                if not whitelist:
+                    return True
+                return (
+                    any(fnmatch.fnmatch(file_path, p) for p in whitelist) or
+                    any(fnmatch.fnmatch(Path(file_path).name, p) for p in whitelist) or
+                    any(file_path.endswith(f'.{p.lstrip("*.")}') for p in whitelist if p.startswith('*.'))
+                )
+            
+            for file_path in _files:
+                if is_whitelisted(file_path) and not is_blacklisted(file_path):
+                    yield file_path
+        
+        # Decide strategy: use glob if whitelist has path patterns (contains / or **)
+        use_glob = whitelist and any('/' in p or '**' in p for p in whitelist)
+        
+        if use_glob:
+            self._log_tool_event(f"Using glob patterns: {whitelist}", "loader")
+            file_iterator = get_files_via_glob()
+        else:
+            file_iterator = get_files_via_scan()
+        
+        def raw_document_generator() -> Generator[Document, None, None]:
+            self._log_tool_event("Reading files...", "loader")
+            processed = 0
+            
+            for file_path in file_iterator:
+                content = self._read_file(file_path)
+                if not content:
+                    continue
+                
+                content_hash = hashlib.sha256(content.encode('utf-8')).hexdigest()
+                processed += 1
+                
+                yield Document(
+                    page_content=content,
+                    metadata={
+                        'file_path': file_path,
+                        'file_name': Path(file_path).name,
+                        'source': file_path,
+                        'commit_hash': content_hash,
+                    }
+                )
+                
+                # Log progress every 100 files
+                if processed % 100 == 0:
+                    logger.debug(f"[loader] Read {processed} files...")
+            
+            self._log_tool_event(f"Loaded {processed} files", "loader")
+        
+        if not chunked:
+            return raw_document_generator()
+        
+        try:
+            from alita_sdk.tools.chunkers.universal_chunker import universal_chunker
+            return universal_chunker(raw_document_generator())
+        except ImportError:
+            logger.warning("Universal chunker not available, returning raw documents")
+            return raw_document_generator()
+    
+    def chunker(self, documents: Generator[Document, None, None]) -> Generator[Document, None, None]:
+        """Apply universal chunker to documents."""
+        try:
+            from alita_sdk.tools.chunkers.universal_chunker import universal_chunker
+            return universal_chunker(documents)
+        except ImportError:
+            return documents
+    
+    def get_files_content(self, file_path: str) -> Optional[str]:
+        """Get file content (compatibility alias for retrieval toolkit)."""
+        return self._read_file(file_path)
 
 
 # Predefined tool presets for common use cases
@@ -761,6 +1511,7 @@ FILESYSTEM_TOOL_PRESETS = {
     'read_only': {
         'exclude_tools': [
             'filesystem_write_file',
+            'filesystem_append_file',
             'filesystem_edit_file',
             'filesystem_apply_patch',
             'filesystem_delete_file',
@@ -775,6 +1526,7 @@ FILESYSTEM_TOOL_PRESETS = {
         'include_tools': [
             'filesystem_read_file',
             'filesystem_write_file',
+            'filesystem_append_file',
             'filesystem_list_directory',
             'filesystem_create_directory',
         ]
@@ -792,20 +1544,21 @@ def get_filesystem_tools(
     base_directory: str,
     include_tools: Optional[List[str]] = None,
     exclude_tools: Optional[List[str]] = None,
-    preset: Optional[str] = None
+    preset: Optional[str] = None,
+    allowed_directories: Optional[List[str]] = None
 ) -> List[BaseTool]:
     """
-    Get filesystem tools for the specified base directory.
+    Get filesystem tools for the specified directories.
     
     Args:
-        base_directory: Absolute or relative path to the directory to restrict access to
+        base_directory: Absolute or relative path to the primary directory to restrict access to
         include_tools: Optional list of tool names to include. If provided, only these tools are returned.
                       If None, all tools are included (unless excluded).
         exclude_tools: Optional list of tool names to exclude. Applied after include_tools.
         preset: Optional preset name to use predefined tool sets. Presets:
                 - 'read_only': Excludes all write/modify operations
                 - 'no_delete': All tools except delete
-                - 'basic': Read, write, list, create directory
+                - 'basic': Read, write, append, list, create directory
                 - 'minimal': Only read and list
                 Note: If preset is used with include_tools or exclude_tools, 
                       preset is applied first, then custom filters.
@@ -818,6 +1571,7 @@ def get_filesystem_tools(
         - filesystem_read_file_chunk
         - filesystem_read_multiple_files
         - filesystem_write_file
+        - filesystem_append_file (for incremental file creation)
         - filesystem_edit_file
         - filesystem_apply_patch
         - filesystem_list_directory
@@ -847,6 +1601,10 @@ def get_filesystem_tools(
         # Use preset and add custom exclusions
         get_filesystem_tools('/path/to/dir', preset='read_only', 
                            exclude_tools=['filesystem_search_files'])
+        
+        # Multiple allowed directories
+        get_filesystem_tools('/path/to/primary', 
+                           allowed_directories=['/path/to/other1', '/path/to/other2'])
     """
     # Apply preset if specified
     preset_include = None
@@ -870,25 +1628,27 @@ def get_filesystem_tools(
         final_exclude.extend(exclude_tools)
     final_exclude = list(set(final_exclude)) if final_exclude else None
     
-    # Resolve to absolute path
+    # Resolve to absolute paths
     base_dir = str(Path(base_directory).resolve())
+    extra_dirs = [str(Path(d).resolve()) for d in (allowed_directories or [])]
     
     # Define all available tools with their names
     all_tools = {
-        'filesystem_read_file': ReadFileTool(base_directory=base_dir),
-        'filesystem_read_file_chunk': ReadFileChunkTool(base_directory=base_dir),
-        'filesystem_read_multiple_files': ReadMultipleFilesTool(base_directory=base_dir),
-        'filesystem_write_file': WriteFileTool(base_directory=base_dir),
-        'filesystem_edit_file': EditFileTool(base_directory=base_dir),
-        'filesystem_apply_patch': ApplyPatchTool(base_directory=base_dir),
-        'filesystem_list_directory': ListDirectoryTool(base_directory=base_dir),
-        'filesystem_directory_tree': DirectoryTreeTool(base_directory=base_dir),
-        'filesystem_search_files': SearchFilesTool(base_directory=base_dir),
-        'filesystem_delete_file': DeleteFileTool(base_directory=base_dir),
-        'filesystem_move_file': MoveFileTool(base_directory=base_dir),
-        'filesystem_create_directory': CreateDirectoryTool(base_directory=base_dir),
-        'filesystem_get_file_info': GetFileInfoTool(base_directory=base_dir),
-        'filesystem_list_allowed_directories': ListAllowedDirectoriesTool(base_directory=base_dir),
+        'filesystem_read_file': ReadFileTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_read_file_chunk': ReadFileChunkTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_read_multiple_files': ReadMultipleFilesTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_write_file': WriteFileTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_append_file': AppendFileTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_edit_file': EditFileTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_apply_patch': ApplyPatchTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_list_directory': ListDirectoryTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_directory_tree': DirectoryTreeTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_search_files': SearchFilesTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_delete_file': DeleteFileTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_move_file': MoveFileTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_create_directory': CreateDirectoryTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_get_file_info': GetFileInfoTool(base_directory=base_dir, allowed_directories=extra_dirs),
+        'filesystem_list_allowed_directories': ListAllowedDirectoriesTool(base_directory=base_dir, allowed_directories=extra_dirs),
     }
     
     # Start with all tools or only included ones