PraisonAI 3.0.0__py3-none-any.whl

praisonai/code/utils/__init__.py

@@ -0,0 +1,46 @@
+"""
+Utility functions for PraisonAI Code module.
+"""
+
+from .file_utils import (
+    add_line_numbers,
+    strip_line_numbers,
+    every_line_has_line_numbers,
+    normalize_line_endings,
+    get_file_extension,
+    is_binary_file,
+    create_directories_for_file,
+    file_exists,
+)
+
+from .text_utils import (
+    normalize_string,
+    unescape_html_entities,
+    get_similarity,
+)
+
+from .ignore_utils import (
+    FileAccessController,
+    load_gitignore_patterns,
+    should_ignore_path,
+)
+
+__all__ = [
+    # File utilities
+    "add_line_numbers",
+    "strip_line_numbers",
+    "every_line_has_line_numbers",
+    "normalize_line_endings",
+    "get_file_extension",
+    "is_binary_file",
+    "create_directories_for_file",
+    "file_exists",
+    # Text utilities
+    "normalize_string",
+    "unescape_html_entities",
+    "get_similarity",
+    # Ignore utilities
+    "FileAccessController",
+    "load_gitignore_patterns",
+    "should_ignore_path",
+]

praisonai/code/utils/file_utils.py

@@ -0,0 +1,307 @@
+"""
+File utility functions for PraisonAI Code module.
+
+Provides utilities for file operations including:
+- Line number handling (add/strip)
+- File existence and type checking
+- Directory creation
+- Line ending normalization
+"""
+
+import os
+import re
+from typing import Optional, Tuple
+
+
+def add_line_numbers(content: str, start_line: int = 1) -> str:
+    """
+    Add line numbers to content in the format: "  N | content"
+    
+    Args:
+        content: The text content to add line numbers to
+        start_line: The starting line number (1-indexed)
+        
+    Returns:
+        Content with line numbers prefixed to each line
+        
+    Example:
+        >>> add_line_numbers("hello\\nworld", 1)
+        '  1 | hello\\n  2 | world'
+    """
+    if not content:
+        return ""
+    
+    lines = content.split('\n')
+    max_line_num = start_line + len(lines) - 1
+    width = len(str(max_line_num))
+    
+    numbered_lines = []
+    for i, line in enumerate(lines):
+        line_num = start_line + i
+        numbered_lines.append(f"{line_num:>{width}} | {line}")
+    
+    return '\n'.join(numbered_lines)
+
+
+def strip_line_numbers(content: str, aggressive: bool = False) -> str:
+    """
+    Strip line numbers from content.
+    
+    Handles formats like:
+    - "  1 | content"
+    - "1| content"
+    - "  1\tcontent"
+    
+    Args:
+        content: Content with line numbers
+        aggressive: If True, also strip formats like "1:" or just leading numbers
+        
+    Returns:
+        Content with line numbers removed
+    """
+    if not content:
+        return ""
+    
+    lines = content.split('\n')
+    stripped_lines = []
+    
+    # Pattern for standard format: optional spaces, digits, optional spaces, pipe/tab, content
+    standard_pattern = re.compile(r'^\s*\d+\s*[|\t]\s?(.*)$')
+    # Aggressive pattern: digits followed by colon or just leading digits with space
+    aggressive_pattern = re.compile(r'^\s*\d+[:\s]\s?(.*)$')
+    
+    for line in lines:
+        match = standard_pattern.match(line)
+        if match:
+            stripped_lines.append(match.group(1))
+        elif aggressive:
+            match = aggressive_pattern.match(line)
+            if match:
+                stripped_lines.append(match.group(1))
+            else:
+                stripped_lines.append(line)
+        else:
+            stripped_lines.append(line)
+    
+    return '\n'.join(stripped_lines)
+
+
+def every_line_has_line_numbers(content: str) -> bool:
+    """
+    Check if every non-empty line in content has line numbers.
+    
+    Args:
+        content: The content to check
+        
+    Returns:
+        True if all non-empty lines have line numbers
+    """
+    if not content or not content.strip():
+        return False
+    
+    lines = content.split('\n')
+    pattern = re.compile(r'^\s*\d+\s*[|\t]')
+    
+    for line in lines:
+        if line.strip():  # Only check non-empty lines
+            if not pattern.match(line):
+                return False
+    
+    return True
+
+
+def normalize_line_endings(content: str, target: str = '\n') -> str:
+    """
+    Normalize line endings to a consistent format.
+    
+    Args:
+        content: The content to normalize
+        target: The target line ending ('\\n' or '\\r\\n')
+        
+    Returns:
+        Content with normalized line endings
+    """
+    # First normalize all to \n, then convert to target
+    normalized = content.replace('\r\n', '\n').replace('\r', '\n')
+    if target == '\r\n':
+        normalized = normalized.replace('\n', '\r\n')
+    return normalized
+
+
+def detect_line_ending(content: str) -> str:
+    """
+    Detect the predominant line ending in content.
+    
+    Args:
+        content: The content to analyze
+        
+    Returns:
+        '\\r\\n' if Windows-style, '\\n' otherwise
+    """
+    if '\r\n' in content:
+        return '\r\n'
+    return '\n'
+
+
+def get_file_extension(file_path: str) -> str:
+    """
+    Get the file extension from a path.
+    
+    Args:
+        file_path: Path to the file
+        
+    Returns:
+        File extension without the dot, or empty string
+    """
+    ext = os.path.splitext(file_path)[1]
+    return ext[1:] if ext else ""
+
+
+def is_binary_file(file_path: str, sample_size: int = 8192) -> bool:
+    """
+    Check if a file is binary by reading a sample.
+    
+    Args:
+        file_path: Path to the file
+        sample_size: Number of bytes to sample
+        
+    Returns:
+        True if the file appears to be binary
+    """
+    try:
+        with open(file_path, 'rb') as f:
+            sample = f.read(sample_size)
+        
+        # Check for null bytes (common in binary files)
+        if b'\x00' in sample:
+            return True
+        
+        # Check the ratio of non-text characters
+        text_chars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
+        non_text = sum(1 for byte in sample if byte not in text_chars)
+        
+        # If more than 30% non-text characters, consider it binary
+        return len(sample) > 0 and (non_text / len(sample)) > 0.30
+        
+    except (IOError, OSError):
+        return False
+
+
+def create_directories_for_file(file_path: str) -> bool:
+    """
+    Create parent directories for a file path if they don't exist.
+    
+    Args:
+        file_path: Path to the file
+        
+    Returns:
+        True if directories were created or already exist
+    """
+    try:
+        parent_dir = os.path.dirname(file_path)
+        if parent_dir:
+            os.makedirs(parent_dir, exist_ok=True)
+        return True
+    except OSError:
+        return False
+
+
+def file_exists(file_path: str) -> bool:
+    """
+    Check if a file exists.
+    
+    Args:
+        file_path: Path to the file
+        
+    Returns:
+        True if the file exists and is a file (not directory)
+    """
+    return os.path.isfile(file_path)
+
+
+def get_relative_path(file_path: str, base_path: str) -> str:
+    """
+    Get the relative path from base_path to file_path.
+    
+    Args:
+        file_path: The target file path
+        base_path: The base directory path
+        
+    Returns:
+        Relative path string
+    """
+    try:
+        return os.path.relpath(file_path, base_path)
+    except ValueError:
+        # On Windows, relpath fails for paths on different drives
+        return file_path
+
+
+def is_path_within_directory(file_path: str, directory: str) -> bool:
+    """
+    Check if a file path is within a directory (prevents path traversal).
+    
+    Args:
+        file_path: The file path to check
+        directory: The directory that should contain the file
+        
+    Returns:
+        True if file_path is within directory
+    """
+    # Resolve to absolute paths
+    abs_file = os.path.abspath(file_path)
+    abs_dir = os.path.abspath(directory)
+    
+    # Ensure directory ends with separator for proper prefix matching
+    if not abs_dir.endswith(os.sep):
+        abs_dir += os.sep
+    
+    return abs_file.startswith(abs_dir) or abs_file == abs_dir.rstrip(os.sep)
+
+
+def read_file_lines(file_path: str, start_line: int = 1, end_line: Optional[int] = None) -> Tuple[str, int]:
+    """
+    Read specific lines from a file.
+    
+    Args:
+        file_path: Path to the file
+        start_line: First line to read (1-indexed)
+        end_line: Last line to read (inclusive), None for all remaining
+        
+    Returns:
+        Tuple of (content, total_lines)
+    """
+    with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
+        lines = f.readlines()
+    
+    total_lines = len(lines)
+    
+    # Convert to 0-indexed
+    start_idx = max(0, start_line - 1)
+    end_idx = end_line if end_line else total_lines
+    
+    selected_lines = lines[start_idx:end_idx]
+    content = ''.join(selected_lines)
+    
+    # Remove trailing newline if present
+    if content.endswith('\n'):
+        content = content[:-1]
+    
+    return content, total_lines
+
+
+def count_file_lines(file_path: str) -> int:
+    """
+    Count the number of lines in a file efficiently.
+    
+    Args:
+        file_path: Path to the file
+        
+    Returns:
+        Number of lines in the file
+    """
+    count = 0
+    with open(file_path, 'rb') as f:
+        for _ in f:
+            count += 1
+    return count

praisonai/code/utils/ignore_utils.py

@@ -0,0 +1,308 @@
+"""
+File access control utilities for PraisonAI Code module.
+
+Provides utilities for:
+- Loading and parsing .gitignore patterns
+- Checking if paths should be ignored
+- Controlling file access within workspaces
+"""
+
+import os
+import fnmatch
+from typing import List, Optional, Set
+
+
+def load_gitignore_patterns(directory: str) -> List[str]:
+    """
+    Load .gitignore patterns from a directory.
+    
+    Args:
+        directory: The directory to search for .gitignore
+        
+    Returns:
+        List of gitignore patterns
+    """
+    patterns = []
+    gitignore_path = os.path.join(directory, '.gitignore')
+    
+    if os.path.isfile(gitignore_path):
+        try:
+            with open(gitignore_path, 'r', encoding='utf-8') as f:
+                for line in f:
+                    line = line.strip()
+                    # Skip empty lines and comments
+                    if line and not line.startswith('#'):
+                        patterns.append(line)
+        except (IOError, OSError):
+            pass
+    
+    return patterns
+
+
+def _pattern_to_regex_parts(pattern: str) -> tuple:
+    """
+    Convert a gitignore pattern to matching components.
+    
+    Args:
+        pattern: A gitignore pattern
+        
+    Returns:
+        Tuple of (is_negation, is_directory_only, cleaned_pattern)
+    """
+    is_negation = pattern.startswith('!')
+    if is_negation:
+        pattern = pattern[1:]
+    
+    is_directory_only = pattern.endswith('/')
+    if is_directory_only:
+        pattern = pattern[:-1]
+    
+    return is_negation, is_directory_only, pattern
+
+
+def should_ignore_path(
+    path: str,
+    patterns: List[str],
+    base_dir: str,
+    is_directory: bool = False
+) -> bool:
+    """
+    Check if a path should be ignored based on gitignore patterns.
+    
+    Args:
+        path: The path to check (relative or absolute)
+        patterns: List of gitignore patterns
+        base_dir: The base directory for relative path calculation
+        is_directory: Whether the path is a directory
+        
+    Returns:
+        True if the path should be ignored
+    """
+    # Get relative path
+    if os.path.isabs(path):
+        try:
+            rel_path = os.path.relpath(path, base_dir)
+        except ValueError:
+            rel_path = path
+    else:
+        rel_path = path
+    
+    # Normalize path separators
+    rel_path = rel_path.replace(os.sep, '/')
+    
+    # Check each pattern
+    ignored = False
+    
+    for pattern in patterns:
+        is_negation, is_directory_only, clean_pattern = _pattern_to_regex_parts(pattern)
+        
+        # Skip directory-only patterns for files
+        if is_directory_only and not is_directory:
+            continue
+        
+        # Check if pattern matches
+        matches = False
+        
+        # Handle patterns with /
+        if '/' in clean_pattern:
+            # Pattern with / matches from root
+            if clean_pattern.startswith('/'):
+                clean_pattern = clean_pattern[1:]
+            matches = fnmatch.fnmatch(rel_path, clean_pattern)
+        else:
+            # Pattern without / matches any path component
+            path_parts = rel_path.split('/')
+            for part in path_parts:
+                if fnmatch.fnmatch(part, clean_pattern):
+                    matches = True
+                    break
+            # Also check full path for ** patterns
+            if not matches and '**' in clean_pattern:
+                matches = fnmatch.fnmatch(rel_path, clean_pattern)
+        
+        if matches:
+            ignored = not is_negation
+    
+    return ignored
+
+
+class FileAccessController:
+    """
+    Controls file access within a workspace.
+    
+    Respects .gitignore patterns and can be configured with
+    additional ignore/allow patterns.
+    
+    Attributes:
+        workspace_root: The root directory of the workspace
+        ignore_patterns: Patterns for files to ignore
+        protected_patterns: Patterns for files that are read-only
+    """
+    
+    def __init__(
+        self,
+        workspace_root: str,
+        load_gitignore: bool = True,
+        additional_ignore_patterns: Optional[List[str]] = None,
+        protected_patterns: Optional[List[str]] = None
+    ):
+        """
+        Initialize the FileAccessController.
+        
+        Args:
+            workspace_root: The root directory of the workspace
+            load_gitignore: Whether to load .gitignore patterns
+            additional_ignore_patterns: Extra patterns to ignore
+            protected_patterns: Patterns for read-only files
+        """
+        self.workspace_root = os.path.abspath(workspace_root)
+        self.ignore_patterns: List[str] = []
+        self.protected_patterns: List[str] = protected_patterns or []
+        self._ignored_cache: Set[str] = set()
+        
+        # Load .gitignore patterns
+        if load_gitignore:
+            self.ignore_patterns.extend(load_gitignore_patterns(self.workspace_root))
+        
+        # Add additional patterns
+        if additional_ignore_patterns:
+            self.ignore_patterns.extend(additional_ignore_patterns)
+        
+        # Always ignore common patterns
+        default_ignores = [
+            '.git',
+            '.git/**',
+            '__pycache__',
+            '__pycache__/**',
+            '*.pyc',
+            '.DS_Store',
+            'node_modules',
+            'node_modules/**',
+        ]
+        for pattern in default_ignores:
+            if pattern not in self.ignore_patterns:
+                self.ignore_patterns.append(pattern)
+    
+    def is_path_allowed(self, path: str) -> bool:
+        """
+        Check if a path is allowed for access.
+        
+        Args:
+            path: The path to check
+            
+        Returns:
+            True if the path is allowed
+        """
+        # Resolve to absolute path
+        if not os.path.isabs(path):
+            abs_path = os.path.abspath(os.path.join(self.workspace_root, path))
+        else:
+            abs_path = os.path.abspath(path)
+        
+        # Check if within workspace
+        if not self._is_within_workspace(abs_path):
+            return False
+        
+        # Check if ignored
+        is_dir = os.path.isdir(abs_path)
+        if should_ignore_path(abs_path, self.ignore_patterns, self.workspace_root, is_dir):
+            return False
+        
+        return True
+    
+    def is_write_protected(self, path: str) -> bool:
+        """
+        Check if a path is write-protected.
+        
+        Args:
+            path: The path to check
+            
+        Returns:
+            True if the path is write-protected
+        """
+        if not self.protected_patterns:
+            return False
+        
+        # Get relative path
+        if os.path.isabs(path):
+            try:
+                rel_path = os.path.relpath(path, self.workspace_root)
+            except ValueError:
+                rel_path = path
+        else:
+            rel_path = path
+        
+        rel_path = rel_path.replace(os.sep, '/')
+        
+        for pattern in self.protected_patterns:
+            if fnmatch.fnmatch(rel_path, pattern):
+                return True
+        
+        return False
+    
+    def validate_access(self, path: str, write: bool = False) -> bool:
+        """
+        Validate if access to a path is allowed.
+        
+        Args:
+            path: The path to validate
+            write: Whether write access is needed
+            
+        Returns:
+            True if access is allowed
+        """
+        if not self.is_path_allowed(path):
+            return False
+        
+        if write and self.is_write_protected(path):
+            return False
+        
+        return True
+    
+    def _is_within_workspace(self, abs_path: str) -> bool:
+        """
+        Check if an absolute path is within the workspace.
+        
+        Args:
+            abs_path: The absolute path to check
+            
+        Returns:
+            True if within workspace
+        """
+        # Normalize paths
+        workspace = os.path.normpath(self.workspace_root)
+        target = os.path.normpath(abs_path)
+        
+        # Check if target starts with workspace path
+        return target.startswith(workspace + os.sep) or target == workspace
+    
+    def get_relative_path(self, path: str) -> str:
+        """
+        Get the relative path from workspace root.
+        
+        Args:
+            path: The path to convert
+            
+        Returns:
+            Relative path string
+        """
+        if os.path.isabs(path):
+            try:
+                return os.path.relpath(path, self.workspace_root)
+            except ValueError:
+                return path
+        return path
+    
+    def get_absolute_path(self, path: str) -> str:
+        """
+        Get the absolute path for a relative path.
+        
+        Args:
+            path: The relative path
+            
+        Returns:
+            Absolute path string
+        """
+        if os.path.isabs(path):
+            return path
+        return os.path.abspath(os.path.join(self.workspace_root, path))