alita-sdk 0.3.351__py3-none-any.whl → 0.3.499__py3-none-any.whl

alita_sdk/tools/utils/content_parser.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 import os
 import re
 import tempfile
@@ -91,21 +92,24 @@ def parse_file_content(file_name=None, file_content=None, is_capture_image: bool
         return ToolException(
             "Not supported type of files entered. Supported types are TXT, DOCX, PDF, PPTX, XLSX and XLS only.")
 
-    if hasattr(loader, 'get_content'):
-        return loader.get_content()
-    else:
-        extension = Path(file_path if file_path else file_name).suffix
-        loader_kwargs = get_loader_kwargs(loaders_map.get(extension), file_name, file_content, is_capture_image, page_number, sheet_name, llm, file_path, excel_by_sheets)
-        if file_content:
-            return load_content_from_bytes(file_content=file_content,
-                                           extension=extension,
-                                           loader_extra_config=loader_kwargs,
-                                           llm=llm)
+    try:
+        if hasattr(loader, 'get_content'):
+            return loader.get_content()
         else:
-            return load_content(file_path=file_path,
-                                extension=extension,
-                                loader_extra_config=loader_kwargs,
-                                llm=llm)
+            extension = Path(file_path if file_path else file_name).suffix
+            loader_kwargs = get_loader_kwargs(loaders_map.get(extension), file_name, file_content, is_capture_image, page_number, sheet_name, llm, file_path, excel_by_sheets)
+            if file_content:
+                return load_content_from_bytes(file_content=file_content,
+                                               extension=extension,
+                                               loader_extra_config=loader_kwargs,
+                                               llm=llm)
+            else:
+                return load_content(file_path=file_path,
+                                    extension=extension,
+                                    loader_extra_config=loader_kwargs,
+                                    llm=llm)
+    except Exception as e:
+        return ToolException(f"Error reading file ({file_name or file_path}) content. Make sure these types are supported: {str(e)}")
 
 def load_file_docs(file_name=None, file_content=None, is_capture_image: bool = False, page_number: int = None,
                        sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False) -> List[Document] | ToolException:
@@ -126,7 +130,7 @@ def load_file_docs(file_name=None, file_content=None, is_capture_image: bool = F
 
 def get_loader_kwargs(loader_object, file_name=None, file_content=None, is_capture_image: bool = False, page_number: int = None,
                     sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False, prompt=None):
-    loader_kwargs = loader_object['kwargs']
+    loader_kwargs = deepcopy(loader_object['kwargs'])
     loader_kwargs.update({
         "file_path": file_path,
         "file_content": file_content,
@@ -152,7 +156,7 @@ def prepare_loader(file_name=None, file_content=None, is_capture_image: bool = F
 
         loader_object = loaders_map.get(extension)
         if not loader_object:
-            return None
+            loader_object = loaders_map.get('.txt')  # Default to text loader if no specific loader found
         loader_kwargs = get_loader_kwargs(loader_object, file_name, file_content, is_capture_image, page_number, sheet_name, llm, file_path, excel_by_sheets, prompt)
         loader = loader_object['class'](**loader_kwargs)
         return loader
@@ -221,58 +225,80 @@ def process_document_by_type(content, extension_source: str, document: Document
             metadata={**document.metadata, 'chunk_id': 1}
         )
         return
+    #
+    chunks_counter = 0
     for chunk in chunks:
+        chunks_counter += 1
+        metadata = {**document.metadata, **chunk.metadata}
+        #
+        # ensure each chunk has a unique chunk_id
+        metadata['chunk_id'] = chunks_counter
+        #
         yield Document(
             page_content=sanitize_for_postgres(chunk.page_content),
-            metadata={**document.metadata, **chunk.metadata}
+            metadata=metadata
         )
 
 
-def process_content_by_type(content, filename: str, llm=None, chunking_config=None) -> \
+def process_content_by_type(content, filename: str, llm=None, chunking_config=None, fallback_extensions=None) -> \
         Generator[Document, None, None]:
     """Process the content of a file based on its type using a configured loader."""
     temp_file_path = None
-    try:
-        match = re.search(r'\.([^.]+)$', filename)
-        extension = f".{match.group(1).lower()}" if match else ".txt"
-
-        with tempfile.NamedTemporaryFile(mode='w+b', suffix=extension, delete=False) as temp_file:
-            temp_file_path = temp_file.name
-            if content is None:
-                logger.warning(
-                    f"'{IndexerKeywords.CONTENT_IN_BYTES.value}' ie expected but not found in document metadata.")
-                return []
-
-            temp_file.write(content)
-            temp_file.flush()
-
-            loader_config = loaders_map.get(extension)
-            if not loader_config:
-                logger.warning(f"No loader found for file extension: {extension}. File: {temp_file_path}")
-                return []
-
-            loader_cls = loader_config['class']
-            loader_kwargs = loader_config['kwargs']
-            # Determine which loader configuration keys are allowed to be overridden by user input.
-            # If 'allowed_to_override' is specified in the loader configuration, use it; otherwise, allow all keys in loader_kwargs.
-            allowed_to_override = loader_config.get('allowed_to_override', list(loader_kwargs.keys()))
-            # If a chunking_config is provided and contains custom configuration for the current file extension,
-            # update loader_kwargs with user-supplied values, but only for keys explicitly permitted in allowed_to_override.
-            # This ensures that only safe and intended parameters can be customized, preventing accidental or unauthorized changes
-            # to critical loader settings.
-            if chunking_config and (users_config_for_extension := chunking_config.get(extension, {})):
-                for key in set(users_config_for_extension.keys()) & set(allowed_to_override):
-                    loader_kwargs[key] = users_config_for_extension[key]
-            if LoaderProperties.LLM.value in loader_kwargs:
-                loader_kwargs[LoaderProperties.LLM.value] = llm
-            if LoaderProperties.PROMPT_DEFAULT.value in loader_kwargs:
-                loader_kwargs.pop(LoaderProperties.PROMPT_DEFAULT.value)
-                loader_kwargs[LoaderProperties.PROMPT.value] = image_processing_prompt
-            loader = loader_cls(file_path=temp_file_path, **loader_kwargs)
-            return loader.load()
-    finally:
-        if temp_file_path and os.path.exists(temp_file_path):
-            os.remove(temp_file_path)
+    extensions = fallback_extensions if fallback_extensions else []
+    match = re.search(r'\.([^.]+)$', filename)
+
+    if match:
+        extensions.insert(0, f".{match.group(1).lower()}")
+    elif not extensions:
+        extensions = [".txt"]
+
+    for extension in extensions:
+        try:
+            with tempfile.NamedTemporaryFile(mode='w+b', suffix=extension, delete=False) as temp_file:
+                temp_file_path = temp_file.name
+                if content is None:
+                    logger.warning(
+                        f"'{IndexerKeywords.CONTENT_IN_BYTES.value}' ie expected but not found in document metadata.")
+                    return []
+    
+                temp_file.write(content)
+                temp_file.flush()
+    
+                loader_config = loaders_map.get(extension)
+                if not loader_config:
+                    logger.warning(f"No loader found for file extension: {extension}. File: {temp_file_path}")
+                    return []
+    
+                loader_cls = loader_config['class']
+                loader_kwargs = loader_config['kwargs']
+                # Determine which loader configuration keys are allowed to be overridden by user input.
+                # If 'allowed_to_override' is specified in the loader configuration, use it; otherwise, allow all keys in loader_kwargs.
+                allowed_to_override = loader_config.get('allowed_to_override', loader_kwargs)
+                # If a chunking_config is provided and contains custom configuration for the current file extension,
+                # update loader_kwargs with user-supplied values, but only for keys explicitly permitted in allowed_to_override and if value differs from default.
+                # This ensures that only safe and intended parameters can be customized, preventing accidental or unauthorized changes
+                # to critical loader settings.
+                if chunking_config and (users_config_for_extension := chunking_config.get(extension, {})):
+                    for key in set(users_config_for_extension.keys()) & set(allowed_to_override.keys()):
+                        if users_config_for_extension[key] != allowed_to_override[key]:
+                            loader_kwargs[key] = users_config_for_extension[key]
+                if LoaderProperties.LLM.value in loader_kwargs and loader_kwargs.pop(LoaderProperties.LLM.value):
+                    loader_kwargs['llm'] = llm
+                if LoaderProperties.PROMPT_DEFAULT.value in loader_kwargs and loader_kwargs.pop(LoaderProperties.PROMPT_DEFAULT.value):
+                    loader_kwargs[LoaderProperties.PROMPT.value] = image_processing_prompt
+                loader = loader_cls(file_path=temp_file_path, **loader_kwargs)
+                yield from loader.load()
+                break
+        except Exception as e:
+            if fallback_extensions:
+                logger.warning(f"Error loading attachment: {str(e)} for file {temp_file_path} (extension: {extension})")
+                logger.warning(f"Continuing with fallback extensions: {fallback_extensions}.")
+                continue
+            else:
+                raise e
+        finally:
+            if temp_file_path and os.path.exists(temp_file_path):
+                os.remove(temp_file_path)
 
 # FIXME copied from langchain_core/utils/strings.py of 0.3.74 version
 # https://github.com/langchain-ai/langchain/pull/32157
@@ -296,7 +322,9 @@ def sanitize_for_postgres(text: str, replacement: str = "") -> str:
     return text.replace("\x00", replacement)
 
 
-def file_extension_by_chunker(chunker_name: str) -> str:
+def file_extension_by_chunker(chunker_name: str) -> str | None:
+    if not chunker_name:
+        return None
     name = chunker_name.lower()
     if name == "markdown":
         return ".md"

alita_sdk/tools/utils/text_operations.py

@@ -0,0 +1,254 @@
+"""
+Shared text operations utilities for file manipulation across toolkits.
+
+Provides common functionality for:
+- Parsing OLD/NEW marker-based edits
+- Text file validation
+- Line-based slicing and partial reads
+- Content searching with context
+"""
+import re
+import logging
+from typing import List, Tuple, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+# Text file extensions that support editing
+TEXT_EDITABLE_EXTENSIONS = {
+    '.md', '.txt', '.csv', '.json', '.xml', '.html', 
+    '.yaml', '.yml', '.ini', '.conf', '.log', '.sh',
+    '.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.go',
+    '.rb', '.php', '.c', '.cpp', '.h', '.hpp', '.cs',
+    '.sql', '.r', '.m', '.swift', '.kt', '.rs', '.scala'
+}
+
+
+def parse_old_new_markers(file_query: str) -> List[Tuple[str, str]]:
+    """
+    Parse OLD/NEW marker-based edit instructions.
+    
+    Extracts pairs of old and new content from a file query using markers:
+    - OLD <<<< ... >>>> OLD
+    - NEW <<<< ... >>>> NEW
+    
+    Args:
+        file_query: String containing marked old and new content sections
+        
+    Returns:
+        List of tuples (old_content, new_content) for each edit pair
+        
+    Example:
+        >>> query = '''
+        ... OLD <<<<
+        ... Hello World
+        ... >>>> OLD
+        ... NEW <<<<
+        ... Hello Mars
+        ... >>>> NEW
+        ... '''
+        >>> parse_old_new_markers(query)
+        [('Hello World', 'Hello Mars')]
+    """
+    # Split the file content by lines
+    code_lines = file_query.split("\n")
+
+    # Initialize lists to hold the contents of OLD and NEW sections
+    old_contents = []
+    new_contents = []
+
+    # Initialize variables to track whether the current line is within an OLD or NEW section
+    in_old_section = False
+    in_new_section = False
+
+    # Temporary storage for the current section's content
+    current_section_content = []
+
+    # Iterate through each line in the file content
+    for line in code_lines:
+        # Check for OLD section start
+        if "OLD <<<" in line:
+            in_old_section = True
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # Check for OLD section end
+        if ">>>> OLD" in line:
+            in_old_section = False
+            old_contents.append("\n".join(current_section_content).strip())  # Add the captured content
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # Check for NEW section start
+        if "NEW <<<" in line:
+            in_new_section = True
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # Check for NEW section end
+        if ">>>> NEW" in line:
+            in_new_section = False
+            new_contents.append("\n".join(current_section_content).strip())  # Add the captured content
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # If currently in an OLD or NEW section, add the line to the current section content
+        if in_old_section or in_new_section:
+            current_section_content.append(line)
+
+    # Pair the OLD and NEW contents
+    paired_contents = list(zip(old_contents, new_contents))
+
+    return paired_contents
+
+
+def is_text_editable(filename: str) -> bool:
+    """
+    Check if a file is editable as text based on its extension.
+    
+    Args:
+        filename: Name or path of the file to check
+        
+    Returns:
+        True if file extension is in the text-editable whitelist
+        
+    Example:
+        >>> is_text_editable("config.json")
+        True
+        >>> is_text_editable("image.png")
+        False
+    """
+    from pathlib import Path
+    ext = Path(filename).suffix.lower()
+    return ext in TEXT_EDITABLE_EXTENSIONS
+
+
+def apply_line_slice(
+    content: str, 
+    offset: Optional[int] = None,
+    limit: Optional[int] = None,
+    head: Optional[int] = None,
+    tail: Optional[int] = None
+) -> str:
+    """
+    Apply line-based slicing to text content.
+    
+    Supports multiple modes:
+    - offset + limit: Read from line `offset` for `limit` lines (1-indexed)
+    - head: Read only first N lines
+    - tail: Read only last N lines
+    - No params: Return full content
+    
+    Args:
+        content: Text content to slice
+        offset: Starting line number (1-indexed, inclusive)
+        limit: Number of lines to read from offset
+        head: Return only first N lines
+        tail: Return only last N lines
+        
+    Returns:
+        Sliced content as string
+        
+    Example:
+        >>> text = "line1\\nline2\\nline3\\nline4\\nline5"
+        >>> apply_line_slice(text, offset=2, limit=2)
+        'line2\\nline3'
+        >>> apply_line_slice(text, head=2)
+        'line1\\nline2'
+        >>> apply_line_slice(text, tail=2)
+        'line4\\nline5'
+    """
+    if not content:
+        return content
+        
+    lines = content.splitlines(keepends=True)
+    
+    # Head mode: first N lines
+    if head is not None:
+        return ''.join(lines[:head])
+    
+    # Tail mode: last N lines
+    if tail is not None:
+        return ''.join(lines[-tail:] if tail > 0 else lines)
+    
+    # Offset + limit mode: slice from offset for limit lines
+    if offset is not None:
+        start_idx = max(0, offset - 1)  # Convert 1-indexed to 0-indexed
+        if limit is not None:
+            end_idx = start_idx + limit
+            return ''.join(lines[start_idx:end_idx])
+        else:
+            return ''.join(lines[start_idx:])
+    
+    # No slicing parameters: return full content
+    return content
+
+
+def search_in_content(
+    content: str,
+    pattern: str,
+    is_regex: bool = True,
+    context_lines: int = 2
+) -> List[Dict[str, any]]:
+    """
+    Search for pattern in content with context lines.
+    
+    Args:
+        content: Text content to search
+        pattern: Search pattern (regex if is_regex=True, else literal string)
+        is_regex: Whether to treat pattern as regex (default True)
+        context_lines: Number of lines before/after match to include (default 2)
+        
+    Returns:
+        List of match dictionaries with keys:
+            - line_number: 1-indexed line number of match
+            - line_content: The matching line
+            - match_text: The actual matched text
+            - context_before: List of lines before match
+            - context_after: List of lines after match
+            
+    Example:
+        >>> text = "line1\\nHello World\\nline3"
+        >>> matches = search_in_content(text, "Hello", is_regex=False)
+        >>> matches[0]['line_number']
+        2
+        >>> matches[0]['match_text']
+        'Hello'
+    """
+    if not content:
+        return []
+    
+    lines = content.splitlines()
+    matches = []
+    
+    # Compile regex pattern or escape for literal search
+    if is_regex:
+        try:
+            regex = re.compile(pattern, re.IGNORECASE)
+        except re.error as e:
+            logger.warning(f"Invalid regex pattern '{pattern}': {e}")
+            return []
+    else:
+        regex = re.compile(re.escape(pattern), re.IGNORECASE)
+    
+    # Search each line
+    for line_idx, line in enumerate(lines):
+        match = regex.search(line)
+        if match:
+            line_number = line_idx + 1  # Convert to 1-indexed
+            
+            # Get context lines
+            context_start = max(0, line_idx - context_lines)
+            context_end = min(len(lines), line_idx + context_lines + 1)
+            
+            context_before = lines[context_start:line_idx]
+            context_after = lines[line_idx + 1:context_end]
+            
+            matches.append({
+                'line_number': line_number,
+                'line_content': line,
+                'match_text': match.group(0),
+                'context_before': context_before,
+                'context_after': context_after,
+            })
+    
+    return matches