alita-sdk 0.3.257__py3-none-any.whl → 0.3.584__py3-none-any.whl

alita_sdk/tools/utils/content_parser.py

@@ -1,13 +1,17 @@
+from copy import deepcopy
 import os
+import re
 import tempfile
 from logging import getLogger
 from pathlib import Path
-from typing import Generator
+from typing import Generator, List
 
 from langchain_core.documents import Document
 from langchain_core.tools import ToolException
 
-from alita_sdk.runtime.langchain.document_loaders.constants import loaders_map
+from alita_sdk.runtime.langchain.document_loaders.constants import loaders_map, LoaderProperties
+from ...runtime.langchain.document_loaders.AlitaTextLoader import AlitaTextLoader
+from ...runtime.utils.utils import IndexerKeywords
 
 logger = getLogger(__name__)
 
@@ -51,11 +55,9 @@ Highlight any visible details that could help in understanding the image.
 Be as precise and thorough as possible in your responses. If something is unclear or illegible, state that explicitly.
 '''
 
-IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'webp', 'svg']
-
 
 def parse_file_content(file_name=None, file_content=None, is_capture_image: bool = False, page_number: int = None,
-                       sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False) -> str | ToolException:
+                       sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False, prompt=None) -> str | ToolException:
     """Parse the content of a file based on its type and return the parsed content.
 
     Args:
@@ -72,18 +74,94 @@ def parse_file_content(file_name=None, file_content=None, is_capture_image: bool
     Raises:
         ToolException: If the file type is not supported or if there is an error reading the file.
         """
+    if not prompt:
+        prompt = image_processing_prompt
+    loader = prepare_loader(
+        file_name=file_name,
+        file_content=file_content,
+        is_capture_image=is_capture_image,
+        page_number=page_number,
+        sheet_name=sheet_name,
+        llm=llm,
+        file_path=file_path,
+        excel_by_sheets=excel_by_sheets,
+        prompt=prompt
+    )
 
-    if (file_path and (file_name or file_content)) or (not file_path and (not file_name or file_content is None)):
-        raise ToolException("Either (file_name and file_content) or file_path must be provided, but not both.")
-
-    extension = Path(file_path if file_path else file_name).suffix
+    if not loader:
+        return ToolException(
+            "Not supported type of files entered. Supported types are TXT, DOCX, PDF, PPTX, XLSX and XLS only.")
 
-    loader_object = loaders_map.get(extension)
-    if not loader_object:
-        logger.warning(f"No loader found for file extension: {extension}. File: {file_path if file_path else file_name}")
+    try:
+        if hasattr(loader, 'get_content'):
+            return loader.get_content()
+        else:
+            extension = Path(file_path if file_path else file_name).suffix
+            loader_kwargs = get_loader_kwargs(loaders_map.get(extension), file_name, file_content, is_capture_image, page_number, sheet_name, llm, file_path, excel_by_sheets)
+            if file_content:
+                return load_content_from_bytes(file_content=file_content,
+                                               extension=extension,
+                                               loader_extra_config=loader_kwargs,
+                                               llm=llm)
+            else:
+                return load_content(file_path=file_path,
+                                    extension=extension,
+                                    loader_extra_config=loader_kwargs,
+                                    llm=llm)
+    except Exception as e:
+        return ToolException(f"Error reading file ({file_name or file_path}) content. Make sure these types are supported: {str(e)}")
+
+def load_file_docs(file_name=None, file_content=None, is_capture_image: bool = False, page_number: int = None,
+                       sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False) -> List[Document] | ToolException:
+    loader = prepare_loader(
+        file_name=file_name,
+        file_content=file_content,
+        is_capture_image=is_capture_image,
+        page_number=page_number,
+        sheet_name=sheet_name,
+        llm=llm,
+        file_path=file_path,
+        excel_by_sheets=excel_by_sheets
+    )
+    if not loader:
         return ToolException(
             "Not supported type of files entered. Supported types are TXT, DOCX, PDF, PPTX, XLSX and XLS only.")
-    loader_kwargs = loader_object['kwargs']
+    return loader.load()
+
+def get_loader_kwargs(loader_object, file_name=None, file_content=None, is_capture_image: bool = False, page_number: int = None,
+                    sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False, prompt=None):
+    """Build loader kwargs safely without deepcopying non-picklable objects like LLMs.
+
+    We avoid copying keys that are going to be overridden by this function anyway
+    (file_path, file_content, file_name, extract_images, llm, page_number,
+    sheet_name, excel_by_sheets, prompt, row_content, json_documents) to
+    prevent errors such as `cannot pickle '_thread.RLock' object` when an LLM
+    or client with internal locks is stored in the original kwargs.
+    """
+    if not loader_object:
+        raise ToolException("Loader configuration is missing.")
+
+    original_kwargs = loader_object.get("kwargs", {}) or {}
+
+    # Keys that will be overwritten below – skip them when copying
+    overridden_keys = {
+        "file_path",
+        "file_content",
+        "file_name",
+        "extract_images",
+        "llm",
+        "page_number",
+        "sheet_name",
+        "excel_by_sheets",
+        "prompt",
+        "row_content",
+        "json_documents",
+    }
+
+    # Build a safe shallow copy without overridden keys to avoid deepcopy
+    # of potentially non-picklable objects (e.g., llm with internal RLock).
+    loader_kwargs = {k: v for k, v in original_kwargs.items() if k not in overridden_keys}
+
     loader_kwargs.update({
         "file_path": file_path,
         "file_content": file_content,
@@ -93,28 +171,26 @@ def parse_file_content(file_name=None, file_content=None, is_capture_image: bool
         "page_number": page_number,
         "sheet_name": sheet_name,
         "excel_by_sheets": excel_by_sheets,
+        "prompt": prompt,
         "row_content": True,
         "json_documents": False
     })
-    loader = loader_object['class'](**loader_kwargs)
+    return loader_kwargs
 
-    if not loader:
-        return ToolException(
-            "Not supported type of files entered. Supported types are TXT, DOCX, PDF, PPTX, XLSX and XLS only.")
+def prepare_loader(file_name=None, file_content=None, is_capture_image: bool = False, page_number: int = None,
+                       sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False,
+                   prompt=None):
+        if (file_path and (file_name or file_content)) or (not file_path and (not file_name or file_content is None)):
+            raise ToolException("Either (file_name and file_content) or file_path must be provided, but not both.")
 
-    if hasattr(loader, 'get_content'):
-        return loader.get_content()
-    else:
-        if file_content:
-            return load_content_from_bytes(file_content=file_content,
-                                           extension=extension,
-                                           loader_extra_config=loader_kwargs,
-                                           llm=llm)
-        else:
-            return load_content(file_path=file_path,
-                                extension=extension,
-                                loader_extra_config=loader_kwargs,
-                                llm=llm)
+        extension = Path(file_path if file_path else file_name).suffix
+
+        loader_object = loaders_map.get(extension)
+        if not loader_object:
+            loader_object = loaders_map.get('.txt')  # Default to text loader if no specific loader found
+        loader_kwargs = get_loader_kwargs(loader_object, file_name, file_content, is_capture_image, page_number, sheet_name, llm, file_path, excel_by_sheets, prompt)
+        loader = loader_object['class'](**loader_kwargs)
+        return loader
 
 # TODO: review usage of this function alongside with functions above
 def load_content(file_path: str, extension: str = None, loader_extra_config: dict = None, llm = None) -> str:
@@ -142,7 +218,7 @@ def load_content(file_path: str, extension: str = None, loader_extra_config: dic
         if "file_path" in loader_kwargs:
             del loader_kwargs["file_path"]
 
-        loader = loader_cls(file_path, **loader_kwargs)
+        loader = loader_cls(file_path=file_path, **loader_kwargs)
         documents = loader.load()
 
         page_contents = [doc.page_content for doc in documents]
@@ -167,38 +243,129 @@ def load_content_from_bytes(file_content: bytes, extension: str = None, loader_e
         if temp_file_path and os.path.exists(temp_file_path):
             os.remove(temp_file_path)
 
-def process_content_by_type(document: Document, content, extension_source: str, llm = None, chunking_config={}) -> Generator[Document, None, None]:
+
+def _load_content_from_bytes_with_prompt(file_content: bytes, extension: str = None, loader_extra_config: dict = None, llm = None, prompt: str = image_processing_prompt) -> str:
+    """Internal helper that behaves like load_content_from_bytes but also propagates prompt.
+
+    This keeps the public load_content_from_bytes API unchanged while allowing newer
+    code paths to pass an explicit prompt through to the loader.
+    """
     temp_file_path = None
     try:
-        extension = "." + extension_source.split('.')[-1].lower()
-
-        with tempfile.NamedTemporaryFile(mode='w+b', suffix=extension, delete=False) as temp_file:
-            temp_file_path = temp_file.name
-            if content is None:
-                logger.warning("'loader_content' ie expected but not found in document metadata.")
-                return
-            
-            temp_file.write(content)
+        with tempfile.NamedTemporaryFile(mode='w+b', delete=False, suffix=extension or '') as temp_file:
+            temp_file.write(file_content)
             temp_file.flush()
+            temp_file_path = temp_file.name
 
-            loader_config = loaders_map.get(extension)
-            if not loader_config:
-                logger.warning(f"No loader found for file extension: {extension}. File: {temp_file_path}")
-                return
-
-            loader_cls = loader_config['class']
-            loader_kwargs = loader_config['kwargs']
+        # Use prepare_loader so that prompt and other kwargs are handled consistently
+        loader = prepare_loader(
+            file_name=None,
+            file_content=None,
+            is_capture_image=loader_extra_config.get('extract_images') if loader_extra_config else False,
+            page_number=loader_extra_config.get('page_number') if loader_extra_config else None,
+            sheet_name=loader_extra_config.get('sheet_name') if loader_extra_config else None,
+            llm=llm or (loader_extra_config.get('llm') if loader_extra_config else None),
+            file_path=temp_file_path,
+            excel_by_sheets=loader_extra_config.get('excel_by_sheets') if loader_extra_config else False,
+            prompt=prompt or (loader_extra_config.get('prompt') if loader_extra_config else image_processing_prompt),
+        )
 
-            loader = loader_cls(file_path=temp_file_path, **loader_kwargs)
-            for chunk in loader.load():
-                yield Document(
-                    page_content=sanitize_for_postgres(chunk.page_content),
-                    metadata={**document.metadata, **chunk.metadata}
-                )
+        documents = loader.load()
+        page_contents = [doc.page_content for doc in documents]
+        return "\n".join(page_contents)
     finally:
         if temp_file_path and os.path.exists(temp_file_path):
             os.remove(temp_file_path)
 
+
+def process_document_by_type(content, extension_source: str, document: Document = None, llm = None, chunking_config=None) \
+        -> Generator[Document, None, None]:
+    """Process the content of a file based on its type using a configured loader cosidering the origin document."""
+    try:
+        chunks = process_content_by_type(content, extension_source, llm, chunking_config)
+    except Exception as e:
+        msg = f"Error during content parsing for file {extension_source}:\n{e}"
+        logger.warning(msg)
+        yield Document(
+            page_content=msg,
+            metadata={**document.metadata, 'chunk_id': 1}
+        )
+        return
+    #
+    chunks_counter = 0
+    for chunk in chunks:
+        chunks_counter += 1
+        metadata = {**document.metadata, **chunk.metadata}
+        #
+        # ensure each chunk has a unique chunk_id
+        metadata['chunk_id'] = chunks_counter
+        #
+        yield Document(
+            page_content=sanitize_for_postgres(chunk.page_content),
+            metadata=metadata
+        )
+
+
+def process_content_by_type(content, filename: str, llm=None, chunking_config=None, fallback_extensions=None) -> \
+        Generator[Document, None, None]:
+    """Process the content of a file based on its type using a configured loader."""
+    temp_file_path = None
+    extensions = fallback_extensions if fallback_extensions else []
+    match = re.search(r'\.([^.]+)$', filename)
+
+    if match:
+        extensions.insert(0, f".{match.group(1).lower()}")
+    elif not extensions:
+        extensions = [".txt"]
+
+    for extension in extensions:
+        try:
+            with tempfile.NamedTemporaryFile(mode='w+b', suffix=extension, delete=False) as temp_file:
+                temp_file_path = temp_file.name
+                if content is None:
+                    logger.warning(
+                        f"'{IndexerKeywords.CONTENT_IN_BYTES.value}' ie expected but not found in document metadata.")
+                    return []
+    
+                temp_file.write(content)
+                temp_file.flush()
+    
+                loader_config = loaders_map.get(extension)
+                if not loader_config:
+                    logger.warning(f"No loader found for file extension: {extension}. File: {temp_file_path}")
+                    return []
+    
+                loader_cls = loader_config['class']
+                loader_kwargs = loader_config['kwargs']
+                # Determine which loader configuration keys are allowed to be overridden by user input.
+                # If 'allowed_to_override' is specified in the loader configuration, use it; otherwise, allow all keys in loader_kwargs.
+                allowed_to_override = loader_config.get('allowed_to_override', loader_kwargs)
+                # If a chunking_config is provided and contains custom configuration for the current file extension,
+                # update loader_kwargs with user-supplied values, but only for keys explicitly permitted in allowed_to_override and if value differs from default.
+                # This ensures that only safe and intended parameters can be customized, preventing accidental or unauthorized changes
+                # to critical loader settings.
+                if chunking_config and (users_config_for_extension := chunking_config.get(extension, {})):
+                    for key in set(users_config_for_extension.keys()) & set(allowed_to_override.keys()):
+                        if users_config_for_extension[key] != allowed_to_override[key]:
+                            loader_kwargs[key] = users_config_for_extension[key]
+                if LoaderProperties.LLM.value in loader_kwargs and loader_kwargs.pop(LoaderProperties.LLM.value):
+                    loader_kwargs['llm'] = llm
+                if LoaderProperties.PROMPT_DEFAULT.value in loader_kwargs and loader_kwargs.pop(LoaderProperties.PROMPT_DEFAULT.value):
+                    loader_kwargs[LoaderProperties.PROMPT.value] = image_processing_prompt
+                loader = loader_cls(file_path=temp_file_path, **loader_kwargs)
+                yield from loader.load()
+                break
+        except Exception as e:
+            if fallback_extensions:
+                logger.warning(f"Error loading attachment: {str(e)} for file {temp_file_path} (extension: {extension})")
+                logger.warning(f"Continuing with fallback extensions: {fallback_extensions}.")
+                continue
+            else:
+                raise e
+        finally:
+            if temp_file_path and os.path.exists(temp_file_path):
+                os.remove(temp_file_path)
+
 # FIXME copied from langchain_core/utils/strings.py of 0.3.74 version
 # https://github.com/langchain-ai/langchain/pull/32157
 # should be used from langchain_core.utils import sanitize_for_postgres once updated to newer version
@@ -218,4 +385,23 @@ def sanitize_for_postgres(text: str, replacement: str = "") -> str:
         >>> sanitize_for_postgres("Hello\\x00world", " ")
         'Hello world'
     """
-    return text.replace("\x00", replacement)
+    return text.replace("\x00", replacement)
+
+
+def file_extension_by_chunker(chunker_name: str) -> str | None:
+    if not chunker_name:
+        return None
+    name = chunker_name.lower()
+    if name == "markdown":
+        return ".md"
+    if name == "json":
+        return ".json"
+    if name == "text" or name == "txt":
+        return ".txt"
+    if name == "html":
+        return ".html"
+    if name == "xml":
+        return ".xml"
+    if name == "csv":
+        return ".csv"
+    return None

alita_sdk/tools/utils/text_operations.py

@@ -0,0 +1,254 @@
+"""
+Shared text operations utilities for file manipulation across toolkits.
+
+Provides common functionality for:
+- Parsing OLD/NEW marker-based edits
+- Text file validation
+- Line-based slicing and partial reads
+- Content searching with context
+"""
+import re
+import logging
+from typing import List, Tuple, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+# Text file extensions that support editing
+TEXT_EDITABLE_EXTENSIONS = {
+    '.md', '.txt', '.csv', '.json', '.xml', '.html', 
+    '.yaml', '.yml', '.ini', '.conf', '.log', '.sh',
+    '.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.go',
+    '.rb', '.php', '.c', '.cpp', '.h', '.hpp', '.cs',
+    '.sql', '.r', '.m', '.swift', '.kt', '.rs', '.scala'
+}
+
+
+def parse_old_new_markers(file_query: str) -> List[Tuple[str, str]]:
+    """
+    Parse OLD/NEW marker-based edit instructions.
+    
+    Extracts pairs of old and new content from a file query using markers:
+    - OLD <<<< ... >>>> OLD
+    - NEW <<<< ... >>>> NEW
+    
+    Args:
+        file_query: String containing marked old and new content sections
+        
+    Returns:
+        List of tuples (old_content, new_content) for each edit pair
+        
+    Example:
+        >>> query = '''
+        ... OLD <<<<
+        ... Hello World
+        ... >>>> OLD
+        ... NEW <<<<
+        ... Hello Mars
+        ... >>>> NEW
+        ... '''
+        >>> parse_old_new_markers(query)
+        [('Hello World', 'Hello Mars')]
+    """
+    # Split the file content by lines
+    code_lines = file_query.split("\n")
+
+    # Initialize lists to hold the contents of OLD and NEW sections
+    old_contents = []
+    new_contents = []
+
+    # Initialize variables to track whether the current line is within an OLD or NEW section
+    in_old_section = False
+    in_new_section = False
+
+    # Temporary storage for the current section's content
+    current_section_content = []
+
+    # Iterate through each line in the file content
+    for line in code_lines:
+        # Check for OLD section start
+        if "OLD <<<" in line:
+            in_old_section = True
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # Check for OLD section end
+        if ">>>> OLD" in line:
+            in_old_section = False
+            old_contents.append("\n".join(current_section_content).strip())  # Add the captured content
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # Check for NEW section start
+        if "NEW <<<" in line:
+            in_new_section = True
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # Check for NEW section end
+        if ">>>> NEW" in line:
+            in_new_section = False
+            new_contents.append("\n".join(current_section_content).strip())  # Add the captured content
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # If currently in an OLD or NEW section, add the line to the current section content
+        if in_old_section or in_new_section:
+            current_section_content.append(line)
+
+    # Pair the OLD and NEW contents
+    paired_contents = list(zip(old_contents, new_contents))
+
+    return paired_contents
+
+
+def is_text_editable(filename: str) -> bool:
+    """
+    Check if a file is editable as text based on its extension.
+    
+    Args:
+        filename: Name or path of the file to check
+        
+    Returns:
+        True if file extension is in the text-editable whitelist
+        
+    Example:
+        >>> is_text_editable("config.json")
+        True
+        >>> is_text_editable("image.png")
+        False
+    """
+    from pathlib import Path
+    ext = Path(filename).suffix.lower()
+    return ext in TEXT_EDITABLE_EXTENSIONS
+
+
+def apply_line_slice(
+    content: str, 
+    offset: Optional[int] = None,
+    limit: Optional[int] = None,
+    head: Optional[int] = None,
+    tail: Optional[int] = None
+) -> str:
+    """
+    Apply line-based slicing to text content.
+    
+    Supports multiple modes:
+    - offset + limit: Read from line `offset` for `limit` lines (1-indexed)
+    - head: Read only first N lines
+    - tail: Read only last N lines
+    - No params: Return full content
+    
+    Args:
+        content: Text content to slice
+        offset: Starting line number (1-indexed, inclusive)
+        limit: Number of lines to read from offset
+        head: Return only first N lines
+        tail: Return only last N lines
+        
+    Returns:
+        Sliced content as string
+        
+    Example:
+        >>> text = "line1\\nline2\\nline3\\nline4\\nline5"
+        >>> apply_line_slice(text, offset=2, limit=2)
+        'line2\\nline3'
+        >>> apply_line_slice(text, head=2)
+        'line1\\nline2'
+        >>> apply_line_slice(text, tail=2)
+        'line4\\nline5'
+    """
+    if not content:
+        return content
+        
+    lines = content.splitlines(keepends=True)
+    
+    # Head mode: first N lines
+    if head is not None:
+        return ''.join(lines[:head])
+    
+    # Tail mode: last N lines
+    if tail is not None:
+        return ''.join(lines[-tail:] if tail > 0 else lines)
+    
+    # Offset + limit mode: slice from offset for limit lines
+    if offset is not None:
+        start_idx = max(0, offset - 1)  # Convert 1-indexed to 0-indexed
+        if limit is not None:
+            end_idx = start_idx + limit
+            return ''.join(lines[start_idx:end_idx])
+        else:
+            return ''.join(lines[start_idx:])
+    
+    # No slicing parameters: return full content
+    return content
+
+
+def search_in_content(
+    content: str,
+    pattern: str,
+    is_regex: bool = True,
+    context_lines: int = 2
+) -> List[Dict[str, any]]:
+    """
+    Search for pattern in content with context lines.
+    
+    Args:
+        content: Text content to search
+        pattern: Search pattern (regex if is_regex=True, else literal string)
+        is_regex: Whether to treat pattern as regex (default True)
+        context_lines: Number of lines before/after match to include (default 2)
+        
+    Returns:
+        List of match dictionaries with keys:
+            - line_number: 1-indexed line number of match
+            - line_content: The matching line
+            - match_text: The actual matched text
+            - context_before: List of lines before match
+            - context_after: List of lines after match
+            
+    Example:
+        >>> text = "line1\\nHello World\\nline3"
+        >>> matches = search_in_content(text, "Hello", is_regex=False)
+        >>> matches[0]['line_number']
+        2
+        >>> matches[0]['match_text']
+        'Hello'
+    """
+    if not content:
+        return []
+    
+    lines = content.splitlines()
+    matches = []
+    
+    # Compile regex pattern or escape for literal search
+    if is_regex:
+        try:
+            regex = re.compile(pattern, re.IGNORECASE)
+        except re.error as e:
+            logger.warning(f"Invalid regex pattern '{pattern}': {e}")
+            return []
+    else:
+        regex = re.compile(re.escape(pattern), re.IGNORECASE)
+    
+    # Search each line
+    for line_idx, line in enumerate(lines):
+        match = regex.search(line)
+        if match:
+            line_number = line_idx + 1  # Convert to 1-indexed
+            
+            # Get context lines
+            context_start = max(0, line_idx - context_lines)
+            context_end = min(len(lines), line_idx + context_lines + 1)
+            
+            context_before = lines[context_start:line_idx]
+            context_after = lines[line_idx + 1:context_end]
+            
+            matches.append({
+                'line_number': line_number,
+                'line_content': line,
+                'match_text': match.group(0),
+                'context_before': context_before,
+                'context_after': context_after,
+            })
+    
+    return matches