alita-sdk 0.3.257__py3-none-any.whl → 0.3.562__py3-none-any.whl

alita_sdk/tools/utils/content_parser.py

@@ -1,13 +1,17 @@
+from copy import deepcopy
 import os
+import re
 import tempfile
 from logging import getLogger
 from pathlib import Path
-from typing import Generator
+from typing import Generator, List
 
 from langchain_core.documents import Document
 from langchain_core.tools import ToolException
 
-from alita_sdk.runtime.langchain.document_loaders.constants import loaders_map
+from alita_sdk.runtime.langchain.document_loaders.constants import loaders_map, LoaderProperties
+from ...runtime.langchain.document_loaders.AlitaTextLoader import AlitaTextLoader
+from ...runtime.utils.utils import IndexerKeywords
 
 logger = getLogger(__name__)
 
@@ -51,11 +55,9 @@ Highlight any visible details that could help in understanding the image.
 Be as precise and thorough as possible in your responses. If something is unclear or illegible, state that explicitly.
 '''
 
-IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'webp', 'svg']
-
 
 def parse_file_content(file_name=None, file_content=None, is_capture_image: bool = False, page_number: int = None,
-                       sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False) -> str | ToolException:
+                       sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False, prompt=None) -> str | ToolException:
     """Parse the content of a file based on its type and return the parsed content.
 
     Args:
@@ -72,18 +74,63 @@ def parse_file_content(file_name=None, file_content=None, is_capture_image: bool
     Raises:
         ToolException: If the file type is not supported or if there is an error reading the file.
         """
+    if not prompt:
+        prompt = image_processing_prompt
+    loader = prepare_loader(
+        file_name=file_name,
+        file_content=file_content,
+        is_capture_image=is_capture_image,
+        page_number=page_number,
+        sheet_name=sheet_name,
+        llm=llm,
+        file_path=file_path,
+        excel_by_sheets=excel_by_sheets,
+        prompt=prompt
+    )
 
-    if (file_path and (file_name or file_content)) or (not file_path and (not file_name or file_content is None)):
-        raise ToolException("Either (file_name and file_content) or file_path must be provided, but not both.")
-
-    extension = Path(file_path if file_path else file_name).suffix
+    if not loader:
+        return ToolException(
+            "Not supported type of files entered. Supported types are TXT, DOCX, PDF, PPTX, XLSX and XLS only.")
 
-    loader_object = loaders_map.get(extension)
-    if not loader_object:
-        logger.warning(f"No loader found for file extension: {extension}. File: {file_path if file_path else file_name}")
+    try:
+        if hasattr(loader, 'get_content'):
+            return loader.get_content()
+        else:
+            extension = Path(file_path if file_path else file_name).suffix
+            loader_kwargs = get_loader_kwargs(loaders_map.get(extension), file_name, file_content, is_capture_image, page_number, sheet_name, llm, file_path, excel_by_sheets)
+            if file_content:
+                return load_content_from_bytes(file_content=file_content,
+                                               extension=extension,
+                                               loader_extra_config=loader_kwargs,
+                                               llm=llm)
+            else:
+                return load_content(file_path=file_path,
+                                    extension=extension,
+                                    loader_extra_config=loader_kwargs,
+                                    llm=llm)
+    except Exception as e:
+        return ToolException(f"Error reading file ({file_name or file_path}) content. Make sure these types are supported: {str(e)}")
+
+def load_file_docs(file_name=None, file_content=None, is_capture_image: bool = False, page_number: int = None,
+                       sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False) -> List[Document] | ToolException:
+    loader = prepare_loader(
+        file_name=file_name,
+        file_content=file_content,
+        is_capture_image=is_capture_image,
+        page_number=page_number,
+        sheet_name=sheet_name,
+        llm=llm,
+        file_path=file_path,
+        excel_by_sheets=excel_by_sheets
+    )
+    if not loader:
         return ToolException(
             "Not supported type of files entered. Supported types are TXT, DOCX, PDF, PPTX, XLSX and XLS only.")
-    loader_kwargs = loader_object['kwargs']
+    return loader.load()
+
+def get_loader_kwargs(loader_object, file_name=None, file_content=None, is_capture_image: bool = False, page_number: int = None,
+                    sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False, prompt=None):
+    loader_kwargs = deepcopy(loader_object['kwargs'])
     loader_kwargs.update({
         "file_path": file_path,
         "file_content": file_content,
@@ -93,28 +140,26 @@ def parse_file_content(file_name=None, file_content=None, is_capture_image: bool
         "page_number": page_number,
         "sheet_name": sheet_name,
         "excel_by_sheets": excel_by_sheets,
+        "prompt": prompt,
         "row_content": True,
         "json_documents": False
     })
-    loader = loader_object['class'](**loader_kwargs)
+    return loader_kwargs
 
-    if not loader:
-        return ToolException(
-            "Not supported type of files entered. Supported types are TXT, DOCX, PDF, PPTX, XLSX and XLS only.")
+def prepare_loader(file_name=None, file_content=None, is_capture_image: bool = False, page_number: int = None,
+                       sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False,
+                   prompt=None):
+        if (file_path and (file_name or file_content)) or (not file_path and (not file_name or file_content is None)):
+            raise ToolException("Either (file_name and file_content) or file_path must be provided, but not both.")
 
-    if hasattr(loader, 'get_content'):
-        return loader.get_content()
-    else:
-        if file_content:
-            return load_content_from_bytes(file_content=file_content,
-                                           extension=extension,
-                                           loader_extra_config=loader_kwargs,
-                                           llm=llm)
-        else:
-            return load_content(file_path=file_path,
-                                extension=extension,
-                                loader_extra_config=loader_kwargs,
-                                llm=llm)
+        extension = Path(file_path if file_path else file_name).suffix
+
+        loader_object = loaders_map.get(extension)
+        if not loader_object:
+            loader_object = loaders_map.get('.txt')  # Default to text loader if no specific loader found
+        loader_kwargs = get_loader_kwargs(loader_object, file_name, file_content, is_capture_image, page_number, sheet_name, llm, file_path, excel_by_sheets, prompt)
+        loader = loader_object['class'](**loader_kwargs)
+        return loader
 
 # TODO: review usage of this function alongside with functions above
 def load_content(file_path: str, extension: str = None, loader_extra_config: dict = None, llm = None) -> str:
@@ -142,7 +187,7 @@ def load_content(file_path: str, extension: str = None, loader_extra_config: dic
         if "file_path" in loader_kwargs:
             del loader_kwargs["file_path"]
 
-        loader = loader_cls(file_path, **loader_kwargs)
+        loader = loader_cls(file_path=file_path, **loader_kwargs)
         documents = loader.load()
 
         page_contents = [doc.page_content for doc in documents]
@@ -167,37 +212,93 @@ def load_content_from_bytes(file_content: bytes, extension: str = None, loader_e
         if temp_file_path and os.path.exists(temp_file_path):
             os.remove(temp_file_path)
 
-def process_content_by_type(document: Document, content, extension_source: str, llm = None, chunking_config={}) -> Generator[Document, None, None]:
-    temp_file_path = None
+def process_document_by_type(content, extension_source: str, document: Document = None, llm = None, chunking_config=None) \
+        -> Generator[Document, None, None]:
+    """Process the content of a file based on its type using a configured loader cosidering the origin document."""
     try:
-        extension = "." + extension_source.split('.')[-1].lower()
-
-        with tempfile.NamedTemporaryFile(mode='w+b', suffix=extension, delete=False) as temp_file:
-            temp_file_path = temp_file.name
-            if content is None:
-                logger.warning("'loader_content' ie expected but not found in document metadata.")
-                return
-            
-            temp_file.write(content)
-            temp_file.flush()
-
-            loader_config = loaders_map.get(extension)
-            if not loader_config:
-                logger.warning(f"No loader found for file extension: {extension}. File: {temp_file_path}")
-                return
-
-            loader_cls = loader_config['class']
-            loader_kwargs = loader_config['kwargs']
-
-            loader = loader_cls(file_path=temp_file_path, **loader_kwargs)
-            for chunk in loader.load():
-                yield Document(
-                    page_content=sanitize_for_postgres(chunk.page_content),
-                    metadata={**document.metadata, **chunk.metadata}
-                )
-    finally:
-        if temp_file_path and os.path.exists(temp_file_path):
-            os.remove(temp_file_path)
+        chunks = process_content_by_type(content, extension_source, llm, chunking_config)
+    except Exception as e:
+        msg = f"Error during content parsing for file {extension_source}:\n{e}"
+        logger.warning(msg)
+        yield Document(
+            page_content=msg,
+            metadata={**document.metadata, 'chunk_id': 1}
+        )
+        return
+    #
+    chunks_counter = 0
+    for chunk in chunks:
+        chunks_counter += 1
+        metadata = {**document.metadata, **chunk.metadata}
+        #
+        # ensure each chunk has a unique chunk_id
+        metadata['chunk_id'] = chunks_counter
+        #
+        yield Document(
+            page_content=sanitize_for_postgres(chunk.page_content),
+            metadata=metadata
+        )
+
+
+def process_content_by_type(content, filename: str, llm=None, chunking_config=None, fallback_extensions=None) -> \
+        Generator[Document, None, None]:
+    """Process the content of a file based on its type using a configured loader."""
+    temp_file_path = None
+    extensions = fallback_extensions if fallback_extensions else []
+    match = re.search(r'\.([^.]+)$', filename)
+
+    if match:
+        extensions.insert(0, f".{match.group(1).lower()}")
+    elif not extensions:
+        extensions = [".txt"]
+
+    for extension in extensions:
+        try:
+            with tempfile.NamedTemporaryFile(mode='w+b', suffix=extension, delete=False) as temp_file:
+                temp_file_path = temp_file.name
+                if content is None:
+                    logger.warning(
+                        f"'{IndexerKeywords.CONTENT_IN_BYTES.value}' ie expected but not found in document metadata.")
+                    return []
+    
+                temp_file.write(content)
+                temp_file.flush()
+    
+                loader_config = loaders_map.get(extension)
+                if not loader_config:
+                    logger.warning(f"No loader found for file extension: {extension}. File: {temp_file_path}")
+                    return []
+    
+                loader_cls = loader_config['class']
+                loader_kwargs = loader_config['kwargs']
+                # Determine which loader configuration keys are allowed to be overridden by user input.
+                # If 'allowed_to_override' is specified in the loader configuration, use it; otherwise, allow all keys in loader_kwargs.
+                allowed_to_override = loader_config.get('allowed_to_override', loader_kwargs)
+                # If a chunking_config is provided and contains custom configuration for the current file extension,
+                # update loader_kwargs with user-supplied values, but only for keys explicitly permitted in allowed_to_override and if value differs from default.
+                # This ensures that only safe and intended parameters can be customized, preventing accidental or unauthorized changes
+                # to critical loader settings.
+                if chunking_config and (users_config_for_extension := chunking_config.get(extension, {})):
+                    for key in set(users_config_for_extension.keys()) & set(allowed_to_override.keys()):
+                        if users_config_for_extension[key] != allowed_to_override[key]:
+                            loader_kwargs[key] = users_config_for_extension[key]
+                if LoaderProperties.LLM.value in loader_kwargs and loader_kwargs.pop(LoaderProperties.LLM.value):
+                    loader_kwargs['llm'] = llm
+                if LoaderProperties.PROMPT_DEFAULT.value in loader_kwargs and loader_kwargs.pop(LoaderProperties.PROMPT_DEFAULT.value):
+                    loader_kwargs[LoaderProperties.PROMPT.value] = image_processing_prompt
+                loader = loader_cls(file_path=temp_file_path, **loader_kwargs)
+                yield from loader.load()
+                break
+        except Exception as e:
+            if fallback_extensions:
+                logger.warning(f"Error loading attachment: {str(e)} for file {temp_file_path} (extension: {extension})")
+                logger.warning(f"Continuing with fallback extensions: {fallback_extensions}.")
+                continue
+            else:
+                raise e
+        finally:
+            if temp_file_path and os.path.exists(temp_file_path):
+                os.remove(temp_file_path)
 
 # FIXME copied from langchain_core/utils/strings.py of 0.3.74 version
 # https://github.com/langchain-ai/langchain/pull/32157
@@ -218,4 +319,23 @@ def sanitize_for_postgres(text: str, replacement: str = "") -> str:
         >>> sanitize_for_postgres("Hello\\x00world", " ")
         'Hello world'
     """
-    return text.replace("\x00", replacement)
+    return text.replace("\x00", replacement)
+
+
+def file_extension_by_chunker(chunker_name: str) -> str | None:
+    if not chunker_name:
+        return None
+    name = chunker_name.lower()
+    if name == "markdown":
+        return ".md"
+    if name == "json":
+        return ".json"
+    if name == "text" or name == "txt":
+        return ".txt"
+    if name == "html":
+        return ".html"
+    if name == "xml":
+        return ".xml"
+    if name == "csv":
+        return ".csv"
+    return None

alita_sdk/tools/utils/text_operations.py

@@ -0,0 +1,254 @@
+"""
+Shared text operations utilities for file manipulation across toolkits.
+
+Provides common functionality for:
+- Parsing OLD/NEW marker-based edits
+- Text file validation
+- Line-based slicing and partial reads
+- Content searching with context
+"""
+import re
+import logging
+from typing import List, Tuple, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+# Text file extensions that support editing
+TEXT_EDITABLE_EXTENSIONS = {
+    '.md', '.txt', '.csv', '.json', '.xml', '.html', 
+    '.yaml', '.yml', '.ini', '.conf', '.log', '.sh',
+    '.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.go',
+    '.rb', '.php', '.c', '.cpp', '.h', '.hpp', '.cs',
+    '.sql', '.r', '.m', '.swift', '.kt', '.rs', '.scala'
+}
+
+
+def parse_old_new_markers(file_query: str) -> List[Tuple[str, str]]:
+    """
+    Parse OLD/NEW marker-based edit instructions.
+    
+    Extracts pairs of old and new content from a file query using markers:
+    - OLD <<<< ... >>>> OLD
+    - NEW <<<< ... >>>> NEW
+    
+    Args:
+        file_query: String containing marked old and new content sections
+        
+    Returns:
+        List of tuples (old_content, new_content) for each edit pair
+        
+    Example:
+        >>> query = '''
+        ... OLD <<<<
+        ... Hello World
+        ... >>>> OLD
+        ... NEW <<<<
+        ... Hello Mars
+        ... >>>> NEW
+        ... '''
+        >>> parse_old_new_markers(query)
+        [('Hello World', 'Hello Mars')]
+    """
+    # Split the file content by lines
+    code_lines = file_query.split("\n")
+
+    # Initialize lists to hold the contents of OLD and NEW sections
+    old_contents = []
+    new_contents = []
+
+    # Initialize variables to track whether the current line is within an OLD or NEW section
+    in_old_section = False
+    in_new_section = False
+
+    # Temporary storage for the current section's content
+    current_section_content = []
+
+    # Iterate through each line in the file content
+    for line in code_lines:
+        # Check for OLD section start
+        if "OLD <<<" in line:
+            in_old_section = True
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # Check for OLD section end
+        if ">>>> OLD" in line:
+            in_old_section = False
+            old_contents.append("\n".join(current_section_content).strip())  # Add the captured content
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # Check for NEW section start
+        if "NEW <<<" in line:
+            in_new_section = True
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # Check for NEW section end
+        if ">>>> NEW" in line:
+            in_new_section = False
+            new_contents.append("\n".join(current_section_content).strip())  # Add the captured content
+            current_section_content = []  # Reset current section content
+            continue  # Skip the line with the marker
+
+        # If currently in an OLD or NEW section, add the line to the current section content
+        if in_old_section or in_new_section:
+            current_section_content.append(line)
+
+    # Pair the OLD and NEW contents
+    paired_contents = list(zip(old_contents, new_contents))
+
+    return paired_contents
+
+
+def is_text_editable(filename: str) -> bool:
+    """
+    Check if a file is editable as text based on its extension.
+    
+    Args:
+        filename: Name or path of the file to check
+        
+    Returns:
+        True if file extension is in the text-editable whitelist
+        
+    Example:
+        >>> is_text_editable("config.json")
+        True
+        >>> is_text_editable("image.png")
+        False
+    """
+    from pathlib import Path
+    ext = Path(filename).suffix.lower()
+    return ext in TEXT_EDITABLE_EXTENSIONS
+
+
+def apply_line_slice(
+    content: str, 
+    offset: Optional[int] = None,
+    limit: Optional[int] = None,
+    head: Optional[int] = None,
+    tail: Optional[int] = None
+) -> str:
+    """
+    Apply line-based slicing to text content.
+    
+    Supports multiple modes:
+    - offset + limit: Read from line `offset` for `limit` lines (1-indexed)
+    - head: Read only first N lines
+    - tail: Read only last N lines
+    - No params: Return full content
+    
+    Args:
+        content: Text content to slice
+        offset: Starting line number (1-indexed, inclusive)
+        limit: Number of lines to read from offset
+        head: Return only first N lines
+        tail: Return only last N lines
+        
+    Returns:
+        Sliced content as string
+        
+    Example:
+        >>> text = "line1\\nline2\\nline3\\nline4\\nline5"
+        >>> apply_line_slice(text, offset=2, limit=2)
+        'line2\\nline3'
+        >>> apply_line_slice(text, head=2)
+        'line1\\nline2'
+        >>> apply_line_slice(text, tail=2)
+        'line4\\nline5'
+    """
+    if not content:
+        return content
+        
+    lines = content.splitlines(keepends=True)
+    
+    # Head mode: first N lines
+    if head is not None:
+        return ''.join(lines[:head])
+    
+    # Tail mode: last N lines
+    if tail is not None:
+        return ''.join(lines[-tail:] if tail > 0 else lines)
+    
+    # Offset + limit mode: slice from offset for limit lines
+    if offset is not None:
+        start_idx = max(0, offset - 1)  # Convert 1-indexed to 0-indexed
+        if limit is not None:
+            end_idx = start_idx + limit
+            return ''.join(lines[start_idx:end_idx])
+        else:
+            return ''.join(lines[start_idx:])
+    
+    # No slicing parameters: return full content
+    return content
+
+
+def search_in_content(
+    content: str,
+    pattern: str,
+    is_regex: bool = True,
+    context_lines: int = 2
+) -> List[Dict[str, any]]:
+    """
+    Search for pattern in content with context lines.
+    
+    Args:
+        content: Text content to search
+        pattern: Search pattern (regex if is_regex=True, else literal string)
+        is_regex: Whether to treat pattern as regex (default True)
+        context_lines: Number of lines before/after match to include (default 2)
+        
+    Returns:
+        List of match dictionaries with keys:
+            - line_number: 1-indexed line number of match
+            - line_content: The matching line
+            - match_text: The actual matched text
+            - context_before: List of lines before match
+            - context_after: List of lines after match
+            
+    Example:
+        >>> text = "line1\\nHello World\\nline3"
+        >>> matches = search_in_content(text, "Hello", is_regex=False)
+        >>> matches[0]['line_number']
+        2
+        >>> matches[0]['match_text']
+        'Hello'
+    """
+    if not content:
+        return []
+    
+    lines = content.splitlines()
+    matches = []
+    
+    # Compile regex pattern or escape for literal search
+    if is_regex:
+        try:
+            regex = re.compile(pattern, re.IGNORECASE)
+        except re.error as e:
+            logger.warning(f"Invalid regex pattern '{pattern}': {e}")
+            return []
+    else:
+        regex = re.compile(re.escape(pattern), re.IGNORECASE)
+    
+    # Search each line
+    for line_idx, line in enumerate(lines):
+        match = regex.search(line)
+        if match:
+            line_number = line_idx + 1  # Convert to 1-indexed
+            
+            # Get context lines
+            context_start = max(0, line_idx - context_lines)
+            context_end = min(len(lines), line_idx + context_lines + 1)
+            
+            context_before = lines[context_start:line_idx]
+            context_after = lines[line_idx + 1:context_end]
+            
+            matches.append({
+                'line_number': line_number,
+                'line_content': line,
+                'match_text': match.group(0),
+                'context_before': context_before,
+                'context_after': context_after,
+            })
+    
+    return matches