ara-cli 0.1.9.94__py3-none-any.whl → 0.1.9.96__py3-none-any.whl

ara_cli/file_loaders/document_readers.py

@@ -0,0 +1,233 @@
+import os
+from abc import ABC, abstractmethod
+from typing import Tuple, Optional
+
+
+class DocumentReader(ABC):
+    """Abstract base class for document readers"""
+    
+    def __init__(self, file_path: str):
+        self.file_path = file_path
+        self.base_dir = os.path.dirname(file_path)
+    
+    @abstractmethod
+    def read(self, extract_images: bool = False) -> str:
+        """Read document and optionally extract images"""
+        pass
+    
+    def create_image_data_dir(self, extension_suffix: str) -> str:
+        """
+        Create data directory for images with file extension suffix to avoid conflicts.
+        
+        Returns:
+            str: Path to images directory
+        """
+        file_name_with_ext = os.path.splitext(os.path.basename(self.file_path))[0] + f"_{extension_suffix}"
+        data_dir = os.path.join(self.base_dir, f"{file_name_with_ext}.data")
+        images_dir = os.path.join(data_dir, "images")
+        if not os.path.exists(images_dir):
+            os.makedirs(images_dir)
+        return images_dir
+    
+    def save_and_describe_image(self, image_data: bytes, image_format: str, 
+                               save_dir: str, image_counter: int) -> Tuple[str, str]:
+        """
+        Save image data and get its description from LLM.
+        
+        Returns:
+            tuple: (relative_image_path, description)
+        """
+        from ara_cli.prompt_handler import describe_image
+        
+        # Save image
+        image_filename = f"{image_counter}.{image_format}"
+        image_path = os.path.join(save_dir, image_filename)
+        
+        with open(image_path, "wb") as image_file:
+            image_file.write(image_data)
+        
+        # Get image description from LLM
+        description = describe_image(image_path)
+        
+        # Get relative path
+        relative_image_path = os.path.relpath(image_path, self.base_dir)
+        
+        return relative_image_path, description
+
+
+class DocxReader(DocumentReader):
+    """Reader for DOCX files"""
+    
+    def read(self, extract_images: bool = False) -> str:
+        import docx
+        
+        doc = docx.Document(self.file_path)
+        text_content = '\n'.join(para.text for para in doc.paragraphs)
+        
+        if not extract_images:
+            return text_content
+        
+        from PIL import Image
+        import io
+        
+        # Create data directory for images
+        images_dir = self.create_image_data_dir("docx")
+        
+        # Extract and process images
+        image_descriptions = []
+        image_counter = 1
+        
+        for rel in doc.part.rels.values():
+            if "image" in rel.reltype:
+                image_data = rel.target_part.blob
+                
+                # Determine image format
+                image = Image.open(io.BytesIO(image_data))
+                image_format = image.format.lower()
+                
+                # Save and describe image
+                relative_path, description = self.save_and_describe_image(
+                    image_data, image_format, images_dir, image_counter
+                )
+                
+                # Add formatted description to list
+                image_description = f"\nImage: {relative_path}\n[{description}]\n"
+                image_descriptions.append(image_description)
+                
+                image_counter += 1
+        
+        # Combine text content with image descriptions
+        if image_descriptions:
+            text_content += "\n\n### Extracted Images\n" + "\n".join(image_descriptions)
+        
+        return text_content
+
+
+class PdfReader(DocumentReader):
+    """Reader for PDF files"""
+    
+    def read(self, extract_images: bool = False) -> str:
+        import pymupdf4llm
+        
+        if not extract_images:
+            return pymupdf4llm.to_markdown(self.file_path, write_images=False)
+        
+        import fitz  # PyMuPDF
+        
+        # Create images directory
+        images_dir = self.create_image_data_dir("pdf")
+        
+        # Extract text without images first
+        text_content = pymupdf4llm.to_markdown(self.file_path, write_images=False)
+        
+        # Extract and process images
+        doc = fitz.open(self.file_path)
+        image_descriptions = []
+        image_counter = 1
+        
+        for page_num, page in enumerate(doc):
+            image_list = page.get_images()
+            
+            for img_index, img in enumerate(image_list):
+                # Extract image
+                xref = img[0]
+                base_image = doc.extract_image(xref)
+                image_bytes = base_image["image"]
+                image_ext = base_image["ext"]
+                
+                # Save and describe image
+                relative_path, description = self.save_and_describe_image(
+                    image_bytes, image_ext, images_dir, image_counter
+                )
+                
+                # Add formatted description to list
+                image_description = f"\nImage: {relative_path}\n[{description}]\n"
+                image_descriptions.append(image_description)
+                
+                image_counter += 1
+        
+        doc.close()
+        
+        # Combine text content with image descriptions
+        if image_descriptions:
+            text_content += "\n\n### Extracted Images\n" + "\n".join(image_descriptions)
+        
+        return text_content
+
+
+class OdtReader(DocumentReader):
+    """Reader for ODT files"""
+    
+    def read(self, extract_images: bool = False) -> str:
+        import pymupdf4llm
+        
+        if not extract_images:
+            return pymupdf4llm.to_markdown(self.file_path, write_images=False)
+        
+        import zipfile
+        from PIL import Image
+        import io
+        
+        # Create data directory for images
+        images_dir = self.create_image_data_dir("odt")
+        
+        # Get text content
+        text_content = pymupdf4llm.to_markdown(self.file_path, write_images=False)
+        
+        # Extract and process images from ODT
+        image_descriptions = []
+        image_counter = 1
+        
+        try:
+            with zipfile.ZipFile(self.file_path, 'r') as odt_zip:
+                # List all files in the Pictures directory
+                picture_files = [f for f in odt_zip.namelist() if f.startswith('Pictures/')]
+                
+                for picture_file in picture_files:
+                    # Extract image data
+                    image_data = odt_zip.read(picture_file)
+                    
+                    # Determine image format
+                    image = Image.open(io.BytesIO(image_data))
+                    image_format = image.format.lower()
+                    
+                    # Save and describe image
+                    relative_path, description = self.save_and_describe_image(
+                        image_data, image_format, images_dir, image_counter
+                    )
+                    
+                    # Add formatted description to list
+                    image_description = f"\nImage: {relative_path}\n[{description}]\n"
+                    image_descriptions.append(image_description)
+                    
+                    image_counter += 1
+        except Exception as e:
+            print(f"Warning: Could not extract images from ODT: {e}")
+        
+        # Combine text content with image descriptions
+        if image_descriptions:
+            text_content += "\n\n### Extracted Images\n" + "\n".join(image_descriptions)
+        
+        return text_content
+
+
+class DocumentReaderFactory:
+    """Factory for creating appropriate document readers"""
+    
+    @staticmethod
+    def create_reader(file_path: str) -> Optional[DocumentReader]:
+        """Create appropriate reader based on file extension"""
+        _, ext = os.path.splitext(file_path)
+        ext = ext.lower()
+        
+        readers = {
+            '.docx': DocxReader,
+            '.pdf': PdfReader,
+            '.odt': OdtReader
+        }
+        
+        reader_class = readers.get(ext)
+        if reader_class:
+            return reader_class(file_path)
+        
+        return None

ara_cli/file_loaders/file_loaders.py

@@ -0,0 +1,123 @@
+import os
+import base64
+from abc import ABC, abstractmethod
+from typing import Optional
+from ara_cli.file_loaders.markdown_reader import MarkdownReader
+from ara_cli.file_loaders.document_readers import DocumentReaderFactory
+
+
+class FileLoader(ABC):
+    """Abstract base class for file loaders"""
+    
+    def __init__(self, chat_instance):
+        self.chat = chat_instance
+        
+    @abstractmethod
+    def load(self, file_path: str, **kwargs) -> bool:
+        """Load file with specific implementation"""
+        pass
+        
+    def add_prompt_tag_if_needed(self):
+        """Add prompt tag to chat if needed"""
+        self.chat.add_prompt_tag_if_needed(self.chat.chat_name)
+
+
+class TextFileLoader(FileLoader):
+    """Loads text files"""
+    
+    def load(self, file_path: str, prefix: str = "", suffix: str = "", 
+             block_delimiter: str = "", extract_images: bool = False) -> bool:
+        """Load text file with optional markdown image extraction"""
+        
+        is_md_file = file_path.lower().endswith('.md')
+        
+        if is_md_file and extract_images:
+            reader = MarkdownReader(file_path)
+            file_content = reader.read(extract_images=True)
+        else:
+            with open(file_path, 'r', encoding='utf-8', errors="replace") as file:
+                file_content = file.read()
+        
+        if block_delimiter:
+            file_content = f"{block_delimiter}\n{file_content}\n{block_delimiter}"
+        
+        write_content = f"{prefix}{file_content}{suffix}\n"
+        
+        with open(self.chat.chat_name, 'a', encoding='utf-8') as chat_file:
+            chat_file.write(write_content)
+            
+        return True
+
+
+class BinaryFileLoader(FileLoader):
+    """Loads binary files (images)"""
+    
+    def load(self, file_path: str, mime_type: str, prefix: str = "", suffix: str = "") -> bool:
+        """Load binary file as base64"""
+        
+        with open(file_path, 'rb') as file:
+            file_content = file.read()
+            
+        base64_image = base64.b64encode(file_content).decode("utf-8")
+        write_content = f"{prefix}![{os.path.basename(file_path)}](data:{mime_type};base64,{base64_image}){suffix}\n"
+        
+        with open(self.chat.chat_name, 'a', encoding='utf-8') as chat_file:
+            chat_file.write(write_content)
+            
+        return True
+
+
+class DocumentFileLoader(FileLoader):
+    """Loads document files (PDF, DOCX, ODT)"""
+    
+    def load(self, file_path: str, prefix: str = "", suffix: str = "", 
+             block_delimiter: str = "```", extract_images: bool = False) -> bool:
+        """Load document file with optional image extraction"""
+        
+        reader = DocumentReaderFactory.create_reader(file_path)
+        
+        if not reader:
+            print("Unsupported document type.")
+            return False
+        
+        text_content = reader.read(extract_images=extract_images)
+        
+        if block_delimiter:
+            text_content = f"{block_delimiter}\n{text_content}\n{block_delimiter}"
+        
+        write_content = f"{prefix}{text_content}{suffix}\n"
+        
+        with open(self.chat.chat_name, 'a', encoding='utf-8') as chat_file:
+            chat_file.write(write_content)
+            
+        return True
+
+
+class FileLoaderFactory:
+    """Factory for creating appropriate file loaders"""
+    
+    BINARY_TYPE_MAPPING = {
+        ".png": "image/png",
+        ".jpg": "image/jpeg",
+        ".jpeg": "image/jpeg",
+    }
+    
+    DOCUMENT_TYPE_EXTENSIONS = [".docx", ".doc", ".odt", ".pdf"]
+    
+    @staticmethod
+    def create_loader(file_name: str, chat_instance) -> Optional[FileLoader]:
+        """Create appropriate loader based on file type"""
+        
+        file_name_lower = file_name.lower()
+        
+        # Check if it's a binary file
+        for extension, mime_type in FileLoaderFactory.BINARY_TYPE_MAPPING.items():
+            if file_name_lower.endswith(extension):
+                return BinaryFileLoader(chat_instance)
+        
+        # Check if it's a document
+        if any(file_name_lower.endswith(ext) for ext in FileLoaderFactory.DOCUMENT_TYPE_EXTENSIONS):
+            return DocumentFileLoader(chat_instance)
+        
+        # Default to text file loader
+        return TextFileLoader(chat_instance)

ara_cli/file_loaders/image_processor.py

@@ -0,0 +1,89 @@
+import os
+import base64
+import tempfile
+import requests
+from typing import Optional, Tuple
+import re
+from ara_cli.prompt_handler import describe_image
+
+
+class ImageProcessor:
+    """Handles image processing operations"""
+    
+    @staticmethod
+    def process_base64_image(image_ref: str, base64_pattern: re.Pattern) -> Optional[Tuple[str, str]]:
+        """Process base64 encoded image and return description"""
+        base64_match = base64_pattern.match(image_ref)
+        if not base64_match:
+            return None
+            
+        image_format = base64_match.group(1)
+        base64_data = base64_match.group(2)
+        image_data = base64.b64decode(base64_data)
+        
+        # Create a temporary file to send to LLM
+        with tempfile.NamedTemporaryFile(suffix=f'.{image_format}', delete=False) as tmp_file:
+            tmp_file.write(image_data)
+            tmp_file_path = tmp_file.name
+        
+        try:
+            description = describe_image(tmp_file_path)
+            return f"Image: (base64 embedded {image_format} image)\n[{description}]", None
+        finally:
+            os.unlink(tmp_file_path)
+    
+    @staticmethod
+    def process_url_image(image_ref: str) -> Tuple[str, Optional[str]]:
+        """Process image from URL and return description"""
+        if not image_ref.startswith(('http://', 'https://')):
+            return "", None
+            
+        try:
+            response = requests.get(image_ref, timeout=10)
+            response.raise_for_status()
+            
+            # Determine file extension from content-type
+            content_type = response.headers.get('content-type', '')
+            ext = ImageProcessor._get_extension_from_content_type(content_type, image_ref)
+            
+            # Create temporary file
+            with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp_file:
+                tmp_file.write(response.content)
+                tmp_file_path = tmp_file.name
+            
+            try:
+                description = describe_image(tmp_file_path)
+                return f"Image: {image_ref}\n[{description}]", None
+            finally:
+                os.unlink(tmp_file_path)
+                
+        except Exception as e:
+            error_msg = f"Could not download image: {str(e)}"
+            return f"Image: {image_ref}\n[{error_msg}]", error_msg
+    
+    @staticmethod
+    def process_local_image(image_ref: str, base_dir: str) -> Tuple[str, Optional[str]]:
+        """Process local image file and return description"""
+        if os.path.isabs(image_ref):
+            local_image_path = image_ref
+        else:
+            local_image_path = os.path.join(base_dir, image_ref)
+        
+        if os.path.exists(local_image_path):
+            description = describe_image(local_image_path)
+            return f"Image: {image_ref}\n[{description}]", None
+        else:
+            error_msg = f"Image file not found"
+            return f"Image: {image_ref}\n[{error_msg}]", f"Local image not found: {local_image_path}"
+    
+    @staticmethod
+    def _get_extension_from_content_type(content_type: str, url: str) -> str:
+        """Determine file extension from content type or URL"""
+        if 'image/jpeg' in content_type:
+            return '.jpg'
+        elif 'image/png' in content_type:
+            return '.png'
+        elif 'image/gif' in content_type:
+            return '.gif'
+        else:
+            return os.path.splitext(url)[1] or '.png'

ara_cli/file_loaders/markdown_reader.py

@@ -0,0 +1,75 @@
+import os
+import re
+from typing import Optional
+from charset_normalizer import from_path
+from ara_cli.file_loaders.image_processor import ImageProcessor
+
+
+class MarkdownReader:
+    """Handles markdown file reading with optional image extraction"""
+
+    def __init__(self, file_path: str):
+        self.file_path = file_path
+        self.base_dir = os.path.dirname(file_path)
+        self.image_processor = ImageProcessor()
+
+    def read(self, extract_images: bool = False) -> str:
+        """Read markdown file and optionally extract/describe images"""
+        # Detect and use the most appropriate encoding
+        result = from_path(self.file_path).best()
+        if not result:
+            print(f"Failed to detect encoding for {self.file_path}")
+            return ""
+        content = str(result)
+
+        if not extract_images:
+            return content
+
+        return self._process_images(content)
+
+    def _process_images(self, content: str) -> str:
+        """Process all images in markdown content"""
+        # Pattern to match markdown images: ![alt text](url or path)
+        image_pattern = re.compile(r"!\[([^\]]*)\]\(([^\)]+)\)")
+        base64_pattern = re.compile(r"data:image/([^;]+);base64,([^)]+)")
+
+        # Process each image reference
+        for match in image_pattern.finditer(content):
+            image_ref = match.group(2)
+            replacement = self._process_single_image(image_ref, base64_pattern)
+
+            if replacement:
+                content = content.replace(match.group(0), replacement, 1)
+
+        return content
+
+    def _process_single_image(
+        self, image_ref: str, base64_pattern: re.Pattern
+    ) -> Optional[str]:
+        """Process a single image reference"""
+        try:
+            # Try base64 first
+            result = self.image_processor.process_base64_image(
+                image_ref, base64_pattern
+            )
+            if result:
+                return result[0]
+
+            # Try URL
+            result, error = self.image_processor.process_url_image(image_ref)
+            if result:
+                if error:
+                    print(f"Warning: {error}")
+                return result
+
+            # Try local file
+            result, error = self.image_processor.process_local_image(
+                image_ref, self.base_dir
+            )
+            if error:
+                print(f"Warning: {error}")
+            return result
+
+        except Exception as e:
+            print(f"Warning: Could not process image {image_ref}: {e}")
+            return None

ara_cli/file_loaders/text_file_loader.py

@@ -4,21 +4,15 @@ import base64
 import tempfile
 from typing import Optional, Tuple
 import requests
+from charset_normalizer import from_path
 from ara_cli.prompt_handler import describe_image
 from ara_cli.file_loaders.file_loader import FileLoader
 
 
 class TextFileLoader(FileLoader):
     """Loads text files"""
-
-    def load(
-        self,
-        file_path: str,
-        prefix: str = "",
-        suffix: str = "",
-        block_delimiter: str = "",
-        extract_images: bool = False
-    ) -> bool:
+    def load(self, file_path: str, prefix: str = "", suffix: str = "",
+             block_delimiter: str = "", extract_images: bool = False, **kwargs) -> bool:
         """Load text file with optional markdown image extraction"""
 
         is_md_file = file_path.lower().endswith('.md')
@@ -27,8 +21,12 @@ class TextFileLoader(FileLoader):
             reader = MarkdownReader(file_path)
             file_content = reader.read(extract_images=True)
         else:
-            with open(file_path, 'r', encoding='utf-8', errors='replace') as file:
-                file_content = file.read()
+            # Use charset-normalizer to detect encoding
+            encoded_content = from_path(file_path).best()
+            if not encoded_content:
+                print(f"Failed to detect encoding for {file_path}")
+                return False
+            file_content = str(encoded_content)
 
         if block_delimiter:
             file_content = f"{block_delimiter}\n{file_content}\n{block_delimiter}"

ara_cli/global_file_lister.py

@@ -0,0 +1,61 @@
+import os
+import fnmatch
+from typing import List, Dict, Any
+
+# Ağaç yapımız için bir tip tanımı yapalım
+DirTree = Dict[str, Any]
+
+def _build_tree(root_path: str, patterns: List[str]) -> DirTree:
+    """Belirtilen yoldaki dizin yapısını temsil eden iç içe bir sözlük oluşturur."""
+    tree: DirTree = {'files': [], 'dirs': {}}
+    try:
+        for item in os.listdir(root_path):
+            item_path = os.path.join(root_path, item)
+            if os.path.isdir(item_path):
+                subtree = _build_tree(item_path, patterns)
+                # Sadece içinde dosya olan veya dosyası olan alt klasörleri ekle
+                if subtree['files'] or subtree['dirs']:
+                    tree['dirs'][item] = subtree
+            elif os.path.isfile(item_path):
+                # Dosyanın verilen desenlerden herhangi biriyle eşleşip eşleşmediğini kontrol et
+                if any(fnmatch.fnmatch(item, pattern) for pattern in patterns):
+                    tree['files'].append(item)
+    except OSError as e:
+        print(f"Warning: Could not access path {root_path}: {e}")
+    return tree
+
+def _write_tree_to_markdown(md_file, tree: DirTree, level: int):
+    """Ağaç veri yapısını markdown formatında dosyaya yazar."""
+    # Dosyaları girintili olarak yaz
+    indent = '    ' * level
+    for filename in sorted(tree['files']):
+        md_file.write(f"{indent}- [] {filename}\n")
+    
+    # Alt dizinler için başlık oluştur ve recursive olarak devam et
+    for dirname, subtree in sorted(tree['dirs'].items()):
+        # Alt başlıklar için girinti yok, sadece başlık seviyesi artıyor
+        md_file.write(f"{'    ' * (level -1)}{'#' * (level + 1)} {dirname}\n")
+        _write_tree_to_markdown(md_file, subtree, level + 1)
+
+def generate_global_markdown_listing(directories: List[str], file_patterns: List[str], output_file: str):
+    """
+    Global dizinler için hiyerarşik bir markdown dosya listesi oluşturur.
+    En üst başlık olarak mutlak yolu kullanır, alt öğeler için göreceli isimler kullanır.
+    """
+    with open(output_file, 'w', encoding='utf-8') as md_file:
+        for directory in directories:
+            abs_dir = os.path.abspath(directory)
+            
+            if not os.path.isdir(abs_dir):
+                print(f"Warning: Global directory not found: {abs_dir}")
+                md_file.write(f"# {directory}\n")
+                md_file.write(f"    - !! UYARI: Dizin bulunamadı: {abs_dir}\n\n")
+                continue
+
+            tree = _build_tree(abs_dir, file_patterns)
+            
+            # Sadece ağaç boş değilse yaz
+            if tree['files'] or tree['dirs']:
+                md_file.write(f"# {abs_dir}\n")
+                _write_tree_to_markdown(md_file, tree, 1)
+                md_file.write("\n")

ara_cli/prompt_extractor.py

@@ -17,7 +17,7 @@ def extract_code_blocks_md(markdown_text):
 
 
 def extract_responses(document_path, relative_to_ara_root=False, force=False, write=False):
-    print(f"Debug: Starting extraction from {document_path}")
+    print(f"Starting extraction from '{document_path}'")
     block_extraction_counter = 0
 
     with open(document_path, 'r', encoding='utf-8', errors='replace') as file: