ara-cli 0.1.9.93__py3-none-any.whl → 0.1.9.95__py3-none-any.whl

ara_cli/chat.py

@@ -2,6 +2,7 @@ import os
 import argparse
 import cmd2
 from ara_cli.prompt_handler import send_prompt
+from ara_cli.file_loaders.markdown_reader import MarkdownReader
 
 from ara_cli.file_loaders.document_file_loader import DocumentFileLoader
 from ara_cli.file_loaders.binary_file_loader import BinaryFileLoader
@@ -9,13 +10,29 @@ from ara_cli.file_loaders.text_file_loader import TextFileLoader
 
 
 extract_parser = argparse.ArgumentParser()
-extract_parser.add_argument('-s', '--skip-queries', action='store_true', help='Force extraction')
+extract_parser.add_argument('-f', '--force', action='store_true', help='Force extraction')
+extract_parser.add_argument('-w','--write', action='store_true', help='Overwrite existing files without using LLM for merging.')
 
 load_parser = argparse.ArgumentParser()
 load_parser.add_argument('file_name', nargs='?', default='', help='File to load')
 load_parser.add_argument('--load-images', action='store_true', help='Extract and describe images from documents')
 
 
+from ara_cli.file_loaders.document_file_loader import DocumentFileLoader
+from ara_cli.file_loaders.binary_file_loader import BinaryFileLoader
+from ara_cli.file_loaders.text_file_loader import TextFileLoader
+
+
+extract_parser = argparse.ArgumentParser()
+extract_parser.add_argument('-f', '--force', action='store_true', help='Force extraction')
+extract_parser.add_argument('-w','--write', action='store_true', help='Overwrite existing files without using LLM for merging.')
+
+load_parser = argparse.ArgumentParser()
+load_parser.add_argument('file_name', nargs='?', default='', help='File to load')
+load_parser.add_argument('--load-images', action='store_true', help='Extract and describe images from documents')
+
+
+
 class Chat(cmd2.Cmd):
     CATEGORY_CHAT_CONTROL = "Chat control commands"
     CATEGORY_LLM_CONTROL = "Language model controls"
@@ -813,51 +830,80 @@ Start chatting (type 'HELP'/'h' for available commands, 'QUIT'/'q' to exit chat
 
         command = ExtractCommand(
             file_name=self.chat_name,
-            skip_queries=args.skip_queries,
+            force=args.force,
+            write=args.write,
             output=self.poutput,
             error_output=self.perror
         )
         command.execute()
 
+    def _find_givens_files(self, file_name: str) -> list[str]:
+        """
+        Finds the givens files to be processed.
+        - If file_name is provided, it resolves that path.
+        - Otherwise, it looks for default givens files.
+        - If no defaults are found, it prompts the user.
+        Returns a list of absolute file paths or an empty list if none are found.
+        """
+        base_directory = os.path.dirname(self.chat_name)
+
+        def resolve_path(name):
+            """Inner helper to resolve a path relative to chat, then absolute."""
+            relative_path = os.path.join(base_directory, name)
+            if os.path.exists(relative_path):
+                return relative_path
+            if os.path.exists(name):
+                return name
+            return None
+
+        if file_name:
+            path = resolve_path(file_name)
+            if path:
+                return [path]
+            relative_path_for_error = os.path.join(base_directory, file_name)
+            self.perror(f"No givens file found at {relative_path_for_error} or {file_name}")
+            return []
+
+        # If no file_name, check for defaults
+        default_files_to_check = [
+            os.path.join(base_directory, "prompt.data", "config.prompt_givens.md"),
+            os.path.join(base_directory, "prompt.data", "config.prompt_global_givens.md")
+        ]
+        existing_defaults = [f for f in default_files_to_check if os.path.exists(f)]
+        if existing_defaults:
+            return existing_defaults
+
+        # No defaults found, prompt user
+        user_input = input("Please specify a givens file: ")
+        if not user_input:
+            self.poutput("Aborting.")
+            return []
+        
+        path = resolve_path(user_input)
+        if path:
+            return [path]
+        self.perror(f"No givens file found at {user_input}. Aborting.")
+        return []
+
     @cmd2.with_category(CATEGORY_CHAT_CONTROL)
     def do_LOAD_GIVENS(self, file_name):
-        """Load all files listed in a ./prompt.data/config.prompt_givens.md"""
-        from ara_cli.directory_navigator import DirectoryNavigator
+        """Load all files listed in a ./prompt.data/config.prompt_givens.md and ./prompt.data/config.prompt_global_givens.md"""
         from ara_cli.prompt_handler import load_givens
 
-        base_directory = os.path.dirname(self.chat_name)
-
-        if file_name == "":
-            file_name = f"{base_directory}/prompt.data/config.prompt_givens.md"
-
-        # Check the relative path first
-        relative_givens_path = os.path.join(base_directory, file_name)
-        if os.path.exists(relative_givens_path):
-            givens_path = relative_givens_path
-        elif os.path.exists(file_name):  # Check the absolute path
-            givens_path = file_name
-        else:
-            print(f"No givens file found at {relative_givens_path} or {file_name}")
-            user_input = input("Please specify a givens file: ")
-            if os.path.exists(os.path.join(base_directory, user_input)):
-                givens_path = os.path.join(base_directory, user_input)
-            elif os.path.exists(user_input):
-                givens_path = user_input
-            else:
-                print(f"No givens file found at {user_input}. Aborting.")
-                return
+        givens_files_to_process = self._find_givens_files(file_name)
+        if not givens_files_to_process:
+            self.poutput("No givens files to load.")
+            return
 
-        cwd = os.getcwd()
-        navigator = DirectoryNavigator()
-        navigator.navigate_to_target()
-        os.chdir('..')
-        content, image_data = load_givens(givens_path)
-        os.chdir(cwd)
+        for givens_path in givens_files_to_process:
+            # The givens_path is absolute, and load_givens reconstructs absolute paths
+            # from the markdown file. No directory change is needed.
+            content, _ = load_givens(givens_path)
 
-        with open(self.chat_name, 'a', encoding='utf-8') as chat_file:
-            chat_file.write(content)
+            with open(self.chat_name, 'a', encoding='utf-8') as chat_file:
+                chat_file.write(content)
 
-        print(f"Loaded files listed and marked in {givens_path}")
+            self.poutput(f"Loaded files listed and marked in {givens_path}")
 
     @cmd2.with_category(CATEGORY_CHAT_CONTROL)
     def do_SEND(self, _):

ara_cli/commands/extract_command.py

@@ -3,15 +3,16 @@ from ara_cli.prompt_extractor import extract_responses
 import os
 
 class ExtractCommand(Command):
-    def __init__(self, file_name, skip_queries=False, output=None, error_output=None):
+    def __init__(self, file_name, force=False, write=False, output=None, error_output=None):
         self.file_name = file_name
-        self.skip_queries = skip_queries
+        self.force = force
+        self.write = write
         self.output = output    # Callable for standard output (optional)
         self.error_output = error_output  # Callable for errors (optional)
 
     def execute(self, *args, **kwargs):
         try:
-            extract_responses(self.file_name, True, skip_queries=self.skip_queries)
+            extract_responses(self.file_name, True, force=self.force, write=self.write)
             if self.output:
                 self.output("End of extraction")
         except Exception as e:

ara_cli/commands/read_command.py

@@ -0,0 +1,104 @@
+from ara_cli.commands.command import Command
+from ara_cli.artefact_reader import ArtefactReader
+from ara_cli.file_classifier import FileClassifier
+from ara_cli.list_filter import ListFilter, filter_list
+from ara_cli.artefact_models.artefact_data_retrieval import (
+    artefact_content_retrieval, 
+    artefact_path_retrieval, 
+    artefact_tags_retrieval
+)
+from ara_cli.artefact_fuzzy_search import suggest_close_name_matches
+import os
+
+
+class ReadCommand(Command):
+    def __init__(
+        self,
+        classifier: str,
+        artefact_name: str,
+        read_mode: str = "default",
+        list_filter: ListFilter = None,
+        output=None
+    ):
+        self.classifier = classifier
+        self.artefact_name = artefact_name
+        self.read_mode = read_mode
+        self.list_filter = list_filter or ListFilter()
+        self.output = output or print
+
+    def execute(self) -> bool:
+        """Execute the read command and return success status."""
+        file_classifier = FileClassifier(os)
+        classified_artefacts = ArtefactReader.read_artefacts()
+        artefacts = classified_artefacts.get(self.classifier, [])
+        all_artefact_names = [a.title for a in artefacts]
+
+        if self.artefact_name not in all_artefact_names:
+            suggest_close_name_matches(
+                self.artefact_name,
+                all_artefact_names
+            )
+            return False
+
+        target_artefact = next(filter(
+            lambda x: x.title == self.artefact_name, artefacts
+        ))
+
+        artefacts_by_classifier = {self.classifier: []}
+
+        try:
+            match self.read_mode:
+                case "branch":
+                    self._handle_branch_mode(
+                        classified_artefacts, artefacts_by_classifier
+                    )
+                case "children":
+                    artefacts_by_classifier = self._handle_children_mode(
+                        classified_artefacts
+                    )
+                case _:
+                    self._handle_default_mode(
+                        target_artefact, artefacts_by_classifier
+                    )
+
+            # Apply filtering and print results
+            filtered_artefacts = self._apply_filtering(artefacts_by_classifier)
+            file_classifier.print_classified_files(
+                filtered_artefacts, print_content=True
+            )
+            return True
+
+        except Exception as e:
+            self.output(f"Error reading artefact: {e}")
+            return False
+
+    def _handle_branch_mode(self, classified_artefacts, artefacts_by_classifier):
+        """Handle branch read mode."""
+        ArtefactReader.step_through_value_chain(
+            artefact_name=self.artefact_name,
+            classifier=self.classifier,
+            artefacts_by_classifier=artefacts_by_classifier,
+            classified_artefacts=classified_artefacts
+        )
+
+    def _handle_children_mode(self, classified_artefacts):
+        """Handle children read mode."""
+        return ArtefactReader.find_children(
+            artefact_name=self.artefact_name,
+            classifier=self.classifier,
+            classified_artefacts=classified_artefacts
+        )
+
+    def _handle_default_mode(self, target_artefact, artefacts_by_classifier):
+        """Handle default read mode."""
+        artefacts_by_classifier[self.classifier].append(target_artefact)
+
+    def _apply_filtering(self, artefacts_by_classifier):
+        """Apply list filtering to artefacts."""
+        return filter_list(
+            list_to_filter=artefacts_by_classifier,
+            list_filter=self.list_filter,
+            content_retrieval_strategy=artefact_content_retrieval,
+            file_path_retrieval=artefact_path_retrieval,
+            tag_retrieval=artefact_tags_retrieval
+        )

ara_cli/file_loaders/document_readers.py

@@ -0,0 +1,233 @@
+import os
+from abc import ABC, abstractmethod
+from typing import Tuple, Optional
+
+
+class DocumentReader(ABC):
+    """Abstract base class for document readers"""
+    
+    def __init__(self, file_path: str):
+        self.file_path = file_path
+        self.base_dir = os.path.dirname(file_path)
+    
+    @abstractmethod
+    def read(self, extract_images: bool = False) -> str:
+        """Read document and optionally extract images"""
+        pass
+    
+    def create_image_data_dir(self, extension_suffix: str) -> str:
+        """
+        Create data directory for images with file extension suffix to avoid conflicts.
+        
+        Returns:
+            str: Path to images directory
+        """
+        file_name_with_ext = os.path.splitext(os.path.basename(self.file_path))[0] + f"_{extension_suffix}"
+        data_dir = os.path.join(self.base_dir, f"{file_name_with_ext}.data")
+        images_dir = os.path.join(data_dir, "images")
+        if not os.path.exists(images_dir):
+            os.makedirs(images_dir)
+        return images_dir
+    
+    def save_and_describe_image(self, image_data: bytes, image_format: str, 
+                               save_dir: str, image_counter: int) -> Tuple[str, str]:
+        """
+        Save image data and get its description from LLM.
+        
+        Returns:
+            tuple: (relative_image_path, description)
+        """
+        from ara_cli.prompt_handler import describe_image
+        
+        # Save image
+        image_filename = f"{image_counter}.{image_format}"
+        image_path = os.path.join(save_dir, image_filename)
+        
+        with open(image_path, "wb") as image_file:
+            image_file.write(image_data)
+        
+        # Get image description from LLM
+        description = describe_image(image_path)
+        
+        # Get relative path
+        relative_image_path = os.path.relpath(image_path, self.base_dir)
+        
+        return relative_image_path, description
+
+
+class DocxReader(DocumentReader):
+    """Reader for DOCX files"""
+    
+    def read(self, extract_images: bool = False) -> str:
+        import docx
+        
+        doc = docx.Document(self.file_path)
+        text_content = '\n'.join(para.text for para in doc.paragraphs)
+        
+        if not extract_images:
+            return text_content
+        
+        from PIL import Image
+        import io
+        
+        # Create data directory for images
+        images_dir = self.create_image_data_dir("docx")
+        
+        # Extract and process images
+        image_descriptions = []
+        image_counter = 1
+        
+        for rel in doc.part.rels.values():
+            if "image" in rel.reltype:
+                image_data = rel.target_part.blob
+                
+                # Determine image format
+                image = Image.open(io.BytesIO(image_data))
+                image_format = image.format.lower()
+                
+                # Save and describe image
+                relative_path, description = self.save_and_describe_image(
+                    image_data, image_format, images_dir, image_counter
+                )
+                
+                # Add formatted description to list
+                image_description = f"\nImage: {relative_path}\n[{description}]\n"
+                image_descriptions.append(image_description)
+                
+                image_counter += 1
+        
+        # Combine text content with image descriptions
+        if image_descriptions:
+            text_content += "\n\n### Extracted Images\n" + "\n".join(image_descriptions)
+        
+        return text_content
+
+
+class PdfReader(DocumentReader):
+    """Reader for PDF files"""
+    
+    def read(self, extract_images: bool = False) -> str:
+        import pymupdf4llm
+        
+        if not extract_images:
+            return pymupdf4llm.to_markdown(self.file_path, write_images=False)
+        
+        import fitz  # PyMuPDF
+        
+        # Create images directory
+        images_dir = self.create_image_data_dir("pdf")
+        
+        # Extract text without images first
+        text_content = pymupdf4llm.to_markdown(self.file_path, write_images=False)
+        
+        # Extract and process images
+        doc = fitz.open(self.file_path)
+        image_descriptions = []
+        image_counter = 1
+        
+        for page_num, page in enumerate(doc):
+            image_list = page.get_images()
+            
+            for img_index, img in enumerate(image_list):
+                # Extract image
+                xref = img[0]
+                base_image = doc.extract_image(xref)
+                image_bytes = base_image["image"]
+                image_ext = base_image["ext"]
+                
+                # Save and describe image
+                relative_path, description = self.save_and_describe_image(
+                    image_bytes, image_ext, images_dir, image_counter
+                )
+                
+                # Add formatted description to list
+                image_description = f"\nImage: {relative_path}\n[{description}]\n"
+                image_descriptions.append(image_description)
+                
+                image_counter += 1
+        
+        doc.close()
+        
+        # Combine text content with image descriptions
+        if image_descriptions:
+            text_content += "\n\n### Extracted Images\n" + "\n".join(image_descriptions)
+        
+        return text_content
+
+
+class OdtReader(DocumentReader):
+    """Reader for ODT files"""
+    
+    def read(self, extract_images: bool = False) -> str:
+        import pymupdf4llm
+        
+        if not extract_images:
+            return pymupdf4llm.to_markdown(self.file_path, write_images=False)
+        
+        import zipfile
+        from PIL import Image
+        import io
+        
+        # Create data directory for images
+        images_dir = self.create_image_data_dir("odt")
+        
+        # Get text content
+        text_content = pymupdf4llm.to_markdown(self.file_path, write_images=False)
+        
+        # Extract and process images from ODT
+        image_descriptions = []
+        image_counter = 1
+        
+        try:
+            with zipfile.ZipFile(self.file_path, 'r') as odt_zip:
+                # List all files in the Pictures directory
+                picture_files = [f for f in odt_zip.namelist() if f.startswith('Pictures/')]
+                
+                for picture_file in picture_files:
+                    # Extract image data
+                    image_data = odt_zip.read(picture_file)
+                    
+                    # Determine image format
+                    image = Image.open(io.BytesIO(image_data))
+                    image_format = image.format.lower()
+                    
+                    # Save and describe image
+                    relative_path, description = self.save_and_describe_image(
+                        image_data, image_format, images_dir, image_counter
+                    )
+                    
+                    # Add formatted description to list
+                    image_description = f"\nImage: {relative_path}\n[{description}]\n"
+                    image_descriptions.append(image_description)
+                    
+                    image_counter += 1
+        except Exception as e:
+            print(f"Warning: Could not extract images from ODT: {e}")
+        
+        # Combine text content with image descriptions
+        if image_descriptions:
+            text_content += "\n\n### Extracted Images\n" + "\n".join(image_descriptions)
+        
+        return text_content
+
+
+class DocumentReaderFactory:
+    """Factory for creating appropriate document readers"""
+    
+    @staticmethod
+    def create_reader(file_path: str) -> Optional[DocumentReader]:
+        """Create appropriate reader based on file extension"""
+        _, ext = os.path.splitext(file_path)
+        ext = ext.lower()
+        
+        readers = {
+            '.docx': DocxReader,
+            '.pdf': PdfReader,
+            '.odt': OdtReader
+        }
+        
+        reader_class = readers.get(ext)
+        if reader_class:
+            return reader_class(file_path)
+        
+        return None

ara_cli/file_loaders/file_loaders.py

@@ -0,0 +1,123 @@
+import os
+import base64
+from abc import ABC, abstractmethod
+from typing import Optional
+from ara_cli.file_loaders.markdown_reader import MarkdownReader
+from ara_cli.file_loaders.document_readers import DocumentReaderFactory
+
+
+class FileLoader(ABC):
+    """Abstract base class for file loaders"""
+    
+    def __init__(self, chat_instance):
+        self.chat = chat_instance
+        
+    @abstractmethod
+    def load(self, file_path: str, **kwargs) -> bool:
+        """Load file with specific implementation"""
+        pass
+        
+    def add_prompt_tag_if_needed(self):
+        """Add prompt tag to chat if needed"""
+        self.chat.add_prompt_tag_if_needed(self.chat.chat_name)
+
+
+class TextFileLoader(FileLoader):
+    """Loads text files"""
+    
+    def load(self, file_path: str, prefix: str = "", suffix: str = "", 
+             block_delimiter: str = "", extract_images: bool = False) -> bool:
+        """Load text file with optional markdown image extraction"""
+        
+        is_md_file = file_path.lower().endswith('.md')
+        
+        if is_md_file and extract_images:
+            reader = MarkdownReader(file_path)
+            file_content = reader.read(extract_images=True)
+        else:
+            with open(file_path, 'r', encoding='utf-8', errors="replace") as file:
+                file_content = file.read()
+        
+        if block_delimiter:
+            file_content = f"{block_delimiter}\n{file_content}\n{block_delimiter}"
+        
+        write_content = f"{prefix}{file_content}{suffix}\n"
+        
+        with open(self.chat.chat_name, 'a', encoding='utf-8') as chat_file:
+            chat_file.write(write_content)
+            
+        return True
+
+
+class BinaryFileLoader(FileLoader):
+    """Loads binary files (images)"""
+    
+    def load(self, file_path: str, mime_type: str, prefix: str = "", suffix: str = "") -> bool:
+        """Load binary file as base64"""
+        
+        with open(file_path, 'rb') as file:
+            file_content = file.read()
+            
+        base64_image = base64.b64encode(file_content).decode("utf-8")
+        write_content = f"{prefix}![{os.path.basename(file_path)}](data:{mime_type};base64,{base64_image}){suffix}\n"
+        
+        with open(self.chat.chat_name, 'a', encoding='utf-8') as chat_file:
+            chat_file.write(write_content)
+            
+        return True
+
+
+class DocumentFileLoader(FileLoader):
+    """Loads document files (PDF, DOCX, ODT)"""
+    
+    def load(self, file_path: str, prefix: str = "", suffix: str = "", 
+             block_delimiter: str = "```", extract_images: bool = False) -> bool:
+        """Load document file with optional image extraction"""
+        
+        reader = DocumentReaderFactory.create_reader(file_path)
+        
+        if not reader:
+            print("Unsupported document type.")
+            return False
+        
+        text_content = reader.read(extract_images=extract_images)
+        
+        if block_delimiter:
+            text_content = f"{block_delimiter}\n{text_content}\n{block_delimiter}"
+        
+        write_content = f"{prefix}{text_content}{suffix}\n"
+        
+        with open(self.chat.chat_name, 'a', encoding='utf-8') as chat_file:
+            chat_file.write(write_content)
+            
+        return True
+
+
+class FileLoaderFactory:
+    """Factory for creating appropriate file loaders"""
+    
+    BINARY_TYPE_MAPPING = {
+        ".png": "image/png",
+        ".jpg": "image/jpeg",
+        ".jpeg": "image/jpeg",
+    }
+    
+    DOCUMENT_TYPE_EXTENSIONS = [".docx", ".doc", ".odt", ".pdf"]
+    
+    @staticmethod
+    def create_loader(file_name: str, chat_instance) -> Optional[FileLoader]:
+        """Create appropriate loader based on file type"""
+        
+        file_name_lower = file_name.lower()
+        
+        # Check if it's a binary file
+        for extension, mime_type in FileLoaderFactory.BINARY_TYPE_MAPPING.items():
+            if file_name_lower.endswith(extension):
+                return BinaryFileLoader(chat_instance)
+        
+        # Check if it's a document
+        if any(file_name_lower.endswith(ext) for ext in FileLoaderFactory.DOCUMENT_TYPE_EXTENSIONS):
+            return DocumentFileLoader(chat_instance)
+        
+        # Default to text file loader
+        return TextFileLoader(chat_instance)