ara-cli 0.1.9.77__py3-none-any.whl → 0.1.10.8__py3-none-any.whl

ara_cli/file_loaders/text_file_loader.py

@@ -0,0 +1,187 @@
+import os
+import re
+import base64
+import tempfile
+from typing import Optional, Tuple
+import requests
+from charset_normalizer import from_path
+from ara_cli.prompt_handler import describe_image
+from ara_cli.file_loaders.file_loader import FileLoader
+
+
+class TextFileLoader(FileLoader):
+    """Loads text files"""
+    def load(self, file_path: str, prefix: str = "", suffix: str = "",
+             block_delimiter: str = "", extract_images: bool = False, **kwargs) -> bool:
+        """Load text file with optional markdown image extraction"""
+
+        is_md_file = file_path.lower().endswith('.md')
+
+        if is_md_file and extract_images:
+            reader = MarkdownReader(file_path)
+            file_content = reader.read(extract_images=True).replace('\r\n', '\n')
+        else:
+            # Use charset-normalizer to detect encoding
+            encoded_content = from_path(file_path).best()
+            if not encoded_content:
+                print(f"Failed to detect encoding for {file_path}")
+                return False
+            file_content = str(encoded_content).replace('\r\n', '\n')
+
+        if block_delimiter:
+            file_content = f"{block_delimiter}\n{file_content}\n{block_delimiter}"
+
+        write_content = f"{prefix}{file_content}{suffix}\n"
+
+        with open(self.chat.chat_name, 'a', encoding='utf-8') as chat_file:
+            chat_file.write(write_content)
+
+        return True
+
+
+class MarkdownReader:
+    """Handles markdown file reading with optional image extraction"""
+
+    def __init__(self, file_path: str):
+        self.file_path = file_path
+        self.base_dir = os.path.dirname(file_path)
+        self.image_processor = ImageProcessor()
+
+    def read(self, extract_images: bool = False) -> str:
+        """Read markdown file and optionally extract/describe images"""
+        with open(self.file_path, 'r', encoding='utf-8') as file:
+            content = file.read()
+
+        if not extract_images:
+            return content
+
+        return self._process_images(content)
+
+    def _process_images(self, content: str) -> str:
+        """Process all images in markdown content"""
+        # Pattern to match markdown images: ![alt text](url or path)
+        image_pattern = re.compile(r'!\[([^\]]*)\]\(([^\)]+)\)')
+        base64_pattern = re.compile(r'data:image/([^;]+);base64,([^)]+)')
+
+        # Process each image reference
+        for match in image_pattern.finditer(content):
+            image_ref = match.group(2)
+            replacement = self._process_single_image(image_ref, base64_pattern)
+
+            if replacement:
+                content = content.replace(match.group(0), replacement, 1)
+
+        return content
+
+    def _process_single_image(self, image_ref: str, base64_pattern: re.Pattern) -> Optional[str]:
+        """Process a single image reference"""
+        try:
+            # Try base64 first
+            result = self.image_processor.process_base64_image(
+                image_ref, base64_pattern)
+            if result:
+                return result[0]
+
+            # Try URL
+            result, error = self.image_processor.process_url_image(image_ref)
+            if result:
+                if error:
+                    print(f"Warning: {error}")
+                return result
+
+            # Try local file
+            result, error = self.image_processor.process_local_image(
+                image_ref, self.base_dir)
+            if error:
+                print(f"Warning: {error}")
+            return result
+
+        except Exception as e:
+            print(f"Warning: Could not process image {image_ref}: {e}")
+            return None
+
+
+class ImageProcessor:
+    """Handles image processing operations"""
+
+    @staticmethod
+    def process_base64_image(
+        image_ref: str,
+        base64_pattern: re.Pattern
+    ) -> Optional[Tuple[str, str]]:
+        """Process base64 encoded image and return description"""
+        base64_match = base64_pattern.match(image_ref)
+        if not base64_match:
+            return None
+
+        image_format = base64_match.group(1)
+        base64_data = base64_match.group(2)
+        image_data = base64.b64decode(base64_data)
+
+        # Create a temporary file to send to LLM
+        with tempfile.NamedTemporaryFile(suffix=f'.{image_format}', delete=False) as tmp_file:
+            tmp_file.write(image_data)
+            tmp_file_path = tmp_file.name
+
+        try:
+            description = describe_image(tmp_file_path)
+            return f"Image: (base64 embedded {image_format} image)\n[{description}]", None
+        finally:
+            os.unlink(tmp_file_path)
+
+    @staticmethod
+    def process_url_image(image_ref: str) -> Tuple[str, Optional[str]]:
+        """Process image from URL and return description"""
+        if not image_ref.startswith(('http://', 'https://')):
+            return "", None
+
+        try:
+            response = requests.get(image_ref, timeout=10)
+            response.raise_for_status()
+
+            # Determine file extension from content-type
+            content_type = response.headers.get('content-type', '')
+            ext = ImageProcessor._get_extension_from_content_type(
+                content_type, image_ref)
+
+            # Create temporary file
+            with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp_file:
+                tmp_file.write(response.content)
+                tmp_file_path = tmp_file.name
+
+            try:
+                description = describe_image(tmp_file_path)
+                return f"Image: {image_ref}\n[{description}]", None
+            finally:
+                os.unlink(tmp_file_path)
+
+        except Exception as e:
+            error_msg = f"Could not download image: {str(e)}"
+            return f"Image: {image_ref}\n[{error_msg}]", error_msg
+
+    @staticmethod
+    def process_local_image(image_ref: str, base_dir: str) -> Tuple[str, Optional[str]]:
+        """Process local image file and return description"""
+        if os.path.isabs(image_ref):
+            local_image_path = image_ref
+        else:
+            local_image_path = os.path.join(base_dir, image_ref)
+
+        if os.path.exists(local_image_path):
+            description = describe_image(local_image_path)
+            return f"Image: {image_ref}\n[{description}]", None
+        else:
+            error_msg = f"Image file not found"
+            return f"Image: {image_ref}\n[{error_msg}]", f"Local image not found: {local_image_path}"
+
+    @staticmethod
+    def _get_extension_from_content_type(content_type: str, url: str) -> str:
+        """Determine file extension from content type or URL"""
+        if 'image/jpeg' in content_type:
+            return '.jpg'
+        elif 'image/png' in content_type:
+            return '.png'
+        elif 'image/gif' in content_type:
+            return '.gif'
+        else:
+            return os.path.splitext(url)[1] or '.png'

ara_cli/global_file_lister.py

@@ -0,0 +1,51 @@
+import os
+import fnmatch
+from typing import List, Dict, Any
+
+DirTree = Dict[str, Any]
+
+def _build_tree(root_path: str, patterns: List[str]) -> DirTree:
+    """Creates a nested dictionary representing the directory structure in the specified path."""
+    tree: DirTree = {'files': [], 'dirs': {}}
+    try:
+        for item in os.listdir(root_path):
+            item_path = os.path.join(root_path, item)
+            if os.path.isdir(item_path):
+                subtree = _build_tree(item_path, patterns)
+                if subtree['files'] or subtree['dirs']:
+                    tree['dirs'][item] = subtree
+            elif os.path.isfile(item_path):
+                if any(fnmatch.fnmatch(item, pattern) for pattern in patterns):
+                    tree['files'].append(item)
+    except OSError as e:
+        print(f"Warning: Could not access path {root_path}: {e}")
+    return tree
+
+def _write_tree_to_markdown(md_file, tree: DirTree, level: int):
+    """Writes the tree data structure to the file in markdown format."""
+    indent = '    ' * level
+    for filename in sorted(tree['files']):
+        md_file.write(f"{indent}- [] {filename}\n")
+    
+    for dirname, subtree in sorted(tree['dirs'].items()):
+        md_file.write(f"{'    ' * (level -1)}{'#' * (level + 1)} {dirname}\n")
+        _write_tree_to_markdown(md_file, subtree, level + 1)
+
+def generate_global_markdown_listing(directories: List[str], file_patterns: List[str], output_file: str):
+    """Creates a hierarchical list of markdown files for global directories. Uses the absolute path as the top heading and relative names for children."""
+    with open(output_file, 'w', encoding='utf-8') as md_file:
+        for directory in directories:
+            abs_dir = os.path.abspath(directory)
+            
+            if not os.path.isdir(abs_dir):
+                print(f"Warning: Global directory not found: {abs_dir}")
+                md_file.write(f"# {directory}\n")
+                md_file.write(f"    - !! Warning: Global directory not found: {abs_dir}\n\n")
+                continue
+
+            tree = _build_tree(abs_dir, file_patterns)
+
+            if tree['files'] or tree['dirs']:
+                md_file.write(f"# {abs_dir}\n")
+                _write_tree_to_markdown(md_file, tree, 1)
+                md_file.write("\n")

ara_cli/prompt_extractor.py

@@ -1,101 +1,207 @@
+import re
+import json
+import os
+import json_repair
+from markdown_it import MarkdownIt
 from ara_cli.prompt_handler import send_prompt, get_file_content
 from ara_cli.classifier import Classifier
 from ara_cli.directory_navigator import DirectoryNavigator
 from ara_cli.artefact_models.artefact_mapping import title_prefix_to_artefact_class
-import re
-import json
-import json_repair
-import os
 
-from markdown_it import MarkdownIt
 
+def _find_extract_token(tokens):
+    """Find the first token that needs to be processed."""
+    for token in tokens:
+        if token.type == 'fence' and token.content.strip().startswith("# [x] extract"):
+            return token
+    return None
 
-def extract_code_blocks_md(markdown_text):
-    md = MarkdownIt()
-    tokens = md.parse(markdown_text)
-    code_blocks = [token.content for token in tokens if token.type == 'fence']
-    return code_blocks
 
+def _extract_file_path(content_lines):
+    """Extract file path from content lines."""
+    if not content_lines:
+        return None
+    file_path_search = re.search(r"# filename: (.+)", content_lines[0])
+    return file_path_search.group(1).strip() if file_path_search else None
 
-def extract_responses(document_path, relative_to_ara_root=False):
-    print(f"Debug: Starting extraction from {document_path}")
-    block_extraction_counter = 0
 
-    with open(document_path, 'r', encoding='utf-8') as file:
-        content = file.read()
+def _find_artefact_class(content_lines):
+    """Find the appropriate artefact class from content lines."""
+    for line in content_lines[:2]:
+        words = line.strip().split(' ')
+        if not words:
+            continue
+        first_word = words[0]
+        if first_word in title_prefix_to_artefact_class:
+            return title_prefix_to_artefact_class[first_word]
+    return None
 
-    cwd = os.getcwd()
-    if relative_to_ara_root:
-        from ara_cli.directory_navigator import DirectoryNavigator
-        navigator = DirectoryNavigator()
-        navigator.navigate_to_target()
-        os.chdir('..')
 
-    code_blocks_found = extract_code_blocks_md(content)
-    updated_content = content
+def _process_file_extraction(file_path, code_content, force, write):
+    """Process file extraction logic."""
+    print(f"Filename extracted: {file_path}")
+    handle_existing_file(file_path, code_content, force, write)
 
-    for block in code_blocks_found:
-        block_lines = block.split('\n')
 
-        if "# [x] extract" not in block_lines[0]:
-            continue
-        print("Block found and processed.")
+def _process_artefact_extraction(artefact_class, content_lines, force, write):
+    """Process artefact extraction logic."""
+    artefact = artefact_class.deserialize('\n'.join(content_lines))
+    serialized_artefact = artefact.serialize()
+
+    original_directory = os.getcwd()
+    directory_navigator = DirectoryNavigator()
+    directory_navigator.navigate_to_target()
 
-        block_lines = block_lines[1:]
+    artefact_path = artefact.file_path
+    directory = os.path.dirname(artefact_path)
+    os.makedirs(directory, exist_ok=True)
+    handle_existing_file(artefact_path, serialized_artefact, force, write)
 
-        file_path_search = re.search(r"# filename: (.+)", block_lines[0])
+    os.chdir(original_directory)
 
-        if file_path_search:
-            file_path = file_path_search.group(1).strip()
-            print(f"Filename extracted: {file_path}")
 
-            block_lines = block_lines[1:]  # Remove first line again after removing filename line
-            block = '\n'.join(block_lines)
+def _perform_extraction_for_block(source_lines, block_start, block_end, force, write):
+    """Helper function to process a single, identified block."""
+    original_block_text = '\n'.join(source_lines[block_start:block_end + 1])
+    block_content_lines = source_lines[block_start + 1:block_end]
+    block_content = '\n'.join(block_content_lines)
 
-            handle_existing_file(file_path, block)
-            block_extraction_counter += 1
+    block_lines = block_content.split('\n')
+    content_lines_after_extract = block_lines[1:]
 
-            # Update the markdown content
-            updated_content = update_markdown(content, block, file_path)
+    file_path = _extract_file_path(content_lines_after_extract)
+
+    if file_path:
+        code_content = '\n'.join(content_lines_after_extract[1:])
+        _process_file_extraction(file_path, code_content, force, write)
+    else:
+        artefact_class = _find_artefact_class(content_lines_after_extract)
+        if artefact_class:
+            _process_artefact_extraction(artefact_class, content_lines_after_extract, force, write)
         else:
-            # Extract artefact
-            artefact_class = None
-            for line in block_lines[:2]:
-                words = line.strip().split(' ')
-                if not words:
-                    continue
-                first_word = words[0]
-                if first_word not in title_prefix_to_artefact_class:
-                    continue
-                artefact_class = title_prefix_to_artefact_class[first_word]
-            if not artefact_class:
-                print("No filename found, skipping this block.")
+            print("No filename or valid artefact found, skipping processing for this block.")
+            return None, None
+
+    modified_block_text = original_block_text.replace("# [x] extract", "# [v] extract", 1)
+    return original_block_text, modified_block_text
+
+
+class FenceDetector:
+    """Helper class to detect and match fence blocks."""
+    
+    def __init__(self, source_lines):
+        self.source_lines = source_lines
+    
+    def is_extract_fence(self, line_num):
+        """Check if line is a fence with extract marker."""
+        line = self.source_lines[line_num]
+        stripped_line = line.strip()
+        
+        is_fence = stripped_line.startswith('```') or stripped_line.startswith('~~~')
+        if not is_fence:
+            return False
+            
+        if not (line_num + 1 < len(self.source_lines)):
+            return False
+            
+        return self.source_lines[line_num + 1].strip().startswith("# [x] extract")
+    
+    def find_matching_fence_end(self, start_line):
+        """Find the matching end fence for a given start fence."""
+        fence_line = self.source_lines[start_line]
+        indentation = len(fence_line) - len(fence_line.lstrip())
+        stripped_fence_line = fence_line.strip()
+        fence_char = stripped_fence_line[0]
+        fence_length = len(stripped_fence_line) - len(stripped_fence_line.lstrip(fence_char))
+
+        for i in range(start_line + 1, len(self.source_lines)):
+            scan_line = self.source_lines[i]
+            stripped_scan_line = scan_line.strip()
+
+            if not stripped_scan_line or stripped_scan_line[0] != fence_char:
+                continue
+            
+            if not all(c == fence_char for c in stripped_scan_line):
                 continue
-            artefact = artefact_class.deserialize('\n'.join(block_lines))
-            serialized_artefact = artefact.serialize()
 
-            original_directory = os.getcwd()
-            directory_navigator = DirectoryNavigator()
-            directory_navigator.navigate_to_target()
+            candidate_indentation = len(scan_line) - len(scan_line.lstrip())
+            candidate_length = len(stripped_scan_line)
+            
+            if candidate_length == fence_length and candidate_indentation == indentation:
+                return i
+        
+        return -1
+
+
+def _process_document_blocks(source_lines, force, write):
+    """Process all extract blocks in the document."""
+    fence_detector = FenceDetector(source_lines)
+    replacements = []
+    line_num = 0
+    
+    while line_num < len(source_lines):
+        if not fence_detector.is_extract_fence(line_num):
+            line_num += 1
+            continue
 
-            artefact_path = artefact.file_path
-            directory = os.path.dirname(artefact_path)
-            os.makedirs(directory, exist_ok=True)
-            with open(artefact_path, 'w', encoding='utf-8') as file:
-                file.write(serialized_artefact)
+        block_start_line = line_num
+        block_end_line = fence_detector.find_matching_fence_end(block_start_line)
+        
+        if block_end_line != -1:
+            print(f"Block found and processed starting on line {block_start_line + 1}.")
+            original, modified = _perform_extraction_for_block(
+                source_lines, block_start_line, block_end_line, force, write
+            )
+            if original and modified:
+                replacements.append((original, modified))
+            line_num = block_end_line + 1
+        else:
+            line_num += 1
+    
+    return replacements
 
-            os.chdir(original_directory)
 
-            # TODO: make update_markdown work block by block instead of updating the whole document at once
-            block_extraction_counter += 1
-            updated_content = update_markdown(content, block, None)
+def _apply_replacements(content, replacements):
+    """Apply all replacements to the content."""
+    updated_content = content
+    for original, modified in replacements:
+        updated_content = updated_content.replace(original, modified, 1)
+    return updated_content
+
+
+def _setup_working_directory(relative_to_ara_root):
+    """Setup working directory and return original cwd."""
+    cwd = os.getcwd()
+    if relative_to_ara_root:
+        navigator = DirectoryNavigator()
+        navigator.navigate_to_target()
+        os.chdir('..')
+    return cwd
+
+
+def extract_responses(document_path, relative_to_ara_root=False, force=False, write=False):
+    print(f"Starting extraction from '{document_path}'")
+    
+    try:
+        with open(document_path, 'r', encoding='utf-8', errors='replace') as file:
+            content = file.read()
+    except FileNotFoundError:
+        print(f"Error: File not found at '{document_path}'. Skipping extraction.")
+        return
+
+    cwd = _setup_working_directory(relative_to_ara_root)
+    
+    source_lines = content.split('\n')
+    replacements = _process_document_blocks(source_lines, force, write)
+    
+    updated_content = _apply_replacements(content, replacements)
 
     os.chdir(cwd)
-    # Save the updated markdown content
     with open(document_path, 'w', encoding='utf-8') as file:
         file.write(updated_content)
 
-    print(f"End of extraction. Found {block_extraction_counter} blocks.")
+    if replacements:
+        print(f"End of extraction. Found and processed {len(replacements)} blocks in '{os.path.basename(document_path)}'.")
 
 
 def modify_and_save_file(response, file_path):
@@ -116,7 +222,7 @@ def modify_and_save_file(response, file_path):
                 print("Skipping block.")
                 return
 
-        with open(file_path, 'w', encoding='utf-8') as file:
+        with open(file_path, 'w', encoding='utf-8', errors='replace') as file:
             file.write(response_data['content'])
             print(f"File {file_path} updated successfully.")
     except json.JSONDecodeError as ex:
@@ -127,14 +233,23 @@ def prompt_user_decision(prompt):
     return input(prompt)
 
 
-def create_file_if_not_exist(filename, content):
+def determine_should_create(skip_query=False):
+    if skip_query:
+        return True
+    user_decision = prompt_user_decision("File does not exist. Create? (y/n): ")
+    if user_decision.lower() in ['y', 'yes']:
+        return True
+    return False
+
+
+def create_file_if_not_exist(filename, content, skip_query=False):
     try:
         if not os.path.exists(filename):
-            user_decision = prompt_user_decision("File does not exist. Create? (y/n): ")
-
-            if user_decision.lower() in ['y', 'yes']:
+            if determine_should_create(skip_query):
                 # Ensure the directory exists
-                os.makedirs(os.path.dirname(filename), exist_ok=True)
+                dir_name = os.path.dirname(filename)
+                if dir_name:
+                    os.makedirs(dir_name, exist_ok=True)
 
                 with open(filename, 'w', encoding='utf-8') as file:
                     file.write(content)
@@ -170,7 +285,7 @@ def create_prompt_for_file_modification(content_str, filename):
     {{
         "filename": "path/filename.filextension",
         "content":  "full content of the modified file in valid json format"
-    }} 
+    }}
     """
 
     # print(f"Debug: modification prompt created: {prompt_text}")
@@ -178,34 +293,46 @@ def create_prompt_for_file_modification(content_str, filename):
     return prompt_text
 
 
-def handle_existing_file(filename, block_content):
+def handle_existing_file(filename, block_content, skip_query=False, write=False):
     if not os.path.isfile(filename):
         print(f"File {filename} does not exist, attempting to create")
-        create_file_if_not_exist(filename, block_content)
+        # Ensure directory exists before writing
+        directory = os.path.dirname(filename)
+        if directory:
+            os.makedirs(directory, exist_ok=True)
+        create_file_if_not_exist(filename, block_content, skip_query)
+
+    elif write:
+        print(f"File {filename} exists. Overwriting without LLM merge as requested.")
+        try:
+            directory = os.path.dirname(filename)
+            if directory:
+                os.makedirs(directory, exist_ok=True)
+            with open(filename, 'w', encoding='utf-8', errors='replace') as file:
+                file.write(block_content)
+            print(f"File {filename} overwritten successfully.")
+        except OSError as e:
+            print(f"Error: {e}")
+            print(f"Failed to overwrite file {filename} due to an OS error")
     else:
         print(f"File {filename} exists, creating modification prompt")
         prompt_text = create_prompt_for_file_modification(block_content, filename)
+        if prompt_text is None:
+            return
+
         messages = [{"role": "user", "content": prompt_text}]
         response = ""
 
-        for chunk in send_prompt(messages):
+        for chunk in send_prompt(messages, purpose='extraction'):
             content = chunk.choices[0].delta.content
             if content:
                 response += content
         modify_and_save_file(response, filename)
 
 
-def extract_and_save_prompt_results(classifier, param):
+def extract_and_save_prompt_results(classifier, param, write=False):
     sub_directory = Classifier.get_sub_directory(classifier)
     prompt_log_file = f"ara/{sub_directory}/{param}.data/{classifier}.prompt_log.md"
     print(f"Extract marked sections from: {prompt_log_file}")
 
-    extract_responses(prompt_log_file)
-
-
-def update_markdown(original_content, block_content, filename):
-    """
-    Update the markdown content by changing the extract block from "# [x] extract" to "# [v] extract"
-    """
-    updated_content = original_content.replace("# [x] extract", "# [v] extract")
-    return updated_content
+    extract_responses(prompt_log_file, write=write)