attachments 0.2.4__tar.gz → 0.3.0__tar.gz

{attachments-0.2.4/src/attachments.egg-info → attachments-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: attachments
-Version: 0.2.4
+Version: 0.3.0
 Summary: A Python library to handle various file types for LLMs.
 Author-email: Maxim Rivest <mrive052@gmail.com>
 License-Expression: MIT
@@ -35,13 +35,16 @@ Requires-Dist: flake8; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
 Requires-Dist: black; extra == "dev"
 Requires-Dist: filelock; extra == "dev"
+Requires-Dist: mystmd; extra == "dev"
 Dynamic: license-file
 
-# Attachments
+# Attachments: The Python Library for Effortless LLM Context
 ## Easiest way to give context to LLMs
 
 Attachments has the ambition to be the general funnel for any files to be transformed into images+text for large language models context. For this to be true we, has a community, have to stop implementing this funnel (file -> images+text) each on our own. If you have a detector + parser + renderer contibute it! This will help you and us. Don't let your code alone in a dark corner of you computer.
 
+From a `path` to a `string` and a `base64` image encoded string. That is the goal.
+
 ## Very quickstart
 
 ```bash
@@ -76,15 +79,11 @@ from attachments import Attachments
 
 pdf_attachment = Attachments("https://github.com/microsoft/markitdown/raw/refs/heads/main/packages/markitdown/tests/test_files/test.pdf")
 
-prompt = f"""
-Analyze the following documents:
-{pdf_attachment}
-"""
-
-content = [{"type": "input_image","image_url": image} for image in pdf_attachment.images] + \
-          [{"type": "input_text", "text": prompt}]
-
-response = OpenAI().responses.create(model="gpt-4.1-nano", input=[{"role": "user", "content": content}])
+response = OpenAI().responses.create(
+    model="gpt-4.1-nano", 
+    input=[{
+        "role": "user",
+        "content": pdf_attachment.to_openai_content("Analyze the following documents:")}])
 response.output_text
 ```
 
@@ -97,27 +96,10 @@ from attachments import Attachments
 pptx_file = Attachments(
     "https://github.com/microsoft/markitdown/raw/refs/heads/main/packages/markitdown/tests/test_files/test.pptx")
 
-prompt = f"""
-Analyze the following documents:
-{pptx_file}
-"""
-
-content = [
-    {
-        "type": "image",
-        "source": {
-            "type": "base64",
-            "media_type": image.split(";")[0].split(":")[1], # this is ugly but it works
-            "data": image.split(",")[1] # this is ugly but it works
-        }
-    }
-    for image in pptx_file.images
-] + [{"type": "text", "text": prompt}]
-
 message = anthropic.Anthropic().messages.create(
     max_tokens=8192,
     model="claude-3-5-haiku-20241022",
-    messages=[{"role": "user", "content": content}])
+    messages=[{"role": "user", "content": pptx_file.to_claude_content("Analyze the following documents:")}])
 print(message.content)
 ```
 
@@ -126,6 +108,7 @@ print(message.content)
 I am not a fan of the object that langchain and anthropic make you create, but hey!, I don't make the rules...
 
 ```python
+#pip install -U langchain-anthropic
 from langchain.chat_models import init_chat_model
 from attachments import Attachments
 
@@ -134,21 +117,7 @@ docx_file = Attachments("https://github.com/microsoft/markitdown/raw/refs/heads/
 llm = init_chat_model("anthropic:claude-3-5-sonnet-latest")
 
 # Creation of a list of object as expected by langchain/anthropic
-content = [{"type": "text", "text": f"Analyze the following documents: {docx_file}"}]
-
-for image in docx_file.images
-    media_type_i = image.split(";")[0].split(":")[1]
-    media_data_i = image.split(",")[1]
-    content.append({
-        "type": "image",
-        "source": {
-            "type": "base64",
-            "media_type": media_type_i, 
-            "data": media_data_i 
-        }
-    })
-
-
+content = docx_file.to_claude_content("Analyze the following documents:")
 message = {
     "role": "user",
     "content": content,

{attachments-0.2.4 → attachments-0.3.0}/README.md

@@ -1,8 +1,10 @@
-# Attachments
+# Attachments: The Python Library for Effortless LLM Context
 ## Easiest way to give context to LLMs
 
 Attachments has the ambition to be the general funnel for any files to be transformed into images+text for large language models context. For this to be true we, has a community, have to stop implementing this funnel (file -> images+text) each on our own. If you have a detector + parser + renderer contibute it! This will help you and us. Don't let your code alone in a dark corner of you computer.
 
+From a `path` to a `string` and a `base64` image encoded string. That is the goal.
+
 ## Very quickstart
 
 ```bash
@@ -37,15 +39,11 @@ from attachments import Attachments
 
 pdf_attachment = Attachments("https://github.com/microsoft/markitdown/raw/refs/heads/main/packages/markitdown/tests/test_files/test.pdf")
 
-prompt = f"""
-Analyze the following documents:
-{pdf_attachment}
-"""
-
-content = [{"type": "input_image","image_url": image} for image in pdf_attachment.images] + \
-          [{"type": "input_text", "text": prompt}]
-
-response = OpenAI().responses.create(model="gpt-4.1-nano", input=[{"role": "user", "content": content}])
+response = OpenAI().responses.create(
+    model="gpt-4.1-nano", 
+    input=[{
+        "role": "user",
+        "content": pdf_attachment.to_openai_content("Analyze the following documents:")}])
 response.output_text
 ```
 
@@ -58,27 +56,10 @@ from attachments import Attachments
 pptx_file = Attachments(
     "https://github.com/microsoft/markitdown/raw/refs/heads/main/packages/markitdown/tests/test_files/test.pptx")
 
-prompt = f"""
-Analyze the following documents:
-{pptx_file}
-"""
-
-content = [
-    {
-        "type": "image",
-        "source": {
-            "type": "base64",
-            "media_type": image.split(";")[0].split(":")[1], # this is ugly but it works
-            "data": image.split(",")[1] # this is ugly but it works
-        }
-    }
-    for image in pptx_file.images
-] + [{"type": "text", "text": prompt}]
-
 message = anthropic.Anthropic().messages.create(
     max_tokens=8192,
     model="claude-3-5-haiku-20241022",
-    messages=[{"role": "user", "content": content}])
+    messages=[{"role": "user", "content": pptx_file.to_claude_content("Analyze the following documents:")}])
 print(message.content)
 ```
 
@@ -87,6 +68,7 @@ print(message.content)
 I am not a fan of the object that langchain and anthropic make you create, but hey!, I don't make the rules...
 
 ```python
+#pip install -U langchain-anthropic
 from langchain.chat_models import init_chat_model
 from attachments import Attachments
 
@@ -95,21 +77,7 @@ docx_file = Attachments("https://github.com/microsoft/markitdown/raw/refs/heads/
 llm = init_chat_model("anthropic:claude-3-5-sonnet-latest")
 
 # Creation of a list of object as expected by langchain/anthropic
-content = [{"type": "text", "text": f"Analyze the following documents: {docx_file}"}]
-
-for image in docx_file.images
-    media_type_i = image.split(";")[0].split(":")[1]
-    media_data_i = image.split(",")[1]
-    content.append({
-        "type": "image",
-        "source": {
-            "type": "base64",
-            "media_type": media_type_i, 
-            "data": media_data_i 
-        }
-    })
-
-
+content = docx_file.to_claude_content("Analyze the following documents:")
 message = {
     "role": "user",
     "content": content,

{attachments-0.2.4 → attachments-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "attachments"
-version = "0.2.4"
+version = "0.3.0"
 authors = [
   { name="Maxim Rivest", email="mrive052@gmail.com" },
 ]
@@ -53,6 +53,7 @@ dev = [
     "mypy",
     "black",
     "filelock",
+    "mystmd",
 ]
 
 [tool.setuptools]

{attachments-0.2.4 → attachments-0.3.0}/src/attachments/core.py

@@ -11,7 +11,7 @@ from typing import List, Dict, Tuple, Optional, Any, Union, cast
 from collections import defaultdict
 
 from .detectors import Detector
-from .parsers import ParserRegistry, PDFParser, PPTXParser, HTMLParser, ImageParser, AudioParser, DOCXParser, ODTParser
+from .parsers import ParserRegistry, PDFParser, PPTXParser, HTMLParser, ImageParser, AudioParser, DOCXParser, ODTParser, TextParser
 from .renderers import RendererRegistry, DefaultXMLRenderer, PlainTextRenderer
 from .exceptions import DetectionError, ParsingError
 from .image_processing import DEFAULT_IMAGE_OUTPUT_FORMAT, DEFAULT_IMAGE_QUALITY
@@ -73,6 +73,7 @@ class Attachments:
         
         self.parser_registry.register('docx', DOCXParser)
         self.parser_registry.register('odt', ODTParser)
+        self.parser_registry.register('txt', TextParser) # Register TextParser
         
         # Register ImageParser class for various image types
         # The registry will create an instance for each, configured with self._config
@@ -188,14 +189,42 @@ class Attachments:
                 
                 file_type = self.detector.detect(path_for_detector_and_parser, content_type=detected_file_type_arg)
                 
-                if not file_type:
-                    print(f"Warning: Could not detect file type for '{path_for_detector_and_parser}' (from input '{path_str}'). Skipping.")
-                    continue
+                parser = None
+                original_detected_type = file_type # Keep track of initially detected type
+
+                if file_type:
+                    try:
+                        parser = self.parser_registry.get_parser(file_type)
+                    except KeyError:
+                        if self.verbose:
+                            print(f"Warning: No parser registered for detected type '{file_type}' for '{path_for_detector_and_parser}' (from input '{path_str}'). Attempting to parse as plain text.")
+                        file_type = 'txt' # Fallback to txt
+                else:
+                    if self.verbose:
+                        print(f"Warning: Could not detect file type for '{path_for_detector_and_parser}' (from input '{path_str}'). Attempting to parse as plain text.")
+                    file_type = 'txt' # Fallback to txt
 
-                parser = self.parser_registry.get_parser(file_type)
+                if not parser: # If parser is still None, it means we are in a fallback scenario or initial detection failed
+                    try:
+                        parser = self.parser_registry.get_parser(file_type) # This should get TextParser if file_type is 'txt'
+                    except KeyError:
+                        # This should ideally not happen if TextParser is registered, but as a safeguard:
+                        print(f"Critical Warning: Fallback TextParser not found or not registered for type '{file_type}'. Cannot process '{path_for_detector_and_parser}'. Skipping.")
+                        # Clean up temp file if it was created for a URL
+                        if is_url and temp_file_path_for_parsing and os.path.exists(temp_file_path_for_parsing):
+                            try:
+                                os.remove(temp_file_path_for_parsing)
+                            except Exception as e_clean_critical:
+                                print(f"Warning: Could not clean up temp file {temp_file_path_for_parsing} during critical skip: {e_clean_critical}")
+                        continue # Skip this file
+                
+                # If file_type was overridden for fallback, ensure the parsed_content reflects that.
+                # However, we might want to store the original_detected_type if it existed.
                 parsed_content = parser.parse(path_for_detector_and_parser, indices=indices)
                 
-                parsed_content['type'] = file_type
+                parsed_content['type'] = file_type # Use the final file_type (could be 'txt')
+                if original_detected_type and original_detected_type != file_type:
+                    parsed_content['original_detected_type'] = original_detected_type
                 parsed_content['id'] = f"{file_type}{i+1}" 
                 parsed_content['original_path_str'] = path_str 
                 parsed_content['file_path'] = original_file_path_or_url 
@@ -278,7 +307,7 @@ class Attachments:
                         'm4a': 'audio/m4a', # mimetypes might say audio/mp4, which is also fine for .m4a
                         'mp4_audio': 'audio/mp4',
                         'wav': 'audio/wav', 
-                        'flac': 'audio/flac', 
+                        'flac': 'audio/flac',
                         'ogg_audio': 'audio/ogg',
                         'oga': 'audio/ogg', 
                         'webm_audio': 'audio/webm', 
@@ -634,6 +663,59 @@ class Attachments:
     def another_method_or_end_of_class_placeholder(self):
         pass
 
+    def to_openai_content(self, prompt: str) -> List[Dict[str, Any]]:
+        """
+        Formats attachments and a prompt for the OpenAI API.
+        Currently supports images.
+
+        Args:
+            prompt: The text prompt to include.
+
+        Returns:
+            A list of dictionaries formatted for the OpenAI API,
+            combining image data and the text prompt.
+        """
+        content = [{"type": "input_image", "image_url": image_data_uri} for image_data_uri in self.images]
+        content.append({"type": "input_text", "text": prompt + "\n\n" + self.__str__()})
+        return content
+
+    def to_claude_content(self, prompt: str) -> List[Dict[str, Any]]:
+        """
+        Formats attachments and a prompt for the Anthropic Claude API.
+        Currently supports images.
+
+        Args:
+            prompt: The text prompt to include.
+
+        Returns:
+            A list of dictionaries formatted for the Anthropic Claude API,
+            combining image data and the text prompt.
+        """
+        content = []
+        for image_data_uri in self.images:
+            try:
+                # Example data URI: "data:image/jpeg;base64,BASE64_STRING"
+                header, base64_data = image_data_uri.split(',', 1)
+                media_type = header.split(';')[0].split(':')[1]
+                
+                content.append({
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": media_type,
+                        "data": base64_data
+                    }
+                })
+            except ValueError as e:
+                # Handle cases where the data URI might not be as expected, though self.images should provide valid ones.
+                if self.verbose:
+                    print(f"Warning: Could not parse image data URI for Claude content: {image_data_uri}. Error: {e}")
+                # Optionally, append an error or skip this image
+                continue # Skip malformed URIs for now
+        
+        content.append({"type": "text", "text": prompt})
+        return content
+
     # Potentially other methods follow, or end of class / file 
 
     def _format_operations_for_markdown(self, operations: Dict[str, Any]) -> str:

{attachments-0.2.4 → attachments-0.3.0}/src/attachments/parsers.py

@@ -150,6 +150,27 @@ class PPTXParser(BaseParser):
         except Exception as e:
             raise ParsingError(message=f"Failed to parse PPTX '{file_path}': {e}", file_path=file_path, parser_name=self.__class__.__name__) from e
 
+class TextParser(BaseParser):
+    """Parses plain text files."""
+    def parse(self, file_path: str, indices: Optional[str] = None) -> Dict[str, Any]:
+        """Reads the file content as UTF-8 text.
+        `indices` argument is ignored for text files.
+        """
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                text_content = f.read()
+            return {
+                "text": text_content,
+                "original_basename": os.path.basename(file_path),
+                "file_path": file_path,
+            }
+        except FileNotFoundError:
+            raise ParsingError(message=f"Text file not found: {file_path}", file_path=file_path, parser_name=self.__class__.__name__) from None
+        except UnicodeDecodeError as e:
+            raise ParsingError(message=f"Error decoding text file '{file_path}' as UTF-8: {e}. The file might not be plain text or uses a different encoding.", file_path=file_path, parser_name=self.__class__.__name__) from e
+        except Exception as e:
+            raise ParsingError(message=f"Failed to parse text file '{file_path}': {e}", file_path=file_path, parser_name=self.__class__.__name__) from e
+
 # ImageParser and AudioParser __init__ are fine from previous step.
 # Their full parse methods will be restored/verified after document parsers.
 class ImageParser(BaseParser):

{attachments-0.2.4 → attachments-0.3.0/src/attachments.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: attachments
-Version: 0.2.4
+Version: 0.3.0
 Summary: A Python library to handle various file types for LLMs.
 Author-email: Maxim Rivest <mrive052@gmail.com>
 License-Expression: MIT
@@ -35,13 +35,16 @@ Requires-Dist: flake8; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
 Requires-Dist: black; extra == "dev"
 Requires-Dist: filelock; extra == "dev"
+Requires-Dist: mystmd; extra == "dev"
 Dynamic: license-file
 
-# Attachments
+# Attachments: The Python Library for Effortless LLM Context
 ## Easiest way to give context to LLMs
 
 Attachments has the ambition to be the general funnel for any files to be transformed into images+text for large language models context. For this to be true we, has a community, have to stop implementing this funnel (file -> images+text) each on our own. If you have a detector + parser + renderer contibute it! This will help you and us. Don't let your code alone in a dark corner of you computer.
 
+From a `path` to a `string` and a `base64` image encoded string. That is the goal.
+
 ## Very quickstart
 
 ```bash
@@ -76,15 +79,11 @@ from attachments import Attachments
 
 pdf_attachment = Attachments("https://github.com/microsoft/markitdown/raw/refs/heads/main/packages/markitdown/tests/test_files/test.pdf")
 
-prompt = f"""
-Analyze the following documents:
-{pdf_attachment}
-"""
-
-content = [{"type": "input_image","image_url": image} for image in pdf_attachment.images] + \
-          [{"type": "input_text", "text": prompt}]
-
-response = OpenAI().responses.create(model="gpt-4.1-nano", input=[{"role": "user", "content": content}])
+response = OpenAI().responses.create(
+    model="gpt-4.1-nano", 
+    input=[{
+        "role": "user",
+        "content": pdf_attachment.to_openai_content("Analyze the following documents:")}])
 response.output_text
 ```
 
@@ -97,27 +96,10 @@ from attachments import Attachments
 pptx_file = Attachments(
     "https://github.com/microsoft/markitdown/raw/refs/heads/main/packages/markitdown/tests/test_files/test.pptx")
 
-prompt = f"""
-Analyze the following documents:
-{pptx_file}
-"""
-
-content = [
-    {
-        "type": "image",
-        "source": {
-            "type": "base64",
-            "media_type": image.split(";")[0].split(":")[1], # this is ugly but it works
-            "data": image.split(",")[1] # this is ugly but it works
-        }
-    }
-    for image in pptx_file.images
-] + [{"type": "text", "text": prompt}]
-
 message = anthropic.Anthropic().messages.create(
     max_tokens=8192,
     model="claude-3-5-haiku-20241022",
-    messages=[{"role": "user", "content": content}])
+    messages=[{"role": "user", "content": pptx_file.to_claude_content("Analyze the following documents:")}])
 print(message.content)
 ```
 
@@ -126,6 +108,7 @@ print(message.content)
 I am not a fan of the object that langchain and anthropic make you create, but hey!, I don't make the rules...
 
 ```python
+#pip install -U langchain-anthropic
 from langchain.chat_models import init_chat_model
 from attachments import Attachments
 
@@ -134,21 +117,7 @@ docx_file = Attachments("https://github.com/microsoft/markitdown/raw/refs/heads/
 llm = init_chat_model("anthropic:claude-3-5-sonnet-latest")
 
 # Creation of a list of object as expected by langchain/anthropic
-content = [{"type": "text", "text": f"Analyze the following documents: {docx_file}"}]
-
-for image in docx_file.images
-    media_type_i = image.split(";")[0].split(":")[1]
-    media_data_i = image.split(",")[1]
-    content.append({
-        "type": "image",
-        "source": {
-            "type": "base64",
-            "media_type": media_type_i, 
-            "data": media_data_i 
-        }
-    })
-
-
+content = docx_file.to_claude_content("Analyze the following documents:")
 message = {
     "role": "user",
     "content": content,

{attachments-0.2.4 → attachments-0.3.0}/src/attachments.egg-info/requires.txt

@@ -23,3 +23,4 @@ flake8
 mypy
 black
 filelock
+mystmd

{attachments-0.2.4 → attachments-0.3.0}/tests/test_core_functionality.py

@@ -282,5 +282,55 @@ class TestCoreFunctionality(unittest.TestCase):
         # It's unclear what this test is intended to check, so it's left unchanged
         pass
 
+    def test_fallback_to_text_parser(self):
+        """Test that files with unknown extensions or no extension fall back to TextParser."""
+        unknown_ext_content = "Unknown extension fallback test."
+        no_ext_content = "No extension fallback test."
+        
+        temp_dir = TEST_DATA_DIR # Use the existing test data directory for temp files
+        unknown_ext_filepath = os.path.join(temp_dir, "test_unknown.unknownext")
+        no_ext_filepath = os.path.join(temp_dir, "test_no_ext_file")
+
+        try:
+            with open(unknown_ext_filepath, "w", encoding="utf-8") as f:
+                f.write(unknown_ext_content)
+            with open(no_ext_filepath, "w", encoding="utf-8") as f:
+                f.write(no_ext_content)
+
+            atts = Attachments(unknown_ext_filepath, no_ext_filepath, verbose=True) # Verbose for debugging if needed
+            self.assertEqual(len(atts.attachments_data), 2, "Should process two files with fallback")
+
+            # Sort by original path to ensure consistent order for assertions
+            sorted_attachments = sorted(atts.attachments_data, key=lambda x: x['original_path_str'])
+
+            # Test the file with no extension
+            # The order depends on how Attachments processes and potentially sorts them, or how listdir returns them.
+            # Assuming Attachments processes in order of input for now, or we find by path.
+            no_ext_data = next((item for item in sorted_attachments if item['original_path_str'] == no_ext_filepath), None)
+            self.assertIsNotNone(no_ext_data, "Data for no-extension file not found")
+            if no_ext_data: # Check to satisfy mypy
+                self.assertEqual(no_ext_data['type'], 'txt', "File with no extension should be parsed as txt")
+                self.assertEqual(no_ext_data['text'], no_ext_content, "Text content mismatch for no-extension file")
+                self.assertIsNone(no_ext_data.get('original_detected_type'), "original_detected_type should be None for no-extension file")
+
+            # Test the file with an unknown extension
+            unknown_ext_data = next((item for item in sorted_attachments if item['original_path_str'] == unknown_ext_filepath), None)
+            self.assertIsNotNone(unknown_ext_data, "Data for unknown-extension file not found")
+            if unknown_ext_data: # Check to satisfy mypy
+                self.assertEqual(unknown_ext_data['type'], 'txt', "File with unknown extension should be parsed as txt")
+                self.assertEqual(unknown_ext_data['text'], unknown_ext_content, "Text content mismatch for unknown-extension file")
+                self.assertIsNone(unknown_ext_data.get('original_detected_type'), "original_detected_type should be None for unknown-extension file")
+
+        finally:
+            # Clean up temporary files
+            if os.path.exists(unknown_ext_filepath):
+                os.remove(unknown_ext_filepath)
+            if os.path.exists(no_ext_filepath):
+                os.remove(no_ext_filepath)
+
+    def test_initialize_attachments_with_list_of_paths(self):
+        """Test initializing Attachments with a list of paths."""
+        # ... existing code ...
+
 if __name__ == '__main__':
     unittest.main()