markitdown-pro 0.1.0__py3-none-any.whl

markitdown_pro/handlers/email_handler.py

@@ -0,0 +1,169 @@
+import os
+import tempfile
+from email.message import Message
+from email.parser import BytesParser
+from email.policy import default
+from typing import Dict, List, Tuple
+
+from ..common.logger import logger
+from ..common.utils import clean_markdown
+from .base_handler import BaseHandler
+
+
+class EmailHandler(BaseHandler):
+    extensions = frozenset([".eml", ".p7s"])
+
+    async def handle(self, file_path, *args, **kwargs) -> str:
+        logger.info(f"Processing email file: {file_path}")
+
+        try:
+            email_data = self._parse_email(file_path)
+            if not email_data:
+                return "# Error: Could not parse email file."
+
+            markdown_content = self._build_markdown(email_data)
+            return markdown_content
+
+        except Exception as e:
+            logger.error(f"Error processing email file: {file_path}: {e}")
+            return f"# Error processing email: {e}"
+
+    def _parse_email(self, file_path: str) -> Dict[str, any]:
+        """
+        Parses the EML/P7S file and extracts relevant information.
+
+        Args:
+            file_path: The path to the email file.
+
+        Returns:
+            A dictionary containing extracted data:
+            {
+                "subject": str,
+                "from": str,
+                "to": str,
+                "date": str,
+                "body": str,  # Plain text or HTML body (plain text preferred)
+                "attachments": List[Tuple[str, str]],  # (filename, temp_file_path)
+            }
+            Returns an empty dictionary if parsing fails.
+        """
+        try:
+            with open(file_path, "rb") as f:
+                # Use BytesParser with the 'default' policy for best compatibility
+                msg: Message = BytesParser(policy=default).parse(f)
+
+            subject = msg.get("Subject", "(No Subject)")
+            from_ = msg.get("From", "(Unknown Sender)")
+            to_ = msg.get("To", "(Unknown Recipient)")
+            date_ = msg.get("Date", "(Unknown Date)")
+
+            body = ""
+            attachments: List[Tuple[str, str]] = []
+
+            # Prefer plain text body
+            if msg.is_multipart():
+                for part in msg.walk():
+                    content_type = part.get_content_type()
+                    if part.get_content_disposition() == "attachment":
+                        filename = part.get_filename()
+                        if filename:
+                            att_data = part.get_payload(decode=True)
+                            # Create a temporary file for the attachment
+                            with tempfile.NamedTemporaryFile(
+                                delete=False, suffix=os.path.splitext(filename)[1]
+                            ) as tmp_file:
+                                tmp_file.write(att_data)
+                            attachments.append((filename, tmp_file.name))
+
+                    elif content_type == "text/plain" and not body:
+                        # Get the charset, default to utf-8 if not specified
+                        charset = part.get_content_charset() or "utf-8"
+                        try:
+                            body = part.get_payload(decode=True).decode(charset, errors="replace")
+                        except Exception as decode_err:
+                            logger.warning(
+                                f"Error decoding text/plain part: {decode_err}, using fallback",
+                            )
+                            body = part.get_payload(decode=True).decode("utf-8", errors="replace")
+
+                # If no plain text, try for HTML
+                if not body:
+                    for part in msg.walk():
+                        if part.get_content_type() == "text/html":
+                            charset = part.get_content_charset() or "utf-8"
+                            try:
+                                body = part.get_payload(decode=True).decode(
+                                    charset, errors="replace"
+                                )
+                                break  # Stop after finding the first HTML part
+                            except Exception as decode_err:
+                                logger.warning(
+                                    f"Error decoding text/html part: {decode_err}, using fallback",
+                                )
+                                body = part.get_payload(decode=True).decode(
+                                    "utf-8", errors="replace"
+                                )
+
+            else:  # Not multipart
+                content_type = msg.get_content_type()
+                if content_type == "text/plain":
+                    charset = msg.get_content_charset() or "utf-8"
+                    body = msg.get_payload(decode=True).decode(charset, "replace")
+
+                elif content_type == "text/html":
+                    charset = msg.get_content_charset() or "utf-8"
+                    body = msg.get_payload(decode=True).decode(charset, errors="replace")
+
+            return {
+                "subject": subject,
+                "from": from_,
+                "to": to_,
+                "date": date_,
+                "body": body,
+                "attachments": attachments,
+            }
+        except Exception as e:
+            logger.error(f"Error parsing email {file_path}: {e}")
+            return {}
+
+    def _build_markdown(self, email_data: Dict[str, any]) -> str:
+        """
+        Builds the Markdown output from the extracted email data.
+
+        Args:
+            email_data: The dictionary returned by _parse_email.
+
+        Returns:
+            The complete Markdown string.
+        """
+        # Use an f-string for more concise header formatting
+        markdown_parts = [
+            f"# Email: {email_data['subject']}",
+            f"**From:** {email_data['from']}",
+            f"**To:** {email_data['to']}",
+            f"**Date:** {email_data['date']}",
+            "",  # Add an empty line for separation
+        ]
+
+        body_content = email_data["body"]
+        if body_content:
+            markdown_parts.append("```")
+            markdown_parts.append(body_content)  # Add the email body
+            markdown_parts.append("```")
+
+        for filename, filepath in email_data["attachments"]:
+            markdown_parts.append(f"\n## Attachment: {filename}\n")
+            try:
+                # todo
+                attachment_markdown = None
+                markdown_parts.append(attachment_markdown)
+            except Exception as e:
+                logger.error(f"Error converting attachment '{filename}' in email: {e}")
+                markdown_parts.append(f"[Error converting attachment: {e}]")
+            finally:
+                try:
+                    os.remove(filepath)  # Clean up the temporary attachment file
+                except OSError as e:
+                    logger.warning(f"Could not remove temporary attachment file '{filepath}': {e}")
+
+        return clean_markdown("\n".join(markdown_parts))

markitdown_pro/handlers/epub_handler.py

@@ -0,0 +1,33 @@
+from ..common.logger import logger
+from ..common.utils import ensure_minimum_content
+from ..converters.unstructured_wrapper import UnstructuredWrapper
+from .base_handler import BaseHandler
+
+
+class EPUBHandler(BaseHandler):
+    extensions = frozenset([".epub"])
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.unstructured = UnstructuredWrapper()
+
+    async def handle(self, file_path, *args, **kwargs) -> str:
+        """
+        Handles EPUB files by converting them to Markdown using Unstructured.
+
+        Args:
+            file_path: Path to the .epub file.
+
+        Returns:
+            Markdown string representing the EPUB content, or an error message.
+        """
+        logger.info(f"Processing EPUB file {file_path}")
+        try:
+            md_content = await self.unstructured.convert(file_path)  # added await
+            if md_content and ensure_minimum_content(md_content):
+                return md_content
+            else:
+                raise RuntimeError(f"EPUB conversion failed or insufficient content: {file_path}")
+        except Exception as e:
+            logger.error(f"Error handling EPUB file '{file_path}': {e}")
+            return "# Error al procesar EPUB"

markitdown_pro/handlers/image_handler.py

@@ -0,0 +1,48 @@
+from ..common.logger import logger
+from ..common.utils import ensure_minimum_content
+from ..converters.gpt4o_mini_vision import GPT4oMiniVisionWrapper
+from .base_handler import BaseHandler
+
+
+class ImageHandler(BaseHandler):
+    extensions = frozenset(
+        [
+            ".bmp",
+            ".gif",
+            ".heic",
+            ".jpeg",
+            ".jpg",
+            ".png",
+            ".prn",
+            ".svg",
+            ".tiff",
+            ".webp",
+            ".heif",
+        ]
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.gpt4o_mini_vision = GPT4oMiniVisionWrapper()
+
+    async def handle(self, file_path, *args, **kwargs) -> str:
+        """
+        Handles image files by converting them to Markdown using GPT-4o-mini Vision.
+
+        Args:
+            file_path: Path to the image file.
+
+        Returns:
+            Markdown string representing the image content (OCR and analysis),
+            or an error message.
+        """
+        logger.info(f"Processing image file: {file_path}")
+        try:
+            md_content = await self.gpt4o_mini_vision.convert(file_path)
+            if md_content and ensure_minimum_content(md_content):
+                return md_content
+            else:
+                raise RuntimeError(f"Image conversion failed or insufficient content: {file_path}")
+        except Exception as e:
+            logger.error(f"Error handling image file '{file_path}': {e}")
+            return "# Error al procesar imagen"

markitdown_pro/handlers/ipynb_handler.py

@@ -0,0 +1,31 @@
+import nbformat
+
+from ..common import utils
+from ..common.logger import logger
+from .base_handler import BaseHandler
+
+
+class IpynbHandler(BaseHandler):
+    """Handler for Jupyter notebooks (.ipynb)."""
+
+    extensions = frozenset([".ipynb"])
+
+    async def handle(self, file_path: str, *args, **kwargs) -> str | None:
+        logger.info(f"Processing notebook: {file_path}")
+        try:
+            nb = nbformat.read(file_path, as_version=4)
+            cells_content = []
+            for cell in nb.cells:
+                if cell.cell_type == "markdown":
+                    cells_content.append(cell.source)
+                elif cell.cell_type == "code":
+                    cells_content.append("```python\n" + cell.source + "\n```")
+            result = "\n\n".join(cells_content)
+
+            if utils.ensure_minimum_content(result):
+                return result
+            else:
+                return None
+        except Exception as e:
+            logger.error(f"Error processing notebook {file_path}: {e}")
+            return None

markitdown_pro/handlers/markup_handler.py

@@ -0,0 +1,75 @@
+import json
+import os
+
+import yaml
+from bs4 import BeautifulSoup
+
+try:
+    import chardet
+except ImportError:
+    chardet = None
+
+from ..common.logger import logger
+from ..common.utils import ensure_minimum_content
+from .base_handler import BaseHandler
+
+
+class MarkupHandler(BaseHandler):
+    """Handler for .html, .xml, .json, .ndjson, .yaml, .yml files."""
+
+    extensions = frozenset([".html", ".htm", ".xml", ".json", ".ndjson", ".yaml", ".yml"])
+
+    async def handle(self, file_path: str, *args, **kwargs) -> str:
+        logger.info(f"Processing markup file: {file_path}")
+        try:
+            ext = os.path.splitext(file_path)[1].lower()
+
+            # Detect encoding
+            encoding = "utf-8"  # Default
+            if chardet:
+                with open(file_path, "rb") as f:
+                    raw_data = f.read()
+                    result = chardet.detect(raw_data)
+                    encoding = result["encoding"]
+                    logger.debug(f"Detected encoding: {encoding}")
+            else:
+                logger.warning("chardet not available, assuming UTF-8 encoding.")
+
+            with open(file_path, "r", encoding=encoding) as f:
+                content = f.read()
+
+            if ext in [".html", ".htm"]:
+                soup = BeautifulSoup(content, "html.parser")
+                text = soup.get_text(separator="\n")
+            elif ext == ".xml":
+                soup = BeautifulSoup(content, "xml")
+                text = soup.get_text(separator="\n")
+            elif ext in [".json", ".ndjson"]:
+                try:
+                    # Attempt to parse as complete JSON
+                    data = json.loads(content)
+                    text = json.dumps(data, indent=2, ensure_ascii=False)
+                except Exception:
+                    # If not, process line by line (ndjson case)
+                    lines = content.splitlines()
+                    parsed_lines = []
+                    for line in lines:
+                        try:
+                            obj = json.loads(line)
+                            parsed_lines.append(json.dumps(obj, indent=2, ensure_ascii=False))
+                        except Exception:
+                            parsed_lines.append(line)
+                    text = "\n".join(parsed_lines)
+            elif ext in [".yaml", ".yml"]:
+                data = yaml.safe_load(content)
+                text = yaml.dump(data, allow_unicode=True)
+            else:
+                text = content
+
+            if ensure_minimum_content(text):
+                return text
+            else:
+                raise RuntimeError(f"Insufficient content after conversion: {file_path}")
+        except Exception as e:
+            logger.error(f"Error processing markup file {file_path}: {e}")
+            raise

markitdown_pro/handlers/office_handler.py

@@ -0,0 +1,47 @@
+from ..common.logger import logger
+from ..common.utils import ensure_minimum_content
+from ..converters.azure_docint import AzureDocIntWrapper
+from ..converters.unstructured_wrapper import UnstructuredWrapper
+from .base_handler import BaseHandler
+
+
+class OfficeHandler(BaseHandler):
+    extensions = frozenset([".doc", ".docx", ".odt", ".rtf", ".ppt", ".pptx"])
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.azure_docint = AzureDocIntWrapper()
+        self.unstructured = UnstructuredWrapper()
+
+    async def handle(self, file_path, *args, **kwargs) -> str:
+        """
+        Handles Office documents (.doc, .docx, .odt, .rtf, .ppt, .pptx) by
+        first trying Azure Document Intelligence and falling back to Unstructured.
+
+        Args:
+            file_path: Path to the Office document file.
+
+        Returns:
+            Markdown string representing the document content, or an error message.
+        """
+        logger.info(f"Processing Office document: {file_path}")
+        try:
+            # First try with Azure Document Intelligence
+            logger.info(f"Attempting conversion with Azure Document Intelligence for: {file_path}")
+            md_content = await self.azure_docint.convert(file_path)
+            if md_content and ensure_minimum_content(md_content):
+                return md_content
+
+            # Fallback to Unstructured if Azure Doc Intelligence fails or returns insufficient content
+            logger.info(f"Falling back to Unstructured for: {file_path}")
+            md_content = await self.unstructured.convert(file_path)
+            if md_content and ensure_minimum_content(md_content):
+                return md_content
+
+            raise RuntimeError(
+                f"Office document conversion failed with both Azure Doc Intelligence and Unstructured: {file_path}"
+            )
+
+        except Exception as e:
+            logger.error(f"Error handling Office document '{file_path}': {e}")
+            return "# Error al procesar documento de Office"

markitdown_pro/handlers/pdf_handler.py

@@ -0,0 +1,122 @@
+from enum import Enum
+
+import fitz
+
+from ..common.logger import logger
+from ..common.utils import ensure_minimum_content
+from ..converters.azure_docint import AzureDocIntWrapper
+from ..converters.gpt4o_mini_vision import GPT4oMiniVisionWrapper
+from ..converters.markitdown_wrapper import MarkitDownWrapper
+from ..converters.pymupdf_wrapper import PyMuPDFWrapper
+from ..converters.unstructured_wrapper import UnstructuredWrapper
+from .base_handler import BaseHandler
+
+
+class PDFType(Enum):
+    """
+    Tipos de PDF detectados por el PDFHandler.
+
+    - TEXT_ONLY:       PDF que contiene solo texto, sin imágenes.
+    - TEXT_PLUS_IMAGES: PDF que contiene texto e imágenes.
+    - ALL_IMAGES:      PDF que contiene solo imágenes (PDF escaneado).
+    """
+
+    TEXT_ONLY = "TEXT_ONLY"
+    TEXT_PLUS_IMAGES = "TEXT_PLUS_IMAGES"
+    ALL_IMAGES = "ALL_IMAGES"
+
+
+class PDFHandler(BaseHandler):
+    extensions = frozenset([".pdf"])
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.markitdown = MarkitDownWrapper()
+        self.unstructured = UnstructuredWrapper()
+        self.azure_docint = AzureDocIntWrapper()
+        self.gpt4o_mini_vision = GPT4oMiniVisionWrapper()
+        self.pymu = PyMuPDFWrapper()
+
+        self.text_pipeline = [
+            self.markitdown,
+            self.unstructured,
+            self.pymu,
+            self.azure_docint,
+        ]
+        self.image_pipeline = [
+            self.gpt4o_mini_vision,
+        ]
+
+    async def handle(self, file_path, *args, **kwargs):
+        try:
+            pdf_type = await self._detect_pdf_type(file_path)
+
+            if pdf_type == PDFType.TEXT_ONLY:
+                pipeline = self.text_pipeline
+            elif pdf_type == PDFType.ALL_IMAGES:
+                pipeline = self.image_pipeline
+            elif pdf_type == PDFType.TEXT_PLUS_IMAGES:
+                pipeline = self.text_pipeline + self.image_pipeline
+            else:
+                pipeline = self.text_pipeline
+
+            for converter in pipeline:
+                logger.info(f"Trying {converter.name} for PDF {file_path}")
+                try:
+                    md_content = await converter.convert(file_path)
+                    if md_content and ensure_minimum_content(md_content):
+                        return md_content
+                except Exception as e:
+                    logger.error(f"Converter {converter.name} failed for PDF {file_path}: {e}")
+
+            raise RuntimeError(f"PDF conversion failed with all converters for {file_path}")
+
+        except Exception as e:
+            logger.error(f"Error handling PDF '{file_path}': {e}")
+            return None
+
+    async def _detect_pdf_type(self, file_path: str) -> PDFType:
+        """
+        Detect the type of PDF file based on its content.
+        """
+        min_text_length_threshold = 50
+
+        try:
+
+            async def open_and_process_doc():
+                with fitz.open(file_path) as doc:
+                    total_pages = doc.page_count
+                    pages_with_text = 0
+                    pages_with_images = 0
+
+                    for page_index in range(total_pages):
+                        page = doc.load_page(page_index)
+                        page_text = page.get_text().strip()
+                        if len(page_text) >= min_text_length_threshold:
+                            pages_with_text += 1
+                        images = page.get_images(full=True)
+                        if images:
+                            pages_with_images += 1
+
+                    logger.debug(f"Pages with text: {pages_with_text}/{total_pages}")
+                    logger.debug(f"Pages with images: {pages_with_images}/{total_pages}")
+
+                    is_text_only = pages_with_text == total_pages and pages_with_images == 0
+                    is_all_images = pages_with_images == total_pages and pages_with_text == 0
+                    has_text_and_images = pages_with_text > 0 and pages_with_images > 0
+
+                    if is_text_only:
+                        return PDFType.TEXT_ONLY
+                    elif is_all_images:
+                        return PDFType.ALL_IMAGES
+                    elif has_text_and_images:
+                        return PDFType.TEXT_PLUS_IMAGES
+                    else:
+                        return PDFType.TEXT_PLUS_IMAGES  # Fallback
+
+            return await open_and_process_doc()  # Run in thread
+        except Exception as e:
+            logger.error(f"Error analyzing PDF '{file_path}': {e}")
+            #  Important to re-raise the exception after logging, so the
+            #  caller knows something went wrong *during the analysis*.
+            raise

markitdown_pro/handlers/pst_handler.py

@@ -0,0 +1,153 @@
+import os
+from typing import Optional
+
+from .base_handler import BaseHandler
+
+try:
+    from libratom.lib.pff import PffArchive
+
+    HAS_LIBRATOM = True
+except ImportError:
+    HAS_LIBRATOM = False
+
+from ..common.logger import logger
+from ..common.utils import clean_markdown, ensure_minimum_content
+
+
+class PSTHandler(BaseHandler):
+    extensions = frozenset([".pst"])
+
+    async def handle(self, file_path, *args, **kwargs) -> str:
+        """
+        Parses a PST file using libratom, extracts messages and attachments,
+        and converts the content to Markdown. Recursively processes attachments.
+
+        Args:
+            file_path: Path to the .pst file.
+
+        Returns:
+            Markdown string representing the PST content, or an error message.
+        """
+        if not HAS_LIBRATOM:
+            logger.error("libratom is not installed. PST processing is disabled.")
+            return "# Error: libratom not installed. Cannot process PST files."
+
+        logger.info(f"Processing PST file: {file_path}")
+        try:
+            markdown_content = self._process_pst(file_path)
+            if markdown_content:
+                return markdown_content
+            else:
+                return "# PST Archive\n\n(No messages found or insufficient content.)"
+        except Exception as e:
+            logger.error(f"Error processing PST file: {file_path}: {e}")
+            return f"# Error processing PST file: {e}"
+
+    def _process_pst(self, file_path: str) -> Optional[str]:
+        """
+        Parses the PST file, extracts messages and attachments, and converts to Markdown.
+
+        Args:
+            file_path: Path to the PST file.
+
+        Returns:
+            Markdown string, or None if no messages are found or content is insufficient.
+        """
+        if not os.path.isfile(file_path):
+            logger.error(f"PST file not found: {file_path}")
+            return None
+
+        all_md_parts = [f"# PST Archive: {os.path.basename(file_path)}\n"]
+
+        try:
+            with PffArchive(file_path) as archive:
+                for folder in archive.folders():
+                    if not folder.name:  # Skip folders with no name
+                        continue
+
+                    # Add folder information, handling None folder names
+                    all_md_parts.append(f"\n## Folder: {folder.name or '(Unnamed Folder)'}\n")
+                    message_count = 0
+
+                    for message in folder.messages():
+                        message_count += 1
+                        try:
+                            message_md = self._process_message(message, message_count)
+                            if message_md:
+                                all_md_parts.extend(message_md)
+                        except Exception as e:
+                            logger.error(
+                                f"Error processing message {message_count} in folder {folder.name}: {e}"
+                            )
+                            all_md_parts.append(
+                                f"### Error processing message {message_count}: {e}"
+                            )
+
+            final_md = clean_markdown("\n\n".join(all_md_parts))
+            return final_md if ensure_minimum_content(final_md) else None
+
+        except Exception as e:
+            logger.error(f"Error opening or processing PST archive {file_path}: {e}")
+            return None
+
+    def _process_message(self, message, message_count: int) -> Optional[list[str]]:
+        """
+        Processes a single message from the PST archive.
+
+        Args:
+            message: The message object from libratom.
+            message_count:  The message number within the folder (for display).
+
+        Returns:
+            A list of Markdown strings representing the message, or None on error.
+        """
+        try:
+            subject = message.subject or "(No Subject)"
+            sender = "Unknown Sender"
+            date_ = "Unknown Date"
+
+            # Extract headers safely, handling potential errors
+            try:
+                headers = message.transport_headers
+                if headers:
+                    if isinstance(headers, bytes):
+                        headers = headers.decode(errors="replace")
+                    for line in headers.splitlines():
+                        if line.lower().startswith("from:"):
+                            sender = line.split(":", 1)[1].strip()
+                        elif line.lower().startswith("date:"):
+                            date_ = line.split(":", 1)[1].strip()
+            except Exception as header_err:
+                logger.warning(f"Error parsing headers: {header_err}")
+
+            # Extract the body, handling different encodings and body types
+            body_content = ""
+            try:
+                if message.plain_text_body:
+                    body_content = message.plain_text_body.decode(errors="replace")
+                elif message.html_body:
+                    body_content = message.html_body.decode(errors="replace")
+                elif message.rtf_body:
+                    body_content = message.rtf_body.decode(errors="replace")
+
+            except Exception as body_err:
+                logger.warning(f"Error decoding message body: {body_err}")
+
+            message_md_parts = [
+                f"### Message {message_count}",
+                f"**Subject:** {subject}",
+                f"**From:** {sender}",
+                f"**Date:** {date_}",
+                "",
+                "```",
+                body_content.strip() if body_content else "[No body text]",
+                "```",
+            ]
+
+            # Handle attachments todo: handle attachments
+
+            return message_md_parts
+
+        except Exception as e:
+            logger.error(f"Error processing individual message: {e}")
+            return None