markitdown-pro 0.1.0__py3-none-any.whl

markitdown_pro/__init__.py

@@ -0,0 +1 @@
+from . import common, converters, handlers, services  # noqa F401

markitdown_pro/common/logger.py

@@ -0,0 +1,6 @@
+import logging
+import os
+
+log_level = int(os.getenv("LOG_LEVEL", "20"))
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO, format="[%(asctime)s] [%(levelname)s] %(message)s")

markitdown_pro/common/utils.py

@@ -0,0 +1,59 @@
+import re
+from pathlib import Path
+
+
+def detect_extension(file_path: str) -> str:
+    """Return the file extension in lowercase."""
+    return Path(file_path).suffix.lower()
+
+
+def is_pdf(file_path: str) -> bool:
+    """Check if the file is a PDF."""
+    return detect_extension(file_path) == ".pdf"
+
+
+def is_image(file_path: str) -> bool:
+    """Check if the file is an image based on its extension."""
+    return detect_extension(file_path) in (
+        ".jpg",
+        ".jpeg",
+        ".png",
+        ".bmp",
+        ".tiff",
+        ".gif",
+        ".heic",
+        ".webp",
+    )
+
+
+def is_audio(file_path: str) -> bool:
+    """Check if the file is an audio file based on its extension."""
+    return detect_extension(file_path) in (".mp3", ".wav", ".m4a", ".ogg", ".flac")
+
+
+def is_zip(file_path: str) -> bool:
+    """Check if the file is a ZIP archive."""
+    return detect_extension(file_path) == ".zip"
+
+
+def is_eml(file_path: str) -> bool:
+    """Check if the file is an EML email file."""
+    return detect_extension(file_path) == ".eml"
+
+
+def clean_markdown(md_text: str) -> str:
+    """
+    Clean up Markdown text by removing trailing spaces and reducing excess newlines.
+    """
+    md_text = re.sub(r"[ \t]+(\r?\n)", r"\1", md_text)
+    md_text = re.sub(r"\n{3,}", "\n\n", md_text)
+    return md_text.strip()
+
+
+def ensure_minimum_content(md_text: str) -> bool:
+    """
+    Check if the Markdown text has non-trivial content.
+    """
+    if not md_text:
+        return False
+    return bool(md_text and len(md_text.strip()) > 30)

markitdown_pro/conversion_pipeline.py

@@ -0,0 +1,212 @@
+import os
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+import requests
+
+from .common.logger import logger
+from .common.utils import detect_extension, ensure_minimum_content, is_image
+from .converters.azure_docint import AzureDocIntWrapper
+from .converters.gpt4o_mini_vision import GPT4oMiniVisionWrapper
+from .converters.markitdown_wrapper import MarkitDownWrapper
+from .converters.unstructured_wrapper import UnstructuredWrapper
+from .handlers.audio_handler import AudioHandler
+from .handlers.base_handler import BaseHandler  # Import BaseHandler
+from .handlers.email_handler import EmailHandler
+from .handlers.epub_handler import EPUBHandler
+from .handlers.image_handler import ImageHandler
+from .handlers.ipynb_handler import IpynbHandler
+from .handlers.markup_handler import MarkupHandler
+from .handlers.office_handler import OfficeHandler
+from .handlers.pdf_handler import PDFHandler
+from .handlers.pst_handler import PSTHandler
+from .handlers.tabular_handler import TabularHandler
+from .handlers.text_handler import TextHandler
+
+
+def _write_md(path: str, content: str) -> str:
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(content)
+    return content
+
+
+class ConversionPipeline:
+    """
+    A class to encapsulate the conversion pipeline, making it easier to use
+    without needing to directly interact with dependency injection in the main
+    functions.  This simplifies testing and usage.
+    """
+
+    def __init__(
+        self,
+    ):
+        self.pdf_handler = PDFHandler()
+        self.audio_handler = AudioHandler()
+        self.image_handler = ImageHandler()
+        self.text_handler = TextHandler()
+        self.tabular_handler = TabularHandler()
+        self.markup_handler = MarkupHandler()
+        self.office_handler = OfficeHandler()
+        self.epub_handler = EPUBHandler()
+        self.email_handler = EmailHandler()
+        self.pst_handler = PSTHandler()
+        self.ipynb_handler = IpynbHandler()
+        self.markitdown_wrapper = MarkitDownWrapper()
+        self.unstructured_wrapper = UnstructuredWrapper()
+        self.azure_docint_wrapper = AzureDocIntWrapper()
+        self.gpt4o_mini_vision_wrapper = GPT4oMiniVisionWrapper()
+
+        # Define the handlers in a dictionary mapping extensions to handlers
+        self.handlers_mapping: dict[str, BaseHandler] = {  # Use BaseHandler type hint
+            ".pdf": self.pdf_handler,
+            ".mp3": self.audio_handler,
+            ".wav": self.audio_handler,
+            ".ogg": self.audio_handler,
+            ".flac": self.audio_handler,
+            ".m4a": self.audio_handler,
+            ".aac": self.audio_handler,
+            ".wma": self.audio_handler,
+            ".webm": self.audio_handler,
+            ".opus": self.audio_handler,
+            ".bmp": self.image_handler,
+            ".gif": self.image_handler,
+            ".heic": self.image_handler,
+            ".jpeg": self.image_handler,
+            ".jpg": self.image_handler,
+            ".png": self.image_handler,
+            ".prn": self.image_handler,
+            ".svg": self.image_handler,
+            ".tiff": self.image_handler,
+            ".webp": self.image_handler,
+            ".heif": self.image_handler,
+            ".txt": self.text_handler,
+            ".md": self.text_handler,
+            ".py": self.text_handler,
+            ".go": self.text_handler,
+            ".csv": self.tabular_handler,
+            ".tsv": self.tabular_handler,
+            ".xls": self.tabular_handler,
+            ".xlsx": self.tabular_handler,
+            ".html": self.markup_handler,
+            ".htm": self.markup_handler,
+            ".xml": self.markup_handler,
+            ".json": self.markup_handler,
+            ".ndjson": self.markup_handler,
+            ".yaml": self.markup_handler,
+            ".yml": self.markup_handler,
+            ".doc": self.office_handler,
+            ".docx": self.office_handler,
+            ".odt": self.office_handler,
+            ".rtf": self.office_handler,
+            ".ppt": self.office_handler,
+            ".pptx": self.office_handler,
+            ".epub": self.epub_handler,
+            ".eml": self.email_handler,
+            ".p7s": self.email_handler,
+            ".msg": self.email_handler,
+            ".pst": self.pst_handler,
+            ".ipynb": self.ipynb_handler,
+        }
+
+    async def convert_document_to_md(self, file_path: str, output_md: Optional[str] = None) -> str:
+        """
+        Convert a document to Markdown by trying a series of specialized handlers.
+        """
+        if not Path(file_path).is_file():
+            raise ValueError(f"The provided path '{file_path}' is not a valid file.")
+
+        if not output_md:
+            base = os.path.splitext(file_path)[0]
+            output_md = base + ".md"
+
+        logger.debug(f"convert_document_to_md: Converting '{file_path}' --> '{output_md}'")
+
+        extension = detect_extension(file_path).lower()
+
+        handler = self.handlers_mapping.get(extension)
+
+        if handler:
+            try:
+                logger.debug(f"convert_document_to_md: Using handler for extension '{extension}'")
+                md_content = await handler.handle(file_path)  # AWAIT the handler!
+                if not md_content:
+                    raise RuntimeError(f"Handler for {extension} returned None for {file_path}")
+
+                logger.debug(
+                    f"convert_document_to_md: Handler returned {len(md_content)} characters"
+                )
+                if md_content and ensure_minimum_content(md_content):
+                    return _write_md(output_md, md_content)
+                else:
+                    raise RuntimeError(
+                        f"Conversion failed for {file_path} using handler for {extension}"
+                    )
+            except Exception as e:
+                logger.error(f"Error in handler for extension {extension}: {e}")
+                raise
+
+        else:
+            logger.warning(f"No specific handler found for '{extension}'. Using default path.")
+            # Use the injected wrappers directly
+            md_out = await self.markitdown_wrapper.process(file_path)
+            if md_out and ensure_minimum_content(md_out):
+                return _write_md(output_md, md_out)
+            md_out = await self.unstructured_wrapper.process(file_path)
+            if md_out and ensure_minimum_content(md_out):
+                return _write_md(output_md, md_out)
+            md_out = await self.azure_docint_wrapper.process(file_path)
+            if md_out and ensure_minimum_content(md_out):
+                return _write_md(output_md, md_out)
+            if is_image(file_path):
+                md_out = await self.gpt4o_mini_vision_wrapper.process(file_path)
+                if md_out and ensure_minimum_content(md_out):
+                    return _write_md(output_md, md_out)
+
+            raise RuntimeError(f"Markdown conversion failed for {file_path} at the default path.")
+
+    async def convert_document_from_url(
+        self, url: str, output_md: Optional[str] = None
+    ) -> str:  # Add async
+        logger.info(f"convert_document_from_url: Downloading '{url}'")
+        resp = requests.get(url, stream=True)  # requests is *synchronous*
+        resp.raise_for_status()
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".download") as tmp:
+            for chunk in resp.iter_content(chunk_size=8192):
+                tmp.write(chunk)
+            tmp.flush()
+            local_path = tmp.name
+
+        try:
+            return await self.convert_document_to_md(
+                local_path, output_md=output_md
+            )  # Await the call
+        finally:
+            Path(local_path).unlink(missing_ok=True)
+
+    async def convert_document_from_stream(
+        self, stream, extension: str, output_md: Optional[str] = None
+    ) -> str:  # Add async
+        if not extension.startswith("."):
+            extension = "." + extension
+        logger.debug(
+            f"convert_document_from_url: Converting from stream with extension '{extension}'"
+        )
+
+        from io import BytesIO
+
+        if not isinstance(stream, BytesIO):
+            raise ValueError("Stream must be a BytesIO object")
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=extension) as tmp:
+            tmp.write(stream.read())
+            tmp.flush()
+            local_path = tmp.name
+
+        try:
+            return await self.convert_document_to_md(
+                local_path, output_md=output_md
+            )  # Await the call
+        finally:
+            Path(local_path).unlink(missing_ok=True)

markitdown_pro/converters/azure_docint.py

@@ -0,0 +1,13 @@
+from typing import Optional
+
+from ..services.azure_service import AzureServices
+from .base import ConverterWrapper
+
+
+class AzureDocIntWrapper(ConverterWrapper):
+    def __init__(self):
+        super().__init__("Azure Document Intelligence")
+        self.azure_services = AzureServices()
+
+    async def convert(self, file_path: str) -> Optional[str]:
+        return self.azure_services.process_azure_doc_intelligence(file_path)

markitdown_pro/converters/base.py

@@ -0,0 +1,26 @@
+from typing import Optional
+
+from ..common.logger import logger
+
+
+class ConverterWrapper:
+    SUPPORTED_FORMATS = ()
+
+    def __init__(self, name: str):
+        self.name = name
+
+    async def convert(self, file_path: str) -> Optional[str]:  # MUST be async
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def process(self, file_path: str) -> Optional[str]:  # Make process async
+        logger.info(f"Processing {self.name} for file: {file_path}")
+        try:
+            result = await self.convert(file_path)  # AWAIT the convert
+            if result:
+                return result
+            else:
+                logger.warning(f"{self.name} conversion returned insufficient content.")
+                return None
+        except Exception as e:
+            logger.error(f"Error during {self.name} conversion: {e}")
+            return None

markitdown_pro/converters/gpt4o_mini_vision.py

@@ -0,0 +1,21 @@
+from typing import Optional
+
+from ..common.utils import is_pdf
+from ..services.openai_services import GPT4oMiniVision
+from .base import ConverterWrapper
+
+
+class GPT4oMiniVisionWrapper(ConverterWrapper):
+    def __init__(self):
+        super().__init__("GPT-4o-mini Vision")
+        self.gpt4o_mini = GPT4oMiniVision()  # Instantiate the actual service
+
+    async def convert(self, file_path: str) -> Optional[str]:
+        if is_pdf(file_path):
+            # Try concurrent first, then simple if it fails
+            result = await self.gpt4o_mini.process_scanned_pdf_concurrent(file_path)
+            if result:
+                return result
+            return await self.gpt4o_mini.process_scanned_pdf_simple(file_path)
+        else:  # Assume it's an image
+            return await self.gpt4o_mini.process_image(file_path)

markitdown_pro/converters/markitdown_wrapper.py

@@ -0,0 +1,44 @@
+import logging
+from typing import Optional
+
+from markitdown import MarkItDown
+
+from ..common.logger import logger
+from ..common.utils import clean_markdown, ensure_minimum_content
+from .base import ConverterWrapper
+
+
+class MarkitDownWrapper(ConverterWrapper):
+    SUPPORTED_FORMATS = [
+        "txt",
+        "md",
+        "html",
+        "pdf",
+        "docx",
+        "xlsx",
+        "pptx",
+        "zip",
+        "wav",
+        "mp3",
+        "jpg",
+        "png",
+        "gif",
+        "bmp",
+        "tiff",
+    ]
+
+    def __init__(self):
+        super().__init__("MarkItDown")
+        self.markitdown = MarkItDown()
+
+    async def convert(self, file_path: str) -> Optional[str]:
+        file_extension = file_path.split(".")[-1].lower()
+        logging.info(f"Processing MarkItDown file: {file_path} with extension: {file_extension}")
+
+        if file_extension not in self.SUPPORTED_FORMATS:
+            logger.warning(f"Unsupported file format: {file_extension}")
+            return None
+
+        result = self.markitdown.convert(file_path)
+        text_md = clean_markdown(result.text_content or None)
+        return text_md if ensure_minimum_content(text_md) else None

markitdown_pro/converters/pymupdf_wrapper.py

@@ -0,0 +1,42 @@
+import logging
+from typing import Optional
+
+import fitz  # PyMuPDF
+
+from ..common.logger import logger
+from ..common.utils import clean_markdown, ensure_minimum_content
+from .base import ConverterWrapper
+
+
+class PyMuPDFWrapper(ConverterWrapper):
+    SUPPORTED_FORMATS = ["pdf", "xps", "epub", "mobi", "fb2", "cbz", "svg"]
+
+    def __init__(self):
+        super().__init__("PyMuPDF")
+
+    async def convert(self, file_path: str) -> Optional[str]:
+        file_extension = file_path.split(".")[-1].lower()
+        logging.info(f"Processing PyMuPDF file: {file_path} with extension: {file_extension}")
+
+        if file_extension not in self.SUPPORTED_FORMATS:
+            logger.warning(f"Unsupported file format: {file_extension}")
+            return None
+
+        try:
+            doc = fitz.open(file_path)
+
+            # Extract text from all pages
+            text_content = ""
+            for page_num in range(len(doc)):
+                page = doc.load_page(page_num)
+                text_content += page.get_text()
+                text_content += "\n\n"
+
+            doc.close()
+
+            text_md = clean_markdown(text_content)
+            return text_md if ensure_minimum_content(text_md) else None
+
+        except Exception as e:
+            logger.error(f"Error processing document with PyMuPDF: {e}")
+            return None

markitdown_pro/converters/unstructured_wrapper.py

@@ -0,0 +1,62 @@
+from typing import Optional
+
+from unstructured.partition.auto import partition
+
+from ..common.logger import logger
+from ..common.utils import clean_markdown, ensure_minimum_content
+from .base import ConverterWrapper
+
+
+class UnstructuredWrapper(ConverterWrapper):
+    SUPPORTED_FORMATS = (
+        "txt",
+        "md",
+        "html",
+        "htm",
+        "xml",
+        "pdf",
+        "docx",
+        "xlsx",
+        "pptx",
+        "odt",
+        "ods",
+        "odp",
+        "rtf",
+        "json",
+        "csv",
+        "tsv",
+        "jpg",
+        "jpeg",
+        "png",
+        "gif",
+        "bmp",
+        "tiff",
+        "svg",
+        "mp3",
+        "wav",
+        "ogg",
+        "flac",
+        "zip",
+        "tar",
+        "gz",
+        "rar",
+        "epub",
+        "eml",
+        "msg",
+    )
+
+    def __init__(self):
+        super().__init__("Unstructured")
+
+    async def convert(self, file_path: str) -> Optional[str]:
+        logger.debug(f"Converting {file_path} to markdown")
+        file_extension = file_path.split(".")[-1].lower()
+
+        if file_extension not in self.SUPPORTED_FORMATS:
+            logger.warning(f"Unsupported file format: {file_extension}")
+            return None
+
+        # Directly call partition.  No asyncio.to_thread
+        elements = partition(filename=file_path, extract_images_in_pdf=True)
+        combined = clean_markdown("\n\n".join(str(el) for el in elements))
+        return combined if ensure_minimum_content(combined) else None

markitdown_pro/converters/youtube_wrapper.py

@@ -0,0 +1,67 @@
+from typing import Optional
+from urllib.parse import parse_qs, urlparse
+
+from markitdown import MarkItDown
+from youtube_transcript_api import YouTubeTranscriptApi
+
+from ..common.logger import logger
+from .base import ConverterWrapper
+
+
+class YouTubeWrapper(ConverterWrapper):
+    def __init__(self):
+        super().__init__("YouTube")
+        self.markitdown = MarkItDown()
+
+    async def convert(self, url: str) -> Optional[str]:
+        logger.info(f"Processing YouTube video: {url}")
+
+        markitdown_result = self.markitdown.convert(url)
+        markitdown_content = markitdown_result.text_content if markitdown_result else None
+
+        if not markitdown_content:
+            logger.warning("MarkItDown processing failed or returned empty content.")
+
+        parsed_url = urlparse(url)
+        params = parse_qs(parsed_url.query)
+        if "v" not in params:
+            logger.warning(f"Invalid YouTube URL: {url}")
+            return markitdown_content
+
+        video_id = params["v"][0]
+        try:
+            transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
+            transcript_text = None
+
+            for lang in [
+                "en",
+                "es",
+                "fr",
+                "de",
+                "it",
+                "pt",
+                "ru",
+                "zh-Hans",
+                "ja",
+                "ko",
+            ]:
+                try:
+                    transcript = transcript_list.find_transcript([lang])
+                    transcript_text = "\n".join([part["text"] for part in transcript.fetch()])
+                except Exception as e:
+                    logger.info(f"No transcript found for language {lang}: {e}")
+                    continue
+
+            if not transcript_text:
+                logger.warning(f"No transcript found for YouTube video: {url}")
+                return markitdown_content
+            full_content = (
+                f"{markitdown_content}\n\n# YouTube Transcript\n\n{transcript_text}"
+                if markitdown_content
+                else f"# YouTube Transcript\n\n{transcript_text}"
+            )
+            return full_content
+
+        except Exception as e:
+            logger.error(f"Error fetching transcript for YouTube video {url}: {e}", exc_info=True)
+            return markitdown_content

markitdown_pro/handlers/audio_handler.py

@@ -0,0 +1,40 @@
+from ..common.logger import logger
+from ..converters.markitdown_wrapper import MarkitDownWrapper
+from ..handlers.base_handler import BaseHandler
+from ..services.azure_service import AzureServices
+
+
+class AudioHandler(BaseHandler):
+    extensions = {".mp3", ".wav", ".m4a", ".ogg", ".flac"}
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.markitdown = MarkitDownWrapper()
+        self.azure_services = AzureServices()
+
+    async def handle(self, file_path, *args, **kwargs) -> str:
+        """
+        Transcribe an audio file using MarkItDown first.
+        If MarkItDown fails, fall back to Azure Speech-to-Text API.
+        """
+        logger.info(f"Processing audio file: {file_path} using MarkItDown")
+
+        try:
+            text: str = await self.markitdown.convert(file_path)
+            if text:
+                return text
+            else:
+                logger.warning(
+                    "MarkItDown conversion returned insufficient content. Falling back to Azure Speech-to-Text API."
+                )
+        except Exception as e:
+            logger.error(f"Error processing audio file with MarkItDown: {e}")
+
+        # Fallback to Azure Speech-to-Text API
+        try:
+            logger.info(f"Transcribing audio file: {file_path} using Azure Speech-to-Text API")
+            text = await self.azure_services.recognize_azure_speech_to_text_from_file(file_path)
+            return text if text else "# Audio File\n\n(Transcription not available.)"
+        except Exception as e:
+            logger.error(f"Error transcribing audio file with Azure Speech-to-Text API: {e}")
+            return "# Audio File\n\n(Transcription failed.)"

markitdown_pro/handlers/base_handler.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+
+
+class BaseHandler:
+    extensions: set = frozenset()
+    pipeline: list = None
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    @classmethod
+    async def is_valid(cls, file_path: str) -> bool:
+        return Path(file_path).is_file() and Path(file_path).suffix in cls.extensions
+
+    async def handle(self, file_path, *args, **kwargs) -> str:
+        raise NotImplementedError("You must implement the handle method")