markitdown-pro 2.0.0__tar.gz → 2.1.1__tar.gz

{markitdown_pro-2.0.0 → markitdown_pro-2.1.1}/.github/workflows/test.yml

@@ -1,16 +1,14 @@
 name: '🧪 Test package'
 
 on:
-  push:
+  pull_request:
     branches:
       - main
     paths:
       - 'markitdown_pro/**'
       - 'tests/**'
       - 'pyproject.toml'
-  pull_request:
-    branches:
-      - main
+  workflow_dispatch:
 
 permissions:
   contents: read

{markitdown_pro-2.0.0 → markitdown_pro-2.1.1}/.gitignore

@@ -24,3 +24,5 @@ docs/
 !./CLAUDE.md
 **/CONTINUITY.md
 **/CHANGELOG.md
+.mcp.json
+.worktrees/

{markitdown_pro-2.0.0 → markitdown_pro-2.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: markitdown-pro
-Version: 2.0.0
+Version: 2.1.1
 Summary: A package that converts almost any file format to Markdown.
 Author: Developer
 License-Expression: MIT

{markitdown_pro-2.0.0 → markitdown_pro-2.1.1}/markitdown_pro/conversion_pipeline.py

@@ -152,21 +152,40 @@ class ConversionPipeline:
                 ".pdf": self.pdf_handler,
                 ".xls": self.tabular_handler,
                 ".xlsx": self.tabular_handler,
-                ".docx": self.office_handler,
-                ".pptx": self.office_handler,
             },
         }
 
+        # Register Office extensions dynamically — the handler decides which
+        # legacy formats it supports based on Gotenberg availability. This
+        # keeps ``handlers_mapping`` (what consumers read) truthful at runtime.
+        for ext in self.office_handler.supported_extensions:
+            self.handlers_categories[ExtensionCategory.FORMATTED_DOCS][ext] = self.office_handler
+
         # Normalize map of extensions to handlers
         self.handlers_mapping: dict[str, BaseHandler] = {}
         for handlers in self.handlers_categories.values():
             for ext, handler in handlers.items():
                 self.handlers_mapping[ext] = handler
 
+        # Cache the read-only view so consumers get a stable object (and we
+        # don't rebuild the frozenset on every ``.supported_extensions`` access).
+        # ``handlers_mapping`` is read-only by convention.
+        self._supported_extensions: frozenset[str] = frozenset(self.handlers_mapping)
+
     # ---------------------------------------------------------------------
     # Public APIs
     # ---------------------------------------------------------------------
 
+    @property
+    def supported_extensions(self) -> frozenset[str]:
+        """Canonical set of extensions this pipeline can route.
+
+        This is the source of truth for consumers advertising supported
+        formats (e.g., a ``/extensions`` endpoint). ``handlers_mapping`` is
+        the routing table; ``supported_extensions`` is its read-only view.
+        """
+        return self._supported_extensions
+
     async def convert_document_to_md(
         self,
         file_path: str | Path,

markitdown_pro-2.1.1/markitdown_pro/converters/gotenberg_converter.py

@@ -0,0 +1,200 @@
+"""
+Convert Office files to PDF via Gotenberg HTTP API, then process through PDFHandler.
+
+Gotenberg (https://gotenberg.dev) is a Docker-based API that wraps LibreOffice
+for document-to-PDF conversion. It handles concurrency internally and scales
+horizontally via container replicas.
+
+Configure via the ``GOTENBERG_URL`` environment variable or pass
+``gotenberg_url=...`` explicitly. If neither is provided, the converter is
+disabled and ``convert()`` returns ``None`` without any network I/O — this lets
+downstream consumers advertise supported formats accurately.
+
+If the Gotenberg service is configured but unreachable at call time,
+``convert()`` returns ``None`` and the caller should fall through to the next
+converter.
+"""
+
+import asyncio
+import contextlib
+import functools
+import os
+import tempfile
+from collections.abc import Awaitable, Callable
+from pathlib import Path
+from typing import TypeVar
+
+import httpx
+
+from ..common.logger import logger
+from .base import Converter
+
+_COMPONENT = "GotenbergConverter"
+_CONVERT_ENDPOINT = "/forms/libreoffice/convert"
+
+# Transient failures worth retrying. ConnectError is intentionally excluded —
+# "service unreachable" is handled as a graceful skip, not a retry.
+_RETRYABLE_EXC: tuple[type[BaseException], ...] = (
+    httpx.TimeoutException,
+    httpx.RemoteProtocolError,
+    httpx.ReadError,
+    httpx.WriteError,
+)
+
+T = TypeVar("T")
+
+
+def _retry_transient(
+    attempts: int = 3, delays: tuple[float, ...] = (1.0, 3.0)
+) -> Callable[[Callable[..., Awaitable[T | None]]], Callable[..., Awaitable[T | None]]]:
+    """Retry an async method on transient httpx errors. Sleeps between attempts
+    are taken from ``delays`` (short, fixed backoff — enough to ride out a
+    worker restart or a brief connection hiccup)."""
+
+    def decorator(
+        fn: Callable[..., Awaitable[T | None]],
+    ) -> Callable[..., Awaitable[T | None]]:
+        @functools.wraps(fn)
+        async def wrapper(self, *args, **kwargs) -> T | None:
+            tag = args[1] if len(args) >= 2 else kwargs.get("tag", _COMPONENT)
+            for i in range(attempts):
+                try:
+                    return await fn(self, *args, **kwargs)
+                except _RETRYABLE_EXC as e:
+                    if i == attempts - 1:
+                        raise
+                    sleep_for = delays[min(i, len(delays) - 1)]
+                    logger.warning(
+                        f"{tag} | transient {type(e).__name__}: {e}; "
+                        f"retry {i + 1}/{attempts - 1} in {sleep_for:.1f}s"
+                    )
+                    await asyncio.sleep(sleep_for)
+            return None
+
+        return wrapper
+
+    return decorator
+
+
+class GotenbergConverter(Converter):
+    """
+    Convert Office files to PDF via Gotenberg, then delegate to PDFHandler
+    for per-page text extraction and OCR.
+
+    This captures embedded images via OCR that neither MarkItDown nor
+    DocIntelligence can extract from Office files directly.
+
+    Requires a running Gotenberg instance. Gracefully returns None if
+    the service is unavailable or if the converter is disabled, allowing
+    fallback to other converters.
+    """
+
+    SUPPORTED_EXTENSIONS = frozenset({".docx", ".pptx", ".doc", ".ppt", ".odt", ".rtf"})
+
+    def __init__(self, gotenberg_url: str | None = None) -> None:
+        super().__init__(_COMPONENT)
+        # gotenberg_url is the single source of truth; ``enabled`` derives from it.
+        # Treat empty/whitespace values as unconfigured to catch a common
+        # ``docker-compose`` misconfiguration (``environment: GOTENBERG_URL=``).
+        raw = (gotenberg_url if gotenberg_url is not None else os.getenv("GOTENBERG_URL")) or ""
+        self.gotenberg_url: str | None = raw.strip() or None
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.gotenberg_url)
+
+    @property
+    def _convert_url(self) -> str:
+        """The fully-qualified Gotenberg endpoint URL.
+
+        Only safe to access when ``self.enabled`` is True. Callers in this
+        class are already guarded by the ``if not self.enabled`` short-circuit
+        in ``convert()``; raising here makes the invariant explicit instead
+        of letting a ``None`` propagate into ``httpx`` or the format string.
+        """
+        if not self.gotenberg_url:
+            raise RuntimeError(
+                "GotenbergConverter is not configured (set GOTENBERG_URL "
+                "or pass gotenberg_url=...). Check ``enabled`` before calling."
+            )
+        return f"{self.gotenberg_url.rstrip('/')}{_CONVERT_ENDPOINT}"
+
+    async def convert(self, file_path: str) -> str | None:
+        if not self.enabled:
+            return None
+
+        file_name = Path(file_path).name
+        tag = f"{_COMPONENT} | {file_name}"
+
+        logger.info(f"{tag} | converting to PDF via Gotenberg ({self.gotenberg_url})")
+
+        try:
+            pdf_bytes = await self._convert_to_pdf(file_path, tag)
+        except _RETRYABLE_EXC as e:
+            logger.error(f"{tag} | Gotenberg error after retries: {type(e).__name__}: {e}")
+            return None
+        if not pdf_bytes:
+            return None
+
+        # Write PDF to temp file and process through PDFHandler
+        tmp_pdf = None
+        try:
+            fd, tmp_pdf = tempfile.mkstemp(suffix=".pdf", prefix="mkdpro_")
+            os.close(fd)
+            Path(tmp_pdf).write_bytes(pdf_bytes)
+
+            logger.info(f"{tag} | PDF received ({len(pdf_bytes)} bytes), delegating to PDFHandler")
+
+            # Import here to avoid circular import
+            from ..handlers.pdf_handler import PDFHandler
+
+            pdf_handler = PDFHandler()
+            try:
+                result = await pdf_handler.handle(tmp_pdf, force_ocr=True)
+                if result:
+                    logger.info(f"{tag} | PDFHandler succeeded ({len(result)} chars)")
+                return result
+            finally:
+                await pdf_handler.aclose()
+
+        except Exception as e:
+            logger.error(f"{tag} | error processing PDF: {e}")
+            return None
+        finally:
+            if tmp_pdf and Path(tmp_pdf).exists():
+                with contextlib.suppress(Exception):
+                    os.unlink(tmp_pdf)
+
+    @_retry_transient(attempts=3, delays=(1.0, 3.0))
+    async def _convert_to_pdf(self, file_path: str, tag: str) -> bytes | None:
+        """Send file to Gotenberg and return PDF bytes."""
+        try:
+            async with httpx.AsyncClient(timeout=120) as client:
+                with open(file_path, "rb") as f:
+                    response = await client.post(
+                        self._convert_url,
+                        files={"files": (Path(file_path).name, f)},
+                        data={
+                            "losslessImageCompression": "true",
+                            "quality": "100",
+                        },
+                    )
+
+                if response.status_code == 200:
+                    return response.content
+
+                logger.error(
+                    f"{tag} | Gotenberg returned {response.status_code}: {response.text[:200]}"
+                )
+                return None
+
+        except httpx.ConnectError:
+            logger.info(f"{tag} | Gotenberg not reachable at {self.gotenberg_url}, skipping")
+            return None
+        except _RETRYABLE_EXC:
+            # Let the retry decorator handle transient errors. If retries are
+            # exhausted it re-raises, and the caller in ``convert()`` logs.
+            raise
+        except Exception as e:
+            logger.error(f"{tag} | Gotenberg error: {e}")
+            return None

{markitdown_pro-2.0.0 → markitdown_pro-2.1.1}/markitdown_pro/converters/gpt_vision_converter.py

@@ -11,6 +11,8 @@ Returns Markdown when successful, or `None` if conversion produced
 insufficient content (as determined by `ensure_minimum_content` in the service).
 """
 
+from pathlib import Path
+
 from ..common import config
 from ..common.utils import is_pdf
 from ..services.openai_services import GPTVision
@@ -115,6 +117,7 @@ class GPTVisionConverter(Converter):
             max_retries=_retries,
             base_delay=base_delay,
             max_delay=max_delay,
+            context_label=Path(file_path).name,
         )
 
     async def aclose(self) -> None:

{markitdown_pro-2.0.0 → markitdown_pro-2.1.1}/markitdown_pro/handlers/office_handler.py

@@ -38,11 +38,22 @@ class OfficeHandler(BaseHandler):
     the pipeline order when initializing the handler.
 
     Supported formats: .docx, .pptx (modern Office XML formats).
-    Old binary formats (.doc, .ppt, .odt, .rtf) supported via Gotenberg only.
+    Legacy binary formats (.doc, .ppt, .odt, .rtf) are supported at runtime
+    only when the configured pipeline contains an enabled ``GotenbergConverter``.
+    Callers should inspect the instance attribute ``supported_extensions`` (not
+    the class constant) to learn which formats this handler will actually
+    process.
     """
 
+    # Class constant stays narrow (the always-supported modern formats) so that
+    # ``BaseHandler.is_valid`` and any class-level introspection never falsely
+    # advertise legacy formats that would fail without Gotenberg. Dynamic
+    # runtime capability lives on the instance attribute ``supported_extensions``
+    # set in ``__init__``.
     SUPPORTED_EXTENSIONS = frozenset({".docx", ".pptx"})
 
+    _GOTENBERG_ONLY_EXTS = frozenset({".doc", ".ppt", ".odt", ".rtf"})
+
     def __init__(
         self,
         pipeline: list[tuple[Converter, str]] | None = None,
@@ -62,6 +73,26 @@ class OfficeHandler(BaseHandler):
             (self.markitdown, "MarkItDown"),
         ]
 
+        # isinstance catches subclasses/proxies users might pass in custom
+        # pipelines. We explicitly check that the imported ``GotenbergConverter``
+        # is still a real class before using it as an isinstance second arg —
+        # if tests patch the name to a MagicMock *instance* (not a class),
+        # we fall back to ``gotenberg_enabled = False`` instead of wrapping
+        # the whole generator in try/except TypeError, which would otherwise
+        # silently swallow a real bug inside an ``enabled`` property.
+        if isinstance(GotenbergConverter, type):
+            gotenberg_enabled = any(
+                isinstance(c, GotenbergConverter) and getattr(c, "enabled", False)
+                for c, _ in self._pipeline
+            )
+        else:
+            gotenberg_enabled = False
+        self.supported_extensions: frozenset[str] = (
+            self.SUPPORTED_EXTENSIONS | self._GOTENBERG_ONLY_EXTS
+            if gotenberg_enabled
+            else self.SUPPORTED_EXTENSIONS
+        )
+
     async def handle(self, file_path: str, *args, **kwargs) -> str | None:
         """
         Convert an Office document to Markdown by trying the configured pipeline in order.

{markitdown_pro-2.0.0 → markitdown_pro-2.1.1}/markitdown_pro/handlers/page_pdf_converter.py

@@ -158,6 +158,7 @@ class PagePDFConverter:
             png_path = await asyncio.to_thread(self._render_page_to_png, file_path, page_idx)
             content = await self.gpt_vision.gpt_vision.process_image(
                 file_or_url=png_path,
+                context_label=f"{Path(file_path).name} page {page_idx + 1}",
             )
             if content:
                 logger.info(f"{tag} | succeeded ({len(content)} chars)")

{markitdown_pro-2.0.0 → markitdown_pro-2.1.1}/markitdown_pro/services/openai_services.py

@@ -240,6 +240,7 @@ class GPTVision:
         base_delay: float = 0.5,
         max_delay: float = 20.0,
         retry_http_statuses: tuple[int, ...] = _RETRY_HTTP_STATUSES,
+        context_label: str | None = None,
     ):
         """
         Call self.lang_client.ainvoke with:
@@ -275,8 +276,9 @@ class GPTVision:
                 raise last_err
             cap = min(max_delay, base_delay * (2 ** (attempt - 1)))
             sleep_for = random.uniform(0, cap)
+            label = context_label or file_or_url
             logger.warning(
-                f"GPTVision: ainvoke transient error {file_or_url}: attempt {attempt}/{max_retries}: "
+                f"GPTVision: ainvoke transient error {label}: attempt {attempt}/{max_retries}: "
                 f"{type(last_err).__name__}: {last_err}. Retrying in {sleep_for:.2f}s"
             )
             await asyncio.sleep(sleep_for)
@@ -331,6 +333,7 @@ class GPTVision:
         max_retries: int = 6,
         base_delay: float = 0.5,
         max_delay: float = 20.0,
+        context_label: str | None = None,
     ) -> str | None:
         """
         OCR a single image (local path or URL) with the vision model.
@@ -373,6 +376,7 @@ class GPTVision:
                     max_retries=max_retries,
                     base_delay=base_delay,
                     max_delay=max_delay,
+                    context_label=context_label,
                 )
             finally:
                 with suppress(Exception):
@@ -469,7 +473,11 @@ class GPTVision:
                             # Per-page hard timeout so one slow page cannot stall the run
                             try:
                                 partial_md = await asyncio.wait_for(
-                                    self.process_image(png_path), timeout=self.page_timeout_s
+                                    self.process_image(
+                                        png_path,
+                                        context_label=f"{file_stem} page {page_index + 1}/{num_pages}",
+                                    ),
+                                    timeout=self.page_timeout_s,
                                 )
                             except TimeoutError:
                                 logger.error(

{markitdown_pro-2.0.0 → markitdown_pro-2.1.1}/tests/data/test_expectations.yaml

@@ -74,6 +74,62 @@ gotenberg_converter:
     expect: ["what is lorem ipsum?", "where does it come from"]
 
 
+# =============================================================================
+# LEGACY OFFICE FORMATS — Gotenberg-gated (.doc / .ppt / .odt / .rtf)
+# Same fixtures as above re-saved / pandoc-converted into legacy formats.
+# Routed through OfficeHandler → Gotenberg → PDF → PDFHandler per-page OCR.
+# Only available iff GOTENBERG_URL is configured on the pipeline.
+# =============================================================================
+legacy_office_formats:
+  # .doc (re-saved from .docx via Word)
+  - file: tests/data/word-legacy/lorem-ipsum-only-text.doc
+    expect: ["what is lorem ipsum?", "where does it come from"]
+  - file: tests/data/word-legacy/lorem-ipsum-text-with-image.doc
+    expect: ["what is lorem ipsum?", "red", "green", "blue"]
+  - file: tests/data/word-legacy/geometry-colors.doc
+    expect: ["red", "green", "blue"]
+  - file: tests/data/word-legacy/lorem-ipsum-scanned-text.doc
+    expect: ["what is lorem ipsum?", "why do we use it?"]
+  - file: tests/data/word-legacy/lorem-ipsum-text-with-scanned-text.doc
+    expect: ["what is lorem ipsum?", "where does it come from"]
+
+  # .ppt (re-saved from .pptx via PowerPoint)
+  - file: tests/data/powerpoint-legacy/lorem-ipsum-only-text.ppt
+    expect: ["what is lorem ipsum?", "where does it come from"]
+  - file: tests/data/powerpoint-legacy/lorem-ipsum-text-with-image.ppt
+    expect: ["what is lorem ipsum", "red", "green", "blue"]
+  - file: tests/data/powerpoint-legacy/geometry-colors.ppt
+    expect: ["red", "green", "blue"]
+  - file: tests/data/powerpoint-legacy/lorem-ipsum-scanned-text.ppt
+    expect: ["what is lorem ipsum", "why do we use it"]
+  - file: tests/data/powerpoint-legacy/lorem-ipsum-text-with-scanned-text.ppt
+    expect: ["what is lorem ipsum?", "where does it come from"]
+
+  # .odt (re-saved from .docx via Word)
+  - file: tests/data/word-legacy/lorem-ipsum-only-text.odt
+    expect: ["what is lorem ipsum?", "where does it come from"]
+  - file: tests/data/word-legacy/lorem-ipsum-text-with-image.odt
+    expect: ["what is lorem ipsum?", "red", "green", "blue"]
+  - file: tests/data/word-legacy/geometry-colors.odt
+    expect: ["red", "green", "blue"]
+  - file: tests/data/word-legacy/lorem-ipsum-scanned-text.odt
+    expect: ["what is lorem ipsum?", "why do we use it?"]
+  - file: tests/data/word-legacy/lorem-ipsum-text-with-scanned-text.odt
+    expect: ["what is lorem ipsum?", "where does it come from"]
+
+  # .rtf (re-saved from .docx via Word)
+  - file: tests/data/word-legacy/lorem-ipsum-only-text.rtf
+    expect: ["what is lorem ipsum?", "where does it come from"]
+  - file: tests/data/word-legacy/lorem-ipsum-text-with-image.rtf
+    expect: ["what is lorem ipsum?", "red", "green", "blue"]
+  - file: tests/data/word-legacy/geometry-colors.rtf
+    expect: ["red", "green", "blue"]
+  - file: tests/data/word-legacy/lorem-ipsum-scanned-text.rtf
+    expect: ["what is lorem ipsum?", "why do we use it?"]
+  - file: tests/data/word-legacy/lorem-ipsum-text-with-scanned-text.rtf
+    expect: ["what is lorem ipsum?", "where does it come from"]
+
+
 # =============================================================================
 # MARKITDOWN CONVERTER — text extraction only, NO OCR, NO image content
 # =============================================================================