markitdown-ocr 0.1.0__py3-none-any.whl

markitdown_ocr/_pptx_converter_with_ocr.py

@@ -0,0 +1,249 @@
+"""
+Enhanced PPTX Converter with improved OCR support.
+Already has LLM-based image description, this enhances it with traditional OCR fallback.
+"""
+
+import io
+import sys
+from typing import Any, BinaryIO, Optional
+
+from typing import BinaryIO, Any, Optional
+
+from markitdown.converters import HtmlConverter
+from markitdown import DocumentConverter, DocumentConverterResult, StreamInfo
+from markitdown._exceptions import (
+    MissingDependencyException,
+    MISSING_DEPENDENCY_MESSAGE,
+)
+from ._ocr_service import LLMVisionOCRService
+
+_dependency_exc_info = None
+try:
+    import pptx
+except ImportError:
+    _dependency_exc_info = sys.exc_info()
+
+
+class PptxConverterWithOCR(DocumentConverter):
+    """Enhanced PPTX Converter with OCR fallback."""
+
+    def __init__(self, ocr_service: Optional[LLMVisionOCRService] = None):
+        super().__init__()
+        self._html_converter = HtmlConverter()
+        self.ocr_service = ocr_service
+
+    def accepts(
+        self,
+        file_stream: BinaryIO,
+        stream_info: StreamInfo,
+        **kwargs: Any,
+    ) -> bool:
+        mimetype = (stream_info.mimetype or "").lower()
+        extension = (stream_info.extension or "").lower()
+
+        if extension == ".pptx":
+            return True
+
+        if mimetype.startswith(
+            "application/vnd.openxmlformats-officedocument.presentationml"
+        ):
+            return True
+
+        return False
+
+    def convert(
+        self,
+        file_stream: BinaryIO,
+        stream_info: StreamInfo,
+        **kwargs: Any,
+    ) -> DocumentConverterResult:
+        if _dependency_exc_info is not None:
+            raise MissingDependencyException(
+                MISSING_DEPENDENCY_MESSAGE.format(
+                    converter=type(self).__name__,
+                    extension=".pptx",
+                    feature="pptx",
+                )
+            ) from _dependency_exc_info[1].with_traceback(
+                _dependency_exc_info[2]
+            )  # type: ignore[union-attr]
+
+        # Get OCR service (from kwargs or instance)
+        ocr_service: Optional[LLMVisionOCRService] = (
+            kwargs.get("ocr_service") or self.ocr_service
+        )
+        llm_client = kwargs.get("llm_client")
+
+        presentation = pptx.Presentation(file_stream)
+        md_content = ""
+        slide_num = 0
+
+        for slide in presentation.slides:
+            slide_num += 1
+            md_content += f"\\n\\n<!-- Slide number: {slide_num} -->\\n"
+
+            title = slide.shapes.title
+
+            def get_shape_content(shape, **kwargs):
+                nonlocal md_content
+
+                # Pictures
+                if self._is_picture(shape):
+                    # Get image data
+                    image_stream = io.BytesIO(shape.image.blob)
+
+                    # Try LLM description first if available
+                    llm_description = ""
+                    if llm_client and kwargs.get("llm_model"):
+                        try:
+                            from ._llm_caption import llm_caption
+
+                            image_filename = shape.image.filename
+                            image_extension = None
+                            if image_filename:
+                                import os
+
+                                image_extension = os.path.splitext(image_filename)[1]
+
+                            image_stream_info = StreamInfo(
+                                mimetype=shape.image.content_type,
+                                extension=image_extension,
+                                filename=image_filename,
+                            )
+
+                            llm_description = llm_caption(
+                                image_stream,
+                                image_stream_info,
+                                client=llm_client,
+                                model=kwargs.get("llm_model"),
+                                prompt=kwargs.get("llm_prompt"),
+                            )
+                        except Exception:
+                            pass
+
+                    # Try OCR if LLM failed or not available
+                    ocr_text = ""
+                    if not llm_description and ocr_service:
+                        try:
+                            image_stream.seek(0)
+                            ocr_result = ocr_service.extract_text(image_stream)
+                            if ocr_result.text.strip():
+                                ocr_text = ocr_result.text.strip()
+                        except Exception:
+                            pass
+
+                    # Format extracted content using unified OCR block format
+                    content = (llm_description or ocr_text or "").strip()
+                    if content:
+                        md_content += f"\n*[Image OCR]\n{content}\n[End OCR]*\n"
+
+                # Tables
+                if self._is_table(shape):
+                    md_content += self._convert_table_to_markdown(shape.table, **kwargs)
+
+                # Charts
+                if shape.has_chart:
+                    md_content += self._convert_chart_to_markdown(shape.chart)
+
+                # Text areas
+                elif shape.has_text_frame:
+                    if shape == title:
+                        md_content += "# " + shape.text.lstrip() + "\\n"
+                    else:
+                        md_content += shape.text + "\\n"
+
+                # Group Shapes
+                if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.GROUP:
+                    sorted_shapes = sorted(
+                        shape.shapes,
+                        key=lambda x: (
+                            float("-inf") if not x.top else x.top,
+                            float("-inf") if not x.left else x.left,
+                        ),
+                    )
+                    for subshape in sorted_shapes:
+                        get_shape_content(subshape, **kwargs)
+
+            sorted_shapes = sorted(
+                slide.shapes,
+                key=lambda x: (
+                    float("-inf") if not x.top else x.top,
+                    float("-inf") if not x.left else x.left,
+                ),
+            )
+            for shape in sorted_shapes:
+                get_shape_content(shape, **kwargs)
+
+            md_content = md_content.strip()
+
+            if slide.has_notes_slide:
+                md_content += "\\n\\n### Notes:\\n"
+                notes_frame = slide.notes_slide.notes_text_frame
+                if notes_frame is not None:
+                    md_content += notes_frame.text
+                md_content = md_content.strip()
+
+        return DocumentConverterResult(markdown=md_content.strip())
+
+    def _is_picture(self, shape):
+        if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.PICTURE:
+            return True
+        if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.PLACEHOLDER:
+            if hasattr(shape, "image"):
+                return True
+        return False
+
+    def _is_table(self, shape):
+        if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.TABLE:
+            return True
+        return False
+
+    def _convert_table_to_markdown(self, table, **kwargs):
+        import html
+
+        html_table = "<html><body><table>"
+        first_row = True
+        for row in table.rows:
+            html_table += "<tr>"
+            for cell in row.cells:
+                if first_row:
+                    html_table += "<th>" + html.escape(cell.text) + "</th>"
+                else:
+                    html_table += "<td>" + html.escape(cell.text) + "</td>"
+            html_table += "</tr>"
+            first_row = False
+        html_table += "</table></body></html>"
+
+        return (
+            self._html_converter.convert_string(html_table, **kwargs).markdown.strip()
+            + "\\n"
+        )
+
+    def _convert_chart_to_markdown(self, chart):
+        try:
+            md = "\\n\\n### Chart"
+            if chart.has_title:
+                md += f": {chart.chart_title.text_frame.text}"
+            md += "\\n\\n"
+            data = []
+            category_names = [c.label for c in chart.plots[0].categories]
+            series_names = [s.name for s in chart.series]
+            data.append(["Category"] + series_names)
+
+            for idx, category in enumerate(category_names):
+                row = [category]
+                for series in chart.series:
+                    row.append(series.values[idx])
+                data.append(row)
+
+            markdown_table = []
+            for row in data:
+                markdown_table.append("| " + " | ".join(map(str, row)) + " |")
+            header = markdown_table[0]
+            separator = "|" + "|".join(["---"] * len(data[0])) + "|"
+            return md + "\\n".join([header, separator] + markdown_table[1:])
+        except ValueError as e:
+            if "unsupported plot type" in str(e):
+                return "\\n\\n[unsupported chart]\\n\\n"
+        except Exception:
+            return "\\n\\n[unsupported chart]\\n\\n"

markitdown_ocr/_xlsx_converter_with_ocr.py

@@ -0,0 +1,225 @@
+"""
+Enhanced XLSX Converter with OCR support for embedded images.
+Extracts images from Excel spreadsheets and performs OCR while maintaining cell context.
+"""
+
+import io
+import sys
+from typing import Any, BinaryIO, Optional
+
+from markitdown.converters import HtmlConverter
+from markitdown import DocumentConverter, DocumentConverterResult, StreamInfo
+from markitdown._exceptions import (
+    MissingDependencyException,
+    MISSING_DEPENDENCY_MESSAGE,
+)
+from ._ocr_service import LLMVisionOCRService
+
+# Try loading dependencies
+_xlsx_dependency_exc_info = None
+try:
+    import pandas as pd
+    from openpyxl import load_workbook
+except ImportError:
+    _xlsx_dependency_exc_info = sys.exc_info()
+
+
+class XlsxConverterWithOCR(DocumentConverter):
+    """
+    Enhanced XLSX Converter with OCR support for embedded images.
+    Extracts images with their cell positions and performs OCR.
+    """
+
+    def __init__(self, ocr_service: Optional[LLMVisionOCRService] = None):
+        super().__init__()
+        self._html_converter = HtmlConverter()
+        self.ocr_service = ocr_service
+
+    def accepts(
+        self,
+        file_stream: BinaryIO,
+        stream_info: StreamInfo,
+        **kwargs: Any,
+    ) -> bool:
+        mimetype = (stream_info.mimetype or "").lower()
+        extension = (stream_info.extension or "").lower()
+
+        if extension == ".xlsx":
+            return True
+
+        if mimetype.startswith(
+            "application/vnd.openxmlformats-officedocument.spreadsheetml"
+        ):
+            return True
+
+        return False
+
+    def convert(
+        self,
+        file_stream: BinaryIO,
+        stream_info: StreamInfo,
+        **kwargs: Any,
+    ) -> DocumentConverterResult:
+        if _xlsx_dependency_exc_info is not None:
+            raise MissingDependencyException(
+                MISSING_DEPENDENCY_MESSAGE.format(
+                    converter=type(self).__name__,
+                    extension=".xlsx",
+                    feature="xlsx",
+                )
+            ) from _xlsx_dependency_exc_info[1].with_traceback(
+                _xlsx_dependency_exc_info[2]
+            )  # type: ignore[union-attr]
+
+        # Get OCR service if available (from kwargs or instance)
+        ocr_service: Optional[LLMVisionOCRService] = (
+            kwargs.get("ocr_service") or self.ocr_service
+        )
+
+        if ocr_service:
+            # Remove ocr_service from kwargs to avoid duplicate argument error
+            kwargs_without_ocr = {k: v for k, v in kwargs.items() if k != "ocr_service"}
+            return self._convert_with_ocr(
+                file_stream, ocr_service, **kwargs_without_ocr
+            )
+        else:
+            return self._convert_standard(file_stream, **kwargs)
+
+    def _convert_standard(
+        self, file_stream: BinaryIO, **kwargs: Any
+    ) -> DocumentConverterResult:
+        """Standard conversion without OCR."""
+        file_stream.seek(0)
+        sheets = pd.read_excel(file_stream, sheet_name=None, engine="openpyxl")
+        md_content = ""
+
+        for sheet_name in sheets:
+            md_content += f"## {sheet_name}\n"
+            html_content = sheets[sheet_name].to_html(index=False)
+            md_content += (
+                self._html_converter.convert_string(
+                    html_content, **kwargs
+                ).markdown.strip()
+                + "\n\n"
+            )
+
+        return DocumentConverterResult(markdown=md_content.strip())
+
+    def _convert_with_ocr(
+        self, file_stream: BinaryIO, ocr_service: LLMVisionOCRService, **kwargs: Any
+    ) -> DocumentConverterResult:
+        """Convert XLSX with image OCR."""
+        file_stream.seek(0)
+        wb = load_workbook(file_stream)
+
+        md_content = ""
+
+        for sheet_name in wb.sheetnames:
+            sheet = wb[sheet_name]
+            md_content += f"## {sheet_name}\n\n"
+
+            # Convert sheet data to markdown table
+            file_stream.seek(0)
+            try:
+                df = pd.read_excel(
+                    file_stream, sheet_name=sheet_name, engine="openpyxl"
+                )
+                html_content = df.to_html(index=False)
+                md_content += (
+                    self._html_converter.convert_string(
+                        html_content, **kwargs
+                    ).markdown.strip()
+                    + "\n\n"
+                )
+            except Exception:
+                # If pandas fails, just skip the table
+                pass
+
+            # Extract and OCR images in this sheet
+            images_with_ocr = self._extract_and_ocr_sheet_images(sheet, ocr_service)
+
+            if images_with_ocr:
+                md_content += "### Images in this sheet:\n\n"
+                for img_info in images_with_ocr:
+                    ocr_text = img_info["ocr_text"]
+                    md_content += f"*[Image OCR]\n{ocr_text}\n[End OCR]*\n\n"
+
+        return DocumentConverterResult(markdown=md_content.strip())
+
+    def _extract_and_ocr_sheet_images(
+        self, sheet: Any, ocr_service: LLMVisionOCRService
+    ) -> list[dict]:
+        """
+        Extract and OCR images from an Excel sheet.
+
+        Args:
+            sheet: openpyxl worksheet
+            ocr_service: OCR service
+
+        Returns:
+            List of dicts with 'cell_ref' and 'ocr_text'
+        """
+        results = []
+
+        try:
+            # Check if sheet has images
+            if hasattr(sheet, "_images"):
+                for img in sheet._images:
+                    try:
+                        # Get image data
+                        if hasattr(img, "_data"):
+                            image_data = img._data()
+                        elif hasattr(img, "image"):
+                            # Some versions store it differently
+                            image_data = img.image
+                        else:
+                            continue
+
+                        # Create image stream
+                        image_stream = io.BytesIO(image_data)
+
+                        # Get cell reference
+                        cell_ref = "unknown"
+                        if hasattr(img, "anchor"):
+                            anchor = img.anchor
+                            if hasattr(anchor, "_from"):
+                                from_cell = anchor._from
+                                if hasattr(from_cell, "col") and hasattr(
+                                    from_cell, "row"
+                                ):
+                                    # Convert column number to letter
+                                    col_letter = self._column_number_to_letter(
+                                        from_cell.col
+                                    )
+                                    cell_ref = f"{col_letter}{from_cell.row + 1}"
+
+                        # Perform OCR
+                        ocr_result = ocr_service.extract_text(image_stream)
+
+                        if ocr_result.text.strip():
+                            results.append(
+                                {
+                                    "cell_ref": cell_ref,
+                                    "ocr_text": ocr_result.text.strip(),
+                                    "backend": ocr_result.backend_used,
+                                }
+                            )
+
+                    except Exception:
+                        continue
+
+        except Exception:
+            pass
+
+        return results
+
+    @staticmethod
+    def _column_number_to_letter(n: int) -> str:
+        """Convert column number to Excel column letter (0-indexed)."""
+        result = ""
+        n = n + 1  # Make 1-indexed
+        while n > 0:
+            n -= 1
+            result = chr(65 + (n % 26)) + result
+            n //= 26
+        return result

markitdown_ocr-0.1.0.dist-info/METADATA

@@ -0,0 +1,233 @@
+Metadata-Version: 2.4
+Name: markitdown-ocr
+Version: 0.1.0
+Summary: OCR plugin for MarkItDown - Extracts text from images in PDF, DOCX, PPTX, and XLSX via LLM Vision
+Project-URL: Documentation, https://github.com/microsoft/markitdown#readme
+Project-URL: Issues, https://github.com/microsoft/markitdown/issues
+Project-URL: Source, https://github.com/microsoft/markitdown
+Author-email: Contributors <noreply@github.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: docx,llm,markitdown,ocr,pdf,pptx,vision,xlsx
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Requires-Python: >=3.10
+Requires-Dist: mammoth~=1.11.0
+Requires-Dist: markitdown>=0.1.0
+Requires-Dist: openpyxl
+Requires-Dist: pandas
+Requires-Dist: pdfminer-six>=20251230
+Requires-Dist: pdfplumber>=0.11.9
+Requires-Dist: pillow>=9.0.0
+Requires-Dist: pymupdf>=1.24.0
+Requires-Dist: python-docx
+Requires-Dist: python-pptx
+Provides-Extra: llm
+Requires-Dist: openai>=1.0.0; extra == 'llm'
+Description-Content-Type: text/markdown
+
+# MarkItDown OCR Plugin
+
+LLM Vision plugin for MarkItDown that extracts text from images embedded in PDF, DOCX, PPTX, and XLSX files.
+
+Uses the same `llm_client` / `llm_model` pattern that MarkItDown already supports for image descriptions — no new ML libraries or binary dependencies required.
+
+## Features
+
+- **Enhanced PDF Converter**: Extracts text from images within PDFs, with full-page OCR fallback for scanned documents
+- **Enhanced DOCX Converter**: OCR for images in Word documents
+- **Enhanced PPTX Converter**: OCR for images in PowerPoint presentations
+- **Enhanced XLSX Converter**: OCR for images in Excel spreadsheets
+- **Context Preservation**: Maintains document structure and flow when inserting extracted text
+
+## Installation
+
+```bash
+pip install markitdown-ocr
+```
+
+The plugin uses whatever OpenAI-compatible client you already have. Install one if you don't have it yet:
+
+```bash
+pip install openai
+```
+
+## Usage
+
+### Command Line
+
+```bash
+markitdown document.pdf --use-plugins --llm-client openai --llm-model gpt-4o
+```
+
+### Python API
+
+Pass `llm_client` and `llm_model` to `MarkItDown()` exactly as you would for image descriptions:
+
+```python
+from markitdown import MarkItDown
+from openai import OpenAI
+
+md = MarkItDown(
+    enable_plugins=True,
+    llm_client=OpenAI(),
+    llm_model="gpt-4o",
+)
+
+result = md.convert("document_with_images.pdf")
+print(result.text_content)
+```
+
+If no `llm_client` is provided the plugin still loads, but OCR is silently skipped — falling back to the standard built-in converter.
+
+### Custom Prompt
+
+Override the default extraction prompt for specialized documents:
+
+```python
+md = MarkItDown(
+    enable_plugins=True,
+    llm_client=OpenAI(),
+    llm_model="gpt-4o",
+    llm_prompt="Extract all text from this image, preserving table structure.",
+)
+```
+
+### Any OpenAI-Compatible Client
+
+Works with any client that follows the OpenAI API:
+
+```python
+from openai import AzureOpenAI
+
+md = MarkItDown(
+    enable_plugins=True,
+    llm_client=AzureOpenAI(
+        api_key="...",
+        azure_endpoint="https://your-resource.openai.azure.com/",
+        api_version="2024-02-01",
+    ),
+    llm_model="gpt-4o",
+)
+```
+
+## How It Works
+
+When `MarkItDown(enable_plugins=True, llm_client=..., llm_model=...)` is called:
+
+1. MarkItDown discovers the plugin via the `markitdown.plugin` entry point group
+2. It calls `register_converters()`, forwarding all kwargs including `llm_client` and `llm_model`
+3. The plugin creates an `LLMVisionOCRService` from those kwargs
+4. Four OCR-enhanced converters are registered at **priority -1.0** — before the built-in converters at priority 0.0
+
+When a file is converted:
+
+1. The OCR converter accepts the file
+2. It extracts embedded images from the document
+3. Each image is sent to the LLM with an extraction prompt
+4. The returned text is inserted inline, preserving document structure
+5. If the LLM call fails, conversion continues without that image's text
+
+## Supported File Formats
+
+### PDF
+
+- Embedded images are extracted by position (via `page.images` / page XObjects) and OCR'd inline, interleaved with the surrounding text in vertical reading order.
+- **Scanned PDFs** (pages with no extractable text) are detected automatically: each page is rendered at 300 DPI and sent to the LLM as a full-page image.
+- **Malformed PDFs** that pdfplumber/pdfminer cannot open (e.g. truncated EOF) are retried with PyMuPDF page rendering, so content is still recovered.
+
+### DOCX
+
+- Images are extracted via document part relationships (`doc.part.rels`).
+- OCR is run before the DOCX→HTML→Markdown pipeline executes: placeholder tokens are injected into the HTML so that the markdown converter does not escape the OCR markers, and the final placeholders are replaced with the formatted `*[Image OCR]...[End OCR]*` blocks after conversion.
+- Document flow (headings, paragraphs, tables) is fully preserved around the OCR blocks.
+
+### PPTX
+
+- Picture shapes, placeholder shapes with images, and images inside groups are all supported.
+- Shapes are processed in top-to-left reading order per slide.
+- If an `llm_client` is configured, the LLM is asked for a description first; OCR is used as the fallback when no description is returned.
+
+### XLSX
+
+- Images embedded in worksheets (`sheet._images`) are extracted per sheet.
+- Cell position is calculated from the image anchor coordinates (column/row → Excel letter notation).
+- Images are listed under a `### Images in this sheet:` section after the sheet's data table — they are not interleaved into the table rows.
+
+### Output format
+
+Every extracted OCR block is wrapped as:
+
+```text
+*[Image OCR]
+<extracted text>
+[End OCR]*
+```
+
+## Troubleshooting
+
+### OCR text missing from output
+
+The most likely cause is a missing `llm_client` or `llm_model`. Verify:
+
+```python
+from openai import OpenAI
+from markitdown import MarkItDown
+
+md = MarkItDown(
+    enable_plugins=True,
+    llm_client=OpenAI(),   # required
+    llm_model="gpt-4o",    # required
+)
+```
+
+### Plugin not loading
+
+Confirm the plugin is installed and discovered:
+
+```bash
+markitdown --list-plugins   # should show: ocr
+```
+
+### API errors
+
+The plugin propagates LLM API errors as warnings and continues conversion. Check your API key, quota, and that the chosen model supports vision inputs.
+
+## Development
+
+### Running Tests
+
+```bash
+cd packages/markitdown-ocr
+pytest tests/ -v
+```
+
+### Building from Source
+
+```bash
+git clone https://github.com/microsoft/markitdown.git
+cd markitdown/packages/markitdown-ocr
+pip install -e .
+```
+
+## Contributing
+
+Contributions are welcome! See the [MarkItDown repository](https://github.com/microsoft/markitdown) for guidelines.
+
+## License
+
+MIT — see [LICENSE](LICENSE).
+
+## Changelog
+
+### 0.1.0 (Initial Release)
+
+- LLM Vision OCR for PDF, DOCX, PPTX, XLSX
+- Full-page OCR fallback for scanned PDFs
+- Context-aware inline text insertion
+- Priority-based converter replacement (no code changes required)