markitdown-pro 1.1.1__tar.gz → 1.2.2__tar.gz

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: markitdown-pro
-Version: 1.1.1
+Version: 1.2.2
 Summary: A package that converts almost any file format to Markdown.
 Author: Developer
 Classifier: Programming Language :: Python :: 3
@@ -26,6 +26,7 @@ Requires-Dist: azure-cognitiveservices-speech==1.42.0
 Requires-Dist: chardet==5.2.0
 Requires-Dist: nbformat
 Requires-Dist: youtube-transcript-api
+Requires-Dist: tabulate
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

markitdown_pro-1.2.2/markitdown_pro/common/schemas.py

@@ -0,0 +1,7 @@
+from io import BytesIO
+
+
+class NamedBytesIO(BytesIO):
+    def __init__(self, data: bytes, name: str):
+        super().__init__(data)
+        self.name = name

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro/common/utils.py

@@ -1,6 +1,8 @@
 import re
 from pathlib import Path
 
+from .schemas import NamedBytesIO
+
 
 def detect_extension(file_path: str) -> str:
     """Return the file extension in lowercase."""
@@ -56,4 +58,48 @@ def ensure_minimum_content(md_text: str) -> bool:
     """
     if not md_text:
         return False
-    return bool(md_text and len(md_text.strip()) > 30)
+    return bool(md_text and len(md_text.strip()) > 10)
+
+
+SAFE_BOM_EXTS = {".csv", ".tsv", ".txt", ".log", ".md", ".rst"}
+AVOID_BOM_EXTS = {".json", ".py", ".js", ".ts", ".sh", ".bat", ".ps1", ".sql"}
+
+
+def has_utf8_bom(b: bytes) -> bool:
+    return b.startswith(b"\xef\xbb\xbf")
+
+
+def should_add_bom(
+    ext: str,
+    detected_enc: str,
+    confidence: float,
+    consumer_defaults_ascii: bool,
+) -> bool:
+    ext = ext.lower()
+    if ext in AVOID_BOM_EXTS:
+        return False
+    if not consumer_defaults_ascii:
+        return False
+    enc = (detected_enc or "").replace("_", "-").lower()
+    # Only add when it's actually UTF-8/ASCII and no BOM present
+    if enc not in ("utf-8", "us-ascii", "ascii", "utf-8-sig"):
+        return False
+    # Use confidence guard so we don't add BOM to files we’re already certain about
+    if confidence >= 0.10:
+        return False
+    return ext in SAFE_BOM_EXTS
+
+
+def prepare_stream_with_optional_bom(
+    path: str,
+    detected_enc: str,
+    confidence: float,
+    consumer_defaults_ascii: bool = True,
+) -> NamedBytesIO:
+    p = Path(path)
+    raw = p.read_bytes()
+    if has_utf8_bom(raw):
+        return NamedBytesIO(raw, name=p.name)
+    if should_add_bom(p.suffix, detected_enc, confidence, consumer_defaults_ascii):
+        return NamedBytesIO(b"\xef\xbb\xbf" + raw, name=p.name)
+    return NamedBytesIO(raw, name=p.name)

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro/conversion_pipeline.py

@@ -50,8 +50,7 @@ from .handlers.markup_handler import MarkupHandler
 from .handlers.office_handler import OfficeHandler
 from .handlers.pdf_handler import PDFHandler
 from .handlers.pst_handler import PSTHandler
-
-# from .handlers.tabular_handler import TabularHandler
+from .handlers.tabular_handler import TabularHandler
 from .handlers.text_handler import TextHandler
 
 
@@ -81,7 +80,7 @@ class ConversionPipeline:
         self.audio_handler = AudioHandler()
         self.image_handler = ImageHandler()
         self.text_handler = TextHandler()
-        # self.tabular_handler = TabularHandler()
+        self.tabular_handler = TabularHandler()
         self.markup_handler = MarkupHandler()
         self.office_handler = OfficeHandler()
         self.epub_handler = EPUBHandler()
@@ -122,8 +121,8 @@ class ConversionPipeline:
             ".py": self.text_handler,
             ".go": self.text_handler,
             # Tabular (via MarkItDown)
-            ".csv": self.markitdown_handler,
-            ".tsv": self.markitdown_handler,
+            ".csv": self.tabular_handler,
+            ".tsv": self.tabular_handler,
             # Office/Spreadsheet/Slides
             ".xls": self.office_handler,
             ".xlsx": self.office_handler,

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro/converters/azure_doc_intel_wrapper.py

@@ -5,9 +5,14 @@ from .base import ConverterWrapper
 
 
 class DocIntelligenceWrapper(ConverterWrapper):
+
+    SUPPORTED_EXTENSIONS = DocumentIntelligenceHandler.SUPPORTED_EXTENSIONS
+
     def __init__(self):
         super().__init__("Azure Document Intelligence")
         self.converter = DocumentIntelligenceHandler()
 
     async def convert(self, file_path: str) -> Optional[str]:
-        return await self.converter.convert_to_md(file_path)
+        markdown = await self.converter.convert_to_md(file_path)
+        await self.converter.aclose()
+        return markdown

markitdown_pro-1.2.2/markitdown_pro/converters/tabular_wrapper.py

@@ -0,0 +1,31 @@
+from typing import Optional
+
+from ..common.logger import logger
+from ..common.utils import clean_markdown
+from ..handlers.tabular_handler import TabularHandler
+from .base import ConverterWrapper
+
+
+class TabularWrapper(ConverterWrapper):
+    SUPPORTED_FORMATS = (
+        "csv",
+        "tsv",
+        "xls",
+        "xlsx",
+    )
+
+    def __init__(self):
+        super().__init__("TabularHandler")
+        self.tabular_handler = TabularHandler()
+
+    async def convert(self, file_path: str) -> Optional[str]:
+        logger.debug(f"TabularWrapper: Converting {file_path} to markdown")
+        file_extension = file_path.split(".")[-1].lower()
+
+        if file_extension not in self.SUPPORTED_FORMATS:
+            logger.warning(f"TabularWrapper: Unsupported file format: {file_extension}")
+            return None
+
+        markdown = await self.tabular_handler.handle(file_path)
+        markdown = clean_markdown(markdown) if markdown else None
+        return markdown

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro/handlers/markup_handler.py

@@ -18,7 +18,7 @@ from .base_handler import BaseHandler
 class MarkupHandler(BaseHandler):
     """Handler for .html, .xml, .json, .ndjson, .yaml, .yml files."""
 
-    extensions = frozenset([".html", ".htm", ".xml", ".json", ".ndjson", ".yaml", ".yml"])
+    SUPPORTED_EXTENSIONS = frozenset([".html", ".htm", ".xml", ".json", ".ndjson", ".yaml", ".yml"])
 
     async def handle(self, file_path: str, *args, **kwargs) -> Optional[str]:
         logger.info(f"Processing markup file: {file_path}")

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro/handlers/office_handler.py

@@ -5,6 +5,7 @@ from ..common.utils import ensure_minimum_content
 from ..converters.azure_doc_intel_wrapper import DocIntelligenceWrapper
 from ..converters.base import ConverterWrapper
 from ..converters.markitdown_wrapper import MarkItDownWrapper
+from ..converters.tabular_wrapper import TabularWrapper
 from ..converters.unstructured_wrapper import UnstructuredWrapper
 from .base_handler import BaseHandler
 
@@ -13,9 +14,10 @@ class OfficeHandler(BaseHandler):
     """
     Convert Office documents to Markdown by trying a series of converters in order:
 
-      1) MarkItDown (fast, local parsing for Office formats)
-      2) Azure Document Intelligence (cloud extraction + OCR for embedded images)
-      3) Unstructured (robust fallback parser)
+      1) TabularHandler (specialized handler for tabular data)
+      2) MarkItDown (fast, local parsing for Office formats)
+      3) Azure Document Intelligence (cloud extraction + OCR for embedded images)
+      4) Unstructured (robust fallback parser)
 
     The first converter that returns non-trivial Markdown (as judged by
     `ensure_minimum_content`) wins. If all fail, `None` is returned.
@@ -34,12 +36,14 @@ class OfficeHandler(BaseHandler):
         super().__init__(*args, **kwargs)
 
         # Ordered, fastest → most comprehensive
+        self.tabular = TabularWrapper()
         self.markitdown = MarkItDownWrapper()
         self.doc_intelligence = DocIntelligenceWrapper()
         self.unstructured = UnstructuredWrapper()
 
         # (converter, human_readable_name) in the exact order to try
         self._pipeline: list[tuple[ConverterWrapper, str]] = [
+            (self.tabular, "TabularHandler"),
             (self.markitdown, "MarkItDown"),
             (self.doc_intelligence, "Azure Document Intelligence"),
             (self.unstructured, "Unstructured"),

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro/handlers/pdf_handler.py

@@ -89,10 +89,10 @@ class PDFHandler(BaseHandler):
             # Choose pipeline based on detected content type
             if pdf_type == PDFType.TEXT_ONLY:
                 pipeline = self.text_pipeline
-            elif pdf_type == PDFType.ALL_IMAGES:
+            elif pdf_type == PDFType.ALL_IMAGES or pdf_type == PDFType.TEXT_PLUS_IMAGES:
                 pipeline = self.image_pipeline
-            elif pdf_type == PDFType.TEXT_PLUS_IMAGES:
-                pipeline = self.text_pipeline + self.image_pipeline
+            # elif pdf_type == PDFType.TEXT_PLUS_IMAGES:
+            #     pipeline = self.text_pipeline + self.image_pipeline
             else:
                 # Fallback to text pipeline if detection returned an unexpected value
                 pipeline = self.text_pipeline

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro/handlers/tabular_handler.py

@@ -4,7 +4,6 @@ from typing import Optional
 import pandas as pd
 
 from ..common.logger import logger
-from ..common.utils import ensure_minimum_content
 from .base_handler import BaseHandler
 
 
@@ -24,12 +23,9 @@ class TabularHandler(BaseHandler):
                 df = pd.read_excel(file_path)
             else:
                 raise RuntimeError("Unsupported tabular format")
-            md = df.to_markdown(index=False)
 
-            if ensure_minimum_content(md):
-                return md
-            else:
-                raise RuntimeError(f"Tabular handler failed or insufficient content: {file_path}.")
+            md = df.to_markdown(index=False)
+            return md
         except Exception as e:
             logger.error(f"Error processing tabular file {file_path}: {e}")
             return None

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro/handlers/text_handler.py

@@ -11,7 +11,7 @@ from .base_handler import BaseHandler
 class TextHandler(BaseHandler):
     """Handler for .txt, .md, .py, .go, and other text/code files."""
 
-    extensions = frozenset([".txt", ".md", ".py", ".go"])
+    SUPPORTED_EXTENSIONS = frozenset([".txt", ".md", ".py", ".go"])
 
     async def handle(self, file_path: str, *args, **kwargs) -> Optional[str]:
         logger.info(f"Processing text file: {file_path}")

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro/services/azure_doc_intelligence.py

@@ -59,7 +59,7 @@ class DocumentIntelligenceHandler:
             ".heic",
         }
     )
-    extensions: frozenset[str] = frozenset(
+    SUPPORTED_EXTENSIONS: frozenset[str] = frozenset(
         list(pdf_extensions) + list(office_extensions) + list(image_extensions)
     )
 
@@ -112,7 +112,7 @@ class DocumentIntelligenceHandler:
             return None
 
         ext = file_path.suffix.lower()
-        if ext not in self.extensions:
+        if ext not in self.SUPPORTED_EXTENSIONS:
             logger.warning(f"DocumentIntelligenceHandler: unsupported extension '{ext}'.")
             return None
 
@@ -178,4 +178,4 @@ class DocumentIntelligenceHandler:
     @classmethod
     def supports(cls, path: str | Path) -> bool:
         """Return True if the file extension is supported by this handler."""
-        return Path(path).suffix.lower() in cls.extensions
+        return Path(path).suffix.lower() in cls.SUPPORTED_EXTENSIONS

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: markitdown-pro
-Version: 1.1.1
+Version: 1.2.2
 Summary: A package that converts almost any file format to Markdown.
 Author: Developer
 Classifier: Programming Language :: Python :: 3
@@ -26,6 +26,7 @@ Requires-Dist: azure-cognitiveservices-speech==1.42.0
 Requires-Dist: chardet==5.2.0
 Requires-Dist: nbformat
 Requires-Dist: youtube-transcript-api
+Requires-Dist: tabulate
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro.egg-info/SOURCES.txt

@@ -12,6 +12,7 @@ markitdown_pro.egg-info/top_level.txt
 markitdown_pro/common/__init__.py
 markitdown_pro/common/isolated_worker.py
 markitdown_pro/common/logger.py
+markitdown_pro/common/schemas.py
 markitdown_pro/common/utils.py
 markitdown_pro/converters/__init__.py
 markitdown_pro/converters/azure_doc_intel_wrapper.py
@@ -20,6 +21,7 @@ markitdown_pro/converters/base.py
 markitdown_pro/converters/gpt_vision_wrapper.py
 markitdown_pro/converters/markitdown_wrapper.py
 markitdown_pro/converters/pymupdf_wrapper.py
+markitdown_pro/converters/tabular_wrapper.py
 markitdown_pro/converters/unstructured_wrapper.py
 markitdown_pro/converters/youtube_wrapper.py
 markitdown_pro/handlers/__init__.py
@@ -44,7 +46,5 @@ tests/test_conversion_pipeline.py
 tests/handlers/__init__.py
 tests/handlers/test_email_handler.py
 tests/handlers/test_markup_handler.py
-tests/handlers/test_office_handler.py
-tests/handlers/test_pdf_handler.py
 tests/handlers/test_pst_handler.py
 tests/handlers/test_text_handler.py

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/markitdown_pro.egg-info/requires.txt

@@ -16,3 +16,4 @@ azure-cognitiveservices-speech==1.42.0
 chardet==5.2.0
 nbformat
 youtube-transcript-api
+tabulate

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="markitdown-pro",
-    version="1.1.1",
+    version="1.2.2",
     author="Developer",
     description="A package that converts almost any file format to Markdown.",
     long_description=open("README.md").read(),
@@ -30,6 +30,7 @@ setup(
         "chardet==5.2.0",
         "nbformat",
         "youtube-transcript-api",
+        "tabulate",
     ],
     classifiers=[
         "Programming Language :: Python :: 3",

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/tests/handlers/test_markup_handler.py

@@ -9,25 +9,15 @@ from tests.utils import list_files
 
 log = logging.getLogger(__name__)
 
-INCLUDE_EXTENSIONS = [
-    ".html",
-    ".htm",
-    ".xml",
-    ".json",
-    ".ndjson",
-    ".yaml",
-    ".yml",
-]
-
 TEST_FILES_PATH = data_path()
-ALL_FILES = list_files(TEST_FILES_PATH, include_ext=INCLUDE_EXTENSIONS, recursive=True)
+ALL_FILES = list_files(TEST_FILES_PATH, include_ext=MarkupHandler.extensions, recursive=True)
 log.info(f"Found {len(ALL_FILES)} test files in {TEST_FILES_PATH}.")
 
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("file_path", ALL_FILES, ids=[pretty_id(p) for p in ALL_FILES])
 async def test_files(file_path: Path, markup_handler: MarkupHandler):
-    log.info(f"Testing file: {file_path}")
+    log.info(f"{MarkupHandler.__name__}: Testing file {file_path}")
 
     markdown_text = await markup_handler.handle(str(file_path))
     assert markdown_text is not None, f"Conversion returned None for {file_path.name}"

{markitdown_pro-1.1.1 → markitdown_pro-1.2.2}/tests/handlers/test_text_handler.py

@@ -9,17 +9,17 @@ from tests.utils import list_files
 
 log = logging.getLogger(__name__)
 
-INCLUDE_EXTENSIONS = [".txt", ".md", ".py", ".go"]
-
 TEST_FILES_PATH = data_path()
-ALL_FILES = list_files(TEST_FILES_PATH, include_ext=INCLUDE_EXTENSIONS, recursive=True)
+ALL_FILES = list_files(
+    TEST_FILES_PATH, include_ext=TextHandler.SUPPORTED_EXTENSIONS, recursive=True
+)
 log.info(f"Found {len(ALL_FILES)} test files in {TEST_FILES_PATH}.")
 
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("file_path", ALL_FILES, ids=[pretty_id(p) for p in ALL_FILES])
 async def test_files(file_path: Path, text_handler: TextHandler):
-    log.info(f"Testing file: {file_path}")
+    log.info(f"{TextHandler.__name__}: Testing file {file_path}")
 
     markdown_text = await text_handler.handle(str(file_path))
     assert markdown_text is not None, f"Conversion returned None for {file_path.name}"

markitdown_pro-1.1.1/tests/handlers/test_office_handler.py

@@ -1,26 +0,0 @@
-import logging
-from pathlib import Path
-
-import pytest
-
-from markitdown_pro.handlers.office_handler import OfficeHandler
-from tests.fixtures import data_path, pretty_id
-from tests.utils import list_files
-
-log = logging.getLogger(__name__)
-
-INCLUDE_EXTENSIONS = [".docx", ".xlsx", ".pptx"]
-
-TEST_FILES_PATH = data_path()
-ALL_FILES = list_files(TEST_FILES_PATH, include_ext=INCLUDE_EXTENSIONS, recursive=True)
-log.info(f"Found {len(ALL_FILES)} test files in {TEST_FILES_PATH}.")
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize("file_path", ALL_FILES, ids=[pretty_id(p) for p in ALL_FILES])
-async def test_files(file_path: Path, office_handler: OfficeHandler):
-    log.info(f"Testing file: {file_path}")
-
-    markdown_text = await office_handler.handle(str(file_path))
-    assert markdown_text is not None, f"Conversion returned None for {file_path.name}"
-    assert isinstance(markdown_text, str), f"Output type is not str for {file_path.name}"

markitdown_pro-1.1.1/tests/handlers/test_pdf_handler.py

@@ -1,26 +0,0 @@
-import logging
-from pathlib import Path
-
-import pytest
-
-from markitdown_pro.handlers.pdf_handler import PDFHandler
-from tests.fixtures import data_path, pretty_id
-from tests.utils import list_files
-
-log = logging.getLogger(__name__)
-
-INCLUDE_EXTENSIONS = [".pdf"]
-
-TEST_FILES_PATH = data_path()
-ALL_FILES = list_files(TEST_FILES_PATH, include_ext=INCLUDE_EXTENSIONS, recursive=True)
-log.info(f"Found {len(ALL_FILES)} test files in {TEST_FILES_PATH}.")
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize("file_path", ALL_FILES, ids=[pretty_id(p) for p in ALL_FILES])
-async def test_files(file_path: Path, pdf_handler: PDFHandler):
-    log.info(f"Testing file: {file_path}")
-
-    markdown_text = await pdf_handler.handle(str(file_path))
-    assert markdown_text is not None, f"Conversion returned None for {file_path.name}"
-    assert isinstance(markdown_text, str), f"Output type is not str for {file_path.name}"