markitdown-pro 1.3.2__tar.gz → 1.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/PKG-INFO +3 -1
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/conversion_pipeline.py +37 -5
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/azure_doc_intel_wrapper.py +4 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/markitdown_wrapper.py +2 -1
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/pymupdf_wrapper.py +2 -5
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/audio_handler.py +5 -19
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/image_handler.py +23 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/office_handler.py +3 -8
- markitdown_pro-1.3.4/markitdown_pro/handlers/pdf_handler.py +377 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/services/azure_doc_intelligence.py +6 -16
- markitdown_pro-1.3.4/markitdown_pro/services/azure_speech.py +473 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro.egg-info/PKG-INFO +3 -1
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro.egg-info/SOURCES.txt +5 -2
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro.egg-info/requires.txt +2 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/setup.py +3 -1
- markitdown_pro-1.3.4/tests/conversion_pipeline/test_conversion_pipeline.py +31 -0
- markitdown_pro-1.3.4/tests/handlers/__init__.py +0 -0
- markitdown_pro-1.3.4/tests/page_count/__init__.py +0 -0
- markitdown_pro-1.3.4/tests/page_count/test_pdf_page_count.py +30 -0
- markitdown_pro-1.3.2/markitdown_pro/handlers/pdf_handler.py +0 -194
- markitdown_pro-1.3.2/markitdown_pro/services/azure_speech.py +0 -274
- markitdown_pro-1.3.2/tests/test_conversion_pipeline.py +0 -25
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/README.md +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/__init__.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/common/__init__.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/common/isolated_worker.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/common/logger.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/common/schemas.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/common/utils.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/__init__.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/azure_speech_wrapper.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/base.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/gpt_vision_wrapper.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/tabular_wrapper.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/unstructured_wrapper.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/youtube_wrapper.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/__init__.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/base_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/email_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/epub_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/ipynb_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/markitdown_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/markup_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/pst_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/tabular_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/text_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/services/__init__.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/services/openai_services.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro.egg-info/dependency_links.txt +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro.egg-info/top_level.txt +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/pyproject.toml +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/setup.cfg +0 -0
- {markitdown_pro-1.3.2/tests/handlers → markitdown_pro-1.3.4/tests/conversion_pipeline}/__init__.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_email_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_epub_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_image_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_ipynb_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_markitdown_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_markup_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_pst_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_tabular_handler.py +0 -0
- {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_text_handler.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: markitdown-pro
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.4
|
|
4
4
|
Summary: A package that converts almost any file format to Markdown.
|
|
5
5
|
Author: Developer
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -29,6 +29,8 @@ Requires-Dist: youtube-transcript-api
|
|
|
29
29
|
Requires-Dist: tabulate
|
|
30
30
|
Requires-Dist: ebooklib
|
|
31
31
|
Requires-Dist: beautifulsoup4
|
|
32
|
+
Requires-Dist: cairosvg
|
|
33
|
+
Requires-Dist: pillow_heif
|
|
32
34
|
Dynamic: author
|
|
33
35
|
Dynamic: classifier
|
|
34
36
|
Dynamic: description
|
|
@@ -39,6 +39,7 @@ import httpx
|
|
|
39
39
|
|
|
40
40
|
from .common.logger import logger
|
|
41
41
|
from .common.utils import clean_markdown, detect_extension, ensure_minimum_content
|
|
42
|
+
from .converters.azure_doc_intel_wrapper import DocIntelligenceWrapper
|
|
42
43
|
from .handlers.audio_handler import AudioHandler
|
|
43
44
|
from .handlers.base_handler import BaseHandler
|
|
44
45
|
from .handlers.email_handler import EmailHandler
|
|
@@ -88,6 +89,7 @@ class ConversionPipeline:
|
|
|
88
89
|
self.pst_handler = PSTHandler()
|
|
89
90
|
self.ipynb_handler = IpynbHandler()
|
|
90
91
|
self.markitdown_handler = MarkItDownHandler()
|
|
92
|
+
self.doc_intel_wrapper = DocIntelligenceWrapper()
|
|
91
93
|
|
|
92
94
|
# Map extensions to handlers (normalized lowercase with dot)
|
|
93
95
|
self.handlers_mapping: dict[str, BaseHandler] = {
|
|
@@ -108,7 +110,7 @@ class ConversionPipeline:
|
|
|
108
110
|
# region Images
|
|
109
111
|
".bmp": self.image_handler,
|
|
110
112
|
".gif": self.image_handler,
|
|
111
|
-
".heic": self.image_handler,
|
|
113
|
+
# ".heic": self.image_handler,
|
|
112
114
|
".jpeg": self.image_handler,
|
|
113
115
|
".jpg": self.image_handler,
|
|
114
116
|
".png": self.image_handler,
|
|
@@ -129,8 +131,8 @@ class ConversionPipeline:
|
|
|
129
131
|
".tsv": self.tabular_handler,
|
|
130
132
|
# endregion
|
|
131
133
|
# region Office/Spreadsheet/Slides
|
|
132
|
-
".xls": self.
|
|
133
|
-
".xlsx": self.
|
|
134
|
+
".xls": self.tabular_handler,
|
|
135
|
+
".xlsx": self.tabular_handler,
|
|
134
136
|
".docx": self.office_handler,
|
|
135
137
|
# ".odt": self.office_handler,
|
|
136
138
|
# ".rtf": self.office_handler,
|
|
@@ -161,7 +163,11 @@ class ConversionPipeline:
|
|
|
161
163
|
# ---------------------------------------------------------------------
|
|
162
164
|
|
|
163
165
|
async def convert_document_to_md(
|
|
164
|
-
self,
|
|
166
|
+
self,
|
|
167
|
+
file_path: str | Path,
|
|
168
|
+
output_md: Optional[str | Path] = None,
|
|
169
|
+
*args,
|
|
170
|
+
**kwargs,
|
|
165
171
|
) -> Optional[str]:
|
|
166
172
|
"""
|
|
167
173
|
Convert a **local file** to Markdown.
|
|
@@ -193,7 +199,7 @@ class ConversionPipeline:
|
|
|
193
199
|
raise RuntimeError(f"No handler for extension '{extension}'.")
|
|
194
200
|
|
|
195
201
|
try:
|
|
196
|
-
md_content = await handler.handle(str(file_path))
|
|
202
|
+
md_content = await handler.handle(str(file_path), *args, **kwargs)
|
|
197
203
|
if not md_content or not ensure_minimum_content(md_content):
|
|
198
204
|
return None
|
|
199
205
|
|
|
@@ -281,3 +287,29 @@ class ConversionPipeline:
|
|
|
281
287
|
return await self.convert_document_to_md(local_path, output_md=output_md)
|
|
282
288
|
finally:
|
|
283
289
|
local_path.unlink(missing_ok=True)
|
|
290
|
+
|
|
291
|
+
async def get_page_count(self, file_path: str | Path) -> Optional[int]:
|
|
292
|
+
"""
|
|
293
|
+
Get the page count of a document if supported by its handler.
|
|
294
|
+
|
|
295
|
+
Parameters
|
|
296
|
+
----------
|
|
297
|
+
file_path : str | Path
|
|
298
|
+
Path to the local file.
|
|
299
|
+
"""
|
|
300
|
+
file_path = Path(file_path)
|
|
301
|
+
if not file_path.is_file():
|
|
302
|
+
raise ValueError(f"The provided path '{file_path}' is not a valid file.")
|
|
303
|
+
|
|
304
|
+
# handle file extensions individually to avoid using the wrong handlers
|
|
305
|
+
extension = detect_extension(str(file_path)).lower()
|
|
306
|
+
|
|
307
|
+
if extension in self.image_handler.SUPPORTED_EXTENSIONS:
|
|
308
|
+
return await self.image_handler.get_page_count(file_path)
|
|
309
|
+
elif extension in self.pdf_handler.SUPPORTED_EXTENSIONS:
|
|
310
|
+
return await self.doc_intel_wrapper.get_page_count(file_path)
|
|
311
|
+
# elif extension in self.doc_intel_wrapper.SUPPORTED_EXTENSIONS:
|
|
312
|
+
# return await self.doc_intel_wrapper.get_page_count(file_path)
|
|
313
|
+
else:
|
|
314
|
+
logger.warning(f"ConversionPipeline: No page count support for extension '{extension}'")
|
|
315
|
+
return None
|
{markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/azure_doc_intel_wrapper.py
RENAMED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from pathlib import Path
|
|
1
2
|
from typing import Optional
|
|
2
3
|
|
|
3
4
|
from ..services.azure_doc_intelligence import DocumentIntelligenceHandler
|
|
@@ -16,3 +17,6 @@ class DocIntelligenceWrapper(ConverterWrapper):
|
|
|
16
17
|
markdown = await self.converter.convert_to_md(file_path)
|
|
17
18
|
await self.converter.aclose()
|
|
18
19
|
return markdown
|
|
20
|
+
|
|
21
|
+
async def get_page_count(self, file_path: str | Path) -> Optional[int]:
|
|
22
|
+
return await self.converter.get_page_count(file_path)
|
{markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/markitdown_wrapper.py
RENAMED
|
@@ -3,6 +3,7 @@ from typing import Optional
|
|
|
3
3
|
from markitdown_pro.handlers.markitdown_handler import MarkItDownHandler
|
|
4
4
|
|
|
5
5
|
from ..common.logger import logger
|
|
6
|
+
from ..common.utils import detect_extension
|
|
6
7
|
from .base import ConverterWrapper
|
|
7
8
|
|
|
8
9
|
|
|
@@ -14,7 +15,7 @@ class MarkItDownWrapper(ConverterWrapper):
|
|
|
14
15
|
self.handler = MarkItDownHandler(*args, **kwargs)
|
|
15
16
|
|
|
16
17
|
async def convert(self, file_path: str) -> Optional[str]:
|
|
17
|
-
file_extension =
|
|
18
|
+
file_extension = detect_extension(file_path)
|
|
18
19
|
if file_extension not in self.SUPPORTED_EXTENSIONS:
|
|
19
20
|
logger.warning(f"MarkItDownWrapper: Unsupported file format: {file_extension}")
|
|
20
21
|
return None
|
|
@@ -3,6 +3,7 @@ from typing import Optional
|
|
|
3
3
|
import fitz # PyMuPDF
|
|
4
4
|
|
|
5
5
|
from ..common.logger import logger
|
|
6
|
+
from ..common.utils import detect_extension
|
|
6
7
|
from .base import ConverterWrapper
|
|
7
8
|
|
|
8
9
|
|
|
@@ -13,11 +14,7 @@ class PyMuPDFWrapper(ConverterWrapper):
|
|
|
13
14
|
super().__init__("PyMuPDF")
|
|
14
15
|
|
|
15
16
|
async def convert(self, file_path: str) -> Optional[str]:
|
|
16
|
-
file_extension = file_path
|
|
17
|
-
logger.info(
|
|
18
|
-
f"PyMuPDFWrapper: Processing file: {file_path} with extension: {file_extension}"
|
|
19
|
-
)
|
|
20
|
-
|
|
17
|
+
file_extension = detect_extension(file_path)
|
|
21
18
|
if file_extension not in self.SUPPORTED_EXTENSIONS:
|
|
22
19
|
logger.warning(f"PyMuPDFWrapper: Unsupported file format: {file_extension}")
|
|
23
20
|
return None
|
|
@@ -3,9 +3,9 @@ from __future__ import annotations
|
|
|
3
3
|
from typing import Optional
|
|
4
4
|
|
|
5
5
|
from ..common.logger import logger
|
|
6
|
+
from ..common.utils import ensure_minimum_content
|
|
6
7
|
from ..converters.azure_speech_wrapper import AzureSpeechWrapper
|
|
7
8
|
from ..converters.base import ConverterWrapper
|
|
8
|
-
from ..converters.markitdown_wrapper import MarkItDownWrapper
|
|
9
9
|
from ..handlers.base_handler import BaseHandler
|
|
10
10
|
|
|
11
11
|
|
|
@@ -13,8 +13,7 @@ class AudioHandler(BaseHandler):
|
|
|
13
13
|
"""
|
|
14
14
|
Convert audio files to Markdown by trying two converters in order:
|
|
15
15
|
|
|
16
|
-
1) **
|
|
17
|
-
2) **Azure Speech** — cloud speech-to-text fallback for actual audio transcription.
|
|
16
|
+
1) **Azure Speech** — cloud speech-to-text fallback for actual audio transcription.
|
|
18
17
|
|
|
19
18
|
The first converter that yields non-trivial Markdown (as determined by
|
|
20
19
|
`ensure_minimum_content`) is returned. If both fail, `None` is returned.
|
|
@@ -26,27 +25,13 @@ class AudioHandler(BaseHandler):
|
|
|
26
25
|
- This handler runs converters **sequentially** to avoid unnecessary API calls.
|
|
27
26
|
"""
|
|
28
27
|
|
|
29
|
-
SUPPORTED_EXTENSIONS =
|
|
30
|
-
{
|
|
31
|
-
".mp3",
|
|
32
|
-
".wav",
|
|
33
|
-
".ogg",
|
|
34
|
-
".flac",
|
|
35
|
-
".m4a",
|
|
36
|
-
".aac",
|
|
37
|
-
".wma",
|
|
38
|
-
".webm",
|
|
39
|
-
".opus",
|
|
40
|
-
}
|
|
41
|
-
)
|
|
28
|
+
SUPPORTED_EXTENSIONS = AzureSpeechWrapper.SUPPORTED_EXTENSIONS
|
|
42
29
|
|
|
43
30
|
def __init__(self, *args, **kwargs) -> None:
|
|
44
31
|
super().__init__(*args, **kwargs)
|
|
45
|
-
self.markitdown = MarkItDownWrapper()
|
|
46
32
|
self.azure_speech = AzureSpeechWrapper()
|
|
47
33
|
|
|
48
34
|
self._pipeline: list[tuple[ConverterWrapper, str]] = [
|
|
49
|
-
(self.markitdown, "MarkItDown"),
|
|
50
35
|
(self.azure_speech, "Azure Speech"),
|
|
51
36
|
]
|
|
52
37
|
|
|
@@ -71,7 +56,8 @@ class AudioHandler(BaseHandler):
|
|
|
71
56
|
logger.info(f"AudioHandler: Trying {name} for '{file_path}'")
|
|
72
57
|
try:
|
|
73
58
|
md = await converter.convert(file_path)
|
|
74
|
-
|
|
59
|
+
if md and ensure_minimum_content(md):
|
|
60
|
+
return md
|
|
75
61
|
except Exception as e:
|
|
76
62
|
logger.error(f"AudioHandler: {name} failed for '{file_path}': {e}")
|
|
77
63
|
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import contextlib
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
from typing import Optional
|
|
3
4
|
|
|
4
5
|
from ..common.logger import logger
|
|
6
|
+
from ..common.utils import detect_extension
|
|
5
7
|
from ..converters.gpt_vision_wrapper import GPTVisionWrapper
|
|
6
8
|
from .base_handler import BaseHandler
|
|
7
9
|
|
|
@@ -37,6 +39,27 @@ class ImageHandler(BaseHandler):
|
|
|
37
39
|
if hasattr(self.gpt_vision, "gpt_vision"):
|
|
38
40
|
await self.gpt_vision.gpt_vision.aclose()
|
|
39
41
|
|
|
42
|
+
async def get_page_count(self, file_path: str | Path) -> Optional[int]:
|
|
43
|
+
"""
|
|
44
|
+
Determines the number of pages in the given image file.
|
|
45
|
+
"""
|
|
46
|
+
p = Path(file_path)
|
|
47
|
+
ext = detect_extension(str(p.absolute()))
|
|
48
|
+
if ext not in self.SUPPORTED_EXTENSIONS:
|
|
49
|
+
logger.warning(f"ImageHandler: Unsupported file format: {ext}")
|
|
50
|
+
return None
|
|
51
|
+
|
|
52
|
+
if ext in {".tif", ".tiff"}:
|
|
53
|
+
try:
|
|
54
|
+
from PIL import Image, ImageSequence # optional dependency
|
|
55
|
+
|
|
56
|
+
with Image.open(str(p)) as im:
|
|
57
|
+
return sum(1 for _ in ImageSequence.Iterator(im)) or 1
|
|
58
|
+
except Exception as e_tiff:
|
|
59
|
+
logger.warning(f"ImageHandler: local TIFF page count failed for '{p}': {e_tiff}")
|
|
60
|
+
else:
|
|
61
|
+
return 1
|
|
62
|
+
|
|
40
63
|
async def aclose(self) -> None:
|
|
41
64
|
if hasattr(self, "gpt_vision") and self.gpt_vision:
|
|
42
65
|
await self.gpt_vision.aclose()
|
|
@@ -4,7 +4,6 @@ from ..common.logger import logger
|
|
|
4
4
|
from ..converters.azure_doc_intel_wrapper import DocIntelligenceWrapper
|
|
5
5
|
from ..converters.base import ConverterWrapper
|
|
6
6
|
from ..converters.markitdown_wrapper import MarkItDownWrapper
|
|
7
|
-
from ..converters.tabular_wrapper import TabularWrapper
|
|
8
7
|
from ..converters.unstructured_wrapper import UnstructuredWrapper
|
|
9
8
|
from .base_handler import BaseHandler
|
|
10
9
|
|
|
@@ -13,10 +12,9 @@ class OfficeHandler(BaseHandler):
|
|
|
13
12
|
"""
|
|
14
13
|
Convert Office documents to Markdown by trying a series of converters in order:
|
|
15
14
|
|
|
16
|
-
1)
|
|
17
|
-
2)
|
|
18
|
-
3)
|
|
19
|
-
4) Unstructured (robust fallback parser)
|
|
15
|
+
1) MarkItDown (fast, local parsing for Office formats)
|
|
16
|
+
2) Azure Document Intelligence (cloud extraction + OCR for embedded images)
|
|
17
|
+
3) Unstructured (robust fallback parser)
|
|
20
18
|
|
|
21
19
|
The first converter that returns non-trivial Markdown (as judged by
|
|
22
20
|
`ensure_minimum_content`) wins. If all fail, `None` is returned.
|
|
@@ -28,21 +26,18 @@ class OfficeHandler(BaseHandler):
|
|
|
28
26
|
avoid unnecessary API calls when earlier options succeed.
|
|
29
27
|
"""
|
|
30
28
|
|
|
31
|
-
# Supported Office-like extensions (lowercase with dot)
|
|
32
29
|
SUPPORTED_EXTENSIONS = frozenset({".doc", ".docx", ".odt", ".rtf", ".ppt", ".pptx"})
|
|
33
30
|
|
|
34
31
|
def __init__(self, *args, **kwargs) -> None:
|
|
35
32
|
super().__init__(*args, **kwargs)
|
|
36
33
|
|
|
37
34
|
# Ordered, fastest → most comprehensive
|
|
38
|
-
self.tabular = TabularWrapper()
|
|
39
35
|
self.markitdown = MarkItDownWrapper()
|
|
40
36
|
self.doc_intelligence = DocIntelligenceWrapper()
|
|
41
37
|
self.unstructured = UnstructuredWrapper()
|
|
42
38
|
|
|
43
39
|
# (converter, human_readable_name) in the exact order to try
|
|
44
40
|
self._pipeline: list[tuple[ConverterWrapper, str]] = [
|
|
45
|
-
(self.tabular, "TabularHandler"),
|
|
46
41
|
(self.markitdown, "MarkItDown"),
|
|
47
42
|
(self.doc_intelligence, "Azure Document Intelligence"),
|
|
48
43
|
(self.unstructured, "Unstructured"),
|
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
import tempfile
|
|
5
|
+
from contextlib import contextmanager
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from itertools import groupby
|
|
8
|
+
from operator import itemgetter
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Optional
|
|
11
|
+
|
|
12
|
+
import fitz
|
|
13
|
+
|
|
14
|
+
from ..common.logger import logger
|
|
15
|
+
from ..common.utils import detect_extension
|
|
16
|
+
from ..converters.azure_doc_intel_wrapper import DocIntelligenceWrapper
|
|
17
|
+
from ..converters.gpt_vision_wrapper import GPTVisionWrapper
|
|
18
|
+
from ..converters.markitdown_wrapper import MarkItDownWrapper
|
|
19
|
+
from ..converters.pymupdf_wrapper import PyMuPDFWrapper
|
|
20
|
+
from ..converters.unstructured_wrapper import UnstructuredWrapper
|
|
21
|
+
from .base_handler import BaseHandler
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ConversionError(Exception):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# region helpers: page parsing / slicing
|
|
29
|
+
def _parse_pages_spec(pages: str | list[int], total_pages: int) -> list[int]:
|
|
30
|
+
"""Return a sorted, de-duped, 0-based list of page indexes within bounds."""
|
|
31
|
+
if pages is None:
|
|
32
|
+
return list(range(total_pages))
|
|
33
|
+
if isinstance(pages, (list, tuple)):
|
|
34
|
+
idxs = [int(p) - 1 if p >= 1 else int(p) for p in pages] # allow 1-based
|
|
35
|
+
else:
|
|
36
|
+
idxs = []
|
|
37
|
+
for tok in re.split(r"[,\s]+", pages.strip()):
|
|
38
|
+
if not tok:
|
|
39
|
+
continue
|
|
40
|
+
if "-" in tok:
|
|
41
|
+
a, b = tok.split("-", 1)
|
|
42
|
+
a, b = int(a), int(b)
|
|
43
|
+
if a > b:
|
|
44
|
+
a, b = b, a
|
|
45
|
+
idxs.extend(range(a - 1, b)) # user passes 1-based
|
|
46
|
+
else:
|
|
47
|
+
idxs.append(int(tok) - 1)
|
|
48
|
+
idxs = sorted(set(i for i in idxs if 0 <= i < total_pages))
|
|
49
|
+
return idxs
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _coalesce_runs(idxs: list[int]) -> list[tuple[int, int]]:
|
|
53
|
+
"""Turn [0,1,2,4,7,8] into [(0,2),(4,4),(7,8)]"""
|
|
54
|
+
runs = []
|
|
55
|
+
for k, g in groupby(enumerate(idxs), key=lambda t: t[0] - t[1]):
|
|
56
|
+
group = list(map(itemgetter(1), g))
|
|
57
|
+
runs.append((group[0], group[-1]))
|
|
58
|
+
return runs
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@contextmanager
|
|
62
|
+
def _maybe_slice_pdf(src_path: str, page_idxs: list[int], total_pages: int):
|
|
63
|
+
"""Yield original pdf if full-range, else a temp sliced pdf with selected pages."""
|
|
64
|
+
full_range = (
|
|
65
|
+
len(page_idxs) == total_pages and page_idxs[0] == 0 and page_idxs[-1] == total_pages - 1
|
|
66
|
+
)
|
|
67
|
+
if full_range:
|
|
68
|
+
yield src_path
|
|
69
|
+
return
|
|
70
|
+
import fitz
|
|
71
|
+
|
|
72
|
+
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf")
|
|
73
|
+
tmp.close()
|
|
74
|
+
try:
|
|
75
|
+
with fitz.open(src_path) as src:
|
|
76
|
+
dst = fitz.open()
|
|
77
|
+
for start, end in _coalesce_runs(page_idxs):
|
|
78
|
+
dst.insert_pdf(src, from_page=start, to_page=end)
|
|
79
|
+
dst.save(tmp.name)
|
|
80
|
+
dst.close()
|
|
81
|
+
yield tmp.name
|
|
82
|
+
finally:
|
|
83
|
+
try:
|
|
84
|
+
os.remove(tmp.name)
|
|
85
|
+
except Exception:
|
|
86
|
+
pass
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# endregion
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class PDFType(Enum):
|
|
93
|
+
"""
|
|
94
|
+
Simple classification of a PDF based on page contents.
|
|
95
|
+
|
|
96
|
+
- TEXT_ONLY: Every page has "enough" extractable text; no pages contain images.
|
|
97
|
+
- TEXT_PLUS_IMAGES: At least one page contains text and at least one page contains images.
|
|
98
|
+
- ALL_IMAGES: Every page has at least one image and no page has "enough" text
|
|
99
|
+
(typical of scanned/image-only PDFs).
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
TEXT_ONLY = "TEXT_ONLY"
|
|
103
|
+
TEXT_PLUS_IMAGES = "TEXT_PLUS_IMAGES"
|
|
104
|
+
ALL_IMAGES = "ALL_IMAGES"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class PDFHandler(BaseHandler):
|
|
108
|
+
"""
|
|
109
|
+
Orchestrates PDF → Markdown conversion by trying a sequence of converters
|
|
110
|
+
chosen from a text-oriented pipeline or an image/OCR pipeline depending
|
|
111
|
+
on a quick content scan of the PDF.
|
|
112
|
+
|
|
113
|
+
The flow is:
|
|
114
|
+
1) `_detect_pdf_type` runs a fast pass over the PDF (offloaded to a thread)
|
|
115
|
+
to count text-bearing and image-bearing pages.
|
|
116
|
+
2) Based on the detected `PDFType`, choose a pipeline:
|
|
117
|
+
- TEXT_ONLY: text pipeline
|
|
118
|
+
- ALL_IMAGES: image pipeline (OCR)
|
|
119
|
+
- TEXT_PLUS_IMAGES: image pipeline (OCR)
|
|
120
|
+
3) Iterate the chosen converters in order until one returns acceptable Markdown.
|
|
121
|
+
|
|
122
|
+
Notes:
|
|
123
|
+
- Converters MUST implement an async `convert(file_path: str) -> Optional[str]`.
|
|
124
|
+
- `ensure_minimum_content` is used to filter out trivial/empty results.
|
|
125
|
+
"""
|
|
126
|
+
|
|
127
|
+
# File extensions handled by this handler
|
|
128
|
+
SUPPORTED_EXTENSIONS = frozenset([".pdf"])
|
|
129
|
+
|
|
130
|
+
def __init__(self, *args, **kwargs) -> None:
|
|
131
|
+
super().__init__(*args, **kwargs)
|
|
132
|
+
|
|
133
|
+
# Text-first converters (fastest/cheapest first, progressively more robust)
|
|
134
|
+
self.markitdown = MarkItDownWrapper()
|
|
135
|
+
self.unstructured = UnstructuredWrapper()
|
|
136
|
+
self.pymu = PyMuPDFWrapper()
|
|
137
|
+
self.azure_docint = DocIntelligenceWrapper()
|
|
138
|
+
|
|
139
|
+
# OCR-based converter for scanned/image-only PDFs
|
|
140
|
+
self.gpt_vision = GPTVisionWrapper()
|
|
141
|
+
|
|
142
|
+
# Pipelines in the order they should be attempted
|
|
143
|
+
self.text_pipeline = [self.markitdown, self.unstructured, self.pymu, self.azure_docint]
|
|
144
|
+
self.image_pipeline = [self.gpt_vision]
|
|
145
|
+
|
|
146
|
+
async def handle(self, file_path: str, *args, **kwargs) -> Optional[str]:
|
|
147
|
+
"""
|
|
148
|
+
Convert a PDF to Markdown, honoring page selection from kwargs only.
|
|
149
|
+
|
|
150
|
+
Expected kwargs:
|
|
151
|
+
- pages: list[int] (1-based page numbers). If absent/invalid → process all pages.
|
|
152
|
+
"""
|
|
153
|
+
try:
|
|
154
|
+
# Resolve selection to 0-based page indexes (offloaded: PyMuPDF is CPU-bound)
|
|
155
|
+
def _resolve():
|
|
156
|
+
with fitz.open(file_path) as doc:
|
|
157
|
+
total = doc.page_count
|
|
158
|
+
pages_kw = kwargs.get("pages", None)
|
|
159
|
+
if isinstance(pages_kw, list) and all(isinstance(p, int) for p in pages_kw):
|
|
160
|
+
idxs = sorted(set(p - 1 for p in pages_kw))
|
|
161
|
+
idxs = [i for i in idxs if 0 <= i < total]
|
|
162
|
+
if not idxs:
|
|
163
|
+
logger.warning(
|
|
164
|
+
f"PDFHandler: 'pages' kwarg produced no valid pages for {file_path}; defaulting to all pages."
|
|
165
|
+
)
|
|
166
|
+
idxs = list(range(total))
|
|
167
|
+
else:
|
|
168
|
+
if pages_kw is not None:
|
|
169
|
+
logger.warning(
|
|
170
|
+
"PDFHandler: 'pages' must be a list[int] (1-based). Defaulting to all pages."
|
|
171
|
+
)
|
|
172
|
+
idxs = list(range(total))
|
|
173
|
+
return idxs, total
|
|
174
|
+
|
|
175
|
+
page_idxs, total_pages = await asyncio.to_thread(_resolve)
|
|
176
|
+
|
|
177
|
+
# Detect type using only the selected pages
|
|
178
|
+
pdf_type = await self._detect_pdf_type(file_path, page_indexes=page_idxs)
|
|
179
|
+
pipeline = (
|
|
180
|
+
self.image_pipeline
|
|
181
|
+
if pdf_type in (PDFType.ALL_IMAGES, PDFType.TEXT_PLUS_IMAGES)
|
|
182
|
+
else self.text_pipeline
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
# Slice once and feed to converters
|
|
186
|
+
with _maybe_slice_pdf(file_path, page_idxs, total_pages) as to_process:
|
|
187
|
+
for converter in pipeline:
|
|
188
|
+
preview = [i + 1 for i in page_idxs[:5]]
|
|
189
|
+
logger.info(
|
|
190
|
+
f"PDFHandler: Trying {converter.name} for {file_path} pages={preview}{'...' if len(page_idxs) > 5 else ''}"
|
|
191
|
+
)
|
|
192
|
+
try:
|
|
193
|
+
md_content = await converter.convert(to_process)
|
|
194
|
+
return md_content
|
|
195
|
+
except Exception as e:
|
|
196
|
+
logger.error(
|
|
197
|
+
f"PDFHandler: Converter {converter.name} failed for subset {file_path}: {e}"
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
raise RuntimeError(f"PDF conversion failed with all converters for {file_path}")
|
|
201
|
+
except Exception as e:
|
|
202
|
+
logger.error(f"PDFHandler: Error handling PDF '{file_path}': {e}")
|
|
203
|
+
return None
|
|
204
|
+
|
|
205
|
+
async def _detect_pdf_type(
|
|
206
|
+
self,
|
|
207
|
+
file_path: str,
|
|
208
|
+
page_indexes: list[int] | None = None,
|
|
209
|
+
) -> PDFType:
|
|
210
|
+
"""
|
|
211
|
+
Quickly scan the PDF to decide which pipeline to run.
|
|
212
|
+
|
|
213
|
+
Heuristic
|
|
214
|
+
---------
|
|
215
|
+
- A page is considered to have "text" if the extracted text length ≥ `min_text_length_threshold`.
|
|
216
|
+
- A page is considered to have "images" if `page.get_images(full=True)` returns any entries.
|
|
217
|
+
- The final classification is derived from counts across all pages.
|
|
218
|
+
|
|
219
|
+
Parameters
|
|
220
|
+
----------
|
|
221
|
+
file_path : str
|
|
222
|
+
Path to the input PDF.
|
|
223
|
+
|
|
224
|
+
Returns
|
|
225
|
+
-------
|
|
226
|
+
PDFType
|
|
227
|
+
Classification result used to pick the processing pipeline.
|
|
228
|
+
|
|
229
|
+
Raises
|
|
230
|
+
------
|
|
231
|
+
Exception
|
|
232
|
+
If the file cannot be opened or scanned.
|
|
233
|
+
"""
|
|
234
|
+
min_text_length_threshold = 20
|
|
235
|
+
try:
|
|
236
|
+
|
|
237
|
+
def _scan() -> PDFType:
|
|
238
|
+
with fitz.open(file_path) as doc:
|
|
239
|
+
total_pages = doc.page_count
|
|
240
|
+
idxs = page_indexes or list(range(total_pages))
|
|
241
|
+
pages_with_text = 0
|
|
242
|
+
pages_with_images = 0
|
|
243
|
+
for i in idxs:
|
|
244
|
+
page = doc.load_page(i)
|
|
245
|
+
if not page:
|
|
246
|
+
continue
|
|
247
|
+
if len((page.get_text() or "").strip()) >= min_text_length_threshold:
|
|
248
|
+
pages_with_text += 1
|
|
249
|
+
if page.get_images(full=True):
|
|
250
|
+
pages_with_images += 1
|
|
251
|
+
is_text_only = pages_with_text == len(idxs) and pages_with_images == 0
|
|
252
|
+
is_all_images = pages_with_images == len(idxs) and pages_with_text == 0
|
|
253
|
+
has_text_and_images = pages_with_text > 0 and pages_with_images > 0
|
|
254
|
+
if is_text_only:
|
|
255
|
+
return PDFType.TEXT_ONLY
|
|
256
|
+
elif is_all_images:
|
|
257
|
+
return PDFType.ALL_IMAGES
|
|
258
|
+
elif has_text_and_images:
|
|
259
|
+
return PDFType.TEXT_PLUS_IMAGES
|
|
260
|
+
return PDFType.TEXT_PLUS_IMAGES
|
|
261
|
+
|
|
262
|
+
return await asyncio.to_thread(_scan)
|
|
263
|
+
except Exception as e:
|
|
264
|
+
logger.error(f"PDFHandler: Error analyzing PDF '{file_path}': {e}")
|
|
265
|
+
raise
|
|
266
|
+
|
|
267
|
+
async def get_page_count(self, file_path: str | os.PathLike) -> Optional[int]:
|
|
268
|
+
p = Path(file_path)
|
|
269
|
+
ext = detect_extension(str(p.absolute()))
|
|
270
|
+
if ext not in self.SUPPORTED_EXTENSIONS:
|
|
271
|
+
logger.warning(f"DocumentIntelligenceHandler: Unsupported file format: {ext}")
|
|
272
|
+
return None
|
|
273
|
+
if ext in self.SUPPORTED_EXTENSIONS:
|
|
274
|
+
try:
|
|
275
|
+
import pypdf as _pypdf
|
|
276
|
+
|
|
277
|
+
try:
|
|
278
|
+
reader = _pypdf.PdfReader(str(p))
|
|
279
|
+
except Exception:
|
|
280
|
+
with open(p, "rb") as fh:
|
|
281
|
+
reader = _pypdf.PdfReader(fh)
|
|
282
|
+
return len(reader.pages)
|
|
283
|
+
except Exception as e_pdf:
|
|
284
|
+
logger.warning(
|
|
285
|
+
f"DocumentIntelligenceHandler: local PDF page count failed for '{p}': {e_pdf}"
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
# async def _run_soffice_convert(
|
|
289
|
+
# self, file_path: Path, filter_name: Optional[str] = None, timeout_s: int = 180
|
|
290
|
+
# ) -> Path:
|
|
291
|
+
# """
|
|
292
|
+
# Generic converter via LibreOffice 'soffice' headless CLI.
|
|
293
|
+
# `filter_name` can be 'writer_pdf_Export' for DOCX or 'impress_pdf_Export' for PPTX.
|
|
294
|
+
# Returns final PDF path placed next to the source file.
|
|
295
|
+
# """
|
|
296
|
+
# soffice = shutil.which("soffice") or shutil.which("libreoffice")
|
|
297
|
+
# if not soffice:
|
|
298
|
+
# raise ConversionError("LibreOffice not found on PATH (need 'soffice').")
|
|
299
|
+
|
|
300
|
+
# if not file_path.exists():
|
|
301
|
+
# raise FileNotFoundError(file_path)
|
|
302
|
+
|
|
303
|
+
# # Output to a temp dir, then move next to source
|
|
304
|
+
# tmp_out = Path(tempfile.mkdtemp())
|
|
305
|
+
# try:
|
|
306
|
+
# convert_to = "pdf" if not filter_name else f"pdf:{filter_name}"
|
|
307
|
+
# cmd = [
|
|
308
|
+
# soffice,
|
|
309
|
+
# "--headless",
|
|
310
|
+
# "--invisible",
|
|
311
|
+
# "--norestore",
|
|
312
|
+
# "--nodefault",
|
|
313
|
+
# "--nolockcheck",
|
|
314
|
+
# "--nofirststartwizard",
|
|
315
|
+
# "--convert-to",
|
|
316
|
+
# convert_to,
|
|
317
|
+
# str(file_path),
|
|
318
|
+
# "--outdir",
|
|
319
|
+
# str(tmp_out),
|
|
320
|
+
# ]
|
|
321
|
+
|
|
322
|
+
# proc = await asyncio.create_subprocess_exec(
|
|
323
|
+
# *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
|
|
324
|
+
# )
|
|
325
|
+
# try:
|
|
326
|
+
# stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout_s)
|
|
327
|
+
# except asyncio.TimeoutError:
|
|
328
|
+
# proc.kill()
|
|
329
|
+
# raise ConversionError("LibreOffice timed out during conversion")
|
|
330
|
+
|
|
331
|
+
# if proc.returncode != 0:
|
|
332
|
+
# raise ConversionError(
|
|
333
|
+
# f"LibreOffice failed (code {proc.returncode}). Stderr: {stderr.decode(errors='ignore')}"
|
|
334
|
+
# )
|
|
335
|
+
|
|
336
|
+
# produced = tmp_out / (file_path.stem + ".pdf")
|
|
337
|
+
# if not produced.exists():
|
|
338
|
+
# # Some LO versions may change the output name; find the first PDF
|
|
339
|
+
# candidates = list(tmp_out.glob("*.pdf"))
|
|
340
|
+
# if not candidates:
|
|
341
|
+
# raise ConversionError("Expected PDF not created by LibreOffice")
|
|
342
|
+
# produced = candidates[0]
|
|
343
|
+
|
|
344
|
+
# final_pdf = file_path.with_suffix(".pdf")
|
|
345
|
+
# if final_pdf.exists():
|
|
346
|
+
# final_pdf.unlink()
|
|
347
|
+
# produced.replace(final_pdf)
|
|
348
|
+
# return final_pdf
|
|
349
|
+
# finally:
|
|
350
|
+
# try:
|
|
351
|
+
# # Clean temp dir
|
|
352
|
+
# for f in tmp_out.glob("*"):
|
|
353
|
+
# try:
|
|
354
|
+
# f.unlink()
|
|
355
|
+
# except Exception:
|
|
356
|
+
# pass
|
|
357
|
+
# tmp_out.rmdir()
|
|
358
|
+
# except Exception:
|
|
359
|
+
# pass
|
|
360
|
+
|
|
361
|
+
# async def docx_to_pdf(self, file_path: str | Path, timeout_s: int = 180) -> Path:
|
|
362
|
+
# src = Path(file_path).resolve()
|
|
363
|
+
# if src.suffix.lower() != ".docx":
|
|
364
|
+
# raise ValueError("docx_to_pdf expects a .docx file")
|
|
365
|
+
# # writer filter produces better pagination on Writer docs
|
|
366
|
+
# return await self._run_soffice_convert(
|
|
367
|
+
# src, filter_name="writer_pdf_Export", timeout_s=timeout_s
|
|
368
|
+
# )
|
|
369
|
+
|
|
370
|
+
# async def pptx_to_pdf(self, file_path: str | Path, timeout_s: int = 180) -> Path:
|
|
371
|
+
# src = Path(file_path).resolve()
|
|
372
|
+
# if src.suffix.lower() != ".pptx":
|
|
373
|
+
# raise ValueError("pptx_to_pdf expects a .pptx file")
|
|
374
|
+
# # impress filter produces better pagination on Impress/PowerPoint
|
|
375
|
+
# return await self._run_soffice_convert(
|
|
376
|
+
# src, filter_name="impress_pdf_Export", timeout_s=timeout_s
|
|
377
|
+
# )
|