markitdown-pro 1.3.2__tar.gz → 1.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/PKG-INFO +3 -1
  2. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/conversion_pipeline.py +37 -5
  3. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/azure_doc_intel_wrapper.py +4 -0
  4. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/markitdown_wrapper.py +2 -1
  5. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/pymupdf_wrapper.py +2 -5
  6. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/audio_handler.py +5 -19
  7. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/image_handler.py +23 -0
  8. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/office_handler.py +3 -8
  9. markitdown_pro-1.3.4/markitdown_pro/handlers/pdf_handler.py +377 -0
  10. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/services/azure_doc_intelligence.py +6 -16
  11. markitdown_pro-1.3.4/markitdown_pro/services/azure_speech.py +473 -0
  12. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro.egg-info/PKG-INFO +3 -1
  13. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro.egg-info/SOURCES.txt +5 -2
  14. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro.egg-info/requires.txt +2 -0
  15. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/setup.py +3 -1
  16. markitdown_pro-1.3.4/tests/conversion_pipeline/test_conversion_pipeline.py +31 -0
  17. markitdown_pro-1.3.4/tests/handlers/__init__.py +0 -0
  18. markitdown_pro-1.3.4/tests/page_count/__init__.py +0 -0
  19. markitdown_pro-1.3.4/tests/page_count/test_pdf_page_count.py +30 -0
  20. markitdown_pro-1.3.2/markitdown_pro/handlers/pdf_handler.py +0 -194
  21. markitdown_pro-1.3.2/markitdown_pro/services/azure_speech.py +0 -274
  22. markitdown_pro-1.3.2/tests/test_conversion_pipeline.py +0 -25
  23. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/README.md +0 -0
  24. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/__init__.py +0 -0
  25. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/common/__init__.py +0 -0
  26. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/common/isolated_worker.py +0 -0
  27. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/common/logger.py +0 -0
  28. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/common/schemas.py +0 -0
  29. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/common/utils.py +0 -0
  30. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/__init__.py +0 -0
  31. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/azure_speech_wrapper.py +0 -0
  32. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/base.py +0 -0
  33. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/gpt_vision_wrapper.py +0 -0
  34. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/tabular_wrapper.py +0 -0
  35. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/unstructured_wrapper.py +0 -0
  36. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/converters/youtube_wrapper.py +0 -0
  37. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/__init__.py +0 -0
  38. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/base_handler.py +0 -0
  39. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/email_handler.py +0 -0
  40. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/epub_handler.py +0 -0
  41. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/ipynb_handler.py +0 -0
  42. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/markitdown_handler.py +0 -0
  43. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/markup_handler.py +0 -0
  44. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/pst_handler.py +0 -0
  45. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/tabular_handler.py +0 -0
  46. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/handlers/text_handler.py +0 -0
  47. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/services/__init__.py +0 -0
  48. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro/services/openai_services.py +0 -0
  49. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro.egg-info/dependency_links.txt +0 -0
  50. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/markitdown_pro.egg-info/top_level.txt +0 -0
  51. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/pyproject.toml +0 -0
  52. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/setup.cfg +0 -0
  53. {markitdown_pro-1.3.2/tests/handlers → markitdown_pro-1.3.4/tests/conversion_pipeline}/__init__.py +0 -0
  54. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_email_handler.py +0 -0
  55. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_epub_handler.py +0 -0
  56. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_image_handler.py +0 -0
  57. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_ipynb_handler.py +0 -0
  58. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_markitdown_handler.py +0 -0
  59. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_markup_handler.py +0 -0
  60. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_pst_handler.py +0 -0
  61. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_tabular_handler.py +0 -0
  62. {markitdown_pro-1.3.2 → markitdown_pro-1.3.4}/tests/handlers/test_text_handler.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: markitdown-pro
3
- Version: 1.3.2
3
+ Version: 1.3.4
4
4
  Summary: A package that converts almost any file format to Markdown.
5
5
  Author: Developer
6
6
  Classifier: Programming Language :: Python :: 3
@@ -29,6 +29,8 @@ Requires-Dist: youtube-transcript-api
29
29
  Requires-Dist: tabulate
30
30
  Requires-Dist: ebooklib
31
31
  Requires-Dist: beautifulsoup4
32
+ Requires-Dist: cairosvg
33
+ Requires-Dist: pillow_heif
32
34
  Dynamic: author
33
35
  Dynamic: classifier
34
36
  Dynamic: description
@@ -39,6 +39,7 @@ import httpx
39
39
 
40
40
  from .common.logger import logger
41
41
  from .common.utils import clean_markdown, detect_extension, ensure_minimum_content
42
+ from .converters.azure_doc_intel_wrapper import DocIntelligenceWrapper
42
43
  from .handlers.audio_handler import AudioHandler
43
44
  from .handlers.base_handler import BaseHandler
44
45
  from .handlers.email_handler import EmailHandler
@@ -88,6 +89,7 @@ class ConversionPipeline:
88
89
  self.pst_handler = PSTHandler()
89
90
  self.ipynb_handler = IpynbHandler()
90
91
  self.markitdown_handler = MarkItDownHandler()
92
+ self.doc_intel_wrapper = DocIntelligenceWrapper()
91
93
 
92
94
  # Map extensions to handlers (normalized lowercase with dot)
93
95
  self.handlers_mapping: dict[str, BaseHandler] = {
@@ -108,7 +110,7 @@ class ConversionPipeline:
108
110
  # region Images
109
111
  ".bmp": self.image_handler,
110
112
  ".gif": self.image_handler,
111
- ".heic": self.image_handler,
113
+ # ".heic": self.image_handler,
112
114
  ".jpeg": self.image_handler,
113
115
  ".jpg": self.image_handler,
114
116
  ".png": self.image_handler,
@@ -129,8 +131,8 @@ class ConversionPipeline:
129
131
  ".tsv": self.tabular_handler,
130
132
  # endregion
131
133
  # region Office/Spreadsheet/Slides
132
- ".xls": self.office_handler,
133
- ".xlsx": self.office_handler,
134
+ ".xls": self.tabular_handler,
135
+ ".xlsx": self.tabular_handler,
134
136
  ".docx": self.office_handler,
135
137
  # ".odt": self.office_handler,
136
138
  # ".rtf": self.office_handler,
@@ -161,7 +163,11 @@ class ConversionPipeline:
161
163
  # ---------------------------------------------------------------------
162
164
 
163
165
  async def convert_document_to_md(
164
- self, file_path: str | Path, output_md: Optional[str | Path] = None
166
+ self,
167
+ file_path: str | Path,
168
+ output_md: Optional[str | Path] = None,
169
+ *args,
170
+ **kwargs,
165
171
  ) -> Optional[str]:
166
172
  """
167
173
  Convert a **local file** to Markdown.
@@ -193,7 +199,7 @@ class ConversionPipeline:
193
199
  raise RuntimeError(f"No handler for extension '{extension}'.")
194
200
 
195
201
  try:
196
- md_content = await handler.handle(str(file_path))
202
+ md_content = await handler.handle(str(file_path), *args, **kwargs)
197
203
  if not md_content or not ensure_minimum_content(md_content):
198
204
  return None
199
205
 
@@ -281,3 +287,29 @@ class ConversionPipeline:
281
287
  return await self.convert_document_to_md(local_path, output_md=output_md)
282
288
  finally:
283
289
  local_path.unlink(missing_ok=True)
290
+
291
+ async def get_page_count(self, file_path: str | Path) -> Optional[int]:
292
+ """
293
+ Get the page count of a document if supported by its handler.
294
+
295
+ Parameters
296
+ ----------
297
+ file_path : str | Path
298
+ Path to the local file.
299
+ """
300
+ file_path = Path(file_path)
301
+ if not file_path.is_file():
302
+ raise ValueError(f"The provided path '{file_path}' is not a valid file.")
303
+
304
+ # handle file extensions individually to avoid using the wrong handlers
305
+ extension = detect_extension(str(file_path)).lower()
306
+
307
+ if extension in self.image_handler.SUPPORTED_EXTENSIONS:
308
+ return await self.image_handler.get_page_count(file_path)
309
+ elif extension in self.pdf_handler.SUPPORTED_EXTENSIONS:
310
+ return await self.doc_intel_wrapper.get_page_count(file_path)
311
+ # elif extension in self.doc_intel_wrapper.SUPPORTED_EXTENSIONS:
312
+ # return await self.doc_intel_wrapper.get_page_count(file_path)
313
+ else:
314
+ logger.warning(f"ConversionPipeline: No page count support for extension '{extension}'")
315
+ return None
@@ -1,3 +1,4 @@
1
+ from pathlib import Path
1
2
  from typing import Optional
2
3
 
3
4
  from ..services.azure_doc_intelligence import DocumentIntelligenceHandler
@@ -16,3 +17,6 @@ class DocIntelligenceWrapper(ConverterWrapper):
16
17
  markdown = await self.converter.convert_to_md(file_path)
17
18
  await self.converter.aclose()
18
19
  return markdown
20
+
21
+ async def get_page_count(self, file_path: str | Path) -> Optional[int]:
22
+ return await self.converter.get_page_count(file_path)
@@ -3,6 +3,7 @@ from typing import Optional
3
3
  from markitdown_pro.handlers.markitdown_handler import MarkItDownHandler
4
4
 
5
5
  from ..common.logger import logger
6
+ from ..common.utils import detect_extension
6
7
  from .base import ConverterWrapper
7
8
 
8
9
 
@@ -14,7 +15,7 @@ class MarkItDownWrapper(ConverterWrapper):
14
15
  self.handler = MarkItDownHandler(*args, **kwargs)
15
16
 
16
17
  async def convert(self, file_path: str) -> Optional[str]:
17
- file_extension = f".{file_path.split('.')[-1].lower()}"
18
+ file_extension = detect_extension(file_path)
18
19
  if file_extension not in self.SUPPORTED_EXTENSIONS:
19
20
  logger.warning(f"MarkItDownWrapper: Unsupported file format: {file_extension}")
20
21
  return None
@@ -3,6 +3,7 @@ from typing import Optional
3
3
  import fitz # PyMuPDF
4
4
 
5
5
  from ..common.logger import logger
6
+ from ..common.utils import detect_extension
6
7
  from .base import ConverterWrapper
7
8
 
8
9
 
@@ -13,11 +14,7 @@ class PyMuPDFWrapper(ConverterWrapper):
13
14
  super().__init__("PyMuPDF")
14
15
 
15
16
  async def convert(self, file_path: str) -> Optional[str]:
16
- file_extension = file_path.split(".")[-1].lower()
17
- logger.info(
18
- f"PyMuPDFWrapper: Processing file: {file_path} with extension: {file_extension}"
19
- )
20
-
17
+ file_extension = detect_extension(file_path)
21
18
  if file_extension not in self.SUPPORTED_EXTENSIONS:
22
19
  logger.warning(f"PyMuPDFWrapper: Unsupported file format: {file_extension}")
23
20
  return None
@@ -3,9 +3,9 @@ from __future__ import annotations
3
3
  from typing import Optional
4
4
 
5
5
  from ..common.logger import logger
6
+ from ..common.utils import ensure_minimum_content
6
7
  from ..converters.azure_speech_wrapper import AzureSpeechWrapper
7
8
  from ..converters.base import ConverterWrapper
8
- from ..converters.markitdown_wrapper import MarkItDownWrapper
9
9
  from ..handlers.base_handler import BaseHandler
10
10
 
11
11
 
@@ -13,8 +13,7 @@ class AudioHandler(BaseHandler):
13
13
  """
14
14
  Convert audio files to Markdown by trying two converters in order:
15
15
 
16
- 1) **MarkItDown** — fast local extractor (works when the file format is supported).
17
- 2) **Azure Speech** — cloud speech-to-text fallback for actual audio transcription.
16
+ 1) **Azure Speech** — cloud speech-to-text fallback for actual audio transcription.
18
17
 
19
18
  The first converter that yields non-trivial Markdown (as determined by
20
19
  `ensure_minimum_content`) is returned. If both fail, `None` is returned.
@@ -26,27 +25,13 @@ class AudioHandler(BaseHandler):
26
25
  - This handler runs converters **sequentially** to avoid unnecessary API calls.
27
26
  """
28
27
 
29
- SUPPORTED_EXTENSIONS = frozenset(
30
- {
31
- ".mp3",
32
- ".wav",
33
- ".ogg",
34
- ".flac",
35
- ".m4a",
36
- ".aac",
37
- ".wma",
38
- ".webm",
39
- ".opus",
40
- }
41
- )
28
+ SUPPORTED_EXTENSIONS = AzureSpeechWrapper.SUPPORTED_EXTENSIONS
42
29
 
43
30
  def __init__(self, *args, **kwargs) -> None:
44
31
  super().__init__(*args, **kwargs)
45
- self.markitdown = MarkItDownWrapper()
46
32
  self.azure_speech = AzureSpeechWrapper()
47
33
 
48
34
  self._pipeline: list[tuple[ConverterWrapper, str]] = [
49
- (self.markitdown, "MarkItDown"),
50
35
  (self.azure_speech, "Azure Speech"),
51
36
  ]
52
37
 
@@ -71,7 +56,8 @@ class AudioHandler(BaseHandler):
71
56
  logger.info(f"AudioHandler: Trying {name} for '{file_path}'")
72
57
  try:
73
58
  md = await converter.convert(file_path)
74
- return md
59
+ if md and ensure_minimum_content(md):
60
+ return md
75
61
  except Exception as e:
76
62
  logger.error(f"AudioHandler: {name} failed for '{file_path}': {e}")
77
63
 
@@ -1,7 +1,9 @@
1
1
  import contextlib
2
+ from pathlib import Path
2
3
  from typing import Optional
3
4
 
4
5
  from ..common.logger import logger
6
+ from ..common.utils import detect_extension
5
7
  from ..converters.gpt_vision_wrapper import GPTVisionWrapper
6
8
  from .base_handler import BaseHandler
7
9
 
@@ -37,6 +39,27 @@ class ImageHandler(BaseHandler):
37
39
  if hasattr(self.gpt_vision, "gpt_vision"):
38
40
  await self.gpt_vision.gpt_vision.aclose()
39
41
 
42
+ async def get_page_count(self, file_path: str | Path) -> Optional[int]:
43
+ """
44
+ Determines the number of pages in the given image file.
45
+ """
46
+ p = Path(file_path)
47
+ ext = detect_extension(str(p.absolute()))
48
+ if ext not in self.SUPPORTED_EXTENSIONS:
49
+ logger.warning(f"ImageHandler: Unsupported file format: {ext}")
50
+ return None
51
+
52
+ if ext in {".tif", ".tiff"}:
53
+ try:
54
+ from PIL import Image, ImageSequence # optional dependency
55
+
56
+ with Image.open(str(p)) as im:
57
+ return sum(1 for _ in ImageSequence.Iterator(im)) or 1
58
+ except Exception as e_tiff:
59
+ logger.warning(f"ImageHandler: local TIFF page count failed for '{p}': {e_tiff}")
60
+ else:
61
+ return 1
62
+
40
63
  async def aclose(self) -> None:
41
64
  if hasattr(self, "gpt_vision") and self.gpt_vision:
42
65
  await self.gpt_vision.aclose()
@@ -4,7 +4,6 @@ from ..common.logger import logger
4
4
  from ..converters.azure_doc_intel_wrapper import DocIntelligenceWrapper
5
5
  from ..converters.base import ConverterWrapper
6
6
  from ..converters.markitdown_wrapper import MarkItDownWrapper
7
- from ..converters.tabular_wrapper import TabularWrapper
8
7
  from ..converters.unstructured_wrapper import UnstructuredWrapper
9
8
  from .base_handler import BaseHandler
10
9
 
@@ -13,10 +12,9 @@ class OfficeHandler(BaseHandler):
13
12
  """
14
13
  Convert Office documents to Markdown by trying a series of converters in order:
15
14
 
16
- 1) TabularHandler (specialized handler for tabular data)
17
- 2) MarkItDown (fast, local parsing for Office formats)
18
- 3) Azure Document Intelligence (cloud extraction + OCR for embedded images)
19
- 4) Unstructured (robust fallback parser)
15
+ 1) MarkItDown (fast, local parsing for Office formats)
16
+ 2) Azure Document Intelligence (cloud extraction + OCR for embedded images)
17
+ 3) Unstructured (robust fallback parser)
20
18
 
21
19
  The first converter that returns non-trivial Markdown (as judged by
22
20
  `ensure_minimum_content`) wins. If all fail, `None` is returned.
@@ -28,21 +26,18 @@ class OfficeHandler(BaseHandler):
28
26
  avoid unnecessary API calls when earlier options succeed.
29
27
  """
30
28
 
31
- # Supported Office-like extensions (lowercase with dot)
32
29
  SUPPORTED_EXTENSIONS = frozenset({".doc", ".docx", ".odt", ".rtf", ".ppt", ".pptx"})
33
30
 
34
31
  def __init__(self, *args, **kwargs) -> None:
35
32
  super().__init__(*args, **kwargs)
36
33
 
37
34
  # Ordered, fastest → most comprehensive
38
- self.tabular = TabularWrapper()
39
35
  self.markitdown = MarkItDownWrapper()
40
36
  self.doc_intelligence = DocIntelligenceWrapper()
41
37
  self.unstructured = UnstructuredWrapper()
42
38
 
43
39
  # (converter, human_readable_name) in the exact order to try
44
40
  self._pipeline: list[tuple[ConverterWrapper, str]] = [
45
- (self.tabular, "TabularHandler"),
46
41
  (self.markitdown, "MarkItDown"),
47
42
  (self.doc_intelligence, "Azure Document Intelligence"),
48
43
  (self.unstructured, "Unstructured"),
@@ -0,0 +1,377 @@
1
+ import asyncio
2
+ import os
3
+ import re
4
+ import tempfile
5
+ from contextlib import contextmanager
6
+ from enum import Enum
7
+ from itertools import groupby
8
+ from operator import itemgetter
9
+ from pathlib import Path
10
+ from typing import Optional
11
+
12
+ import fitz
13
+
14
+ from ..common.logger import logger
15
+ from ..common.utils import detect_extension
16
+ from ..converters.azure_doc_intel_wrapper import DocIntelligenceWrapper
17
+ from ..converters.gpt_vision_wrapper import GPTVisionWrapper
18
+ from ..converters.markitdown_wrapper import MarkItDownWrapper
19
+ from ..converters.pymupdf_wrapper import PyMuPDFWrapper
20
+ from ..converters.unstructured_wrapper import UnstructuredWrapper
21
+ from .base_handler import BaseHandler
22
+
23
+
24
+ class ConversionError(Exception):
25
+ pass
26
+
27
+
28
+ # region helpers: page parsing / slicing
29
+ def _parse_pages_spec(pages: str | list[int], total_pages: int) -> list[int]:
30
+ """Return a sorted, de-duped, 0-based list of page indexes within bounds."""
31
+ if pages is None:
32
+ return list(range(total_pages))
33
+ if isinstance(pages, (list, tuple)):
34
+ idxs = [int(p) - 1 if p >= 1 else int(p) for p in pages] # allow 1-based
35
+ else:
36
+ idxs = []
37
+ for tok in re.split(r"[,\s]+", pages.strip()):
38
+ if not tok:
39
+ continue
40
+ if "-" in tok:
41
+ a, b = tok.split("-", 1)
42
+ a, b = int(a), int(b)
43
+ if a > b:
44
+ a, b = b, a
45
+ idxs.extend(range(a - 1, b)) # user passes 1-based
46
+ else:
47
+ idxs.append(int(tok) - 1)
48
+ idxs = sorted(set(i for i in idxs if 0 <= i < total_pages))
49
+ return idxs
50
+
51
+
52
+ def _coalesce_runs(idxs: list[int]) -> list[tuple[int, int]]:
53
+ """Turn [0,1,2,4,7,8] into [(0,2),(4,4),(7,8)]"""
54
+ runs = []
55
+ for k, g in groupby(enumerate(idxs), key=lambda t: t[0] - t[1]):
56
+ group = list(map(itemgetter(1), g))
57
+ runs.append((group[0], group[-1]))
58
+ return runs
59
+
60
+
61
+ @contextmanager
62
+ def _maybe_slice_pdf(src_path: str, page_idxs: list[int], total_pages: int):
63
+ """Yield original pdf if full-range, else a temp sliced pdf with selected pages."""
64
+ full_range = (
65
+ len(page_idxs) == total_pages and page_idxs[0] == 0 and page_idxs[-1] == total_pages - 1
66
+ )
67
+ if full_range:
68
+ yield src_path
69
+ return
70
+ import fitz
71
+
72
+ tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf")
73
+ tmp.close()
74
+ try:
75
+ with fitz.open(src_path) as src:
76
+ dst = fitz.open()
77
+ for start, end in _coalesce_runs(page_idxs):
78
+ dst.insert_pdf(src, from_page=start, to_page=end)
79
+ dst.save(tmp.name)
80
+ dst.close()
81
+ yield tmp.name
82
+ finally:
83
+ try:
84
+ os.remove(tmp.name)
85
+ except Exception:
86
+ pass
87
+
88
+
89
+ # endregion
90
+
91
+
92
+ class PDFType(Enum):
93
+ """
94
+ Simple classification of a PDF based on page contents.
95
+
96
+ - TEXT_ONLY: Every page has "enough" extractable text; no pages contain images.
97
+ - TEXT_PLUS_IMAGES: At least one page contains text and at least one page contains images.
98
+ - ALL_IMAGES: Every page has at least one image and no page has "enough" text
99
+ (typical of scanned/image-only PDFs).
100
+ """
101
+
102
+ TEXT_ONLY = "TEXT_ONLY"
103
+ TEXT_PLUS_IMAGES = "TEXT_PLUS_IMAGES"
104
+ ALL_IMAGES = "ALL_IMAGES"
105
+
106
+
107
+ class PDFHandler(BaseHandler):
108
+ """
109
+ Orchestrates PDF → Markdown conversion by trying a sequence of converters
110
+ chosen from a text-oriented pipeline or an image/OCR pipeline depending
111
+ on a quick content scan of the PDF.
112
+
113
+ The flow is:
114
+ 1) `_detect_pdf_type` runs a fast pass over the PDF (offloaded to a thread)
115
+ to count text-bearing and image-bearing pages.
116
+ 2) Based on the detected `PDFType`, choose a pipeline:
117
+ - TEXT_ONLY: text pipeline
118
+ - ALL_IMAGES: image pipeline (OCR)
119
+ - TEXT_PLUS_IMAGES: image pipeline (OCR)
120
+ 3) Iterate the chosen converters in order until one returns acceptable Markdown.
121
+
122
+ Notes:
123
+ - Converters MUST implement an async `convert(file_path: str) -> Optional[str]`.
124
+ - `ensure_minimum_content` is used to filter out trivial/empty results.
125
+ """
126
+
127
+ # File extensions handled by this handler
128
+ SUPPORTED_EXTENSIONS = frozenset([".pdf"])
129
+
130
+ def __init__(self, *args, **kwargs) -> None:
131
+ super().__init__(*args, **kwargs)
132
+
133
+ # Text-first converters (fastest/cheapest first, progressively more robust)
134
+ self.markitdown = MarkItDownWrapper()
135
+ self.unstructured = UnstructuredWrapper()
136
+ self.pymu = PyMuPDFWrapper()
137
+ self.azure_docint = DocIntelligenceWrapper()
138
+
139
+ # OCR-based converter for scanned/image-only PDFs
140
+ self.gpt_vision = GPTVisionWrapper()
141
+
142
+ # Pipelines in the order they should be attempted
143
+ self.text_pipeline = [self.markitdown, self.unstructured, self.pymu, self.azure_docint]
144
+ self.image_pipeline = [self.gpt_vision]
145
+
146
+ async def handle(self, file_path: str, *args, **kwargs) -> Optional[str]:
147
+ """
148
+ Convert a PDF to Markdown, honoring page selection from kwargs only.
149
+
150
+ Expected kwargs:
151
+ - pages: list[int] (1-based page numbers). If absent/invalid → process all pages.
152
+ """
153
+ try:
154
+ # Resolve selection to 0-based page indexes (offloaded: PyMuPDF is CPU-bound)
155
+ def _resolve():
156
+ with fitz.open(file_path) as doc:
157
+ total = doc.page_count
158
+ pages_kw = kwargs.get("pages", None)
159
+ if isinstance(pages_kw, list) and all(isinstance(p, int) for p in pages_kw):
160
+ idxs = sorted(set(p - 1 for p in pages_kw))
161
+ idxs = [i for i in idxs if 0 <= i < total]
162
+ if not idxs:
163
+ logger.warning(
164
+ f"PDFHandler: 'pages' kwarg produced no valid pages for {file_path}; defaulting to all pages."
165
+ )
166
+ idxs = list(range(total))
167
+ else:
168
+ if pages_kw is not None:
169
+ logger.warning(
170
+ "PDFHandler: 'pages' must be a list[int] (1-based). Defaulting to all pages."
171
+ )
172
+ idxs = list(range(total))
173
+ return idxs, total
174
+
175
+ page_idxs, total_pages = await asyncio.to_thread(_resolve)
176
+
177
+ # Detect type using only the selected pages
178
+ pdf_type = await self._detect_pdf_type(file_path, page_indexes=page_idxs)
179
+ pipeline = (
180
+ self.image_pipeline
181
+ if pdf_type in (PDFType.ALL_IMAGES, PDFType.TEXT_PLUS_IMAGES)
182
+ else self.text_pipeline
183
+ )
184
+
185
+ # Slice once and feed to converters
186
+ with _maybe_slice_pdf(file_path, page_idxs, total_pages) as to_process:
187
+ for converter in pipeline:
188
+ preview = [i + 1 for i in page_idxs[:5]]
189
+ logger.info(
190
+ f"PDFHandler: Trying {converter.name} for {file_path} pages={preview}{'...' if len(page_idxs) > 5 else ''}"
191
+ )
192
+ try:
193
+ md_content = await converter.convert(to_process)
194
+ return md_content
195
+ except Exception as e:
196
+ logger.error(
197
+ f"PDFHandler: Converter {converter.name} failed for subset {file_path}: {e}"
198
+ )
199
+
200
+ raise RuntimeError(f"PDF conversion failed with all converters for {file_path}")
201
+ except Exception as e:
202
+ logger.error(f"PDFHandler: Error handling PDF '{file_path}': {e}")
203
+ return None
204
+
205
+ async def _detect_pdf_type(
206
+ self,
207
+ file_path: str,
208
+ page_indexes: list[int] | None = None,
209
+ ) -> PDFType:
210
+ """
211
+ Quickly scan the PDF to decide which pipeline to run.
212
+
213
+ Heuristic
214
+ ---------
215
+ - A page is considered to have "text" if the extracted text length ≥ `min_text_length_threshold`.
216
+ - A page is considered to have "images" if `page.get_images(full=True)` returns any entries.
217
+ - The final classification is derived from counts across all pages.
218
+
219
+ Parameters
220
+ ----------
221
+ file_path : str
222
+ Path to the input PDF.
223
+
224
+ Returns
225
+ -------
226
+ PDFType
227
+ Classification result used to pick the processing pipeline.
228
+
229
+ Raises
230
+ ------
231
+ Exception
232
+ If the file cannot be opened or scanned.
233
+ """
234
+ min_text_length_threshold = 20
235
+ try:
236
+
237
+ def _scan() -> PDFType:
238
+ with fitz.open(file_path) as doc:
239
+ total_pages = doc.page_count
240
+ idxs = page_indexes or list(range(total_pages))
241
+ pages_with_text = 0
242
+ pages_with_images = 0
243
+ for i in idxs:
244
+ page = doc.load_page(i)
245
+ if not page:
246
+ continue
247
+ if len((page.get_text() or "").strip()) >= min_text_length_threshold:
248
+ pages_with_text += 1
249
+ if page.get_images(full=True):
250
+ pages_with_images += 1
251
+ is_text_only = pages_with_text == len(idxs) and pages_with_images == 0
252
+ is_all_images = pages_with_images == len(idxs) and pages_with_text == 0
253
+ has_text_and_images = pages_with_text > 0 and pages_with_images > 0
254
+ if is_text_only:
255
+ return PDFType.TEXT_ONLY
256
+ elif is_all_images:
257
+ return PDFType.ALL_IMAGES
258
+ elif has_text_and_images:
259
+ return PDFType.TEXT_PLUS_IMAGES
260
+ return PDFType.TEXT_PLUS_IMAGES
261
+
262
+ return await asyncio.to_thread(_scan)
263
+ except Exception as e:
264
+ logger.error(f"PDFHandler: Error analyzing PDF '{file_path}': {e}")
265
+ raise
266
+
267
+ async def get_page_count(self, file_path: str | os.PathLike) -> Optional[int]:
268
+ p = Path(file_path)
269
+ ext = detect_extension(str(p.absolute()))
270
+ if ext not in self.SUPPORTED_EXTENSIONS:
271
+ logger.warning(f"DocumentIntelligenceHandler: Unsupported file format: {ext}")
272
+ return None
273
+ if ext in self.SUPPORTED_EXTENSIONS:
274
+ try:
275
+ import pypdf as _pypdf
276
+
277
+ try:
278
+ reader = _pypdf.PdfReader(str(p))
279
+ except Exception:
280
+ with open(p, "rb") as fh:
281
+ reader = _pypdf.PdfReader(fh)
282
+ return len(reader.pages)
283
+ except Exception as e_pdf:
284
+ logger.warning(
285
+ f"DocumentIntelligenceHandler: local PDF page count failed for '{p}': {e_pdf}"
286
+ )
287
+
288
+ # async def _run_soffice_convert(
289
+ # self, file_path: Path, filter_name: Optional[str] = None, timeout_s: int = 180
290
+ # ) -> Path:
291
+ # """
292
+ # Generic converter via LibreOffice 'soffice' headless CLI.
293
+ # `filter_name` can be 'writer_pdf_Export' for DOCX or 'impress_pdf_Export' for PPTX.
294
+ # Returns final PDF path placed next to the source file.
295
+ # """
296
+ # soffice = shutil.which("soffice") or shutil.which("libreoffice")
297
+ # if not soffice:
298
+ # raise ConversionError("LibreOffice not found on PATH (need 'soffice').")
299
+
300
+ # if not file_path.exists():
301
+ # raise FileNotFoundError(file_path)
302
+
303
+ # # Output to a temp dir, then move next to source
304
+ # tmp_out = Path(tempfile.mkdtemp())
305
+ # try:
306
+ # convert_to = "pdf" if not filter_name else f"pdf:{filter_name}"
307
+ # cmd = [
308
+ # soffice,
309
+ # "--headless",
310
+ # "--invisible",
311
+ # "--norestore",
312
+ # "--nodefault",
313
+ # "--nolockcheck",
314
+ # "--nofirststartwizard",
315
+ # "--convert-to",
316
+ # convert_to,
317
+ # str(file_path),
318
+ # "--outdir",
319
+ # str(tmp_out),
320
+ # ]
321
+
322
+ # proc = await asyncio.create_subprocess_exec(
323
+ # *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
324
+ # )
325
+ # try:
326
+ # stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout_s)
327
+ # except asyncio.TimeoutError:
328
+ # proc.kill()
329
+ # raise ConversionError("LibreOffice timed out during conversion")
330
+
331
+ # if proc.returncode != 0:
332
+ # raise ConversionError(
333
+ # f"LibreOffice failed (code {proc.returncode}). Stderr: {stderr.decode(errors='ignore')}"
334
+ # )
335
+
336
+ # produced = tmp_out / (file_path.stem + ".pdf")
337
+ # if not produced.exists():
338
+ # # Some LO versions may change the output name; find the first PDF
339
+ # candidates = list(tmp_out.glob("*.pdf"))
340
+ # if not candidates:
341
+ # raise ConversionError("Expected PDF not created by LibreOffice")
342
+ # produced = candidates[0]
343
+
344
+ # final_pdf = file_path.with_suffix(".pdf")
345
+ # if final_pdf.exists():
346
+ # final_pdf.unlink()
347
+ # produced.replace(final_pdf)
348
+ # return final_pdf
349
+ # finally:
350
+ # try:
351
+ # # Clean temp dir
352
+ # for f in tmp_out.glob("*"):
353
+ # try:
354
+ # f.unlink()
355
+ # except Exception:
356
+ # pass
357
+ # tmp_out.rmdir()
358
+ # except Exception:
359
+ # pass
360
+
361
+ # async def docx_to_pdf(self, file_path: str | Path, timeout_s: int = 180) -> Path:
362
+ # src = Path(file_path).resolve()
363
+ # if src.suffix.lower() != ".docx":
364
+ # raise ValueError("docx_to_pdf expects a .docx file")
365
+ # # writer filter produces better pagination on Writer docs
366
+ # return await self._run_soffice_convert(
367
+ # src, filter_name="writer_pdf_Export", timeout_s=timeout_s
368
+ # )
369
+
370
+ # async def pptx_to_pdf(self, file_path: str | Path, timeout_s: int = 180) -> Path:
371
+ # src = Path(file_path).resolve()
372
+ # if src.suffix.lower() != ".pptx":
373
+ # raise ValueError("pptx_to_pdf expects a .pptx file")
374
+ # # impress filter produces better pagination on Impress/PowerPoint
375
+ # return await self._run_soffice_convert(
376
+ # src, filter_name="impress_pdf_Export", timeout_s=timeout_s
377
+ # )