markitdown-pro 1.2.2__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/PKG-INFO +1 -1
  2. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/conversion_pipeline.py +22 -14
  3. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/azure_speech_wrapper.py +2 -0
  4. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/base.py +1 -1
  5. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/gpt_vision_wrapper.py +17 -0
  6. markitdown_pro-1.3.0/markitdown_pro/converters/markitdown_wrapper.py +23 -0
  7. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/pymupdf_wrapper.py +8 -10
  8. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/tabular_wrapper.py +0 -2
  9. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/unstructured_wrapper.py +2 -3
  10. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/audio_handler.py +2 -6
  11. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/base_handler.py +2 -2
  12. markitdown_pro-1.3.0/markitdown_pro/handlers/email_handler.py +301 -0
  13. markitdown_pro-1.3.0/markitdown_pro/handlers/epub_handler.py +99 -0
  14. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/image_handler.py +13 -21
  15. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/ipynb_handler.py +4 -9
  16. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/markitdown_handler.py +1 -5
  17. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/markup_handler.py +5 -9
  18. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/office_handler.py +2 -8
  19. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/pdf_handler.py +9 -9
  20. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/pst_handler.py +3 -4
  21. markitdown_pro-1.3.0/markitdown_pro/handlers/tabular_handler.py +118 -0
  22. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/text_handler.py +4 -13
  23. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/services/azure_doc_intelligence.py +1 -8
  24. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/services/azure_speech.py +4 -5
  25. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/services/openai_services.py +74 -31
  26. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro.egg-info/PKG-INFO +1 -1
  27. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro.egg-info/SOURCES.txt +5 -0
  28. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/setup.py +1 -1
  29. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/tests/handlers/test_email_handler.py +7 -5
  30. markitdown_pro-1.3.0/tests/handlers/test_epub_handler.py +28 -0
  31. markitdown_pro-1.3.0/tests/handlers/test_image_handler.py +28 -0
  32. markitdown_pro-1.3.0/tests/handlers/test_ipynb_handler.py +28 -0
  33. markitdown_pro-1.3.0/tests/handlers/test_markitdown_handler.py +28 -0
  34. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/tests/handlers/test_markup_handler.py +7 -3
  35. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/tests/handlers/test_pst_handler.py +3 -3
  36. markitdown_pro-1.3.0/tests/handlers/test_tabular_handler.py +28 -0
  37. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/tests/handlers/test_text_handler.py +2 -0
  38. markitdown_pro-1.3.0/tests/test_conversion_pipeline.py +25 -0
  39. markitdown_pro-1.2.2/markitdown_pro/converters/markitdown_wrapper.py +0 -44
  40. markitdown_pro-1.2.2/markitdown_pro/handlers/email_handler.py +0 -169
  41. markitdown_pro-1.2.2/markitdown_pro/handlers/epub_handler.py +0 -33
  42. markitdown_pro-1.2.2/markitdown_pro/handlers/tabular_handler.py +0 -31
  43. markitdown_pro-1.2.2/tests/test_conversion_pipeline.py +0 -25
  44. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/README.md +0 -0
  45. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/__init__.py +0 -0
  46. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/common/__init__.py +0 -0
  47. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/common/isolated_worker.py +0 -0
  48. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/common/logger.py +0 -0
  49. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/common/schemas.py +0 -0
  50. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/common/utils.py +0 -0
  51. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/__init__.py +0 -0
  52. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/azure_doc_intel_wrapper.py +0 -0
  53. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/youtube_wrapper.py +0 -0
  54. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/__init__.py +0 -0
  55. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/services/__init__.py +0 -0
  56. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro.egg-info/dependency_links.txt +0 -0
  57. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro.egg-info/requires.txt +0 -0
  58. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro.egg-info/top_level.txt +0 -0
  59. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/pyproject.toml +0 -0
  60. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/setup.cfg +0 -0
  61. {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/tests/handlers/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: markitdown-pro
3
- Version: 1.2.2
3
+ Version: 1.3.0
4
4
  Summary: A package that converts almost any file format to Markdown.
5
5
  Author: Developer
6
6
  Classifier: Programming Language :: Python :: 3
@@ -38,7 +38,7 @@ from typing import Optional
38
38
  import httpx
39
39
 
40
40
  from .common.logger import logger
41
- from .common.utils import detect_extension, ensure_minimum_content
41
+ from .common.utils import clean_markdown, detect_extension, ensure_minimum_content
42
42
  from .handlers.audio_handler import AudioHandler
43
43
  from .handlers.base_handler import BaseHandler
44
44
  from .handlers.email_handler import EmailHandler
@@ -91,9 +91,10 @@ class ConversionPipeline:
91
91
 
92
92
  # Map extensions to handlers (normalized lowercase with dot)
93
93
  self.handlers_mapping: dict[str, BaseHandler] = {
94
- # PDFs
94
+ # region PDFs
95
95
  ".pdf": self.pdf_handler,
96
- # Audio
96
+ # endregion
97
+ # region Audio
97
98
  ".mp3": self.audio_handler,
98
99
  ".wav": self.audio_handler,
99
100
  # ".ogg": self.audio_handler,
@@ -103,36 +104,40 @@ class ConversionPipeline:
103
104
  # ".wma": self.audio_handler,
104
105
  # ".webm": self.audio_handler,
105
106
  # ".opus": self.audio_handler,
106
- # Images
107
+ # endregion
108
+ # region Images
107
109
  ".bmp": self.image_handler,
108
110
  ".gif": self.image_handler,
109
111
  ".heic": self.image_handler,
110
112
  ".jpeg": self.image_handler,
111
113
  ".jpg": self.image_handler,
112
114
  ".png": self.image_handler,
113
- ".prn": self.image_handler,
115
+ # ".prn": self.image_handler,
114
116
  ".svg": self.image_handler,
115
117
  ".tiff": self.image_handler,
116
118
  ".webp": self.image_handler,
117
119
  ".heif": self.image_handler,
118
- # Plain/code text
120
+ # endregion
121
+ # region Plain/code text
119
122
  ".txt": self.text_handler,
120
123
  ".md": self.text_handler,
121
124
  ".py": self.text_handler,
122
125
  ".go": self.text_handler,
123
- # Tabular (via MarkItDown)
126
+ # endregion
127
+ # region Tabular (via MarkItDown)
124
128
  ".csv": self.tabular_handler,
125
129
  ".tsv": self.tabular_handler,
126
- # Office/Spreadsheet/Slides
130
+ # endregion
131
+ # region Office/Spreadsheet/Slides
127
132
  ".xls": self.office_handler,
128
133
  ".xlsx": self.office_handler,
129
- ".doc": self.office_handler,
130
134
  ".docx": self.office_handler,
131
- ".odt": self.office_handler,
132
- ".rtf": self.office_handler,
133
- ".ppt": self.office_handler,
135
+ # ".odt": self.office_handler,
136
+ # ".rtf": self.office_handler,
137
+ # ".ppt": self.office_handler,
134
138
  ".pptx": self.office_handler,
135
- # Markup/serialization
139
+ # endregion
140
+ # region Markup/serialization
136
141
  ".html": self.markup_handler,
137
142
  ".htm": self.markup_handler,
138
143
  ".xml": self.markup_handler,
@@ -140,13 +145,15 @@ class ConversionPipeline:
140
145
  ".ndjson": self.markup_handler,
141
146
  ".yaml": self.markup_handler,
142
147
  ".yml": self.markup_handler,
143
- # Others
148
+ # endregion
149
+ # region Others
144
150
  ".epub": self.epub_handler,
145
151
  ".eml": self.email_handler,
146
152
  ".p7s": self.email_handler,
147
153
  ".msg": self.email_handler,
148
154
  ".pst": self.pst_handler,
149
155
  ".ipynb": self.ipynb_handler,
156
+ # endregion
150
157
  }
151
158
 
152
159
  # ---------------------------------------------------------------------
@@ -190,6 +197,7 @@ class ConversionPipeline:
190
197
  if not md_content or not ensure_minimum_content(md_content):
191
198
  return None
192
199
 
200
+ md_content = clean_markdown(md_content)
193
201
  logger.debug(
194
202
  f"convert_document_to_md: {file_path} returned {len(md_content)} characters"
195
203
  )
@@ -5,6 +5,8 @@ from .base import ConverterWrapper
5
5
 
6
6
 
7
7
  class AzureSpeechWrapper(ConverterWrapper):
8
+ SUPPORTED_EXTENSIONS = AzureSpeechService.SUPPORTED_EXTENSIONS
9
+
8
10
  def __init__(self):
9
11
  super().__init__("Azure Speech Service")
10
12
  self.converter = AzureSpeechService()
@@ -4,7 +4,7 @@ from ..common.logger import logger
4
4
 
5
5
 
6
6
  class ConverterWrapper:
7
- SUPPORTED_FORMATS = ()
7
+ SUPPORTED_EXTENSIONS = ()
8
8
 
9
9
  def __init__(self, name: str):
10
10
  self.name = name
@@ -44,6 +44,20 @@ class GPTVisionWrapper(ConverterWrapper):
44
44
  add that logic in `convert()` (see inline TODO in that method).
45
45
  """
46
46
 
47
+ SUPPORTED_EXTENSIONS = frozenset(
48
+ {
49
+ ".bmp",
50
+ ".gif",
51
+ ".jpeg",
52
+ ".jpg",
53
+ ".png",
54
+ ".tiff",
55
+ ".webp",
56
+ ".heic",
57
+ ".heif",
58
+ }
59
+ )
60
+
47
61
  def __init__(
48
62
  self,
49
63
  model_name: str = "gpt-4.1-mini",
@@ -82,3 +96,6 @@ class GPTVisionWrapper(ConverterWrapper):
82
96
  return await self.gpt_vision.process_scanned_pdf_concurrent(file_path)
83
97
  # Otherwise, assume it's an image and run single-image OCR/analysis.
84
98
  return await self.gpt_vision.process_image(file_path)
99
+
100
+ async def aclose(self) -> None:
101
+ await self.gpt_vision.aclose()
@@ -0,0 +1,23 @@
1
+ from typing import Optional
2
+
3
+ from markitdown_pro.handlers.markitdown_handler import MarkItDownHandler
4
+
5
+ from ..common.logger import logger
6
+ from .base import ConverterWrapper
7
+
8
+
9
+ class MarkItDownWrapper(ConverterWrapper):
10
+ SUPPORTED_EXTENSIONS = MarkItDownHandler.SUPPORTED_EXTENSIONS
11
+
12
+ def __init__(self, *args, **kwargs):
13
+ super().__init__(*args, **kwargs)
14
+ self.handler = MarkItDownHandler(*args, **kwargs)
15
+
16
+ async def convert(self, file_path: str) -> Optional[str]:
17
+ file_extension = file_path.split(".")[-1].lower()
18
+ if file_extension not in self.SUPPORTED_EXTENSIONS:
19
+ logger.warning(f"MarkItDownWrapper: Unsupported file format: {file_extension}")
20
+ return None
21
+
22
+ md_content = await self.handler.handle(file_path)
23
+ return md_content
@@ -1,25 +1,25 @@
1
- import logging
2
1
  from typing import Optional
3
2
 
4
3
  import fitz # PyMuPDF
5
4
 
6
5
  from ..common.logger import logger
7
- from ..common.utils import clean_markdown, ensure_minimum_content
8
6
  from .base import ConverterWrapper
9
7
 
10
8
 
11
9
  class PyMuPDFWrapper(ConverterWrapper):
12
- SUPPORTED_FORMATS = ["pdf", "xps", "epub", "mobi", "fb2", "cbz", "svg"]
10
+ SUPPORTED_EXTENSIONS = frozenset({"pdf", "xps", "epub", "mobi", "fb2", "cbz", "svg"})
13
11
 
14
12
  def __init__(self):
15
13
  super().__init__("PyMuPDF")
16
14
 
17
15
  async def convert(self, file_path: str) -> Optional[str]:
18
16
  file_extension = file_path.split(".")[-1].lower()
19
- logging.info(f"Processing PyMuPDF file: {file_path} with extension: {file_extension}")
17
+ logger.info(
18
+ f"PyMuPDFWrapper: Processing file: {file_path} with extension: {file_extension}"
19
+ )
20
20
 
21
- if file_extension not in self.SUPPORTED_FORMATS:
22
- logger.warning(f"Unsupported file format: {file_extension}")
21
+ if file_extension not in self.SUPPORTED_EXTENSIONS:
22
+ logger.warning(f"PyMuPDFWrapper: Unsupported file format: {file_extension}")
23
23
  return None
24
24
 
25
25
  try:
@@ -33,10 +33,8 @@ class PyMuPDFWrapper(ConverterWrapper):
33
33
  text_content += "\n\n"
34
34
 
35
35
  doc.close()
36
-
37
- text_md = clean_markdown(text_content)
38
- return text_md if ensure_minimum_content(text_md) else None
36
+ return text_content
39
37
 
40
38
  except Exception as e:
41
- logger.error(f"Error processing document with PyMuPDF: {e}")
39
+ logger.error(f"PyMuPDFWrapper: Error processing document with PyMuPDF: {e}")
42
40
  return None
@@ -1,7 +1,6 @@
1
1
  from typing import Optional
2
2
 
3
3
  from ..common.logger import logger
4
- from ..common.utils import clean_markdown
5
4
  from ..handlers.tabular_handler import TabularHandler
6
5
  from .base import ConverterWrapper
7
6
 
@@ -27,5 +26,4 @@ class TabularWrapper(ConverterWrapper):
27
26
  return None
28
27
 
29
28
  markdown = await self.tabular_handler.handle(file_path)
30
- markdown = clean_markdown(markdown) if markdown else None
31
29
  return markdown
@@ -3,7 +3,6 @@ from typing import Optional
3
3
  from unstructured.partition.auto import partition
4
4
 
5
5
  from ..common.logger import logger
6
- from ..common.utils import clean_markdown, ensure_minimum_content
7
6
  from .base import ConverterWrapper
8
7
 
9
8
 
@@ -58,5 +57,5 @@ class UnstructuredWrapper(ConverterWrapper):
58
57
 
59
58
  # Directly call partition. No asyncio.to_thread
60
59
  elements = partition(filename=file_path, extract_images_in_pdf=True)
61
- combined = clean_markdown("\n\n".join(str(el) for el in elements))
62
- return combined if ensure_minimum_content(combined) else None
60
+ combined = "\n\n".join(str(el) for el in elements)
61
+ return combined
@@ -3,7 +3,6 @@ from __future__ import annotations
3
3
  from typing import Optional
4
4
 
5
5
  from ..common.logger import logger
6
- from ..common.utils import ensure_minimum_content
7
6
  from ..converters.azure_speech_wrapper import AzureSpeechWrapper
8
7
  from ..converters.base import ConverterWrapper
9
8
  from ..converters.markitdown_wrapper import MarkItDownWrapper
@@ -27,7 +26,7 @@ class AudioHandler(BaseHandler):
27
26
  - This handler runs converters **sequentially** to avoid unnecessary API calls.
28
27
  """
29
28
 
30
- extensions = frozenset(
29
+ SUPPORTED_EXTENSIONS = frozenset(
31
30
  {
32
31
  ".mp3",
33
32
  ".wav",
@@ -72,10 +71,7 @@ class AudioHandler(BaseHandler):
72
71
  logger.info(f"AudioHandler: Trying {name} for '{file_path}'")
73
72
  try:
74
73
  md = await converter.convert(file_path)
75
- if md and ensure_minimum_content(md):
76
- logger.info(f"AudioHandler: {name} succeeded for '{file_path}'")
77
- return md
78
- logger.debug(f"AudioHandler: {name} returned insufficient content")
74
+ return md
79
75
  except Exception as e:
80
76
  logger.error(f"AudioHandler: {name} failed for '{file_path}': {e}")
81
77
 
@@ -3,7 +3,7 @@ from typing import Optional
3
3
 
4
4
 
5
5
  class BaseHandler:
6
- extensions: frozenset = frozenset()
6
+ SUPPORTED_EXTENSIONS: frozenset = frozenset()
7
7
  pipeline: Optional[list] = None
8
8
 
9
9
  def __init__(self, *args, **kwargs):
@@ -11,7 +11,7 @@ class BaseHandler:
11
11
 
12
12
  @classmethod
13
13
  async def is_valid(cls, file_path: str) -> bool:
14
- return Path(file_path).is_file() and Path(file_path).suffix in cls.extensions
14
+ return Path(file_path).is_file() and Path(file_path).suffix in cls.SUPPORTED_EXTENSIONS
15
15
 
16
16
  async def handle(self, file_path, *args, **kwargs) -> Optional[str]:
17
17
  raise NotImplementedError("You must implement the handle method")
@@ -0,0 +1,301 @@
1
+ import asyncio
2
+ import html
3
+ import io
4
+ import os
5
+ import re
6
+ import tempfile
7
+ import uuid
8
+ from email.header import decode_header, make_header
9
+ from email.message import Message
10
+ from email.parser import BytesParser
11
+ from email.policy import default
12
+ from typing import Any, Callable, Dict, List, Optional
13
+
14
+ from ..common.logger import logger
15
+ from .base_handler import BaseHandler
16
+
17
+ _MAX_ATTACHMENT_BYTES = 50 * 1024 * 1024 # 50MB safety cap
18
+ _READ_CHUNK = 1024 * 1024 # 1MB
19
+ _PARSE_TIMEOUT_SEC = 30
20
+ _CONVERT_TIMEOUT_SEC = 60
21
+
22
+
23
+ def _safe_decode_header(value: Optional[str]) -> str:
24
+ try:
25
+ if not value:
26
+ return ""
27
+ return str(make_header(decode_header(value)))
28
+ except Exception:
29
+ return value or ""
30
+
31
+
32
+ def _coerce_charset(s: bytes, charset: Optional[str]) -> str:
33
+ for cs in [charset, "utf-8", "latin-1"]:
34
+ if not cs:
35
+ continue
36
+ try:
37
+ return s.decode(cs, errors="replace")
38
+ except Exception:
39
+ continue
40
+ return s.decode("utf-8", errors="replace")
41
+
42
+
43
+ def _basic_html_to_md(html_text: str) -> str:
44
+ # Minimal, dependency-free HTML → Markdown-ish conversion that preserves readability
45
+ txt = html_text
46
+ txt = re.sub(r"(?is)<(script|style).*?>.*?</\1>", "", txt)
47
+ txt = re.sub(r"(?i)<br\s*/?>", "\n", txt)
48
+ txt = re.sub(r"(?i)</p\s*>", "\n\n", txt)
49
+ txt = re.sub(r"(?i)<li\s*>", "- ", txt)
50
+ txt = re.sub(r"(?i)</h1\s*>", "\n\n", re.sub(r"(?i)<h1\s*>(.*?)</h1\s*>", r"# \1\n\n", txt))
51
+ txt = re.sub(r"(?i)</h2\s*>", "\n\n", re.sub(r"(?i)<h2\s*>(.*?)</h2\s*>", r"## \1\n\n", txt))
52
+ txt = re.sub(r"(?i)</h3\s*>", "\n\n", re.sub(r"(?i)<h3\s*>(.*?)</h3\s*>", r"### \1\n\n", txt))
53
+ txt = re.sub(r"(?i)<a\s+[^>]*href=['\"]([^'\"]+)['\"][^>]*>(.*?)</a>", r"[\2](\1)", txt)
54
+ txt = re.sub(
55
+ r"(?i)<img\s+[^>]*alt=['\"]([^'\"]*)['\"][^>]*src=['\"]([^'\"]+)['\"][^>]*>",
56
+ r"![\1](\2)",
57
+ txt,
58
+ )
59
+ txt = re.sub(r"(?is)<[^>]+>", "", txt)
60
+ txt = html.unescape(txt)
61
+ # Normalize whitespace
62
+ txt = re.sub(r"[ \t]+\n", "\n", txt)
63
+ txt = re.sub(r"\n{3,}", "\n\n", txt).strip()
64
+ return txt
65
+
66
+
67
+ class EmailHandler(BaseHandler):
68
+ SUPPORTED_EXTENSIONS = frozenset({".eml", ".p7s", ".msg"})
69
+
70
+ async def handle(self, file_path: str, *args, **kwargs) -> Optional[str]:
71
+ logger.info(f"EmailHandler: Processing email file: {file_path}")
72
+ attachment_converter: Optional[Callable[[str, str], "asyncio.Future[str]"]] = kwargs.get(
73
+ "attachment_converter"
74
+ )
75
+ try:
76
+ email_data = await asyncio.wait_for(
77
+ asyncio.to_thread(self._parse_email, file_path), timeout=_PARSE_TIMEOUT_SEC
78
+ )
79
+ if not email_data:
80
+ return None
81
+ markdown_content = await self._build_markdown(email_data, attachment_converter)
82
+ return markdown_content
83
+ except asyncio.TimeoutError:
84
+ logger.error(f"EmailHandler: Timeout while processing email file: {file_path}")
85
+ return None
86
+ except Exception as e:
87
+ logger.exception(f"EmailHandler: Error processing email file: {file_path}: {e}")
88
+ return None
89
+
90
+ def _parse_email(self, file_path: str) -> Dict[str, Any]:
91
+ """
92
+ Parses the EML/P7S file and extracts relevant information.
93
+ Returns:
94
+ {
95
+ "subject": str,
96
+ "from": str,
97
+ "to": str,
98
+ "cc": str,
99
+ "bcc": str,
100
+ "date": str,
101
+ "body_text": str, # text/plain best-effort
102
+ "body_html": str, # raw HTML if present
103
+ "attachments": List[Dict[str, str]] # [{"filename":..., "path":..., "content_type":..., "cid":...}]
104
+ }
105
+ """
106
+ try:
107
+ with open(file_path, "rb") as f:
108
+ msg: Message = BytesParser(policy=default).parse(f)
109
+ subject = _safe_decode_header(msg.get("Subject", "(No Subject)")) or "(No Subject)"
110
+ from_ = _safe_decode_header(msg.get("From", "(Unknown Sender)")) or "(Unknown Sender)"
111
+ to_ = _safe_decode_header(msg.get("To", "")) or ""
112
+ cc_ = _safe_decode_header(msg.get("Cc", "")) or ""
113
+ bcc_ = _safe_decode_header(msg.get("Bcc", "")) or ""
114
+ date_ = msg.get("Date", "(Unknown Date)") or "(Unknown Date)"
115
+ body_text = ""
116
+ body_html = ""
117
+ attachments: List[Dict[str, str]] = []
118
+
119
+ def _should_skip(part: Message) -> bool:
120
+ ctype = (part.get_content_type() or "").lower()
121
+ # Skip S/MIME signatures and pkcs7 blobs as attachments
122
+ if ctype in ("application/pkcs7-signature", "application/pkcs7-mime"):
123
+ return True
124
+ return False
125
+
126
+ if msg.is_multipart():
127
+ for part in msg.walk():
128
+ if part.is_multipart():
129
+ continue
130
+ if _should_skip(part):
131
+ continue
132
+ disp = part.get_content_disposition() or ""
133
+ ctype = (part.get_content_type() or "").lower()
134
+ cid = (part.get("Content-ID") or "").strip().strip("<>")
135
+ payload = part.get_payload(decode=True) or b""
136
+ if disp == "attachment" or (
137
+ disp == "inline" and cid
138
+ ): # treat inline with CID as attachment (e.g., images)
139
+ filename = part.get_filename()
140
+ filename = (
141
+ _safe_decode_header(filename)
142
+ if filename
143
+ else f"attachment-{uuid.uuid4().hex}"
144
+ )
145
+ # Write safely with size cap
146
+ suffix = os.path.splitext(filename)[1][:10] if filename else ""
147
+ with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp_file:
148
+ written = 0
149
+ stream = io.BytesIO(payload)
150
+ while True:
151
+ chunk = stream.read(_READ_CHUNK)
152
+ if not chunk:
153
+ break
154
+ written += len(chunk)
155
+ if written > _MAX_ATTACHMENT_BYTES:
156
+ raise ValueError(
157
+ f"Attachment '{filename}' exceeds {_MAX_ATTACHMENT_BYTES} bytes"
158
+ )
159
+ tmp_file.write(chunk)
160
+ attachments.append(
161
+ {
162
+ "filename": filename,
163
+ "path": tmp_file.name,
164
+ "content_type": ctype,
165
+ "cid": cid,
166
+ }
167
+ )
168
+ continue
169
+ if ctype == "text/plain" and not body_text:
170
+ body_text = _coerce_charset(payload, part.get_content_charset() or "utf-8")
171
+ elif ctype == "text/html" and not body_html:
172
+ body_html = _coerce_charset(payload, part.get_content_charset() or "utf-8")
173
+ else:
174
+ ctype = (msg.get_content_type() or "").lower()
175
+ if ctype == "text/plain":
176
+ body_text = _coerce_charset(
177
+ msg.get_payload(decode=True) or b"", msg.get_content_charset() or "utf-8"
178
+ )
179
+ elif ctype == "text/html":
180
+ body_html = _coerce_charset(
181
+ msg.get_payload(decode=True) or b"", msg.get_content_charset() or "utf-8"
182
+ )
183
+ return {
184
+ "subject": subject,
185
+ "from": from_,
186
+ "to": to_,
187
+ "cc": cc_,
188
+ "bcc": bcc_,
189
+ "date": date_,
190
+ "body_text": body_text.strip(),
191
+ "body_html": body_html.strip(),
192
+ "attachments": attachments,
193
+ }
194
+ except Exception as e:
195
+ logger.exception(f"EmailHandler: Error parsing email {file_path}: {e}")
196
+ return {}
197
+
198
+ async def _build_markdown(
199
+ self,
200
+ email_data: Dict[str, Any],
201
+ attachment_converter: Optional[Callable[[str, str], "asyncio.Future[str]"]] = None,
202
+ ) -> str:
203
+ # Resolve inline CID references in HTML to temp file paths (if any)
204
+ cid_map = {a["cid"]: a for a in email_data.get("attachments", []) if a.get("cid")}
205
+ body_md = ""
206
+ if email_data.get("body_text"):
207
+ body_md = email_data["body_text"]
208
+ elif email_data.get("body_html"):
209
+ html_body = email_data["body_html"]
210
+ if cid_map:
211
+ for cid, meta in cid_map.items():
212
+ html_body = re.sub(
213
+ rf"(?i)src=['\"]cid:{re.escape(cid)}['\"]",
214
+ f"src=\"{meta['path']}\"",
215
+ html_body,
216
+ )
217
+ body_md = _basic_html_to_md(html_body)
218
+ header_lines = [
219
+ f"# Email: {email_data.get('subject', '(No Subject)')}",
220
+ f"**From:** {email_data.get('from', '')}",
221
+ f"**To:** {email_data.get('to', '')}",
222
+ ]
223
+ if email_data.get("cc"):
224
+ header_lines.append(f"**Cc:** {email_data['cc']}")
225
+ if email_data.get("bcc"):
226
+ header_lines.append(f"**Bcc:** {email_data['bcc']}")
227
+ header_lines.append(f"**Date:** {email_data.get('date', '')}")
228
+ markdown_parts: List[str] = ["\n".join(header_lines), ""]
229
+ if body_md:
230
+ markdown_parts.append("```")
231
+ markdown_parts.append(body_md)
232
+ markdown_parts.append("```")
233
+ # Convert attachments (non-CID inline images will also appear here)
234
+ attachments = email_data.get("attachments", [])
235
+ for att in attachments:
236
+ filename = att.get("filename", "attachment")
237
+ path = att.get("path")
238
+ ctype = att.get("content_type", "")
239
+ if att.get("cid"): # Already linked in body if referenced; still list it
240
+ markdown_parts.append(f"\n## Inline asset: {filename} ({ctype})\n")
241
+ markdown_parts.append(
242
+ f"![{filename}]({path})"
243
+ if ctype.startswith("image/")
244
+ else f"[{filename}]({path})"
245
+ )
246
+ continue
247
+ markdown_parts.append(f"\n## Attachment: {filename} ({ctype})\n")
248
+ try:
249
+ if attachment_converter:
250
+ converted = await asyncio.wait_for(
251
+ attachment_converter(path, ctype), timeout=_CONVERT_TIMEOUT_SEC
252
+ )
253
+ markdown_parts.append(converted if converted else f"[Saved attachment]({path})")
254
+ else:
255
+ # Built-in light handling when no converter provided
256
+ if ctype.startswith("image/"):
257
+ markdown_parts.append(f"![{filename}]({path})")
258
+ elif ctype.startswith("text/"):
259
+ try:
260
+ with open(path, "rb") as f:
261
+ content = f.read(_MAX_ATTACHMENT_BYTES + 1)
262
+ if len(content) > _MAX_ATTACHMENT_BYTES:
263
+ raise ValueError(
264
+ f"Attachment too large to inline (> {_MAX_ATTACHMENT_BYTES} bytes)"
265
+ )
266
+ text = _coerce_charset(content, "utf-8")
267
+ markdown_parts.append("\n```")
268
+ markdown_parts.append(text.strip())
269
+ markdown_parts.append("```")
270
+ except Exception as e:
271
+ logger.warning(
272
+ f"EmailHandler: Inline text fallback failed for {filename}: {e}"
273
+ )
274
+ markdown_parts.append(f"[Saved attachment]({path})")
275
+ elif ctype == "text/html":
276
+ try:
277
+ with open(path, "rb") as f:
278
+ content = f.read(min(_MAX_ATTACHMENT_BYTES, 5 * 1024 * 1024))
279
+ text = _coerce_charset(content, "utf-8")
280
+ markdown_parts.append(_basic_html_to_md(text))
281
+ except Exception as e:
282
+ logger.warning(
283
+ f"EmailHandler: HTML to MD fallback failed for {filename}: {e}"
284
+ )
285
+ markdown_parts.append(f"[Saved attachment]({path})")
286
+ else:
287
+ markdown_parts.append(f"[Saved attachment]({path})")
288
+ except asyncio.TimeoutError:
289
+ logger.error(f"EmailHandler: Timeout converting attachment '{filename}'")
290
+ markdown_parts.append(f"[Attachment conversion timed out: {filename}]")
291
+ except Exception as e:
292
+ logger.error(f"EmailHandler: Error converting attachment '{filename}': {e}")
293
+ markdown_parts.append(f"[Error converting attachment: {e}]")
294
+ finally:
295
+ try:
296
+ os.remove(path)
297
+ except OSError as e:
298
+ logger.warning(
299
+ f"EmailHandler: Could not remove temporary attachment file '{path}': {e}"
300
+ )
301
+ return "\n".join(markdown_parts)