markitdown-pro 1.2.2__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/PKG-INFO +1 -1
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/conversion_pipeline.py +22 -14
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/azure_speech_wrapper.py +2 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/base.py +1 -1
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/gpt_vision_wrapper.py +17 -0
- markitdown_pro-1.3.0/markitdown_pro/converters/markitdown_wrapper.py +23 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/pymupdf_wrapper.py +8 -10
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/tabular_wrapper.py +0 -2
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/unstructured_wrapper.py +2 -3
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/audio_handler.py +2 -6
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/base_handler.py +2 -2
- markitdown_pro-1.3.0/markitdown_pro/handlers/email_handler.py +301 -0
- markitdown_pro-1.3.0/markitdown_pro/handlers/epub_handler.py +99 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/image_handler.py +13 -21
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/ipynb_handler.py +4 -9
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/markitdown_handler.py +1 -5
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/markup_handler.py +5 -9
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/office_handler.py +2 -8
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/pdf_handler.py +9 -9
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/pst_handler.py +3 -4
- markitdown_pro-1.3.0/markitdown_pro/handlers/tabular_handler.py +118 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/text_handler.py +4 -13
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/services/azure_doc_intelligence.py +1 -8
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/services/azure_speech.py +4 -5
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/services/openai_services.py +74 -31
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro.egg-info/PKG-INFO +1 -1
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro.egg-info/SOURCES.txt +5 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/setup.py +1 -1
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/tests/handlers/test_email_handler.py +7 -5
- markitdown_pro-1.3.0/tests/handlers/test_epub_handler.py +28 -0
- markitdown_pro-1.3.0/tests/handlers/test_image_handler.py +28 -0
- markitdown_pro-1.3.0/tests/handlers/test_ipynb_handler.py +28 -0
- markitdown_pro-1.3.0/tests/handlers/test_markitdown_handler.py +28 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/tests/handlers/test_markup_handler.py +7 -3
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/tests/handlers/test_pst_handler.py +3 -3
- markitdown_pro-1.3.0/tests/handlers/test_tabular_handler.py +28 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/tests/handlers/test_text_handler.py +2 -0
- markitdown_pro-1.3.0/tests/test_conversion_pipeline.py +25 -0
- markitdown_pro-1.2.2/markitdown_pro/converters/markitdown_wrapper.py +0 -44
- markitdown_pro-1.2.2/markitdown_pro/handlers/email_handler.py +0 -169
- markitdown_pro-1.2.2/markitdown_pro/handlers/epub_handler.py +0 -33
- markitdown_pro-1.2.2/markitdown_pro/handlers/tabular_handler.py +0 -31
- markitdown_pro-1.2.2/tests/test_conversion_pipeline.py +0 -25
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/README.md +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/__init__.py +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/common/__init__.py +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/common/isolated_worker.py +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/common/logger.py +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/common/schemas.py +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/common/utils.py +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/__init__.py +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/azure_doc_intel_wrapper.py +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/youtube_wrapper.py +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/handlers/__init__.py +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/services/__init__.py +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro.egg-info/dependency_links.txt +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro.egg-info/requires.txt +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro.egg-info/top_level.txt +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/pyproject.toml +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/setup.cfg +0 -0
- {markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/tests/handlers/__init__.py +0 -0
|
@@ -38,7 +38,7 @@ from typing import Optional
|
|
|
38
38
|
import httpx
|
|
39
39
|
|
|
40
40
|
from .common.logger import logger
|
|
41
|
-
from .common.utils import detect_extension, ensure_minimum_content
|
|
41
|
+
from .common.utils import clean_markdown, detect_extension, ensure_minimum_content
|
|
42
42
|
from .handlers.audio_handler import AudioHandler
|
|
43
43
|
from .handlers.base_handler import BaseHandler
|
|
44
44
|
from .handlers.email_handler import EmailHandler
|
|
@@ -91,9 +91,10 @@ class ConversionPipeline:
|
|
|
91
91
|
|
|
92
92
|
# Map extensions to handlers (normalized lowercase with dot)
|
|
93
93
|
self.handlers_mapping: dict[str, BaseHandler] = {
|
|
94
|
-
# PDFs
|
|
94
|
+
# region PDFs
|
|
95
95
|
".pdf": self.pdf_handler,
|
|
96
|
-
#
|
|
96
|
+
# endregion
|
|
97
|
+
# region Audio
|
|
97
98
|
".mp3": self.audio_handler,
|
|
98
99
|
".wav": self.audio_handler,
|
|
99
100
|
# ".ogg": self.audio_handler,
|
|
@@ -103,36 +104,40 @@ class ConversionPipeline:
|
|
|
103
104
|
# ".wma": self.audio_handler,
|
|
104
105
|
# ".webm": self.audio_handler,
|
|
105
106
|
# ".opus": self.audio_handler,
|
|
106
|
-
#
|
|
107
|
+
# endregion
|
|
108
|
+
# region Images
|
|
107
109
|
".bmp": self.image_handler,
|
|
108
110
|
".gif": self.image_handler,
|
|
109
111
|
".heic": self.image_handler,
|
|
110
112
|
".jpeg": self.image_handler,
|
|
111
113
|
".jpg": self.image_handler,
|
|
112
114
|
".png": self.image_handler,
|
|
113
|
-
".prn": self.image_handler,
|
|
115
|
+
# ".prn": self.image_handler,
|
|
114
116
|
".svg": self.image_handler,
|
|
115
117
|
".tiff": self.image_handler,
|
|
116
118
|
".webp": self.image_handler,
|
|
117
119
|
".heif": self.image_handler,
|
|
118
|
-
#
|
|
120
|
+
# endregion
|
|
121
|
+
# region Plain/code text
|
|
119
122
|
".txt": self.text_handler,
|
|
120
123
|
".md": self.text_handler,
|
|
121
124
|
".py": self.text_handler,
|
|
122
125
|
".go": self.text_handler,
|
|
123
|
-
#
|
|
126
|
+
# endregion
|
|
127
|
+
# region Tabular (via MarkItDown)
|
|
124
128
|
".csv": self.tabular_handler,
|
|
125
129
|
".tsv": self.tabular_handler,
|
|
126
|
-
#
|
|
130
|
+
# endregion
|
|
131
|
+
# region Office/Spreadsheet/Slides
|
|
127
132
|
".xls": self.office_handler,
|
|
128
133
|
".xlsx": self.office_handler,
|
|
129
|
-
".doc": self.office_handler,
|
|
130
134
|
".docx": self.office_handler,
|
|
131
|
-
".odt": self.office_handler,
|
|
132
|
-
".rtf": self.office_handler,
|
|
133
|
-
".ppt": self.office_handler,
|
|
135
|
+
# ".odt": self.office_handler,
|
|
136
|
+
# ".rtf": self.office_handler,
|
|
137
|
+
# ".ppt": self.office_handler,
|
|
134
138
|
".pptx": self.office_handler,
|
|
135
|
-
#
|
|
139
|
+
# endregion
|
|
140
|
+
# region Markup/serialization
|
|
136
141
|
".html": self.markup_handler,
|
|
137
142
|
".htm": self.markup_handler,
|
|
138
143
|
".xml": self.markup_handler,
|
|
@@ -140,13 +145,15 @@ class ConversionPipeline:
|
|
|
140
145
|
".ndjson": self.markup_handler,
|
|
141
146
|
".yaml": self.markup_handler,
|
|
142
147
|
".yml": self.markup_handler,
|
|
143
|
-
#
|
|
148
|
+
# endregion
|
|
149
|
+
# region Others
|
|
144
150
|
".epub": self.epub_handler,
|
|
145
151
|
".eml": self.email_handler,
|
|
146
152
|
".p7s": self.email_handler,
|
|
147
153
|
".msg": self.email_handler,
|
|
148
154
|
".pst": self.pst_handler,
|
|
149
155
|
".ipynb": self.ipynb_handler,
|
|
156
|
+
# endregion
|
|
150
157
|
}
|
|
151
158
|
|
|
152
159
|
# ---------------------------------------------------------------------
|
|
@@ -190,6 +197,7 @@ class ConversionPipeline:
|
|
|
190
197
|
if not md_content or not ensure_minimum_content(md_content):
|
|
191
198
|
return None
|
|
192
199
|
|
|
200
|
+
md_content = clean_markdown(md_content)
|
|
193
201
|
logger.debug(
|
|
194
202
|
f"convert_document_to_md: {file_path} returned {len(md_content)} characters"
|
|
195
203
|
)
|
{markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/gpt_vision_wrapper.py
RENAMED
|
@@ -44,6 +44,20 @@ class GPTVisionWrapper(ConverterWrapper):
|
|
|
44
44
|
add that logic in `convert()` (see inline TODO in that method).
|
|
45
45
|
"""
|
|
46
46
|
|
|
47
|
+
SUPPORTED_EXTENSIONS = frozenset(
|
|
48
|
+
{
|
|
49
|
+
".bmp",
|
|
50
|
+
".gif",
|
|
51
|
+
".jpeg",
|
|
52
|
+
".jpg",
|
|
53
|
+
".png",
|
|
54
|
+
".tiff",
|
|
55
|
+
".webp",
|
|
56
|
+
".heic",
|
|
57
|
+
".heif",
|
|
58
|
+
}
|
|
59
|
+
)
|
|
60
|
+
|
|
47
61
|
def __init__(
|
|
48
62
|
self,
|
|
49
63
|
model_name: str = "gpt-4.1-mini",
|
|
@@ -82,3 +96,6 @@ class GPTVisionWrapper(ConverterWrapper):
|
|
|
82
96
|
return await self.gpt_vision.process_scanned_pdf_concurrent(file_path)
|
|
83
97
|
# Otherwise, assume it's an image and run single-image OCR/analysis.
|
|
84
98
|
return await self.gpt_vision.process_image(file_path)
|
|
99
|
+
|
|
100
|
+
async def aclose(self) -> None:
|
|
101
|
+
await self.gpt_vision.aclose()
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
from markitdown_pro.handlers.markitdown_handler import MarkItDownHandler
|
|
4
|
+
|
|
5
|
+
from ..common.logger import logger
|
|
6
|
+
from .base import ConverterWrapper
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class MarkItDownWrapper(ConverterWrapper):
|
|
10
|
+
SUPPORTED_EXTENSIONS = MarkItDownHandler.SUPPORTED_EXTENSIONS
|
|
11
|
+
|
|
12
|
+
def __init__(self, *args, **kwargs):
|
|
13
|
+
super().__init__(*args, **kwargs)
|
|
14
|
+
self.handler = MarkItDownHandler(*args, **kwargs)
|
|
15
|
+
|
|
16
|
+
async def convert(self, file_path: str) -> Optional[str]:
|
|
17
|
+
file_extension = file_path.split(".")[-1].lower()
|
|
18
|
+
if file_extension not in self.SUPPORTED_EXTENSIONS:
|
|
19
|
+
logger.warning(f"MarkItDownWrapper: Unsupported file format: {file_extension}")
|
|
20
|
+
return None
|
|
21
|
+
|
|
22
|
+
md_content = await self.handler.handle(file_path)
|
|
23
|
+
return md_content
|
|
@@ -1,25 +1,25 @@
|
|
|
1
|
-
import logging
|
|
2
1
|
from typing import Optional
|
|
3
2
|
|
|
4
3
|
import fitz # PyMuPDF
|
|
5
4
|
|
|
6
5
|
from ..common.logger import logger
|
|
7
|
-
from ..common.utils import clean_markdown, ensure_minimum_content
|
|
8
6
|
from .base import ConverterWrapper
|
|
9
7
|
|
|
10
8
|
|
|
11
9
|
class PyMuPDFWrapper(ConverterWrapper):
|
|
12
|
-
|
|
10
|
+
SUPPORTED_EXTENSIONS = frozenset({"pdf", "xps", "epub", "mobi", "fb2", "cbz", "svg"})
|
|
13
11
|
|
|
14
12
|
def __init__(self):
|
|
15
13
|
super().__init__("PyMuPDF")
|
|
16
14
|
|
|
17
15
|
async def convert(self, file_path: str) -> Optional[str]:
|
|
18
16
|
file_extension = file_path.split(".")[-1].lower()
|
|
19
|
-
|
|
17
|
+
logger.info(
|
|
18
|
+
f"PyMuPDFWrapper: Processing file: {file_path} with extension: {file_extension}"
|
|
19
|
+
)
|
|
20
20
|
|
|
21
|
-
if file_extension not in self.
|
|
22
|
-
logger.warning(f"Unsupported file format: {file_extension}")
|
|
21
|
+
if file_extension not in self.SUPPORTED_EXTENSIONS:
|
|
22
|
+
logger.warning(f"PyMuPDFWrapper: Unsupported file format: {file_extension}")
|
|
23
23
|
return None
|
|
24
24
|
|
|
25
25
|
try:
|
|
@@ -33,10 +33,8 @@ class PyMuPDFWrapper(ConverterWrapper):
|
|
|
33
33
|
text_content += "\n\n"
|
|
34
34
|
|
|
35
35
|
doc.close()
|
|
36
|
-
|
|
37
|
-
text_md = clean_markdown(text_content)
|
|
38
|
-
return text_md if ensure_minimum_content(text_md) else None
|
|
36
|
+
return text_content
|
|
39
37
|
|
|
40
38
|
except Exception as e:
|
|
41
|
-
logger.error(f"Error processing document with PyMuPDF: {e}")
|
|
39
|
+
logger.error(f"PyMuPDFWrapper: Error processing document with PyMuPDF: {e}")
|
|
42
40
|
return None
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
3
|
from ..common.logger import logger
|
|
4
|
-
from ..common.utils import clean_markdown
|
|
5
4
|
from ..handlers.tabular_handler import TabularHandler
|
|
6
5
|
from .base import ConverterWrapper
|
|
7
6
|
|
|
@@ -27,5 +26,4 @@ class TabularWrapper(ConverterWrapper):
|
|
|
27
26
|
return None
|
|
28
27
|
|
|
29
28
|
markdown = await self.tabular_handler.handle(file_path)
|
|
30
|
-
markdown = clean_markdown(markdown) if markdown else None
|
|
31
29
|
return markdown
|
{markitdown_pro-1.2.2 → markitdown_pro-1.3.0}/markitdown_pro/converters/unstructured_wrapper.py
RENAMED
|
@@ -3,7 +3,6 @@ from typing import Optional
|
|
|
3
3
|
from unstructured.partition.auto import partition
|
|
4
4
|
|
|
5
5
|
from ..common.logger import logger
|
|
6
|
-
from ..common.utils import clean_markdown, ensure_minimum_content
|
|
7
6
|
from .base import ConverterWrapper
|
|
8
7
|
|
|
9
8
|
|
|
@@ -58,5 +57,5 @@ class UnstructuredWrapper(ConverterWrapper):
|
|
|
58
57
|
|
|
59
58
|
# Directly call partition. No asyncio.to_thread
|
|
60
59
|
elements = partition(filename=file_path, extract_images_in_pdf=True)
|
|
61
|
-
combined =
|
|
62
|
-
return combined
|
|
60
|
+
combined = "\n\n".join(str(el) for el in elements)
|
|
61
|
+
return combined
|
|
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|
|
3
3
|
from typing import Optional
|
|
4
4
|
|
|
5
5
|
from ..common.logger import logger
|
|
6
|
-
from ..common.utils import ensure_minimum_content
|
|
7
6
|
from ..converters.azure_speech_wrapper import AzureSpeechWrapper
|
|
8
7
|
from ..converters.base import ConverterWrapper
|
|
9
8
|
from ..converters.markitdown_wrapper import MarkItDownWrapper
|
|
@@ -27,7 +26,7 @@ class AudioHandler(BaseHandler):
|
|
|
27
26
|
- This handler runs converters **sequentially** to avoid unnecessary API calls.
|
|
28
27
|
"""
|
|
29
28
|
|
|
30
|
-
|
|
29
|
+
SUPPORTED_EXTENSIONS = frozenset(
|
|
31
30
|
{
|
|
32
31
|
".mp3",
|
|
33
32
|
".wav",
|
|
@@ -72,10 +71,7 @@ class AudioHandler(BaseHandler):
|
|
|
72
71
|
logger.info(f"AudioHandler: Trying {name} for '{file_path}'")
|
|
73
72
|
try:
|
|
74
73
|
md = await converter.convert(file_path)
|
|
75
|
-
|
|
76
|
-
logger.info(f"AudioHandler: {name} succeeded for '{file_path}'")
|
|
77
|
-
return md
|
|
78
|
-
logger.debug(f"AudioHandler: {name} returned insufficient content")
|
|
74
|
+
return md
|
|
79
75
|
except Exception as e:
|
|
80
76
|
logger.error(f"AudioHandler: {name} failed for '{file_path}': {e}")
|
|
81
77
|
|
|
@@ -3,7 +3,7 @@ from typing import Optional
|
|
|
3
3
|
|
|
4
4
|
|
|
5
5
|
class BaseHandler:
|
|
6
|
-
|
|
6
|
+
SUPPORTED_EXTENSIONS: frozenset = frozenset()
|
|
7
7
|
pipeline: Optional[list] = None
|
|
8
8
|
|
|
9
9
|
def __init__(self, *args, **kwargs):
|
|
@@ -11,7 +11,7 @@ class BaseHandler:
|
|
|
11
11
|
|
|
12
12
|
@classmethod
|
|
13
13
|
async def is_valid(cls, file_path: str) -> bool:
|
|
14
|
-
return Path(file_path).is_file() and Path(file_path).suffix in cls.
|
|
14
|
+
return Path(file_path).is_file() and Path(file_path).suffix in cls.SUPPORTED_EXTENSIONS
|
|
15
15
|
|
|
16
16
|
async def handle(self, file_path, *args, **kwargs) -> Optional[str]:
|
|
17
17
|
raise NotImplementedError("You must implement the handle method")
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import html
|
|
3
|
+
import io
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import tempfile
|
|
7
|
+
import uuid
|
|
8
|
+
from email.header import decode_header, make_header
|
|
9
|
+
from email.message import Message
|
|
10
|
+
from email.parser import BytesParser
|
|
11
|
+
from email.policy import default
|
|
12
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
13
|
+
|
|
14
|
+
from ..common.logger import logger
|
|
15
|
+
from .base_handler import BaseHandler
|
|
16
|
+
|
|
17
|
+
_MAX_ATTACHMENT_BYTES = 50 * 1024 * 1024 # 50MB safety cap
|
|
18
|
+
_READ_CHUNK = 1024 * 1024 # 1MB
|
|
19
|
+
_PARSE_TIMEOUT_SEC = 30
|
|
20
|
+
_CONVERT_TIMEOUT_SEC = 60
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _safe_decode_header(value: Optional[str]) -> str:
|
|
24
|
+
try:
|
|
25
|
+
if not value:
|
|
26
|
+
return ""
|
|
27
|
+
return str(make_header(decode_header(value)))
|
|
28
|
+
except Exception:
|
|
29
|
+
return value or ""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _coerce_charset(s: bytes, charset: Optional[str]) -> str:
|
|
33
|
+
for cs in [charset, "utf-8", "latin-1"]:
|
|
34
|
+
if not cs:
|
|
35
|
+
continue
|
|
36
|
+
try:
|
|
37
|
+
return s.decode(cs, errors="replace")
|
|
38
|
+
except Exception:
|
|
39
|
+
continue
|
|
40
|
+
return s.decode("utf-8", errors="replace")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _basic_html_to_md(html_text: str) -> str:
|
|
44
|
+
# Minimal, dependency-free HTML → Markdown-ish conversion that preserves readability
|
|
45
|
+
txt = html_text
|
|
46
|
+
txt = re.sub(r"(?is)<(script|style).*?>.*?</\1>", "", txt)
|
|
47
|
+
txt = re.sub(r"(?i)<br\s*/?>", "\n", txt)
|
|
48
|
+
txt = re.sub(r"(?i)</p\s*>", "\n\n", txt)
|
|
49
|
+
txt = re.sub(r"(?i)<li\s*>", "- ", txt)
|
|
50
|
+
txt = re.sub(r"(?i)</h1\s*>", "\n\n", re.sub(r"(?i)<h1\s*>(.*?)</h1\s*>", r"# \1\n\n", txt))
|
|
51
|
+
txt = re.sub(r"(?i)</h2\s*>", "\n\n", re.sub(r"(?i)<h2\s*>(.*?)</h2\s*>", r"## \1\n\n", txt))
|
|
52
|
+
txt = re.sub(r"(?i)</h3\s*>", "\n\n", re.sub(r"(?i)<h3\s*>(.*?)</h3\s*>", r"### \1\n\n", txt))
|
|
53
|
+
txt = re.sub(r"(?i)<a\s+[^>]*href=['\"]([^'\"]+)['\"][^>]*>(.*?)</a>", r"[\2](\1)", txt)
|
|
54
|
+
txt = re.sub(
|
|
55
|
+
r"(?i)<img\s+[^>]*alt=['\"]([^'\"]*)['\"][^>]*src=['\"]([^'\"]+)['\"][^>]*>",
|
|
56
|
+
r"",
|
|
57
|
+
txt,
|
|
58
|
+
)
|
|
59
|
+
txt = re.sub(r"(?is)<[^>]+>", "", txt)
|
|
60
|
+
txt = html.unescape(txt)
|
|
61
|
+
# Normalize whitespace
|
|
62
|
+
txt = re.sub(r"[ \t]+\n", "\n", txt)
|
|
63
|
+
txt = re.sub(r"\n{3,}", "\n\n", txt).strip()
|
|
64
|
+
return txt
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class EmailHandler(BaseHandler):
|
|
68
|
+
SUPPORTED_EXTENSIONS = frozenset({".eml", ".p7s", ".msg"})
|
|
69
|
+
|
|
70
|
+
async def handle(self, file_path: str, *args, **kwargs) -> Optional[str]:
|
|
71
|
+
logger.info(f"EmailHandler: Processing email file: {file_path}")
|
|
72
|
+
attachment_converter: Optional[Callable[[str, str], "asyncio.Future[str]"]] = kwargs.get(
|
|
73
|
+
"attachment_converter"
|
|
74
|
+
)
|
|
75
|
+
try:
|
|
76
|
+
email_data = await asyncio.wait_for(
|
|
77
|
+
asyncio.to_thread(self._parse_email, file_path), timeout=_PARSE_TIMEOUT_SEC
|
|
78
|
+
)
|
|
79
|
+
if not email_data:
|
|
80
|
+
return None
|
|
81
|
+
markdown_content = await self._build_markdown(email_data, attachment_converter)
|
|
82
|
+
return markdown_content
|
|
83
|
+
except asyncio.TimeoutError:
|
|
84
|
+
logger.error(f"EmailHandler: Timeout while processing email file: {file_path}")
|
|
85
|
+
return None
|
|
86
|
+
except Exception as e:
|
|
87
|
+
logger.exception(f"EmailHandler: Error processing email file: {file_path}: {e}")
|
|
88
|
+
return None
|
|
89
|
+
|
|
90
|
+
def _parse_email(self, file_path: str) -> Dict[str, Any]:
|
|
91
|
+
"""
|
|
92
|
+
Parses the EML/P7S file and extracts relevant information.
|
|
93
|
+
Returns:
|
|
94
|
+
{
|
|
95
|
+
"subject": str,
|
|
96
|
+
"from": str,
|
|
97
|
+
"to": str,
|
|
98
|
+
"cc": str,
|
|
99
|
+
"bcc": str,
|
|
100
|
+
"date": str,
|
|
101
|
+
"body_text": str, # text/plain best-effort
|
|
102
|
+
"body_html": str, # raw HTML if present
|
|
103
|
+
"attachments": List[Dict[str, str]] # [{"filename":..., "path":..., "content_type":..., "cid":...}]
|
|
104
|
+
}
|
|
105
|
+
"""
|
|
106
|
+
try:
|
|
107
|
+
with open(file_path, "rb") as f:
|
|
108
|
+
msg: Message = BytesParser(policy=default).parse(f)
|
|
109
|
+
subject = _safe_decode_header(msg.get("Subject", "(No Subject)")) or "(No Subject)"
|
|
110
|
+
from_ = _safe_decode_header(msg.get("From", "(Unknown Sender)")) or "(Unknown Sender)"
|
|
111
|
+
to_ = _safe_decode_header(msg.get("To", "")) or ""
|
|
112
|
+
cc_ = _safe_decode_header(msg.get("Cc", "")) or ""
|
|
113
|
+
bcc_ = _safe_decode_header(msg.get("Bcc", "")) or ""
|
|
114
|
+
date_ = msg.get("Date", "(Unknown Date)") or "(Unknown Date)"
|
|
115
|
+
body_text = ""
|
|
116
|
+
body_html = ""
|
|
117
|
+
attachments: List[Dict[str, str]] = []
|
|
118
|
+
|
|
119
|
+
def _should_skip(part: Message) -> bool:
|
|
120
|
+
ctype = (part.get_content_type() or "").lower()
|
|
121
|
+
# Skip S/MIME signatures and pkcs7 blobs as attachments
|
|
122
|
+
if ctype in ("application/pkcs7-signature", "application/pkcs7-mime"):
|
|
123
|
+
return True
|
|
124
|
+
return False
|
|
125
|
+
|
|
126
|
+
if msg.is_multipart():
|
|
127
|
+
for part in msg.walk():
|
|
128
|
+
if part.is_multipart():
|
|
129
|
+
continue
|
|
130
|
+
if _should_skip(part):
|
|
131
|
+
continue
|
|
132
|
+
disp = part.get_content_disposition() or ""
|
|
133
|
+
ctype = (part.get_content_type() or "").lower()
|
|
134
|
+
cid = (part.get("Content-ID") or "").strip().strip("<>")
|
|
135
|
+
payload = part.get_payload(decode=True) or b""
|
|
136
|
+
if disp == "attachment" or (
|
|
137
|
+
disp == "inline" and cid
|
|
138
|
+
): # treat inline with CID as attachment (e.g., images)
|
|
139
|
+
filename = part.get_filename()
|
|
140
|
+
filename = (
|
|
141
|
+
_safe_decode_header(filename)
|
|
142
|
+
if filename
|
|
143
|
+
else f"attachment-{uuid.uuid4().hex}"
|
|
144
|
+
)
|
|
145
|
+
# Write safely with size cap
|
|
146
|
+
suffix = os.path.splitext(filename)[1][:10] if filename else ""
|
|
147
|
+
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp_file:
|
|
148
|
+
written = 0
|
|
149
|
+
stream = io.BytesIO(payload)
|
|
150
|
+
while True:
|
|
151
|
+
chunk = stream.read(_READ_CHUNK)
|
|
152
|
+
if not chunk:
|
|
153
|
+
break
|
|
154
|
+
written += len(chunk)
|
|
155
|
+
if written > _MAX_ATTACHMENT_BYTES:
|
|
156
|
+
raise ValueError(
|
|
157
|
+
f"Attachment '{filename}' exceeds {_MAX_ATTACHMENT_BYTES} bytes"
|
|
158
|
+
)
|
|
159
|
+
tmp_file.write(chunk)
|
|
160
|
+
attachments.append(
|
|
161
|
+
{
|
|
162
|
+
"filename": filename,
|
|
163
|
+
"path": tmp_file.name,
|
|
164
|
+
"content_type": ctype,
|
|
165
|
+
"cid": cid,
|
|
166
|
+
}
|
|
167
|
+
)
|
|
168
|
+
continue
|
|
169
|
+
if ctype == "text/plain" and not body_text:
|
|
170
|
+
body_text = _coerce_charset(payload, part.get_content_charset() or "utf-8")
|
|
171
|
+
elif ctype == "text/html" and not body_html:
|
|
172
|
+
body_html = _coerce_charset(payload, part.get_content_charset() or "utf-8")
|
|
173
|
+
else:
|
|
174
|
+
ctype = (msg.get_content_type() or "").lower()
|
|
175
|
+
if ctype == "text/plain":
|
|
176
|
+
body_text = _coerce_charset(
|
|
177
|
+
msg.get_payload(decode=True) or b"", msg.get_content_charset() or "utf-8"
|
|
178
|
+
)
|
|
179
|
+
elif ctype == "text/html":
|
|
180
|
+
body_html = _coerce_charset(
|
|
181
|
+
msg.get_payload(decode=True) or b"", msg.get_content_charset() or "utf-8"
|
|
182
|
+
)
|
|
183
|
+
return {
|
|
184
|
+
"subject": subject,
|
|
185
|
+
"from": from_,
|
|
186
|
+
"to": to_,
|
|
187
|
+
"cc": cc_,
|
|
188
|
+
"bcc": bcc_,
|
|
189
|
+
"date": date_,
|
|
190
|
+
"body_text": body_text.strip(),
|
|
191
|
+
"body_html": body_html.strip(),
|
|
192
|
+
"attachments": attachments,
|
|
193
|
+
}
|
|
194
|
+
except Exception as e:
|
|
195
|
+
logger.exception(f"EmailHandler: Error parsing email {file_path}: {e}")
|
|
196
|
+
return {}
|
|
197
|
+
|
|
198
|
+
async def _build_markdown(
|
|
199
|
+
self,
|
|
200
|
+
email_data: Dict[str, Any],
|
|
201
|
+
attachment_converter: Optional[Callable[[str, str], "asyncio.Future[str]"]] = None,
|
|
202
|
+
) -> str:
|
|
203
|
+
# Resolve inline CID references in HTML to temp file paths (if any)
|
|
204
|
+
cid_map = {a["cid"]: a for a in email_data.get("attachments", []) if a.get("cid")}
|
|
205
|
+
body_md = ""
|
|
206
|
+
if email_data.get("body_text"):
|
|
207
|
+
body_md = email_data["body_text"]
|
|
208
|
+
elif email_data.get("body_html"):
|
|
209
|
+
html_body = email_data["body_html"]
|
|
210
|
+
if cid_map:
|
|
211
|
+
for cid, meta in cid_map.items():
|
|
212
|
+
html_body = re.sub(
|
|
213
|
+
rf"(?i)src=['\"]cid:{re.escape(cid)}['\"]",
|
|
214
|
+
f"src=\"{meta['path']}\"",
|
|
215
|
+
html_body,
|
|
216
|
+
)
|
|
217
|
+
body_md = _basic_html_to_md(html_body)
|
|
218
|
+
header_lines = [
|
|
219
|
+
f"# Email: {email_data.get('subject', '(No Subject)')}",
|
|
220
|
+
f"**From:** {email_data.get('from', '')}",
|
|
221
|
+
f"**To:** {email_data.get('to', '')}",
|
|
222
|
+
]
|
|
223
|
+
if email_data.get("cc"):
|
|
224
|
+
header_lines.append(f"**Cc:** {email_data['cc']}")
|
|
225
|
+
if email_data.get("bcc"):
|
|
226
|
+
header_lines.append(f"**Bcc:** {email_data['bcc']}")
|
|
227
|
+
header_lines.append(f"**Date:** {email_data.get('date', '')}")
|
|
228
|
+
markdown_parts: List[str] = ["\n".join(header_lines), ""]
|
|
229
|
+
if body_md:
|
|
230
|
+
markdown_parts.append("```")
|
|
231
|
+
markdown_parts.append(body_md)
|
|
232
|
+
markdown_parts.append("```")
|
|
233
|
+
# Convert attachments (non-CID inline images will also appear here)
|
|
234
|
+
attachments = email_data.get("attachments", [])
|
|
235
|
+
for att in attachments:
|
|
236
|
+
filename = att.get("filename", "attachment")
|
|
237
|
+
path = att.get("path")
|
|
238
|
+
ctype = att.get("content_type", "")
|
|
239
|
+
if att.get("cid"): # Already linked in body if referenced; still list it
|
|
240
|
+
markdown_parts.append(f"\n## Inline asset: {filename} ({ctype})\n")
|
|
241
|
+
markdown_parts.append(
|
|
242
|
+
f""
|
|
243
|
+
if ctype.startswith("image/")
|
|
244
|
+
else f"[{filename}]({path})"
|
|
245
|
+
)
|
|
246
|
+
continue
|
|
247
|
+
markdown_parts.append(f"\n## Attachment: {filename} ({ctype})\n")
|
|
248
|
+
try:
|
|
249
|
+
if attachment_converter:
|
|
250
|
+
converted = await asyncio.wait_for(
|
|
251
|
+
attachment_converter(path, ctype), timeout=_CONVERT_TIMEOUT_SEC
|
|
252
|
+
)
|
|
253
|
+
markdown_parts.append(converted if converted else f"[Saved attachment]({path})")
|
|
254
|
+
else:
|
|
255
|
+
# Built-in light handling when no converter provided
|
|
256
|
+
if ctype.startswith("image/"):
|
|
257
|
+
markdown_parts.append(f"")
|
|
258
|
+
elif ctype.startswith("text/"):
|
|
259
|
+
try:
|
|
260
|
+
with open(path, "rb") as f:
|
|
261
|
+
content = f.read(_MAX_ATTACHMENT_BYTES + 1)
|
|
262
|
+
if len(content) > _MAX_ATTACHMENT_BYTES:
|
|
263
|
+
raise ValueError(
|
|
264
|
+
f"Attachment too large to inline (> {_MAX_ATTACHMENT_BYTES} bytes)"
|
|
265
|
+
)
|
|
266
|
+
text = _coerce_charset(content, "utf-8")
|
|
267
|
+
markdown_parts.append("\n```")
|
|
268
|
+
markdown_parts.append(text.strip())
|
|
269
|
+
markdown_parts.append("```")
|
|
270
|
+
except Exception as e:
|
|
271
|
+
logger.warning(
|
|
272
|
+
f"EmailHandler: Inline text fallback failed for {filename}: {e}"
|
|
273
|
+
)
|
|
274
|
+
markdown_parts.append(f"[Saved attachment]({path})")
|
|
275
|
+
elif ctype == "text/html":
|
|
276
|
+
try:
|
|
277
|
+
with open(path, "rb") as f:
|
|
278
|
+
content = f.read(min(_MAX_ATTACHMENT_BYTES, 5 * 1024 * 1024))
|
|
279
|
+
text = _coerce_charset(content, "utf-8")
|
|
280
|
+
markdown_parts.append(_basic_html_to_md(text))
|
|
281
|
+
except Exception as e:
|
|
282
|
+
logger.warning(
|
|
283
|
+
f"EmailHandler: HTML to MD fallback failed for {filename}: {e}"
|
|
284
|
+
)
|
|
285
|
+
markdown_parts.append(f"[Saved attachment]({path})")
|
|
286
|
+
else:
|
|
287
|
+
markdown_parts.append(f"[Saved attachment]({path})")
|
|
288
|
+
except asyncio.TimeoutError:
|
|
289
|
+
logger.error(f"EmailHandler: Timeout converting attachment '{filename}'")
|
|
290
|
+
markdown_parts.append(f"[Attachment conversion timed out: {filename}]")
|
|
291
|
+
except Exception as e:
|
|
292
|
+
logger.error(f"EmailHandler: Error converting attachment '{filename}': {e}")
|
|
293
|
+
markdown_parts.append(f"[Error converting attachment: {e}]")
|
|
294
|
+
finally:
|
|
295
|
+
try:
|
|
296
|
+
os.remove(path)
|
|
297
|
+
except OSError as e:
|
|
298
|
+
logger.warning(
|
|
299
|
+
f"EmailHandler: Could not remove temporary attachment file '{path}': {e}"
|
|
300
|
+
)
|
|
301
|
+
return "\n".join(markdown_parts)
|