thunderbird-mcp 0.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- thunderbird_mcp/__init__.py +8 -0
- thunderbird_mcp/attachments.py +302 -0
- thunderbird_mcp/automation.py +441 -0
- thunderbird_mcp/cli.py +450 -0
- thunderbird_mcp/enrollment.py +226 -0
- thunderbird_mcp/exports.py +189 -0
- thunderbird_mcp/imap.py +353 -0
- thunderbird_mcp/index.py +547 -0
- thunderbird_mcp/insights.py +156 -0
- thunderbird_mcp/live.py +506 -0
- thunderbird_mcp/mailbox.py +295 -0
- thunderbird_mcp/maildir.py +206 -0
- thunderbird_mcp/messages.py +448 -0
- thunderbird_mcp/oauth.py +296 -0
- thunderbird_mcp/ocr.py +240 -0
- thunderbird_mcp/operations.py +288 -0
- thunderbird_mcp/outgoing.py +1245 -0
- thunderbird_mcp/pagination.py +146 -0
- thunderbird_mcp/policy.py +336 -0
- thunderbird_mcp/profile.py +151 -0
- thunderbird_mcp/providers.py +543 -0
- thunderbird_mcp/remote.py +185 -0
- thunderbird_mcp/runtime.py +62 -0
- thunderbird_mcp/search.py +541 -0
- thunderbird_mcp/server.py +756 -0
- thunderbird_mcp/service.py +73 -0
- thunderbird_mcp/setup.py +379 -0
- thunderbird_mcp/status.py +115 -0
- thunderbird_mcp/web.html +173 -0
- thunderbird_mcp/web.py +109 -0
- thunderbird_mcp-0.4.1.dist-info/METADATA +591 -0
- thunderbird_mcp-0.4.1.dist-info/RECORD +35 -0
- thunderbird_mcp-0.4.1.dist-info/WHEEL +4 -0
- thunderbird_mcp-0.4.1.dist-info/entry_points.txt +10 -0
- thunderbird_mcp-0.4.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
"""Bounded, local attachment text extraction. Files are never executed or uploaded."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import io
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
from pathlib import PurePosixPath
|
|
10
|
+
import re
|
|
11
|
+
import subprocess
|
|
12
|
+
import sys
|
|
13
|
+
import zipfile
|
|
14
|
+
from html.parser import HTMLParser
|
|
15
|
+
|
|
16
|
+
MAX_ATTACHMENT_BYTES = 8 * 1024 * 1024
|
|
17
|
+
MAX_MESSAGE_BYTES = 24 * 1024 * 1024
|
|
18
|
+
MAX_EXPANDED_BYTES = 16 * 1024 * 1024
|
|
19
|
+
MAX_ZIP_ENTRIES = 2048
|
|
20
|
+
MAX_ATTACHMENT_CHARS = 25_000
|
|
21
|
+
MAX_MESSAGE_CHARS = 100_000
|
|
22
|
+
MAX_ATTACHMENTS = 64
|
|
23
|
+
MAX_PDF_PAGES = 50
|
|
24
|
+
PDF_TIMEOUT_SECONDS = 10
|
|
25
|
+
|
|
26
|
+
_TEXT_SUFFIXES = {".txt", ".csv", ".tsv", ".json", ".md", ".log", ".yaml", ".yml", ".ics"}
|
|
27
|
+
_OOXML_TYPES = {
|
|
28
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
|
|
29
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
|
|
30
|
+
"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class ExtractionLimit(ValueError):
|
|
35
|
+
"""A document exceeded a fixed extraction limit."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class EncryptedDocument(ValueError):
|
|
39
|
+
"""Encrypted attachments require opening explicitly in another application."""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class _HTMLText(HTMLParser):
|
|
43
|
+
def __init__(self):
|
|
44
|
+
super().__init__(convert_charrefs=True)
|
|
45
|
+
self.parts = []
|
|
46
|
+
self.skip = 0
|
|
47
|
+
|
|
48
|
+
def handle_starttag(self, tag, attrs):
|
|
49
|
+
if tag in {"script", "style", "head"}:
|
|
50
|
+
self.skip += 1
|
|
51
|
+
if tag in {"p", "br", "div", "li", "tr"}:
|
|
52
|
+
self.parts.append(" ")
|
|
53
|
+
|
|
54
|
+
def handle_endtag(self, tag):
|
|
55
|
+
if tag in {"script", "style", "head"} and self.skip:
|
|
56
|
+
self.skip -= 1
|
|
57
|
+
self.parts.append(" ")
|
|
58
|
+
|
|
59
|
+
def handle_data(self, value):
|
|
60
|
+
if not self.skip:
|
|
61
|
+
self.parts.append(value)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _plain(value: str) -> str:
|
|
65
|
+
return " ".join(re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", " ", value).split())
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _decode(payload: bytes, charset: str | None) -> str:
|
|
69
|
+
if payload.startswith((b"\xff\xfe", b"\xfe\xff")):
|
|
70
|
+
return payload.decode("utf-16", "replace")
|
|
71
|
+
try:
|
|
72
|
+
return payload.decode(charset or "utf-8-sig", "replace")
|
|
73
|
+
except LookupError:
|
|
74
|
+
return payload.decode("utf-8", "replace")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _xml(payload: bytes):
|
|
78
|
+
from defusedxml.ElementTree import fromstring
|
|
79
|
+
return fromstring(payload, forbid_dtd=True, forbid_entities=True, forbid_external=True)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _office(payload: bytes, suffix: str) -> str:
|
|
83
|
+
with zipfile.ZipFile(io.BytesIO(payload)) as archive:
|
|
84
|
+
entries = archive.infolist()
|
|
85
|
+
if len(entries) > MAX_ZIP_ENTRIES:
|
|
86
|
+
raise ExtractionLimit("Archive has too many entries")
|
|
87
|
+
if any(entry.flag_bits & 1 for entry in entries):
|
|
88
|
+
raise EncryptedDocument("Encrypted archive")
|
|
89
|
+
if sum(entry.file_size for entry in entries) > MAX_EXPANDED_BYTES:
|
|
90
|
+
raise ExtractionLimit("Archive exceeds expanded byte limit")
|
|
91
|
+
if any(entry.file_size > MAX_EXPANDED_BYTES for entry in entries):
|
|
92
|
+
raise ExtractionLimit("Archive entry exceeds byte limit")
|
|
93
|
+
# Read only document XML. No files, relationships, links, or macros run.
|
|
94
|
+
names = set(archive.namelist())
|
|
95
|
+
if suffix == ".docx":
|
|
96
|
+
selected = sorted(name for name in names if re.fullmatch(
|
|
97
|
+
r"word/(document|header\d+|footer\d+|footnotes|endnotes|comments)\.xml", name))
|
|
98
|
+
elif suffix == ".pptx":
|
|
99
|
+
selected = sorted(name for name in names if re.fullmatch(
|
|
100
|
+
r"ppt/(slides/slide\d+|notesSlides/notesSlide\d+)\.xml", name))
|
|
101
|
+
else:
|
|
102
|
+
selected = sorted(name for name in names if re.fullmatch(
|
|
103
|
+
r"xl/(sharedStrings|worksheets/sheet\d+|comments\d+)\.xml", name))
|
|
104
|
+
if not selected:
|
|
105
|
+
raise ValueError("Document XML missing")
|
|
106
|
+
parts, chars, expanded = [], 0, 0
|
|
107
|
+
for name in selected:
|
|
108
|
+
# ZipFile checks the central-directory length, and a bounded read
|
|
109
|
+
# also protects against inconsistent or malformed declarations.
|
|
110
|
+
with archive.open(name) as member:
|
|
111
|
+
data = member.read(MAX_EXPANDED_BYTES - expanded + 1)
|
|
112
|
+
expanded += len(data)
|
|
113
|
+
if expanded > MAX_EXPANDED_BYTES:
|
|
114
|
+
raise ExtractionLimit("Document exceeds expanded byte limit")
|
|
115
|
+
root = _xml(data)
|
|
116
|
+
# Ignore spreadsheet formulas; retain their cached values and text.
|
|
117
|
+
tags = {"t", "v"} if suffix == ".xlsx" else {"t"}
|
|
118
|
+
for node in root.iter():
|
|
119
|
+
if node.tag.rsplit("}", 1)[-1] in tags and node.text:
|
|
120
|
+
parts.append(node.text)
|
|
121
|
+
chars += len(node.text) + 1
|
|
122
|
+
if chars > MAX_ATTACHMENT_CHARS:
|
|
123
|
+
return " ".join(parts)
|
|
124
|
+
return " ".join(parts)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _pdf_document(payload: bytes) -> dict:
|
|
128
|
+
"""Run only in the bounded child process below."""
|
|
129
|
+
from pypdf import PdfReader
|
|
130
|
+
from pypdf import filters
|
|
131
|
+
# pypdf also bounds its Flate decompressor; our process limits contain
|
|
132
|
+
# resource use in other PDF encodings and damaged object graphs.
|
|
133
|
+
filters.ZLIB_MAX_OUTPUT_LENGTH = MAX_EXPANDED_BYTES
|
|
134
|
+
reader = PdfReader(io.BytesIO(payload), strict=False)
|
|
135
|
+
if reader.is_encrypted:
|
|
136
|
+
raise EncryptedDocument("Encrypted PDF")
|
|
137
|
+
parts, size = [], 0
|
|
138
|
+
truncated = len(reader.pages) > MAX_PDF_PAGES
|
|
139
|
+
for page in reader.pages[:MAX_PDF_PAGES]:
|
|
140
|
+
contents = page.get_contents()
|
|
141
|
+
if contents is not None and len(contents.get_data()) > MAX_EXPANDED_BYTES:
|
|
142
|
+
raise ExtractionLimit("PDF page exceeds expanded byte limit")
|
|
143
|
+
value = page.extract_text() or ""
|
|
144
|
+
parts.append(value)
|
|
145
|
+
size += len(value)
|
|
146
|
+
if size > MAX_ATTACHMENT_CHARS:
|
|
147
|
+
truncated = True
|
|
148
|
+
break
|
|
149
|
+
return {"text": "\n".join(parts)[:MAX_ATTACHMENT_CHARS + 1], "truncated": truncated}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _pdf(payload: bytes) -> dict:
|
|
153
|
+
try:
|
|
154
|
+
result = subprocess.run(
|
|
155
|
+
[sys.executable, "-m", "thunderbird_mcp.attachments", "--pdf-worker"],
|
|
156
|
+
input=payload, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
|
157
|
+
timeout=PDF_TIMEOUT_SECONDS, check=False,
|
|
158
|
+
)
|
|
159
|
+
except subprocess.TimeoutExpired as exc:
|
|
160
|
+
raise ExtractionLimit("PDF extraction timed out") from exc
|
|
161
|
+
if result.returncode != 0:
|
|
162
|
+
raise ExtractionLimit("PDF extraction exceeded process resource limits")
|
|
163
|
+
data = json.loads(result.stdout)
|
|
164
|
+
if data.get("status") == "encrypted":
|
|
165
|
+
raise EncryptedDocument("Encrypted PDF")
|
|
166
|
+
if data.get("status") == "too_large":
|
|
167
|
+
raise ExtractionLimit("PDF exceeds extraction limits")
|
|
168
|
+
if data.get("status") != "extracted":
|
|
169
|
+
raise ValueError("PDF could not be parsed")
|
|
170
|
+
return data
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def extract_text(payload: bytes, filename: str = "", content_type: str = "", *,
|
|
174
|
+
charset: str | None = None) -> dict:
|
|
175
|
+
"""Return bounded text and a redacted, machine-readable extraction status."""
|
|
176
|
+
result = {"text": "", "status": "unsupported", "truncated": False}
|
|
177
|
+
if len(payload) > MAX_ATTACHMENT_BYTES:
|
|
178
|
+
return {**result, "status": "too_large", "reason": "attachment_byte_limit"}
|
|
179
|
+
suffix = PurePosixPath(filename.lower()).suffix
|
|
180
|
+
ctype = content_type.lower().split(";", 1)[0]
|
|
181
|
+
try:
|
|
182
|
+
truncated = False
|
|
183
|
+
if suffix == ".pdf" or ctype == "application/pdf":
|
|
184
|
+
pdf = _pdf(payload)
|
|
185
|
+
value, truncated = pdf["text"], pdf["truncated"]
|
|
186
|
+
if not value.strip():
|
|
187
|
+
from . import ocr
|
|
188
|
+
if ocr.enabled():
|
|
189
|
+
return ocr.extract_text(payload, pdf=True)
|
|
190
|
+
elif suffix in {".docx", ".xlsx", ".pptx"} or ctype in _OOXML_TYPES:
|
|
191
|
+
value = _office(payload, _OOXML_TYPES.get(ctype, suffix))
|
|
192
|
+
elif suffix in {".html", ".htm"} or ctype == "text/html":
|
|
193
|
+
parser = _HTMLText()
|
|
194
|
+
parser.feed(_decode(payload, charset))
|
|
195
|
+
value = " ".join(parser.parts)
|
|
196
|
+
elif suffix == ".xml" or ctype in {"text/xml", "application/xml"} or ctype.endswith("+xml"):
|
|
197
|
+
value = " ".join(_xml(payload).itertext())
|
|
198
|
+
elif suffix in _TEXT_SUFFIXES or ctype.startswith("text/") or ctype in {"application/json", "application/ld+json"}:
|
|
199
|
+
value = _decode(payload, charset)
|
|
200
|
+
elif suffix in {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".gif", ".webp"} or ctype in {
|
|
201
|
+
"image/png", "image/jpeg", "image/tiff", "image/bmp", "image/gif", "image/webp"}:
|
|
202
|
+
from . import ocr
|
|
203
|
+
return ocr.extract_text(payload)
|
|
204
|
+
else:
|
|
205
|
+
return {**result, "reason": "format_not_supported"}
|
|
206
|
+
value = _plain(value)
|
|
207
|
+
return {"text": value[:MAX_ATTACHMENT_CHARS], "status": "extracted" if value else "empty",
|
|
208
|
+
"truncated": truncated or len(value) > MAX_ATTACHMENT_CHARS,
|
|
209
|
+
**({"reason": "no_extractable_text_or_scanned_document"} if not value else {})}
|
|
210
|
+
except EncryptedDocument:
|
|
211
|
+
return {**result, "status": "encrypted", "reason": "password_required"}
|
|
212
|
+
except ExtractionLimit:
|
|
213
|
+
return {**result, "status": "too_large", "reason": "extraction_resource_limit"}
|
|
214
|
+
except Exception:
|
|
215
|
+
# Parser errors can contain document text or local paths. Keep them
|
|
216
|
+
# private while retaining a useful status for the caller.
|
|
217
|
+
return {**result, "status": "error", "reason": "document_could_not_be_parsed"}
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _filename(value: str | None, number: int) -> str:
|
|
221
|
+
value = (value or f"attachment-{number}").replace("\\", "/").rsplit("/", 1)[-1]
|
|
222
|
+
return _plain(value)[:255] or f"attachment-{number}"
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def extract_message(msg) -> list[dict]:
|
|
226
|
+
"""Extract up to 64 MIME attachments, with shared byte and text budgets."""
|
|
227
|
+
result, used_bytes, used_chars = [], 0, 0
|
|
228
|
+
for part in msg.walk():
|
|
229
|
+
if part.is_multipart() or not (part.get_content_disposition() == "attachment" or part.get_filename()):
|
|
230
|
+
continue
|
|
231
|
+
number = len(result) + 1
|
|
232
|
+
if number > MAX_ATTACHMENTS:
|
|
233
|
+
result.append({"id": number, "filename": "", "content_type": "", "size": None,
|
|
234
|
+
"text": "", "status": "too_many", "truncated": True,
|
|
235
|
+
"reason": "message_attachment_limit"})
|
|
236
|
+
break
|
|
237
|
+
filename = _filename(part.get_filename(), number)
|
|
238
|
+
meta = {"id": number, "filename": filename, "content_type": part.get_content_type(),
|
|
239
|
+
"size": None, "text": "", "status": "too_large", "truncated": False}
|
|
240
|
+
raw = part.get_payload()
|
|
241
|
+
# Check before base64 decoding. The email parser already holds encoded
|
|
242
|
+
# content; a second unbounded decoded allocation is unnecessary.
|
|
243
|
+
if not isinstance(raw, str) or len(raw) > MAX_ATTACHMENT_BYTES * 2:
|
|
244
|
+
result.append({**meta, "reason": "attachment_byte_limit"})
|
|
245
|
+
continue
|
|
246
|
+
if used_bytes >= MAX_MESSAGE_BYTES or used_chars >= MAX_MESSAGE_CHARS:
|
|
247
|
+
result.append({**meta, "reason": "message_extraction_limit"})
|
|
248
|
+
continue
|
|
249
|
+
payload = part.get_payload(decode=True) or b""
|
|
250
|
+
meta["size"] = len(payload)
|
|
251
|
+
if len(payload) > MAX_ATTACHMENT_BYTES or used_bytes + len(payload) > MAX_MESSAGE_BYTES:
|
|
252
|
+
result.append({**meta, "reason": "attachment_or_message_byte_limit"})
|
|
253
|
+
continue
|
|
254
|
+
used_bytes += len(payload)
|
|
255
|
+
meta["sha256"] = hashlib.sha256(payload).hexdigest()
|
|
256
|
+
extracted = extract_text(payload, filename, meta["content_type"], charset=part.get_content_charset())
|
|
257
|
+
separator = 1 if used_chars and extracted["text"] else 0
|
|
258
|
+
remaining = max(0, MAX_MESSAGE_CHARS - used_chars - separator)
|
|
259
|
+
if len(extracted["text"]) > remaining:
|
|
260
|
+
extracted.update(text=extracted["text"][:remaining], truncated=True, reason="message_text_limit")
|
|
261
|
+
used_chars += len(extracted["text"]) + (separator if extracted["text"] else 0)
|
|
262
|
+
extracted["text_chars"] = len(extracted["text"])
|
|
263
|
+
result.append({**meta, **extracted})
|
|
264
|
+
return result
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def summarise_message(msg) -> dict:
|
|
268
|
+
entries = extract_message(msg)
|
|
269
|
+
return {
|
|
270
|
+
"attachment_text": "\n".join(entry["text"] for entry in entries if entry["text"]),
|
|
271
|
+
"attachment_names": "\n".join(entry["filename"] for entry in entries if entry["filename"]),
|
|
272
|
+
"attachments_json": json.dumps([{k: v for k, v in entry.items() if k != "text"} for entry in entries], ensure_ascii=False),
|
|
273
|
+
"attachments_indexed": 1,
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _worker() -> None:
|
|
278
|
+
if os.name == "posix":
|
|
279
|
+
import resource
|
|
280
|
+
# Darwin's shared-library virtual mappings can already exceed this
|
|
281
|
+
# Linux-oriented cap before parsing begins. Keep portable CPU/wall and
|
|
282
|
+
# document limits there; use the address-space cap where it is reliable.
|
|
283
|
+
if sys.platform.startswith("linux"):
|
|
284
|
+
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024,) * 2)
|
|
285
|
+
resource.setrlimit(resource.RLIMIT_CPU, (8, 8))
|
|
286
|
+
resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0))
|
|
287
|
+
try:
|
|
288
|
+
payload = sys.stdin.buffer.read(MAX_ATTACHMENT_BYTES + 1)
|
|
289
|
+
if len(payload) > MAX_ATTACHMENT_BYTES:
|
|
290
|
+
raise ExtractionLimit("PDF too large")
|
|
291
|
+
result = {"status": "extracted", **_pdf_document(payload)}
|
|
292
|
+
except EncryptedDocument:
|
|
293
|
+
result = {"status": "encrypted"}
|
|
294
|
+
except (ExtractionLimit, MemoryError):
|
|
295
|
+
result = {"status": "too_large"}
|
|
296
|
+
except Exception:
|
|
297
|
+
result = {"status": "error"}
|
|
298
|
+
sys.stdout.write(json.dumps(result))
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
if __name__ == "__main__" and sys.argv[1:] == ["--pdf-worker"]:
|
|
302
|
+
_worker()
|