multixtract 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- multixtract/__init__.py +52 -0
- multixtract/chunking.py +297 -0
- multixtract/cli.py +75 -0
- multixtract/extraction.py +44 -0
- multixtract/extractors/__init__.py +84 -0
- multixtract/extractors/_image_utils.py +157 -0
- multixtract/extractors/docx.py +375 -0
- multixtract/extractors/eml.py +187 -0
- multixtract/extractors/epub.py +142 -0
- multixtract/extractors/excel.py +321 -0
- multixtract/extractors/html.py +127 -0
- multixtract/extractors/image.py +149 -0
- multixtract/extractors/legacy.py +148 -0
- multixtract/extractors/markdown.py +104 -0
- multixtract/extractors/pdf.py +347 -0
- multixtract/extractors/pptx.py +303 -0
- multixtract/extractors/registry.py +72 -0
- multixtract/extractors/rtf.py +54 -0
- multixtract/extractors/text.py +50 -0
- multixtract/filters.py +181 -0
- multixtract/interfaces.py +151 -0
- multixtract/pipeline.py +250 -0
- multixtract/providers/__init__.py +38 -0
- multixtract/providers/_utils.py +32 -0
- multixtract/providers/azure.py +91 -0
- multixtract/providers/llama.py +151 -0
- multixtract/providers/openai.py +125 -0
- multixtract/providers/qwen2vl.py +158 -0
- multixtract/providers/smolvlm.py +152 -0
- multixtract/providers/storage.py +97 -0
- multixtract/py.typed +1 -0
- multixtract/vision.py +120 -0
- multixtract-0.1.1.dist-info/METADATA +419 -0
- multixtract-0.1.1.dist-info/RECORD +37 -0
- multixtract-0.1.1.dist-info/WHEEL +4 -0
- multixtract-0.1.1.dist-info/entry_points.txt +2 -0
- multixtract-0.1.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
"""PDF extractor (PyMuPDF + pdfplumber).
|
|
2
|
+
|
|
3
|
+
Extracts text and tables with pdfplumber and images with PyMuPDF, applying
|
|
4
|
+
cross-page deduplication via xref tracking. This is the reference
|
|
5
|
+
implementation of the :class:`~multixtract.interfaces.DocumentExtractor`
|
|
6
|
+
protocol; behaviour is identical to the original ``extract_document``.
|
|
7
|
+
|
|
8
|
+
Requires the ``[pdf]`` extra (PyMuPDF + pdfplumber). These are imported lazily
|
|
9
|
+
inside :meth:`PdfExtractor.extract` so importing ``multixtract`` (and the
|
|
10
|
+
registry) never pulls in the heavy PDF stack until a PDF is actually parsed.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import io
|
|
15
|
+
import logging
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
from typing import Any, Dict, List, Optional, Set, Tuple
|
|
19
|
+
|
|
20
|
+
from ..filters import ImageFilterPipeline
|
|
21
|
+
from ._image_utils import (
|
|
22
|
+
VECTOR_EXTS,
|
|
23
|
+
WDP_EXTS,
|
|
24
|
+
batch_convert_vectors_to_png,
|
|
25
|
+
decode_wdp_to_png,
|
|
26
|
+
ensure_rgb_png,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
log = logging.getLogger("multixtract.extractors.pdf")
|
|
30
|
+
|
|
31
|
+
# PDF date tokens: "D:20231015120000+05'30'" or "D:20231015120000Z" etc.
|
|
32
|
+
_PDF_DATE_RE = re.compile(
|
|
33
|
+
r"D:(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})"
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _parse_pdf_date(value: Any) -> Optional[str]:
|
|
38
|
+
"""Convert a raw PDF date string to ISO-8601 (best-effort). Returns None on failure."""
|
|
39
|
+
if not isinstance(value, str):
|
|
40
|
+
return None
|
|
41
|
+
date_match = _PDF_DATE_RE.search(value)
|
|
42
|
+
if not date_match:
|
|
43
|
+
return None
|
|
44
|
+
year, month, day, hour, minute, second = date_match.groups()
|
|
45
|
+
return f"{year}-{month}-{day}T{hour}:{minute}:{second}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _normalize_pdf_metadata(
|
|
49
|
+
raw: Dict[str, Any],
|
|
50
|
+
page_count: int,
|
|
51
|
+
table_count: int,
|
|
52
|
+
) -> Dict[str, Any]:
|
|
53
|
+
"""Normalize raw pdfplumber metadata to the shared schema.
|
|
54
|
+
|
|
55
|
+
Maps common PDF info-dict keys (case-insensitively, with or without leading
|
|
56
|
+
slash) to canonical names, parses PDF date strings, and appends computed
|
|
57
|
+
aggregates so the shape matches the DOCX metadata dict.
|
|
58
|
+
"""
|
|
59
|
+
def _get(*keys: str) -> Any:
|
|
60
|
+
for k in keys:
|
|
61
|
+
for candidate in (k, k.lstrip("/"), f"/{k}"):
|
|
62
|
+
if candidate in raw:
|
|
63
|
+
return raw[candidate]
|
|
64
|
+
if candidate.lower() in {rk.lower(): rv for rk, rv in raw.items()}:
|
|
65
|
+
for rk, rv in raw.items():
|
|
66
|
+
if rk.lower() == candidate.lower():
|
|
67
|
+
return rv
|
|
68
|
+
return None
|
|
69
|
+
|
|
70
|
+
return {
|
|
71
|
+
"author": _get("Author", "/Author") or None,
|
|
72
|
+
"creator": _get("Creator", "/Creator") or None,
|
|
73
|
+
"producer": _get("Producer", "/Producer") or None,
|
|
74
|
+
"title": _get("Title", "/Title") or None,
|
|
75
|
+
"subject": _get("Subject", "/Subject") or None,
|
|
76
|
+
"keywords": _get("Keywords", "/Keywords") or None,
|
|
77
|
+
"created": _parse_pdf_date(_get("CreationDate", "/CreationDate")),
|
|
78
|
+
"modified": _parse_pdf_date(_get("ModDate", "/ModDate")),
|
|
79
|
+
"page_count": page_count,
|
|
80
|
+
"table_count": table_count,
|
|
81
|
+
"raw": raw,
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _is_blank_table(rows: List[List[str]]) -> bool:
|
|
86
|
+
"""Return True if every cell in the table is empty/whitespace/None."""
|
|
87
|
+
return all(not (cell or "").strip() for row in rows for cell in row)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _extract_page_elements(page: Any) -> List[Dict[str, Any]]:
|
|
91
|
+
"""Return page content as an ordered elements list.
|
|
92
|
+
|
|
93
|
+
Uses pdfplumber ``find_tables()`` to locate table regions with bboxes, then
|
|
94
|
+
crops the page into strips to extract prose text. Uses column-aware cropping:
|
|
95
|
+
for each table, text to the LEFT and RIGHT of the table at the same y-band
|
|
96
|
+
is also captured as separate strips. This handles multi-column layouts where
|
|
97
|
+
prose and tables sit side-by-side at the same vertical position.
|
|
98
|
+
|
|
99
|
+
The result is a list of dicts in vertical (reading) order::
|
|
100
|
+
|
|
101
|
+
{"type": "text", "content": "<stripped text>"}
|
|
102
|
+
{"type": "table", "rows": [["header", ...], ["row", ...], ...]}
|
|
103
|
+
|
|
104
|
+
Tables are never duplicated into the text stream; each character on the page
|
|
105
|
+
appears in exactly one element.
|
|
106
|
+
"""
|
|
107
|
+
# pdfplumber top-left origin: bbox = (x0, top, x1, bottom), top < bottom.
|
|
108
|
+
tables_sorted = sorted(page.find_tables(), key=lambda t: t.bbox[1])
|
|
109
|
+
|
|
110
|
+
elements: List[Dict[str, Any]] = []
|
|
111
|
+
prev_bottom = 0.0
|
|
112
|
+
|
|
113
|
+
for table in tables_sorted:
|
|
114
|
+
x0, t_top, x1, t_bottom = table.bbox
|
|
115
|
+
|
|
116
|
+
# 1. Full-width strip ABOVE this table (skip negligible gaps < 2 pt)
|
|
117
|
+
if t_top > prev_bottom + 2:
|
|
118
|
+
strip = page.crop((0, prev_bottom, page.width, t_top))
|
|
119
|
+
text = strip.extract_text()
|
|
120
|
+
if text and text.strip():
|
|
121
|
+
elements.append({"type": "text", "content": text.strip()})
|
|
122
|
+
|
|
123
|
+
# 2. LEFT strip beside table — prose in the left column at same y-band
|
|
124
|
+
if x0 > 10:
|
|
125
|
+
strip = page.crop((0, t_top, x0, t_bottom))
|
|
126
|
+
text = strip.extract_text()
|
|
127
|
+
if text and text.strip():
|
|
128
|
+
elements.append({"type": "text", "content": text.strip()})
|
|
129
|
+
|
|
130
|
+
# 3. Table — clean None cells to empty string; skip all-blank tables
|
|
131
|
+
# (chart legend boxes and axis tick tables produce blank table regions)
|
|
132
|
+
rows = table.extract() or []
|
|
133
|
+
clean_rows = [
|
|
134
|
+
[c if c is not None else "" for c in row]
|
|
135
|
+
for row in rows
|
|
136
|
+
]
|
|
137
|
+
if clean_rows and not _is_blank_table(clean_rows):
|
|
138
|
+
elements.append({"type": "table", "rows": clean_rows})
|
|
139
|
+
|
|
140
|
+
# 4. RIGHT strip beside table — prose in the right column at same y-band
|
|
141
|
+
if x1 < page.width - 10:
|
|
142
|
+
strip = page.crop((x1, t_top, page.width, t_bottom))
|
|
143
|
+
text = strip.extract_text()
|
|
144
|
+
if text and text.strip():
|
|
145
|
+
elements.append({"type": "text", "content": text.strip()})
|
|
146
|
+
|
|
147
|
+
prev_bottom = max(prev_bottom, t_bottom)
|
|
148
|
+
|
|
149
|
+
# Text after last table (or full page when there are no tables)
|
|
150
|
+
if prev_bottom < page.height - 2:
|
|
151
|
+
strip = page.crop((0, prev_bottom, page.width, page.height))
|
|
152
|
+
text = strip.extract_text()
|
|
153
|
+
if text and text.strip():
|
|
154
|
+
elements.append({"type": "text", "content": text.strip()})
|
|
155
|
+
|
|
156
|
+
return elements
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class PdfExtractor:
|
|
160
|
+
"""DocumentExtractor for ``.pdf`` files."""
|
|
161
|
+
|
|
162
|
+
extensions: Tuple[str, ...] = (".pdf",)
|
|
163
|
+
|
|
164
|
+
def __init__(self, vector_timeout: int = 120) -> None:
|
|
165
|
+
self.vector_timeout = vector_timeout
|
|
166
|
+
|
|
167
|
+
def extract(
|
|
168
|
+
self,
|
|
169
|
+
path: str,
|
|
170
|
+
image_filter: Optional[ImageFilterPipeline] = None,
|
|
171
|
+
) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
|
|
172
|
+
try:
|
|
173
|
+
import fitz # PyMuPDF
|
|
174
|
+
import pdfplumber
|
|
175
|
+
except ImportError as e:
|
|
176
|
+
raise ImportError(
|
|
177
|
+
"PDF support requires PyMuPDF + pdfplumber: pip install 'multixtract[pdf]'"
|
|
178
|
+
) from e
|
|
179
|
+
from PIL import Image
|
|
180
|
+
|
|
181
|
+
if image_filter is None:
|
|
182
|
+
image_filter = ImageFilterPipeline()
|
|
183
|
+
image_filter.reset()
|
|
184
|
+
|
|
185
|
+
base_name = os.path.splitext(os.path.basename(path))[0]
|
|
186
|
+
document: Dict[str, Any] = {"metadata": {}, "pgs": []}
|
|
187
|
+
prepared_images: List[Dict[str, Any]] = []
|
|
188
|
+
seen_xrefs: Set[int] = set()
|
|
189
|
+
doc_fitz = None
|
|
190
|
+
|
|
191
|
+
try:
|
|
192
|
+
with pdfplumber.open(path) as pdf: # raises FileNotFoundError/PDFSyntaxError on bad path
|
|
193
|
+
_raw_meta = dict(pdf.metadata or {})
|
|
194
|
+
doc_fitz = fitz.open(path)
|
|
195
|
+
|
|
196
|
+
# Pass 1: build page dicts and collect xrefs in first-appearance order.
|
|
197
|
+
# xref_order stores (xref, page_idx, img_idx_on_page) for each unique xref.
|
|
198
|
+
xref_order: List[Tuple[int, int, int]] = []
|
|
199
|
+
|
|
200
|
+
for page_idx, page in enumerate(pdf.pages):
|
|
201
|
+
# Guard: PyMuPDF and pdfplumber use independent parsers and
|
|
202
|
+
# can disagree on page count for corrupted/non-linear PDFs.
|
|
203
|
+
fitz_page = doc_fitz[page_idx] if page_idx < len(doc_fitz) else None
|
|
204
|
+
|
|
205
|
+
hyperlinks: List[str] = []
|
|
206
|
+
if fitz_page is not None:
|
|
207
|
+
for _link in fitz_page.get_links():
|
|
208
|
+
_uri = _link.get("uri", "")
|
|
209
|
+
if _uri and _uri.startswith(("http://", "https://", "ftp://")):
|
|
210
|
+
hyperlinks.append(_uri)
|
|
211
|
+
hyperlinks = list(dict.fromkeys(hyperlinks))
|
|
212
|
+
|
|
213
|
+
# Use _extract_page_elements for bbox-clean extraction (no
|
|
214
|
+
# text/table overlap, Y-position ordered), then flatten to
|
|
215
|
+
# the shared {txt, tables} schema used by all four extractors.
|
|
216
|
+
_elems = _extract_page_elements(page)
|
|
217
|
+
|
|
218
|
+
document["pgs"].append({
|
|
219
|
+
"pg_num": page_idx + 1,
|
|
220
|
+
"kind": "page",
|
|
221
|
+
"title": "",
|
|
222
|
+
"elements": _elems,
|
|
223
|
+
"txt": "\n\n".join(
|
|
224
|
+
e["content"] for e in _elems if e["type"] == "text"
|
|
225
|
+
),
|
|
226
|
+
"tables": [
|
|
227
|
+
e["rows"] for e in _elems if e["type"] == "table"
|
|
228
|
+
],
|
|
229
|
+
"imgs": [],
|
|
230
|
+
"hyperlinks": hyperlinks,
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
if fitz_page is None:
|
|
234
|
+
log.debug(
|
|
235
|
+
"pg %d of %s has no fitz page (parser mismatch); skipping images",
|
|
236
|
+
page_idx + 1, base_name,
|
|
237
|
+
)
|
|
238
|
+
continue
|
|
239
|
+
|
|
240
|
+
img_counter = 0
|
|
241
|
+
for img in fitz_page.get_images(full=True):
|
|
242
|
+
xref = img[0]
|
|
243
|
+
if xref in seen_xrefs:
|
|
244
|
+
image_filter.note_duplicate()
|
|
245
|
+
continue
|
|
246
|
+
seen_xrefs.add(xref)
|
|
247
|
+
xref_order.append((xref, page_idx, img_counter))
|
|
248
|
+
img_counter += 1
|
|
249
|
+
|
|
250
|
+
# Minimum-page guarantee: ensure downstream always has at least one page.
|
|
251
|
+
if not document["pgs"]:
|
|
252
|
+
document["pgs"].append({
|
|
253
|
+
"pg_num": 1, "kind": "page", "title": "", "elements": [],
|
|
254
|
+
"txt": "", "tables": [], "hyperlinks": [], "imgs": [],
|
|
255
|
+
})
|
|
256
|
+
|
|
257
|
+
# Normalize metadata now that aggregates are known.
|
|
258
|
+
_table_count = sum(len(pg["tables"]) for pg in document["pgs"])
|
|
259
|
+
document["metadata"] = _normalize_pdf_metadata(
|
|
260
|
+
_raw_meta, len(document["pgs"]), _table_count
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
# Pass 2: extract raw bytes and categorize into vector / WDP / raster.
|
|
264
|
+
vector_items: List[Tuple[str, bytes]] = []
|
|
265
|
+
wdp_items: List[Tuple[str, bytes]] = []
|
|
266
|
+
raster_cache: Dict[int, Any] = {} # xref -> fitz base_image dict
|
|
267
|
+
xref_fake_path: Dict[int, str] = {} # xref -> key used in `converted`
|
|
268
|
+
|
|
269
|
+
for xref, page_idx, _img_idx in xref_order:
|
|
270
|
+
try:
|
|
271
|
+
base_image = doc_fitz.extract_image(xref)
|
|
272
|
+
except Exception as exc:
|
|
273
|
+
log.debug(
|
|
274
|
+
"xref %d image extraction failed on pg %d of %s: %s",
|
|
275
|
+
xref, page_idx + 1, base_name, exc,
|
|
276
|
+
)
|
|
277
|
+
continue
|
|
278
|
+
ext = f".{base_image['ext'].lower()}"
|
|
279
|
+
# Use a synthetic path as the dict key so batch_convert_vectors_to_png
|
|
280
|
+
# can write a temp file with a sensible name and extension.
|
|
281
|
+
fake_path = f"xref_{xref}{ext}"
|
|
282
|
+
xref_fake_path[xref] = fake_path
|
|
283
|
+
if ext in VECTOR_EXTS:
|
|
284
|
+
vector_items.append((fake_path, base_image["image"]))
|
|
285
|
+
elif ext in WDP_EXTS:
|
|
286
|
+
wdp_items.append((fake_path, base_image["image"]))
|
|
287
|
+
else:
|
|
288
|
+
raster_cache[xref] = base_image
|
|
289
|
+
|
|
290
|
+
# Pass 3: batch-convert all vector (EMF/WMF/SVG) and WDP images to PNG.
|
|
291
|
+
converted = batch_convert_vectors_to_png(vector_items, self.vector_timeout)
|
|
292
|
+
converted.update(decode_wdp_to_png(wdp_items))
|
|
293
|
+
|
|
294
|
+
# Pass 4: run prepare_image for every image that survived.
|
|
295
|
+
for xref, page_idx, img_idx in xref_order:
|
|
296
|
+
fake_path = xref_fake_path.get(xref)
|
|
297
|
+
if fake_path is None:
|
|
298
|
+
continue # extraction failed in pass 2
|
|
299
|
+
|
|
300
|
+
if fake_path in converted:
|
|
301
|
+
image_bytes = converted[fake_path]
|
|
302
|
+
ext_out = "png"
|
|
303
|
+
elif xref in raster_cache:
|
|
304
|
+
base_image = raster_cache[xref]
|
|
305
|
+
image_bytes = base_image["image"]
|
|
306
|
+
ext_out = base_image["ext"]
|
|
307
|
+
if ext_out == "png" and not image_bytes[:4].startswith(b"\x89PNG"):
|
|
308
|
+
fixed = ensure_rgb_png(image_bytes)
|
|
309
|
+
if fixed is None:
|
|
310
|
+
continue
|
|
311
|
+
image_bytes = fixed
|
|
312
|
+
else:
|
|
313
|
+
continue # vector/WDP conversion failed; skip
|
|
314
|
+
|
|
315
|
+
try:
|
|
316
|
+
with Image.open(io.BytesIO(image_bytes)) as image:
|
|
317
|
+
width, height = image.size
|
|
318
|
+
except Exception as exc:
|
|
319
|
+
log.debug(
|
|
320
|
+
"image decode failed for xref %d on pg %d of %s: %s",
|
|
321
|
+
xref, page_idx + 1, base_name, exc,
|
|
322
|
+
)
|
|
323
|
+
continue
|
|
324
|
+
|
|
325
|
+
prepared = image_filter.prepare_image(
|
|
326
|
+
image_bytes=image_bytes,
|
|
327
|
+
ext=ext_out,
|
|
328
|
+
width=width,
|
|
329
|
+
height=height,
|
|
330
|
+
image_id=f"page_{page_idx + 1}_img_{img_idx}",
|
|
331
|
+
page_number=page_idx + 1,
|
|
332
|
+
img_idx=img_idx,
|
|
333
|
+
)
|
|
334
|
+
if prepared is not None:
|
|
335
|
+
prepared_images.append(prepared)
|
|
336
|
+
except ImportError:
|
|
337
|
+
raise
|
|
338
|
+
except Exception:
|
|
339
|
+
log.warning("PdfExtractor failed for %s", path, exc_info=True)
|
|
340
|
+
document["_base_name"] = base_name
|
|
341
|
+
return document, []
|
|
342
|
+
finally:
|
|
343
|
+
if doc_fitz is not None:
|
|
344
|
+
doc_fitz.close()
|
|
345
|
+
|
|
346
|
+
document["_base_name"] = base_name
|
|
347
|
+
return document, prepared_images
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
"""Native PowerPoint (.pptx) extractor.
|
|
2
|
+
|
|
3
|
+
Ported from the ZF PPT + GPT-4o Vision pipeline. Extracts, with python-pptx:
|
|
4
|
+
* presentation metadata (slide count + dimensions)
|
|
5
|
+
* per-slide title, text (with GroupShape recursion + SmartArt), tables,
|
|
6
|
+
and hyperlinks (single pass over slide shapes)
|
|
7
|
+
* images from ``ppt/media/`` mapped to their slide via slide relationships
|
|
8
|
+
|
|
9
|
+
Vector images (EMF/WMF/SVG, plus EMF-in-.bin) are converted to PNG in one
|
|
10
|
+
headless LibreOffice batch call; JPEG XR (.wdp) via imagecodecs when available.
|
|
11
|
+
Images are filtered through the shared :class:`ImageFilterPipeline` and returned
|
|
12
|
+
as ``prepared_images`` for downstream vision analysis.
|
|
13
|
+
|
|
14
|
+
Requires the ``[pptx]`` extra (python-pptx). EMF/WMF/SVG need system LibreOffice;
|
|
15
|
+
.wdp needs optional ``imagecodecs``. Both degrade gracefully.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import io
|
|
20
|
+
import logging
|
|
21
|
+
import os
|
|
22
|
+
import xml.etree.ElementTree as ET
|
|
23
|
+
import zipfile
|
|
24
|
+
from collections import defaultdict
|
|
25
|
+
from typing import Any, Dict, List, Optional, Set, Tuple
|
|
26
|
+
|
|
27
|
+
from ..filters import ImageFilterPipeline
|
|
28
|
+
from ._image_utils import (
|
|
29
|
+
IMAGE_EXTS,
|
|
30
|
+
VECTOR_EXTS,
|
|
31
|
+
WDP_EXTS,
|
|
32
|
+
batch_convert_vectors_to_png,
|
|
33
|
+
decode_wdp_to_png,
|
|
34
|
+
ensure_rgb_png,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
log = logging.getLogger("multixtract.extractors.pptx")
|
|
38
|
+
|
|
39
|
+
_A_NS = "http://schemas.openxmlformats.org/drawingml/2006/main"
|
|
40
|
+
_PKG_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _iter_all_shapes(shapes, mso):
|
|
44
|
+
"""Recursively yield shapes, descending into GroupShapes."""
|
|
45
|
+
for shape in shapes:
|
|
46
|
+
if shape.shape_type == mso.GROUP:
|
|
47
|
+
yield from _iter_all_shapes(shape.shapes, mso)
|
|
48
|
+
else:
|
|
49
|
+
yield shape
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _extract_smartart_text(shape) -> Optional[str]:
|
|
53
|
+
"""Extract SmartArt text from drawingml <a:t> nodes."""
|
|
54
|
+
try:
|
|
55
|
+
root = ET.fromstring(shape.element.xml)
|
|
56
|
+
texts = [(t.text or "").strip() for t in root.iter(f"{{{_A_NS}}}t")]
|
|
57
|
+
texts = [t for t in texts if t]
|
|
58
|
+
if texts:
|
|
59
|
+
return " | ".join(texts)
|
|
60
|
+
except Exception:
|
|
61
|
+
pass
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _extract_slide_content(slide, mso) -> Tuple[str, str, List[List[List[str]]], List[str]]:
|
|
66
|
+
"""Single-pass slide extraction: (text, title, tables, hyperlinks)."""
|
|
67
|
+
texts: List[str] = []
|
|
68
|
+
title = ""
|
|
69
|
+
tables: List[List[List[str]]] = []
|
|
70
|
+
hyperlinks: List[str] = []
|
|
71
|
+
|
|
72
|
+
for shape in _iter_all_shapes(slide.shapes, mso):
|
|
73
|
+
shape_type = shape.shape_type
|
|
74
|
+
if shape_type == mso.EMBEDDED_OLE_OBJECT:
|
|
75
|
+
continue
|
|
76
|
+
|
|
77
|
+
if shape.has_text_frame:
|
|
78
|
+
parts = []
|
|
79
|
+
for para in shape.text_frame.paragraphs:
|
|
80
|
+
pt = para.text.strip()
|
|
81
|
+
if pt:
|
|
82
|
+
parts.append(pt)
|
|
83
|
+
for run in para.runs:
|
|
84
|
+
try:
|
|
85
|
+
if run.hyperlink and run.hyperlink.address:
|
|
86
|
+
hyperlinks.append(run.hyperlink.address)
|
|
87
|
+
except Exception:
|
|
88
|
+
pass
|
|
89
|
+
shape_text = "\n".join(parts)
|
|
90
|
+
if not shape_text:
|
|
91
|
+
continue
|
|
92
|
+
try:
|
|
93
|
+
if shape.is_placeholder and shape.placeholder_format.idx == 0:
|
|
94
|
+
title = shape_text
|
|
95
|
+
except Exception:
|
|
96
|
+
pass
|
|
97
|
+
texts.append(shape_text)
|
|
98
|
+
|
|
99
|
+
elif shape.has_table:
|
|
100
|
+
table_data = [[cell.text.strip() for cell in row.cells] for row in shape.table.rows]
|
|
101
|
+
if table_data:
|
|
102
|
+
tables.append(table_data)
|
|
103
|
+
|
|
104
|
+
elif shape_type == mso.PICTURE:
|
|
105
|
+
pass # images handled separately
|
|
106
|
+
else:
|
|
107
|
+
smartart_text = _extract_smartart_text(shape)
|
|
108
|
+
if smartart_text:
|
|
109
|
+
texts.append(f"[SmartArt] {smartart_text}")
|
|
110
|
+
|
|
111
|
+
return "\n".join(texts), title, tables, hyperlinks
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _build_slide_media_map(zf: zipfile.ZipFile, n_slides: int) -> Dict[int, List[str]]:
|
|
115
|
+
"""Map slide number -> [media paths] from each slide's relationships."""
|
|
116
|
+
slide_media: Dict[int, List[str]] = defaultdict(list)
|
|
117
|
+
namelist = set(zf.namelist())
|
|
118
|
+
for slide_num in range(1, n_slides + 1):
|
|
119
|
+
rels_path = f"ppt/slides/_rels/slide{slide_num}.xml.rels"
|
|
120
|
+
if rels_path not in namelist:
|
|
121
|
+
continue
|
|
122
|
+
try:
|
|
123
|
+
root = ET.fromstring(zf.read(rels_path).decode("utf-8"))
|
|
124
|
+
for rel in root.findall(f"{{{_PKG_REL_NS}}}Relationship"):
|
|
125
|
+
target = rel.get("Target", "")
|
|
126
|
+
if "../media/" in target:
|
|
127
|
+
slide_media[slide_num].append(f"ppt/media/{target.replace('../media/', '')}")
|
|
128
|
+
except Exception as exc:
|
|
129
|
+
log.debug("rels parse failed for slide %d: %s", slide_num, exc)
|
|
130
|
+
return slide_media
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _looks_like_emf_bin(raw: bytes) -> bool:
|
|
134
|
+
"""Detect EMF metafiles stored with a .bin extension."""
|
|
135
|
+
return len(raw) >= 44 and raw[:4] == b"\x01\x00\x00\x00" and raw[40:44] == b" EMF"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class PptxExtractor:
|
|
139
|
+
"""DocumentExtractor for ``.pptx`` (native python-pptx extraction)."""
|
|
140
|
+
|
|
141
|
+
extensions: Tuple[str, ...] = (".pptx",)
|
|
142
|
+
|
|
143
|
+
def __init__(self, vector_timeout: int = 120) -> None:
|
|
144
|
+
self.vector_timeout = vector_timeout
|
|
145
|
+
|
|
146
|
+
def extract(
|
|
147
|
+
self,
|
|
148
|
+
path: str,
|
|
149
|
+
image_filter: Optional[ImageFilterPipeline] = None,
|
|
150
|
+
) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
|
|
151
|
+
try:
|
|
152
|
+
from pptx import Presentation
|
|
153
|
+
from pptx.enum.shapes import MSO_SHAPE_TYPE as mso
|
|
154
|
+
except ImportError as e:
|
|
155
|
+
raise ImportError(
|
|
156
|
+
"PowerPoint support requires python-pptx: pip install 'multixtract[pptx]'"
|
|
157
|
+
) from e
|
|
158
|
+
from PIL import Image
|
|
159
|
+
|
|
160
|
+
if image_filter is None:
|
|
161
|
+
image_filter = ImageFilterPipeline()
|
|
162
|
+
image_filter.reset()
|
|
163
|
+
|
|
164
|
+
base_name = os.path.splitext(os.path.basename(path))[0]
|
|
165
|
+
empty: Dict[str, Any] = {"_base_name": base_name, "metadata": {}, "pgs": []}
|
|
166
|
+
try:
|
|
167
|
+
prs = Presentation(path)
|
|
168
|
+
n_slides = len(prs.slides)
|
|
169
|
+
|
|
170
|
+
document: Dict[str, Any] = {
|
|
171
|
+
"metadata": {
|
|
172
|
+
"slide_count": n_slides,
|
|
173
|
+
"slide_width": prs.slide_width,
|
|
174
|
+
"slide_height": prs.slide_height,
|
|
175
|
+
},
|
|
176
|
+
"_base_name": base_name,
|
|
177
|
+
"pgs": [],
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
prepared_images: List[Dict[str, Any]] = []
|
|
181
|
+
try:
|
|
182
|
+
zf = zipfile.ZipFile(path, "r")
|
|
183
|
+
except Exception as exc:
|
|
184
|
+
log.warning("could not open %s as ZIP for image extraction: %s", base_name, exc)
|
|
185
|
+
# Still return slide text even if the archive can't be opened.
|
|
186
|
+
for idx, slide in enumerate(prs.slides):
|
|
187
|
+
txt, title, tables, links = _extract_slide_content(slide, mso)
|
|
188
|
+
document["pgs"].append({
|
|
189
|
+
"pg_num": idx + 1, "kind": "slide", "title": title,
|
|
190
|
+
"txt": txt, "tables": tables,
|
|
191
|
+
"hyperlinks": list(dict.fromkeys(links)), "imgs": [],
|
|
192
|
+
})
|
|
193
|
+
return document, prepared_images
|
|
194
|
+
|
|
195
|
+
try:
|
|
196
|
+
slide_media_map = _build_slide_media_map(zf, n_slides)
|
|
197
|
+
seen_media: Set[str] = set()
|
|
198
|
+
|
|
199
|
+
# Pre-scan: collect vector / WDP images that need conversion.
|
|
200
|
+
vector_items, wdp_items = [], []
|
|
201
|
+
for slide_num in range(1, n_slides + 1):
|
|
202
|
+
for media_path in slide_media_map.get(slide_num, []):
|
|
203
|
+
if media_path in seen_media:
|
|
204
|
+
continue
|
|
205
|
+
ext = os.path.splitext(media_path)[1].lower()
|
|
206
|
+
try:
|
|
207
|
+
raw = zf.read(media_path)
|
|
208
|
+
except KeyError:
|
|
209
|
+
continue
|
|
210
|
+
if ext in VECTOR_EXTS:
|
|
211
|
+
vector_items.append((media_path, raw))
|
|
212
|
+
seen_media.add(media_path)
|
|
213
|
+
elif ext in WDP_EXTS:
|
|
214
|
+
wdp_items.append((media_path, raw))
|
|
215
|
+
seen_media.add(media_path)
|
|
216
|
+
elif ext == ".bin" and _looks_like_emf_bin(raw):
|
|
217
|
+
vector_items.append((media_path, raw))
|
|
218
|
+
seen_media.add(media_path)
|
|
219
|
+
converted = batch_convert_vectors_to_png(vector_items, self.vector_timeout)
|
|
220
|
+
converted.update(decode_wdp_to_png(wdp_items))
|
|
221
|
+
|
|
222
|
+
# Track media paths already sent to prepare_image (deduplicates
|
|
223
|
+
# converted vectors that appear on multiple slides — Bug 8).
|
|
224
|
+
processed_media: Set[str] = set()
|
|
225
|
+
|
|
226
|
+
# Process slides: content + per-slide images.
|
|
227
|
+
for slide_idx, slide in enumerate(prs.slides):
|
|
228
|
+
slide_num = slide_idx + 1
|
|
229
|
+
txt, title, tables, links = _extract_slide_content(slide, mso)
|
|
230
|
+
document["pgs"].append({
|
|
231
|
+
"pg_num": slide_num,
|
|
232
|
+
"kind": "slide",
|
|
233
|
+
"title": title,
|
|
234
|
+
"txt": txt,
|
|
235
|
+
"tables": tables,
|
|
236
|
+
"hyperlinks": list(dict.fromkeys(links)),
|
|
237
|
+
"imgs": [],
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
img_idx = 0
|
|
241
|
+
for media_path in slide_media_map.get(slide_num, []):
|
|
242
|
+
ext = os.path.splitext(media_path)[1].lower()
|
|
243
|
+
if ext not in IMAGE_EXTS:
|
|
244
|
+
continue
|
|
245
|
+
|
|
246
|
+
# Deduplicate across all slides (covers both converted vectors
|
|
247
|
+
# and rasters that appear in multiple slide relationships).
|
|
248
|
+
if media_path in processed_media:
|
|
249
|
+
image_filter.note_duplicate()
|
|
250
|
+
continue
|
|
251
|
+
|
|
252
|
+
if media_path in converted:
|
|
253
|
+
image_bytes, ext_out = converted[media_path], "png"
|
|
254
|
+
elif ext in VECTOR_EXTS or ext in WDP_EXTS or ext == ".bin":
|
|
255
|
+
continue # conversion failed or non-image .bin
|
|
256
|
+
else:
|
|
257
|
+
if media_path in seen_media:
|
|
258
|
+
image_filter.note_duplicate()
|
|
259
|
+
continue # raster already handled on an earlier slide
|
|
260
|
+
try:
|
|
261
|
+
image_bytes = zf.read(media_path)
|
|
262
|
+
except KeyError:
|
|
263
|
+
continue
|
|
264
|
+
seen_media.add(media_path)
|
|
265
|
+
ext_out = ext.lstrip(".")
|
|
266
|
+
ext_out = {"tif": "tiff", "jpg": "jpeg"}.get(ext_out, ext_out)
|
|
267
|
+
if ext_out == "png" and not image_bytes[:4].startswith(b"\x89PNG"):
|
|
268
|
+
fixed = ensure_rgb_png(image_bytes)
|
|
269
|
+
if fixed is None:
|
|
270
|
+
continue
|
|
271
|
+
image_bytes = fixed
|
|
272
|
+
|
|
273
|
+
processed_media.add(media_path)
|
|
274
|
+
|
|
275
|
+
try:
|
|
276
|
+
with Image.open(io.BytesIO(image_bytes)) as image:
|
|
277
|
+
width, height = image.size
|
|
278
|
+
except Exception as exc:
|
|
279
|
+
log.debug("image decode failed for %s on slide %d of %s: %s",
|
|
280
|
+
media_path, slide_num, base_name, exc)
|
|
281
|
+
continue
|
|
282
|
+
|
|
283
|
+
prepared = image_filter.prepare_image(
|
|
284
|
+
image_bytes=image_bytes,
|
|
285
|
+
ext=ext_out,
|
|
286
|
+
width=width,
|
|
287
|
+
height=height,
|
|
288
|
+
image_id=f"page_{slide_num}_img_{img_idx}",
|
|
289
|
+
page_number=slide_num,
|
|
290
|
+
img_idx=img_idx,
|
|
291
|
+
)
|
|
292
|
+
if prepared is not None:
|
|
293
|
+
prepared_images.append(prepared)
|
|
294
|
+
img_idx += 1 # only advance for images that pass filtering
|
|
295
|
+
finally:
|
|
296
|
+
zf.close()
|
|
297
|
+
|
|
298
|
+
return document, prepared_images
|
|
299
|
+
except ImportError:
|
|
300
|
+
raise
|
|
301
|
+
except Exception:
|
|
302
|
+
log.warning("PptxExtractor failed for %s", path, exc_info=True)
|
|
303
|
+
return empty, []
|