super-document-processor 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- super_document_processor/__init__.py +28 -0
- super_document_processor/document_processors/process_docx.py +824 -0
- super_document_processor/document_processors/process_images.py +39 -0
- super_document_processor/document_processors/process_msg.py +62 -0
- super_document_processor/document_processors/process_pdf.py +715 -0
- super_document_processor/document_processors/process_plain_text.py +25 -0
- super_document_processor/document_processors/process_pptx.py +367 -0
- super_document_processor/main.py +826 -0
- super_document_processor/processor.py +135 -0
- super_document_processor/utils/aspose_slide_renderer.py +103 -0
- super_document_processor/utils/chunker.py +127 -0
- super_document_processor/utils/docx_image_utils.py +167 -0
- super_document_processor/utils/docx_xml_parser.py +73 -0
- super_document_processor/utils/optimizer.py +49 -0
- super_document_processor/utils/pdf_image_utils.py +90 -0
- super_document_processor/utils/pdf_table_utils.py +627 -0
- super_document_processor/utils/pptx_components_extract.py +2197 -0
- super_document_processor/utils/pptx_converter.py +170 -0
- super_document_processor/utils/vlm.py +176 -0
- super_document_processor-0.1.3.dist-info/METADATA +376 -0
- super_document_processor-0.1.3.dist-info/RECORD +25 -0
- super_document_processor-0.1.3.dist-info/WHEEL +5 -0
- super_document_processor-0.1.3.dist-info/entry_points.txt +2 -0
- super_document_processor-0.1.3.dist-info/licenses/LICENSE +21 -0
- super_document_processor-0.1.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2197 @@
|
|
|
1
|
+
import zipfile
|
|
2
|
+
import re
|
|
3
|
+
import io
|
|
4
|
+
import struct
|
|
5
|
+
import datetime
|
|
6
|
+
import xml.etree.ElementTree as ET
|
|
7
|
+
from collections import defaultdict, Counter
|
|
8
|
+
from pptx.enum.shapes import MSO_SHAPE_TYPE
|
|
9
|
+
from pptx.enum.dml import MSO_FILL, MSO_COLOR_TYPE
|
|
10
|
+
|
|
11
|
+
PUA_GLYPH_MAP = {
|
|
12
|
+
'\uf0b7': '•',
|
|
13
|
+
'\uf06f': 'o',
|
|
14
|
+
'\uf0a7': '▪',
|
|
15
|
+
'\uf0a8': '▫',
|
|
16
|
+
'\uf0d8': '➢',
|
|
17
|
+
'\uf0fc': '✔',
|
|
18
|
+
'\uf075': '◆',
|
|
19
|
+
'\u25cf': '•',
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
def sanitize_table_cell(text):
|
|
23
|
+
if not text:
|
|
24
|
+
return ""
|
|
25
|
+
text_str = str(text)
|
|
26
|
+
clean = text_str.replace('\r\n', '\n').replace('\r', '\n').replace('\x0b', '\n').replace('\v', '\n').replace('\x0c', '\n').replace('\u2028', '\n').replace('\u2029', '\n')
|
|
27
|
+
lines = [l.strip() for l in clean.split('\n') if l.strip()]
|
|
28
|
+
return "<br>".join(lines).replace('|', '¦').strip()
|
|
29
|
+
|
|
30
|
+
def format_table_markdown(table):
|
|
31
|
+
rows = []
|
|
32
|
+
for i, row in enumerate(table.rows):
|
|
33
|
+
cells = [sanitize_table_cell(cell.text) for cell in row.cells]
|
|
34
|
+
row_line = "| " + " | ".join(cells) + " |"
|
|
35
|
+
if i == 0:
|
|
36
|
+
separator = "| " + " | ".join("---" for _ in cells) + " |"
|
|
37
|
+
rows.append(row_line)
|
|
38
|
+
rows.append(separator)
|
|
39
|
+
else:
|
|
40
|
+
rows.append(row_line)
|
|
41
|
+
return "\n".join(rows)
|
|
42
|
+
|
|
43
|
+
def format_run_text(run):
|
|
44
|
+
txt = getattr(run, 'text', '') or ''
|
|
45
|
+
if not txt:
|
|
46
|
+
return ""
|
|
47
|
+
try:
|
|
48
|
+
hlink = getattr(run, 'hyperlink', None)
|
|
49
|
+
address = getattr(hlink, 'address', None) if hlink else None
|
|
50
|
+
if address and address.strip():
|
|
51
|
+
addr = address.strip()
|
|
52
|
+
stripped_txt = txt.strip()
|
|
53
|
+
if stripped_txt == addr or stripped_txt.startswith(('http://', 'https://', 'mailto:')):
|
|
54
|
+
return txt
|
|
55
|
+
return f"[{txt}]({addr})"
|
|
56
|
+
except Exception:
|
|
57
|
+
pass
|
|
58
|
+
return txt
|
|
59
|
+
|
|
60
|
+
def get_layout_placeholder(shape):
|
|
61
|
+
if not getattr(shape, 'is_placeholder', False):
|
|
62
|
+
return None
|
|
63
|
+
try:
|
|
64
|
+
slide = shape.part
|
|
65
|
+
slide_layout = getattr(slide, 'slide_layout', None)
|
|
66
|
+
if not slide_layout:
|
|
67
|
+
return None
|
|
68
|
+
ph = shape.placeholder_format
|
|
69
|
+
idx = ph.idx
|
|
70
|
+
for l_ph in slide_layout.placeholders:
|
|
71
|
+
if l_ph.placeholder_format.idx == idx:
|
|
72
|
+
return l_ph
|
|
73
|
+
for l_ph in slide_layout.placeholders:
|
|
74
|
+
if l_ph.placeholder_format.type == ph.type:
|
|
75
|
+
return l_ph
|
|
76
|
+
except Exception:
|
|
77
|
+
pass
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
def int_to_roman(n):
|
|
81
|
+
if not isinstance(n, int) or n <= 0:
|
|
82
|
+
return str(n)
|
|
83
|
+
val = [1000, 900, 500, 400, 100, 90, 50, 40, 10, 9, 5, 4, 1]
|
|
84
|
+
syb = ["M", "CM", "D", "CD", "C", "XC", "L", "XL", "X", "IX", "V", "IV", "I"]
|
|
85
|
+
roman_num = ""
|
|
86
|
+
i = 0
|
|
87
|
+
while n > 0 and i < len(val):
|
|
88
|
+
for _ in range(n // val[i]):
|
|
89
|
+
roman_num += syb[i]
|
|
90
|
+
n -= val[i]
|
|
91
|
+
i += 1
|
|
92
|
+
return roman_num
|
|
93
|
+
|
|
94
|
+
def int_to_alpha(n):
|
|
95
|
+
if not isinstance(n, int) or n <= 0:
|
|
96
|
+
return str(n)
|
|
97
|
+
result = ""
|
|
98
|
+
while n > 0:
|
|
99
|
+
n, remainder = divmod(n - 1, 26)
|
|
100
|
+
result = chr(65 + remainder) + result
|
|
101
|
+
return result
|
|
102
|
+
|
|
103
|
+
def format_autonum_prefix(num_type, index):
|
|
104
|
+
if not num_type:
|
|
105
|
+
return f"{index}."
|
|
106
|
+
|
|
107
|
+
# Roman Upper
|
|
108
|
+
if num_type in ('romanUcPeriod', 'romanUpperLetter'):
|
|
109
|
+
return f"{int_to_roman(index)}."
|
|
110
|
+
elif num_type == 'romanUcParenRight':
|
|
111
|
+
return f"{int_to_roman(index)})"
|
|
112
|
+
elif num_type == 'romanUcParenBoth':
|
|
113
|
+
return f"({int_to_roman(index)})"
|
|
114
|
+
|
|
115
|
+
# Roman Lower
|
|
116
|
+
elif num_type in ('romanLcPeriod', 'romanLowerLetter'):
|
|
117
|
+
return f"{int_to_roman(index).lower()}."
|
|
118
|
+
elif num_type == 'romanLcParenRight':
|
|
119
|
+
return f"{int_to_roman(index).lower()})"
|
|
120
|
+
elif num_type == 'romanLcParenBoth':
|
|
121
|
+
return f"({int_to_roman(index).lower()})"
|
|
122
|
+
|
|
123
|
+
# Alpha Upper
|
|
124
|
+
elif num_type in ('alphaUcPeriod', 'alphaUpperLetter'):
|
|
125
|
+
return f"{int_to_alpha(index)}."
|
|
126
|
+
elif num_type == 'alphaUcParenRight':
|
|
127
|
+
return f"{int_to_alpha(index)})"
|
|
128
|
+
elif num_type == 'alphaUcParenBoth':
|
|
129
|
+
return f"({int_to_alpha(index)})"
|
|
130
|
+
|
|
131
|
+
# Alpha Lower
|
|
132
|
+
elif num_type in ('alphaLcPeriod', 'alphaLowerLetter'):
|
|
133
|
+
return f"{int_to_alpha(index).lower()}."
|
|
134
|
+
elif num_type == 'alphaLcParenRight':
|
|
135
|
+
return f"{int_to_alpha(index).lower()})"
|
|
136
|
+
elif num_type == 'alphaLcParenBoth':
|
|
137
|
+
return f"({int_to_alpha(index).lower()})"
|
|
138
|
+
|
|
139
|
+
# Arabic / numbers
|
|
140
|
+
elif num_type in ('arabicPeriod', 'arabicPlain', 'arabic1Period'):
|
|
141
|
+
return f"{index}."
|
|
142
|
+
elif num_type == 'arabicParenRight':
|
|
143
|
+
return f"{index})"
|
|
144
|
+
elif num_type == 'arabicParenBoth':
|
|
145
|
+
return f"({index})"
|
|
146
|
+
|
|
147
|
+
# Fallback prefixes
|
|
148
|
+
elif num_type.startswith('romanUc'):
|
|
149
|
+
return f"{int_to_roman(index)}."
|
|
150
|
+
elif num_type.startswith('romanLc'):
|
|
151
|
+
return f"{int_to_roman(index).lower()}."
|
|
152
|
+
elif num_type.startswith('alphaUc'):
|
|
153
|
+
return f"{int_to_alpha(index)}."
|
|
154
|
+
elif num_type.startswith('alphaLc'):
|
|
155
|
+
return f"{int_to_alpha(index).lower()}."
|
|
156
|
+
else:
|
|
157
|
+
return f"{index}."
|
|
158
|
+
|
|
159
|
+
def resolve_paragraph_bullet_format(shape, p):
|
|
160
|
+
"""
|
|
161
|
+
Resolves bullet formatting for a paragraph using OpenXML presentation inheritance:
|
|
162
|
+
1. Paragraph pPr (<a:buNone/> or <a:buChar/> / <a:buAutoNum/> / <a:buFont/>)
|
|
163
|
+
2. Shape txBody <a:lstStyle>
|
|
164
|
+
3. Layout placeholder <a:lstStyle> and <a:pPr>
|
|
165
|
+
4. PresentationML default (body placeholders default to bullets; title/other default to none)
|
|
166
|
+
Returns tuple: (is_bullet: bool, bu_char: str or None, autonum_type: str or None, start_at: int or None)
|
|
167
|
+
"""
|
|
168
|
+
level = getattr(p, 'level', 0) or 0
|
|
169
|
+
lvl_tag = f'{{http://schemas.openxmlformats.org/drawingml/2006/main}}lvl{level+1}pPr'
|
|
170
|
+
bu_none_tag = '{http://schemas.openxmlformats.org/drawingml/2006/main}buNone'
|
|
171
|
+
bu_char_tag = '{http://schemas.openxmlformats.org/drawingml/2006/main}buChar'
|
|
172
|
+
bu_auto_tag = '{http://schemas.openxmlformats.org/drawingml/2006/main}buAutoNum'
|
|
173
|
+
bu_font_tag = '{http://schemas.openxmlformats.org/drawingml/2006/main}buFont'
|
|
174
|
+
lst_style_tag = '{http://schemas.openxmlformats.org/drawingml/2006/main}lstStyle'
|
|
175
|
+
p_tag = '{http://schemas.openxmlformats.org/drawingml/2006/main}p'
|
|
176
|
+
p_pr_tag = '{http://schemas.openxmlformats.org/drawingml/2006/main}pPr'
|
|
177
|
+
|
|
178
|
+
# 1. Paragraph-level pPr
|
|
179
|
+
try:
|
|
180
|
+
pPr = p._p.pPr
|
|
181
|
+
if pPr is not None:
|
|
182
|
+
if pPr.find(bu_none_tag) is not None:
|
|
183
|
+
return False, None, None, None
|
|
184
|
+
buChar_el = pPr.find(bu_char_tag)
|
|
185
|
+
if buChar_el is not None:
|
|
186
|
+
return True, buChar_el.attrib.get('char'), None, None
|
|
187
|
+
buAuto_el = pPr.find(bu_auto_tag)
|
|
188
|
+
if buAuto_el is not None:
|
|
189
|
+
num_type = buAuto_el.attrib.get('type')
|
|
190
|
+
start_at_raw = buAuto_el.attrib.get('startAt')
|
|
191
|
+
start_at = int(start_at_raw) if start_at_raw and start_at_raw.isdigit() else None
|
|
192
|
+
return True, None, num_type, start_at
|
|
193
|
+
if pPr.find(bu_font_tag) is not None:
|
|
194
|
+
return True, None, None, None
|
|
195
|
+
except Exception:
|
|
196
|
+
pass
|
|
197
|
+
|
|
198
|
+
# 2. Shape-level lstStyle
|
|
199
|
+
if shape is not None:
|
|
200
|
+
try:
|
|
201
|
+
txBody = shape._element.find('{http://schemas.openxmlformats.org/drawingml/2006/main}txBody') or getattr(shape._element, 'txBody', None)
|
|
202
|
+
if txBody is not None:
|
|
203
|
+
lstStyle = txBody.find(lst_style_tag)
|
|
204
|
+
if lstStyle is not None:
|
|
205
|
+
lvl_el = lstStyle.find(lvl_tag)
|
|
206
|
+
if lvl_el is not None:
|
|
207
|
+
if lvl_el.find(bu_none_tag) is not None:
|
|
208
|
+
return False, None, None, None
|
|
209
|
+
buChar_el = lvl_el.find(bu_char_tag)
|
|
210
|
+
if buChar_el is not None:
|
|
211
|
+
return True, buChar_el.attrib.get('char'), None, None
|
|
212
|
+
buAuto_el = lvl_el.find(bu_auto_tag)
|
|
213
|
+
if buAuto_el is not None:
|
|
214
|
+
num_type = buAuto_el.attrib.get('type')
|
|
215
|
+
start_at_raw = buAuto_el.attrib.get('startAt')
|
|
216
|
+
start_at = int(start_at_raw) if start_at_raw and start_at_raw.isdigit() else None
|
|
217
|
+
return True, None, num_type, start_at
|
|
218
|
+
if lvl_el.find(bu_font_tag) is not None:
|
|
219
|
+
return True, None, None, None
|
|
220
|
+
except Exception:
|
|
221
|
+
pass
|
|
222
|
+
|
|
223
|
+
# 3. Slide layout placeholder lstStyle and pPr
|
|
224
|
+
if shape is not None:
|
|
225
|
+
try:
|
|
226
|
+
l_ph = get_layout_placeholder(shape)
|
|
227
|
+
if l_ph is not None:
|
|
228
|
+
l_txBody = l_ph._element.find('{http://schemas.openxmlformats.org/drawingml/2006/main}txBody') or getattr(l_ph._element, 'txBody', None)
|
|
229
|
+
if l_txBody is not None:
|
|
230
|
+
lstStyle = l_txBody.find(lst_style_tag)
|
|
231
|
+
if lstStyle is not None:
|
|
232
|
+
lvl_el = lstStyle.find(lvl_tag)
|
|
233
|
+
if lvl_el is not None:
|
|
234
|
+
if lvl_el.find(bu_none_tag) is not None:
|
|
235
|
+
return False, None, None, None
|
|
236
|
+
buChar_el = lvl_el.find(bu_char_tag)
|
|
237
|
+
if buChar_el is not None:
|
|
238
|
+
return True, buChar_el.attrib.get('char'), None, None
|
|
239
|
+
buAuto_el = lvl_el.find(bu_auto_tag)
|
|
240
|
+
if buAuto_el is not None:
|
|
241
|
+
num_type = buAuto_el.attrib.get('type')
|
|
242
|
+
start_at_raw = buAuto_el.attrib.get('startAt')
|
|
243
|
+
start_at = int(start_at_raw) if start_at_raw and start_at_raw.isdigit() else None
|
|
244
|
+
return True, None, num_type, start_at
|
|
245
|
+
for lp in l_txBody.findall(p_tag):
|
|
246
|
+
lpPr = lp.find(p_pr_tag)
|
|
247
|
+
if lpPr is not None:
|
|
248
|
+
l_lvl = int(lpPr.attrib.get('lvl', '0'))
|
|
249
|
+
if l_lvl == level:
|
|
250
|
+
if lpPr.find(bu_none_tag) is not None:
|
|
251
|
+
return False, None, None, None
|
|
252
|
+
buChar_el = lpPr.find(bu_char_tag)
|
|
253
|
+
if buChar_el is not None:
|
|
254
|
+
return True, buChar_el.attrib.get('char'), None, None
|
|
255
|
+
buAuto_el = lpPr.find(bu_auto_tag)
|
|
256
|
+
if buAuto_el is not None:
|
|
257
|
+
num_type = buAuto_el.attrib.get('type')
|
|
258
|
+
start_at_raw = buAuto_el.attrib.get('startAt')
|
|
259
|
+
start_at = int(start_at_raw) if start_at_raw and start_at_raw.isdigit() else None
|
|
260
|
+
return True, None, num_type, start_at
|
|
261
|
+
except Exception:
|
|
262
|
+
pass
|
|
263
|
+
|
|
264
|
+
# 4. Body / content placeholders default to bullet points in PowerPoint
|
|
265
|
+
if shape is not None and getattr(shape, 'is_placeholder', False):
|
|
266
|
+
try:
|
|
267
|
+
ph_type = shape.placeholder_format.type
|
|
268
|
+
if ph_type in (2, 7, 12, 13) or (ph_type not in (1, 3, 4, 5, 6, 8) and ph_type is not None):
|
|
269
|
+
return True, None, None, None
|
|
270
|
+
except Exception:
|
|
271
|
+
pass
|
|
272
|
+
|
|
273
|
+
return (level > 0), None, None, None
|
|
274
|
+
|
|
275
|
+
def format_paragraph_text(p, shape=None, bullet_info=None):
|
|
276
|
+
p_text = "".join(format_run_text(run) for run in p.runs).strip()
|
|
277
|
+
if not p_text:
|
|
278
|
+
return ""
|
|
279
|
+
|
|
280
|
+
level = getattr(p, 'level', 0) or 0
|
|
281
|
+
indent = " " * level
|
|
282
|
+
|
|
283
|
+
ALL_BULLETS_PATTERN = (
|
|
284
|
+
r'^(?:([•*➢➤➔➜►▶→⇒»›✓✔★✦✧\u25cf\u25a0\u25a1\u25aa\u25ab\u25cb\u25e6\u25c6\u25c7\u25c8\u2756\u2219'
|
|
285
|
+
r'\uf0b7\u27a2\uf0d8\uf0fc\uf06f\uf0a7\uf0a8\uf075]|[-–—]\s+|o\s+|\d{1,3}[.)]\s+(?!\d|%|k\b|m\b|cr\b|bn\b|lakh\b|x\b|\$)|'
|
|
286
|
+
r'\([a-zA-Z0-9]{1,3}\)\s+|\[\d{1,3}\]\s+|'
|
|
287
|
+
r'(?!(?:e\.g\.|i\.e\.|etc\.|vs\.|w\.r\.t\.)\b)[a-z][.)]\s+|[A-Z][.)]\s+(?=[A-Z0-9])|(?:[ivxIVX]+)[.)]\s+)\s*)(.*)$'
|
|
288
|
+
)
|
|
289
|
+
m_bullet = re.match(ALL_BULLETS_PATTERN, p_text)
|
|
290
|
+
|
|
291
|
+
if m_bullet:
|
|
292
|
+
raw_glyph = m_bullet.group(1).strip()
|
|
293
|
+
rest = m_bullet.group(2).strip()
|
|
294
|
+
glyph = PUA_GLYPH_MAP.get(raw_glyph, raw_glyph)
|
|
295
|
+
if level == 0:
|
|
296
|
+
return f"{glyph} {rest}" if rest else glyph
|
|
297
|
+
elif level == 1:
|
|
298
|
+
return f"{indent}o {rest}" if glyph in ('•', '-', '*') else f"{indent}{glyph} {rest}"
|
|
299
|
+
else:
|
|
300
|
+
return f"{indent}▪ {rest}" if glyph in ('•', '-', '*') else f"{indent}{glyph} {rest}"
|
|
301
|
+
|
|
302
|
+
if bullet_info is not None:
|
|
303
|
+
is_bullet, bu_char, autonum_type, autonum_idx = bullet_info
|
|
304
|
+
else:
|
|
305
|
+
res = resolve_paragraph_bullet_format(shape, p)
|
|
306
|
+
if len(res) == 4:
|
|
307
|
+
is_bullet, bu_char, autonum_type, autonum_start = res
|
|
308
|
+
autonum_idx = autonum_start or 1
|
|
309
|
+
else:
|
|
310
|
+
is_bullet, bu_char = res
|
|
311
|
+
autonum_type, autonum_idx = None, None
|
|
312
|
+
|
|
313
|
+
if autonum_type is not None and autonum_idx is not None:
|
|
314
|
+
prefix = format_autonum_prefix(autonum_type, autonum_idx)
|
|
315
|
+
return f"{indent}{prefix} {p_text}"
|
|
316
|
+
|
|
317
|
+
if is_bullet:
|
|
318
|
+
if level == 0:
|
|
319
|
+
bullet_char = PUA_GLYPH_MAP.get(bu_char, '•') if bu_char else '•'
|
|
320
|
+
return f"{bullet_char} {p_text}"
|
|
321
|
+
elif level == 1:
|
|
322
|
+
return f"{indent}o {p_text}"
|
|
323
|
+
else:
|
|
324
|
+
return f"{indent}▪ {p_text}"
|
|
325
|
+
|
|
326
|
+
return f"{indent}{p_text}"
|
|
327
|
+
|
|
328
|
+
def is_slide_header_shape(shape, slide_height=6858000, slide_width=12192000):
|
|
329
|
+
"""
|
|
330
|
+
Determines if a PowerPoint shape represents the slide title / main header using pure structural properties:
|
|
331
|
+
1. Placeholder type is TITLE or CENTER_TITLE.
|
|
332
|
+
2. Positioned in the top 14% of the slide and starting in the left-hand portion (<= 35% width).
|
|
333
|
+
3. Contains 1-2 short lines with no bullet formatting, excluding corner annotations/units.
|
|
334
|
+
"""
|
|
335
|
+
try:
|
|
336
|
+
if getattr(shape, 'is_placeholder', False):
|
|
337
|
+
ph_format = getattr(shape, 'placeholder_format', None)
|
|
338
|
+
if ph_format is not None:
|
|
339
|
+
ph_type = getattr(ph_format, 'type', None)
|
|
340
|
+
if ph_type in (1, 3): # PP_PLACEHOLDER.TITLE or CENTER_TITLE
|
|
341
|
+
return True
|
|
342
|
+
except Exception:
|
|
343
|
+
pass
|
|
344
|
+
|
|
345
|
+
try:
|
|
346
|
+
top = getattr(shape, 'top', None)
|
|
347
|
+
left = getattr(shape, 'left', 0) or 0
|
|
348
|
+
if top is not None and top <= 0.14 * (slide_height or 6858000):
|
|
349
|
+
# Exclude right-aligned corner notes and right-side columns
|
|
350
|
+
if left > 0.35 * (slide_width or 12192000):
|
|
351
|
+
return False
|
|
352
|
+
if getattr(shape, 'has_text_frame', False):
|
|
353
|
+
tf = shape.text_frame
|
|
354
|
+
paras = [p for p in tf.paragraphs if p.text.strip()]
|
|
355
|
+
if 1 <= len(paras) <= 2:
|
|
356
|
+
p = paras[0]
|
|
357
|
+
p_text = p.text.strip()
|
|
358
|
+
is_bul, _, auto_type, _ = resolve_paragraph_bullet_format(shape, p)
|
|
359
|
+
if not is_bul and auto_type is None and not p_text.startswith(('•', '-', '*', '1.', '2.', '3.', '4.', '5.')):
|
|
360
|
+
if len(p_text) <= 140:
|
|
361
|
+
return True
|
|
362
|
+
except Exception:
|
|
363
|
+
pass
|
|
364
|
+
|
|
365
|
+
return False
|
|
366
|
+
|
|
367
|
+
def extract_text_from_shape(shape, is_header=False):
|
|
368
|
+
# Ignore invisible shapes (e.g. transparent background with white font color)
|
|
369
|
+
try:
|
|
370
|
+
if getattr(shape, 'has_text_frame', False):
|
|
371
|
+
has_no_fill = False
|
|
372
|
+
sp_fill = getattr(shape, 'fill', None)
|
|
373
|
+
if sp_fill is not None and getattr(sp_fill, 'type', None) == MSO_FILL.BACKGROUND:
|
|
374
|
+
has_no_fill = True
|
|
375
|
+
if hasattr(shape, '_element'):
|
|
376
|
+
spPr = shape._element.find('{http://schemas.openxmlformats.org/presentationml/2006/main}spPr')
|
|
377
|
+
if spPr is not None and spPr.find('{http://schemas.openxmlformats.org/drawingml/2006/main}noFill') is not None:
|
|
378
|
+
has_no_fill = True
|
|
379
|
+
if has_no_fill:
|
|
380
|
+
runs = [r for p in shape.text_frame.paragraphs for r in p.runs if r.text.strip()]
|
|
381
|
+
if runs and all(
|
|
382
|
+
getattr(r.font, 'color', None) is not None and
|
|
383
|
+
getattr(r.font.color, 'type', None) == MSO_COLOR_TYPE.RGB and
|
|
384
|
+
str(getattr(r.font.color, 'rgb', '')).upper() == 'FFFFFF'
|
|
385
|
+
for r in runs
|
|
386
|
+
):
|
|
387
|
+
return ""
|
|
388
|
+
except Exception:
|
|
389
|
+
pass
|
|
390
|
+
|
|
391
|
+
if getattr(shape, 'shape_type', None) == MSO_SHAPE_TYPE.GROUP:
|
|
392
|
+
sub_texts = []
|
|
393
|
+
for s in shape.shapes:
|
|
394
|
+
t = extract_text_from_shape(s)
|
|
395
|
+
if t:
|
|
396
|
+
sub_texts.append(t)
|
|
397
|
+
return "\n".join(sub_texts)
|
|
398
|
+
elif getattr(shape, 'has_text_frame', False):
|
|
399
|
+
lines = []
|
|
400
|
+
autonum_counters = {}
|
|
401
|
+
prefix_parent = {}
|
|
402
|
+
prev_level = 0
|
|
403
|
+
for p in shape.text_frame.paragraphs:
|
|
404
|
+
p_text_raw = "".join(format_run_text(run) for run in p.runs).strip()
|
|
405
|
+
if not p_text_raw:
|
|
406
|
+
continue
|
|
407
|
+
level = getattr(p, 'level', 0) or 0
|
|
408
|
+
|
|
409
|
+
res = resolve_paragraph_bullet_format(shape, p)
|
|
410
|
+
if len(res) == 4:
|
|
411
|
+
is_bullet, bu_char, autonum_type, start_at = res
|
|
412
|
+
else:
|
|
413
|
+
is_bullet, bu_char = res
|
|
414
|
+
autonum_type, start_at = None, None
|
|
415
|
+
|
|
416
|
+
if autonum_type is not None:
|
|
417
|
+
if level < prev_level:
|
|
418
|
+
for k in list(prefix_parent.keys()):
|
|
419
|
+
if k >= level:
|
|
420
|
+
del prefix_parent[k]
|
|
421
|
+
|
|
422
|
+
if start_at is not None:
|
|
423
|
+
autonum_counters[level] = start_at
|
|
424
|
+
else:
|
|
425
|
+
autonum_counters[level] = autonum_counters.get(level, 0) + 1
|
|
426
|
+
curr_val = autonum_counters[level]
|
|
427
|
+
|
|
428
|
+
is_arabic = (autonum_type in ('arabicPeriod', 'arabicPlain', 'arabic1Period', 'arabicParenRight', 'arabicParenBoth') or str(autonum_type).startswith('arabic'))
|
|
429
|
+
|
|
430
|
+
# When a child list starts with the same bullet number as its immediate parent (e.g. 1 under 1)
|
|
431
|
+
parent_val = autonum_counters.get(level - 1)
|
|
432
|
+
if is_arabic and level > 0 and level > prev_level and parent_val is not None:
|
|
433
|
+
if curr_val == parent_val:
|
|
434
|
+
prefix_parent[level] = parent_val
|
|
435
|
+
|
|
436
|
+
if level in prefix_parent:
|
|
437
|
+
curr_idx = f"{prefix_parent[level]}.{curr_val}"
|
|
438
|
+
else:
|
|
439
|
+
curr_idx = curr_val
|
|
440
|
+
prev_level = level
|
|
441
|
+
else:
|
|
442
|
+
curr_idx = None
|
|
443
|
+
|
|
444
|
+
formatted = format_paragraph_text(p, shape=shape, bullet_info=(is_bullet, bu_char, autonum_type, curr_idx))
|
|
445
|
+
if formatted:
|
|
446
|
+
lines.append(formatted)
|
|
447
|
+
if not lines:
|
|
448
|
+
return ""
|
|
449
|
+
if is_header:
|
|
450
|
+
header_lines = []
|
|
451
|
+
for i, line in enumerate(lines):
|
|
452
|
+
cleaned = line.strip()
|
|
453
|
+
if i == 0 and not cleaned.startswith('#'):
|
|
454
|
+
header_lines.append(f"## {cleaned}")
|
|
455
|
+
else:
|
|
456
|
+
header_lines.append(line)
|
|
457
|
+
return "\n".join(header_lines)
|
|
458
|
+
return "\n".join(lines)
|
|
459
|
+
return ""
|
|
460
|
+
|
|
461
|
+
def get_pptx_card_boxes(slide):
|
|
462
|
+
"""
|
|
463
|
+
Extracts visual container/card boxes from PPTX shapes.
|
|
464
|
+
Identifies shapes that act as visual cards (e.g. rounded rectangles, callouts, cards enclosing text).
|
|
465
|
+
"""
|
|
466
|
+
raw_boxes = []
|
|
467
|
+
slide_w = getattr(slide, 'width', 12192000) or 12192000
|
|
468
|
+
slide_h = getattr(slide, 'height', 6858000) or 6858000
|
|
469
|
+
|
|
470
|
+
for s in slide.shapes:
|
|
471
|
+
try:
|
|
472
|
+
top = getattr(s, 'top', None)
|
|
473
|
+
left = getattr(s, 'left', None)
|
|
474
|
+
width = getattr(s, 'width', 0) or 0
|
|
475
|
+
height = getattr(s, 'height', 0) or 0
|
|
476
|
+
if top is None or left is None or width < 800000 or height < 400000:
|
|
477
|
+
continue
|
|
478
|
+
|
|
479
|
+
if width >= slide_w * 0.92 and height >= slide_h * 0.92:
|
|
480
|
+
continue
|
|
481
|
+
|
|
482
|
+
is_container = False
|
|
483
|
+
if s.shape_type == MSO_SHAPE_TYPE.GROUP:
|
|
484
|
+
is_container = True
|
|
485
|
+
elif s.shape_type == MSO_SHAPE_TYPE.AUTO_SHAPE:
|
|
486
|
+
is_container = True
|
|
487
|
+
|
|
488
|
+
if is_container:
|
|
489
|
+
box = {
|
|
490
|
+
'left': int(left),
|
|
491
|
+
'top': int(top),
|
|
492
|
+
'right': int(left) + int(width),
|
|
493
|
+
'bottom': int(top) + int(height),
|
|
494
|
+
'width': int(width),
|
|
495
|
+
'height': int(height)
|
|
496
|
+
}
|
|
497
|
+
raw_boxes.append(box)
|
|
498
|
+
except Exception:
|
|
499
|
+
continue
|
|
500
|
+
|
|
501
|
+
unique_boxes = []
|
|
502
|
+
for b in raw_boxes:
|
|
503
|
+
if not any(
|
|
504
|
+
abs(ub['left'] - b['left']) < 50000 and
|
|
505
|
+
abs(ub['top'] - b['top']) < 50000 and
|
|
506
|
+
abs(ub['right'] - b['right']) < 50000 and
|
|
507
|
+
abs(ub['bottom'] - b['bottom']) < 50000
|
|
508
|
+
for ub in unique_boxes
|
|
509
|
+
):
|
|
510
|
+
unique_boxes.append(b)
|
|
511
|
+
|
|
512
|
+
clean_boxes = []
|
|
513
|
+
for b in unique_boxes:
|
|
514
|
+
children = [
|
|
515
|
+
other for other in unique_boxes
|
|
516
|
+
if other != b and
|
|
517
|
+
b['left'] <= other['left'] + 50000 and
|
|
518
|
+
b['top'] <= other['top'] + 50000 and
|
|
519
|
+
b['right'] >= other['right'] - 50000 and
|
|
520
|
+
b['bottom'] >= other['bottom'] - 50000 and
|
|
521
|
+
(other['width'] * other['height']) < 0.8 * (b['width'] * b['height'])
|
|
522
|
+
]
|
|
523
|
+
if len(children) >= 2:
|
|
524
|
+
continue
|
|
525
|
+
clean_boxes.append(b)
|
|
526
|
+
|
|
527
|
+
return clean_boxes
|
|
528
|
+
|
|
529
|
+
def apply_pptx_box_decorations(ordered_elements, card_boxes):
|
|
530
|
+
"""
|
|
531
|
+
Identifies elements enclosed inside PowerPoint card boxes / callouts and frames them with '---'
|
|
532
|
+
without creating consecutive duplicate separators.
|
|
533
|
+
"""
|
|
534
|
+
if not card_boxes:
|
|
535
|
+
clean = []
|
|
536
|
+
for e in ordered_elements:
|
|
537
|
+
t = e if isinstance(e, str) else e.get('text', '')
|
|
538
|
+
if t.strip() == '---':
|
|
539
|
+
if clean and (clean[-1] if isinstance(clean[-1], str) else clean[-1].get('text', '')).strip() == '---':
|
|
540
|
+
continue
|
|
541
|
+
clean.append(t if isinstance(e, dict) else e)
|
|
542
|
+
return clean
|
|
543
|
+
|
|
544
|
+
decorated = []
|
|
545
|
+
current_box_elements = []
|
|
546
|
+
active_box = None
|
|
547
|
+
|
|
548
|
+
for el in ordered_elements:
|
|
549
|
+
matching_box = None
|
|
550
|
+
if isinstance(el, dict) and el.get('type') != 'header' and 'left' in el and 'top' in el:
|
|
551
|
+
el_cx = el['left'] + el.get('width', 0) / 2.0
|
|
552
|
+
el_cy = el['top'] + el.get('height', 0) / 2.0
|
|
553
|
+
for box in card_boxes:
|
|
554
|
+
if (box['left'] - 50000 <= el_cx <= box['right'] + 50000 and
|
|
555
|
+
box['top'] - 50000 <= el_cy <= box['bottom'] + 50000):
|
|
556
|
+
matching_box = box
|
|
557
|
+
break
|
|
558
|
+
|
|
559
|
+
el_text = el['text'] if isinstance(el, dict) else str(el)
|
|
560
|
+
if not el_text.strip():
|
|
561
|
+
continue
|
|
562
|
+
|
|
563
|
+
if matching_box != active_box:
|
|
564
|
+
if active_box is not None and current_box_elements:
|
|
565
|
+
decorated.append('---')
|
|
566
|
+
decorated.extend(current_box_elements)
|
|
567
|
+
decorated.append('---')
|
|
568
|
+
current_box_elements = []
|
|
569
|
+
active_box = matching_box
|
|
570
|
+
|
|
571
|
+
if active_box is not None:
|
|
572
|
+
current_box_elements.append(el_text)
|
|
573
|
+
else:
|
|
574
|
+
decorated.append(el_text)
|
|
575
|
+
|
|
576
|
+
if active_box is not None and current_box_elements:
|
|
577
|
+
decorated.append('---')
|
|
578
|
+
decorated.extend(current_box_elements)
|
|
579
|
+
decorated.append('---')
|
|
580
|
+
|
|
581
|
+
clean_decorated = []
|
|
582
|
+
for text in decorated:
|
|
583
|
+
if text.strip() == '---':
|
|
584
|
+
if clean_decorated and clean_decorated[-1].strip() == '---':
|
|
585
|
+
continue
|
|
586
|
+
clean_decorated.append(text)
|
|
587
|
+
|
|
588
|
+
return clean_decorated
|
|
589
|
+
|
|
590
|
+
def get_all_shapes_flattened(shapes):
|
|
591
|
+
"""
|
|
592
|
+
Recursively unpacks Group shapes into their individual shapes so every text frame,
|
|
593
|
+
table, or shape is individually placed in reading order and extracted.
|
|
594
|
+
Computes absolute slide coordinates for children in group coordinate transforms.
|
|
595
|
+
"""
|
|
596
|
+
flat = []
|
|
597
|
+
for s in shapes:
|
|
598
|
+
if getattr(s, 'shape_type', None) == MSO_SHAPE_TYPE.GROUP or hasattr(s, 'shapes'):
|
|
599
|
+
try:
|
|
600
|
+
grp_spPr = s._element.xpath('./p:grpSpPr') if hasattr(s, '_element') else []
|
|
601
|
+
off_x, off_y = 0, 0
|
|
602
|
+
chOff_x, chOff_y = 0, 0
|
|
603
|
+
scale_x, scale_y = 1.0, 1.0
|
|
604
|
+
if grp_spPr:
|
|
605
|
+
xfrm = grp_spPr[0].xpath('./a:xfrm')
|
|
606
|
+
if xfrm:
|
|
607
|
+
off = xfrm[0].xpath('./a:off')
|
|
608
|
+
chOff = xfrm[0].xpath('./a:chOff')
|
|
609
|
+
ext = xfrm[0].xpath('./a:ext')
|
|
610
|
+
chExt = xfrm[0].xpath('./a:chExt')
|
|
611
|
+
if off:
|
|
612
|
+
off_x = int(off[0].get('x', 0))
|
|
613
|
+
off_y = int(off[0].get('y', 0))
|
|
614
|
+
if chOff:
|
|
615
|
+
chOff_x = int(chOff[0].get('x', 0))
|
|
616
|
+
chOff_y = int(chOff[0].get('y', 0))
|
|
617
|
+
if ext and chExt:
|
|
618
|
+
cx = int(ext[0].get('cx', 1))
|
|
619
|
+
cy = int(ext[0].get('cy', 1))
|
|
620
|
+
ch_cx = int(chExt[0].get('cx', 1))
|
|
621
|
+
ch_cy = int(chExt[0].get('cy', 1))
|
|
622
|
+
scale_x = cx / ch_cx if ch_cx else 1.0
|
|
623
|
+
scale_y = cy / ch_cy if ch_cy else 1.0
|
|
624
|
+
|
|
625
|
+
for cs in s.shapes:
|
|
626
|
+
if getattr(cs, 'shape_type', None) == MSO_SHAPE_TYPE.GROUP or hasattr(cs, 'shapes'):
|
|
627
|
+
flat.extend(get_all_shapes_flattened([cs]))
|
|
628
|
+
else:
|
|
629
|
+
try:
|
|
630
|
+
orig_l = cs.left
|
|
631
|
+
orig_t = cs.top
|
|
632
|
+
abs_l = off_x + int((orig_l - chOff_x) * scale_x)
|
|
633
|
+
abs_t = off_y + int((orig_t - chOff_y) * scale_y)
|
|
634
|
+
cs.left = abs_l
|
|
635
|
+
cs.top = abs_t
|
|
636
|
+
except Exception:
|
|
637
|
+
pass
|
|
638
|
+
flat.append(cs)
|
|
639
|
+
except Exception:
|
|
640
|
+
flat.append(s)
|
|
641
|
+
else:
|
|
642
|
+
flat.append(s)
|
|
643
|
+
return flat
|
|
644
|
+
|
|
645
|
+
def get_spatial_shape_reading_order(shapes, slide_h=6858000, slide_w=12192000):
|
|
646
|
+
"""
|
|
647
|
+
Sorts PowerPoint slide shapes into natural 2D reading order (Recursive XY-Cut):
|
|
648
|
+
1. Header/Title at the top (peeled first).
|
|
649
|
+
2. Spanning footer at the bottom.
|
|
650
|
+
3. Distinct vertical columns (left-to-right) with top-to-bottom shape order.
|
|
651
|
+
"""
|
|
652
|
+
all_shapes = get_all_shapes_flattened(shapes)
|
|
653
|
+
valid_shapes = []
|
|
654
|
+
unpositioned = []
|
|
655
|
+
|
|
656
|
+
for s in all_shapes:
|
|
657
|
+
try:
|
|
658
|
+
top = getattr(s, 'top', None)
|
|
659
|
+
left = getattr(s, 'left', None)
|
|
660
|
+
width = getattr(s, 'width', 0) or 0
|
|
661
|
+
height = getattr(s, 'height', 0) or 0
|
|
662
|
+
if top is not None and left is not None:
|
|
663
|
+
# Filter out purely decorative background shapes with no readable content
|
|
664
|
+
has_content = False
|
|
665
|
+
if getattr(s, 'has_table', False) or getattr(s, 'shape_type', None) == MSO_SHAPE_TYPE.TABLE:
|
|
666
|
+
has_content = True
|
|
667
|
+
elif getattr(s, 'has_text_frame', False):
|
|
668
|
+
if s.text_frame.text.strip():
|
|
669
|
+
has_content = True
|
|
670
|
+
elif getattr(s, 'shape_type', None) == MSO_SHAPE_TYPE.GROUP or hasattr(s, 'shapes'):
|
|
671
|
+
group_text = extract_text_from_shape(s)
|
|
672
|
+
if group_text.strip():
|
|
673
|
+
has_content = True
|
|
674
|
+
elif hasattr(s, '_element') and s._element is not None:
|
|
675
|
+
# Check for embedded media / blip (EMF/WMF vector drawings, charts, OLE tables)
|
|
676
|
+
if len(s._element.xpath('.//a:blip')) > 0:
|
|
677
|
+
has_content = True
|
|
678
|
+
|
|
679
|
+
if not has_content:
|
|
680
|
+
continue
|
|
681
|
+
|
|
682
|
+
is_hdr = is_slide_header_shape(s, slide_h, slide_w)
|
|
683
|
+
effective_w = width
|
|
684
|
+
if is_hdr and getattr(s, 'has_text_frame', False):
|
|
685
|
+
t_len = len(s.text_frame.text.strip())
|
|
686
|
+
if t_len < 40 and width > 0.5 * (slide_w or 12192000):
|
|
687
|
+
effective_w = min(width, int((slide_w or 12192000) * 0.4))
|
|
688
|
+
|
|
689
|
+
valid_shapes.append({
|
|
690
|
+
'shape': s,
|
|
691
|
+
'left': int(left),
|
|
692
|
+
'top': int(top),
|
|
693
|
+
'width': int(width),
|
|
694
|
+
'height': int(height),
|
|
695
|
+
'right': int(left) + int(width),
|
|
696
|
+
'bottom': int(top) + int(height),
|
|
697
|
+
'is_header': is_hdr
|
|
698
|
+
})
|
|
699
|
+
else:
|
|
700
|
+
unpositioned.append(s)
|
|
701
|
+
except Exception:
|
|
702
|
+
unpositioned.append(s)
|
|
703
|
+
|
|
704
|
+
if not valid_shapes:
|
|
705
|
+
return list(all_shapes)
|
|
706
|
+
|
|
707
|
+
def order_shape_items(items, min_gutter_x=100000, min_band_gap_y=120000):
|
|
708
|
+
if len(items) <= 1:
|
|
709
|
+
return items
|
|
710
|
+
|
|
711
|
+
min_x = min(item['left'] for item in items)
|
|
712
|
+
max_x = max(item['right'] for item in items)
|
|
713
|
+
total_w = max_x - min_x
|
|
714
|
+
|
|
715
|
+
sorted_by_y = sorted(items, key=lambda it: (it['top'], it['left']))
|
|
716
|
+
|
|
717
|
+
# 1a. Header at top (peeled first if slide title or strictly above everything)
|
|
718
|
+
slide_hdrs = [
|
|
719
|
+
it for it in items
|
|
720
|
+
if it.get('is_header')
|
|
721
|
+
and it['left'] <= 0.35 * (slide_w or 12192000)
|
|
722
|
+
and it['top'] <= 0.20 * (slide_h or 6858000)
|
|
723
|
+
and (
|
|
724
|
+
(getattr(it['shape'], 'is_placeholder', False) and getattr(it['shape'], 'placeholder_format', None) and getattr(it['shape'].placeholder_format, 'type', None) in (1, 3))
|
|
725
|
+
or len(items) == len(valid_shapes)
|
|
726
|
+
)
|
|
727
|
+
]
|
|
728
|
+
if slide_hdrs:
|
|
729
|
+
hdr = min(slide_hdrs, key=lambda it: (it['top'], it['left']))
|
|
730
|
+
rest = [it for it in items if it is not hdr]
|
|
731
|
+
return [hdr] + order_shape_items(rest, min_gutter_x, min_band_gap_y)
|
|
732
|
+
|
|
733
|
+
top_item = sorted_by_y[0]
|
|
734
|
+
if len(sorted_by_y) > 1:
|
|
735
|
+
other_min_top = min(it['top'] for it in sorted_by_y[1:])
|
|
736
|
+
if top_item.get('is_header') and (top_item['left'] <= 0.35 * (slide_w or 12192000)):
|
|
737
|
+
rest = [it for it in items if it is not top_item]
|
|
738
|
+
return [top_item] + order_shape_items(rest, min_gutter_x, min_band_gap_y)
|
|
739
|
+
elif top_item['bottom'] <= other_min_top:
|
|
740
|
+
rest = [it for it in items if it is not top_item]
|
|
741
|
+
return [top_item] + order_shape_items(rest, min_gutter_x, min_band_gap_y)
|
|
742
|
+
|
|
743
|
+
# 1b. Spanning Footer at bottom
|
|
744
|
+
bottom_item = sorted_by_y[-1]
|
|
745
|
+
if bottom_item['width'] >= 0.35 * total_w and len(sorted_by_y) > 1:
|
|
746
|
+
other_max_bottom = max(it['bottom'] for it in sorted_by_y[:-1])
|
|
747
|
+
y_gap = bottom_item['top'] - other_max_bottom
|
|
748
|
+
if y_gap >= min_band_gap_y * 0.5:
|
|
749
|
+
rest = [it for it in items if it is not bottom_item]
|
|
750
|
+
return order_shape_items(rest, min_gutter_x, min_band_gap_y) + [bottom_item]
|
|
751
|
+
|
|
752
|
+
# 2. Check for horizontal band split (rows) FIRST
|
|
753
|
+
y_cuts = []
|
|
754
|
+
cur_bottom = sorted_by_y[0]['bottom']
|
|
755
|
+
cur_group = [sorted_by_y[0]]
|
|
756
|
+
|
|
757
|
+
for it in sorted_by_y[1:]:
|
|
758
|
+
if it['top'] >= cur_bottom + min_band_gap_y:
|
|
759
|
+
y_cuts.append(cur_group)
|
|
760
|
+
cur_group = [it]
|
|
761
|
+
cur_bottom = it['bottom']
|
|
762
|
+
else:
|
|
763
|
+
cur_group.append(it)
|
|
764
|
+
cur_bottom = max(cur_bottom, it['bottom'])
|
|
765
|
+
if cur_group:
|
|
766
|
+
y_cuts.append(cur_group)
|
|
767
|
+
|
|
768
|
+
if len(y_cuts) > 1:
|
|
769
|
+
result = []
|
|
770
|
+
for group in y_cuts:
|
|
771
|
+
result.extend(order_shape_items(group, min_gutter_x, min_band_gap_y))
|
|
772
|
+
return result
|
|
773
|
+
|
|
774
|
+
# 3. Check for vertical column split (gutters)
|
|
775
|
+
sorted_by_x = sorted(items, key=lambda it: (it['left'], it['top']))
|
|
776
|
+
columns = []
|
|
777
|
+
cur_col = [sorted_by_x[0]]
|
|
778
|
+
cur_right = sorted_by_x[0]['right']
|
|
779
|
+
|
|
780
|
+
for it in sorted_by_x[1:]:
|
|
781
|
+
if it['left'] >= cur_right + min_gutter_x:
|
|
782
|
+
columns.append(cur_col)
|
|
783
|
+
cur_col = [it]
|
|
784
|
+
cur_right = it['right']
|
|
785
|
+
else:
|
|
786
|
+
cur_col.append(it)
|
|
787
|
+
cur_right = max(cur_right, it['right'])
|
|
788
|
+
if cur_col:
|
|
789
|
+
columns.append(cur_col)
|
|
790
|
+
|
|
791
|
+
if len(columns) > 1:
|
|
792
|
+
ordered = []
|
|
793
|
+
for col in columns:
|
|
794
|
+
ordered.extend(order_shape_items(col, min_gutter_x, min_band_gap_y))
|
|
795
|
+
return ordered
|
|
796
|
+
|
|
797
|
+
# 3b. Check for distinct left-anchor column clustering
|
|
798
|
+
# (Handles staircase/chevron diagrams where wide arrow banners overlap horizontally)
|
|
799
|
+
distinct_lefts = sorted(set(it['left'] for it in items))
|
|
800
|
+
if len(distinct_lefts) >= 2 and total_w > min_gutter_x * 4:
|
|
801
|
+
left_clusters = []
|
|
802
|
+
cur_group = [distinct_lefts[0]]
|
|
803
|
+
for l in distinct_lefts[1:]:
|
|
804
|
+
if l - cur_group[-1] <= min_gutter_x * 0.75:
|
|
805
|
+
cur_group.append(l)
|
|
806
|
+
else:
|
|
807
|
+
left_clusters.append((min(cur_group), max(cur_group)))
|
|
808
|
+
cur_group = [l]
|
|
809
|
+
if cur_group:
|
|
810
|
+
left_clusters.append((min(cur_group), max(cur_group)))
|
|
811
|
+
|
|
812
|
+
if len(left_clusters) >= 2:
|
|
813
|
+
clustered_cols = [[] for _ in left_clusters]
|
|
814
|
+
for it in items:
|
|
815
|
+
best_idx = 0
|
|
816
|
+
min_dist = float('inf')
|
|
817
|
+
for ci, (c_min, c_max) in enumerate(left_clusters):
|
|
818
|
+
dist = abs(it['left'] - (c_min + c_max) / 2.0)
|
|
819
|
+
if dist < min_dist:
|
|
820
|
+
min_dist = dist
|
|
821
|
+
best_idx = ci
|
|
822
|
+
clustered_cols[best_idx].append(it)
|
|
823
|
+
|
|
824
|
+
if all(len(c) > 0 for c in clustered_cols):
|
|
825
|
+
# Verify clusters are genuinely separate columns rather than nested/overlapping elements
|
|
826
|
+
col_spans = [(min(it['left'] for it in c), max(it['right'] for it in c)) for c in clustered_cols]
|
|
827
|
+
overlapping = any(
|
|
828
|
+
not (col_spans[i][1] <= col_spans[j][0] + min_gutter_x or col_spans[j][1] <= col_spans[i][0] + min_gutter_x)
|
|
829
|
+
for i in range(len(col_spans))
|
|
830
|
+
for j in range(i + 1, len(col_spans))
|
|
831
|
+
)
|
|
832
|
+
if not overlapping:
|
|
833
|
+
ordered = []
|
|
834
|
+
for col in clustered_cols:
|
|
835
|
+
ordered.extend(order_shape_items(col, min_gutter_x, min_band_gap_y))
|
|
836
|
+
return ordered
|
|
837
|
+
|
|
838
|
+
return sorted(items, key=lambda it: (it['top'], it['left']))
|
|
839
|
+
|
|
840
|
+
ordered_items = order_shape_items(valid_shapes)
|
|
841
|
+
return [it['shape'] for it in ordered_items] + unpositioned
|
|
842
|
+
|
|
843
|
+
def extract_smartart_texts_per_slide(filename, page_num):
|
|
844
|
+
try:
|
|
845
|
+
with zipfile.ZipFile(filename, 'r') as pptx:
|
|
846
|
+
slide_diagram_map = defaultdict(list)
|
|
847
|
+
|
|
848
|
+
for file in pptx.namelist():
|
|
849
|
+
if file.startswith('ppt/slides/_rels/') and file.endswith('.rels'):
|
|
850
|
+
with pptx.open(file) as rels_file:
|
|
851
|
+
rels_tree = ET.parse(rels_file)
|
|
852
|
+
rels_root = rels_tree.getroot()
|
|
853
|
+
for rel in rels_root.findall('{http://schemas.openxmlformats.org/package/2006/relationships}Relationship'):
|
|
854
|
+
target = rel.attrib.get('Target', '')
|
|
855
|
+
if 'diagrams/' in target:
|
|
856
|
+
slide_num = int(file.split('/')[-1].replace('slide', '').replace('.xml.rels', ''))
|
|
857
|
+
diagram_path = 'ppt/' + target.replace('../', '')
|
|
858
|
+
if diagram_path in pptx.namelist() and diagram_path not in slide_diagram_map[slide_num]:
|
|
859
|
+
slide_diagram_map[slide_num].append(diagram_path)
|
|
860
|
+
|
|
861
|
+
namespaces = {
|
|
862
|
+
'dgm': 'http://schemas.openxmlformats.org/drawingml/2006/diagram',
|
|
863
|
+
'a': 'http://schemas.openxmlformats.org/drawingml/2006/main'
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
slide_texts = {}
|
|
867
|
+
|
|
868
|
+
for slide_num, diagram_files in slide_diagram_map.items():
|
|
869
|
+
if page_num is not None and slide_num != page_num:
|
|
870
|
+
continue
|
|
871
|
+
|
|
872
|
+
texts_in_slide = []
|
|
873
|
+
data_files = [f for f in diagram_files if 'diagrams/data' in f]
|
|
874
|
+
other_files = [f for f in diagram_files if 'diagrams/data' not in f]
|
|
875
|
+
|
|
876
|
+
# 1. Attempt structured hierarchical extraction from SmartArt data model
|
|
877
|
+
for df in data_files:
|
|
878
|
+
if df not in pptx.namelist():
|
|
879
|
+
continue
|
|
880
|
+
try:
|
|
881
|
+
with pptx.open(df) as f:
|
|
882
|
+
root = ET.parse(f).getroot()
|
|
883
|
+
|
|
884
|
+
pts = {}
|
|
885
|
+
for pt in root.findall('.//dgm:pt', namespaces):
|
|
886
|
+
mid = pt.attrib.get('modelId')
|
|
887
|
+
p_texts = [''.join(t.text for t in p.findall('.//a:t', namespaces) if t.text).strip() for p in pt.findall('.//a:p', namespaces)]
|
|
888
|
+
txt = ' '.join(t for t in p_texts if t).strip()
|
|
889
|
+
if txt:
|
|
890
|
+
pts[mid] = txt
|
|
891
|
+
|
|
892
|
+
cxns = defaultdict(list)
|
|
893
|
+
for cxn in root.findall('.//dgm:cxn', namespaces):
|
|
894
|
+
c_type = cxn.attrib.get('type')
|
|
895
|
+
if c_type is None or c_type == 'None':
|
|
896
|
+
src = cxn.attrib.get('srcId')
|
|
897
|
+
dest = cxn.attrib.get('destId')
|
|
898
|
+
src_ord = int(cxn.attrib.get('srcOrd', 0))
|
|
899
|
+
cxns[src].append((src_ord, dest))
|
|
900
|
+
|
|
901
|
+
incoming = {d for dsts in cxns.values() for _, d in dsts}
|
|
902
|
+
roots = [mid for mid in cxns if mid not in incoming]
|
|
903
|
+
has_sub = any(cxns.get(ch) for r in roots for _, ch in cxns.get(r, []))
|
|
904
|
+
|
|
905
|
+
if has_sub:
|
|
906
|
+
for r in roots:
|
|
907
|
+
for _, top_id in sorted(cxns.get(r, [])):
|
|
908
|
+
top_text = pts.get(top_id, '')
|
|
909
|
+
children = [pts.get(ch_id, '') for _, ch_id in sorted(cxns.get(top_id, [])) if pts.get(ch_id, '')]
|
|
910
|
+
if top_text and children:
|
|
911
|
+
texts_in_slide.append(f"### {top_text}")
|
|
912
|
+
for ch in children:
|
|
913
|
+
texts_in_slide.append(f"• {ch}")
|
|
914
|
+
elif top_text:
|
|
915
|
+
texts_in_slide.append(f"• {top_text}")
|
|
916
|
+
else:
|
|
917
|
+
for r in roots:
|
|
918
|
+
for _, ch_id in sorted(cxns.get(r, [])):
|
|
919
|
+
txt = pts.get(ch_id, '')
|
|
920
|
+
if txt and txt not in texts_in_slide:
|
|
921
|
+
texts_in_slide.append(txt)
|
|
922
|
+
except Exception:
|
|
923
|
+
pass
|
|
924
|
+
|
|
925
|
+
# 2. Fallback to drawing files or legacy extraction if no structured text found
|
|
926
|
+
if not texts_in_slide:
|
|
927
|
+
for diagram_file in other_files or diagram_files:
|
|
928
|
+
if diagram_file in pptx.namelist():
|
|
929
|
+
with pptx.open(diagram_file) as f:
|
|
930
|
+
tree = ET.parse(f)
|
|
931
|
+
root = tree.getroot()
|
|
932
|
+
|
|
933
|
+
for paragraph in root.findall('.//a:p', namespaces):
|
|
934
|
+
text_parts = paragraph.findall('.//a:t', namespaces)
|
|
935
|
+
combined_text = ''.join(t.text for t in text_parts if t.text)
|
|
936
|
+
cleaned_text = combined_text.replace('\n', ' ').strip()
|
|
937
|
+
if cleaned_text and cleaned_text not in texts_in_slide:
|
|
938
|
+
texts_in_slide.append(cleaned_text)
|
|
939
|
+
|
|
940
|
+
for dgm_t in root.findall('.//dgm:t', namespaces):
|
|
941
|
+
if dgm_t.text and dgm_t.text.strip():
|
|
942
|
+
cleaned_text = dgm_t.text.replace('\n', ' ').strip()
|
|
943
|
+
if cleaned_text and cleaned_text not in texts_in_slide:
|
|
944
|
+
texts_in_slide.append(cleaned_text)
|
|
945
|
+
|
|
946
|
+
slide_texts[slide_num] = texts_in_slide
|
|
947
|
+
|
|
948
|
+
return slide_texts
|
|
949
|
+
except Exception:
|
|
950
|
+
return {}
|
|
951
|
+
|
|
952
|
+
def extract_ole_streams(raw_bytes):
|
|
953
|
+
"""
|
|
954
|
+
Parses a Microsoft Compound File Binary (CFBF / OLE2) and extracts all streams by name.
|
|
955
|
+
"""
|
|
956
|
+
import struct
|
|
957
|
+
if len(raw_bytes) < 512 or raw_bytes[:8] != b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1":
|
|
958
|
+
return {}
|
|
959
|
+
|
|
960
|
+
try:
|
|
961
|
+
sector_shift = struct.unpack_from("<H", raw_bytes, 30)[0]
|
|
962
|
+
sector_size = 1 << sector_shift
|
|
963
|
+
mini_sector_shift = struct.unpack_from("<H", raw_bytes, 32)[0]
|
|
964
|
+
mini_sector_size = 1 << mini_sector_shift
|
|
965
|
+
|
|
966
|
+
c_fat = struct.unpack_from("<I", raw_bytes, 44)[0]
|
|
967
|
+
first_dir_sect = struct.unpack_from("<I", raw_bytes, 48)[0]
|
|
968
|
+
first_mini_fat = struct.unpack_from("<I", raw_bytes, 60)[0]
|
|
969
|
+
c_mini_fat = struct.unpack_from("<I", raw_bytes, 64)[0]
|
|
970
|
+
|
|
971
|
+
difat = list(struct.unpack_from("<109I", raw_bytes, 76))
|
|
972
|
+
fat = []
|
|
973
|
+
for fat_sect in difat[:c_fat]:
|
|
974
|
+
if fat_sect in (0xFFFFFFFE, 0xFFFFFFFF):
|
|
975
|
+
break
|
|
976
|
+
offset = (fat_sect + 1) * sector_size
|
|
977
|
+
fat.extend(struct.unpack_from(f"<{sector_size // 4}I", raw_bytes, offset))
|
|
978
|
+
|
|
979
|
+
def get_chain(start_sect):
|
|
980
|
+
chain = []
|
|
981
|
+
curr = start_sect
|
|
982
|
+
while curr < len(fat) and curr < 0xFFFFFFFC:
|
|
983
|
+
chain.append(curr)
|
|
984
|
+
curr = fat[curr]
|
|
985
|
+
return chain
|
|
986
|
+
|
|
987
|
+
dir_chain = get_chain(first_dir_sect)
|
|
988
|
+
dir_bytes = b"".join(raw_bytes[(s + 1) * sector_size:(s + 2) * sector_size] for s in dir_chain)
|
|
989
|
+
|
|
990
|
+
entries = []
|
|
991
|
+
for i in range(0, len(dir_bytes), 128):
|
|
992
|
+
entry_raw = dir_bytes[i:i + 128]
|
|
993
|
+
if len(entry_raw) < 128:
|
|
994
|
+
break
|
|
995
|
+
name_len = struct.unpack_from("<H", entry_raw, 64)[0]
|
|
996
|
+
name = entry_raw[:name_len].decode("utf-16le", errors="ignore").rstrip("\x00")
|
|
997
|
+
entry_type = entry_raw[66]
|
|
998
|
+
start_sect = struct.unpack_from("<I", entry_raw, 116)[0]
|
|
999
|
+
stream_size = struct.unpack_from("<Q", entry_raw, 120)[0]
|
|
1000
|
+
if entry_type in (2, 5) and name:
|
|
1001
|
+
entries.append((name, entry_type, start_sect, stream_size))
|
|
1002
|
+
|
|
1003
|
+
root_entry = next((e for e in entries if e[1] == 5), None)
|
|
1004
|
+
mini_stream = b""
|
|
1005
|
+
if root_entry:
|
|
1006
|
+
mini_chain = get_chain(root_entry[2])
|
|
1007
|
+
mini_stream = b"".join(raw_bytes[(s + 1) * sector_size:(s + 2) * sector_size] for s in mini_chain)[:root_entry[3]]
|
|
1008
|
+
|
|
1009
|
+
mini_fat = []
|
|
1010
|
+
if first_mini_fat < 0xFFFFFFFC:
|
|
1011
|
+
mf_chain = get_chain(first_mini_fat)
|
|
1012
|
+
for s in mf_chain[:c_mini_fat]:
|
|
1013
|
+
offset = (s + 1) * sector_size
|
|
1014
|
+
mini_fat.extend(struct.unpack_from(f"<{sector_size // 4}I", raw_bytes, offset))
|
|
1015
|
+
|
|
1016
|
+
def get_mini_chain(start_sect):
|
|
1017
|
+
chain = []
|
|
1018
|
+
curr = start_sect
|
|
1019
|
+
while curr < len(mini_fat) and curr < 0xFFFFFFFC:
|
|
1020
|
+
chain.append(curr)
|
|
1021
|
+
curr = mini_fat[curr]
|
|
1022
|
+
return chain
|
|
1023
|
+
|
|
1024
|
+
streams = {}
|
|
1025
|
+
for name, entry_type, start_sect, size in entries:
|
|
1026
|
+
if entry_type == 2 and size > 0:
|
|
1027
|
+
if size < 4096 and mini_stream and mini_fat:
|
|
1028
|
+
chain = get_mini_chain(start_sect)
|
|
1029
|
+
stream_data = b"".join(mini_stream[s * mini_sector_size:(s + 1) * mini_sector_size] for s in chain)[:size]
|
|
1030
|
+
else:
|
|
1031
|
+
chain = get_chain(start_sect)
|
|
1032
|
+
stream_data = b"".join(raw_bytes[(s + 1) * sector_size:(s + 2) * sector_size] for s in chain)[:size]
|
|
1033
|
+
streams[name] = stream_data
|
|
1034
|
+
|
|
1035
|
+
return streams
|
|
1036
|
+
except Exception:
|
|
1037
|
+
return {}
|
|
1038
|
+
|
|
1039
|
+
def decode_biff_rk(rk_int):
|
|
1040
|
+
"""Decodes a BIFF8 RK number to float."""
|
|
1041
|
+
import struct
|
|
1042
|
+
if rk_int & 0x02:
|
|
1043
|
+
val = float(rk_int >> 2)
|
|
1044
|
+
else:
|
|
1045
|
+
val = struct.unpack("<d", struct.pack("<I", 0) + struct.pack("<I", rk_int & 0xFFFFFFFC))[0]
|
|
1046
|
+
if rk_int & 0x01:
|
|
1047
|
+
val /= 100.0
|
|
1048
|
+
return val
|
|
1049
|
+
|
|
1050
|
+
def parse_biff8_workbook(wb_bytes):
|
|
1051
|
+
"""
|
|
1052
|
+
Parses a BIFF8 (Excel 97-2003) Workbook stream to extract shared strings and cell values into a Markdown table.
|
|
1053
|
+
"""
|
|
1054
|
+
import struct
|
|
1055
|
+
pos = 0
|
|
1056
|
+
sst = []
|
|
1057
|
+
rows_dict = defaultdict(dict)
|
|
1058
|
+
max_col = 0
|
|
1059
|
+
|
|
1060
|
+
try:
|
|
1061
|
+
while pos + 4 <= len(wb_bytes):
|
|
1062
|
+
rec_type, rec_len = struct.unpack_from("<HH", wb_bytes, pos)
|
|
1063
|
+
pos += 4
|
|
1064
|
+
rec_data = wb_bytes[pos:pos + rec_len]
|
|
1065
|
+
pos += rec_len
|
|
1066
|
+
|
|
1067
|
+
# SST (Shared String Table) record: 0x00FC
|
|
1068
|
+
if rec_type == 0x00FC and len(rec_data) >= 8:
|
|
1069
|
+
c_strings = struct.unpack_from("<I", rec_data, 4)[0]
|
|
1070
|
+
str_pos = 8
|
|
1071
|
+
for _ in range(c_strings):
|
|
1072
|
+
if str_pos + 3 > len(rec_data):
|
|
1073
|
+
break
|
|
1074
|
+
c_chars = struct.unpack_from("<H", rec_data, str_pos)[0]
|
|
1075
|
+
flags = rec_data[str_pos + 2]
|
|
1076
|
+
is_unicode = bool(flags & 0x01)
|
|
1077
|
+
str_pos += 3
|
|
1078
|
+
if flags & 0x08:
|
|
1079
|
+
str_pos += 2
|
|
1080
|
+
if flags & 0x04:
|
|
1081
|
+
str_pos += 4
|
|
1082
|
+
char_byte_len = c_chars * 2 if is_unicode else c_chars
|
|
1083
|
+
if str_pos + char_byte_len > len(rec_data):
|
|
1084
|
+
break
|
|
1085
|
+
s_bytes = rec_data[str_pos:str_pos + char_byte_len]
|
|
1086
|
+
str_pos += char_byte_len
|
|
1087
|
+
encoding = "utf-16le" if is_unicode else "latin-1"
|
|
1088
|
+
sst.append(s_bytes.decode(encoding, errors="ignore"))
|
|
1089
|
+
|
|
1090
|
+
# LABELSST (0x00FD)
|
|
1091
|
+
elif rec_type == 0x00FD and len(rec_data) >= 10:
|
|
1092
|
+
row, col, _, sst_idx = struct.unpack_from("<HHHI", rec_data, 0)
|
|
1093
|
+
if sst_idx < len(sst):
|
|
1094
|
+
rows_dict[row][col] = sst[sst_idx]
|
|
1095
|
+
max_col = max(max_col, col)
|
|
1096
|
+
|
|
1097
|
+
# LABEL (0x0204)
|
|
1098
|
+
elif rec_type == 0x0204 and len(rec_data) >= 8:
|
|
1099
|
+
row, col, _, str_len = struct.unpack_from("<HHHH", rec_data, 0)
|
|
1100
|
+
str_bytes = rec_data[8:8 + str_len]
|
|
1101
|
+
rows_dict[row][col] = str_bytes.decode("latin-1", errors="ignore")
|
|
1102
|
+
max_col = max(max_col, col)
|
|
1103
|
+
|
|
1104
|
+
# NUMBER (0x0203)
|
|
1105
|
+
elif rec_type == 0x0203 and len(rec_data) >= 14:
|
|
1106
|
+
row, col, _ = struct.unpack_from("<HHH", rec_data, 0)
|
|
1107
|
+
num_val = struct.unpack_from("<d", rec_data, 6)[0]
|
|
1108
|
+
val_str = f"{num_val:.3f}".rstrip('0').rstrip('.') if not num_val.is_integer() else str(int(num_val))
|
|
1109
|
+
rows_dict[row][col] = val_str
|
|
1110
|
+
max_col = max(max_col, col)
|
|
1111
|
+
|
|
1112
|
+
# RK (0x027E)
|
|
1113
|
+
elif rec_type == 0x027E and len(rec_data) >= 10:
|
|
1114
|
+
row, col, _, rk_raw = struct.unpack_from("<HHHI", rec_data, 0)
|
|
1115
|
+
num_val = decode_biff_rk(rk_raw)
|
|
1116
|
+
val_str = f"{num_val:.3f}".rstrip('0').rstrip('.') if not num_val.is_integer() else str(int(num_val))
|
|
1117
|
+
rows_dict[row][col] = val_str
|
|
1118
|
+
max_col = max(max_col, col)
|
|
1119
|
+
|
|
1120
|
+
# MULRK (0x00BD)
|
|
1121
|
+
elif rec_type == 0x00BD and len(rec_data) >= 6:
|
|
1122
|
+
row, first_col = struct.unpack_from("<HH", rec_data, 0)
|
|
1123
|
+
last_col = struct.unpack_from("<H", rec_data, len(rec_data) - 2)[0]
|
|
1124
|
+
rk_pos = 4
|
|
1125
|
+
for col in range(first_col, last_col + 1):
|
|
1126
|
+
if rk_pos + 6 > len(rec_data):
|
|
1127
|
+
break
|
|
1128
|
+
_, rk_raw = struct.unpack_from("<HI", rec_data, rk_pos)
|
|
1129
|
+
rk_pos += 6
|
|
1130
|
+
num_val = decode_biff_rk(rk_raw)
|
|
1131
|
+
val_str = f"{num_val:.3f}".rstrip('0').rstrip('.') if not num_val.is_integer() else str(int(num_val))
|
|
1132
|
+
rows_dict[row][col] = val_str
|
|
1133
|
+
max_col = max(max_col, col)
|
|
1134
|
+
|
|
1135
|
+
if not rows_dict:
|
|
1136
|
+
return ""
|
|
1137
|
+
|
|
1138
|
+
sorted_rows = sorted(rows_dict.keys())
|
|
1139
|
+
num_cols = max_col + 1
|
|
1140
|
+
grid = []
|
|
1141
|
+
for r in sorted_rows:
|
|
1142
|
+
row_cells = [rows_dict[r].get(c, "") for c in range(num_cols)]
|
|
1143
|
+
if any(str(cell).strip() for cell in row_cells):
|
|
1144
|
+
grid.append(row_cells)
|
|
1145
|
+
|
|
1146
|
+
if not grid:
|
|
1147
|
+
return ""
|
|
1148
|
+
|
|
1149
|
+
header = "| " + " | ".join(sanitize_table_cell(str(c)) for c in grid[0]) + " |"
|
|
1150
|
+
sep = "| " + " | ".join("---" for _ in range(num_cols)) + " |"
|
|
1151
|
+
table_lines = [header, sep]
|
|
1152
|
+
for row in grid[1:]:
|
|
1153
|
+
table_lines.append("| " + " | ".join(sanitize_table_cell(str(c)) for c in row) + " |")
|
|
1154
|
+
|
|
1155
|
+
return "\n".join(table_lines)
|
|
1156
|
+
except Exception:
|
|
1157
|
+
return ""
|
|
1158
|
+
|
|
1159
|
+
def _col_ref_to_index(cell_ref):
|
|
1160
|
+
"""Converts an Excel column reference like 'A1' or 'BC12' to a 0-based column index."""
|
|
1161
|
+
col_str = ''.join(c for c in cell_ref if c.isalpha()).upper()
|
|
1162
|
+
idx = 0
|
|
1163
|
+
for c in col_str:
|
|
1164
|
+
idx = idx * 26 + (ord(c) - ord('A') + 1)
|
|
1165
|
+
return max(0, idx - 1)
|
|
1166
|
+
|
|
1167
|
+
def _parse_xlsx_stream(zip_stream):
|
|
1168
|
+
"""Parses an open XML zip stream into Markdown tables."""
|
|
1169
|
+
tables = []
|
|
1170
|
+
with zipfile.ZipFile(zip_stream, 'r') as xlsx:
|
|
1171
|
+
shared_strings = []
|
|
1172
|
+
if 'xl/sharedStrings.xml' in xlsx.namelist():
|
|
1173
|
+
sst_tree = ET.parse(xlsx.open('xl/sharedStrings.xml'))
|
|
1174
|
+
ns_ss = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
|
|
1175
|
+
for si in sst_tree.getroot().findall('.//s:si', ns_ss):
|
|
1176
|
+
t_parts = [t.text for t in si.findall('.//s:t', ns_ss) if t.text]
|
|
1177
|
+
shared_strings.append("".join(t_parts))
|
|
1178
|
+
|
|
1179
|
+
sheet_files = sorted([f for f in xlsx.namelist() if f.startswith('xl/worksheets/sheet') and f.endswith('.xml')])
|
|
1180
|
+
for sheet_file in sheet_files:
|
|
1181
|
+
sheet_tree = ET.parse(xlsx.open(sheet_file))
|
|
1182
|
+
ns_s = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
|
|
1183
|
+
rows_dict = defaultdict(dict)
|
|
1184
|
+
max_col_idx = 0
|
|
1185
|
+
|
|
1186
|
+
for row_el in sheet_tree.getroot().findall('.//s:sheetData/s:row', ns_s):
|
|
1187
|
+
row_idx = int(row_el.attrib.get('r', len(rows_dict) + 1))
|
|
1188
|
+
for c in row_el.findall('s:c', ns_s):
|
|
1189
|
+
cell_ref = c.attrib.get('r', '')
|
|
1190
|
+
col_idx = _col_ref_to_index(cell_ref) if cell_ref else len(rows_dict[row_idx])
|
|
1191
|
+
max_col_idx = max(max_col_idx, col_idx)
|
|
1192
|
+
|
|
1193
|
+
c_type = c.attrib.get('t', '')
|
|
1194
|
+
val = ""
|
|
1195
|
+
if c_type == 's':
|
|
1196
|
+
v_el = c.find('s:v', ns_s)
|
|
1197
|
+
if v_el is not None and v_el.text and v_el.text.strip().isdigit():
|
|
1198
|
+
s_idx = int(v_el.text.strip())
|
|
1199
|
+
if s_idx < len(shared_strings):
|
|
1200
|
+
val = shared_strings[s_idx]
|
|
1201
|
+
elif c_type == 'inlineStr':
|
|
1202
|
+
is_t = c.find('.//s:t', ns_s)
|
|
1203
|
+
if is_t is not None and is_t.text:
|
|
1204
|
+
val = is_t.text
|
|
1205
|
+
else:
|
|
1206
|
+
v_el = c.find('s:v', ns_s)
|
|
1207
|
+
if v_el is not None and v_el.text:
|
|
1208
|
+
val = v_el.text
|
|
1209
|
+
|
|
1210
|
+
if val:
|
|
1211
|
+
rows_dict[row_idx][col_idx] = val
|
|
1212
|
+
|
|
1213
|
+
if rows_dict:
|
|
1214
|
+
sorted_row_indices = sorted(rows_dict.keys())
|
|
1215
|
+
grid = []
|
|
1216
|
+
num_cols = max_col_idx + 1
|
|
1217
|
+
|
|
1218
|
+
for r_i in sorted_row_indices:
|
|
1219
|
+
row_data = [rows_dict[r_i].get(c_i, "") for c_i in range(num_cols)]
|
|
1220
|
+
if any(str(c).strip() for c in row_data):
|
|
1221
|
+
grid.append(row_data)
|
|
1222
|
+
|
|
1223
|
+
if grid:
|
|
1224
|
+
header = "| " + " | ".join(sanitize_table_cell(c) for c in grid[0]) + " |"
|
|
1225
|
+
sep = "| " + " | ".join("---" for _ in range(num_cols)) + " |"
|
|
1226
|
+
table_lines = [header, sep]
|
|
1227
|
+
for row in grid[1:]:
|
|
1228
|
+
table_lines.append("| " + " | ".join(sanitize_table_cell(c) for c in row) + " |")
|
|
1229
|
+
tables.append("\n".join(table_lines))
|
|
1230
|
+
return tables
|
|
1231
|
+
|
|
1232
|
+
def extract_embedded_excel_tables(pptx_file, target_slide=None):
|
|
1233
|
+
"""
|
|
1234
|
+
Extracts tables from embedded Excel worksheets (.xlsx, .xlsm, .bin OLE objects) associated with slides.
|
|
1235
|
+
"""
|
|
1236
|
+
slide_excel_map = defaultdict(list)
|
|
1237
|
+
try:
|
|
1238
|
+
with zipfile.ZipFile(pptx_file, 'r') as pptx:
|
|
1239
|
+
for file in pptx.namelist():
|
|
1240
|
+
if file.startswith('ppt/slides/_rels/slide') and file.endswith('.rels'):
|
|
1241
|
+
slide_num = int(file.split('/')[-1].replace('slide', '').replace('.xml.rels', ''))
|
|
1242
|
+
if target_slide is not None and slide_num != target_slide:
|
|
1243
|
+
continue
|
|
1244
|
+
with pptx.open(file) as rels_file:
|
|
1245
|
+
rels_tree = ET.parse(rels_file)
|
|
1246
|
+
rels_root = rels_tree.getroot()
|
|
1247
|
+
for rel in rels_root.findall('{http://schemas.openxmlformats.org/package/2006/relationships}Relationship'):
|
|
1248
|
+
target = rel.attrib.get('Target', '')
|
|
1249
|
+
if 'embeddings/' in target or 'package' in rel.attrib.get('Type', '').lower() or 'oleobject' in rel.attrib.get('Type', '').lower():
|
|
1250
|
+
emb_path = 'ppt/' + target.replace('../', '') if not target.startswith('ppt/') else target
|
|
1251
|
+
if emb_path in pptx.namelist() and emb_path not in slide_excel_map[slide_num]:
|
|
1252
|
+
slide_excel_map[slide_num].append(emb_path)
|
|
1253
|
+
|
|
1254
|
+
slide_excel_tables = {}
|
|
1255
|
+
for slide_num, emb_files in slide_excel_map.items():
|
|
1256
|
+
tables = []
|
|
1257
|
+
for emb_path in emb_files:
|
|
1258
|
+
try:
|
|
1259
|
+
raw_data = pptx.read(emb_path)
|
|
1260
|
+
|
|
1261
|
+
# 1. Try Direct ZIP/XLSX
|
|
1262
|
+
if raw_data.startswith(b"PK\x03\x04"):
|
|
1263
|
+
parsed = _parse_xlsx_stream(io.BytesIO(raw_data))
|
|
1264
|
+
tables.extend(parsed)
|
|
1265
|
+
continue
|
|
1266
|
+
|
|
1267
|
+
# 2. Try OLE2 CFBF Container (Package / Workbook streams)
|
|
1268
|
+
if raw_data.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):
|
|
1269
|
+
ole_streams = extract_ole_streams(raw_data)
|
|
1270
|
+
if "Package" in ole_streams:
|
|
1271
|
+
parsed = _parse_xlsx_stream(io.BytesIO(ole_streams["Package"]))
|
|
1272
|
+
tables.extend(parsed)
|
|
1273
|
+
continue
|
|
1274
|
+
elif "Workbook" in ole_streams:
|
|
1275
|
+
biff_md = parse_biff8_workbook(ole_streams["Workbook"])
|
|
1276
|
+
if biff_md:
|
|
1277
|
+
tables.append(biff_md)
|
|
1278
|
+
continue
|
|
1279
|
+
elif "Book" in ole_streams:
|
|
1280
|
+
biff_md = parse_biff8_workbook(ole_streams["Book"])
|
|
1281
|
+
if biff_md:
|
|
1282
|
+
tables.append(biff_md)
|
|
1283
|
+
continue
|
|
1284
|
+
|
|
1285
|
+
# 3. Fallback: Search for embedded PK (ZIP) header anywhere in binary
|
|
1286
|
+
pk_idx = raw_data.find(b"PK\x03\x04")
|
|
1287
|
+
if pk_idx != -1:
|
|
1288
|
+
parsed = _parse_xlsx_stream(io.BytesIO(raw_data[pk_idx:]))
|
|
1289
|
+
tables.extend(parsed)
|
|
1290
|
+
continue
|
|
1291
|
+
|
|
1292
|
+
except Exception:
|
|
1293
|
+
pass
|
|
1294
|
+
if tables:
|
|
1295
|
+
slide_excel_tables[slide_num] = tables
|
|
1296
|
+
return slide_excel_tables
|
|
1297
|
+
except Exception:
|
|
1298
|
+
return {}
|
|
1299
|
+
|
|
1300
|
+
def format_chart_category_val(val_text, format_code=''):
|
|
1301
|
+
if not val_text:
|
|
1302
|
+
return ""
|
|
1303
|
+
is_date_fmt = any(k in format_code.lower() for k in ['yy', 'mmm', 'dd', 'd-', 'm-']) if format_code else False
|
|
1304
|
+
try:
|
|
1305
|
+
f_val = float(val_text)
|
|
1306
|
+
if (is_date_fmt and 1 <= f_val <= 100000) or (not format_code and 35000 <= f_val <= 60000):
|
|
1307
|
+
dt = datetime.datetime(1899, 12, 30) + datetime.timedelta(days=f_val)
|
|
1308
|
+
if format_code and ('d' in format_code.lower()):
|
|
1309
|
+
return dt.strftime('%d-%b-%y')
|
|
1310
|
+
return dt.strftime('%b-%y')
|
|
1311
|
+
except Exception:
|
|
1312
|
+
pass
|
|
1313
|
+
return str(val_text)
|
|
1314
|
+
|
|
1315
|
+
def format_chart_num_val(val_str, is_percent=False):
|
|
1316
|
+
if not val_str:
|
|
1317
|
+
return ""
|
|
1318
|
+
try:
|
|
1319
|
+
val_float = float(val_str)
|
|
1320
|
+
if is_percent:
|
|
1321
|
+
pct_val = val_float * 100.0 if abs(val_float) <= 1.0 else val_float
|
|
1322
|
+
if pct_val.is_integer():
|
|
1323
|
+
return f"{int(pct_val)}%"
|
|
1324
|
+
formatted = f"{pct_val:.3f}".rstrip('0').rstrip('.')
|
|
1325
|
+
return f"{formatted}%"
|
|
1326
|
+
|
|
1327
|
+
if val_float.is_integer():
|
|
1328
|
+
return str(int(val_float))
|
|
1329
|
+
formatted = f"{val_float:.3f}".rstrip('0').rstrip('.')
|
|
1330
|
+
return formatted
|
|
1331
|
+
except Exception:
|
|
1332
|
+
return str(val_str)
|
|
1333
|
+
|
|
1334
|
+
def extract_chart_data_from_pptx(pptx_file, target_slide=None):
|
|
1335
|
+
with zipfile.ZipFile(pptx_file, 'r') as pptx:
|
|
1336
|
+
slide_chart_map = defaultdict(list)
|
|
1337
|
+
|
|
1338
|
+
for file in pptx.namelist():
|
|
1339
|
+
if file.startswith('ppt/slides/_rels/slide') and file.endswith('.xml.rels'):
|
|
1340
|
+
slide_num = int(file.split('/')[-1].replace('slide', '').replace('.xml.rels', ''))
|
|
1341
|
+
if target_slide is not None and slide_num != target_slide:
|
|
1342
|
+
continue
|
|
1343
|
+
|
|
1344
|
+
with pptx.open(file) as rels_file:
|
|
1345
|
+
rels_tree = ET.parse(rels_file)
|
|
1346
|
+
rels_root = rels_tree.getroot()
|
|
1347
|
+
for rel in rels_root.findall('{http://schemas.openxmlformats.org/package/2006/relationships}Relationship'):
|
|
1348
|
+
target = rel.attrib.get('Target', '')
|
|
1349
|
+
if 'charts/chart' in target:
|
|
1350
|
+
chart_path = 'ppt/' + target.replace('../', '')
|
|
1351
|
+
slide_chart_map[slide_num].append(chart_path)
|
|
1352
|
+
|
|
1353
|
+
namespaces = {
|
|
1354
|
+
'c': 'http://schemas.openxmlformats.org/drawingml/2006/chart',
|
|
1355
|
+
'a': 'http://schemas.openxmlformats.org/drawingml/2006/main',
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
slide_chart_data = {}
|
|
1359
|
+
|
|
1360
|
+
for slide_num, chart_files in slide_chart_map.items():
|
|
1361
|
+
charts_in_slide = []
|
|
1362
|
+
|
|
1363
|
+
for chart_file in chart_files:
|
|
1364
|
+
if chart_file not in pptx.namelist():
|
|
1365
|
+
continue
|
|
1366
|
+
|
|
1367
|
+
with pptx.open(chart_file) as f:
|
|
1368
|
+
tree = ET.parse(f)
|
|
1369
|
+
root = tree.getroot()
|
|
1370
|
+
|
|
1371
|
+
# Extract Chart Title if present
|
|
1372
|
+
chart_title = None
|
|
1373
|
+
title_el = root.find('.//c:chart//c:title', namespaces)
|
|
1374
|
+
if title_el is not None:
|
|
1375
|
+
title_parts = [t.text for t in title_el.findall('.//a:t', namespaces) if t.text]
|
|
1376
|
+
if not title_parts:
|
|
1377
|
+
title_parts = [v.text for v in title_el.findall('.//c:v', namespaces) if v.text]
|
|
1378
|
+
if title_parts:
|
|
1379
|
+
chart_title = "".join(title_parts).strip()
|
|
1380
|
+
|
|
1381
|
+
chart_series_list = []
|
|
1382
|
+
|
|
1383
|
+
for ser in root.findall('.//c:ser', namespaces):
|
|
1384
|
+
series_name = ser.find('.//c:tx//c:v', namespaces)
|
|
1385
|
+
if series_name is None:
|
|
1386
|
+
series_name = ser.find('.//c:tx//a:t', namespaces)
|
|
1387
|
+
series_name = series_name.text if series_name is not None else 'Series'
|
|
1388
|
+
|
|
1389
|
+
# Check for percentage format code in OpenXML or explicit '%' in series name
|
|
1390
|
+
format_code = ""
|
|
1391
|
+
num_fmt_el = ser.find('.//c:numFmt', namespaces) or ser.find('.//c:formatCode', namespaces)
|
|
1392
|
+
if num_fmt_el is not None:
|
|
1393
|
+
format_code = num_fmt_el.attrib.get('formatCode', '') or num_fmt_el.text or ''
|
|
1394
|
+
|
|
1395
|
+
is_percent = '%' in format_code or '%' in series_name
|
|
1396
|
+
|
|
1397
|
+
cat_el = ser.find('.//c:cat', namespaces)
|
|
1398
|
+
cat_format_code = ""
|
|
1399
|
+
if cat_el is not None:
|
|
1400
|
+
cat_fmt_el = cat_el.find('.//c:numFmt', namespaces) or cat_el.find('.//c:formatCode', namespaces)
|
|
1401
|
+
if cat_fmt_el is not None:
|
|
1402
|
+
cat_format_code = cat_fmt_el.attrib.get('formatCode', '') or cat_fmt_el.text or ''
|
|
1403
|
+
|
|
1404
|
+
categories = [
|
|
1405
|
+
format_chart_category_val(cat_pt.text, cat_format_code) for cat_pt in ser.findall('.//c:cat//c:pt//c:v', namespaces)
|
|
1406
|
+
]
|
|
1407
|
+
raw_values = [
|
|
1408
|
+
val_el.text for val_el in ser.findall('.//c:val//c:pt//c:v', namespaces)
|
|
1409
|
+
]
|
|
1410
|
+
|
|
1411
|
+
formatted_values = [
|
|
1412
|
+
format_chart_num_val(v, is_percent=is_percent) for v in raw_values
|
|
1413
|
+
]
|
|
1414
|
+
|
|
1415
|
+
data_points = list(zip(categories, formatted_values)) if categories else formatted_values
|
|
1416
|
+
chart_series_list.append({
|
|
1417
|
+
'series': series_name,
|
|
1418
|
+
'data': data_points
|
|
1419
|
+
})
|
|
1420
|
+
|
|
1421
|
+
if chart_series_list:
|
|
1422
|
+
all_cats = []
|
|
1423
|
+
for s in chart_series_list:
|
|
1424
|
+
for dp in s['data']:
|
|
1425
|
+
if isinstance(dp, tuple) and dp[0] not in all_cats:
|
|
1426
|
+
all_cats.append(dp[0])
|
|
1427
|
+
|
|
1428
|
+
table_parts = []
|
|
1429
|
+
if chart_title:
|
|
1430
|
+
table_parts.append(chart_title)
|
|
1431
|
+
|
|
1432
|
+
if all_cats:
|
|
1433
|
+
header = "| " + " | ".join(["Series"] + all_cats) + " |"
|
|
1434
|
+
sep = "| " + " | ".join(["---"] + ["---" for _ in all_cats]) + " |"
|
|
1435
|
+
rows = [header, sep]
|
|
1436
|
+
for s in chart_series_list:
|
|
1437
|
+
val_dict = {dp[0]: dp[1] for dp in s['data'] if isinstance(dp, tuple)}
|
|
1438
|
+
row_cells = [s['series']] + [val_dict.get(c, "-") for c in all_cats]
|
|
1439
|
+
rows.append("| " + " | ".join(row_cells) + " |")
|
|
1440
|
+
table_parts.append("\n".join(rows))
|
|
1441
|
+
else:
|
|
1442
|
+
lines = []
|
|
1443
|
+
for s in chart_series_list:
|
|
1444
|
+
lines.append(f"• {s['series']}: " + ", ".join(str(d) for d in s['data']))
|
|
1445
|
+
table_parts.append("\n".join(lines))
|
|
1446
|
+
|
|
1447
|
+
charts_in_slide.append("\n\n".join(table_parts))
|
|
1448
|
+
|
|
1449
|
+
slide_chart_data[slide_num] = charts_in_slide
|
|
1450
|
+
|
|
1451
|
+
return slide_chart_data
|
|
1452
|
+
|
|
1453
|
+
from collections import namedtuple
|
|
1454
|
+
|
|
1455
|
+
# TextItem preserves backward compatibility where it[0]=x, it[1]=y, it[2]=text
|
|
1456
|
+
# while exposing it.right and it.bottom for exact physical bounding boxes.
|
|
1457
|
+
TextItem = namedtuple('TextItem', ['x', 'y', 'text', 'right', 'bottom'])
|
|
1458
|
+
|
|
1459
|
+
def extract_wmf_text_records(wmf_bytes):
|
|
1460
|
+
"""
|
|
1461
|
+
Decodes vector text records from a 16-bit Windows Metafile (WMF) binary stream:
|
|
1462
|
+
- META_EXTTEXTOUT (0x0A32 / 2610)
|
|
1463
|
+
- META_TEXTOUT (0x0521 / 1313)
|
|
1464
|
+
Returns a list of TextItem(x, y, text, right, bottom) tuples.
|
|
1465
|
+
"""
|
|
1466
|
+
items = []
|
|
1467
|
+
offset = 0
|
|
1468
|
+
# Check Placeable Header (magic: 0x9AC6CDD7)
|
|
1469
|
+
if len(wmf_bytes) > 22 and wmf_bytes[:4] == b'\xd7\xcd\xc6\x9a':
|
|
1470
|
+
offset = 22
|
|
1471
|
+
if len(wmf_bytes) < offset + 18:
|
|
1472
|
+
return items
|
|
1473
|
+
offset += 18
|
|
1474
|
+
while offset < len(wmf_bytes) - 6:
|
|
1475
|
+
rd_size, rd_func = struct.unpack_from('<IH', wmf_bytes, offset)
|
|
1476
|
+
if rd_size == 0:
|
|
1477
|
+
break
|
|
1478
|
+
rec_len_bytes = rd_size * 2
|
|
1479
|
+
if offset + rec_len_bytes > len(wmf_bytes):
|
|
1480
|
+
break
|
|
1481
|
+
rec_data = wmf_bytes[offset:offset+rec_len_bytes]
|
|
1482
|
+
if rd_func == 0x0A32 and len(rec_data) >= 12:
|
|
1483
|
+
y, x, count, fw_opts = struct.unpack_from('<hhHH', rec_data, 6)
|
|
1484
|
+
str_offset = 14
|
|
1485
|
+
rec_rect = None
|
|
1486
|
+
if (fw_opts & 0x0006) != 0 and len(rec_data) >= 22:
|
|
1487
|
+
r_left, r_top, r_right, r_bottom = struct.unpack_from('<hhhh', rec_data, 14)
|
|
1488
|
+
rec_rect = (int(r_left), int(r_top), int(r_right), int(r_bottom))
|
|
1489
|
+
str_offset = 22
|
|
1490
|
+
if str_offset + count <= len(rec_data):
|
|
1491
|
+
raw_s = rec_data[str_offset:str_offset+count]
|
|
1492
|
+
try:
|
|
1493
|
+
s = raw_s.decode('utf-8')
|
|
1494
|
+
except Exception:
|
|
1495
|
+
s = raw_s.decode('latin1', errors='ignore')
|
|
1496
|
+
s = s.strip()
|
|
1497
|
+
if s:
|
|
1498
|
+
if rec_rect and rec_rect[2] > rec_rect[0]:
|
|
1499
|
+
items.append(TextItem(rec_rect[0], rec_rect[1], s, rec_rect[2], rec_rect[3]))
|
|
1500
|
+
else:
|
|
1501
|
+
items.append(TextItem(int(x), int(y), s, int(x) + len(s) * 7, int(y) + 14))
|
|
1502
|
+
elif rd_func == 0x0521 and len(rec_data) >= 12:
|
|
1503
|
+
count = struct.unpack_from('<H', rec_data, 6)[0]
|
|
1504
|
+
str_offset = 8
|
|
1505
|
+
pad_count = count + (count % 2)
|
|
1506
|
+
if str_offset + pad_count + 4 <= len(rec_data):
|
|
1507
|
+
y, x = struct.unpack_from('<hh', rec_data, str_offset + pad_count)
|
|
1508
|
+
raw_s = rec_data[str_offset:str_offset+count]
|
|
1509
|
+
try:
|
|
1510
|
+
s = raw_s.decode('utf-8')
|
|
1511
|
+
except Exception:
|
|
1512
|
+
s = raw_s.decode('latin1', errors='ignore')
|
|
1513
|
+
s = s.strip()
|
|
1514
|
+
if s:
|
|
1515
|
+
items.append(TextItem(int(x), int(y), s, int(x) + len(s) * 7, int(y) + 14))
|
|
1516
|
+
offset += rec_len_bytes
|
|
1517
|
+
return items
|
|
1518
|
+
|
|
1519
|
+
def extract_emf_text_records(emf_bytes):
|
|
1520
|
+
"""
|
|
1521
|
+
Decodes vector text records from an Enhanced Metafile (EMF) binary stream:
|
|
1522
|
+
- EMR_EXTTEXTOUTW (0x54 / 84) - UTF-16LE text
|
|
1523
|
+
- EMR_EXTTEXTOUTA (0x53 / 83) - ANSI/Latin1 text
|
|
1524
|
+
Extracts the true physical bounding box [left, top, right, bottom] written by Windows GDI.
|
|
1525
|
+
Returns a list of TextItem(x, y, text, right, bottom) tuples.
|
|
1526
|
+
"""
|
|
1527
|
+
offset = 0
|
|
1528
|
+
items = []
|
|
1529
|
+
cur_dx, cur_dy = 0.0, 0.0
|
|
1530
|
+
while offset < len(emf_bytes) - 8:
|
|
1531
|
+
itype, nsize = struct.unpack_from('<II', emf_bytes, offset)
|
|
1532
|
+
if nsize <= 0 or offset + nsize > len(emf_bytes):
|
|
1533
|
+
break
|
|
1534
|
+
rec_data = emf_bytes[offset:offset+nsize]
|
|
1535
|
+
if itype in (35, 36) and len(rec_data) >= 32: # EMR_SETWORLDTRANSFORM / EMR_MODIFYWORLDTRANSFORM
|
|
1536
|
+
try:
|
|
1537
|
+
_, _, _, _, eDx, eDy = struct.unpack_from('<ffffff', rec_data, 8)
|
|
1538
|
+
cur_dx, cur_dy = float(eDx), float(eDy)
|
|
1539
|
+
except Exception:
|
|
1540
|
+
pass
|
|
1541
|
+
elif itype in (83, 84) and len(rec_data) >= 76: # EMR_EXTTEXTOUTW / EMR_EXTTEXTOUTA
|
|
1542
|
+
rcl_left, rcl_top, rcl_right, rcl_bottom = struct.unpack_from('<iiii', rec_data, 8)
|
|
1543
|
+
ref_x, ref_y = struct.unpack_from('<ii', rec_data, 36)
|
|
1544
|
+
nchars, off_str = struct.unpack_from('<II', rec_data, 44)
|
|
1545
|
+
is_w = (itype == 84)
|
|
1546
|
+
char_len = nchars * 2 if is_w else nchars
|
|
1547
|
+
if off_str + char_len <= len(rec_data):
|
|
1548
|
+
raw_bytes = rec_data[off_str:off_str + char_len]
|
|
1549
|
+
s = raw_bytes.decode('utf-16le' if is_w else 'latin1', errors='ignore').strip()
|
|
1550
|
+
if s:
|
|
1551
|
+
eff_x = cur_dx if (ref_x == 0 and cur_dx != 0.0) else float(ref_x)
|
|
1552
|
+
eff_y = cur_dy if (ref_y == 0 and cur_dy != 0.0) else float(ref_y)
|
|
1553
|
+
if rcl_right > rcl_left:
|
|
1554
|
+
item_l = float(rcl_left)
|
|
1555
|
+
item_r = float(rcl_right)
|
|
1556
|
+
item_t = float(rcl_top)
|
|
1557
|
+
item_b = float(rcl_bottom)
|
|
1558
|
+
else:
|
|
1559
|
+
item_l = eff_x
|
|
1560
|
+
item_t = eff_y
|
|
1561
|
+
item_r = eff_x + len(s) * 7.0
|
|
1562
|
+
item_b = eff_y + 14.0
|
|
1563
|
+
items.append(TextItem(item_l, item_t, s, item_r, item_b))
|
|
1564
|
+
offset += nsize
|
|
1565
|
+
return items
|
|
1566
|
+
|
|
1567
|
+
|
|
1568
|
+
def extract_metafile_text_records(data):
|
|
1569
|
+
"""
|
|
1570
|
+
Dispatches decoding between EMF (32-bit) and WMF (16-bit) vector formats.
|
|
1571
|
+
"""
|
|
1572
|
+
if not data or len(data) < 18:
|
|
1573
|
+
return []
|
|
1574
|
+
# EMF header magic (record type 1 at offset 0)
|
|
1575
|
+
if len(data) >= 4 and data[:4] == b'\x01\x00\x00\x00':
|
|
1576
|
+
return extract_emf_text_records(data)
|
|
1577
|
+
# WMF placeable magic or standard header
|
|
1578
|
+
if (len(data) >= 4 and data[:4] == b'\xd7\xcd\xc6\x9a') or (len(data) >= 2 and data[:2] == b'\x01\x00'):
|
|
1579
|
+
return extract_wmf_text_records(data)
|
|
1580
|
+
# Fallback try EMF first then WMF
|
|
1581
|
+
res = extract_emf_text_records(data)
|
|
1582
|
+
if not res:
|
|
1583
|
+
res = extract_wmf_text_records(data)
|
|
1584
|
+
return res
|
|
1585
|
+
|
|
1586
|
+
def split_side_by_side_blocks(items):
|
|
1587
|
+
"""
|
|
1588
|
+
Geometrically detects and splits side-by-side sub-tables/blocks in vector drawings
|
|
1589
|
+
by finding column gutters separating parallel table headers/blocks.
|
|
1590
|
+
"""
|
|
1591
|
+
if not items or len(items) < 10:
|
|
1592
|
+
return [items] if items else []
|
|
1593
|
+
sorted_by_y = sorted(items, key=lambda it: (it[1], it[0]))
|
|
1594
|
+
min_y = sorted_by_y[0][1]
|
|
1595
|
+
|
|
1596
|
+
# Find top-row section titles (e.g. 2 or 3 parallel titles at the top edge)
|
|
1597
|
+
top_items = [it for it in sorted_by_y if it[1] <= min_y + 8 and any(c.isalpha() for c in it[2]) and len(it[2].strip()) >= 2]
|
|
1598
|
+
top_items.sort(key=lambda it: it[0])
|
|
1599
|
+
|
|
1600
|
+
# If top row has > 3 items, it is a multi-column table header row (e.g. wide single table), NOT table titles!
|
|
1601
|
+
if len(top_items) < 2 or len(top_items) > 3:
|
|
1602
|
+
return [items]
|
|
1603
|
+
|
|
1604
|
+
candidate_splits = []
|
|
1605
|
+
for i in range(len(top_items) - 1):
|
|
1606
|
+
t1_x = top_items[i][0]
|
|
1607
|
+
t2_x = top_items[i+1][0]
|
|
1608
|
+
if t2_x - t1_x > 200:
|
|
1609
|
+
window_xs = sorted(set(it[0] for it in items if t1_x <= it[0] <= t2_x + 50))
|
|
1610
|
+
gaps = []
|
|
1611
|
+
for j in range(len(window_xs) - 1):
|
|
1612
|
+
g = window_xs[j+1] - window_xs[j]
|
|
1613
|
+
if g >= 35:
|
|
1614
|
+
gaps.append((window_xs[j], window_xs[j+1], g))
|
|
1615
|
+
|
|
1616
|
+
if gaps:
|
|
1617
|
+
chosen_split = None
|
|
1618
|
+
for x_left, x_right, _ in reversed(gaps):
|
|
1619
|
+
col_items = [it for it in items if abs(it[0] - x_right) <= 20]
|
|
1620
|
+
if x_right <= t2_x:
|
|
1621
|
+
if len(col_items) >= 2 or x_right >= t2_x - 30:
|
|
1622
|
+
chosen_split = (x_left + x_right) / 2
|
|
1623
|
+
if len(col_items) >= 2:
|
|
1624
|
+
break
|
|
1625
|
+
if chosen_split is None:
|
|
1626
|
+
chosen_split = (gaps[-1][0] + gaps[-1][1]) / 2
|
|
1627
|
+
candidate_splits.append(chosen_split)
|
|
1628
|
+
|
|
1629
|
+
if not candidate_splits or len(candidate_splits) != len(top_items) - 1:
|
|
1630
|
+
return [items]
|
|
1631
|
+
|
|
1632
|
+
def count_cols_in_block(b_items):
|
|
1633
|
+
col_means = []
|
|
1634
|
+
for it in sorted(b_items, key=lambda x: x[0]):
|
|
1635
|
+
x = it[0]
|
|
1636
|
+
matched = False
|
|
1637
|
+
for c in col_means:
|
|
1638
|
+
if abs(c['mean'] - x) <= 30:
|
|
1639
|
+
c['items'].append(it)
|
|
1640
|
+
c['mean'] = sum(i[0] for i in c['items']) / len(c['items'])
|
|
1641
|
+
matched = True
|
|
1642
|
+
break
|
|
1643
|
+
if not matched:
|
|
1644
|
+
col_means.append({'mean': x, 'items': [it]})
|
|
1645
|
+
return len(col_means)
|
|
1646
|
+
|
|
1647
|
+
bounds = [0] + sorted(candidate_splits) + [999999]
|
|
1648
|
+
blocks = []
|
|
1649
|
+
for i in range(len(bounds) - 1):
|
|
1650
|
+
b_items = [it for it in items if bounds[i] <= it[0] < bounds[i+1]]
|
|
1651
|
+
if len(b_items) < 10 or count_cols_in_block(b_items) < 2:
|
|
1652
|
+
return [items]
|
|
1653
|
+
blocks.append(b_items)
|
|
1654
|
+
|
|
1655
|
+
# Validate that each block represents an independent sub-table with its own row labels/categories.
|
|
1656
|
+
# If blocks lack text labels in data rows (e.g. they only contain numbers/values under a spanning header banner),
|
|
1657
|
+
# they are columns of a single unified table with zero space between sections, NOT separate tables!
|
|
1658
|
+
for b in blocks:
|
|
1659
|
+
rg_list = []
|
|
1660
|
+
for it in sorted(b, key=lambda x: (x[1], x[0])):
|
|
1661
|
+
if not rg_list or abs(rg_list[-1]['y'] - it[1]) > 8:
|
|
1662
|
+
rg_list.append({'y': it[1], 'items': [it]})
|
|
1663
|
+
else:
|
|
1664
|
+
rg_list[-1]['items'].append(it)
|
|
1665
|
+
|
|
1666
|
+
def is_num(t):
|
|
1667
|
+
s = t.strip().rstrip('%').lstrip('$').lstrip('₹').replace(',', '').replace('(', '-').replace(')', '')
|
|
1668
|
+
try:
|
|
1669
|
+
float(s)
|
|
1670
|
+
return True
|
|
1671
|
+
except ValueError:
|
|
1672
|
+
return False
|
|
1673
|
+
|
|
1674
|
+
first_data_y = None
|
|
1675
|
+
for rg in rg_list:
|
|
1676
|
+
if sum(1 for it in rg['items'] if is_num(it[2])) >= 1:
|
|
1677
|
+
first_data_y = rg['y']
|
|
1678
|
+
break
|
|
1679
|
+
|
|
1680
|
+
if first_data_y is not None:
|
|
1681
|
+
text_labels = [it[2] for it in b if it[1] >= first_data_y and any(c.isalpha() for c in it[2]) and it[2] != '-']
|
|
1682
|
+
if not text_labels:
|
|
1683
|
+
return [items]
|
|
1684
|
+
|
|
1685
|
+
return blocks if blocks else [items]
|
|
1686
|
+
|
|
1687
|
+
def build_table_from_emf_items(sub_items):
|
|
1688
|
+
"""
|
|
1689
|
+
Converts a clustered block of 2D EMF/WMF vector text coordinates into a Markdown table or structured text.
|
|
1690
|
+
Handles multi-tier table headers with spanning banner forward-fill and parenthesized subheaders.
|
|
1691
|
+
"""
|
|
1692
|
+
if not sub_items:
|
|
1693
|
+
return ""
|
|
1694
|
+
sorted_by_y = sorted(sub_items, key=lambda it: (it[1], it[0]))
|
|
1695
|
+
min_y = sorted_by_y[0][1]
|
|
1696
|
+
|
|
1697
|
+
# Filter out single rogue digits at top edge and zero-height placeholder dashes
|
|
1698
|
+
clean_items = [it for it in sorted_by_y if not (it[1] <= min_y + 10 and len(it[2].strip()) == 1 and it[2].strip().isdigit()) and not (it[2] == '-' and it.bottom <= it[1])]
|
|
1699
|
+
if not clean_items:
|
|
1700
|
+
return ""
|
|
1701
|
+
min_y = clean_items[0][1]
|
|
1702
|
+
|
|
1703
|
+
# Title detection
|
|
1704
|
+
title = ""
|
|
1705
|
+
min_x = min(it[0] for it in clean_items) if clean_items else 0
|
|
1706
|
+
max_x = max(it[0] for it in clean_items) if clean_items else 0
|
|
1707
|
+
width = max(max_x - min_x, 1)
|
|
1708
|
+
top_row = [it for it in clean_items if it[1] <= min_y + 8 and any(c.isalpha() for c in it[2])]
|
|
1709
|
+
if len(top_row) == 1 and (top_row[0][0] - min_x) <= width * 0.7:
|
|
1710
|
+
title = top_row[0][2]
|
|
1711
|
+
remaining = [it for it in clean_items if it not in top_row]
|
|
1712
|
+
else:
|
|
1713
|
+
remaining = clean_items
|
|
1714
|
+
|
|
1715
|
+
# Separate footnote lines at the bottom (lone item at bottom edge starting with * or Note)
|
|
1716
|
+
footnotes = []
|
|
1717
|
+
if not remaining:
|
|
1718
|
+
return ""
|
|
1719
|
+
max_y = max(it[1] for it in remaining)
|
|
1720
|
+
bottom_items = [it for it in remaining if it[1] >= max_y - 10]
|
|
1721
|
+
if len(bottom_items) == 1 and bottom_items[0][2].startswith(('*', 'Note:', 'Note -', '#')):
|
|
1722
|
+
footnotes.append(bottom_items[0][2])
|
|
1723
|
+
remaining = [it for it in remaining if it not in bottom_items]
|
|
1724
|
+
|
|
1725
|
+
# Group into rows by Y (threshold 8px)
|
|
1726
|
+
row_groups = []
|
|
1727
|
+
for it in remaining:
|
|
1728
|
+
if not row_groups or abs(row_groups[-1]['y'] - it[1]) > 8:
|
|
1729
|
+
row_groups.append({'y': it[1], 'items': [it]})
|
|
1730
|
+
else:
|
|
1731
|
+
row_groups[-1]['items'].append(it)
|
|
1732
|
+
|
|
1733
|
+
# Merge wrapped multiline rows (continuation lines)
|
|
1734
|
+
merged = []
|
|
1735
|
+
i = 0
|
|
1736
|
+
while i < len(row_groups):
|
|
1737
|
+
rg = row_groups[i]
|
|
1738
|
+
# Lookahead: single-item line followed by multi-item row where next row lacks item at this X
|
|
1739
|
+
if len(rg['items']) == 1 and i + 1 < len(row_groups) and len(row_groups[i+1]['items']) >= 2:
|
|
1740
|
+
next_rg = row_groups[i+1]
|
|
1741
|
+
it_x = rg['items'][0][0]
|
|
1742
|
+
if all(abs(it[0] - it_x) > 30 for it in next_rg['items']) and (next_rg['y'] - rg['y']) <= 25:
|
|
1743
|
+
next_rg['items'].append(rg['items'][0])
|
|
1744
|
+
next_rg['items'].sort(key=lambda x: x[0])
|
|
1745
|
+
i += 1
|
|
1746
|
+
continue
|
|
1747
|
+
# Lookahead: multi-item row followed by single-item continuation line
|
|
1748
|
+
if len(rg['items']) >= 2 and i + 1 < len(row_groups) and len(row_groups[i+1]['items']) == 1:
|
|
1749
|
+
next_rg = row_groups[i+1]
|
|
1750
|
+
it_next = next_rg['items'][0]
|
|
1751
|
+
matched_item = None
|
|
1752
|
+
for it in rg['items']:
|
|
1753
|
+
if abs(it[0] - it_next[0]) <= 30:
|
|
1754
|
+
matched_item = it
|
|
1755
|
+
break
|
|
1756
|
+
if matched_item is not None and (next_rg['y'] - rg['y']) <= 25:
|
|
1757
|
+
idx_in_rg = rg['items'].index(matched_item)
|
|
1758
|
+
merged_text = f"{matched_item[2]} {it_next[2]}"
|
|
1759
|
+
merged_right = max(matched_item.right, it_next.right)
|
|
1760
|
+
merged_bottom = max(matched_item.bottom, it_next.bottom)
|
|
1761
|
+
merged_it = TextItem(matched_item[0], matched_item[1], merged_text, merged_right, merged_bottom)
|
|
1762
|
+
rg['items'][idx_in_rg] = merged_it
|
|
1763
|
+
merged.append(rg)
|
|
1764
|
+
i += 2
|
|
1765
|
+
continue
|
|
1766
|
+
merged.append(rg)
|
|
1767
|
+
i += 1
|
|
1768
|
+
row_groups = merged
|
|
1769
|
+
|
|
1770
|
+
# Filter pure separator lines (rows where all items are '-')
|
|
1771
|
+
row_groups = [r for r in row_groups if not all(it[2] == '-' for it in r['items'])]
|
|
1772
|
+
|
|
1773
|
+
if not row_groups:
|
|
1774
|
+
return title
|
|
1775
|
+
|
|
1776
|
+
for r in row_groups:
|
|
1777
|
+
r['items'].sort(key=lambda it: it[0])
|
|
1778
|
+
|
|
1779
|
+
if len(row_groups) <= 1 or max(len(rg['items']) for rg in row_groups) <= 1:
|
|
1780
|
+
lines = []
|
|
1781
|
+
if title:
|
|
1782
|
+
lines.append(f"### {title}\n")
|
|
1783
|
+
for rg in row_groups:
|
|
1784
|
+
r_text = " ".join(it[2] for it in rg['items'])
|
|
1785
|
+
if r_text:
|
|
1786
|
+
lines.append(r_text)
|
|
1787
|
+
if footnotes:
|
|
1788
|
+
lines.append("\n" + "\n".join(footnotes))
|
|
1789
|
+
return "\n".join(lines)
|
|
1790
|
+
|
|
1791
|
+
# Separate Header Rows vs Data Rows
|
|
1792
|
+
def is_cell_numeric(text):
|
|
1793
|
+
if not text:
|
|
1794
|
+
return False
|
|
1795
|
+
s = text.strip().rstrip('%').lstrip('$').lstrip('₹').lstrip('€').lstrip('£').strip()
|
|
1796
|
+
s = s.replace(',', '').replace('(', '-').replace(')', '')
|
|
1797
|
+
try:
|
|
1798
|
+
float(s)
|
|
1799
|
+
return True
|
|
1800
|
+
except ValueError:
|
|
1801
|
+
return False
|
|
1802
|
+
|
|
1803
|
+
first_data_idx = None
|
|
1804
|
+
for idx, rg in enumerate(row_groups):
|
|
1805
|
+
num_numeric = sum(1 for it in rg['items'] if is_cell_numeric(it[2]))
|
|
1806
|
+
if num_numeric >= 2 or (num_numeric >= 1 and idx >= 1):
|
|
1807
|
+
first_data_idx = idx
|
|
1808
|
+
break
|
|
1809
|
+
|
|
1810
|
+
if first_data_idx is None:
|
|
1811
|
+
header_rows = [row_groups[0]]
|
|
1812
|
+
data_rows = row_groups[1:]
|
|
1813
|
+
elif first_data_idx == 0:
|
|
1814
|
+
header_rows = []
|
|
1815
|
+
data_rows = row_groups
|
|
1816
|
+
else:
|
|
1817
|
+
if first_data_idx > 1 and len(row_groups[first_data_idx - 1]['items']) == 1 and len(row_groups[first_data_idx - 2]['items']) >= 3:
|
|
1818
|
+
first_data_idx -= 1
|
|
1819
|
+
header_rows = row_groups[:first_data_idx]
|
|
1820
|
+
data_rows = row_groups[first_data_idx:]
|
|
1821
|
+
|
|
1822
|
+
# Check for unaligned header (missing stub header like in Slide 9 Actuals where data has a column far to the left of all header items)
|
|
1823
|
+
if header_rows and data_rows:
|
|
1824
|
+
min_hdr_x = min(it[0] for r in header_rows for it in r['items'])
|
|
1825
|
+
data_xs_left_of_hdr = [it[0] for r in data_rows for it in r['items'] if it[0] < min_hdr_x - 100]
|
|
1826
|
+
if len(data_xs_left_of_hdr) >= max(1, len(data_rows) * 0.5):
|
|
1827
|
+
left_x = min(data_xs_left_of_hdr)
|
|
1828
|
+
for r in header_rows:
|
|
1829
|
+
stub_y = r['items'][0][1]
|
|
1830
|
+
stub_bottom = r['items'][0].bottom if hasattr(r['items'][0], 'bottom') else stub_y + 14.0
|
|
1831
|
+
r['items'].insert(0, TextItem(left_x, stub_y, '-', left_x + 10.0, stub_bottom))
|
|
1832
|
+
|
|
1833
|
+
# Derive column clusters from all header items using column spacing threshold
|
|
1834
|
+
all_header_items = [it for hr in header_rows for it in hr['items']]
|
|
1835
|
+
sh_clusters = []
|
|
1836
|
+
if all_header_items:
|
|
1837
|
+
data_items = [it for r in data_rows for it in r['items']]
|
|
1838
|
+
cluster_gap = 18.0
|
|
1839
|
+
if data_items:
|
|
1840
|
+
data_xs = sorted(it[0] for it in data_items)
|
|
1841
|
+
d_cl = []
|
|
1842
|
+
for x in data_xs:
|
|
1843
|
+
if not d_cl or abs(d_cl[-1][-1] - x) > 30:
|
|
1844
|
+
d_cl.append([x])
|
|
1845
|
+
else:
|
|
1846
|
+
d_cl[-1].append(x)
|
|
1847
|
+
data_centers = [sum(c) / len(c) for c in d_cl]
|
|
1848
|
+
d_spacings = [data_centers[c + 1] - data_centers[c] for c in range(len(data_centers) - 1)]
|
|
1849
|
+
if d_spacings:
|
|
1850
|
+
cluster_gap = max(18.0, min(d_spacings) * 0.45)
|
|
1851
|
+
|
|
1852
|
+
all_header_items_sorted = sorted(all_header_items, key=lambda x: x[0])
|
|
1853
|
+
for it in all_header_items_sorted:
|
|
1854
|
+
if not sh_clusters or (it[0] - sh_clusters[-1][-1][0] > cluster_gap):
|
|
1855
|
+
sh_clusters.append([it])
|
|
1856
|
+
else:
|
|
1857
|
+
sh_clusters[-1].append(it)
|
|
1858
|
+
|
|
1859
|
+
valid_lens = [len(r['items']) for r in (header_rows + data_rows) if len(r['items']) >= 2]
|
|
1860
|
+
max_cols = max(valid_lens) if valid_lens else (len(row_groups[0]['items']) if row_groups else 1)
|
|
1861
|
+
|
|
1862
|
+
# Collect column items for each column from full-width rows to establish geometric column extents
|
|
1863
|
+
full_rows = [r['items'] for r in data_rows if len(r['items']) == max_cols]
|
|
1864
|
+
if len(full_rows) == 1 and any('total' in it[2].lower() for it in full_rows[0]):
|
|
1865
|
+
non_total_rows = [r['items'] for r in data_rows if not any('total' in it[2].lower() for it in r['items'])]
|
|
1866
|
+
if non_total_rows:
|
|
1867
|
+
total_first_x = full_rows[0][0][0]
|
|
1868
|
+
stubs_in_data = [it for it in non_total_rows[0] if it[0] < total_first_x]
|
|
1869
|
+
if stubs_in_data:
|
|
1870
|
+
full_rows = []
|
|
1871
|
+
|
|
1872
|
+
if not full_rows:
|
|
1873
|
+
fallback_rows = [r['items'] for r in (header_rows + data_rows) if len(r['items']) == max_cols]
|
|
1874
|
+
# Do not fall back to the single summary row that was just disqualified
|
|
1875
|
+
if not (len(fallback_rows) == 1 and any('total' in it[2].lower() for it in fallback_rows[0])):
|
|
1876
|
+
full_rows = fallback_rows
|
|
1877
|
+
|
|
1878
|
+
if full_rows:
|
|
1879
|
+
full_xs = [sum(r[c][0] for r in full_rows) / len(full_rows) for c in range(len(full_rows[0]))]
|
|
1880
|
+
full_rights = [sum(r[c].right for r in full_rows) / len(full_rows) for c in range(len(full_rows[0]))]
|
|
1881
|
+
unmatched = []
|
|
1882
|
+
for r in data_rows:
|
|
1883
|
+
if len(r['items']) >= 2:
|
|
1884
|
+
for it in r['items']:
|
|
1885
|
+
if not any(abs(it[0] - fx) < 25 or abs(it.right - fr) < 20 for fx, fr in zip(full_xs, full_rights)):
|
|
1886
|
+
unmatched.append(it)
|
|
1887
|
+
persistent = []
|
|
1888
|
+
if unmatched:
|
|
1889
|
+
unmatched.sort(key=lambda it: it[0])
|
|
1890
|
+
u_clusters = []
|
|
1891
|
+
for u in unmatched:
|
|
1892
|
+
if not u_clusters or abs(u[0] - u_clusters[-1][-1][0]) > 25:
|
|
1893
|
+
u_clusters.append([u])
|
|
1894
|
+
else:
|
|
1895
|
+
u_clusters[-1].append(u)
|
|
1896
|
+
persistent = [cl for cl in u_clusters if len(cl) >= 2]
|
|
1897
|
+
|
|
1898
|
+
if persistent:
|
|
1899
|
+
all_col_centers = sorted(full_xs + [sum(u[0] for u in cl) / len(cl) for cl in persistent])
|
|
1900
|
+
num_cols = len(all_col_centers)
|
|
1901
|
+
col_items_by_c = [[] for _ in range(num_cols)]
|
|
1902
|
+
for r in data_rows:
|
|
1903
|
+
for it in r['items']:
|
|
1904
|
+
best_c = min(range(num_cols), key=lambda c: abs(it[0] - all_col_centers[c]))
|
|
1905
|
+
if abs(it[0] - all_col_centers[best_c]) < 35:
|
|
1906
|
+
col_items_by_c[best_c].append(it)
|
|
1907
|
+
col_boxes = []
|
|
1908
|
+
for c_idx in range(num_cols):
|
|
1909
|
+
items_c = col_items_by_c[c_idx]
|
|
1910
|
+
if items_c:
|
|
1911
|
+
mn = min(it.x for it in items_c)
|
|
1912
|
+
mx = max(it.right for it in items_c)
|
|
1913
|
+
col_boxes.append((mn, mx, (mn + mx) / 2.0))
|
|
1914
|
+
else:
|
|
1915
|
+
col_boxes.append((all_col_centers[c_idx], all_col_centers[c_idx] + 20, all_col_centers[c_idx] + 10))
|
|
1916
|
+
else:
|
|
1917
|
+
num_cols = len(full_rows[0])
|
|
1918
|
+
col_items_by_c = [[] for _ in range(num_cols)]
|
|
1919
|
+
for r_items in full_rows:
|
|
1920
|
+
for c_idx, it in enumerate(r_items):
|
|
1921
|
+
col_items_by_c[c_idx].append(it)
|
|
1922
|
+
col_boxes = []
|
|
1923
|
+
for c_idx in range(num_cols):
|
|
1924
|
+
items_c = col_items_by_c[c_idx]
|
|
1925
|
+
if items_c:
|
|
1926
|
+
mn = min(it.x for it in items_c)
|
|
1927
|
+
mx = max(it.right for it in items_c)
|
|
1928
|
+
col_boxes.append((mn, mx, (mn + mx) / 2.0))
|
|
1929
|
+
else:
|
|
1930
|
+
col_boxes.append((0, 0, 0))
|
|
1931
|
+
elif sh_clusters:
|
|
1932
|
+
num_cols = len(sh_clusters)
|
|
1933
|
+
col_boxes = []
|
|
1934
|
+
for cl in sh_clusters:
|
|
1935
|
+
mn = min(it.x for it in cl)
|
|
1936
|
+
mx = max(it.right for it in cl)
|
|
1937
|
+
col_boxes.append((mn, mx, (mn + mx) / 2.0))
|
|
1938
|
+
else:
|
|
1939
|
+
num_cols = max_cols
|
|
1940
|
+
col_boxes = [(0, 0, 0) for _ in range(num_cols)]
|
|
1941
|
+
|
|
1942
|
+
# Calculate exact geometric dividers between adjacent columns
|
|
1943
|
+
dividers = []
|
|
1944
|
+
for c in range(num_cols - 1):
|
|
1945
|
+
end_c = col_boxes[c][1]
|
|
1946
|
+
start_next = col_boxes[c+1][0]
|
|
1947
|
+
if start_next > end_c:
|
|
1948
|
+
div = (end_c + start_next) / 2.0
|
|
1949
|
+
else:
|
|
1950
|
+
div = (col_boxes[c][0] + start_next) / 2.0
|
|
1951
|
+
dividers.append(div)
|
|
1952
|
+
|
|
1953
|
+
col_bounds = []
|
|
1954
|
+
for c in range(num_cols):
|
|
1955
|
+
l = -float('inf') if c == 0 else dividers[c - 1]
|
|
1956
|
+
r = float('inf') if c == num_cols - 1 else dividers[c]
|
|
1957
|
+
col_bounds.append((l, r))
|
|
1958
|
+
|
|
1959
|
+
def get_col(it):
|
|
1960
|
+
center = (it.x + it.right) / 2.0
|
|
1961
|
+
return [ci for ci, (l, r) in enumerate(col_bounds) if l <= center < r][0]
|
|
1962
|
+
|
|
1963
|
+
def get_col_for_x(x):
|
|
1964
|
+
return [ci for ci, (l, r) in enumerate(col_bounds) if l <= x < r][0]
|
|
1965
|
+
|
|
1966
|
+
# Detect true multi-column banners in the top header row
|
|
1967
|
+
# A true banner has a physical text box [it.x, it.right] that spans across multiple column boundaries
|
|
1968
|
+
# or whose text width significantly exceeds the column width (> 1.1x col_w)
|
|
1969
|
+
banner_spans = {}
|
|
1970
|
+
if num_cols >= 2 and len(header_rows) >= 2:
|
|
1971
|
+
top_items = header_rows[0]['items']
|
|
1972
|
+
banner_candidates = []
|
|
1973
|
+
for it in top_items:
|
|
1974
|
+
c_start = get_col_for_x(it.x)
|
|
1975
|
+
c_end = get_col_for_x(it.right)
|
|
1976
|
+
col_w_start = col_bounds[c_start][1] - col_bounds[c_start][0]
|
|
1977
|
+
# True multi-column banner: physically spans across columns or width exceeds column width
|
|
1978
|
+
if (c_end > c_start and (it.right - it.x) > col_w_start * 1.1) or (c_end - c_start >= 1 and (it.right - col_bounds[c_start][1]) > col_w_start * 0.15):
|
|
1979
|
+
sub_cols = set(get_col(sub) for r in header_rows[1:] for sub in r['items'] if not (sub.right <= it.x or sub.x >= it.right))
|
|
1980
|
+
if len(sub_cols) >= 2 or c_end - c_start >= 2 or (c_end > c_start and (it.right - it.x) > 100):
|
|
1981
|
+
banner_candidates.append(it)
|
|
1982
|
+
if banner_candidates:
|
|
1983
|
+
sorted_banners = sorted(banner_candidates, key=lambda it: it.x)
|
|
1984
|
+
banner_centers = [(it.x + it.right) / 2.0 for it in sorted_banners]
|
|
1985
|
+
first_b_start_c = get_col_for_x(sorted_banners[0].x)
|
|
1986
|
+
if first_b_start_c > 0 and len(header_rows) >= 2:
|
|
1987
|
+
upper_rows = header_rows[:-1]
|
|
1988
|
+
for c_prev in range(first_b_start_c - 1, 0, -1):
|
|
1989
|
+
# Stop if column has an independent header in any row above the leaf header row
|
|
1990
|
+
has_upper = any(get_col(it) == c_prev for r in upper_rows for it in r['items'] if it not in banner_candidates)
|
|
1991
|
+
if has_upper:
|
|
1992
|
+
break
|
|
1993
|
+
first_b_start_c = c_prev
|
|
1994
|
+
for c in range(first_b_start_c, num_cols):
|
|
1995
|
+
col_c = (col_boxes[c][0] + col_boxes[c][1]) / 2.0
|
|
1996
|
+
best_idx = min(range(len(sorted_banners)), key=lambda bi: abs(col_c - banner_centers[bi]))
|
|
1997
|
+
b_cand = sorted_banners[best_idx]
|
|
1998
|
+
# Check if this column physically overlaps the banner
|
|
1999
|
+
c_overlaps = max(b_cand.x, col_bounds[c][0]) < min(b_cand.right, col_bounds[c][1])
|
|
2000
|
+
# Check column's own header items in lower rows
|
|
2001
|
+
col_sub_text = ' '.join(it[2] for r in header_rows[1:] for it in r['items'] if get_col(it) == c and it not in banner_candidates).strip()
|
|
2002
|
+
# If column does not physically overlap the banner and its own header is an independent metric contained in the banner, do not span
|
|
2003
|
+
if not c_overlaps and col_sub_text and len(col_sub_text) >= 3 and col_sub_text.lower() in b_cand[2].lower():
|
|
2004
|
+
if best_idx > 0 and col_c < banner_centers[best_idx]:
|
|
2005
|
+
b_cand = sorted_banners[best_idx - 1]
|
|
2006
|
+
c_overlaps = max(b_cand.x, col_bounds[c][0]) < min(b_cand.right, col_bounds[c][1])
|
|
2007
|
+
if not c_overlaps and col_sub_text and len(col_sub_text) >= 3 and col_sub_text.lower() in b_cand[2].lower():
|
|
2008
|
+
continue
|
|
2009
|
+
else:
|
|
2010
|
+
continue
|
|
2011
|
+
banner_spans.setdefault(b_cand, []).append(c)
|
|
2012
|
+
|
|
2013
|
+
# Build Header Matrix: map banners across their spanned columns, and standard items into physical columns
|
|
2014
|
+
h_matrix = [['' for _ in range(num_cols)] for _ in range(len(header_rows))]
|
|
2015
|
+
for b_it, cols in banner_spans.items():
|
|
2016
|
+
for c in cols:
|
|
2017
|
+
h_matrix[0][c] = b_it[2]
|
|
2018
|
+
|
|
2019
|
+
for r_idx in range(len(header_rows) - 1, -1, -1):
|
|
2020
|
+
for it in header_rows[r_idx]['items']:
|
|
2021
|
+
if it in banner_spans:
|
|
2022
|
+
continue
|
|
2023
|
+
c_idx = get_col(it)
|
|
2024
|
+
if h_matrix[r_idx][c_idx]:
|
|
2025
|
+
h_matrix[r_idx][c_idx] += ' ' + it[2]
|
|
2026
|
+
else:
|
|
2027
|
+
h_matrix[r_idx][c_idx] = it[2]
|
|
2028
|
+
|
|
2029
|
+
# Multi-tier sub-banner forward-fill across parent banner columns
|
|
2030
|
+
if len(header_rows) >= 3 and banner_spans:
|
|
2031
|
+
for b_it, cols in banner_spans.items():
|
|
2032
|
+
for r_i in range(1, len(header_rows) - 1):
|
|
2033
|
+
sub_items_in_cols = [it for it in header_rows[r_i]['items'] if any(col_bounds[c][0] <= (it.x + it.right)/2.0 < col_bounds[c][1] for c in cols)]
|
|
2034
|
+
if len(sub_items_in_cols) == 1:
|
|
2035
|
+
sub_b = sub_items_in_cols[0]
|
|
2036
|
+
# Verify that this is a true sub-tier banner overarching the columns below it,
|
|
2037
|
+
# i.e., lower header rows must contain child subheaders covering all columns in cols
|
|
2038
|
+
lower_cols = set(get_col(lit) for lr in range(r_i + 1, len(header_rows)) for lit in header_rows[lr]['items'] if any(col_bounds[c][0] <= (lit.x + lit.right)/2.0 < col_bounds[c][1] for c in cols))
|
|
2039
|
+
if lower_cols == set(cols):
|
|
2040
|
+
for c in cols:
|
|
2041
|
+
h_matrix[r_i][c] = sub_b[2]
|
|
2042
|
+
|
|
2043
|
+
# Multi-tier Header Merging
|
|
2044
|
+
if len(header_rows) >= 2:
|
|
2045
|
+
header_names = []
|
|
2046
|
+
banner_texts = set(b_it[2] for b_it in banner_spans.keys())
|
|
2047
|
+
for c in range(num_cols):
|
|
2048
|
+
parts = [h_matrix[r][c] for r in range(len(header_rows)) if h_matrix[r][c]]
|
|
2049
|
+
if not parts:
|
|
2050
|
+
header_names.append('-')
|
|
2051
|
+
elif len(parts) == 1:
|
|
2052
|
+
header_names.append(parts[0])
|
|
2053
|
+
else:
|
|
2054
|
+
top = parts[0]
|
|
2055
|
+
sub = ' '.join(parts[1:])
|
|
2056
|
+
if sub.lower() in top.lower():
|
|
2057
|
+
header_names.append(top)
|
|
2058
|
+
elif top.lower() in sub.lower():
|
|
2059
|
+
header_names.append(sub)
|
|
2060
|
+
elif top not in banner_texts:
|
|
2061
|
+
header_names.append(f'{top} {sub}')
|
|
2062
|
+
elif sub.startswith('('):
|
|
2063
|
+
header_names.append(f'{top} {sub}')
|
|
2064
|
+
elif '(' in sub and ')' in sub:
|
|
2065
|
+
header_names.append(f'{top} {sub}')
|
|
2066
|
+
else:
|
|
2067
|
+
header_names.append(f'{top} ({sub})')
|
|
2068
|
+
elif len(header_rows) == 1:
|
|
2069
|
+
header_names = [h_matrix[0][c] if h_matrix[0][c] else '-' for c in range(num_cols)]
|
|
2070
|
+
else:
|
|
2071
|
+
header_names = ['-' for _ in range(num_cols)]
|
|
2072
|
+
|
|
2073
|
+
if title and all(h in ('-', '') for h in header_names):
|
|
2074
|
+
header_names = [title] + [""] * (num_cols - 1)
|
|
2075
|
+
title = ""
|
|
2076
|
+
|
|
2077
|
+
# Map Data Rows into Grid
|
|
2078
|
+
grid = []
|
|
2079
|
+
for rg in data_rows:
|
|
2080
|
+
row_cells = [""] * num_cols
|
|
2081
|
+
for it in rg['items']:
|
|
2082
|
+
c_idx = get_col(it)
|
|
2083
|
+
if row_cells[c_idx]:
|
|
2084
|
+
row_cells[c_idx] += ' ' + it[2]
|
|
2085
|
+
else:
|
|
2086
|
+
row_cells[c_idx] = it[2]
|
|
2087
|
+
grid.append(row_cells)
|
|
2088
|
+
|
|
2089
|
+
separator = ["---"] * len(header_names)
|
|
2090
|
+
|
|
2091
|
+
md_lines = []
|
|
2092
|
+
if title:
|
|
2093
|
+
md_lines.append(f"### {title}\n")
|
|
2094
|
+
md_lines.append("| " + " | ".join(header_names) + " |")
|
|
2095
|
+
md_lines.append("| " + " | ".join(separator) + " |")
|
|
2096
|
+
|
|
2097
|
+
for r in grid:
|
|
2098
|
+
md_lines.append("| " + " | ".join(r) + " |")
|
|
2099
|
+
|
|
2100
|
+
if footnotes:
|
|
2101
|
+
md_lines.append("\n" + "\n".join(footnotes))
|
|
2102
|
+
|
|
2103
|
+
return "\n".join(md_lines)
|
|
2104
|
+
|
|
2105
|
+
def extract_emf_tables_and_text(media_bytes):
|
|
2106
|
+
"""
|
|
2107
|
+
Parses an EMF or WMF byte stream and returns formatted Markdown tables / text.
|
|
2108
|
+
Builds standard multi-column Markdown tables using geometric row/column clustering.
|
|
2109
|
+
"""
|
|
2110
|
+
items = extract_metafile_text_records(media_bytes)
|
|
2111
|
+
if not items:
|
|
2112
|
+
return ""
|
|
2113
|
+
|
|
2114
|
+
blocks = split_side_by_side_blocks(items)
|
|
2115
|
+
results = []
|
|
2116
|
+
for b in blocks:
|
|
2117
|
+
t = build_table_from_emf_items(b)
|
|
2118
|
+
if t.strip():
|
|
2119
|
+
results.append(t.strip())
|
|
2120
|
+
return "\n\n".join(results)
|
|
2121
|
+
|
|
2122
|
+
def extract_emf_content_per_slide(pptx_file, target_slide=None):
|
|
2123
|
+
"""
|
|
2124
|
+
Extracts all EMF/WMF vector tables and text associated with slides in a PPTX archive.
|
|
2125
|
+
Returns a dict mapping slide_num -> list of markdown strings ordered spatially.
|
|
2126
|
+
"""
|
|
2127
|
+
slide_emf_map = defaultdict(list)
|
|
2128
|
+
try:
|
|
2129
|
+
with zipfile.ZipFile(pptx_file, 'r') as pptx:
|
|
2130
|
+
for file in pptx.namelist():
|
|
2131
|
+
if file.startswith('ppt/slides/_rels/slide') and file.endswith('.xml.rels'):
|
|
2132
|
+
slide_num = int(file.split('/')[-1].replace('slide', '').replace('.xml.rels', ''))
|
|
2133
|
+
if target_slide is not None and slide_num != target_slide:
|
|
2134
|
+
continue
|
|
2135
|
+
|
|
2136
|
+
rid_to_target = {}
|
|
2137
|
+
with pptx.open(file) as rels_file:
|
|
2138
|
+
rels_tree = ET.parse(rels_file)
|
|
2139
|
+
rels_root = rels_tree.getroot()
|
|
2140
|
+
for rel in rels_root.findall('{http://schemas.openxmlformats.org/package/2006/relationships}Relationship'):
|
|
2141
|
+
rid = rel.attrib.get('Id')
|
|
2142
|
+
target = rel.attrib.get('Target', '')
|
|
2143
|
+
ext = target.lower().split('.')[-1]
|
|
2144
|
+
if ext in ('emf', 'wmf'):
|
|
2145
|
+
media_path = 'ppt/' + target.replace('../', '') if not target.startswith('ppt/') else target
|
|
2146
|
+
if media_path in pptx.namelist():
|
|
2147
|
+
rid_to_target[rid] = media_path
|
|
2148
|
+
|
|
2149
|
+
# Parse slide xml to sort metafiles by shape 2D reading order (Row-by-row: Y then X)
|
|
2150
|
+
slide_xml_name = f"ppt/slides/slide{slide_num}.xml"
|
|
2151
|
+
ordered_emfs = []
|
|
2152
|
+
if slide_xml_name in pptx.namelist():
|
|
2153
|
+
try:
|
|
2154
|
+
slide_root = ET.fromstring(pptx.read(slide_xml_name))
|
|
2155
|
+
for elem in slide_root.iter():
|
|
2156
|
+
if elem.tag.endswith('}graphicFrame') or elem.tag.endswith('}pic') or elem.tag.endswith('}sp'):
|
|
2157
|
+
blip = elem.find('.//{http://schemas.openxmlformats.org/drawingml/2006/main}blip')
|
|
2158
|
+
if blip is not None:
|
|
2159
|
+
embed_rid = blip.get('{http://schemas.openxmlformats.org/officeDocument/2006/relationships}embed')
|
|
2160
|
+
if embed_rid in rid_to_target:
|
|
2161
|
+
media_path = rid_to_target[embed_rid]
|
|
2162
|
+
xfrm = elem.find('.//{http://schemas.openxmlformats.org/drawingml/2006/main}xfrm')
|
|
2163
|
+
off = xfrm.find('{http://schemas.openxmlformats.org/drawingml/2006/main}off') if xfrm is not None else None
|
|
2164
|
+
x = int(off.get('x', 0)) if off is not None else 0
|
|
2165
|
+
y = int(off.get('y', 0)) if off is not None else 0
|
|
2166
|
+
if not any(item[2] == media_path for item in ordered_emfs):
|
|
2167
|
+
ordered_emfs.append((y, x, media_path))
|
|
2168
|
+
except Exception:
|
|
2169
|
+
pass
|
|
2170
|
+
|
|
2171
|
+
if ordered_emfs:
|
|
2172
|
+
# Cluster rows within ~15% slide height tolerance then order left to right
|
|
2173
|
+
ordered_emfs.sort(key=lambda item: (round(item[0] / 1000000), item[1]))
|
|
2174
|
+
for _, _, media_path in ordered_emfs:
|
|
2175
|
+
if media_path not in slide_emf_map[slide_num]:
|
|
2176
|
+
slide_emf_map[slide_num].append(media_path)
|
|
2177
|
+
else:
|
|
2178
|
+
for media_path in rid_to_target.values():
|
|
2179
|
+
if media_path not in slide_emf_map[slide_num]:
|
|
2180
|
+
slide_emf_map[slide_num].append(media_path)
|
|
2181
|
+
|
|
2182
|
+
slide_emf_data = {}
|
|
2183
|
+
for slide_num, emf_paths in slide_emf_map.items():
|
|
2184
|
+
slide_results = []
|
|
2185
|
+
for media_path in emf_paths:
|
|
2186
|
+
try:
|
|
2187
|
+
raw_data = pptx.read(media_path)
|
|
2188
|
+
content = extract_emf_tables_and_text(raw_data)
|
|
2189
|
+
if content.strip():
|
|
2190
|
+
slide_results.append(content.strip())
|
|
2191
|
+
except Exception:
|
|
2192
|
+
pass
|
|
2193
|
+
if slide_results:
|
|
2194
|
+
slide_emf_data[slide_num] = slide_results
|
|
2195
|
+
return slide_emf_data
|
|
2196
|
+
except Exception:
|
|
2197
|
+
return {}
|