super-document-processor 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- super_document_processor/__init__.py +28 -0
- super_document_processor/document_processors/process_docx.py +824 -0
- super_document_processor/document_processors/process_images.py +39 -0
- super_document_processor/document_processors/process_msg.py +62 -0
- super_document_processor/document_processors/process_pdf.py +715 -0
- super_document_processor/document_processors/process_plain_text.py +25 -0
- super_document_processor/document_processors/process_pptx.py +367 -0
- super_document_processor/main.py +826 -0
- super_document_processor/processor.py +135 -0
- super_document_processor/utils/aspose_slide_renderer.py +103 -0
- super_document_processor/utils/chunker.py +127 -0
- super_document_processor/utils/docx_image_utils.py +167 -0
- super_document_processor/utils/docx_xml_parser.py +73 -0
- super_document_processor/utils/optimizer.py +49 -0
- super_document_processor/utils/pdf_image_utils.py +90 -0
- super_document_processor/utils/pdf_table_utils.py +627 -0
- super_document_processor/utils/pptx_components_extract.py +2197 -0
- super_document_processor/utils/pptx_converter.py +170 -0
- super_document_processor/utils/vlm.py +176 -0
- super_document_processor-0.1.3.dist-info/METADATA +376 -0
- super_document_processor-0.1.3.dist-info/RECORD +25 -0
- super_document_processor-0.1.3.dist-info/WHEEL +5 -0
- super_document_processor-0.1.3.dist-info/entry_points.txt +2 -0
- super_document_processor-0.1.3.dist-info/licenses/LICENSE +21 -0
- super_document_processor-0.1.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,826 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
import json
|
|
4
|
+
import time
|
|
5
|
+
import uuid
|
|
6
|
+
import logging
|
|
7
|
+
import argparse
|
|
8
|
+
import tempfile
|
|
9
|
+
import threading
|
|
10
|
+
from typing import Optional, List, Dict, Any
|
|
11
|
+
|
|
12
|
+
# Ensure standard output and error support UTF-8 on Windows
|
|
13
|
+
if sys.platform == "win32":
|
|
14
|
+
try:
|
|
15
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
16
|
+
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
|
17
|
+
except AttributeError:
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
from processor import process_file, SUPPORTED_EXTENSIONS, IMAGE_EXTENSIONS, PORT, HOST
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger("super_document_processor")
|
|
23
|
+
|
|
24
|
+
def create_app():
|
|
25
|
+
"""Dynamically import FastAPI dependencies and build the REST API application."""
|
|
26
|
+
from fastapi import FastAPI, UploadFile, File, Request, HTTPException, Response
|
|
27
|
+
from fastapi.responses import Response
|
|
28
|
+
from fastapi.middleware.cors import CORSMiddleware
|
|
29
|
+
from pydantic import BaseModel
|
|
30
|
+
|
|
31
|
+
app = FastAPI(
|
|
32
|
+
title="Super Document Processor API",
|
|
33
|
+
description="High-fidelity multi-format document parsing and chunking REST API for RAG pipelines."
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
app.add_middleware(
|
|
37
|
+
CORSMiddleware,
|
|
38
|
+
allow_origins=["*"],
|
|
39
|
+
allow_credentials=True,
|
|
40
|
+
allow_methods=["*"],
|
|
41
|
+
allow_headers=["*"],
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
class HealthResponse(BaseModel):
|
|
45
|
+
status: str
|
|
46
|
+
supported_extensions_count: int
|
|
47
|
+
supported_extensions: List[str]
|
|
48
|
+
|
|
49
|
+
class ProcessPathRequest(BaseModel):
|
|
50
|
+
file_path: str
|
|
51
|
+
process_images: Optional[bool] = None
|
|
52
|
+
model: Optional[str] = None
|
|
53
|
+
|
|
54
|
+
class ProcessResponse(BaseModel):
|
|
55
|
+
filename: str
|
|
56
|
+
total_chunks: int
|
|
57
|
+
chunks: List[Dict[str, Any]]
|
|
58
|
+
|
|
59
|
+
class AsyncJobSubmitResponse(BaseModel):
|
|
60
|
+
job_id: str
|
|
61
|
+
filename: str
|
|
62
|
+
status: str
|
|
63
|
+
poll_url: str
|
|
64
|
+
|
|
65
|
+
class AsyncJobStatusResponse(BaseModel):
|
|
66
|
+
job_id: str
|
|
67
|
+
filename: str
|
|
68
|
+
status: str
|
|
69
|
+
progress: int
|
|
70
|
+
created_at: float
|
|
71
|
+
completed_at: Optional[float] = None
|
|
72
|
+
total_chunks: Optional[int] = None
|
|
73
|
+
chunks: Optional[List[Dict[str, Any]]] = None
|
|
74
|
+
error: Optional[str] = None
|
|
75
|
+
|
|
76
|
+
# Resolve shared jobs storage location (supports Docker volume /data and local dev)
|
|
77
|
+
jobs_dir = os.getenv("JOBS_DIR")
|
|
78
|
+
if not jobs_dir:
|
|
79
|
+
if os.path.exists("/data") and os.access("/data", os.W_OK):
|
|
80
|
+
jobs_dir = "/data/jobs"
|
|
81
|
+
else:
|
|
82
|
+
jobs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "jobs")
|
|
83
|
+
os.makedirs(jobs_dir, exist_ok=True)
|
|
84
|
+
temp_uploads_dir = os.path.join(os.path.dirname(jobs_dir), "temp_uploads")
|
|
85
|
+
os.makedirs(temp_uploads_dir, exist_ok=True)
|
|
86
|
+
|
|
87
|
+
def cleanup_old_jobs(max_age_seconds: int = 900):
|
|
88
|
+
"""Auto-purge abandoned job files older than 15 minutes to prevent disk bloat."""
|
|
89
|
+
try:
|
|
90
|
+
now = time.time()
|
|
91
|
+
for fname in os.listdir(jobs_dir):
|
|
92
|
+
if fname.endswith(".json"):
|
|
93
|
+
fpath = os.path.join(jobs_dir, fname)
|
|
94
|
+
if os.path.isfile(fpath) and (now - os.path.getmtime(fpath)) > max_age_seconds:
|
|
95
|
+
try:
|
|
96
|
+
os.remove(fpath)
|
|
97
|
+
except Exception:
|
|
98
|
+
pass
|
|
99
|
+
except Exception:
|
|
100
|
+
pass
|
|
101
|
+
|
|
102
|
+
@app.get("/", tags=["General"])
|
|
103
|
+
def root():
|
|
104
|
+
return {
|
|
105
|
+
"service": "Super Document Processor API",
|
|
106
|
+
"health": "/health",
|
|
107
|
+
"docs": "/docs",
|
|
108
|
+
"process_sync": "/process",
|
|
109
|
+
"process_async": "/process-async",
|
|
110
|
+
"job_poll": "/jobs/{job_id}",
|
|
111
|
+
"convert_pptx": "/convert/pptx",
|
|
112
|
+
"supported_extensions_count": len(SUPPORTED_EXTENSIONS)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
@app.get("/health", response_model=HealthResponse, tags=["General"])
|
|
116
|
+
def health_check():
|
|
117
|
+
return {
|
|
118
|
+
"status": "healthy",
|
|
119
|
+
"supported_extensions_count": len(SUPPORTED_EXTENSIONS),
|
|
120
|
+
"supported_extensions": sorted(list(SUPPORTED_EXTENSIONS))
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
@app.post(
|
|
124
|
+
"/process",
|
|
125
|
+
response_model=ProcessResponse,
|
|
126
|
+
tags=["Processing"],
|
|
127
|
+
summary="Process Document",
|
|
128
|
+
description="Unified document processing endpoint. Provide either an uploaded file (`file`) as multipart/form-data OR a server/volume file path (`file_path` as JSON or form field). Providing both or neither will return a 400 error."
|
|
129
|
+
)
|
|
130
|
+
async def process_document(request: Request):
|
|
131
|
+
content_type = request.headers.get("content-type", "").lower()
|
|
132
|
+
file_path = None
|
|
133
|
+
upload_file = None
|
|
134
|
+
process_images = None
|
|
135
|
+
model = None
|
|
136
|
+
|
|
137
|
+
if "application/json" in content_type:
|
|
138
|
+
try:
|
|
139
|
+
body = await request.json()
|
|
140
|
+
except Exception:
|
|
141
|
+
raise HTTPException(status_code=400, detail="Invalid JSON payload.")
|
|
142
|
+
file_path = body.get("file_path")
|
|
143
|
+
process_images = body.get("process_images")
|
|
144
|
+
model = body.get("model")
|
|
145
|
+
else:
|
|
146
|
+
try:
|
|
147
|
+
form = await request.form()
|
|
148
|
+
except Exception:
|
|
149
|
+
form = {}
|
|
150
|
+
raw_file = form.get("file")
|
|
151
|
+
if raw_file is not None and hasattr(raw_file, "filename") and raw_file.filename:
|
|
152
|
+
upload_file = raw_file
|
|
153
|
+
|
|
154
|
+
raw_path = form.get("file_path")
|
|
155
|
+
if raw_path and str(raw_path).strip():
|
|
156
|
+
file_path = str(raw_path).strip()
|
|
157
|
+
|
|
158
|
+
form_proc_img = form.get("process_images")
|
|
159
|
+
if form_proc_img is not None:
|
|
160
|
+
if isinstance(form_proc_img, str):
|
|
161
|
+
process_images = form_proc_img.lower() in ("true", "1", "yes")
|
|
162
|
+
else:
|
|
163
|
+
process_images = bool(form_proc_img)
|
|
164
|
+
form_model = form.get("model")
|
|
165
|
+
if form_model:
|
|
166
|
+
model = str(form_model)
|
|
167
|
+
|
|
168
|
+
# Fallback to query parameters if not specified in body/form
|
|
169
|
+
if process_images is None and "process_images" in request.query_params:
|
|
170
|
+
process_images = request.query_params["process_images"].lower() in ("true", "1", "yes")
|
|
171
|
+
if not model and "model" in request.query_params:
|
|
172
|
+
model = request.query_params["model"]
|
|
173
|
+
if not file_path and not upload_file and "file_path" in request.query_params:
|
|
174
|
+
file_path = request.query_params["file_path"]
|
|
175
|
+
|
|
176
|
+
# Validate mutual exclusivity: cannot provide both
|
|
177
|
+
if upload_file and file_path:
|
|
178
|
+
raise HTTPException(
|
|
179
|
+
status_code=400,
|
|
180
|
+
detail="Cannot provide both an uploaded 'file' and a 'file_path'. Please provide only one."
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
# Validate presence: must provide at least one
|
|
184
|
+
if not upload_file and not file_path:
|
|
185
|
+
raise HTTPException(
|
|
186
|
+
status_code=400,
|
|
187
|
+
detail="Either an uploaded 'file' or a 'file_path' must be provided."
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
from starlette.concurrency import run_in_threadpool
|
|
191
|
+
|
|
192
|
+
if upload_file:
|
|
193
|
+
if not upload_file.filename:
|
|
194
|
+
raise HTTPException(status_code=400, detail="Uploaded file must have a filename.")
|
|
195
|
+
ext = os.path.splitext(upload_file.filename)[1].lower()
|
|
196
|
+
if ext not in SUPPORTED_EXTENSIONS:
|
|
197
|
+
raise HTTPException(
|
|
198
|
+
status_code=400,
|
|
199
|
+
detail=f"Unsupported file type '{ext}'. Supported: {sorted(list(SUPPORTED_EXTENSIONS))}"
|
|
200
|
+
)
|
|
201
|
+
with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp:
|
|
202
|
+
tmp_path = tmp.name
|
|
203
|
+
content = await upload_file.read()
|
|
204
|
+
tmp.write(content)
|
|
205
|
+
|
|
206
|
+
try:
|
|
207
|
+
chunks = await run_in_threadpool(
|
|
208
|
+
process_file,
|
|
209
|
+
tmp_path,
|
|
210
|
+
original_filename=upload_file.filename,
|
|
211
|
+
process_images=process_images,
|
|
212
|
+
model=model
|
|
213
|
+
)
|
|
214
|
+
return {
|
|
215
|
+
"filename": upload_file.filename,
|
|
216
|
+
"total_chunks": len(chunks),
|
|
217
|
+
"chunks": chunks
|
|
218
|
+
}
|
|
219
|
+
except ValueError as e:
|
|
220
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
221
|
+
except Exception as e:
|
|
222
|
+
raise HTTPException(status_code=500, detail=f"Processing failed: {str(e)}")
|
|
223
|
+
finally:
|
|
224
|
+
if os.path.exists(tmp_path):
|
|
225
|
+
try:
|
|
226
|
+
os.remove(tmp_path)
|
|
227
|
+
except Exception:
|
|
228
|
+
pass
|
|
229
|
+
else:
|
|
230
|
+
if not os.path.exists(file_path):
|
|
231
|
+
raise HTTPException(status_code=404, detail=f"File not found: {file_path}")
|
|
232
|
+
ext = os.path.splitext(file_path)[1].lower()
|
|
233
|
+
if ext not in SUPPORTED_EXTENSIONS:
|
|
234
|
+
raise HTTPException(
|
|
235
|
+
status_code=400,
|
|
236
|
+
detail=f"Unsupported file type '{ext}'. Supported: {sorted(list(SUPPORTED_EXTENSIONS))}"
|
|
237
|
+
)
|
|
238
|
+
try:
|
|
239
|
+
chunks = await run_in_threadpool(
|
|
240
|
+
process_file,
|
|
241
|
+
file_path,
|
|
242
|
+
process_images=process_images,
|
|
243
|
+
model=model
|
|
244
|
+
)
|
|
245
|
+
return {
|
|
246
|
+
"filename": os.path.basename(file_path),
|
|
247
|
+
"total_chunks": len(chunks),
|
|
248
|
+
"chunks": chunks
|
|
249
|
+
}
|
|
250
|
+
except ValueError as e:
|
|
251
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
252
|
+
except Exception as e:
|
|
253
|
+
raise HTTPException(status_code=500, detail=f"Processing failed: {str(e)}")
|
|
254
|
+
|
|
255
|
+
@app.post("/process-path", response_model=ProcessResponse, tags=["Processing"], include_in_schema=False)
|
|
256
|
+
async def process_local_document_path_legacy(request: Request):
|
|
257
|
+
"""Backward-compatible alias for /process using file_path."""
|
|
258
|
+
return await process_document(request)
|
|
259
|
+
|
|
260
|
+
@app.post("/process-async", response_model=AsyncJobSubmitResponse, tags=["Async Processing"])
|
|
261
|
+
async def process_document_async(
|
|
262
|
+
file: UploadFile = File(...),
|
|
263
|
+
process_images: Optional[bool] = None,
|
|
264
|
+
model: Optional[str] = None
|
|
265
|
+
):
|
|
266
|
+
if not file.filename:
|
|
267
|
+
raise HTTPException(status_code=400, detail="Uploaded file must have a filename.")
|
|
268
|
+
|
|
269
|
+
ext = os.path.splitext(file.filename)[1].lower()
|
|
270
|
+
if ext not in SUPPORTED_EXTENSIONS:
|
|
271
|
+
raise HTTPException(
|
|
272
|
+
status_code=400,
|
|
273
|
+
detail=f"Unsupported file type '{ext}'. Supported: {sorted(list(SUPPORTED_EXTENSIONS))}"
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
job_id = str(uuid.uuid4())
|
|
277
|
+
created_at = time.time()
|
|
278
|
+
job_file = os.path.join(jobs_dir, f"{job_id}.json")
|
|
279
|
+
upload_path = os.path.join(temp_uploads_dir, f"{job_id}_{file.filename}")
|
|
280
|
+
|
|
281
|
+
# Save uploaded file to shared storage
|
|
282
|
+
with open(upload_path, "wb") as f:
|
|
283
|
+
content = await file.read()
|
|
284
|
+
f.write(content)
|
|
285
|
+
|
|
286
|
+
# Initialize job metadata on shared storage
|
|
287
|
+
initial_payload = {
|
|
288
|
+
"job_id": job_id,
|
|
289
|
+
"filename": file.filename,
|
|
290
|
+
"status": "processing",
|
|
291
|
+
"progress": 0,
|
|
292
|
+
"created_at": created_at,
|
|
293
|
+
"completed_at": None,
|
|
294
|
+
"total_chunks": None,
|
|
295
|
+
"chunks": None,
|
|
296
|
+
"error": None
|
|
297
|
+
}
|
|
298
|
+
with open(job_file, "w", encoding="utf-8") as f:
|
|
299
|
+
json.dump(initial_payload, f, ensure_ascii=False)
|
|
300
|
+
|
|
301
|
+
def _worker():
|
|
302
|
+
try:
|
|
303
|
+
chunks = process_file(
|
|
304
|
+
upload_path,
|
|
305
|
+
original_filename=file.filename,
|
|
306
|
+
process_images=process_images,
|
|
307
|
+
model=model
|
|
308
|
+
)
|
|
309
|
+
done_payload = {
|
|
310
|
+
"job_id": job_id,
|
|
311
|
+
"filename": file.filename,
|
|
312
|
+
"status": "completed",
|
|
313
|
+
"progress": 100,
|
|
314
|
+
"created_at": created_at,
|
|
315
|
+
"completed_at": time.time(),
|
|
316
|
+
"total_chunks": len(chunks),
|
|
317
|
+
"chunks": chunks,
|
|
318
|
+
"error": None
|
|
319
|
+
}
|
|
320
|
+
tmp_job = f"{job_file}.tmp"
|
|
321
|
+
with open(tmp_job, "w", encoding="utf-8") as f:
|
|
322
|
+
json.dump(done_payload, f, ensure_ascii=False)
|
|
323
|
+
os.replace(tmp_job, job_file)
|
|
324
|
+
except Exception as e:
|
|
325
|
+
err_payload = {
|
|
326
|
+
"job_id": job_id,
|
|
327
|
+
"filename": file.filename,
|
|
328
|
+
"status": "failed",
|
|
329
|
+
"progress": 0,
|
|
330
|
+
"created_at": created_at,
|
|
331
|
+
"completed_at": time.time(),
|
|
332
|
+
"total_chunks": 0,
|
|
333
|
+
"chunks": None,
|
|
334
|
+
"error": str(e)
|
|
335
|
+
}
|
|
336
|
+
tmp_job = f"{job_file}.tmp"
|
|
337
|
+
with open(tmp_job, "w", encoding="utf-8") as f:
|
|
338
|
+
json.dump(err_payload, f, ensure_ascii=False)
|
|
339
|
+
os.replace(tmp_job, job_file)
|
|
340
|
+
finally:
|
|
341
|
+
if os.path.exists(upload_path):
|
|
342
|
+
try:
|
|
343
|
+
os.remove(upload_path)
|
|
344
|
+
except Exception:
|
|
345
|
+
pass
|
|
346
|
+
cleanup_old_jobs()
|
|
347
|
+
|
|
348
|
+
threading.Thread(target=_worker, daemon=True).start()
|
|
349
|
+
|
|
350
|
+
return {
|
|
351
|
+
"job_id": job_id,
|
|
352
|
+
"filename": file.filename,
|
|
353
|
+
"status": "processing",
|
|
354
|
+
"poll_url": f"/jobs/{job_id}"
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
@app.get("/jobs/{job_id}", response_model=AsyncJobStatusResponse, tags=["Async Processing"])
|
|
358
|
+
def get_job_status(job_id: str):
|
|
359
|
+
job_file = os.path.join(jobs_dir, f"{job_id}.json")
|
|
360
|
+
if not os.path.exists(job_file):
|
|
361
|
+
raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.")
|
|
362
|
+
try:
|
|
363
|
+
with open(job_file, "r", encoding="utf-8") as f:
|
|
364
|
+
data = json.load(f)
|
|
365
|
+
|
|
366
|
+
# Once the client retrieves the final result (completed or failed), delete the file immediately
|
|
367
|
+
if data.get("status") in ("completed", "failed"):
|
|
368
|
+
try:
|
|
369
|
+
os.remove(job_file)
|
|
370
|
+
except Exception:
|
|
371
|
+
pass
|
|
372
|
+
|
|
373
|
+
return data
|
|
374
|
+
except HTTPException:
|
|
375
|
+
raise
|
|
376
|
+
except Exception as e:
|
|
377
|
+
raise HTTPException(status_code=500, detail=f"Failed to read job state: {str(e)}")
|
|
378
|
+
|
|
379
|
+
@app.post(
|
|
380
|
+
"/convert/pptx",
|
|
381
|
+
tags=["Conversion"],
|
|
382
|
+
summary="Convert PPTX to SVG or PNG Images",
|
|
383
|
+
description="Renders slides of a PowerPoint (.pptx) file to clean SVG or PNG images with Aspose evaluation watermarks stripped. Supports single slide streaming, JSON base64 payloads, or zip archive download."
|
|
384
|
+
)
|
|
385
|
+
async def convert_pptx_endpoint(request: Request):
|
|
386
|
+
content_type = request.headers.get("content-type", "").lower()
|
|
387
|
+
file_path = None
|
|
388
|
+
upload_file = None
|
|
389
|
+
fmt = "svg"
|
|
390
|
+
slide = None
|
|
391
|
+
scale = 2.0
|
|
392
|
+
as_zip = False
|
|
393
|
+
response_type = None
|
|
394
|
+
|
|
395
|
+
if "application/json" in content_type:
|
|
396
|
+
try:
|
|
397
|
+
body = await request.json()
|
|
398
|
+
except Exception:
|
|
399
|
+
raise HTTPException(status_code=400, detail="Invalid JSON payload.")
|
|
400
|
+
file_path = body.get("file_path")
|
|
401
|
+
fmt = body.get("format", "svg")
|
|
402
|
+
slide = body.get("slide")
|
|
403
|
+
try:
|
|
404
|
+
scale = float(body.get("scale", 2.0))
|
|
405
|
+
except (ValueError, TypeError):
|
|
406
|
+
scale = 2.0
|
|
407
|
+
as_zip = bool(body.get("as_zip", False))
|
|
408
|
+
response_type = body.get("response_type")
|
|
409
|
+
else:
|
|
410
|
+
try:
|
|
411
|
+
form = await request.form()
|
|
412
|
+
except Exception:
|
|
413
|
+
form = {}
|
|
414
|
+
raw_file = form.get("file")
|
|
415
|
+
if raw_file is not None and hasattr(raw_file, "filename") and raw_file.filename:
|
|
416
|
+
upload_file = raw_file
|
|
417
|
+
|
|
418
|
+
raw_path = form.get("file_path")
|
|
419
|
+
if raw_path and str(raw_path).strip():
|
|
420
|
+
file_path = str(raw_path).strip()
|
|
421
|
+
|
|
422
|
+
if form.get("format"):
|
|
423
|
+
fmt = str(form.get("format"))
|
|
424
|
+
if form.get("slide"):
|
|
425
|
+
try:
|
|
426
|
+
slide = int(form.get("slide"))
|
|
427
|
+
except (ValueError, TypeError):
|
|
428
|
+
pass
|
|
429
|
+
if form.get("scale"):
|
|
430
|
+
try:
|
|
431
|
+
scale = float(form.get("scale"))
|
|
432
|
+
except (ValueError, TypeError):
|
|
433
|
+
pass
|
|
434
|
+
if form.get("as_zip") is not None:
|
|
435
|
+
as_zip = str(form.get("as_zip")).lower() in ("true", "1", "yes")
|
|
436
|
+
if form.get("response_type"):
|
|
437
|
+
response_type = str(form.get("response_type"))
|
|
438
|
+
|
|
439
|
+
# Fallback to query parameters
|
|
440
|
+
if "format" in request.query_params:
|
|
441
|
+
fmt = request.query_params["format"]
|
|
442
|
+
if "slide" in request.query_params:
|
|
443
|
+
try:
|
|
444
|
+
slide = int(request.query_params["slide"])
|
|
445
|
+
except (ValueError, TypeError):
|
|
446
|
+
pass
|
|
447
|
+
if "scale" in request.query_params:
|
|
448
|
+
try:
|
|
449
|
+
scale = float(request.query_params["scale"])
|
|
450
|
+
except (ValueError, TypeError):
|
|
451
|
+
pass
|
|
452
|
+
if "as_zip" in request.query_params:
|
|
453
|
+
as_zip = request.query_params["as_zip"].lower() in ("true", "1", "yes")
|
|
454
|
+
if "response_type" in request.query_params:
|
|
455
|
+
response_type = request.query_params["response_type"]
|
|
456
|
+
if not file_path and not upload_file and "file_path" in request.query_params:
|
|
457
|
+
file_path = request.query_params["file_path"]
|
|
458
|
+
|
|
459
|
+
if upload_file and file_path:
|
|
460
|
+
raise HTTPException(
|
|
461
|
+
status_code=400,
|
|
462
|
+
detail="Cannot provide both an uploaded 'file' and a 'file_path'. Please provide only one."
|
|
463
|
+
)
|
|
464
|
+
if not upload_file and not file_path:
|
|
465
|
+
raise HTTPException(
|
|
466
|
+
status_code=400,
|
|
467
|
+
detail="Either an uploaded 'file' or a 'file_path' must be provided."
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
from starlette.concurrency import run_in_threadpool
|
|
471
|
+
try:
|
|
472
|
+
from super_document_processor.utils.pptx_converter import convert_pptx, create_pptx_render_zip
|
|
473
|
+
except ImportError:
|
|
474
|
+
from utils.pptx_converter import convert_pptx, create_pptx_render_zip
|
|
475
|
+
|
|
476
|
+
norm_fmt = "png" if fmt.lower() in ("png", "img", "image") else "svg"
|
|
477
|
+
|
|
478
|
+
pptx_source = None
|
|
479
|
+
filename = "presentation.pptx"
|
|
480
|
+
|
|
481
|
+
if upload_file:
|
|
482
|
+
if not upload_file.filename or not upload_file.filename.lower().endswith(".pptx"):
|
|
483
|
+
raise HTTPException(status_code=400, detail="Uploaded file must be a .pptx file.")
|
|
484
|
+
filename = upload_file.filename
|
|
485
|
+
content = await upload_file.read()
|
|
486
|
+
pptx_source = content
|
|
487
|
+
else:
|
|
488
|
+
if not os.path.exists(file_path):
|
|
489
|
+
raise HTTPException(status_code=404, detail=f"File not found: {file_path}")
|
|
490
|
+
if not file_path.lower().endswith(".pptx"):
|
|
491
|
+
raise HTTPException(status_code=400, detail="Target file must be a .pptx file.")
|
|
492
|
+
filename = os.path.basename(file_path)
|
|
493
|
+
pptx_source = file_path
|
|
494
|
+
|
|
495
|
+
try:
|
|
496
|
+
# 1. Download all as ZIP archive if requested
|
|
497
|
+
if as_zip:
|
|
498
|
+
zip_bytes = await run_in_threadpool(create_pptx_render_zip, pptx_source, format=norm_fmt, scale=scale)
|
|
499
|
+
base_name = os.path.splitext(filename)[0]
|
|
500
|
+
return Response(
|
|
501
|
+
content=zip_bytes,
|
|
502
|
+
media_type="application/zip",
|
|
503
|
+
headers={"Content-Disposition": f'attachment; filename="{base_name}_{norm_fmt}.zip"'}
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
# 2. Convert PPTX
|
|
507
|
+
render_res = await run_in_threadpool(
|
|
508
|
+
convert_pptx,
|
|
509
|
+
pptx_source,
|
|
510
|
+
format=norm_fmt,
|
|
511
|
+
slide=slide,
|
|
512
|
+
scale=scale
|
|
513
|
+
)
|
|
514
|
+
|
|
515
|
+
# 3. If single slide requested and binary direct streaming is selected (default for single slide unless response_type=json)
|
|
516
|
+
if slide is not None and response_type != "json":
|
|
517
|
+
media_type = "image/svg+xml" if norm_fmt == "svg" else "image/png"
|
|
518
|
+
return Response(content=render_res["raw_bytes"], media_type=media_type)
|
|
519
|
+
|
|
520
|
+
# 4. Return clean JSON response
|
|
521
|
+
if "slides" in render_res:
|
|
522
|
+
clean_slides = []
|
|
523
|
+
for s in render_res["slides"]:
|
|
524
|
+
item = {"slide": s["slide"], "format": s["format"]}
|
|
525
|
+
if s["format"] == "svg":
|
|
526
|
+
item["svg"] = s["content"]
|
|
527
|
+
else:
|
|
528
|
+
item["png_base64"] = s["content_base64"]
|
|
529
|
+
clean_slides.append(item)
|
|
530
|
+
return {
|
|
531
|
+
"filename": filename,
|
|
532
|
+
"format": render_res["format"],
|
|
533
|
+
"total_slides": render_res["total_slides"],
|
|
534
|
+
"slides": clean_slides
|
|
535
|
+
}
|
|
536
|
+
else:
|
|
537
|
+
resp = {
|
|
538
|
+
"filename": filename,
|
|
539
|
+
"slide": render_res["slide"],
|
|
540
|
+
"total_slides": render_res["total_slides"],
|
|
541
|
+
"format": render_res["format"]
|
|
542
|
+
}
|
|
543
|
+
if render_res["format"] == "svg":
|
|
544
|
+
resp["svg"] = render_res["content"]
|
|
545
|
+
else:
|
|
546
|
+
resp["png_base64"] = render_res["content_base64"]
|
|
547
|
+
return resp
|
|
548
|
+
|
|
549
|
+
except ValueError as e:
|
|
550
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
551
|
+
except Exception as e:
|
|
552
|
+
raise HTTPException(status_code=500, detail=f"PPTX conversion failed: {str(e)}")
|
|
553
|
+
|
|
554
|
+
return app
|
|
555
|
+
|
|
556
|
+
def start_server(host: Optional[str] = None, port: Optional[int] = None):
|
|
557
|
+
"""Dynamically launch the FastAPI server via uvicorn."""
|
|
558
|
+
import uvicorn
|
|
559
|
+
h = host or HOST
|
|
560
|
+
p = port or PORT
|
|
561
|
+
logger.info(f"Starting Super Document Processor API on http://{h}:{p}")
|
|
562
|
+
app = create_app()
|
|
563
|
+
uvicorn.run(app, host=h, port=p)
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def __getattr__(name: str):
|
|
567
|
+
"""Module-level attribute getter supporting ASGI servers (e.g. uvicorn main:app)."""
|
|
568
|
+
if name == "app":
|
|
569
|
+
global _app
|
|
570
|
+
if "_app" not in globals():
|
|
571
|
+
_app = create_app()
|
|
572
|
+
return _app
|
|
573
|
+
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def main():
|
|
577
|
+
parser = argparse.ArgumentParser(
|
|
578
|
+
description="Super Document Processor - High-fidelity multi-format document parser & chunker for RAG"
|
|
579
|
+
)
|
|
580
|
+
parser.add_argument(
|
|
581
|
+
"file",
|
|
582
|
+
nargs="?",
|
|
583
|
+
help="Path to the document to process (PDF, DOCX, PPTX, XLSX, images, plain text, etc.)"
|
|
584
|
+
)
|
|
585
|
+
parser.add_argument(
|
|
586
|
+
"--serve",
|
|
587
|
+
"-s",
|
|
588
|
+
action="store_true",
|
|
589
|
+
help="Start the FastAPI HTTP REST API server"
|
|
590
|
+
)
|
|
591
|
+
parser.add_argument(
|
|
592
|
+
"--port",
|
|
593
|
+
"-p",
|
|
594
|
+
type=int,
|
|
595
|
+
default=PORT,
|
|
596
|
+
help=f"Port to listen on when in server mode (default: env PORT, if specified)"
|
|
597
|
+
)
|
|
598
|
+
parser.add_argument(
|
|
599
|
+
"--host",
|
|
600
|
+
type=str,
|
|
601
|
+
default=HOST,
|
|
602
|
+
help="Host address to bind to when in server mode (default: 0.0.0.0)"
|
|
603
|
+
)
|
|
604
|
+
parser.add_argument(
|
|
605
|
+
"--json",
|
|
606
|
+
action="store_true",
|
|
607
|
+
help="Output raw JSON array of chunks"
|
|
608
|
+
)
|
|
609
|
+
parser.add_argument(
|
|
610
|
+
"--output",
|
|
611
|
+
"-o",
|
|
612
|
+
type=str,
|
|
613
|
+
default=None,
|
|
614
|
+
help="Path to save output (JSON or text depending on --json flag)"
|
|
615
|
+
)
|
|
616
|
+
parser.add_argument(
|
|
617
|
+
"--process-images",
|
|
618
|
+
dest="process_images",
|
|
619
|
+
action="store_true",
|
|
620
|
+
default=False,
|
|
621
|
+
help="Enable AI vision processing of embedded images and tables (default: False)"
|
|
622
|
+
)
|
|
623
|
+
parser.add_argument(
|
|
624
|
+
"--no-process-images",
|
|
625
|
+
dest="process_images",
|
|
626
|
+
action="store_false",
|
|
627
|
+
help="Explicitly disable AI vision processing of embedded images and tables"
|
|
628
|
+
)
|
|
629
|
+
parser.add_argument(
|
|
630
|
+
"--model",
|
|
631
|
+
"-m",
|
|
632
|
+
type=str,
|
|
633
|
+
default=None,
|
|
634
|
+
help="Vision model name for image and table extraction (e.g. gpt-4o, google-gemini-3-flash)"
|
|
635
|
+
)
|
|
636
|
+
parser.add_argument(
|
|
637
|
+
"--convert-pptx",
|
|
638
|
+
"-c",
|
|
639
|
+
dest="convert_pptx",
|
|
640
|
+
action="store_true",
|
|
641
|
+
default=False,
|
|
642
|
+
help="Convert PPTX slides to clean SVG or PNG images instead of text chunking"
|
|
643
|
+
)
|
|
644
|
+
parser.add_argument(
|
|
645
|
+
"--format",
|
|
646
|
+
"-f",
|
|
647
|
+
dest="format",
|
|
648
|
+
type=str,
|
|
649
|
+
default="svg",
|
|
650
|
+
choices=["svg", "png"],
|
|
651
|
+
help="Target format for PPTX slide conversion (svg or png, default: svg)"
|
|
652
|
+
)
|
|
653
|
+
parser.add_argument(
|
|
654
|
+
"--slide",
|
|
655
|
+
dest="slide",
|
|
656
|
+
type=int,
|
|
657
|
+
default=None,
|
|
658
|
+
help="1-based slide index to convert (default: all slides)"
|
|
659
|
+
)
|
|
660
|
+
parser.add_argument(
|
|
661
|
+
"--scale",
|
|
662
|
+
dest="scale",
|
|
663
|
+
type=float,
|
|
664
|
+
default=2.0,
|
|
665
|
+
help="Image resolution scale multiplier for PPTX rendering (default: 2.0)"
|
|
666
|
+
)
|
|
667
|
+
parser.add_argument(
|
|
668
|
+
"--as-zip",
|
|
669
|
+
dest="as_zip",
|
|
670
|
+
action="store_true",
|
|
671
|
+
default=False,
|
|
672
|
+
help="Bundle all converted slides into a single .zip archive file"
|
|
673
|
+
)
|
|
674
|
+
|
|
675
|
+
args = parser.parse_args()
|
|
676
|
+
|
|
677
|
+
# Mode 1: Server Mode
|
|
678
|
+
if args.serve:
|
|
679
|
+
start_server(host=args.host, port=args.port)
|
|
680
|
+
return
|
|
681
|
+
|
|
682
|
+
# Mode 2: No arguments -> print help
|
|
683
|
+
if not args.file:
|
|
684
|
+
parser.print_help()
|
|
685
|
+
print("\nSupported file extensions:")
|
|
686
|
+
print(", ".join(sorted(SUPPORTED_EXTENSIONS)))
|
|
687
|
+
sys.exit(0)
|
|
688
|
+
|
|
689
|
+
# Mode 3: File validation
|
|
690
|
+
file_path = args.file
|
|
691
|
+
if not os.path.exists(file_path):
|
|
692
|
+
print(f"Error: File not found: {file_path}", file=sys.stderr)
|
|
693
|
+
sys.exit(1)
|
|
694
|
+
|
|
695
|
+
file_ext = os.path.splitext(file_path)[1].lower()
|
|
696
|
+
|
|
697
|
+
# Mode 4: CLI PPTX Conversion Mode
|
|
698
|
+
if args.convert_pptx:
|
|
699
|
+
if file_ext != ".pptx":
|
|
700
|
+
print(f"Error: --convert-pptx only supports .pptx files, got '{file_ext}'.", file=sys.stderr)
|
|
701
|
+
sys.exit(1)
|
|
702
|
+
|
|
703
|
+
try:
|
|
704
|
+
from super_document_processor.utils.pptx_converter import convert_pptx, create_pptx_render_zip
|
|
705
|
+
except ImportError:
|
|
706
|
+
from utils.pptx_converter import convert_pptx, create_pptx_render_zip
|
|
707
|
+
|
|
708
|
+
try:
|
|
709
|
+
norm_fmt = "png" if args.format.lower() in ("png", "img", "image") else "svg"
|
|
710
|
+
if args.as_zip:
|
|
711
|
+
zip_bytes = create_pptx_render_zip(file_path, format=norm_fmt, scale=args.scale)
|
|
712
|
+
out_path = args.output or f"{os.path.splitext(file_path)[0]}_{norm_fmt}.zip"
|
|
713
|
+
with open(out_path, "wb") as f:
|
|
714
|
+
f.write(zip_bytes)
|
|
715
|
+
print(f"Saved {norm_fmt.upper()} ZIP archive to: {out_path}")
|
|
716
|
+
return
|
|
717
|
+
|
|
718
|
+
render_res = convert_pptx(file_path, format=norm_fmt, slide=args.slide, scale=args.scale)
|
|
719
|
+
|
|
720
|
+
if args.json:
|
|
721
|
+
clean_slides = []
|
|
722
|
+
if "slides" in render_res:
|
|
723
|
+
for s in render_res["slides"]:
|
|
724
|
+
item = {"slide": s["slide"], "format": s["format"]}
|
|
725
|
+
if s["format"] == "svg":
|
|
726
|
+
item["svg"] = s["content"]
|
|
727
|
+
else:
|
|
728
|
+
item["png_base64"] = s["content_base64"]
|
|
729
|
+
clean_slides.append(item)
|
|
730
|
+
out_json = {
|
|
731
|
+
"filename": os.path.basename(file_path),
|
|
732
|
+
"format": render_res["format"],
|
|
733
|
+
"total_slides": render_res["total_slides"],
|
|
734
|
+
"slides": clean_slides
|
|
735
|
+
}
|
|
736
|
+
else:
|
|
737
|
+
out_json = {
|
|
738
|
+
"filename": os.path.basename(file_path),
|
|
739
|
+
"slide": render_res["slide"],
|
|
740
|
+
"total_slides": render_res["total_slides"],
|
|
741
|
+
"format": render_res["format"]
|
|
742
|
+
}
|
|
743
|
+
if render_res["format"] == "svg":
|
|
744
|
+
out_json["svg"] = render_res["content"]
|
|
745
|
+
else:
|
|
746
|
+
out_json["png_base64"] = render_res["content_base64"]
|
|
747
|
+
output_str = json.dumps(out_json, indent=2, ensure_ascii=False)
|
|
748
|
+
if args.output:
|
|
749
|
+
with open(args.output, "w", encoding="utf-8") as f:
|
|
750
|
+
f.write(output_str)
|
|
751
|
+
print(f"Results written to: {args.output}")
|
|
752
|
+
else:
|
|
753
|
+
print(output_str)
|
|
754
|
+
return
|
|
755
|
+
|
|
756
|
+
# Saving raw files (SVG or PNG)
|
|
757
|
+
if "slides" in render_res:
|
|
758
|
+
out_dir = args.output or os.path.splitext(file_path)[0] + f"_{norm_fmt}"
|
|
759
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
760
|
+
ext = "svg" if render_res["format"] == "svg" else "png"
|
|
761
|
+
for item in render_res["slides"]:
|
|
762
|
+
slide_path = os.path.join(out_dir, f"slide_{item['slide']}.{ext}")
|
|
763
|
+
mode = "w" if ext == "svg" else "wb"
|
|
764
|
+
content = item["content"] if ext == "svg" else item["raw_bytes"]
|
|
765
|
+
encoding = "utf-8" if ext == "svg" else None
|
|
766
|
+
with open(slide_path, mode, encoding=encoding) as f:
|
|
767
|
+
f.write(content)
|
|
768
|
+
print(f"Exported {len(render_res['slides'])} {ext.upper()} slides to directory: {out_dir}")
|
|
769
|
+
else:
|
|
770
|
+
ext = "svg" if render_res["format"] == "svg" else "png"
|
|
771
|
+
out_path = args.output or f"{os.path.splitext(file_path)[0]}_slide_{render_res['slide']}.{ext}"
|
|
772
|
+
mode = "w" if ext == "svg" else "wb"
|
|
773
|
+
content = render_res["content"] if ext == "svg" else render_res["raw_bytes"]
|
|
774
|
+
encoding = "utf-8" if ext == "svg" else None
|
|
775
|
+
with open(out_path, mode, encoding=encoding) as f:
|
|
776
|
+
f.write(content)
|
|
777
|
+
print(f"Exported slide {render_res['slide']} to: {out_path}")
|
|
778
|
+
return
|
|
779
|
+
except Exception as e:
|
|
780
|
+
print(f"Error converting PPTX '{file_path}': {e}", file=sys.stderr)
|
|
781
|
+
sys.exit(1)
|
|
782
|
+
if file_ext in IMAGE_EXTENSIONS and not args.process_images:
|
|
783
|
+
print(f"Error: Processing image file '{os.path.basename(file_path)}' requires --process-images to be enabled.", file=sys.stderr)
|
|
784
|
+
sys.exit(1)
|
|
785
|
+
|
|
786
|
+
if args.process_images and not args.model:
|
|
787
|
+
print("Error: --process-images requires a vision model. Please specify one using --model / -m (e.g., --model gpt-4o).", file=sys.stderr)
|
|
788
|
+
sys.exit(1)
|
|
789
|
+
|
|
790
|
+
if args.model and not args.process_images:
|
|
791
|
+
print("Error: --model was specified, but --process-images is not enabled. Add --process-images to process images/tables with vision, or remove --model.", file=sys.stderr)
|
|
792
|
+
sys.exit(1)
|
|
793
|
+
|
|
794
|
+
try:
|
|
795
|
+
chunks = process_file(file_path, process_images=args.process_images, model=args.model)
|
|
796
|
+
except Exception as e:
|
|
797
|
+
print(f"Error processing file '{file_path}': {e}", file=sys.stderr)
|
|
798
|
+
sys.exit(1)
|
|
799
|
+
|
|
800
|
+
if args.json:
|
|
801
|
+
output_str = json.dumps(chunks, indent=2, ensure_ascii=False)
|
|
802
|
+
else:
|
|
803
|
+
lines = [
|
|
804
|
+
f"Processed: {file_path}",
|
|
805
|
+
f"Total Chunks: {len(chunks)}",
|
|
806
|
+
"=" * 60
|
|
807
|
+
]
|
|
808
|
+
for chunk in chunks:
|
|
809
|
+
idx = chunk.get("chunkIndex", 0)
|
|
810
|
+
name = chunk.get("filename", "")
|
|
811
|
+
content = chunk.get("content", "")
|
|
812
|
+
lines.append(f"\n--- [Chunk {idx}] ({name}) [{len(content)} chars] ---")
|
|
813
|
+
lines.append(content)
|
|
814
|
+
lines.append("-" * 60)
|
|
815
|
+
output_str = "\n".join(lines)
|
|
816
|
+
|
|
817
|
+
if args.output:
|
|
818
|
+
with open(args.output, "w", encoding="utf-8") as f:
|
|
819
|
+
f.write(output_str)
|
|
820
|
+
print(f"Results written to: {args.output}")
|
|
821
|
+
else:
|
|
822
|
+
print(output_str)
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
if __name__ == "__main__":
|
|
826
|
+
main()
|