docler 2.0.0__tar.gz → 2.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {docler-2.0.0 → docler-2.0.1}/PKG-INFO +1 -1
- {docler-2.0.0 → docler-2.0.1}/pyproject.toml +1 -1
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/aggregated_converter.py +1 -5
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/base.py +55 -14
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/datalab_provider/provider.py +1 -5
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/docling_remote_provider/provider.py +1 -5
- {docler-2.0.0 → docler-2.0.1}/src/docler_api/routes.py +33 -110
- {docler-2.0.0 → docler-2.0.1}/LICENSE +0 -0
- {docler-2.0.0 → docler-2.0.1}/README.md +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/__main__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/annotators/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/annotators/ai_document_annotator.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/annotators/ai_image_annotator.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/annotators/base.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/ai_chunker/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/ai_chunker/chunker.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/ai_chunker/models.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/ai_chunker/utils.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/base.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/llamaindex_chunker/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/llamaindex_chunker/chunker.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/markdown_chunker/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/markdown_chunker/chunker.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/markdown_chunker/utils.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/token_chunker/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/chunkers/token_chunker/chunker.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/common_types.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/configs/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/configs/annotator_configs.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/configs/chunker_configs.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/configs/converter_configs.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/configs/processor_configs.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/azure_provider/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/azure_provider/provider.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/azure_provider/utils.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/datalab_provider/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/datalab_provider/utils.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/dir_converter.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/docling_provider/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/docling_provider/provider.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/docling_provider/utils.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/docling_remote_provider/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/docling_remote_provider/utils.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/exceptions.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/llamaparse_provider/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/llamaparse_provider/provider.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/llamaparse_provider/utils.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/llm_provider/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/llm_provider/provider.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/marker_provider/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/marker_provider/provider.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/markitdown_provider/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/markitdown_provider/provider.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/mistral_provider/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/mistral_provider/provider.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/mistral_provider/utils.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/registry.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/upstage_provider/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/converters/upstage_provider/provider.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/diffs.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/log.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/mime_types.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/models.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/pdf_utils.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/process_runner.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/processors/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/processors/ai_processor.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/processors/base.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/provider.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/py.typed +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/resources/pdf_sample.pdf +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/resources/samplepptx.pptx +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/resources/test.docx +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/resources/test.jpg +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/resources/test.pptx +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/resources/test.xls +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/resources/test.xlsx +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler/utils.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_api/README.md +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_api/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_api/main.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_api/py.typed +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_mcp/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_mcp/py.typed +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_mcp/server.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_streamlit/__init__.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_streamlit/chunk_app.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_streamlit/chunkers.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_streamlit/converters.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_streamlit/ocr_app.py +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_streamlit/py.typed +0 -0
- {docler-2.0.0 → docler-2.0.1}/src/docler_streamlit/utils.py +0 -0
|
@@ -84,11 +84,7 @@ class AggregatedConverter(DocumentConverter[AggregatedConverterConfig]):
|
|
|
84
84
|
"""
|
|
85
85
|
self._registry.set_preference(mime_or_extension, converter_name)
|
|
86
86
|
|
|
87
|
-
async def _convert_async(
|
|
88
|
-
self,
|
|
89
|
-
data: BytesIO,
|
|
90
|
-
mime_type: MimeType,
|
|
91
|
-
) -> ConverterResult:
|
|
87
|
+
async def _convert_async(self, data: BytesIO, mime_type: MimeType) -> ConverterResult:
|
|
92
88
|
"""Delegate conversion to the appropriate converter.
|
|
93
89
|
|
|
94
90
|
Note: We need a filename/extension to select the right converter,
|
|
@@ -122,6 +122,7 @@ class DocumentConverter[TConfig: BaseModel = Any](BaseProvider[TConfig], ABC):
|
|
|
122
122
|
*,
|
|
123
123
|
source_path: str | None = None,
|
|
124
124
|
title: str | None = None,
|
|
125
|
+
pdf_password: str | None = None,
|
|
125
126
|
) -> Document:
|
|
126
127
|
"""Convert document content directly from bytes, BytesIO, or base64 string.
|
|
127
128
|
|
|
@@ -130,6 +131,7 @@ class DocumentConverter[TConfig: BaseModel = Any](BaseProvider[TConfig], ABC):
|
|
|
130
131
|
mime_type: MIME type of the content.
|
|
131
132
|
source_path: Optional source path for metadata.
|
|
132
133
|
title: Optional title. If None, uses "Untitled".
|
|
134
|
+
pdf_password: Password for encrypted PDF files.
|
|
133
135
|
|
|
134
136
|
Returns:
|
|
135
137
|
Converted document with extracted text and images.
|
|
@@ -161,6 +163,42 @@ class DocumentConverter[TConfig: BaseModel = Any](BaseProvider[TConfig], ABC):
|
|
|
161
163
|
msg = f"Unsupported content type: {type(content)}"
|
|
162
164
|
raise TypeError(msg)
|
|
163
165
|
|
|
166
|
+
# Handle PDF decryption if needed
|
|
167
|
+
if mime_type == "application/pdf":
|
|
168
|
+
from docler.pdf_utils import decrypt_pdf, get_pdf_info
|
|
169
|
+
|
|
170
|
+
try:
|
|
171
|
+
pdf_info = get_pdf_info(data.getvalue())
|
|
172
|
+
if pdf_info.is_encrypted:
|
|
173
|
+
if pdf_password is None:
|
|
174
|
+
# Try empty password first
|
|
175
|
+
try:
|
|
176
|
+
decrypted_content = decrypt_pdf(data.getvalue(), None)
|
|
177
|
+
data = BytesIO(decrypted_content)
|
|
178
|
+
except ValueError as e:
|
|
179
|
+
if "requires a password" in str(e):
|
|
180
|
+
msg = (
|
|
181
|
+
"PDF is encrypted but no password provided. "
|
|
182
|
+
"Please provide pdf_password parameter."
|
|
183
|
+
)
|
|
184
|
+
raise ValueError(msg) from e
|
|
185
|
+
raise
|
|
186
|
+
else:
|
|
187
|
+
# Decrypt the PDF with provided password
|
|
188
|
+
try:
|
|
189
|
+
decrypted_content = decrypt_pdf(data.getvalue(), pdf_password)
|
|
190
|
+
data = BytesIO(decrypted_content)
|
|
191
|
+
except ValueError as e:
|
|
192
|
+
if "Incorrect password" in str(e):
|
|
193
|
+
raise ValueError("Incorrect PDF password") from e
|
|
194
|
+
raise
|
|
195
|
+
except ValueError as e:
|
|
196
|
+
if "encrypted" not in str(e).lower():
|
|
197
|
+
# If it's not an encryption issue, let it pass through to the converter
|
|
198
|
+
pass
|
|
199
|
+
else:
|
|
200
|
+
raise
|
|
201
|
+
|
|
164
202
|
result = await self._convert_threaded(data, mime_type)
|
|
165
203
|
|
|
166
204
|
# Assemble final Document
|
|
@@ -186,11 +224,14 @@ class DocumentConverter[TConfig: BaseModel = Any](BaseProvider[TConfig], ABC):
|
|
|
186
224
|
|
|
187
225
|
return document
|
|
188
226
|
|
|
189
|
-
async def convert_files(
|
|
227
|
+
async def convert_files(
|
|
228
|
+
self, file_paths: Sequence[JoinablePathLike], *, pdf_password: str | None = None
|
|
229
|
+
) -> list[Document]:
|
|
190
230
|
"""Convert multiple document files in parallel.
|
|
191
231
|
|
|
192
232
|
Args:
|
|
193
233
|
file_paths: Sequence of paths to documents to convert.
|
|
234
|
+
pdf_password: Password for encrypted PDF files.
|
|
194
235
|
|
|
195
236
|
Returns:
|
|
196
237
|
List of converted documents in the same order as the input paths.
|
|
@@ -199,16 +240,19 @@ class DocumentConverter[TConfig: BaseModel = Any](BaseProvider[TConfig], ABC):
|
|
|
199
240
|
FileNotFoundError: If any file doesn't exist.
|
|
200
241
|
ValueError: If any file format is not supported.
|
|
201
242
|
"""
|
|
202
|
-
tasks = [self.convert_file(path) for path in file_paths]
|
|
243
|
+
tasks = [self.convert_file(path, pdf_password=pdf_password) for path in file_paths]
|
|
203
244
|
return list(await anyenv.gather(*tasks))
|
|
204
245
|
|
|
205
|
-
async def convert_file(
|
|
246
|
+
async def convert_file(
|
|
247
|
+
self, file_path: JoinablePathLike, *, pdf_password: str | None = None
|
|
248
|
+
) -> Document:
|
|
206
249
|
"""Convert a document file.
|
|
207
250
|
|
|
208
251
|
Supports both local and remote files through fsspec/upath.
|
|
209
252
|
|
|
210
253
|
Args:
|
|
211
254
|
file_path: Path to the file to process (local or remote URI).
|
|
255
|
+
pdf_password: Password for encrypted PDF files.
|
|
212
256
|
|
|
213
257
|
Returns:
|
|
214
258
|
Converted document with extracted text and images.
|
|
@@ -239,13 +283,10 @@ class DocumentConverter[TConfig: BaseModel = Any](BaseProvider[TConfig], ABC):
|
|
|
239
283
|
mime_type=mime_type,
|
|
240
284
|
source_path=str(path),
|
|
241
285
|
title=path.stem,
|
|
286
|
+
pdf_password=pdf_password,
|
|
242
287
|
)
|
|
243
288
|
|
|
244
|
-
async def _convert_threaded(
|
|
245
|
-
self,
|
|
246
|
-
data: BytesIO,
|
|
247
|
-
mime_type: MimeType,
|
|
248
|
-
) -> ConverterResult:
|
|
289
|
+
async def _convert_threaded(self, data: BytesIO, mime_type: MimeType) -> ConverterResult:
|
|
249
290
|
"""Internal method to handle conversion routing.
|
|
250
291
|
|
|
251
292
|
Will use _convert_async if implemented, otherwise falls back to
|
|
@@ -268,11 +309,7 @@ class DocumentConverter[TConfig: BaseModel = Any](BaseProvider[TConfig], ABC):
|
|
|
268
309
|
"""
|
|
269
310
|
raise NotImplementedError
|
|
270
311
|
|
|
271
|
-
async def _convert_async(
|
|
272
|
-
self,
|
|
273
|
-
data: BytesIO,
|
|
274
|
-
mime_type: MimeType,
|
|
275
|
-
) -> ConverterResult:
|
|
312
|
+
async def _convert_async(self, data: BytesIO, mime_type: MimeType) -> ConverterResult:
|
|
276
313
|
"""Asynchronous implementation for IO-bound operations.
|
|
277
314
|
|
|
278
315
|
Args:
|
|
@@ -312,6 +349,7 @@ class DocumentConverter[TConfig: BaseModel = Any](BaseProvider[TConfig], ABC):
|
|
|
312
349
|
exclude: list[str] | None = None,
|
|
313
350
|
max_depth: int | None = None,
|
|
314
351
|
chunk_size: int = 50,
|
|
352
|
+
pdf_password: str | None = None,
|
|
315
353
|
) -> dict[str, Document]:
|
|
316
354
|
"""Convert all supported files in a directory.
|
|
317
355
|
|
|
@@ -322,6 +360,7 @@ class DocumentConverter[TConfig: BaseModel = Any](BaseProvider[TConfig], ABC):
|
|
|
322
360
|
exclude: List of patterns to exclude.
|
|
323
361
|
max_depth: Maximum directory depth for recursive search.
|
|
324
362
|
chunk_size: Number of files to convert in parallel.
|
|
363
|
+
pdf_password: Password for encrypted PDF files.
|
|
325
364
|
|
|
326
365
|
Returns:
|
|
327
366
|
Mapping of relative paths to converted documents.
|
|
@@ -363,7 +402,9 @@ class DocumentConverter[TConfig: BaseModel = Any](BaseProvider[TConfig], ABC):
|
|
|
363
402
|
for i in range(0, len(supported_files), chunk_size):
|
|
364
403
|
chunk = supported_files[i : i + chunk_size]
|
|
365
404
|
# Convert using full paths
|
|
366
|
-
documents = await self.convert_files(
|
|
405
|
+
documents = await self.convert_files(
|
|
406
|
+
[path for _, path in chunk], pdf_password=pdf_password
|
|
407
|
+
)
|
|
367
408
|
|
|
368
409
|
# Store results with relative paths as keys
|
|
369
410
|
for (rel_path, _), doc in zip(chunk, documents):
|
|
@@ -81,11 +81,7 @@ class DataLabConverter(DocumentConverter[DataLabConfig]):
|
|
|
81
81
|
"""Price per page in USD."""
|
|
82
82
|
return 0.003 if self.use_llm else 0.0015
|
|
83
83
|
|
|
84
|
-
async def _convert_async(
|
|
85
|
-
self,
|
|
86
|
-
data: BytesIO,
|
|
87
|
-
mime_type: MimeType,
|
|
88
|
-
) -> ConverterResult:
|
|
84
|
+
async def _convert_async(self, data: BytesIO, mime_type: MimeType) -> ConverterResult:
|
|
89
85
|
"""Convert a file using DataLab's API.
|
|
90
86
|
|
|
91
87
|
Args:
|
|
@@ -141,11 +141,7 @@ class DoclingRemoteConverter(DocumentConverter[DoclingRemoteConfig]):
|
|
|
141
141
|
if languages:
|
|
142
142
|
self.config["ocr_lang"] = languages
|
|
143
143
|
|
|
144
|
-
async def _convert_async(
|
|
145
|
-
self,
|
|
146
|
-
data: BytesIO,
|
|
147
|
-
mime_type: MimeType,
|
|
148
|
-
) -> ConverterResult:
|
|
144
|
+
async def _convert_async(self, data: BytesIO, mime_type: MimeType) -> ConverterResult:
|
|
149
145
|
"""Convert a document using remote Docling service.
|
|
150
146
|
|
|
151
147
|
Args:
|
|
@@ -3,17 +3,15 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import mimetypes
|
|
6
|
-
import tempfile
|
|
7
6
|
from typing import TYPE_CHECKING, Annotated, Any
|
|
8
7
|
|
|
9
8
|
import anyenv
|
|
10
9
|
from fastapi import Body, File, Form, HTTPException, Query
|
|
11
10
|
from pydantic import TypeAdapter
|
|
12
|
-
import upath
|
|
13
11
|
|
|
14
12
|
from docler.configs.converter_configs import ConverterConfig
|
|
15
13
|
from docler.models import ChunkedDocument, PageMetadata # noqa: TC001
|
|
16
|
-
from docler.pdf_utils import
|
|
14
|
+
from docler.pdf_utils import get_pdf_info
|
|
17
15
|
|
|
18
16
|
|
|
19
17
|
if TYPE_CHECKING:
|
|
@@ -44,65 +42,33 @@ async def convert_document(
|
|
|
44
42
|
|
|
45
43
|
content = await file.read()
|
|
46
44
|
|
|
47
|
-
# Handle PDF decryption if needed
|
|
48
|
-
if file.content_type == "application/pdf" or (
|
|
49
|
-
file.filename and file.filename.lower().endswith(".pdf")
|
|
50
|
-
):
|
|
51
|
-
# Check if PDF is encrypted
|
|
52
|
-
try:
|
|
53
|
-
pdf_info = get_pdf_info(content)
|
|
54
|
-
if pdf_info.is_encrypted:
|
|
55
|
-
if pdf_password is None:
|
|
56
|
-
# Try empty password first
|
|
57
|
-
try:
|
|
58
|
-
content = decrypt_pdf(content, None)
|
|
59
|
-
except ValueError as e:
|
|
60
|
-
if "requires a password" in str(e):
|
|
61
|
-
raise HTTPException(
|
|
62
|
-
status_code=400,
|
|
63
|
-
detail=(
|
|
64
|
-
"PDF is encrypted but no password provided. "
|
|
65
|
-
"Please provide pdf_password parameter."
|
|
66
|
-
),
|
|
67
|
-
) from e
|
|
68
|
-
raise HTTPException(
|
|
69
|
-
status_code=400, detail=f"Failed to decrypt PDF: {e}"
|
|
70
|
-
) from e
|
|
71
|
-
else:
|
|
72
|
-
# Decrypt the PDF with provided password
|
|
73
|
-
try:
|
|
74
|
-
content = decrypt_pdf(content, pdf_password)
|
|
75
|
-
except ValueError as e:
|
|
76
|
-
if "Incorrect password" in str(e):
|
|
77
|
-
raise HTTPException( # noqa: B904
|
|
78
|
-
status_code=401, detail="Incorrect PDF password"
|
|
79
|
-
)
|
|
80
|
-
raise HTTPException(
|
|
81
|
-
status_code=400, detail=f"Failed to decrypt PDF: {e}"
|
|
82
|
-
) from e
|
|
83
|
-
except ValueError as e:
|
|
84
|
-
if "encrypted" not in str(e).lower():
|
|
85
|
-
# If it's not an encryption issue, let it pass through to the converter
|
|
86
|
-
pass
|
|
87
|
-
|
|
88
|
-
with tempfile.NamedTemporaryFile(suffix=f"_{file.filename}", delete=False) as temp_file:
|
|
89
|
-
temp_path = temp_file.name
|
|
90
|
-
temp_file.write(content)
|
|
91
|
-
|
|
92
45
|
try:
|
|
93
46
|
mime_type, _ = mimetypes.guess_type(file.filename or "")
|
|
94
47
|
if not mime_type:
|
|
95
|
-
mime_type = file.content_type
|
|
48
|
+
mime_type = file.content_type or "application/octet-stream"
|
|
96
49
|
|
|
97
50
|
# Create converter from config
|
|
98
51
|
converter = parsed.get_provider()
|
|
99
|
-
document = await converter.
|
|
52
|
+
document = await converter.convert_content(
|
|
53
|
+
content=content,
|
|
54
|
+
mime_type=mime_type,
|
|
55
|
+
source_path=file.filename,
|
|
56
|
+
title=file.filename,
|
|
57
|
+
pdf_password=pdf_password,
|
|
58
|
+
)
|
|
100
59
|
|
|
101
60
|
# Always include images as base64
|
|
102
61
|
for image in document.images:
|
|
103
62
|
if isinstance(image.content, bytes):
|
|
104
63
|
# Convert bytes to base64 string
|
|
105
64
|
image.content = image.to_base64()
|
|
65
|
+
except ValueError as e:
|
|
66
|
+
if "password" in str(e).lower():
|
|
67
|
+
status_code = 401 if "incorrect" in str(e).lower() else 400
|
|
68
|
+
raise HTTPException(status_code=status_code, detail=str(e)) from e
|
|
69
|
+
raise HTTPException(
|
|
70
|
+
status_code=400, detail=f"Error during document conversion: {e!s}"
|
|
71
|
+
) from e
|
|
106
72
|
except Exception as e:
|
|
107
73
|
if not isinstance(e, HTTPException):
|
|
108
74
|
raise HTTPException(
|
|
@@ -111,10 +77,6 @@ async def convert_document(
|
|
|
111
77
|
raise
|
|
112
78
|
else:
|
|
113
79
|
return document
|
|
114
|
-
finally:
|
|
115
|
-
# Clean up the temporary file
|
|
116
|
-
path = upath.UPath(temp_path)
|
|
117
|
-
path.unlink(missing_ok=True)
|
|
118
80
|
|
|
119
81
|
|
|
120
82
|
async def chunk_document(
|
|
@@ -145,59 +107,19 @@ async def chunk_document(
|
|
|
145
107
|
"""
|
|
146
108
|
content = await file.read()
|
|
147
109
|
|
|
148
|
-
# Handle PDF decryption if needed
|
|
149
|
-
if file.content_type == "application/pdf" or (
|
|
150
|
-
file.filename and file.filename.lower().endswith(".pdf")
|
|
151
|
-
):
|
|
152
|
-
# Check if PDF is encrypted
|
|
153
|
-
try:
|
|
154
|
-
pdf_info = get_pdf_info(content)
|
|
155
|
-
if pdf_info.is_encrypted:
|
|
156
|
-
if pdf_password is None:
|
|
157
|
-
# Try empty password first
|
|
158
|
-
try:
|
|
159
|
-
content = decrypt_pdf(content, None)
|
|
160
|
-
except ValueError as e:
|
|
161
|
-
if "requires a password" in str(e):
|
|
162
|
-
raise HTTPException(
|
|
163
|
-
status_code=400,
|
|
164
|
-
detail="PDF is encrypted but no password provided.",
|
|
165
|
-
) from e
|
|
166
|
-
raise HTTPException(
|
|
167
|
-
status_code=400, detail=f"Failed to decrypt PDF: {e}"
|
|
168
|
-
) from e
|
|
169
|
-
else:
|
|
170
|
-
# Decrypt the PDF with provided password
|
|
171
|
-
try:
|
|
172
|
-
content = decrypt_pdf(content, pdf_password)
|
|
173
|
-
except ValueError as e:
|
|
174
|
-
if "Incorrect password" in str(e):
|
|
175
|
-
raise HTTPException( # noqa: B904
|
|
176
|
-
status_code=401, detail="Incorrect PDF password"
|
|
177
|
-
)
|
|
178
|
-
raise HTTPException(
|
|
179
|
-
status_code=400, detail=f"Failed to decrypt PDF: {e}"
|
|
180
|
-
) from e
|
|
181
|
-
except ValueError as e:
|
|
182
|
-
if "encrypted" not in str(e).lower():
|
|
183
|
-
# If it's not an encryption issue, let it pass through to the converter
|
|
184
|
-
pass
|
|
185
|
-
|
|
186
|
-
with tempfile.NamedTemporaryFile(suffix=f"_{file.filename}", delete=False) as temp_file:
|
|
187
|
-
temp_path = temp_file.name
|
|
188
|
-
temp_file.write(content)
|
|
189
|
-
|
|
190
110
|
try:
|
|
191
111
|
mime_type, _ = mimetypes.guess_type(file.filename or "")
|
|
192
112
|
if not mime_type:
|
|
193
|
-
mime_type = file.content_type
|
|
194
|
-
|
|
195
|
-
# Get converter and chunker instances
|
|
113
|
+
mime_type = file.content_type or "application/octet-stream"
|
|
196
114
|
converter = converter_config.get_provider()
|
|
197
115
|
chunker = chunker_config.get_provider()
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
116
|
+
document = await converter.convert_content(
|
|
117
|
+
content=content,
|
|
118
|
+
mime_type=mime_type,
|
|
119
|
+
source_path=file.filename,
|
|
120
|
+
title=file.filename,
|
|
121
|
+
pdf_password=pdf_password,
|
|
122
|
+
)
|
|
201
123
|
|
|
202
124
|
# Chunk the document
|
|
203
125
|
chunked_document = await chunker.chunk(document)
|
|
@@ -212,19 +134,20 @@ async def chunk_document(
|
|
|
212
134
|
if isinstance(image.content, bytes):
|
|
213
135
|
# Convert bytes to base64 string
|
|
214
136
|
image.content = image.to_base64()
|
|
137
|
+
except ValueError as e:
|
|
138
|
+
if "password" in str(e).lower():
|
|
139
|
+
status_code = 401 if "incorrect" in str(e).lower() else 400
|
|
140
|
+
raise HTTPException(status_code=status_code, detail=str(e)) from e
|
|
141
|
+
raise HTTPException(
|
|
142
|
+
status_code=400, detail=f"Error during document conversion or chunking: {e!s}"
|
|
143
|
+
) from e
|
|
215
144
|
except Exception as e:
|
|
216
145
|
if not isinstance(e, HTTPException):
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
detail=f"Error during document conversion or chunking: {e!s}",
|
|
220
|
-
) from e
|
|
146
|
+
msg = f"Error during document conversion or chunking: {e!s}"
|
|
147
|
+
raise HTTPException(status_code=500, detail=msg) from e
|
|
221
148
|
raise
|
|
222
149
|
else:
|
|
223
150
|
return chunked_document
|
|
224
|
-
finally:
|
|
225
|
-
# Clean up the temporary file
|
|
226
|
-
path = upath.UPath(temp_path)
|
|
227
|
-
path.unlink(missing_ok=True)
|
|
228
151
|
|
|
229
152
|
|
|
230
153
|
async def list_converters() -> dict[str, list[dict[str, Any]]]:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|