th2rag 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- th2rag/__init__.py +0 -0
- th2rag/auth/__init__.py +0 -0
- th2rag/auth/exceptions.py +2 -0
- th2rag/auth/router.py +43 -0
- th2rag/auth/schemas.py +15 -0
- th2rag/auth/service.py +115 -0
- th2rag/clients/__init__.py +0 -0
- th2rag/clients/storage/config.py +13 -0
- th2rag/clients/storage/local.py +48 -0
- th2rag/clients/storage/s3.py +93 -0
- th2rag/config.py +267 -0
- th2rag/conversations/__init__.py +0 -0
- th2rag/conversations/dependencies.py +30 -0
- th2rag/conversations/exceptions.py +6 -0
- th2rag/conversations/router.py +47 -0
- th2rag/conversations/schemas.py +43 -0
- th2rag/conversations/service.py +51 -0
- th2rag/database.py +72 -0
- th2rag/dataset/__init__.py +0 -0
- th2rag/dataset/dependencies.py +25 -0
- th2rag/dataset/exceptions.py +2 -0
- th2rag/dataset/router.py +53 -0
- th2rag/dataset/schemas.py +23 -0
- th2rag/dataset/service.py +78 -0
- th2rag/dependencies.py +143 -0
- th2rag/knowledge/__init__.py +0 -0
- th2rag/knowledge/dependencies.py +25 -0
- th2rag/knowledge/exceptions.py +2 -0
- th2rag/knowledge/mage_client.py +505 -0
- th2rag/knowledge/router.py +714 -0
- th2rag/knowledge/schemas.py +108 -0
- th2rag/knowledge/service.py +392 -0
- th2rag/logger.py +21 -0
- th2rag/main.py +51 -0
- th2rag/messages/__init__.py +0 -0
- th2rag/messages/exceptions.py +2 -0
- th2rag/messages/router.py +63 -0
- th2rag/messages/schemas.py +68 -0
- th2rag/messages/service.py +210 -0
- th2rag/models.py +215 -0
- th2rag/pagination.py +68 -0
- th2rag/rag/__init__.py +0 -0
- th2rag/rag/chunkers/__init__.py +0 -0
- th2rag/rag/chunkers/hybrid_chunker.py +23 -0
- th2rag/rag/constants.py +15 -0
- th2rag/rag/converters/__init__.py +0 -0
- th2rag/rag/converters/documents_converter.py +123 -0
- th2rag/rag/converters/json_converter.py +521 -0
- th2rag/rag/converters/legacy_converter.py +124 -0
- th2rag/rag/converters/pdf_converter.py +384 -0
- th2rag/rag/converters/spreadsheet_converter.py +159 -0
- th2rag/rag/converters/txt_converter.py +98 -0
- th2rag/rag/embeddings/__init__.py +0 -0
- th2rag/rag/embeddings/base.py +21 -0
- th2rag/rag/embeddings/exceptions.py +2 -0
- th2rag/rag/embeddings/sentence_transformer.py +97 -0
- th2rag/rag/generation/base.py +22 -0
- th2rag/rag/generation/exceptions.py +2 -0
- th2rag/rag/generation/litellm_generator.py +143 -0
- th2rag/rag/generation/mistral_generator.py +269 -0
- th2rag/rag/generation/test_litellm.py +33 -0
- th2rag/rag/retrieval/__init__.py +0 -0
- th2rag/rag/retrieval/base.py +11 -0
- th2rag/rag/retrieval/exceptions.py +10 -0
- th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
- th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
- th2rag/rag/retrieval/lancedb/schemas.py +19 -0
- th2rag/rag/services/__init__.py +0 -0
- th2rag/rag/services/rag_service.py +99 -0
- th2rag/rag/services/rag_service_builder.py +42 -0
- th2rag/rag/tasks/process_pdf.py +586 -0
- th2rag/users/dependencies.py +44 -0
- th2rag/users/exceptions.py +6 -0
- th2rag/users/router.py +92 -0
- th2rag/users/schemas.py +61 -0
- th2rag/users/service.py +140 -0
- th2rag/utils/__init__.py +0 -0
- th2rag/utils/helpers.py +40 -0
- th2rag/utils/logger.py +15 -0
- th2rag/utils/security.py +76 -0
- th2rag/utils/welcome_router.py +98 -0
- th2rag/webhooks/__init__.py +0 -0
- th2rag/webhooks/sender.py +126 -0
- th2rag-0.0.2.dist-info/METADATA +1002 -0
- th2rag-0.0.2.dist-info/RECORD +87 -0
- th2rag-0.0.2.dist-info/WHEEL +4 -0
- th2rag-0.0.2.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Conversion de formats legacy via LibreOffice headless.
|
|
3
|
+
|
|
4
|
+
Formats supportés : .doc → .docx, .ppt → .pptx, .xls → .xlsx
|
|
5
|
+
Pré-requis runtime : soffice ou libreoffice dans le PATH.
|
|
6
|
+
|
|
7
|
+
Si LibreOffice est absent, lève ValueError avec message explicite.
|
|
8
|
+
Ne crashe jamais avec FileNotFoundError ou CalledProcessError.
|
|
9
|
+
"""
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
import shutil
|
|
13
|
+
import subprocess
|
|
14
|
+
import tempfile
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
# Mapping extension legacy → extension cible Docling-compatible
|
|
19
|
+
_LEGACY_TO_TARGET: dict[str, str] = {
|
|
20
|
+
".doc": ".docx",
|
|
21
|
+
".ppt": ".pptx",
|
|
22
|
+
".xls": ".xlsx",
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
LEGACY_EXTENSIONS: frozenset[str] = frozenset(_LEGACY_TO_TARGET.keys())
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _find_soffice() -> str | None:
|
|
29
|
+
"""Retourne le chemin de soffice/libreoffice, ou None si absent."""
|
|
30
|
+
return shutil.which("soffice") or shutil.which("libreoffice")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def convert_with_libreoffice(source: str) -> str:
|
|
34
|
+
"""
|
|
35
|
+
Convertit un fichier legacy (.doc/.ppt/.xls) en format moderne
|
|
36
|
+
via LibreOffice headless, dans un répertoire temporaire.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
source: Chemin absolu vers le fichier source.
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
Chemin absolu vers le fichier converti (dans un tmpdir).
|
|
43
|
+
L'appelant est responsable de nettoyer ce tmpdir.
|
|
44
|
+
|
|
45
|
+
Raises:
|
|
46
|
+
ValueError: Si LibreOffice est absent du PATH.
|
|
47
|
+
ValueError: Si la conversion échoue (returncode != 0 ou fichier absent).
|
|
48
|
+
"""
|
|
49
|
+
soffice = _find_soffice()
|
|
50
|
+
if soffice is None:
|
|
51
|
+
ext = os.path.splitext(source)[1].lower()
|
|
52
|
+
target = _LEGACY_TO_TARGET.get(ext, ".docx")
|
|
53
|
+
raise ValueError(
|
|
54
|
+
f"Format {ext} non supporté sans LibreOffice. "
|
|
55
|
+
f"Installez LibreOffice (apt install libreoffice) pour convertir "
|
|
56
|
+
f"{ext} → {target}. "
|
|
57
|
+
f"Formats acceptés sans LibreOffice : PDF, DOCX, PPTX, XLSX, CSV, TXT, "
|
|
58
|
+
f"HTML, Markdown, images, AsciiDoc, JSON."
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
ext = os.path.splitext(source)[1].lower()
|
|
62
|
+
if ext not in _LEGACY_TO_TARGET:
|
|
63
|
+
raise ValueError(
|
|
64
|
+
f"convert_with_libreoffice: extension {ext} non supportée. "
|
|
65
|
+
f"Extensions legacy supportées : {sorted(LEGACY_EXTENSIONS)}"
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
target_ext = _LEGACY_TO_TARGET[ext]
|
|
69
|
+
tmpdir = tempfile.mkdtemp(prefix="th2rag_legacy_")
|
|
70
|
+
|
|
71
|
+
logger.info(f"Converting legacy {ext} → {target_ext} via LibreOffice: {source}")
|
|
72
|
+
|
|
73
|
+
_success = False
|
|
74
|
+
try:
|
|
75
|
+
result = subprocess.run(
|
|
76
|
+
[soffice, "--headless", "--convert-to", target_ext.lstrip("."),
|
|
77
|
+
"--outdir", tmpdir, source],
|
|
78
|
+
capture_output=True,
|
|
79
|
+
text=True,
|
|
80
|
+
timeout=120,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
if result.returncode != 0:
|
|
84
|
+
raise ValueError(
|
|
85
|
+
f"LibreOffice conversion failed for {os.path.basename(source)} "
|
|
86
|
+
f"(returncode={result.returncode}): {result.stderr[:500]}"
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# Chercher le fichier converti dans tmpdir
|
|
90
|
+
basename = os.path.splitext(os.path.basename(source))[0]
|
|
91
|
+
converted_path = os.path.join(tmpdir, basename + target_ext)
|
|
92
|
+
|
|
93
|
+
if not os.path.exists(converted_path):
|
|
94
|
+
# Parfois soffice crée le fichier sans l'extension demandée
|
|
95
|
+
candidates = [
|
|
96
|
+
f for f in os.listdir(tmpdir)
|
|
97
|
+
if f.startswith(basename) and f.endswith(target_ext)
|
|
98
|
+
]
|
|
99
|
+
if candidates:
|
|
100
|
+
converted_path = os.path.join(tmpdir, candidates[0])
|
|
101
|
+
else:
|
|
102
|
+
raise ValueError(
|
|
103
|
+
f"LibreOffice n'a pas produit le fichier attendu : {converted_path}. "
|
|
104
|
+
f"Contenu de {tmpdir}: {os.listdir(tmpdir)}"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
logger.info(f"LibreOffice conversion successful: {converted_path}")
|
|
108
|
+
_success = True
|
|
109
|
+
return converted_path
|
|
110
|
+
|
|
111
|
+
except subprocess.TimeoutExpired as e:
|
|
112
|
+
raise ValueError(
|
|
113
|
+
f"LibreOffice conversion timeout (120s) pour {os.path.basename(source)}"
|
|
114
|
+
) from e
|
|
115
|
+
except ValueError:
|
|
116
|
+
raise
|
|
117
|
+
except Exception as e:
|
|
118
|
+
raise ValueError(
|
|
119
|
+
f"Erreur inattendue lors de la conversion LibreOffice de "
|
|
120
|
+
f"{os.path.basename(source)}: {e}"
|
|
121
|
+
) from e
|
|
122
|
+
finally:
|
|
123
|
+
if not _success:
|
|
124
|
+
shutil.rmtree(tmpdir, ignore_errors=True)
|
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
import threading
|
|
4
|
+
import uuid
|
|
5
|
+
from io import BytesIO
|
|
6
|
+
|
|
7
|
+
from docling.datamodel.base_models import InputFormat
|
|
8
|
+
from docling.datamodel.document import DocumentStream
|
|
9
|
+
from docling.datamodel.pipeline_options import (
|
|
10
|
+
PdfPipelineOptions,
|
|
11
|
+
ThreadedPdfPipelineOptions,
|
|
12
|
+
)
|
|
13
|
+
from docling.document_converter import (
|
|
14
|
+
AsciiDocFormatOption,
|
|
15
|
+
DocumentConverter,
|
|
16
|
+
HTMLFormatOption,
|
|
17
|
+
ImageFormatOption,
|
|
18
|
+
MarkdownFormatOption,
|
|
19
|
+
PdfFormatOption,
|
|
20
|
+
PowerpointFormatOption,
|
|
21
|
+
WordFormatOption,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
from th2rag.clients.storage.s3 import upload_file_to_s3
|
|
25
|
+
from th2rag.config import settings
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
TOKEN_THRESHOLD = 50
|
|
30
|
+
|
|
31
|
+
# Mapping from extension to Docling InputFormat
|
|
32
|
+
EXT_TO_INPUT_FORMAT: dict[str, InputFormat] = {
|
|
33
|
+
".pdf": InputFormat.PDF,
|
|
34
|
+
".docx": InputFormat.DOCX,
|
|
35
|
+
".pptx": InputFormat.PPTX,
|
|
36
|
+
".html": InputFormat.HTML,
|
|
37
|
+
".htm": InputFormat.HTML,
|
|
38
|
+
".md": InputFormat.MD,
|
|
39
|
+
".markdown": InputFormat.MD,
|
|
40
|
+
".png": InputFormat.IMAGE,
|
|
41
|
+
".jpg": InputFormat.IMAGE,
|
|
42
|
+
".jpeg": InputFormat.IMAGE,
|
|
43
|
+
".tiff": InputFormat.IMAGE,
|
|
44
|
+
".tif": InputFormat.IMAGE,
|
|
45
|
+
".bmp": InputFormat.IMAGE,
|
|
46
|
+
".adoc": InputFormat.ASCIIDOC,
|
|
47
|
+
".asciidoc": InputFormat.ASCIIDOC,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
# Canonical extension per InputFormat (used when the real extension is an alias
|
|
51
|
+
# that Docling does not recognise natively, e.g. .markdown, .htm, .jpeg).
|
|
52
|
+
_CANONICAL_EXT: dict[InputFormat, str] = {
|
|
53
|
+
InputFormat.PDF: ".pdf",
|
|
54
|
+
InputFormat.DOCX: ".docx",
|
|
55
|
+
InputFormat.PPTX: ".pptx",
|
|
56
|
+
InputFormat.HTML: ".html",
|
|
57
|
+
InputFormat.MD: ".md",
|
|
58
|
+
InputFormat.IMAGE: ".png",
|
|
59
|
+
InputFormat.ASCIIDOC: ".adoc",
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def count_tokens_simple(doc) -> int:
|
|
64
|
+
text = doc.export_to_markdown()
|
|
65
|
+
return len(text.split()) if text else 0
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def get_pipeline_options(enable_ocr: bool) -> PdfPipelineOptions:
|
|
69
|
+
options = PdfPipelineOptions()
|
|
70
|
+
options.do_table_structure = True
|
|
71
|
+
options.table_structure_options.mode = "fast"
|
|
72
|
+
options.do_ocr = enable_ocr
|
|
73
|
+
options.generate_picture_images = True
|
|
74
|
+
|
|
75
|
+
if settings.handle_img_mode == "describe":
|
|
76
|
+
options.do_picture_description = True
|
|
77
|
+
else:
|
|
78
|
+
options.do_picture_description = False
|
|
79
|
+
return options
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def get_image_pipeline_options(enable_ocr: bool) -> ThreadedPdfPipelineOptions:
|
|
83
|
+
"""Build pipeline options for image documents (ThreadedPdfPipelineOptions)."""
|
|
84
|
+
options = ThreadedPdfPipelineOptions()
|
|
85
|
+
options.do_ocr = enable_ocr
|
|
86
|
+
options.do_table_structure = False
|
|
87
|
+
options.generate_picture_images = False
|
|
88
|
+
return options
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def enable_ocr(doc) -> bool:
|
|
92
|
+
"""
|
|
93
|
+
Return True when any page has fewer than TOKEN_THRESHOLD words,
|
|
94
|
+
signalling a scanned/image-only document.
|
|
95
|
+
"""
|
|
96
|
+
if not doc.pages:
|
|
97
|
+
return True
|
|
98
|
+
|
|
99
|
+
for page_no, _page in doc.pages.items():
|
|
100
|
+
text_items = [
|
|
101
|
+
item.text
|
|
102
|
+
for item in doc.texts
|
|
103
|
+
if item.prov and item.prov[0].page_no == page_no
|
|
104
|
+
]
|
|
105
|
+
page_text = " ".join(text_items)
|
|
106
|
+
word_count = len(page_text.split())
|
|
107
|
+
|
|
108
|
+
if word_count < TOKEN_THRESHOLD:
|
|
109
|
+
logger.warning(
|
|
110
|
+
f"Page {page_no} has only {word_count} tokens. Triggering OCR."
|
|
111
|
+
)
|
|
112
|
+
return True
|
|
113
|
+
|
|
114
|
+
return False
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def handle_images(doc, source_filename: str, doc_id: int = None) -> list[dict]:
|
|
118
|
+
processed_images = []
|
|
119
|
+
if not hasattr(doc, "pictures"):
|
|
120
|
+
return processed_images
|
|
121
|
+
|
|
122
|
+
batch_id = str(uuid.uuid4())[:8]
|
|
123
|
+
|
|
124
|
+
for i, picture in enumerate(doc.pictures):
|
|
125
|
+
try:
|
|
126
|
+
image_obj = picture.get_image(doc)
|
|
127
|
+
if image_obj is None:
|
|
128
|
+
logger.warning(f"Image {i} could not be retrieved/generated.")
|
|
129
|
+
continue
|
|
130
|
+
|
|
131
|
+
img_byte_arr = BytesIO()
|
|
132
|
+
image_obj.save(img_byte_arr, format="PNG")
|
|
133
|
+
img_bytes = img_byte_arr.getvalue()
|
|
134
|
+
|
|
135
|
+
base_name = os.path.basename(source_filename)
|
|
136
|
+
name_without_ext, _ = os.path.splitext(base_name)
|
|
137
|
+
img_filename = f"{name_without_ext}_img_{batch_id}_{i}.png"
|
|
138
|
+
|
|
139
|
+
image_link = upload_file_to_s3(img_bytes, img_filename, doc_id=doc_id)
|
|
140
|
+
|
|
141
|
+
page_number = picture.prov[0].page_no if picture.prov else 1
|
|
142
|
+
description = f"Image extracted from page {page_number}"
|
|
143
|
+
|
|
144
|
+
processed_images.append(
|
|
145
|
+
{
|
|
146
|
+
"image_index": i,
|
|
147
|
+
"link": image_link,
|
|
148
|
+
"description": description,
|
|
149
|
+
"page": page_number,
|
|
150
|
+
}
|
|
151
|
+
)
|
|
152
|
+
logger.info(f"Image processed: {image_link}")
|
|
153
|
+
|
|
154
|
+
except Exception as e:
|
|
155
|
+
logger.error(f"Failed to process image {i}: {e}")
|
|
156
|
+
|
|
157
|
+
return processed_images
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _build_format_options(input_format: InputFormat, enable_ocr_flag: bool) -> dict:
|
|
161
|
+
"""Return format_options dict for DocumentConverter based on input_format."""
|
|
162
|
+
if input_format == InputFormat.PDF:
|
|
163
|
+
return {
|
|
164
|
+
InputFormat.PDF: PdfFormatOption(
|
|
165
|
+
pipeline_options=get_pipeline_options(enable_ocr_flag)
|
|
166
|
+
)
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
if input_format == InputFormat.DOCX:
|
|
170
|
+
return {InputFormat.DOCX: WordFormatOption()}
|
|
171
|
+
|
|
172
|
+
if input_format == InputFormat.PPTX:
|
|
173
|
+
return {InputFormat.PPTX: PowerpointFormatOption()}
|
|
174
|
+
|
|
175
|
+
if input_format == InputFormat.IMAGE:
|
|
176
|
+
return {
|
|
177
|
+
InputFormat.IMAGE: ImageFormatOption(
|
|
178
|
+
pipeline_options=get_image_pipeline_options(enable_ocr_flag)
|
|
179
|
+
)
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
if input_format == InputFormat.HTML:
|
|
183
|
+
return {InputFormat.HTML: HTMLFormatOption()}
|
|
184
|
+
|
|
185
|
+
if input_format == InputFormat.MD:
|
|
186
|
+
return {InputFormat.MD: MarkdownFormatOption()}
|
|
187
|
+
|
|
188
|
+
if input_format == InputFormat.ASCIIDOC:
|
|
189
|
+
return {InputFormat.ASCIIDOC: AsciiDocFormatOption()}
|
|
190
|
+
|
|
191
|
+
return {}
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
# DocumentConverter charge des modeles PyTorch (layout / table-structure / OCR).
|
|
195
|
+
# L instancier a chaque conversion provoquait, en lot concurrent, la meme race
|
|
196
|
+
# meta-device que l embedding. On met en cache une instance par configuration
|
|
197
|
+
# (input_format, ocr, restriction de formats) et on la reutilise sous verrou.
|
|
198
|
+
_CONVERTER_CACHE: dict = {}
|
|
199
|
+
_CONVERTER_LOCK = threading.Lock()
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _get_converter(
|
|
203
|
+
input_format: InputFormat,
|
|
204
|
+
enable_ocr_flag: bool,
|
|
205
|
+
restrict_format: bool,
|
|
206
|
+
) -> DocumentConverter:
|
|
207
|
+
key = (input_format, enable_ocr_flag, restrict_format)
|
|
208
|
+
conv = _CONVERTER_CACHE.get(key)
|
|
209
|
+
if conv is not None:
|
|
210
|
+
return conv
|
|
211
|
+
with _CONVERTER_LOCK:
|
|
212
|
+
conv = _CONVERTER_CACHE.get(key)
|
|
213
|
+
if conv is None:
|
|
214
|
+
kwargs = {
|
|
215
|
+
"format_options": _build_format_options(input_format, enable_ocr_flag)
|
|
216
|
+
}
|
|
217
|
+
if restrict_format:
|
|
218
|
+
kwargs["allowed_formats"] = [input_format]
|
|
219
|
+
conv = DocumentConverter(**kwargs)
|
|
220
|
+
_CONVERTER_CACHE[key] = conv
|
|
221
|
+
return conv
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def convert_with_docling(
|
|
225
|
+
source: str,
|
|
226
|
+
doc_id: int = None,
|
|
227
|
+
input_format: InputFormat | None = None,
|
|
228
|
+
) -> dict:
|
|
229
|
+
"""
|
|
230
|
+
Generic Docling conversion for any supported format.
|
|
231
|
+
|
|
232
|
+
For PDF: two-pass strategy (fast scan then OCR if needed).
|
|
233
|
+
For images: OCR enabled by default (image documents are always raster).
|
|
234
|
+
For DOCX/PPTX: native text extraction; OCR on embedded images requires
|
|
235
|
+
LibreOffice (not available in this environment — logged as warning).
|
|
236
|
+
For HTML/Markdown/AsciiDoc: direct conversion via Docling pipelines.
|
|
237
|
+
|
|
238
|
+
Args:
|
|
239
|
+
source: Local file path.
|
|
240
|
+
doc_id: Optional knowledge DB id for S3 image naming.
|
|
241
|
+
input_format: Docling InputFormat. Inferred from extension if None.
|
|
242
|
+
|
|
243
|
+
Returns:
|
|
244
|
+
dict with keys: "document", "metadata".
|
|
245
|
+
"""
|
|
246
|
+
if input_format is None:
|
|
247
|
+
ext = os.path.splitext(source)[1].lower()
|
|
248
|
+
input_format = EXT_TO_INPUT_FORMAT.get(ext)
|
|
249
|
+
if input_format is None:
|
|
250
|
+
raise ValueError(f"Cannot infer Docling InputFormat from extension: {ext}")
|
|
251
|
+
|
|
252
|
+
doc_type = input_format.value
|
|
253
|
+
|
|
254
|
+
# Docling detects file format from the file extension in the path.
|
|
255
|
+
# Alias extensions (e.g. .markdown, .htm, .jpeg, .adoc) may not be
|
|
256
|
+
# recognised natively. We wrap the file in a DocumentStream with the
|
|
257
|
+
# canonical extension so Docling always gets a name it understands.
|
|
258
|
+
real_ext = os.path.splitext(source)[1].lower()
|
|
259
|
+
canonical_ext = _CANONICAL_EXT.get(input_format, real_ext)
|
|
260
|
+
if real_ext != canonical_ext:
|
|
261
|
+
base_name = os.path.splitext(os.path.basename(source))[0] + canonical_ext
|
|
262
|
+
with open(source, "rb") as fh:
|
|
263
|
+
docling_source: str | DocumentStream = DocumentStream(
|
|
264
|
+
name=base_name, stream=BytesIO(fh.read())
|
|
265
|
+
)
|
|
266
|
+
logger.debug(
|
|
267
|
+
f"Alias extension {real_ext} → stream with canonical name {base_name}"
|
|
268
|
+
)
|
|
269
|
+
else:
|
|
270
|
+
docling_source = source
|
|
271
|
+
|
|
272
|
+
# --- PDF: two-pass with adaptive OCR ---
|
|
273
|
+
if input_format == InputFormat.PDF:
|
|
274
|
+
logger.info(f"Pass 1: Fast scan for {source}")
|
|
275
|
+
converter = _get_converter(InputFormat.PDF, False, restrict_format=False)
|
|
276
|
+
result = converter.convert(docling_source)
|
|
277
|
+
doc = result.document
|
|
278
|
+
ocr_used = False
|
|
279
|
+
|
|
280
|
+
if enable_ocr(doc):
|
|
281
|
+
logger.info("Sparse text detected. Retrying with OCR...")
|
|
282
|
+
# Re-open the file for the second pass when using DocumentStream
|
|
283
|
+
if isinstance(docling_source, DocumentStream):
|
|
284
|
+
with open(source, "rb") as fh:
|
|
285
|
+
docling_source_2: str | DocumentStream = DocumentStream(
|
|
286
|
+
name=docling_source.name, stream=BytesIO(fh.read())
|
|
287
|
+
)
|
|
288
|
+
else:
|
|
289
|
+
docling_source_2 = docling_source
|
|
290
|
+
converter_ocr = _get_converter(InputFormat.PDF, True, restrict_format=False)
|
|
291
|
+
result = converter_ocr.convert(docling_source_2)
|
|
292
|
+
doc = result.document
|
|
293
|
+
ocr_used = True
|
|
294
|
+
|
|
295
|
+
extracted_images = handle_images(doc, source, doc_id=doc_id)
|
|
296
|
+
full_text = doc.export_to_markdown()
|
|
297
|
+
return {
|
|
298
|
+
"document": doc,
|
|
299
|
+
"metadata": {
|
|
300
|
+
"source": source,
|
|
301
|
+
"type": doc_type,
|
|
302
|
+
"ocr_used": ocr_used,
|
|
303
|
+
"token_count": len(full_text.split()) if full_text else 0,
|
|
304
|
+
"images": extracted_images,
|
|
305
|
+
},
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
# --- DOCX / PPTX: native text extraction + image extraction ---
|
|
309
|
+
if input_format in (InputFormat.DOCX, InputFormat.PPTX):
|
|
310
|
+
logger.info(
|
|
311
|
+
f"Converting {doc_type.upper()} via Docling native backend: {source}"
|
|
312
|
+
)
|
|
313
|
+
# Note: OCR on embedded DrawingML images requires LibreOffice (docx_to_pdf converter).
|
|
314
|
+
# get_docx_to_pdf_converter() returns None on this VM (LibreOffice not installed).
|
|
315
|
+
# Docling logs a warning and skips those images; native text is always extracted.
|
|
316
|
+
converter = _get_converter(input_format, False, restrict_format=True)
|
|
317
|
+
result = converter.convert(docling_source)
|
|
318
|
+
doc = result.document
|
|
319
|
+
extracted_images = handle_images(doc, source, doc_id=doc_id)
|
|
320
|
+
full_text = doc.export_to_markdown()
|
|
321
|
+
|
|
322
|
+
if not full_text.strip():
|
|
323
|
+
logger.warning(
|
|
324
|
+
f"{doc_type.upper()} produced 0 text. Document may be graphical-only. "
|
|
325
|
+
"LibreOffice is required for DOCX→PDF→OCR fallback (not installed)."
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
return {
|
|
329
|
+
"document": doc,
|
|
330
|
+
"metadata": {
|
|
331
|
+
"source": source,
|
|
332
|
+
"type": doc_type,
|
|
333
|
+
"ocr_used": False,
|
|
334
|
+
"token_count": len(full_text.split()) if full_text else 0,
|
|
335
|
+
"images": extracted_images,
|
|
336
|
+
},
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
# --- Images: OCR enabled by default ---
|
|
340
|
+
if input_format == InputFormat.IMAGE:
|
|
341
|
+
logger.info(f"Converting image with OCR: {source}")
|
|
342
|
+
converter = _get_converter(InputFormat.IMAGE, True, restrict_format=True)
|
|
343
|
+
result = converter.convert(docling_source)
|
|
344
|
+
doc = result.document
|
|
345
|
+
full_text = doc.export_to_markdown()
|
|
346
|
+
return {
|
|
347
|
+
"document": doc,
|
|
348
|
+
"metadata": {
|
|
349
|
+
"source": source,
|
|
350
|
+
"type": "image",
|
|
351
|
+
"ocr_used": True,
|
|
352
|
+
"token_count": len(full_text.split()) if full_text else 0,
|
|
353
|
+
"images": [],
|
|
354
|
+
},
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
# --- HTML / Markdown / AsciiDoc: simple pipeline ---
|
|
358
|
+
logger.info(f"Converting {doc_type} via Docling simple pipeline: {source}")
|
|
359
|
+
converter = _get_converter(input_format, False, restrict_format=True)
|
|
360
|
+
result = converter.convert(docling_source)
|
|
361
|
+
doc = result.document
|
|
362
|
+
full_text = doc.export_to_markdown()
|
|
363
|
+
return {
|
|
364
|
+
"document": doc,
|
|
365
|
+
"metadata": {
|
|
366
|
+
"source": source,
|
|
367
|
+
"type": doc_type,
|
|
368
|
+
"ocr_used": False,
|
|
369
|
+
"token_count": len(full_text.split()) if full_text else 0,
|
|
370
|
+
"images": [],
|
|
371
|
+
},
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def convert_pdf(source: str, doc_id: int = None) -> dict:
|
|
376
|
+
"""
|
|
377
|
+
Convert a PDF to an internal document representation.
|
|
378
|
+
|
|
379
|
+
Delegates to convert_with_docling for consistency.
|
|
380
|
+
Kept for backward compatibility with existing callers.
|
|
381
|
+
"""
|
|
382
|
+
return convert_with_docling(
|
|
383
|
+
source=source, doc_id=doc_id, input_format=InputFormat.PDF
|
|
384
|
+
)
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
from docling_core.types.doc import DoclingDocument
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def _try_read_csv(source: str) -> pd.DataFrame:
|
|
6
|
+
"""
|
|
7
|
+
Attempt to read a CSV file with multiple fallback strategies to handle
|
|
8
|
+
malformed files, inconsistent columns, mixed encodings, etc.
|
|
9
|
+
"""
|
|
10
|
+
encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252', 'iso-8859-1']
|
|
11
|
+
separators = [None, ',', ';', '\t', '|'] # None = auto-detect
|
|
12
|
+
|
|
13
|
+
last_error = None
|
|
14
|
+
|
|
15
|
+
for encoding in encodings:
|
|
16
|
+
for sep in separators:
|
|
17
|
+
try:
|
|
18
|
+
kwargs = dict(
|
|
19
|
+
encoding=encoding,
|
|
20
|
+
encoding_errors='replace', # never crash on bad bytes
|
|
21
|
+
on_bad_lines='skip', # skip rows with wrong column count
|
|
22
|
+
engine='python', # most tolerant engine
|
|
23
|
+
skip_blank_lines=True,
|
|
24
|
+
dtype=str, # read everything as string first — safest
|
|
25
|
+
)
|
|
26
|
+
if sep is not None:
|
|
27
|
+
kwargs['sep'] = sep
|
|
28
|
+
else:
|
|
29
|
+
kwargs['sep'] = None # python engine auto-detects
|
|
30
|
+
|
|
31
|
+
df = pd.read_csv(source, **kwargs)
|
|
32
|
+
|
|
33
|
+
# If we got a single-column DataFrame, the separator is probably wrong — try next
|
|
34
|
+
if len(df.columns) == 1 and sep is None:
|
|
35
|
+
# Could be a real single-col file; keep it but continue checking
|
|
36
|
+
pass
|
|
37
|
+
|
|
38
|
+
# Drop columns that are entirely unnamed AND entirely empty
|
|
39
|
+
unnamed_empty = [
|
|
40
|
+
col for col in df.columns
|
|
41
|
+
if str(col).startswith('Unnamed:') and df[col].isna().all()
|
|
42
|
+
]
|
|
43
|
+
if unnamed_empty:
|
|
44
|
+
df = df.drop(columns=unnamed_empty)
|
|
45
|
+
|
|
46
|
+
# Drop rows that are entirely empty
|
|
47
|
+
df = df.dropna(how='all')
|
|
48
|
+
|
|
49
|
+
# Strip whitespace from column names
|
|
50
|
+
df.columns = [str(c).strip() for c in df.columns]
|
|
51
|
+
|
|
52
|
+
# Remove completely duplicate columns
|
|
53
|
+
df = df.loc[:, ~df.columns.duplicated()]
|
|
54
|
+
|
|
55
|
+
if df.empty or len(df.columns) == 0:
|
|
56
|
+
raise ValueError("DataFrame is empty after cleaning")
|
|
57
|
+
|
|
58
|
+
return df
|
|
59
|
+
|
|
60
|
+
except (pd.errors.ParserError, pd.errors.EmptyDataError, ValueError) as e:
|
|
61
|
+
last_error = e
|
|
62
|
+
continue
|
|
63
|
+
except UnicodeDecodeError as e:
|
|
64
|
+
last_error = e
|
|
65
|
+
continue
|
|
66
|
+
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f"Could not parse CSV file after trying all encodings and separators. "
|
|
69
|
+
f"Last error: {last_error}"
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def convert_spreadsheet(source: str) -> dict:
|
|
74
|
+
try:
|
|
75
|
+
# Validate file extension
|
|
76
|
+
if not (source.lower().endswith('.csv') or source.lower().endswith('.xlsx')):
|
|
77
|
+
raise ValueError(f"Unsupported spreadsheet format: {source}")
|
|
78
|
+
|
|
79
|
+
# Read the file based on extension
|
|
80
|
+
if source.lower().endswith('.csv'):
|
|
81
|
+
df = _try_read_csv(source)
|
|
82
|
+
|
|
83
|
+
elif source.lower().endswith('.xlsx'):
|
|
84
|
+
try:
|
|
85
|
+
df = pd.read_excel(source, engine='openpyxl', dtype=str)
|
|
86
|
+
except Exception:
|
|
87
|
+
# Fallback: try reading without dtype constraint
|
|
88
|
+
df = pd.read_excel(source, engine='openpyxl')
|
|
89
|
+
|
|
90
|
+
# Same cleanup as CSV
|
|
91
|
+
unnamed_empty = [
|
|
92
|
+
col for col in df.columns
|
|
93
|
+
if str(col).startswith('Unnamed:') and df[col].isna().all()
|
|
94
|
+
]
|
|
95
|
+
if unnamed_empty:
|
|
96
|
+
df = df.drop(columns=unnamed_empty)
|
|
97
|
+
df = df.dropna(how='all')
|
|
98
|
+
df.columns = [str(c).strip() for c in df.columns]
|
|
99
|
+
df = df.loc[:, ~df.columns.duplicated()]
|
|
100
|
+
|
|
101
|
+
if df.empty:
|
|
102
|
+
raise ValueError(f"Spreadsheet is empty or has no usable data: {source}")
|
|
103
|
+
|
|
104
|
+
MAX_ROWS = 10000
|
|
105
|
+
if len(df) > MAX_ROWS:
|
|
106
|
+
print(f"Warning: Spreadsheet has {len(df)} rows. Truncating to {MAX_ROWS} rows.")
|
|
107
|
+
df = df.head(MAX_ROWS)
|
|
108
|
+
|
|
109
|
+
# Convert to markdown
|
|
110
|
+
try:
|
|
111
|
+
# Fill NaN with empty string for cleaner markdown output
|
|
112
|
+
text_content = df.fillna('').to_markdown(index=False)
|
|
113
|
+
except Exception as e:
|
|
114
|
+
print(f"Warning: to_markdown failed ({e}), falling back to CSV string.")
|
|
115
|
+
text_content = df.fillna('').to_csv(index=False)
|
|
116
|
+
|
|
117
|
+
# Create DoclingDocument
|
|
118
|
+
doc = DoclingDocument(name=source)
|
|
119
|
+
|
|
120
|
+
if len(text_content) > 5000:
|
|
121
|
+
lines = text_content.split('\n')
|
|
122
|
+
|
|
123
|
+
# Always include the header rows (first 2 lines for markdown: header + separator)
|
|
124
|
+
HEADER_LINES = 2
|
|
125
|
+
if lines:
|
|
126
|
+
header_text = '\n'.join(lines[:HEADER_LINES])
|
|
127
|
+
doc.add_text(text=header_text, label="text")
|
|
128
|
+
|
|
129
|
+
chunk_size = 50
|
|
130
|
+
for i in range(HEADER_LINES, len(lines), chunk_size):
|
|
131
|
+
chunk = '\n'.join(lines[i:i + chunk_size])
|
|
132
|
+
if chunk.strip():
|
|
133
|
+
doc.add_text(text=chunk, label="text")
|
|
134
|
+
else:
|
|
135
|
+
doc.add_text(text=text_content, label="text")
|
|
136
|
+
|
|
137
|
+
#Metadata
|
|
138
|
+
metadata = {
|
|
139
|
+
"source": source,
|
|
140
|
+
"type": "spreadsheet",
|
|
141
|
+
"rows": len(df),
|
|
142
|
+
"columns": len(df.columns),
|
|
143
|
+
"column_names": list(df.columns),
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
return {
|
|
147
|
+
"document": doc,
|
|
148
|
+
"metadata": metadata,
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
except pd.errors.EmptyDataError:
|
|
152
|
+
raise ValueError(f"Spreadsheet file is empty or invalid: {source}")
|
|
153
|
+
except pd.errors.ParserError as e:
|
|
154
|
+
raise ValueError(f"Failed to parse spreadsheet: {source}. Error: {str(e)}")
|
|
155
|
+
except FileNotFoundError:
|
|
156
|
+
raise FileNotFoundError(f"Spreadsheet file not found: {source}")
|
|
157
|
+
except Exception as e:
|
|
158
|
+
print(f"Error reading spreadsheet {source}: {e}")
|
|
159
|
+
raise Exception(f"Failed to convert spreadsheet: {str(e)}")
|