th2rag 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. th2rag/__init__.py +0 -0
  2. th2rag/auth/__init__.py +0 -0
  3. th2rag/auth/exceptions.py +2 -0
  4. th2rag/auth/router.py +43 -0
  5. th2rag/auth/schemas.py +15 -0
  6. th2rag/auth/service.py +115 -0
  7. th2rag/clients/__init__.py +0 -0
  8. th2rag/clients/storage/config.py +13 -0
  9. th2rag/clients/storage/local.py +48 -0
  10. th2rag/clients/storage/s3.py +93 -0
  11. th2rag/config.py +267 -0
  12. th2rag/conversations/__init__.py +0 -0
  13. th2rag/conversations/dependencies.py +30 -0
  14. th2rag/conversations/exceptions.py +6 -0
  15. th2rag/conversations/router.py +47 -0
  16. th2rag/conversations/schemas.py +43 -0
  17. th2rag/conversations/service.py +51 -0
  18. th2rag/database.py +72 -0
  19. th2rag/dataset/__init__.py +0 -0
  20. th2rag/dataset/dependencies.py +25 -0
  21. th2rag/dataset/exceptions.py +2 -0
  22. th2rag/dataset/router.py +53 -0
  23. th2rag/dataset/schemas.py +23 -0
  24. th2rag/dataset/service.py +78 -0
  25. th2rag/dependencies.py +143 -0
  26. th2rag/knowledge/__init__.py +0 -0
  27. th2rag/knowledge/dependencies.py +25 -0
  28. th2rag/knowledge/exceptions.py +2 -0
  29. th2rag/knowledge/mage_client.py +505 -0
  30. th2rag/knowledge/router.py +714 -0
  31. th2rag/knowledge/schemas.py +108 -0
  32. th2rag/knowledge/service.py +392 -0
  33. th2rag/logger.py +21 -0
  34. th2rag/main.py +51 -0
  35. th2rag/messages/__init__.py +0 -0
  36. th2rag/messages/exceptions.py +2 -0
  37. th2rag/messages/router.py +63 -0
  38. th2rag/messages/schemas.py +68 -0
  39. th2rag/messages/service.py +210 -0
  40. th2rag/models.py +215 -0
  41. th2rag/pagination.py +68 -0
  42. th2rag/rag/__init__.py +0 -0
  43. th2rag/rag/chunkers/__init__.py +0 -0
  44. th2rag/rag/chunkers/hybrid_chunker.py +23 -0
  45. th2rag/rag/constants.py +15 -0
  46. th2rag/rag/converters/__init__.py +0 -0
  47. th2rag/rag/converters/documents_converter.py +123 -0
  48. th2rag/rag/converters/json_converter.py +521 -0
  49. th2rag/rag/converters/legacy_converter.py +124 -0
  50. th2rag/rag/converters/pdf_converter.py +384 -0
  51. th2rag/rag/converters/spreadsheet_converter.py +159 -0
  52. th2rag/rag/converters/txt_converter.py +98 -0
  53. th2rag/rag/embeddings/__init__.py +0 -0
  54. th2rag/rag/embeddings/base.py +21 -0
  55. th2rag/rag/embeddings/exceptions.py +2 -0
  56. th2rag/rag/embeddings/sentence_transformer.py +97 -0
  57. th2rag/rag/generation/base.py +22 -0
  58. th2rag/rag/generation/exceptions.py +2 -0
  59. th2rag/rag/generation/litellm_generator.py +143 -0
  60. th2rag/rag/generation/mistral_generator.py +269 -0
  61. th2rag/rag/generation/test_litellm.py +33 -0
  62. th2rag/rag/retrieval/__init__.py +0 -0
  63. th2rag/rag/retrieval/base.py +11 -0
  64. th2rag/rag/retrieval/exceptions.py +10 -0
  65. th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
  66. th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
  67. th2rag/rag/retrieval/lancedb/schemas.py +19 -0
  68. th2rag/rag/services/__init__.py +0 -0
  69. th2rag/rag/services/rag_service.py +99 -0
  70. th2rag/rag/services/rag_service_builder.py +42 -0
  71. th2rag/rag/tasks/process_pdf.py +586 -0
  72. th2rag/users/dependencies.py +44 -0
  73. th2rag/users/exceptions.py +6 -0
  74. th2rag/users/router.py +92 -0
  75. th2rag/users/schemas.py +61 -0
  76. th2rag/users/service.py +140 -0
  77. th2rag/utils/__init__.py +0 -0
  78. th2rag/utils/helpers.py +40 -0
  79. th2rag/utils/logger.py +15 -0
  80. th2rag/utils/security.py +76 -0
  81. th2rag/utils/welcome_router.py +98 -0
  82. th2rag/webhooks/__init__.py +0 -0
  83. th2rag/webhooks/sender.py +126 -0
  84. th2rag-0.0.2.dist-info/METADATA +1002 -0
  85. th2rag-0.0.2.dist-info/RECORD +87 -0
  86. th2rag-0.0.2.dist-info/WHEEL +4 -0
  87. th2rag-0.0.2.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,124 @@
1
+ """
2
+ Conversion de formats legacy via LibreOffice headless.
3
+
4
+ Formats supportés : .doc → .docx, .ppt → .pptx, .xls → .xlsx
5
+ Pré-requis runtime : soffice ou libreoffice dans le PATH.
6
+
7
+ Si LibreOffice est absent, lève ValueError avec message explicite.
8
+ Ne crashe jamais avec FileNotFoundError ou CalledProcessError.
9
+ """
10
+ import logging
11
+ import os
12
+ import shutil
13
+ import subprocess
14
+ import tempfile
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ # Mapping extension legacy → extension cible Docling-compatible
19
+ _LEGACY_TO_TARGET: dict[str, str] = {
20
+ ".doc": ".docx",
21
+ ".ppt": ".pptx",
22
+ ".xls": ".xlsx",
23
+ }
24
+
25
+ LEGACY_EXTENSIONS: frozenset[str] = frozenset(_LEGACY_TO_TARGET.keys())
26
+
27
+
28
+ def _find_soffice() -> str | None:
29
+ """Retourne le chemin de soffice/libreoffice, ou None si absent."""
30
+ return shutil.which("soffice") or shutil.which("libreoffice")
31
+
32
+
33
+ def convert_with_libreoffice(source: str) -> str:
34
+ """
35
+ Convertit un fichier legacy (.doc/.ppt/.xls) en format moderne
36
+ via LibreOffice headless, dans un répertoire temporaire.
37
+
38
+ Args:
39
+ source: Chemin absolu vers le fichier source.
40
+
41
+ Returns:
42
+ Chemin absolu vers le fichier converti (dans un tmpdir).
43
+ L'appelant est responsable de nettoyer ce tmpdir.
44
+
45
+ Raises:
46
+ ValueError: Si LibreOffice est absent du PATH.
47
+ ValueError: Si la conversion échoue (returncode != 0 ou fichier absent).
48
+ """
49
+ soffice = _find_soffice()
50
+ if soffice is None:
51
+ ext = os.path.splitext(source)[1].lower()
52
+ target = _LEGACY_TO_TARGET.get(ext, ".docx")
53
+ raise ValueError(
54
+ f"Format {ext} non supporté sans LibreOffice. "
55
+ f"Installez LibreOffice (apt install libreoffice) pour convertir "
56
+ f"{ext} → {target}. "
57
+ f"Formats acceptés sans LibreOffice : PDF, DOCX, PPTX, XLSX, CSV, TXT, "
58
+ f"HTML, Markdown, images, AsciiDoc, JSON."
59
+ )
60
+
61
+ ext = os.path.splitext(source)[1].lower()
62
+ if ext not in _LEGACY_TO_TARGET:
63
+ raise ValueError(
64
+ f"convert_with_libreoffice: extension {ext} non supportée. "
65
+ f"Extensions legacy supportées : {sorted(LEGACY_EXTENSIONS)}"
66
+ )
67
+
68
+ target_ext = _LEGACY_TO_TARGET[ext]
69
+ tmpdir = tempfile.mkdtemp(prefix="th2rag_legacy_")
70
+
71
+ logger.info(f"Converting legacy {ext} → {target_ext} via LibreOffice: {source}")
72
+
73
+ _success = False
74
+ try:
75
+ result = subprocess.run(
76
+ [soffice, "--headless", "--convert-to", target_ext.lstrip("."),
77
+ "--outdir", tmpdir, source],
78
+ capture_output=True,
79
+ text=True,
80
+ timeout=120,
81
+ )
82
+
83
+ if result.returncode != 0:
84
+ raise ValueError(
85
+ f"LibreOffice conversion failed for {os.path.basename(source)} "
86
+ f"(returncode={result.returncode}): {result.stderr[:500]}"
87
+ )
88
+
89
+ # Chercher le fichier converti dans tmpdir
90
+ basename = os.path.splitext(os.path.basename(source))[0]
91
+ converted_path = os.path.join(tmpdir, basename + target_ext)
92
+
93
+ if not os.path.exists(converted_path):
94
+ # Parfois soffice crée le fichier sans l'extension demandée
95
+ candidates = [
96
+ f for f in os.listdir(tmpdir)
97
+ if f.startswith(basename) and f.endswith(target_ext)
98
+ ]
99
+ if candidates:
100
+ converted_path = os.path.join(tmpdir, candidates[0])
101
+ else:
102
+ raise ValueError(
103
+ f"LibreOffice n'a pas produit le fichier attendu : {converted_path}. "
104
+ f"Contenu de {tmpdir}: {os.listdir(tmpdir)}"
105
+ )
106
+
107
+ logger.info(f"LibreOffice conversion successful: {converted_path}")
108
+ _success = True
109
+ return converted_path
110
+
111
+ except subprocess.TimeoutExpired as e:
112
+ raise ValueError(
113
+ f"LibreOffice conversion timeout (120s) pour {os.path.basename(source)}"
114
+ ) from e
115
+ except ValueError:
116
+ raise
117
+ except Exception as e:
118
+ raise ValueError(
119
+ f"Erreur inattendue lors de la conversion LibreOffice de "
120
+ f"{os.path.basename(source)}: {e}"
121
+ ) from e
122
+ finally:
123
+ if not _success:
124
+ shutil.rmtree(tmpdir, ignore_errors=True)
@@ -0,0 +1,384 @@
1
+ import logging
2
+ import os
3
+ import threading
4
+ import uuid
5
+ from io import BytesIO
6
+
7
+ from docling.datamodel.base_models import InputFormat
8
+ from docling.datamodel.document import DocumentStream
9
+ from docling.datamodel.pipeline_options import (
10
+ PdfPipelineOptions,
11
+ ThreadedPdfPipelineOptions,
12
+ )
13
+ from docling.document_converter import (
14
+ AsciiDocFormatOption,
15
+ DocumentConverter,
16
+ HTMLFormatOption,
17
+ ImageFormatOption,
18
+ MarkdownFormatOption,
19
+ PdfFormatOption,
20
+ PowerpointFormatOption,
21
+ WordFormatOption,
22
+ )
23
+
24
+ from th2rag.clients.storage.s3 import upload_file_to_s3
25
+ from th2rag.config import settings
26
+
27
+ logger = logging.getLogger(__name__)
28
+
29
+ TOKEN_THRESHOLD = 50
30
+
31
+ # Mapping from extension to Docling InputFormat
32
+ EXT_TO_INPUT_FORMAT: dict[str, InputFormat] = {
33
+ ".pdf": InputFormat.PDF,
34
+ ".docx": InputFormat.DOCX,
35
+ ".pptx": InputFormat.PPTX,
36
+ ".html": InputFormat.HTML,
37
+ ".htm": InputFormat.HTML,
38
+ ".md": InputFormat.MD,
39
+ ".markdown": InputFormat.MD,
40
+ ".png": InputFormat.IMAGE,
41
+ ".jpg": InputFormat.IMAGE,
42
+ ".jpeg": InputFormat.IMAGE,
43
+ ".tiff": InputFormat.IMAGE,
44
+ ".tif": InputFormat.IMAGE,
45
+ ".bmp": InputFormat.IMAGE,
46
+ ".adoc": InputFormat.ASCIIDOC,
47
+ ".asciidoc": InputFormat.ASCIIDOC,
48
+ }
49
+
50
+ # Canonical extension per InputFormat (used when the real extension is an alias
51
+ # that Docling does not recognise natively, e.g. .markdown, .htm, .jpeg).
52
+ _CANONICAL_EXT: dict[InputFormat, str] = {
53
+ InputFormat.PDF: ".pdf",
54
+ InputFormat.DOCX: ".docx",
55
+ InputFormat.PPTX: ".pptx",
56
+ InputFormat.HTML: ".html",
57
+ InputFormat.MD: ".md",
58
+ InputFormat.IMAGE: ".png",
59
+ InputFormat.ASCIIDOC: ".adoc",
60
+ }
61
+
62
+
63
+ def count_tokens_simple(doc) -> int:
64
+ text = doc.export_to_markdown()
65
+ return len(text.split()) if text else 0
66
+
67
+
68
+ def get_pipeline_options(enable_ocr: bool) -> PdfPipelineOptions:
69
+ options = PdfPipelineOptions()
70
+ options.do_table_structure = True
71
+ options.table_structure_options.mode = "fast"
72
+ options.do_ocr = enable_ocr
73
+ options.generate_picture_images = True
74
+
75
+ if settings.handle_img_mode == "describe":
76
+ options.do_picture_description = True
77
+ else:
78
+ options.do_picture_description = False
79
+ return options
80
+
81
+
82
+ def get_image_pipeline_options(enable_ocr: bool) -> ThreadedPdfPipelineOptions:
83
+ """Build pipeline options for image documents (ThreadedPdfPipelineOptions)."""
84
+ options = ThreadedPdfPipelineOptions()
85
+ options.do_ocr = enable_ocr
86
+ options.do_table_structure = False
87
+ options.generate_picture_images = False
88
+ return options
89
+
90
+
91
+ def enable_ocr(doc) -> bool:
92
+ """
93
+ Return True when any page has fewer than TOKEN_THRESHOLD words,
94
+ signalling a scanned/image-only document.
95
+ """
96
+ if not doc.pages:
97
+ return True
98
+
99
+ for page_no, _page in doc.pages.items():
100
+ text_items = [
101
+ item.text
102
+ for item in doc.texts
103
+ if item.prov and item.prov[0].page_no == page_no
104
+ ]
105
+ page_text = " ".join(text_items)
106
+ word_count = len(page_text.split())
107
+
108
+ if word_count < TOKEN_THRESHOLD:
109
+ logger.warning(
110
+ f"Page {page_no} has only {word_count} tokens. Triggering OCR."
111
+ )
112
+ return True
113
+
114
+ return False
115
+
116
+
117
+ def handle_images(doc, source_filename: str, doc_id: int = None) -> list[dict]:
118
+ processed_images = []
119
+ if not hasattr(doc, "pictures"):
120
+ return processed_images
121
+
122
+ batch_id = str(uuid.uuid4())[:8]
123
+
124
+ for i, picture in enumerate(doc.pictures):
125
+ try:
126
+ image_obj = picture.get_image(doc)
127
+ if image_obj is None:
128
+ logger.warning(f"Image {i} could not be retrieved/generated.")
129
+ continue
130
+
131
+ img_byte_arr = BytesIO()
132
+ image_obj.save(img_byte_arr, format="PNG")
133
+ img_bytes = img_byte_arr.getvalue()
134
+
135
+ base_name = os.path.basename(source_filename)
136
+ name_without_ext, _ = os.path.splitext(base_name)
137
+ img_filename = f"{name_without_ext}_img_{batch_id}_{i}.png"
138
+
139
+ image_link = upload_file_to_s3(img_bytes, img_filename, doc_id=doc_id)
140
+
141
+ page_number = picture.prov[0].page_no if picture.prov else 1
142
+ description = f"Image extracted from page {page_number}"
143
+
144
+ processed_images.append(
145
+ {
146
+ "image_index": i,
147
+ "link": image_link,
148
+ "description": description,
149
+ "page": page_number,
150
+ }
151
+ )
152
+ logger.info(f"Image processed: {image_link}")
153
+
154
+ except Exception as e:
155
+ logger.error(f"Failed to process image {i}: {e}")
156
+
157
+ return processed_images
158
+
159
+
160
+ def _build_format_options(input_format: InputFormat, enable_ocr_flag: bool) -> dict:
161
+ """Return format_options dict for DocumentConverter based on input_format."""
162
+ if input_format == InputFormat.PDF:
163
+ return {
164
+ InputFormat.PDF: PdfFormatOption(
165
+ pipeline_options=get_pipeline_options(enable_ocr_flag)
166
+ )
167
+ }
168
+
169
+ if input_format == InputFormat.DOCX:
170
+ return {InputFormat.DOCX: WordFormatOption()}
171
+
172
+ if input_format == InputFormat.PPTX:
173
+ return {InputFormat.PPTX: PowerpointFormatOption()}
174
+
175
+ if input_format == InputFormat.IMAGE:
176
+ return {
177
+ InputFormat.IMAGE: ImageFormatOption(
178
+ pipeline_options=get_image_pipeline_options(enable_ocr_flag)
179
+ )
180
+ }
181
+
182
+ if input_format == InputFormat.HTML:
183
+ return {InputFormat.HTML: HTMLFormatOption()}
184
+
185
+ if input_format == InputFormat.MD:
186
+ return {InputFormat.MD: MarkdownFormatOption()}
187
+
188
+ if input_format == InputFormat.ASCIIDOC:
189
+ return {InputFormat.ASCIIDOC: AsciiDocFormatOption()}
190
+
191
+ return {}
192
+
193
+
194
+ # DocumentConverter charge des modeles PyTorch (layout / table-structure / OCR).
195
+ # L instancier a chaque conversion provoquait, en lot concurrent, la meme race
196
+ # meta-device que l embedding. On met en cache une instance par configuration
197
+ # (input_format, ocr, restriction de formats) et on la reutilise sous verrou.
198
+ _CONVERTER_CACHE: dict = {}
199
+ _CONVERTER_LOCK = threading.Lock()
200
+
201
+
202
+ def _get_converter(
203
+ input_format: InputFormat,
204
+ enable_ocr_flag: bool,
205
+ restrict_format: bool,
206
+ ) -> DocumentConverter:
207
+ key = (input_format, enable_ocr_flag, restrict_format)
208
+ conv = _CONVERTER_CACHE.get(key)
209
+ if conv is not None:
210
+ return conv
211
+ with _CONVERTER_LOCK:
212
+ conv = _CONVERTER_CACHE.get(key)
213
+ if conv is None:
214
+ kwargs = {
215
+ "format_options": _build_format_options(input_format, enable_ocr_flag)
216
+ }
217
+ if restrict_format:
218
+ kwargs["allowed_formats"] = [input_format]
219
+ conv = DocumentConverter(**kwargs)
220
+ _CONVERTER_CACHE[key] = conv
221
+ return conv
222
+
223
+
224
+ def convert_with_docling(
225
+ source: str,
226
+ doc_id: int = None,
227
+ input_format: InputFormat | None = None,
228
+ ) -> dict:
229
+ """
230
+ Generic Docling conversion for any supported format.
231
+
232
+ For PDF: two-pass strategy (fast scan then OCR if needed).
233
+ For images: OCR enabled by default (image documents are always raster).
234
+ For DOCX/PPTX: native text extraction; OCR on embedded images requires
235
+ LibreOffice (not available in this environment — logged as warning).
236
+ For HTML/Markdown/AsciiDoc: direct conversion via Docling pipelines.
237
+
238
+ Args:
239
+ source: Local file path.
240
+ doc_id: Optional knowledge DB id for S3 image naming.
241
+ input_format: Docling InputFormat. Inferred from extension if None.
242
+
243
+ Returns:
244
+ dict with keys: "document", "metadata".
245
+ """
246
+ if input_format is None:
247
+ ext = os.path.splitext(source)[1].lower()
248
+ input_format = EXT_TO_INPUT_FORMAT.get(ext)
249
+ if input_format is None:
250
+ raise ValueError(f"Cannot infer Docling InputFormat from extension: {ext}")
251
+
252
+ doc_type = input_format.value
253
+
254
+ # Docling detects file format from the file extension in the path.
255
+ # Alias extensions (e.g. .markdown, .htm, .jpeg, .adoc) may not be
256
+ # recognised natively. We wrap the file in a DocumentStream with the
257
+ # canonical extension so Docling always gets a name it understands.
258
+ real_ext = os.path.splitext(source)[1].lower()
259
+ canonical_ext = _CANONICAL_EXT.get(input_format, real_ext)
260
+ if real_ext != canonical_ext:
261
+ base_name = os.path.splitext(os.path.basename(source))[0] + canonical_ext
262
+ with open(source, "rb") as fh:
263
+ docling_source: str | DocumentStream = DocumentStream(
264
+ name=base_name, stream=BytesIO(fh.read())
265
+ )
266
+ logger.debug(
267
+ f"Alias extension {real_ext} → stream with canonical name {base_name}"
268
+ )
269
+ else:
270
+ docling_source = source
271
+
272
+ # --- PDF: two-pass with adaptive OCR ---
273
+ if input_format == InputFormat.PDF:
274
+ logger.info(f"Pass 1: Fast scan for {source}")
275
+ converter = _get_converter(InputFormat.PDF, False, restrict_format=False)
276
+ result = converter.convert(docling_source)
277
+ doc = result.document
278
+ ocr_used = False
279
+
280
+ if enable_ocr(doc):
281
+ logger.info("Sparse text detected. Retrying with OCR...")
282
+ # Re-open the file for the second pass when using DocumentStream
283
+ if isinstance(docling_source, DocumentStream):
284
+ with open(source, "rb") as fh:
285
+ docling_source_2: str | DocumentStream = DocumentStream(
286
+ name=docling_source.name, stream=BytesIO(fh.read())
287
+ )
288
+ else:
289
+ docling_source_2 = docling_source
290
+ converter_ocr = _get_converter(InputFormat.PDF, True, restrict_format=False)
291
+ result = converter_ocr.convert(docling_source_2)
292
+ doc = result.document
293
+ ocr_used = True
294
+
295
+ extracted_images = handle_images(doc, source, doc_id=doc_id)
296
+ full_text = doc.export_to_markdown()
297
+ return {
298
+ "document": doc,
299
+ "metadata": {
300
+ "source": source,
301
+ "type": doc_type,
302
+ "ocr_used": ocr_used,
303
+ "token_count": len(full_text.split()) if full_text else 0,
304
+ "images": extracted_images,
305
+ },
306
+ }
307
+
308
+ # --- DOCX / PPTX: native text extraction + image extraction ---
309
+ if input_format in (InputFormat.DOCX, InputFormat.PPTX):
310
+ logger.info(
311
+ f"Converting {doc_type.upper()} via Docling native backend: {source}"
312
+ )
313
+ # Note: OCR on embedded DrawingML images requires LibreOffice (docx_to_pdf converter).
314
+ # get_docx_to_pdf_converter() returns None on this VM (LibreOffice not installed).
315
+ # Docling logs a warning and skips those images; native text is always extracted.
316
+ converter = _get_converter(input_format, False, restrict_format=True)
317
+ result = converter.convert(docling_source)
318
+ doc = result.document
319
+ extracted_images = handle_images(doc, source, doc_id=doc_id)
320
+ full_text = doc.export_to_markdown()
321
+
322
+ if not full_text.strip():
323
+ logger.warning(
324
+ f"{doc_type.upper()} produced 0 text. Document may be graphical-only. "
325
+ "LibreOffice is required for DOCX→PDF→OCR fallback (not installed)."
326
+ )
327
+
328
+ return {
329
+ "document": doc,
330
+ "metadata": {
331
+ "source": source,
332
+ "type": doc_type,
333
+ "ocr_used": False,
334
+ "token_count": len(full_text.split()) if full_text else 0,
335
+ "images": extracted_images,
336
+ },
337
+ }
338
+
339
+ # --- Images: OCR enabled by default ---
340
+ if input_format == InputFormat.IMAGE:
341
+ logger.info(f"Converting image with OCR: {source}")
342
+ converter = _get_converter(InputFormat.IMAGE, True, restrict_format=True)
343
+ result = converter.convert(docling_source)
344
+ doc = result.document
345
+ full_text = doc.export_to_markdown()
346
+ return {
347
+ "document": doc,
348
+ "metadata": {
349
+ "source": source,
350
+ "type": "image",
351
+ "ocr_used": True,
352
+ "token_count": len(full_text.split()) if full_text else 0,
353
+ "images": [],
354
+ },
355
+ }
356
+
357
+ # --- HTML / Markdown / AsciiDoc: simple pipeline ---
358
+ logger.info(f"Converting {doc_type} via Docling simple pipeline: {source}")
359
+ converter = _get_converter(input_format, False, restrict_format=True)
360
+ result = converter.convert(docling_source)
361
+ doc = result.document
362
+ full_text = doc.export_to_markdown()
363
+ return {
364
+ "document": doc,
365
+ "metadata": {
366
+ "source": source,
367
+ "type": doc_type,
368
+ "ocr_used": False,
369
+ "token_count": len(full_text.split()) if full_text else 0,
370
+ "images": [],
371
+ },
372
+ }
373
+
374
+
375
+ def convert_pdf(source: str, doc_id: int = None) -> dict:
376
+ """
377
+ Convert a PDF to an internal document representation.
378
+
379
+ Delegates to convert_with_docling for consistency.
380
+ Kept for backward compatibility with existing callers.
381
+ """
382
+ return convert_with_docling(
383
+ source=source, doc_id=doc_id, input_format=InputFormat.PDF
384
+ )
@@ -0,0 +1,159 @@
1
+ import pandas as pd
2
+ from docling_core.types.doc import DoclingDocument
3
+
4
+
5
+ def _try_read_csv(source: str) -> pd.DataFrame:
6
+ """
7
+ Attempt to read a CSV file with multiple fallback strategies to handle
8
+ malformed files, inconsistent columns, mixed encodings, etc.
9
+ """
10
+ encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252', 'iso-8859-1']
11
+ separators = [None, ',', ';', '\t', '|'] # None = auto-detect
12
+
13
+ last_error = None
14
+
15
+ for encoding in encodings:
16
+ for sep in separators:
17
+ try:
18
+ kwargs = dict(
19
+ encoding=encoding,
20
+ encoding_errors='replace', # never crash on bad bytes
21
+ on_bad_lines='skip', # skip rows with wrong column count
22
+ engine='python', # most tolerant engine
23
+ skip_blank_lines=True,
24
+ dtype=str, # read everything as string first — safest
25
+ )
26
+ if sep is not None:
27
+ kwargs['sep'] = sep
28
+ else:
29
+ kwargs['sep'] = None # python engine auto-detects
30
+
31
+ df = pd.read_csv(source, **kwargs)
32
+
33
+ # If we got a single-column DataFrame, the separator is probably wrong — try next
34
+ if len(df.columns) == 1 and sep is None:
35
+ # Could be a real single-col file; keep it but continue checking
36
+ pass
37
+
38
+ # Drop columns that are entirely unnamed AND entirely empty
39
+ unnamed_empty = [
40
+ col for col in df.columns
41
+ if str(col).startswith('Unnamed:') and df[col].isna().all()
42
+ ]
43
+ if unnamed_empty:
44
+ df = df.drop(columns=unnamed_empty)
45
+
46
+ # Drop rows that are entirely empty
47
+ df = df.dropna(how='all')
48
+
49
+ # Strip whitespace from column names
50
+ df.columns = [str(c).strip() for c in df.columns]
51
+
52
+ # Remove completely duplicate columns
53
+ df = df.loc[:, ~df.columns.duplicated()]
54
+
55
+ if df.empty or len(df.columns) == 0:
56
+ raise ValueError("DataFrame is empty after cleaning")
57
+
58
+ return df
59
+
60
+ except (pd.errors.ParserError, pd.errors.EmptyDataError, ValueError) as e:
61
+ last_error = e
62
+ continue
63
+ except UnicodeDecodeError as e:
64
+ last_error = e
65
+ continue
66
+
67
+ raise ValueError(
68
+ f"Could not parse CSV file after trying all encodings and separators. "
69
+ f"Last error: {last_error}"
70
+ )
71
+
72
+
73
+ def convert_spreadsheet(source: str) -> dict:
74
+ try:
75
+ # Validate file extension
76
+ if not (source.lower().endswith('.csv') or source.lower().endswith('.xlsx')):
77
+ raise ValueError(f"Unsupported spreadsheet format: {source}")
78
+
79
+ # Read the file based on extension
80
+ if source.lower().endswith('.csv'):
81
+ df = _try_read_csv(source)
82
+
83
+ elif source.lower().endswith('.xlsx'):
84
+ try:
85
+ df = pd.read_excel(source, engine='openpyxl', dtype=str)
86
+ except Exception:
87
+ # Fallback: try reading without dtype constraint
88
+ df = pd.read_excel(source, engine='openpyxl')
89
+
90
+ # Same cleanup as CSV
91
+ unnamed_empty = [
92
+ col for col in df.columns
93
+ if str(col).startswith('Unnamed:') and df[col].isna().all()
94
+ ]
95
+ if unnamed_empty:
96
+ df = df.drop(columns=unnamed_empty)
97
+ df = df.dropna(how='all')
98
+ df.columns = [str(c).strip() for c in df.columns]
99
+ df = df.loc[:, ~df.columns.duplicated()]
100
+
101
+ if df.empty:
102
+ raise ValueError(f"Spreadsheet is empty or has no usable data: {source}")
103
+
104
+ MAX_ROWS = 10000
105
+ if len(df) > MAX_ROWS:
106
+ print(f"Warning: Spreadsheet has {len(df)} rows. Truncating to {MAX_ROWS} rows.")
107
+ df = df.head(MAX_ROWS)
108
+
109
+ # Convert to markdown
110
+ try:
111
+ # Fill NaN with empty string for cleaner markdown output
112
+ text_content = df.fillna('').to_markdown(index=False)
113
+ except Exception as e:
114
+ print(f"Warning: to_markdown failed ({e}), falling back to CSV string.")
115
+ text_content = df.fillna('').to_csv(index=False)
116
+
117
+ # Create DoclingDocument
118
+ doc = DoclingDocument(name=source)
119
+
120
+ if len(text_content) > 5000:
121
+ lines = text_content.split('\n')
122
+
123
+ # Always include the header rows (first 2 lines for markdown: header + separator)
124
+ HEADER_LINES = 2
125
+ if lines:
126
+ header_text = '\n'.join(lines[:HEADER_LINES])
127
+ doc.add_text(text=header_text, label="text")
128
+
129
+ chunk_size = 50
130
+ for i in range(HEADER_LINES, len(lines), chunk_size):
131
+ chunk = '\n'.join(lines[i:i + chunk_size])
132
+ if chunk.strip():
133
+ doc.add_text(text=chunk, label="text")
134
+ else:
135
+ doc.add_text(text=text_content, label="text")
136
+
137
+ #Metadata
138
+ metadata = {
139
+ "source": source,
140
+ "type": "spreadsheet",
141
+ "rows": len(df),
142
+ "columns": len(df.columns),
143
+ "column_names": list(df.columns),
144
+ }
145
+
146
+ return {
147
+ "document": doc,
148
+ "metadata": metadata,
149
+ }
150
+
151
+ except pd.errors.EmptyDataError:
152
+ raise ValueError(f"Spreadsheet file is empty or invalid: {source}")
153
+ except pd.errors.ParserError as e:
154
+ raise ValueError(f"Failed to parse spreadsheet: {source}. Error: {str(e)}")
155
+ except FileNotFoundError:
156
+ raise FileNotFoundError(f"Spreadsheet file not found: {source}")
157
+ except Exception as e:
158
+ print(f"Error reading spreadsheet {source}: {e}")
159
+ raise Exception(f"Failed to convert spreadsheet: {str(e)}")