th2rag 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. th2rag/__init__.py +0 -0
  2. th2rag/auth/__init__.py +0 -0
  3. th2rag/auth/exceptions.py +2 -0
  4. th2rag/auth/router.py +43 -0
  5. th2rag/auth/schemas.py +15 -0
  6. th2rag/auth/service.py +115 -0
  7. th2rag/clients/__init__.py +0 -0
  8. th2rag/clients/storage/config.py +13 -0
  9. th2rag/clients/storage/local.py +48 -0
  10. th2rag/clients/storage/s3.py +93 -0
  11. th2rag/config.py +267 -0
  12. th2rag/conversations/__init__.py +0 -0
  13. th2rag/conversations/dependencies.py +30 -0
  14. th2rag/conversations/exceptions.py +6 -0
  15. th2rag/conversations/router.py +47 -0
  16. th2rag/conversations/schemas.py +43 -0
  17. th2rag/conversations/service.py +51 -0
  18. th2rag/database.py +72 -0
  19. th2rag/dataset/__init__.py +0 -0
  20. th2rag/dataset/dependencies.py +25 -0
  21. th2rag/dataset/exceptions.py +2 -0
  22. th2rag/dataset/router.py +53 -0
  23. th2rag/dataset/schemas.py +23 -0
  24. th2rag/dataset/service.py +78 -0
  25. th2rag/dependencies.py +143 -0
  26. th2rag/knowledge/__init__.py +0 -0
  27. th2rag/knowledge/dependencies.py +25 -0
  28. th2rag/knowledge/exceptions.py +2 -0
  29. th2rag/knowledge/mage_client.py +505 -0
  30. th2rag/knowledge/router.py +714 -0
  31. th2rag/knowledge/schemas.py +108 -0
  32. th2rag/knowledge/service.py +392 -0
  33. th2rag/logger.py +21 -0
  34. th2rag/main.py +51 -0
  35. th2rag/messages/__init__.py +0 -0
  36. th2rag/messages/exceptions.py +2 -0
  37. th2rag/messages/router.py +63 -0
  38. th2rag/messages/schemas.py +68 -0
  39. th2rag/messages/service.py +210 -0
  40. th2rag/models.py +215 -0
  41. th2rag/pagination.py +68 -0
  42. th2rag/rag/__init__.py +0 -0
  43. th2rag/rag/chunkers/__init__.py +0 -0
  44. th2rag/rag/chunkers/hybrid_chunker.py +23 -0
  45. th2rag/rag/constants.py +15 -0
  46. th2rag/rag/converters/__init__.py +0 -0
  47. th2rag/rag/converters/documents_converter.py +123 -0
  48. th2rag/rag/converters/json_converter.py +521 -0
  49. th2rag/rag/converters/legacy_converter.py +124 -0
  50. th2rag/rag/converters/pdf_converter.py +384 -0
  51. th2rag/rag/converters/spreadsheet_converter.py +159 -0
  52. th2rag/rag/converters/txt_converter.py +98 -0
  53. th2rag/rag/embeddings/__init__.py +0 -0
  54. th2rag/rag/embeddings/base.py +21 -0
  55. th2rag/rag/embeddings/exceptions.py +2 -0
  56. th2rag/rag/embeddings/sentence_transformer.py +97 -0
  57. th2rag/rag/generation/base.py +22 -0
  58. th2rag/rag/generation/exceptions.py +2 -0
  59. th2rag/rag/generation/litellm_generator.py +143 -0
  60. th2rag/rag/generation/mistral_generator.py +269 -0
  61. th2rag/rag/generation/test_litellm.py +33 -0
  62. th2rag/rag/retrieval/__init__.py +0 -0
  63. th2rag/rag/retrieval/base.py +11 -0
  64. th2rag/rag/retrieval/exceptions.py +10 -0
  65. th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
  66. th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
  67. th2rag/rag/retrieval/lancedb/schemas.py +19 -0
  68. th2rag/rag/services/__init__.py +0 -0
  69. th2rag/rag/services/rag_service.py +99 -0
  70. th2rag/rag/services/rag_service_builder.py +42 -0
  71. th2rag/rag/tasks/process_pdf.py +586 -0
  72. th2rag/users/dependencies.py +44 -0
  73. th2rag/users/exceptions.py +6 -0
  74. th2rag/users/router.py +92 -0
  75. th2rag/users/schemas.py +61 -0
  76. th2rag/users/service.py +140 -0
  77. th2rag/utils/__init__.py +0 -0
  78. th2rag/utils/helpers.py +40 -0
  79. th2rag/utils/logger.py +15 -0
  80. th2rag/utils/security.py +76 -0
  81. th2rag/utils/welcome_router.py +98 -0
  82. th2rag/webhooks/__init__.py +0 -0
  83. th2rag/webhooks/sender.py +126 -0
  84. th2rag-0.0.2.dist-info/METADATA +1002 -0
  85. th2rag-0.0.2.dist-info/RECORD +87 -0
  86. th2rag-0.0.2.dist-info/WHEEL +4 -0
  87. th2rag-0.0.2.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,123 @@
1
+ import logging
2
+ import os
3
+
4
+ from th2rag.rag.converters.json_converter import convert_json
5
+ from th2rag.rag.converters.legacy_converter import (
6
+ LEGACY_EXTENSIONS,
7
+ convert_with_libreoffice,
8
+ )
9
+ from th2rag.rag.converters.pdf_converter import (
10
+ EXT_TO_INPUT_FORMAT,
11
+ convert_with_docling,
12
+ )
13
+ from th2rag.rag.converters.spreadsheet_converter import convert_spreadsheet
14
+ from th2rag.rag.converters.txt_converter import convert_txt
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ # Extensions handled natively by Docling (via convert_with_docling)
19
+ _DOCLING_EXTENSIONS: frozenset[str] = frozenset(EXT_TO_INPUT_FORMAT.keys())
20
+
21
+ # Human-readable list for error messages
22
+ _ALL_SUPPORTED: list[str] = sorted(
23
+ _DOCLING_EXTENSIONS | {".csv", ".xlsx", ".json", ".txt"} | LEGACY_EXTENSIONS
24
+ )
25
+
26
+
27
+ def _convert_legacy_with_soffice(source: str, doc_id: int | None) -> dict:
28
+ """
29
+ Conversion via LibreOffice puis Docling.
30
+ Exposée séparément pour faciliter le mock dans les tests.
31
+ Lève ValueError si LibreOffice est absent ou si la conversion échoue.
32
+ """
33
+ converted_path = convert_with_libreoffice(source)
34
+ try:
35
+ result = convert_with_docling(source=converted_path, doc_id=doc_id)
36
+ finally:
37
+ # Nettoyage du tmpdir créé par convert_with_libreoffice
38
+ tmpdir = os.path.dirname(converted_path)
39
+ try:
40
+ import shutil
41
+ shutil.rmtree(tmpdir, ignore_errors=True)
42
+ except Exception:
43
+ pass
44
+ return result
45
+
46
+
47
+ def convert_any_doc(source: str, doc_id: int = None) -> dict:
48
+ """
49
+ Convert any supported document format to a standardised internal representation.
50
+
51
+ Supported formats
52
+ -----------------
53
+ Via Docling (convert_with_docling):
54
+ PDF, DOCX, PPTX, HTML, HTM, MD, MARKDOWN,
55
+ PNG, JPG, JPEG, TIFF, TIF, BMP, ADOC, ASCIIDOC
56
+
57
+ Via dedicated converters:
58
+ CSV, XLSX → convert_spreadsheet
59
+ JSON → convert_json
60
+ TXT → convert_txt
61
+
62
+ Via LibreOffice + Docling (si LibreOffice installé) :
63
+ DOC → DOCX, PPT → PPTX, XLS → XLSX
64
+ Si LibreOffice absent : ValueError avec message explicite.
65
+ """
66
+ if not os.path.exists(source):
67
+ raise FileNotFoundError(f"Source file not found: {source}")
68
+
69
+ ext = os.path.splitext(source)[1].lower()
70
+ logger.debug(f"Converting file: {os.path.basename(source)} (extension: {ext})")
71
+
72
+ if not ext:
73
+ raise ValueError(f"File has no extension: {source}")
74
+
75
+ logger.info(f"Converting document with extension: {ext} from {source}")
76
+
77
+ try:
78
+ if ext in _DOCLING_EXTENSIONS:
79
+ logger.debug(f"Routing {ext} to Docling converter")
80
+ result = convert_with_docling(source=source, doc_id=doc_id)
81
+
82
+ elif ext in (".csv", ".xlsx"):
83
+ logger.debug(f"Using spreadsheet converter for {ext}")
84
+ result = convert_spreadsheet(source=source)
85
+
86
+ elif ext == ".json":
87
+ logger.debug(f"Using JSON converter for {ext}")
88
+ result = convert_json(source=source)
89
+
90
+ elif ext == ".txt":
91
+ logger.debug(f"Using TXT converter for {ext}")
92
+ result = convert_txt(source=source)
93
+
94
+ elif ext in LEGACY_EXTENSIONS:
95
+ logger.debug(f"Routing legacy {ext} through LibreOffice → Docling")
96
+ result = _convert_legacy_with_soffice(source, doc_id)
97
+
98
+ else:
99
+ raise ValueError(
100
+ f"Unsupported file type: {ext}. "
101
+ f"Supported types: {', '.join(_ALL_SUPPORTED)}"
102
+ )
103
+
104
+ if not isinstance(result, dict):
105
+ raise ValueError(f"Converter returned invalid type: {type(result)}")
106
+
107
+ if "document" not in result:
108
+ raise ValueError("Converter result missing 'document' key")
109
+
110
+ logger.info(f"Successfully converted {ext} document")
111
+ return result
112
+
113
+ except ValueError as e:
114
+ logger.error(f"Validation error converting {source}: {e}")
115
+ raise
116
+
117
+ except FileNotFoundError:
118
+ logger.error(f"File not found: {source}")
119
+ raise
120
+
121
+ except Exception as e:
122
+ logger.error(f"Unexpected error converting {source}: {e}", exc_info=True)
123
+ raise Exception(f"Failed to convert {ext} document: {str(e)}") from e
@@ -0,0 +1,521 @@
1
+ import json
2
+ import logging
3
+ import os
4
+ from typing import Any
5
+
6
+ from docling_core.types.doc import DoclingDocument
7
+
8
+ logger = logging.getLogger(__name__)
9
+
10
+ # A document is treated as the article/JDS schema if it carries ANY of these
11
+ # markers (content OR identity). For such documents only the whitelisted
12
+ # metadata is embedded — never a raw dump of _source, to avoid leaking
13
+ # internal fields into RAG-retrievable content.
14
+ _ARTICLE_MARKERS = ("titre", "chapeau", "texte", "id_article", "id_element", "etat")
15
+
16
+
17
+ def load_json_source(source: str) -> Any:
18
+ """Load a JSON file, falling back to NDJSON (one object per line).
19
+
20
+ Plain ``json.load`` chokes on newline-delimited JSON (``{...}\n{...}``)
21
+ with an "Extra data" error. When the whole-file parse fails and the file
22
+ has more than one non-empty line, retry line by line and return the list
23
+ of objects.
24
+ """
25
+ with open(source, encoding="utf-8") as f:
26
+ text = f.read()
27
+ try:
28
+ return json.loads(text)
29
+ except json.JSONDecodeError as exc:
30
+ lines = [ln for ln in text.splitlines() if ln.strip()]
31
+ if len(lines) > 1:
32
+ try:
33
+ return [json.loads(ln) for ln in lines]
34
+ except json.JSONDecodeError:
35
+ pass
36
+ raise json.JSONDecodeError(
37
+ f"Invalid JSON syntax in file {source}: {str(exc)}", exc.doc, exc.pos
38
+ )
39
+
40
+
41
+ def render_generic_content(obj: Any, prefix: str = "") -> str:
42
+ """Render an arbitrary JSON value as readable ``key: value`` lines.
43
+
44
+ Used when a document has none of the article fields (titre/chapeau/texte),
45
+ so a plain JSON object or array still produces meaningful embedding content
46
+ instead of an empty métadonnée line.
47
+ """
48
+ lines: list[str] = []
49
+ if isinstance(obj, dict):
50
+ for key, value in obj.items():
51
+ if isinstance(value, (dict, list)):
52
+ nested = render_generic_content(value, f"{prefix}{key}.")
53
+ # Keep empty containers visible (distinguish from absent keys).
54
+ lines.append(nested if nested else f"{prefix}{key}: {value}")
55
+ else:
56
+ lines.append(f"{prefix}{key}: {value}")
57
+ elif isinstance(obj, list):
58
+ for idx, value in enumerate(obj):
59
+ if isinstance(value, (dict, list)):
60
+ nested = render_generic_content(value, f"{prefix}{idx}.")
61
+ lines.append(nested if nested else f"{prefix}{idx}: {value}")
62
+ else:
63
+ lines.append(f"{prefix}{idx}: {value}")
64
+ else:
65
+ lines.append(f"{prefix}{obj}")
66
+ return "\n".join(line for line in lines if line)
67
+
68
+
69
+ def build_embedding_content(doc: dict, max_length: int = 512) -> str:
70
+ """
71
+ Build content for embedding with length limit to avoid token overflow.
72
+
73
+ Args:
74
+ doc: Document dictionary with _source key
75
+ max_length: Maximum character length (roughly translates to ~100-150 tokens)
76
+
77
+ Returns:
78
+ Truncated content string
79
+ """
80
+ source = doc["_source"]
81
+
82
+ parts = []
83
+
84
+ if source.get("titre"):
85
+ parts.append(f"Title: {source['titre']}")
86
+
87
+ if source.get("chapeau"):
88
+ parts.append(f"Summary: {source['chapeau']}")
89
+
90
+ if source.get("texte"):
91
+ parts.append(f"Content:\n{source['texte']}")
92
+
93
+ if any(source.get(k) for k in _ARTICLE_MARKERS):
94
+ # Article/JDS schema (even if title/body are empty): only the
95
+ # whitelisted metadata is embedded — never a raw dump of _source.
96
+ parts.append(f"métadonnée {build_metadata(doc)}")
97
+ else:
98
+ # Genuinely generic JSON object (no article markers): render its
99
+ # keys/values so the content is embeddable instead of an empty line.
100
+ parts.append(render_generic_content(source))
101
+
102
+ # Join and truncate if necessary
103
+ full_content = "\n\n".join(parts)
104
+
105
+ if len(full_content) > max_length:
106
+ # Truncate and add indicator
107
+ full_content = full_content[:max_length] + "\n... [content truncated]"
108
+
109
+ return full_content
110
+
111
+
112
+ def build_metadata(doc: dict) -> dict:
113
+ source = doc["_source"]
114
+
115
+ return {
116
+ "id_article": source.get("id_article", ""),
117
+ "id_element": source.get("id_element", ""),
118
+ "etat": source.get("etat", ""),
119
+ "created_at": source.get("created_at", ""),
120
+ "updated_at": source.get("updated_at", ""),
121
+ "index": doc.get("_index", ""),
122
+ "doc_id": doc.get("_id", ""),
123
+ }
124
+
125
+
126
+ def normalize_to_wrapped_format(doc: Any) -> dict:
127
+ """
128
+ Normalize a document to the wrapped format with _source key.
129
+
130
+ Args:
131
+ doc: Document (may be dict or other type)
132
+
133
+ Returns:
134
+ Document in wrapped format with _source key
135
+
136
+ Raises:
137
+ ValueError: If doc is not a dictionary
138
+ """
139
+ if not isinstance(doc, dict):
140
+ raise ValueError(f"Expected dictionary, got {type(doc).__name__}: {doc}")
141
+
142
+ # Already in wrapped format
143
+ if "_source" in doc:
144
+ return doc
145
+
146
+ # Convert direct format to wrapped format
147
+ return {"_source": doc}
148
+
149
+
150
+ def detect_multi_doc(raw_content: Any) -> bool | int:
151
+ """
152
+ Auto-detect if the JSON contains multiple documents.
153
+ """
154
+ # Case 1: Array at top level with more than one item
155
+ if isinstance(raw_content, list):
156
+ count = len(raw_content)
157
+ return count if count > 1 else False
158
+
159
+ # Ensure it's a dictionary for other cases
160
+ if not isinstance(raw_content, dict):
161
+ return False
162
+
163
+ # Case 2: Elasticsearch response format
164
+ if "hits" in raw_content and isinstance(raw_content["hits"], dict):
165
+ hits = raw_content["hits"].get("hits", [])
166
+ count = len(hits)
167
+ return count if count > 1 else False
168
+
169
+ # Case 3: Dictionary with multiple numeric keys
170
+ numeric_keys = [k for k in raw_content.keys() if str(k).isdigit()]
171
+ if numeric_keys:
172
+ count = len(numeric_keys)
173
+ return count if count > 1 else False
174
+
175
+ # Case 4: Dictionary with multiple non-metadata keys that are dicts
176
+ metadata_keys = {"took", "timed_out", "_shards", "hits"}
177
+ dict_values = [
178
+ v for k, v in raw_content.items()
179
+ if isinstance(v, dict) and k not in metadata_keys
180
+ ]
181
+ count = len(dict_values)
182
+ return count if count > 1 else False
183
+
184
+
185
+ def extract_first_document(raw_content: Any) -> dict:
186
+ """
187
+ Extract the first valid document from various JSON formats.
188
+
189
+ Args:
190
+ raw_content: Parsed JSON content
191
+
192
+ Returns:
193
+ First document in wrapped format
194
+
195
+ Raises:
196
+ ValueError: If no valid document found
197
+ """
198
+ # Case 1: Array at top level
199
+ if isinstance(raw_content, list):
200
+ if len(raw_content) == 0:
201
+ raise ValueError("Empty array (no documents)")
202
+ first_item = raw_content[0]
203
+ if not isinstance(first_item, dict):
204
+ raise ValueError(f"First array item is not a dict: {type(first_item).__name__}")
205
+ return normalize_to_wrapped_format(first_item)
206
+
207
+ # Ensure it's a dictionary
208
+ if not isinstance(raw_content, dict):
209
+ raise ValueError(f"Expected dict or list at top level, got {type(raw_content).__name__}")
210
+
211
+ # Case 2: Elasticsearch response format
212
+ if "hits" in raw_content and isinstance(raw_content["hits"], dict):
213
+ hits = raw_content["hits"].get("hits", [])
214
+ if not hits:
215
+ raise ValueError("Elasticsearch format: no documents in hits array")
216
+ return hits[0] # Already in wrapped format from ES
217
+
218
+ # Case 3: Dictionary with values - try to find first valid dict value
219
+ # Iterate through values until we find a dictionary
220
+ for key, value in raw_content.items():
221
+ if isinstance(value, dict):
222
+ return normalize_to_wrapped_format(value)
223
+
224
+ # Case 4: fallback — treat the entire dict as a single generic document
225
+ # (a plain JSON object with no nested doc values and no article markers).
226
+ if raw_content:
227
+ logger.warning(
228
+ "json_converter: no article/document markers found; indexing the "
229
+ "whole JSON object as a single generic document (top-level keys: %s). "
230
+ "An accidental upload (e.g. an API error payload) would still be indexed.",
231
+ list(raw_content.keys())[:10],
232
+ )
233
+ return normalize_to_wrapped_format(raw_content)
234
+
235
+ raise ValueError("No valid document found: empty JSON object")
236
+
237
+
238
+ def extract_all_documents(raw_content: Any) -> list[dict]:
239
+ """
240
+ Extract all valid documents from various JSON formats.
241
+
242
+ Args:
243
+ raw_content: Parsed JSON content
244
+
245
+ Returns:
246
+ List of documents in wrapped format
247
+
248
+ Raises:
249
+ ValueError: If no valid documents found
250
+ """
251
+ documents = []
252
+
253
+ # Case 1: Array at top level
254
+ if isinstance(raw_content, list):
255
+ if len(raw_content) == 0:
256
+ raise ValueError("Empty array (no documents)")
257
+ for item in raw_content:
258
+ if isinstance(item, dict):
259
+ documents.append(normalize_to_wrapped_format(item))
260
+ if not documents:
261
+ raise ValueError("Array contains no dictionary items")
262
+ return documents
263
+
264
+ # Ensure it's a dictionary
265
+ if not isinstance(raw_content, dict):
266
+ raise ValueError(f"Expected dict or list, got {type(raw_content).__name__}")
267
+
268
+ # Case 2: Elasticsearch response format
269
+ if "hits" in raw_content and isinstance(raw_content["hits"], dict):
270
+ hits = raw_content["hits"].get("hits", [])
271
+ if not hits:
272
+ raise ValueError("Elasticsearch format: no documents in hits array")
273
+ return hits # Already in wrapped format from ES
274
+
275
+ # Case 3: Dictionary with numeric keys (sorted)
276
+ if all(str(k).isdigit() for k in raw_content.keys()):
277
+ sorted_keys = sorted(raw_content.keys(), key=lambda x: int(x))
278
+ for key in sorted_keys:
279
+ value = raw_content[key]
280
+ if isinstance(value, dict):
281
+ documents.append(normalize_to_wrapped_format(value))
282
+ if not documents:
283
+ raise ValueError("Numeric key format: no dictionary values found")
284
+ return documents
285
+
286
+ # Case 4: Dictionary with arbitrary keys - extract all dict values
287
+ for key, value in raw_content.items():
288
+ if isinstance(value, dict):
289
+ documents.append(normalize_to_wrapped_format(value))
290
+
291
+ if documents:
292
+ return documents
293
+
294
+ # Case 5: fallback — the entire dict is a single generic document
295
+ if raw_content:
296
+ logger.warning(
297
+ "json_converter: no article/document markers found; indexing the "
298
+ "whole JSON object as a single generic document (top-level keys: %s). "
299
+ "An accidental upload (e.g. an API error payload) would still be indexed.",
300
+ list(raw_content.keys())[:10],
301
+ )
302
+ return [normalize_to_wrapped_format(raw_content)]
303
+
304
+ raise ValueError("No valid documents found: empty JSON object")
305
+
306
+
307
+ def combine_documents(docs_list: list[dict], source_path: str) -> dict:
308
+ """
309
+ Combine multiple documents into a single document.
310
+ """
311
+ if not docs_list:
312
+ raise ValueError("Cannot combine empty document list")
313
+
314
+ # If only one document, return it as-is
315
+ if len(docs_list) == 1:
316
+ return docs_list[0]
317
+
318
+ # Create a combined DoclingDocument
319
+ combined_doc = DoclingDocument(name=source_path)
320
+
321
+ # Collect all metadata
322
+ all_metadata = {
323
+ "source": source_path,
324
+ "document_count": len(docs_list),
325
+ "combined": True,
326
+ "sub_documents": []
327
+ }
328
+
329
+ # Combine all text from all documents
330
+ for idx, doc_dict in enumerate(docs_list):
331
+ docling_doc = doc_dict["document"]
332
+ metadata = doc_dict["metadata"]
333
+
334
+ # Add metadata reference
335
+ all_metadata["sub_documents"].append({
336
+ "index": idx,
337
+ "id_article": metadata.get("id_article"),
338
+ "id_element": metadata.get("id_element")
339
+ })
340
+
341
+ # Add all texts from this document
342
+ for text_item in docling_doc.texts:
343
+ # FIX: Access TextItem object attributes using dot notation, not dict syntax
344
+ # Use the original label value as DocItemLabel is a strict enum
345
+ combined_doc.add_text(
346
+ text=text_item.text, # Changed from text_item["text"]
347
+ label=text_item.label
348
+ )
349
+
350
+ return {
351
+ "document": combined_doc,
352
+ "metadata": all_metadata
353
+ }
354
+
355
+
356
+ def process_single_document(doc: dict, source_path: str) -> dict:
357
+ """
358
+ Process a single document.
359
+
360
+ Args:
361
+ doc: Document in wrapped format
362
+ source_path: Path to source file
363
+
364
+ Returns:
365
+ Dict with DoclingDocument and metadata
366
+ """
367
+ metadata = build_metadata(doc)
368
+ metadata["source"] = source_path
369
+
370
+ # Create a DoclingDocument
371
+ text_content = build_embedding_content(doc)
372
+ docling_doc = DoclingDocument(name=source_path)
373
+
374
+ # Split content into smaller paragraphs to help with chunking
375
+ paragraphs = text_content.split('\n\n')
376
+
377
+ for para in paragraphs:
378
+ if para.strip(): # Only add non-empty paragraphs
379
+ docling_doc.add_text(text=para.strip(), label="paragraph")
380
+
381
+ rag_doc = {
382
+ "document": docling_doc,
383
+ "metadata": metadata,
384
+ }
385
+ return rag_doc
386
+
387
+
388
+ def special_json_jds(source: str) -> dict:
389
+ """
390
+ Process special JSON format for JDS documents (single document).
391
+
392
+ Args:
393
+ source: Path to JSON file
394
+
395
+ Returns:
396
+ Dict with document and metadata
397
+ """
398
+ # Check if file exists
399
+ if not os.path.exists(source):
400
+ raise FileNotFoundError(f"JSON file not found: {source}")
401
+
402
+ # Check if file is empty
403
+ if os.path.getsize(source) == 0:
404
+ raise ValueError(f"JSON file is empty (0 bytes): {source}")
405
+
406
+ # Parse JSON (NDJSON-aware: falls back to one object per line)
407
+ raw_content = load_json_source(source)
408
+
409
+ # Extract first document (this handles all the format variations)
410
+ doc = extract_first_document(raw_content)
411
+
412
+ # Process and return
413
+ return process_single_document(doc, source)
414
+
415
+
416
+ def special_json_jds_multi(source: str) -> list[dict]:
417
+ """
418
+ Process special JSON format for JDS documents (multiple documents).
419
+
420
+ Args:
421
+ source: Path to JSON file
422
+
423
+ Returns:
424
+ List of dicts with document and metadata
425
+ """
426
+ # Check if file exists
427
+ if not os.path.exists(source):
428
+ raise FileNotFoundError(f"JSON file not found: {source}")
429
+
430
+ # Check if file is empty
431
+ if os.path.getsize(source) == 0:
432
+ raise ValueError(f"JSON file is empty (0 bytes): {source}")
433
+
434
+ # Parse JSON (NDJSON-aware: falls back to one object per line)
435
+ raw_content = load_json_source(source)
436
+
437
+ # Extract all documents
438
+ documents = extract_all_documents(raw_content)
439
+
440
+ # Process all documents
441
+ results = []
442
+ failed_count = 0
443
+
444
+ for i, doc in enumerate(documents):
445
+ try:
446
+ results.append(process_single_document(doc, source))
447
+ except Exception as e:
448
+ # Log the error but continue processing other documents
449
+ failed_count += 1
450
+ print(f"Warning: Failed to process document {i+1}/{len(documents)}: {str(e)}")
451
+ continue
452
+
453
+ if not results:
454
+ raise ValueError(
455
+ f"All {len(documents)} document(s) failed to process. "
456
+ f"Check document structure and required fields."
457
+ )
458
+
459
+ if failed_count > 0:
460
+ print(f"Warning: {failed_count}/{len(documents)} document(s) failed to process")
461
+
462
+ return results
463
+
464
+
465
+ def convert_json(
466
+ source: str,
467
+ special_format: str = "jds",
468
+ multi_doc: bool = None
469
+ ) -> dict:
470
+ """
471
+ Convert JSON file to document format.
472
+
473
+ Args:
474
+ source: Path to JSON file
475
+ special_format: Format type (default: "jds")
476
+ multi_doc: If True, process all documents and combine; if False, process first only;
477
+ if None (default), auto-detect based on file content
478
+
479
+ Returns:
480
+ Dict with document and metadata (always returns a single combined document)
481
+
482
+ Raises:
483
+ ValueError: If JSON format is invalid or file is empty
484
+ FileNotFoundError: If source file doesn't exist
485
+ json.JSONDecodeError: If JSON syntax is invalid
486
+ """
487
+ # Check file existence and size first
488
+ if not os.path.exists(source):
489
+ raise FileNotFoundError(f"JSON file not found: {source}")
490
+
491
+ if os.path.getsize(source) == 0:
492
+ raise ValueError(f"JSON file is empty (0 bytes): {source}")
493
+
494
+ # Parse JSON (NDJSON-aware: falls back to one object per line)
495
+ raw_content = load_json_source(source)
496
+
497
+ # Auto-detect multi_doc if not specified
498
+ if multi_doc is None:
499
+ result = detect_multi_doc(raw_content)
500
+ if result: # result is either False or int > 1
501
+ multi_doc = True
502
+ else:
503
+ multi_doc = False
504
+
505
+ if special_format == "jds":
506
+ if multi_doc:
507
+ # Process multiple documents and combine into one
508
+ docs_list = special_json_jds_multi(source)
509
+ return combine_documents(docs_list, source)
510
+ return special_json_jds(source)
511
+
512
+ # Generic JSON handling (for future extensibility)
513
+ if multi_doc:
514
+ documents = extract_all_documents(raw_content)
515
+ results = []
516
+ for doc in documents:
517
+ results.append(process_single_document(doc, source))
518
+ return combine_documents(results, source)
519
+ else:
520
+ doc = extract_first_document(raw_content)
521
+ return process_single_document(doc, source)