th2rag 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- th2rag/__init__.py +0 -0
- th2rag/auth/__init__.py +0 -0
- th2rag/auth/exceptions.py +2 -0
- th2rag/auth/router.py +43 -0
- th2rag/auth/schemas.py +15 -0
- th2rag/auth/service.py +115 -0
- th2rag/clients/__init__.py +0 -0
- th2rag/clients/storage/config.py +13 -0
- th2rag/clients/storage/local.py +48 -0
- th2rag/clients/storage/s3.py +93 -0
- th2rag/config.py +267 -0
- th2rag/conversations/__init__.py +0 -0
- th2rag/conversations/dependencies.py +30 -0
- th2rag/conversations/exceptions.py +6 -0
- th2rag/conversations/router.py +47 -0
- th2rag/conversations/schemas.py +43 -0
- th2rag/conversations/service.py +51 -0
- th2rag/database.py +72 -0
- th2rag/dataset/__init__.py +0 -0
- th2rag/dataset/dependencies.py +25 -0
- th2rag/dataset/exceptions.py +2 -0
- th2rag/dataset/router.py +53 -0
- th2rag/dataset/schemas.py +23 -0
- th2rag/dataset/service.py +78 -0
- th2rag/dependencies.py +143 -0
- th2rag/knowledge/__init__.py +0 -0
- th2rag/knowledge/dependencies.py +25 -0
- th2rag/knowledge/exceptions.py +2 -0
- th2rag/knowledge/mage_client.py +505 -0
- th2rag/knowledge/router.py +714 -0
- th2rag/knowledge/schemas.py +108 -0
- th2rag/knowledge/service.py +392 -0
- th2rag/logger.py +21 -0
- th2rag/main.py +51 -0
- th2rag/messages/__init__.py +0 -0
- th2rag/messages/exceptions.py +2 -0
- th2rag/messages/router.py +63 -0
- th2rag/messages/schemas.py +68 -0
- th2rag/messages/service.py +210 -0
- th2rag/models.py +215 -0
- th2rag/pagination.py +68 -0
- th2rag/rag/__init__.py +0 -0
- th2rag/rag/chunkers/__init__.py +0 -0
- th2rag/rag/chunkers/hybrid_chunker.py +23 -0
- th2rag/rag/constants.py +15 -0
- th2rag/rag/converters/__init__.py +0 -0
- th2rag/rag/converters/documents_converter.py +123 -0
- th2rag/rag/converters/json_converter.py +521 -0
- th2rag/rag/converters/legacy_converter.py +124 -0
- th2rag/rag/converters/pdf_converter.py +384 -0
- th2rag/rag/converters/spreadsheet_converter.py +159 -0
- th2rag/rag/converters/txt_converter.py +98 -0
- th2rag/rag/embeddings/__init__.py +0 -0
- th2rag/rag/embeddings/base.py +21 -0
- th2rag/rag/embeddings/exceptions.py +2 -0
- th2rag/rag/embeddings/sentence_transformer.py +97 -0
- th2rag/rag/generation/base.py +22 -0
- th2rag/rag/generation/exceptions.py +2 -0
- th2rag/rag/generation/litellm_generator.py +143 -0
- th2rag/rag/generation/mistral_generator.py +269 -0
- th2rag/rag/generation/test_litellm.py +33 -0
- th2rag/rag/retrieval/__init__.py +0 -0
- th2rag/rag/retrieval/base.py +11 -0
- th2rag/rag/retrieval/exceptions.py +10 -0
- th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
- th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
- th2rag/rag/retrieval/lancedb/schemas.py +19 -0
- th2rag/rag/services/__init__.py +0 -0
- th2rag/rag/services/rag_service.py +99 -0
- th2rag/rag/services/rag_service_builder.py +42 -0
- th2rag/rag/tasks/process_pdf.py +586 -0
- th2rag/users/dependencies.py +44 -0
- th2rag/users/exceptions.py +6 -0
- th2rag/users/router.py +92 -0
- th2rag/users/schemas.py +61 -0
- th2rag/users/service.py +140 -0
- th2rag/utils/__init__.py +0 -0
- th2rag/utils/helpers.py +40 -0
- th2rag/utils/logger.py +15 -0
- th2rag/utils/security.py +76 -0
- th2rag/utils/welcome_router.py +98 -0
- th2rag/webhooks/__init__.py +0 -0
- th2rag/webhooks/sender.py +126 -0
- th2rag-0.0.2.dist-info/METADATA +1002 -0
- th2rag-0.0.2.dist-info/RECORD +87 -0
- th2rag-0.0.2.dist-info/WHEEL +4 -0
- th2rag-0.0.2.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from th2rag.rag.converters.json_converter import convert_json
|
|
5
|
+
from th2rag.rag.converters.legacy_converter import (
|
|
6
|
+
LEGACY_EXTENSIONS,
|
|
7
|
+
convert_with_libreoffice,
|
|
8
|
+
)
|
|
9
|
+
from th2rag.rag.converters.pdf_converter import (
|
|
10
|
+
EXT_TO_INPUT_FORMAT,
|
|
11
|
+
convert_with_docling,
|
|
12
|
+
)
|
|
13
|
+
from th2rag.rag.converters.spreadsheet_converter import convert_spreadsheet
|
|
14
|
+
from th2rag.rag.converters.txt_converter import convert_txt
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
# Extensions handled natively by Docling (via convert_with_docling)
|
|
19
|
+
_DOCLING_EXTENSIONS: frozenset[str] = frozenset(EXT_TO_INPUT_FORMAT.keys())
|
|
20
|
+
|
|
21
|
+
# Human-readable list for error messages
|
|
22
|
+
_ALL_SUPPORTED: list[str] = sorted(
|
|
23
|
+
_DOCLING_EXTENSIONS | {".csv", ".xlsx", ".json", ".txt"} | LEGACY_EXTENSIONS
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _convert_legacy_with_soffice(source: str, doc_id: int | None) -> dict:
|
|
28
|
+
"""
|
|
29
|
+
Conversion via LibreOffice puis Docling.
|
|
30
|
+
Exposée séparément pour faciliter le mock dans les tests.
|
|
31
|
+
Lève ValueError si LibreOffice est absent ou si la conversion échoue.
|
|
32
|
+
"""
|
|
33
|
+
converted_path = convert_with_libreoffice(source)
|
|
34
|
+
try:
|
|
35
|
+
result = convert_with_docling(source=converted_path, doc_id=doc_id)
|
|
36
|
+
finally:
|
|
37
|
+
# Nettoyage du tmpdir créé par convert_with_libreoffice
|
|
38
|
+
tmpdir = os.path.dirname(converted_path)
|
|
39
|
+
try:
|
|
40
|
+
import shutil
|
|
41
|
+
shutil.rmtree(tmpdir, ignore_errors=True)
|
|
42
|
+
except Exception:
|
|
43
|
+
pass
|
|
44
|
+
return result
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def convert_any_doc(source: str, doc_id: int = None) -> dict:
|
|
48
|
+
"""
|
|
49
|
+
Convert any supported document format to a standardised internal representation.
|
|
50
|
+
|
|
51
|
+
Supported formats
|
|
52
|
+
-----------------
|
|
53
|
+
Via Docling (convert_with_docling):
|
|
54
|
+
PDF, DOCX, PPTX, HTML, HTM, MD, MARKDOWN,
|
|
55
|
+
PNG, JPG, JPEG, TIFF, TIF, BMP, ADOC, ASCIIDOC
|
|
56
|
+
|
|
57
|
+
Via dedicated converters:
|
|
58
|
+
CSV, XLSX → convert_spreadsheet
|
|
59
|
+
JSON → convert_json
|
|
60
|
+
TXT → convert_txt
|
|
61
|
+
|
|
62
|
+
Via LibreOffice + Docling (si LibreOffice installé) :
|
|
63
|
+
DOC → DOCX, PPT → PPTX, XLS → XLSX
|
|
64
|
+
Si LibreOffice absent : ValueError avec message explicite.
|
|
65
|
+
"""
|
|
66
|
+
if not os.path.exists(source):
|
|
67
|
+
raise FileNotFoundError(f"Source file not found: {source}")
|
|
68
|
+
|
|
69
|
+
ext = os.path.splitext(source)[1].lower()
|
|
70
|
+
logger.debug(f"Converting file: {os.path.basename(source)} (extension: {ext})")
|
|
71
|
+
|
|
72
|
+
if not ext:
|
|
73
|
+
raise ValueError(f"File has no extension: {source}")
|
|
74
|
+
|
|
75
|
+
logger.info(f"Converting document with extension: {ext} from {source}")
|
|
76
|
+
|
|
77
|
+
try:
|
|
78
|
+
if ext in _DOCLING_EXTENSIONS:
|
|
79
|
+
logger.debug(f"Routing {ext} to Docling converter")
|
|
80
|
+
result = convert_with_docling(source=source, doc_id=doc_id)
|
|
81
|
+
|
|
82
|
+
elif ext in (".csv", ".xlsx"):
|
|
83
|
+
logger.debug(f"Using spreadsheet converter for {ext}")
|
|
84
|
+
result = convert_spreadsheet(source=source)
|
|
85
|
+
|
|
86
|
+
elif ext == ".json":
|
|
87
|
+
logger.debug(f"Using JSON converter for {ext}")
|
|
88
|
+
result = convert_json(source=source)
|
|
89
|
+
|
|
90
|
+
elif ext == ".txt":
|
|
91
|
+
logger.debug(f"Using TXT converter for {ext}")
|
|
92
|
+
result = convert_txt(source=source)
|
|
93
|
+
|
|
94
|
+
elif ext in LEGACY_EXTENSIONS:
|
|
95
|
+
logger.debug(f"Routing legacy {ext} through LibreOffice → Docling")
|
|
96
|
+
result = _convert_legacy_with_soffice(source, doc_id)
|
|
97
|
+
|
|
98
|
+
else:
|
|
99
|
+
raise ValueError(
|
|
100
|
+
f"Unsupported file type: {ext}. "
|
|
101
|
+
f"Supported types: {', '.join(_ALL_SUPPORTED)}"
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
if not isinstance(result, dict):
|
|
105
|
+
raise ValueError(f"Converter returned invalid type: {type(result)}")
|
|
106
|
+
|
|
107
|
+
if "document" not in result:
|
|
108
|
+
raise ValueError("Converter result missing 'document' key")
|
|
109
|
+
|
|
110
|
+
logger.info(f"Successfully converted {ext} document")
|
|
111
|
+
return result
|
|
112
|
+
|
|
113
|
+
except ValueError as e:
|
|
114
|
+
logger.error(f"Validation error converting {source}: {e}")
|
|
115
|
+
raise
|
|
116
|
+
|
|
117
|
+
except FileNotFoundError:
|
|
118
|
+
logger.error(f"File not found: {source}")
|
|
119
|
+
raise
|
|
120
|
+
|
|
121
|
+
except Exception as e:
|
|
122
|
+
logger.error(f"Unexpected error converting {source}: {e}", exc_info=True)
|
|
123
|
+
raise Exception(f"Failed to convert {ext} document: {str(e)}") from e
|
|
@@ -0,0 +1,521 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from docling_core.types.doc import DoclingDocument
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
# A document is treated as the article/JDS schema if it carries ANY of these
|
|
11
|
+
# markers (content OR identity). For such documents only the whitelisted
|
|
12
|
+
# metadata is embedded — never a raw dump of _source, to avoid leaking
|
|
13
|
+
# internal fields into RAG-retrievable content.
|
|
14
|
+
_ARTICLE_MARKERS = ("titre", "chapeau", "texte", "id_article", "id_element", "etat")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def load_json_source(source: str) -> Any:
|
|
18
|
+
"""Load a JSON file, falling back to NDJSON (one object per line).
|
|
19
|
+
|
|
20
|
+
Plain ``json.load`` chokes on newline-delimited JSON (``{...}\n{...}``)
|
|
21
|
+
with an "Extra data" error. When the whole-file parse fails and the file
|
|
22
|
+
has more than one non-empty line, retry line by line and return the list
|
|
23
|
+
of objects.
|
|
24
|
+
"""
|
|
25
|
+
with open(source, encoding="utf-8") as f:
|
|
26
|
+
text = f.read()
|
|
27
|
+
try:
|
|
28
|
+
return json.loads(text)
|
|
29
|
+
except json.JSONDecodeError as exc:
|
|
30
|
+
lines = [ln for ln in text.splitlines() if ln.strip()]
|
|
31
|
+
if len(lines) > 1:
|
|
32
|
+
try:
|
|
33
|
+
return [json.loads(ln) for ln in lines]
|
|
34
|
+
except json.JSONDecodeError:
|
|
35
|
+
pass
|
|
36
|
+
raise json.JSONDecodeError(
|
|
37
|
+
f"Invalid JSON syntax in file {source}: {str(exc)}", exc.doc, exc.pos
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def render_generic_content(obj: Any, prefix: str = "") -> str:
|
|
42
|
+
"""Render an arbitrary JSON value as readable ``key: value`` lines.
|
|
43
|
+
|
|
44
|
+
Used when a document has none of the article fields (titre/chapeau/texte),
|
|
45
|
+
so a plain JSON object or array still produces meaningful embedding content
|
|
46
|
+
instead of an empty métadonnée line.
|
|
47
|
+
"""
|
|
48
|
+
lines: list[str] = []
|
|
49
|
+
if isinstance(obj, dict):
|
|
50
|
+
for key, value in obj.items():
|
|
51
|
+
if isinstance(value, (dict, list)):
|
|
52
|
+
nested = render_generic_content(value, f"{prefix}{key}.")
|
|
53
|
+
# Keep empty containers visible (distinguish from absent keys).
|
|
54
|
+
lines.append(nested if nested else f"{prefix}{key}: {value}")
|
|
55
|
+
else:
|
|
56
|
+
lines.append(f"{prefix}{key}: {value}")
|
|
57
|
+
elif isinstance(obj, list):
|
|
58
|
+
for idx, value in enumerate(obj):
|
|
59
|
+
if isinstance(value, (dict, list)):
|
|
60
|
+
nested = render_generic_content(value, f"{prefix}{idx}.")
|
|
61
|
+
lines.append(nested if nested else f"{prefix}{idx}: {value}")
|
|
62
|
+
else:
|
|
63
|
+
lines.append(f"{prefix}{idx}: {value}")
|
|
64
|
+
else:
|
|
65
|
+
lines.append(f"{prefix}{obj}")
|
|
66
|
+
return "\n".join(line for line in lines if line)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def build_embedding_content(doc: dict, max_length: int = 512) -> str:
|
|
70
|
+
"""
|
|
71
|
+
Build content for embedding with length limit to avoid token overflow.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
doc: Document dictionary with _source key
|
|
75
|
+
max_length: Maximum character length (roughly translates to ~100-150 tokens)
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
Truncated content string
|
|
79
|
+
"""
|
|
80
|
+
source = doc["_source"]
|
|
81
|
+
|
|
82
|
+
parts = []
|
|
83
|
+
|
|
84
|
+
if source.get("titre"):
|
|
85
|
+
parts.append(f"Title: {source['titre']}")
|
|
86
|
+
|
|
87
|
+
if source.get("chapeau"):
|
|
88
|
+
parts.append(f"Summary: {source['chapeau']}")
|
|
89
|
+
|
|
90
|
+
if source.get("texte"):
|
|
91
|
+
parts.append(f"Content:\n{source['texte']}")
|
|
92
|
+
|
|
93
|
+
if any(source.get(k) for k in _ARTICLE_MARKERS):
|
|
94
|
+
# Article/JDS schema (even if title/body are empty): only the
|
|
95
|
+
# whitelisted metadata is embedded — never a raw dump of _source.
|
|
96
|
+
parts.append(f"métadonnée {build_metadata(doc)}")
|
|
97
|
+
else:
|
|
98
|
+
# Genuinely generic JSON object (no article markers): render its
|
|
99
|
+
# keys/values so the content is embeddable instead of an empty line.
|
|
100
|
+
parts.append(render_generic_content(source))
|
|
101
|
+
|
|
102
|
+
# Join and truncate if necessary
|
|
103
|
+
full_content = "\n\n".join(parts)
|
|
104
|
+
|
|
105
|
+
if len(full_content) > max_length:
|
|
106
|
+
# Truncate and add indicator
|
|
107
|
+
full_content = full_content[:max_length] + "\n... [content truncated]"
|
|
108
|
+
|
|
109
|
+
return full_content
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def build_metadata(doc: dict) -> dict:
|
|
113
|
+
source = doc["_source"]
|
|
114
|
+
|
|
115
|
+
return {
|
|
116
|
+
"id_article": source.get("id_article", ""),
|
|
117
|
+
"id_element": source.get("id_element", ""),
|
|
118
|
+
"etat": source.get("etat", ""),
|
|
119
|
+
"created_at": source.get("created_at", ""),
|
|
120
|
+
"updated_at": source.get("updated_at", ""),
|
|
121
|
+
"index": doc.get("_index", ""),
|
|
122
|
+
"doc_id": doc.get("_id", ""),
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def normalize_to_wrapped_format(doc: Any) -> dict:
|
|
127
|
+
"""
|
|
128
|
+
Normalize a document to the wrapped format with _source key.
|
|
129
|
+
|
|
130
|
+
Args:
|
|
131
|
+
doc: Document (may be dict or other type)
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
Document in wrapped format with _source key
|
|
135
|
+
|
|
136
|
+
Raises:
|
|
137
|
+
ValueError: If doc is not a dictionary
|
|
138
|
+
"""
|
|
139
|
+
if not isinstance(doc, dict):
|
|
140
|
+
raise ValueError(f"Expected dictionary, got {type(doc).__name__}: {doc}")
|
|
141
|
+
|
|
142
|
+
# Already in wrapped format
|
|
143
|
+
if "_source" in doc:
|
|
144
|
+
return doc
|
|
145
|
+
|
|
146
|
+
# Convert direct format to wrapped format
|
|
147
|
+
return {"_source": doc}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def detect_multi_doc(raw_content: Any) -> bool | int:
|
|
151
|
+
"""
|
|
152
|
+
Auto-detect if the JSON contains multiple documents.
|
|
153
|
+
"""
|
|
154
|
+
# Case 1: Array at top level with more than one item
|
|
155
|
+
if isinstance(raw_content, list):
|
|
156
|
+
count = len(raw_content)
|
|
157
|
+
return count if count > 1 else False
|
|
158
|
+
|
|
159
|
+
# Ensure it's a dictionary for other cases
|
|
160
|
+
if not isinstance(raw_content, dict):
|
|
161
|
+
return False
|
|
162
|
+
|
|
163
|
+
# Case 2: Elasticsearch response format
|
|
164
|
+
if "hits" in raw_content and isinstance(raw_content["hits"], dict):
|
|
165
|
+
hits = raw_content["hits"].get("hits", [])
|
|
166
|
+
count = len(hits)
|
|
167
|
+
return count if count > 1 else False
|
|
168
|
+
|
|
169
|
+
# Case 3: Dictionary with multiple numeric keys
|
|
170
|
+
numeric_keys = [k for k in raw_content.keys() if str(k).isdigit()]
|
|
171
|
+
if numeric_keys:
|
|
172
|
+
count = len(numeric_keys)
|
|
173
|
+
return count if count > 1 else False
|
|
174
|
+
|
|
175
|
+
# Case 4: Dictionary with multiple non-metadata keys that are dicts
|
|
176
|
+
metadata_keys = {"took", "timed_out", "_shards", "hits"}
|
|
177
|
+
dict_values = [
|
|
178
|
+
v for k, v in raw_content.items()
|
|
179
|
+
if isinstance(v, dict) and k not in metadata_keys
|
|
180
|
+
]
|
|
181
|
+
count = len(dict_values)
|
|
182
|
+
return count if count > 1 else False
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def extract_first_document(raw_content: Any) -> dict:
|
|
186
|
+
"""
|
|
187
|
+
Extract the first valid document from various JSON formats.
|
|
188
|
+
|
|
189
|
+
Args:
|
|
190
|
+
raw_content: Parsed JSON content
|
|
191
|
+
|
|
192
|
+
Returns:
|
|
193
|
+
First document in wrapped format
|
|
194
|
+
|
|
195
|
+
Raises:
|
|
196
|
+
ValueError: If no valid document found
|
|
197
|
+
"""
|
|
198
|
+
# Case 1: Array at top level
|
|
199
|
+
if isinstance(raw_content, list):
|
|
200
|
+
if len(raw_content) == 0:
|
|
201
|
+
raise ValueError("Empty array (no documents)")
|
|
202
|
+
first_item = raw_content[0]
|
|
203
|
+
if not isinstance(first_item, dict):
|
|
204
|
+
raise ValueError(f"First array item is not a dict: {type(first_item).__name__}")
|
|
205
|
+
return normalize_to_wrapped_format(first_item)
|
|
206
|
+
|
|
207
|
+
# Ensure it's a dictionary
|
|
208
|
+
if not isinstance(raw_content, dict):
|
|
209
|
+
raise ValueError(f"Expected dict or list at top level, got {type(raw_content).__name__}")
|
|
210
|
+
|
|
211
|
+
# Case 2: Elasticsearch response format
|
|
212
|
+
if "hits" in raw_content and isinstance(raw_content["hits"], dict):
|
|
213
|
+
hits = raw_content["hits"].get("hits", [])
|
|
214
|
+
if not hits:
|
|
215
|
+
raise ValueError("Elasticsearch format: no documents in hits array")
|
|
216
|
+
return hits[0] # Already in wrapped format from ES
|
|
217
|
+
|
|
218
|
+
# Case 3: Dictionary with values - try to find first valid dict value
|
|
219
|
+
# Iterate through values until we find a dictionary
|
|
220
|
+
for key, value in raw_content.items():
|
|
221
|
+
if isinstance(value, dict):
|
|
222
|
+
return normalize_to_wrapped_format(value)
|
|
223
|
+
|
|
224
|
+
# Case 4: fallback — treat the entire dict as a single generic document
|
|
225
|
+
# (a plain JSON object with no nested doc values and no article markers).
|
|
226
|
+
if raw_content:
|
|
227
|
+
logger.warning(
|
|
228
|
+
"json_converter: no article/document markers found; indexing the "
|
|
229
|
+
"whole JSON object as a single generic document (top-level keys: %s). "
|
|
230
|
+
"An accidental upload (e.g. an API error payload) would still be indexed.",
|
|
231
|
+
list(raw_content.keys())[:10],
|
|
232
|
+
)
|
|
233
|
+
return normalize_to_wrapped_format(raw_content)
|
|
234
|
+
|
|
235
|
+
raise ValueError("No valid document found: empty JSON object")
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def extract_all_documents(raw_content: Any) -> list[dict]:
|
|
239
|
+
"""
|
|
240
|
+
Extract all valid documents from various JSON formats.
|
|
241
|
+
|
|
242
|
+
Args:
|
|
243
|
+
raw_content: Parsed JSON content
|
|
244
|
+
|
|
245
|
+
Returns:
|
|
246
|
+
List of documents in wrapped format
|
|
247
|
+
|
|
248
|
+
Raises:
|
|
249
|
+
ValueError: If no valid documents found
|
|
250
|
+
"""
|
|
251
|
+
documents = []
|
|
252
|
+
|
|
253
|
+
# Case 1: Array at top level
|
|
254
|
+
if isinstance(raw_content, list):
|
|
255
|
+
if len(raw_content) == 0:
|
|
256
|
+
raise ValueError("Empty array (no documents)")
|
|
257
|
+
for item in raw_content:
|
|
258
|
+
if isinstance(item, dict):
|
|
259
|
+
documents.append(normalize_to_wrapped_format(item))
|
|
260
|
+
if not documents:
|
|
261
|
+
raise ValueError("Array contains no dictionary items")
|
|
262
|
+
return documents
|
|
263
|
+
|
|
264
|
+
# Ensure it's a dictionary
|
|
265
|
+
if not isinstance(raw_content, dict):
|
|
266
|
+
raise ValueError(f"Expected dict or list, got {type(raw_content).__name__}")
|
|
267
|
+
|
|
268
|
+
# Case 2: Elasticsearch response format
|
|
269
|
+
if "hits" in raw_content and isinstance(raw_content["hits"], dict):
|
|
270
|
+
hits = raw_content["hits"].get("hits", [])
|
|
271
|
+
if not hits:
|
|
272
|
+
raise ValueError("Elasticsearch format: no documents in hits array")
|
|
273
|
+
return hits # Already in wrapped format from ES
|
|
274
|
+
|
|
275
|
+
# Case 3: Dictionary with numeric keys (sorted)
|
|
276
|
+
if all(str(k).isdigit() for k in raw_content.keys()):
|
|
277
|
+
sorted_keys = sorted(raw_content.keys(), key=lambda x: int(x))
|
|
278
|
+
for key in sorted_keys:
|
|
279
|
+
value = raw_content[key]
|
|
280
|
+
if isinstance(value, dict):
|
|
281
|
+
documents.append(normalize_to_wrapped_format(value))
|
|
282
|
+
if not documents:
|
|
283
|
+
raise ValueError("Numeric key format: no dictionary values found")
|
|
284
|
+
return documents
|
|
285
|
+
|
|
286
|
+
# Case 4: Dictionary with arbitrary keys - extract all dict values
|
|
287
|
+
for key, value in raw_content.items():
|
|
288
|
+
if isinstance(value, dict):
|
|
289
|
+
documents.append(normalize_to_wrapped_format(value))
|
|
290
|
+
|
|
291
|
+
if documents:
|
|
292
|
+
return documents
|
|
293
|
+
|
|
294
|
+
# Case 5: fallback — the entire dict is a single generic document
|
|
295
|
+
if raw_content:
|
|
296
|
+
logger.warning(
|
|
297
|
+
"json_converter: no article/document markers found; indexing the "
|
|
298
|
+
"whole JSON object as a single generic document (top-level keys: %s). "
|
|
299
|
+
"An accidental upload (e.g. an API error payload) would still be indexed.",
|
|
300
|
+
list(raw_content.keys())[:10],
|
|
301
|
+
)
|
|
302
|
+
return [normalize_to_wrapped_format(raw_content)]
|
|
303
|
+
|
|
304
|
+
raise ValueError("No valid documents found: empty JSON object")
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def combine_documents(docs_list: list[dict], source_path: str) -> dict:
|
|
308
|
+
"""
|
|
309
|
+
Combine multiple documents into a single document.
|
|
310
|
+
"""
|
|
311
|
+
if not docs_list:
|
|
312
|
+
raise ValueError("Cannot combine empty document list")
|
|
313
|
+
|
|
314
|
+
# If only one document, return it as-is
|
|
315
|
+
if len(docs_list) == 1:
|
|
316
|
+
return docs_list[0]
|
|
317
|
+
|
|
318
|
+
# Create a combined DoclingDocument
|
|
319
|
+
combined_doc = DoclingDocument(name=source_path)
|
|
320
|
+
|
|
321
|
+
# Collect all metadata
|
|
322
|
+
all_metadata = {
|
|
323
|
+
"source": source_path,
|
|
324
|
+
"document_count": len(docs_list),
|
|
325
|
+
"combined": True,
|
|
326
|
+
"sub_documents": []
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
# Combine all text from all documents
|
|
330
|
+
for idx, doc_dict in enumerate(docs_list):
|
|
331
|
+
docling_doc = doc_dict["document"]
|
|
332
|
+
metadata = doc_dict["metadata"]
|
|
333
|
+
|
|
334
|
+
# Add metadata reference
|
|
335
|
+
all_metadata["sub_documents"].append({
|
|
336
|
+
"index": idx,
|
|
337
|
+
"id_article": metadata.get("id_article"),
|
|
338
|
+
"id_element": metadata.get("id_element")
|
|
339
|
+
})
|
|
340
|
+
|
|
341
|
+
# Add all texts from this document
|
|
342
|
+
for text_item in docling_doc.texts:
|
|
343
|
+
# FIX: Access TextItem object attributes using dot notation, not dict syntax
|
|
344
|
+
# Use the original label value as DocItemLabel is a strict enum
|
|
345
|
+
combined_doc.add_text(
|
|
346
|
+
text=text_item.text, # Changed from text_item["text"]
|
|
347
|
+
label=text_item.label
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
return {
|
|
351
|
+
"document": combined_doc,
|
|
352
|
+
"metadata": all_metadata
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def process_single_document(doc: dict, source_path: str) -> dict:
|
|
357
|
+
"""
|
|
358
|
+
Process a single document.
|
|
359
|
+
|
|
360
|
+
Args:
|
|
361
|
+
doc: Document in wrapped format
|
|
362
|
+
source_path: Path to source file
|
|
363
|
+
|
|
364
|
+
Returns:
|
|
365
|
+
Dict with DoclingDocument and metadata
|
|
366
|
+
"""
|
|
367
|
+
metadata = build_metadata(doc)
|
|
368
|
+
metadata["source"] = source_path
|
|
369
|
+
|
|
370
|
+
# Create a DoclingDocument
|
|
371
|
+
text_content = build_embedding_content(doc)
|
|
372
|
+
docling_doc = DoclingDocument(name=source_path)
|
|
373
|
+
|
|
374
|
+
# Split content into smaller paragraphs to help with chunking
|
|
375
|
+
paragraphs = text_content.split('\n\n')
|
|
376
|
+
|
|
377
|
+
for para in paragraphs:
|
|
378
|
+
if para.strip(): # Only add non-empty paragraphs
|
|
379
|
+
docling_doc.add_text(text=para.strip(), label="paragraph")
|
|
380
|
+
|
|
381
|
+
rag_doc = {
|
|
382
|
+
"document": docling_doc,
|
|
383
|
+
"metadata": metadata,
|
|
384
|
+
}
|
|
385
|
+
return rag_doc
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def special_json_jds(source: str) -> dict:
|
|
389
|
+
"""
|
|
390
|
+
Process special JSON format for JDS documents (single document).
|
|
391
|
+
|
|
392
|
+
Args:
|
|
393
|
+
source: Path to JSON file
|
|
394
|
+
|
|
395
|
+
Returns:
|
|
396
|
+
Dict with document and metadata
|
|
397
|
+
"""
|
|
398
|
+
# Check if file exists
|
|
399
|
+
if not os.path.exists(source):
|
|
400
|
+
raise FileNotFoundError(f"JSON file not found: {source}")
|
|
401
|
+
|
|
402
|
+
# Check if file is empty
|
|
403
|
+
if os.path.getsize(source) == 0:
|
|
404
|
+
raise ValueError(f"JSON file is empty (0 bytes): {source}")
|
|
405
|
+
|
|
406
|
+
# Parse JSON (NDJSON-aware: falls back to one object per line)
|
|
407
|
+
raw_content = load_json_source(source)
|
|
408
|
+
|
|
409
|
+
# Extract first document (this handles all the format variations)
|
|
410
|
+
doc = extract_first_document(raw_content)
|
|
411
|
+
|
|
412
|
+
# Process and return
|
|
413
|
+
return process_single_document(doc, source)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def special_json_jds_multi(source: str) -> list[dict]:
|
|
417
|
+
"""
|
|
418
|
+
Process special JSON format for JDS documents (multiple documents).
|
|
419
|
+
|
|
420
|
+
Args:
|
|
421
|
+
source: Path to JSON file
|
|
422
|
+
|
|
423
|
+
Returns:
|
|
424
|
+
List of dicts with document and metadata
|
|
425
|
+
"""
|
|
426
|
+
# Check if file exists
|
|
427
|
+
if not os.path.exists(source):
|
|
428
|
+
raise FileNotFoundError(f"JSON file not found: {source}")
|
|
429
|
+
|
|
430
|
+
# Check if file is empty
|
|
431
|
+
if os.path.getsize(source) == 0:
|
|
432
|
+
raise ValueError(f"JSON file is empty (0 bytes): {source}")
|
|
433
|
+
|
|
434
|
+
# Parse JSON (NDJSON-aware: falls back to one object per line)
|
|
435
|
+
raw_content = load_json_source(source)
|
|
436
|
+
|
|
437
|
+
# Extract all documents
|
|
438
|
+
documents = extract_all_documents(raw_content)
|
|
439
|
+
|
|
440
|
+
# Process all documents
|
|
441
|
+
results = []
|
|
442
|
+
failed_count = 0
|
|
443
|
+
|
|
444
|
+
for i, doc in enumerate(documents):
|
|
445
|
+
try:
|
|
446
|
+
results.append(process_single_document(doc, source))
|
|
447
|
+
except Exception as e:
|
|
448
|
+
# Log the error but continue processing other documents
|
|
449
|
+
failed_count += 1
|
|
450
|
+
print(f"Warning: Failed to process document {i+1}/{len(documents)}: {str(e)}")
|
|
451
|
+
continue
|
|
452
|
+
|
|
453
|
+
if not results:
|
|
454
|
+
raise ValueError(
|
|
455
|
+
f"All {len(documents)} document(s) failed to process. "
|
|
456
|
+
f"Check document structure and required fields."
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
if failed_count > 0:
|
|
460
|
+
print(f"Warning: {failed_count}/{len(documents)} document(s) failed to process")
|
|
461
|
+
|
|
462
|
+
return results
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def convert_json(
|
|
466
|
+
source: str,
|
|
467
|
+
special_format: str = "jds",
|
|
468
|
+
multi_doc: bool = None
|
|
469
|
+
) -> dict:
|
|
470
|
+
"""
|
|
471
|
+
Convert JSON file to document format.
|
|
472
|
+
|
|
473
|
+
Args:
|
|
474
|
+
source: Path to JSON file
|
|
475
|
+
special_format: Format type (default: "jds")
|
|
476
|
+
multi_doc: If True, process all documents and combine; if False, process first only;
|
|
477
|
+
if None (default), auto-detect based on file content
|
|
478
|
+
|
|
479
|
+
Returns:
|
|
480
|
+
Dict with document and metadata (always returns a single combined document)
|
|
481
|
+
|
|
482
|
+
Raises:
|
|
483
|
+
ValueError: If JSON format is invalid or file is empty
|
|
484
|
+
FileNotFoundError: If source file doesn't exist
|
|
485
|
+
json.JSONDecodeError: If JSON syntax is invalid
|
|
486
|
+
"""
|
|
487
|
+
# Check file existence and size first
|
|
488
|
+
if not os.path.exists(source):
|
|
489
|
+
raise FileNotFoundError(f"JSON file not found: {source}")
|
|
490
|
+
|
|
491
|
+
if os.path.getsize(source) == 0:
|
|
492
|
+
raise ValueError(f"JSON file is empty (0 bytes): {source}")
|
|
493
|
+
|
|
494
|
+
# Parse JSON (NDJSON-aware: falls back to one object per line)
|
|
495
|
+
raw_content = load_json_source(source)
|
|
496
|
+
|
|
497
|
+
# Auto-detect multi_doc if not specified
|
|
498
|
+
if multi_doc is None:
|
|
499
|
+
result = detect_multi_doc(raw_content)
|
|
500
|
+
if result: # result is either False or int > 1
|
|
501
|
+
multi_doc = True
|
|
502
|
+
else:
|
|
503
|
+
multi_doc = False
|
|
504
|
+
|
|
505
|
+
if special_format == "jds":
|
|
506
|
+
if multi_doc:
|
|
507
|
+
# Process multiple documents and combine into one
|
|
508
|
+
docs_list = special_json_jds_multi(source)
|
|
509
|
+
return combine_documents(docs_list, source)
|
|
510
|
+
return special_json_jds(source)
|
|
511
|
+
|
|
512
|
+
# Generic JSON handling (for future extensibility)
|
|
513
|
+
if multi_doc:
|
|
514
|
+
documents = extract_all_documents(raw_content)
|
|
515
|
+
results = []
|
|
516
|
+
for doc in documents:
|
|
517
|
+
results.append(process_single_document(doc, source))
|
|
518
|
+
return combine_documents(results, source)
|
|
519
|
+
else:
|
|
520
|
+
doc = extract_first_document(raw_content)
|
|
521
|
+
return process_single_document(doc, source)
|