OpenRAG 0.4.1.dev15__py3-none-any.whl → 0.4.1.dev16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api/settings.py +9 -2
- api/upload.py +17 -29
- config/embedding_constants.py +4 -0
- config/model_constants.py +0 -2
- config/settings.py +51 -44
- connectors/service.py +7 -7
- main.py +57 -42
- models/processors.py +37 -20
- {openrag-0.4.1.dev15.dist-info → openrag-0.4.1.dev16.dist-info}/METADATA +4 -2
- {openrag-0.4.1.dev15.dist-info → openrag-0.4.1.dev16.dist-info}/RECORD +21 -20
- services/document_service.py +15 -9
- services/models_service.py +150 -13
- services/search_service.py +62 -25
- services/task_service.py +5 -1
- session_manager.py +2 -0
- utils/embedding_fields.py +4 -2
- utils/embeddings.py +2 -159
- {openrag-0.4.1.dev15.dist-info → openrag-0.4.1.dev16.dist-info}/WHEEL +0 -0
- {openrag-0.4.1.dev15.dist-info → openrag-0.4.1.dev16.dist-info}/entry_points.txt +0 -0
- {openrag-0.4.1.dev15.dist-info → openrag-0.4.1.dev16.dist-info}/licenses/LICENSE +0 -0
- {openrag-0.4.1.dev15.dist-info → openrag-0.4.1.dev16.dist-info}/top_level.txt +0 -0
api/settings.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from dependencies import get_models_service
|
|
1
2
|
import asyncio
|
|
2
3
|
import json
|
|
3
4
|
import platform
|
|
@@ -9,7 +10,6 @@ from utils.telemetry import TelemetryClient, Category, MessageId
|
|
|
9
10
|
from utils.version_utils import OPENRAG_VERSION
|
|
10
11
|
from config.settings import (
|
|
11
12
|
DEFAULT_DOCS_URL,
|
|
12
|
-
DISABLE_INGEST_WITH_LANGFLOW,
|
|
13
13
|
INGEST_SAMPLE_DATA,
|
|
14
14
|
LANGFLOW_URL,
|
|
15
15
|
LANGFLOW_CHAT_FLOW_ID,
|
|
@@ -885,6 +885,7 @@ async def onboarding(
|
|
|
885
885
|
flows_service=Depends(get_flows_service),
|
|
886
886
|
session_manager=Depends(get_session_manager),
|
|
887
887
|
document_service=Depends(get_document_service),
|
|
888
|
+
models_service=Depends(get_models_service),
|
|
888
889
|
task_service=Depends(get_task_service),
|
|
889
890
|
langflow_file_service=Depends(get_langflow_file_service),
|
|
890
891
|
knowledge_filter_service=Depends(get_knowledge_filter_service),
|
|
@@ -936,7 +937,7 @@ async def onboarding(
|
|
|
936
937
|
embedding_model_selected = None
|
|
937
938
|
embedding_provider_selected = None
|
|
938
939
|
|
|
939
|
-
if body.embedding_model
|
|
940
|
+
if body.embedding_model:
|
|
940
941
|
embedding_model_selected = body.embedding_model.strip()
|
|
941
942
|
current_config.knowledge.embedding_model = embedding_model_selected
|
|
942
943
|
config_updated = True
|
|
@@ -1151,8 +1152,12 @@ async def onboarding(
|
|
|
1151
1152
|
# Import the function here to avoid circular imports
|
|
1152
1153
|
from main import ingest_default_documents_when_ready
|
|
1153
1154
|
|
|
1155
|
+
if not config_manager.save_config_file(current_config):
|
|
1156
|
+
logger.error("Failed to save embedding model to config")
|
|
1157
|
+
|
|
1154
1158
|
task_id = await ingest_default_documents_when_ready(
|
|
1155
1159
|
document_service,
|
|
1160
|
+
models_service,
|
|
1156
1161
|
task_service,
|
|
1157
1162
|
langflow_file_service,
|
|
1158
1163
|
session_manager,
|
|
@@ -1888,6 +1893,7 @@ async def update_docling_preset(
|
|
|
1888
1893
|
async def refresh_openrag_docs(
|
|
1889
1894
|
document_service=Depends(get_document_service),
|
|
1890
1895
|
task_service=Depends(get_task_service),
|
|
1896
|
+
models_service=Depends(get_models_service),
|
|
1891
1897
|
langflow_file_service=Depends(get_langflow_file_service),
|
|
1892
1898
|
session_manager=Depends(get_session_manager),
|
|
1893
1899
|
user: User = Depends(get_current_user),
|
|
@@ -1898,6 +1904,7 @@ async def refresh_openrag_docs(
|
|
|
1898
1904
|
|
|
1899
1905
|
refreshed = await refresh_default_openrag_docs(
|
|
1900
1906
|
document_service=document_service,
|
|
1907
|
+
models_service=models_service,
|
|
1901
1908
|
task_service=task_service,
|
|
1902
1909
|
langflow_file_service=langflow_file_service,
|
|
1903
1910
|
session_manager=session_manager,
|
api/upload.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from dependencies import get_models_service
|
|
1
2
|
import os
|
|
2
3
|
from typing import Optional
|
|
3
4
|
from urllib.parse import urlparse
|
|
@@ -35,15 +36,10 @@ async def upload(
|
|
|
35
36
|
try:
|
|
36
37
|
|
|
37
38
|
from config.settings import is_no_auth_mode
|
|
38
|
-
|
|
39
|
-
if
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
owner_email = None
|
|
43
|
-
else:
|
|
44
|
-
owner_user_id = user.user_id
|
|
45
|
-
owner_name = user.name
|
|
46
|
-
owner_email = user.email
|
|
39
|
+
is_no_auth = is_no_auth_mode()
|
|
40
|
+
owner_user_id = user.user_id if (user and not is_no_auth) else None
|
|
41
|
+
owner_name = user.name if user else None
|
|
42
|
+
owner_email = user.email if user else None
|
|
47
43
|
|
|
48
44
|
result = await document_service.process_upload_file(
|
|
49
45
|
file,
|
|
@@ -84,15 +80,10 @@ async def upload_path(
|
|
|
84
80
|
jwt_token = user.jwt_token
|
|
85
81
|
|
|
86
82
|
from config.settings import is_no_auth_mode
|
|
87
|
-
|
|
88
|
-
if
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
owner_email = None
|
|
92
|
-
else:
|
|
93
|
-
owner_user_id = user.user_id
|
|
94
|
-
owner_name = user.name
|
|
95
|
-
owner_email = user.email
|
|
83
|
+
is_no_auth = is_no_auth_mode()
|
|
84
|
+
owner_user_id = user.user_id if (user and not is_no_auth) else None
|
|
85
|
+
owner_name = user.name if user else None
|
|
86
|
+
owner_email = user.email if user else None
|
|
96
87
|
|
|
97
88
|
from api.documents import _ensure_index_exists
|
|
98
89
|
await _ensure_index_exists()
|
|
@@ -163,6 +154,7 @@ async def upload_options(
|
|
|
163
154
|
async def upload_bucket(
|
|
164
155
|
body: UploadBucketBody,
|
|
165
156
|
task_service=Depends(get_task_service),
|
|
157
|
+
models_service=Depends(get_models_service),
|
|
166
158
|
session_manager=Depends(get_session_manager),
|
|
167
159
|
user: User = Depends(get_current_user),
|
|
168
160
|
):
|
|
@@ -194,18 +186,13 @@ async def upload_bucket(
|
|
|
194
186
|
jwt_token = user.jwt_token
|
|
195
187
|
|
|
196
188
|
from models.processors import S3FileProcessor
|
|
197
|
-
from config.settings import is_no_auth_mode
|
|
198
189
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
else
|
|
205
|
-
owner_user_id = user.user_id
|
|
206
|
-
owner_name = user.name
|
|
207
|
-
owner_email = user.email
|
|
208
|
-
task_user_id = user.user_id
|
|
190
|
+
from config.settings import is_no_auth_mode
|
|
191
|
+
is_no_auth = is_no_auth_mode()
|
|
192
|
+
owner_user_id = user.user_id if (user and not is_no_auth) else None
|
|
193
|
+
owner_name = user.name if user else None
|
|
194
|
+
owner_email = user.email if user else None
|
|
195
|
+
task_user_id = user.user_id if (user and not is_no_auth) else None
|
|
209
196
|
|
|
210
197
|
from api.documents import _ensure_index_exists
|
|
211
198
|
await _ensure_index_exists()
|
|
@@ -213,6 +200,7 @@ async def upload_bucket(
|
|
|
213
200
|
processor = S3FileProcessor(
|
|
214
201
|
task_service.document_service,
|
|
215
202
|
bucket,
|
|
203
|
+
models_service=models_service,
|
|
216
204
|
s3_client=s3_client,
|
|
217
205
|
owner_user_id=owner_user_id,
|
|
218
206
|
jwt_token=jwt_token,
|
config/model_constants.py
CHANGED
|
@@ -31,8 +31,6 @@ OPENAI_VALIDATION_MODELS = [
|
|
|
31
31
|
]
|
|
32
32
|
|
|
33
33
|
OPENAI_DEFAULT_LANGUAGE_MODEL = "gpt-4o"
|
|
34
|
-
OPENAI_DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small"
|
|
35
|
-
OPENAI_EMBEDDING_MODEL_PREFIX = "text-embedding"
|
|
36
34
|
|
|
37
35
|
ANTHROPIC_DEFAULT_LANGUAGE_MODEL = "claude-sonnet-4-5-20250929"
|
|
38
36
|
|
config/settings.py
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import os
|
|
3
|
+
import threading
|
|
4
|
+
import concurrent.futures
|
|
3
5
|
from utils.env_utils import get_env_int, get_env_float
|
|
4
6
|
|
|
5
7
|
import httpx
|
|
8
|
+
from agentd.patch import patch_openai_with_mcp
|
|
6
9
|
from dotenv import load_dotenv
|
|
7
10
|
from openai import AsyncOpenAI
|
|
8
11
|
from opensearchpy import AsyncOpenSearch
|
|
9
12
|
from opensearchpy._async.http_aiohttp import AIOHttpConnection
|
|
13
|
+
from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL
|
|
10
14
|
|
|
11
15
|
from utils.container_utils import get_container_host
|
|
12
16
|
from utils.logging_config import get_logger
|
|
@@ -106,26 +110,6 @@ WEBHOOK_BASE_URL = os.getenv(
|
|
|
106
110
|
VECTOR_DIM = 1536
|
|
107
111
|
KNN_EF_CONSTRUCTION = 100
|
|
108
112
|
KNN_M = 16
|
|
109
|
-
EMBED_MODEL = "text-embedding-3-small"
|
|
110
|
-
|
|
111
|
-
OPENAI_EMBEDDING_DIMENSIONS = {
|
|
112
|
-
"text-embedding-3-small": 1536,
|
|
113
|
-
"text-embedding-3-large": 3072,
|
|
114
|
-
"text-embedding-ada-002": 1536,
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
WATSONX_EMBEDDING_DIMENSIONS = {
|
|
118
|
-
# IBM Models
|
|
119
|
-
"ibm/granite-embedding-107m-multilingual": 384,
|
|
120
|
-
"ibm/granite-embedding-278m-multilingual": 1024,
|
|
121
|
-
"ibm/slate-125m-english-rtrvr": 768,
|
|
122
|
-
"ibm/slate-125m-english-rtrvr-v2": 768,
|
|
123
|
-
"ibm/slate-30m-english-rtrvr": 384,
|
|
124
|
-
"ibm/slate-30m-english-rtrvr-v2": 384,
|
|
125
|
-
# Third Party Models
|
|
126
|
-
"intfloat/multilingual-e5-large": 1024,
|
|
127
|
-
"sentence-transformers/all-minilm-l6-v2": 384,
|
|
128
|
-
}
|
|
129
113
|
|
|
130
114
|
INDEX_BODY = {
|
|
131
115
|
"settings": {
|
|
@@ -319,7 +303,7 @@ class AppClients:
|
|
|
319
303
|
self.langflow_client = None
|
|
320
304
|
self.langflow_http_client = None
|
|
321
305
|
self._patched_async_client = None # Private attribute - single client for all providers
|
|
322
|
-
self._client_init_lock =
|
|
306
|
+
self._client_init_lock = threading.Lock() # Lock for thread-safe initialization
|
|
323
307
|
self.docling_http_client = None
|
|
324
308
|
|
|
325
309
|
async def initialize(self):
|
|
@@ -454,6 +438,11 @@ class AppClients:
|
|
|
454
438
|
if config.providers.openai.api_key:
|
|
455
439
|
os.environ["OPENAI_API_KEY"] = config.providers.openai.api_key
|
|
456
440
|
logger.debug("Loaded OpenAI API key from config")
|
|
441
|
+
elif not os.environ.get("OPENAI_API_KEY"):
|
|
442
|
+
# Provide dummy key to satisfy AsyncOpenAI constructor;
|
|
443
|
+
# LiteLLM/MCP will handle routing to other providers if needed.
|
|
444
|
+
os.environ["OPENAI_API_KEY"] = "no-key-required"
|
|
445
|
+
logger.debug("Using dummy OpenAI API key to satisfy client constructor")
|
|
457
446
|
|
|
458
447
|
# Set Anthropic credentials
|
|
459
448
|
if config.providers.anthropic.api_key:
|
|
@@ -477,14 +466,23 @@ class AppClients:
|
|
|
477
466
|
os.environ["OLLAMA_ENDPOINT"] = config.providers.ollama.endpoint
|
|
478
467
|
logger.debug("Loaded Ollama endpoint from config")
|
|
479
468
|
|
|
469
|
+
# Determine model and provider for both probe and production client
|
|
470
|
+
model_name = config.knowledge.embedding_model or OPENAI_DEFAULT_EMBEDDING_MODEL
|
|
471
|
+
provider = config.knowledge.embedding_provider or "openai"
|
|
480
472
|
except Exception as e:
|
|
481
473
|
logger.debug("Could not load provider credentials from config", error=str(e))
|
|
474
|
+
# Provide fallbacks if config loading failed
|
|
475
|
+
model_name = OPENAI_DEFAULT_EMBEDDING_MODEL
|
|
476
|
+
provider = "openai"
|
|
477
|
+
# Ensure a dummy key is available to satisfy the AsyncOpenAI constructor
|
|
478
|
+
# and avoid AuthenticationError if config loading failed.
|
|
479
|
+
if not os.environ.get("OPENAI_API_KEY"):
|
|
480
|
+
os.environ["OPENAI_API_KEY"] = "no-key-required"
|
|
481
|
+
logger.debug("Using dummy OpenAI API key fallback (config load failed)")
|
|
482
|
+
|
|
482
483
|
|
|
483
|
-
#
|
|
484
|
-
|
|
485
|
-
import asyncio
|
|
486
|
-
import concurrent.futures
|
|
487
|
-
import threading
|
|
484
|
+
# API key for AsyncOpenAI constructor
|
|
485
|
+
api_key = os.environ.get("OPENAI_API_KEY")
|
|
488
486
|
|
|
489
487
|
async def probe_http2():
|
|
490
488
|
"""Returns True if HTTP/2 works, False to fall back to HTTP/1.1.
|
|
@@ -494,20 +492,21 @@ class AppClients:
|
|
|
494
492
|
production client is created after this thread exits, in the
|
|
495
493
|
caller's event loop, avoiding cross-loop SSL transport errors.
|
|
496
494
|
"""
|
|
497
|
-
client
|
|
498
|
-
|
|
495
|
+
# Use a standard OpenAI client for the probe (only runs for OpenAI provider)
|
|
496
|
+
client = AsyncOpenAI(api_key=api_key)
|
|
497
|
+
logger.info(f"Probing client with HTTP/2 using model {model_name}...")
|
|
499
498
|
try:
|
|
500
499
|
await asyncio.wait_for(
|
|
501
500
|
client.embeddings.create(
|
|
502
|
-
model=
|
|
501
|
+
model=model_name,
|
|
503
502
|
input=['test']
|
|
504
503
|
),
|
|
505
504
|
timeout=5.0
|
|
506
505
|
)
|
|
507
|
-
logger.info("HTTP/2 probe successful")
|
|
506
|
+
logger.info(f"HTTP/2 probe successful with {model_name}")
|
|
508
507
|
return True
|
|
509
508
|
except (asyncio.TimeoutError, Exception) as probe_error:
|
|
510
|
-
logger.warning("HTTP/2 probe failed, falling back to HTTP/1.1", error=str(probe_error))
|
|
509
|
+
logger.warning(f"HTTP/2 probe failed with {model_name}, falling back to HTTP/1.1", error=str(probe_error))
|
|
511
510
|
return False
|
|
512
511
|
finally:
|
|
513
512
|
# Always close the probe client so its connections are fully
|
|
@@ -527,24 +526,32 @@ class AppClients:
|
|
|
527
526
|
loop.close()
|
|
528
527
|
|
|
529
528
|
try:
|
|
530
|
-
# Run the probe
|
|
531
|
-
#
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
529
|
+
# Run the probe only for OpenAI provider; local and other providers
|
|
530
|
+
# (Ollama, WatsonX) typically use HTTP/1.1 for reliability.
|
|
531
|
+
if provider.lower() == "openai":
|
|
532
|
+
# Run the probe in a separate thread with its own event loop.
|
|
533
|
+
# Only the probe result (bool) crosses the thread boundary;
|
|
534
|
+
# the production client is created here so its connections are
|
|
535
|
+
# bound to the caller's event loop, not the (now closed) probe loop.
|
|
536
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
|
537
|
+
future = executor.submit(run_probe_in_thread)
|
|
538
|
+
use_http2 = future.result(timeout=15)
|
|
539
|
+
else:
|
|
540
|
+
use_http2 = False
|
|
541
|
+
logger.debug(f"Skipping HTTP/2 probe for provider: {provider}")
|
|
537
542
|
|
|
538
543
|
if use_http2:
|
|
539
|
-
self._patched_async_client = AsyncOpenAI()
|
|
540
|
-
logger.info("OpenAI client initialized with HTTP/2")
|
|
544
|
+
self._patched_async_client = patch_openai_with_mcp(AsyncOpenAI(api_key=api_key))
|
|
545
|
+
logger.info(f"OpenAI-compatible client initialized with HTTP/2 (model: {model_name})")
|
|
541
546
|
else:
|
|
542
547
|
http_client = httpx.AsyncClient(
|
|
543
548
|
http2=False,
|
|
544
549
|
timeout=httpx.Timeout(60.0, connect=10.0)
|
|
545
550
|
)
|
|
546
|
-
self._patched_async_client =
|
|
547
|
-
|
|
551
|
+
self._patched_async_client = patch_openai_with_mcp(
|
|
552
|
+
AsyncOpenAI(api_key=api_key, http_client=http_client)
|
|
553
|
+
)
|
|
554
|
+
logger.info(f"OpenAI-compatible client initialized with HTTP/1.1 fallback (model: {model_name})")
|
|
548
555
|
logger.info("Successfully initialized OpenAI client")
|
|
549
556
|
except Exception as e:
|
|
550
557
|
logger.error(f"Failed to initialize OpenAI client: {e.__class__.__name__}: {str(e)}")
|
|
@@ -861,7 +868,7 @@ def get_openrag_config():
|
|
|
861
868
|
# Expose configuration settings for backward compatibility and easy access
|
|
862
869
|
def get_provider_config():
|
|
863
870
|
"""Get provider configuration."""
|
|
864
|
-
return get_openrag_config().
|
|
871
|
+
return get_openrag_config().providers
|
|
865
872
|
|
|
866
873
|
|
|
867
874
|
def get_knowledge_config():
|
|
@@ -876,7 +883,7 @@ def get_agent_config():
|
|
|
876
883
|
|
|
877
884
|
def get_embedding_model() -> str:
|
|
878
885
|
"""Return the currently configured embedding model."""
|
|
879
|
-
return get_openrag_config().knowledge.embedding_model or
|
|
886
|
+
return get_openrag_config().knowledge.embedding_model or (OPENAI_DEFAULT_EMBEDDING_MODEL if DISABLE_INGEST_WITH_LANGFLOW else "")
|
|
880
887
|
|
|
881
888
|
|
|
882
889
|
def get_index_name() -> str:
|
connectors/service.py
CHANGED
|
@@ -20,6 +20,8 @@ class ConnectorService:
|
|
|
20
20
|
index_name: str,
|
|
21
21
|
task_service=None,
|
|
22
22
|
session_manager=None,
|
|
23
|
+
models_service=None,
|
|
24
|
+
document_service=None,
|
|
23
25
|
):
|
|
24
26
|
self.clients = patched_async_client
|
|
25
27
|
self.embed_model = embed_model
|
|
@@ -27,6 +29,8 @@ class ConnectorService:
|
|
|
27
29
|
self.task_service = task_service
|
|
28
30
|
self.session_manager = session_manager
|
|
29
31
|
self.connection_manager = ConnectionManager()
|
|
32
|
+
self.models_service = models_service
|
|
33
|
+
self.document_service = document_service
|
|
30
34
|
|
|
31
35
|
async def initialize(self):
|
|
32
36
|
"""Initialize the service by loading existing connections"""
|
|
@@ -57,18 +61,12 @@ class ConnectorService:
|
|
|
57
61
|
with open(tmp_path, "wb") as f:
|
|
58
62
|
f.write(document.content)
|
|
59
63
|
|
|
60
|
-
# Use existing process_file_common function with connector document metadata
|
|
61
|
-
# We'll use the document service's process_file_common method
|
|
62
|
-
from services.document_service import DocumentService
|
|
63
|
-
|
|
64
|
-
doc_service = DocumentService(session_manager=self.session_manager)
|
|
65
|
-
|
|
66
64
|
logger.debug("Processing connector document", document_id=document.id)
|
|
67
65
|
|
|
68
66
|
# Process using consolidated processing pipeline
|
|
69
67
|
from models.processors import TaskProcessor
|
|
70
68
|
|
|
71
|
-
processor = TaskProcessor(document_service=
|
|
69
|
+
processor = TaskProcessor(document_service=self.document_service, models_service=self.models_service)
|
|
72
70
|
result = await processor.process_document_standard(
|
|
73
71
|
file_path=tmp_path,
|
|
74
72
|
file_hash=document.id, # Use connector document ID as hash
|
|
@@ -284,6 +282,7 @@ class ConnectorService:
|
|
|
284
282
|
if self.task_service and self.task_service.document_service
|
|
285
283
|
else DocumentService(session_manager=self.session_manager)
|
|
286
284
|
),
|
|
285
|
+
models_service=self.models_service,
|
|
287
286
|
)
|
|
288
287
|
|
|
289
288
|
# Use file IDs as items (no more fake file paths!)
|
|
@@ -415,6 +414,7 @@ class ConnectorService:
|
|
|
415
414
|
if self.task_service and self.task_service.document_service
|
|
416
415
|
else DocumentService(session_manager=self.session_manager)
|
|
417
416
|
),
|
|
417
|
+
models_service=self.models_service,
|
|
418
418
|
)
|
|
419
419
|
|
|
420
420
|
# Create custom task using TaskService
|
main.py
CHANGED
|
@@ -15,7 +15,7 @@ from connectors.langflow_connector_service import LangflowConnectorService
|
|
|
15
15
|
from connectors.service import ConnectorService
|
|
16
16
|
from services.flows_service import FlowsService
|
|
17
17
|
from utils.container_utils import detect_container_environment
|
|
18
|
-
from utils.embeddings import
|
|
18
|
+
from utils.embeddings import create_index_body
|
|
19
19
|
from utils.logging_config import configure_from_env, get_logger
|
|
20
20
|
from utils.telemetry import TelemetryClient, Category, MessageId
|
|
21
21
|
from fastapi import FastAPI, Request
|
|
@@ -100,7 +100,7 @@ from services.knowledge_filter_service import KnowledgeFilterService
|
|
|
100
100
|
from services.langflow_file_service import LangflowFileService
|
|
101
101
|
from services.models_service import ModelsService
|
|
102
102
|
from services.monitor_service import MonitorService
|
|
103
|
-
from services.search_service import SearchService
|
|
103
|
+
from services.search_service import SearchService, register_search_service
|
|
104
104
|
from services.task_service import TaskService
|
|
105
105
|
from session_manager import SessionManager
|
|
106
106
|
|
|
@@ -221,22 +221,15 @@ async def init_index():
|
|
|
221
221
|
# Get the configured embedding model from user configuration
|
|
222
222
|
config = get_openrag_config()
|
|
223
223
|
embedding_model = config.knowledge.embedding_model
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
# Create dynamic index body based on the configured embedding model
|
|
228
|
-
# Pass provider and endpoint for dynamic dimension resolution (Ollama probing)
|
|
229
|
-
dynamic_index_body = await create_dynamic_index_body(
|
|
230
|
-
embedding_model,
|
|
231
|
-
provider=embedding_provider,
|
|
232
|
-
endpoint=getattr(embedding_provider_config, "endpoint", None),
|
|
233
|
-
)
|
|
224
|
+
|
|
225
|
+
# Create index body
|
|
226
|
+
index_body = await create_index_body()
|
|
234
227
|
|
|
235
228
|
# Create documents index
|
|
236
229
|
index_name = get_index_name()
|
|
237
230
|
if not await clients.opensearch.indices.exists(index=index_name):
|
|
238
231
|
await clients.opensearch.indices.create(
|
|
239
|
-
index=index_name, body=
|
|
232
|
+
index=index_name, body=index_body
|
|
240
233
|
)
|
|
241
234
|
logger.info(
|
|
242
235
|
"Created OpenSearch index",
|
|
@@ -435,7 +428,7 @@ def _should_use_url_default_docs_ingest() -> bool:
|
|
|
435
428
|
|
|
436
429
|
|
|
437
430
|
async def ingest_openrag_docs_when_ready(
|
|
438
|
-
document_service, task_service, langflow_file_service, session_manager
|
|
431
|
+
document_service, models_service, task_service, langflow_file_service, session_manager
|
|
439
432
|
):
|
|
440
433
|
"""Ingest OpenRAG docs during onboarding."""
|
|
441
434
|
use_url_ingest = _should_use_url_default_docs_ingest()
|
|
@@ -448,6 +441,7 @@ async def ingest_openrag_docs_when_ready(
|
|
|
448
441
|
if DISABLE_INGEST_WITH_LANGFLOW:
|
|
449
442
|
task_id = await _ingest_default_documents_url(
|
|
450
443
|
document_service=document_service,
|
|
444
|
+
models_service=models_service,
|
|
451
445
|
docs_url=DEFAULT_DOCS_URL,
|
|
452
446
|
crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,
|
|
453
447
|
)
|
|
@@ -475,7 +469,7 @@ async def ingest_openrag_docs_when_ready(
|
|
|
475
469
|
|
|
476
470
|
|
|
477
471
|
async def ingest_default_documents_when_ready(
|
|
478
|
-
document_service, task_service, langflow_file_service, session_manager
|
|
472
|
+
document_service, models_service, task_service, langflow_file_service, session_manager
|
|
479
473
|
):
|
|
480
474
|
"""Ingest default OpenRAG docs during onboarding."""
|
|
481
475
|
try:
|
|
@@ -487,11 +481,10 @@ async def ingest_default_documents_when_ready(
|
|
|
487
481
|
await TelemetryClient.send_event(
|
|
488
482
|
Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START
|
|
489
483
|
)
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
)
|
|
484
|
+
|
|
485
|
+
task_id = await ingest_openrag_docs_when_ready(
|
|
486
|
+
document_service, models_service, task_service, langflow_file_service, session_manager
|
|
487
|
+
)
|
|
495
488
|
|
|
496
489
|
base_dir = _get_documents_dir()
|
|
497
490
|
if not os.path.isdir(base_dir):
|
|
@@ -515,7 +508,7 @@ async def ingest_default_documents_when_ready(
|
|
|
515
508
|
|
|
516
509
|
if DISABLE_INGEST_WITH_LANGFLOW:
|
|
517
510
|
new_task_id = await _ingest_default_documents_openrag(
|
|
518
|
-
document_service, task_service, file_paths, existing_task_id=task_id, connector_type="local"
|
|
511
|
+
document_service, models_service, task_service, file_paths, existing_task_id=task_id, connector_type="local"
|
|
519
512
|
)
|
|
520
513
|
task_id = new_task_id or task_id
|
|
521
514
|
else:
|
|
@@ -670,6 +663,7 @@ async def _ingest_default_documents_url_langflow(
|
|
|
670
663
|
|
|
671
664
|
async def _ingest_default_documents_url(
|
|
672
665
|
document_service,
|
|
666
|
+
models_service,
|
|
673
667
|
docs_url: str,
|
|
674
668
|
crawl_depth: int,
|
|
675
669
|
):
|
|
@@ -690,12 +684,17 @@ async def _ingest_default_documents_url(
|
|
|
690
684
|
from models.processors import DocumentFileProcessor
|
|
691
685
|
from utils.hash_utils import hash_id
|
|
692
686
|
|
|
687
|
+
from session_manager import AnonymousUser
|
|
688
|
+
|
|
689
|
+
anonymous_user = AnonymousUser()
|
|
690
|
+
|
|
693
691
|
processor = DocumentFileProcessor(
|
|
694
692
|
document_service,
|
|
693
|
+
models_service=models_service,
|
|
695
694
|
owner_user_id=None,
|
|
696
695
|
jwt_token=None,
|
|
697
|
-
owner_name=
|
|
698
|
-
owner_email=
|
|
696
|
+
owner_name=anonymous_user.name,
|
|
697
|
+
owner_email=anonymous_user.email,
|
|
699
698
|
is_sample_data=True,
|
|
700
699
|
connector_type="openrag_docs",
|
|
701
700
|
)
|
|
@@ -705,8 +704,8 @@ async def _ingest_default_documents_url(
|
|
|
705
704
|
owner_user_id=None,
|
|
706
705
|
original_filename="openrag-url-default.txt",
|
|
707
706
|
jwt_token=None,
|
|
708
|
-
owner_name=
|
|
709
|
-
owner_email=
|
|
707
|
+
owner_name=anonymous_user.name,
|
|
708
|
+
owner_email=anonymous_user.email,
|
|
710
709
|
file_size=os.path.getsize(temp_file_path),
|
|
711
710
|
connector_type="openrag_docs",
|
|
712
711
|
is_sample_data=True,
|
|
@@ -801,16 +800,6 @@ async def _delete_existing_default_docs(session_manager, connector_type: str):
|
|
|
801
800
|
]
|
|
802
801
|
}
|
|
803
802
|
},
|
|
804
|
-
# Legacy file-based default docs were ingested as local and
|
|
805
|
-
# marked with is_sample_data=true.
|
|
806
|
-
{
|
|
807
|
-
"bool": {
|
|
808
|
-
"must": [
|
|
809
|
-
{"term": {"connector_type": "local"}},
|
|
810
|
-
{"term": {"is_sample_data": "true"}},
|
|
811
|
-
]
|
|
812
|
-
}
|
|
813
|
-
},
|
|
814
803
|
],
|
|
815
804
|
"minimum_should_match": 1,
|
|
816
805
|
}
|
|
@@ -829,6 +818,7 @@ async def _delete_existing_default_docs(session_manager, connector_type: str):
|
|
|
829
818
|
|
|
830
819
|
async def _reingest_default_docs_on_upgrade_if_needed(
|
|
831
820
|
document_service,
|
|
821
|
+
models_service,
|
|
832
822
|
task_service,
|
|
833
823
|
langflow_file_service,
|
|
834
824
|
session_manager,
|
|
@@ -857,6 +847,7 @@ async def _reingest_default_docs_on_upgrade_if_needed(
|
|
|
857
847
|
await _delete_existing_default_docs(session_manager, connector_type="openrag_docs")
|
|
858
848
|
await ingest_openrag_docs_when_ready(
|
|
859
849
|
document_service,
|
|
850
|
+
models_service,
|
|
860
851
|
task_service,
|
|
861
852
|
langflow_file_service,
|
|
862
853
|
session_manager,
|
|
@@ -926,6 +917,7 @@ async def _get_remote_docs_signature(docs_url: str):
|
|
|
926
917
|
|
|
927
918
|
async def refresh_default_openrag_docs(
|
|
928
919
|
document_service,
|
|
920
|
+
models_service,
|
|
929
921
|
task_service,
|
|
930
922
|
langflow_file_service,
|
|
931
923
|
session_manager,
|
|
@@ -1015,6 +1007,7 @@ async def refresh_default_openrag_docs(
|
|
|
1015
1007
|
await _delete_existing_default_docs(session_manager, connector_type="openrag_docs")
|
|
1016
1008
|
await ingest_openrag_docs_when_ready(
|
|
1017
1009
|
document_service,
|
|
1010
|
+
models_service,
|
|
1018
1011
|
task_service,
|
|
1019
1012
|
langflow_file_service,
|
|
1020
1013
|
session_manager,
|
|
@@ -1075,7 +1068,7 @@ async def opensearch_health_ready(request):
|
|
|
1075
1068
|
|
|
1076
1069
|
|
|
1077
1070
|
async def _ingest_default_documents_openrag(
|
|
1078
|
-
document_service, task_service, file_paths, existing_task_id: str = None, connector_type: str = "openrag_docs"
|
|
1071
|
+
document_service, models_service, task_service, file_paths, existing_task_id: str = None, connector_type: str = "openrag_docs"
|
|
1079
1072
|
):
|
|
1080
1073
|
"""Ingest default documents using traditional OpenRAG processor."""
|
|
1081
1074
|
logger.info(
|
|
@@ -1086,12 +1079,16 @@ async def _ingest_default_documents_openrag(
|
|
|
1086
1079
|
# Build a processor that DOES NOT set 'owner' on documents (owner_user_id=None)
|
|
1087
1080
|
from models.processors import DocumentFileProcessor
|
|
1088
1081
|
|
|
1082
|
+
from session_manager import AnonymousUser
|
|
1083
|
+
anonymous_user = AnonymousUser()
|
|
1084
|
+
|
|
1089
1085
|
processor = DocumentFileProcessor(
|
|
1090
1086
|
document_service,
|
|
1087
|
+
models_service=models_service,
|
|
1091
1088
|
owner_user_id=None,
|
|
1092
1089
|
jwt_token=None,
|
|
1093
|
-
owner_name=
|
|
1094
|
-
owner_email=
|
|
1090
|
+
owner_name=anonymous_user.name,
|
|
1091
|
+
owner_email=anonymous_user.email,
|
|
1095
1092
|
is_sample_data=True, # Mark as sample data
|
|
1096
1093
|
connector_type=connector_type,
|
|
1097
1094
|
)
|
|
@@ -1174,6 +1171,19 @@ async def startup_tasks(services):
|
|
|
1174
1171
|
await TelemetryClient.send_event(
|
|
1175
1172
|
Category.APPLICATION_STARTUP, MessageId.ORB_APP_START_INIT
|
|
1176
1173
|
)
|
|
1174
|
+
|
|
1175
|
+
# Update model registry to allow further search calls to be instant
|
|
1176
|
+
try:
|
|
1177
|
+
models_service = services["models_service"]
|
|
1178
|
+
await models_service.update_model_registry()
|
|
1179
|
+
except Exception as e:
|
|
1180
|
+
logger.error(
|
|
1181
|
+
"Failed to update model registry at startup — "
|
|
1182
|
+
"models may be missing until the next restart",
|
|
1183
|
+
error=str(e),
|
|
1184
|
+
)
|
|
1185
|
+
|
|
1186
|
+
|
|
1177
1187
|
# Only initialize basic OpenSearch connection, not the index
|
|
1178
1188
|
# Index will be created after onboarding when we know the embedding model
|
|
1179
1189
|
await wait_for_opensearch()
|
|
@@ -1216,6 +1226,7 @@ async def startup_tasks(services):
|
|
|
1216
1226
|
try:
|
|
1217
1227
|
upgrade_reingested = await _reingest_default_docs_on_upgrade_if_needed(
|
|
1218
1228
|
services["document_service"],
|
|
1229
|
+
services["models_service"],
|
|
1219
1230
|
services["task_service"],
|
|
1220
1231
|
services["langflow_file_service"],
|
|
1221
1232
|
services["session_manager"],
|
|
@@ -1227,6 +1238,7 @@ async def startup_tasks(services):
|
|
|
1227
1238
|
try:
|
|
1228
1239
|
await refresh_default_openrag_docs(
|
|
1229
1240
|
services["document_service"],
|
|
1241
|
+
services["models_service"],
|
|
1230
1242
|
services["task_service"],
|
|
1231
1243
|
services["langflow_file_service"],
|
|
1232
1244
|
services["session_manager"],
|
|
@@ -1316,13 +1328,14 @@ async def initialize_services():
|
|
|
1316
1328
|
session_manager = SessionManager(SESSION_SECRET)
|
|
1317
1329
|
|
|
1318
1330
|
# Initialize services
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1331
|
+
models_service = ModelsService()
|
|
1332
|
+
document_service = DocumentService(session_manager=session_manager, models_service=models_service)
|
|
1333
|
+
search_service = SearchService(session_manager, models_service)
|
|
1334
|
+
register_search_service(search_service)
|
|
1335
|
+
task_service = TaskService(document_service, models_service, ingestion_timeout=INGESTION_TIMEOUT)
|
|
1322
1336
|
flows_service = FlowsService()
|
|
1323
1337
|
chat_service = ChatService(flows_service=flows_service)
|
|
1324
1338
|
knowledge_filter_service = KnowledgeFilterService(session_manager)
|
|
1325
|
-
models_service = ModelsService()
|
|
1326
1339
|
monitor_service = MonitorService(session_manager)
|
|
1327
1340
|
langflow_file_service = LangflowFileService(flows_service=flows_service)
|
|
1328
1341
|
|
|
@@ -1337,6 +1350,8 @@ async def initialize_services():
|
|
|
1337
1350
|
index_name=get_index_name(),
|
|
1338
1351
|
task_service=task_service,
|
|
1339
1352
|
session_manager=session_manager,
|
|
1353
|
+
models_service=models_service,
|
|
1354
|
+
document_service=document_service,
|
|
1340
1355
|
)
|
|
1341
1356
|
|
|
1342
1357
|
# Create connector router that chooses based on configuration
|