OpenRAG 0.1.25__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent.py +669 -0
- api/__init__.py +0 -0
- api/auth.py +84 -0
- api/chat.py +181 -0
- api/connector_router.py +78 -0
- api/connectors.py +437 -0
- api/docling.py +120 -0
- api/documents.py +103 -0
- api/flows.py +66 -0
- api/knowledge_filter.py +468 -0
- api/langflow_files.py +296 -0
- api/models.py +148 -0
- api/nudges.py +44 -0
- api/oidc.py +111 -0
- api/router.py +206 -0
- api/search.py +53 -0
- api/settings.py +762 -0
- api/tasks.py +35 -0
- api/upload.py +202 -0
- auth_context.py +75 -0
- auth_middleware.py +84 -0
- config/__init__.py +0 -0
- config/config_manager.py +214 -0
- config/settings.py +602 -0
- connectors/__init__.py +11 -0
- connectors/base.py +149 -0
- connectors/connection_manager.py +532 -0
- connectors/google_drive/__init__.py +4 -0
- connectors/google_drive/connector.py +1112 -0
- connectors/google_drive/oauth.py +152 -0
- connectors/langflow_connector_service.py +342 -0
- connectors/onedrive/__init__.py +4 -0
- connectors/onedrive/connector.py +494 -0
- connectors/onedrive/oauth.py +322 -0
- connectors/service.py +402 -0
- connectors/sharepoint/__init__.py +4 -0
- connectors/sharepoint/connector.py +567 -0
- connectors/sharepoint/oauth.py +329 -0
- main.py +1186 -0
- models/__init__.py +0 -0
- models/processors.py +769 -0
- models/tasks.py +46 -0
- openrag-0.1.25.dist-info/METADATA +166 -0
- openrag-0.1.25.dist-info/RECORD +112 -0
- openrag-0.1.25.dist-info/WHEEL +5 -0
- openrag-0.1.25.dist-info/entry_points.txt +2 -0
- openrag-0.1.25.dist-info/licenses/LICENSE +201 -0
- openrag-0.1.25.dist-info/top_level.txt +12 -0
- services/__init__.py +0 -0
- services/auth_service.py +401 -0
- services/chat_service.py +537 -0
- services/conversation_persistence_service.py +130 -0
- services/document_service.py +213 -0
- services/flows_service.py +906 -0
- services/knowledge_filter_service.py +347 -0
- services/langflow_file_service.py +338 -0
- services/langflow_history_service.py +172 -0
- services/langflow_mcp_service.py +275 -0
- services/models_service.py +372 -0
- services/monitor_service.py +305 -0
- services/search_service.py +477 -0
- services/session_ownership_service.py +109 -0
- services/task_service.py +435 -0
- session_manager.py +235 -0
- tui/__init__.py +8 -0
- tui/_assets/docker-compose-cpu.yml +127 -0
- tui/_assets/docker-compose.yml +126 -0
- tui/_assets/documents/docling.pdf +0 -0
- tui/_assets/documents/ibm_anthropic.pdf +0 -0
- tui/_assets/documents/openrag-documentation.pdf +0 -0
- tui/_assets/documents/warmup_ocr.pdf +0 -0
- tui/_assets/flows/components/ollama_embedding.json +142 -0
- tui/_assets/flows/components/ollama_llm.json +682 -0
- tui/_assets/flows/components/ollama_llm_text.json +683 -0
- tui/_assets/flows/components/watsonx_embedding.json +207 -0
- tui/_assets/flows/components/watsonx_llm.json +521 -0
- tui/_assets/flows/components/watsonx_llm_text.json +521 -0
- tui/_assets/flows/ingestion_flow.json +4199 -0
- tui/_assets/flows/openrag_agent.json +2875 -0
- tui/_assets/flows/openrag_nudges.json +2350 -0
- tui/_assets/flows/openrag_url_mcp.json +3621 -0
- tui/main.py +521 -0
- tui/managers/__init__.py +1 -0
- tui/managers/container_manager.py +892 -0
- tui/managers/docling_manager.py +488 -0
- tui/managers/env_manager.py +442 -0
- tui/screens/__init__.py +1 -0
- tui/screens/config.py +690 -0
- tui/screens/diagnostics.py +572 -0
- tui/screens/logs.py +356 -0
- tui/screens/monitor.py +858 -0
- tui/screens/welcome.py +446 -0
- tui/utils/__init__.py +1 -0
- tui/utils/clipboard.py +50 -0
- tui/utils/platform.py +209 -0
- tui/utils/validation.py +139 -0
- tui/widgets/__init__.py +3 -0
- tui/widgets/command_modal.py +379 -0
- tui/widgets/diagnostics_notification.py +40 -0
- tui/widgets/error_notification.py +40 -0
- tui/widgets/waves.py +146 -0
- utils/__init__.py +0 -0
- utils/container_utils.py +298 -0
- utils/document_processing.py +370 -0
- utils/embedding_fields.py +178 -0
- utils/embeddings.py +198 -0
- utils/file_utils.py +60 -0
- utils/gpu_detection.py +47 -0
- utils/hash_utils.py +76 -0
- utils/logging_config.py +117 -0
- utils/opensearch_queries.py +55 -0
- utils/process_pool.py +13 -0
api/auth.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from starlette.requests import Request
|
|
2
|
+
from starlette.responses import JSONResponse
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
async def auth_init(request: Request, auth_service, session_manager):
|
|
6
|
+
"""Initialize OAuth flow for authentication or data source connection"""
|
|
7
|
+
try:
|
|
8
|
+
data = await request.json()
|
|
9
|
+
connector_type = data.get("connector_type")
|
|
10
|
+
purpose = data.get("purpose", "data_source")
|
|
11
|
+
connection_name = data.get("name", f"{connector_type}_{purpose}")
|
|
12
|
+
redirect_uri = data.get("redirect_uri")
|
|
13
|
+
|
|
14
|
+
user = getattr(request.state, "user", None)
|
|
15
|
+
user_id = user.user_id if user else None
|
|
16
|
+
|
|
17
|
+
result = await auth_service.init_oauth(
|
|
18
|
+
connector_type, purpose, connection_name, redirect_uri, user_id
|
|
19
|
+
)
|
|
20
|
+
return JSONResponse(result)
|
|
21
|
+
|
|
22
|
+
except Exception as e:
|
|
23
|
+
import traceback
|
|
24
|
+
|
|
25
|
+
traceback.print_exc()
|
|
26
|
+
return JSONResponse(
|
|
27
|
+
{"error": f"Failed to initialize OAuth: {str(e)}"}, status_code=500
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
async def auth_callback(request: Request, auth_service, session_manager):
|
|
32
|
+
"""Handle OAuth callback - exchange authorization code for tokens"""
|
|
33
|
+
try:
|
|
34
|
+
data = await request.json()
|
|
35
|
+
connection_id = data.get("connection_id")
|
|
36
|
+
authorization_code = data.get("authorization_code")
|
|
37
|
+
state = data.get("state")
|
|
38
|
+
|
|
39
|
+
result = await auth_service.handle_oauth_callback(
|
|
40
|
+
connection_id, authorization_code, state, request
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# If this is app auth, set JWT cookie
|
|
44
|
+
if result.get("purpose") == "app_auth" and result.get("jwt_token"):
|
|
45
|
+
response = JSONResponse(
|
|
46
|
+
{k: v for k, v in result.items() if k != "jwt_token"}
|
|
47
|
+
)
|
|
48
|
+
response.set_cookie(
|
|
49
|
+
key="auth_token",
|
|
50
|
+
value=result["jwt_token"],
|
|
51
|
+
httponly=True,
|
|
52
|
+
secure=False,
|
|
53
|
+
samesite="lax",
|
|
54
|
+
max_age=7 * 24 * 60 * 60, # 7 days
|
|
55
|
+
)
|
|
56
|
+
return response
|
|
57
|
+
else:
|
|
58
|
+
return JSONResponse(result)
|
|
59
|
+
|
|
60
|
+
except Exception as e:
|
|
61
|
+
import traceback
|
|
62
|
+
|
|
63
|
+
traceback.print_exc()
|
|
64
|
+
return JSONResponse({"error": f"Callback failed: {str(e)}"}, status_code=500)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
async def auth_me(request: Request, auth_service, session_manager):
|
|
68
|
+
"""Get current user information"""
|
|
69
|
+
result = await auth_service.get_user_info(request)
|
|
70
|
+
return JSONResponse(result)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
async def auth_logout(request: Request, auth_service, session_manager):
|
|
74
|
+
"""Logout user by clearing auth cookie"""
|
|
75
|
+
response = JSONResponse(
|
|
76
|
+
{"status": "logged_out", "message": "Successfully logged out"}
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
# Clear the auth cookie
|
|
80
|
+
response.delete_cookie(
|
|
81
|
+
key="auth_token", httponly=True, secure=False, samesite="lax"
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
return response
|
api/chat.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
from starlette.requests import Request
|
|
2
|
+
from starlette.responses import JSONResponse, StreamingResponse
|
|
3
|
+
from utils.logging_config import get_logger
|
|
4
|
+
|
|
5
|
+
logger = get_logger(__name__)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
async def chat_endpoint(request: Request, chat_service, session_manager):
|
|
9
|
+
"""Handle chat requests"""
|
|
10
|
+
data = await request.json()
|
|
11
|
+
prompt = data.get("prompt", "")
|
|
12
|
+
previous_response_id = data.get("previous_response_id")
|
|
13
|
+
stream = data.get("stream", False)
|
|
14
|
+
filters = data.get("filters")
|
|
15
|
+
limit = data.get("limit", 10)
|
|
16
|
+
score_threshold = data.get("scoreThreshold", 0)
|
|
17
|
+
|
|
18
|
+
user = request.state.user
|
|
19
|
+
user_id = user.user_id
|
|
20
|
+
|
|
21
|
+
jwt_token = session_manager.get_effective_jwt_token(user_id, request.state.jwt_token)
|
|
22
|
+
|
|
23
|
+
if not prompt:
|
|
24
|
+
return JSONResponse({"error": "Prompt is required"}, status_code=400)
|
|
25
|
+
|
|
26
|
+
# Set context variables for search tool (similar to search endpoint)
|
|
27
|
+
if filters:
|
|
28
|
+
from auth_context import set_search_filters
|
|
29
|
+
|
|
30
|
+
set_search_filters(filters)
|
|
31
|
+
|
|
32
|
+
from auth_context import set_search_limit, set_score_threshold
|
|
33
|
+
|
|
34
|
+
set_search_limit(limit)
|
|
35
|
+
set_score_threshold(score_threshold)
|
|
36
|
+
|
|
37
|
+
if stream:
|
|
38
|
+
return StreamingResponse(
|
|
39
|
+
await chat_service.chat(
|
|
40
|
+
prompt,
|
|
41
|
+
user_id,
|
|
42
|
+
jwt_token,
|
|
43
|
+
previous_response_id=previous_response_id,
|
|
44
|
+
stream=True,
|
|
45
|
+
),
|
|
46
|
+
media_type="text/event-stream",
|
|
47
|
+
headers={
|
|
48
|
+
"Cache-Control": "no-cache",
|
|
49
|
+
"Connection": "keep-alive",
|
|
50
|
+
"Access-Control-Allow-Origin": "*",
|
|
51
|
+
"Access-Control-Allow-Headers": "Cache-Control",
|
|
52
|
+
},
|
|
53
|
+
)
|
|
54
|
+
else:
|
|
55
|
+
result = await chat_service.chat(
|
|
56
|
+
prompt,
|
|
57
|
+
user_id,
|
|
58
|
+
jwt_token,
|
|
59
|
+
previous_response_id=previous_response_id,
|
|
60
|
+
stream=False,
|
|
61
|
+
)
|
|
62
|
+
return JSONResponse(result)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def langflow_endpoint(request: Request, chat_service, session_manager):
|
|
66
|
+
"""Handle Langflow chat requests"""
|
|
67
|
+
data = await request.json()
|
|
68
|
+
prompt = data.get("prompt", "")
|
|
69
|
+
previous_response_id = data.get("previous_response_id")
|
|
70
|
+
stream = data.get("stream", False)
|
|
71
|
+
filters = data.get("filters")
|
|
72
|
+
limit = data.get("limit", 10)
|
|
73
|
+
score_threshold = data.get("scoreThreshold", 0)
|
|
74
|
+
|
|
75
|
+
user = request.state.user
|
|
76
|
+
user_id = user.user_id
|
|
77
|
+
|
|
78
|
+
jwt_token = session_manager.get_effective_jwt_token(user_id, request.state.jwt_token)
|
|
79
|
+
|
|
80
|
+
if not prompt:
|
|
81
|
+
return JSONResponse({"error": "Prompt is required"}, status_code=400)
|
|
82
|
+
|
|
83
|
+
# Set context variables for search tool (similar to chat endpoint)
|
|
84
|
+
if filters:
|
|
85
|
+
from auth_context import set_search_filters
|
|
86
|
+
|
|
87
|
+
set_search_filters(filters)
|
|
88
|
+
|
|
89
|
+
from auth_context import set_search_limit, set_score_threshold
|
|
90
|
+
|
|
91
|
+
set_search_limit(limit)
|
|
92
|
+
set_score_threshold(score_threshold)
|
|
93
|
+
|
|
94
|
+
try:
|
|
95
|
+
if stream:
|
|
96
|
+
return StreamingResponse(
|
|
97
|
+
await chat_service.langflow_chat(
|
|
98
|
+
prompt,
|
|
99
|
+
user_id,
|
|
100
|
+
jwt_token,
|
|
101
|
+
previous_response_id=previous_response_id,
|
|
102
|
+
stream=True,
|
|
103
|
+
),
|
|
104
|
+
media_type="text/event-stream",
|
|
105
|
+
headers={
|
|
106
|
+
"Cache-Control": "no-cache",
|
|
107
|
+
"Connection": "keep-alive",
|
|
108
|
+
"Access-Control-Allow-Origin": "*",
|
|
109
|
+
"Access-Control-Allow-Headers": "Cache-Control",
|
|
110
|
+
},
|
|
111
|
+
)
|
|
112
|
+
else:
|
|
113
|
+
result = await chat_service.langflow_chat(
|
|
114
|
+
prompt,
|
|
115
|
+
user_id,
|
|
116
|
+
jwt_token,
|
|
117
|
+
previous_response_id=previous_response_id,
|
|
118
|
+
stream=False,
|
|
119
|
+
)
|
|
120
|
+
return JSONResponse(result)
|
|
121
|
+
|
|
122
|
+
except Exception as e:
|
|
123
|
+
import traceback
|
|
124
|
+
|
|
125
|
+
traceback.print_exc()
|
|
126
|
+
logger.error("Langflow request failed", error=str(e))
|
|
127
|
+
return JSONResponse(
|
|
128
|
+
{"error": f"Langflow request failed: {str(e)}"}, status_code=500
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
async def chat_history_endpoint(request: Request, chat_service, session_manager):
|
|
133
|
+
"""Get chat history for a user"""
|
|
134
|
+
user = request.state.user
|
|
135
|
+
user_id = user.user_id
|
|
136
|
+
|
|
137
|
+
try:
|
|
138
|
+
history = await chat_service.get_chat_history(user_id)
|
|
139
|
+
return JSONResponse(history)
|
|
140
|
+
except Exception as e:
|
|
141
|
+
return JSONResponse(
|
|
142
|
+
{"error": f"Failed to get chat history: {str(e)}"}, status_code=500
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
async def langflow_history_endpoint(request: Request, chat_service, session_manager):
|
|
147
|
+
"""Get langflow chat history for a user"""
|
|
148
|
+
user = request.state.user
|
|
149
|
+
user_id = user.user_id
|
|
150
|
+
|
|
151
|
+
try:
|
|
152
|
+
history = await chat_service.get_langflow_history(user_id)
|
|
153
|
+
return JSONResponse(history)
|
|
154
|
+
except Exception as e:
|
|
155
|
+
return JSONResponse(
|
|
156
|
+
{"error": f"Failed to get langflow history: {str(e)}"}, status_code=500
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
async def delete_session_endpoint(request: Request, chat_service, session_manager):
|
|
161
|
+
"""Delete a chat session"""
|
|
162
|
+
user = request.state.user
|
|
163
|
+
user_id = user.user_id
|
|
164
|
+
session_id = request.path_params["session_id"]
|
|
165
|
+
|
|
166
|
+
try:
|
|
167
|
+
# Delete from both local storage and Langflow
|
|
168
|
+
result = await chat_service.delete_session(user_id, session_id)
|
|
169
|
+
|
|
170
|
+
if result.get("success"):
|
|
171
|
+
return JSONResponse({"message": "Session deleted successfully"})
|
|
172
|
+
else:
|
|
173
|
+
return JSONResponse(
|
|
174
|
+
{"error": result.get("error", "Failed to delete session")},
|
|
175
|
+
status_code=500
|
|
176
|
+
)
|
|
177
|
+
except Exception as e:
|
|
178
|
+
logger.error(f"Error deleting session: {e}")
|
|
179
|
+
return JSONResponse(
|
|
180
|
+
{"error": f"Failed to delete session: {str(e)}"}, status_code=500
|
|
181
|
+
)
|
api/connector_router.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Connector router that automatically routes based on configuration settings."""
|
|
2
|
+
|
|
3
|
+
from starlette.requests import Request
|
|
4
|
+
|
|
5
|
+
from config.settings import (
|
|
6
|
+
DISABLE_INGEST_WITH_LANGFLOW,
|
|
7
|
+
clients,
|
|
8
|
+
INDEX_NAME,
|
|
9
|
+
INDEX_BODY,
|
|
10
|
+
)
|
|
11
|
+
from utils.logging_config import get_logger
|
|
12
|
+
|
|
13
|
+
logger = get_logger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ConnectorRouter:
|
|
17
|
+
"""
|
|
18
|
+
Router that automatically chooses between LangflowConnectorService and ConnectorService
|
|
19
|
+
based on the DISABLE_INGEST_WITH_LANGFLOW configuration.
|
|
20
|
+
|
|
21
|
+
- If DISABLE_INGEST_WITH_LANGFLOW is False (default): uses LangflowConnectorService
|
|
22
|
+
- If DISABLE_INGEST_WITH_LANGFLOW is True: uses traditional ConnectorService
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, langflow_connector_service, openrag_connector_service):
|
|
26
|
+
self.langflow_connector_service = langflow_connector_service
|
|
27
|
+
self.openrag_connector_service = openrag_connector_service
|
|
28
|
+
logger.debug(
|
|
29
|
+
"ConnectorRouter initialized",
|
|
30
|
+
disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
def get_active_service(self):
|
|
34
|
+
"""Get the currently active connector service based on configuration."""
|
|
35
|
+
if DISABLE_INGEST_WITH_LANGFLOW:
|
|
36
|
+
logger.debug("Using traditional OpenRAG connector service")
|
|
37
|
+
return self.openrag_connector_service
|
|
38
|
+
else:
|
|
39
|
+
logger.debug("Using Langflow connector service")
|
|
40
|
+
return self.langflow_connector_service
|
|
41
|
+
|
|
42
|
+
# Proxy all connector service methods to the active service
|
|
43
|
+
|
|
44
|
+
async def initialize(self):
|
|
45
|
+
"""Initialize the active connector service."""
|
|
46
|
+
# Initialize OpenSearch index if using traditional OpenRAG connector service
|
|
47
|
+
|
|
48
|
+
return await self.get_active_service().initialize()
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def connection_manager(self):
|
|
52
|
+
"""Get the connection manager from the active service."""
|
|
53
|
+
return self.get_active_service().connection_manager
|
|
54
|
+
|
|
55
|
+
async def get_connector(self, connection_id: str):
|
|
56
|
+
"""Get a connector instance from the active service."""
|
|
57
|
+
return await self.get_active_service().get_connector(connection_id)
|
|
58
|
+
|
|
59
|
+
async def sync_specific_files(
|
|
60
|
+
self, connection_id: str, user_id: str, file_list: list, jwt_token: str = None
|
|
61
|
+
):
|
|
62
|
+
"""Sync specific files using the active service."""
|
|
63
|
+
return await self.get_active_service().sync_specific_files(
|
|
64
|
+
connection_id, user_id, file_list, jwt_token
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
def __getattr__(self, name):
|
|
68
|
+
"""
|
|
69
|
+
Proxy any other method calls to the active service.
|
|
70
|
+
This ensures compatibility with any methods we might have missed.
|
|
71
|
+
"""
|
|
72
|
+
active_service = self.get_active_service()
|
|
73
|
+
if hasattr(active_service, name):
|
|
74
|
+
return getattr(active_service, name)
|
|
75
|
+
else:
|
|
76
|
+
raise AttributeError(
|
|
77
|
+
f"'{type(active_service).__name__}' object has no attribute '{name}'"
|
|
78
|
+
)
|