OpenRAG 0.1.25__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent.py +669 -0
- api/__init__.py +0 -0
- api/auth.py +84 -0
- api/chat.py +181 -0
- api/connector_router.py +78 -0
- api/connectors.py +437 -0
- api/docling.py +120 -0
- api/documents.py +103 -0
- api/flows.py +66 -0
- api/knowledge_filter.py +468 -0
- api/langflow_files.py +296 -0
- api/models.py +148 -0
- api/nudges.py +44 -0
- api/oidc.py +111 -0
- api/router.py +206 -0
- api/search.py +53 -0
- api/settings.py +762 -0
- api/tasks.py +35 -0
- api/upload.py +202 -0
- auth_context.py +75 -0
- auth_middleware.py +84 -0
- config/__init__.py +0 -0
- config/config_manager.py +214 -0
- config/settings.py +602 -0
- connectors/__init__.py +11 -0
- connectors/base.py +149 -0
- connectors/connection_manager.py +532 -0
- connectors/google_drive/__init__.py +4 -0
- connectors/google_drive/connector.py +1112 -0
- connectors/google_drive/oauth.py +152 -0
- connectors/langflow_connector_service.py +342 -0
- connectors/onedrive/__init__.py +4 -0
- connectors/onedrive/connector.py +494 -0
- connectors/onedrive/oauth.py +322 -0
- connectors/service.py +402 -0
- connectors/sharepoint/__init__.py +4 -0
- connectors/sharepoint/connector.py +567 -0
- connectors/sharepoint/oauth.py +329 -0
- main.py +1186 -0
- models/__init__.py +0 -0
- models/processors.py +769 -0
- models/tasks.py +46 -0
- openrag-0.1.25.dist-info/METADATA +166 -0
- openrag-0.1.25.dist-info/RECORD +112 -0
- openrag-0.1.25.dist-info/WHEEL +5 -0
- openrag-0.1.25.dist-info/entry_points.txt +2 -0
- openrag-0.1.25.dist-info/licenses/LICENSE +201 -0
- openrag-0.1.25.dist-info/top_level.txt +12 -0
- services/__init__.py +0 -0
- services/auth_service.py +401 -0
- services/chat_service.py +537 -0
- services/conversation_persistence_service.py +130 -0
- services/document_service.py +213 -0
- services/flows_service.py +906 -0
- services/knowledge_filter_service.py +347 -0
- services/langflow_file_service.py +338 -0
- services/langflow_history_service.py +172 -0
- services/langflow_mcp_service.py +275 -0
- services/models_service.py +372 -0
- services/monitor_service.py +305 -0
- services/search_service.py +477 -0
- services/session_ownership_service.py +109 -0
- services/task_service.py +435 -0
- session_manager.py +235 -0
- tui/__init__.py +8 -0
- tui/_assets/docker-compose-cpu.yml +127 -0
- tui/_assets/docker-compose.yml +126 -0
- tui/_assets/documents/docling.pdf +0 -0
- tui/_assets/documents/ibm_anthropic.pdf +0 -0
- tui/_assets/documents/openrag-documentation.pdf +0 -0
- tui/_assets/documents/warmup_ocr.pdf +0 -0
- tui/_assets/flows/components/ollama_embedding.json +142 -0
- tui/_assets/flows/components/ollama_llm.json +682 -0
- tui/_assets/flows/components/ollama_llm_text.json +683 -0
- tui/_assets/flows/components/watsonx_embedding.json +207 -0
- tui/_assets/flows/components/watsonx_llm.json +521 -0
- tui/_assets/flows/components/watsonx_llm_text.json +521 -0
- tui/_assets/flows/ingestion_flow.json +4199 -0
- tui/_assets/flows/openrag_agent.json +2875 -0
- tui/_assets/flows/openrag_nudges.json +2350 -0
- tui/_assets/flows/openrag_url_mcp.json +3621 -0
- tui/main.py +521 -0
- tui/managers/__init__.py +1 -0
- tui/managers/container_manager.py +892 -0
- tui/managers/docling_manager.py +488 -0
- tui/managers/env_manager.py +442 -0
- tui/screens/__init__.py +1 -0
- tui/screens/config.py +690 -0
- tui/screens/diagnostics.py +572 -0
- tui/screens/logs.py +356 -0
- tui/screens/monitor.py +858 -0
- tui/screens/welcome.py +446 -0
- tui/utils/__init__.py +1 -0
- tui/utils/clipboard.py +50 -0
- tui/utils/platform.py +209 -0
- tui/utils/validation.py +139 -0
- tui/widgets/__init__.py +3 -0
- tui/widgets/command_modal.py +379 -0
- tui/widgets/diagnostics_notification.py +40 -0
- tui/widgets/error_notification.py +40 -0
- tui/widgets/waves.py +146 -0
- utils/__init__.py +0 -0
- utils/container_utils.py +298 -0
- utils/document_processing.py +370 -0
- utils/embedding_fields.py +178 -0
- utils/embeddings.py +198 -0
- utils/file_utils.py +60 -0
- utils/gpu_detection.py +47 -0
- utils/hash_utils.py +76 -0
- utils/logging_config.py +117 -0
- utils/opensearch_queries.py +55 -0
- utils/process_pool.py +13 -0
api/connectors.py
ADDED
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
from starlette.requests import Request
|
|
2
|
+
from starlette.responses import JSONResponse, PlainTextResponse
|
|
3
|
+
from utils.logging_config import get_logger
|
|
4
|
+
|
|
5
|
+
logger = get_logger(__name__)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
async def list_connectors(request: Request, connector_service, session_manager):
|
|
9
|
+
"""List available connector types with metadata"""
|
|
10
|
+
try:
|
|
11
|
+
connector_types = (
|
|
12
|
+
connector_service.connection_manager.get_available_connector_types()
|
|
13
|
+
)
|
|
14
|
+
return JSONResponse({"connectors": connector_types})
|
|
15
|
+
except Exception as e:
|
|
16
|
+
logger.info("Error listing connectors", error=str(e))
|
|
17
|
+
return JSONResponse({"connectors": []})
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
async def connector_sync(request: Request, connector_service, session_manager):
|
|
21
|
+
"""Sync files from all active connections of a connector type"""
|
|
22
|
+
connector_type = request.path_params.get("connector_type", "google_drive")
|
|
23
|
+
data = await request.json()
|
|
24
|
+
max_files = data.get("max_files")
|
|
25
|
+
selected_files = data.get("selected_files")
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
logger.debug(
|
|
29
|
+
"Starting connector sync",
|
|
30
|
+
connector_type=connector_type,
|
|
31
|
+
max_files=max_files,
|
|
32
|
+
)
|
|
33
|
+
user = request.state.user
|
|
34
|
+
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
|
35
|
+
|
|
36
|
+
# Get all active connections for this connector type and user
|
|
37
|
+
connections = await connector_service.connection_manager.list_connections(
|
|
38
|
+
user_id=user.user_id, connector_type=connector_type
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
active_connections = [conn for conn in connections if conn.is_active]
|
|
42
|
+
if not active_connections:
|
|
43
|
+
return JSONResponse(
|
|
44
|
+
{"error": f"No active {connector_type} connections found"},
|
|
45
|
+
status_code=404,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
# Find the first connection that actually works
|
|
49
|
+
working_connection = None
|
|
50
|
+
for connection in active_connections:
|
|
51
|
+
logger.debug(
|
|
52
|
+
"Testing connection authentication",
|
|
53
|
+
connection_id=connection.connection_id,
|
|
54
|
+
)
|
|
55
|
+
try:
|
|
56
|
+
# Get the connector instance and test authentication
|
|
57
|
+
connector = await connector_service.get_connector(connection.connection_id)
|
|
58
|
+
if connector and await connector.authenticate():
|
|
59
|
+
working_connection = connection
|
|
60
|
+
logger.debug(
|
|
61
|
+
"Found working connection",
|
|
62
|
+
connection_id=connection.connection_id,
|
|
63
|
+
)
|
|
64
|
+
break
|
|
65
|
+
else:
|
|
66
|
+
logger.debug(
|
|
67
|
+
"Connection authentication failed",
|
|
68
|
+
connection_id=connection.connection_id,
|
|
69
|
+
)
|
|
70
|
+
except Exception as e:
|
|
71
|
+
logger.debug(
|
|
72
|
+
"Connection validation failed",
|
|
73
|
+
connection_id=connection.connection_id,
|
|
74
|
+
error=str(e),
|
|
75
|
+
)
|
|
76
|
+
continue
|
|
77
|
+
|
|
78
|
+
if not working_connection:
|
|
79
|
+
return JSONResponse(
|
|
80
|
+
{"error": f"No working {connector_type} connections found"},
|
|
81
|
+
status_code=404,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
# Use the working connection
|
|
85
|
+
logger.debug(
|
|
86
|
+
"Starting sync with working connection",
|
|
87
|
+
connection_id=working_connection.connection_id,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
if selected_files:
|
|
91
|
+
task_id = await connector_service.sync_specific_files(
|
|
92
|
+
working_connection.connection_id,
|
|
93
|
+
user.user_id,
|
|
94
|
+
selected_files,
|
|
95
|
+
jwt_token=jwt_token,
|
|
96
|
+
)
|
|
97
|
+
else:
|
|
98
|
+
task_id = await connector_service.sync_connector_files(
|
|
99
|
+
working_connection.connection_id,
|
|
100
|
+
user.user_id,
|
|
101
|
+
max_files,
|
|
102
|
+
jwt_token=jwt_token,
|
|
103
|
+
)
|
|
104
|
+
task_ids = [task_id]
|
|
105
|
+
return JSONResponse(
|
|
106
|
+
{
|
|
107
|
+
"task_ids": task_ids,
|
|
108
|
+
"status": "sync_started",
|
|
109
|
+
"message": f"Started syncing files from {len(active_connections)} {connector_type} connection(s)",
|
|
110
|
+
"connections_synced": len(active_connections),
|
|
111
|
+
},
|
|
112
|
+
status_code=201,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
except Exception as e:
|
|
116
|
+
logger.error("Connector sync failed", error=str(e))
|
|
117
|
+
return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
async def connector_status(request: Request, connector_service, session_manager):
|
|
121
|
+
"""Get connector status for authenticated user"""
|
|
122
|
+
connector_type = request.path_params.get("connector_type", "google_drive")
|
|
123
|
+
user = request.state.user
|
|
124
|
+
|
|
125
|
+
# Get connections for this connector type and user
|
|
126
|
+
connections = await connector_service.connection_manager.list_connections(
|
|
127
|
+
user_id=user.user_id, connector_type=connector_type
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
# Get the connector for each connection
|
|
131
|
+
connection_client_ids = {}
|
|
132
|
+
for connection in connections:
|
|
133
|
+
try:
|
|
134
|
+
connector = await connector_service._get_connector(connection.connection_id)
|
|
135
|
+
if connector is not None:
|
|
136
|
+
connection_client_ids[connection.connection_id] = connector.get_client_id()
|
|
137
|
+
else:
|
|
138
|
+
connection_client_ids[connection.connection_id] = None
|
|
139
|
+
except Exception as e:
|
|
140
|
+
logger.warning(
|
|
141
|
+
"Could not get connector for connection",
|
|
142
|
+
connection_id=connection.connection_id,
|
|
143
|
+
error=str(e),
|
|
144
|
+
)
|
|
145
|
+
connection.connector = None
|
|
146
|
+
|
|
147
|
+
# Check if there are any active connections
|
|
148
|
+
active_connections = [conn for conn in connections if conn.is_active]
|
|
149
|
+
has_authenticated_connection = len(active_connections) > 0
|
|
150
|
+
|
|
151
|
+
return JSONResponse(
|
|
152
|
+
{
|
|
153
|
+
"connector_type": connector_type,
|
|
154
|
+
"authenticated": has_authenticated_connection,
|
|
155
|
+
"status": "connected" if has_authenticated_connection else "not_connected",
|
|
156
|
+
"connections": [
|
|
157
|
+
{
|
|
158
|
+
"connection_id": conn.connection_id,
|
|
159
|
+
"name": conn.name,
|
|
160
|
+
"client_id": connection_client_ids.get(conn.connection_id),
|
|
161
|
+
"is_active": conn.is_active,
|
|
162
|
+
"created_at": conn.created_at.isoformat(),
|
|
163
|
+
"last_sync": conn.last_sync.isoformat() if conn.last_sync else None,
|
|
164
|
+
}
|
|
165
|
+
for conn in connections
|
|
166
|
+
],
|
|
167
|
+
}
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
async def connector_webhook(request: Request, connector_service, session_manager):
|
|
172
|
+
"""Handle webhook notifications from any connector type"""
|
|
173
|
+
connector_type = request.path_params.get("connector_type")
|
|
174
|
+
if connector_type is None:
|
|
175
|
+
connector_type = "unknown"
|
|
176
|
+
|
|
177
|
+
# Handle webhook validation (connector-specific)
|
|
178
|
+
temp_config = {"token_file": "temp.json"}
|
|
179
|
+
from connectors.connection_manager import ConnectionConfig
|
|
180
|
+
|
|
181
|
+
temp_connection = ConnectionConfig(
|
|
182
|
+
connection_id="temp",
|
|
183
|
+
connector_type=str(connector_type),
|
|
184
|
+
name="temp",
|
|
185
|
+
config=temp_config,
|
|
186
|
+
)
|
|
187
|
+
try:
|
|
188
|
+
temp_connector = connector_service.connection_manager._create_connector(
|
|
189
|
+
temp_connection
|
|
190
|
+
)
|
|
191
|
+
validation_response = temp_connector.handle_webhook_validation(
|
|
192
|
+
request.method, dict(request.headers), dict(request.query_params)
|
|
193
|
+
)
|
|
194
|
+
if validation_response:
|
|
195
|
+
return PlainTextResponse(validation_response)
|
|
196
|
+
except (NotImplementedError, ValueError):
|
|
197
|
+
# Connector type not found or validation not needed
|
|
198
|
+
pass
|
|
199
|
+
|
|
200
|
+
try:
|
|
201
|
+
# Get the raw payload and headers
|
|
202
|
+
payload = {}
|
|
203
|
+
headers = dict(request.headers)
|
|
204
|
+
|
|
205
|
+
if request.method == "POST":
|
|
206
|
+
content_type = headers.get("content-type", "").lower()
|
|
207
|
+
if "application/json" in content_type:
|
|
208
|
+
payload = await request.json()
|
|
209
|
+
else:
|
|
210
|
+
# Some webhooks send form data or plain text
|
|
211
|
+
body = await request.body()
|
|
212
|
+
payload = {"raw_body": body.decode("utf-8") if body else ""}
|
|
213
|
+
else:
|
|
214
|
+
# GET webhooks use query params
|
|
215
|
+
payload = dict(request.query_params)
|
|
216
|
+
|
|
217
|
+
# Add headers to payload for connector processing
|
|
218
|
+
payload["_headers"] = headers
|
|
219
|
+
payload["_method"] = request.method
|
|
220
|
+
|
|
221
|
+
logger.info("Webhook notification received", connector_type=connector_type)
|
|
222
|
+
|
|
223
|
+
# Extract channel/subscription ID using connector-specific method
|
|
224
|
+
try:
|
|
225
|
+
temp_connector = connector_service.connection_manager._create_connector(
|
|
226
|
+
temp_connection
|
|
227
|
+
)
|
|
228
|
+
channel_id = temp_connector.extract_webhook_channel_id(payload, headers)
|
|
229
|
+
except (NotImplementedError, ValueError):
|
|
230
|
+
channel_id = None
|
|
231
|
+
|
|
232
|
+
if not channel_id:
|
|
233
|
+
logger.warning(
|
|
234
|
+
"No channel ID found in webhook", connector_type=connector_type
|
|
235
|
+
)
|
|
236
|
+
return JSONResponse({"status": "ignored", "reason": "no_channel_id"})
|
|
237
|
+
|
|
238
|
+
# Find the specific connection for this webhook
|
|
239
|
+
connection = (
|
|
240
|
+
await connector_service.connection_manager.get_connection_by_webhook_id(
|
|
241
|
+
channel_id
|
|
242
|
+
)
|
|
243
|
+
)
|
|
244
|
+
if not connection or not connection.is_active:
|
|
245
|
+
logger.info(
|
|
246
|
+
"Unknown webhook channel, will auto-expire", channel_id=channel_id
|
|
247
|
+
)
|
|
248
|
+
return JSONResponse(
|
|
249
|
+
{"status": "ignored_unknown_channel", "channel_id": channel_id}
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
# Process webhook for the specific connection
|
|
253
|
+
try:
|
|
254
|
+
# Get the connector instance
|
|
255
|
+
connector = await connector_service._get_connector(connection.connection_id)
|
|
256
|
+
if not connector:
|
|
257
|
+
logger.error(
|
|
258
|
+
"Could not get connector for connection",
|
|
259
|
+
connection_id=connection.connection_id,
|
|
260
|
+
)
|
|
261
|
+
return JSONResponse(
|
|
262
|
+
{"status": "error", "reason": "connector_not_found"}
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
# Let the connector handle the webhook and return affected file IDs
|
|
266
|
+
affected_files = await connector.handle_webhook(payload)
|
|
267
|
+
|
|
268
|
+
if affected_files:
|
|
269
|
+
logger.info(
|
|
270
|
+
"Webhook connection files affected",
|
|
271
|
+
connection_id=connection.connection_id,
|
|
272
|
+
affected_count=len(affected_files),
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
# Generate JWT token for the user (needed for OpenSearch authentication)
|
|
276
|
+
user = session_manager.get_user(connection.user_id)
|
|
277
|
+
if user:
|
|
278
|
+
jwt_token = session_manager.create_jwt_token(user)
|
|
279
|
+
else:
|
|
280
|
+
jwt_token = None
|
|
281
|
+
|
|
282
|
+
# Trigger incremental sync for affected files
|
|
283
|
+
task_id = await connector_service.sync_specific_files(
|
|
284
|
+
connection.connection_id,
|
|
285
|
+
connection.user_id,
|
|
286
|
+
affected_files,
|
|
287
|
+
jwt_token=jwt_token,
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
result = {
|
|
291
|
+
"connection_id": connection.connection_id,
|
|
292
|
+
"task_id": task_id,
|
|
293
|
+
"affected_files": len(affected_files),
|
|
294
|
+
}
|
|
295
|
+
else:
|
|
296
|
+
# No specific files identified - just log the webhook
|
|
297
|
+
logger.info(
|
|
298
|
+
"Webhook general change detected, no specific files",
|
|
299
|
+
connection_id=connection.connection_id,
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
result = {
|
|
303
|
+
"connection_id": connection.connection_id,
|
|
304
|
+
"action": "logged_only",
|
|
305
|
+
"reason": "no_specific_files",
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
return JSONResponse(
|
|
309
|
+
{
|
|
310
|
+
"status": "processed",
|
|
311
|
+
"connector_type": connector_type,
|
|
312
|
+
"channel_id": channel_id,
|
|
313
|
+
**result,
|
|
314
|
+
}
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
except Exception as e:
|
|
318
|
+
logger.error(
|
|
319
|
+
"Failed to process webhook for connection",
|
|
320
|
+
connection_id=connection.connection_id,
|
|
321
|
+
error=str(e),
|
|
322
|
+
)
|
|
323
|
+
import traceback
|
|
324
|
+
|
|
325
|
+
traceback.print_exc()
|
|
326
|
+
|
|
327
|
+
return JSONResponse(
|
|
328
|
+
{
|
|
329
|
+
"status": "error",
|
|
330
|
+
"connector_type": connector_type,
|
|
331
|
+
"channel_id": channel_id,
|
|
332
|
+
"error": str(e),
|
|
333
|
+
},
|
|
334
|
+
status_code=500,
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
except Exception as e:
|
|
338
|
+
logger.error("Webhook processing failed", error=str(e))
|
|
339
|
+
return JSONResponse(
|
|
340
|
+
{"error": f"Webhook processing failed: {str(e)}"}, status_code=500
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
async def connector_token(request: Request, connector_service, session_manager):
|
|
344
|
+
"""Get access token for connector API calls (e.g., Pickers)."""
|
|
345
|
+
url_connector_type = request.path_params.get("connector_type")
|
|
346
|
+
connection_id = request.query_params.get("connection_id")
|
|
347
|
+
|
|
348
|
+
if not connection_id:
|
|
349
|
+
return JSONResponse({"error": "connection_id is required"}, status_code=400)
|
|
350
|
+
|
|
351
|
+
user = request.state.user
|
|
352
|
+
|
|
353
|
+
try:
|
|
354
|
+
# 1) Load the connection and verify ownership
|
|
355
|
+
connection = await connector_service.connection_manager.get_connection(connection_id)
|
|
356
|
+
if not connection or connection.user_id != user.user_id:
|
|
357
|
+
return JSONResponse({"error": "Connection not found"}, status_code=404)
|
|
358
|
+
|
|
359
|
+
# 2) Get the ACTUAL connector instance/type for this connection_id
|
|
360
|
+
connector = await connector_service._get_connector(connection_id)
|
|
361
|
+
if not connector:
|
|
362
|
+
return JSONResponse(
|
|
363
|
+
{"error": f"Connector not available - authentication may have failed for {url_connector_type}"},
|
|
364
|
+
status_code=404,
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
real_type = getattr(connector, "type", None) or getattr(connection, "connector_type", None)
|
|
368
|
+
if real_type is None:
|
|
369
|
+
return JSONResponse({"error": "Unable to determine connector type"}, status_code=500)
|
|
370
|
+
|
|
371
|
+
# Optional: warn if URL path type disagrees with real type
|
|
372
|
+
if url_connector_type and url_connector_type != real_type:
|
|
373
|
+
# You can downgrade this to debug if you expect cross-routing.
|
|
374
|
+
return JSONResponse(
|
|
375
|
+
{
|
|
376
|
+
"error": "Connector type mismatch",
|
|
377
|
+
"detail": {
|
|
378
|
+
"requested_type": url_connector_type,
|
|
379
|
+
"actual_type": real_type,
|
|
380
|
+
"hint": "Call the token endpoint using the correct connector_type for this connection_id.",
|
|
381
|
+
},
|
|
382
|
+
},
|
|
383
|
+
status_code=400,
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
# 3) Branch by the actual connector type
|
|
387
|
+
# GOOGLE DRIVE (google-auth)
|
|
388
|
+
if real_type == "google_drive" and hasattr(connector, "oauth"):
|
|
389
|
+
await connector.oauth.load_credentials()
|
|
390
|
+
if connector.oauth.creds and connector.oauth.creds.valid:
|
|
391
|
+
expires_in = None
|
|
392
|
+
try:
|
|
393
|
+
if connector.oauth.creds.expiry:
|
|
394
|
+
import time
|
|
395
|
+
expires_in = max(0, int(connector.oauth.creds.expiry.timestamp() - time.time()))
|
|
396
|
+
except Exception:
|
|
397
|
+
expires_in = None
|
|
398
|
+
|
|
399
|
+
return JSONResponse(
|
|
400
|
+
{
|
|
401
|
+
"access_token": connector.oauth.creds.token,
|
|
402
|
+
"expires_in": expires_in,
|
|
403
|
+
}
|
|
404
|
+
)
|
|
405
|
+
return JSONResponse({"error": "Invalid or expired credentials"}, status_code=401)
|
|
406
|
+
|
|
407
|
+
# ONEDRIVE / SHAREPOINT (MSAL or custom)
|
|
408
|
+
if real_type in ("onedrive", "sharepoint") and hasattr(connector, "oauth"):
|
|
409
|
+
# Ensure cache/credentials are loaded before trying to use them
|
|
410
|
+
try:
|
|
411
|
+
# Prefer a dedicated is_authenticated() that loads cache internally
|
|
412
|
+
if hasattr(connector.oauth, "is_authenticated"):
|
|
413
|
+
ok = await connector.oauth.is_authenticated()
|
|
414
|
+
else:
|
|
415
|
+
# Fallback: try to load credentials explicitly if available
|
|
416
|
+
ok = True
|
|
417
|
+
if hasattr(connector.oauth, "load_credentials"):
|
|
418
|
+
ok = await connector.oauth.load_credentials()
|
|
419
|
+
|
|
420
|
+
if not ok:
|
|
421
|
+
return JSONResponse({"error": "Not authenticated"}, status_code=401)
|
|
422
|
+
|
|
423
|
+
# Now safe to fetch access token
|
|
424
|
+
access_token = connector.oauth.get_access_token()
|
|
425
|
+
# MSAL result has expiry, but we’re returning a raw token; keep expires_in None for simplicity
|
|
426
|
+
return JSONResponse({"access_token": access_token, "expires_in": None})
|
|
427
|
+
except ValueError as e:
|
|
428
|
+
# Typical when acquire_token_silent fails (e.g., needs re-auth)
|
|
429
|
+
return JSONResponse({"error": f"Failed to get access token: {str(e)}"}, status_code=401)
|
|
430
|
+
except Exception as e:
|
|
431
|
+
return JSONResponse({"error": f"Authentication error: {str(e)}"}, status_code=500)
|
|
432
|
+
|
|
433
|
+
return JSONResponse({"error": "Token not available for this connector type"}, status_code=400)
|
|
434
|
+
|
|
435
|
+
except Exception as e:
|
|
436
|
+
logger.error("Error getting connector token", exc_info=True)
|
|
437
|
+
return JSONResponse({"error": str(e)}, status_code=500)
|
api/docling.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""Docling service proxy endpoints."""
|
|
2
|
+
|
|
3
|
+
import socket
|
|
4
|
+
import struct
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
from starlette.requests import Request
|
|
9
|
+
from starlette.responses import JSONResponse
|
|
10
|
+
|
|
11
|
+
from utils.container_utils import (
|
|
12
|
+
detect_container_environment,
|
|
13
|
+
get_container_host,
|
|
14
|
+
guess_host_ip_for_containers,
|
|
15
|
+
)
|
|
16
|
+
from utils.logging_config import get_logger
|
|
17
|
+
|
|
18
|
+
logger = get_logger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _get_gateway_ip_from_route() -> str | None:
|
|
22
|
+
"""Return the default gateway IP visible from the current network namespace."""
|
|
23
|
+
try:
|
|
24
|
+
with Path("/proc/net/route").open() as route_table:
|
|
25
|
+
next(route_table) # Skip header
|
|
26
|
+
for line in route_table:
|
|
27
|
+
fields = line.strip().split()
|
|
28
|
+
min_fields = 3 # interface, destination, gateway
|
|
29
|
+
if len(fields) >= min_fields and fields[1] == "00000000":
|
|
30
|
+
gateway_hex = fields[2]
|
|
31
|
+
gw_int = int(gateway_hex, 16)
|
|
32
|
+
gateway_ip = socket.inet_ntoa(struct.pack("<L", gw_int))
|
|
33
|
+
return gateway_ip
|
|
34
|
+
except (FileNotFoundError, PermissionError, IndexError, ValueError) as err:
|
|
35
|
+
logger.warning("Could not read routing table: %s", err)
|
|
36
|
+
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def determine_docling_host() -> str:
|
|
41
|
+
"""Determine the host address used for docling health checks."""
|
|
42
|
+
container_type = detect_container_environment()
|
|
43
|
+
if container_type:
|
|
44
|
+
# Try HOST_DOCKER_INTERNAL env var first
|
|
45
|
+
container_host = get_container_host()
|
|
46
|
+
if container_host:
|
|
47
|
+
logger.info("Using container-aware host '%s'", container_host)
|
|
48
|
+
return container_host
|
|
49
|
+
|
|
50
|
+
# Try special hostnames (Docker Desktop and rootless podman)
|
|
51
|
+
import socket
|
|
52
|
+
for hostname in ["host.docker.internal", "host.containers.internal"]:
|
|
53
|
+
try:
|
|
54
|
+
socket.getaddrinfo(hostname, None)
|
|
55
|
+
logger.info("Using %s for container-to-host communication", hostname)
|
|
56
|
+
return hostname
|
|
57
|
+
except socket.gaierror:
|
|
58
|
+
logger.debug("%s not available", hostname)
|
|
59
|
+
|
|
60
|
+
# Try gateway IP detection (Docker on Linux)
|
|
61
|
+
gateway_ip = _get_gateway_ip_from_route()
|
|
62
|
+
if gateway_ip:
|
|
63
|
+
logger.info("Detected host gateway IP: %s", gateway_ip)
|
|
64
|
+
return gateway_ip
|
|
65
|
+
|
|
66
|
+
# Fallback to bridge IP
|
|
67
|
+
fallback_ip = guess_host_ip_for_containers(logger=logger)
|
|
68
|
+
logger.info("Falling back to container bridge host %s", fallback_ip)
|
|
69
|
+
return fallback_ip
|
|
70
|
+
|
|
71
|
+
# Running outside a container
|
|
72
|
+
logger.info("Running outside a container; using localhost")
|
|
73
|
+
return "localhost"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# Detect the host IP once at startup
|
|
77
|
+
HOST_IP = determine_docling_host()
|
|
78
|
+
DOCLING_SERVICE_URL = f"http://{HOST_IP}:5001"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
async def health(request: Request) -> JSONResponse:
|
|
82
|
+
"""
|
|
83
|
+
Proxy health check to docling-serve.
|
|
84
|
+
This allows the frontend to check docling status via same-origin request.
|
|
85
|
+
"""
|
|
86
|
+
health_url = f"{DOCLING_SERVICE_URL}/health"
|
|
87
|
+
try:
|
|
88
|
+
async with httpx.AsyncClient() as client:
|
|
89
|
+
response = await client.get(
|
|
90
|
+
health_url,
|
|
91
|
+
timeout=2.0
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
if response.status_code == 200:
|
|
95
|
+
return JSONResponse({
|
|
96
|
+
"status": "healthy",
|
|
97
|
+
"host": HOST_IP
|
|
98
|
+
})
|
|
99
|
+
else:
|
|
100
|
+
logger.warning("Docling health check failed", url=health_url, status_code=response.status_code)
|
|
101
|
+
return JSONResponse({
|
|
102
|
+
"status": "unhealthy",
|
|
103
|
+
"message": f"Health check failed with status: {response.status_code}",
|
|
104
|
+
"host": HOST_IP
|
|
105
|
+
}, status_code=503)
|
|
106
|
+
|
|
107
|
+
except httpx.TimeoutException:
|
|
108
|
+
logger.warning("Docling health check timeout", url=health_url)
|
|
109
|
+
return JSONResponse({
|
|
110
|
+
"status": "unhealthy",
|
|
111
|
+
"message": "Connection timeout",
|
|
112
|
+
"host": HOST_IP
|
|
113
|
+
}, status_code=503)
|
|
114
|
+
except Exception as e:
|
|
115
|
+
logger.error("Docling health check failed", url=health_url, error=str(e))
|
|
116
|
+
return JSONResponse({
|
|
117
|
+
"status": "unhealthy",
|
|
118
|
+
"message": str(e),
|
|
119
|
+
"host": HOST_IP
|
|
120
|
+
}, status_code=503)
|
api/documents.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from starlette.requests import Request
|
|
2
|
+
from starlette.responses import JSONResponse
|
|
3
|
+
from utils.logging_config import get_logger
|
|
4
|
+
from config.settings import INDEX_NAME
|
|
5
|
+
|
|
6
|
+
logger = get_logger(__name__)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
async def check_filename_exists(request: Request, document_service, session_manager):
|
|
10
|
+
"""Check if a document with a specific filename already exists"""
|
|
11
|
+
filename = request.query_params.get("filename")
|
|
12
|
+
|
|
13
|
+
if not filename:
|
|
14
|
+
return JSONResponse({"error": "filename parameter is required"}, status_code=400)
|
|
15
|
+
|
|
16
|
+
user = request.state.user
|
|
17
|
+
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
|
18
|
+
|
|
19
|
+
try:
|
|
20
|
+
# Get user's OpenSearch client
|
|
21
|
+
opensearch_client = session_manager.get_user_opensearch_client(
|
|
22
|
+
user.user_id, jwt_token
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
# Search for any document with this exact filename
|
|
26
|
+
from utils.opensearch_queries import build_filename_search_body
|
|
27
|
+
|
|
28
|
+
search_body = build_filename_search_body(filename, size=1, source=["filename"])
|
|
29
|
+
|
|
30
|
+
logger.debug(f"Checking filename existence: {filename}")
|
|
31
|
+
|
|
32
|
+
response = await opensearch_client.search(
|
|
33
|
+
index=INDEX_NAME,
|
|
34
|
+
body=search_body
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
# Check if any hits were found
|
|
38
|
+
hits = response.get("hits", {}).get("hits", [])
|
|
39
|
+
exists = len(hits) > 0
|
|
40
|
+
|
|
41
|
+
logger.debug(f"Filename check result - exists: {exists}, hits: {len(hits)}")
|
|
42
|
+
|
|
43
|
+
return JSONResponse({
|
|
44
|
+
"exists": exists,
|
|
45
|
+
"filename": filename
|
|
46
|
+
}, status_code=200)
|
|
47
|
+
|
|
48
|
+
except Exception as e:
|
|
49
|
+
logger.error("Error checking filename existence", filename=filename, error=str(e))
|
|
50
|
+
error_str = str(e)
|
|
51
|
+
if "AuthenticationException" in error_str:
|
|
52
|
+
return JSONResponse({"error": "Access denied: insufficient permissions"}, status_code=403)
|
|
53
|
+
else:
|
|
54
|
+
return JSONResponse({"error": str(e)}, status_code=500)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
async def delete_documents_by_filename(request: Request, document_service, session_manager):
|
|
58
|
+
"""Delete all documents with a specific filename"""
|
|
59
|
+
data = await request.json()
|
|
60
|
+
filename = data.get("filename")
|
|
61
|
+
|
|
62
|
+
if not filename:
|
|
63
|
+
return JSONResponse({"error": "filename is required"}, status_code=400)
|
|
64
|
+
|
|
65
|
+
user = request.state.user
|
|
66
|
+
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
|
67
|
+
|
|
68
|
+
try:
|
|
69
|
+
# Get user's OpenSearch client
|
|
70
|
+
opensearch_client = session_manager.get_user_opensearch_client(
|
|
71
|
+
user.user_id, jwt_token
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
# Delete by query to remove all chunks of this document
|
|
75
|
+
from utils.opensearch_queries import build_filename_delete_body
|
|
76
|
+
|
|
77
|
+
delete_query = build_filename_delete_body(filename)
|
|
78
|
+
|
|
79
|
+
logger.debug(f"Deleting documents with filename: {filename}")
|
|
80
|
+
|
|
81
|
+
result = await opensearch_client.delete_by_query(
|
|
82
|
+
index=INDEX_NAME,
|
|
83
|
+
body=delete_query,
|
|
84
|
+
conflicts="proceed"
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
deleted_count = result.get("deleted", 0)
|
|
88
|
+
logger.info(f"Deleted {deleted_count} chunks for filename {filename}", user_id=user.user_id)
|
|
89
|
+
|
|
90
|
+
return JSONResponse({
|
|
91
|
+
"success": True,
|
|
92
|
+
"deleted_chunks": deleted_count,
|
|
93
|
+
"filename": filename,
|
|
94
|
+
"message": f"All documents with filename '{filename}' deleted successfully"
|
|
95
|
+
}, status_code=200)
|
|
96
|
+
|
|
97
|
+
except Exception as e:
|
|
98
|
+
logger.error("Error deleting documents by filename", filename=filename, error=str(e))
|
|
99
|
+
error_str = str(e)
|
|
100
|
+
if "AuthenticationException" in error_str:
|
|
101
|
+
return JSONResponse({"error": "Access denied: insufficient permissions"}, status_code=403)
|
|
102
|
+
else:
|
|
103
|
+
return JSONResponse({"error": str(e)}, status_code=500)
|