OpenRAG 0.1.25__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. agent.py +669 -0
  2. api/__init__.py +0 -0
  3. api/auth.py +84 -0
  4. api/chat.py +181 -0
  5. api/connector_router.py +78 -0
  6. api/connectors.py +437 -0
  7. api/docling.py +120 -0
  8. api/documents.py +103 -0
  9. api/flows.py +66 -0
  10. api/knowledge_filter.py +468 -0
  11. api/langflow_files.py +296 -0
  12. api/models.py +148 -0
  13. api/nudges.py +44 -0
  14. api/oidc.py +111 -0
  15. api/router.py +206 -0
  16. api/search.py +53 -0
  17. api/settings.py +762 -0
  18. api/tasks.py +35 -0
  19. api/upload.py +202 -0
  20. auth_context.py +75 -0
  21. auth_middleware.py +84 -0
  22. config/__init__.py +0 -0
  23. config/config_manager.py +214 -0
  24. config/settings.py +602 -0
  25. connectors/__init__.py +11 -0
  26. connectors/base.py +149 -0
  27. connectors/connection_manager.py +532 -0
  28. connectors/google_drive/__init__.py +4 -0
  29. connectors/google_drive/connector.py +1112 -0
  30. connectors/google_drive/oauth.py +152 -0
  31. connectors/langflow_connector_service.py +342 -0
  32. connectors/onedrive/__init__.py +4 -0
  33. connectors/onedrive/connector.py +494 -0
  34. connectors/onedrive/oauth.py +322 -0
  35. connectors/service.py +402 -0
  36. connectors/sharepoint/__init__.py +4 -0
  37. connectors/sharepoint/connector.py +567 -0
  38. connectors/sharepoint/oauth.py +329 -0
  39. main.py +1186 -0
  40. models/__init__.py +0 -0
  41. models/processors.py +769 -0
  42. models/tasks.py +46 -0
  43. openrag-0.1.25.dist-info/METADATA +166 -0
  44. openrag-0.1.25.dist-info/RECORD +112 -0
  45. openrag-0.1.25.dist-info/WHEEL +5 -0
  46. openrag-0.1.25.dist-info/entry_points.txt +2 -0
  47. openrag-0.1.25.dist-info/licenses/LICENSE +201 -0
  48. openrag-0.1.25.dist-info/top_level.txt +12 -0
  49. services/__init__.py +0 -0
  50. services/auth_service.py +401 -0
  51. services/chat_service.py +537 -0
  52. services/conversation_persistence_service.py +130 -0
  53. services/document_service.py +213 -0
  54. services/flows_service.py +906 -0
  55. services/knowledge_filter_service.py +347 -0
  56. services/langflow_file_service.py +338 -0
  57. services/langflow_history_service.py +172 -0
  58. services/langflow_mcp_service.py +275 -0
  59. services/models_service.py +372 -0
  60. services/monitor_service.py +305 -0
  61. services/search_service.py +477 -0
  62. services/session_ownership_service.py +109 -0
  63. services/task_service.py +435 -0
  64. session_manager.py +235 -0
  65. tui/__init__.py +8 -0
  66. tui/_assets/docker-compose-cpu.yml +127 -0
  67. tui/_assets/docker-compose.yml +126 -0
  68. tui/_assets/documents/docling.pdf +0 -0
  69. tui/_assets/documents/ibm_anthropic.pdf +0 -0
  70. tui/_assets/documents/openrag-documentation.pdf +0 -0
  71. tui/_assets/documents/warmup_ocr.pdf +0 -0
  72. tui/_assets/flows/components/ollama_embedding.json +142 -0
  73. tui/_assets/flows/components/ollama_llm.json +682 -0
  74. tui/_assets/flows/components/ollama_llm_text.json +683 -0
  75. tui/_assets/flows/components/watsonx_embedding.json +207 -0
  76. tui/_assets/flows/components/watsonx_llm.json +521 -0
  77. tui/_assets/flows/components/watsonx_llm_text.json +521 -0
  78. tui/_assets/flows/ingestion_flow.json +4199 -0
  79. tui/_assets/flows/openrag_agent.json +2875 -0
  80. tui/_assets/flows/openrag_nudges.json +2350 -0
  81. tui/_assets/flows/openrag_url_mcp.json +3621 -0
  82. tui/main.py +521 -0
  83. tui/managers/__init__.py +1 -0
  84. tui/managers/container_manager.py +892 -0
  85. tui/managers/docling_manager.py +488 -0
  86. tui/managers/env_manager.py +442 -0
  87. tui/screens/__init__.py +1 -0
  88. tui/screens/config.py +690 -0
  89. tui/screens/diagnostics.py +572 -0
  90. tui/screens/logs.py +356 -0
  91. tui/screens/monitor.py +858 -0
  92. tui/screens/welcome.py +446 -0
  93. tui/utils/__init__.py +1 -0
  94. tui/utils/clipboard.py +50 -0
  95. tui/utils/platform.py +209 -0
  96. tui/utils/validation.py +139 -0
  97. tui/widgets/__init__.py +3 -0
  98. tui/widgets/command_modal.py +379 -0
  99. tui/widgets/diagnostics_notification.py +40 -0
  100. tui/widgets/error_notification.py +40 -0
  101. tui/widgets/waves.py +146 -0
  102. utils/__init__.py +0 -0
  103. utils/container_utils.py +298 -0
  104. utils/document_processing.py +370 -0
  105. utils/embedding_fields.py +178 -0
  106. utils/embeddings.py +198 -0
  107. utils/file_utils.py +60 -0
  108. utils/gpu_detection.py +47 -0
  109. utils/hash_utils.py +76 -0
  110. utils/logging_config.py +117 -0
  111. utils/opensearch_queries.py +55 -0
  112. utils/process_pool.py +13 -0
api/connectors.py ADDED
@@ -0,0 +1,437 @@
1
+ from starlette.requests import Request
2
+ from starlette.responses import JSONResponse, PlainTextResponse
3
+ from utils.logging_config import get_logger
4
+
5
+ logger = get_logger(__name__)
6
+
7
+
8
+ async def list_connectors(request: Request, connector_service, session_manager):
9
+ """List available connector types with metadata"""
10
+ try:
11
+ connector_types = (
12
+ connector_service.connection_manager.get_available_connector_types()
13
+ )
14
+ return JSONResponse({"connectors": connector_types})
15
+ except Exception as e:
16
+ logger.info("Error listing connectors", error=str(e))
17
+ return JSONResponse({"connectors": []})
18
+
19
+
20
+ async def connector_sync(request: Request, connector_service, session_manager):
21
+ """Sync files from all active connections of a connector type"""
22
+ connector_type = request.path_params.get("connector_type", "google_drive")
23
+ data = await request.json()
24
+ max_files = data.get("max_files")
25
+ selected_files = data.get("selected_files")
26
+
27
+ try:
28
+ logger.debug(
29
+ "Starting connector sync",
30
+ connector_type=connector_type,
31
+ max_files=max_files,
32
+ )
33
+ user = request.state.user
34
+ jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
35
+
36
+ # Get all active connections for this connector type and user
37
+ connections = await connector_service.connection_manager.list_connections(
38
+ user_id=user.user_id, connector_type=connector_type
39
+ )
40
+
41
+ active_connections = [conn for conn in connections if conn.is_active]
42
+ if not active_connections:
43
+ return JSONResponse(
44
+ {"error": f"No active {connector_type} connections found"},
45
+ status_code=404,
46
+ )
47
+
48
+ # Find the first connection that actually works
49
+ working_connection = None
50
+ for connection in active_connections:
51
+ logger.debug(
52
+ "Testing connection authentication",
53
+ connection_id=connection.connection_id,
54
+ )
55
+ try:
56
+ # Get the connector instance and test authentication
57
+ connector = await connector_service.get_connector(connection.connection_id)
58
+ if connector and await connector.authenticate():
59
+ working_connection = connection
60
+ logger.debug(
61
+ "Found working connection",
62
+ connection_id=connection.connection_id,
63
+ )
64
+ break
65
+ else:
66
+ logger.debug(
67
+ "Connection authentication failed",
68
+ connection_id=connection.connection_id,
69
+ )
70
+ except Exception as e:
71
+ logger.debug(
72
+ "Connection validation failed",
73
+ connection_id=connection.connection_id,
74
+ error=str(e),
75
+ )
76
+ continue
77
+
78
+ if not working_connection:
79
+ return JSONResponse(
80
+ {"error": f"No working {connector_type} connections found"},
81
+ status_code=404,
82
+ )
83
+
84
+ # Use the working connection
85
+ logger.debug(
86
+ "Starting sync with working connection",
87
+ connection_id=working_connection.connection_id,
88
+ )
89
+
90
+ if selected_files:
91
+ task_id = await connector_service.sync_specific_files(
92
+ working_connection.connection_id,
93
+ user.user_id,
94
+ selected_files,
95
+ jwt_token=jwt_token,
96
+ )
97
+ else:
98
+ task_id = await connector_service.sync_connector_files(
99
+ working_connection.connection_id,
100
+ user.user_id,
101
+ max_files,
102
+ jwt_token=jwt_token,
103
+ )
104
+ task_ids = [task_id]
105
+ return JSONResponse(
106
+ {
107
+ "task_ids": task_ids,
108
+ "status": "sync_started",
109
+ "message": f"Started syncing files from {len(active_connections)} {connector_type} connection(s)",
110
+ "connections_synced": len(active_connections),
111
+ },
112
+ status_code=201,
113
+ )
114
+
115
+ except Exception as e:
116
+ logger.error("Connector sync failed", error=str(e))
117
+ return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)
118
+
119
+
120
+ async def connector_status(request: Request, connector_service, session_manager):
121
+ """Get connector status for authenticated user"""
122
+ connector_type = request.path_params.get("connector_type", "google_drive")
123
+ user = request.state.user
124
+
125
+ # Get connections for this connector type and user
126
+ connections = await connector_service.connection_manager.list_connections(
127
+ user_id=user.user_id, connector_type=connector_type
128
+ )
129
+
130
+ # Get the connector for each connection
131
+ connection_client_ids = {}
132
+ for connection in connections:
133
+ try:
134
+ connector = await connector_service._get_connector(connection.connection_id)
135
+ if connector is not None:
136
+ connection_client_ids[connection.connection_id] = connector.get_client_id()
137
+ else:
138
+ connection_client_ids[connection.connection_id] = None
139
+ except Exception as e:
140
+ logger.warning(
141
+ "Could not get connector for connection",
142
+ connection_id=connection.connection_id,
143
+ error=str(e),
144
+ )
145
+ connection.connector = None
146
+
147
+ # Check if there are any active connections
148
+ active_connections = [conn for conn in connections if conn.is_active]
149
+ has_authenticated_connection = len(active_connections) > 0
150
+
151
+ return JSONResponse(
152
+ {
153
+ "connector_type": connector_type,
154
+ "authenticated": has_authenticated_connection,
155
+ "status": "connected" if has_authenticated_connection else "not_connected",
156
+ "connections": [
157
+ {
158
+ "connection_id": conn.connection_id,
159
+ "name": conn.name,
160
+ "client_id": connection_client_ids.get(conn.connection_id),
161
+ "is_active": conn.is_active,
162
+ "created_at": conn.created_at.isoformat(),
163
+ "last_sync": conn.last_sync.isoformat() if conn.last_sync else None,
164
+ }
165
+ for conn in connections
166
+ ],
167
+ }
168
+ )
169
+
170
+
171
+ async def connector_webhook(request: Request, connector_service, session_manager):
172
+ """Handle webhook notifications from any connector type"""
173
+ connector_type = request.path_params.get("connector_type")
174
+ if connector_type is None:
175
+ connector_type = "unknown"
176
+
177
+ # Handle webhook validation (connector-specific)
178
+ temp_config = {"token_file": "temp.json"}
179
+ from connectors.connection_manager import ConnectionConfig
180
+
181
+ temp_connection = ConnectionConfig(
182
+ connection_id="temp",
183
+ connector_type=str(connector_type),
184
+ name="temp",
185
+ config=temp_config,
186
+ )
187
+ try:
188
+ temp_connector = connector_service.connection_manager._create_connector(
189
+ temp_connection
190
+ )
191
+ validation_response = temp_connector.handle_webhook_validation(
192
+ request.method, dict(request.headers), dict(request.query_params)
193
+ )
194
+ if validation_response:
195
+ return PlainTextResponse(validation_response)
196
+ except (NotImplementedError, ValueError):
197
+ # Connector type not found or validation not needed
198
+ pass
199
+
200
+ try:
201
+ # Get the raw payload and headers
202
+ payload = {}
203
+ headers = dict(request.headers)
204
+
205
+ if request.method == "POST":
206
+ content_type = headers.get("content-type", "").lower()
207
+ if "application/json" in content_type:
208
+ payload = await request.json()
209
+ else:
210
+ # Some webhooks send form data or plain text
211
+ body = await request.body()
212
+ payload = {"raw_body": body.decode("utf-8") if body else ""}
213
+ else:
214
+ # GET webhooks use query params
215
+ payload = dict(request.query_params)
216
+
217
+ # Add headers to payload for connector processing
218
+ payload["_headers"] = headers
219
+ payload["_method"] = request.method
220
+
221
+ logger.info("Webhook notification received", connector_type=connector_type)
222
+
223
+ # Extract channel/subscription ID using connector-specific method
224
+ try:
225
+ temp_connector = connector_service.connection_manager._create_connector(
226
+ temp_connection
227
+ )
228
+ channel_id = temp_connector.extract_webhook_channel_id(payload, headers)
229
+ except (NotImplementedError, ValueError):
230
+ channel_id = None
231
+
232
+ if not channel_id:
233
+ logger.warning(
234
+ "No channel ID found in webhook", connector_type=connector_type
235
+ )
236
+ return JSONResponse({"status": "ignored", "reason": "no_channel_id"})
237
+
238
+ # Find the specific connection for this webhook
239
+ connection = (
240
+ await connector_service.connection_manager.get_connection_by_webhook_id(
241
+ channel_id
242
+ )
243
+ )
244
+ if not connection or not connection.is_active:
245
+ logger.info(
246
+ "Unknown webhook channel, will auto-expire", channel_id=channel_id
247
+ )
248
+ return JSONResponse(
249
+ {"status": "ignored_unknown_channel", "channel_id": channel_id}
250
+ )
251
+
252
+ # Process webhook for the specific connection
253
+ try:
254
+ # Get the connector instance
255
+ connector = await connector_service._get_connector(connection.connection_id)
256
+ if not connector:
257
+ logger.error(
258
+ "Could not get connector for connection",
259
+ connection_id=connection.connection_id,
260
+ )
261
+ return JSONResponse(
262
+ {"status": "error", "reason": "connector_not_found"}
263
+ )
264
+
265
+ # Let the connector handle the webhook and return affected file IDs
266
+ affected_files = await connector.handle_webhook(payload)
267
+
268
+ if affected_files:
269
+ logger.info(
270
+ "Webhook connection files affected",
271
+ connection_id=connection.connection_id,
272
+ affected_count=len(affected_files),
273
+ )
274
+
275
+ # Generate JWT token for the user (needed for OpenSearch authentication)
276
+ user = session_manager.get_user(connection.user_id)
277
+ if user:
278
+ jwt_token = session_manager.create_jwt_token(user)
279
+ else:
280
+ jwt_token = None
281
+
282
+ # Trigger incremental sync for affected files
283
+ task_id = await connector_service.sync_specific_files(
284
+ connection.connection_id,
285
+ connection.user_id,
286
+ affected_files,
287
+ jwt_token=jwt_token,
288
+ )
289
+
290
+ result = {
291
+ "connection_id": connection.connection_id,
292
+ "task_id": task_id,
293
+ "affected_files": len(affected_files),
294
+ }
295
+ else:
296
+ # No specific files identified - just log the webhook
297
+ logger.info(
298
+ "Webhook general change detected, no specific files",
299
+ connection_id=connection.connection_id,
300
+ )
301
+
302
+ result = {
303
+ "connection_id": connection.connection_id,
304
+ "action": "logged_only",
305
+ "reason": "no_specific_files",
306
+ }
307
+
308
+ return JSONResponse(
309
+ {
310
+ "status": "processed",
311
+ "connector_type": connector_type,
312
+ "channel_id": channel_id,
313
+ **result,
314
+ }
315
+ )
316
+
317
+ except Exception as e:
318
+ logger.error(
319
+ "Failed to process webhook for connection",
320
+ connection_id=connection.connection_id,
321
+ error=str(e),
322
+ )
323
+ import traceback
324
+
325
+ traceback.print_exc()
326
+
327
+ return JSONResponse(
328
+ {
329
+ "status": "error",
330
+ "connector_type": connector_type,
331
+ "channel_id": channel_id,
332
+ "error": str(e),
333
+ },
334
+ status_code=500,
335
+ )
336
+
337
+ except Exception as e:
338
+ logger.error("Webhook processing failed", error=str(e))
339
+ return JSONResponse(
340
+ {"error": f"Webhook processing failed: {str(e)}"}, status_code=500
341
+ )
342
+
343
+ async def connector_token(request: Request, connector_service, session_manager):
344
+ """Get access token for connector API calls (e.g., Pickers)."""
345
+ url_connector_type = request.path_params.get("connector_type")
346
+ connection_id = request.query_params.get("connection_id")
347
+
348
+ if not connection_id:
349
+ return JSONResponse({"error": "connection_id is required"}, status_code=400)
350
+
351
+ user = request.state.user
352
+
353
+ try:
354
+ # 1) Load the connection and verify ownership
355
+ connection = await connector_service.connection_manager.get_connection(connection_id)
356
+ if not connection or connection.user_id != user.user_id:
357
+ return JSONResponse({"error": "Connection not found"}, status_code=404)
358
+
359
+ # 2) Get the ACTUAL connector instance/type for this connection_id
360
+ connector = await connector_service._get_connector(connection_id)
361
+ if not connector:
362
+ return JSONResponse(
363
+ {"error": f"Connector not available - authentication may have failed for {url_connector_type}"},
364
+ status_code=404,
365
+ )
366
+
367
+ real_type = getattr(connector, "type", None) or getattr(connection, "connector_type", None)
368
+ if real_type is None:
369
+ return JSONResponse({"error": "Unable to determine connector type"}, status_code=500)
370
+
371
+ # Optional: warn if URL path type disagrees with real type
372
+ if url_connector_type and url_connector_type != real_type:
373
+ # You can downgrade this to debug if you expect cross-routing.
374
+ return JSONResponse(
375
+ {
376
+ "error": "Connector type mismatch",
377
+ "detail": {
378
+ "requested_type": url_connector_type,
379
+ "actual_type": real_type,
380
+ "hint": "Call the token endpoint using the correct connector_type for this connection_id.",
381
+ },
382
+ },
383
+ status_code=400,
384
+ )
385
+
386
+ # 3) Branch by the actual connector type
387
+ # GOOGLE DRIVE (google-auth)
388
+ if real_type == "google_drive" and hasattr(connector, "oauth"):
389
+ await connector.oauth.load_credentials()
390
+ if connector.oauth.creds and connector.oauth.creds.valid:
391
+ expires_in = None
392
+ try:
393
+ if connector.oauth.creds.expiry:
394
+ import time
395
+ expires_in = max(0, int(connector.oauth.creds.expiry.timestamp() - time.time()))
396
+ except Exception:
397
+ expires_in = None
398
+
399
+ return JSONResponse(
400
+ {
401
+ "access_token": connector.oauth.creds.token,
402
+ "expires_in": expires_in,
403
+ }
404
+ )
405
+ return JSONResponse({"error": "Invalid or expired credentials"}, status_code=401)
406
+
407
+ # ONEDRIVE / SHAREPOINT (MSAL or custom)
408
+ if real_type in ("onedrive", "sharepoint") and hasattr(connector, "oauth"):
409
+ # Ensure cache/credentials are loaded before trying to use them
410
+ try:
411
+ # Prefer a dedicated is_authenticated() that loads cache internally
412
+ if hasattr(connector.oauth, "is_authenticated"):
413
+ ok = await connector.oauth.is_authenticated()
414
+ else:
415
+ # Fallback: try to load credentials explicitly if available
416
+ ok = True
417
+ if hasattr(connector.oauth, "load_credentials"):
418
+ ok = await connector.oauth.load_credentials()
419
+
420
+ if not ok:
421
+ return JSONResponse({"error": "Not authenticated"}, status_code=401)
422
+
423
+ # Now safe to fetch access token
424
+ access_token = connector.oauth.get_access_token()
425
+ # MSAL result has expiry, but we’re returning a raw token; keep expires_in None for simplicity
426
+ return JSONResponse({"access_token": access_token, "expires_in": None})
427
+ except ValueError as e:
428
+ # Typical when acquire_token_silent fails (e.g., needs re-auth)
429
+ return JSONResponse({"error": f"Failed to get access token: {str(e)}"}, status_code=401)
430
+ except Exception as e:
431
+ return JSONResponse({"error": f"Authentication error: {str(e)}"}, status_code=500)
432
+
433
+ return JSONResponse({"error": "Token not available for this connector type"}, status_code=400)
434
+
435
+ except Exception as e:
436
+ logger.error("Error getting connector token", exc_info=True)
437
+ return JSONResponse({"error": str(e)}, status_code=500)
api/docling.py ADDED
@@ -0,0 +1,120 @@
1
+ """Docling service proxy endpoints."""
2
+
3
+ import socket
4
+ import struct
5
+ from pathlib import Path
6
+
7
+ import httpx
8
+ from starlette.requests import Request
9
+ from starlette.responses import JSONResponse
10
+
11
+ from utils.container_utils import (
12
+ detect_container_environment,
13
+ get_container_host,
14
+ guess_host_ip_for_containers,
15
+ )
16
+ from utils.logging_config import get_logger
17
+
18
+ logger = get_logger(__name__)
19
+
20
+
21
+ def _get_gateway_ip_from_route() -> str | None:
22
+ """Return the default gateway IP visible from the current network namespace."""
23
+ try:
24
+ with Path("/proc/net/route").open() as route_table:
25
+ next(route_table) # Skip header
26
+ for line in route_table:
27
+ fields = line.strip().split()
28
+ min_fields = 3 # interface, destination, gateway
29
+ if len(fields) >= min_fields and fields[1] == "00000000":
30
+ gateway_hex = fields[2]
31
+ gw_int = int(gateway_hex, 16)
32
+ gateway_ip = socket.inet_ntoa(struct.pack("<L", gw_int))
33
+ return gateway_ip
34
+ except (FileNotFoundError, PermissionError, IndexError, ValueError) as err:
35
+ logger.warning("Could not read routing table: %s", err)
36
+
37
+ return None
38
+
39
+
40
+ def determine_docling_host() -> str:
41
+ """Determine the host address used for docling health checks."""
42
+ container_type = detect_container_environment()
43
+ if container_type:
44
+ # Try HOST_DOCKER_INTERNAL env var first
45
+ container_host = get_container_host()
46
+ if container_host:
47
+ logger.info("Using container-aware host '%s'", container_host)
48
+ return container_host
49
+
50
+ # Try special hostnames (Docker Desktop and rootless podman)
51
+ import socket
52
+ for hostname in ["host.docker.internal", "host.containers.internal"]:
53
+ try:
54
+ socket.getaddrinfo(hostname, None)
55
+ logger.info("Using %s for container-to-host communication", hostname)
56
+ return hostname
57
+ except socket.gaierror:
58
+ logger.debug("%s not available", hostname)
59
+
60
+ # Try gateway IP detection (Docker on Linux)
61
+ gateway_ip = _get_gateway_ip_from_route()
62
+ if gateway_ip:
63
+ logger.info("Detected host gateway IP: %s", gateway_ip)
64
+ return gateway_ip
65
+
66
+ # Fallback to bridge IP
67
+ fallback_ip = guess_host_ip_for_containers(logger=logger)
68
+ logger.info("Falling back to container bridge host %s", fallback_ip)
69
+ return fallback_ip
70
+
71
+ # Running outside a container
72
+ logger.info("Running outside a container; using localhost")
73
+ return "localhost"
74
+
75
+
76
+ # Detect the host IP once at startup
77
+ HOST_IP = determine_docling_host()
78
+ DOCLING_SERVICE_URL = f"http://{HOST_IP}:5001"
79
+
80
+
81
+ async def health(request: Request) -> JSONResponse:
82
+ """
83
+ Proxy health check to docling-serve.
84
+ This allows the frontend to check docling status via same-origin request.
85
+ """
86
+ health_url = f"{DOCLING_SERVICE_URL}/health"
87
+ try:
88
+ async with httpx.AsyncClient() as client:
89
+ response = await client.get(
90
+ health_url,
91
+ timeout=2.0
92
+ )
93
+
94
+ if response.status_code == 200:
95
+ return JSONResponse({
96
+ "status": "healthy",
97
+ "host": HOST_IP
98
+ })
99
+ else:
100
+ logger.warning("Docling health check failed", url=health_url, status_code=response.status_code)
101
+ return JSONResponse({
102
+ "status": "unhealthy",
103
+ "message": f"Health check failed with status: {response.status_code}",
104
+ "host": HOST_IP
105
+ }, status_code=503)
106
+
107
+ except httpx.TimeoutException:
108
+ logger.warning("Docling health check timeout", url=health_url)
109
+ return JSONResponse({
110
+ "status": "unhealthy",
111
+ "message": "Connection timeout",
112
+ "host": HOST_IP
113
+ }, status_code=503)
114
+ except Exception as e:
115
+ logger.error("Docling health check failed", url=health_url, error=str(e))
116
+ return JSONResponse({
117
+ "status": "unhealthy",
118
+ "message": str(e),
119
+ "host": HOST_IP
120
+ }, status_code=503)
api/documents.py ADDED
@@ -0,0 +1,103 @@
1
+ from starlette.requests import Request
2
+ from starlette.responses import JSONResponse
3
+ from utils.logging_config import get_logger
4
+ from config.settings import INDEX_NAME
5
+
6
+ logger = get_logger(__name__)
7
+
8
+
9
+ async def check_filename_exists(request: Request, document_service, session_manager):
10
+ """Check if a document with a specific filename already exists"""
11
+ filename = request.query_params.get("filename")
12
+
13
+ if not filename:
14
+ return JSONResponse({"error": "filename parameter is required"}, status_code=400)
15
+
16
+ user = request.state.user
17
+ jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
18
+
19
+ try:
20
+ # Get user's OpenSearch client
21
+ opensearch_client = session_manager.get_user_opensearch_client(
22
+ user.user_id, jwt_token
23
+ )
24
+
25
+ # Search for any document with this exact filename
26
+ from utils.opensearch_queries import build_filename_search_body
27
+
28
+ search_body = build_filename_search_body(filename, size=1, source=["filename"])
29
+
30
+ logger.debug(f"Checking filename existence: {filename}")
31
+
32
+ response = await opensearch_client.search(
33
+ index=INDEX_NAME,
34
+ body=search_body
35
+ )
36
+
37
+ # Check if any hits were found
38
+ hits = response.get("hits", {}).get("hits", [])
39
+ exists = len(hits) > 0
40
+
41
+ logger.debug(f"Filename check result - exists: {exists}, hits: {len(hits)}")
42
+
43
+ return JSONResponse({
44
+ "exists": exists,
45
+ "filename": filename
46
+ }, status_code=200)
47
+
48
+ except Exception as e:
49
+ logger.error("Error checking filename existence", filename=filename, error=str(e))
50
+ error_str = str(e)
51
+ if "AuthenticationException" in error_str:
52
+ return JSONResponse({"error": "Access denied: insufficient permissions"}, status_code=403)
53
+ else:
54
+ return JSONResponse({"error": str(e)}, status_code=500)
55
+
56
+
57
+ async def delete_documents_by_filename(request: Request, document_service, session_manager):
58
+ """Delete all documents with a specific filename"""
59
+ data = await request.json()
60
+ filename = data.get("filename")
61
+
62
+ if not filename:
63
+ return JSONResponse({"error": "filename is required"}, status_code=400)
64
+
65
+ user = request.state.user
66
+ jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
67
+
68
+ try:
69
+ # Get user's OpenSearch client
70
+ opensearch_client = session_manager.get_user_opensearch_client(
71
+ user.user_id, jwt_token
72
+ )
73
+
74
+ # Delete by query to remove all chunks of this document
75
+ from utils.opensearch_queries import build_filename_delete_body
76
+
77
+ delete_query = build_filename_delete_body(filename)
78
+
79
+ logger.debug(f"Deleting documents with filename: {filename}")
80
+
81
+ result = await opensearch_client.delete_by_query(
82
+ index=INDEX_NAME,
83
+ body=delete_query,
84
+ conflicts="proceed"
85
+ )
86
+
87
+ deleted_count = result.get("deleted", 0)
88
+ logger.info(f"Deleted {deleted_count} chunks for filename {filename}", user_id=user.user_id)
89
+
90
+ return JSONResponse({
91
+ "success": True,
92
+ "deleted_chunks": deleted_count,
93
+ "filename": filename,
94
+ "message": f"All documents with filename '{filename}' deleted successfully"
95
+ }, status_code=200)
96
+
97
+ except Exception as e:
98
+ logger.error("Error deleting documents by filename", filename=filename, error=str(e))
99
+ error_str = str(e)
100
+ if "AuthenticationException" in error_str:
101
+ return JSONResponse({"error": "Access denied: insufficient permissions"}, status_code=403)
102
+ else:
103
+ return JSONResponse({"error": str(e)}, status_code=500)