OpenRAG 0.1.25__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. agent.py +669 -0
  2. api/__init__.py +0 -0
  3. api/auth.py +84 -0
  4. api/chat.py +181 -0
  5. api/connector_router.py +78 -0
  6. api/connectors.py +437 -0
  7. api/docling.py +120 -0
  8. api/documents.py +103 -0
  9. api/flows.py +66 -0
  10. api/knowledge_filter.py +468 -0
  11. api/langflow_files.py +296 -0
  12. api/models.py +148 -0
  13. api/nudges.py +44 -0
  14. api/oidc.py +111 -0
  15. api/router.py +206 -0
  16. api/search.py +53 -0
  17. api/settings.py +762 -0
  18. api/tasks.py +35 -0
  19. api/upload.py +202 -0
  20. auth_context.py +75 -0
  21. auth_middleware.py +84 -0
  22. config/__init__.py +0 -0
  23. config/config_manager.py +214 -0
  24. config/settings.py +602 -0
  25. connectors/__init__.py +11 -0
  26. connectors/base.py +149 -0
  27. connectors/connection_manager.py +532 -0
  28. connectors/google_drive/__init__.py +4 -0
  29. connectors/google_drive/connector.py +1112 -0
  30. connectors/google_drive/oauth.py +152 -0
  31. connectors/langflow_connector_service.py +342 -0
  32. connectors/onedrive/__init__.py +4 -0
  33. connectors/onedrive/connector.py +494 -0
  34. connectors/onedrive/oauth.py +322 -0
  35. connectors/service.py +402 -0
  36. connectors/sharepoint/__init__.py +4 -0
  37. connectors/sharepoint/connector.py +567 -0
  38. connectors/sharepoint/oauth.py +329 -0
  39. main.py +1186 -0
  40. models/__init__.py +0 -0
  41. models/processors.py +769 -0
  42. models/tasks.py +46 -0
  43. openrag-0.1.25.dist-info/METADATA +166 -0
  44. openrag-0.1.25.dist-info/RECORD +112 -0
  45. openrag-0.1.25.dist-info/WHEEL +5 -0
  46. openrag-0.1.25.dist-info/entry_points.txt +2 -0
  47. openrag-0.1.25.dist-info/licenses/LICENSE +201 -0
  48. openrag-0.1.25.dist-info/top_level.txt +12 -0
  49. services/__init__.py +0 -0
  50. services/auth_service.py +401 -0
  51. services/chat_service.py +537 -0
  52. services/conversation_persistence_service.py +130 -0
  53. services/document_service.py +213 -0
  54. services/flows_service.py +906 -0
  55. services/knowledge_filter_service.py +347 -0
  56. services/langflow_file_service.py +338 -0
  57. services/langflow_history_service.py +172 -0
  58. services/langflow_mcp_service.py +275 -0
  59. services/models_service.py +372 -0
  60. services/monitor_service.py +305 -0
  61. services/search_service.py +477 -0
  62. services/session_ownership_service.py +109 -0
  63. services/task_service.py +435 -0
  64. session_manager.py +235 -0
  65. tui/__init__.py +8 -0
  66. tui/_assets/docker-compose-cpu.yml +127 -0
  67. tui/_assets/docker-compose.yml +126 -0
  68. tui/_assets/documents/docling.pdf +0 -0
  69. tui/_assets/documents/ibm_anthropic.pdf +0 -0
  70. tui/_assets/documents/openrag-documentation.pdf +0 -0
  71. tui/_assets/documents/warmup_ocr.pdf +0 -0
  72. tui/_assets/flows/components/ollama_embedding.json +142 -0
  73. tui/_assets/flows/components/ollama_llm.json +682 -0
  74. tui/_assets/flows/components/ollama_llm_text.json +683 -0
  75. tui/_assets/flows/components/watsonx_embedding.json +207 -0
  76. tui/_assets/flows/components/watsonx_llm.json +521 -0
  77. tui/_assets/flows/components/watsonx_llm_text.json +521 -0
  78. tui/_assets/flows/ingestion_flow.json +4199 -0
  79. tui/_assets/flows/openrag_agent.json +2875 -0
  80. tui/_assets/flows/openrag_nudges.json +2350 -0
  81. tui/_assets/flows/openrag_url_mcp.json +3621 -0
  82. tui/main.py +521 -0
  83. tui/managers/__init__.py +1 -0
  84. tui/managers/container_manager.py +892 -0
  85. tui/managers/docling_manager.py +488 -0
  86. tui/managers/env_manager.py +442 -0
  87. tui/screens/__init__.py +1 -0
  88. tui/screens/config.py +690 -0
  89. tui/screens/diagnostics.py +572 -0
  90. tui/screens/logs.py +356 -0
  91. tui/screens/monitor.py +858 -0
  92. tui/screens/welcome.py +446 -0
  93. tui/utils/__init__.py +1 -0
  94. tui/utils/clipboard.py +50 -0
  95. tui/utils/platform.py +209 -0
  96. tui/utils/validation.py +139 -0
  97. tui/widgets/__init__.py +3 -0
  98. tui/widgets/command_modal.py +379 -0
  99. tui/widgets/diagnostics_notification.py +40 -0
  100. tui/widgets/error_notification.py +40 -0
  101. tui/widgets/waves.py +146 -0
  102. utils/__init__.py +0 -0
  103. utils/container_utils.py +298 -0
  104. utils/document_processing.py +370 -0
  105. utils/embedding_fields.py +178 -0
  106. utils/embeddings.py +198 -0
  107. utils/file_utils.py +60 -0
  108. utils/gpu_detection.py +47 -0
  109. utils/hash_utils.py +76 -0
  110. utils/logging_config.py +117 -0
  111. utils/opensearch_queries.py +55 -0
  112. utils/process_pool.py +13 -0
api/auth.py ADDED
@@ -0,0 +1,84 @@
1
+ from starlette.requests import Request
2
+ from starlette.responses import JSONResponse
3
+
4
+
5
+ async def auth_init(request: Request, auth_service, session_manager):
6
+ """Initialize OAuth flow for authentication or data source connection"""
7
+ try:
8
+ data = await request.json()
9
+ connector_type = data.get("connector_type")
10
+ purpose = data.get("purpose", "data_source")
11
+ connection_name = data.get("name", f"{connector_type}_{purpose}")
12
+ redirect_uri = data.get("redirect_uri")
13
+
14
+ user = getattr(request.state, "user", None)
15
+ user_id = user.user_id if user else None
16
+
17
+ result = await auth_service.init_oauth(
18
+ connector_type, purpose, connection_name, redirect_uri, user_id
19
+ )
20
+ return JSONResponse(result)
21
+
22
+ except Exception as e:
23
+ import traceback
24
+
25
+ traceback.print_exc()
26
+ return JSONResponse(
27
+ {"error": f"Failed to initialize OAuth: {str(e)}"}, status_code=500
28
+ )
29
+
30
+
31
+ async def auth_callback(request: Request, auth_service, session_manager):
32
+ """Handle OAuth callback - exchange authorization code for tokens"""
33
+ try:
34
+ data = await request.json()
35
+ connection_id = data.get("connection_id")
36
+ authorization_code = data.get("authorization_code")
37
+ state = data.get("state")
38
+
39
+ result = await auth_service.handle_oauth_callback(
40
+ connection_id, authorization_code, state, request
41
+ )
42
+
43
+ # If this is app auth, set JWT cookie
44
+ if result.get("purpose") == "app_auth" and result.get("jwt_token"):
45
+ response = JSONResponse(
46
+ {k: v for k, v in result.items() if k != "jwt_token"}
47
+ )
48
+ response.set_cookie(
49
+ key="auth_token",
50
+ value=result["jwt_token"],
51
+ httponly=True,
52
+ secure=False,
53
+ samesite="lax",
54
+ max_age=7 * 24 * 60 * 60, # 7 days
55
+ )
56
+ return response
57
+ else:
58
+ return JSONResponse(result)
59
+
60
+ except Exception as e:
61
+ import traceback
62
+
63
+ traceback.print_exc()
64
+ return JSONResponse({"error": f"Callback failed: {str(e)}"}, status_code=500)
65
+
66
+
67
+ async def auth_me(request: Request, auth_service, session_manager):
68
+ """Get current user information"""
69
+ result = await auth_service.get_user_info(request)
70
+ return JSONResponse(result)
71
+
72
+
73
+ async def auth_logout(request: Request, auth_service, session_manager):
74
+ """Logout user by clearing auth cookie"""
75
+ response = JSONResponse(
76
+ {"status": "logged_out", "message": "Successfully logged out"}
77
+ )
78
+
79
+ # Clear the auth cookie
80
+ response.delete_cookie(
81
+ key="auth_token", httponly=True, secure=False, samesite="lax"
82
+ )
83
+
84
+ return response
api/chat.py ADDED
@@ -0,0 +1,181 @@
1
+ from starlette.requests import Request
2
+ from starlette.responses import JSONResponse, StreamingResponse
3
+ from utils.logging_config import get_logger
4
+
5
+ logger = get_logger(__name__)
6
+
7
+
8
+ async def chat_endpoint(request: Request, chat_service, session_manager):
9
+ """Handle chat requests"""
10
+ data = await request.json()
11
+ prompt = data.get("prompt", "")
12
+ previous_response_id = data.get("previous_response_id")
13
+ stream = data.get("stream", False)
14
+ filters = data.get("filters")
15
+ limit = data.get("limit", 10)
16
+ score_threshold = data.get("scoreThreshold", 0)
17
+
18
+ user = request.state.user
19
+ user_id = user.user_id
20
+
21
+ jwt_token = session_manager.get_effective_jwt_token(user_id, request.state.jwt_token)
22
+
23
+ if not prompt:
24
+ return JSONResponse({"error": "Prompt is required"}, status_code=400)
25
+
26
+ # Set context variables for search tool (similar to search endpoint)
27
+ if filters:
28
+ from auth_context import set_search_filters
29
+
30
+ set_search_filters(filters)
31
+
32
+ from auth_context import set_search_limit, set_score_threshold
33
+
34
+ set_search_limit(limit)
35
+ set_score_threshold(score_threshold)
36
+
37
+ if stream:
38
+ return StreamingResponse(
39
+ await chat_service.chat(
40
+ prompt,
41
+ user_id,
42
+ jwt_token,
43
+ previous_response_id=previous_response_id,
44
+ stream=True,
45
+ ),
46
+ media_type="text/event-stream",
47
+ headers={
48
+ "Cache-Control": "no-cache",
49
+ "Connection": "keep-alive",
50
+ "Access-Control-Allow-Origin": "*",
51
+ "Access-Control-Allow-Headers": "Cache-Control",
52
+ },
53
+ )
54
+ else:
55
+ result = await chat_service.chat(
56
+ prompt,
57
+ user_id,
58
+ jwt_token,
59
+ previous_response_id=previous_response_id,
60
+ stream=False,
61
+ )
62
+ return JSONResponse(result)
63
+
64
+
65
+ async def langflow_endpoint(request: Request, chat_service, session_manager):
66
+ """Handle Langflow chat requests"""
67
+ data = await request.json()
68
+ prompt = data.get("prompt", "")
69
+ previous_response_id = data.get("previous_response_id")
70
+ stream = data.get("stream", False)
71
+ filters = data.get("filters")
72
+ limit = data.get("limit", 10)
73
+ score_threshold = data.get("scoreThreshold", 0)
74
+
75
+ user = request.state.user
76
+ user_id = user.user_id
77
+
78
+ jwt_token = session_manager.get_effective_jwt_token(user_id, request.state.jwt_token)
79
+
80
+ if not prompt:
81
+ return JSONResponse({"error": "Prompt is required"}, status_code=400)
82
+
83
+ # Set context variables for search tool (similar to chat endpoint)
84
+ if filters:
85
+ from auth_context import set_search_filters
86
+
87
+ set_search_filters(filters)
88
+
89
+ from auth_context import set_search_limit, set_score_threshold
90
+
91
+ set_search_limit(limit)
92
+ set_score_threshold(score_threshold)
93
+
94
+ try:
95
+ if stream:
96
+ return StreamingResponse(
97
+ await chat_service.langflow_chat(
98
+ prompt,
99
+ user_id,
100
+ jwt_token,
101
+ previous_response_id=previous_response_id,
102
+ stream=True,
103
+ ),
104
+ media_type="text/event-stream",
105
+ headers={
106
+ "Cache-Control": "no-cache",
107
+ "Connection": "keep-alive",
108
+ "Access-Control-Allow-Origin": "*",
109
+ "Access-Control-Allow-Headers": "Cache-Control",
110
+ },
111
+ )
112
+ else:
113
+ result = await chat_service.langflow_chat(
114
+ prompt,
115
+ user_id,
116
+ jwt_token,
117
+ previous_response_id=previous_response_id,
118
+ stream=False,
119
+ )
120
+ return JSONResponse(result)
121
+
122
+ except Exception as e:
123
+ import traceback
124
+
125
+ traceback.print_exc()
126
+ logger.error("Langflow request failed", error=str(e))
127
+ return JSONResponse(
128
+ {"error": f"Langflow request failed: {str(e)}"}, status_code=500
129
+ )
130
+
131
+
132
+ async def chat_history_endpoint(request: Request, chat_service, session_manager):
133
+ """Get chat history for a user"""
134
+ user = request.state.user
135
+ user_id = user.user_id
136
+
137
+ try:
138
+ history = await chat_service.get_chat_history(user_id)
139
+ return JSONResponse(history)
140
+ except Exception as e:
141
+ return JSONResponse(
142
+ {"error": f"Failed to get chat history: {str(e)}"}, status_code=500
143
+ )
144
+
145
+
146
+ async def langflow_history_endpoint(request: Request, chat_service, session_manager):
147
+ """Get langflow chat history for a user"""
148
+ user = request.state.user
149
+ user_id = user.user_id
150
+
151
+ try:
152
+ history = await chat_service.get_langflow_history(user_id)
153
+ return JSONResponse(history)
154
+ except Exception as e:
155
+ return JSONResponse(
156
+ {"error": f"Failed to get langflow history: {str(e)}"}, status_code=500
157
+ )
158
+
159
+
160
+ async def delete_session_endpoint(request: Request, chat_service, session_manager):
161
+ """Delete a chat session"""
162
+ user = request.state.user
163
+ user_id = user.user_id
164
+ session_id = request.path_params["session_id"]
165
+
166
+ try:
167
+ # Delete from both local storage and Langflow
168
+ result = await chat_service.delete_session(user_id, session_id)
169
+
170
+ if result.get("success"):
171
+ return JSONResponse({"message": "Session deleted successfully"})
172
+ else:
173
+ return JSONResponse(
174
+ {"error": result.get("error", "Failed to delete session")},
175
+ status_code=500
176
+ )
177
+ except Exception as e:
178
+ logger.error(f"Error deleting session: {e}")
179
+ return JSONResponse(
180
+ {"error": f"Failed to delete session: {str(e)}"}, status_code=500
181
+ )
@@ -0,0 +1,78 @@
1
+ """Connector router that automatically routes based on configuration settings."""
2
+
3
+ from starlette.requests import Request
4
+
5
+ from config.settings import (
6
+ DISABLE_INGEST_WITH_LANGFLOW,
7
+ clients,
8
+ INDEX_NAME,
9
+ INDEX_BODY,
10
+ )
11
+ from utils.logging_config import get_logger
12
+
13
+ logger = get_logger(__name__)
14
+
15
+
16
+ class ConnectorRouter:
17
+ """
18
+ Router that automatically chooses between LangflowConnectorService and ConnectorService
19
+ based on the DISABLE_INGEST_WITH_LANGFLOW configuration.
20
+
21
+ - If DISABLE_INGEST_WITH_LANGFLOW is False (default): uses LangflowConnectorService
22
+ - If DISABLE_INGEST_WITH_LANGFLOW is True: uses traditional ConnectorService
23
+ """
24
+
25
+ def __init__(self, langflow_connector_service, openrag_connector_service):
26
+ self.langflow_connector_service = langflow_connector_service
27
+ self.openrag_connector_service = openrag_connector_service
28
+ logger.debug(
29
+ "ConnectorRouter initialized",
30
+ disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,
31
+ )
32
+
33
+ def get_active_service(self):
34
+ """Get the currently active connector service based on configuration."""
35
+ if DISABLE_INGEST_WITH_LANGFLOW:
36
+ logger.debug("Using traditional OpenRAG connector service")
37
+ return self.openrag_connector_service
38
+ else:
39
+ logger.debug("Using Langflow connector service")
40
+ return self.langflow_connector_service
41
+
42
+ # Proxy all connector service methods to the active service
43
+
44
+ async def initialize(self):
45
+ """Initialize the active connector service."""
46
+ # Initialize OpenSearch index if using traditional OpenRAG connector service
47
+
48
+ return await self.get_active_service().initialize()
49
+
50
+ @property
51
+ def connection_manager(self):
52
+ """Get the connection manager from the active service."""
53
+ return self.get_active_service().connection_manager
54
+
55
+ async def get_connector(self, connection_id: str):
56
+ """Get a connector instance from the active service."""
57
+ return await self.get_active_service().get_connector(connection_id)
58
+
59
+ async def sync_specific_files(
60
+ self, connection_id: str, user_id: str, file_list: list, jwt_token: str = None
61
+ ):
62
+ """Sync specific files using the active service."""
63
+ return await self.get_active_service().sync_specific_files(
64
+ connection_id, user_id, file_list, jwt_token
65
+ )
66
+
67
+ def __getattr__(self, name):
68
+ """
69
+ Proxy any other method calls to the active service.
70
+ This ensures compatibility with any methods we might have missed.
71
+ """
72
+ active_service = self.get_active_service()
73
+ if hasattr(active_service, name):
74
+ return getattr(active_service, name)
75
+ else:
76
+ raise AttributeError(
77
+ f"'{type(active_service).__name__}' object has no attribute '{name}'"
78
+ )