OpenRAG 0.5.1.dev21__py3-none-any.whl → 0.6.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. agent.py +148 -112
  2. api/acl.py +44 -0
  3. api/auth.py +36 -28
  4. api/chat.py +48 -10
  5. api/config.py +46 -0
  6. api/connectors.py +1081 -144
  7. api/docling.py +46 -104
  8. api/documents.py +139 -44
  9. api/files.py +95 -0
  10. api/flows.py +2 -2
  11. api/health.py +80 -0
  12. api/ingest_preview.py +67 -0
  13. api/keys.py +4 -4
  14. api/knowledge_filter.py +86 -79
  15. api/langflow_files.py +5 -10
  16. api/langflow_ingest.py +119 -0
  17. api/models.py +24 -23
  18. api/nudges.py +15 -1
  19. api/oidc.py +25 -17
  20. api/provider_health.py +92 -29
  21. api/router.py +150 -25
  22. api/schemas/tasks.py +22 -0
  23. api/search.py +7 -2
  24. api/settings/__init__.py +88 -0
  25. api/{settings.py → settings/endpoints.py} +540 -1005
  26. api/settings/helpers.py +230 -0
  27. api/settings/langflow_sync.py +323 -0
  28. api/settings/models.py +242 -0
  29. api/tasks.py +62 -6
  30. api/upload.py +51 -41
  31. api/users.py +166 -0
  32. api/v1/_filter_resolution.py +100 -0
  33. api/v1/chat.py +126 -44
  34. api/v1/documents.py +154 -21
  35. api/v1/knowledge_filters.py +8 -6
  36. api/v1/models.py +26 -10
  37. api/v1/search.py +43 -14
  38. api/v1/settings.py +34 -18
  39. app/__init__.py +0 -0
  40. app/container.py +240 -0
  41. app/factory.py +55 -0
  42. app/lifespan.py +435 -0
  43. app/middleware.py +69 -0
  44. app/router_app.py +158 -0
  45. app/routes/__init__.py +23 -0
  46. app/routes/internal.py +456 -0
  47. app/routes/mcp.py +29 -0
  48. app/routes/public_v1.py +132 -0
  49. auth/ibm_auth.py +48 -2
  50. auth/jwt_roles.py +102 -0
  51. auth/permission_dependencies.py +143 -0
  52. auth/request_identity.py +698 -0
  53. auth/user_identity_cache.py +138 -0
  54. bootstrap.py +12 -0
  55. config/config_manager.py +185 -42
  56. config/model_constants.py +11 -14
  57. config/settings.py +908 -126
  58. config/storage_mode.py +61 -0
  59. config/utils.py +202 -0
  60. connectors/__init__.py +2 -4
  61. connectors/aws_s3/connector.py +47 -14
  62. connectors/aws_s3/models.py +7 -7
  63. connectors/aws_s3/support.py +13 -6
  64. connectors/base.py +141 -25
  65. connectors/connection_manager.py +334 -194
  66. connectors/google_drive/connector.py +201 -30
  67. connectors/google_drive/oauth.py +92 -28
  68. connectors/google_drive_acl.py +408 -0
  69. connectors/microsoft_graph_acl.py +395 -0
  70. connectors/microsoft_oauth_utils.py +54 -0
  71. connectors/onedrive/connector.py +325 -53
  72. connectors/onedrive/oauth.py +17 -4
  73. connectors/registry.py +73 -0
  74. connectors/service.py +160 -13
  75. connectors/sharepoint/connector.py +302 -34
  76. connectors/sharepoint/oauth.py +148 -57
  77. db/__init__.py +21 -0
  78. db/base.py +19 -0
  79. db/engine.py +82 -0
  80. db/migrations_runtime.py +351 -0
  81. db/models/__init__.py +33 -0
  82. db/models/api_key.py +27 -0
  83. db/models/audit_log.py +23 -0
  84. db/models/conversation.py +30 -0
  85. db/models/migration_status.py +13 -0
  86. db/models/permission.py +13 -0
  87. db/models/role.py +14 -0
  88. db/models/role_permission.py +16 -0
  89. db/models/session_ownership.py +23 -0
  90. db/models/user.py +38 -0
  91. db/models/user_preferences.py +51 -0
  92. db/models/user_role.py +20 -0
  93. db/models/workspace_config.py +30 -0
  94. db/repositories/__init__.py +27 -0
  95. db/repositories/_helpers.py +9 -0
  96. db/repositories/api_key_repo.py +43 -0
  97. db/repositories/audit_repo.py +36 -0
  98. db/repositories/conversation_repo.py +104 -0
  99. db/repositories/permission_repo.py +23 -0
  100. db/repositories/preferences_repo.py +44 -0
  101. db/repositories/role_repo.py +68 -0
  102. db/repositories/session_ownership_repo.py +85 -0
  103. db/repositories/user_repo.py +74 -0
  104. db/repositories/workspace_config_repo.py +60 -0
  105. db/seed.py +251 -0
  106. db/types.py +61 -0
  107. dependencies.py +216 -330
  108. main.py +82 -2198
  109. mcp_http/server.py +62 -27
  110. models/processors.py +744 -209
  111. models/tasks.py +37 -5
  112. models/url.py +8 -11
  113. {openrag-0.5.1.dev21.dist-info → openrag-0.6.0.dev1.dist-info}/METADATA +32 -19
  114. openrag-0.6.0.dev1.dist-info/RECORD +247 -0
  115. {openrag-0.5.1.dev21.dist-info → openrag-0.6.0.dev1.dist-info}/top_level.txt +2 -0
  116. services/api_key_service.py +65 -35
  117. services/auth_service.py +153 -157
  118. services/chat_service.py +301 -92
  119. services/connector_access_service.py +108 -0
  120. services/connector_oauth_config_service.py +197 -0
  121. services/conversation_persistence_service.py +295 -106
  122. services/default_docs_service.py +696 -0
  123. services/default_role_sync.py +307 -0
  124. services/dev_role_toggle.py +48 -0
  125. services/dls_principal_service.py +449 -0
  126. services/docling_polling_service.py +224 -0
  127. services/docling_service.py +474 -0
  128. services/document_index_writer.py +327 -0
  129. services/document_service.py +36 -25
  130. services/file_service.py +245 -0
  131. services/flows_service.py +228 -114
  132. services/group_acl_service.py +117 -0
  133. services/ingest_preview_service.py +186 -0
  134. services/knowledge_filter_service.py +114 -65
  135. services/langflow_file_service.py +697 -245
  136. services/langflow_ingest_token_service.py +203 -0
  137. services/monitor_service.py +65 -53
  138. services/onboarding_reset.py +151 -0
  139. services/rbac_service.py +141 -0
  140. services/search_service.py +117 -113
  141. services/session_ownership_service.py +219 -85
  142. services/startup_orchestrator.py +221 -0
  143. services/task_service.py +790 -53
  144. services/user_service.py +324 -0
  145. services/workspace_config_service.py +411 -0
  146. session_manager.py +134 -62
  147. tui/_assets/docker-compose.yml +77 -23
  148. tui/_assets/flows/ingestion_flow.json +1589 -3982
  149. tui/_assets/flows/openrag_agent.json +194 -477
  150. tui/_assets/flows/openrag_nudges.json +147 -375
  151. tui/_assets/flows/openrag_url_mcp.json +619 -1880
  152. tui/config_fields.py +370 -232
  153. tui/managers/container_manager.py +248 -210
  154. tui/managers/docling_manager.py +20 -9
  155. tui/managers/env_manager.py +149 -56
  156. tui/utils/validation.py +21 -5
  157. utils/acl_utils.py +110 -112
  158. utils/container_utils.py +63 -0
  159. utils/document_processing.py +67 -29
  160. utils/embeddings.py +56 -25
  161. utils/env_utils.py +21 -4
  162. utils/file_utils.py +17 -12
  163. utils/group_acl.py +161 -0
  164. utils/ingest_preview_flag.py +21 -0
  165. utils/jwt_keygen.py +55 -0
  166. utils/jwt_verification.py +443 -0
  167. utils/langflow_headers.py +51 -11
  168. utils/logging_config.py +98 -78
  169. utils/opensearch_delete.py +0 -30
  170. utils/opensearch_init.py +434 -0
  171. utils/opensearch_queries.py +66 -18
  172. utils/opensearch_utils.py +194 -58
  173. utils/provider_health_cache.py +115 -0
  174. utils/run_mode_utils.py +1 -0
  175. utils/telemetry/message_id.py +30 -28
  176. utils/url_content_fetcher.py +72 -0
  177. utils/version_utils.py +8 -6
  178. connectors/ibm_cos/__init__.py +0 -17
  179. connectors/ibm_cos/api.py +0 -202
  180. connectors/ibm_cos/auth.py +0 -183
  181. connectors/ibm_cos/connector.py +0 -370
  182. connectors/ibm_cos/models.py +0 -20
  183. connectors/ibm_cos/support.py +0 -68
  184. openrag-0.5.1.dev21.dist-info/RECORD +0 -165
  185. {openrag-0.5.1.dev21.dist-info → openrag-0.6.0.dev1.dist-info}/WHEEL +0 -0
  186. {openrag-0.5.1.dev21.dist-info → openrag-0.6.0.dev1.dist-info}/entry_points.txt +0 -0
  187. {openrag-0.5.1.dev21.dist-info → openrag-0.6.0.dev1.dist-info}/licenses/LICENSE +0 -0
agent.py CHANGED
@@ -1,19 +1,17 @@
1
- from http.client import HTTPException
1
+ from typing import Any
2
2
 
3
+ from services.conversation_persistence_service import conversation_persistence
3
4
  from utils.logging_config import get_logger
4
5
 
5
6
  logger = get_logger(__name__)
6
7
 
7
- # Import persistent storage
8
- from services.conversation_persistence_service import conversation_persistence
9
-
10
8
  # In-memory storage for active conversation threads (preserves function calls)
11
- active_conversations = {}
9
+ active_conversations: dict[str, dict[str, Any]] = {}
12
10
 
13
11
 
14
- def get_user_conversations(user_id: str):
15
- """Get conversation metadata for a user from persistent storage"""
16
- return conversation_persistence.get_user_conversations(user_id)
12
+ async def get_user_conversations(user_id: str):
13
+ """Get conversation metadata for a user from persistent storage."""
14
+ return await conversation_persistence.get_user_conversations(user_id)
17
15
 
18
16
 
19
17
  def get_conversation_thread(user_id: str, previous_response_id: str = None):
@@ -36,7 +34,7 @@ def get_conversation_thread(user_id: str, previous_response_id: str = None):
36
34
  "messages": [
37
35
  {
38
36
  "role": "system",
39
- "content": "You are the OpenRAG Agent. You answer questions using retrieval, reasoning, and tool use.\nYou have access to several tools. Your job is to determine **which tool to use and when**.\n### Available Tools\n- OpenSearch Retrieval Tool:\n Use this to search the indexed knowledge base. Use when the user asks about product details, internal concepts, processes, architecture, documentation, roadmaps, or anything that may be stored in the index.\n- Conversation History:\n Use this to maintain continuity when the user is referring to previous turns. \n Do not treat history as a factual source.\n- Conversation File Context:\n Use this when the user asks about a document they uploaded or refers directly to its contents.\n **IMPORTANT**: If you receive confirmation that a file was uploaded (e.g., \"Confirm that you received this file\"), the file content is already available in the conversation context. Do NOT attempt to ingest it as a URL.\n Simply acknowledge the file and answer questions about it directly from the context.\n- URL Ingestion Tool:\n Use this **only** when the user explicitly asks you to read, summarize, or analyze the content of a web URL (http:// or https://).\n **Do NOT use this tool for filenames** (e.g., README.md, document.pdf, data.txt). These are file uploads, not URLs.\n Only use this tool for actual web addresses that the user explicitly provides.\n If unclear → ask a clarifying question.\n- Calculator / Expression Evaluation Tool:\n Use this when the user asks to compare numbers, compute estimates, calculate totals, analyze pricing, or answer any question requiring mathematics or quantitative reasoning.\n If the answer requires arithmetic, call the calculator tool rather than calculating internally.\n### Retrieval Decision Rules\nUse OpenSearch **whenever**:\n1. The question may be answered from internal or indexed data.\n2. The user references team names, product names, release plans, configurations, requirements, or official information.\n3. The user needs a factual, grounded answer.\nDo **not** use retrieval if:\n- The question is purely creative (e.g., storytelling, analogies) or personal preference.\n- The user simply wants text reformatted or rewritten from what is already present in the conversation.\nWhen uncertain → **Retrieve.** Retrieval is low risk and improves grounding.\n### File Upload vs URL Distinction\n**File uploads** (already in context):\n- Filenames like: README.md, document.pdf, notes.txt, data.csv\n- When you see file confirmation messages\n- Use conversation context directly - do NOT call URL tool\n**Web URLs** (need ingestion):\n- Start with http:// or https://\n- Examples: https://example.com, http://docs.site.org\n- User explicitly asks to fetch from web\n### Calculator Usage Rules\nUse the calculator when:\n- Performing arithmetic\n- Estimating totals\n- Comparing values\n- Modeling cost, time, effort, scale, or projections\nDo not perform math internally. **Call the calculator tool instead.**\n### Answer Construction Rules\n1. When asked: \"What is OpenRAG\", answer the following:\n\"OpenRAG is an open-source package for building agentic RAG systems. It supports integration with a wide range of orchestration tools, vector databases, and LLM providers. OpenRAG connects and amplifies three popular, proven open-source projects into one powerful platform:\n**Langflow** – Langflow is a powerful tool to build and deploy AI agents and MCP servers. [Read more](https://www.langflow.org/)\n**OpenSearch** – OpenSearch is an open source, search and observability suite that brings order to unstructured data at scale. [Read more](https://opensearch.org/)\n**Docling** – Docling simplifies document processing with advanced PDF understanding, OCR support, and seamless AI integrations. Parse PDFs, DOCX, PPTX, images & more. [Read more](https://www.docling.ai/)\"\n2. Synthesize retrieved or ingested content in your own words.\n3. Support factual claims with citations in the format:\n (Source: <document_name_or_id>)\n4. If no supporting evidence is found:\n Say: \"No relevant supporting sources were found for that request.\"\n5. Never invent facts or hallucinate details.\n6. Be concise, direct, and confident. \n7. Do not reveal internal chain-of-thought.",
37
+ "content": 'You are the OpenRAG Agent. You answer questions using retrieval, reasoning, and tool use.\nYou have access to several tools. Your job is to determine **which tool to use and when**.\n### Available Tools\n- OpenSearch Retrieval Tool:\n Use this to search the indexed knowledge base. Use when the user asks about product details, internal concepts, processes, architecture, documentation, roadmaps, or anything that may be stored in the index.\n- Conversation History:\n Use this to maintain continuity when the user is referring to previous turns. \n Do not treat history as a factual source.\n- Conversation File Context:\n Use this when the user asks about a document they uploaded or refers directly to its contents.\n **IMPORTANT**: If you receive confirmation that a file was uploaded (e.g., "Confirm that you received this file"), the file content is already available in the conversation context. Do NOT attempt to ingest it as a URL.\n Simply acknowledge the file and answer questions about it directly from the context.\n- URL Ingestion Tool:\n Use this **only** when the user explicitly asks you to read, summarize, or analyze the content of a web URL (http:// or https://).\n **Do NOT use this tool for filenames** (e.g., README.md, document.pdf, data.txt). These are file uploads, not URLs.\n Only use this tool for actual web addresses that the user explicitly provides.\n If unclear → ask a clarifying question.\n- Calculator / Expression Evaluation Tool:\n Use this when the user asks to compare numbers, compute estimates, calculate totals, analyze pricing, or answer any question requiring mathematics or quantitative reasoning.\n If the answer requires arithmetic, call the calculator tool rather than calculating internally.\n### Retrieval Decision Rules\nUse OpenSearch **whenever**:\n1. The question may be answered from internal or indexed data.\n2. The user references team names, product names, release plans, configurations, requirements, or official information.\n3. The user needs a factual, grounded answer.\nDo **not** use retrieval if:\n- The question is purely creative (e.g., storytelling, analogies) or personal preference.\n- The user simply wants text reformatted or rewritten from what is already present in the conversation.\nWhen uncertain → **Retrieve.** Retrieval is low risk and improves grounding.\n### File Upload vs URL Distinction\n**File uploads** (already in context):\n- Filenames like: README.md, document.pdf, notes.txt, data.csv\n- When you see file confirmation messages\n- Use conversation context directly - do NOT call URL tool\n**Web URLs** (need ingestion):\n- Start with http:// or https://\n- Examples: https://example.com, http://docs.site.org\n- User explicitly asks to fetch from web\n### Calculator Usage Rules\nUse the calculator when:\n- Performing arithmetic\n- Estimating totals\n- Comparing values\n- Modeling cost, time, effort, scale, or projections\nDo not perform math internally. **Call the calculator tool instead.**\n### Answer Construction Rules\n1. When asked: "What is OpenRAG", answer the following:\n"OpenRAG is an open-source package for building agentic RAG systems. It supports integration with a wide range of orchestration tools, vector databases, and LLM providers. OpenRAG connects and amplifies three popular, proven open-source projects into one powerful platform:\n**Langflow** – Langflow is a powerful tool to build and deploy AI agents and MCP servers. [Read more](https://www.langflow.org/)\n**OpenSearch** – OpenSearch is an open source, search and observability suite that brings order to unstructured data at scale. [Read more](https://opensearch.org/)\n**Docling** – Docling simplifies document processing with advanced PDF understanding, OCR support, and seamless AI integrations. Parse PDFs, DOCX, PPTX, images & more. [Read more](https://www.docling.ai/)"\n2. Synthesize retrieved or ingested content in your own words.\n3. Support factual claims with citations in the format: (Source: <chunk_id>) placed exactly where the claim occurs (e.g., at the end of the sentence or clause making the claim). If multiple sources support a claim, cite them sequentially like: (Source: chunk_id_1)(Source: chunk_id_2). Use the exact chunk_id or id provided in the retrieved source block.\n4. If no supporting evidence is found:\n Say: "No relevant supporting sources were found for that request."\n5. Never invent facts or hallucinate details.\n6. Be concise, direct, and confident. \n7. Do not reveal internal chain-of-thought.',
40
38
  }
41
39
  ],
42
40
  "previous_response_id": previous_response_id, # Parent response_id for branching
@@ -76,9 +74,7 @@ async def store_conversation_thread(user_id: str, response_id: str, conversation
76
74
  # Don't store actual messages - Langflow has them
77
75
  }
78
76
 
79
- await conversation_persistence.store_conversation_thread(
80
- user_id, response_id, metadata_only
81
- )
77
+ await conversation_persistence.store_conversation_thread(user_id, response_id, metadata_only)
82
78
 
83
79
 
84
80
  # Legacy function for backward compatibility
@@ -92,8 +88,12 @@ def get_user_conversation(user_id: str):
92
88
  )
93
89
  return active_conversations[user_id][latest_response_id]
94
90
 
95
- # Fallback to metadata-only conversations
96
- conversations = get_user_conversations(user_id)
91
+ # Fallback to metadata-only conversations. This is a SYNC legacy
92
+ # caller, so we read the eagerly-loaded JSON cache directly instead
93
+ # of awaiting the async service. The dict is populated in the
94
+ # service's __init__ for all storage modes; in `db` mode it stays
95
+ # empty but `active_conversations` above already covered fresh data.
96
+ conversations = conversation_persistence._conversations.get(user_id, {})
97
97
  if not conversations:
98
98
  return get_conversation_thread(user_id)
99
99
 
@@ -157,14 +157,18 @@ async def async_response_stream(
157
157
  full_response += delta_text
158
158
 
159
159
  # Enhanced logging for tool call detection (Granite 3.3 8b investigation)
160
- chunk_attrs = dir(chunk) if hasattr(chunk, '__dict__') else []
161
- tool_related_attrs = [attr for attr in chunk_attrs if 'tool' in attr.lower() or 'call' in attr.lower() or 'retrieval' in attr.lower()]
160
+ chunk_attrs = dir(chunk) if hasattr(chunk, "__dict__") else []
161
+ tool_related_attrs = [
162
+ attr
163
+ for attr in chunk_attrs
164
+ if "tool" in attr.lower() or "call" in attr.lower() or "retrieval" in attr.lower()
165
+ ]
162
166
  if tool_related_attrs:
163
167
  logger.info(
164
168
  "Tool-related attributes found in chunk",
165
169
  chunk_count=chunk_count,
166
170
  attributes=tool_related_attrs,
167
- chunk_type=type(chunk).__name__
171
+ chunk_type=type(chunk).__name__,
168
172
  )
169
173
 
170
174
  # Send the raw event as JSON followed by newline for easy parsing
@@ -176,7 +180,7 @@ async def async_response_stream(
176
180
  # openai SDK's event models declare/expect delta to be str so model_dump() will
177
181
  # emit noisy logs (PydanticSerializationUnexpectedValue) for every chunk. Basically
178
182
  # just reattach the raw (unvalidated) value afterwards to preserve same shape.
179
- # TODO: replace this with just chunk.model_dump() if langflow's /response SSE matches openai's
183
+ # TODO: replace this with just chunk.model_dump() if langflow's /response SSE matches openai's
180
184
  # SDK's delta: str schema
181
185
  chunk_data = chunk.model_dump(exclude={"delta"})
182
186
  if hasattr(chunk, "delta"):
@@ -190,15 +194,26 @@ async def async_response_stream(
190
194
  if isinstance(chunk_data, dict):
191
195
  # Check for any fields that might indicate tool usage
192
196
  potential_tool_fields = {
193
- k: v for k, v in chunk_data.items()
194
- if any(keyword in str(k).lower() for keyword in ['tool', 'call', 'retrieval', 'function', 'result', 'output'])
197
+ k: v
198
+ for k, v in chunk_data.items()
199
+ if any(
200
+ keyword in str(k).lower()
201
+ for keyword in [
202
+ "tool",
203
+ "call",
204
+ "retrieval",
205
+ "function",
206
+ "result",
207
+ "output",
208
+ ]
209
+ )
195
210
  }
196
211
  if potential_tool_fields:
197
212
  logger.info(
198
213
  "Potential tool-related fields in chunk",
199
214
  chunk_count=chunk_count,
200
215
  fields=list(potential_tool_fields.keys()),
201
- sample_data=str(potential_tool_fields)[:500]
216
+ sample_data=str(potential_tool_fields)[:500],
202
217
  )
203
218
 
204
219
  # Detect response.completed event and log usage
@@ -217,17 +232,19 @@ async def async_response_stream(
217
232
  # This helps Granite 3.3 8b and other models that don't emit standard markers
218
233
  if isinstance(chunk_data, dict) and not detected_tool_call:
219
234
  # Check if this chunk contains retrieval results
220
- has_results = any([
221
- 'results' in chunk_data and isinstance(chunk_data.get('results'), list),
222
- 'outputs' in chunk_data and isinstance(chunk_data.get('outputs'), list),
223
- 'retrieved_documents' in chunk_data,
224
- 'retrieval_results' in chunk_data,
225
- ])
235
+ has_results = any(
236
+ [
237
+ "results" in chunk_data and isinstance(chunk_data.get("results"), list),
238
+ "outputs" in chunk_data and isinstance(chunk_data.get("outputs"), list),
239
+ "retrieved_documents" in chunk_data,
240
+ "retrieval_results" in chunk_data,
241
+ ]
242
+ )
226
243
 
227
244
  if has_results:
228
245
  logger.info(
229
246
  "Detected implicit tool call in backend, injecting synthetic event",
230
- chunk_fields=list(chunk_data.keys())
247
+ chunk_fields=list(chunk_data.keys()),
231
248
  )
232
249
  # Inject a synthetic tool call event before this chunk
233
250
  synthetic_event = {
@@ -239,10 +256,12 @@ async def async_response_stream(
239
256
  "tool_name": "Retrieval",
240
257
  "status": "completed",
241
258
  "inputs": {"implicit": True, "backend_detected": True},
242
- "results": chunk_data.get('results') or chunk_data.get('outputs') or
243
- chunk_data.get('retrieved_documents') or
244
- chunk_data.get('retrieval_results') or []
245
- }
259
+ "results": chunk_data.get("results")
260
+ or chunk_data.get("outputs")
261
+ or chunk_data.get("retrieved_documents")
262
+ or chunk_data.get("retrieval_results")
263
+ or [],
264
+ },
246
265
  }
247
266
  # Send the synthetic event first
248
267
  yield (json.dumps(synthetic_event, default=str) + "\n").encode("utf-8")
@@ -253,16 +272,13 @@ async def async_response_stream(
253
272
  # Fallback to string representation
254
273
  logger.warning("JSON serialization failed", error=str(e))
255
274
  yield (
256
- json.dumps(
257
- {"error": f"Serialization failed: {e}", "raw": str(chunk)}
258
- )
259
- + "\n"
275
+ json.dumps({"error": f"Serialization failed: {e}", "raw": str(chunk)}) + "\n"
260
276
  ).encode("utf-8")
261
277
 
262
278
  logger.debug("Stream complete", total_chunks=chunk_count)
263
279
  logger.info("Response generated", log_prefix=log_prefix, response=full_response)
264
280
 
265
- except Exception as e:
281
+ except Exception:
266
282
  logger.exception("[AGENT] Streaming failed")
267
283
  raise
268
284
 
@@ -276,6 +292,11 @@ async def async_response(
276
292
  previous_response_id: str = None,
277
293
  log_prefix: str = "response",
278
294
  ):
295
+ # extra_headers/request_params carry raw secrets (JWT, provider API keys,
296
+ # x-api-key — see langflow_headers.py). Never pass them to a logger call
297
+ # (e.g. logger.info(..., extra_headers=extra_headers)) — that logs the
298
+ # secrets directly, bypassing the show_locals=False traceback protection
299
+ # in utils/logging_config.py.
279
300
  try:
280
301
  logger.info("User prompt received", prompt=prompt)
281
302
 
@@ -297,16 +318,17 @@ async def async_response(
297
318
 
298
319
  response = await client.responses.create(**request_params)
299
320
 
300
- # Check if response has output_text using getattr to avoid issues with special objects
301
- output_text = getattr(response, "output_text", None)
321
+ try:
322
+ output_text = response.output_text
323
+ except (AttributeError, TypeError):
324
+ output_text = None
325
+
302
326
  if output_text is not None:
303
327
  response_text = output_text
304
328
  logger.info("Response generated", log_prefix=log_prefix, response=response_text)
305
329
 
306
330
  # Extract and store response_id if available
307
- response_id = getattr(response, "id", None) or getattr(
308
- response, "response_id", None
309
- )
331
+ response_id = getattr(response, "id", None) or getattr(response, "response_id", None)
310
332
 
311
333
  return response_text, response_id, response
312
334
  else:
@@ -317,7 +339,7 @@ async def async_response(
317
339
  if error_msg:
318
340
  msg = error_msg
319
341
  raise ValueError(msg)
320
- except Exception as e:
342
+ except Exception:
321
343
  logger.exception("[AGENT] Non-streaming response failed")
322
344
  raise
323
345
 
@@ -377,7 +399,7 @@ async def async_langflow_stream(
377
399
  flow_id,
378
400
  extra_headers=extra_headers,
379
401
  previous_response_id=previous_response_id,
380
- log_prefix="langflow",
402
+ log_prefix="langflow",
381
403
  ):
382
404
  logger.debug(
383
405
  "Yielding chunk from langflow stream",
@@ -385,7 +407,7 @@ async def async_langflow_stream(
385
407
  )
386
408
  yield chunk
387
409
  logger.debug("Langflow stream completed")
388
- except Exception as e:
410
+ except Exception:
389
411
  logger.exception("[AGENT] Langflow stream failed")
390
412
  raise
391
413
 
@@ -399,24 +421,18 @@ async def async_chat(
399
421
  previous_response_id: str = None,
400
422
  filter_id: str = None,
401
423
  ):
402
- logger.debug(
403
- "async_chat called", user_id=user_id, previous_response_id=previous_response_id
404
- )
424
+ logger.debug("async_chat called", user_id=user_id, previous_response_id=previous_response_id)
405
425
 
406
426
  # Get the specific conversation thread (or create new one)
407
427
  conversation_state = get_conversation_thread(user_id, previous_response_id)
408
- logger.debug(
409
- "Got conversation state", message_count=len(conversation_state["messages"])
410
- )
428
+ logger.debug("Got conversation state", message_count=len(conversation_state["messages"]))
411
429
 
412
430
  # Add user message to conversation with timestamp
413
431
  from datetime import datetime
414
432
 
415
433
  user_message = {"role": "user", "content": prompt, "timestamp": datetime.now()}
416
434
  conversation_state["messages"].append(user_message)
417
- logger.debug(
418
- "Added user message", message_count=len(conversation_state["messages"])
419
- )
435
+ logger.debug("Added user message", message_count=len(conversation_state["messages"]))
420
436
 
421
437
  # Store filter_id in conversation state if provided
422
438
  if filter_id:
@@ -429,9 +445,7 @@ async def async_chat(
429
445
  previous_response_id=previous_response_id,
430
446
  log_prefix="agent",
431
447
  )
432
- logger.debug(
433
- "Got response", response_preview=response_text[:50], response_id=response_id
434
- )
448
+ logger.debug("Got response", response_preview=response_text[:50], response_id=response_id)
435
449
 
436
450
  # Add assistant response to conversation with response_id, timestamp, and full response object
437
451
  assistant_message = {
@@ -444,20 +458,16 @@ async def async_chat(
444
458
  else str(response_obj), # Store complete response for function calls
445
459
  }
446
460
  conversation_state["messages"].append(assistant_message)
447
- logger.debug(
448
- "Added assistant message", message_count=len(conversation_state["messages"])
449
- )
461
+ logger.debug("Added assistant message", message_count=len(conversation_state["messages"]))
450
462
 
451
463
  # Store the conversation thread with its response_id
452
464
  if response_id:
453
465
  conversation_state["last_activity"] = datetime.now()
454
466
  await store_conversation_thread(user_id, response_id, conversation_state)
455
- logger.debug(
456
- "Stored conversation thread", user_id=user_id, response_id=response_id
457
- )
467
+ logger.debug("Stored conversation thread", user_id=user_id, response_id=response_id)
458
468
 
459
469
  # Debug: Check what's in user_conversations now
460
- conversations = get_user_conversations(user_id)
470
+ conversations = await get_user_conversations(user_id)
461
471
  logger.debug(
462
472
  "User conversations updated",
463
473
  user_id=user_id,
@@ -518,7 +528,7 @@ async def async_chat_stream(
518
528
  if chunk_data.get("type") == "response.completed":
519
529
  response_obj = chunk_data.get("response", {})
520
530
  usage_data = response_obj.get("usage")
521
- except:
531
+ except Exception:
522
532
  pass
523
533
  yield chunk
524
534
 
@@ -634,13 +644,21 @@ async def async_langflow_chat(
634
644
  else (result if isinstance(result, dict) else {})
635
645
  )
636
646
  if "text" in rd:
637
- sources.append({
638
- "filename": rd.get("filename", ""),
639
- "text": rd.get("text", ""),
640
- "score": rd.get("score", 0),
641
- "page": rd.get("page"),
642
- "mimetype": rd.get("mimetype"),
643
- })
647
+ sources.append(
648
+ {
649
+ "filename": rd.get("filename", ""),
650
+ "text": rd.get("text", ""),
651
+ "score": rd.get("score", 0),
652
+ "page": rd.get("page"),
653
+ "mimetype": rd.get("mimetype"),
654
+ "chunk_id": rd.get("chunk_id") or rd.get("id") or "",
655
+ "id": rd.get("id") or rd.get("chunk_id") or "",
656
+ "embedding_model": rd.get("embedding_model"),
657
+ "parser": rd.get("parser"),
658
+ "chunk_size": rd.get("chunk_size"),
659
+ "chunk_overlap": rd.get("chunk_overlap"),
660
+ }
661
+ )
644
662
 
645
663
  # Layer 2: Top-level dict inspection (mirrors streaming middleware in async_response_stream).
646
664
  # Langflow may embed retrieval results directly in the response dict rather than
@@ -661,13 +679,21 @@ async def async_langflow_chat(
661
679
  if isinstance(implicit_results, list):
662
680
  for result in implicit_results:
663
681
  if isinstance(result, dict) and "text" in result:
664
- sources.append({
665
- "filename": result.get("filename", ""),
666
- "text": result.get("text", ""),
667
- "score": result.get("score", 0),
668
- "page": result.get("page"),
669
- "mimetype": result.get("mimetype"),
670
- })
682
+ sources.append(
683
+ {
684
+ "filename": result.get("filename", ""),
685
+ "text": result.get("text", ""),
686
+ "score": result.get("score", 0),
687
+ "page": result.get("page"),
688
+ "mimetype": result.get("mimetype"),
689
+ "chunk_id": result.get("chunk_id") or result.get("id") or "",
690
+ "id": result.get("id") or result.get("chunk_id") or "",
691
+ "embedding_model": result.get("embedding_model"),
692
+ "parser": result.get("parser"),
693
+ "chunk_size": result.get("chunk_size"),
694
+ "chunk_overlap": result.get("chunk_overlap"),
695
+ }
696
+ )
671
697
 
672
698
  # Layer 3: Citation-text fallback.
673
699
  # Parse "(Source: filename)" patterns emitted by the LLM when it cites documents.
@@ -675,14 +701,17 @@ async def async_langflow_chat(
675
701
  # structured retrieval data.
676
702
  if not sources:
677
703
  import re
704
+
678
705
  for match in re.finditer(r"\(Source:\s*([^\)]+)\)", response_text):
679
- sources.append({
680
- "filename": match.group(1).strip(),
681
- "text": "",
682
- "score": 0,
683
- "page": None,
684
- "mimetype": None,
685
- })
706
+ sources.append(
707
+ {
708
+ "filename": match.group(1).strip(),
709
+ "text": "",
710
+ "score": 0,
711
+ "page": None,
712
+ "mimetype": None,
713
+ }
714
+ )
686
715
 
687
716
  if not store_conversation:
688
717
  return response_text, response_id, sources
@@ -696,7 +725,7 @@ async def async_langflow_chat(
696
725
  try:
697
726
  from services.session_ownership_service import session_ownership_service
698
727
 
699
- session_ownership_service.claim_session(user_id, response_id)
728
+ await session_ownership_service.claim_session(user_id, response_id)
700
729
  logger.debug(f"Claimed session {response_id} for user {user_id}")
701
730
  except Exception as e:
702
731
  logger.warning(f"Failed to claim session ownership: {e}")
@@ -711,7 +740,7 @@ async def async_langflow_chat(
711
740
  )
712
741
 
713
742
  # Debug: Check what's in user_conversations now
714
- conversations = get_user_conversations(user_id)
743
+ conversations = await get_user_conversations(user_id)
715
744
  logger.debug(
716
745
  "User conversations updated",
717
746
  user_id=user_id,
@@ -784,44 +813,46 @@ async def async_langflow_chat_stream(
784
813
  response_id = chunk_data["response_id"]
785
814
 
786
815
  # Check for error status
787
- if chunk_data.get("finish_reason") == "error" or chunk_data.get("status") == "failed":
816
+ if (
817
+ chunk_data.get("finish_reason") == "error"
818
+ or chunk_data.get("status") == "failed"
819
+ ):
788
820
  error_occurred = True
789
821
  logger.error("Error detected in Langflow stream chunk")
790
822
  # Capture usage from response.completed event
791
823
  if chunk_data.get("type") == "response.completed":
792
824
  response_obj = chunk_data.get("response", {})
793
825
  usage_data = response_obj.get("usage")
794
- except:
826
+ except Exception:
795
827
  pass
796
828
  yield chunk
797
829
 
798
830
  # Add the complete assistant response to message history with response_id, timestamp, and function call data
799
- assistant_message = {
800
- "role": "assistant",
801
- "content": full_response,
802
- "response_id": response_id,
803
- "timestamp": datetime.now(),
804
- "chunks": collected_chunks, # Store complete chunk data for function calls
805
- "error": error_occurred, # Mark if this was an error response
806
- }
807
- # Store usage data if available (from response.completed event)
808
- if usage_data:
809
- assistant_message["response_data"] = {"usage": usage_data}
810
- conversation_state["messages"].append(assistant_message)
831
+ if full_response:
832
+ assistant_message = {
833
+ "role": "assistant",
834
+ "content": full_response,
835
+ "response_id": response_id,
836
+ "timestamp": datetime.now(),
837
+ "chunks": collected_chunks,
838
+ "error": error_occurred,
839
+ }
840
+ if usage_data:
841
+ assistant_message["response_data"] = {"usage": usage_data}
842
+ conversation_state["messages"].append(assistant_message)
811
843
 
812
844
  # Store the conversation thread with its response_id
813
845
  if response_id:
814
846
  conversation_state["last_activity"] = datetime.now()
815
847
  await store_conversation_thread(user_id, response_id, conversation_state)
816
848
 
817
- # Claim session ownership for this user
818
- try:
819
- from services.session_ownership_service import session_ownership_service
849
+ try:
850
+ from services.session_ownership_service import session_ownership_service
820
851
 
821
- session_ownership_service.claim_session(user_id, response_id)
822
- logger.debug(f"Claimed session {response_id} for user {user_id}")
823
- except Exception as e:
824
- logger.warning(f"Failed to claim session ownership: {e}")
852
+ await session_ownership_service.claim_session(user_id, response_id)
853
+ logger.debug(f"Claimed session {response_id} for user {user_id}")
854
+ except Exception as e:
855
+ logger.warning(f"Failed to claim session ownership: {e}")
825
856
 
826
857
  logger.debug(
827
858
  f"Stored langflow conversation thread for user {user_id} with response_id: {response_id}"
@@ -876,15 +907,20 @@ async def delete_user_conversation(user_id: str, response_id: str) -> bool:
876
907
  deleted = True
877
908
 
878
909
  # Delete from persistent storage — let real errors propagate to the caller
879
- conversation_deleted = await conversation_persistence.delete_conversation_thread(user_id, response_id)
910
+ conversation_deleted = await conversation_persistence.delete_conversation_thread(
911
+ user_id, response_id
912
+ )
880
913
  if conversation_deleted:
881
- logger.debug(f"Deleted conversation {response_id} from persistent storage for user {user_id}")
914
+ logger.debug(
915
+ f"Deleted conversation {response_id} from persistent storage for user {user_id}"
916
+ )
882
917
  deleted = True
883
918
 
884
919
  # Release session ownership (best-effort; never masks storage errors above)
885
920
  try:
886
921
  from services.session_ownership_service import session_ownership_service
887
- session_ownership_service.release_session(user_id, response_id)
922
+
923
+ await session_ownership_service.release_session(user_id, response_id)
888
924
  logger.debug(f"Released session ownership for {response_id} for user {user_id}")
889
925
  except Exception as e:
890
926
  logger.warning(f"Failed to release session ownership: {e}")
api/acl.py ADDED
@@ -0,0 +1,44 @@
1
+ from fastapi import Depends
2
+ from fastapi.responses import JSONResponse
3
+
4
+ from config.settings import get_index_name
5
+ from dependencies import get_current_user, get_session_manager
6
+ from session_manager import User
7
+
8
+
9
+ async def get_document_acl(
10
+ filename: str,
11
+ session_manager=Depends(get_session_manager),
12
+ user: User = Depends(get_current_user),
13
+ ):
14
+ """GET /documents/acl?filename=... — return ACL for a document."""
15
+ opensearch_client = session_manager.get_user_opensearch_client(user.user_id, user.jwt_token)
16
+ response = await opensearch_client.search(
17
+ index=get_index_name(),
18
+ body={
19
+ "query": {"term": {"filename": filename}},
20
+ "size": 1,
21
+ "_source": [
22
+ "owner",
23
+ "allowed_users",
24
+ "allowed_groups",
25
+ "allowed_principal_labels",
26
+ ],
27
+ },
28
+ )
29
+ hits = response.get("hits", {}).get("hits", [])
30
+ if not hits:
31
+ return JSONResponse(
32
+ {"error": f"Document '{filename}' not found"},
33
+ status_code=404,
34
+ )
35
+
36
+ source = hits[0]["_source"]
37
+ return JSONResponse(
38
+ {
39
+ "owner": source.get("owner"),
40
+ "allowed_users": source.get("allowed_users", []),
41
+ "allowed_groups": source.get("allowed_groups", []),
42
+ "allowed_principal_labels": source.get("allowed_principal_labels", []),
43
+ }
44
+ )