OpenRAG 0.5.1.dev21__py3-none-any.whl → 0.6.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent.py +148 -112
- api/acl.py +44 -0
- api/auth.py +36 -28
- api/chat.py +48 -10
- api/config.py +46 -0
- api/connectors.py +1081 -144
- api/docling.py +46 -104
- api/documents.py +139 -44
- api/files.py +95 -0
- api/flows.py +2 -2
- api/health.py +80 -0
- api/ingest_preview.py +67 -0
- api/keys.py +4 -4
- api/knowledge_filter.py +86 -79
- api/langflow_files.py +5 -10
- api/langflow_ingest.py +119 -0
- api/models.py +24 -23
- api/nudges.py +15 -1
- api/oidc.py +25 -17
- api/provider_health.py +92 -29
- api/router.py +150 -25
- api/schemas/tasks.py +22 -0
- api/search.py +7 -2
- api/settings/__init__.py +88 -0
- api/{settings.py → settings/endpoints.py} +540 -1005
- api/settings/helpers.py +230 -0
- api/settings/langflow_sync.py +323 -0
- api/settings/models.py +242 -0
- api/tasks.py +62 -6
- api/upload.py +51 -41
- api/users.py +166 -0
- api/v1/_filter_resolution.py +100 -0
- api/v1/chat.py +126 -44
- api/v1/documents.py +154 -21
- api/v1/knowledge_filters.py +8 -6
- api/v1/models.py +26 -10
- api/v1/search.py +43 -14
- api/v1/settings.py +34 -18
- app/__init__.py +0 -0
- app/container.py +240 -0
- app/factory.py +55 -0
- app/lifespan.py +435 -0
- app/middleware.py +69 -0
- app/router_app.py +158 -0
- app/routes/__init__.py +23 -0
- app/routes/internal.py +456 -0
- app/routes/mcp.py +29 -0
- app/routes/public_v1.py +132 -0
- auth/ibm_auth.py +48 -2
- auth/jwt_roles.py +102 -0
- auth/permission_dependencies.py +143 -0
- auth/request_identity.py +698 -0
- auth/user_identity_cache.py +138 -0
- bootstrap.py +12 -0
- config/config_manager.py +185 -42
- config/model_constants.py +11 -14
- config/settings.py +908 -126
- config/storage_mode.py +61 -0
- config/utils.py +202 -0
- connectors/__init__.py +2 -4
- connectors/aws_s3/connector.py +47 -14
- connectors/aws_s3/models.py +7 -7
- connectors/aws_s3/support.py +13 -6
- connectors/base.py +141 -25
- connectors/connection_manager.py +334 -194
- connectors/google_drive/connector.py +201 -30
- connectors/google_drive/oauth.py +92 -28
- connectors/google_drive_acl.py +408 -0
- connectors/microsoft_graph_acl.py +395 -0
- connectors/microsoft_oauth_utils.py +54 -0
- connectors/onedrive/connector.py +325 -53
- connectors/onedrive/oauth.py +17 -4
- connectors/registry.py +73 -0
- connectors/service.py +160 -13
- connectors/sharepoint/connector.py +302 -34
- connectors/sharepoint/oauth.py +148 -57
- db/__init__.py +21 -0
- db/base.py +19 -0
- db/engine.py +82 -0
- db/migrations_runtime.py +351 -0
- db/models/__init__.py +33 -0
- db/models/api_key.py +27 -0
- db/models/audit_log.py +23 -0
- db/models/conversation.py +30 -0
- db/models/migration_status.py +13 -0
- db/models/permission.py +13 -0
- db/models/role.py +14 -0
- db/models/role_permission.py +16 -0
- db/models/session_ownership.py +23 -0
- db/models/user.py +38 -0
- db/models/user_preferences.py +51 -0
- db/models/user_role.py +20 -0
- db/models/workspace_config.py +30 -0
- db/repositories/__init__.py +27 -0
- db/repositories/_helpers.py +9 -0
- db/repositories/api_key_repo.py +43 -0
- db/repositories/audit_repo.py +36 -0
- db/repositories/conversation_repo.py +104 -0
- db/repositories/permission_repo.py +23 -0
- db/repositories/preferences_repo.py +44 -0
- db/repositories/role_repo.py +68 -0
- db/repositories/session_ownership_repo.py +85 -0
- db/repositories/user_repo.py +74 -0
- db/repositories/workspace_config_repo.py +60 -0
- db/seed.py +251 -0
- db/types.py +61 -0
- dependencies.py +216 -330
- main.py +82 -2198
- mcp_http/server.py +62 -27
- models/processors.py +744 -209
- models/tasks.py +37 -5
- models/url.py +8 -11
- {openrag-0.5.1.dev21.dist-info → openrag-0.6.0.dev1.dist-info}/METADATA +32 -19
- openrag-0.6.0.dev1.dist-info/RECORD +247 -0
- {openrag-0.5.1.dev21.dist-info → openrag-0.6.0.dev1.dist-info}/top_level.txt +2 -0
- services/api_key_service.py +65 -35
- services/auth_service.py +153 -157
- services/chat_service.py +301 -92
- services/connector_access_service.py +108 -0
- services/connector_oauth_config_service.py +197 -0
- services/conversation_persistence_service.py +295 -106
- services/default_docs_service.py +696 -0
- services/default_role_sync.py +307 -0
- services/dev_role_toggle.py +48 -0
- services/dls_principal_service.py +449 -0
- services/docling_polling_service.py +224 -0
- services/docling_service.py +474 -0
- services/document_index_writer.py +327 -0
- services/document_service.py +36 -25
- services/file_service.py +245 -0
- services/flows_service.py +228 -114
- services/group_acl_service.py +117 -0
- services/ingest_preview_service.py +186 -0
- services/knowledge_filter_service.py +114 -65
- services/langflow_file_service.py +697 -245
- services/langflow_ingest_token_service.py +203 -0
- services/monitor_service.py +65 -53
- services/onboarding_reset.py +151 -0
- services/rbac_service.py +141 -0
- services/search_service.py +117 -113
- services/session_ownership_service.py +219 -85
- services/startup_orchestrator.py +221 -0
- services/task_service.py +790 -53
- services/user_service.py +324 -0
- services/workspace_config_service.py +411 -0
- session_manager.py +134 -62
- tui/_assets/docker-compose.yml +77 -23
- tui/_assets/flows/ingestion_flow.json +1589 -3982
- tui/_assets/flows/openrag_agent.json +194 -477
- tui/_assets/flows/openrag_nudges.json +147 -375
- tui/_assets/flows/openrag_url_mcp.json +619 -1880
- tui/config_fields.py +370 -232
- tui/managers/container_manager.py +248 -210
- tui/managers/docling_manager.py +20 -9
- tui/managers/env_manager.py +149 -56
- tui/utils/validation.py +21 -5
- utils/acl_utils.py +110 -112
- utils/container_utils.py +63 -0
- utils/document_processing.py +67 -29
- utils/embeddings.py +56 -25
- utils/env_utils.py +21 -4
- utils/file_utils.py +17 -12
- utils/group_acl.py +161 -0
- utils/ingest_preview_flag.py +21 -0
- utils/jwt_keygen.py +55 -0
- utils/jwt_verification.py +443 -0
- utils/langflow_headers.py +51 -11
- utils/logging_config.py +98 -78
- utils/opensearch_delete.py +0 -30
- utils/opensearch_init.py +434 -0
- utils/opensearch_queries.py +66 -18
- utils/opensearch_utils.py +194 -58
- utils/provider_health_cache.py +115 -0
- utils/run_mode_utils.py +1 -0
- utils/telemetry/message_id.py +30 -28
- utils/url_content_fetcher.py +72 -0
- utils/version_utils.py +8 -6
- connectors/ibm_cos/__init__.py +0 -17
- connectors/ibm_cos/api.py +0 -202
- connectors/ibm_cos/auth.py +0 -183
- connectors/ibm_cos/connector.py +0 -370
- connectors/ibm_cos/models.py +0 -20
- connectors/ibm_cos/support.py +0 -68
- openrag-0.5.1.dev21.dist-info/RECORD +0 -165
- {openrag-0.5.1.dev21.dist-info → openrag-0.6.0.dev1.dist-info}/WHEEL +0 -0
- {openrag-0.5.1.dev21.dist-info → openrag-0.6.0.dev1.dist-info}/entry_points.txt +0 -0
- {openrag-0.5.1.dev21.dist-info → openrag-0.6.0.dev1.dist-info}/licenses/LICENSE +0 -0
agent.py
CHANGED
|
@@ -1,19 +1,17 @@
|
|
|
1
|
-
from
|
|
1
|
+
from typing import Any
|
|
2
2
|
|
|
3
|
+
from services.conversation_persistence_service import conversation_persistence
|
|
3
4
|
from utils.logging_config import get_logger
|
|
4
5
|
|
|
5
6
|
logger = get_logger(__name__)
|
|
6
7
|
|
|
7
|
-
# Import persistent storage
|
|
8
|
-
from services.conversation_persistence_service import conversation_persistence
|
|
9
|
-
|
|
10
8
|
# In-memory storage for active conversation threads (preserves function calls)
|
|
11
|
-
active_conversations = {}
|
|
9
|
+
active_conversations: dict[str, dict[str, Any]] = {}
|
|
12
10
|
|
|
13
11
|
|
|
14
|
-
def get_user_conversations(user_id: str):
|
|
15
|
-
"""Get conversation metadata for a user from persistent storage"""
|
|
16
|
-
return conversation_persistence.get_user_conversations(user_id)
|
|
12
|
+
async def get_user_conversations(user_id: str):
|
|
13
|
+
"""Get conversation metadata for a user from persistent storage."""
|
|
14
|
+
return await conversation_persistence.get_user_conversations(user_id)
|
|
17
15
|
|
|
18
16
|
|
|
19
17
|
def get_conversation_thread(user_id: str, previous_response_id: str = None):
|
|
@@ -36,7 +34,7 @@ def get_conversation_thread(user_id: str, previous_response_id: str = None):
|
|
|
36
34
|
"messages": [
|
|
37
35
|
{
|
|
38
36
|
"role": "system",
|
|
39
|
-
"content":
|
|
37
|
+
"content": 'You are the OpenRAG Agent. You answer questions using retrieval, reasoning, and tool use.\nYou have access to several tools. Your job is to determine **which tool to use and when**.\n### Available Tools\n- OpenSearch Retrieval Tool:\n Use this to search the indexed knowledge base. Use when the user asks about product details, internal concepts, processes, architecture, documentation, roadmaps, or anything that may be stored in the index.\n- Conversation History:\n Use this to maintain continuity when the user is referring to previous turns. \n Do not treat history as a factual source.\n- Conversation File Context:\n Use this when the user asks about a document they uploaded or refers directly to its contents.\n **IMPORTANT**: If you receive confirmation that a file was uploaded (e.g., "Confirm that you received this file"), the file content is already available in the conversation context. Do NOT attempt to ingest it as a URL.\n Simply acknowledge the file and answer questions about it directly from the context.\n- URL Ingestion Tool:\n Use this **only** when the user explicitly asks you to read, summarize, or analyze the content of a web URL (http:// or https://).\n **Do NOT use this tool for filenames** (e.g., README.md, document.pdf, data.txt). These are file uploads, not URLs.\n Only use this tool for actual web addresses that the user explicitly provides.\n If unclear → ask a clarifying question.\n- Calculator / Expression Evaluation Tool:\n Use this when the user asks to compare numbers, compute estimates, calculate totals, analyze pricing, or answer any question requiring mathematics or quantitative reasoning.\n If the answer requires arithmetic, call the calculator tool rather than calculating internally.\n### Retrieval Decision Rules\nUse OpenSearch **whenever**:\n1. The question may be answered from internal or indexed data.\n2. The user references team names, product names, release plans, configurations, requirements, or official information.\n3. The user needs a factual, grounded answer.\nDo **not** use retrieval if:\n- The question is purely creative (e.g., storytelling, analogies) or personal preference.\n- The user simply wants text reformatted or rewritten from what is already present in the conversation.\nWhen uncertain → **Retrieve.** Retrieval is low risk and improves grounding.\n### File Upload vs URL Distinction\n**File uploads** (already in context):\n- Filenames like: README.md, document.pdf, notes.txt, data.csv\n- When you see file confirmation messages\n- Use conversation context directly - do NOT call URL tool\n**Web URLs** (need ingestion):\n- Start with http:// or https://\n- Examples: https://example.com, http://docs.site.org\n- User explicitly asks to fetch from web\n### Calculator Usage Rules\nUse the calculator when:\n- Performing arithmetic\n- Estimating totals\n- Comparing values\n- Modeling cost, time, effort, scale, or projections\nDo not perform math internally. **Call the calculator tool instead.**\n### Answer Construction Rules\n1. When asked: "What is OpenRAG", answer the following:\n"OpenRAG is an open-source package for building agentic RAG systems. It supports integration with a wide range of orchestration tools, vector databases, and LLM providers. OpenRAG connects and amplifies three popular, proven open-source projects into one powerful platform:\n**Langflow** – Langflow is a powerful tool to build and deploy AI agents and MCP servers. [Read more](https://www.langflow.org/)\n**OpenSearch** – OpenSearch is an open source, search and observability suite that brings order to unstructured data at scale. [Read more](https://opensearch.org/)\n**Docling** – Docling simplifies document processing with advanced PDF understanding, OCR support, and seamless AI integrations. Parse PDFs, DOCX, PPTX, images & more. [Read more](https://www.docling.ai/)"\n2. Synthesize retrieved or ingested content in your own words.\n3. Support factual claims with citations in the format: (Source: <chunk_id>) placed exactly where the claim occurs (e.g., at the end of the sentence or clause making the claim). If multiple sources support a claim, cite them sequentially like: (Source: chunk_id_1)(Source: chunk_id_2). Use the exact chunk_id or id provided in the retrieved source block.\n4. If no supporting evidence is found:\n Say: "No relevant supporting sources were found for that request."\n5. Never invent facts or hallucinate details.\n6. Be concise, direct, and confident. \n7. Do not reveal internal chain-of-thought.',
|
|
40
38
|
}
|
|
41
39
|
],
|
|
42
40
|
"previous_response_id": previous_response_id, # Parent response_id for branching
|
|
@@ -76,9 +74,7 @@ async def store_conversation_thread(user_id: str, response_id: str, conversation
|
|
|
76
74
|
# Don't store actual messages - Langflow has them
|
|
77
75
|
}
|
|
78
76
|
|
|
79
|
-
await conversation_persistence.store_conversation_thread(
|
|
80
|
-
user_id, response_id, metadata_only
|
|
81
|
-
)
|
|
77
|
+
await conversation_persistence.store_conversation_thread(user_id, response_id, metadata_only)
|
|
82
78
|
|
|
83
79
|
|
|
84
80
|
# Legacy function for backward compatibility
|
|
@@ -92,8 +88,12 @@ def get_user_conversation(user_id: str):
|
|
|
92
88
|
)
|
|
93
89
|
return active_conversations[user_id][latest_response_id]
|
|
94
90
|
|
|
95
|
-
# Fallback to metadata-only conversations
|
|
96
|
-
|
|
91
|
+
# Fallback to metadata-only conversations. This is a SYNC legacy
|
|
92
|
+
# caller, so we read the eagerly-loaded JSON cache directly instead
|
|
93
|
+
# of awaiting the async service. The dict is populated in the
|
|
94
|
+
# service's __init__ for all storage modes; in `db` mode it stays
|
|
95
|
+
# empty but `active_conversations` above already covered fresh data.
|
|
96
|
+
conversations = conversation_persistence._conversations.get(user_id, {})
|
|
97
97
|
if not conversations:
|
|
98
98
|
return get_conversation_thread(user_id)
|
|
99
99
|
|
|
@@ -157,14 +157,18 @@ async def async_response_stream(
|
|
|
157
157
|
full_response += delta_text
|
|
158
158
|
|
|
159
159
|
# Enhanced logging for tool call detection (Granite 3.3 8b investigation)
|
|
160
|
-
chunk_attrs = dir(chunk) if hasattr(chunk,
|
|
161
|
-
tool_related_attrs = [
|
|
160
|
+
chunk_attrs = dir(chunk) if hasattr(chunk, "__dict__") else []
|
|
161
|
+
tool_related_attrs = [
|
|
162
|
+
attr
|
|
163
|
+
for attr in chunk_attrs
|
|
164
|
+
if "tool" in attr.lower() or "call" in attr.lower() or "retrieval" in attr.lower()
|
|
165
|
+
]
|
|
162
166
|
if tool_related_attrs:
|
|
163
167
|
logger.info(
|
|
164
168
|
"Tool-related attributes found in chunk",
|
|
165
169
|
chunk_count=chunk_count,
|
|
166
170
|
attributes=tool_related_attrs,
|
|
167
|
-
chunk_type=type(chunk).__name__
|
|
171
|
+
chunk_type=type(chunk).__name__,
|
|
168
172
|
)
|
|
169
173
|
|
|
170
174
|
# Send the raw event as JSON followed by newline for easy parsing
|
|
@@ -176,7 +180,7 @@ async def async_response_stream(
|
|
|
176
180
|
# openai SDK's event models declare/expect delta to be str so model_dump() will
|
|
177
181
|
# emit noisy logs (PydanticSerializationUnexpectedValue) for every chunk. Basically
|
|
178
182
|
# just reattach the raw (unvalidated) value afterwards to preserve same shape.
|
|
179
|
-
# TODO: replace this with just chunk.model_dump() if langflow's /response SSE matches openai's
|
|
183
|
+
# TODO: replace this with just chunk.model_dump() if langflow's /response SSE matches openai's
|
|
180
184
|
# SDK's delta: str schema
|
|
181
185
|
chunk_data = chunk.model_dump(exclude={"delta"})
|
|
182
186
|
if hasattr(chunk, "delta"):
|
|
@@ -190,15 +194,26 @@ async def async_response_stream(
|
|
|
190
194
|
if isinstance(chunk_data, dict):
|
|
191
195
|
# Check for any fields that might indicate tool usage
|
|
192
196
|
potential_tool_fields = {
|
|
193
|
-
k: v
|
|
194
|
-
|
|
197
|
+
k: v
|
|
198
|
+
for k, v in chunk_data.items()
|
|
199
|
+
if any(
|
|
200
|
+
keyword in str(k).lower()
|
|
201
|
+
for keyword in [
|
|
202
|
+
"tool",
|
|
203
|
+
"call",
|
|
204
|
+
"retrieval",
|
|
205
|
+
"function",
|
|
206
|
+
"result",
|
|
207
|
+
"output",
|
|
208
|
+
]
|
|
209
|
+
)
|
|
195
210
|
}
|
|
196
211
|
if potential_tool_fields:
|
|
197
212
|
logger.info(
|
|
198
213
|
"Potential tool-related fields in chunk",
|
|
199
214
|
chunk_count=chunk_count,
|
|
200
215
|
fields=list(potential_tool_fields.keys()),
|
|
201
|
-
sample_data=str(potential_tool_fields)[:500]
|
|
216
|
+
sample_data=str(potential_tool_fields)[:500],
|
|
202
217
|
)
|
|
203
218
|
|
|
204
219
|
# Detect response.completed event and log usage
|
|
@@ -217,17 +232,19 @@ async def async_response_stream(
|
|
|
217
232
|
# This helps Granite 3.3 8b and other models that don't emit standard markers
|
|
218
233
|
if isinstance(chunk_data, dict) and not detected_tool_call:
|
|
219
234
|
# Check if this chunk contains retrieval results
|
|
220
|
-
has_results = any(
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
235
|
+
has_results = any(
|
|
236
|
+
[
|
|
237
|
+
"results" in chunk_data and isinstance(chunk_data.get("results"), list),
|
|
238
|
+
"outputs" in chunk_data and isinstance(chunk_data.get("outputs"), list),
|
|
239
|
+
"retrieved_documents" in chunk_data,
|
|
240
|
+
"retrieval_results" in chunk_data,
|
|
241
|
+
]
|
|
242
|
+
)
|
|
226
243
|
|
|
227
244
|
if has_results:
|
|
228
245
|
logger.info(
|
|
229
246
|
"Detected implicit tool call in backend, injecting synthetic event",
|
|
230
|
-
chunk_fields=list(chunk_data.keys())
|
|
247
|
+
chunk_fields=list(chunk_data.keys()),
|
|
231
248
|
)
|
|
232
249
|
# Inject a synthetic tool call event before this chunk
|
|
233
250
|
synthetic_event = {
|
|
@@ -239,10 +256,12 @@ async def async_response_stream(
|
|
|
239
256
|
"tool_name": "Retrieval",
|
|
240
257
|
"status": "completed",
|
|
241
258
|
"inputs": {"implicit": True, "backend_detected": True},
|
|
242
|
-
"results": chunk_data.get(
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
259
|
+
"results": chunk_data.get("results")
|
|
260
|
+
or chunk_data.get("outputs")
|
|
261
|
+
or chunk_data.get("retrieved_documents")
|
|
262
|
+
or chunk_data.get("retrieval_results")
|
|
263
|
+
or [],
|
|
264
|
+
},
|
|
246
265
|
}
|
|
247
266
|
# Send the synthetic event first
|
|
248
267
|
yield (json.dumps(synthetic_event, default=str) + "\n").encode("utf-8")
|
|
@@ -253,16 +272,13 @@ async def async_response_stream(
|
|
|
253
272
|
# Fallback to string representation
|
|
254
273
|
logger.warning("JSON serialization failed", error=str(e))
|
|
255
274
|
yield (
|
|
256
|
-
json.dumps(
|
|
257
|
-
{"error": f"Serialization failed: {e}", "raw": str(chunk)}
|
|
258
|
-
)
|
|
259
|
-
+ "\n"
|
|
275
|
+
json.dumps({"error": f"Serialization failed: {e}", "raw": str(chunk)}) + "\n"
|
|
260
276
|
).encode("utf-8")
|
|
261
277
|
|
|
262
278
|
logger.debug("Stream complete", total_chunks=chunk_count)
|
|
263
279
|
logger.info("Response generated", log_prefix=log_prefix, response=full_response)
|
|
264
280
|
|
|
265
|
-
except Exception
|
|
281
|
+
except Exception:
|
|
266
282
|
logger.exception("[AGENT] Streaming failed")
|
|
267
283
|
raise
|
|
268
284
|
|
|
@@ -276,6 +292,11 @@ async def async_response(
|
|
|
276
292
|
previous_response_id: str = None,
|
|
277
293
|
log_prefix: str = "response",
|
|
278
294
|
):
|
|
295
|
+
# extra_headers/request_params carry raw secrets (JWT, provider API keys,
|
|
296
|
+
# x-api-key — see langflow_headers.py). Never pass them to a logger call
|
|
297
|
+
# (e.g. logger.info(..., extra_headers=extra_headers)) — that logs the
|
|
298
|
+
# secrets directly, bypassing the show_locals=False traceback protection
|
|
299
|
+
# in utils/logging_config.py.
|
|
279
300
|
try:
|
|
280
301
|
logger.info("User prompt received", prompt=prompt)
|
|
281
302
|
|
|
@@ -297,16 +318,17 @@ async def async_response(
|
|
|
297
318
|
|
|
298
319
|
response = await client.responses.create(**request_params)
|
|
299
320
|
|
|
300
|
-
|
|
301
|
-
|
|
321
|
+
try:
|
|
322
|
+
output_text = response.output_text
|
|
323
|
+
except (AttributeError, TypeError):
|
|
324
|
+
output_text = None
|
|
325
|
+
|
|
302
326
|
if output_text is not None:
|
|
303
327
|
response_text = output_text
|
|
304
328
|
logger.info("Response generated", log_prefix=log_prefix, response=response_text)
|
|
305
329
|
|
|
306
330
|
# Extract and store response_id if available
|
|
307
|
-
response_id = getattr(response, "id", None) or getattr(
|
|
308
|
-
response, "response_id", None
|
|
309
|
-
)
|
|
331
|
+
response_id = getattr(response, "id", None) or getattr(response, "response_id", None)
|
|
310
332
|
|
|
311
333
|
return response_text, response_id, response
|
|
312
334
|
else:
|
|
@@ -317,7 +339,7 @@ async def async_response(
|
|
|
317
339
|
if error_msg:
|
|
318
340
|
msg = error_msg
|
|
319
341
|
raise ValueError(msg)
|
|
320
|
-
except Exception
|
|
342
|
+
except Exception:
|
|
321
343
|
logger.exception("[AGENT] Non-streaming response failed")
|
|
322
344
|
raise
|
|
323
345
|
|
|
@@ -377,7 +399,7 @@ async def async_langflow_stream(
|
|
|
377
399
|
flow_id,
|
|
378
400
|
extra_headers=extra_headers,
|
|
379
401
|
previous_response_id=previous_response_id,
|
|
380
|
-
|
|
402
|
+
log_prefix="langflow",
|
|
381
403
|
):
|
|
382
404
|
logger.debug(
|
|
383
405
|
"Yielding chunk from langflow stream",
|
|
@@ -385,7 +407,7 @@ async def async_langflow_stream(
|
|
|
385
407
|
)
|
|
386
408
|
yield chunk
|
|
387
409
|
logger.debug("Langflow stream completed")
|
|
388
|
-
except Exception
|
|
410
|
+
except Exception:
|
|
389
411
|
logger.exception("[AGENT] Langflow stream failed")
|
|
390
412
|
raise
|
|
391
413
|
|
|
@@ -399,24 +421,18 @@ async def async_chat(
|
|
|
399
421
|
previous_response_id: str = None,
|
|
400
422
|
filter_id: str = None,
|
|
401
423
|
):
|
|
402
|
-
logger.debug(
|
|
403
|
-
"async_chat called", user_id=user_id, previous_response_id=previous_response_id
|
|
404
|
-
)
|
|
424
|
+
logger.debug("async_chat called", user_id=user_id, previous_response_id=previous_response_id)
|
|
405
425
|
|
|
406
426
|
# Get the specific conversation thread (or create new one)
|
|
407
427
|
conversation_state = get_conversation_thread(user_id, previous_response_id)
|
|
408
|
-
logger.debug(
|
|
409
|
-
"Got conversation state", message_count=len(conversation_state["messages"])
|
|
410
|
-
)
|
|
428
|
+
logger.debug("Got conversation state", message_count=len(conversation_state["messages"]))
|
|
411
429
|
|
|
412
430
|
# Add user message to conversation with timestamp
|
|
413
431
|
from datetime import datetime
|
|
414
432
|
|
|
415
433
|
user_message = {"role": "user", "content": prompt, "timestamp": datetime.now()}
|
|
416
434
|
conversation_state["messages"].append(user_message)
|
|
417
|
-
logger.debug(
|
|
418
|
-
"Added user message", message_count=len(conversation_state["messages"])
|
|
419
|
-
)
|
|
435
|
+
logger.debug("Added user message", message_count=len(conversation_state["messages"]))
|
|
420
436
|
|
|
421
437
|
# Store filter_id in conversation state if provided
|
|
422
438
|
if filter_id:
|
|
@@ -429,9 +445,7 @@ async def async_chat(
|
|
|
429
445
|
previous_response_id=previous_response_id,
|
|
430
446
|
log_prefix="agent",
|
|
431
447
|
)
|
|
432
|
-
logger.debug(
|
|
433
|
-
"Got response", response_preview=response_text[:50], response_id=response_id
|
|
434
|
-
)
|
|
448
|
+
logger.debug("Got response", response_preview=response_text[:50], response_id=response_id)
|
|
435
449
|
|
|
436
450
|
# Add assistant response to conversation with response_id, timestamp, and full response object
|
|
437
451
|
assistant_message = {
|
|
@@ -444,20 +458,16 @@ async def async_chat(
|
|
|
444
458
|
else str(response_obj), # Store complete response for function calls
|
|
445
459
|
}
|
|
446
460
|
conversation_state["messages"].append(assistant_message)
|
|
447
|
-
logger.debug(
|
|
448
|
-
"Added assistant message", message_count=len(conversation_state["messages"])
|
|
449
|
-
)
|
|
461
|
+
logger.debug("Added assistant message", message_count=len(conversation_state["messages"]))
|
|
450
462
|
|
|
451
463
|
# Store the conversation thread with its response_id
|
|
452
464
|
if response_id:
|
|
453
465
|
conversation_state["last_activity"] = datetime.now()
|
|
454
466
|
await store_conversation_thread(user_id, response_id, conversation_state)
|
|
455
|
-
logger.debug(
|
|
456
|
-
"Stored conversation thread", user_id=user_id, response_id=response_id
|
|
457
|
-
)
|
|
467
|
+
logger.debug("Stored conversation thread", user_id=user_id, response_id=response_id)
|
|
458
468
|
|
|
459
469
|
# Debug: Check what's in user_conversations now
|
|
460
|
-
conversations = get_user_conversations(user_id)
|
|
470
|
+
conversations = await get_user_conversations(user_id)
|
|
461
471
|
logger.debug(
|
|
462
472
|
"User conversations updated",
|
|
463
473
|
user_id=user_id,
|
|
@@ -518,7 +528,7 @@ async def async_chat_stream(
|
|
|
518
528
|
if chunk_data.get("type") == "response.completed":
|
|
519
529
|
response_obj = chunk_data.get("response", {})
|
|
520
530
|
usage_data = response_obj.get("usage")
|
|
521
|
-
except:
|
|
531
|
+
except Exception:
|
|
522
532
|
pass
|
|
523
533
|
yield chunk
|
|
524
534
|
|
|
@@ -634,13 +644,21 @@ async def async_langflow_chat(
|
|
|
634
644
|
else (result if isinstance(result, dict) else {})
|
|
635
645
|
)
|
|
636
646
|
if "text" in rd:
|
|
637
|
-
sources.append(
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
647
|
+
sources.append(
|
|
648
|
+
{
|
|
649
|
+
"filename": rd.get("filename", ""),
|
|
650
|
+
"text": rd.get("text", ""),
|
|
651
|
+
"score": rd.get("score", 0),
|
|
652
|
+
"page": rd.get("page"),
|
|
653
|
+
"mimetype": rd.get("mimetype"),
|
|
654
|
+
"chunk_id": rd.get("chunk_id") or rd.get("id") or "",
|
|
655
|
+
"id": rd.get("id") or rd.get("chunk_id") or "",
|
|
656
|
+
"embedding_model": rd.get("embedding_model"),
|
|
657
|
+
"parser": rd.get("parser"),
|
|
658
|
+
"chunk_size": rd.get("chunk_size"),
|
|
659
|
+
"chunk_overlap": rd.get("chunk_overlap"),
|
|
660
|
+
}
|
|
661
|
+
)
|
|
644
662
|
|
|
645
663
|
# Layer 2: Top-level dict inspection (mirrors streaming middleware in async_response_stream).
|
|
646
664
|
# Langflow may embed retrieval results directly in the response dict rather than
|
|
@@ -661,13 +679,21 @@ async def async_langflow_chat(
|
|
|
661
679
|
if isinstance(implicit_results, list):
|
|
662
680
|
for result in implicit_results:
|
|
663
681
|
if isinstance(result, dict) and "text" in result:
|
|
664
|
-
sources.append(
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
682
|
+
sources.append(
|
|
683
|
+
{
|
|
684
|
+
"filename": result.get("filename", ""),
|
|
685
|
+
"text": result.get("text", ""),
|
|
686
|
+
"score": result.get("score", 0),
|
|
687
|
+
"page": result.get("page"),
|
|
688
|
+
"mimetype": result.get("mimetype"),
|
|
689
|
+
"chunk_id": result.get("chunk_id") or result.get("id") or "",
|
|
690
|
+
"id": result.get("id") or result.get("chunk_id") or "",
|
|
691
|
+
"embedding_model": result.get("embedding_model"),
|
|
692
|
+
"parser": result.get("parser"),
|
|
693
|
+
"chunk_size": result.get("chunk_size"),
|
|
694
|
+
"chunk_overlap": result.get("chunk_overlap"),
|
|
695
|
+
}
|
|
696
|
+
)
|
|
671
697
|
|
|
672
698
|
# Layer 3: Citation-text fallback.
|
|
673
699
|
# Parse "(Source: filename)" patterns emitted by the LLM when it cites documents.
|
|
@@ -675,14 +701,17 @@ async def async_langflow_chat(
|
|
|
675
701
|
# structured retrieval data.
|
|
676
702
|
if not sources:
|
|
677
703
|
import re
|
|
704
|
+
|
|
678
705
|
for match in re.finditer(r"\(Source:\s*([^\)]+)\)", response_text):
|
|
679
|
-
sources.append(
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
706
|
+
sources.append(
|
|
707
|
+
{
|
|
708
|
+
"filename": match.group(1).strip(),
|
|
709
|
+
"text": "",
|
|
710
|
+
"score": 0,
|
|
711
|
+
"page": None,
|
|
712
|
+
"mimetype": None,
|
|
713
|
+
}
|
|
714
|
+
)
|
|
686
715
|
|
|
687
716
|
if not store_conversation:
|
|
688
717
|
return response_text, response_id, sources
|
|
@@ -696,7 +725,7 @@ async def async_langflow_chat(
|
|
|
696
725
|
try:
|
|
697
726
|
from services.session_ownership_service import session_ownership_service
|
|
698
727
|
|
|
699
|
-
session_ownership_service.claim_session(user_id, response_id)
|
|
728
|
+
await session_ownership_service.claim_session(user_id, response_id)
|
|
700
729
|
logger.debug(f"Claimed session {response_id} for user {user_id}")
|
|
701
730
|
except Exception as e:
|
|
702
731
|
logger.warning(f"Failed to claim session ownership: {e}")
|
|
@@ -711,7 +740,7 @@ async def async_langflow_chat(
|
|
|
711
740
|
)
|
|
712
741
|
|
|
713
742
|
# Debug: Check what's in user_conversations now
|
|
714
|
-
conversations = get_user_conversations(user_id)
|
|
743
|
+
conversations = await get_user_conversations(user_id)
|
|
715
744
|
logger.debug(
|
|
716
745
|
"User conversations updated",
|
|
717
746
|
user_id=user_id,
|
|
@@ -784,44 +813,46 @@ async def async_langflow_chat_stream(
|
|
|
784
813
|
response_id = chunk_data["response_id"]
|
|
785
814
|
|
|
786
815
|
# Check for error status
|
|
787
|
-
if
|
|
816
|
+
if (
|
|
817
|
+
chunk_data.get("finish_reason") == "error"
|
|
818
|
+
or chunk_data.get("status") == "failed"
|
|
819
|
+
):
|
|
788
820
|
error_occurred = True
|
|
789
821
|
logger.error("Error detected in Langflow stream chunk")
|
|
790
822
|
# Capture usage from response.completed event
|
|
791
823
|
if chunk_data.get("type") == "response.completed":
|
|
792
824
|
response_obj = chunk_data.get("response", {})
|
|
793
825
|
usage_data = response_obj.get("usage")
|
|
794
|
-
except:
|
|
826
|
+
except Exception:
|
|
795
827
|
pass
|
|
796
828
|
yield chunk
|
|
797
829
|
|
|
798
830
|
# Add the complete assistant response to message history with response_id, timestamp, and function call data
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
831
|
+
if full_response:
|
|
832
|
+
assistant_message = {
|
|
833
|
+
"role": "assistant",
|
|
834
|
+
"content": full_response,
|
|
835
|
+
"response_id": response_id,
|
|
836
|
+
"timestamp": datetime.now(),
|
|
837
|
+
"chunks": collected_chunks,
|
|
838
|
+
"error": error_occurred,
|
|
839
|
+
}
|
|
840
|
+
if usage_data:
|
|
841
|
+
assistant_message["response_data"] = {"usage": usage_data}
|
|
842
|
+
conversation_state["messages"].append(assistant_message)
|
|
811
843
|
|
|
812
844
|
# Store the conversation thread with its response_id
|
|
813
845
|
if response_id:
|
|
814
846
|
conversation_state["last_activity"] = datetime.now()
|
|
815
847
|
await store_conversation_thread(user_id, response_id, conversation_state)
|
|
816
848
|
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
from services.session_ownership_service import session_ownership_service
|
|
849
|
+
try:
|
|
850
|
+
from services.session_ownership_service import session_ownership_service
|
|
820
851
|
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
852
|
+
await session_ownership_service.claim_session(user_id, response_id)
|
|
853
|
+
logger.debug(f"Claimed session {response_id} for user {user_id}")
|
|
854
|
+
except Exception as e:
|
|
855
|
+
logger.warning(f"Failed to claim session ownership: {e}")
|
|
825
856
|
|
|
826
857
|
logger.debug(
|
|
827
858
|
f"Stored langflow conversation thread for user {user_id} with response_id: {response_id}"
|
|
@@ -876,15 +907,20 @@ async def delete_user_conversation(user_id: str, response_id: str) -> bool:
|
|
|
876
907
|
deleted = True
|
|
877
908
|
|
|
878
909
|
# Delete from persistent storage — let real errors propagate to the caller
|
|
879
|
-
conversation_deleted = await conversation_persistence.delete_conversation_thread(
|
|
910
|
+
conversation_deleted = await conversation_persistence.delete_conversation_thread(
|
|
911
|
+
user_id, response_id
|
|
912
|
+
)
|
|
880
913
|
if conversation_deleted:
|
|
881
|
-
logger.debug(
|
|
914
|
+
logger.debug(
|
|
915
|
+
f"Deleted conversation {response_id} from persistent storage for user {user_id}"
|
|
916
|
+
)
|
|
882
917
|
deleted = True
|
|
883
918
|
|
|
884
919
|
# Release session ownership (best-effort; never masks storage errors above)
|
|
885
920
|
try:
|
|
886
921
|
from services.session_ownership_service import session_ownership_service
|
|
887
|
-
|
|
922
|
+
|
|
923
|
+
await session_ownership_service.release_session(user_id, response_id)
|
|
888
924
|
logger.debug(f"Released session ownership for {response_id} for user {user_id}")
|
|
889
925
|
except Exception as e:
|
|
890
926
|
logger.warning(f"Failed to release session ownership: {e}")
|
api/acl.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from fastapi import Depends
|
|
2
|
+
from fastapi.responses import JSONResponse
|
|
3
|
+
|
|
4
|
+
from config.settings import get_index_name
|
|
5
|
+
from dependencies import get_current_user, get_session_manager
|
|
6
|
+
from session_manager import User
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
async def get_document_acl(
|
|
10
|
+
filename: str,
|
|
11
|
+
session_manager=Depends(get_session_manager),
|
|
12
|
+
user: User = Depends(get_current_user),
|
|
13
|
+
):
|
|
14
|
+
"""GET /documents/acl?filename=... — return ACL for a document."""
|
|
15
|
+
opensearch_client = session_manager.get_user_opensearch_client(user.user_id, user.jwt_token)
|
|
16
|
+
response = await opensearch_client.search(
|
|
17
|
+
index=get_index_name(),
|
|
18
|
+
body={
|
|
19
|
+
"query": {"term": {"filename": filename}},
|
|
20
|
+
"size": 1,
|
|
21
|
+
"_source": [
|
|
22
|
+
"owner",
|
|
23
|
+
"allowed_users",
|
|
24
|
+
"allowed_groups",
|
|
25
|
+
"allowed_principal_labels",
|
|
26
|
+
],
|
|
27
|
+
},
|
|
28
|
+
)
|
|
29
|
+
hits = response.get("hits", {}).get("hits", [])
|
|
30
|
+
if not hits:
|
|
31
|
+
return JSONResponse(
|
|
32
|
+
{"error": f"Document '{filename}' not found"},
|
|
33
|
+
status_code=404,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
source = hits[0]["_source"]
|
|
37
|
+
return JSONResponse(
|
|
38
|
+
{
|
|
39
|
+
"owner": source.get("owner"),
|
|
40
|
+
"allowed_users": source.get("allowed_users", []),
|
|
41
|
+
"allowed_groups": source.get("allowed_groups", []),
|
|
42
|
+
"allowed_principal_labels": source.get("allowed_principal_labels", []),
|
|
43
|
+
}
|
|
44
|
+
)
|