OpenRAG 0.3.4__tar.gz → 0.4.0__tar.gz

{openrag-0.3.4/src/openrag.egg-info → openrag-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrag
-Version: 0.3.4
+Version: 0.4.0
 Summary: OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations.
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
@@ -28,6 +28,7 @@ Requires-Dist: python-multipart>=0.0.20
 Requires-Dist: fastapi>=0.115.0
 Requires-Dist: uvicorn>=0.35.0
 Requires-Dist: boto3>=1.35.0
+Requires-Dist: ibm-cos-sdk>=2.13.0
 Requires-Dist: psutil>=7.0.0
 Requires-Dist: rich>=13.0.0
 Requires-Dist: textual>=0.45.0

{openrag-0.3.4 → openrag-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "openrag"
-version = "0.3.4"
+version = "0.4.0"
 description = "OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -34,6 +34,7 @@ dependencies = [
     "fastapi>=0.115.0",
     "uvicorn>=0.35.0",
     "boto3>=1.35.0",
+    "ibm-cos-sdk>=2.13.0",
     "psutil>=7.0.0",
     "rich>=13.0.0",
     "textual>=0.45.0",
@@ -43,7 +44,7 @@ dependencies = [
     "zxcvbn>=4.5.0",
     "openai>=1.0.0",
     "pyyaml>=6.0",
-    "tiktoken>=0.7.0"
+    "tiktoken>=0.7.0",
 ]
 
 [dependency-groups]

{openrag-0.3.4 → openrag-0.4.0}/src/agent.py

@@ -815,31 +815,36 @@ async def async_langflow_chat_stream(
 
 
 async def delete_user_conversation(user_id: str, response_id: str) -> bool:
-    """Delete a conversation for a user from both memory and persistent storage (async, non-blocking)"""
+    """Delete a conversation for a user from both memory and persistent storage.
+
+    Returns:
+        True  — conversation was found and deleted from at least one store.
+        False — conversation did not exist in any store (confirmed not-found).
+
+    Raises:
+        Exception — on unexpected storage errors so callers can distinguish
+                    a confirmed "not found" from a backend failure.
+    """
     deleted = False
 
-    try:
-        # Delete from in-memory storage
-        if user_id in active_conversations and response_id in active_conversations[user_id]:
-            del active_conversations[user_id][response_id]
-            logger.debug(f"Deleted conversation {response_id} from memory for user {user_id}")
-            deleted = True
-
-        # Delete from persistent storage
-        conversation_deleted = await conversation_persistence.delete_conversation_thread(user_id, response_id)
-        if conversation_deleted:
-            logger.debug(f"Deleted conversation {response_id} from persistent storage for user {user_id}")
-            deleted = True
-
-        # Release session ownership
-        try:
-            from services.session_ownership_service import session_ownership_service
-            session_ownership_service.release_session(user_id, response_id)
-            logger.debug(f"Released session ownership for {response_id} for user {user_id}")
-        except Exception as e:
-            logger.warning(f"Failed to release session ownership: {e}")
+    # Delete from in-memory storage (cannot raise)
+    if user_id in active_conversations and response_id in active_conversations[user_id]:
+        del active_conversations[user_id][response_id]
+        logger.debug(f"Deleted conversation {response_id} from memory for user {user_id}")
+        deleted = True
+
+    # Delete from persistent storage — let real errors propagate to the caller
+    conversation_deleted = await conversation_persistence.delete_conversation_thread(user_id, response_id)
+    if conversation_deleted:
+        logger.debug(f"Deleted conversation {response_id} from persistent storage for user {user_id}")
+        deleted = True
 
-        return deleted
+    # Release session ownership (best-effort; never masks storage errors above)
+    try:
+        from services.session_ownership_service import session_ownership_service
+        session_ownership_service.release_session(user_id, response_id)
+        logger.debug(f"Released session ownership for {response_id} for user {user_id}")
     except Exception as e:
-        logger.error(f"Error deleting conversation {response_id} for user {user_id}: {e}")
-        return False
+        logger.warning(f"Failed to release session ownership: {e}")
+
+    return deleted

{openrag-0.3.4 → openrag-0.4.0}/src/api/connectors.py

@@ -88,6 +88,11 @@ async def get_synced_file_ids_for_connector(
 class ConnectorSyncBody(BaseModel):
     max_files: Optional[int] = None
     selected_files: Optional[List[Any]] = None
+    # When True, ingest ALL files from the connector (bypasses the existing-files gate).
+    # Used by direct-sync providers like IBM COS on initial ingest.
+    sync_all: bool = False
+    # When set, only ingest files from these buckets (IBM COS specific).
+    bucket_filter: Optional[List[str]] = None
 
 
 async def list_connectors(
@@ -96,8 +101,8 @@ async def list_connectors(
 ):
     """List available connector types with metadata"""
     try:
-        connector_types = (
-            connector_service.connection_manager.get_available_connector_types()
+        connector_types = connector_service.connection_manager.get_available_connector_types(
+            user_id=user.user_id
         )
         return JSONResponse({"connectors": connector_types})
     except Exception as e:
@@ -200,6 +205,51 @@ async def connector_sync(
                 jwt_token=jwt_token,
                 file_infos=file_infos,
             )
+        elif body.sync_all or body.bucket_filter:
+            # Full ingest: discover and ingest all files (or files from specific buckets).
+            # Used by direct-sync providers (IBM COS) on initial ingest or per-bucket sync.
+            logger.info(
+                "Full connector ingest requested",
+                connector_type=connector_type,
+                bucket_filter=body.bucket_filter,
+            )
+            connector = await connector_service.get_connector(working_connection.connection_id)
+            if body.bucket_filter:
+                # List only files from the requested buckets, then sync_specific_files
+                original_buckets = connector.bucket_names
+                connector.bucket_names = body.bucket_filter
+                try:
+                    all_file_ids = []
+                    page_token = None
+                    while True:
+                        result = await connector.list_files(page_token=page_token)
+                        for f in result.get("files", []):
+                            all_file_ids.append(f["id"])
+                        page_token = result.get("next_page_token")
+                        if not page_token:
+                            break
+                finally:
+                    connector.bucket_names = original_buckets
+
+                if not all_file_ids:
+                    return JSONResponse(
+                        {"status": "no_files", "message": "No files found in the selected buckets."},
+                        status_code=200,
+                    )
+                task_id = await connector_service.sync_specific_files(
+                    working_connection.connection_id,
+                    user.user_id,
+                    all_file_ids,
+                    jwt_token=jwt_token,
+                )
+            else:
+                # sync_all: ingest everything the connector can see
+                task_id = await connector_service.sync_connector_files(
+                    working_connection.connection_id,
+                    user.user_id,
+                    max_files=max_files,
+                    jwt_token=jwt_token,
+                )
         else:
             # No files specified - sync only files already in OpenSearch for this connector
             # This ensures deleted files stay deleted
@@ -209,7 +259,7 @@ async def connector_sync(
                 session_manager=session_manager,
                 jwt_token=jwt_token,
             )
-            
+
             if not existing_file_ids and not existing_filenames:
                 return JSONResponse(
                     {
@@ -218,7 +268,7 @@ async def connector_sync(
                     },
                     status_code=200,
                 )
-            
+
             # If we have document_ids (connector file IDs), use sync_specific_files
             # Otherwise, use filename filtering with sync_connector_files
             if existing_file_ids:
@@ -602,6 +652,8 @@ async def connector_disconnect(
         )
 
 
+# ---------------------------------------------------------------------------
+
 async def sync_all_connectors(
     connector_service=Depends(get_connector_service),
     session_manager=Depends(get_session_manager),
@@ -615,7 +667,7 @@ async def sync_all_connectors(
         jwt_token = user.jwt_token
 
         # Cloud connector types to sync
-        cloud_connector_types = ["google_drive", "onedrive", "sharepoint"]
+        cloud_connector_types = ["google_drive", "onedrive", "sharepoint", "ibm_cos", "aws_s3"]
         
         all_task_ids = []
         synced_connectors = []

{openrag-0.3.4 → openrag-0.4.0}/src/api/documents.py

@@ -121,16 +121,26 @@ async def check_filename_exists(
         )
 
         from utils.opensearch_queries import build_filename_search_body
+        from utils.file_utils import get_filename_aliases
 
-        search_body = build_filename_search_body(filename, size=1, source=["filename"])
+        candidate_filenames = get_filename_aliases(filename)
+        if not candidate_filenames:
+            return JSONResponse({"exists": False, "filename": filename}, status_code=200)
 
         logger.debug("Checking filename existence", filename=filename, index_name=get_index_name())
+        exists = False
 
         try:
-            response = await opensearch_client.search(
-                index=get_index_name(),
-                body=search_body
-            )
+            for candidate in candidate_filenames:
+                search_body = build_filename_search_body(candidate, size=1, source=["filename"])
+                response = await opensearch_client.search(
+                    index=get_index_name(),
+                    body=search_body
+                )
+                hits = response.get("hits", {}).get("hits", [])
+                if hits:
+                    exists = True
+                    break
         except Exception as search_err:
             if "index_not_found_exception" in str(search_err):
                 logger.info("Index does not exist, creating it now before upload")
@@ -138,9 +148,6 @@ async def check_filename_exists(
                 return JSONResponse({"exists": False, "filename": filename}, status_code=200)
             raise
 
-        hits = response.get("hits", {}).get("hits", [])
-        exists = len(hits) > 0
-
         return JSONResponse({"exists": exists, "filename": filename}, status_code=200)
 
     except Exception as e:

{openrag-0.3.4 → openrag-0.4.0}/src/api/settings.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import platform
 from fastapi import Depends, Request, HTTPException
@@ -36,6 +37,7 @@ from dependencies import (
 from session_manager import User
 
 logger = get_logger(__name__)
+_background_tasks: set[asyncio.Task] = set()
 
 
 class SettingsUpdateBody(BaseModel):
@@ -209,6 +211,9 @@ class RollbackResponse(BaseModel):
     cancelled_tasks: int
     deleted_files: int
 
+class RollbackBody(BaseModel):
+    embedding_only: bool = False
+
 
 # Docling preset configurations
 def get_docling_preset_configs(
@@ -605,6 +610,12 @@ async def update_settings(
                 logger.error(f"Failed to update docling settings in flow: {str(e)}")
 
         if body.chunk_size is not None:
+            effective_overlap = body.chunk_overlap if body.chunk_overlap is not None else current_config.knowledge.chunk_overlap
+            if effective_overlap >= body.chunk_size:
+                raise HTTPException(
+                    status_code=422,
+                    detail="chunk_overlap must be less than chunk_size"
+                )
             current_config.knowledge.chunk_size = body.chunk_size
             config_updated = True
             await TelemetryClient.send_event(
@@ -625,6 +636,12 @@ async def update_settings(
                 # The config will still be saved
 
         if body.chunk_overlap is not None:
+            effective_chunk_size = body.chunk_size if body.chunk_size is not None else current_config.knowledge.chunk_size
+            if body.chunk_overlap >= effective_chunk_size:
+                raise HTTPException(
+                    status_code=422,
+                    detail="chunk_overlap must be less than chunk_size"
+                )
             current_config.knowledge.chunk_overlap = body.chunk_overlap
             config_updated = True
             await TelemetryClient.send_event(
@@ -815,30 +832,31 @@ async def update_settings(
                 {"error": "Failed to save configuration"}, status_code=500
             )
 
-        # Update Langflow global variables and model values if provider settings changed
+        # Refresh patched client immediately so subsequent requests pick up latest config.
         await clients.refresh_patched_client()
 
+        # Run expensive Langflow sync in the background to keep settings updates responsive.
         if should_validate or provider_updated:
-            try:
-                flows_service = _get_flows_service()
-
-                # Update global variables
-                await _update_langflow_global_variables(current_config, flows_service=flows_service)
-
-                # Update LLM client credentials when embedding selection changes
-                if body.embedding_provider is not None or body.embedding_model is not None:
-                    await _update_mcp_servers_with_provider_credentials(
-                        current_config, session_manager, flows_service=flows_service
-                    )
-
-                # Update model values if provider or model changed (including removals that trigger fallback)
-                if body.llm_provider is not None or body.llm_model is not None or body.embedding_provider is not None or body.embedding_model is not None or provider_updated:
-                    await _update_langflow_model_values(current_config, flows_service)
-
-            except Exception as e:
-                logger.error(f"Failed to update Langflow settings: {str(e)}")
-                # Don't fail the entire settings update if Langflow update fails
-                # The config was still saved
+            task = asyncio.create_task(
+                _run_async_post_save_langflow_updates(
+                    session_manager=session_manager,
+                    update_mcp_servers=(
+                        body.embedding_provider is not None
+                        or body.embedding_model is not None
+                        or provider_updated
+                    ),
+                    update_model_values=(
+                        body.llm_provider is not None
+                        or body.llm_model is not None
+                        or body.embedding_provider is not None
+                        or body.embedding_model is not None
+                        or provider_updated
+                    ),
+                )
+            )
+            # Keep a strong reference until completion to avoid premature GC cancellation.
+            _background_tasks.add(task)
+            task.add_done_callback(_background_tasks.discard)
 
 
         set_fields = [k for k, v in body.model_dump().items() if v is not None]
@@ -1380,6 +1398,37 @@ async def _update_langflow_global_variables(config, flows_service=None):
         raise
 
 
+async def _run_async_post_save_langflow_updates(
+    session_manager,
+    update_mcp_servers: bool,
+    update_model_values: bool,
+) -> None:
+    """Apply post-save Langflow synchronization asynchronously."""
+    try:
+        current_config = get_openrag_config()
+        flows_service = _get_flows_service()
+
+        # Update global variables
+        await _update_langflow_global_variables(
+            current_config, flows_service=flows_service
+        )
+
+        # Update LLM client credentials when embedding selection changes
+        if update_mcp_servers:
+            await _update_mcp_servers_with_provider_credentials(
+                current_config, session_manager, flows_service=flows_service
+            )
+
+        # Update model values if provider/model changed (including removals/fallbacks)
+        if update_model_values:
+            await _update_langflow_model_values(current_config, flows_service)
+
+        logger.info("Completed asynchronous Langflow post-save sync")
+    except Exception as e:
+        # Do not fail user request if async sync fails; keep parity with existing behavior.
+        logger.error(f"Failed to update Langflow settings asynchronously: {str(e)}")
+
+
 async def _update_mcp_servers_with_provider_credentials(config, session_manager = None, flows_service=None):
     # Update MCP servers with provider credentials
     try:
@@ -1592,8 +1641,10 @@ async def reapply_all_settings(session_manager = None):
 
 async def rollback_onboarding(
     request: Request,
+    body: Optional[RollbackBody] = None,
     session_manager=Depends(get_session_manager),
     task_service=Depends(get_task_service),
+    knowledge_filter_service=Depends(get_knowledge_filter_service),
     user: User = Depends(get_current_user),
 ) -> RollbackResponse:
     """Rollback onboarding configuration when sample data files fail.
@@ -1623,7 +1674,36 @@ async def rollback_onboarding(
         cancelled_tasks = []
         deleted_files = []
 
+        # Delete knowledge filters created during onboarding
+        try:
+            async def remove_filter(filter_id: Optional[str]):
+                if filter_id and knowledge_filter_service:
+                    try:
+                        result = await knowledge_filter_service.delete_knowledge_filter(
+                            filter_id, user.user_id, user.jwt_token
+                        )
+                        if result and result.get("success"):
+                            logger.info(f"Deleted knowledge filter {filter_id}")
+                        else:
+                            error_msg = result.get("error") if result else "Unknown error"
+                            logger.warning(f"Could not delete knowledge filter {filter_id}: {error_msg}")
+                    except Exception as e:
+                        logger.warning(f"Exception deleting knowledge filter {filter_id}: {str(e)}")
+
+            if getattr(current_config.onboarding, 'openrag_docs_filter_id', None):
+                await remove_filter(current_config.onboarding.openrag_docs_filter_id)
+                current_config.onboarding.openrag_docs_filter_id = None
+                
+            if getattr(current_config.onboarding, 'user_doc_filter_id', None):
+                await remove_filter(current_config.onboarding.user_doc_filter_id)
+                current_config.onboarding.user_doc_filter_id = None
+        except Exception as e:
+            logger.error(f"Error while cleaning up knowledge filters: {e}")
+
         # Cancel all active tasks and collect successfully ingested files
+        from session_manager import AnonymousUser
+        anonymous_user_id = AnonymousUser().user_id
+
         for task_data in all_tasks:
             task_id = task_data.get("task_id")
             task_status = task_data.get("status")
@@ -1638,41 +1718,40 @@ async def rollback_onboarding(
                 except Exception as e:
                     logger.error(f"Failed to cancel task {task_id}: {str(e)}")
 
-            # For completed tasks, find successfully ingested files and delete them
-            elif task_status == "completed":
-                files = task_data.get("files", {})
-                if isinstance(files, dict):
-                    for file_path, file_info in files.items():
-                        # Check if file was successfully ingested
-                        if isinstance(file_info, dict):
-                            file_status = file_info.get("status")
-                            filename = file_info.get("filename") or file_path.split("/")[-1]
-
-                            if file_status == "completed" and filename:
-                                try:
-                                    # Get user's OpenSearch client
-                                    opensearch_client = session_manager.get_user_opensearch_client(
-                                        user.user_id, jwt_token
-                                    )
-
-                                    # Delete documents by filename
-                                    from utils.opensearch_queries import build_filename_delete_body
-                                    from config.settings import get_index_name
-
-                                    delete_query = build_filename_delete_body(filename)
-
-                                    result = await opensearch_client.delete_by_query(
-                                        index=get_index_name(),
-                                        body=delete_query,
-                                        conflicts="proceed"
-                                    )
-
-                                    deleted_count = result.get("deleted", 0)
-                                    if deleted_count > 0:
-                                        deleted_files.append(filename)
-                                        logger.info(f"Deleted {deleted_count} chunks for filename {filename}")
-                                except Exception as e:
-                                    logger.error(f"Failed to delete documents for {filename}: {str(e)}")
+            # Delete all files associated with any task, regardless of whether 
+            # the task failed or completed, to ensure no partial chunks remain in OpenSearch.
+            files = task_data.get("files", {})
+            if isinstance(files, dict):
+                for file_path, file_info in files.items():
+                    if isinstance(file_info, dict):
+                        filename = file_info.get("filename") or file_path.split("/")[-1]
+                        if filename:
+                            try:
+                                opensearch_client = session_manager.get_user_opensearch_client(
+                                    user.user_id, jwt_token
+                                )
+                                from utils.opensearch_queries import build_filename_delete_body
+                                from config.settings import get_index_name
+
+                                delete_query = build_filename_delete_body(filename)
+                                result = await opensearch_client.delete_by_query(
+                                    index=get_index_name(),
+                                    body=delete_query,
+                                    conflicts="proceed"
+                                )
+                                deleted_count = result.get("deleted", 0)
+                                if deleted_count > 0:
+                                    deleted_files.append(filename)
+                                    logger.info(f"Deleted {deleted_count} chunks for filename {filename}")
+                            except Exception as e:
+                                logger.error(f"Failed to delete documents for {filename}: {str(e)}")
+
+            # Wipe the task completely from memory so the frontend doesn't see it anymore
+            for check_user_id in [user.user_id, anonymous_user_id]:
+                if check_user_id in task_service.task_store and task_id in task_service.task_store[check_user_id]:
+                    task_service._task_locks.pop(task_id, None)
+                    task_service.task_store[check_user_id].pop(task_id, None)
+                    logger.info(f"Purged task {task_id} completely from task_store for user {check_user_id}")
 
         # Clear embedding provider and model settings
         current_config.knowledge.embedding_provider = "openai"  # Reset to default
@@ -1680,11 +1759,21 @@ async def rollback_onboarding(
         current_config.onboarding.openrag_docs_ingested_version = None
         current_config.onboarding.openrag_docs_remote_signature = None
 
+        embedding_only = body.embedding_only if body else False
+
         # Mark config as not edited so user can go through onboarding again
-        current_config.edited = False
-        current_config.onboarding.current_step = 0
+        if not embedding_only:
+            current_config.edited = False
+            current_config.onboarding.current_step = 0
+            # Also clear LLM provider and model settings when doing a full rollback
+            current_config.agent.llm_provider = "openai"  # Reset to default
+            current_config.agent.llm_model = ""
+        else:
+            # When rolling back embedding only, we keep edited=True
+            # and set current_step to 1 (which is the embedding step)
+            current_config.onboarding.current_step = 1
 
-        # Save the rolled back configuration manually to avoid save_config_file setting edited=True
+        # Save the rolled back configuration manually
         try:
             import yaml
             config_file = config_manager.config_file
@@ -1692,14 +1781,14 @@ async def rollback_onboarding(
             # Ensure directory exists
             config_file.parent.mkdir(parents=True, exist_ok=True)
 
-            # Save config with edited=False
+            # Save config with current edited state
             with open(config_file, "w") as f:
                 yaml.dump(current_config.to_dict(), f, default_flow_style=False, indent=2)
 
             # Update cached config
             config_manager._config = current_config
 
-            logger.info("Successfully saved rolled back configuration with edited=False")
+            logger.info(f"Successfully saved rolled back configuration with edited={current_config.edited}")
         except Exception as e:
             logger.error(f"Failed to save rolled back configuration: {e}")
             return JSONResponse(

{openrag-0.3.4 → openrag-0.4.0}/src/api/v1/chat.py

@@ -226,6 +226,8 @@ async def chat_delete_endpoint(
     """Delete a conversation. DELETE /v1/chat/{chat_id}"""
     try:
         result = await chat_service.delete_session(user.user_id, chat_id)
+        if result.get("not_found"):
+            return JSONResponse({"error": "Conversation not found"}, status_code=404)
         if result.get("success"):
             return JSONResponse({"success": True})
         else:

{openrag-0.3.4 → openrag-0.4.0}/src/config/settings.py

@@ -50,6 +50,8 @@ GOOGLE_OAUTH_CLIENT_ID = os.getenv("GOOGLE_OAUTH_CLIENT_ID")
 GOOGLE_OAUTH_CLIENT_SECRET = os.getenv("GOOGLE_OAUTH_CLIENT_SECRET")
 DOCLING_OCR_ENGINE = os.getenv("DOCLING_OCR_ENGINE")
 
+IBM_AUTH_ENABLED = os.getenv("IBM_AUTH_ENABLED", "false").lower() in ("true", "1", "yes")
+
 # Ingestion configuration
 DISABLE_INGEST_WITH_LANGFLOW = os.getenv(
     "DISABLE_INGEST_WITH_LANGFLOW", "false"

{openrag-0.3.4 → openrag-0.4.0}/src/connectors/__init__.py

@@ -2,10 +2,14 @@ from .base import BaseConnector
 from .google_drive import GoogleDriveConnector
 from .sharepoint import SharePointConnector
 from .onedrive import OneDriveConnector
+from .ibm_cos import IBMCOSConnector
+from .aws_s3 import S3Connector
 
 __all__ = [
     "BaseConnector",
     "GoogleDriveConnector",
     "SharePointConnector",
     "OneDriveConnector",
+    "IBMCOSConnector",
+    "S3Connector",
 ]

openrag-0.4.0/src/connectors/aws_s3/__init__.py

@@ -0,0 +1,19 @@
+"""Amazon S3 / S3-compatible connector for OpenRAG."""
+
+from .connector import S3Connector
+from .models import S3ConfigureBody
+from .api import (
+    s3_defaults,
+    s3_configure,
+    s3_list_buckets,
+    s3_bucket_status,
+)
+
+__all__ = [
+    "S3Connector",
+    "S3ConfigureBody",
+    "s3_defaults",
+    "s3_configure",
+    "s3_list_buckets",
+    "s3_bucket_status",
+]