openrag 0.4.0.dev6__tar.gz → 0.4.0.dev8__tar.gz

{openrag-0.4.0.dev6/src/openrag.egg-info → openrag-0.4.0.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrag
-Version: 0.4.0.dev6
+Version: 0.4.0.dev8
 Summary: OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations.
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console

{openrag-0.4.0.dev6 → openrag-0.4.0.dev8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "openrag"
-version = "0.4.0.dev6"
+version = "0.4.0.dev8"
 description = "OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations."
 readme = "README.md"
 requires-python = ">=3.13"

{openrag-0.4.0.dev6 → openrag-0.4.0.dev8}/src/api/settings.py

@@ -211,6 +211,9 @@ class RollbackResponse(BaseModel):
     cancelled_tasks: int
     deleted_files: int
 
+class RollbackBody(BaseModel):
+    embedding_only: bool = False
+
 
 # Docling preset configurations
 def get_docling_preset_configs(
@@ -1638,8 +1641,10 @@ async def reapply_all_settings(session_manager = None):
 
 async def rollback_onboarding(
     request: Request,
+    body: Optional[RollbackBody] = None,
     session_manager=Depends(get_session_manager),
     task_service=Depends(get_task_service),
+    knowledge_filter_service=Depends(get_knowledge_filter_service),
     user: User = Depends(get_current_user),
 ) -> RollbackResponse:
     """Rollback onboarding configuration when sample data files fail.
@@ -1669,7 +1674,36 @@ async def rollback_onboarding(
         cancelled_tasks = []
         deleted_files = []
 
+        # Delete knowledge filters created during onboarding
+        try:
+            async def remove_filter(filter_id: Optional[str]):
+                if filter_id and knowledge_filter_service:
+                    try:
+                        result = await knowledge_filter_service.delete_knowledge_filter(
+                            filter_id, user.user_id, user.jwt_token
+                        )
+                        if result and result.get("success"):
+                            logger.info(f"Deleted knowledge filter {filter_id}")
+                        else:
+                            error_msg = result.get("error") if result else "Unknown error"
+                            logger.warning(f"Could not delete knowledge filter {filter_id}: {error_msg}")
+                    except Exception as e:
+                        logger.warning(f"Exception deleting knowledge filter {filter_id}: {str(e)}")
+
+            if getattr(current_config.onboarding, 'openrag_docs_filter_id', None):
+                await remove_filter(current_config.onboarding.openrag_docs_filter_id)
+                current_config.onboarding.openrag_docs_filter_id = None
+                
+            if getattr(current_config.onboarding, 'user_doc_filter_id', None):
+                await remove_filter(current_config.onboarding.user_doc_filter_id)
+                current_config.onboarding.user_doc_filter_id = None
+        except Exception as e:
+            logger.error(f"Error while cleaning up knowledge filters: {e}")
+
         # Cancel all active tasks and collect successfully ingested files
+        from session_manager import AnonymousUser
+        anonymous_user_id = AnonymousUser().user_id
+
         for task_data in all_tasks:
             task_id = task_data.get("task_id")
             task_status = task_data.get("status")
@@ -1684,41 +1718,40 @@ async def rollback_onboarding(
                 except Exception as e:
                     logger.error(f"Failed to cancel task {task_id}: {str(e)}")
 
-            # For completed tasks, find successfully ingested files and delete them
-            elif task_status == "completed":
-                files = task_data.get("files", {})
-                if isinstance(files, dict):
-                    for file_path, file_info in files.items():
-                        # Check if file was successfully ingested
-                        if isinstance(file_info, dict):
-                            file_status = file_info.get("status")
-                            filename = file_info.get("filename") or file_path.split("/")[-1]
-
-                            if file_status == "completed" and filename:
-                                try:
-                                    # Get user's OpenSearch client
-                                    opensearch_client = session_manager.get_user_opensearch_client(
-                                        user.user_id, jwt_token
-                                    )
-
-                                    # Delete documents by filename
-                                    from utils.opensearch_queries import build_filename_delete_body
-                                    from config.settings import get_index_name
-
-                                    delete_query = build_filename_delete_body(filename)
-
-                                    result = await opensearch_client.delete_by_query(
-                                        index=get_index_name(),
-                                        body=delete_query,
-                                        conflicts="proceed"
-                                    )
-
-                                    deleted_count = result.get("deleted", 0)
-                                    if deleted_count > 0:
-                                        deleted_files.append(filename)
-                                        logger.info(f"Deleted {deleted_count} chunks for filename {filename}")
-                                except Exception as e:
-                                    logger.error(f"Failed to delete documents for {filename}: {str(e)}")
+            # Delete all files associated with any task, regardless of whether 
+            # the task failed or completed, to ensure no partial chunks remain in OpenSearch.
+            files = task_data.get("files", {})
+            if isinstance(files, dict):
+                for file_path, file_info in files.items():
+                    if isinstance(file_info, dict):
+                        filename = file_info.get("filename") or file_path.split("/")[-1]
+                        if filename:
+                            try:
+                                opensearch_client = session_manager.get_user_opensearch_client(
+                                    user.user_id, jwt_token
+                                )
+                                from utils.opensearch_queries import build_filename_delete_body
+                                from config.settings import get_index_name
+
+                                delete_query = build_filename_delete_body(filename)
+                                result = await opensearch_client.delete_by_query(
+                                    index=get_index_name(),
+                                    body=delete_query,
+                                    conflicts="proceed"
+                                )
+                                deleted_count = result.get("deleted", 0)
+                                if deleted_count > 0:
+                                    deleted_files.append(filename)
+                                    logger.info(f"Deleted {deleted_count} chunks for filename {filename}")
+                            except Exception as e:
+                                logger.error(f"Failed to delete documents for {filename}: {str(e)}")
+
+            # Wipe the task completely from memory so the frontend doesn't see it anymore
+            for check_user_id in [user.user_id, anonymous_user_id]:
+                if check_user_id in task_service.task_store and task_id in task_service.task_store[check_user_id]:
+                    task_service._task_locks.pop(task_id, None)
+                    task_service.task_store[check_user_id].pop(task_id, None)
+                    logger.info(f"Purged task {task_id} completely from task_store for user {check_user_id}")
 
         # Clear embedding provider and model settings
         current_config.knowledge.embedding_provider = "openai"  # Reset to default
@@ -1726,11 +1759,21 @@ async def rollback_onboarding(
         current_config.onboarding.openrag_docs_ingested_version = None
         current_config.onboarding.openrag_docs_remote_signature = None
 
+        embedding_only = body.embedding_only if body else False
+
         # Mark config as not edited so user can go through onboarding again
-        current_config.edited = False
-        current_config.onboarding.current_step = 0
+        if not embedding_only:
+            current_config.edited = False
+            current_config.onboarding.current_step = 0
+            # Also clear LLM provider and model settings when doing a full rollback
+            current_config.agent.llm_provider = "openai"  # Reset to default
+            current_config.agent.llm_model = ""
+        else:
+            # When rolling back embedding only, we keep edited=True
+            # and set current_step to 1 (which is the embedding step)
+            current_config.onboarding.current_step = 1
 
-        # Save the rolled back configuration manually to avoid save_config_file setting edited=True
+        # Save the rolled back configuration manually
         try:
             import yaml
             config_file = config_manager.config_file
@@ -1738,14 +1781,14 @@ async def rollback_onboarding(
             # Ensure directory exists
             config_file.parent.mkdir(parents=True, exist_ok=True)
 
-            # Save config with edited=False
+            # Save config with current edited state
             with open(config_file, "w") as f:
                 yaml.dump(current_config.to_dict(), f, default_flow_style=False, indent=2)
 
             # Update cached config
             config_manager._config = current_config
 
-            logger.info("Successfully saved rolled back configuration with edited=False")
+            logger.info(f"Successfully saved rolled back configuration with edited={current_config.edited}")
         except Exception as e:
             logger.error(f"Failed to save rolled back configuration: {e}")
             return JSONResponse(

{openrag-0.4.0.dev6 → openrag-0.4.0.dev8}/src/main.py

@@ -410,13 +410,14 @@ async def ingest_openrag_docs_when_ready(
 ):
     """Ingest OpenRAG docs during onboarding."""
     use_url_ingest = _should_use_url_default_docs_ingest()
+    task_id = None
     if use_url_ingest:
         try:
             await TelemetryClient.send_event(
                 Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_START
             )
             if DISABLE_INGEST_WITH_LANGFLOW:
-                await _ingest_default_documents_url(
+                task_id = await _ingest_default_documents_url(
                     document_service=document_service,
                     docs_url=DEFAULT_DOCS_URL,
                     crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,
@@ -426,7 +427,7 @@ async def ingest_openrag_docs_when_ready(
                     "Ingesting default documents using Langflow",
                     docs_url=DEFAULT_DOCS_URL,
                 )
-                await _ingest_default_documents_url_langflow(
+                task_id = await _ingest_default_documents_url_langflow(
                     langflow_file_service=langflow_file_service,
                     session_manager=session_manager,
                     task_service=task_service,
@@ -441,6 +442,7 @@ async def ingest_openrag_docs_when_ready(
             await TelemetryClient.send_event(
                 Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_FAILED
             )
+    return task_id
 
 
 async def ingest_default_documents_when_ready(
@@ -456,9 +458,11 @@ async def ingest_default_documents_when_ready(
         await TelemetryClient.send_event(
             Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START
         )
-        await ingest_openrag_docs_when_ready(
-            document_service, task_service, langflow_file_service, session_manager
-        )
+        task_id = None
+        if _should_use_url_default_docs_ingest():
+            task_id = await ingest_openrag_docs_when_ready(
+                document_service, task_service, langflow_file_service, session_manager
+            )
 
         base_dir = _get_documents_dir()
         if not os.path.isdir(base_dir):
@@ -469,8 +473,7 @@ async def ingest_default_documents_when_ready(
         excluded_files = set(EXCLUDED_INGESTION_FILES)
         if _should_use_url_default_docs_ingest():
             excluded_files.update(URL_INGEST_EXCLUDED_INGESTION_FILES)
-
-        # Collect files recursively, excluding warmup files and URL-ingested docs
+        
         file_paths = [
             os.path.join(root, fn)
             for root, _, files in os.walk(base_dir)
@@ -482,18 +485,22 @@ async def ingest_default_documents_when_ready(
             raise FileNotFoundError(f"No default documents found in {base_dir}")
 
         if DISABLE_INGEST_WITH_LANGFLOW:
-            await _ingest_default_documents_openrag(
-                document_service, task_service, file_paths
+            new_task_id = await _ingest_default_documents_openrag(
+                document_service, task_service, file_paths, existing_task_id=task_id, connector_type="local"
             )
+            task_id = new_task_id or task_id
         else:
-            await _ingest_default_documents_langflow(
-                langflow_file_service, session_manager, task_service, file_paths
+            new_task_id = await _ingest_default_documents_langflow(
+                langflow_file_service, session_manager, task_service, file_paths, existing_task_id=task_id, connector_type="local"
             )
+            task_id = new_task_id or task_id
 
         await TelemetryClient.send_event(
             Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE
         )
 
+        return task_id
+
     except Exception as e:
         logger.error("Default documents ingestion failed", error=str(e))
         await TelemetryClient.send_event(
@@ -503,7 +510,7 @@ async def ingest_default_documents_when_ready(
 
 
 async def _ingest_default_documents_langflow(
-    langflow_file_service, session_manager, task_service, file_paths
+    langflow_file_service, session_manager, task_service, file_paths, existing_task_id: str = None, connector_type: str = "openrag_docs"
 ):
     """Ingest default documents using Langflow upload-ingest-delete pipeline."""
 
@@ -538,7 +545,7 @@ async def _ingest_default_documents_langflow(
                 {"key": "owner", "value": None},
                 {"key": "owner_name", "value": anonymous_user.name},
                 {"key": "owner_email", "value": anonymous_user.email},
-                {"key": "connector_type", "value": "system_default"},
+                {"key": "connector_type", "value": "openrag_docs"},
                 {"key": "is_sample_data", "value": "true"},
             ]
         }
@@ -558,6 +565,8 @@ async def _ingest_default_documents_langflow(
         settings=None,  # Use default ingestion settings
         delete_after_ingest=True,  # Clean up after ingestion
         replace_duplicates=True,
+        connector_type=connector_type,
+        existing_task_id=existing_task_id,
     )
 
     logger.info(
@@ -565,6 +574,7 @@ async def _ingest_default_documents_langflow(
         task_id=task_id,
         file_count=len(file_paths),
     )
+    return task_id
 
 
 async def _ingest_default_documents_url_langflow(
@@ -626,6 +636,7 @@ async def _ingest_default_documents_url_langflow(
         task_id=task_id,
         docs_url=docs_url,
     )
+    return task_id
 
 
 async def _ingest_default_documents_url(
@@ -657,7 +668,7 @@ async def _ingest_default_documents_url(
             owner_name=None,
             owner_email=None,
             is_sample_data=True,
-            connector_type="system_default",
+            connector_type="openrag_docs",
         )
         await processor.process_document_standard(
             file_path=temp_file_path,
@@ -668,7 +679,7 @@ async def _ingest_default_documents_url(
             owner_name=None,
             owner_email=None,
             file_size=os.path.getsize(temp_file_path),
-            connector_type="system_default",
+            connector_type="openrag_docs",
             is_sample_data=True,
         )
     finally:
@@ -1035,7 +1046,7 @@ async def opensearch_health_ready(request):
 
 
 async def _ingest_default_documents_openrag(
-    document_service, task_service, file_paths, connector_type: str = "local"
+    document_service, task_service, file_paths, existing_task_id: str = None, connector_type: str = "openrag_docs"
 ):
     """Ingest default documents using traditional OpenRAG processor."""
     logger.info(
@@ -1056,12 +1067,13 @@ async def _ingest_default_documents_openrag(
         connector_type=connector_type,
     )
 
-    task_id = await task_service.create_custom_task("anonymous", file_paths, processor)
+    task_id = await task_service.create_custom_task("anonymous", file_paths, processor, existing_task_id=existing_task_id)
     logger.info(
         "Started traditional OpenRAG ingestion task",
         task_id=task_id,
         file_count=len(file_paths),
     )
+    return task_id
 
 
 async def _update_mcp_servers_with_provider_credentials(services):
@@ -1198,6 +1210,20 @@ async def startup_tasks(services):
     # Update MCP servers with provider credentials (especially important for no-auth mode)
     await _update_mcp_servers_with_provider_credentials(services)
 
+    # Ensure all configured flows exist in Langflow (create-only, never overwrites).
+    # This replaces LANGFLOW_LOAD_FLOWS_PATH, which performed a blind upsert on
+    # every container start and discarded any user edits made in the Langflow UI.
+    newly_created: set[str] = set()
+    try:
+        flows_service = services["flows_service"]
+        newly_created = await flows_service.ensure_flows_exist()
+    except Exception as e:
+        logger.error(
+            "Failed to ensure Langflow flows exist at startup — "
+            "flows may be missing until the next restart",
+            error=str(e),
+        )
+
     # Check if flows were reset and reapply settings if config is edited
     try:
         config = get_openrag_config()
@@ -1205,6 +1231,9 @@ async def startup_tasks(services):
             logger.info("Checking if Langflow flows were reset")
             flows_service = services["flows_service"]
             reset_flows = await flows_service.check_flows_reset()
+            # Exclude flows that were just seeded — they match the JSON by design,
+            # not because they were externally reset.
+            reset_flows = [f for f in reset_flows if f not in newly_created]
 
             if reset_flows:
                 logger.info(

{openrag-0.4.0.dev6 → openrag-0.4.0.dev8}/src/models/processors.py

@@ -730,6 +730,7 @@ class LangflowFileProcessor(TaskProcessor):
         settings: dict = None,
         delete_after_ingest: bool = True,
         replace_duplicates: bool = False,
+        connector_type: str = "local",
     ):
         super().__init__()
         self.langflow_file_service = langflow_file_service
@@ -743,6 +744,7 @@ class LangflowFileProcessor(TaskProcessor):
         self.settings = settings
         self.delete_after_ingest = delete_after_ingest
         self.replace_duplicates = replace_duplicates
+        self.connector_type = connector_type
 
     async def process_item(
         self, upload_task: UploadTask, item: str, file_task: FileTask
@@ -828,8 +830,7 @@ class LangflowFileProcessor(TaskProcessor):
                 owner=self.owner_user_id,
                 owner_name=self.owner_name,
                 owner_email=self.owner_email,
-                connector_type="local",
-
+                connector_type=self.connector_type,
             )
 
             # Update task with success

{openrag-0.4.0.dev6 → openrag-0.4.0.dev8/src/openrag.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrag
-Version: 0.4.0.dev6
+Version: 0.4.0.dev8
 Summary: OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations.
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console