openrag 0.4.0.dev7__tar.gz → 0.4.0.dev9__tar.gz

{openrag-0.4.0.dev7/src/openrag.egg-info → openrag-0.4.0.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrag
-Version: 0.4.0.dev7
+Version: 0.4.0.dev9
 Summary: OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations.
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console

{openrag-0.4.0.dev7 → openrag-0.4.0.dev9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "openrag"
-version = "0.4.0.dev7"
+version = "0.4.0.dev9"
 description = "OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations."
 readme = "README.md"
 requires-python = ">=3.13"

{openrag-0.4.0.dev7 → openrag-0.4.0.dev9}/src/api/settings.py

@@ -211,6 +211,9 @@ class RollbackResponse(BaseModel):
     cancelled_tasks: int
     deleted_files: int
 
+class RollbackBody(BaseModel):
+    embedding_only: bool = False
+
 
 # Docling preset configurations
 def get_docling_preset_configs(
@@ -1638,8 +1641,10 @@ async def reapply_all_settings(session_manager = None):
 
 async def rollback_onboarding(
     request: Request,
+    body: Optional[RollbackBody] = None,
     session_manager=Depends(get_session_manager),
     task_service=Depends(get_task_service),
+    knowledge_filter_service=Depends(get_knowledge_filter_service),
     user: User = Depends(get_current_user),
 ) -> RollbackResponse:
     """Rollback onboarding configuration when sample data files fail.
@@ -1669,7 +1674,36 @@ async def rollback_onboarding(
         cancelled_tasks = []
         deleted_files = []
 
+        # Delete knowledge filters created during onboarding
+        try:
+            async def remove_filter(filter_id: Optional[str]):
+                if filter_id and knowledge_filter_service:
+                    try:
+                        result = await knowledge_filter_service.delete_knowledge_filter(
+                            filter_id, user.user_id, user.jwt_token
+                        )
+                        if result and result.get("success"):
+                            logger.info(f"Deleted knowledge filter {filter_id}")
+                        else:
+                            error_msg = result.get("error") if result else "Unknown error"
+                            logger.warning(f"Could not delete knowledge filter {filter_id}: {error_msg}")
+                    except Exception as e:
+                        logger.warning(f"Exception deleting knowledge filter {filter_id}: {str(e)}")
+
+            if getattr(current_config.onboarding, 'openrag_docs_filter_id', None):
+                await remove_filter(current_config.onboarding.openrag_docs_filter_id)
+                current_config.onboarding.openrag_docs_filter_id = None
+                
+            if getattr(current_config.onboarding, 'user_doc_filter_id', None):
+                await remove_filter(current_config.onboarding.user_doc_filter_id)
+                current_config.onboarding.user_doc_filter_id = None
+        except Exception as e:
+            logger.error(f"Error while cleaning up knowledge filters: {e}")
+
         # Cancel all active tasks and collect successfully ingested files
+        from session_manager import AnonymousUser
+        anonymous_user_id = AnonymousUser().user_id
+
         for task_data in all_tasks:
             task_id = task_data.get("task_id")
             task_status = task_data.get("status")
@@ -1684,41 +1718,40 @@ async def rollback_onboarding(
                 except Exception as e:
                     logger.error(f"Failed to cancel task {task_id}: {str(e)}")
 
-            # For completed tasks, find successfully ingested files and delete them
-            elif task_status == "completed":
-                files = task_data.get("files", {})
-                if isinstance(files, dict):
-                    for file_path, file_info in files.items():
-                        # Check if file was successfully ingested
-                        if isinstance(file_info, dict):
-                            file_status = file_info.get("status")
-                            filename = file_info.get("filename") or file_path.split("/")[-1]
-
-                            if file_status == "completed" and filename:
-                                try:
-                                    # Get user's OpenSearch client
-                                    opensearch_client = session_manager.get_user_opensearch_client(
-                                        user.user_id, jwt_token
-                                    )
-
-                                    # Delete documents by filename
-                                    from utils.opensearch_queries import build_filename_delete_body
-                                    from config.settings import get_index_name
-
-                                    delete_query = build_filename_delete_body(filename)
-
-                                    result = await opensearch_client.delete_by_query(
-                                        index=get_index_name(),
-                                        body=delete_query,
-                                        conflicts="proceed"
-                                    )
-
-                                    deleted_count = result.get("deleted", 0)
-                                    if deleted_count > 0:
-                                        deleted_files.append(filename)
-                                        logger.info(f"Deleted {deleted_count} chunks for filename {filename}")
-                                except Exception as e:
-                                    logger.error(f"Failed to delete documents for {filename}: {str(e)}")
+            # Delete all files associated with any task, regardless of whether 
+            # the task failed or completed, to ensure no partial chunks remain in OpenSearch.
+            files = task_data.get("files", {})
+            if isinstance(files, dict):
+                for file_path, file_info in files.items():
+                    if isinstance(file_info, dict):
+                        filename = file_info.get("filename") or file_path.split("/")[-1]
+                        if filename:
+                            try:
+                                opensearch_client = session_manager.get_user_opensearch_client(
+                                    user.user_id, jwt_token
+                                )
+                                from utils.opensearch_queries import build_filename_delete_body
+                                from config.settings import get_index_name
+
+                                delete_query = build_filename_delete_body(filename)
+                                result = await opensearch_client.delete_by_query(
+                                    index=get_index_name(),
+                                    body=delete_query,
+                                    conflicts="proceed"
+                                )
+                                deleted_count = result.get("deleted", 0)
+                                if deleted_count > 0:
+                                    deleted_files.append(filename)
+                                    logger.info(f"Deleted {deleted_count} chunks for filename {filename}")
+                            except Exception as e:
+                                logger.error(f"Failed to delete documents for {filename}: {str(e)}")
+
+            # Wipe the task completely from memory so the frontend doesn't see it anymore
+            for check_user_id in [user.user_id, anonymous_user_id]:
+                if check_user_id in task_service.task_store and task_id in task_service.task_store[check_user_id]:
+                    task_service._task_locks.pop(task_id, None)
+                    task_service.task_store[check_user_id].pop(task_id, None)
+                    logger.info(f"Purged task {task_id} completely from task_store for user {check_user_id}")
 
         # Clear embedding provider and model settings
         current_config.knowledge.embedding_provider = "openai"  # Reset to default
@@ -1726,11 +1759,21 @@ async def rollback_onboarding(
         current_config.onboarding.openrag_docs_ingested_version = None
         current_config.onboarding.openrag_docs_remote_signature = None
 
+        embedding_only = body.embedding_only if body else False
+
         # Mark config as not edited so user can go through onboarding again
-        current_config.edited = False
-        current_config.onboarding.current_step = 0
+        if not embedding_only:
+            current_config.edited = False
+            current_config.onboarding.current_step = 0
+            # Also clear LLM provider and model settings when doing a full rollback
+            current_config.agent.llm_provider = "openai"  # Reset to default
+            current_config.agent.llm_model = ""
+        else:
+            # When rolling back embedding only, we keep edited=True
+            # and set current_step to 1 (which is the embedding step)
+            current_config.onboarding.current_step = 1
 
-        # Save the rolled back configuration manually to avoid save_config_file setting edited=True
+        # Save the rolled back configuration manually
         try:
             import yaml
             config_file = config_manager.config_file
@@ -1738,14 +1781,14 @@ async def rollback_onboarding(
             # Ensure directory exists
             config_file.parent.mkdir(parents=True, exist_ok=True)
 
-            # Save config with edited=False
+            # Save config with current edited state
             with open(config_file, "w") as f:
                 yaml.dump(current_config.to_dict(), f, default_flow_style=False, indent=2)
 
             # Update cached config
             config_manager._config = current_config
 
-            logger.info("Successfully saved rolled back configuration with edited=False")
+            logger.info(f"Successfully saved rolled back configuration with edited={current_config.edited}")
         except Exception as e:
             logger.error(f"Failed to save rolled back configuration: {e}")
             return JSONResponse(

{openrag-0.4.0.dev7 → openrag-0.4.0.dev9}/src/main.py

@@ -410,13 +410,14 @@ async def ingest_openrag_docs_when_ready(
 ):
     """Ingest OpenRAG docs during onboarding."""
     use_url_ingest = _should_use_url_default_docs_ingest()
+    task_id = None
     if use_url_ingest:
         try:
             await TelemetryClient.send_event(
                 Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_START
             )
             if DISABLE_INGEST_WITH_LANGFLOW:
-                await _ingest_default_documents_url(
+                task_id = await _ingest_default_documents_url(
                     document_service=document_service,
                     docs_url=DEFAULT_DOCS_URL,
                     crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,
@@ -426,7 +427,7 @@ async def ingest_openrag_docs_when_ready(
                     "Ingesting default documents using Langflow",
                     docs_url=DEFAULT_DOCS_URL,
                 )
-                await _ingest_default_documents_url_langflow(
+                task_id = await _ingest_default_documents_url_langflow(
                     langflow_file_service=langflow_file_service,
                     session_manager=session_manager,
                     task_service=task_service,
@@ -441,6 +442,7 @@ async def ingest_openrag_docs_when_ready(
             await TelemetryClient.send_event(
                 Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_FAILED
             )
+    return task_id
 
 
 async def ingest_default_documents_when_ready(
@@ -456,9 +458,11 @@ async def ingest_default_documents_when_ready(
         await TelemetryClient.send_event(
             Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START
         )
-        await ingest_openrag_docs_when_ready(
-            document_service, task_service, langflow_file_service, session_manager
-        )
+        task_id = None
+        if _should_use_url_default_docs_ingest():
+            task_id = await ingest_openrag_docs_when_ready(
+                document_service, task_service, langflow_file_service, session_manager
+            )
 
         base_dir = _get_documents_dir()
         if not os.path.isdir(base_dir):
@@ -469,8 +473,7 @@ async def ingest_default_documents_when_ready(
         excluded_files = set(EXCLUDED_INGESTION_FILES)
         if _should_use_url_default_docs_ingest():
             excluded_files.update(URL_INGEST_EXCLUDED_INGESTION_FILES)
-
-        # Collect files recursively, excluding warmup files and URL-ingested docs
+        
         file_paths = [
             os.path.join(root, fn)
             for root, _, files in os.walk(base_dir)
@@ -482,18 +485,22 @@ async def ingest_default_documents_when_ready(
             raise FileNotFoundError(f"No default documents found in {base_dir}")
 
         if DISABLE_INGEST_WITH_LANGFLOW:
-            await _ingest_default_documents_openrag(
-                document_service, task_service, file_paths
+            new_task_id = await _ingest_default_documents_openrag(
+                document_service, task_service, file_paths, existing_task_id=task_id, connector_type="local"
             )
+            task_id = new_task_id or task_id
         else:
-            await _ingest_default_documents_langflow(
-                langflow_file_service, session_manager, task_service, file_paths
+            new_task_id = await _ingest_default_documents_langflow(
+                langflow_file_service, session_manager, task_service, file_paths, existing_task_id=task_id, connector_type="local"
             )
+            task_id = new_task_id or task_id
 
         await TelemetryClient.send_event(
             Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE
         )
 
+        return task_id
+
     except Exception as e:
         logger.error("Default documents ingestion failed", error=str(e))
         await TelemetryClient.send_event(
@@ -503,7 +510,7 @@ async def ingest_default_documents_when_ready(
 
 
 async def _ingest_default_documents_langflow(
-    langflow_file_service, session_manager, task_service, file_paths
+    langflow_file_service, session_manager, task_service, file_paths, existing_task_id: str = None, connector_type: str = "openrag_docs"
 ):
     """Ingest default documents using Langflow upload-ingest-delete pipeline."""
 
@@ -538,7 +545,7 @@ async def _ingest_default_documents_langflow(
                 {"key": "owner", "value": None},
                 {"key": "owner_name", "value": anonymous_user.name},
                 {"key": "owner_email", "value": anonymous_user.email},
-                {"key": "connector_type", "value": "system_default"},
+                {"key": "connector_type", "value": "openrag_docs"},
                 {"key": "is_sample_data", "value": "true"},
             ]
         }
@@ -558,6 +565,8 @@ async def _ingest_default_documents_langflow(
         settings=None,  # Use default ingestion settings
         delete_after_ingest=True,  # Clean up after ingestion
         replace_duplicates=True,
+        connector_type=connector_type,
+        existing_task_id=existing_task_id,
     )
 
     logger.info(
@@ -565,6 +574,7 @@ async def _ingest_default_documents_langflow(
         task_id=task_id,
         file_count=len(file_paths),
     )
+    return task_id
 
 
 async def _ingest_default_documents_url_langflow(
@@ -626,6 +636,7 @@ async def _ingest_default_documents_url_langflow(
         task_id=task_id,
         docs_url=docs_url,
     )
+    return task_id
 
 
 async def _ingest_default_documents_url(
@@ -657,7 +668,7 @@ async def _ingest_default_documents_url(
             owner_name=None,
             owner_email=None,
             is_sample_data=True,
-            connector_type="system_default",
+            connector_type="openrag_docs",
         )
         await processor.process_document_standard(
             file_path=temp_file_path,
@@ -668,7 +679,7 @@ async def _ingest_default_documents_url(
             owner_name=None,
             owner_email=None,
             file_size=os.path.getsize(temp_file_path),
-            connector_type="system_default",
+            connector_type="openrag_docs",
             is_sample_data=True,
         )
     finally:
@@ -1035,7 +1046,7 @@ async def opensearch_health_ready(request):
 
 
 async def _ingest_default_documents_openrag(
-    document_service, task_service, file_paths, connector_type: str = "local"
+    document_service, task_service, file_paths, existing_task_id: str = None, connector_type: str = "openrag_docs"
 ):
     """Ingest default documents using traditional OpenRAG processor."""
     logger.info(
@@ -1056,12 +1067,13 @@ async def _ingest_default_documents_openrag(
         connector_type=connector_type,
     )
 
-    task_id = await task_service.create_custom_task("anonymous", file_paths, processor)
+    task_id = await task_service.create_custom_task("anonymous", file_paths, processor, existing_task_id=existing_task_id)
     logger.info(
         "Started traditional OpenRAG ingestion task",
         task_id=task_id,
         file_count=len(file_paths),
     )
+    return task_id
 
 
 async def _update_mcp_servers_with_provider_credentials(services):

{openrag-0.4.0.dev7 → openrag-0.4.0.dev9}/src/models/processors.py

@@ -730,6 +730,7 @@ class LangflowFileProcessor(TaskProcessor):
         settings: dict = None,
         delete_after_ingest: bool = True,
         replace_duplicates: bool = False,
+        connector_type: str = "local",
     ):
         super().__init__()
         self.langflow_file_service = langflow_file_service
@@ -743,6 +744,7 @@ class LangflowFileProcessor(TaskProcessor):
         self.settings = settings
         self.delete_after_ingest = delete_after_ingest
         self.replace_duplicates = replace_duplicates
+        self.connector_type = connector_type
 
     async def process_item(
         self, upload_task: UploadTask, item: str, file_task: FileTask
@@ -828,8 +830,7 @@ class LangflowFileProcessor(TaskProcessor):
                 owner=self.owner_user_id,
                 owner_name=self.owner_name,
                 owner_email=self.owner_email,
-                connector_type="local",
-
+                connector_type=self.connector_type,
             )
 
             # Update task with success

{openrag-0.4.0.dev7 → openrag-0.4.0.dev9/src/openrag.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrag
-Version: 0.4.0.dev7
+Version: 0.4.0.dev9
 Summary: OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations.
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console

{openrag-0.4.0.dev7 → openrag-0.4.0.dev9}/src/openrag.egg-info/SOURCES.txt

@@ -152,7 +152,6 @@ src/utils/hash_utils.py
 src/utils/langflow_headers.py
 src/utils/langflow_utils.py
 src/utils/logging_config.py
-src/utils/opensearch_filter_merge.py
 src/utils/opensearch_queries.py
 src/utils/opensearch_utils.py
 src/utils/paths.py

{openrag-0.4.0.dev7 → openrag-0.4.0.dev9}/src/services/task_service.py

@@ -137,6 +137,8 @@ class TaskService:
         settings: dict = None,
         delete_after_ingest: bool = True,
         replace_duplicates: bool = False,
+        connector_type: str = "local",
+        existing_task_id: str = None,
     ) -> str:
         """Create a new upload task for Langflow file processing with upload and ingest"""
         # Use LangflowFileProcessor with user context
@@ -154,8 +156,9 @@ class TaskService:
             settings=settings,
             delete_after_ingest=delete_after_ingest,
             replace_duplicates=replace_duplicates,
+            connector_type=connector_type,
         )
-        return await self.create_custom_task(user_id, file_paths, processor, original_filenames)
+        return await self.create_custom_task(user_id, file_paths, processor, original_filenames, existing_task_id=existing_task_id)
 
     async def create_langflow_url_upload_task(
         self,
@@ -170,6 +173,7 @@ class TaskService:
         connector_type: str = "openrag_docs",
         prevent_outside: bool = True,
         tweaks: dict = None,
+        existing_task_id: str = None,
     ) -> str:
         """Create a new upload task for Langflow URL ingestion."""
         from models.url import LangflowUrlProcessor
@@ -187,14 +191,14 @@ class TaskService:
             prevent_outside=prevent_outside,
             tweaks=tweaks,
         )
-        return await self.create_custom_task(owner_user_id, [docs_url], processor)
+        return await self.create_custom_task(owner_user_id, [docs_url], processor, existing_task_id=existing_task_id)
 
-    async def create_custom_task(self, user_id: str, items: list, processor, original_filenames: dict | None = None) -> str:
+    async def create_custom_task(self, user_id: str, items: list, processor, original_filenames: dict | None = None, existing_task_id: str = None) -> str:
         """Create a new task with custom processor for any type of items"""
         import os
         # Store anonymous tasks under a stable key so they can be retrieved later
         store_user_id = user_id or AnonymousUser().user_id
-        task_id = str(uuid.uuid4())
+        task_id = existing_task_id or str(uuid.uuid4())
 
         # Create file tasks with original filenames if provided
         normalized_originals = (
@@ -210,28 +214,32 @@ class TaskService:
             for item in items
         }
 
-        upload_task = UploadTask(
-            task_id=task_id,
-            total_files=len(items),
-            file_tasks=file_tasks,
-        )
-
-        # Attach the custom processor to the task
-        upload_task.processor = processor
-
-        if store_user_id not in self.task_store:
-            self.task_store[store_user_id] = {}
-        self.task_store[store_user_id][task_id] = upload_task
+        if existing_task_id and store_user_id in self.task_store and existing_task_id in self.task_store[store_user_id]:
+            upload_task = self.task_store[store_user_id][existing_task_id]
+            upload_task.file_tasks.update(file_tasks)
+            upload_task.total_files += len(items)
+            upload_task.status = TaskStatus.RUNNING
+        else:
+            upload_task = UploadTask(
+                task_id=task_id,
+                total_files=len(items),
+                file_tasks=file_tasks,
+            )
+            upload_task.processor = processor
+            if store_user_id not in self.task_store:
+                self.task_store[store_user_id] = {}
+            self.task_store[store_user_id][task_id] = upload_task
 
         # Start background processing
         background_task = asyncio.create_task(
-            self.background_custom_processor(store_user_id, task_id, items)
+            self.background_custom_processor(store_user_id, task_id, items, processor)
         )
         self.background_tasks.add(background_task)
         background_task.add_done_callback(self.background_tasks.discard)
 
-        # Store reference to background task for cancellation
-        upload_task.background_task = background_task
+        # Store reference to background task for cancellation if newly created
+        if not existing_task_id:
+            upload_task.background_task = background_task
 
         # Send telemetry event for task creation with metadata
         asyncio.create_task(
@@ -282,7 +290,7 @@ class TaskService:
         return f"{hours}h {mins}m {secs}s"
 
     async def background_custom_processor(
-        self, user_id: str, task_id: str, items: list
+        self, user_id: str, task_id: str, items: list, processor=None
     ) -> None:
         """Background task to process items using custom processor"""
         try:
@@ -290,7 +298,7 @@ class TaskService:
             upload_task.status = TaskStatus.RUNNING
             upload_task.updated_at = time.time()
 
-            processor = upload_task.processor
+            processor = processor or upload_task.processor
 
             logger.info(
                 "Upload / ingestion task started",
@@ -401,9 +409,10 @@ class TaskService:
 
             await asyncio.gather(*tasks, return_exceptions=True)
 
-            # Mark task as completed
-            upload_task.status = TaskStatus.COMPLETED
-            upload_task.updated_at = time.time()
+            # Mark task as completed if all files (including appended ones) are done
+            if upload_task.processed_files >= upload_task.total_files:
+                upload_task.status = TaskStatus.COMPLETED
+                upload_task.updated_at = time.time()
 
             status: str = "FAILED"