openrag 0.4.0.dev1__tar.gz → 0.4.0.dev3__tar.gz

{openrag-0.4.0.dev1/src/openrag.egg-info → openrag-0.4.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrag
-Version: 0.4.0.dev1
+Version: 0.4.0.dev3
 Summary: OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations.
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console

{openrag-0.4.0.dev1 → openrag-0.4.0.dev3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "openrag"
-version = "0.4.0.dev1"
+version = "0.4.0.dev3"
 description = "OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations."
 readme = "README.md"
 requires-python = ">=3.13"

{openrag-0.4.0.dev1 → openrag-0.4.0.dev3}/src/api/documents.py

@@ -121,16 +121,26 @@ async def check_filename_exists(
         )
 
         from utils.opensearch_queries import build_filename_search_body
+        from utils.file_utils import get_filename_aliases
 
-        search_body = build_filename_search_body(filename, size=1, source=["filename"])
+        candidate_filenames = get_filename_aliases(filename)
+        if not candidate_filenames:
+            return JSONResponse({"exists": False, "filename": filename}, status_code=200)
 
         logger.debug("Checking filename existence", filename=filename, index_name=get_index_name())
+        exists = False
 
         try:
-            response = await opensearch_client.search(
-                index=get_index_name(),
-                body=search_body
-            )
+            for candidate in candidate_filenames:
+                search_body = build_filename_search_body(candidate, size=1, source=["filename"])
+                response = await opensearch_client.search(
+                    index=get_index_name(),
+                    body=search_body
+                )
+                hits = response.get("hits", {}).get("hits", [])
+                if hits:
+                    exists = True
+                    break
         except Exception as search_err:
             if "index_not_found_exception" in str(search_err):
                 logger.info("Index does not exist, creating it now before upload")
@@ -138,9 +148,6 @@ async def check_filename_exists(
                 return JSONResponse({"exists": False, "filename": filename}, status_code=200)
             raise
 
-        hits = response.get("hits", {}).get("hits", [])
-        exists = len(hits) > 0
-
         return JSONResponse({"exists": exists, "filename": filename}, status_code=200)
 
     except Exception as e:

{openrag-0.4.0.dev1 → openrag-0.4.0.dev3}/src/api/settings.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import platform
 from fastapi import Depends, Request, HTTPException
@@ -36,6 +37,7 @@ from dependencies import (
 from session_manager import User
 
 logger = get_logger(__name__)
+_background_tasks: set[asyncio.Task] = set()
 
 
 class SettingsUpdateBody(BaseModel):
@@ -815,30 +817,31 @@ async def update_settings(
                 {"error": "Failed to save configuration"}, status_code=500
             )
 
-        # Update Langflow global variables and model values if provider settings changed
+        # Refresh patched client immediately so subsequent requests pick up latest config.
         await clients.refresh_patched_client()
 
+        # Run expensive Langflow sync in the background to keep settings updates responsive.
         if should_validate or provider_updated:
-            try:
-                flows_service = _get_flows_service()
-
-                # Update global variables
-                await _update_langflow_global_variables(current_config, flows_service=flows_service)
-
-                # Update LLM client credentials when embedding selection changes
-                if body.embedding_provider is not None or body.embedding_model is not None:
-                    await _update_mcp_servers_with_provider_credentials(
-                        current_config, session_manager, flows_service=flows_service
-                    )
-
-                # Update model values if provider or model changed (including removals that trigger fallback)
-                if body.llm_provider is not None or body.llm_model is not None or body.embedding_provider is not None or body.embedding_model is not None or provider_updated:
-                    await _update_langflow_model_values(current_config, flows_service)
-
-            except Exception as e:
-                logger.error(f"Failed to update Langflow settings: {str(e)}")
-                # Don't fail the entire settings update if Langflow update fails
-                # The config was still saved
+            task = asyncio.create_task(
+                _run_async_post_save_langflow_updates(
+                    session_manager=session_manager,
+                    update_mcp_servers=(
+                        body.embedding_provider is not None
+                        or body.embedding_model is not None
+                        or provider_updated
+                    ),
+                    update_model_values=(
+                        body.llm_provider is not None
+                        or body.llm_model is not None
+                        or body.embedding_provider is not None
+                        or body.embedding_model is not None
+                        or provider_updated
+                    ),
+                )
+            )
+            # Keep a strong reference until completion to avoid premature GC cancellation.
+            _background_tasks.add(task)
+            task.add_done_callback(_background_tasks.discard)
 
 
         set_fields = [k for k, v in body.model_dump().items() if v is not None]
@@ -1380,6 +1383,37 @@ async def _update_langflow_global_variables(config, flows_service=None):
         raise
 
 
+async def _run_async_post_save_langflow_updates(
+    session_manager,
+    update_mcp_servers: bool,
+    update_model_values: bool,
+) -> None:
+    """Apply post-save Langflow synchronization asynchronously."""
+    try:
+        current_config = get_openrag_config()
+        flows_service = _get_flows_service()
+
+        # Update global variables
+        await _update_langflow_global_variables(
+            current_config, flows_service=flows_service
+        )
+
+        # Update LLM client credentials when embedding selection changes
+        if update_mcp_servers:
+            await _update_mcp_servers_with_provider_credentials(
+                current_config, session_manager, flows_service=flows_service
+            )
+
+        # Update model values if provider/model changed (including removals/fallbacks)
+        if update_model_values:
+            await _update_langflow_model_values(current_config, flows_service)
+
+        logger.info("Completed asynchronous Langflow post-save sync")
+    except Exception as e:
+        # Do not fail user request if async sync fails; keep parity with existing behavior.
+        logger.error(f"Failed to update Langflow settings asynchronously: {str(e)}")
+
+
 async def _update_mcp_servers_with_provider_credentials(config, session_manager = None, flows_service=None):
     # Update MCP servers with provider credentials
     try:

{openrag-0.4.0.dev1 → openrag-0.4.0.dev3}/src/models/processors.py

@@ -1,7 +1,11 @@
 from typing import Any
 from .tasks import UploadTask, FileTask
 from utils.logging_config import get_logger
-from utils.file_utils import get_file_extension, clean_connector_filename
+from utils.file_utils import (
+    get_file_extension,
+    clean_connector_filename,
+    get_filename_aliases,
+)
 
 logger = get_logger(__name__)
 
@@ -72,19 +76,37 @@ class TaskProcessor:
         max_retries = 3
         retry_delay = 1.0
 
+        candidate_filenames = get_filename_aliases(filename)
+        if not candidate_filenames:
+            return False
+        # Keep track of aliases that still need checking across retries.
+        # If one alias was already checked successfully with no hits, we avoid
+        # re-querying it when another alias fails transiently.
+        pending_candidates = list(candidate_filenames)
+        # Retry strategy: only retry aliases that have not completed successfully.
+        # This avoids re-querying aliases already checked with no hits when a later
+        # alias fails transiently (e.g., timeout).
+
         for attempt in range(max_retries):
             try:
-                # Search for any document with this exact filename
-                search_body = build_filename_search_body(filename, size=1, source=False)
-
-                response = await opensearch_client.search(
-                    index=get_index_name(),
-                    body=search_body
-                )
-
-                # Check if any hits were found
-                hits = response.get("hits", {}).get("hits", [])
-                return len(hits) > 0
+                i = 0
+                while i < len(pending_candidates):
+                    candidate = pending_candidates[i]
+                    search_body = build_filename_search_body(
+                        candidate, size=1, source=False
+                    )
+                    response = await opensearch_client.search(
+                        index=get_index_name(),
+                        body=search_body
+                    )
+                    hits = response.get("hits", {}).get("hits", [])
+                    if hits:
+                        return True
+                    # Successfully checked this alias with no hits; don't
+                    # re-query it on future retries.
+                    pending_candidates.pop(i)
+                    continue
+                return False
 
             except (asyncio.TimeoutError, Exception) as e:
                 if attempt == max_retries - 1:
@@ -123,15 +145,21 @@ class TaskProcessor:
         from utils.opensearch_queries import build_filename_delete_body
 
         try:
-            # Delete all documents with this filename
-            delete_body = build_filename_delete_body(filename)
-
-            response = await opensearch_client.delete_by_query(
-                index=get_index_name(),
-                body=delete_body
-            )
-
-            deleted_count = response.get("deleted", 0)
+            deleted_count = 0
+            candidate_filenames = get_filename_aliases(filename)
+            if not candidate_filenames:
+                logger.info(
+                    "Skipped delete_by_filename due to empty filename input",
+                    filename=filename,
+                )
+                return
+            for candidate in candidate_filenames:
+                delete_body = build_filename_delete_body(candidate)
+                response = await opensearch_client.delete_by_query(
+                    index=get_index_name(),
+                    body=delete_body
+                )
+                deleted_count += response.get("deleted", 0)
             logger.info(
                 "Deleted existing document chunks",
                 filename=filename,

{openrag-0.4.0.dev1 → openrag-0.4.0.dev3/src/openrag.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrag
-Version: 0.4.0.dev1
+Version: 0.4.0.dev3
 Summary: OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations.
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console

{openrag-0.4.0.dev1 → openrag-0.4.0.dev3}/src/services/search_service.py

@@ -1,5 +1,6 @@
 import copy
 import json
+from collections import Counter
 from typing import Any, Dict
 from agentd.tool_decorator import tool
 from config.settings import EMBED_MODEL, clients, get_embedding_model, get_index_name, WATSONX_EMBEDDING_DIMENSIONS
@@ -317,10 +318,25 @@ class SearchService:
                                 "query": query,
                                 "fields": ["text^2", "filename^1.5"],
                                 "type": "best_fields",
-                                "fuzziness": "AUTO",
+                                "operator": "or",
+                                "fuzziness": "AUTO:4,7",
                                 "boost": 0.3,  # 30% weight for keyword search
                             }
                         },
+                        {
+                            # Prefix fallback for partial input (e.g. "vita" -> "vitamin").
+                            # Avoid bool_prefix here because our current mappings are:
+                            # - text: standard "text" (not search_as_you_type / edge-ngram)
+                            # - filename: "keyword"
+                            # match_phrase_prefix with a bounded expansion is safer.
+                            "match_phrase_prefix": {
+                                "text": {
+                                    "query": query,
+                                    "max_expansions": 50,
+                                    "boost": 0.25,
+                                }
+                            }
+                        },
                     ],
                     "minimum_should_match": 1,
                     "filter": all_filters,
@@ -481,15 +497,62 @@ class SearchService:
                 }
             )
 
+        # If query text appears verbatim in one subset of files, prefer those files
+        # to avoid broad semantic spillover for unique lookups.
+        normalized_query = query.strip().lower()
+        aggregations = results.get("aggregations", {})
+        if (
+            normalized_query
+            and not is_wildcard_match_all
+            and len(normalized_query) >= 4
+        ):
+            exact_files = {
+                filename
+                for chunk in chunks
+                for filename in [chunk.get("filename")]
+                if isinstance(filename, str)
+                and (
+                    normalized_query in filename.lower()
+                    or (
+                        isinstance(chunk.get("text"), str)
+                        and normalized_query in chunk.get("text", "").lower()
+                    )
+                )
+            }
+            if exact_files:
+                chunks = [chunk for chunk in chunks if chunk.get("filename") in exact_files]
+
+                def _build_terms_agg(field: str) -> Dict[str, Any]:
+                    counts = Counter(
+                        value
+                        for chunk in chunks
+                        for value in [chunk.get(field)]
+                        if isinstance(value, str) and value
+                    )
+                    return {
+                        "doc_count_error_upper_bound": 0,
+                        "sum_other_doc_count": 0,
+                        "buckets": [
+                            {"key": key, "doc_count": count}
+                            for key, count in counts.most_common()
+                        ],
+                    }
+
+                # Keep aggregations consistent with the post-filtered result set.
+                aggregations = {
+                    **aggregations,
+                    "data_sources": _build_terms_agg("filename"),
+                    "document_types": _build_terms_agg("mimetype"),
+                    "owners": _build_terms_agg("owner"),
+                    "connector_types": _build_terms_agg("connector_type"),
+                    "embedding_models": _build_terms_agg("embedding_model"),
+                }
+
         # Return both transformed results and aggregations
         return {
             "results": chunks,
-            "aggregations": results.get("aggregations", {}),
-            "total": (
-                results.get("hits", {}).get("total", {}).get("value")
-                if isinstance(results.get("hits", {}).get("total"), dict)
-                else results.get("hits", {}).get("total")
-            ),
+            "aggregations": aggregations,
+            "total": len(chunks),
         }
 
     async def search(

{openrag-0.4.0.dev1 → openrag-0.4.0.dev3}/src/utils/file_utils.py

@@ -98,4 +98,32 @@ def clean_connector_filename(filename: str, mimetype: str) -> str:
         return clean_name
     if not clean_name.lower().endswith(suffix.lower()):
         return clean_name + suffix
-    return clean_name
+    return clean_name
+
+
+def get_filename_aliases(filename: str) -> list[str]:
+    """Return equivalent filename variants used by ingestion/indexing.
+
+    Legacy Langflow ingest indexes `.txt` uploads as `.md` (see
+    `LangflowFileProcessor`). The alias always uses a lowercase extension
+    to match the rename behavior:
+      `original_filename[:-4] + ".md"`
+    So `"FOO.TXT"` aliases to `"FOO.md"`, not `"FOO.MD"`.
+
+    This helper keeps duplicate detection/deletion consistent by checking
+    both `.txt` and `.md` forms.
+    """
+    normalized = (filename or "").strip()
+    if not normalized:
+        return []
+
+    aliases = [normalized]
+    lower_name = normalized.lower()
+
+    if lower_name.endswith(".txt"):
+        aliases.append(normalized[:-4] + ".md")
+    elif lower_name.endswith(".md"):
+        aliases.append(normalized[:-3] + ".txt")
+
+    # Keep order stable while removing duplicates.
+    return list(dict.fromkeys(aliases))