orchestrator-core 4.5.0a6__py3-none-any.whl → 4.5.0a8__py3-none-any.whl

orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "4.5.0a6"
+__version__ = "4.5.0a8"
 
 
 from structlog import get_logger
@@ -25,18 +25,9 @@ logger.info("Starting the orchestrator", version=__version__)
 from orchestrator.llm_settings import llm_settings
 from orchestrator.settings import app_settings
 
-if llm_settings.LLM_ENABLED:
-    try:
-        from importlib import import_module
+if llm_settings.SEARCH_ENABLED or llm_settings.AGENT_ENABLED:
 
-        import_module("pydantic_ai")
-        from orchestrator.agentic_app import AgenticOrchestratorCore as OrchestratorCore
-
-    except ImportError:
-        logger.error(
-            "Unable to import 'pydantic_ai' module, please install the orchestrator with llm dependencies. `pip install orchestrator-core[llm]",
-        )
-        exit(1)
+    from orchestrator.agentic_app import LLMOrchestratorCore as OrchestratorCore
 else:
     from orchestrator.app import OrchestratorCore  # type: ignore[assignment]
 

orchestrator/agentic_app.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 """The main application module.
 
-This module contains the main `AgenticOrchestratorCore` class for the `FastAPI` backend and
-provides the ability to run the CLI.
+This module contains the main `LLMOrchestratorCore` class for the `FastAPI` backend and
+provides the ability to run the CLI with LLM features (search and/or agent).
 """
 # Copyright 2019-2025 SURF
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,65 +16,84 @@ provides the ability to run the CLI.
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import typer
-from pydantic_ai.models.openai import OpenAIModel
-from pydantic_ai.toolsets import FunctionToolset
 from structlog import get_logger
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.cli.main import app as cli_app
 from orchestrator.llm_settings import LLMSettings, llm_settings
 
+if TYPE_CHECKING:
+    from pydantic_ai.models.openai import OpenAIModel
+    from pydantic_ai.toolsets import FunctionToolset
+
 logger = get_logger(__name__)
 
 
-class AgenticOrchestratorCore(OrchestratorCore):
+class LLMOrchestratorCore(OrchestratorCore):
     def __init__(
         self,
         *args: Any,
-        llm_model: OpenAIModel | str = "gpt-4o-mini",
         llm_settings: LLMSettings = llm_settings,
-        agent_tools: list[FunctionToolset] | None = None,
+        agent_model: "OpenAIModel | str | None" = None,
+        agent_tools: "list[FunctionToolset] | None" = None,
         **kwargs: Any,
     ) -> None:
-        """Initialize the `AgenticOrchestratorCore` class.
+        """Initialize the `LLMOrchestratorCore` class.
 
-        This class takes the same arguments as the `OrchestratorCore` class.
+        This class extends `OrchestratorCore` with LLM features (search and agent).
+        It runs the search migration and mounts the agent endpoint based on feature flags.
 
         Args:
             *args: All the normal arguments passed to the `OrchestratorCore` class.
-            llm_model: An OpenAI model class or string, not limited to OpenAI models (gpt-4o-mini etc)
             llm_settings: A class of settings for the LLM
+            agent_model: Override the agent model (defaults to llm_settings.AGENT_MODEL)
             agent_tools: A list of tools that can be used by the agent
             **kwargs: Additional arguments passed to the `OrchestratorCore` class.
 
         Returns:
             None
         """
-        self.llm_model = llm_model
-        self.agent_tools = agent_tools
         self.llm_settings = llm_settings
+        self.agent_model = agent_model or llm_settings.AGENT_MODEL
+        self.agent_tools = agent_tools
 
         super().__init__(*args, **kwargs)
 
-        logger.info("Mounting the agent")
-        self.register_llm_integration()
-
-    def register_llm_integration(self) -> None:
-        """Mount the Agent endpoint.
-
-        This helper mounts the agent endpoint on the application.
-
-        Returns:
-            None
-
-        """
-        from orchestrator.search.agent import build_agent_app
-
-        agent_app = build_agent_app(self.llm_model, self.agent_tools)
-        self.mount("/agent", agent_app)
+        # Run search migration if search or agent is enabled
+        if self.llm_settings.SEARCH_ENABLED or self.llm_settings.AGENT_ENABLED:
+            logger.info("Running search migration")
+            try:
+                from orchestrator.db import db
+                from orchestrator.search.llm_migration import run_migration
+
+                with db.engine.begin() as connection:
+                    run_migration(connection)
+            except ImportError as e:
+                logger.error(
+                    "Unable to run search migration. Please install search dependencies: "
+                    "`pip install orchestrator-core[search]`",
+                    error=str(e),
+                )
+                raise
+
+        # Mount agent endpoint if agent is enabled
+        if self.llm_settings.AGENT_ENABLED:
+            logger.info("Initializing agent features", model=self.agent_model)
+            try:
+                from orchestrator.search.agent import build_agent_router
+
+                agent_app = build_agent_router(self.agent_model, self.agent_tools)
+                self.mount("/agent", agent_app)
+            except ImportError as e:
+                logger.error(
+                    "Unable to initialize agent features. Please install agent dependencies: "
+                    "`pip install orchestrator-core[agent]`",
+                    error=str(e),
+                )
+                raise
 
 
 main_typer_app = typer.Typer()

orchestrator/api/api_v1/api.py

@@ -89,7 +89,7 @@ api_router.include_router(
     ws.router, prefix="/ws", tags=["Core", "Events"]
 )  # Auth on the websocket is handled in the Websocket Manager
 
-if llm_settings.LLM_ENABLED:
+if llm_settings.SEARCH_ENABLED:
     from orchestrator.api.api_v1.endpoints import search
 
     api_router.include_router(

orchestrator/api/api_v1/endpoints/search.py

@@ -24,6 +24,7 @@ from orchestrator.db import (
     db,
 )
 from orchestrator.domain.base import SubscriptionModel
+from orchestrator.domain.context_cache import cache_subscription_models
 from orchestrator.schemas.search import (
     PageInfoSchema,
     PathsResponse,
@@ -192,17 +193,22 @@ async def search_subscriptions(
     page_info = PageInfoSchema(has_next_page=has_next_page, next_page_cursor=next_page_cursor)
 
     search_info_map = {res.entity_id: res for res in search_response.results}
-    results_data = []
-    for sub_id, search_info in search_info_map.items():
-        subscription_model = SubscriptionModel.from_subscription(sub_id)
-        sub_data = subscription_model.model_dump(exclude_unset=False)
-        search_result_item = SubscriptionSearchResult(
-            subscription=format_special_types(sub_data),
+
+    with cache_subscription_models():
+        subscriptions_data = {
+            sub_id: SubscriptionModel.from_subscription(sub_id).model_dump(exclude_unset=False)
+            for sub_id in search_info_map
+        }
+
+    results_data = [
+        SubscriptionSearchResult(
+            subscription=format_special_types(subscriptions_data[sub_id]),
             score=search_info.score,
             perfect_match=search_info.perfect_match,
             matching_field=search_info.matching_field,
         )
-        results_data.append(search_result_item)
+        for sub_id, search_info in search_info_map.items()
+    ]
 
     return SearchResultsSchema(data=results_data, page_info=page_info, search_metadata=search_response.metadata)
 

orchestrator/cli/main.py

@@ -25,8 +25,8 @@ app.add_typer(scheduler.app, name="scheduler", help="Access all the scheduler fu
 app.add_typer(database.app, name="db", help="Interact with the application database")
 app.add_typer(generate.app, name="generate", help="Generate products, workflows and other artifacts")
 
-if llm_settings.LLM_ENABLED:
-    from orchestrator.cli import index_llm, resize_embedding, search_explore, speedtest
+if llm_settings.SEARCH_ENABLED:
+    from orchestrator.cli.search import index_llm, resize_embedding, search_explore, speedtest
 
     app.add_typer(index_llm.app, name="index", help="(Re-)Index the search table.")
     app.add_typer(search_explore.app, name="search", help="Try out different search types.")

orchestrator/cli/search/__init__.py

@@ -0,0 +1,32 @@
+# Copyright 2019-2020 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typer
+
+from orchestrator.cli.search import index_llm, resize_embedding, search_explore, speedtest
+
+
+def register_commands(app: typer.Typer) -> None:
+    """Register all LLM/search related commands to the main app."""
+    app.add_typer(index_llm.app, name="index", help="(Re-)Index the search table.")
+    app.add_typer(search_explore.app, name="search", help="Try out different search types.")
+    app.add_typer(
+        resize_embedding.app,
+        name="embedding",
+        help="Resize the vector dimension of the embedding column in the search table.",
+    )
+    app.add_typer(
+        speedtest.app,
+        name="speedtest",
+        help="Search performance testing and analysis.",
+    )

orchestrator/llm_settings.py

@@ -18,7 +18,10 @@ logger = get_logger(__name__)
 
 
 class LLMSettings(BaseSettings):
-    LLM_ENABLED: bool = False  # Default to false
+    # Feature flags for LLM functionality
+    SEARCH_ENABLED: bool = False  # Enable search/indexing with embeddings
+    AGENT_ENABLED: bool = False  # Enable agentic functionality
+
     # Pydantic-ai Agent settings
     AGENT_MODEL: str = "gpt-4o-mini"  # See pydantic-ai docs for supported models.
     AGENT_MODEL_VERSION: str = "2025-01-01-preview"
@@ -30,11 +33,11 @@ class LLMSettings(BaseSettings):
         0.1, description="Safety margin as a percentage (e.g., 0.1 for 10%) for token budgeting.", ge=0, le=1
     )
 
-    # The following settings are only needed for local models.
+    # The following settings are only needed for local models or system constraints.
     # By default, they are set conservative assuming a small model like All-MiniLM-L6-V2.
     OPENAI_BASE_URL: str | None = None
     EMBEDDING_FALLBACK_MAX_TOKENS: int | None = 512
-    EMBEDDING_MAX_BATCH_SIZE: int | None = 32
+    EMBEDDING_MAX_BATCH_SIZE: int | None = None
 
     # General LiteLLM settings
     LLM_MAX_RETRIES: int = 3

orchestrator/search/agent/__init__.py

@@ -14,8 +14,8 @@
 # This module requires: pydantic-ai==0.7.0, ag-ui-protocol>=0.1.8
 
 
-from orchestrator.search.agent.agent import build_agent_app
+from orchestrator.search.agent.agent import build_agent_router
 
 __all__ = [
-    "build_agent_app",
+    "build_agent_router",
 ]

orchestrator/search/agent/agent.py

@@ -14,13 +14,13 @@
 from typing import Any
 
 import structlog
-from fastapi import FastAPI, HTTPException
-from pydantic_ai.ag_ui import StateDeps
+from fastapi import APIRouter, HTTPException, Request
+from pydantic_ai.ag_ui import StateDeps, handle_ag_ui_request
 from pydantic_ai.agent import Agent
 from pydantic_ai.models.openai import OpenAIModel
 from pydantic_ai.settings import ModelSettings
 from pydantic_ai.toolsets import FunctionToolset
-from starlette.types import ASGIApp
+from starlette.responses import Response
 
 from orchestrator.search.agent.prompts import get_base_instructions, get_dynamic_instructions
 from orchestrator.search.agent.state import SearchState
@@ -29,17 +29,9 @@ from orchestrator.search.agent.tools import search_toolset
 logger = structlog.get_logger(__name__)
 
 
-def _disabled_agent_app(reason: str) -> FastAPI:
-    app = FastAPI(title="Agent disabled")
+def build_agent_router(model: str | OpenAIModel, toolsets: list[FunctionToolset[Any]] | None = None) -> APIRouter:
+    router = APIRouter()
 
-    @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"])
-    async def _disabled(path: str) -> None:
-        raise HTTPException(status_code=503, detail=f"Agent disabled: {reason}")
-
-    return app
-
-
-def build_agent_app(model: str | OpenAIModel, toolsets: list[FunctionToolset[Any]] | None = None) -> ASGIApp:
     try:
         toolsets = toolsets + [search_toolset] if toolsets else [search_toolset]
 
@@ -54,7 +46,17 @@ def build_agent_app(model: str | OpenAIModel, toolsets: list[FunctionToolset[Any
         agent.instructions(get_base_instructions)
         agent.instructions(get_dynamic_instructions)
 
-        return agent.to_ag_ui(deps=StateDeps(SearchState()))
+        @router.post("/")
+        async def agent_endpoint(request: Request) -> Response:
+            return await handle_ag_ui_request(agent, request, deps=StateDeps(SearchState()))
+
+        return router
     except Exception as e:
         logger.error("Agent init failed; serving disabled stub.", error=str(e))
-        return _disabled_agent_app(str(e))
+        error_msg = f"Agent disabled: {str(e)}"
+
+        @router.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"])
+        async def _disabled(path: str) -> None:
+            raise HTTPException(status_code=503, detail=error_msg)
+
+        return router

orchestrator/search/agent/prompts.py

@@ -62,7 +62,7 @@ async def get_base_instructions() -> str:
         ---
         ### 4. Critical Rules
 
-        - **NEVER GUESS PATHS**: You *must* verify every filter path by calling `discover_filter_paths` first. If a path does not exist, you must inform the user and not include it in the `FilterTree`.
+        - **NEVER GUESS PATHS IN THE DATABASE**: You *must* verify every filter path by calling `discover_filter_paths` first. If a path does not exist, you may attempt to map the question on an existing paths that are valid and available from `discover_filter_paths`. If you cannot infer a match, inform the user and do not include it in the `FilterTree`.
         - **USE FULL PATHS**: Always use the full, unambiguous path returned by the discovery tool.
         - **MATCH OPERATORS**: Only use operators that are compatible with the field type as confirmed by `get_filter_operators`.
         """

orchestrator/search/docs/running_local_text_embedding_inference.md

@@ -18,6 +18,7 @@ Point your backend to the local endpoint and declare the new vector size:
 ```env
 OPENAI_BASE_URL=http://localhost:8080/v1
 EMBEDDING_DIMENSION=384
+EMBEDDING_MAX_BATCH_SIZE=32 # Not required when using OpenAI embeddings
 ```
 
 Depending on the model, you might want to change the `EMBEDDING_FALLBACK_MAX_TOKENS` and `EMBEDDING_MAX_BATCH_SIZE` settings, which are set conservatively and according to the requirements of the setup used in this example.

orchestrator/search/filters/base.py

@@ -226,6 +226,38 @@ class FilterTree(BaseModel):
                 leaves.extend(child.get_all_leaves())
         return leaves
 
+    @staticmethod
+    def _build_correlates(
+        alias: Any, entity_id_col: SQLAColumn, entity_type_value: str | None
+    ) -> list[ColumnElement[bool]]:
+        """Build the correlation predicates that link the subquery to the outer query."""
+        correlates = [alias.entity_id == entity_id_col]
+        if entity_type_value is not None:
+            correlates.append(alias.entity_type == entity_type_value)
+        return correlates
+
+    @staticmethod
+    def _handle_ltree_filter(pf: PathFilter, alias: Any, correlates: list[ColumnElement[bool]]) -> ColumnElement[bool]:
+        """Handle path-only filters (has_component, not_has_component, ends_with)."""
+        # row-level predicate is always positive
+        positive = pf.condition.to_expression(alias.path, pf.path)
+        subq = select(1).select_from(alias).where(and_(*correlates, positive))
+        if pf.condition.op == FilterOp.NOT_HAS_COMPONENT:
+            return ~exists(subq)  # NOT at the entity level
+        return exists(subq)
+
+    @staticmethod
+    def _handle_value_filter(pf: PathFilter, alias: Any, correlates: list[ColumnElement[bool]]) -> ColumnElement[bool]:
+        """Handle value-based filters (equality, comparison, etc)."""
+        if "." not in pf.path:
+            path_pred = LtreeFilter(op=FilterOp.ENDS_WITH, value=pf.path).to_expression(alias.path, "")
+        else:
+            path_pred = alias.path == Ltree(pf.path)
+
+        value_pred = pf.to_expression(alias.value, alias.value_type)
+        subq = select(1).select_from(alias).where(and_(*correlates, path_pred, value_pred))
+        return exists(subq)
+
     def to_expression(
         self,
         entity_id_col: SQLAColumn,
@@ -241,35 +273,18 @@ class FilterTree(BaseModel):
         Returns:
             ColumnElement[bool]: A SQLAlchemy expression suitable for use in a WHERE clause.
         """
+        from sqlalchemy.orm import aliased
 
         alias_idx = count(1)
 
         def leaf_exists(pf: PathFilter) -> ColumnElement[bool]:
-            from sqlalchemy.orm import aliased
-
+            """Convert a PathFilter into an EXISTS subquery."""
             alias = aliased(AiSearchIndex, name=f"flt_{next(alias_idx)}")
-
-            correlates = [alias.entity_id == entity_id_col]
-            if entity_type_value is not None:
-                correlates.append(alias.entity_type == entity_type_value)
+            correlates = self._build_correlates(alias, entity_id_col, entity_type_value)
 
             if isinstance(pf.condition, LtreeFilter):
-                # row-level predicate is always positive
-                positive = pf.condition.to_expression(alias.path, pf.path)
-                subq = select(1).select_from(alias).where(and_(*correlates, positive))
-                if pf.condition.op == FilterOp.NOT_HAS_COMPONENT:
-                    return ~exists(subq)  # NOT at the entity level
-                return exists(subq)
-
-            # value leaf: path predicate + typed value compare
-            if "." not in pf.path:
-                path_pred = LtreeFilter(op=FilterOp.ENDS_WITH, value=pf.path).to_expression(alias.path, "")
-            else:
-                path_pred = alias.path == Ltree(pf.path)
-
-            value_pred = pf.to_expression(alias.value, alias.value_type)
-            subq = select(1).select_from(alias).where(and_(*correlates, path_pred, value_pred))
-            return exists(subq)
+                return self._handle_ltree_filter(pf, alias, correlates)
+            return self._handle_value_filter(pf, alias, correlates)
 
         def compile_node(node: FilterTree | PathFilter) -> ColumnElement[bool]:
             if isinstance(node, FilterTree):

orchestrator/search/indexing/indexer.py

@@ -226,9 +226,7 @@ class Indexer:
         safe_margin = int(max_ctx * llm_settings.EMBEDDING_SAFE_MARGIN_PERCENT)
         token_budget = max(1, max_ctx - safe_margin)
 
-        max_batch_size = None
-        if llm_settings.OPENAI_BASE_URL:  # We are using a local model
-            max_batch_size = llm_settings.EMBEDDING_MAX_BATCH_SIZE
+        max_batch_size = llm_settings.EMBEDDING_MAX_BATCH_SIZE
 
         for entity_id, field in fields_to_upsert:
             if field.value_type.is_embeddable(field.value):

orchestrator/search/indexing/tasks.py

@@ -15,6 +15,7 @@ import structlog
 from sqlalchemy.orm import Query
 
 from orchestrator.db import db
+from orchestrator.domain.context_cache import cache_subscription_models
 from orchestrator.search.core.types import EntityType
 from orchestrator.search.indexing.indexer import Indexer
 from orchestrator.search.indexing.registry import ENTITY_CONFIG_REGISTRY
@@ -63,4 +64,6 @@ def run_indexing_for_entity(
     entities = db.session.execute(stmt).scalars()
 
     indexer = Indexer(config=config, dry_run=dry_run, force_index=force_index, chunk_size=chunk_size)
-    indexer.run(entities)
+
+    with cache_subscription_models():
+        indexer.run(entities)

orchestrator/search/llm_migration.py

@@ -0,0 +1,102 @@
+# Copyright 2019-2025 SURF
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Simple search migration function that runs when SEARCH_ENABLED = True."""
+
+from sqlalchemy import text
+from sqlalchemy.engine import Connection
+from structlog import get_logger
+
+from orchestrator.search.core.types import FieldType
+
+logger = get_logger(__name__)
+
+TABLE = "ai_search_index"
+TARGET_DIM = 1536
+
+
+def run_migration(connection: Connection) -> None:
+    """Run LLM migration with ON CONFLICT DO NOTHING pattern."""
+    logger.info("Running LLM migration")
+
+    try:
+        # Create PostgreSQL extensions
+        connection.execute(text("CREATE EXTENSION IF NOT EXISTS ltree;"))
+        connection.execute(text("CREATE EXTENSION IF NOT EXISTS unaccent;"))
+        connection.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm;"))
+        connection.execute(text("CREATE EXTENSION IF NOT EXISTS vector;"))
+
+        # Create field_type enum
+        field_type_values = "', '".join([ft.value for ft in FieldType])
+        connection.execute(
+            text(
+                f"""
+            DO $$
+            BEGIN
+                IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'field_type') THEN
+                    CREATE TYPE field_type AS ENUM ('{field_type_values}');
+                END IF;
+            END $$;
+        """
+            )
+        )
+
+        # Create table with ON CONFLICT DO NOTHING pattern
+        connection.execute(
+            text(
+                f"""
+            CREATE TABLE IF NOT EXISTS {TABLE} (
+                entity_type TEXT NOT NULL,
+                entity_id UUID NOT NULL,
+                path LTREE NOT NULL,
+                value TEXT NOT NULL,
+                embedding VECTOR({TARGET_DIM}),
+                content_hash VARCHAR(64) NOT NULL,
+                value_type field_type NOT NULL DEFAULT '{FieldType.STRING.value}',
+                CONSTRAINT pk_ai_search_index PRIMARY KEY (entity_id, path)
+            );
+        """
+            )
+        )
+
+        # Drop default
+        connection.execute(text(f"ALTER TABLE {TABLE} ALTER COLUMN value_type DROP DEFAULT;"))
+
+        # Create indexes with IF NOT EXISTS
+        connection.execute(text(f"CREATE INDEX IF NOT EXISTS ix_ai_search_index_entity_id ON {TABLE} (entity_id);"))
+        connection.execute(
+            text(f"CREATE INDEX IF NOT EXISTS idx_ai_search_index_content_hash ON {TABLE} (content_hash);")
+        )
+        connection.execute(
+            text(f"CREATE INDEX IF NOT EXISTS ix_flat_path_gist ON {TABLE} USING GIST (path gist_ltree_ops);")
+        )
+        connection.execute(text(f"CREATE INDEX IF NOT EXISTS ix_flat_path_btree ON {TABLE} (path);"))
+        connection.execute(
+            text(f"CREATE INDEX IF NOT EXISTS ix_flat_value_trgm ON {TABLE} USING GIN (value gin_trgm_ops);")
+        )
+        connection.execute(
+            text(
+                f"CREATE INDEX IF NOT EXISTS ix_flat_embed_hnsw ON {TABLE} USING HNSW (embedding vector_l2_ops) WITH (m = 16, ef_construction = 64);"
+            )
+        )
+
+        connection.commit()
+        logger.info("LLM migration completed successfully")
+
+    except Exception as e:
+        logger.error("LLM migration failed", error=str(e))
+        raise Exception(
+            f"LLM migration failed. This likely means the pgvector extension "
+            f"is not installed. Please install pgvector and ensure your PostgreSQL "
+            f"version supports it. Error: {e}"
+        ) from e

orchestrator/search/retrieval/engine.py

@@ -86,11 +86,7 @@ def _format_response(
 
 
 def _extract_matching_field_from_filters(filters: FilterTree) -> MatchingField | None:
-    """Extract the first path filter to use as matching field for structured searches.
-
-    TODO: Should we allow a list of matched fields in the MatchingField model?
-    We need a different approach, probably a cross join in StructuredRetriever.
-    """
+    """Extract the first path filter to use as matching field for structured searches."""
     leaves = filters.get_all_leaves()
     if len(leaves) != 1:
         return None

orchestrator/search/retrieval/retrievers/base.py

@@ -112,7 +112,8 @@ class Retriever(ABC):
 
     def _quantize_score_for_pagination(self, score_value: float) -> BindParameter[Decimal]:
         """Convert score value to properly quantized Decimal parameter for pagination."""
-        pas_dec = Decimal(str(score_value)).quantize(Decimal("0.000000000001"))
+        quantizer = Decimal(1).scaleb(-self.SCORE_PRECISION)
+        pas_dec = Decimal(str(score_value)).quantize(quantizer)
         return literal(pas_dec, type_=self.SCORE_NUMERIC_TYPE)
 
     @property

orchestrator/search/retrieval/retrievers/hybrid.py

@@ -11,8 +11,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import TypedDict
+
 from sqlalchemy import BindParameter, Select, and_, bindparam, case, cast, func, literal, or_, select
-from sqlalchemy.sql.expression import ColumnElement
+from sqlalchemy.sql.expression import ColumnElement, Label
+from sqlalchemy.types import TypeEngine
 
 from orchestrator.db.models import AiSearchIndex
 from orchestrator.search.core.types import SearchMetadata
@@ -21,6 +24,102 @@ from ..pagination import PaginationParams
 from .base import Retriever
 
 
+class RrfScoreSqlComponents(TypedDict):
+    """SQL expression components of the RRF hybrid score calculation."""
+
+    rrf_num: ColumnElement
+    perfect: Label
+    beta: ColumnElement
+    rrf_max: ColumnElement
+    fused_num: ColumnElement
+    normalized_score: ColumnElement
+
+
+def compute_rrf_hybrid_score_sql(
+    sem_rank_col: ColumnElement,
+    fuzzy_rank_col: ColumnElement,
+    avg_fuzzy_score_col: ColumnElement,
+    k: int,
+    perfect_threshold: float,
+    n_sources: int = 2,
+    margin_factor: float = 0.05,
+    score_numeric_type: TypeEngine | None = None,
+) -> RrfScoreSqlComponents:
+    """Compute RRF (Reciprocal Rank Fusion) hybrid score as SQL expressions for database execution.
+
+    This function implements the core scoring logic for hybrid search combining semantic
+    and fuzzy ranking. It computes:
+    1. Base RRF score from both ranks
+    2. Perfect match detection and boosting
+    3. Dynamic beta parameter based on k and n_sources
+    4. Normalized final score in [0, 1] range
+
+    Args:
+        sem_rank_col: SQLAlchemy column expression for semantic rank
+        fuzzy_rank_col: SQLAlchemy column expression for fuzzy rank
+        avg_fuzzy_score_col: SQLAlchemy column expression for average fuzzy score
+        k: RRF constant controlling rank influence (typically 60)
+        perfect_threshold: Threshold for perfect match boost (typically 0.9)
+        n_sources: Number of ranking sources being fused (default: 2 for semantic + fuzzy)
+        margin_factor: Margin above rrf_max as fraction (default: 0.05 = 5%)
+        score_numeric_type: SQLAlchemy numeric type for casting scores
+
+    Returns:
+        RrfScoreSqlComponents: Dictionary of SQL expressions for score components
+            - rrf_num: Raw RRF score (cast to numeric type if provided)
+            - perfect: Perfect match flag (1 if avg_fuzzy_score >= threshold, else 0)
+            - beta: Boost amount for perfect matches
+            - rrf_max: Maximum possible RRF score
+            - fused_num: RRF + perfect boost
+            - normalized_score: Final score normalized to [0, 1]
+
+    Note:
+        -   Keep margin_factor small to avoid compressing perfects near 1 after normalization.
+
+        -   The `beta` boost is calculated to be greater than the maximum possible standard
+            RRF score (`rrf_max`). This guarantees that any item flagged as a "perfect" match
+            will always rank above any non-perfect match.
+
+        -   This function assumes that rank columns do not
+            contain `NULL` values. A `NULL` in any rank column will result in a `NULL` final score
+            for that item.
+    """
+    # RRF (rank-based): sum of 1/(k + rank_i) for each ranking source
+    rrf_raw = (1.0 / (k + sem_rank_col)) + (1.0 / (k + fuzzy_rank_col))
+    rrf_num = cast(rrf_raw, score_numeric_type) if score_numeric_type else rrf_raw
+
+    # Perfect flag to boost near perfect fuzzy matches
+    perfect = case((avg_fuzzy_score_col >= perfect_threshold, 1), else_=0).label("perfect_match")
+
+    # Dynamic beta based on k and number of sources
+    # rrf_max = n_sources / (k + 1)
+    k_num = literal(float(k), type_=score_numeric_type) if score_numeric_type else literal(float(k))
+    n_sources_lit = (
+        literal(float(n_sources), type_=score_numeric_type) if score_numeric_type else literal(float(n_sources))
+    )
+    rrf_max = n_sources_lit / (k_num + literal(1.0, type_=score_numeric_type if score_numeric_type else None))
+
+    margin = rrf_max * literal(margin_factor, type_=score_numeric_type if score_numeric_type else None)
+    beta = rrf_max + margin
+
+    # Fused score: RRF + perfect match boost
+    perfect_casted = cast(perfect, score_numeric_type) if score_numeric_type else perfect
+    fused_num = rrf_num + beta * perfect_casted
+
+    # Normalize to [0,1] via the theoretical max (beta + rrf_max)
+    norm_den = beta + rrf_max
+    normalized_score = fused_num / norm_den
+
+    return RrfScoreSqlComponents(
+        rrf_num=rrf_num,
+        perfect=perfect,
+        beta=beta,
+        rrf_max=rrf_max,
+        fused_num=fused_num,
+        normalized_score=normalized_score,
+    )
+
+
 class RrfHybridRetriever(Retriever):
     """Reciprocal Rank Fusion of semantic and fuzzy ranking with parent-child retrieval."""
 
@@ -122,30 +221,20 @@ class RrfHybridRetriever(Retriever):
             )
         ).cte("ranked_results")
 
-        # RRF (rank-based)
-        rrf_raw = (1.0 / (self.k + ranked.c.sem_rank)) + (1.0 / (self.k + ranked.c.fuzzy_rank))
-        rrf_num = cast(rrf_raw, self.SCORE_NUMERIC_TYPE)
-
-        # Perfect flag to boost near perfect fuzzy matches as this most likely indicates the desired record.
-        perfect = case((ranked.c.avg_fuzzy_score >= 0.9, 1), else_=0).label("perfect_match")
-
-        # Dynamic beta based on k (and number of sources)
-        # rrf_max = n_sources / (k + 1)
-        k_num = literal(float(self.k), type_=self.SCORE_NUMERIC_TYPE)
-        n_sources = literal(2.0, type_=self.SCORE_NUMERIC_TYPE)  # semantic + fuzzy
-        rrf_max = n_sources / (k_num + literal(1.0, type_=self.SCORE_NUMERIC_TYPE))
-
-        # Choose a small positive margin above rrf_max to ensure strict separation
-        # Keep it small to avoid compressing perfects near 1 after normalization
-        margin = rrf_max * literal(0.05, type_=self.SCORE_NUMERIC_TYPE)  # 5% above bound
-        beta = rrf_max + margin
-
-        fused_num = rrf_num + beta * cast(perfect, self.SCORE_NUMERIC_TYPE)
+        # Compute RRF hybrid score
+        score_components = compute_rrf_hybrid_score_sql(
+            sem_rank_col=ranked.c.sem_rank,
+            fuzzy_rank_col=ranked.c.fuzzy_rank,
+            avg_fuzzy_score_col=ranked.c.avg_fuzzy_score,
+            k=self.k,
+            perfect_threshold=0.9,
+            score_numeric_type=self.SCORE_NUMERIC_TYPE,
+        )
 
-        # Normalize to [0,1] via the theoretical max (beta + rrf_max)
-        norm_den = beta + rrf_max
-        normalized_score = fused_num / norm_den
+        perfect = score_components["perfect"]
+        normalized_score = score_components["normalized_score"]
 
+        # Round to configured precision
         score = cast(
             func.round(cast(normalized_score, self.SCORE_NUMERIC_TYPE), self.SCORE_PRECISION),
             self.SCORE_NUMERIC_TYPE,

orchestrator/workflows/steps.py

@@ -156,7 +156,7 @@ def refresh_subscription_search_index(subscription: SubscriptionModel | None) ->
     """
     try:
         reset_search_index()
-        if llm_settings.LLM_ENABLED and subscription:
+        if llm_settings.SEARCH_ENABLED and subscription:
             run_indexing_for_entity(EntityType.SUBSCRIPTION, str(subscription.subscription_id))
     except Exception:
         # Don't fail workflow in case of unexpected error

{orchestrator_core-4.5.0a6.dist-info → orchestrator_core-4.5.0a8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 4.5.0a6
+Version: 4.5.0a8
 Summary: This is the orchestrator workflow engine.
 Author-email: SURF <automation-beheer@surf.nl>
 Requires-Python: >=3.11,<3.14
@@ -63,15 +63,17 @@ Requires-Dist: structlog>=25.4.0
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: typer==0.15.4
 Requires-Dist: uvicorn[standard]~=0.34.0
+Requires-Dist: pydantic-ai-slim ==0.7.0 ; extra == "agent"
+Requires-Dist: ag-ui-protocol>=0.1.8 ; extra == "agent"
+Requires-Dist: litellm>=1.75.7 ; extra == "agent"
 Requires-Dist: celery~=5.5.1 ; extra == "celery"
-Requires-Dist: pydantic-ai-slim ==0.7.0 ; extra == "llm"
-Requires-Dist: ag-ui-protocol>=0.1.8 ; extra == "llm"
-Requires-Dist: litellm>=1.75.7 ; extra == "llm"
+Requires-Dist: litellm>=1.75.7 ; extra == "search"
 Project-URL: Documentation, https://workfloworchestrator.org/orchestrator-core
 Project-URL: Homepage, https://workfloworchestrator.org/orchestrator-core
 Project-URL: Source, https://github.com/workfloworchestrator/orchestrator-core
+Provides-Extra: agent
 Provides-Extra: celery
-Provides-Extra: llm
+Provides-Extra: search
 
 # Orchestrator-Core
 

{orchestrator_core-4.5.0a6.dist-info → orchestrator_core-4.5.0a8.dist-info}/RECORD

@@ -1,8 +1,8 @@
-orchestrator/__init__.py,sha256=F2rD2p-hLcFjUslWKOUIoLQwd-MMglQNUA3EPCszEdk,1732
-orchestrator/agentic_app.py,sha256=bBMuH9Ub42nb8oFG0U00SzW_uQqnAayUX2tNs6yz1BM,2810
+orchestrator/__init__.py,sha256=bfe5yk8RiIvi9Pr2-Ym6xKS3wO9qHR93paLma0iVR9A,1449
+orchestrator/agentic_app.py,sha256=op7osw7KJRww90iYuWBt_bB5qI-sAkpG0fyr0liubQw,3968
 orchestrator/app.py,sha256=UPKQuDpg8MWNC6r3SRRbp6l9RBzwb00IMIaGRk-jbCU,13203
 orchestrator/exception_handlers.py,sha256=UsW3dw8q0QQlNLcV359bIotah8DYjMsj2Ts1LfX4ClY,1268
-orchestrator/llm_settings.py,sha256=PJ3vf5aEugVigHFU7iw9haQon_bC7Y268GTFhfFaQHs,2075
+orchestrator/llm_settings.py,sha256=RjOY-FRVd648HYa6Im8ni4h76KlLFja36zlGD14sPeY,2231
 orchestrator/log_config.py,sha256=1cPl_OXT4tEUyNxG8cwIWXrmadUm1E81vq0mdtrV-v4,1912
 orchestrator/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 orchestrator/security.py,sha256=iXFxGxab54aav7oHEKLAVkTgrQMJGHy6IYLojEnD7gI,2422
@@ -16,14 +16,14 @@ orchestrator/api/error_handling.py,sha256=YrPCxSa-DSa9KwqIMlXI-KGBGnbGIW5ukOPiik
 orchestrator/api/helpers.py,sha256=s0QRHYw8AvEmlkmRhuEzz9xixaZKUF3YuPzUVHkcoXk,6933
 orchestrator/api/models.py,sha256=z9BDBx7uI4KBHWbD_LVrLsqNQ0_w-Mg9Qiy7PR_rZhk,5996
 orchestrator/api/api_v1/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
-orchestrator/api/api_v1/api.py,sha256=7_alDkK-UfqCauCSCFfQzDIwZHVI_CrRvLCMuCR0408,3276
+orchestrator/api/api_v1/api.py,sha256=bWsvWgLap7b6ltu1BvwZpW7X2dEE6cQ7-WY0HcY7Yoo,3279
 orchestrator/api/api_v1/endpoints/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
 orchestrator/api/api_v1/endpoints/health.py,sha256=iaxs1XX1_250_gKNsspuULCV2GEMBjbtjsmfQTOvMAI,1284
 orchestrator/api/api_v1/endpoints/processes.py,sha256=OVbt6FgFnJ4aHaYGIg0cPoim8mxDpdzJ4TGAyfB_kPw,16269
 orchestrator/api/api_v1/endpoints/product_blocks.py,sha256=kZ6ywIOsS_S2qGq7RvZ4KzjvaS1LmwbGWR37AKRvWOw,2146
 orchestrator/api/api_v1/endpoints/products.py,sha256=BfFtwu9dZXEQbtKxYj9icc73GKGvAGMR5ytyf41nQlQ,3081
 orchestrator/api/api_v1/endpoints/resource_types.py,sha256=gGyuaDyOD0TAVoeFGaGmjDGnQ8eQQArOxKrrk4MaDzA,2145
-orchestrator/api/api_v1/endpoints/search.py,sha256=sxiS2n1I0ghfQifo8Vf4PsAoxxGjKKqPTtHcAiuWrPE,10780
+orchestrator/api/api_v1/endpoints/search.py,sha256=NooZcMXmlnD1NxdhFWlqF3jhmixF1DZYuUG8XtEVGjo,10885
 orchestrator/api/api_v1/endpoints/settings.py,sha256=5s-k169podZjgGHUbVDmSQwpY_3Cs_Bbf2PPtZIkBcw,6184
 orchestrator/api/api_v1/endpoints/subscription_customer_descriptions.py,sha256=1_6LtgQleoq3M6z_W-Qz__Bj3OFUweoPrUqHMwSH6AM,3288
 orchestrator/api/api_v1/endpoints/subscriptions.py,sha256=7KaodccUiMkcVnrFnK2azp_V_-hGudcIyhov5WwVGQY,9810
@@ -34,16 +34,12 @@ orchestrator/api/api_v1/endpoints/ws.py,sha256=1l7E0ag_sZ6UMfQPHlmew7ENwxjm6fflB
 orchestrator/cli/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
 orchestrator/cli/database.py,sha256=YkYAbCY2VPAa6mDW0PpNKG5wL4FuAQYD2CGl1_DQtEk,19595
 orchestrator/cli/generate.py,sha256=SBaYfRijXPF9r3VxarPKTiDzDcB6GBMMQvecQIb_ZLQ,7377
-orchestrator/cli/index_llm.py,sha256=RWPkFz5bxiznjpN1vMsSWeqcvYKB90DLL4pXQ92QJNI,2239
-orchestrator/cli/main.py,sha256=U4eAG_iT3JhmeB6yZnogB6KTM6kFlDUo7zY4qBdIHv4,1648
+orchestrator/cli/main.py,sha256=xGLc_cS2LoSIbK5qkMFE7GCnZoOi5kATgtmQDFNQU7E,1658
 orchestrator/cli/migrate_domain_models.py,sha256=WRXy_1OnziQwpsCFZXvjB30nDJtjj0ikVXy8YNLque4,20928
 orchestrator/cli/migrate_tasks.py,sha256=bju8XColjSZD0v3rS4kl-24dLr8En_H4-6enBmqd494,7255
 orchestrator/cli/migrate_workflows.py,sha256=nxUpx0vgEIc_8aJrjAyrw3E9Dt8JmaamTts8oiQ4vHY,8923
 orchestrator/cli/migration_helpers.py,sha256=C5tpkP5WEBr7G9S-1k1hgSI8ili6xd9Z5ygc9notaK0,4110
-orchestrator/cli/resize_embedding.py,sha256=ds830T26ADOD9vZS7psRHJVF_u2xar2d4vvIH1AOlww,4216
 orchestrator/cli/scheduler.py,sha256=2q6xT_XVOodY3e_qzIV98MWNvKvrbFpOJajWesj1fcs,1911
-orchestrator/cli/search_explore.py,sha256=SDn1DMN8a4roSPodIHl-KrNAvdHo5jTDUvMUFLVh1P4,8602
-orchestrator/cli/speedtest.py,sha256=QkQ_YhKh7TnNRX4lKjgrmF7DyufU9teLqw4CWkm52ko,4972
 orchestrator/cli/domain_gen_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 orchestrator/cli/domain_gen_helpers/fixed_input_helpers.py,sha256=uzpwsaau81hHSxNMOS9-o7kF-9_78R0f_UE0AvWooZQ,6775
 orchestrator/cli/domain_gen_helpers/helpers.py,sha256=tIPxn8ezED_xYZxH7ZAtQLwkDc6RNmLZVxWAoJ3a9lw,4203
@@ -106,6 +102,11 @@ orchestrator/cli/generator/templates/validate_product.j2,sha256=_gPNYS8dGOSpRm2E
 orchestrator/cli/helpers/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
 orchestrator/cli/helpers/input_helpers.py,sha256=pv5GTMuIWLzBE_bKNhn1XD_gxoqB0s1ZN4cnKkIIu5I,1139
 orchestrator/cli/helpers/print_helpers.py,sha256=b3ePg6HfBLKPYBBVr5XOA__JnFEMI5HBjbjov3CP8Po,859
+orchestrator/cli/search/__init__.py,sha256=K15_iW9ogR7xtX7qHDal4H09tmwVGnOBZWyPBLWhuzc,1274
+orchestrator/cli/search/index_llm.py,sha256=RWPkFz5bxiznjpN1vMsSWeqcvYKB90DLL4pXQ92QJNI,2239
+orchestrator/cli/search/resize_embedding.py,sha256=ds830T26ADOD9vZS7psRHJVF_u2xar2d4vvIH1AOlww,4216
+orchestrator/cli/search/search_explore.py,sha256=SDn1DMN8a4roSPodIHl-KrNAvdHo5jTDUvMUFLVh1P4,8602
+orchestrator/cli/search/speedtest.py,sha256=QkQ_YhKh7TnNRX4lKjgrmF7DyufU9teLqw4CWkm52ko,4972
 orchestrator/config/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
 orchestrator/config/assignee.py,sha256=9mFFe9hoi2NCkXFOKL2pU2aveBzcZhljSvqUnh55vrk,780
 orchestrator/db/__init__.py,sha256=41_v-oeX5SMnwH2uIeBzscoST3FRXdpkEFDE5AoQR1E,3498
@@ -254,7 +255,6 @@ orchestrator/migrations/versions/schema/2025-05-08_161918133bec_add_is_task_to_w
 orchestrator/migrations/versions/schema/2025-07-01_93fc5834c7e5_changed_timestamping_fields_in_process_steps.py,sha256=Oezd8b2qaI1Kyq-sZFVFmdzd4d9NjXrf6HtJGk11fy0,1914
 orchestrator/migrations/versions/schema/2025-07-04_4b58e336d1bf_deprecating_workflow_target_in_.py,sha256=xnD6w-97R4ClS7rbmXQEXc36K3fdcXKhCy7ZZNy_FX4,742
 orchestrator/migrations/versions/schema/2025-07-28_850dccac3b02_update_description_of_resume_workflows_.py,sha256=R6Qoga83DJ1IL0WYPu0u5u2ZvAmqGlDmUMv_KtJyOhQ,812
-orchestrator/migrations/versions/schema/2025-08-12_52b37b5b2714_search_index_model_for_llm_integration.py,sha256=6lRbUd1hJBjG8KM4Ow_J4pk2qwlRVhTKczS7XmoW-q4,3304
 orchestrator/schedules/__init__.py,sha256=Zy0fTOBMGIRFoh5iVFDLF9_PRAFaONYDThGK9EsysWo,981
 orchestrator/schedules/resume_workflows.py,sha256=jRnVRWDy687pQu-gtk80ecwiLSdrvtL15tG3U2zWA6I,891
 orchestrator/schedules/scheduler.py,sha256=nnuehZnBbtC90MsFP_Q6kqcD1ihsq08vr1ALJ6jHF_s,5833
@@ -276,9 +276,10 @@ orchestrator/schemas/subscription.py,sha256=-jXyHZIed9Xlia18ksSDyenblNN6Q2yM2FlG
 orchestrator/schemas/subscription_descriptions.py,sha256=Ft_jw1U0bf9Z0U8O4OWfLlcl0mXCVT_qYVagBP3GbIQ,1262
 orchestrator/schemas/workflow.py,sha256=StVoRGyNT2iIeq3z8BIlTPt0bcafzbeYxXRrCucR6LU,2146
 orchestrator/search/__init__.py,sha256=2uhTQexKx-cdBP1retV3CYSNCs02s8WL3fhGvupRGZk,571
-orchestrator/search/agent/__init__.py,sha256=guIh2VaXHvR8wkdOJMhoFmSQCLju4qTXyZDuLc12zrA,746
-orchestrator/search/agent/agent.py,sha256=iejwwBl4YPDVpYIBy9NzNdOSW7xbOUZedfC_7iGHpSg,2320
-orchestrator/search/agent/prompts.py,sha256=2VS0Ho-jq9u2kqrlcCIAHRqs35eww1eC0tZXPlhTbaE,4401
+orchestrator/search/llm_migration.py,sha256=UvFyzLGhlfPulA9T1pcpq8HnRd7Uu2ssKqW_N5NMeQk,3962
+orchestrator/search/agent/__init__.py,sha256=_b7Q43peWSi2bb3-69CThAqt_sxgoaMbHeq6erLGR00,752
+orchestrator/search/agent/agent.py,sha256=zhDyXwRf118vH96CmKRbo5O8GKl_mnLJTDNfWgvsKeE,2450
+orchestrator/search/agent/prompts.py,sha256=-1VLYwPecC6xroKQTc9AE9MTtg_ffAUfHUi8ZATyUMg,4556
 orchestrator/search/agent/state.py,sha256=1WHYol5UlYpq2QZz-BVsBFYrJZms5P18ohN2Ur8P2F4,783
 orchestrator/search/agent/tools.py,sha256=4kvY0tG7i5-w8C-ZMuSabxb_sJmd_TpFl3F4xeGgzok,9513
 orchestrator/search/core/__init__.py,sha256=q5G0z3nKjIHKFs1PkEG3nvTUy3Wp4kCyBtCbqUITj3A,579
@@ -287,29 +288,29 @@ orchestrator/search/core/exceptions.py,sha256=qp7ZdyDvN5b2HD5_oZXMgoLJgy79krpCls
 orchestrator/search/core/types.py,sha256=Gaf77cKUqnE8vJNCpk-g3h2U5912axhIgZZnF_0_O48,8831
 orchestrator/search/core/validators.py,sha256=zktY5A3RTBmfdARJoxoz9rnnyTZj7L30Kbmh9UTQz2o,1204
 orchestrator/search/docs/index.md,sha256=zKzE2fbtHDfYTKaHg628wAsqCTOJ5yFUWV0ucFH3pAg,863
-orchestrator/search/docs/running_local_text_embedding_inference.md,sha256=KlFxyAjHfLyCeV9fXAFVUqZOFWYwGPH-_oBjWx2Vgng,1255
+orchestrator/search/docs/running_local_text_embedding_inference.md,sha256=OR0NVZMb8DqpgXYxlwDUrJwfRk0bYOk1-LkDMqsV6bU,1327
 orchestrator/search/filters/__init__.py,sha256=Yutr21lv8RtZf5OKaBozlYufgmmV2QHuzAPPjvUamLE,1222
-orchestrator/search/filters/base.py,sha256=sk9m65M_viJV08YpBUm8CrLwmb-o9GcqAMXztqVAVtU,11612
+orchestrator/search/filters/base.py,sha256=lUr0eW0zi4oIMVUHuRD3GAQ9xEbHiFUl_EfAI6ABPVo,12456
 orchestrator/search/filters/date_filters.py,sha256=0a6nbUTK647_Qf4XXZMLDvBLVjF5Qqy9eJ-9SrTGaGg,3040
 orchestrator/search/filters/definitions.py,sha256=wl2HiXlTWXQN4JmuSq2SBuhTMvyIeonTtUZoCrJAK6M,4093
 orchestrator/search/filters/ltree_filters.py,sha256=1OOmM5K90NsGBQmTqyoDlphdAOGd9r2rmz1rNItm8yk,2341
 orchestrator/search/filters/numeric_filter.py,sha256=lcOAOpPNTwA0SW8QPiMOs1oKTYZLwGDQSrwFydXgMUU,2774
 orchestrator/search/indexing/__init__.py,sha256=Or78bizNPiuNOgwLGJQ0mspCF1G_gSe5C9Ap7qi0MZk,662
-orchestrator/search/indexing/indexer.py,sha256=Iw9BZWWpMPdhMffMwpZt1QrACSmmMVFg29R04zYuQgA,14971
+orchestrator/search/indexing/indexer.py,sha256=puYOL7IXyJi7A7huT1jQ_2G3YZimeivkQJF2BZR4apQ,14866
 orchestrator/search/indexing/registry.py,sha256=zEOUmQDmZHJ4xzT63VSJzuuHWVTnuBSvhZg4l6lFTUU,3048
-orchestrator/search/indexing/tasks.py,sha256=UKG5EKE_QZGrYMSuVZB8UWw14U3f6enanTzQVbxS3Is,2363
+orchestrator/search/indexing/tasks.py,sha256=vmS1nnprPF74yitS0xGpP1dhSDis2nekMYF0v_jduDE,2478
 orchestrator/search/indexing/traverse.py,sha256=NKkKSri-if1d1vwzTQlDCF0hvBdB2IbWWuMdPrQ78Jg,14330
 orchestrator/search/retrieval/__init__.py,sha256=JP5WGYhmjd2RKXEExorvU6koMBLsTLdlDGCR_r1t8ug,645
 orchestrator/search/retrieval/builder.py,sha256=70cEvbsWI1dj-4H-LJq4o6Q71e3WERd-V6bzlZhGtHw,4607
-orchestrator/search/retrieval/engine.py,sha256=b7qEw-DA-JEnOTXNm9baKNqLFpNAy9RHQF9s4oEFzvo,6312
+orchestrator/search/retrieval/engine.py,sha256=jHxKuULcsqkdTyh9NEzBCsOnBaZzlbvcGseJoJec1yw,6147
 orchestrator/search/retrieval/exceptions.py,sha256=oHoLGLLxxmVcV-W36uK0V-Pn4vf_iw6hajpQbap3NqI,3588
 orchestrator/search/retrieval/pagination.py,sha256=bRcXtWxxWvOhCQyhjwfJ7S6q_Dn3pYm8TCg7ofjVP44,3353
 orchestrator/search/retrieval/utils.py,sha256=svhF9YfMClq2MVPArS3ir3pg5_e_bremquv_l6tTsOQ,4597
 orchestrator/search/retrieval/validation.py,sha256=AjhttVJWlZDaT1_pUL_LaypQV11U21JpTCE4OwnpoqA,5849
 orchestrator/search/retrieval/retrievers/__init__.py,sha256=1bGmbae0GYRM6e1vxf0ww79NaTSmfOMe9S0pPVmh3CM,897
-orchestrator/search/retrieval/retrievers/base.py,sha256=zg1Sbz_YKBXIfyn69tiTrW8quFvwmeFGNNMU8QQvHfU,4525
+orchestrator/search/retrieval/retrievers/base.py,sha256=Sp8h992lw_7vigE4s2QB0gqtqMACEOA8nDnhuXXHtxA,4570
 orchestrator/search/retrieval/retrievers/fuzzy.py,sha256=U_WNAaxSUVUlVrmFrYFt-s0ebw9ift1Z2zBHG8TSPLE,3839
-orchestrator/search/retrieval/retrievers/hybrid.py,sha256=iQ977OF7GeAv-Pv2eALAxz-KwOL6EUmooSVnHZl5Dsw,7628
+orchestrator/search/retrieval/retrievers/hybrid.py,sha256=YriY3gF6E7pQUumqdSDSyFJvYQbZZ6vSsMUhM5JHGpg,11102
 orchestrator/search/retrieval/retrievers/semantic.py,sha256=oWNJ9DuqM16BXYXUwmRmkfDmp_2vQH2PySNMk8TcvVk,3961
 orchestrator/search/retrieval/retrievers/structured.py,sha256=OHsHEjjLg1QwtEytQNeyWcCBQd8rJxHVf59HxvA9_vc,1452
 orchestrator/search/schemas/__init__.py,sha256=q5G0z3nKjIHKFs1PkEG3nvTUy3Wp4kCyBtCbqUITj3A,579
@@ -360,7 +361,7 @@ orchestrator/websocket/managers/memory_websocket_manager.py,sha256=lF5EEx1iFMCGE
 orchestrator/workflows/__init__.py,sha256=NzIGGI-8SNAwCk2YqH6sHhEWbgAY457ntDwjO15N8v4,4131
 orchestrator/workflows/modify_note.py,sha256=eXt5KQvrkOXf-3YEXCn2XbBLP9N-n1pUYRW2t8Odupo,2150
 orchestrator/workflows/removed_workflow.py,sha256=V0Da5TEdfLdZZKD38ig-MTp3_IuE7VGqzHHzvPYQmLI,909
-orchestrator/workflows/steps.py,sha256=RX1ioybVUGy0m5oRubP0QshupXZk3e2fDKeq-Pjs6mY,6991
+orchestrator/workflows/steps.py,sha256=teis7vHLOEAchMrzw_pvPPQ6pRFliKZRpe02vsv3AZY,6994
 orchestrator/workflows/utils.py,sha256=VUCDoIl5XAKtIeAJpVpyW2pCIg3PoVWfwGn28BYlYhA,15424
 orchestrator/workflows/tasks/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
 orchestrator/workflows/tasks/cleanup_tasks_log.py,sha256=BfWYbPXhnLAHUJ0mlODDnjZnQQAvKCZJDVTwbwOWI04,1624
@@ -368,7 +369,7 @@ orchestrator/workflows/tasks/resume_workflows.py,sha256=T3iobSJjVgiupe0rClD34kUZ
 orchestrator/workflows/tasks/validate_product_type.py,sha256=paG-NAY1bdde3Adt8zItkcBKf5Pxw6f5ngGW6an6dYU,3192
 orchestrator/workflows/tasks/validate_products.py,sha256=kXBGZTkobfYH8e_crhdErT-ypdouH0a3_WLImmbKXcE,8523
 orchestrator/workflows/translations/en-GB.json,sha256=ST53HxkphFLTMjFHonykDBOZ7-P_KxksktZU3GbxLt0,846
-orchestrator_core-4.5.0a6.dist-info/licenses/LICENSE,sha256=b-aA5OZQuuBATmLKo_mln8CQrDPPhg3ghLzjPjLn4Tg,11409
-orchestrator_core-4.5.0a6.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-orchestrator_core-4.5.0a6.dist-info/METADATA,sha256=o1E60Zu0I3TwNWdQqrZPeRrhiwtTmsIkHSAq0jeKsSM,6170
-orchestrator_core-4.5.0a6.dist-info/RECORD,,
+orchestrator_core-4.5.0a8.dist-info/licenses/LICENSE,sha256=b-aA5OZQuuBATmLKo_mln8CQrDPPhg3ghLzjPjLn4Tg,11409
+orchestrator_core-4.5.0a8.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+orchestrator_core-4.5.0a8.dist-info/METADATA,sha256=kPLUhDg7J-mr72Nwo-oB9hEFmhp_xihUJ7JFWHFq64A,6252
+orchestrator_core-4.5.0a8.dist-info/RECORD,,

orchestrator/migrations/versions/schema/2025-08-12_52b37b5b2714_search_index_model_for_llm_integration.py

@@ -1,95 +0,0 @@
-"""Search index model for llm integration.
-
-Revision ID: 52b37b5b2714
-Revises: 850dccac3b02
-Create Date: 2025-08-12 22:34:26.694750
-
-"""
-
-import sqlalchemy as sa
-from alembic import op
-from pgvector.sqlalchemy import Vector
-from sqlalchemy.dialects import postgresql
-from sqlalchemy_utils import LtreeType
-
-from orchestrator.search.core.types import FieldType
-
-# revision identifiers, used by Alembic.
-revision = "52b37b5b2714"
-down_revision = "850dccac3b02"
-branch_labels = None
-depends_on = None
-
-TABLE = "ai_search_index"
-IDX_EMBED_HNSW = "ix_flat_embed_hnsw"
-IDX_PATH_GIST = "ix_flat_path_gist"
-IDX_PATH_BTREE = "ix_flat_path_btree"
-IDX_VALUE_TRGM = "ix_flat_value_trgm"
-IDX_CONTENT_HASH = "idx_ai_search_index_content_hash"
-
-TARGET_DIM = 1536
-
-
-def upgrade() -> None:
-    # Create PostgreSQL extensions
-    op.execute("CREATE EXTENSION IF NOT EXISTS ltree;")
-    op.execute("CREATE EXTENSION IF NOT EXISTS unaccent;")
-    op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
-    op.execute("CREATE EXTENSION IF NOT EXISTS vector;")
-
-    # Create the ai_search_index table
-    op.create_table(
-        TABLE,
-        sa.Column("entity_type", sa.Text, nullable=False),
-        sa.Column("entity_id", postgresql.UUID, nullable=False),
-        sa.Column("path", LtreeType, nullable=False),
-        sa.Column("value", sa.Text, nullable=False),
-        sa.Column("embedding", Vector(TARGET_DIM), nullable=True),
-        sa.Column("content_hash", sa.String(64), nullable=False),
-        sa.PrimaryKeyConstraint("entity_id", "path", name="pk_ai_search_index"),
-    )
-
-    field_type_enum = sa.Enum(*[ft.value for ft in FieldType], name="field_type")
-    field_type_enum.create(op.get_bind(), checkfirst=True)
-    op.add_column(
-        TABLE,
-        sa.Column("value_type", field_type_enum, nullable=False, server_default=FieldType.STRING.value),
-    )
-    op.alter_column(TABLE, "value_type", server_default=None)
-
-    op.create_index(op.f("ix_ai_search_index_entity_id"), TABLE, ["entity_id"], unique=False)
-    op.create_index(IDX_CONTENT_HASH, TABLE, ["content_hash"])
-
-    op.create_index(
-        IDX_PATH_GIST,
-        TABLE,
-        ["path"],
-        postgresql_using="GIST",
-        postgresql_ops={"path": "gist_ltree_ops"},
-    )
-    op.create_index(IDX_PATH_BTREE, TABLE, ["path"])
-    op.create_index(IDX_VALUE_TRGM, TABLE, ["value"], postgresql_using="GIN", postgresql_ops={"value": "gin_trgm_ops"})
-
-    op.create_index(
-        IDX_EMBED_HNSW,
-        TABLE,
-        ["embedding"],
-        postgresql_using="HNSW",
-        postgresql_with={"m": 16, "ef_construction": 64},
-        postgresql_ops={"embedding": "vector_l2_ops"},
-    )
-
-
-def downgrade() -> None:
-    # Drop all indexes
-    op.drop_index(IDX_EMBED_HNSW, table_name=TABLE, if_exists=True)
-    op.drop_index(IDX_VALUE_TRGM, table_name=TABLE, if_exists=True)
-    op.drop_index(IDX_PATH_BTREE, table_name=TABLE, if_exists=True)
-    op.drop_index(IDX_PATH_GIST, table_name=TABLE, if_exists=True)
-    op.drop_index(IDX_CONTENT_HASH, table_name=TABLE, if_exists=True)
-    op.drop_index(op.f("ix_ai_search_index_entity_id"), table_name=TABLE, if_exists=True)
-
-    # Drop table and enum
-    op.drop_table(TABLE, if_exists=True)
-    field_type_enum = sa.Enum(name="field_type")
-    field_type_enum.drop(op.get_bind(), checkfirst=True)