orchestrator-core 4.5.0a2__py3-none-any.whl → 4.5.0a3__py3-none-any.whl

orchestrator/search/indexing/traverse.py

@@ -1,3 +1,16 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import re
 from abc import ABC, abstractmethod
 from collections.abc import Iterable
@@ -19,7 +32,7 @@ from orchestrator.domain.lifecycle import (
 from orchestrator.schemas.process import ProcessSchema
 from orchestrator.schemas.workflow import WorkflowSchema
 from orchestrator.search.core.exceptions import ModelLoadError, ProductNotInRegistryError
-from orchestrator.search.core.types import ExtractedField, FieldType
+from orchestrator.search.core.types import LTREE_SEPARATOR, ExtractedField, FieldType
 from orchestrator.types import SubscriptionLifecycle
 
 logger = structlog.get_logger(__name__)
@@ -30,7 +43,6 @@ DatabaseEntity = SubscriptionTable | ProductTable | ProcessTable | WorkflowTable
 class BaseTraverser(ABC):
     """Base class for traversing database models and extracting searchable fields."""
 
-    _LTREE_SEPARATOR = "."
     _MAX_DEPTH = 40
 
     @classmethod
@@ -62,7 +74,7 @@ class BaseTraverser(ABC):
             except Exception as e:
                 logger.error(f"Failed to access field '{name}' on {model_class.__name__}", error=str(e))
                 continue
-            new_path = f"{path}{cls._LTREE_SEPARATOR}{name}" if path else name
+            new_path = f"{path}{LTREE_SEPARATOR}{name}" if path else name
             annotation = field.annotation if hasattr(field, "annotation") else field.return_type
             yield from cls._yield_fields_for_value(value, new_path, annotation)
 
@@ -197,7 +209,7 @@ class ProductTraverser(BaseTraverser):
         fields = []
 
         # Add the block itself as a BLOCK type
-        block_name = block_path.split(cls._LTREE_SEPARATOR)[-1]
+        block_name = block_path.split(LTREE_SEPARATOR)[-1]
         fields.append(ExtractedField(path=block_path, value=block_name, value_type=FieldType.BLOCK))
 
         # Extract all field names from the block as RESOURCE_TYPE
@@ -223,7 +235,7 @@ class ProductTraverser(BaseTraverser):
                             ExtractedField(path=field_path, value=field_name, value_type=FieldType.RESOURCE_TYPE)
                         )
                         # And potentially traverse the first item for schema
-                        first_item_path = f"{field_path}{cls._LTREE_SEPARATOR}0"
+                        first_item_path = f"{field_path}{LTREE_SEPARATOR}0"
                         nested_fields = cls._extract_block_schema(field_value[0], first_item_path)
                         fields.extend(nested_fields)
                     else:

orchestrator/search/retrieval/__init__.py

@@ -1,3 +1,16 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from .engine import execute_search
 
 __all__ = ["execute_search"]

orchestrator/search/retrieval/builder.py

@@ -1,3 +1,16 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from collections import defaultdict
 from typing import Sequence
 
@@ -23,16 +36,13 @@ def build_candidate_query(params: BaseSearchParameters) -> Select:
     from the index table for the given entity type, applying any structured
     filters from the provided search parameters.
 
-    Parameters
-    ----------
-    params : BaseSearchParameters
-        The search parameters containing the entity type and optional filters.
+    Args:
+        params (BaseSearchParameters): The search parameters containing the entity type and optional filters.
 
     Returns:
-    -------
-    Select
-        The SQLAlchemy `Select` object representing the query.
+        Select: The SQLAlchemy `Select` object representing the query.
     """
+
     stmt = select(AiSearchIndex.entity_id).where(AiSearchIndex.entity_type == params.entity_type.value).distinct()
 
     if params.filters is not None:

orchestrator/search/retrieval/engine.py

@@ -1,3 +1,16 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from collections.abc import Sequence
 
 import structlog
@@ -11,7 +24,7 @@ from orchestrator.search.schemas.results import MatchingField, SearchResponse, S
 
 from .builder import build_candidate_query
 from .pagination import PaginationParams
-from .retriever import Retriever
+from .retrievers import Retriever
 from .utils import generate_highlight_indices
 
 logger = structlog.get_logger(__name__)
@@ -25,16 +38,14 @@ def _format_response(
     Converts raw SQLAlchemy `RowMapping` objects into `SearchResult` instances,
     including highlight metadata if present in the database results.
 
-    Parameters
-    ----------
-    db_rows : Sequence[RowMapping]
-        The rows returned from the executed SQLAlchemy query.
+    Args:
+        db_rows (Sequence[RowMapping]): The rows returned from the executed SQLAlchemy query.
+        search_params (BaseSearchParameters): The search parameters, including query text and filters.
+        metadata (SearchMetadata): Metadata about the search execution.
 
     Returns:
-    -------
-    SearchResponse
-        A list of `SearchResult` objects containing entity IDs, scores, and
-        optional highlight information.
+        SearchResponse: A list of `SearchResult` objects containing entity IDs, scores,
+        and optional highlight information.
     """
 
     if not db_rows:
@@ -46,11 +57,11 @@ def _format_response(
     for row in db_rows:
         matching_field = None
 
-        if user_query and row.get("highlight_text") and row.get("highlight_path"):
-            # Text/semantic searches
-            text = row.highlight_text
-            path = row.highlight_path
-
+        if (
+            user_query
+            and (text := row.get(Retriever.HIGHLIGHT_TEXT_LABEL))
+            and (path := row.get(Retriever.HIGHLIGHT_PATH_LABEL))
+        ):
             if not isinstance(text, str):
                 text = str(text)
             if not isinstance(path, str):
@@ -114,26 +125,21 @@ async def execute_search(
     applies the appropriate ranking strategy, and executes the final ranked
     query to retrieve results.
 
-    Parameters
-    ----------
-    search_params : BaseSearchParameters
-        The search parameters specifying vector, fuzzy, or filter criteria.
-    db_session : Session
-        The active SQLAlchemy session for executing the query.
-    limit : int, optional
-        The maximum number of search results to return, by default 5.
+    Args:
+        search_params (BaseSearchParameters): The search parameters specifying vector, fuzzy, or filter criteria.
+        db_session (Session): The active SQLAlchemy session for executing the query.
+        pagination_params (PaginationParams): Parameters controlling pagination of the search results.
+        limit (int, optional): The maximum number of search results to return, by default 5.
 
     Returns:
-    -------
-    SearchResponse
-        A list of `SearchResult` objects containing entity IDs, scores, and
-        optional highlight metadata.
+        SearchResponse: A list of `SearchResult` objects containing entity IDs, scores,
+        and optional highlight metadata.
 
     Notes:
-    -----
-    If no vector query, filters, or fuzzy term are provided, a warning is logged
-    and an empty result set is returned.
+        If no vector query, filters, or fuzzy term are provided, a warning is logged
+        and an empty result set is returned.
     """
+
     if not search_params.vector_query and not search_params.filters and not search_params.fuzzy_term:
         logger.warning("No search criteria provided (vector_query, fuzzy_term, or filters).")
         return SearchResponse(results=[], metadata=SearchMetadata.empty())

orchestrator/search/retrieval/exceptions.py

@@ -0,0 +1,90 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from orchestrator.search.core.types import FilterOp
+
+
+class FilterValidationError(Exception):
+    """Base exception for filter validation errors."""
+
+    pass
+
+
+class InvalidLtreePatternError(FilterValidationError):
+    """Raised when an ltree pattern has invalid ltree query syntax."""
+
+    def __init__(self, pattern: str) -> None:
+        message = f"Ltree pattern '{pattern}' has invalid syntax. Use valid PostgreSQL ltree lquery syntax."
+        super().__init__(message)
+
+
+class EmptyFilterPathError(FilterValidationError):
+    """Raised when a filter path is empty or contains only whitespace."""
+
+    def __init__(self) -> None:
+        message = (
+            "Filter path cannot be empty. Provide a valid path like 'subscription.product.name' or 'workflow.name'."
+        )
+        super().__init__(message)
+
+
+class PathNotFoundError(FilterValidationError):
+    """Raised when a filter path doesn't exist in the database schema.
+
+    Examples:
+        Using a non-existent filter path:
+
+        >>> print(PathNotFoundError('subscription.nonexistent.field'))
+        Path 'subscription.nonexistent.field' does not exist in the database.
+    """
+
+    def __init__(self, path: str) -> None:
+        message = f"Path '{path}' does not exist in the database."
+        super().__init__(message)
+
+
+class IncompatibleFilterTypeError(FilterValidationError):
+    """Raised when a filter operator is incompatible with the field's data type.
+
+    Examples:
+        Using a numeric comparison operator on a string field:
+
+        >>> print(IncompatibleFilterTypeError(
+        ...     operator='gt',
+        ...     field_type='string',
+        ...     path='subscription.customer_name',
+        ...     expected_operators=[FilterOp.EQ, FilterOp.NEQ, FilterOp.LIKE],
+        ... ))
+        Operator 'gt' is not compatible with field type 'string' for path 'subscription.customer_name'. Valid operators for 'string': [eq, neq, like]
+    """
+
+    def __init__(self, operator: str, field_type: str, path: str, expected_operators: list[FilterOp]) -> None:
+        valid_ops_str = ", ".join([op.value for op in expected_operators])
+        message = f"Operator '{operator}' is not compatible with field type '{field_type}' for path '{path}'. Valid operators for '{field_type}': [{valid_ops_str}]"
+
+        super().__init__(message)
+
+
+class InvalidEntityPrefixError(FilterValidationError):
+    """Raised when a filter path doesn't have the correct entity type prefix.
+
+    Examples:
+        Using wrong entity prefix in filter path:
+
+        >>> print(InvalidEntityPrefixError('workflow.name', 'subscription.', 'SUBSCRIPTION'))
+        Filter path 'workflow.name' must start with 'subscription.' for SUBSCRIPTION searches, or use '*' for wildcard paths.
+    """
+
+    def __init__(self, path: str, expected_prefix: str, entity_type: str) -> None:
+        message = f"Filter path '{path}' must start with '{expected_prefix}' for {entity_type} searches, or use '*' for wildcard paths."
+        super().__init__(message)

orchestrator/search/retrieval/pagination.py

@@ -1,3 +1,16 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import array
 import base64
 from dataclasses import dataclass

orchestrator/search/retrieval/retrievers/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .base import Retriever
+from .fuzzy import FuzzyRetriever
+from .hybrid import RrfHybridRetriever
+from .semantic import SemanticRetriever
+from .structured import StructuredRetriever
+
+__all__ = [
+    "Retriever",
+    "FuzzyRetriever",
+    "RrfHybridRetriever",
+    "SemanticRetriever",
+    "StructuredRetriever",
+]

orchestrator/search/retrieval/retrievers/base.py

@@ -0,0 +1,122 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from decimal import Decimal
+
+import structlog
+from sqlalchemy import BindParameter, Numeric, Select, literal
+
+from orchestrator.search.core.types import FieldType, SearchMetadata
+from orchestrator.search.schemas.parameters import BaseSearchParameters
+
+from ..pagination import PaginationParams
+
+logger = structlog.get_logger(__name__)
+
+
+class Retriever(ABC):
+    """Abstract base class for applying a ranking strategy to a search query."""
+
+    SCORE_PRECISION = 12
+    SCORE_NUMERIC_TYPE = Numeric(38, 12)
+    HIGHLIGHT_TEXT_LABEL = "highlight_text"
+    HIGHLIGHT_PATH_LABEL = "highlight_path"
+    SCORE_LABEL = "score"
+    SEARCHABLE_FIELD_TYPES = [
+        FieldType.STRING.value,
+        FieldType.UUID.value,
+        FieldType.BLOCK.value,
+        FieldType.RESOURCE_TYPE.value,
+    ]
+
+    @classmethod
+    async def from_params(
+        cls,
+        params: BaseSearchParameters,
+        pagination_params: PaginationParams,
+    ) -> "Retriever":
+        """Create the appropriate retriever instance from search parameters.
+
+        Args:
+            params (BaseSearchParameters): Search parameters including vector queries, fuzzy terms, and filters.
+            pagination_params (PaginationParams): Pagination parameters for cursor-based paging.
+
+        Returns:
+            Retriever: A concrete retriever instance (semantic, fuzzy, hybrid, or structured).
+        """
+
+        from .fuzzy import FuzzyRetriever
+        from .hybrid import RrfHybridRetriever
+        from .semantic import SemanticRetriever
+        from .structured import StructuredRetriever
+
+        fuzzy_term = params.fuzzy_term
+        q_vec = await cls._get_query_vector(params.vector_query, pagination_params.q_vec_override)
+
+        # If semantic search was attempted but failed, fall back to fuzzy with the full query
+        fallback_fuzzy_term = fuzzy_term
+        if q_vec is None and params.vector_query is not None and params.query is not None:
+            fallback_fuzzy_term = params.query
+
+        if q_vec is not None and fallback_fuzzy_term is not None:
+            return RrfHybridRetriever(q_vec, fallback_fuzzy_term, pagination_params)
+        if q_vec is not None:
+            return SemanticRetriever(q_vec, pagination_params)
+        if fallback_fuzzy_term is not None:
+            return FuzzyRetriever(fallback_fuzzy_term, pagination_params)
+
+        return StructuredRetriever(pagination_params)
+
+    @classmethod
+    async def _get_query_vector(
+        cls, vector_query: str | None, q_vec_override: list[float] | None
+    ) -> list[float] | None:
+        """Get query vector either from override or by generating from text."""
+        if q_vec_override:
+            return q_vec_override
+
+        if not vector_query:
+            return None
+
+        from orchestrator.search.core.embedding import QueryEmbedder
+
+        q_vec = await QueryEmbedder.generate_for_text_async(vector_query)
+        if not q_vec:
+            logger.warning("Embedding generation failed; using non-semantic retriever")
+            return None
+
+        return q_vec
+
+    @abstractmethod
+    def apply(self, candidate_query: Select) -> Select:
+        """Apply the ranking logic to the given candidate query.
+
+        Args:
+            candidate_query (Select): A SQLAlchemy `Select` statement returning candidate entity IDs.
+
+        Returns:
+            Select: A new `Select` statement with ranking expressions applied.
+        """
+        ...
+
+    def _quantize_score_for_pagination(self, score_value: float) -> BindParameter[Decimal]:
+        """Convert score value to properly quantized Decimal parameter for pagination."""
+        pas_dec = Decimal(str(score_value)).quantize(Decimal("0.000000000001"))
+        return literal(pas_dec, type_=self.SCORE_NUMERIC_TYPE)
+
+    @property
+    @abstractmethod
+    def metadata(self) -> SearchMetadata:
+        """Return metadata describing this search strategy."""
+        ...

orchestrator/search/retrieval/retrievers/fuzzy.py

@@ -0,0 +1,94 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sqlalchemy import Select, and_, cast, func, literal, or_, select
+from sqlalchemy.sql.expression import ColumnElement
+
+from orchestrator.db.models import AiSearchIndex
+from orchestrator.search.core.types import SearchMetadata
+
+from ..pagination import PaginationParams
+from .base import Retriever
+
+
+class FuzzyRetriever(Retriever):
+    """Ranks results based on the max of fuzzy text similarity scores."""
+
+    def __init__(self, fuzzy_term: str, pagination_params: PaginationParams) -> None:
+        self.fuzzy_term = fuzzy_term
+        self.page_after_score = pagination_params.page_after_score
+        self.page_after_id = pagination_params.page_after_id
+
+    def apply(self, candidate_query: Select) -> Select:
+        cand = candidate_query.subquery()
+
+        similarity_expr = func.word_similarity(self.fuzzy_term, AiSearchIndex.value)
+
+        raw_max = func.max(similarity_expr).over(partition_by=AiSearchIndex.entity_id)
+        score = cast(
+            func.round(cast(raw_max, self.SCORE_NUMERIC_TYPE), self.SCORE_PRECISION), self.SCORE_NUMERIC_TYPE
+        ).label(self.SCORE_LABEL)
+
+        combined_query = (
+            select(
+                AiSearchIndex.entity_id,
+                score,
+                func.first_value(AiSearchIndex.value)
+                .over(partition_by=AiSearchIndex.entity_id, order_by=[similarity_expr.desc(), AiSearchIndex.path.asc()])
+                .label(self.HIGHLIGHT_TEXT_LABEL),
+                func.first_value(AiSearchIndex.path)
+                .over(partition_by=AiSearchIndex.entity_id, order_by=[similarity_expr.desc(), AiSearchIndex.path.asc()])
+                .label(self.HIGHLIGHT_PATH_LABEL),
+            )
+            .select_from(AiSearchIndex)
+            .join(cand, cand.c.entity_id == AiSearchIndex.entity_id)
+            .where(
+                and_(
+                    AiSearchIndex.value_type.in_(self.SEARCHABLE_FIELD_TYPES),
+                    literal(self.fuzzy_term).op("<%")(AiSearchIndex.value),
+                )
+            )
+            .distinct(AiSearchIndex.entity_id)
+        )
+        final_query = combined_query.subquery("ranked_fuzzy")
+
+        stmt = select(
+            final_query.c.entity_id,
+            final_query.c.score,
+            final_query.c.highlight_text,
+            final_query.c.highlight_path,
+        ).select_from(final_query)
+
+        stmt = self._apply_score_pagination(stmt, final_query.c.score, final_query.c.entity_id)
+
+        return stmt.order_by(final_query.c.score.desc().nulls_last(), final_query.c.entity_id.asc())
+
+    @property
+    def metadata(self) -> SearchMetadata:
+        return SearchMetadata.fuzzy()
+
+    def _apply_score_pagination(
+        self, stmt: Select, score_column: ColumnElement, entity_id_column: ColumnElement
+    ) -> Select:
+        """Apply standard score + entity_id pagination."""
+        if self.page_after_score is not None and self.page_after_id is not None:
+            stmt = stmt.where(
+                or_(
+                    score_column < self.page_after_score,
+                    and_(
+                        score_column == self.page_after_score,
+                        entity_id_column > self.page_after_id,
+                    ),
+                )
+            )
+        return stmt