orchestrator-core 4.6.2__py3-none-any.whl → 4.6.3__py3-none-any.whl

orchestrator/search/query/results.py

@@ -0,0 +1,252 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Query execution result models and formatting functions."""
+
+from collections.abc import Sequence
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+from sqlalchemy.engine.row import RowMapping
+
+from orchestrator.search.core.types import EntityType, FilterOp, SearchMetadata
+from orchestrator.search.filters import FilterTree
+
+from .queries import AggregateQuery, CountQuery, ExportQuery, SelectQuery
+
+
+class VisualizationType(BaseModel):
+    """Visualization type for aggregation results.
+
+    Choose based on the query context:
+    - 'pie': For categorical distributions (e.g., subscriptions by status, products by type)
+    - 'line': For time-series data (e.g., subscriptions created per month, trends over time)
+    - 'table': For detailed data or multiple grouping dimensions (default)
+    """
+
+    type: Literal["pie", "line", "table"] = Field(
+        default="table",
+        description="Visualization render type: 'pie' for categorical distributions, 'line' for time-series, 'table' for detailed data",
+    )
+
+
+class MatchingField(BaseModel):
+    """Contains the field that contributed most to the (fuzzy) search result."""
+
+    text: str
+    path: str
+    highlight_indices: list[tuple[int, int]] | None = None
+
+
+class SearchResult(BaseModel):
+    """Represents a single search result item."""
+
+    entity_id: str
+    entity_type: EntityType
+    entity_title: str
+    score: float
+    perfect_match: int = 0
+    matching_field: MatchingField | None = None
+
+
+class SearchResponse(BaseModel):
+    """Response containing search results and metadata."""
+
+    results: list[SearchResult]
+    metadata: SearchMetadata
+    query_embedding: list[float] | None = None
+    has_more: bool = False
+
+
+class AggregationResult(BaseModel):
+    """Represents a single aggregation result row."""
+
+    group_values: dict[str, str] = Field(default_factory=dict)  # group_by field -> value
+    aggregations: dict[str, float | int] = Field(default_factory=dict)  # alias -> computed value
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AggregationResponse(BaseModel):
+    """Response containing aggregation results."""
+
+    results: list[AggregationResult]
+    total_groups: int
+    metadata: SearchMetadata
+    visualization_type: VisualizationType = Field(default_factory=VisualizationType)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ExportData(BaseModel):
+    """Export metadata for download."""
+
+    action: str = "export"
+    query_id: str
+    download_url: str
+    message: str
+
+
+def format_aggregation_response(
+    result_rows: Sequence[RowMapping],
+    group_column_names: list[str],
+    query: CountQuery | AggregateQuery,
+) -> AggregationResponse:
+    """Format raw aggregation query results into AggregationResponse.
+
+    Args:
+        result_rows: Raw database result rows
+        group_column_names: List of column names that are grouping columns
+        query: Query plan for metadata
+
+    Returns:
+        AggregationResponse with formatted results and metadata
+    """
+    results = []
+    for row in result_rows:
+        group_values = {}
+        aggregations = {}
+
+        for key, value in row.items():
+            if key in group_column_names:
+                # It's a grouping column
+                group_values[key] = str(value) if value is not None else ""
+            else:
+                # It's an aggregation result
+                aggregations[key] = value if value is not None else 0
+
+        results.append(AggregationResult(group_values=group_values, aggregations=aggregations))
+
+    metadata = SearchMetadata(
+        search_type="aggregation",
+        description=f"Aggregation query with {len(query.group_by or [])} grouping dimension(s)",
+    )
+
+    return AggregationResponse(
+        results=results,
+        total_groups=len(results),
+        metadata=metadata,
+    )
+
+
+def generate_highlight_indices(text: str, term: str) -> list[tuple[int, int]]:
+    """Finds all occurrences of individual words from the term, including both word boundary and substring matches."""
+    import re
+
+    if not text or not term:
+        return []
+
+    all_matches = []
+    words = [w.strip() for w in term.split() if w.strip()]
+
+    for word in words:
+        # First find word boundary matches
+        word_boundary_pattern = rf"\b{re.escape(word)}\b"
+        word_matches = list(re.finditer(word_boundary_pattern, text, re.IGNORECASE))
+        all_matches.extend([(m.start(), m.end()) for m in word_matches])
+
+        # Then find all substring matches
+        substring_pattern = re.escape(word)
+        substring_matches = list(re.finditer(substring_pattern, text, re.IGNORECASE))
+        all_matches.extend([(m.start(), m.end()) for m in substring_matches])
+
+    return sorted(set(all_matches))
+
+
+def format_search_response(
+    db_rows: Sequence[RowMapping], query: "SelectQuery | ExportQuery", metadata: SearchMetadata
+) -> SearchResponse:
+    """Format database query results into a `SearchResponse`.
+
+    Converts raw SQLAlchemy `RowMapping` objects into `SearchResult` instances,
+    including highlight metadata if present in the database results.
+
+    Args:
+        db_rows: The rows returned from the executed SQLAlchemy query.
+        query: SelectQuery or ExportQuery with search criteria.
+        metadata: Metadata about the search execution.
+
+    Returns:
+        SearchResponse: A list of `SearchResult` objects containing entity IDs, scores,
+        and optional highlight information.
+    """
+    from orchestrator.search.retrieval.retrievers import Retriever
+
+    if not db_rows:
+        return SearchResponse(results=[], metadata=metadata)
+
+    user_query = query.query_text
+
+    results = []
+    for row in db_rows:
+        matching_field = None
+
+        if (
+            user_query
+            and (text := row.get(Retriever.HIGHLIGHT_TEXT_LABEL)) is not None
+            and (path := row.get(Retriever.HIGHLIGHT_PATH_LABEL)) is not None
+        ):
+            if not isinstance(text, str):
+                text = str(text)
+            if not isinstance(path, str):
+                path = str(path)
+
+            highlight_indices = generate_highlight_indices(text, user_query) or None
+            matching_field = MatchingField(text=text, path=path, highlight_indices=highlight_indices)
+
+        elif not user_query and query.filters and metadata.search_type == "structured":
+            # Structured search (filter-only)
+            matching_field = _extract_matching_field_from_filters(query.filters)
+
+        entity_title = row.get("entity_title", "")
+        if not isinstance(entity_title, str):
+            entity_title = str(entity_title) if entity_title is not None else ""
+
+        results.append(
+            SearchResult(
+                entity_id=str(row.entity_id),
+                entity_type=query.entity_type,
+                entity_title=entity_title,
+                score=row.score,
+                perfect_match=row.get("perfect_match", 0),
+                matching_field=matching_field,
+            )
+        )
+    return SearchResponse(results=results, metadata=metadata)
+
+
+def _extract_matching_field_from_filters(filters: "FilterTree") -> MatchingField | None:
+    """Extract the first path filter to use as matching field for structured searches."""
+    from orchestrator.search.filters import LtreeFilter
+
+    leaves = filters.get_all_leaves()
+    if len(leaves) != 1:
+        return None
+
+    pf = leaves[0]
+
+    if isinstance(pf.condition, LtreeFilter):
+        op = pf.condition.op
+        # Prefer the original component/pattern (validator may set path="*" and move the value)
+        display = str(getattr(pf.condition, "value", "") or pf.path)
+
+        # There can be no match for abscence.
+        if op == FilterOp.NOT_HAS_COMPONENT:
+            return None
+
+        return MatchingField(text=display, path=display, highlight_indices=[(0, len(display))])
+
+    # Everything thats not Ltree
+    val = getattr(pf.condition, "value", "")
+    text = "" if val is None else str(val)
+    return MatchingField(text=text, path=pf.path, highlight_indices=[(0, len(text))])

orchestrator/search/{retrieval/query_state.py → query/state.py}

@@ -11,39 +11,47 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Generic, TypeVar, cast
 from uuid import UUID
 
+import structlog
 from pydantic import BaseModel, ConfigDict, Field
 
 from orchestrator.db import SearchQueryTable, db
 from orchestrator.search.core.exceptions import QueryStateNotFoundError
-from orchestrator.search.schemas.parameters import SearchParameters
+from orchestrator.search.query.queries import BaseQuery, Query
 
+logger = structlog.get_logger(__name__)
 
-class SearchQueryState(BaseModel):
-    """State of a search query including parameters and embedding.
+T = TypeVar("T", bound=Query)
 
-    This model provides a complete snapshot of what was searched and how.
-    Used for both agent and regular API searches.
+
+class QueryState(BaseModel, Generic[T]):
+    """State of a query including parameters and embedding.
+
+    Thin wrapper around SearchQueryTable that stores query as JSONB blob.
+    Generic over query type for type-safe loading.
+    Used for both agent and regular API queries.
     """
 
-    parameters: SearchParameters = Field(discriminator="entity_type")
+    query: T
     query_embedding: list[float] | None = Field(default=None, description="The embedding vector for semantic search")
 
     model_config = ConfigDict(from_attributes=True)
 
     @classmethod
-    def load_from_id(cls, query_id: UUID | str) -> "SearchQueryState":
-        """Load query state from database by query_id.
+    def load_from_id(cls, query_id: UUID | str, expected_type: type[T]) -> "QueryState[T]":
+        """Load query state from database by query_id with type validation.
 
         Args:
             query_id: UUID or string UUID of the saved query
+            expected_type: Expected query type class (SelectQuery, ExportQuery, etc.)
 
         Returns:
-            SearchQueryState loaded from database
+            QueryState with validated query type
 
         Raises:
-            ValueError: If query_id format is invalid
+            ValueError: If query_id format is invalid or query type doesn't match expected
             QueryStateNotFoundError: If query not found in database
         """
         if isinstance(query_id, UUID):
@@ -58,4 +66,16 @@ class SearchQueryState(BaseModel):
         if not search_query:
             raise QueryStateNotFoundError(f"Query {query_uuid} not found in database")
 
-        return cls.model_validate(search_query)
+        # Clamp limit to valid range to handle legacy queries outside the current limits
+        if "limit" in search_query.parameters and search_query.parameters["limit"] > BaseQuery.MAX_LIMIT:
+            logger.warning(
+                "Loaded query limit exceeds maximum, clamping to MAX_LIMIT",
+                query_id=query_uuid,
+                original_limit=search_query.parameters["limit"],
+                clamped_to=BaseQuery.MAX_LIMIT,
+            )
+            search_query.parameters["limit"] = BaseQuery.MAX_LIMIT
+
+        query = cast(T, expected_type.from_dict(search_query.parameters))
+
+        return cls(query=query, query_embedding=search_query.query_embedding)

orchestrator/search/{retrieval → query}/validation.py

@@ -18,12 +18,15 @@ from sqlalchemy_utils import Ltree
 from orchestrator.db import db
 from orchestrator.db.database import WrappedSession
 from orchestrator.db.models import AiSearchIndex
+from orchestrator.search.aggregations import AggregationType
 from orchestrator.search.core.types import EntityType, FieldType
 from orchestrator.search.filters import FilterCondition, FilterTree, LtreeFilter, PathFilter
 from orchestrator.search.filters.definitions import operators_for
-from orchestrator.search.retrieval.exceptions import (
+from orchestrator.search.query.exceptions import (
     EmptyFilterPathError,
+    IncompatibleAggregationTypeError,
     IncompatibleFilterTypeError,
+    IncompatibleTemporalGroupingTypeError,
     InvalidEntityPrefixError,
     InvalidLtreePatternError,
     PathNotFoundError,
@@ -150,3 +153,57 @@ async def validate_filter_tree(filters: FilterTree | None, entity_type: EntityTy
         return
     for leaf in filters.get_all_leaves():
         await complete_filter_validation(leaf, entity_type)
+
+
+def validate_aggregation_field(agg_type: AggregationType, field_path: str) -> None:
+    """Validate that an aggregation field exists and is compatible with the aggregation type.
+
+    Note: Only for FieldAggregations (SUM, AVG, MIN, MAX). COUNT does not require field validation.
+
+    Args:
+        agg_type: The aggregation type enum
+        field_path: The field path to validate
+
+    Raises:
+        PathNotFoundError: If the field doesn't exist in the database.
+        IncompatibleAggregationTypeError: If the field type is incompatible with the aggregation type.
+    """
+    # Check if field exists in database
+    field_type_str = validate_filter_path(field_path)
+    if field_type_str is None:
+        raise PathNotFoundError(field_path)
+
+    # Validate field type compatibility with aggregation type
+    if agg_type in (AggregationType.SUM, AggregationType.AVG):
+        if field_type_str not in (FieldType.INTEGER.value, FieldType.FLOAT.value):
+            raise IncompatibleAggregationTypeError(
+                agg_type.value, field_type_str, field_path, [FieldType.INTEGER.value, FieldType.FLOAT.value]
+            )
+    elif agg_type in (AggregationType.MIN, AggregationType.MAX):
+        if field_type_str not in (FieldType.INTEGER.value, FieldType.FLOAT.value, FieldType.DATETIME.value):
+            raise IncompatibleAggregationTypeError(
+                agg_type.value,
+                field_type_str,
+                field_path,
+                [FieldType.INTEGER.value, FieldType.FLOAT.value, FieldType.DATETIME.value],
+            )
+
+
+def validate_temporal_grouping_field(field_path: str) -> None:
+    """Validate that a field exists and is a datetime type for temporal grouping.
+
+    Args:
+        field_path: The field path to validate
+
+    Raises:
+        PathNotFoundError: If the field doesn't exist in the database
+        IncompatibleTemporalGroupingTypeError: If the field is not a datetime type
+    """
+    # Check if field exists in database
+    field_type_str = validate_filter_path(field_path)
+    if field_type_str is None:
+        raise PathNotFoundError(field_path)
+
+    # Validate field type is datetime
+    if field_type_str != FieldType.DATETIME.value:
+        raise IncompatibleTemporalGroupingTypeError(field_path, field_type_str)

orchestrator/search/retrieval/__init__.py

@@ -10,8 +10,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from .engine import execute_search, execute_search_for_export
-from .query_state import SearchQueryState
-
-__all__ = ["execute_search", "execute_search_for_export", "SearchQueryState"]

orchestrator/search/retrieval/pagination.py

@@ -18,8 +18,8 @@ from pydantic import BaseModel
 
 from orchestrator.db import SearchQueryTable, db
 from orchestrator.search.core.exceptions import InvalidCursorError
-from orchestrator.search.schemas.parameters import SearchParameters
-from orchestrator.search.schemas.results import SearchResponse
+from orchestrator.search.query.queries import SelectQuery
+from orchestrator.search.query.results import SearchResponse
 
 
 class PageCursor(BaseModel):
@@ -45,7 +45,7 @@ class PageCursor(BaseModel):
 def encode_next_page_cursor(
     search_response: SearchResponse,
     cursor: PageCursor | None,
-    search_params: SearchParameters,
+    query: SelectQuery,
 ) -> str | None:
     """Create next page cursor if there are more results.
 
@@ -55,20 +55,19 @@ def encode_next_page_cursor(
     Args:
         search_response: SearchResponse containing results and query_embedding
         cursor: Current page cursor (None for first page, PageCursor for subsequent pages)
-        search_params: Search parameters to save for pagination consistency
+        query: SelectQuery for search operation to save for pagination consistency
 
     Returns:
         Encoded cursor for next page, or None if no more results
     """
-    from orchestrator.search.retrieval.query_state import SearchQueryState
+    from orchestrator.search.query.state import QueryState
 
-    has_next_page = len(search_response.results) == search_params.limit and search_params.limit > 0
-    if not has_next_page:
+    if not search_response.has_more:
         return None
 
     # If this is the first page, save query state to database
     if cursor is None:
-        query_state = SearchQueryState(parameters=search_params, query_embedding=search_response.query_embedding)
+        query_state = QueryState(query=query, query_embedding=search_response.query_embedding)
         search_query = SearchQueryTable.from_state(state=query_state)
 
         db.session.add(search_query)

orchestrator/search/retrieval/retrievers/base.py

@@ -18,7 +18,7 @@ import structlog
 from sqlalchemy import BindParameter, Numeric, Select, literal
 
 from orchestrator.search.core.types import FieldType, SearchMetadata
-from orchestrator.search.schemas.parameters import BaseSearchParameters
+from orchestrator.search.query.queries import ExportQuery, SelectQuery
 
 from ..pagination import PageCursor
 
@@ -41,13 +41,13 @@ class Retriever(ABC):
     ]
 
     @classmethod
-    async def route(
+    def route(
         cls,
-        params: BaseSearchParameters,
+        query: "SelectQuery | ExportQuery",
         cursor: PageCursor | None,
         query_embedding: list[float] | None = None,
     ) -> "Retriever":
-        """Route to the appropriate retriever instance based on search parameters.
+        """Route to the appropriate retriever instance based on query plan.
 
         Selects the retriever type based on available search criteria:
         - Hybrid: both embedding and fuzzy term available
@@ -56,7 +56,7 @@ class Retriever(ABC):
         - Structured: only filters available
 
         Args:
-            params: Search parameters including vector queries, fuzzy terms, and filters
+            query: SelectQuery or ExportQuery with search criteria
             cursor: Pagination cursor for cursor-based paging
             query_embedding: Query embedding for semantic search, or None if not available
 
@@ -68,11 +68,11 @@ class Retriever(ABC):
         from .semantic import SemanticRetriever
         from .structured import StructuredRetriever
 
-        fuzzy_term = params.fuzzy_term
+        fuzzy_term = query.fuzzy_term
 
-        # If vector_query exists but embedding generation failed, fall back to fuzzy search with full query
-        if query_embedding is None and params.vector_query is not None and params.query is not None:
-            fuzzy_term = params.query
+        # If vector_query exists but embedding generation failed, fall back to fuzzy search with full query text
+        if query_embedding is None and query.vector_query is not None and query.query_text is not None:
+            fuzzy_term = query.query_text
 
         # Select retriever based on available search criteria
         if query_embedding is not None and fuzzy_term is not None:

orchestrator/workflows/translations/en-GB.json

@@ -6,6 +6,7 @@
             "subscription_id": "Subscription",
             "version": "Version",
             "subscription_id_info": "The subscription for this action",
+            "vlan_info": "The vlan id for this service port",
             "product_type": "Product Type"
         }
     },

{orchestrator_core-4.6.2.dist-info → orchestrator_core-4.6.3.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 4.6.2
+Version: 4.6.3
 Summary: This is the orchestrator workflow engine.
 Author-email: SURF <automation-beheer@surf.nl>
-Requires-Python: >=3.11,<3.14
+Requires-Python: >=3.11,<3.15
 Description-Content-Type: text/markdown
 License-Expression: Apache-2.0
 Classifier: Development Status :: 5 - Production/Stable
@@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python
 Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
 Classifier: Topic :: Internet :: WWW/HTTP
@@ -30,40 +31,40 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 License-File: LICENSE
-Requires-Dist: alembic==1.16.1
+Requires-Dist: alembic==1.17.2
 Requires-Dist: anyio>=3.7.0
 Requires-Dist: apscheduler>=3.11.0
 Requires-Dist: click==8.*
 Requires-Dist: deepmerge==2.0
 Requires-Dist: deprecated>=1.2.18
-Requires-Dist: fastapi~=0.115.14
+Requires-Dist: fastapi~=0.121.1
 Requires-Dist: fastapi-etag==0.4.0
 Requires-Dist: itsdangerous>=2.2.0
 Requires-Dist: jinja2==3.1.6
-Requires-Dist: more-itertools~=10.7.0
+Requires-Dist: more-itertools~=10.8.0
 Requires-Dist: nwa-stdlib~=1.10.3
 Requires-Dist: oauth2-lib>=2.5.0
-Requires-Dist: orjson==3.10.18
+Requires-Dist: orjson==3.11.4
 Requires-Dist: pgvector>=0.4.1
-Requires-Dist: prometheus-client==0.22.1
-Requires-Dist: psycopg2-binary==2.9.10
+Requires-Dist: prometheus-client==0.23.1
+Requires-Dist: psycopg2-binary==2.9.11
 Requires-Dist: pydantic-forms>=1.4.0
-Requires-Dist: pydantic-settings~=2.9.1
-Requires-Dist: pydantic[email]~=2.11.7
-Requires-Dist: python-dateutil==2.8.2
-Requires-Dist: python-rapidjson>=1.18,<1.21
+Requires-Dist: pydantic-settings~=2.12.0
+Requires-Dist: pydantic[email]~=2.12.4
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: python-rapidjson>=1.22,<1.23
 Requires-Dist: pytz==2025.2
 Requires-Dist: redis==5.1.1
 Requires-Dist: semver==3.0.4
 Requires-Dist: sentry-sdk[fastapi]>=2.29.1
-Requires-Dist: sqlalchemy==2.0.41
+Requires-Dist: sqlalchemy==2.0.44
 Requires-Dist: sqlalchemy-utils==0.41.2
-Requires-Dist: strawberry-graphql>=0.281.0
+Requires-Dist: strawberry-graphql>=0.281.0,<0.285.0
 Requires-Dist: structlog>=25.4.0
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: typer==0.15.4
 Requires-Dist: uvicorn[standard]~=0.34.0
-Requires-Dist: pydantic-ai-slim >=1.3.0 ; extra == "agent"
+Requires-Dist: pydantic-ai-slim >=1.9.0 ; extra == "agent"
 Requires-Dist: ag-ui-protocol>=0.1.8 ; extra == "agent"
 Requires-Dist: litellm>=1.75.7 ; extra == "agent"
 Requires-Dist: celery~=5.5.1 ; extra == "celery"