orchestrator-core 4.4.1__py3-none-any.whl → 4.5.0a2__py3-none-any.whl

orchestrator/search/filters/base.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+from itertools import count
+from typing import Any, ClassVar, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from sqlalchemy import BinaryExpression, and_, cast, exists, literal, or_, select
+from sqlalchemy.dialects.postgresql import BOOLEAN
+from sqlalchemy.sql.elements import ColumnElement
+from sqlalchemy_utils.types.ltree import Ltree
+
+from orchestrator.db.models import AiSearchIndex
+from orchestrator.search.core.types import BooleanOperator, FieldType, FilterOp, SQLAColumn, UIType
+
+from .date_filters import DateFilter
+from .ltree_filters import LtreeFilter
+from .numeric_filter import NumericFilter
+
+
+class EqualityFilter(BaseModel):
+    op: Literal[FilterOp.EQ, FilterOp.NEQ]
+    value: Any
+
+    def to_expression(self, column: SQLAColumn, path: str) -> BinaryExpression[bool] | ColumnElement[bool]:
+        if isinstance(self.value, bool):
+            colb = cast(column, BOOLEAN)
+            return colb.is_(self.value) if self.op == FilterOp.EQ else ~colb.is_(self.value)
+        sv = str(self.value)
+        return (column == sv) if self.op == FilterOp.EQ else (column != sv)
+
+
+class StringFilter(BaseModel):
+    op: Literal[FilterOp.LIKE]
+    value: str
+
+    def to_expression(self, column: SQLAColumn, path: str) -> ColumnElement[bool]:
+        return column.like(self.value)
+
+    @model_validator(mode="after")
+    def validate_like_pattern(self) -> StringFilter:
+        """If the operation is 'like', the value must contain a wildcard."""
+        if self.op == FilterOp.LIKE:
+            if "%" not in self.value and "_" not in self.value:
+                raise ValueError("The value for a 'like' operation must contain a wildcard character ('%' or '_').")
+        return self
+
+
+# Order matters! Ambiguous ops (like 'eq') are resolved by first matching filter
+FilterCondition = (
+    DateFilter  # DATETIME
+    | NumericFilter  # INT/FLOAT
+    | StringFilter  # STRING TODO: convert to hybrid search?
+    | LtreeFilter  # Path
+    | EqualityFilter  # BOOLEAN/UUID/BLOCK/RESOURCE_TYPE - most generic, try last
+)
+
+
+class PathFilter(BaseModel):
+
+    path: str = Field(description="The ltree path of the field to filter on, e.g., 'subscription.customer_id'.")
+    condition: FilterCondition = Field(description="The filter condition to apply.")
+
+    value_kind: UIType
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "examples": [
+                {"path": "subscription.status", "condition": {"op": "eq", "value": "active"}, "value_kind": "string"},
+                {
+                    "path": "subscription.customer_id",
+                    "condition": {"op": "neq", "value": "acme"},
+                    "value_kind": "string",
+                },
+                {
+                    "path": "subscription.start_date",
+                    "condition": {"op": "gt", "value": "2025-01-01"},
+                    "value_kind": "datetime",
+                },
+                {
+                    "path": "subscription.end_date",
+                    "condition": {
+                        "op": "between",
+                        "value": {"start": "2025-06-01", "end": "2025-07-01"},
+                    },
+                    "value_kind": "datetime",
+                },
+                {
+                    "path": "subscription",
+                    "condition": {"op": "has_component", "value": "node"},
+                    "value_kind": "component",
+                },
+            ]
+        }
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def _transfer_path_to_value_if_needed(cls, data: Any) -> Any:
+        """Transform for path-only filters.
+
+        If `op` is `has_component`, `not_has_component`, or `ends_with` and no `value` is
+        provided in the `condition`, this validator will automatically use the `path`
+        field as the `value` and set the `path` to a wildcard '*' for the query.
+        """
+        if isinstance(data, dict):
+            path = data.get("path")
+            condition = data.get("condition")
+
+            if path and isinstance(condition, dict):
+                op = condition.get("op")
+                value = condition.get("value")
+
+                path_only_ops = [FilterOp.HAS_COMPONENT, FilterOp.NOT_HAS_COMPONENT, FilterOp.ENDS_WITH]
+
+                if op in path_only_ops and value is None:
+                    condition["value"] = path
+                    data["path"] = "*"
+        return data
+
+    def to_expression(self, value_column: SQLAColumn, value_type_column: SQLAColumn) -> ColumnElement[bool]:
+        """Convert the path filter into a SQLAlchemy expression with type safety.
+
+        This method creates a type guard to ensure we only match compatible field types,
+        then delegates to the specific filter condition.
+
+        Parameters
+        ----------
+        value_column : ColumnElement
+            The SQLAlchemy column element representing the value to be filtered.
+        value_type_column : ColumnElement
+            The SQLAlchemy column element representing the field type.
+
+        Returns:
+        -------
+        ColumnElement[bool]
+            A SQLAlchemy boolean expression that can be used in a ``WHERE`` clause.
+        """
+        # Type guard - only match compatible field types
+        allowed_field_types = [ft.value for ft in FieldType if UIType.from_field_type(ft) == self.value_kind]
+        type_guard = value_type_column.in_(allowed_field_types) if allowed_field_types else literal(True)
+
+        return and_(type_guard, self.condition.to_expression(value_column, self.path))
+
+
+class FilterTree(BaseModel):
+    op: BooleanOperator = Field(
+        description="Operator for grouping conditions in uppercase.", default=BooleanOperator.AND
+    )
+
+    children: list[FilterTree | PathFilter] = Field(min_length=1, description="Path filters or nested groups.")
+
+    MAX_DEPTH: ClassVar[int] = 5
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "description": (
+                "Boolean filter tree. Operators must be UPPERCASE: AND / OR.\n"
+                "Node shapes:\n"
+                "  • Group: {'op':'AND'|'OR', 'children': [<PathFilter|FilterTree>, ...]}\n"
+                "  • Leaf (PathFilter): {'path':'<ltree>', 'condition': {...}}\n"
+                "Rules:\n"
+                "  • Do NOT put 'op' or 'children' inside a leaf 'condition'.\n"
+                f"  • Max depth = {MAX_DEPTH}.\n"
+            ),
+            "examples": [
+                {
+                    "description": "Simple filters",
+                    "op": "AND",
+                    "children": [
+                        {"path": "subscription.status", "condition": {"op": "eq", "value": "active"}},
+                        {"path": "subscription.start_date", "condition": {"op": "gt", "value": "2021-01-01"}},
+                    ],
+                },
+                {
+                    "description": "Complex filters with OR group",
+                    "op": "AND",
+                    "children": [
+                        {"path": "subscription.start_date", "condition": {"op": "gte", "value": "2024-01-01"}},
+                        {
+                            "op": "OR",
+                            "children": [
+                                {"path": "subscription.product.name", "condition": {"op": "like", "value": "%fiber%"}},
+                                {"path": "subscription.customer_id", "condition": {"op": "eq", "value": "Surf"}},
+                            ],
+                        },
+                    ],
+                },
+            ],
+        }
+    )
+
+    @model_validator(mode="after")
+    def _validate_depth(self) -> FilterTree:
+        def depth(node: "FilterTree | PathFilter") -> int:
+            return 1 + max(depth(c) for c in node.children) if isinstance(node, FilterTree) else 1
+
+        if depth(self) > self.MAX_DEPTH:
+            raise ValueError(f"FilterTree nesting exceeds MAX_DEPTH={self.MAX_DEPTH}")
+        return self
+
+    @classmethod
+    def from_flat_and(cls, filters: list[PathFilter]) -> FilterTree | None:
+        """Wrap a flat list of PathFilter into an AND group (or None)."""
+        return None if not filters else cls(op=BooleanOperator.AND, children=list(filters))
+
+    def get_all_paths(self) -> set[str]:
+        """Collects all unique paths from the PathFilter leaves in the tree."""
+        return {leaf.path for leaf in self.get_all_leaves()}
+
+    def get_all_leaves(self) -> list[PathFilter]:
+        """Collect all PathFilter leaves in the tree."""
+        leaves: list[PathFilter] = []
+        for child in self.children:
+            if isinstance(child, PathFilter):
+                leaves.append(child)
+            else:
+                leaves.extend(child.get_all_leaves())
+        return leaves
+
+    def to_expression(
+        self,
+        entity_id_col: SQLAColumn,
+        *,
+        entity_type_value: str | None = None,
+    ) -> ColumnElement[bool]:
+        """Compile this tree into a SQLAlchemy boolean expression.
+
+        Parameters
+        ----------
+        entity_id_col : SQLAColumn
+            Column in the outer query representing the entity ID.
+        entity_type_value : str, optional
+            If provided, each subquery is additionally constrained to this entity type.
+
+        Returns:
+        -------
+        ColumnElement[bool]
+            A SQLAlchemy expression suitable for use in a WHERE clause.
+        """
+        alias_idx = count(1)
+
+        def leaf_exists(pf: PathFilter) -> ColumnElement[bool]:
+            from sqlalchemy.orm import aliased
+
+            alias = aliased(AiSearchIndex, name=f"flt_{next(alias_idx)}")
+
+            correlates = [alias.entity_id == entity_id_col]
+            if entity_type_value is not None:
+                correlates.append(alias.entity_type == entity_type_value)
+
+            if isinstance(pf.condition, LtreeFilter):
+                # row-level predicate is always positive
+                positive = pf.condition.to_expression(alias.path, pf.path)
+                subq = select(1).select_from(alias).where(and_(*correlates, positive))
+                if pf.condition.op == FilterOp.NOT_HAS_COMPONENT:
+                    return ~exists(subq)  # NOT at the entity level
+                return exists(subq)
+
+            # value leaf: path predicate + typed value compare
+            if "." not in pf.path:
+                path_pred = LtreeFilter(op=FilterOp.ENDS_WITH, value=pf.path).to_expression(alias.path, "")
+            else:
+                path_pred = alias.path == Ltree(pf.path)
+
+            value_pred = pf.to_expression(alias.value, alias.value_type)
+            subq = select(1).select_from(alias).where(and_(*correlates, path_pred, value_pred))
+            return exists(subq)
+
+        def compile_node(node: FilterTree | PathFilter) -> ColumnElement[bool]:
+            if isinstance(node, FilterTree):
+                compiled = [compile_node(ch) for ch in node.children]
+                return and_(*compiled) if node.op == BooleanOperator.AND else or_(*compiled)
+            return leaf_exists(node)
+
+        return compile_node(self)

orchestrator/search/filters/date_filters.py

@@ -0,0 +1,75 @@
+from datetime import date, datetime
+from typing import Annotated, Any, Literal
+
+from dateutil.parser import parse as dt_parse
+from pydantic import BaseModel, BeforeValidator, Field, model_validator
+from sqlalchemy import TIMESTAMP, and_
+from sqlalchemy import cast as sa_cast
+from sqlalchemy.sql.elements import ColumnElement
+
+from orchestrator.search.core.types import FilterOp, SQLAColumn
+
+
+def _validate_date_string(v: Any) -> Any:
+    if not isinstance(v, str):
+        return v
+    try:
+        dt_parse(v)
+        return v
+    except Exception as exc:
+        raise ValueError("is not a valid date or datetime string") from exc
+
+
+DateValue = datetime | date | str
+ValidatedDateValue = Annotated[DateValue, BeforeValidator(_validate_date_string)]
+
+
+class DateRange(BaseModel):
+
+    start: ValidatedDateValue
+    end: ValidatedDateValue
+
+    @model_validator(mode="after")
+    def _order(self) -> "DateRange":
+        to_datetime = dt_parse(str(self.end))
+        from_datetime = dt_parse(str(self.start))
+        if to_datetime <= from_datetime:
+            raise ValueError("'to' must be after 'from'")
+        return self
+
+
+class DateValueFilter(BaseModel):
+    """A filter that operates on a single date value."""
+
+    op: Literal[FilterOp.EQ, FilterOp.NEQ, FilterOp.LT, FilterOp.LTE, FilterOp.GT, FilterOp.GTE]
+    value: ValidatedDateValue
+
+    def to_expression(self, column: SQLAColumn, path: str) -> ColumnElement[bool]:
+        date_column = sa_cast(column, TIMESTAMP(timezone=True))
+        match self.op:
+            case FilterOp.EQ:
+                return date_column == self.value
+            case FilterOp.NEQ:
+                return date_column != self.value
+            case FilterOp.LT:
+                return date_column < self.value
+            case FilterOp.LTE:
+                return date_column <= self.value
+            case FilterOp.GT:
+                return date_column > self.value
+            case FilterOp.GTE:
+                return date_column >= self.value
+
+
+class DateRangeFilter(BaseModel):
+    """A filter that operates on a range of dates."""
+
+    op: Literal[FilterOp.BETWEEN]
+    value: DateRange
+
+    def to_expression(self, column: SQLAColumn, path: str) -> ColumnElement[bool]:
+        date_column = sa_cast(column, TIMESTAMP(timezone=True))
+        return and_(date_column >= self.value.start, date_column < self.value.end)
+
+
+DateFilter = Annotated[DateValueFilter | DateRangeFilter, Field(discriminator="op")]

orchestrator/search/filters/definitions.py

@@ -0,0 +1,93 @@
+from orchestrator.search.core.types import FieldType, FilterOp, UIType
+from orchestrator.search.schemas.results import TypeDefinition, ValueSchema
+
+
+def operators_for(ft: FieldType) -> list[FilterOp]:
+    """Return the list of valid operators for a given FieldType."""
+    return list(value_schema_for(ft).keys())
+
+
+def component_operators() -> dict[FilterOp, ValueSchema]:
+    """Return operators available for path components."""
+    return {
+        FilterOp.HAS_COMPONENT: ValueSchema(kind=UIType.COMPONENT),
+        FilterOp.NOT_HAS_COMPONENT: ValueSchema(kind=UIType.COMPONENT),
+    }
+
+
+def value_schema_for(ft: FieldType) -> dict[FilterOp, ValueSchema]:
+    """Return the value schema map for a given FieldType."""
+    if ft in (FieldType.INTEGER, FieldType.FLOAT):
+        return {
+            FilterOp.EQ: ValueSchema(kind=UIType.NUMBER),
+            FilterOp.NEQ: ValueSchema(kind=UIType.NUMBER),
+            FilterOp.LT: ValueSchema(kind=UIType.NUMBER),
+            FilterOp.LTE: ValueSchema(kind=UIType.NUMBER),
+            FilterOp.GT: ValueSchema(kind=UIType.NUMBER),
+            FilterOp.GTE: ValueSchema(kind=UIType.NUMBER),
+            FilterOp.BETWEEN: ValueSchema(
+                kind="object",
+                fields={
+                    "start": ValueSchema(kind=UIType.NUMBER),
+                    "end": ValueSchema(kind=UIType.NUMBER),
+                },
+            ),
+        }
+
+    if ft == FieldType.BOOLEAN:
+        return {
+            FilterOp.EQ: ValueSchema(kind=UIType.BOOLEAN),
+            FilterOp.NEQ: ValueSchema(kind=UIType.BOOLEAN),
+        }
+
+    if ft == FieldType.DATETIME:
+        return {
+            FilterOp.EQ: ValueSchema(kind=UIType.DATETIME),
+            FilterOp.NEQ: ValueSchema(kind=UIType.DATETIME),
+            FilterOp.LT: ValueSchema(kind=UIType.DATETIME),
+            FilterOp.LTE: ValueSchema(kind=UIType.DATETIME),
+            FilterOp.GT: ValueSchema(kind=UIType.DATETIME),
+            FilterOp.GTE: ValueSchema(kind=UIType.DATETIME),
+            FilterOp.BETWEEN: ValueSchema(
+                kind="object",
+                fields={
+                    "start": ValueSchema(kind=UIType.DATETIME),
+                    "end": ValueSchema(kind=UIType.DATETIME),
+                },
+            ),
+        }
+
+    return {
+        FilterOp.EQ: ValueSchema(kind=UIType.STRING),
+        FilterOp.NEQ: ValueSchema(kind=UIType.STRING),
+    }
+
+
+def generate_definitions() -> dict[UIType, TypeDefinition]:
+    """Generate the full definitions dictionary for all UI types."""
+    definitions: dict[UIType, TypeDefinition] = {}
+
+    for ui_type in UIType:
+        if ui_type == UIType.COMPONENT:
+            # Special case for component filtering
+            comp_ops = component_operators()
+            definitions[ui_type] = TypeDefinition(
+                operators=list(comp_ops.keys()),
+                valueSchema=comp_ops,
+            )
+        else:
+            # Regular field types
+            if ui_type == UIType.NUMBER:
+                rep_ft = FieldType.INTEGER
+            elif ui_type == UIType.DATETIME:
+                rep_ft = FieldType.DATETIME
+            elif ui_type == UIType.BOOLEAN:
+                rep_ft = FieldType.BOOLEAN
+            else:
+                rep_ft = FieldType.STRING
+
+            definitions[ui_type] = TypeDefinition(
+                operators=operators_for(rep_ft),
+                valueSchema=value_schema_for(rep_ft),
+            )
+    return definitions

orchestrator/search/filters/ltree_filters.py

@@ -0,0 +1,43 @@
+from typing import Literal
+
+from pydantic import BaseModel, Field
+from sqlalchemy import TEXT, bindparam
+from sqlalchemy.sql.elements import ColumnElement
+from sqlalchemy_utils.types.ltree import Ltree
+
+from orchestrator.search.core.types import FilterOp, SQLAColumn
+
+
+class LtreeFilter(BaseModel):
+    """Filter for ltree path operations."""
+
+    op: Literal[
+        FilterOp.MATCHES_LQUERY,
+        FilterOp.IS_ANCESTOR,
+        FilterOp.IS_DESCENDANT,
+        FilterOp.PATH_MATCH,
+        FilterOp.HAS_COMPONENT,
+        FilterOp.NOT_HAS_COMPONENT,
+        FilterOp.ENDS_WITH,
+    ]
+    value: str = Field(description="The ltree path or lquery pattern to compare against.")
+
+    def to_expression(self, column: SQLAColumn, path: str) -> ColumnElement[bool]:
+        """Converts the filter condition into a SQLAlchemy expression."""
+        match self.op:
+            case FilterOp.IS_DESCENDANT:
+                ltree_value = Ltree(self.value)
+                return column.op("<@")(ltree_value)
+            case FilterOp.IS_ANCESTOR:
+                ltree_value = Ltree(self.value)
+                return column.op("@>")(ltree_value)
+            case FilterOp.MATCHES_LQUERY:
+                param = bindparam(None, self.value, type_=TEXT)
+                return column.op("~")(param)
+            case FilterOp.PATH_MATCH:
+                ltree_value = Ltree(path)
+                return column == ltree_value
+            case FilterOp.HAS_COMPONENT | FilterOp.NOT_HAS_COMPONENT:
+                return column.op("~")(bindparam(None, f"*.{self.value}.*", type_=TEXT))
+            case FilterOp.ENDS_WITH:
+                return column.op("~")(bindparam(None, f"*.{self.value}", type_=TEXT))

orchestrator/search/filters/numeric_filter.py

@@ -0,0 +1,60 @@
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, Field, model_validator
+from sqlalchemy import DOUBLE_PRECISION, INTEGER, and_
+from sqlalchemy import cast as sa_cast
+from sqlalchemy.sql.elements import ColumnElement
+from typing_extensions import Self
+
+from orchestrator.search.core.types import FilterOp, SQLAColumn
+
+
+class NumericRange(BaseModel):
+    start: int | float
+    end: int | float
+
+    @model_validator(mode="after")
+    def validate_order(self) -> Self:
+        if self.end <= self.start:
+            raise ValueError("'end' must be greater than 'start'")
+        return self
+
+
+class NumericValueFilter(BaseModel):
+    """A filter for single numeric value comparisons (int or float)."""
+
+    op: Literal[FilterOp.EQ, FilterOp.NEQ, FilterOp.LT, FilterOp.LTE, FilterOp.GT, FilterOp.GTE]
+    value: int | float
+
+    def to_expression(self, column: SQLAColumn, path: str) -> ColumnElement[bool]:
+        cast_type = INTEGER if isinstance(self.value, int) else DOUBLE_PRECISION
+        numeric_column: ColumnElement[Any] = sa_cast(column, cast_type)
+        match self.op:
+
+            case FilterOp.EQ:
+                return numeric_column == self.value
+            case FilterOp.NEQ:
+                return numeric_column != self.value
+            case FilterOp.LT:
+                return numeric_column < self.value
+            case FilterOp.LTE:
+                return numeric_column <= self.value
+            case FilterOp.GT:
+                return numeric_column > self.value
+            case FilterOp.GTE:
+                return numeric_column >= self.value
+
+
+class NumericRangeFilter(BaseModel):
+    """A filter for a range of numeric values (int or float)."""
+
+    op: Literal[FilterOp.BETWEEN]
+    value: NumericRange
+
+    def to_expression(self, column: SQLAColumn, path: str) -> ColumnElement[bool]:
+        cast_type = INTEGER if isinstance(self.value.start, int) else DOUBLE_PRECISION
+        numeric_column: ColumnElement[Any] = sa_cast(column, cast_type)
+        return and_(numeric_column >= self.value.start, numeric_column <= self.value.end)
+
+
+NumericFilter = Annotated[NumericValueFilter | NumericRangeFilter, Field(discriminator="op")]

orchestrator/search/indexing/__init__.py

@@ -0,0 +1,3 @@
+from .tasks import run_indexing_for_entity
+
+__all__ = ["run_indexing_for_entity"]