orchestrator-core 4.4.1__py3-none-any.whl → 4.5.1a1__py3-none-any.whl

orchestrator/search/core/types.py

@@ -0,0 +1,281 @@
+from dataclasses import dataclass
+from datetime import date, datetime
+from enum import Enum, IntEnum
+from typing import Annotated, Any, Literal, NamedTuple, TypeAlias, TypedDict, get_args, get_origin
+from uuid import UUID
+
+from sqlalchemy.orm.attributes import InstrumentedAttribute
+from sqlalchemy.sql.elements import ColumnElement
+from sqlalchemy_utils.types.ltree import Ltree
+
+from orchestrator.types import filter_nonetype, get_origin_and_args, is_optional_type, is_union_type
+
+from .validators import is_bool_string, is_iso_date, is_uuid
+
+SQLAColumn: TypeAlias = ColumnElement[Any] | InstrumentedAttribute[Any]
+
+
+@dataclass
+class SearchMetadata:
+    """Metadata about the search operation performed."""
+
+    search_type: str
+    description: str
+
+    @classmethod
+    def structured(cls) -> "SearchMetadata":
+        return cls(
+            search_type="structured", description="This search performs a filter-based search using structured queries."
+        )
+
+    @classmethod
+    def fuzzy(cls) -> "SearchMetadata":
+        return cls(
+            search_type="fuzzy",
+            description="This search performs a trigram similarity search.",
+        )
+
+    @classmethod
+    def semantic(cls) -> "SearchMetadata":
+        return cls(
+            search_type="semantic",
+            description="This search performs a vector similarity search, using L2 distance on embeddings with minimum distance scoring (normalized).",
+        )
+
+    @classmethod
+    def hybrid(cls) -> "SearchMetadata":
+        return cls(
+            search_type="hybrid",
+            description="This search performs reciprocal rank fusion combining trigram similarity, word_similarity, and L2 vector distance.",
+        )
+
+    @classmethod
+    def empty(cls) -> "SearchMetadata":
+        return cls(search_type="empty", description="Empty search - no criteria provided")
+
+
+class BooleanOperator(str, Enum):
+    AND = "AND"
+    OR = "OR"
+
+
+class FilterOp(str, Enum):
+    EQ = "eq"
+    NEQ = "neq"
+    LT = "lt"
+    LIKE = "like"
+    LTE = "lte"
+    GT = "gt"
+    GTE = "gte"
+    BETWEEN = "between"
+
+    MATCHES_LQUERY = "matches_lquery"  # The ~ operator for wildcard matching
+    IS_ANCESTOR = "is_ancestor"  # The @> operator
+    IS_DESCENDANT = "is_descendant"  # The <@ operator
+    PATH_MATCH = "path_match"
+
+    HAS_COMPONENT = "has_component"  # Path contains this segment
+    NOT_HAS_COMPONENT = "not_has_component"  # Path doesn't contain segment
+    ENDS_WITH = "ends_with"
+
+
+class EntityType(str, Enum):
+    SUBSCRIPTION = "SUBSCRIPTION"
+    PRODUCT = "PRODUCT"
+    WORKFLOW = "WORKFLOW"
+    PROCESS = "PROCESS"
+
+
+class ActionType(str, Enum):
+    """Defines the explicit, safe actions the agent can request."""
+
+    SELECT = "select"  # Retrieve a list of matching records.
+    # COUNT = "count"  # For phase1; the agent will not support this yet.
+
+
+class UIType(str, Enum):
+    STRING = "string"
+    NUMBER = "number"
+    BOOLEAN = "boolean"
+    DATETIME = "datetime"
+    COMPONENT = "component"
+
+    @classmethod
+    def from_field_type(cls, ft: "FieldType") -> "UIType":
+        """Create a UIType from a backend FieldType to indicate how a value must be rendered."""
+        if ft in (FieldType.INTEGER, FieldType.FLOAT):
+            return cls.NUMBER
+        if ft == FieldType.BOOLEAN:
+            return cls.BOOLEAN
+        if ft == FieldType.DATETIME:
+            return cls.DATETIME
+        return cls.STRING
+
+
+class FieldType(str, Enum):
+    STRING = "string"
+    INTEGER = "integer"
+    FLOAT = "float"
+    BOOLEAN = "boolean"
+    DATETIME = "datetime"
+    UUID = "uuid"
+    BLOCK = "block"
+    RESOURCE_TYPE = "resource_type"
+
+    @classmethod
+    def infer(cls, val: Any) -> "FieldType":
+        if isinstance(val, TypedValue):
+            return cls._infer_typed_value(val)
+
+        if isinstance(val, bool):
+            return cls.BOOLEAN
+        if isinstance(val, int):
+            return cls.INTEGER
+        if isinstance(val, float):
+            return cls.FLOAT
+        if isinstance(val, UUID):
+            return cls.UUID
+        if isinstance(val, (datetime, date)):
+            return cls.DATETIME
+        if isinstance(val, str):
+            return cls._infer_from_str(val)
+
+        return cls.STRING
+
+    @classmethod
+    def _infer_typed_value(cls, val: "TypedValue") -> "FieldType":
+        if val.type == cls.BLOCK:
+            return cls.BLOCK
+        if val.type == cls.RESOURCE_TYPE:
+            return cls.RESOURCE_TYPE
+        return cls.STRING
+
+    @classmethod
+    def _infer_from_str(cls, val: str) -> "FieldType":
+        if is_uuid(val):
+            return cls.UUID
+        if is_iso_date(val):
+            return cls.DATETIME
+        if is_bool_string(val):
+            return cls.BOOLEAN
+        if val.isdigit():
+            return cls.INTEGER
+        try:
+            float(val)
+            return cls.FLOAT
+        except ValueError:
+            return cls.STRING
+
+    @classmethod
+    def from_type_hint(cls, type_hint: object) -> "FieldType":
+        """Convert type hint to FieldType."""
+        _type_mapping = {
+            int: cls.INTEGER,
+            float: cls.FLOAT,
+            bool: cls.BOOLEAN,
+            str: cls.STRING,
+            datetime: cls.DATETIME,
+            UUID: cls.UUID,
+        }
+
+        if type_hint in _type_mapping:
+            return _type_mapping[type_hint]  # type: ignore[index]
+
+        if get_origin(type_hint) is Annotated:
+            inner_type = get_args(type_hint)[0]
+            return cls.from_type_hint(inner_type)
+
+        origin, args = get_origin_and_args(type_hint)
+
+        if origin is list:
+            return cls._handle_list_type(args)
+
+        if origin is Literal:
+            return cls._handle_literal_type(args)
+
+        if is_optional_type(type_hint) or is_union_type(type_hint):
+            return cls._handle_union_type(args)
+
+        if isinstance(type_hint, type):
+            return cls._handle_class_type(type_hint)
+
+        return cls.STRING
+
+    @classmethod
+    def _handle_list_type(cls, args: tuple) -> "FieldType":
+        if args:
+            element_type = args[0]
+            return cls.from_type_hint(element_type)
+        return cls.STRING
+
+    @classmethod
+    def _handle_literal_type(cls, args: tuple) -> "FieldType":
+        if not args:
+            return cls.STRING
+        first_value = args[0]
+        if isinstance(first_value, bool):
+            return cls.BOOLEAN
+        if isinstance(first_value, int):
+            return cls.INTEGER
+        if isinstance(first_value, str):
+            return cls.STRING
+        if isinstance(first_value, float):
+            return cls.FLOAT
+        return cls.STRING
+
+    @classmethod
+    def _handle_union_type(cls, args: tuple) -> "FieldType":
+        non_none_types = list(filter_nonetype(args))
+        if non_none_types:
+            return cls.from_type_hint(non_none_types[0])
+        return cls.STRING
+
+    @classmethod
+    def _handle_class_type(cls, type_hint: type) -> "FieldType":
+        if issubclass(type_hint, IntEnum):
+            return cls.INTEGER
+        if issubclass(type_hint, Enum):
+            return cls.STRING
+
+        from orchestrator.domain.base import ProductBlockModel
+
+        if issubclass(type_hint, ProductBlockModel):
+            return cls.BLOCK
+
+        return cls.STRING
+
+    def is_embeddable(self, value: str | None) -> bool:
+        """Check if a field should be embedded."""
+        if value is None:
+            return False
+
+        # If inference suggests it's not actually a string, don't embed it
+        return FieldType._infer_from_str(value) == FieldType.STRING
+
+
+@dataclass(frozen=True)
+class TypedValue:
+    value: Any
+    type: FieldType
+
+
+class ExtractedField(NamedTuple):
+    path: str
+    value: str
+    value_type: FieldType
+
+    @classmethod
+    def from_raw(cls, path: str, raw_value: Any) -> "ExtractedField":
+        value = str(raw_value.value if isinstance(raw_value, TypedValue) else raw_value)
+        value_type = FieldType.infer(raw_value)
+        return cls(path=path, value=value, value_type=value_type)
+
+
+class IndexableRecord(TypedDict):
+    entity_id: str
+    entity_type: str
+    path: Ltree
+    value: Any
+    value_type: Any
+    content_hash: str
+    embedding: list[float] | None

orchestrator/search/core/validators.py

@@ -0,0 +1,27 @@
+import uuid
+
+from dateutil.parser import isoparse
+
+
+def is_uuid(value: str) -> bool:
+    """Check if a string is a valid UUID."""
+    try:
+        uuid.UUID(value)
+        return True
+    except (ValueError, TypeError):
+        return False
+
+
+def is_iso_date(value: str) -> bool:
+    """Check if a string is a valid ISO 8601 date."""
+    try:
+        isoparse(value)
+        return True
+    except (ValueError, TypeError):
+        return False
+
+
+def is_bool_string(value: str) -> bool:
+    """Check if a string explicitly represents a boolean value with true/false."""
+
+    return value.strip().lower() in {"true", "false"}

orchestrator/search/docs/index.md

@@ -0,0 +1,37 @@
+# Search Indexing CLI
+
+Typer-based CLI for maintaining search indexes (subscriptions, products, processes, workflows).
+
+## Usage
+
+Run from project root:
+
+```
+dotenv run python main.py index [COMMAND] [OPTIONS]
+```
+
+### Commands
+
+- `subscriptions` – index `subscription_search_index`
+- `products` – index `product_search_index`
+- `processes` – index `process_search_index`
+- `workflows` – index `workflow_search_index`
+
+### Options
+
+- `--<id>` – UUID of a specific entity (default: all)
+- `--dry-run` – no DB writes
+- `--force-index` – re-index even if unchanged
+
+### Examples
+
+```
+# Index all subscriptions
+dotenv run python main.py index subscriptions
+
+# Re-index all subscriptions
+dotenv run python main.py index subscriptions --force-index
+
+# Index a single subscription
+dotenv run python main.py index subscriptions --subscription-id=<UUID>
+```

orchestrator/search/docs/running_local_text_embedding_inference.md

@@ -0,0 +1,45 @@
+# Running a local MiniLM embedding server with Hugging Face TEI
+
+Only **OpenAI-compatible endpoints** are supported locally.
+
+You can spin up a embedding API based on **sentence-transformers/all-MiniLM-L6-v2** using [Hugging Face TEI](https://github.com/huggingface/text-embeddings-inference):
+
+```bash
+docker run --rm -p 8080:80 ghcr.io/huggingface/text-embeddings-inference:cpu-1.8 \
+    --model-id sentence-transformers/all-MiniLM-L6-v2
+```
+
+---
+
+## Environment variables
+
+Point your backend to the local endpoint and declare the new vector size:
+
+```env
+OPENAI_BASE_URL=http://localhost:8080/v1
+EMBEDDING_DIMENSION=384
+```
+
+Depending on the model, you might want to change the `EMBEDDING_FALLBACK_MAX_TOKENS` and `EMBEDDING_MAX_BATCH_SIZE` settings, which are set conservatively and according to the requirements of the setup used in this example.
+
+---
+
+## Apply the schema change
+
+With these new settings run:
+
+```bash
+dotenv run python main.py embedding resize
+```
+
+**Note** that this will delete all records and you will have to re-index.
+
+---
+
+## Re-index embeddings
+
+```bash
+dotenv run python main.py index subscriptions
+```
+
+The search index now uses **384-dimension MiniLM vectors** served from your local Docker container. That’s it! 🚀

orchestrator/search/filters/__init__.py

@@ -0,0 +1,27 @@
+from .base import (
+    EqualityFilter,
+    FilterCondition,
+    FilterTree,
+    PathFilter,
+    StringFilter,
+)
+from .date_filters import DateFilter, DateRangeFilter, DateValueFilter
+from .ltree_filters import LtreeFilter
+from .numeric_filter import NumericFilter, NumericRangeFilter, NumericValueFilter
+
+__all__ = [
+    # Base filter classes
+    "PathFilter",
+    "FilterTree",
+    "FilterCondition",
+    "StringFilter",
+    "EqualityFilter",
+    # Filters for specific value types
+    "NumericValueFilter",
+    "NumericRangeFilter",
+    "DateValueFilter",
+    "DateRangeFilter",
+    "DateFilter",
+    "LtreeFilter",
+    "NumericFilter",
+]

orchestrator/search/filters/base.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+from itertools import count
+from typing import Any, ClassVar, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from sqlalchemy import BinaryExpression, and_, cast, exists, literal, or_, select
+from sqlalchemy.dialects.postgresql import BOOLEAN
+from sqlalchemy.sql.elements import ColumnElement
+from sqlalchemy_utils.types.ltree import Ltree
+
+from orchestrator.db.models import AiSearchIndex
+from orchestrator.search.core.types import BooleanOperator, FieldType, FilterOp, SQLAColumn, UIType
+
+from .date_filters import DateFilter
+from .ltree_filters import LtreeFilter
+from .numeric_filter import NumericFilter
+
+
+class EqualityFilter(BaseModel):
+    op: Literal[FilterOp.EQ, FilterOp.NEQ]
+    value: Any
+
+    def to_expression(self, column: SQLAColumn, path: str) -> BinaryExpression[bool] | ColumnElement[bool]:
+        if isinstance(self.value, bool):
+            colb = cast(column, BOOLEAN)
+            return colb.is_(self.value) if self.op == FilterOp.EQ else ~colb.is_(self.value)
+        sv = str(self.value)
+        return (column == sv) if self.op == FilterOp.EQ else (column != sv)
+
+
+class StringFilter(BaseModel):
+    op: Literal[FilterOp.LIKE]
+    value: str
+
+    def to_expression(self, column: SQLAColumn, path: str) -> ColumnElement[bool]:
+        return column.like(self.value)
+
+    @model_validator(mode="after")
+    def validate_like_pattern(self) -> StringFilter:
+        """If the operation is 'like', the value must contain a wildcard."""
+        if self.op == FilterOp.LIKE:
+            if "%" not in self.value and "_" not in self.value:
+                raise ValueError("The value for a 'like' operation must contain a wildcard character ('%' or '_').")
+        return self
+
+
+FilterCondition = (
+    DateFilter  # DATETIME
+    | NumericFilter  # INT/FLOAT
+    | EqualityFilter  # BOOLEAN/UUID/BLOCK/RESOURCE_TYPE
+    | StringFilter  # STRING TODO: convert to hybrid search
+    | LtreeFilter  # Path
+)
+
+
+class PathFilter(BaseModel):
+
+    path: str = Field(description="The ltree path of the field to filter on, e.g., 'subscription.customer_id'.")
+    condition: FilterCondition = Field(description="The filter condition to apply.")
+
+    value_kind: UIType
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "examples": [
+                {
+                    "path": "subscription.status",
+                    "condition": {"op": "eq", "value": "active"},
+                },
+                {
+                    "path": "subscription.customer_id",
+                    "condition": {"op": "ne", "value": "acme"},
+                },
+                {
+                    "path": "subscription.start_date",
+                    "condition": {"op": "gt", "value": "2025-01-01"},
+                },
+                {
+                    "path": "subscription.end_date",
+                    "condition": {
+                        "op": "between",
+                        "value": {"from": "2025-06-01", "to": "2025-07-01"},
+                    },
+                },
+                {
+                    "path": "subscription.*.name",
+                    "condition": {"op": "matches_lquery", "value": "*.foo_*"},
+                },
+            ]
+        }
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def _transfer_path_to_value_if_needed(cls, data: Any) -> Any:
+        """Transform for path-only filters.
+
+        If `op` is `has_component`, `not_has_component`, or `ends_with` and no `value` is
+        provided in the `condition`, this validator will automatically use the `path`
+        field as the `value` and set the `path` to a wildcard '*' for the query.
+        """
+        if isinstance(data, dict):
+            path = data.get("path")
+            condition = data.get("condition")
+
+            if path and isinstance(condition, dict):
+                op = condition.get("op")
+                value = condition.get("value")
+
+                path_only_ops = [FilterOp.HAS_COMPONENT, FilterOp.NOT_HAS_COMPONENT, FilterOp.ENDS_WITH]
+
+                if op in path_only_ops and value is None:
+                    condition["value"] = path
+                    data["path"] = "*"
+        return data
+
+    def to_expression(self, value_column: SQLAColumn, value_type_column: SQLAColumn) -> ColumnElement[bool]:
+        """Convert the path filter into a SQLAlchemy expression with type safety.
+
+        This method creates a type guard to ensure we only match compatible field types,
+        then delegates to the specific filter condition.
+
+        Parameters
+        ----------
+        value_column : ColumnElement
+            The SQLAlchemy column element representing the value to be filtered.
+        value_type_column : ColumnElement
+            The SQLAlchemy column element representing the field type.
+
+        Returns:
+        -------
+        ColumnElement[bool]
+            A SQLAlchemy boolean expression that can be used in a ``WHERE`` clause.
+        """
+        # Type guard - only match compatible field types
+        allowed_field_types = [ft.value for ft in FieldType if UIType.from_field_type(ft) == self.value_kind]
+        type_guard = value_type_column.in_(allowed_field_types) if allowed_field_types else literal(True)
+
+        return and_(type_guard, self.condition.to_expression(value_column, self.path))
+
+
+class FilterTree(BaseModel):
+    model_config = ConfigDict(
+        json_schema_extra={
+            "description": (
+                "Boolean filter tree. Operators must be UPPERCASE: AND / OR.\n"
+                "Node shapes:\n"
+                "  • Group: {'op':'AND'|'OR', 'children': [<PathFilter|FilterTree>, ...]}\n"
+                "  • Leaf (PathFilter): {'path':'<ltree>', 'condition': {...}}\n"
+                "Rules:\n"
+                "  • Do NOT put 'op' or 'children' inside a leaf 'condition'.\n"
+                "  • Max depth = 5.\n"
+                "  • Use from_flat_and() for a flat list of leaves."
+            ),
+            "examples": [
+                {
+                    "op": "AND",
+                    "children": [
+                        {"path": "subscription.status", "condition": {"op": "eq", "value": "active"}},
+                        {"path": "subscription.start_date", "condition": {"op": "gt", "value": "2021-01-01"}},
+                    ],
+                },
+                {
+                    "op": "AND",
+                    "children": [
+                        {"path": "subscription.start_date", "condition": {"op": "gte", "value": "2024-01-01"}},
+                        {
+                            "op": "OR",
+                            "children": [
+                                {"path": "subscription.product_name", "condition": {"op": "like", "value": "%fiber%"}},
+                                {"path": "subscription.customer_id", "condition": {"op": "eq", "value": "Surf"}},
+                            ],
+                        },
+                    ],
+                },
+            ],
+        }
+    )
+
+    op: BooleanOperator = Field(
+        description="Operator for grouping conditions in uppercase.", default=BooleanOperator.AND
+    )
+
+    children: list[FilterTree | PathFilter] = Field(min_length=1, description="Path filters or nested groups.")
+
+    MAX_DEPTH: ClassVar[int] = 5
+
+    @model_validator(mode="after")
+    def _validate_depth(self) -> FilterTree:
+        def depth(node: "FilterTree | PathFilter") -> int:
+            return 1 + max(depth(c) for c in node.children) if isinstance(node, FilterTree) else 1
+
+        if depth(self) > self.MAX_DEPTH:
+            raise ValueError(f"FilterTree nesting exceeds MAX_DEPTH={self.MAX_DEPTH}")
+        return self
+
+    @classmethod
+    def from_flat_and(cls, filters: list[PathFilter]) -> FilterTree | None:
+        """Wrap a flat list of PathFilter into an AND group (or None)."""
+        return None if not filters else cls(op=BooleanOperator.AND, children=list(filters))
+
+    def get_all_paths(self) -> set[str]:
+        """Collects all unique paths from the PathFilter leaves in the tree."""
+        return {leaf.path for leaf in self.get_all_leaves()}
+
+    def get_all_leaves(self) -> list[PathFilter]:
+        """Collect all PathFilter leaves in the tree."""
+        leaves: list[PathFilter] = []
+        for child in self.children:
+            if isinstance(child, PathFilter):
+                leaves.append(child)
+            else:
+                leaves.extend(child.get_all_leaves())
+        return leaves
+
+    def to_expression(
+        self,
+        entity_id_col: SQLAColumn,
+        *,
+        entity_type_value: str | None = None,
+    ) -> ColumnElement[bool]:
+        """Compile this tree into a SQLAlchemy boolean expression.
+
+        Parameters
+        ----------
+        entity_id_col : SQLAColumn
+            Column in the outer query representing the entity ID.
+        entity_type_value : str, optional
+            If provided, each subquery is additionally constrained to this entity type.
+
+        Returns:
+        -------
+        ColumnElement[bool]
+            A SQLAlchemy expression suitable for use in a WHERE clause.
+        """
+        alias_idx = count(1)
+
+        def leaf_exists(pf: PathFilter) -> ColumnElement[bool]:
+            from sqlalchemy.orm import aliased
+
+            alias = aliased(AiSearchIndex, name=f"flt_{next(alias_idx)}")
+
+            correlates = [alias.entity_id == entity_id_col]
+            if entity_type_value is not None:
+                correlates.append(alias.entity_type == entity_type_value)
+
+            if isinstance(pf.condition, LtreeFilter):
+                # row-level predicate is always positive
+                positive = pf.condition.to_expression(alias.path, pf.path)
+                subq = select(1).select_from(alias).where(and_(*correlates, positive))
+                if pf.condition.op == FilterOp.NOT_HAS_COMPONENT:
+                    return ~exists(subq)  # NOT at the entity level
+                return exists(subq)
+
+            # value leaf: path predicate + typed value compare
+            if "." not in pf.path:
+                path_pred = LtreeFilter(op=FilterOp.ENDS_WITH, value=pf.path).to_expression(alias.path, "")
+            else:
+                path_pred = alias.path == Ltree(pf.path)
+
+            value_pred = pf.to_expression(alias.value, alias.value_type)
+            subq = select(1).select_from(alias).where(and_(*correlates, path_pred, value_pred))
+            return exists(subq)
+
+        def compile_node(node: FilterTree | PathFilter) -> ColumnElement[bool]:
+            if isinstance(node, FilterTree):
+                compiled = [compile_node(ch) for ch in node.children]
+                return and_(*compiled) if node.op == BooleanOperator.AND else or_(*compiled)
+            return leaf_exists(node)
+
+        return compile_node(self)