orchestrator-core 4.4.0rc1__py3-none-any.whl → 5.0.0a1__py3-none-any.whl

orchestrator/cli/search_explore.py

@@ -0,0 +1,203 @@
+import asyncio
+
+import structlog
+import typer
+from pydantic import ValidationError
+
+from orchestrator.db import db
+from orchestrator.search.core.types import EntityType, FilterOp
+from orchestrator.search.filters import EqualityFilter, FilterTree, LtreeFilter, PathFilter
+from orchestrator.search.retrieval import execute_search
+from orchestrator.search.retrieval.utils import display_filtered_paths_only, display_results
+from orchestrator.search.retrieval.validation import get_structured_filter_schema
+from orchestrator.search.schemas.parameters import BaseSearchParameters
+
+app = typer.Typer(help="Experiment with the subscription search indexes.")
+
+logger = structlog.getLogger(__name__)
+
+
+@app.command()
+def structured(path: str, value: str, entity_type: EntityType = EntityType.SUBSCRIPTION, limit: int = 10) -> None:
+    """Finds subscriptions where a specific field path contains an exact value.
+
+    Example:
+        dotenv run python main.py search structured "subscription.status" "provisioning"
+        ...
+        {
+            "path": "subscription.status",
+            "value": "provisioning"
+        },
+        ...
+    """
+    path_filter = PathFilter(path=path, condition=EqualityFilter(op=FilterOp.EQ, value=value))
+    search_params = BaseSearchParameters.create(
+        entity_type=entity_type, filters=FilterTree.from_flat_and([path_filter])
+    )
+    results = asyncio.run(execute_search(search_params=search_params, db_session=db.session, limit=limit))
+    display_filtered_paths_only(results, search_params, db.session)
+    display_results(results, db.session, "Match")
+
+
+@app.command()
+def semantic(query: str, entity_type: EntityType = EntityType.SUBSCRIPTION, limit: int = 10) -> None:
+    """Finds subscriptions that are conceptually most similar to the query text.
+
+    Example:
+        dotenv run python main.py search semantic "Shop for an alligator store"
+        ...
+        {
+            "path": "subscription.shop.shop_description",
+            "value": "Kingswood reptiles shop"
+        },
+        ...
+    """
+    search_params = BaseSearchParameters.create(entity_type=entity_type, query=query)
+    results = asyncio.run(execute_search(search_params=search_params, db_session=db.session, limit=limit))
+    display_results(results, db.session, "Distance")
+
+
+@app.command()
+def fuzzy(term: str, entity_type: EntityType = EntityType.SUBSCRIPTION, limit: int = 10) -> None:
+    """Finds subscriptions containing text similar to the query, tolerating typos.
+
+    Example:
+        dotenv run python main.py search fuzzy "Colonel"
+        ...
+        {
+          "path": "description",
+          "value": "X Follower WF for TimCoronel"
+        },
+        ...
+    """
+    search_params = BaseSearchParameters.create(entity_type=entity_type, query=term)
+    results = asyncio.run(execute_search(search_params=search_params, db_session=db.session, limit=limit))
+    display_results(results, db.session, "Similarity")
+
+
+@app.command()
+def hierarchical(
+    op: str = typer.Argument(..., help="The hierarchical operation to perform."),
+    path: str = typer.Argument(..., help="The ltree path or lquery pattern for the operation."),
+    query: str | None = typer.Option(None, "--query", "-f", help="An optional fuzzy term to rank the results."),
+    entity_type: EntityType = EntityType.SUBSCRIPTION,
+    limit: int = 10,
+) -> None:
+    """Performs a hierarchical search, optionally combined with fuzzy ranking.
+
+    Examples:
+        dotenv run python main.py search hierarchical is_descendant "subscription.shop" --query "Kingwood"
+        dotenv run python main.py search hierarchical matches_lquery "*.x_follower.x_follower_status*"
+    """
+    try:
+        condition = LtreeFilter(value=path, op=op)  # type: ignore[arg-type]
+    except (ValueError, ValidationError) as e:
+        raise typer.BadParameter(f"Invalid filter: {e}")
+
+    path_filter = PathFilter(path="ltree_hierarchical_filter", condition=condition)
+
+    search_params = BaseSearchParameters.create(
+        entity_type=entity_type, filters=[FilterTree.from_flat_and([path_filter])], query=query
+    )
+    results = asyncio.run(execute_search(search_params=search_params, db_session=db.session, limit=limit))
+    display_results(results, db.session, "Hierarchical Score")
+
+
+@app.command()
+def hybrid(query: str, term: str, entity_type: EntityType = EntityType.SUBSCRIPTION, limit: int = 10) -> None:
+    """Performs a hybrid search, combining semantic and fuzzy matching.
+
+    Example:
+        dotenv run python main.py search hybrid "reptile store" "Kingswood"
+    """
+    search_params = BaseSearchParameters.create(entity_type=entity_type, query=query)
+    logger.info("Executing Hybrid Search", query=query, term=term)
+    results = asyncio.run(execute_search(search_params=search_params, db_session=db.session, limit=limit))
+    display_results(results, db.session, "Hybrid Score")
+
+
+@app.command("generate-schema")
+def generate_schema() -> None:
+    """Generates and prints the dynamic filter schema from the live search index.
+
+    This queries the index for all distinct non-string paths to be used as
+    context for the LLM agent.
+
+    Example:
+        dotenv run python main.py search generate-schema
+    """
+
+    schema_map = get_structured_filter_schema()
+
+    if not schema_map:
+        logger.warning("No filterable paths found in the search index.")
+        return
+
+    logger.info("\nAvailable Structured Filters:\n")
+    for path, value_type in schema_map.items():
+        logger.info(f"- {path}: {value_type}")
+
+    logger.info("Successfully generated dynamic schema.", path_count=len(schema_map))
+
+
+@app.command("nested-demo")
+def nested_demo(entity_type: EntityType = EntityType.SUBSCRIPTION, limit: int = 10) -> None:
+    tree = FilterTree.model_validate(
+        {
+            "op": "AND",
+            "children": [
+                {
+                    "op": "OR",
+                    "children": [
+                        # First OR case: Active subscriptions from 2024
+                        {
+                            "op": "AND",
+                            "children": [
+                                {
+                                    "path": "subscription.status",
+                                    "condition": {"op": "eq", "value": "active"},
+                                },
+                                {
+                                    "path": "subscription.start_date",
+                                    "condition": {
+                                        "op": "between",
+                                        "value": {
+                                            "start": "2024-01-01T00:00:00Z",
+                                            "end": "2024-12-31T23:59:59Z",
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                        # Second OR case: Terminated subscriptions before 2026
+                        {
+                            "op": "AND",
+                            "children": [
+                                {
+                                    "path": "subscription.status",
+                                    "condition": {"op": "eq", "value": "terminated"},
+                                },
+                                {
+                                    "path": "subscription.end_date",
+                                    "condition": {"op": "lte", "value": "2025-12-31"},
+                                },
+                            ],
+                        },
+                    ],
+                },
+                {
+                    "path": "subscription.*.port_mode",
+                    "condition": {"op": "matches_lquery", "value": "*.port_mode"},
+                },
+            ],
+        }
+    )
+
+    params = BaseSearchParameters.create(entity_type=entity_type, filters=tree)
+    results = asyncio.run(execute_search(params, db.session, limit=limit))
+
+    display_results(results, db.session, "Score")
+
+
+if __name__ == "__main__":
+    app()

orchestrator/db/models.py

@@ -20,7 +20,9 @@ from uuid import UUID
 import sqlalchemy
 import structlog
 from more_itertools import first_true
+from pgvector.sqlalchemy import Vector
 from sqlalchemy import (
+    TEXT,
     TIMESTAMP,
     Boolean,
     CheckConstraint,
@@ -29,6 +31,7 @@ from sqlalchemy import (
     ForeignKey,
     Index,
     Integer,
+    PrimaryKeyConstraint,
     Select,
     String,
     Table,
@@ -45,10 +48,12 @@ from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.ext.orderinglist import ordering_list
 from sqlalchemy.orm import Mapped, deferred, mapped_column, object_session, relationship, undefer
 from sqlalchemy.sql.functions import GenericFunction
-from sqlalchemy_utils import TSVectorType, UUIDType
+from sqlalchemy_utils import LtreeType, TSVectorType, UUIDType
 
 from orchestrator.config.assignee import Assignee
 from orchestrator.db.database import BaseModel, SearchQuery
+from orchestrator.search.core.types import FieldType
+from orchestrator.settings import app_settings
 from orchestrator.targets import Target
 from orchestrator.utils.datetime import nowtz
 from orchestrator.version import GIT_COMMIT_HASH
@@ -685,3 +690,34 @@ class SubscriptionInstanceAsJsonFunction(GenericFunction):
 
     def __init__(self, sub_inst_id: UUID):
         super().__init__(sub_inst_id)
+
+
+class AiSearchIndex(BaseModel):
+
+    __tablename__ = "ai_search_index"
+
+    entity_type = mapped_column(
+        TEXT,
+        nullable=False,
+        index=True,
+    )
+    entity_id = mapped_column(
+        UUIDType,
+        nullable=False,
+    )
+
+    # Ltree path for hierarchical data
+    path = mapped_column(LtreeType, nullable=False, index=True)
+    value = mapped_column(TEXT, nullable=False)
+
+    value_type = mapped_column(
+        Enum(FieldType, name="field_type", values_callable=lambda obj: [e.value for e in obj]), nullable=False
+    )
+
+    # Embedding
+    embedding = mapped_column(Vector(app_settings.EMBEDDING_DIMENSION), nullable=True)
+
+    # SHA-256
+    content_hash = mapped_column(String(64), nullable=False, index=True)
+
+    __table_args__ = (PrimaryKeyConstraint("entity_id", "path", name="pk_ai_search_index"),)

orchestrator/graphql/schema.py

@@ -51,14 +51,12 @@ from orchestrator.graphql.resolvers import (
     resolve_version,
     resolve_workflows,
 )
-from orchestrator.graphql.resolvers.scheduled_tasks import resolve_scheduled_tasks
 from orchestrator.graphql.schemas import DEFAULT_GRAPHQL_MODELS
 from orchestrator.graphql.schemas.customer import CustomerType
 from orchestrator.graphql.schemas.process import ProcessType
 from orchestrator.graphql.schemas.product import ProductType
 from orchestrator.graphql.schemas.product_block import ProductBlock
 from orchestrator.graphql.schemas.resource_type import ResourceType
-from orchestrator.graphql.schemas.scheduled_task import ScheduledTaskGraphql
 from orchestrator.graphql.schemas.settings import StatusType
 from orchestrator.graphql.schemas.subscription import SubscriptionInterface
 from orchestrator.graphql.schemas.version import VersionType
@@ -101,9 +99,6 @@ class OrchestratorQuery:
         description="Returns information about cache, workers, and global engine settings",
     )
     version: VersionType = authenticated_field(resolver=resolve_version, description="Returns version information")
-    scheduled_tasks: Connection[ScheduledTaskGraphql] = authenticated_field(
-        resolver=resolve_scheduled_tasks, description="Returns scheduled job information"
-    )
 
 
 @strawberry.federation.type(description="Orchestrator customer Query")

orchestrator/graphql/schemas/process.py

@@ -6,7 +6,7 @@ from strawberry.federation.schema_directives import Key
 from strawberry.scalars import JSON
 
 from oauth2_lib.strawberry import authenticated_field
-from orchestrator.api.api_v1.endpoints.processes import get_auth_callbacks, get_steps_to_evaluate_for_rbac
+from orchestrator.api.api_v1.endpoints.processes import get_auth_callbacks, get_current_steps
 from orchestrator.db import ProcessTable, ProductTable, db
 from orchestrator.graphql.pagination import EMPTY_PAGE, Connection
 from orchestrator.graphql.schemas.customer import CustomerType
@@ -86,7 +86,7 @@ class ProcessType:
         oidc_user = info.context.get_current_user
         workflow = get_workflow(self.workflow_name)
         process = load_process(db.session.get(ProcessTable, self.process_id))  # type: ignore[arg-type]
-        auth_resume, auth_retry = get_auth_callbacks(get_steps_to_evaluate_for_rbac(process), workflow)  # type: ignore[arg-type]
+        auth_resume, auth_retry = get_auth_callbacks(get_current_steps(process), workflow)  # type: ignore[arg-type]
 
         return FormUserPermissionsType(
             retryAllowed=auth_retry and auth_retry(oidc_user),  # type: ignore[arg-type]

orchestrator/graphql/utils/create_resolver_error_handler.py

@@ -25,6 +25,6 @@ def _format_context(context: dict) -> str:
 
 def create_resolver_error_handler(info: OrchestratorInfo) -> CallableErrorHandler:
     def handle_error(message: str, **context) -> None:  # type: ignore
-        return register_error(f"{message} {_format_context(context)}", info, error_type=ErrorType.BAD_REQUEST)
+        return register_error(" ".join([message, _format_context(context)]), info, error_type=ErrorType.BAD_REQUEST)
 
     return handle_error

orchestrator/migrations/versions/schema/2025-08-12_52b37b5b2714_search_index_model_for_llm_integration.py

@@ -0,0 +1,95 @@
+"""Search index model for llm integration.
+
+Revision ID: 52b37b5b2714
+Revises: 850dccac3b02
+Create Date: 2025-08-12 22:34:26.694750
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from pgvector.sqlalchemy import Vector
+from sqlalchemy.dialects import postgresql
+from sqlalchemy_utils import LtreeType
+
+from orchestrator.search.core.types import FieldType
+
+# revision identifiers, used by Alembic.
+revision = "52b37b5b2714"
+down_revision = "850dccac3b02"
+branch_labels = None
+depends_on = None
+
+TABLE = "ai_search_index"
+IDX_EMBED_HNSW = "ix_flat_embed_hnsw"
+IDX_PATH_GIST = "ix_flat_path_gist"
+IDX_PATH_BTREE = "ix_flat_path_btree"
+IDX_VALUE_TRGM = "ix_flat_value_trgm"
+IDX_CONTENT_HASH = "idx_ai_search_index_content_hash"
+
+TARGET_DIM = 1536
+
+
+def upgrade() -> None:
+    # Create PostgreSQL extensions
+    op.execute("CREATE EXTENSION IF NOT EXISTS ltree;")
+    op.execute("CREATE EXTENSION IF NOT EXISTS unaccent;")
+    op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
+    op.execute("CREATE EXTENSION IF NOT EXISTS vector;")
+
+    # Create the ai_search_index table
+    op.create_table(
+        TABLE,
+        sa.Column("entity_type", sa.Text, nullable=False),
+        sa.Column("entity_id", postgresql.UUID, nullable=False),
+        sa.Column("path", LtreeType, nullable=False),
+        sa.Column("value", sa.Text, nullable=False),
+        sa.Column("embedding", Vector(TARGET_DIM), nullable=True),
+        sa.Column("content_hash", sa.String(64), nullable=False),
+        sa.PrimaryKeyConstraint("entity_id", "path", name="pk_ai_search_index"),
+    )
+
+    field_type_enum = sa.Enum(*[ft.value for ft in FieldType], name="field_type")
+    field_type_enum.create(op.get_bind(), checkfirst=True)
+    op.add_column(
+        TABLE,
+        sa.Column("value_type", field_type_enum, nullable=False, server_default=FieldType.STRING.value),
+    )
+    op.alter_column(TABLE, "value_type", server_default=None)
+
+    op.create_index(op.f("ix_ai_search_index_entity_id"), TABLE, ["entity_id"], unique=False)
+    op.create_index(IDX_CONTENT_HASH, TABLE, ["content_hash"])
+
+    op.create_index(
+        IDX_PATH_GIST,
+        TABLE,
+        ["path"],
+        postgresql_using="GIST",
+        postgresql_ops={"path": "gist_ltree_ops"},
+    )
+    op.create_index(IDX_PATH_BTREE, TABLE, ["path"])
+    op.create_index(IDX_VALUE_TRGM, TABLE, ["value"], postgresql_using="GIN", postgresql_ops={"value": "gin_trgm_ops"})
+
+    op.create_index(
+        IDX_EMBED_HNSW,
+        TABLE,
+        ["embedding"],
+        postgresql_using="HNSW",
+        postgresql_with={"m": 16, "ef_construction": 64},
+        postgresql_ops={"embedding": "vector_l2_ops"},
+    )
+
+
+def downgrade() -> None:
+    # Drop all indexes
+    op.drop_index(IDX_EMBED_HNSW, table_name=TABLE, if_exists=True)
+    op.drop_index(IDX_VALUE_TRGM, table_name=TABLE, if_exists=True)
+    op.drop_index(IDX_PATH_BTREE, table_name=TABLE, if_exists=True)
+    op.drop_index(IDX_PATH_GIST, table_name=TABLE, if_exists=True)
+    op.drop_index(IDX_CONTENT_HASH, table_name=TABLE, if_exists=True)
+    op.drop_index(op.f("ix_ai_search_index_entity_id"), table_name=TABLE, if_exists=True)
+
+    # Drop table and enum
+    op.drop_table(TABLE, if_exists=True)
+    field_type_enum = sa.Enum(name="field_type")
+    field_type_enum.drop(op.get_bind(), checkfirst=True)

orchestrator/schedules/__init__.py

@@ -13,11 +13,12 @@
 
 
 from orchestrator.schedules.resume_workflows import run_resume_workflows
+from orchestrator.schedules.scheduling import SchedulingFunction
 from orchestrator.schedules.task_vacuum import vacuum_tasks
 from orchestrator.schedules.validate_products import validate_products
 from orchestrator.schedules.validate_subscriptions import validate_subscriptions
 
-ALL_SCHEDULERS: list = [
+ALL_SCHEDULERS: list[SchedulingFunction] = [
     run_resume_workflows,
     vacuum_tasks,
     validate_subscriptions,

orchestrator/schedules/resume_workflows.py

@@ -12,10 +12,10 @@
 # limitations under the License.
 
 
-from orchestrator.schedules.scheduler import scheduler
+from orchestrator.schedules.scheduling import scheduler
 from orchestrator.services.processes import start_process
 
 
-@scheduler.scheduled_job(id="resume-workflows", name="Resume workflows", trigger="interval", hours=1)  # type: ignore[misc]
+@scheduler(name="Resume workflows", time_unit="hour", period=1)
 def run_resume_workflows() -> None:
     start_process("task_resume_workflows")

orchestrator/schedules/scheduling.py

@@ -12,77 +12,37 @@
 # limitations under the License.
 
 from collections.abc import Callable
-from typing import TypeVar
+from typing import Protocol, cast
 
-from apscheduler.schedulers.base import BaseScheduler
-from deprecated import deprecated
+from schedule import CancelJob
 
-from orchestrator.schedules.scheduler import scheduler as default_scheduler  # your global scheduler instance
 
-F = TypeVar("F", bound=Callable[..., object])
+class SchedulingFunction(Protocol):
+    __name__: str
+    name: str
+    time_unit: str
+    period: int | None
+    at: str | None
 
+    def __call__(self) -> CancelJob | None: ...
 
-@deprecated(
-    reason="We changed from scheduler to apscheduler which has its own decoractor, use `@scheduler.scheduled_job()` from `from orchestrator.scheduling.scheduler import scheduler`"
-)
-def scheduler(
-    name: str,
-    time_unit: str,
-    period: int = 1,
-    at: str | None = None,
-    *,
-    id: str | None = None,
-    scheduler: BaseScheduler = default_scheduler,
-) -> Callable[[F], F]:
-    """APScheduler-compatible decorator to schedule a function.
 
-    id is necessary with apscheduler, if left empty it takes the function name.
+def scheduler(
+    name: str, time_unit: str, period: int = 1, at: str | None = None
+) -> Callable[[Callable[[], CancelJob | None]], SchedulingFunction]:
+    """Create schedule.
 
-    - `time_unit = "hours", period = 12`  → every 12 hours
-    - `time_unit = "day", at = "01:00"`   → every day at 1 AM
+    Either specify the period or the at. Examples:
+    time_unit = "hours", period = 12 -> will run every 12 hours
+    time_unit = "day", at="01:00" -> will run every day at 1 o'clock
     """
 
-    def decorator(func: F) -> F:
-        job_id = id or func.__name__
-
-        trigger = "interval"
-        kwargs: dict[str, int] = {}
-        if time_unit == "day" and at:
-            trigger = "cron"
-            try:
-                hour, minute = map(int, at.split(":"))
-            except ValueError:
-                raise ValueError(f"Invalid time format for 'at': {at}, expected 'HH:MM'")
-
-            kwargs = {
-                "hour": hour,
-                "minute": minute,
-            }
-        else:
-            # Map string units to timedelta kwargs for IntervalTrigger
-            unit_map = {
-                "seconds": "seconds",
-                "second": "seconds",
-                "minutes": "minutes",
-                "minute": "minutes",
-                "hours": "hours",
-                "hour": "hours",
-                "days": "days",
-                "day": "days",
-            }
-
-            interval_arg = unit_map.get(time_unit.lower(), time_unit.lower())
-            kwargs = {interval_arg: period}
-
-        scheduler.add_job(
-            func,
-            trigger=trigger,
-            id=job_id,
-            name=name,
-            replace_existing=True,
-            **kwargs,
-        )
-
-        return func
+    def _scheduler(f: Callable[[], CancelJob | None]) -> SchedulingFunction:
+        schedule = cast(SchedulingFunction, f)
+        schedule.name = name
+        schedule.time_unit = time_unit
+        schedule.period = period
+        schedule.at = at
+        return schedule
 
-    return decorator
+    return _scheduler

orchestrator/schedules/task_vacuum.py

@@ -12,10 +12,10 @@
 # limitations under the License.
 
 
-from orchestrator.schedules.scheduler import scheduler
+from orchestrator.schedules.scheduling import scheduler
 from orchestrator.services.processes import start_process
 
 
-@scheduler.scheduled_job(id="clean-tasks", name="Clean up tasks", trigger="interval", hours=6)  # type: ignore[misc]
+@scheduler(name="Clean up tasks", time_unit="hours", period=6)
 def vacuum_tasks() -> None:
     start_process("task_clean_up_tasks")

orchestrator/schedules/validate_products.py

@@ -14,17 +14,11 @@ from sqlalchemy import func, select
 
 from orchestrator.db import db
 from orchestrator.db.models import ProcessTable
-from orchestrator.schedules.scheduler import scheduler
+from orchestrator.schedules.scheduling import scheduler
 from orchestrator.services.processes import start_process
 
 
-@scheduler.scheduled_job(  # type: ignore[misc]
-    id="validate-products",
-    name="Validate Products and inactive subscriptions",
-    trigger="cron",
-    hour=2,
-    minute=30,
-)
+@scheduler(name="Validate Products and inactive subscriptions", time_unit="day", at="02:30")
 def validate_products() -> None:
     uncompleted_products = db.session.scalar(
         select(func.count())

orchestrator/schedules/validate_subscriptions.py

@@ -16,7 +16,7 @@ from threading import BoundedSemaphore
 
 import structlog
 
-from orchestrator.schedules.scheduler import scheduler
+from orchestrator.schedules.scheduling import scheduler
 from orchestrator.services.subscriptions import (
     get_subscriptions_on_product_table,
     get_subscriptions_on_product_table_in_sync,
@@ -33,7 +33,7 @@ logger = structlog.get_logger(__name__)
 task_semaphore = BoundedSemaphore(value=2)
 
 
-@scheduler.scheduled_job(id="subscriptions-validator", name="Subscriptions Validator", trigger="cron", hour=0, minute=10)  # type: ignore[misc]
+@scheduler(name="Subscriptions Validator", time_unit="day", at="00:10")
 def validate_subscriptions() -> None:
     if app_settings.VALIDATE_OUT_OF_SYNC_SUBSCRIPTIONS:
         # Automatically re-validate out-of-sync subscriptions. This is not recommended for production.