orchestrator-core 4.4.1__py3-none-any.whl → 4.5.0a2__py3-none-any.whl

orchestrator/__init__.py

@@ -13,15 +13,39 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "4.4.1"
+__version__ = "4.5.0a2"
 
-from orchestrator.app import OrchestratorCore
+
+from structlog import get_logger
+
+logger = get_logger(__name__)
+
+logger.info("Starting the orchestrator", version=__version__)
+
+from orchestrator.llm_settings import llm_settings
 from orchestrator.settings import app_settings
+
+if llm_settings.LLM_ENABLED:
+    try:
+        from importlib import import_module
+
+        import_module("pydantic_ai")
+        from orchestrator.agentic_app import AgenticOrchestratorCore as OrchestratorCore
+
+    except ImportError:
+        logger.error(
+            "Unable to import 'pydantic_ai' module, please install the orchestrator with llm dependencies. `pip install orchestrator-core[llm]",
+        )
+        exit(1)
+else:
+    from orchestrator.app import OrchestratorCore  # type: ignore[assignment]
+
 from orchestrator.workflow import begin, conditional, done, focussteps, inputstep, retrystep, step, steplens, workflow
 
 __all__ = [
     "OrchestratorCore",
     "app_settings",
+    "llm_settings",
     "step",
     "inputstep",
     "workflow",

orchestrator/agentic_app.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""The main application module.
+
+This module contains the main `AgenticOrchestratorCore` class for the `FastAPI` backend and
+provides the ability to run the CLI.
+"""
+# Copyright 2019-2025 SURF
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any
+
+import typer
+from pydantic_ai.models.openai import OpenAIModel
+from pydantic_ai.toolsets import FunctionToolset
+from structlog import get_logger
+
+from orchestrator.app import OrchestratorCore
+from orchestrator.cli.main import app as cli_app
+from orchestrator.llm_settings import LLMSettings, llm_settings
+
+logger = get_logger(__name__)
+
+
+class AgenticOrchestratorCore(OrchestratorCore):
+    def __init__(
+        self,
+        *args: Any,
+        llm_model: OpenAIModel | str = "gpt-4o-mini",
+        llm_settings: LLMSettings = llm_settings,
+        agent_tools: list[FunctionToolset] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize the `AgenticOrchestratorCore` class.
+
+        This class takes the same arguments as the `OrchestratorCore` class.
+
+        Args:
+            *args: All the normal arguments passed to the `OrchestratorCore` class.
+            llm_model: An OpenAI model class or string, not limited to OpenAI models (gpt-4o-mini etc)
+            llm_settings: A class of settings for the LLM
+            agent_tools: A list of tools that can be used by the agent
+            **kwargs: Additional arguments passed to the `OrchestratorCore` class.
+
+        Returns:
+            None
+        """
+        self.llm_model = llm_model
+        self.agent_tools = agent_tools
+        self.llm_settings = llm_settings
+
+        super().__init__(*args, **kwargs)
+
+        logger.info("Mounting the agent")
+        self.register_llm_integration()
+
+    def register_llm_integration(self) -> None:
+        """Mount the Agent endpoint.
+
+        This helper mounts the agent endpoint on the application.
+
+        Returns:
+            None
+
+        """
+        from orchestrator.search.agent import build_agent_app
+
+        agent_app = build_agent_app(self.llm_model, self.agent_tools)
+        self.mount("/agent", agent_app)
+
+
+main_typer_app = typer.Typer()
+main_typer_app.add_typer(cli_app, name="orchestrator", help="The orchestrator CLI commands")
+
+if __name__ == "__main__":
+    main_typer_app()

orchestrator/api/api_v1/api.py

@@ -30,6 +30,7 @@ from orchestrator.api.api_v1.endpoints import (
     workflows,
     ws,
 )
+from orchestrator.llm_settings import llm_settings
 from orchestrator.security import authorize
 
 api_router = APIRouter()
@@ -83,3 +84,12 @@ api_router.include_router(
     tags=["Core", "Translations"],
 )
 api_router.include_router(ws.router, prefix="/ws", tags=["Core", "Events"])
+
+if llm_settings.LLM_ENABLED:
+    from orchestrator.api.api_v1.endpoints import search
+
+    api_router.include_router(
+        search.router,
+        prefix="/search",
+        tags=["Core", "Search"],
+    )

orchestrator/api/api_v1/endpoints/search.py

@@ -0,0 +1,277 @@
+from typing import Any, Literal, overload
+
+from fastapi import APIRouter, HTTPException, Query, status
+from sqlalchemy import case, select
+from sqlalchemy.orm import selectinload
+
+from orchestrator.db import (
+    ProcessTable,
+    ProductTable,
+    WorkflowTable,
+    db,
+)
+from orchestrator.domain.base import SubscriptionModel
+from orchestrator.schemas.search import (
+    PageInfoSchema,
+    PathsResponse,
+    ProcessSearchResult,
+    ProcessSearchSchema,
+    ProductSearchResult,
+    ProductSearchSchema,
+    SearchResultsSchema,
+    SubscriptionSearchResult,
+    WorkflowSearchResult,
+    WorkflowSearchSchema,
+)
+from orchestrator.search.core.exceptions import InvalidCursorError
+from orchestrator.search.core.types import EntityType, UIType
+from orchestrator.search.filters.definitions import generate_definitions
+from orchestrator.search.indexing.registry import ENTITY_CONFIG_REGISTRY
+from orchestrator.search.retrieval import execute_search
+from orchestrator.search.retrieval.builder import build_paths_query, create_path_autocomplete_lquery, process_path_rows
+from orchestrator.search.retrieval.pagination import (
+    create_next_page_cursor,
+    process_pagination_cursor,
+)
+from orchestrator.search.retrieval.validation import is_lquery_syntactically_valid
+from orchestrator.search.schemas.parameters import (
+    BaseSearchParameters,
+    ProcessSearchParameters,
+    ProductSearchParameters,
+    SubscriptionSearchParameters,
+    WorkflowSearchParameters,
+)
+from orchestrator.search.schemas.results import SearchResult, TypeDefinition
+from orchestrator.services.subscriptions import format_special_types
+
+router = APIRouter()
+
+
+def _create_search_result_item(
+    entity: WorkflowTable | ProductTable | ProcessTable, entity_type: EntityType, search_info: SearchResult
+) -> WorkflowSearchResult | ProductSearchResult | ProcessSearchResult | None:
+    match entity_type:
+        case EntityType.WORKFLOW:
+            workflow_data = WorkflowSearchSchema.model_validate(entity)
+            return WorkflowSearchResult(
+                workflow=workflow_data,
+                score=search_info.score,
+                perfect_match=search_info.perfect_match,
+                matching_field=search_info.matching_field,
+            )
+        case EntityType.PRODUCT:
+            product_data = ProductSearchSchema.model_validate(entity)
+            return ProductSearchResult(
+                product=product_data,
+                score=search_info.score,
+                perfect_match=search_info.perfect_match,
+                matching_field=search_info.matching_field,
+            )
+        case EntityType.PROCESS:
+            process_data = ProcessSearchSchema.model_validate(entity)
+            return ProcessSearchResult(
+                process=process_data,
+                score=search_info.score,
+                perfect_match=search_info.perfect_match,
+                matching_field=search_info.matching_field,
+            )
+        case _:
+            return None
+
+
+@overload
+async def _perform_search_and_fetch(
+    search_params: BaseSearchParameters,
+    entity_type: Literal[EntityType.WORKFLOW],
+    eager_loads: list[Any],
+    cursor: str | None = None,
+) -> SearchResultsSchema[WorkflowSearchResult]: ...
+
+
+@overload
+async def _perform_search_and_fetch(
+    search_params: BaseSearchParameters,
+    entity_type: Literal[EntityType.PRODUCT],
+    eager_loads: list[Any],
+    cursor: str | None = None,
+) -> SearchResultsSchema[ProductSearchResult]: ...
+
+
+@overload
+async def _perform_search_and_fetch(
+    search_params: BaseSearchParameters,
+    entity_type: Literal[EntityType.PROCESS],
+    eager_loads: list[Any],
+    cursor: str | None = None,
+) -> SearchResultsSchema[ProcessSearchResult]: ...
+
+
+async def _perform_search_and_fetch(
+    search_params: BaseSearchParameters,
+    entity_type: EntityType,
+    eager_loads: list[Any],
+    cursor: str | None = None,
+) -> SearchResultsSchema[Any]:
+    try:
+        pagination_params = await process_pagination_cursor(cursor, search_params)
+    except InvalidCursorError:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor")
+
+    search_response = await execute_search(
+        search_params=search_params,
+        db_session=db.session,
+        pagination_params=pagination_params,
+    )
+    if not search_response.results:
+        return SearchResultsSchema(search_metadata=search_response.metadata)
+
+    next_page_cursor = create_next_page_cursor(search_response.results, pagination_params, search_params.limit)
+    has_next_page = next_page_cursor is not None
+    page_info = PageInfoSchema(has_next_page=has_next_page, next_page_cursor=next_page_cursor)
+
+    config = ENTITY_CONFIG_REGISTRY[entity_type]
+    entity_ids = [res.entity_id for res in search_response.results]
+    pk_column = getattr(config.table, config.pk_name)
+    ordering_case = case({entity_id: i for i, entity_id in enumerate(entity_ids)}, value=pk_column)
+
+    stmt = select(config.table).options(*eager_loads).filter(pk_column.in_(entity_ids)).order_by(ordering_case)
+    entities = db.session.scalars(stmt).all()
+
+    search_info_map = {res.entity_id: res for res in search_response.results}
+    data = []
+    for entity in entities:
+        entity_id = getattr(entity, config.pk_name)
+        search_info = search_info_map.get(str(entity_id))
+        if not search_info:
+            continue
+
+        search_result_item = _create_search_result_item(entity, entity_type, search_info)
+        if search_result_item:
+            data.append(search_result_item)
+
+    return SearchResultsSchema(data=data, page_info=page_info, search_metadata=search_response.metadata)
+
+
+@router.post(
+    "/subscriptions",
+    response_model=SearchResultsSchema[SubscriptionSearchResult],
+)
+async def search_subscriptions(
+    search_params: SubscriptionSearchParameters,
+    cursor: str | None = None,
+) -> SearchResultsSchema[SubscriptionSearchResult]:
+    try:
+        pagination_params = await process_pagination_cursor(cursor, search_params)
+    except InvalidCursorError:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor")
+
+    search_response = await execute_search(
+        search_params=search_params,
+        db_session=db.session,
+        pagination_params=pagination_params,
+    )
+
+    if not search_response.results:
+        return SearchResultsSchema(search_metadata=search_response.metadata)
+
+    next_page_cursor = create_next_page_cursor(search_response.results, pagination_params, search_params.limit)
+    has_next_page = next_page_cursor is not None
+    page_info = PageInfoSchema(has_next_page=has_next_page, next_page_cursor=next_page_cursor)
+
+    search_info_map = {res.entity_id: res for res in search_response.results}
+    results_data = []
+    for sub_id, search_info in search_info_map.items():
+        subscription_model = SubscriptionModel.from_subscription(sub_id)
+        sub_data = subscription_model.model_dump(exclude_unset=False)
+        search_result_item = SubscriptionSearchResult(
+            subscription=format_special_types(sub_data),
+            score=search_info.score,
+            perfect_match=search_info.perfect_match,
+            matching_field=search_info.matching_field,
+        )
+        results_data.append(search_result_item)
+
+    return SearchResultsSchema(data=results_data, page_info=page_info, search_metadata=search_response.metadata)
+
+
+@router.post("/workflows", response_model=SearchResultsSchema[WorkflowSearchResult])
+async def search_workflows(
+    search_params: WorkflowSearchParameters,
+    cursor: str | None = None,
+) -> SearchResultsSchema[WorkflowSearchResult]:
+    return await _perform_search_and_fetch(
+        search_params=search_params,
+        entity_type=EntityType.WORKFLOW,
+        eager_loads=[selectinload(WorkflowTable.products)],
+        cursor=cursor,
+    )
+
+
+@router.post("/products", response_model=SearchResultsSchema[ProductSearchResult])
+async def search_products(
+    search_params: ProductSearchParameters,
+    cursor: str | None = None,
+) -> SearchResultsSchema[ProductSearchResult]:
+    return await _perform_search_and_fetch(
+        search_params=search_params,
+        entity_type=EntityType.PRODUCT,
+        eager_loads=[
+            selectinload(ProductTable.workflows),
+            selectinload(ProductTable.fixed_inputs),
+            selectinload(ProductTable.product_blocks),
+        ],
+        cursor=cursor,
+    )
+
+
+@router.post("/processes", response_model=SearchResultsSchema[ProcessSearchResult])
+async def search_processes(
+    search_params: ProcessSearchParameters,
+    cursor: str | None = None,
+) -> SearchResultsSchema[ProcessSearchResult]:
+    return await _perform_search_and_fetch(
+        search_params=search_params,
+        entity_type=EntityType.PROCESS,
+        eager_loads=[
+            selectinload(ProcessTable.workflow),
+        ],
+        cursor=cursor,
+    )
+
+
+@router.get(
+    "/paths",
+    response_model=PathsResponse,
+    response_model_exclude_none=True,
+)
+async def list_paths(
+    prefix: str = Query("", min_length=0),
+    q: str | None = Query(None, description="Query for path suggestions"),
+    entity_type: EntityType = Query(EntityType.SUBSCRIPTION),
+    limit: int = Query(10, ge=1, le=10),
+) -> PathsResponse:
+
+    if prefix:
+        lquery_pattern = create_path_autocomplete_lquery(prefix)
+
+        if not is_lquery_syntactically_valid(lquery_pattern, db.session):
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=f"Prefix '{prefix}' creates an invalid search pattern.",
+            )
+    stmt = build_paths_query(entity_type=entity_type, prefix=prefix, q=q)
+    stmt = stmt.limit(limit)
+    rows = db.session.execute(stmt).all()
+
+    leaves, components = process_path_rows(rows)
+    return PathsResponse(leaves=leaves, components=components)
+
+
+@router.get(
+    "/definitions",
+    response_model=dict[UIType, TypeDefinition],
+    response_model_exclude_none=True,
+)
+async def get_definitions() -> dict[UIType, TypeDefinition]:
+    """Provide a static definition of operators and schemas for each UI type."""
+    return generate_definitions()

orchestrator/app.py

@@ -90,6 +90,22 @@ class OrchestratorCore(FastAPI):
         base_settings: AppSettings = app_settings,
         **kwargs: Any,
     ) -> None:
+        """Initialize the Orchestrator.
+
+        Args:
+            title: Name of the application.
+            description: Description of the application.
+            openapi_url: Location of the OpenAPI endpoint.
+            docs_url: Location of the docs endpoint.
+            redoc_url: Location of the redoc endpoint.
+            version: Version of the application.
+            default_response_class: Override the default response class.
+            base_settings: Settings for the application.
+            **kwargs: Any additional keyword arguments are sent to the
+
+        Returns:
+            None
+        """
         initialise_logging(LOGGER_OVERRIDES)
         init_model_loaders()
         if base_settings.ENABLE_GRAPHQL_STATS_EXTENSION:
@@ -163,6 +179,22 @@ class OrchestratorCore(FastAPI):
         release: str | None = GIT_COMMIT_HASH,
         **sentry_kwargs: Any,
     ) -> None:
+        """Register sentry to your application.
+
+        Sentry is an application monitoring toolkit.
+
+        Args:
+            sentry_dsn: The location where sentry traces are posted to.
+            trace_sample_rate: The sample rate
+            server_name: The name of the application
+            environment: Production or development
+            release: Version of the application
+            **sentry_kwargs: Any sentry keyword arguments
+
+        Returns:
+            None
+
+        """
         logger.info("Adding Sentry middleware to app", app=self.title)
         if self.base_settings.EXECUTOR == ExecutorType.WORKER:
             from sentry_sdk.integrations.celery import CeleryIntegration

orchestrator/cli/index_llm.py

@@ -0,0 +1,73 @@
+import typer
+
+from orchestrator.search.core.types import EntityType
+from orchestrator.search.indexing import run_indexing_for_entity
+
+app = typer.Typer(
+    name="index",
+    help="Index search indexes",
+)
+
+
+@app.command("subscriptions")
+def subscriptions_command(
+    subscription_id: str | None = typer.Option(None, help="UUID (default = all)"),
+    dry_run: bool = typer.Option(False, help="No DB writes"),
+    force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+) -> None:
+    """Index subscription_search_index."""
+    run_indexing_for_entity(
+        entity_kind=EntityType.SUBSCRIPTION,
+        entity_id=subscription_id,
+        dry_run=dry_run,
+        force_index=force_index,
+    )
+
+
+@app.command("products")
+def products_command(
+    product_id: str | None = typer.Option(None, help="UUID (default = all)"),
+    dry_run: bool = typer.Option(False, help="No DB writes"),
+    force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+) -> None:
+    """Index product_search_index."""
+    run_indexing_for_entity(
+        entity_kind=EntityType.PRODUCT,
+        entity_id=product_id,
+        dry_run=dry_run,
+        force_index=force_index,
+    )
+
+
+@app.command("processes")
+def processes_command(
+    process_id: str | None = typer.Option(None, help="UUID (default = all)"),
+    dry_run: bool = typer.Option(False, help="No DB writes"),
+    force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+) -> None:
+    """Index process_search_index."""
+    run_indexing_for_entity(
+        entity_kind=EntityType.PROCESS,
+        entity_id=process_id,
+        dry_run=dry_run,
+        force_index=force_index,
+    )
+
+
+@app.command("workflows")
+def workflows_command(
+    workflow_id: str | None = typer.Option(None, help="UUID (default = all)"),
+    dry_run: bool = typer.Option(False, help="No DB writes"),
+    force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+) -> None:
+    """Index workflow_search_index."""
+    run_indexing_for_entity(
+        entity_kind=EntityType.WORKFLOW,
+        entity_id=workflow_id,
+        dry_run=dry_run,
+        force_index=force_index,
+    )
+
+
+if __name__ == "__main__":
+    app()

orchestrator/cli/main.py

@@ -13,13 +13,34 @@
 
 import typer
 
-from orchestrator.cli import database, generate, scheduler
+from orchestrator.cli import (
+    database,
+    generate,
+    scheduler,
+)
+from orchestrator.llm_settings import llm_settings
 
 app = typer.Typer()
 app.add_typer(scheduler.app, name="scheduler", help="Access all the scheduler functions")
 app.add_typer(database.app, name="db", help="Interact with the application database")
 app.add_typer(generate.app, name="generate", help="Generate products, workflows and other artifacts")
 
+if llm_settings.LLM_ENABLED:
+    from orchestrator.cli import index_llm, resize_embedding, search_explore, speedtest
+
+    app.add_typer(index_llm.app, name="index", help="(Re-)Index the search table.")
+    app.add_typer(search_explore.app, name="search", help="Try out different search types.")
+    app.add_typer(
+        resize_embedding.app,
+        name="embedding",
+        help="Resize the vector dimension of the embedding column in the search table.",
+    )
+    app.add_typer(
+        speedtest.app,
+        name="speedtest",
+        help="Search performance testing and analysis.",
+    )
+
 
 if __name__ == "__main__":
     app()