orchestrator-core 4.4.2__py3-none-any.whl → 4.5.0__py3-none-any.whl

orchestrator/__init__.py

@@ -13,15 +13,30 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "4.4.2"
+__version__ = "4.5.0"
 
-from orchestrator.app import OrchestratorCore
+
+from structlog import get_logger
+
+logger = get_logger(__name__)
+
+logger.info("Starting the orchestrator", version=__version__)
+
+from orchestrator.llm_settings import llm_settings
 from orchestrator.settings import app_settings
+
+if llm_settings.SEARCH_ENABLED or llm_settings.AGENT_ENABLED:
+
+    from orchestrator.agentic_app import LLMOrchestratorCore as OrchestratorCore
+else:
+    from orchestrator.app import OrchestratorCore  # type: ignore[assignment]
+
 from orchestrator.workflow import begin, conditional, done, focussteps, inputstep, retrystep, step, steplens, workflow
 
 __all__ = [
     "OrchestratorCore",
     "app_settings",
+    "llm_settings",
     "step",
     "inputstep",
     "workflow",

orchestrator/agentic_app.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""The main application module.
+
+This module contains the main `LLMOrchestratorCore` class for the `FastAPI` backend and
+provides the ability to run the CLI with LLM features (search and/or agent).
+"""
+# Copyright 2019-2025 SURF
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING, Any
+
+import typer
+from structlog import get_logger
+
+from orchestrator.app import OrchestratorCore
+from orchestrator.cli.main import app as cli_app
+from orchestrator.llm_settings import LLMSettings, llm_settings
+
+if TYPE_CHECKING:
+    from pydantic_ai.models.openai import OpenAIModel
+    from pydantic_ai.toolsets import FunctionToolset
+
+logger = get_logger(__name__)
+
+
+class LLMOrchestratorCore(OrchestratorCore):
+    def __init__(
+        self,
+        *args: Any,
+        llm_settings: LLMSettings = llm_settings,
+        agent_model: "OpenAIModel | str | None" = None,
+        agent_tools: "list[FunctionToolset] | None" = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize the `LLMOrchestratorCore` class.
+
+        This class extends `OrchestratorCore` with LLM features (search and agent).
+        It runs the search migration and mounts the agent endpoint based on feature flags.
+
+        Args:
+            *args: All the normal arguments passed to the `OrchestratorCore` class.
+            llm_settings: A class of settings for the LLM
+            agent_model: Override the agent model (defaults to llm_settings.AGENT_MODEL)
+            agent_tools: A list of tools that can be used by the agent
+            **kwargs: Additional arguments passed to the `OrchestratorCore` class.
+
+        Returns:
+            None
+        """
+        self.llm_settings = llm_settings
+        self.agent_model = agent_model or llm_settings.AGENT_MODEL
+        self.agent_tools = agent_tools
+
+        super().__init__(*args, **kwargs)
+
+        # Run search migration if search or agent is enabled
+        if self.llm_settings.SEARCH_ENABLED or self.llm_settings.AGENT_ENABLED:
+            logger.info("Running search migration")
+            try:
+                from orchestrator.db import db
+                from orchestrator.search.llm_migration import run_migration
+
+                with db.engine.begin() as connection:
+                    run_migration(connection)
+            except ImportError as e:
+                logger.error(
+                    "Unable to run search migration. Please install search dependencies: "
+                    "`pip install orchestrator-core[search]`",
+                    error=str(e),
+                )
+                raise
+
+        # Mount agent endpoint if agent is enabled
+        if self.llm_settings.AGENT_ENABLED:
+            logger.info("Initializing agent features", model=self.agent_model)
+            try:
+                from orchestrator.search.agent import build_agent_router
+
+                agent_app = build_agent_router(self.agent_model, self.agent_tools)
+                self.mount("/agent", agent_app)
+            except ImportError as e:
+                logger.error(
+                    "Unable to initialize agent features. Please install agent dependencies: "
+                    "`pip install orchestrator-core[agent]`",
+                    error=str(e),
+                )
+                raise
+
+
+main_typer_app = typer.Typer()
+main_typer_app.add_typer(cli_app, name="orchestrator", help="The orchestrator CLI commands")
+
+if __name__ == "__main__":
+    main_typer_app()

orchestrator/api/api_v1/api.py

@@ -30,6 +30,7 @@ from orchestrator.api.api_v1.endpoints import (
     workflows,
     ws,
 )
+from orchestrator.llm_settings import llm_settings
 from orchestrator.security import authorize
 
 api_router = APIRouter()
@@ -75,11 +76,22 @@ api_router.include_router(user.router, prefix="/user", tags=["Core", "User"], de
 api_router.include_router(
     settings.router, prefix="/settings", tags=["Core", "Settings"], dependencies=[Depends(authorize)]
 )
-api_router.include_router(settings.ws_router, prefix="/settings", tags=["Core", "Settings"])
+api_router.include_router(
+    settings.ws_router, prefix="/settings", tags=["Core", "Settings"]
+)  # Auth on the websocket is handled in the Websocket Manager
 api_router.include_router(health.router, prefix="/health", tags=["Core"])
 api_router.include_router(
     translations.router,
     prefix="/translations",
     tags=["Core", "Translations"],
 )
-api_router.include_router(ws.router, prefix="/ws", tags=["Core", "Events"])
+api_router.include_router(
+    ws.router, prefix="/ws", tags=["Core", "Events"]
+)  # Auth on the websocket is handled in the Websocket Manager
+
+if llm_settings.SEARCH_ENABLED:
+    from orchestrator.api.api_v1.endpoints import search
+
+    api_router.include_router(
+        search.router, prefix="/search", tags=["Core", "Search"], dependencies=[Depends(authorize)]
+    )

orchestrator/api/api_v1/endpoints/search.py

@@ -0,0 +1,296 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Literal, overload
+
+from fastapi import APIRouter, HTTPException, Query, status
+from sqlalchemy import case, select
+from sqlalchemy.orm import selectinload
+
+from orchestrator.db import (
+    ProcessTable,
+    ProductTable,
+    WorkflowTable,
+    db,
+)
+from orchestrator.domain.base import SubscriptionModel
+from orchestrator.domain.context_cache import cache_subscription_models
+from orchestrator.schemas.search import (
+    PageInfoSchema,
+    PathsResponse,
+    ProcessSearchResult,
+    ProcessSearchSchema,
+    ProductSearchResult,
+    ProductSearchSchema,
+    SearchResultsSchema,
+    SubscriptionSearchResult,
+    WorkflowSearchResult,
+    WorkflowSearchSchema,
+)
+from orchestrator.search.core.exceptions import InvalidCursorError
+from orchestrator.search.core.types import EntityType, UIType
+from orchestrator.search.filters.definitions import generate_definitions
+from orchestrator.search.indexing.registry import ENTITY_CONFIG_REGISTRY
+from orchestrator.search.retrieval import execute_search
+from orchestrator.search.retrieval.builder import build_paths_query, create_path_autocomplete_lquery, process_path_rows
+from orchestrator.search.retrieval.pagination import (
+    create_next_page_cursor,
+    process_pagination_cursor,
+)
+from orchestrator.search.retrieval.validation import is_lquery_syntactically_valid
+from orchestrator.search.schemas.parameters import (
+    BaseSearchParameters,
+    ProcessSearchParameters,
+    ProductSearchParameters,
+    SubscriptionSearchParameters,
+    WorkflowSearchParameters,
+)
+from orchestrator.search.schemas.results import SearchResult, TypeDefinition
+from orchestrator.services.subscriptions import format_special_types
+
+router = APIRouter()
+
+
+def _create_search_result_item(
+    entity: WorkflowTable | ProductTable | ProcessTable, entity_type: EntityType, search_info: SearchResult
+) -> WorkflowSearchResult | ProductSearchResult | ProcessSearchResult | None:
+    match entity_type:
+        case EntityType.WORKFLOW:
+            workflow_data = WorkflowSearchSchema.model_validate(entity)
+            return WorkflowSearchResult(
+                workflow=workflow_data,
+                score=search_info.score,
+                perfect_match=search_info.perfect_match,
+                matching_field=search_info.matching_field,
+            )
+        case EntityType.PRODUCT:
+            product_data = ProductSearchSchema.model_validate(entity)
+            return ProductSearchResult(
+                product=product_data,
+                score=search_info.score,
+                perfect_match=search_info.perfect_match,
+                matching_field=search_info.matching_field,
+            )
+        case EntityType.PROCESS:
+            process_data = ProcessSearchSchema.model_validate(entity)
+            return ProcessSearchResult(
+                process=process_data,
+                score=search_info.score,
+                perfect_match=search_info.perfect_match,
+                matching_field=search_info.matching_field,
+            )
+        case _:
+            return None
+
+
+@overload
+async def _perform_search_and_fetch(
+    search_params: BaseSearchParameters,
+    entity_type: Literal[EntityType.WORKFLOW],
+    eager_loads: list[Any],
+    cursor: str | None = None,
+) -> SearchResultsSchema[WorkflowSearchResult]: ...
+
+
+@overload
+async def _perform_search_and_fetch(
+    search_params: BaseSearchParameters,
+    entity_type: Literal[EntityType.PRODUCT],
+    eager_loads: list[Any],
+    cursor: str | None = None,
+) -> SearchResultsSchema[ProductSearchResult]: ...
+
+
+@overload
+async def _perform_search_and_fetch(
+    search_params: BaseSearchParameters,
+    entity_type: Literal[EntityType.PROCESS],
+    eager_loads: list[Any],
+    cursor: str | None = None,
+) -> SearchResultsSchema[ProcessSearchResult]: ...
+
+
+async def _perform_search_and_fetch(
+    search_params: BaseSearchParameters,
+    entity_type: EntityType,
+    eager_loads: list[Any],
+    cursor: str | None = None,
+) -> SearchResultsSchema[Any]:
+    try:
+        pagination_params = await process_pagination_cursor(cursor, search_params)
+    except InvalidCursorError:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor")
+
+    search_response = await execute_search(
+        search_params=search_params,
+        db_session=db.session,
+        pagination_params=pagination_params,
+    )
+    if not search_response.results:
+        return SearchResultsSchema(search_metadata=search_response.metadata)
+
+    next_page_cursor = create_next_page_cursor(search_response.results, pagination_params, search_params.limit)
+    has_next_page = next_page_cursor is not None
+    page_info = PageInfoSchema(has_next_page=has_next_page, next_page_cursor=next_page_cursor)
+
+    config = ENTITY_CONFIG_REGISTRY[entity_type]
+    entity_ids = [res.entity_id for res in search_response.results]
+    pk_column = getattr(config.table, config.pk_name)
+    ordering_case = case({entity_id: i for i, entity_id in enumerate(entity_ids)}, value=pk_column)
+
+    stmt = select(config.table).options(*eager_loads).filter(pk_column.in_(entity_ids)).order_by(ordering_case)
+    entities = db.session.scalars(stmt).all()
+
+    search_info_map = {res.entity_id: res for res in search_response.results}
+    data = []
+    for entity in entities:
+        entity_id = getattr(entity, config.pk_name)
+        search_info = search_info_map.get(str(entity_id))
+        if not search_info:
+            continue
+
+        search_result_item = _create_search_result_item(entity, entity_type, search_info)
+        if search_result_item:
+            data.append(search_result_item)
+
+    return SearchResultsSchema(data=data, page_info=page_info, search_metadata=search_response.metadata)
+
+
+@router.post(
+    "/subscriptions",
+    response_model=SearchResultsSchema[SubscriptionSearchResult],
+)
+async def search_subscriptions(
+    search_params: SubscriptionSearchParameters,
+    cursor: str | None = None,
+) -> SearchResultsSchema[SubscriptionSearchResult]:
+    try:
+        pagination_params = await process_pagination_cursor(cursor, search_params)
+    except InvalidCursorError:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor")
+
+    search_response = await execute_search(
+        search_params=search_params,
+        db_session=db.session,
+        pagination_params=pagination_params,
+    )
+
+    if not search_response.results:
+        return SearchResultsSchema(search_metadata=search_response.metadata)
+
+    next_page_cursor = create_next_page_cursor(search_response.results, pagination_params, search_params.limit)
+    has_next_page = next_page_cursor is not None
+    page_info = PageInfoSchema(has_next_page=has_next_page, next_page_cursor=next_page_cursor)
+
+    search_info_map = {res.entity_id: res for res in search_response.results}
+
+    with cache_subscription_models():
+        subscriptions_data = {
+            sub_id: SubscriptionModel.from_subscription(sub_id).model_dump(exclude_unset=False)
+            for sub_id in search_info_map
+        }
+
+    results_data = [
+        SubscriptionSearchResult(
+            subscription=format_special_types(subscriptions_data[sub_id]),
+            score=search_info.score,
+            perfect_match=search_info.perfect_match,
+            matching_field=search_info.matching_field,
+        )
+        for sub_id, search_info in search_info_map.items()
+    ]
+
+    return SearchResultsSchema(data=results_data, page_info=page_info, search_metadata=search_response.metadata)
+
+
+@router.post("/workflows", response_model=SearchResultsSchema[WorkflowSearchResult])
+async def search_workflows(
+    search_params: WorkflowSearchParameters,
+    cursor: str | None = None,
+) -> SearchResultsSchema[WorkflowSearchResult]:
+    return await _perform_search_and_fetch(
+        search_params=search_params,
+        entity_type=EntityType.WORKFLOW,
+        eager_loads=[selectinload(WorkflowTable.products)],
+        cursor=cursor,
+    )
+
+
+@router.post("/products", response_model=SearchResultsSchema[ProductSearchResult])
+async def search_products(
+    search_params: ProductSearchParameters,
+    cursor: str | None = None,
+) -> SearchResultsSchema[ProductSearchResult]:
+    return await _perform_search_and_fetch(
+        search_params=search_params,
+        entity_type=EntityType.PRODUCT,
+        eager_loads=[
+            selectinload(ProductTable.workflows),
+            selectinload(ProductTable.fixed_inputs),
+            selectinload(ProductTable.product_blocks),
+        ],
+        cursor=cursor,
+    )
+
+
+@router.post("/processes", response_model=SearchResultsSchema[ProcessSearchResult])
+async def search_processes(
+    search_params: ProcessSearchParameters,
+    cursor: str | None = None,
+) -> SearchResultsSchema[ProcessSearchResult]:
+    return await _perform_search_and_fetch(
+        search_params=search_params,
+        entity_type=EntityType.PROCESS,
+        eager_loads=[
+            selectinload(ProcessTable.workflow),
+        ],
+        cursor=cursor,
+    )
+
+
+@router.get(
+    "/paths",
+    response_model=PathsResponse,
+    response_model_exclude_none=True,
+)
+async def list_paths(
+    prefix: str = Query("", min_length=0),
+    q: str | None = Query(None, description="Query for path suggestions"),
+    entity_type: EntityType = Query(EntityType.SUBSCRIPTION),
+    limit: int = Query(10, ge=1, le=10),
+) -> PathsResponse:
+
+    if prefix:
+        lquery_pattern = create_path_autocomplete_lquery(prefix)
+
+        if not is_lquery_syntactically_valid(lquery_pattern, db.session):
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=f"Prefix '{prefix}' creates an invalid search pattern.",
+            )
+    stmt = build_paths_query(entity_type=entity_type, prefix=prefix, q=q)
+    stmt = stmt.limit(limit)
+    rows = db.session.execute(stmt).all()
+
+    leaves, components = process_path_rows(rows)
+    return PathsResponse(leaves=leaves, components=components)
+
+
+@router.get(
+    "/definitions",
+    response_model=dict[UIType, TypeDefinition],
+    response_model_exclude_none=True,
+)
+async def get_definitions() -> dict[UIType, TypeDefinition]:
+    """Provide a static definition of operators and schemas for each UI type."""
+    return generate_definitions()

orchestrator/app.py

@@ -90,6 +90,22 @@ class OrchestratorCore(FastAPI):
         base_settings: AppSettings = app_settings,
         **kwargs: Any,
     ) -> None:
+        """Initialize the Orchestrator.
+
+        Args:
+            title: Name of the application.
+            description: Description of the application.
+            openapi_url: Location of the OpenAPI endpoint.
+            docs_url: Location of the docs endpoint.
+            redoc_url: Location of the redoc endpoint.
+            version: Version of the application.
+            default_response_class: Override the default response class.
+            base_settings: Settings for the application.
+            **kwargs: Any additional keyword arguments are sent to the
+
+        Returns:
+            None
+        """
         initialise_logging(LOGGER_OVERRIDES)
         init_model_loaders()
         if base_settings.ENABLE_GRAPHQL_STATS_EXTENSION:
@@ -163,6 +179,22 @@ class OrchestratorCore(FastAPI):
         release: str | None = GIT_COMMIT_HASH,
         **sentry_kwargs: Any,
     ) -> None:
+        """Register sentry to your application.
+
+        Sentry is an application monitoring toolkit.
+
+        Args:
+            sentry_dsn: The location where sentry traces are posted to.
+            trace_sample_rate: The sample rate
+            server_name: The name of the application
+            environment: Production or development
+            release: Version of the application
+            **sentry_kwargs: Any sentry keyword arguments
+
+        Returns:
+            None
+
+        """
         logger.info("Adding Sentry middleware to app", app=self.title)
         if self.base_settings.EXECUTOR == ExecutorType.WORKER:
             from sentry_sdk.integrations.celery import CeleryIntegration

orchestrator/cli/main.py

@@ -13,13 +13,34 @@
 
 import typer
 
-from orchestrator.cli import database, generate, scheduler
+from orchestrator.cli import (
+    database,
+    generate,
+    scheduler,
+)
+from orchestrator.llm_settings import llm_settings
 
 app = typer.Typer()
 app.add_typer(scheduler.app, name="scheduler", help="Access all the scheduler functions")
 app.add_typer(database.app, name="db", help="Interact with the application database")
 app.add_typer(generate.app, name="generate", help="Generate products, workflows and other artifacts")
 
+if llm_settings.SEARCH_ENABLED:
+    from orchestrator.cli.search import index_llm, resize_embedding, search_explore, speedtest
+
+    app.add_typer(index_llm.app, name="index", help="(Re-)Index the search table.")
+    app.add_typer(search_explore.app, name="search", help="Try out different search types.")
+    app.add_typer(
+        resize_embedding.app,
+        name="embedding",
+        help="Resize the vector dimension of the embedding column in the search table.",
+    )
+    app.add_typer(
+        speedtest.app,
+        name="speedtest",
+        help="Search performance testing and analysis.",
+    )
+
 
 if __name__ == "__main__":
     app()

orchestrator/cli/search/__init__.py

@@ -0,0 +1,32 @@
+# Copyright 2019-2020 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typer
+
+from orchestrator.cli.search import index_llm, resize_embedding, search_explore, speedtest
+
+
+def register_commands(app: typer.Typer) -> None:
+    """Register all LLM/search related commands to the main app."""
+    app.add_typer(index_llm.app, name="index", help="(Re-)Index the search table.")
+    app.add_typer(search_explore.app, name="search", help="Try out different search types.")
+    app.add_typer(
+        resize_embedding.app,
+        name="embedding",
+        help="Resize the vector dimension of the embedding column in the search table.",
+    )
+    app.add_typer(
+        speedtest.app,
+        name="speedtest",
+        help="Search performance testing and analysis.",
+    )

orchestrator/cli/search/index_llm.py

@@ -0,0 +1,73 @@
+import typer
+
+from orchestrator.search.core.types import EntityType
+from orchestrator.search.indexing import run_indexing_for_entity
+
+app = typer.Typer(
+    name="index",
+    help="Index search indexes",
+)
+
+
+@app.command("subscriptions")
+def subscriptions_command(
+    subscription_id: str | None = typer.Option(None, help="UUID (default = all)"),
+    dry_run: bool = typer.Option(False, help="No DB writes"),
+    force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+) -> None:
+    """Index subscription_search_index."""
+    run_indexing_for_entity(
+        entity_kind=EntityType.SUBSCRIPTION,
+        entity_id=subscription_id,
+        dry_run=dry_run,
+        force_index=force_index,
+    )
+
+
+@app.command("products")
+def products_command(
+    product_id: str | None = typer.Option(None, help="UUID (default = all)"),
+    dry_run: bool = typer.Option(False, help="No DB writes"),
+    force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+) -> None:
+    """Index product_search_index."""
+    run_indexing_for_entity(
+        entity_kind=EntityType.PRODUCT,
+        entity_id=product_id,
+        dry_run=dry_run,
+        force_index=force_index,
+    )
+
+
+@app.command("processes")
+def processes_command(
+    process_id: str | None = typer.Option(None, help="UUID (default = all)"),
+    dry_run: bool = typer.Option(False, help="No DB writes"),
+    force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+) -> None:
+    """Index process_search_index."""
+    run_indexing_for_entity(
+        entity_kind=EntityType.PROCESS,
+        entity_id=process_id,
+        dry_run=dry_run,
+        force_index=force_index,
+    )
+
+
+@app.command("workflows")
+def workflows_command(
+    workflow_id: str | None = typer.Option(None, help="UUID (default = all)"),
+    dry_run: bool = typer.Option(False, help="No DB writes"),
+    force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+) -> None:
+    """Index workflow_search_index."""
+    run_indexing_for_entity(
+        entity_kind=EntityType.WORKFLOW,
+        entity_id=workflow_id,
+        dry_run=dry_run,
+        force_index=force_index,
+    )
+
+
+if __name__ == "__main__":
+    app()