orchestrator-core 4.6.3rc1__tar.gz → 4.6.5__tar.gz

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 4.6.3rc1
+Version: 4.6.5
 Summary: This is the orchestrator workflow engine.
 Author-email: SURF <automation-beheer@surf.nl>
-Requires-Python: >=3.11,<3.14
+Requires-Python: >=3.11,<3.15
 Description-Content-Type: text/markdown
 License-Expression: Apache-2.0
 Classifier: Development Status :: 5 - Production/Stable
@@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python
 Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
 Classifier: Topic :: Internet :: WWW/HTTP
@@ -30,39 +31,39 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 License-File: LICENSE
-Requires-Dist: alembic==1.16.1
+Requires-Dist: alembic==1.17.2
 Requires-Dist: anyio>=3.7.0
 Requires-Dist: apscheduler>=3.11.0
 Requires-Dist: click==8.*
 Requires-Dist: deepmerge==2.0
 Requires-Dist: deprecated>=1.2.18
-Requires-Dist: fastapi~=0.120.4
+Requires-Dist: fastapi~=0.121.1
 Requires-Dist: fastapi-etag==0.4.0
 Requires-Dist: itsdangerous>=2.2.0
 Requires-Dist: jinja2==3.1.6
-Requires-Dist: more-itertools~=10.7.0
-Requires-Dist: nwa-stdlib~=1.10.3
+Requires-Dist: more-itertools~=10.8.0
+Requires-Dist: nwa-stdlib~=1.11.0
 Requires-Dist: oauth2-lib>=2.5.0
-Requires-Dist: orjson==3.10.18
+Requires-Dist: orjson==3.11.4
 Requires-Dist: pgvector>=0.4.1
-Requires-Dist: prometheus-client==0.22.1
+Requires-Dist: prometheus-client==0.23.1
 Requires-Dist: psycopg2-binary==2.9.11
 Requires-Dist: pydantic-forms>=1.4.0
-Requires-Dist: pydantic-settings~=2.9.1
-Requires-Dist: pydantic[email]~=2.11.7
-Requires-Dist: python-dateutil==2.8.2
-Requires-Dist: python-rapidjson>=1.18,<1.21
+Requires-Dist: pydantic-settings~=2.12.0
+Requires-Dist: pydantic[email]~=2.12.4
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: python-rapidjson>=1.22,<1.23
 Requires-Dist: pytz==2025.2
-Requires-Dist: redis==5.1.1
+Requires-Dist: redis==5.3.1
 Requires-Dist: semver==3.0.4
 Requires-Dist: sentry-sdk[fastapi]>=2.29.1
 Requires-Dist: sqlalchemy==2.0.44
-Requires-Dist: sqlalchemy-utils==0.41.2
+Requires-Dist: sqlalchemy-utils==0.42.0
 Requires-Dist: strawberry-graphql>=0.281.0,<0.285.0
 Requires-Dist: structlog>=25.4.0
 Requires-Dist: tabulate==0.9.0
-Requires-Dist: typer==0.15.4
-Requires-Dist: uvicorn[standard]~=0.34.0
+Requires-Dist: typer==0.20.0
+Requires-Dist: uvicorn[standard]~=0.38.0
 Requires-Dist: pydantic-ai-slim >=1.9.0 ; extra == "agent"
 Requires-Dist: ag-ui-protocol>=0.1.8 ; extra == "agent"
 Requires-Dist: litellm>=1.75.7 ; extra == "agent"

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "4.6.3rc1"
+__version__ = "4.6.5"
 
 
 from structlog import get_logger

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/cli/search/index_llm.py

@@ -14,6 +14,7 @@ def subscriptions_command(
     subscription_id: str | None = typer.Option(None, help="UUID (default = all)"),
     dry_run: bool = typer.Option(False, help="No DB writes"),
     force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+    show_progress: bool = typer.Option(False, help="Show per-entity progress"),
 ) -> None:
     """Index subscription_search_index."""
     run_indexing_for_entity(
@@ -21,6 +22,7 @@ def subscriptions_command(
         entity_id=subscription_id,
         dry_run=dry_run,
         force_index=force_index,
+        show_progress=show_progress,
     )
 
 
@@ -29,6 +31,7 @@ def products_command(
     product_id: str | None = typer.Option(None, help="UUID (default = all)"),
     dry_run: bool = typer.Option(False, help="No DB writes"),
     force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+    show_progress: bool = typer.Option(False, help="Show per-entity progress"),
 ) -> None:
     """Index product_search_index."""
     run_indexing_for_entity(
@@ -36,6 +39,7 @@ def products_command(
         entity_id=product_id,
         dry_run=dry_run,
         force_index=force_index,
+        show_progress=show_progress,
     )
 
 
@@ -44,6 +48,7 @@ def processes_command(
     process_id: str | None = typer.Option(None, help="UUID (default = all)"),
     dry_run: bool = typer.Option(False, help="No DB writes"),
     force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+    show_progress: bool = typer.Option(False, help="Show per-entity progress"),
 ) -> None:
     """Index process_search_index."""
     run_indexing_for_entity(
@@ -51,6 +56,7 @@ def processes_command(
         entity_id=process_id,
         dry_run=dry_run,
         force_index=force_index,
+        show_progress=show_progress,
     )
 
 
@@ -59,6 +65,7 @@ def workflows_command(
     workflow_id: str | None = typer.Option(None, help="UUID (default = all)"),
     dry_run: bool = typer.Option(False, help="No DB writes"),
     force_index: bool = typer.Option(False, help="Force re-index (ignore hash cache)"),
+    show_progress: bool = typer.Option(False, help="Show per-entity progress"),
 ) -> None:
     """Index workflow_search_index."""
     run_indexing_for_entity(
@@ -66,6 +73,7 @@ def workflows_command(
         entity_id=workflow_id,
         dry_run=dry_run,
         force_index=force_index,
+        show_progress=show_progress,
     )
 
 

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/graphql/resolvers/helpers.py

@@ -1,11 +1,17 @@
 from collections.abc import Sequence
+from functools import wraps
+from typing import Callable, Coroutine
 
+import structlog
 from sqlalchemy import CompoundSelect, Select, select
 from sqlalchemy.orm.strategy_options import _AbstractLoad
+from starlette.concurrency import run_in_threadpool
 
 from orchestrator.db import db
 from orchestrator.db.database import BaseModel
 
+logger = structlog.get_logger(__name__)
+
 
 def rows_from_statement(
     stmt: Select | CompoundSelect,
@@ -19,3 +25,12 @@ def rows_from_statement(
     result = db.session.scalars(from_stmt)
     uresult = result.unique() if unique else result
     return uresult.all()
+
+
+def make_async(f: Callable):  # type: ignore
+    @wraps(f)
+    async def wrapper(*args, **kwargs) -> Coroutine:  # type: ignore
+        logger.debug(f"**async, calling fn {f.__name__}")
+        return await run_in_threadpool(f, *args, **kwargs)
+
+    return wrapper

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/graphql/resolvers/process.py

@@ -25,7 +25,7 @@ from orchestrator.db.range import apply_range_to_statement
 from orchestrator.db.sorting import Sort
 from orchestrator.db.sorting.process import process_sort_fields, sort_processes
 from orchestrator.graphql.pagination import Connection
-from orchestrator.graphql.resolvers.helpers import rows_from_statement
+from orchestrator.graphql.resolvers.helpers import make_async, rows_from_statement
 from orchestrator.graphql.schemas.process import ProcessType
 from orchestrator.graphql.types import GraphqlFilter, GraphqlSort, OrchestratorInfo
 from orchestrator.graphql.utils import (
@@ -55,7 +55,8 @@ def _enrich_process(process: ProcessTable, with_details: bool = False) -> Proces
     return ProcessSchema(**process_data)
 
 
-async def resolve_process(info: OrchestratorInfo, process_id: UUID) -> ProcessType | None:
+@make_async
+def resolve_process(info: OrchestratorInfo, process_id: UUID) -> ProcessType | None:
     query_loaders = get_query_loaders_for_gql_fields(ProcessTable, info)
     stmt = select(ProcessTable).options(*query_loaders).where(ProcessTable.process_id == process_id)
     if process := db.session.scalar(stmt):
@@ -64,7 +65,8 @@ async def resolve_process(info: OrchestratorInfo, process_id: UUID) -> ProcessTy
     return None
 
 
-async def resolve_processes(
+@make_async
+def resolve_processes(
     info: OrchestratorInfo,
     filter_by: list[GraphqlFilter] | None = None,
     sort_by: list[GraphqlSort] | None = None,

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/graphql/resolvers/product.py

@@ -9,7 +9,7 @@ from orchestrator.db.range.range import apply_range_to_statement
 from orchestrator.db.sorting import Sort
 from orchestrator.db.sorting.product import product_sort_fields, sort_products
 from orchestrator.graphql.pagination import Connection
-from orchestrator.graphql.resolvers.helpers import rows_from_statement
+from orchestrator.graphql.resolvers.helpers import make_async, rows_from_statement
 from orchestrator.graphql.schemas.product import ProductType
 from orchestrator.graphql.types import GraphqlFilter, GraphqlSort, OrchestratorInfo
 from orchestrator.graphql.utils import create_resolver_error_handler, is_querying_page_data, to_graphql_result_page
@@ -19,7 +19,8 @@ from orchestrator.utils.search_query import create_sqlalchemy_select
 logger = structlog.get_logger(__name__)
 
 
-async def resolve_products(
+@make_async
+def resolve_products(
     info: OrchestratorInfo,
     filter_by: list[GraphqlFilter] | None = None,
     sort_by: list[GraphqlSort] | None = None,

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/graphql/resolvers/product_block.py

@@ -13,7 +13,7 @@ from orchestrator.db.range.range import apply_range_to_statement
 from orchestrator.db.sorting import Sort
 from orchestrator.db.sorting.product_block import product_block_sort_fields, sort_product_blocks
 from orchestrator.graphql.pagination import Connection
-from orchestrator.graphql.resolvers.helpers import rows_from_statement
+from orchestrator.graphql.resolvers.helpers import make_async, rows_from_statement
 from orchestrator.graphql.schemas.product_block import ProductBlock
 from orchestrator.graphql.types import GraphqlFilter, GraphqlSort, OrchestratorInfo
 from orchestrator.graphql.utils import create_resolver_error_handler, is_querying_page_data, to_graphql_result_page
@@ -23,7 +23,8 @@ from orchestrator.utils.search_query import create_sqlalchemy_select
 logger = structlog.get_logger(__name__)
 
 
-async def resolve_product_blocks(
+@make_async
+def resolve_product_blocks(
     info: OrchestratorInfo,
     filter_by: list[GraphqlFilter] | None = None,
     sort_by: list[GraphqlSort] | None = None,

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/graphql/resolvers/resource_type.py

@@ -13,7 +13,7 @@ from orchestrator.db.range import apply_range_to_statement
 from orchestrator.db.sorting import Sort
 from orchestrator.db.sorting.resource_type import resource_type_sort_fields, sort_resource_types
 from orchestrator.graphql.pagination import Connection
-from orchestrator.graphql.resolvers.helpers import rows_from_statement
+from orchestrator.graphql.resolvers.helpers import make_async, rows_from_statement
 from orchestrator.graphql.schemas.resource_type import ResourceType
 from orchestrator.graphql.types import GraphqlFilter, GraphqlSort, OrchestratorInfo
 from orchestrator.graphql.utils import create_resolver_error_handler, is_querying_page_data, to_graphql_result_page
@@ -23,7 +23,8 @@ from orchestrator.utils.search_query import create_sqlalchemy_select
 logger = structlog.get_logger(__name__)
 
 
-async def resolve_resource_types(
+@make_async
+def resolve_resource_types(
     info: OrchestratorInfo,
     filter_by: list[GraphqlFilter] | None = None,
     sort_by: list[GraphqlSort] | None = None,

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/graphql/resolvers/scheduled_tasks.py

@@ -3,6 +3,7 @@ import structlog
 from orchestrator.db.filters import Filter
 from orchestrator.db.sorting import Sort
 from orchestrator.graphql.pagination import Connection
+from orchestrator.graphql.resolvers.helpers import make_async
 from orchestrator.graphql.schemas.scheduled_task import ScheduledTaskGraphql
 from orchestrator.graphql.types import GraphqlFilter, GraphqlSort, OrchestratorInfo
 from orchestrator.graphql.utils import create_resolver_error_handler, to_graphql_result_page
@@ -12,7 +13,8 @@ from orchestrator.schedules.scheduler import get_scheduler_tasks, scheduled_task
 logger = structlog.get_logger(__name__)
 
 
-async def resolve_scheduled_tasks(
+@make_async
+def resolve_scheduled_tasks(
     info: OrchestratorInfo,
     filter_by: list[GraphqlFilter] | None = None,
     sort_by: list[GraphqlSort] | None = None,

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/graphql/resolvers/settings.py

@@ -4,6 +4,7 @@ from redis.asyncio import Redis as AIORedis
 
 from oauth2_lib.strawberry import authenticated_mutation_field
 from orchestrator.api.api_v1.endpoints.settings import generate_engine_status_response
+from orchestrator.graphql.resolvers.helpers import make_async
 from orchestrator.graphql.schemas.errors import Error
 from orchestrator.graphql.schemas.settings import (
     CACHE_FLUSH_OPTIONS,
@@ -27,6 +28,7 @@ logger = structlog.get_logger(__name__)
 
 
 # Queries
+@make_async
 def resolve_settings(info: OrchestratorInfo) -> StatusType:
     selected_fields = get_selected_fields(info)
 

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/graphql/resolvers/subscription.py

@@ -18,6 +18,7 @@ from graphql import GraphQLError
 from pydantic.alias_generators import to_camel as to_lower_camel
 from sqlalchemy import Select, func, select
 from sqlalchemy.orm import contains_eager
+from starlette.concurrency import run_in_threadpool
 from strawberry.experimental.pydantic.conversion_types import StrawberryTypeFromPydantic
 
 from nwastdlib.asyncio import gather_nice
@@ -101,7 +102,7 @@ async def format_subscription(info: OrchestratorInfo, subscription: Subscription
 async def resolve_subscription(info: OrchestratorInfo, id: UUID) -> SubscriptionInterface | None:
     stmt = select(SubscriptionTable).where(SubscriptionTable.subscription_id == id)
 
-    if subscription := db.session.scalar(stmt):
+    if subscription := await run_in_threadpool(db.session.scalar, stmt):
         return await format_subscription(info, subscription)
     return None
 
@@ -141,12 +142,13 @@ async def resolve_subscriptions(
         stmt = filter_by_query_string(stmt, query)
 
     stmt = cast(Select, sort_subscriptions(stmt, pydantic_sort_by, _error_handler))
-    total = db.session.scalar(select(func.count()).select_from(stmt.subquery()))
+    total = await run_in_threadpool(db.session.scalar, select(func.count()).select_from(stmt.subquery()))
     stmt = apply_range_to_statement(stmt, after, after + first + 1)
 
     graphql_subscriptions: list[SubscriptionInterface] = []
     if is_querying_page_data(info):
-        subscriptions = db.session.scalars(stmt).all()
+        scalars = await run_in_threadpool(db.session.scalars, stmt)
+        subscriptions = scalars.all()
         graphql_subscriptions = list(await gather_nice((format_subscription(info, p) for p in subscriptions)))  # type: ignore
     logger.info("Resolve subscriptions", filter_by=filter_by, total=total)
 

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/graphql/resolvers/version.py

@@ -1,6 +1,7 @@
 from structlog import get_logger
 
 from orchestrator import __version__
+from orchestrator.graphql.resolvers.helpers import make_async
 from orchestrator.graphql.schemas.version import VersionType
 from orchestrator.graphql.types import OrchestratorInfo
 from orchestrator.graphql.utils import create_resolver_error_handler
@@ -11,6 +12,7 @@ logger = get_logger(__name__)
 VERSIONS = [f"orchestrator-core: {__version__}"]
 
 
+@make_async
 def resolve_version(info: OrchestratorInfo) -> VersionType | None:
     logger.debug("resolve_version() called")
     _error_handler = create_resolver_error_handler(info)

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/graphql/resolvers/workflow.py

@@ -9,7 +9,7 @@ from orchestrator.db.range.range import apply_range_to_statement
 from orchestrator.db.sorting import Sort
 from orchestrator.db.sorting.workflow import sort_workflows, workflow_sort_fields
 from orchestrator.graphql.pagination import Connection
-from orchestrator.graphql.resolvers.helpers import rows_from_statement
+from orchestrator.graphql.resolvers.helpers import make_async, rows_from_statement
 from orchestrator.graphql.schemas.workflow import Workflow
 from orchestrator.graphql.types import GraphqlFilter, GraphqlSort, OrchestratorInfo
 from orchestrator.graphql.utils import create_resolver_error_handler, is_querying_page_data, to_graphql_result_page
@@ -19,7 +19,8 @@ from orchestrator.utils.search_query import create_sqlalchemy_select
 logger = structlog.get_logger(__name__)
 
 
-async def resolve_workflows(
+@make_async
+def resolve_workflows(
     info: OrchestratorInfo,
     filter_by: list[GraphqlFilter] | None = None,
     sort_by: list[GraphqlSort] | None = None,

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/search/filters/date_filters.py

@@ -14,7 +14,6 @@
 from datetime import date, datetime
 from typing import Annotated, Any, Literal
 
-from dateutil.parser import parse as dt_parse
 from pydantic import BaseModel, BeforeValidator, Field, model_validator
 from sqlalchemy import TIMESTAMP, and_
 from sqlalchemy import cast as sa_cast
@@ -27,10 +26,10 @@ def _validate_date_string(v: Any) -> Any:
     if not isinstance(v, str):
         return v
     try:
-        dt_parse(v)
+        datetime.fromisoformat(v)
         return v
     except Exception as exc:
-        raise ValueError("is not a valid date or datetime string") from exc
+        raise ValueError("is not a valid ISO-8601 date or datetime string") from exc
 
 
 DateValue = datetime | date | str
@@ -44,8 +43,8 @@ class DateRange(BaseModel):
 
     @model_validator(mode="after")
     def _order(self) -> "DateRange":
-        to_datetime = dt_parse(str(self.end))
-        from_datetime = dt_parse(str(self.start))
+        to_datetime = datetime.fromisoformat(str(self.end))
+        from_datetime = datetime.fromisoformat(str(self.start))
         if to_datetime <= from_datetime:
             raise ValueError("'to' must be after 'from'")
         return self

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/search/indexing/indexer.py

@@ -45,6 +45,23 @@ def _maybe_begin(session: Session | None) -> Iterator[None]:
             yield
 
 
+@contextmanager
+def _maybe_progress(show_progress: bool, total_count: int | None, label: str) -> Iterator[Any]:
+    """Context manager that optionally creates a progress bar."""
+    if show_progress:
+        import typer
+
+        with typer.progressbar(
+            length=total_count,
+            label=label,
+            show_eta=True,
+            show_percent=bool(total_count),
+        ) as progress:
+            yield progress
+    else:
+        yield None
+
+
 class Indexer:
     """Index entities into `AiSearchIndex` using streaming reads and batched writes.
 
@@ -89,11 +106,21 @@ class Indexer:
         8) Repeat until the stream is exhausted.
     """
 
-    def __init__(self, config: EntityConfig, dry_run: bool, force_index: bool, chunk_size: int = 1000) -> None:
+    def __init__(
+        self,
+        config: EntityConfig,
+        dry_run: bool,
+        force_index: bool,
+        chunk_size: int = 1000,
+        show_progress: bool = False,
+        total_count: int | None = None,
+    ) -> None:
         self.config = config
         self.dry_run = dry_run
         self.force_index = force_index
         self.chunk_size = chunk_size
+        self.show_progress = show_progress
+        self.total_count = total_count
         self.embedding_model = llm_settings.EMBEDDING_MODEL
         self.logger = logger.bind(entity_kind=config.entity_kind.value)
         self._entity_titles: dict[str, str] = {}
@@ -116,13 +143,22 @@ class Indexer:
 
         with write_scope as database:
             session: Session | None = getattr(database, "session", None)
-            for entity in entities:
-                chunk.append(entity)
-                if len(chunk) >= self.chunk_size:
-                    flush()
 
-            if chunk:
-                flush()
+            with _maybe_progress(
+                self.show_progress, self.total_count, f"Indexing {self.config.entity_kind.value}"
+            ) as progress:
+                for entity in entities:
+                    chunk.append(entity)
+
+                    if len(chunk) >= self.chunk_size:
+                        flush()
+                        if progress:
+                            progress.update(self.chunk_size)
+
+                if chunk:
+                    flush()
+                    if progress:
+                        progress.update(len(chunk))
 
         final_log_message = (
             f"processed {total_records_processed} records and skipped {total_identical_records} identical records."

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/search/indexing/registry.py

@@ -66,6 +66,21 @@ class EntityConfig(Generic[ModelT]):
         return "UNKNOWN"
 
 
+@dataclass(frozen=True)
+class ProcessConfig(EntityConfig[ProcessTable]):
+    """Processes need to eager load workflow for workflow_name field."""
+
+    def get_all_query(self, entity_id: str | None = None) -> Query | Select:
+        from sqlalchemy.orm import selectinload
+
+        # Only load workflow, not subscriptions (keeps it lightweight)
+        query = self.table.query.options(selectinload(ProcessTable.workflow))
+        if entity_id:
+            pk_column = getattr(self.table, self.pk_name)
+            query = query.filter(pk_column == UUID(entity_id))
+        return query
+
+
 @dataclass(frozen=True)
 class WorkflowConfig(EntityConfig[WorkflowTable]):
     """Workflows have a custom select() function that filters out deleted workflows."""
@@ -95,7 +110,7 @@ ENTITY_CONFIG_REGISTRY: dict[EntityType, EntityConfig] = {
         root_name="product",
         title_paths=["product.description", "product.name"],
     ),
-    EntityType.PROCESS: EntityConfig(
+    EntityType.PROCESS: ProcessConfig(
         entity_kind=EntityType.PROCESS,
         table=ProcessTable,
         traverser=ProcessTraverser,

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/search/indexing/tasks.py

@@ -11,7 +11,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Any
+
 import structlog
+from sqlalchemy import func, select
 from sqlalchemy.orm import Query
 
 from orchestrator.db import db
@@ -23,12 +26,20 @@ from orchestrator.search.indexing.registry import ENTITY_CONFIG_REGISTRY
 logger = structlog.get_logger(__name__)
 
 
+def _get_entity_count(stmt: Any) -> int | None:
+    """Get total count of entities from a select statement."""
+
+    count_stmt = select(func.count()).select_from(stmt.subquery())
+    return db.session.execute(count_stmt).scalar()
+
+
 def run_indexing_for_entity(
     entity_kind: EntityType,
     entity_id: str | None = None,
     dry_run: bool = False,
     force_index: bool = False,
     chunk_size: int = 1000,
+    show_progress: bool = False,
 ) -> None:
     """Stream and index entities for the given kind.
 
@@ -46,6 +57,7 @@ def run_indexing_for_entity(
             existing hashes.
         chunk_size (int): Number of rows fetched per round-trip and passed to
             the indexer per batch.
+        show_progress (bool): When True, logs progress for each processed entity.
 
     Returns:
         None
@@ -60,10 +72,19 @@ def run_indexing_for_entity(
     else:
         stmt = q
 
+    total_count = _get_entity_count(stmt) if show_progress else None
+
     stmt = stmt.execution_options(stream_results=True, yield_per=chunk_size)
     entities = db.session.execute(stmt).scalars()
 
-    indexer = Indexer(config=config, dry_run=dry_run, force_index=force_index, chunk_size=chunk_size)
+    indexer = Indexer(
+        config=config,
+        dry_run=dry_run,
+        force_index=force_index,
+        chunk_size=chunk_size,
+        show_progress=show_progress,
+        total_count=total_count,
+    )
 
     with cache_subscription_models():
         indexer.run(entities)

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/search/indexing/traverse.py

@@ -29,7 +29,7 @@ from orchestrator.domain.base import ProductBlockModel, ProductModel
 from orchestrator.domain.lifecycle import (
     lookup_specialized_type,
 )
-from orchestrator.schemas.process import ProcessSchema
+from orchestrator.schemas.process import ProcessBaseSchema
 from orchestrator.schemas.workflow import WorkflowSchema
 from orchestrator.search.core.exceptions import ModelLoadError, ProductNotInRegistryError
 from orchestrator.search.core.types import LTREE_SEPARATOR, ExtractedField, FieldType
@@ -307,17 +307,39 @@ class ProductTraverser(BaseTraverser):
 
 
 class ProcessTraverser(BaseTraverser):
-    """Traverser for process entities using ProcessSchema model.
+    """Traverser for process entities using ProcessBaseSchema.
 
-    Note: Currently extracts only top-level process fields. Could be extended to include:
-    - Related subscriptions (entity.subscriptions)
-    - Related workflow information beyond workflow_name
+    Only indexes top-level process fields (no subscriptions or steps)
+    to keep the index size manageable.
     """
 
+    EXCLUDED_FIELDS = {"traceback", "failed_reason"}
+
+    @classmethod
+    def _load_model(cls, entity: ProcessTable) -> ProcessBaseSchema | None:
+        return cls._load_model_with_schema(entity, ProcessBaseSchema, "process_id")
+
     @classmethod
-    def _load_model(cls, process: ProcessTable) -> ProcessSchema:
-        """Load process model using ProcessSchema."""
-        return cls._load_model_with_schema(process, ProcessSchema, "process_id")
+    def get_fields(cls, entity: ProcessTable, pk_name: str, root_name: str) -> list[ExtractedField]:  # type: ignore[override]
+        """Extract fields from process, excluding fields in EXCLUDED_FIELDS."""
+        try:
+            model = cls._load_model(entity)
+            if model is None:
+                return []
+
+            return sorted(
+                (
+                    field
+                    for field in cls.traverse(model, root_name)
+                    if field.path.split(LTREE_SEPARATOR)[-1] not in cls.EXCLUDED_FIELDS
+                ),
+                key=lambda f: f.path,
+            )
+
+        except (ProductNotInRegistryError, ModelLoadError) as e:
+            entity_id = getattr(entity, pk_name, "unknown")
+            logger.error(f"Failed to extract fields from {entity.__class__.__name__}", id=str(entity_id), error=str(e))
+            return []
 
 
 class WorkflowTraverser(BaseTraverser):

{orchestrator_core-4.6.3rc1 → orchestrator_core-4.6.5}/orchestrator/search/query/builder.py

@@ -282,4 +282,7 @@ def build_aggregation_query(query: CountQuery | AggregateQuery, base_query: Sele
     if group_cols:
         stmt = stmt.group_by(*group_cols)
 
+    if query.temporal_group_by:
+        stmt = stmt.order_by(*group_cols)
+
     return stmt, group_col_names