orchestrator-core 3.1.1__tar.gz → 3.1.2rc3__tar.gz

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.1.1
+current_version = 3.1.2rc3
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(rc(?P<build>\d+))?

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/.github/workflows/run-linting-tests.yml

@@ -5,9 +5,9 @@ name: Linting tests
 
 on:
   push:
-  pull_request:
-    branches: [master]
+    branches: [ main ]
   workflow_call:
+  pull_request:
 
 jobs:
   build:

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/.github/workflows/run-unit-tests.yml

@@ -1,7 +1,9 @@
 name: Unit tests
 on:
   push:
+    branches: [ main ]
   workflow_call:
+  pull_request:
 
 jobs:
   container_job:

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 3.1.1
+Version: 3.1.2rc3
 Summary: This is the orchestrator workflow engine.
 Requires-Python: >=3.11,<3.14
 Classifier: Intended Audience :: Information Technology
@@ -28,7 +28,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
 Classifier: Topic :: Internet :: WWW/HTTP
 License-File: LICENSE
-Requires-Dist: alembic==1.14.1
+Requires-Dist: alembic==1.15.1
 Requires-Dist: anyio>=3.7.0
 Requires-Dist: click==8.*
 Requires-Dist: deprecated
@@ -37,18 +37,18 @@ Requires-Dist: fastapi~=0.115.2
 Requires-Dist: fastapi-etag==0.4.0
 Requires-Dist: more-itertools~=10.6.0
 Requires-Dist: itsdangerous
-Requires-Dist: Jinja2==3.1.5
+Requires-Dist: Jinja2==3.1.6
 Requires-Dist: orjson==3.10.15
-Requires-Dist: psycopg[binary]==3.2.5
-Requires-Dist: pydantic[email]~=2.10.6
+Requires-Dist: psycopg[binary]==3.2.6
+Requires-Dist: pydantic[email]~=2.8.2
 Requires-Dist: pydantic-settings~=2.8.0
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: python-rapidjson>=1.18,<1.21
 Requires-Dist: pytz==2025.1
-Requires-Dist: redis==5.0.3
+Requires-Dist: redis==5.1.1
 Requires-Dist: schedule==1.1.0
 Requires-Dist: sentry-sdk[fastapi]~=2.22.0
-Requires-Dist: SQLAlchemy==2.0.38
+Requires-Dist: SQLAlchemy==2.0.39
 Requires-Dist: SQLAlchemy-Utils==0.41.2
 Requires-Dist: structlog
 Requires-Dist: typer==0.15.2

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/docs/reference-docs/app/scaling.md

@@ -108,6 +108,112 @@ The code above sets our local Celery instance (which initializes the workflows)
 going to be used by the orchestrator-core. Without this code, the orchestrator-core would be only aware of a limited
 set of workflows that are part of orchestrator-core itself.
 
+### An example implementation
+
+When using Celery and Websockets you can use the following example and change it to your needs.
+
+```python
+"""This module contains functions and classes necessary for celery worker processes.
+
+When the orchestrator-core's thread process executor is specified as "celery", the `OrchestratorCore` FastAPI
+application registers celery-specific task functions and `start_process` and `resume_process` now defer to the
+celery task queue.
+
+Celery's task queue enables features like nightly validations by providing a task queue and workers to execute
+workflows that are all started in parallel, which would crash a single-threaded orchestrator-core.
+
+The application flow looks like this when "celery" is the executor (and websockets are enabled):
+
+- FastAPI application validates form input, and places a task on celery queue (create new process).
+  - If websockets are enabled, a connection should exist already b/t the client and backend.
+- FastAPI application begins watching redis pubsub channel for process updates from celery.
+- Celery worker picks up task from queue and begins executing.
+- On each step completion, it publishes state information to redis pubsub channel.
+- FastAPI application grabs this information and publishes it to the client websocket connection.
+
+A celery worker container will start by calling this module instead of `main.py` like so:
+```sh
+celery -A esnetorch.celery_worker worker -E -l INFO -Q new_tasks,resume_tasks,new_workflows,resume_workflows
+```
+
+* `-A` points to this module where the worker class is defined
+* `-E` sends task-related events (capturable and monitorable)
+* `-l` is the short flag for --loglevel
+* `-Q` specifies the queues which the worker should watch for new tasks
+
+See https://workfloworchestrator.org/orchestrator-core/reference-docs/app/scaling for more information.
+"""
+
+from uuid import UUID
+
+from celery import Celery
+from celery.signals import worker_shutting_down
+from orchestrator.db import init_database
+from orchestrator.domain import SUBSCRIPTION_MODEL_REGISTRY
+from orchestrator.services.tasks import initialise_celery
+from orchestrator.types import BroadcastFunc
+from orchestrator.websocket import broadcast_process_update_to_websocket, init_websocket_manager
+from orchestrator.websocket.websocket_manager import WebSocketManager
+from orchestrator.workflows import ALL_WORKFLOWS
+from structlog import get_logger
+
+from esnetorch.settings import app_settings
+
+
+logger = get_logger(__name__)
+
+
+def process_broadcast_fn(process_id: UUID) -> None:
+    # Catch all exceptions as broadcasting failure is noncritical to workflow completion
+    try:
+        broadcast_process_update_to_websocket(process_id)
+    except Exception as e:
+        logger.exception(e)
+
+
+class OrchestratorWorker(Celery):
+    websocket_manager: WebSocketManager
+    process_broadcast_fn: BroadcastFunc
+
+    def on_init(self) -> None:
+        init_database(app_settings)
+
+        # Prepare the wrapped_websocket_manager
+        # Note: cannot prepare the redis connections here as broadcasting is async
+        self.websocket_manager = init_websocket_manager(app_settings)
+        self.process_broadcast_fn = process_broadcast_fn
+
+        # Load the products and load the workflows
+        import esnetorch.products  # noqa: F401  Side-effects
+        import esnetorch.workflows  # noqa: F401  Side-effects
+
+        logger.info(
+            "Loaded the workflows and products",
+            workflows=len(ALL_WORKFLOWS.values()),
+            products=len(SUBSCRIPTION_MODEL_REGISTRY.values()),
+        )
+
+    def close(self) -> None:
+        super().close()
+
+
+celery = OrchestratorWorker(
+    f"{app_settings.SERVICE_NAME}-worker", broker=str(app_settings.CACHE_URI), include=["orchestrator.services.tasks"]
+)
+
+if app_settings.TESTING:
+    celery.conf.update(backend=str(app_settings.CACHE_URI), task_ignore_result=False)
+else:
+    celery.conf.update(task_ignore_result=True)
+
+celery.conf.update(
+    result_expires=3600,
+    worker_prefetch_multiplier=1,
+    worker_send_task_event=True,
+    task_send_sent_event=True,
+)
+```
+
 ### Running locally
 
 If you want to test your application locally you have to start both the orchestrator-api and one or more workers.

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/docs/reference-docs/workflows/callbacks.md

@@ -168,3 +168,11 @@ def create_l2vpn() -> StepList:
         >> set_status(SubscriptionLifecycle.ACTIVE)
     )
 ```
+
+## Callback progress during execution
+
+For long-running jobs, such as executing Terraform or Ansible playbooks, the Orchestrator allows callback jobs to send real-time progress updates which can be used to provide operators with feedback on the progress of the running task.
+
+Progress updates should be delivered to `{callback_route}/progress`, where `"/progress"` is a fixed endpoint appended to the callback URL.
+
+The remote service should send a JSON or plain string payload with each progress callback, this replaces the previous progress update and refreshes the UI. Once the final callback is triggered and the job completes, progress updates are removed, leaving only the callback result. Therefore, all troubleshooting or diagnostic information must be included in the final callback payload, as progress updates are not retained for debugging.

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "3.1.1"
+__version__ = "3.1.2rc3"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/orchestrator/api/api_v1/endpoints/processes.py

@@ -58,6 +58,7 @@ from orchestrator.services.processes import (
     load_process,
     resume_process,
     start_process,
+    update_awaiting_process_progress,
 )
 from orchestrator.services.settings import get_engine_settings
 from orchestrator.settings import app_settings
@@ -197,6 +198,28 @@ def continue_awaiting_process_endpoint(
         raise_status(HTTPStatus.NOT_FOUND, str(e))
 
 
+@router.post(
+    "/{process_id}/callback/{token}/progress",
+    response_model=None,
+    status_code=HTTPStatus.OK,
+    dependencies=[Depends(check_global_lock, use_cache=False)],
+)
+def update_progress_on_awaiting_process_endpoint(
+    process_id: UUID,
+    token: str,
+    data: str | State = Body(...),
+) -> None:
+    process = _get_process(process_id)
+
+    if process.last_status != ProcessStatus.AWAITING_CALLBACK:
+        raise_status(HTTPStatus.CONFLICT, "This process is not in an awaiting state.")
+
+    try:
+        update_awaiting_process_progress(process, token=token, data=data)
+    except AssertionError as exc:
+        raise_status(HTTPStatus.NOT_FOUND, str(exc))
+
+
 @router.put(
     "/resume-all", response_model=ProcessResumeAllSchema, dependencies=[Depends(check_global_lock, use_cache=False)]
 )

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/orchestrator/db/models.py

@@ -92,7 +92,7 @@ class InputStateTable(BaseModel):
 
     input_state_id = mapped_column(UUIDType, primary_key=True, server_default=text("uuid_generate_v4()"), index=True)
     process_id = mapped_column("pid", UUIDType, ForeignKey("processes.pid"), nullable=False)
-    input_state = mapped_column(pg.JSONB(), nullable=False)  # type: ignore
+    input_state = mapped_column(pg.JSONB(), nullable=False)
     input_time = mapped_column(UtcTimestamp, server_default=text("current_timestamp()"), nullable=False)
     input_type = mapped_column(Enum(InputType), nullable=False)
 
@@ -137,7 +137,7 @@ class ProcessStepTable(BaseModel):
     )
     name = mapped_column(String(), nullable=False)
     status = mapped_column(String(50), nullable=False)
-    state = mapped_column(pg.JSONB(), nullable=False)  # type: ignore
+    state = mapped_column(pg.JSONB(), nullable=False)
     created_by = mapped_column(String(255), nullable=True)
     executed_at = mapped_column(UtcTimestamp, server_default=text("statement_timestamp()"), nullable=False)
     commit_hash = mapped_column(String(40), nullable=True, default=GIT_COMMIT_HASH)
@@ -642,7 +642,7 @@ class SubscriptionMetadataTable(BaseModel):
         primary_key=True,
         index=True,
     )
-    metadata_ = mapped_column("metadata", pg.JSONB(), nullable=False)  # type: ignore
+    metadata_ = mapped_column("metadata", pg.JSONB(), nullable=False)
 
     @staticmethod
     def find_by_subscription_id(subscription_id: str) -> SubscriptionMetadataTable | None:
@@ -651,6 +651,8 @@ class SubscriptionMetadataTable(BaseModel):
 
 class SubscriptionSearchView(BaseModel):
     __tablename__ = "subscriptions_search"
+    __table_args__ = {"info": {"materialized_view": True}}
+
     subscription_id = mapped_column(
         UUIDType, ForeignKey("subscriptions.subscription_id"), nullable=False, index=True, primary_key=True
     )

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/orchestrator/migrations/env.py

@@ -41,6 +41,14 @@ target_metadata = BaseModel.metadata
 # ... etc.
 
 
+def include_object(object, name, type_, reflected, compare_to):  # type: ignore
+    """Determines if an object should be included."""
+
+    if type_ == "table" and object.info.get("materialized_view", False):
+        return False
+    return True
+
+
 def run_migrations_offline() -> None:
     """Run migrations in 'offline' mode.
 
@@ -55,7 +63,11 @@ def run_migrations_offline() -> None:
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        include_object=include_object,
     )
 
     with context.begin_transaction():
@@ -90,6 +102,7 @@ def run_migrations_online() -> None:
         target_metadata=target_metadata,
         process_revision_directives=process_revision_directives,
         compare_type=True,
+        include_object=include_object,
     )
 
     try:

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/orchestrator/migrations/versions/schema/2020-10-19_c112305b07d3_initial_schema_migration.py

@@ -157,7 +157,7 @@ def upgrade() -> None:
         sa.Column("pid", sqlalchemy_utils.types.uuid.UUIDType(), nullable=False),
         sa.Column("name", sa.String(), nullable=False),
         sa.Column("status", sa.String(length=50), nullable=False),
-        sa.Column("state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),  # type: ignore
+        sa.Column("state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
         sa.Column("created_by", sa.String(length=255), nullable=True),
         sa.Column(
             "executed_at",

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/orchestrator/migrations/versions/schema/2023-05-25_b1970225392d_add_subscription_metadata_workflow.py

@@ -29,7 +29,7 @@ def upgrade() -> None:
             nullable=False,
             index=True,
         ),
-        sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False),  # type: ignore
+        sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
         sa.ForeignKeyConstraint(["subscription_id"], ["subscriptions.subscription_id"], ondelete="CASCADE"),
     )
 

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/orchestrator/migrations/versions/schema/2025-02-12_bac6be6f2b4f_added_input_state_table.py

@@ -31,7 +31,7 @@ def upgrade() -> None:
             nullable=False,
         ),
         sa.Column("pid", sqlalchemy_utils.types.uuid.UUIDType(), nullable=False),
-        sa.Column("input_state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),  # type: ignore
+        sa.Column("input_state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
         sa.Column(
             "input_time",
             db.models.UtcTimestamp(timezone=True),

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/orchestrator/services/processes.py

@@ -43,9 +43,10 @@ from orchestrator.targets import Target
 from orchestrator.types import BroadcastFunc
 from orchestrator.utils.datetime import nowtz
 from orchestrator.utils.errors import error_state_to_dict
-from orchestrator.websocket import broadcast_invalidate_status_counts
+from orchestrator.websocket import broadcast_invalidate_status_counts, broadcast_process_update_to_websocket
 from orchestrator.workflow import (
     CALLBACK_TOKEN_KEY,
+    DEFAULT_CALLBACK_PROGRESS_KEY,
     Failed,
     ProcessStat,
     ProcessStatus,
@@ -566,6 +567,39 @@ def resume_process(
     return resume_func(process, user_inputs=user_inputs, user=user, broadcast_func=broadcast_func)
 
 
+def ensure_correct_callback_token(pstat: ProcessStat, *, token: str) -> None:
+    """Ensure that a callback token matches the expected value in state.
+
+    Args:
+        pstat: ProcessStat of process.
+        token: The token which was generated for the process.
+
+    Raises:
+        AssertionError: if the supplied token does not match the generated process token.
+
+    """
+    state = pstat.state.unwrap()
+
+    # Check if the token matches
+    token_from_state = state.get(CALLBACK_TOKEN_KEY)
+    if token != token_from_state:
+        raise AssertionError("Invalid token")
+
+
+def replace_current_step_state(process: ProcessTable, *, new_state: State) -> None:
+    """Replace the state of the current step in a process.
+
+    Args:
+        process: Process from database
+        new_state: The new state
+
+    """
+    current_step = process.steps[-1]
+    current_step.state = new_state
+    db.session.add(current_step)
+    db.session.commit()
+
+
 def continue_awaiting_process(
     process: ProcessTable,
     *,
@@ -589,10 +623,7 @@ def continue_awaiting_process(
     pstat = load_process(process)
     state = pstat.state.unwrap()
 
-    # Check if the token matches
-    token_from_state = state.get(CALLBACK_TOKEN_KEY)
-    if token != token_from_state:
-        raise AssertionError("Invalid token")
+    ensure_correct_callback_token(pstat, token=token)
 
     # We need to pass the callback data to the worker executor. Currently, this is not supported.
     # Therefore, we update the step state in the db and kick-off resume_workflow
@@ -600,16 +631,47 @@ def continue_awaiting_process(
     result_key = state.get("__callback_result_key", "callback_result")
     state = {**state, result_key: input_data}
 
-    current_step = process.steps[-1]
-    current_step.state = state
-    db.session.add(current_step)
-    db.session.commit()
+    replace_current_step_state(process, new_state=state)
 
     # Continue the workflow
     resume_func = get_execution_context()["resume"]
     return resume_func(process, broadcast_func=broadcast_func)
 
 
+def update_awaiting_process_progress(
+    process: ProcessTable,
+    *,
+    token: str,
+    data: str | State,
+) -> UUID:
+    """Update progress for a process awaiting data from a callback.
+
+    Args:
+        process: Process from database
+        token: The token which was generated for the process. This must match.
+        data: Progress data posted to the callback
+
+    Returns:
+        process id
+
+    Raises:
+        AssertionError: if the supplied token does not match the generated process token.
+
+    """
+    pstat = load_process(process)
+
+    ensure_correct_callback_token(pstat, token=token)
+
+    state = pstat.state.unwrap()
+    progress_key = state.get(DEFAULT_CALLBACK_PROGRESS_KEY, "callback_progress")
+    state = {**state, progress_key: data} | {"__remove_keys": [progress_key]}
+
+    replace_current_step_state(process, new_state=state)
+    broadcast_process_update_to_websocket(process.process_id)
+
+    return process.process_id
+
+
 async def _async_resume_processes(
     processes: Sequence[ProcessTable],
     user_name: str,

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/orchestrator/utils/state.py

@@ -150,65 +150,85 @@ def _build_arguments(func: StepFunc | InputStepFunc, state: State) -> list:  # n
 
     Raises:
          KeyError: if requested argument is not in the state, or cannot be reconstructed as an initial domain model.
+         ValueError: if requested argument cannot be converted to the expected type.
 
     """
+
     sig = inspect.signature(func)
+    if not sig.parameters:
+        return []
+
+    def _convert_to_uuid(v: Any) -> UUID:
+        """Converts the value to a UUID instance if it is not already one."""
+        return v if isinstance(v, UUID) else UUID(v)
+
     arguments: list[Any] = []
-    if sig.parameters:
-        for name, param in sig.parameters.items():
-            # Ignore dynamic arguments. Mostly need to deal with `const`
-            if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
-                logger.warning("*args and **kwargs are not supported as step params")
-                continue
-
-            # If we find an argument named "state" we use the whole state as argument to
-            # This is mainly to be backward compatible with code that needs the whole state...
-            # TODO: Remove this construction
-            if name == "state":
-                arguments.append(state)
-                continue
-
-            # Workaround for the fact that you can't call issubclass on typing types
+    for name, param in sig.parameters.items():
+        # Ignore dynamic arguments. Mostly need to deal with `const`
+        if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
+            logger.warning("*args and **kwargs are not supported as step params")
+            continue
+
+        # If we find an argument named "state" we use the whole state as argument to
+        # This is mainly to be backward compatible with code that needs the whole state...
+        # TODO: Remove this construction
+        if name == "state":
+            arguments.append(state)
+            continue
+
+        # Workaround for the fact that you can't call issubclass on typing types
+        try:
+            is_subscription_model_type = issubclass(param.annotation, SubscriptionModel)
+        except Exception:
+            is_subscription_model_type = False
+
+        if is_subscription_model_type:
+            subscription_id = _get_sub_id(state.get(name))
+            if subscription_id:
+                sub_mod = param.annotation.from_subscription(subscription_id)
+                arguments.append(sub_mod)
+            else:
+                logger.error("Could not find key in state.", key=name, state=state)
+                raise KeyError(f"Could not find key '{name}' in state.")
+        elif is_list_type(param.annotation, SubscriptionModel):
+            subscription_ids = [_get_sub_id(item) for item in state.get(name, [])]
+            # Actual type is first argument from list type
+            if (actual_type := get_args(param.annotation)[0]) == Any:
+                raise ValueError(
+                    f"Step function argument '{param.name}' cannot be serialized from database with type 'Any'"
+                )
+            subscriptions = [actual_type.from_subscription(subscription_id) for subscription_id in subscription_ids]
+            arguments.append(subscriptions)
+        elif is_optional_type(param.annotation, SubscriptionModel):
+            subscription_id = _get_sub_id(state.get(name))
+            if subscription_id:
+                # Actual type is first argument from optional type
+                sub_mod = get_args(param.annotation)[0].from_subscription(subscription_id)
+                arguments.append(sub_mod)
+            else:
+                arguments.append(None)
+        elif param.default is not inspect.Parameter.empty:
+            arguments.append(state.get(name, param.default))
+        else:
             try:
-                is_subscription_model_type = issubclass(param.annotation, SubscriptionModel)
-            except Exception:
-                is_subscription_model_type = False
-
-            if is_subscription_model_type:
-                subscription_id = _get_sub_id(state.get(name))
-                if subscription_id:
-                    sub_mod = param.annotation.from_subscription(subscription_id)
-                    arguments.append(sub_mod)
-                else:
-                    logger.error("Could not find key in state.", key=name, state=state)
-                    raise KeyError(f"Could not find key '{name}' in state.")
-            elif is_list_type(param.annotation, SubscriptionModel):
-                subscription_ids = map(_get_sub_id, state.get(name, []))
-                # Actual type is first argument from list type
-                if (actual_type := get_args(param.annotation)[0]) == Any:
-                    raise ValueError(
-                        f"Step function argument '{param.name}' cannot be serialized from database with type 'Any'"
-                    )
-                subscriptions = [actual_type.from_subscription(subscription_id) for subscription_id in subscription_ids]
-                arguments.append(subscriptions)
-            elif is_optional_type(param.annotation, SubscriptionModel):
-                subscription_id = _get_sub_id(state.get(name))
-                if subscription_id:
-                    # Actual type is first argument from optional type
-                    sub_mod = get_args(param.annotation)[0].from_subscription(subscription_id)
-                    arguments.append(sub_mod)
+                value = state[name]
+                if param.annotation == UUID:
+                    arguments.append(_convert_to_uuid(value))
+                elif is_list_type(param.annotation, UUID):
+                    arguments.append([_convert_to_uuid(item) for item in value])
+                elif is_optional_type(param.annotation, UUID):
+                    arguments.append(None if value is None else _convert_to_uuid(value))
                 else:
-                    arguments.append(None)
-            elif param.default is not inspect.Parameter.empty:
-                arguments.append(state.get(name, param.default))
-            else:
-                try:
-                    arguments.append(state[name])
-                except KeyError as key_error:
-                    logger.error("Could not find key in state.", key=name, state=state)
-                    raise KeyError(
-                        f"Could not find key '{name}' in state. for function {func.__module__}.{func.__qualname__}"
-                    ) from key_error
+                    arguments.append(value)
+            except KeyError as key_error:
+                logger.error("Could not find key in state.", key=name, state=state)
+                raise KeyError(
+                    f"Could not find key '{name}' in state. for function {func.__module__}.{func.__qualname__}"
+                ) from key_error
+            except ValueError as value_error:
+                logger.error("Could not convert value to expected type.", key=name, state=state, value=state[name])
+                raise ValueError(f"Could not convert value '{state[name]}' to {param.annotation}") from value_error
+
     return arguments
 
 

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/orchestrator/workflow.py

@@ -69,6 +69,7 @@ step_log_fn_var: contextvars.ContextVar[StepLogFuncInternal] = contextvars.Conte
 
 DEFAULT_CALLBACK_ROUTE_KEY = "callback_route"
 CALLBACK_TOKEN_KEY = "__callback_token"  # noqa: S105
+DEFAULT_CALLBACK_PROGRESS_KEY = "callback_progress"  # noqa: S105
 
 
 @runtime_checkable

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/pyproject.toml

@@ -36,7 +36,7 @@ classifiers = [
     "Topic :: Internet :: WWW/HTTP",
 ]
 dependencies = [
-    "alembic==1.14.1",
+    "alembic==1.15.1",
     "anyio>=3.7.0",
     "click==8.*",
     "deprecated",
@@ -45,18 +45,18 @@ dependencies = [
     "fastapi-etag==0.4.0",
     "more-itertools~=10.6.0",
     "itsdangerous",
-    "Jinja2==3.1.5",
+    "Jinja2==3.1.6",
     "orjson==3.10.15",
-    "psycopg[binary]==3.2.5",
-    "pydantic[email]~=2.10.6",
+    "psycopg[binary]==3.2.6",
+    "pydantic[email]~=2.8.2",
     "pydantic-settings~=2.8.0",
     "python-dateutil==2.8.2",
     "python-rapidjson>=1.18,<1.21",
     "pytz==2025.1",
-    "redis==5.0.3",
+    "redis==5.1.1",
     "schedule==1.1.0",
     "sentry-sdk[fastapi]~=2.22.0",
-    "SQLAlchemy==2.0.38",
+    "SQLAlchemy==2.0.39",
     "SQLAlchemy-Utils==0.41.2",
     "structlog",
     "typer==0.15.2",

{orchestrator_core-3.1.1 → orchestrator_core-3.1.2rc3}/test/unit_tests/cli/data/generate/workflows/shared.py

@@ -8,7 +8,7 @@ from pydantic import ConfigDict
 
 
 def summary_form(product_name: str, summary_data: dict) -> Generator:
-    ProductSummary: TypeAlias = cast(type[MigrationSummary], migration_summary(summary_data))
+    ProductSummary: TypeAlias = cast("type[MigrationSummary]", migration_summary(summary_data))
 
     class SummaryForm(FormPage):
         model_config = ConfigDict(title=f"{product_name} summary")