orchestrator-core 4.2.0rc2__py3-none-any.whl → 4.3.0rc1__py3-none-any.whl

orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "4.2.0rc2"
+__version__ = "4.3.0rc1"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings

orchestrator/api/api_v1/endpoints/processes.py

@@ -48,6 +48,7 @@ from orchestrator.services.processes import (
     _async_resume_processes,
     _get_process,
     abort_process,
+    can_be_resumed,
     continue_awaiting_process,
     load_process,
     resume_process,
@@ -120,16 +121,6 @@ def get_auth_callbacks(steps: StepList, workflow: Workflow) -> tuple[Authorizer
     return auth_resume, auth_retry
 
 
-def can_be_resumed(status: ProcessStatus) -> bool:
-    return status in (
-        ProcessStatus.SUSPENDED,  # Can be resumed
-        ProcessStatus.WAITING,  # Can be retried
-        ProcessStatus.FAILED,  # Can be retried
-        ProcessStatus.API_UNAVAILABLE,  # subtype of FAILED
-        ProcessStatus.INCONSISTENT_DATA,  # subtype of FAILED
-    )
-
-
 def resolve_user_name(
     *,
     reporter: Reporter | None,

orchestrator/migrations/versions/schema/2025-07-28_850dccac3b02_update_description_of_resume_workflows_.py

@@ -0,0 +1,35 @@
+"""Update description of resume workflows task.
+
+Revision ID: 850dccac3b02
+Revises: 93fc5834c7e5
+Create Date: 2025-07-28 15:38:57.211087
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "850dccac3b02"
+down_revision = "93fc5834c7e5"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        UPDATE workflows
+        SET description = 'Resume all workflows that are stuck on tasks with the status ''waiting'', ''created'' or ''resumed'''
+        WHERE name = 'task_resume_workflows';
+    """
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        """
+        UPDATE workflows
+        SET description = 'Resume all workflows that are stuck on tasks with the status ''waiting'''
+        WHERE name = 'task_resume_workflows';
+    """
+    )

orchestrator/services/executors/__init__.py

@@ -0,0 +1,12 @@
+# Copyright 2019-2020 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

orchestrator/services/{celery.py → executors/celery.py}

@@ -18,19 +18,22 @@ from uuid import UUID
 import structlog
 from celery.result import AsyncResult
 from kombu.exceptions import ConnectionError, OperationalError
+from sqlalchemy import select
 
-from oauth2_lib.fastapi import OIDCUserModel
 from orchestrator import app_settings
 from orchestrator.api.error_handling import raise_status
 from orchestrator.db import ProcessTable, db
-from orchestrator.services.input_state import store_input_state
-from orchestrator.services.processes import create_process, delete_process
+from orchestrator.services.processes import (
+    SYSTEM_USER,
+    can_be_resumed,
+    create_process,
+    delete_process,
+    set_process_status,
+)
 from orchestrator.services.workflows import get_workflow_by_name
-from orchestrator.workflows import get_workflow
+from orchestrator.workflow import ProcessStat, ProcessStatus
 from pydantic_forms.types import State
 
-SYSTEM_USER = "SYSTEM"
-
 logger = structlog.get_logger(__name__)
 
 
@@ -42,29 +45,17 @@ def _block_when_testing(task_result: AsyncResult) -> None:
             raise RuntimeError("Celery worker has failed to resume process")
 
 
-def _celery_start_process(
-    workflow_key: str,
-    user_inputs: list[State] | None,
-    user: str = SYSTEM_USER,
-    user_model: OIDCUserModel | None = None,
-    **kwargs: Any,
-) -> UUID:
+def _celery_start_process(pstat: ProcessStat, user: str = SYSTEM_USER, **kwargs: Any) -> UUID:
     """Client side call of Celery."""
     from orchestrator.services.tasks import NEW_TASK, NEW_WORKFLOW, get_celery_task
 
-    workflow = get_workflow(workflow_key)
-    if not workflow:
-        raise_status(HTTPStatus.NOT_FOUND, "Workflow does not exist")
-
-    wf_table = get_workflow_by_name(workflow.name)
-    if not wf_table:
+    if not (wf_table := get_workflow_by_name(pstat.workflow.name)):
         raise_status(HTTPStatus.NOT_FOUND, "Workflow in Database does not exist")
 
     task_name = NEW_TASK if wf_table.is_task else NEW_WORKFLOW
     trigger_task = get_celery_task(task_name)
-    pstat = create_process(workflow_key, user_inputs=user_inputs, user=user, user_model=user_model)
     try:
-        result = trigger_task.delay(pstat.process_id, workflow_key, user)
+        result = trigger_task.delay(pstat.process_id, user)
         _block_when_testing(result)
         return pstat.process_id
     except (ConnectionError, OperationalError) as e:
@@ -77,65 +68,62 @@ def _celery_start_process(
 def _celery_resume_process(
     process: ProcessTable,
     *,
-    user_inputs: list[State] | None = None,
     user: str | None = None,
     **kwargs: Any,
-) -> UUID:
+) -> bool:
     """Client side call of Celery."""
-    from orchestrator.services.processes import load_process
     from orchestrator.services.tasks import RESUME_TASK, RESUME_WORKFLOW, get_celery_task
 
-    pstat = load_process(process)
     last_process_status = process.last_status
-    workflow = pstat.workflow
-
-    wf_table = get_workflow_by_name(workflow.name)
-    if not workflow or not wf_table:
-        raise_status(HTTPStatus.NOT_FOUND, "Workflow does not exist")
 
-    task_name = RESUME_TASK if wf_table.is_task else RESUME_WORKFLOW
+    task_name = RESUME_TASK if process.workflow.is_task else RESUME_WORKFLOW
     trigger_task = get_celery_task(task_name)
 
-    user_inputs = user_inputs or [{}]
-    store_input_state(pstat.process_id, user_inputs, "user_input")
+    _celery_set_process_status_resumed(process.process_id)
+
     try:
-        _celery_set_process_status_resumed(process)
-        result = trigger_task.delay(pstat.process_id, user)
+        result = trigger_task.delay(process.process_id, user)
         _block_when_testing(result)
 
-        return pstat.process_id
+        return process.process_id
     except (ConnectionError, OperationalError) as e:
         logger.warning(
             "Connection error when submitting task to celery. Resetting process status back",
             current_status=process.last_status,
             last_status=last_process_status,
         )
-        _celery_set_process_status(process, last_process_status)
+        set_process_status(process.process_id, last_process_status)
         raise e
 
 
-def _celery_set_process_status(process: ProcessTable, status: str) -> None:
-    process.last_status = status
-    db.session.add(process)
-    db.session.commit()
-
+def _celery_set_process_status_resumed(process_id: UUID) -> None:
+    """Set the process status to RESUMED to show its waiting to be picked up by a worker.
 
-def _celery_set_process_status_resumed(process: ProcessTable) -> None:
-    """Set the process status to RESUMED to prevent re-adding to task queue.
+    uses with_for_update to lock the subscription in a transaction, preventing other changes.
+    rolls back transation and raises an exception when it can't change to RESUMED to prevent it from being added to the queue.
 
     Args:
-        process: Process from database
+        process_id: Process ID to fetch process from DB
     """
-    from orchestrator.db import db
-    from orchestrator.workflow import ProcessStatus
+    stmt = select(ProcessTable).where(ProcessTable.process_id == process_id).with_for_update()
+
+    result = db.session.execute(stmt)
+    locked_process = result.scalar_one_or_none()
+
+    if not locked_process:
+        raise ValueError(f"Process not found: {process_id}")
 
-    process.last_status = ProcessStatus.RESUMED
-    db.session.add(process)
-    db.session.commit()
+    if can_be_resumed(locked_process.last_status):
+        locked_process.last_status = ProcessStatus.RESUMED
+        db.session.commit()
+    else:
+        db.session.rollback()
+        raise ValueError(f"Process has incorrect status to resume: {locked_process.last_status}")
 
 
 def _celery_validate(validation_workflow: str, json: list[State] | None) -> None:
-    _celery_start_process(validation_workflow, user_inputs=json)
+    pstat = create_process(validation_workflow, user_inputs=json)
+    _celery_start_process(pstat)
 
 
 CELERY_EXECUTION_CONTEXT: dict[str, Callable] = {

orchestrator/services/executors/threadpool.py

@@ -0,0 +1,128 @@
+# Copyright 2019-2025 SURF, GÉANT, ESnet.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from collections.abc import Callable
+from functools import partial
+from uuid import UUID
+
+import structlog
+from sqlalchemy import select
+
+from oauth2_lib.fastapi import OIDCUserModel
+from orchestrator.db import ProcessTable, db
+from orchestrator.services.input_state import InputType, retrieve_input_state
+from orchestrator.services.processes import (
+    RESUME_WORKFLOW_REMOVED_ERROR_MSG,
+    START_WORKFLOW_REMOVED_ERROR_MSG,
+    SYSTEM_USER,
+    StateMerger,
+    _run_process_async,
+    create_process,
+    load_process,
+    safe_logstep,
+)
+from orchestrator.types import BroadcastFunc
+from orchestrator.workflow import (
+    ProcessStat,
+    ProcessStatus,
+    runwf,
+)
+from orchestrator.workflows.removed_workflow import removed_workflow
+from pydantic_forms.types import State
+
+logger = structlog.get_logger(__name__)
+
+
+def _set_process_status_running(process_id: UUID) -> None:
+    """Set the process status to RUNNING to prevent it from being picked up by mutliple workers.
+
+    uses with_for_update to lock the subscription in a transaction, preventing other changes.
+    rolls back transation and raises an exception when its already on status RUNNING to prevent worker from running an already running process
+
+    Args:
+        process_id: Process ID to fetch process from DB
+    """
+    stmt = select(ProcessTable).where(ProcessTable.process_id == process_id).with_for_update()
+
+    result = db.session.execute(stmt)
+    locked_process = result.scalar_one_or_none()
+
+    if not locked_process:
+        db.session.rollback()
+        raise ValueError(f"Process not found: {process_id}")
+
+    if locked_process.last_status is not ProcessStatus.RUNNING:
+        locked_process.last_status = ProcessStatus.RUNNING
+        db.session.commit()
+    else:
+        db.session.rollback()
+        raise Exception("Process is already running")
+
+
+def thread_start_process(
+    pstat: ProcessStat,
+    user: str = SYSTEM_USER,
+    user_model: OIDCUserModel | None = None,
+    broadcast_func: BroadcastFunc | None = None,
+) -> UUID:
+    if pstat.workflow == removed_workflow:
+        raise ValueError(START_WORKFLOW_REMOVED_ERROR_MSG)
+
+    # enforce an update to the process status to properly show the process
+    _set_process_status_running(pstat.process_id)
+
+    input_data = retrieve_input_state(pstat.process_id, "initial_state", False)
+    pstat.update(state=pstat.state.map(lambda state: StateMerger.merge(state, input_data.input_state)))
+
+    _safe_logstep_with_func = partial(safe_logstep, broadcast_func=broadcast_func)
+    return _run_process_async(pstat.process_id, lambda: runwf(pstat, _safe_logstep_with_func))
+
+
+def thread_resume_process(
+    process: ProcessTable,
+    *,
+    user: str | None = None,
+    user_model: OIDCUserModel | None = None,
+    broadcast_func: BroadcastFunc | None = None,
+) -> UUID:
+    # ATTENTION!! When modifying this function make sure you make similar changes to `resume_workflow` in the test code
+    pstat = load_process(process)
+    if pstat.workflow == removed_workflow:
+        raise ValueError(RESUME_WORKFLOW_REMOVED_ERROR_MSG)
+
+    if user:
+        pstat.update(current_user=user)
+
+    # retrieve_input_str is for the edge case when workflow engine stops whilst there is an existing 'CREATED' process queue'ed.
+    # It will have become a `RUNNING` process that gets resumed and this should fetch initial_state instead of user_input.
+    retrieve_input_str: InputType = "user_input" if process.steps else "initial_state"
+    input_data = retrieve_input_state(process.process_id, retrieve_input_str, False)
+    pstat.update(state=pstat.state.map(lambda state: StateMerger.merge(state, input_data.input_state)))
+
+    # enforce an update to the process status to properly show the process
+    _set_process_status_running(process.process_id)
+
+    _safe_logstep_prep = partial(safe_logstep, broadcast_func=broadcast_func)
+    _run_process_async(pstat.process_id, lambda: runwf(pstat, _safe_logstep_prep))
+    return pstat.process_id
+
+
+def thread_validate_workflow(validation_workflow: str, json: list[State] | None) -> UUID:
+    pstat = create_process(validation_workflow, user_inputs=json)
+    return thread_start_process(pstat)
+
+
+THREADPOOL_EXECUTION_CONTEXT: dict[str, Callable] = {
+    "start": thread_start_process,
+    "resume": thread_resume_process,
+    "validate": thread_validate_workflow,
+}

orchestrator/services/input_state.py

@@ -24,12 +24,13 @@ logger = structlog.get_logger(__name__)
 InputType = Literal["initial_state", "user_input"]
 
 
-def retrieve_input_state(process_id: UUID, input_type: InputType) -> InputStateTable:
+def retrieve_input_state(process_id: UUID, input_type: InputType, raise_exception: bool = True) -> InputStateTable:
     """Get user input.
 
     Args:
         process_id: Process ID
         input_type: The type of the input.
+        raise_exception: boolean to throw error when not finding data or not
 
     Returns:
         User input table
@@ -46,7 +47,9 @@ def retrieve_input_state(process_id: UUID, input_type: InputType) -> InputStateT
     if res:
         logger.debug("Retrieved input state", process_id=process_id, input_state=res, input_type=input_type)
         return res
-    raise ValueError(f"No input state for pid: {process_id}")
+    if raise_exception:
+        raise ValueError(f"No input state for pid: {process_id}")
+    return InputStateTable(input_state={})
 
 
 def store_input_state(

orchestrator/services/processes.py

@@ -51,7 +51,6 @@ from orchestrator.workflow import (
     Success,
     Workflow,
     abort_wf,
-    runwf,
 )
 from orchestrator.workflow import Process as WFProcess
 from orchestrator.workflows import get_workflow
@@ -68,13 +67,18 @@ SYSTEM_USER = "SYSTEM"
 
 _workflow_executor = None
 
+START_WORKFLOW_REMOVED_ERROR_MSG = "This workflow cannot be started because it has been removed"
+RESUME_WORKFLOW_REMOVED_ERROR_MSG = "This workflow cannot be resumed because it has been removed"
+
 
 def get_execution_context() -> dict[str, Callable]:
     if app_settings.EXECUTOR == ExecutorType.WORKER:
-        from orchestrator.services.celery import CELERY_EXECUTION_CONTEXT
+        from orchestrator.services.executors.celery import CELERY_EXECUTION_CONTEXT
 
         return CELERY_EXECUTION_CONTEXT
 
+    from orchestrator.services.executors.threadpool import THREADPOOL_EXECUTION_CONTEXT
+
     return THREADPOOL_EXECUTION_CONTEXT
 
 
@@ -449,7 +453,6 @@ def create_process(
     }
 
     try:
-
         state = post_form(workflow.initial_input_form, initial_state, user_inputs)
     except FormValidationError:
         logger.exception("Validation errors", user_inputs=user_inputs)
@@ -469,19 +472,6 @@ def create_process(
     return pstat
 
 
-def thread_start_process(
-    workflow_key: str,
-    user_inputs: list[State] | None = None,
-    user: str = SYSTEM_USER,
-    user_model: OIDCUserModel | None = None,
-    broadcast_func: BroadcastFunc | None = None,
-) -> UUID:
-    pstat = create_process(workflow_key, user_inputs=user_inputs, user=user, user_model=user_model)
-
-    _safe_logstep_with_func = partial(safe_logstep, broadcast_func=broadcast_func)
-    return _run_process_async(pstat.process_id, lambda: runwf(pstat, _safe_logstep_with_func))
-
-
 def start_process(
     workflow_key: str,
     user_inputs: list[State] | None = None,
@@ -502,57 +492,47 @@ def start_process(
         process id
 
     """
+    pstat = create_process(workflow_key, user_inputs=user_inputs, user=user)
+
     start_func = get_execution_context()["start"]
-    return start_func(
-        workflow_key, user_inputs=user_inputs, user=user, user_model=user_model, broadcast_func=broadcast_func
-    )
+    return start_func(pstat, user=user, user_model=user_model, broadcast_func=broadcast_func)
 
 
-def thread_resume_process(
+def restart_process(
     process: ProcessTable,
     *,
-    user_inputs: list[State] | None = None,
     user: str | None = None,
     broadcast_func: BroadcastFunc | None = None,
 ) -> UUID:
-    # ATTENTION!! When modifying this function make sure you make similar changes to `resume_workflow` in the test code
-
-    if user_inputs is None:
-        user_inputs = [{}]
-
-    pstat = load_process(process)
-
-    if pstat.workflow == removed_workflow:
-        raise ValueError("This workflow cannot be resumed")
+    """Restart a process that is stuck on status CREATED.
 
-    form = pstat.log[0].form
-
-    user_input = post_form(form, pstat.state.unwrap(), user_inputs)
+    Args:
+        process: Process from database
+        user: user who resumed this process
+        broadcast_func: Optional function to broadcast process data
 
-    if user:
-        pstat.update(current_user=user)
+    Returns:
+        process id
 
-    if user_input:
-        pstat.update(state=pstat.state.map(lambda state: StateMerger.merge(state, user_input)))
-    store_input_state(pstat.process_id, user_input, "user_input")
-    # enforce an update to the process status to properly show the process
-    process.last_status = ProcessStatus.RUNNING
-    db.session.add(process)
-    db.session.commit()
+    """
+    pstat = load_process(process)
 
-    _safe_logstep_prep = partial(safe_logstep, broadcast_func=broadcast_func)
-    return _run_process_async(pstat.process_id, lambda: runwf(pstat, _safe_logstep_prep))
+    start_func = get_execution_context()["start"]
+    return start_func(pstat, user=user, broadcast_func=broadcast_func)
 
 
-def thread_validate_workflow(validation_workflow: str, json: list[State] | None) -> UUID:
-    return thread_start_process(validation_workflow, user_inputs=json)
+RESUMABLE_STATUSES = (
+    ProcessStatus.SUSPENDED,  # Can be resumed
+    ProcessStatus.WAITING,  # Can be retried
+    ProcessStatus.FAILED,  # Can be retried
+    ProcessStatus.API_UNAVAILABLE,  # subtype of FAILED
+    ProcessStatus.INCONSISTENT_DATA,  # subtype of FAILED
+    ProcessStatus.RESUMED,  # re-resume stuck process
+)
 
 
-THREADPOOL_EXECUTION_CONTEXT: dict[str, Callable] = {
-    "start": thread_start_process,
-    "resume": thread_resume_process,
-    "validate": thread_validate_workflow,
-}
+def can_be_resumed(status: ProcessStatus) -> bool:
+    return status in RESUMABLE_STATUSES
 
 
 def resume_process(
@@ -576,14 +556,19 @@ def resume_process(
     """
     pstat = load_process(process)
 
+    if pstat.workflow == removed_workflow:
+        raise ValueError(RESUME_WORKFLOW_REMOVED_ERROR_MSG)
+
     try:
-        post_form(pstat.log[0].form, pstat.state.unwrap(), user_inputs=user_inputs or [])
+        user_input = post_form(pstat.log[0].form, pstat.state.unwrap(), user_inputs=user_inputs or [{}])
     except FormValidationError:
         logger.exception("Validation errors", user_inputs=user_inputs)
         raise
 
+    store_input_state(pstat.process_id, user_input, "user_input")
+
     resume_func = get_execution_context()["resume"]
-    return resume_func(process, user_inputs=user_inputs, user=user, broadcast_func=broadcast_func)
+    return resume_func(process, user=user, broadcast_func=broadcast_func)
 
 
 def ensure_correct_callback_token(pstat: ProcessStat, *, token: str) -> None:
@@ -809,6 +794,12 @@ def _get_running_processes() -> list[ProcessTable]:
     return list(db.session.scalars(stmt))
 
 
+def set_process_status(process: ProcessTable, status: ProcessStatus) -> None:
+    process.last_status = status
+    db.session.add(process)
+    db.session.commit()
+
+
 def marshall_processes(engine_settings: EngineSettingsTable, new_global_lock: bool) -> EngineSettingsTable | None:
     """Manage processes depending on the engine status.
 
@@ -831,6 +822,7 @@ def marshall_processes(engine_settings: EngineSettingsTable, new_global_lock: bo
 
             # Resume all the running processes
             for process in _get_running_processes():
+                set_process_status(process, ProcessStatus.RESUMED)
                 resume_process(process, user=SYSTEM_USER)
 
         elif not engine_settings.global_lock and new_global_lock:

orchestrator/services/settings_env_variables.py

@@ -14,7 +14,7 @@
 from typing import Any, Dict, Type
 
 from pydantic import SecretStr as PydanticSecretStr
-from pydantic_core import MultiHostUrl
+from pydantic_core import MultiHostUrl, Url
 from pydantic_settings import BaseSettings
 
 from orchestrator.utils.expose_settings import SecretStr as OrchSecretStr
@@ -32,21 +32,9 @@ def expose_settings(settings_name: str, base_settings: Type[BaseSettings]) -> Ty
 
 def mask_value(key: str, value: Any) -> Any:
     key_lower = key.lower()
+    is_sensitive_key = "secret" in key_lower or "password" in key_lower
 
-    if "secret" in key_lower or "password" in key_lower:
-        # Mask sensitive information
-        return MASK
-
-    if isinstance(value, PydanticSecretStr):
-        # Need to convert SecretStr to str for serialization
-        return str(value)
-
-    if isinstance(value, OrchSecretStr):
-        return MASK
-
-    # PostgresDsn is just MultiHostUrl with extra metadata (annotations)
-    if isinstance(value, MultiHostUrl):
-        # Convert PostgresDsn to str for serialization
+    if is_sensitive_key or isinstance(value, (OrchSecretStr, PydanticSecretStr, MultiHostUrl, Url)):
         return MASK
 
     return value

orchestrator/services/tasks.py

@@ -11,8 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from functools import partial
-from http import HTTPStatus
-from typing import Any
 from uuid import UUID
 
 import structlog
@@ -21,21 +19,12 @@ from celery.app.control import Inspect
 from celery.utils.log import get_task_logger
 from kombu.serialization import registry
 
-from orchestrator.api.error_handling import raise_status
 from orchestrator.schemas.engine_settings import WorkerStatus
-from orchestrator.services.input_state import retrieve_input_state
-from orchestrator.services.processes import (
-    _get_process,
-    _run_process_async,
-    ensure_correct_process_status,
-    safe_logstep,
-    thread_resume_process,
-)
+from orchestrator.services.executors.threadpool import thread_resume_process, thread_start_process
+from orchestrator.services.processes import _get_process, ensure_correct_process_status, load_process
 from orchestrator.types import BroadcastFunc
 from orchestrator.utils.json import json_dumps, json_loads
-from orchestrator.workflow import ProcessStat, ProcessStatus, Success, runwf
-from orchestrator.workflows import get_workflow
-from pydantic_forms.types import State
+from orchestrator.workflow import ProcessStatus
 
 logger = get_task_logger(__name__)
 
@@ -78,24 +67,12 @@ def initialise_celery(celery: Celery) -> None:  # noqa: C901
 
     process_broadcast_fn: BroadcastFunc | None = getattr(celery, "process_broadcast_fn", None)
 
-    def start_process(process_id: UUID, workflow_key: str, state: dict[str, Any], user: str) -> UUID | None:
+    def start_process(process_id: UUID, user: str) -> UUID | None:
         try:
-            workflow = get_workflow(workflow_key)
-
-            if not workflow:
-                raise_status(HTTPStatus.NOT_FOUND, "Workflow does not exist")
-
-            pstat = ProcessStat(
-                process_id,
-                workflow=workflow,
-                state=Success(state),
-                log=workflow.steps,
-                current_user=user,
-            )
-
+            process = _get_process(process_id)
+            pstat = load_process(process)
             ensure_correct_process_status(process_id, ProcessStatus.CREATED)
-            safe_logstep_with_func = partial(safe_logstep, broadcast_func=process_broadcast_fn)
-            process_id = _run_process_async(pstat.process_id, lambda: runwf(pstat, safe_logstep_with_func))
+            thread_start_process(pstat, user)
 
         except Exception as exc:
             local_logger.error("Worker failed to execute workflow", process_id=process_id, details=str(exc))
@@ -103,13 +80,11 @@ def initialise_celery(celery: Celery) -> None:  # noqa: C901
         else:
             return process_id
 
-    def resume_process(process_id: UUID, user_inputs: list[State] | None, user: str) -> UUID | None:
+    def resume_process(process_id: UUID, user: str) -> UUID | None:
         try:
             process = _get_process(process_id)
             ensure_correct_process_status(process_id, ProcessStatus.RESUMED)
-            process_id = thread_resume_process(
-                process, user_inputs=user_inputs, user=user, broadcast_func=process_broadcast_fn
-            )
+            thread_resume_process(process, user=user, broadcast_func=process_broadcast_fn)
         except Exception as exc:
             local_logger.error("Worker failed to resume workflow", process_id=process_id, details=str(exc))
             return None
@@ -119,28 +94,24 @@ def initialise_celery(celery: Celery) -> None:  # noqa: C901
     celery_task = partial(celery.task, log=local_logger, serializer="orchestrator-json")
 
     @celery_task(name=NEW_TASK)  # type: ignore
-    def new_task(process_id, workflow_key: str, user: str) -> UUID | None:
-        local_logger.info("Start task", process_id=process_id, workflow_key=workflow_key)
-        state = retrieve_input_state(process_id, "initial_state").input_state
-        return start_process(process_id, workflow_key, state=state, user=user)
+    def new_task(process_id: UUID, user: str) -> UUID | None:
+        local_logger.info("Start task", process_id=process_id)
+        return start_process(process_id, user=user)
 
     @celery_task(name=NEW_WORKFLOW)  # type: ignore
-    def new_workflow(process_id, workflow_key: str, user: str) -> UUID | None:
-        local_logger.info("Start workflow", process_id=process_id, workflow_key=workflow_key)
-        state = retrieve_input_state(process_id, "initial_state").input_state
-        return start_process(process_id, workflow_key, state=state, user=user)
+    def new_workflow(process_id: UUID, user: str) -> UUID | None:
+        local_logger.info("Start workflow", process_id=process_id)
+        return start_process(process_id, user=user)
 
     @celery_task(name=RESUME_TASK)  # type: ignore
     def resume_task(process_id: UUID, user: str) -> UUID | None:
         local_logger.info("Resume task", process_id=process_id)
-        state = retrieve_input_state(process_id, "user_input").input_state
-        return resume_process(process_id, user_inputs=state, user=user)
+        return resume_process(process_id, user=user)
 
     @celery_task(name=RESUME_WORKFLOW)  # type: ignore
     def resume_workflow(process_id: UUID, user: str) -> UUID | None:
         local_logger.info("Resume workflow", process_id=process_id)
-        state = retrieve_input_state(process_id, "user_input").input_state
-        return resume_process(process_id, user_inputs=state, user=user)
+        return resume_process(process_id, user=user)
 
 
 class CeleryJobWorkerStatus(WorkerStatus):

orchestrator/settings.py

@@ -72,7 +72,7 @@ class AppSettings(BaseSettings):
     TRACING_ENABLED: bool = False
     TRACE_HOST: str = "http://localhost:4317"
     TRANSLATIONS_DIR: Path | None = None
-    WEBSOCKET_BROADCASTER_URL: str = "memory://"
+    WEBSOCKET_BROADCASTER_URL: OrchSecretStr = "memory://"  # type: ignore
     ENABLE_WEBSOCKETS: bool = True
     DISABLE_INSYNC_CHECK: bool = False
     DEFAULT_PRODUCT_WORKFLOWS: list[str] = ["modify_note"]

orchestrator/workflows/tasks/resume_workflows.py

@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from uuid import UUID
 
 import structlog
 from sqlalchemy import select
@@ -24,36 +24,92 @@ from pydantic_forms.types import State, UUIDstr
 logger = structlog.get_logger(__name__)
 
 
+def get_process_ids_by_process_statuses(process_statuses: list[ProcessStatus], exclude_ids: list[UUID]) -> list:
+    return list(
+        db.session.scalars(
+            select(ProcessTable.process_id).filter(
+                ProcessTable.last_status.in_(process_statuses), ProcessTable.process_id.not_in(exclude_ids)
+            )
+        )
+    )
+
+
 @step("Find waiting workflows")
-def find_waiting_workflows() -> State:
-    waiting_processes = db.session.scalars(
-        select(ProcessTable).filter(ProcessTable.last_status == ProcessStatus.WAITING)
-    ).all()
-    waiting_process_ids = [str(process.process_id) for process in waiting_processes]
-    return {"number_of_waiting_processes": len(waiting_process_ids), "waiting_process_ids": waiting_process_ids}
+def find_waiting_workflows(process_id: UUID) -> State:
+    created_process_ids = get_process_ids_by_process_statuses([ProcessStatus.CREATED], exclude_ids=[process_id])
+    resumed_process_ids = get_process_ids_by_process_statuses([ProcessStatus.RESUMED], exclude_ids=[process_id])
+    waiting_process_ids = get_process_ids_by_process_statuses([ProcessStatus.WAITING], exclude_ids=[process_id])
+
+    return {
+        "number_of_waiting_processes": len(waiting_process_ids),
+        "waiting_process_ids": waiting_process_ids,
+        "created_processes_stuck": len(created_process_ids),
+        "created_state_process_ids": created_process_ids,
+        "resumed_processes_stuck": len(resumed_process_ids),
+        "resumed_state_process_ids": resumed_process_ids,
+    }
 
 
 @step("Resume found workflows")
-def resume_found_workflows(waiting_process_ids: list[UUIDstr]) -> State:
-    resumed_process_ids = []
-    for process_id in waiting_process_ids:
+def resume_found_workflows(
+    waiting_process_ids: list[UUIDstr],
+    resumed_state_process_ids: list[UUIDstr],
+) -> State:
+    resume_processes = waiting_process_ids + resumed_state_process_ids
+
+    resumed_process_ids: list = []
+    for process_id in resume_processes:
         try:
             process = db.session.get(ProcessTable, process_id)
             if not process:
                 continue
+
             # Workaround the commit disable function
             db.session.info["disabled"] = False
+
             processes.resume_process(process)
             resumed_process_ids.append(process_id)
-        except Exception:
-            logger.exception()
+        except Exception as exc:
+            logger.warning("Could not resume process", process_id=process_id, error=str(exc))
+        finally:
+            # Make sure to turn it on again
+            db.session.info["disabled"] = True
+
+    return {
+        "number_of_resumed_process_ids": len(resumed_process_ids),
+        "resumed_process_ids": resumed_process_ids,
+    }
+
+
+@step("Restart found CREATED workflows")
+def restart_created_workflows(created_state_process_ids: list[UUIDstr]) -> State:
+    started_process_ids = []
+    for process_id in created_state_process_ids:
+        try:
+            process = db.session.get(ProcessTable, process_id)
+            if not process:
+                continue
+
+            # Workaround the commit disable function
+            db.session.info["disabled"] = False
+
+            processes.restart_process(process)
+            started_process_ids.append(process_id)
+        except Exception as exc:
+            logger.warning("Could not resume process", process_id=process_id, error=str(exc))
         finally:
             # Make sure to turn it on again
             db.session.info["disabled"] = True
 
-    return {"number_of_resumed_process_ids": len(resumed_process_ids), "resumed_process_ids": resumed_process_ids}
+    return {
+        "number_of_started_process_ids": len(started_process_ids),
+        "started_process_ids": started_process_ids,
+    }
 
 
-@workflow("Resume all workflows that are stuck on tasks with the status 'waiting'", target=Target.SYSTEM)
+@workflow(
+    "Resume all workflows that are stuck on tasks with the status 'waiting', 'created' or 'resumed'",
+    target=Target.SYSTEM,
+)
 def task_resume_workflows() -> StepList:
-    return init >> find_waiting_workflows >> resume_found_workflows >> done
+    return init >> find_waiting_workflows >> resume_found_workflows >> restart_created_workflows >> done

{orchestrator_core-4.2.0rc2.dist-info → orchestrator_core-4.3.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 4.2.0rc2
+Version: 4.3.0rc1
 Summary: This is the orchestrator workflow engine.
 Author-email: SURF <automation-beheer@surf.nl>
 Requires-Python: >=3.11,<3.14

{orchestrator_core-4.2.0rc2.dist-info → orchestrator_core-4.3.0rc1.dist-info}/RECORD

@@ -1,10 +1,10 @@
-orchestrator/__init__.py,sha256=ISi38WvUi9c8FugLGPcfGKKTKIlvD_jMwdA7L1SAGzQ,1066
+orchestrator/__init__.py,sha256=AwwISw88tXUMDIxHF1Kl8PnfRUj8xQIIe6WpuvTlxw4,1066
 orchestrator/app.py,sha256=7UrXKjBKNSEaSSXAd5ww_RdMFhFqE4yvfj8faS2MzAA,12089
 orchestrator/exception_handlers.py,sha256=UsW3dw8q0QQlNLcV359bIotah8DYjMsj2Ts1LfX4ClY,1268
 orchestrator/log_config.py,sha256=1tPRX5q65e57a6a_zEii_PFK8SzWT0mnA5w2sKg4hh8,1853
 orchestrator/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 orchestrator/security.py,sha256=iXFxGxab54aav7oHEKLAVkTgrQMJGHy6IYLojEnD7gI,2422
-orchestrator/settings.py,sha256=TFIv09JIKY-lXqd04lH_XEcijEEyheaz3zTcgeG8DEI,4339
+orchestrator/settings.py,sha256=2Kgc6m3qUCcSM3Z_IVUeehfgO0QphMFkLrS0RC3sU-U,4365
 orchestrator/targets.py,sha256=WizBgnp8hWX9YLFUIju7ewSubiwQqinCvyiYNcXHbHI,802
 orchestrator/types.py,sha256=qzs7xx5AYRmKbpYRyJJP3wuDb0W0bcAzefCN0RWLAco,15459
 orchestrator/version.py,sha256=b58e08lxs47wUNXv0jXFO_ykpksmytuzEXD4La4W-NQ,1366
@@ -17,7 +17,7 @@ orchestrator/api/api_v1/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n
 orchestrator/api/api_v1/api.py,sha256=m4iDktsSpzxUDaudkdgXeZ83a6B4wfc3pczQsa-Pb-8,2866
 orchestrator/api/api_v1/endpoints/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
 orchestrator/api/api_v1/endpoints/health.py,sha256=iaxs1XX1_250_gKNsspuULCV2GEMBjbtjsmfQTOvMAI,1284
-orchestrator/api/api_v1/endpoints/processes.py,sha256=79_nrXEfEREc_9P-aIpcJn6idfwQzMutHsPEcyrNzBA,16223
+orchestrator/api/api_v1/endpoints/processes.py,sha256=kWz_jL8_sTNwl44tU17VwkwZGjBIw1IIW5pYCCSHwgs,15891
 orchestrator/api/api_v1/endpoints/product_blocks.py,sha256=kZ6ywIOsS_S2qGq7RvZ4KzjvaS1LmwbGWR37AKRvWOw,2146
 orchestrator/api/api_v1/endpoints/products.py,sha256=BfFtwu9dZXEQbtKxYj9icc73GKGvAGMR5ytyf41nQlQ,3081
 orchestrator/api/api_v1/endpoints/resource_types.py,sha256=gGyuaDyOD0TAVoeFGaGmjDGnQ8eQQArOxKrrk4MaDzA,2145
@@ -243,6 +243,7 @@ orchestrator/migrations/versions/schema/2025-04-09_fc5c993a4b4a_add_cascade_cons
 orchestrator/migrations/versions/schema/2025-05-08_161918133bec_add_is_task_to_workflow.py,sha256=VLFDHFYRWn5ktUba0KuSPWyvjYJdfN1WypWmOPqIW18,721
 orchestrator/migrations/versions/schema/2025-07-01_93fc5834c7e5_changed_timestamping_fields_in_process_steps.py,sha256=Oezd8b2qaI1Kyq-sZFVFmdzd4d9NjXrf6HtJGk11fy0,1914
 orchestrator/migrations/versions/schema/2025-07-04_4b58e336d1bf_deprecating_workflow_target_in_.py,sha256=xnD6w-97R4ClS7rbmXQEXc36K3fdcXKhCy7ZZNy_FX4,742
+orchestrator/migrations/versions/schema/2025-07-28_850dccac3b02_update_description_of_resume_workflows_.py,sha256=R6Qoga83DJ1IL0WYPu0u5u2ZvAmqGlDmUMv_KtJyOhQ,812
 orchestrator/schedules/__init__.py,sha256=JnnaglfK1qYUBKI6Dd9taV-tCZIPlAdAkHtnkJDMXxY,1066
 orchestrator/schedules/resume_workflows.py,sha256=kSotzTAXjX7p9fpSYiGOpuxuTQfv54eRFAe0YSG0DHc,832
 orchestrator/schedules/scheduling.py,sha256=ehtwgpbvMOk1jhn-hHgVzg_9wLJkI6l3mRY3DcO9ZVY,1526
@@ -262,20 +263,22 @@ orchestrator/schemas/subscription.py,sha256=-jXyHZIed9Xlia18ksSDyenblNN6Q2yM2FlG
 orchestrator/schemas/subscription_descriptions.py,sha256=Ft_jw1U0bf9Z0U8O4OWfLlcl0mXCVT_qYVagBP3GbIQ,1262
 orchestrator/schemas/workflow.py,sha256=VqQ9XfV4fVd6MjY0LRRQzWBJHmlPsAamWfTwDx1cZkg,2102
 orchestrator/services/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
-orchestrator/services/celery.py,sha256=PsIgRBJsmA3vKwAUaqPq9ynLwDsXHY2ggDWc-nQAwgM,5232
 orchestrator/services/fixed_inputs.py,sha256=kyz7s2HLzyDulvcq-ZqefTw1om86COvyvTjz0_5CmgI,876
-orchestrator/services/input_state.py,sha256=HF7wl9fWdaAW8pdCCqbuYoKyNj8dY0g8Ff8vXis8z5A,2211
+orchestrator/services/input_state.py,sha256=WfiQrvatPtE1jR8xArE0XamONt1nemTv5iaROgSiJpM,2389
 orchestrator/services/process_broadcast_thread.py,sha256=D44YbjF8mRqGuznkRUV4SoRn1J0lfy_x1H508GnSVlU,4649
-orchestrator/services/processes.py,sha256=W-MCuGxRXLNIx3zn_jQQWXXFIBUrjJgXyBMsx2E0FbQ,30490
+orchestrator/services/processes.py,sha256=JGM9vWbUjvEpy-IpTIgaYaqcTBKMI-CWTY8SJKBf3eI,30153
 orchestrator/services/products.py,sha256=BP4KyE8zO-8z7Trrs5T6zKBOw53S9BfBJnHWI3p6u5Y,1943
 orchestrator/services/resource_types.py,sha256=_QBy_JOW_X3aSTqH0CuLrq4zBJL0p7Q-UDJUcuK2_qc,884
 orchestrator/services/settings.py,sha256=HEWfFulgoEDwgfxGEO__QTr5fDiwNBEj1UhAeTAdbLQ,3159
-orchestrator/services/settings_env_variables.py,sha256=U2xz0h3bOAE8wWQF2iCp5jf-2v0NUOOCTecNTouB1jY,2496
+orchestrator/services/settings_env_variables.py,sha256=iPErQjqPQCxKs0sPhefB16d8SBBVUi6eiRnFBK5bgqA,2196
 orchestrator/services/subscription_relations.py,sha256=9C126TUfFvyBe7y4x007kH_dvxJ9pZ1zSnaWeH6HC5k,12261
 orchestrator/services/subscriptions.py,sha256=nr2HI89nC0lYjzTh2j-lEQ5cPQK43LNZv3gvP6jbepw,27189
-orchestrator/services/tasks.py,sha256=NjPkuauQoh9UJDcjA7OcKFgPk0i6NoKdDO7HlpGbBJ8,6575
+orchestrator/services/tasks.py,sha256=Mq-C7EDrkv-ohhzD9H03Oh540heL19dxvyauHi_0pNI,5246
 orchestrator/services/translations.py,sha256=GyP8soUFGej8AS8uulBsk10CCK6Kwfjv9AHMFm3ElQY,1713
 orchestrator/services/workflows.py,sha256=iEkt2OBuTwkDru4V6ZSKatnw0b96ZdPV-VQqeZ9EOgU,4015
+orchestrator/services/executors/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
+orchestrator/services/executors/celery.py,sha256=j5xJo7sZAdTtc0GmmJzoYVfzuYKiqAdAe5QbtPv0bPI,4937
+orchestrator/services/executors/threadpool.py,sha256=SA0Lns17fP7qp5Y0bLZB7YzZ-sYKrmHQdYTeqs9dnV0,4931
 orchestrator/utils/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
 orchestrator/utils/auth.py,sha256=IWn0amdquLobt1mRNwhgKT0ErCBjLGDtLdsDuaY8rlE,309
 orchestrator/utils/crypt.py,sha256=18eNamYWMllPkxyRtWIde3FDr3rSF74R5SAL6WsCj9Y,5584
@@ -308,11 +311,11 @@ orchestrator/workflows/steps.py,sha256=CZxfzkG5ANJYwuYTkQ4da2RpQqIjXCtey_Uy1ezRA
 orchestrator/workflows/utils.py,sha256=bhX9vm3oc9k6RSaESl34v4Nrh40G4Ys91INoTjZ0XVM,13966
 orchestrator/workflows/tasks/__init__.py,sha256=GyHNfEFCGKQwRiN6rQmvSRH2iYX7npjMZn97n8XzmLU,571
 orchestrator/workflows/tasks/cleanup_tasks_log.py,sha256=BfWYbPXhnLAHUJ0mlODDnjZnQQAvKCZJDVTwbwOWI04,1624
-orchestrator/workflows/tasks/resume_workflows.py,sha256=MzJqlSXUvKStkT7NGzxZyRlfAer_ezYm-kjUqaZi0yc,2359
+orchestrator/workflows/tasks/resume_workflows.py,sha256=T3iobSJjVgiupe0rClD34kUZ7KF4pL5yK2AVeRLZog8,4313
 orchestrator/workflows/tasks/validate_product_type.py,sha256=paG-NAY1bdde3Adt8zItkcBKf5Pxw6f5ngGW6an6dYU,3192
 orchestrator/workflows/tasks/validate_products.py,sha256=GZJBoFF-WMphS7ghMs2-gqvV2iL1F0POhk0uSNt93n0,8510
 orchestrator/workflows/translations/en-GB.json,sha256=ST53HxkphFLTMjFHonykDBOZ7-P_KxksktZU3GbxLt0,846
-orchestrator_core-4.2.0rc2.dist-info/licenses/LICENSE,sha256=b-aA5OZQuuBATmLKo_mln8CQrDPPhg3ghLzjPjLn4Tg,11409
-orchestrator_core-4.2.0rc2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-orchestrator_core-4.2.0rc2.dist-info/METADATA,sha256=0DZbEp0UG3r_U8GA73q2lQqFJS9AQp7Q8UeGDXb0BJ8,5963
-orchestrator_core-4.2.0rc2.dist-info/RECORD,,
+orchestrator_core-4.3.0rc1.dist-info/licenses/LICENSE,sha256=b-aA5OZQuuBATmLKo_mln8CQrDPPhg3ghLzjPjLn4Tg,11409
+orchestrator_core-4.3.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+orchestrator_core-4.3.0rc1.dist-info/METADATA,sha256=LjswuvIeH5R2RgHE84Fr0MhOMfg9lo9g_F9YtTcyjbE,5963
+orchestrator_core-4.3.0rc1.dist-info/RECORD,,