orchestrator-core 3.0.0rc1__tar.gz → 3.1.0rc1__tar.gz

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.0.0rc1
+current_version = 3.1.0rc1
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(rc(?P<build>\d+))?

orchestrator_core-3.1.0rc1/.github/workflows/run-codspeed-tests.yml

@@ -0,0 +1,71 @@
+name: CodSpeed
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  codspeed:
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.11
+      options: --privileged
+    services:
+      postgres:
+        image: postgres:15-alpine
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: nwa
+          POSTGRES_USER: nwa
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        # Docker Hub image
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      # Downloads a copy of the code in your repository before running CI tests
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install curl git build-essential libpq-dev libffi-dev -y
+          python -m pip install --upgrade pip
+          pip install flit
+          flit install --deps develop --symlink
+          echo "GIT_COMMIT_HASH=\"test\"" > orchestrator/version.py
+        env:
+          FLIT_ROOT_INSTALL: 1
+
+      # Prevent error "repository path is not owned by the current user"
+      - name: Fix git owner
+        run: git config --global --add safe.directory "*"
+
+      # Speculatively add the cargo binary directory to the PATH because codspeed's installer script somehow doesn't
+      - name: Add $HOME/.cargo/bin to PATH
+        run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+
+      - uses: CodSpeedHQ/action@v3
+        with:
+          run: CACHE_URI=redis://redis DATABASE_URI=postgresql+psycopg://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB pytest test/unit_tests --codspeed
+          token: ${{ secrets.CODSPEED_TOKEN }}
+        env:
+          POSTGRES_DB: orchestrator-core-test
+          POSTGRES_USER: nwa
+          POSTGRES_PASSWORD: nwa
+          POSTGRES_HOST: postgres
+          ENVIRONMENT: TESTING

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/.github/workflows/run-unit-tests.yml

@@ -51,7 +51,7 @@ jobs:
         env:
           FLIT_ROOT_INSTALL: 1
       - name: Run Unit tests
-        run: CACHE_URI=redis://redis DATABASE_URI=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB  pytest --cov-branch --cov=orchestrator --cov-report=xml --ignore=test --ignore=orchestrator/devtools --ignore=examples --ignore=docs --ignore=orchestrator/vendor
+        run: CACHE_URI=redis://redis DATABASE_URI=postgresql+psycopg://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB  pytest --cov-branch --cov=orchestrator --cov-report=xml --ignore=test --ignore=orchestrator/devtools --ignore=examples --ignore=docs --ignore=orchestrator/vendor
         env:
           POSTGRES_DB: orchestrator-core-test
           POSTGRES_USER: nwa

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 3.0.0rc1
+Version: 3.1.0rc1
 Summary: This is the orchestrator workflow engine.
 Requires-Python: >=3.11,<3.14
 Classifier: Intended Audience :: Information Technology
@@ -27,26 +27,27 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
 Classifier: Topic :: Internet :: WWW/HTTP
+License-File: LICENSE
 Requires-Dist: alembic==1.14.1
 Requires-Dist: anyio>=3.7.0
 Requires-Dist: click==8.*
 Requires-Dist: deprecated
-Requires-Dist: deepmerge==1.1.1
+Requires-Dist: deepmerge==2.0
 Requires-Dist: fastapi~=0.115.2
 Requires-Dist: fastapi-etag==0.4.0
 Requires-Dist: more-itertools~=10.6.0
 Requires-Dist: itsdangerous
 Requires-Dist: Jinja2==3.1.5
 Requires-Dist: orjson==3.10.15
-Requires-Dist: psycopg2-binary==2.9.10
+Requires-Dist: psycopg[binary]==3.2.5
 Requires-Dist: pydantic[email]~=2.8.2
-Requires-Dist: pydantic-settings~=2.7.1
+Requires-Dist: pydantic-settings~=2.8.0
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: python-rapidjson>=1.18,<1.21
-Requires-Dist: pytz==2024.1
+Requires-Dist: pytz==2025.1
 Requires-Dist: redis==5.0.3
 Requires-Dist: schedule==1.1.0
-Requires-Dist: sentry-sdk[fastapi]~=2.18.0
+Requires-Dist: sentry-sdk[fastapi]~=2.22.0
 Requires-Dist: SQLAlchemy==2.0.38
 Requires-Dist: SQLAlchemy-Utils==0.41.2
 Requires-Dist: structlog
@@ -56,7 +57,7 @@ Requires-Dist: nwa-stdlib~=1.9.0
 Requires-Dist: oauth2-lib~=2.4.0
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: strawberry-graphql>=0.246.2
-Requires-Dist: pydantic-forms~=1.2.1
+Requires-Dist: pydantic-forms~=1.3.0
 Requires-Dist: celery~=5.4.0 ; extra == "celery"
 Requires-Dist: toml ; extra == "dev"
 Requires-Dist: bumpversion ; extra == "dev"
@@ -81,8 +82,9 @@ Requires-Dist: dirty-equals ; extra == "test"
 Requires-Dist: jsonref ; extra == "test"
 Requires-Dist: mypy==1.9 ; extra == "test"
 Requires-Dist: pyinstrument ; extra == "test"
-Requires-Dist: pytest==8.3.2 ; extra == "test"
+Requires-Dist: pytest==8.3.4 ; extra == "test"
 Requires-Dist: pytest-asyncio==0.21.2 ; extra == "test"
+Requires-Dist: pytest-codspeed ; extra == "test"
 Requires-Dist: pytest-cov ; extra == "test"
 Requires-Dist: pytest-httpx ; extra == "test"
 Requires-Dist: pytest-xdist ; extra == "test"

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/README.md

@@ -191,3 +191,10 @@ You can do the necessary change with a clean, e.g. every change committed, branc
 ```shell
 bumpversion patch --new-version 0.4.1-rc3
 ```
+
+### Changing the Core database schema
+When you would like to change the core database schema, execute the following steps.
+
+- Create the new model `orchestrator/database/models.py`
+- `cd orchestrator/migrations`
+- `alembic revision --autogenerate -m "Name of the migratioin"`

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/docs/contributing/guidelines.md

@@ -6,6 +6,20 @@ free to raise an issue in the project. We will strive to reply to your enquiry A
 ## Documentation
 We use [**MKDOCS**](https://www.mkdocs.org) as a documentation tool. Please create a PR if you have any additions or contributions to make. All docs can be written in MD or html. Full guidelines on how to set this up can be found [here](development.md).
 
+## Pre-commit hooks
+We use pre-commit hooks to ensure that the code is formatted correctly and that the tests pass. To install the
+pre-commit hooks, run the following command:
+
+```shell
+pre-commit install
+```
+
+To run the pre-commit hooks manually, run the following command:
+
+```shell
+pre-commit run --all-files
+```
+
 ## Orchestrator release
 The `orchestrator-core` has no release schedule but is actively used and maintained by the workflow orchestrator group.
 Creating a new release is done by the developers of the project and the procedure is as follows.

orchestrator_core-3.1.0rc1/docs/migration-guide/3.0.md

@@ -0,0 +1,18 @@
+---
+hide:
+  - navigation
+---
+# 3.0 Migration Guide
+
+In this document we describe the steps that should be taken to migrate from `orchestrator-core` v2 to v3.
+
+## About 3.0
+
+In this release, deprecated import statements from the `orchestrator.types` module are removed, that now come from
+`pydantic-forms.types` instead. These will have to be updated in your implementation of the orchestrator as well.
+
+## Steps
+
+To update the import statements you may have in your implementation of Workflow Orchestrator, we offer a migration
+script that can be run as follows: `python -m orchestrator.devtools.scripts.migrate_30 <dir>` where `<dir>` points to
+your orchestrator implementation.

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/mkdocs.yml

@@ -189,7 +189,9 @@ nav:
                 # - Workflow Lifecycles: reference-docs/workflows/workflow-lifecycles.md
                 - Callbacks: reference-docs/workflows/callbacks.md
           - Websockets: reference-docs/websockets.md
-          - Migration guide: migration-guide/2.0.md
+          - Migration guides:
+                - 2.x: migration-guide/2.0.md
+                - 3.x: migration-guide/3.0.md
 
     - Workshops:
           #    - Beginner:

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "3.0.0rc1"
+__version__ = "3.1.0rc1"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/orchestrator/api/api_v1/endpoints/settings.py

@@ -28,6 +28,7 @@ from orchestrator.services import processes, settings
 from orchestrator.settings import ExecutorType, app_settings
 from orchestrator.utils.json import json_dumps
 from orchestrator.utils.redis import delete_keys_matching_pattern
+from orchestrator.utils.redis_client import create_redis_asyncio_client
 from orchestrator.websocket import WS_CHANNELS, broadcast_invalidate_cache, websocket_manager
 
 router = APIRouter()
@@ -41,7 +42,7 @@ CACHE_FLUSH_OPTIONS: dict[str, str] = {
 
 @router.delete("/cache/{name}")
 async def clear_cache(name: str) -> int | None:
-    cache: AIORedis = AIORedis.from_url(str(app_settings.CACHE_URI))
+    cache: AIORedis = create_redis_asyncio_client(app_settings.CACHE_URI)
     if name not in CACHE_FLUSH_OPTIONS:
         raise_status(HTTPStatus.BAD_REQUEST, "Invalid cache name")
 

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/orchestrator/cli/generator/generator/migration.py

@@ -31,13 +31,16 @@ from orchestrator.cli.generator.generator.helpers import (
     sort_product_blocks_by_dependencies,
 )
 from orchestrator.cli.generator.generator.settings import product_generator_settings as settings
+from orchestrator.settings import convert_database_uri
 
 logger = structlog.getLogger(__name__)
 
 
 def create_migration_file(message: str, head: str) -> Path | None:
-    if not environ.get("DATABASE_URI"):
-        environ.update({"DATABASE_URI": "postgresql://nwa:nwa@localhost/orchestrator-core"})
+    if environ.get("DATABASE_URI"):
+        environ.update({"DATABASE_URI": convert_database_uri(environ["DATABASE_URI"])})
+    else:
+        environ.update({"DATABASE_URI": "postgresql+psycopg://nwa:nwa@localhost/orchestrator-core"})
     if not environ.get("PYTHONPATH"):
         environ.update({"PYTHONPATH": "."})
     logger.info(

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/orchestrator/db/models.py

@@ -13,6 +13,7 @@
 
 from __future__ import annotations
 
+import enum
 from datetime import datetime, timezone
 
 import sqlalchemy
@@ -23,6 +24,7 @@ from sqlalchemy import (
     Boolean,
     CheckConstraint,
     Column,
+    Enum,
     ForeignKey,
     Index,
     Integer,
@@ -81,6 +83,20 @@ class UtcTimestamp(TypeDecorator):
         return value.astimezone(timezone.utc) if value else value
 
 
+class InputStateTable(BaseModel):
+    __tablename__ = "input_states"
+
+    class InputType(enum.Enum):
+        user_input = "user_input"
+        initial_state = "initial_state"
+
+    input_state_id = mapped_column(UUIDType, primary_key=True, server_default=text("uuid_generate_v4()"), index=True)
+    process_id = mapped_column("pid", UUIDType, ForeignKey("processes.pid"), nullable=False)
+    input_state = mapped_column(pg.JSONB(), nullable=False)  # type: ignore
+    input_time = mapped_column(UtcTimestamp, server_default=text("current_timestamp()"), nullable=False)
+    input_type = mapped_column(Enum(InputType), nullable=False)
+
+
 class ProcessTable(BaseModel):
     __tablename__ = "processes"
 
@@ -101,6 +117,7 @@ class ProcessTable(BaseModel):
     steps = relationship(
         "ProcessStepTable", cascade="delete", passive_deletes=True, order_by="asc(ProcessStepTable.executed_at)"
     )
+    input_states = relationship("InputStateTable", cascade="delete", order_by="desc(InputStateTable.input_time)")
     process_subscriptions = relationship("ProcessSubscriptionTable", back_populates="process", passive_deletes=True)
     workflow = relationship("WorkflowTable", back_populates="processes")
 

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/orchestrator/distlock/managers/redis_distlock_manager.py

@@ -20,6 +20,7 @@ from redis.lock import Lock as SyncLock
 from structlog import get_logger
 
 from orchestrator.settings import app_settings
+from orchestrator.utils.redis_client import create_redis_asyncio_client, create_redis_client
 
 logger = get_logger(__name__)
 
@@ -37,7 +38,7 @@ class RedisDistLockManager:
         self.redis_address = redis_address
 
     async def connect_redis(self) -> None:
-        self.redis_conn = AIORedis.from_url(str(self.redis_address))
+        self.redis_conn = create_redis_asyncio_client(self.redis_address)
 
     async def disconnect_redis(self) -> None:
         if self.redis_conn:
@@ -78,7 +79,7 @@ class RedisDistLockManager:
     def release_sync(self, lock: Lock) -> None:
         redis_conn: Redis | None = None
         try:
-            redis_conn = Redis.from_url(str(app_settings.CACHE_URI))
+            redis_conn = create_redis_client(app_settings.CACHE_URI)
             sync_lock: SyncLock = SyncLock(
                 redis=redis_conn,
                 name=lock.name,  # type: ignore

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/orchestrator/graphql/resolvers/settings.py

@@ -21,6 +21,7 @@ from orchestrator.services.processes import SYSTEM_USER, ThreadPoolWorkerStatus,
 from orchestrator.services.settings import get_engine_settings, get_engine_settings_for_update, post_update_to_slack
 from orchestrator.settings import ExecutorType, app_settings
 from orchestrator.utils.redis import delete_keys_matching_pattern
+from orchestrator.utils.redis_client import create_redis_asyncio_client
 
 logger = structlog.get_logger(__name__)
 
@@ -57,7 +58,7 @@ def resolve_settings(info: OrchestratorInfo) -> StatusType:
 
 # Mutations
 async def clear_cache(info: OrchestratorInfo, name: str) -> CacheClearSuccess | Error:
-    cache: AIORedis = AIORedis.from_url(str(app_settings.CACHE_URI))
+    cache: AIORedis = create_redis_asyncio_client(app_settings.CACHE_URI)
     if name not in CACHE_FLUSH_OPTIONS:
         return Error(message="Invalid cache name")
 

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/orchestrator/graphql/schemas/product.py

@@ -1,11 +1,11 @@
-from typing import TYPE_CHECKING, Annotated
+from typing import TYPE_CHECKING, Annotated, Iterable
 
 import strawberry
 from strawberry import UNSET
 from strawberry.federation.schema_directives import Key
 
 from oauth2_lib.strawberry import authenticated_field
-from orchestrator.db import ProductTable
+from orchestrator.db import ProductBlockTable, ProductTable
 from orchestrator.domain.base import ProductModel
 from orchestrator.graphql.pagination import Connection
 from orchestrator.graphql.schemas.fixed_input import FixedInput
@@ -51,6 +51,23 @@ class ProductType:
         filter_by_with_related_subscriptions = (filter_by or []) + [GraphqlFilter(field="product", value=self.name)]
         return await resolve_subscriptions(info, filter_by_with_related_subscriptions, sort_by, first, after)
 
+    @strawberry.field(description="Returns list of all nested productblock names")  # type: ignore
+    async def all_pb_names(self) -> list[str]:
+
+        model = get_original_model(self, ProductTable)
+
+        def get_all_pb_names(product_blocks: list[ProductBlockTable]) -> Iterable[str]:
+            for product_block in product_blocks:
+                yield product_block.name
+
+                if product_block.depends_on:
+                    yield from get_all_pb_names(product_block.depends_on)
+
+        names: list[str] = list(get_all_pb_names(model.product_blocks))
+        names.sort()
+
+        return names
+
     @strawberry.field(description="Return product blocks")  # type: ignore
     async def product_blocks(self) -> list[Annotated["ProductBlock", strawberry.lazy(".product_block")]]:
         from orchestrator.graphql.schemas.product_block import ProductBlock

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/orchestrator/migrations/helpers.py

@@ -880,10 +880,10 @@ def delete_product(conn: sa.engine.Connection, name: str) -> None:
                 RETURNING  product_id
             ),
             deleted_p_pb AS (
-                DELETE FROM product_product_blocks WHERE product_id IN (SELECT product_id FROM deleted_p)
+                DELETE FROM product_product_blocks WHERE product_id = ANY(SELECT product_id FROM deleted_p)
             ),
             deleted_pb_rt AS (
-                DELETE FROM products_workflows WHERE product_id IN (SELECT product_id FROM deleted_p)
+                DELETE FROM products_workflows WHERE product_id = ANY(SELECT product_id FROM deleted_p)
             )
             SELECT * from deleted_p;
             """
@@ -911,10 +911,10 @@ def delete_product_block(conn: sa.engine.Connection, name: str) -> None:
                 RETURNING  product_block_id
             ),
             deleted_p_pb AS (
-                DELETE FROM product_product_blocks WHERE product_block_id IN (SELECT product_block_id FROM deleted_pb)
+                DELETE FROM product_product_blocks WHERE product_block_id =ANY(SELECT product_block_id FROM deleted_pb)
             ),
             deleted_pb_rt AS (
-                DELETE FROM product_block_resource_types WHERE product_block_id IN (SELECT product_block_id FROM deleted_pb)
+                DELETE FROM product_block_resource_types WHERE product_block_id =ANY(SELECT product_block_id FROM deleted_pb)
             )
             SELECT * from deleted_pb;
             """
@@ -968,7 +968,7 @@ def delete_resource_type(conn: sa.engine.Connection, resource_type: str) -> None
                 RETURNING  resource_type_id
             ),
             deleted_pb_rt AS (
-                DELETE FROM product_block_resource_types WHERE resource_type_id IN (SELECT resource_type_id FROM deleted_pb)
+                DELETE FROM product_block_resource_types WHERE resource_type_id =ANY(SELECT resource_type_id FROM deleted_pb)
             )
             SELECT * from deleted_pb;
             """

orchestrator_core-3.1.0rc1/orchestrator/migrations/versions/schema/2025-02-12_bac6be6f2b4f_added_input_state_table.py

@@ -0,0 +1,56 @@
+"""Added Input State Table.
+
+Revision ID: bac6be6f2b4f
+Revises: 4fjdn13f83ga
+Create Date: 2025-02-12 14:39:53.664284
+
+"""
+
+import sqlalchemy as sa
+import sqlalchemy_utils
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+from orchestrator import db
+
+# revision identifiers, used by Alembic.
+revision = "bac6be6f2b4f"
+down_revision = "4fjdn13f83ga"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "input_states",
+        sa.Column(
+            "input_state_id",
+            sqlalchemy_utils.types.uuid.UUIDType(),
+            server_default=sa.text("uuid_generate_v4()"),
+            nullable=False,
+        ),
+        sa.Column("pid", sqlalchemy_utils.types.uuid.UUIDType(), nullable=False),
+        sa.Column("input_state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),  # type: ignore
+        sa.Column(
+            "input_time",
+            db.models.UtcTimestamp(timezone=True),
+            server_default=sa.text("current_timestamp"),
+            nullable=False,
+        ),
+        sa.Column("input_type", sa.Enum("user_input", "initial_state", name="inputtype"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["pid"],
+            ["processes.pid"],
+        ),
+        sa.PrimaryKeyConstraint("input_state_id"),
+    )
+    op.create_index(op.f("ix_input_state_input_state_id"), "input_states", ["input_state_id"], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_input_state_input_state_id"), table_name="input_states")
+    op.drop_table("input_statse")
+    # ### end Alembic commands ###

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/orchestrator/services/celery.py

@@ -22,6 +22,7 @@ from kombu.exceptions import ConnectionError, OperationalError
 from orchestrator import app_settings
 from orchestrator.api.error_handling import raise_status
 from orchestrator.db import ProcessTable, db
+from orchestrator.services.input_state import store_input_state
 from orchestrator.services.processes import create_process, delete_process
 from orchestrator.targets import Target
 from orchestrator.workflows import get_workflow
@@ -53,9 +54,8 @@ def _celery_start_process(
     task_name = NEW_TASK if workflow.target == Target.SYSTEM else NEW_WORKFLOW
     trigger_task = get_celery_task(task_name)
     pstat = create_process(workflow_key, user_inputs, user)
-    tasks = pstat.state.s
     try:
-        result = trigger_task.delay(pstat.process_id, workflow_key, tasks, user)
+        result = trigger_task.delay(pstat.process_id, workflow_key, user)
         _block_when_testing(result)
         return pstat.process_id
     except (ConnectionError, OperationalError) as e:
@@ -82,9 +82,12 @@ def _celery_resume_process(
 
     task_name = RESUME_TASK if workflow.target == Target.SYSTEM else RESUME_WORKFLOW
     trigger_task = get_celery_task(task_name)
+
+    user_inputs = user_inputs or [{}]
+    store_input_state(pstat.process_id, user_inputs, "user_input")
     try:
         _celery_set_process_status_resumed(process)
-        result = trigger_task.delay(pstat.process_id, user_inputs, user)
+        result = trigger_task.delay(pstat.process_id, user)
         _block_when_testing(result)
 
         return pstat.process_id

orchestrator_core-3.1.0rc1/orchestrator/services/input_state.py

@@ -0,0 +1,76 @@
+# Copyright 2019-2025 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Literal
+from uuid import UUID
+
+import structlog
+from sqlalchemy import select
+
+from orchestrator.db import db
+from orchestrator.db.models import InputStateTable
+
+logger = structlog.get_logger(__name__)
+
+InputType = Literal["initial_state", "user_input"]
+
+
+def retrieve_input_state(process_id: UUID, input_type: InputType) -> InputStateTable:
+    """Get user input.
+
+    Args:
+        process_id: Process ID
+        input_type: The type of the input.
+
+    Returns:
+        User input table
+
+    """
+
+    res: InputStateTable | None = db.session.scalars(
+        select(InputStateTable)
+        .filter(InputStateTable.process_id == process_id)
+        .filter(InputStateTable.input_type == input_type)
+        .order_by(InputStateTable.input_time.asc())
+    ).first()
+
+    if res:
+        logger.debug("Retrieved input state", process_id=process_id, input_state=res, input_type=input_type)
+        return res
+    raise ValueError(f"No input state for pid: {process_id}")
+
+
+def store_input_state(
+    process_id: UUID,
+    input_state: dict[str, Any] | list[dict[str, Any]],
+    input_type: InputType,
+) -> None:
+    """Store user input state.
+
+    Args:
+        process_id: Process ID
+        input_state: Dictionary of user input state
+        input_type: The type of the input.
+
+    Returns:
+        None
+
+    """
+    logger.debug("Store input state", process_id=process_id, input_state=input_state, input_type=input_type)
+    db.session.add(
+        InputStateTable(
+            process_id=process_id,
+            input_state=input_state,
+            input_type=input_type,
+        )
+    )
+    db.session.commit()

{orchestrator_core-3.0.0rc1 → orchestrator_core-3.1.0rc1}/orchestrator/services/processes.py

@@ -18,7 +18,7 @@ from typing import Any
 from uuid import UUID, uuid4
 
 import structlog
-from deepmerge import Merger
+from deepmerge.merger import Merger
 from sqlalchemy import delete, select
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import joinedload
@@ -35,6 +35,7 @@ from orchestrator.db import (
 )
 from orchestrator.distlock import distlock_manager
 from orchestrator.schemas.engine_settings import WorkerStatus
+from orchestrator.services.input_state import store_input_state
 from orchestrator.services.settings import get_engine_settings_for_update
 from orchestrator.services.workflows import get_workflow_by_name
 from orchestrator.settings import ExecutorType, app_settings
@@ -450,7 +451,7 @@ def create_process(
     )
 
     _db_create_process(pstat)
-
+    store_input_state(process_id, state | initial_state, "initial_state")
     return pstat
 
 
@@ -514,7 +515,7 @@ def thread_resume_process(
 
     if user_input:
         pstat.update(state=pstat.state.map(lambda state: StateMerger.merge(state, user_input)))
-
+    store_input_state(pstat.process_id, user_input, "user_input")
     # enforce an update to the process status to properly show the process
     process.last_status = ProcessStatus.RUNNING
     db.session.add(process)
@@ -529,8 +530,8 @@ def thread_validate_workflow(validation_workflow: str, json: list[State] | None)
 
 
 THREADPOOL_EXECUTION_CONTEXT: dict[str, Callable] = {
-    "start": lambda *args, **kwargs: thread_start_process(*args, **kwargs),
-    "resume": lambda *args, **kwargs: thread_resume_process(*args, **kwargs),
+    "start": thread_start_process,
+    "resume": thread_resume_process,
     "validate": thread_validate_workflow,
 }