orchestrator-core 2.10.0rc1__py3-none-any.whl → 3.0.0__py3-none-any.whl

orchestrator/services/processes.py

@@ -18,7 +18,7 @@ from typing import Any
 from uuid import UUID, uuid4
 
 import structlog
-from deepmerge import Merger
+from deepmerge.merger import Merger
 from sqlalchemy import delete, select
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import joinedload
@@ -35,11 +35,12 @@ from orchestrator.db import (
 )
 from orchestrator.distlock import distlock_manager
 from orchestrator.schemas.engine_settings import WorkerStatus
+from orchestrator.services.input_state import store_input_state
 from orchestrator.services.settings import get_engine_settings_for_update
 from orchestrator.services.workflows import get_workflow_by_name
 from orchestrator.settings import ExecutorType, app_settings
 from orchestrator.targets import Target
-from orchestrator.types import BroadcastFunc, State
+from orchestrator.types import BroadcastFunc
 from orchestrator.utils.datetime import nowtz
 from orchestrator.utils.errors import error_state_to_dict
 from orchestrator.websocket import broadcast_invalidate_status_counts
@@ -60,6 +61,7 @@ from orchestrator.workflows import get_workflow
 from orchestrator.workflows.removed_workflow import removed_workflow
 from pydantic_forms.core import post_form
 from pydantic_forms.exceptions import FormValidationError
+from pydantic_forms.types import State
 
 logger = structlog.get_logger(__name__)
 
@@ -449,7 +451,7 @@ def create_process(
     )
 
     _db_create_process(pstat)
-
+    store_input_state(process_id, state | initial_state, "initial_state")
     return pstat
 
 
@@ -513,7 +515,7 @@ def thread_resume_process(
 
     if user_input:
         pstat.update(state=pstat.state.map(lambda state: StateMerger.merge(state, user_input)))
-
+    store_input_state(pstat.process_id, user_input, "user_input")
     # enforce an update to the process status to properly show the process
     process.last_status = ProcessStatus.RUNNING
     db.session.add(process)
@@ -528,8 +530,8 @@ def thread_validate_workflow(validation_workflow: str, json: list[State] | None)
 
 
 THREADPOOL_EXECUTION_CONTEXT: dict[str, Callable] = {
-    "start": lambda *args, **kwargs: thread_start_process(*args, **kwargs),
-    "resume": lambda *args, **kwargs: thread_resume_process(*args, **kwargs),
+    "start": thread_start_process,
+    "resume": thread_resume_process,
     "validate": thread_validate_workflow,
 }
 

orchestrator/services/products.py

@@ -18,7 +18,7 @@ from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 
 from orchestrator.db import ProductTable, db
-from orchestrator.types import UUIDstr
+from pydantic_forms.types import UUIDstr
 
 
 def get_products(*, filters: list | None = None) -> list[ProductTable]:

orchestrator/services/subscriptions.py

@@ -43,9 +43,10 @@ from orchestrator.db.models import (
 )
 from orchestrator.domain.base import SubscriptionModel
 from orchestrator.targets import Target
-from orchestrator.types import SubscriptionLifecycle, UUIDstr
+from orchestrator.types import SubscriptionLifecycle
 from orchestrator.utils.datetime import nowtz
 from orchestrator.utils.helpers import is_ipaddress_type
+from pydantic_forms.types import UUIDstr
 
 logger = structlog.get_logger(__name__)
 

orchestrator/services/tasks.py

@@ -23,6 +23,7 @@ from kombu.serialization import registry
 
 from orchestrator.api.error_handling import raise_status
 from orchestrator.schemas.engine_settings import WorkerStatus
+from orchestrator.services.input_state import retrieve_input_state
 from orchestrator.services.processes import (
     _get_process,
     _run_process_async,
@@ -30,10 +31,11 @@ from orchestrator.services.processes import (
     safe_logstep,
     thread_resume_process,
 )
-from orchestrator.types import BroadcastFunc, State
+from orchestrator.types import BroadcastFunc
 from orchestrator.utils.json import json_dumps, json_loads
 from orchestrator.workflow import ProcessStat, ProcessStatus, Success, runwf
 from orchestrator.workflows import get_workflow
+from pydantic_forms.types import State
 
 logger = get_task_logger(__name__)
 
@@ -117,24 +119,28 @@ def initialise_celery(celery: Celery) -> None:  # noqa: C901
     celery_task = partial(celery.task, log=local_logger, serializer="orchestrator-json")
 
     @celery_task(name=NEW_TASK)  # type: ignore
-    def new_task(process_id, workflow_key: str, state: dict[str, Any], user: str) -> UUID | None:
+    def new_task(process_id, workflow_key: str, user: str) -> UUID | None:
         local_logger.info("Start task", process_id=process_id, workflow_key=workflow_key)
+        state = retrieve_input_state(process_id, "initial_state").input_state
         return start_process(process_id, workflow_key, state=state, user=user)
 
     @celery_task(name=NEW_WORKFLOW)  # type: ignore
-    def new_workflow(process_id, workflow_key: str, state: dict[str, Any], user: str) -> UUID | None:
+    def new_workflow(process_id, workflow_key: str, user: str) -> UUID | None:
         local_logger.info("Start workflow", process_id=process_id, workflow_key=workflow_key)
+        state = retrieve_input_state(process_id, "initial_state").input_state
         return start_process(process_id, workflow_key, state=state, user=user)
 
     @celery_task(name=RESUME_TASK)  # type: ignore
-    def resume_task(process_id: UUID, user_inputs: list[State] | None, user: str) -> UUID | None:
+    def resume_task(process_id: UUID, user: str) -> UUID | None:
         local_logger.info("Resume task", process_id=process_id)
-        return resume_process(process_id, user_inputs=user_inputs, user=user)
+        state = retrieve_input_state(process_id, "user_input").input_state
+        return resume_process(process_id, user_inputs=state, user=user)
 
     @celery_task(name=RESUME_WORKFLOW)  # type: ignore
-    def resume_workflow(process_id: UUID, user_inputs: list[State] | None, user: str) -> UUID | None:
+    def resume_workflow(process_id: UUID, user: str) -> UUID | None:
         local_logger.info("Resume workflow", process_id=process_id)
-        return resume_process(process_id, user_inputs=user_inputs, user=user)
+        state = retrieve_input_state(process_id, "user_input").input_state
+        return resume_process(process_id, user_inputs=state, user=user)
 
 
 class CeleryJobWorkerStatus(WorkerStatus):

orchestrator/services/workflows.py

@@ -1,3 +1,16 @@
+# Copyright 2019-2025 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from collections.abc import Iterable
 
 from sqlalchemy import Select, select

orchestrator/settings.py

@@ -16,11 +16,11 @@ import string
 from pathlib import Path
 from typing import Literal
 
-from pydantic import PostgresDsn, RedisDsn
+from pydantic import Field, NonNegativeInt, PostgresDsn, RedisDsn
 from pydantic_settings import BaseSettings
 
 from oauth2_lib.settings import oauth2lib_settings
-from orchestrator.types import strEnum
+from pydantic_forms.types import strEnum
 
 
 class ExecutorType(strEnum):
@@ -57,6 +57,9 @@ class AppSettings(BaseSettings):
     CACHE_URI: RedisDsn = "redis://localhost:6379/0"  # type: ignore
     CACHE_DOMAIN_MODELS: bool = False
     CACHE_HMAC_SECRET: str | None = None  # HMAC signing key, used when pickling results in the cache
+    REDIS_RETRY_COUNT: NonNegativeInt = Field(
+        2, description="Number of retries for redis connection errors/timeouts, 0 to disable"
+    )  # More info: https://redis-py.readthedocs.io/en/stable/retry.html
     ENABLE_DISTLOCK_MANAGER: bool = True
     DISTLOCK_BACKEND: str = "memory"
     CC_NOC: int = 0

orchestrator/targets.py

@@ -13,7 +13,7 @@
 
 import strawberry
 
-from orchestrator.types import strEnum
+from pydantic_forms.types import strEnum
 
 
 @strawberry.enum

orchestrator/types.py

@@ -34,59 +34,24 @@ from annotated_types import Len, MaxLen, MinLen
 from more_itertools import first, last
 from pydantic.fields import FieldInfo
 
-# TODO #428: eventually enforce code migration for downstream users to import
-# these types from pydantic_forms themselves
-from pydantic_forms.types import (
-    JSON,
-    AcceptData,
-    AcceptItemType,
-    FormGenerator,
-    FormGeneratorAsync,
-    InputForm,
-    InputFormGenerator,
-    InputStepFunc,
-    SimpleInputFormGenerator,
-    State,
-    StateInputFormGenerator,
-    StateInputFormGeneratorAsync,
-    StateInputStepFunc,
-    StateSimpleInputFormGenerator,
-    SubscriptionMapping,
-    SummaryData,
-    UUIDstr,
-    strEnum,
-)
+from pydantic_forms.types import InputForm, State, strEnum
 
 __all__ = [
-    "JSON",
+    "SAFE_USED_BY_TRANSITIONS_FOR_STATUS",
     "BroadcastFunc",
-    "AcceptData",
-    "AcceptItemType",
     "ErrorDict",
     "ErrorState",
-    "FormGenerator",
-    "FormGeneratorAsync",
-    "InputForm",
-    "InputFormGenerator",
-    "InputStepFunc",
-    "SimpleInputFormGenerator",
-    "State",
-    "StateInputFormGenerator",
-    "StateInputFormGeneratorAsync",
-    "StateInputStepFunc",
-    "StateSimpleInputFormGenerator",
     "StateStepFunc",
     "StepFunc",
     "SubscriptionLifecycle",
-    "SubscriptionMapping",
-    "SummaryData",
-    "UUIDstr",
+    "filter_nonetype",
+    "get_origin_and_args",
+    "get_possible_product_block_types",
     "is_list_type",
     "is_of_type",
     "is_optional_type",
     "is_union_type",
-    "get_possible_product_block_types",
-    "strEnum",
+    "list_factory",
 ]
 
 if TYPE_CHECKING:
@@ -97,8 +62,8 @@ def is_union(tp: type[Any] | None) -> bool:
     return tp is Union or tp is types.UnionType  # type: ignore[comparison-overlap]
 
 
-# ErrorState is either a string containing an error message, a catched Exception or a tuple containing a message and
-# a HTTP status code
+# ErrorState is either a string containing an error message, a caught Exception, or a tuple containing a message and
+# an HTTP status code
 ErrorState = Union[str, Exception, tuple[str, Union[int, HTTPStatus]]]
 # An ErrorDict should have the following keys:
 # error: str  # A message describing the error

orchestrator/utils/errors.py

@@ -18,7 +18,8 @@ from typing import Any, cast
 import structlog
 
 from nwastdlib.ex import show_ex
-from orchestrator.types import JSON, ErrorDict
+from orchestrator.types import ErrorDict
+from pydantic_forms.types import JSON
 
 logger = structlog.get_logger(__name__)
 

orchestrator/utils/redis.py

@@ -17,22 +17,22 @@ from os import getenv
 from typing import Any, Callable
 from uuid import UUID
 
-import redis.exceptions
 from anyio import CancelScope, get_cancelled_exc_class
-from redis import Redis
 from redis.asyncio import Redis as AIORedis
 from redis.asyncio.client import Pipeline, PubSub
-from redis.asyncio.retry import Retry
-from redis.backoff import EqualJitterBackoff
 from structlog import get_logger
 
 from orchestrator.services.subscriptions import _generate_etag
 from orchestrator.settings import app_settings
 from orchestrator.utils.json import PY_JSON_TYPES, json_dumps, json_loads
+from orchestrator.utils.redis_client import (
+    create_redis_asyncio_client,
+    create_redis_client,
+)
 
 logger = get_logger(__name__)
 
-cache = Redis.from_url(str(app_settings.CACHE_URI))
+cache = create_redis_client(app_settings.CACHE_URI)
 
 ONE_WEEK = 3600 * 24 * 7
 
@@ -136,12 +136,7 @@ class RedisBroadcast:
     client: AIORedis
 
     def __init__(self, redis_url: str):
-        self.client = AIORedis.from_url(
-            redis_url,
-            retry_on_error=[redis.exceptions.ConnectionError],
-            retry_on_timeout=True,
-            retry=Retry(EqualJitterBackoff(base=0.05), 2),
-        )
+        self.client = create_redis_asyncio_client(redis_url)
         self.redis_url = redis_url
 
     @asynccontextmanager

orchestrator/utils/redis_client.py

@@ -0,0 +1,35 @@
+import redis.asyncio
+import redis.client
+import redis.exceptions
+from pydantic import RedisDsn
+from redis import Redis
+from redis.asyncio import Redis as AIORedis
+from redis.asyncio.retry import Retry as AIORetry
+from redis.backoff import EqualJitterBackoff
+from redis.retry import Retry
+
+from orchestrator.settings import app_settings
+
+REDIS_RETRY_ON_ERROR = [redis.exceptions.ConnectionError]
+REDIS_RETRY_ON_TIMEOUT = True
+REDIS_RETRY_BACKOFF = EqualJitterBackoff(base=0.05)
+
+
+def create_redis_client(redis_url: str | RedisDsn) -> redis.client.Redis:
+    """Create sync Redis client for the given Redis DSN with retry handling for connection errors and timeouts."""
+    return Redis.from_url(
+        str(redis_url),
+        retry_on_error=REDIS_RETRY_ON_ERROR,  # type: ignore[arg-type]
+        retry_on_timeout=REDIS_RETRY_ON_TIMEOUT,
+        retry=Retry(REDIS_RETRY_BACKOFF, app_settings.REDIS_RETRY_COUNT),
+    )
+
+
+def create_redis_asyncio_client(redis_url: str | RedisDsn) -> redis.asyncio.client.Redis:
+    """Create async Redis client for the given Redis DSN with retry handling for connection errors and timeouts."""
+    return AIORedis.from_url(
+        str(redis_url),
+        retry_on_error=REDIS_RETRY_ON_ERROR,  # type: ignore[arg-type]
+        retry_on_timeout=REDIS_RETRY_ON_TIMEOUT,
+        retry=AIORetry(REDIS_RETRY_BACKOFF, app_settings.REDIS_RETRY_COUNT),
+    )

orchestrator/utils/state.py

@@ -19,7 +19,7 @@ from typing import Any, cast, get_args
 from uuid import UUID
 
 from orchestrator.domain.base import SubscriptionModel
-from orchestrator.types import State, StepFunc, is_list_type, is_optional_type
+from orchestrator.types import StepFunc, is_list_type, is_optional_type
 from orchestrator.utils.functional import logger
 from pydantic_forms.types import (
     FormGenerator,
@@ -27,6 +27,7 @@ from pydantic_forms.types import (
     InputFormGenerator,
     InputStepFunc,
     SimpleInputFormGenerator,
+    State,
     StateInputStepFunc,
 )
 

orchestrator/workflow.py

@@ -43,7 +43,7 @@ from orchestrator.config.assignee import Assignee
 from orchestrator.db import db, transactional
 from orchestrator.services.settings import get_engine_settings
 from orchestrator.targets import Target
-from orchestrator.types import ErrorDict, State, StepFunc, strEnum
+from orchestrator.types import ErrorDict, StepFunc
 from orchestrator.utils.docs import make_workflow_doc
 from orchestrator.utils.errors import error_state_to_dict
 from orchestrator.utils.state import form_inject_args, inject_args
@@ -52,9 +52,11 @@ from pydantic_forms.types import (
     FormGenerator,
     InputFormGenerator,
     InputStepFunc,
+    State,
     StateInputFormGenerator,
     StateInputStepFunc,
     StateSimpleInputFormGenerator,
+    strEnum,
 )
 
 logger = structlog.get_logger(__name__)

orchestrator/workflows/modify_note.py

@@ -15,12 +15,11 @@ from orchestrator.forms import SubmitFormPage
 from orchestrator.services import subscriptions
 from orchestrator.settings import app_settings
 from orchestrator.targets import Target
-from orchestrator.types import UUIDstr
 from orchestrator.utils.json import to_serializable
 from orchestrator.workflow import StepList, conditional, done, init, step, workflow
 from orchestrator.workflows.steps import cache_domain_models, store_process_subscription
 from orchestrator.workflows.utils import wrap_modify_initial_input_form
-from pydantic_forms.types import FormGenerator, State
+from pydantic_forms.types import FormGenerator, State, UUIDstr
 from pydantic_forms.validators import LongText
 
 

orchestrator/workflows/steps.py

@@ -23,11 +23,12 @@ from orchestrator.domain.base import ProductBlockModel, SubscriptionModel
 from orchestrator.services.settings import reset_search_index
 from orchestrator.services.subscriptions import build_extended_domain_model, get_subscription
 from orchestrator.targets import Target
-from orchestrator.types import State, SubscriptionLifecycle, UUIDstr
+from orchestrator.types import SubscriptionLifecycle
 from orchestrator.utils.json import to_serializable
 from orchestrator.utils.redis import delete_from_redis, to_redis
 from orchestrator.websocket import sync_invalidate_subscription_cache
 from orchestrator.workflow import Step, step
+from pydantic_forms.types import State, UUIDstr
 
 logger = structlog.get_logger(__name__)
 

orchestrator/workflows/tasks/cleanup_tasks_log.py

@@ -19,9 +19,9 @@ from sqlalchemy import select
 from orchestrator.db import ProcessTable, db
 from orchestrator.settings import app_settings
 from orchestrator.targets import Target
-from orchestrator.types import State
 from orchestrator.utils.datetime import nowtz
 from orchestrator.workflow import ProcessStatus, StepList, done, init, step, workflow
+from pydantic_forms.types import State
 
 
 @step("Clean up completed tasks older than TASK_LOG_RETENTION_DAYS")

orchestrator/workflows/tasks/resume_workflows.py

@@ -18,8 +18,8 @@ from sqlalchemy import select
 from orchestrator.db import ProcessTable, db
 from orchestrator.services import processes
 from orchestrator.targets import Target
-from orchestrator.types import State, UUIDstr
 from orchestrator.workflow import ProcessStatus, StepList, done, init, step, workflow
+from pydantic_forms.types import State, UUIDstr
 
 logger = structlog.get_logger(__name__)
 

orchestrator/workflows/tasks/validate_product_type.py

@@ -26,8 +26,8 @@ from orchestrator.services.workflows import (
     start_validation_workflow_for_workflows,
 )
 from orchestrator.targets import Target
-from orchestrator.types import FormGenerator, State
 from orchestrator.workflow import StepList, done, init, step, workflow
+from pydantic_forms.types import FormGenerator, State
 
 logger = structlog.get_logger(__name__)
 

orchestrator/workflows/tasks/validate_products.py

@@ -27,10 +27,10 @@ from orchestrator.services.products import get_products
 from orchestrator.services.translations import generate_translations
 from orchestrator.services.workflows import get_workflow_by_name, get_workflows
 from orchestrator.targets import Target
-from orchestrator.types import State
 from orchestrator.utils.errors import ProcessFailureError
 from orchestrator.utils.fixed_inputs import fixed_input_configuration as fi_configuration
 from orchestrator.workflow import StepList, done, init, step, workflow
+from pydantic_forms.types import State
 
 # Since these errors are probably programming failures we should not throw AssertionErrors
 

orchestrator/workflows/utils.py

@@ -25,7 +25,7 @@ from orchestrator.forms.validators import ProductId
 from orchestrator.services import subscriptions
 from orchestrator.settings import app_settings
 from orchestrator.targets import Target
-from orchestrator.types import State, SubscriptionLifecycle
+from orchestrator.types import SubscriptionLifecycle
 from orchestrator.utils.errors import StaleDataError
 from orchestrator.utils.redis import caching_models_enabled
 from orchestrator.utils.state import form_inject_args
@@ -41,7 +41,7 @@ from orchestrator.workflows.steps import (
     unsync_unchecked,
 )
 from pydantic_forms.core import FormPage
-from pydantic_forms.types import FormGenerator, InputForm, InputStepFunc, StateInputStepFunc
+from pydantic_forms.types import FormGenerator, InputForm, InputStepFunc, State, StateInputStepFunc
 
 
 def _generate_new_subscription_form(_workflow_target: str, workflow_name: str) -> InputForm:

{orchestrator_core-2.10.0rc1.dist-info → orchestrator_core-3.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 2.10.0rc1
+Version: 3.0.0
 Summary: This is the orchestrator workflow engine.
 Requires-Python: >=3.11,<3.14
 Classifier: Intended Audience :: Information Technology
@@ -27,11 +27,12 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
 Classifier: Topic :: Internet :: WWW/HTTP
+License-File: LICENSE
 Requires-Dist: alembic==1.14.1
 Requires-Dist: anyio>=3.7.0
 Requires-Dist: click==8.*
 Requires-Dist: deprecated
-Requires-Dist: deepmerge==1.1.1
+Requires-Dist: deepmerge==2.0
 Requires-Dist: fastapi~=0.115.2
 Requires-Dist: fastapi-etag==0.4.0
 Requires-Dist: more-itertools~=10.6.0
@@ -40,13 +41,13 @@ Requires-Dist: Jinja2==3.1.5
 Requires-Dist: orjson==3.10.15
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: pydantic[email]~=2.8.2
-Requires-Dist: pydantic-settings~=2.7.1
+Requires-Dist: pydantic-settings~=2.8.0
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: python-rapidjson>=1.18,<1.21
-Requires-Dist: pytz==2024.1
+Requires-Dist: pytz==2025.1
 Requires-Dist: redis==5.0.3
 Requires-Dist: schedule==1.1.0
-Requires-Dist: sentry-sdk[fastapi]~=2.18.0
+Requires-Dist: sentry-sdk[fastapi]~=2.22.0
 Requires-Dist: SQLAlchemy==2.0.38
 Requires-Dist: SQLAlchemy-Utils==0.41.2
 Requires-Dist: structlog
@@ -56,7 +57,7 @@ Requires-Dist: nwa-stdlib~=1.9.0
 Requires-Dist: oauth2-lib~=2.4.0
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: strawberry-graphql>=0.246.2
-Requires-Dist: pydantic-forms~=1.2.1
+Requires-Dist: pydantic-forms~=1.3.0
 Requires-Dist: celery~=5.4.0 ; extra == "celery"
 Requires-Dist: toml ; extra == "dev"
 Requires-Dist: bumpversion ; extra == "dev"
@@ -81,8 +82,9 @@ Requires-Dist: dirty-equals ; extra == "test"
 Requires-Dist: jsonref ; extra == "test"
 Requires-Dist: mypy==1.9 ; extra == "test"
 Requires-Dist: pyinstrument ; extra == "test"
-Requires-Dist: pytest==8.3.2 ; extra == "test"
+Requires-Dist: pytest==8.3.4 ; extra == "test"
 Requires-Dist: pytest-asyncio==0.21.2 ; extra == "test"
+Requires-Dist: pytest-codspeed ; extra == "test"
 Requires-Dist: pytest-cov ; extra == "test"
 Requires-Dist: pytest-httpx ; extra == "test"
 Requires-Dist: pytest-xdist ; extra == "test"