orchestrator-core 3.1.2rc1__tar.gz → 3.1.2rc3__tar.gz

{orchestrator_core-3.1.2rc1 → orchestrator_core-3.1.2rc3}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.1.1rc1
+current_version = 3.1.2rc3
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(rc(?P<build>\d+))?

orchestrator_core-3.1.2rc3/PKG-INFO

@@ -0,0 +1,116 @@
+Metadata-Version: 2.4
+Name: orchestrator-core
+Version: 3.1.2rc3
+Summary: This is the orchestrator workflow engine.
+Requires-Python: >=3.11,<3.14
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python
+Classifier: Topic :: Internet
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development
+Classifier: Typing :: Typed
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Web Environment
+Classifier: Framework :: AsyncIO
+Classifier: Framework :: FastAPI
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Telecommunications Industry
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
+Classifier: Topic :: Internet :: WWW/HTTP
+License-File: LICENSE
+Requires-Dist: alembic==1.15.1
+Requires-Dist: anyio>=3.7.0
+Requires-Dist: click==8.*
+Requires-Dist: deprecated
+Requires-Dist: deepmerge==2.0
+Requires-Dist: fastapi~=0.115.2
+Requires-Dist: fastapi-etag==0.4.0
+Requires-Dist: more-itertools~=10.6.0
+Requires-Dist: itsdangerous
+Requires-Dist: Jinja2==3.1.6
+Requires-Dist: orjson==3.10.15
+Requires-Dist: psycopg[binary]==3.2.6
+Requires-Dist: pydantic[email]~=2.8.2
+Requires-Dist: pydantic-settings~=2.8.0
+Requires-Dist: python-dateutil==2.8.2
+Requires-Dist: python-rapidjson>=1.18,<1.21
+Requires-Dist: pytz==2025.1
+Requires-Dist: redis==5.1.1
+Requires-Dist: schedule==1.1.0
+Requires-Dist: sentry-sdk[fastapi]~=2.22.0
+Requires-Dist: SQLAlchemy==2.0.39
+Requires-Dist: SQLAlchemy-Utils==0.41.2
+Requires-Dist: structlog
+Requires-Dist: typer==0.15.2
+Requires-Dist: uvicorn[standard]~=0.34.0
+Requires-Dist: nwa-stdlib~=1.9.0
+Requires-Dist: oauth2-lib~=2.4.0
+Requires-Dist: tabulate==0.9.0
+Requires-Dist: strawberry-graphql>=0.246.2
+Requires-Dist: pydantic-forms~=1.4.0
+Requires-Dist: celery~=5.4.0 ; extra == "celery"
+Requires-Dist: toml ; extra == "dev"
+Requires-Dist: bumpversion ; extra == "dev"
+Requires-Dist: mypy_extensions ; extra == "dev"
+Requires-Dist: pre-commit ; extra == "dev"
+Requires-Dist: pydocstyle ; extra == "dev"
+Requires-Dist: python-dotenv ; extra == "dev"
+Requires-Dist: watchdog ; extra == "dev"
+Requires-Dist: mkdocs ; extra == "doc"
+Requires-Dist: mkdocs-material[imaging] ; extra == "doc"
+Requires-Dist: mkdocs-render-swagger-plugin ; extra == "doc"
+Requires-Dist: mkdocs-include-markdown-plugin ; extra == "doc"
+Requires-Dist: mkdocstrings[python] ; extra == "doc"
+Requires-Dist: mkdocs-open-in-new-tab ; extra == "doc"
+Requires-Dist: mkdocs-macros-plugin ; extra == "doc"
+Requires-Dist: mkdocs-embed-external-markdown ; extra == "doc"
+Requires-Dist: apache-license-check ; extra == "test"
+Requires-Dist: black ; extra == "test"
+Requires-Dist: blinker ; extra == "test"
+Requires-Dist: deepdiff ; extra == "test"
+Requires-Dist: dirty-equals ; extra == "test"
+Requires-Dist: jsonref ; extra == "test"
+Requires-Dist: mypy==1.9 ; extra == "test"
+Requires-Dist: pyinstrument ; extra == "test"
+Requires-Dist: pytest==8.3.5 ; extra == "test"
+Requires-Dist: pytest-asyncio==0.21.2 ; extra == "test"
+Requires-Dist: pytest-codspeed ; extra == "test"
+Requires-Dist: pytest-cov ; extra == "test"
+Requires-Dist: pytest-httpx ; extra == "test"
+Requires-Dist: pytest-xdist ; extra == "test"
+Requires-Dist: requests-mock ; extra == "test"
+Requires-Dist: ruff ; extra == "test"
+Requires-Dist: sqlalchemy[mypy] ; extra == "test"
+Requires-Dist: urllib3-mock ; extra == "test"
+Requires-Dist: types-Deprecated ; extra == "test"
+Requires-Dist: types-Jinja2 ; extra == "test"
+Requires-Dist: types-aiofiles ; extra == "test"
+Requires-Dist: types-certifi ; extra == "test"
+Requires-Dist: types-click ; extra == "test"
+Requires-Dist: types-itsdangerous ; extra == "test"
+Requires-Dist: types-orjson ; extra == "test"
+Requires-Dist: types-python-dateutil ; extra == "test"
+Requires-Dist: types-pytz ; extra == "test"
+Requires-Dist: types-redis ; extra == "test"
+Requires-Dist: types-requests ; extra == "test"
+Requires-Dist: types-setuptools ; extra == "test"
+Requires-Dist: types-tabulate ; extra == "test"
+Requires-Dist: types-toml ; extra == "test"
+Requires-Dist: types-ujson ; extra == "test"
+Requires-Dist: types-PyYAML ; extra == "test"
+Project-URL: Documentation, https://workfloworchestrator.org/orchestrator-core/
+Project-URL: Source, https://github.com/workfloworchestrator/orchestrator-core
+Provides-Extra: celery
+Provides-Extra: dev
+Provides-Extra: doc
+Provides-Extra: test

{orchestrator_core-3.1.2rc1 → orchestrator_core-3.1.2rc3}/docs/reference-docs/workflows/callbacks.md

@@ -168,3 +168,11 @@ def create_l2vpn() -> StepList:
         >> set_status(SubscriptionLifecycle.ACTIVE)
     )
 ```
+
+## Callback progress during execution
+
+For long-running jobs, such as executing Terraform or Ansible playbooks, the Orchestrator allows callback jobs to send real-time progress updates which can be used to provide operators with feedback on the progress of the running task.
+
+Progress updates should be delivered to `{callback_route}/progress`, where `"/progress"` is a fixed endpoint appended to the callback URL.
+
+The remote service should send a JSON or plain string payload with each progress callback, this replaces the previous progress update and refreshes the UI. Once the final callback is triggered and the job completes, progress updates are removed, leaving only the callback result. Therefore, all troubleshooting or diagnostic information must be included in the final callback payload, as progress updates are not retained for debugging.

{orchestrator_core-3.1.2rc1 → orchestrator_core-3.1.2rc3}/orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "3.1.2rc1"
+__version__ = "3.1.2rc3"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings

{orchestrator_core-3.1.2rc1 → orchestrator_core-3.1.2rc3}/orchestrator/api/api_v1/endpoints/processes.py

@@ -58,6 +58,7 @@ from orchestrator.services.processes import (
     load_process,
     resume_process,
     start_process,
+    update_awaiting_process_progress,
 )
 from orchestrator.services.settings import get_engine_settings
 from orchestrator.settings import app_settings
@@ -197,6 +198,28 @@ def continue_awaiting_process_endpoint(
         raise_status(HTTPStatus.NOT_FOUND, str(e))
 
 
+@router.post(
+    "/{process_id}/callback/{token}/progress",
+    response_model=None,
+    status_code=HTTPStatus.OK,
+    dependencies=[Depends(check_global_lock, use_cache=False)],
+)
+def update_progress_on_awaiting_process_endpoint(
+    process_id: UUID,
+    token: str,
+    data: str | State = Body(...),
+) -> None:
+    process = _get_process(process_id)
+
+    if process.last_status != ProcessStatus.AWAITING_CALLBACK:
+        raise_status(HTTPStatus.CONFLICT, "This process is not in an awaiting state.")
+
+    try:
+        update_awaiting_process_progress(process, token=token, data=data)
+    except AssertionError as exc:
+        raise_status(HTTPStatus.NOT_FOUND, str(exc))
+
+
 @router.put(
     "/resume-all", response_model=ProcessResumeAllSchema, dependencies=[Depends(check_global_lock, use_cache=False)]
 )

{orchestrator_core-3.1.2rc1 → orchestrator_core-3.1.2rc3}/orchestrator/services/processes.py

@@ -43,9 +43,10 @@ from orchestrator.targets import Target
 from orchestrator.types import BroadcastFunc
 from orchestrator.utils.datetime import nowtz
 from orchestrator.utils.errors import error_state_to_dict
-from orchestrator.websocket import broadcast_invalidate_status_counts
+from orchestrator.websocket import broadcast_invalidate_status_counts, broadcast_process_update_to_websocket
 from orchestrator.workflow import (
     CALLBACK_TOKEN_KEY,
+    DEFAULT_CALLBACK_PROGRESS_KEY,
     Failed,
     ProcessStat,
     ProcessStatus,
@@ -566,6 +567,39 @@ def resume_process(
     return resume_func(process, user_inputs=user_inputs, user=user, broadcast_func=broadcast_func)
 
 
+def ensure_correct_callback_token(pstat: ProcessStat, *, token: str) -> None:
+    """Ensure that a callback token matches the expected value in state.
+
+    Args:
+        pstat: ProcessStat of process.
+        token: The token which was generated for the process.
+
+    Raises:
+        AssertionError: if the supplied token does not match the generated process token.
+
+    """
+    state = pstat.state.unwrap()
+
+    # Check if the token matches
+    token_from_state = state.get(CALLBACK_TOKEN_KEY)
+    if token != token_from_state:
+        raise AssertionError("Invalid token")
+
+
+def replace_current_step_state(process: ProcessTable, *, new_state: State) -> None:
+    """Replace the state of the current step in a process.
+
+    Args:
+        process: Process from database
+        new_state: The new state
+
+    """
+    current_step = process.steps[-1]
+    current_step.state = new_state
+    db.session.add(current_step)
+    db.session.commit()
+
+
 def continue_awaiting_process(
     process: ProcessTable,
     *,
@@ -589,10 +623,7 @@ def continue_awaiting_process(
     pstat = load_process(process)
     state = pstat.state.unwrap()
 
-    # Check if the token matches
-    token_from_state = state.get(CALLBACK_TOKEN_KEY)
-    if token != token_from_state:
-        raise AssertionError("Invalid token")
+    ensure_correct_callback_token(pstat, token=token)
 
     # We need to pass the callback data to the worker executor. Currently, this is not supported.
     # Therefore, we update the step state in the db and kick-off resume_workflow
@@ -600,16 +631,47 @@ def continue_awaiting_process(
     result_key = state.get("__callback_result_key", "callback_result")
     state = {**state, result_key: input_data}
 
-    current_step = process.steps[-1]
-    current_step.state = state
-    db.session.add(current_step)
-    db.session.commit()
+    replace_current_step_state(process, new_state=state)
 
     # Continue the workflow
     resume_func = get_execution_context()["resume"]
     return resume_func(process, broadcast_func=broadcast_func)
 
 
+def update_awaiting_process_progress(
+    process: ProcessTable,
+    *,
+    token: str,
+    data: str | State,
+) -> UUID:
+    """Update progress for a process awaiting data from a callback.
+
+    Args:
+        process: Process from database
+        token: The token which was generated for the process. This must match.
+        data: Progress data posted to the callback
+
+    Returns:
+        process id
+
+    Raises:
+        AssertionError: if the supplied token does not match the generated process token.
+
+    """
+    pstat = load_process(process)
+
+    ensure_correct_callback_token(pstat, token=token)
+
+    state = pstat.state.unwrap()
+    progress_key = state.get(DEFAULT_CALLBACK_PROGRESS_KEY, "callback_progress")
+    state = {**state, progress_key: data} | {"__remove_keys": [progress_key]}
+
+    replace_current_step_state(process, new_state=state)
+    broadcast_process_update_to_websocket(process.process_id)
+
+    return process.process_id
+
+
 async def _async_resume_processes(
     processes: Sequence[ProcessTable],
     user_name: str,

{orchestrator_core-3.1.2rc1 → orchestrator_core-3.1.2rc3}/orchestrator/workflow.py

@@ -69,6 +69,7 @@ step_log_fn_var: contextvars.ContextVar[StepLogFuncInternal] = contextvars.Conte
 
 DEFAULT_CALLBACK_ROUTE_KEY = "callback_route"
 CALLBACK_TOKEN_KEY = "__callback_token"  # noqa: S105
+DEFAULT_CALLBACK_PROGRESS_KEY = "callback_progress"  # noqa: S105
 
 
 @runtime_checkable

{orchestrator_core-3.1.2rc1 → orchestrator_core-3.1.2rc3}/pyproject.toml

@@ -6,9 +6,9 @@ requires = ["flit_core >=3.2,<4"]
 [project]
 name = "orchestrator-core"
 dynamic = ['version', 'description']
-authors = [
-    {name = "SURF", email = "automation-beheer@surf.nl"},
-]
+author = "SURF"
+author-email = "automation-beheer@surf.nl"
+home-page = "https://github.com/workfloworchestrator/orchestrator-core"
 classifiers = [
     "Intended Audience :: Information Technology",
     "Intended Audience :: System Administrators",
@@ -27,6 +27,7 @@ classifiers = [
     "Framework :: FastAPI",
     "Intended Audience :: Developers",
     "Intended Audience :: Telecommunications Industry",
+    "License :: OSI Approved :: Apache Software License",
     "Programming Language :: Python :: 3 :: Only",
     "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: 3.12",
@@ -47,7 +48,7 @@ dependencies = [
     "Jinja2==3.1.6",
     "orjson==3.10.15",
     "psycopg[binary]==3.2.6",
-    "pydantic[email]~=2.10.6",
+    "pydantic[email]~=2.8.2",
     "pydantic-settings~=2.8.0",
     "python-dateutil==2.8.2",
     "python-rapidjson>=1.18,<1.21",
@@ -67,15 +68,12 @@ dependencies = [
     "pydantic-forms~=1.4.0",
 ]
 
-readme = "README.md"
+description-file = "README.md"
 requires-python = ">=3.11,<3.14"
-license = "Apache-2.0"
-license-files = [ "LICENSE" ]
 
 [project.urls]
-documentation = "https://workfloworchestrator.org/orchestrator-core/"
-source = "https://github.com/workfloworchestrator/orchestrator-core"
-releasenotes = "https://github.com/workfloworchestrator/orchestrator-core/releases"
+Documentation = "https://workfloworchestrator.org/orchestrator-core/"
+Source = "https://github.com/workfloworchestrator/orchestrator-core"
 
 [project.optional-dependencies]
 celery = [

{orchestrator_core-3.1.2rc1 → orchestrator_core-3.1.2rc3}/test/unit_tests/forms/test_display_subscription.py

@@ -59,7 +59,11 @@ def test_display_only_schema():
                 "type": "string",
             },
             "summary": {
-                "$ref": "#/$defs/MigrationSummaryValue",
+                "allOf": [
+                    {
+                        "$ref": "#/$defs/MigrationSummaryValue",
+                    },
+                ],
                 "format": "summary",
                 "default": None,
                 "type": "string",

{orchestrator_core-3.1.2rc1 → orchestrator_core-3.1.2rc3}/test/unit_tests/graphql/test_subscriptions.py

@@ -1335,7 +1335,7 @@ def test_single_subscription_schema(
             "customer_id": {"title": "Customer Id", "type": "string"},
             "subscription_id": {"format": "uuid", "title": "Subscription Id", "type": "string"},
             "description": {"default": "Initial subscription", "title": "Description", "type": "string"},
-            "status": {"$ref": "#/$defs/SubscriptionLifecycle", "default": "initial"},
+            "status": {"allOf": [{"$ref": "#/$defs/SubscriptionLifecycle"}], "default": "initial"},
             "insync": {"default": False, "title": "Insync", "type": "boolean"},
             "start_date": {
                 "anyOf": [{"format": "date-time", "type": "string"}, {"type": "null"}],

{orchestrator_core-3.1.2rc1 → orchestrator_core-3.1.2rc3}/test/unit_tests/workflows/test_async_workflow.py

@@ -1,5 +1,6 @@
 from orchestrator.targets import Target
 from orchestrator.workflow import (
+    DEFAULT_CALLBACK_PROGRESS_KEY,
     DEFAULT_CALLBACK_ROUTE_KEY,
     AwaitingCallback,
     begin,
@@ -165,3 +166,118 @@ def test_wf_with_multiple_callback_steps(test_client):
         assert state["ext_data"] == "56789"
 
         assert state["phase"] is None
+
+
+def test_wf_callback_progress_on_completed_process(test_client):
+    # given
+    @workflow("Test wf", target=Target.CREATE)
+    def test_wf():
+        return begin >> step1 >> done
+
+    with WorkflowInstanceForTests(test_wf, "test_wf"):
+        response = test_client.post("/api/processes/test_wf", json=[{}])
+        assert response.status_code == 201
+        process_id = response.json()["id"]
+
+        response = test_client.get(f"api/processes/{process_id}")
+        response_data = response.json()
+        assert response_data["last_status"] == "completed"
+
+        # when
+        response = test_client.post(f"api/processes/{process_id}/callback/token123/progress", json={})
+        response_data = response.json()
+
+        # then
+        assert response.status_code == 409
+        assert response_data["detail"] == "This process is not in an awaiting state."
+
+
+def test_wf_callback_progress_with_incorrect_token(test_client):
+    # given
+    @workflow("Test wf", target=Target.CREATE)
+    def test_wf():
+        return begin >> cb1 >> done
+
+    with WorkflowInstanceForTests(test_wf, "test_wf"):
+        response = test_client.post("/api/processes/test_wf", json=[{}])
+        assert response.status_code == 201
+        process_id = response.json()["id"]
+
+        response = test_client.get(f"api/processes/{process_id}")
+        response_data = response.json()
+        assert response_data["last_status"] == "awaiting_callback"
+
+        # when
+        response = test_client.post(f"api/processes/{process_id}/callback/incorrectoken/progress", json={})
+        response_data = response.json()
+
+        # then
+        assert response.status_code == 404
+        assert response_data["detail"] == "Invalid token"
+
+
+def test_wf_callback_progress_with_multiple_callback_steps(test_client):
+    with WorkflowInstanceForTests(multiple_callback_wf, "multiple_callback_wf"):
+        # Start workflow
+        response = test_client.post("/api/processes/multiple_callback_wf", json=[{}])
+        assert response.status_code == 201
+        process_id = response.json()["id"]
+
+        # Check process status
+        response = test_client.get(f"api/processes/{process_id}")
+        response_data = response.json()
+        assert response_data["last_status"] == "awaiting_callback"
+        state = response_data["current_state"]
+
+        # Update progress 1 using dict
+        callback_route1 = state["callback_route"]
+        response = test_client.post(f"{callback_route1}/progress", json={"update": "123"})
+        assert response.status_code == 200
+        response_data = response.json()
+
+        # Check process status
+        response = test_client.get(f"api/processes/{process_id}")
+        response_data = response.json()
+        state = response_data["current_state"]
+        assert state[DEFAULT_CALLBACK_PROGRESS_KEY] == {"update": "123"}
+
+        # Continue workflow 1
+        callback_route1 = state["callback_route"]
+        response = test_client.post(callback_route1, json={"ext_data": "12345"})
+        assert response.status_code == 200
+
+        # Check process status
+        response = test_client.get(f"api/processes/{process_id}")
+        response_data = response.json()
+        assert response_data["last_status"] == "awaiting_callback"
+        state = response_data["current_state"]
+        assert state["dr_ext_data"] == "12345"
+        assert DEFAULT_CALLBACK_PROGRESS_KEY not in state
+
+        # Update progress 2 using string
+        callback_route2 = state["callback_route"]
+        response = test_client.post(
+            f"{callback_route2}/progress", data="update 567", headers={"Content-Type": "text/plain; charset=utf-8"}
+        )
+        assert response.status_code == 200
+        response_data = response.json()
+
+        # Check process status
+        response = test_client.get(f"api/processes/{process_id}")
+        response_data = response.json()
+        state = response_data["current_state"]
+        assert state[DEFAULT_CALLBACK_PROGRESS_KEY] == "update 567"
+
+        # Continue workflow 2
+        response = test_client.post(callback_route2, json={"ext_data": "56789"})
+        assert response.status_code == 200
+
+        # Final check
+        response = test_client.get(f"api/processes/{process_id}")
+        response_data = response.json()
+
+        assert response_data["last_status"] == "completed"
+
+        state = response_data["current_state"]
+        assert state["ext_data"] == "56789"
+        assert DEFAULT_CALLBACK_PROGRESS_KEY not in state