orchestrator-core 4.1.0rc2__py3-none-any.whl → 4.2.0__py3-none-any.whl

orchestrator/migrations/versions/schema/2025-07-01_93fc5834c7e5_changed_timestamping_fields_in_process_steps.py

@@ -0,0 +1,65 @@
+"""Changed timestamping fields in process_steps.
+
+Revision ID: 93fc5834c7e5
+Revises: 4b58e336d1bf
+Create Date: 2025-07-01 14:20:44.755694
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+from orchestrator import db
+
+# revision identifiers, used by Alembic.
+revision = "93fc5834c7e5"
+down_revision = "4b58e336d1bf"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "process_steps",
+        sa.Column(
+            "started_at",
+            db.UtcTimestamp(timezone=True),
+            server_default=sa.text("statement_timestamp()"),
+            nullable=False,
+        ),
+    )
+    op.alter_column("process_steps", "executed_at", new_column_name="completed_at")
+    # conn = op.get_bind()
+    # sa.select
+    # ### end Alembic commands ###
+    # Backfill started_at field correctly using proper aliasing
+    op.execute(
+        """
+        WITH backfill_started_at AS (
+            SELECT
+                ps1.stepid,
+                COALESCE(prev.completed_at, p.started_at) AS new_started_at
+            FROM process_steps ps1
+            JOIN processes p ON ps1.pid = p.pid
+            LEFT JOIN LATERAL (
+                SELECT ps2.completed_at
+                FROM process_steps ps2
+                WHERE ps2.pid = ps1.pid AND ps2.completed_at < ps1.completed_at
+                ORDER BY ps2.completed_at DESC
+                LIMIT 1
+            ) prev ON true
+        )
+        UPDATE process_steps
+        SET started_at = b.new_started_at
+        FROM backfill_started_at b
+        WHERE process_steps.stepid = b.stepid;
+    """
+    )
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("process_steps", "started_at")
+    op.alter_column("process_steps", "completed_at", new_column_name="executed_at")
+    # ### end Alembic commands ###

orchestrator/migrations/versions/schema/2025-07-04_4b58e336d1bf_deprecating_workflow_target_in_.py

@@ -0,0 +1,30 @@
+"""Deprecating workflow target in ProcessSubscriptionTable.
+
+Revision ID: 4b58e336d1bf
+Revises: 161918133bec
+Create Date: 2025-07-04 15:27:23.814954
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4b58e336d1bf"
+down_revision = "161918133bec"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.alter_column("processes_subscriptions", "workflow_target", existing_type=sa.VARCHAR(length=255), nullable=True)
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "processes_subscriptions",
+        "workflow_target",
+        existing_type=sa.VARCHAR(length=255),
+        nullable=False,
+        existing_server_default=sa.text("'CREATE'::character varying"),
+    )

orchestrator/schemas/process.py

@@ -49,7 +49,11 @@ class ProcessStepSchema(OrchestratorBaseModel):
     name: str
     status: str
     created_by: str | None = None
-    executed: datetime | None = None
+    executed: datetime | None = Field(
+        None, deprecated="Deprecated, use 'started' and 'completed' for step start and completion times"
+    )
+    started: datetime | None = None
+    completed: datetime | None = None
     commit_hash: str | None = None
     state: dict[str, Any] | None = None
     state_delta: dict[str, Any] | None = None

orchestrator/services/processes.py

@@ -12,6 +12,7 @@
 # limitations under the License.
 from collections.abc import Callable, Sequence
 from concurrent.futures.thread import ThreadPoolExecutor
+from datetime import datetime
 from functools import partial
 from http import HTTPStatus
 from typing import Any
@@ -19,6 +20,7 @@ from uuid import UUID, uuid4
 
 import structlog
 from deepmerge.merger import Merger
+from pytz import utc
 from sqlalchemy import delete, select
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import joinedload
@@ -206,6 +208,10 @@ def _get_current_step_to_update(
     finally:
         step_state.pop("__remove_keys", None)
 
+    # We don't have __last_step_started in __remove_keys because the way __remove_keys is populated appears like it would overwrite
+    # what's put there in the step decorator in certain cases (step groups and callback steps)
+    step_start_time = step_state.pop("__last_step_started_at", None)
+
     if process_state.isfailed() or process_state.iswaiting():
         if (
             last_db_step is not None
@@ -216,7 +222,7 @@ def _get_current_step_to_update(
         ):
             state_ex_info = {
                 "retries": last_db_step.state.get("retries", 0) + 1,
-                "executed_at": last_db_step.state.get("executed_at", []) + [str(last_db_step.executed_at)],
+                "completed_at": last_db_step.state.get("completed_at", []) + [str(last_db_step.completed_at)],
             }
 
             # write new state info and execution date
@@ -236,10 +242,13 @@ def _get_current_step_to_update(
             state=step_state,
             created_by=stat.current_user,
         )
+    # Since the Start step does not have a __last_step_started_at in it's state, we effectively assume it is instantaneous.
+    now = nowtz()
+    current_step.started_at = datetime.fromtimestamp(step_start_time or now.timestamp(), tz=utc)
 
     # Always explicitly set this instead of leaving it to the database to prevent failing tests
     # Test will fail if multiple steps have the same timestamp
-    current_step.executed_at = nowtz()
+    current_step.completed_at = now
     return current_step
 
 

orchestrator/services/settings_env_variables.py

@@ -14,7 +14,7 @@
 from typing import Any, Dict, Type
 
 from pydantic import SecretStr as PydanticSecretStr
-from pydantic_core import MultiHostUrl
+from pydantic_core import MultiHostUrl, Url
 from pydantic_settings import BaseSettings
 
 from orchestrator.utils.expose_settings import SecretStr as OrchSecretStr
@@ -32,21 +32,9 @@ def expose_settings(settings_name: str, base_settings: Type[BaseSettings]) -> Ty
 
 def mask_value(key: str, value: Any) -> Any:
     key_lower = key.lower()
+    is_sensitive_key = "secret" in key_lower or "password" in key_lower
 
-    if "secret" in key_lower or "password" in key_lower:
-        # Mask sensitive information
-        return MASK
-
-    if isinstance(value, PydanticSecretStr):
-        # Need to convert SecretStr to str for serialization
-        return str(value)
-
-    if isinstance(value, OrchSecretStr):
-        return MASK
-
-    # PostgresDsn is just MultiHostUrl with extra metadata (annotations)
-    if isinstance(value, MultiHostUrl):
-        # Convert PostgresDsn to str for serialization
+    if is_sensitive_key or isinstance(value, (OrchSecretStr, PydanticSecretStr, MultiHostUrl, Url)):
         return MASK
 
     return value

orchestrator/settings.py

@@ -72,7 +72,7 @@ class AppSettings(BaseSettings):
     TRACING_ENABLED: bool = False
     TRACE_HOST: str = "http://localhost:4317"
     TRANSLATIONS_DIR: Path | None = None
-    WEBSOCKET_BROADCASTER_URL: str = "memory://"
+    WEBSOCKET_BROADCASTER_URL: OrchSecretStr = "memory://"  # type: ignore
     ENABLE_WEBSOCKETS: bool = True
     DISABLE_INSYNC_CHECK: bool = False
     DEFAULT_PRODUCT_WORKFLOWS: list[str] = ["modify_note"]

orchestrator/utils/enrich_process.py

@@ -57,7 +57,9 @@ def enrich_step_details(step: ProcessStepTable, previous_step: ProcessStepTable
 
     return {
         "name": step.name,
-        "executed": step.executed_at.timestamp(),
+        "executed": step.completed_at.timestamp(),
+        "started": step.started_at.timestamp(),
+        "completed": step.completed_at.timestamp(),
         "status": step.status,
         "state": step.state,
         "created_by": step.created_by,
@@ -103,7 +105,7 @@ def enrich_process(process: ProcessTable, p_stat: ProcessStat | None = None) ->
         "is_task": process.is_task,
         "workflow_id": process.workflow_id,
         "workflow_name": process.workflow.name,
-        "workflow_target": process.process_subscriptions[0].workflow_target if process.process_subscriptions else None,
+        "workflow_target": process.workflow.target,
         "failed_reason": process.failed_reason,
         "created_by": process.created_by,
         "started_at": process.started_at,

orchestrator/utils/errors.py

@@ -128,12 +128,13 @@ def _(err: Exception) -> ErrorDict:
     # We can't dispatch on ApiException, see is_api_exception docstring
     if is_api_exception(err):
         err = cast(ApiException, err)
+        headers = err.headers or {}
         return {
             "class": type(err).__name__,
             "error": err.reason,
             "status_code": err.status,
             "body": err.body,
-            "headers": "\n".join(f"{k}: {v}" for k, v in err.headers.items()),
+            "headers": "\n".join(f"{k}: {v}" for k, v in headers.items()),
             "traceback": show_ex(err),
         }
 

orchestrator/workflow.py

@@ -46,6 +46,7 @@ from orchestrator.services.settings import get_engine_settings
 from orchestrator.targets import Target
 from orchestrator.types import ErrorDict, StepFunc
 from orchestrator.utils.auth import Authorizer
+from orchestrator.utils.datetime import nowtz
 from orchestrator.utils.docs import make_workflow_doc
 from orchestrator.utils.errors import error_state_to_dict
 from orchestrator.utils.state import form_inject_args, inject_args
@@ -381,11 +382,13 @@ def step_group(name: str, steps: StepList, extract_form: bool = True) -> Step:
                 p = p.map(lambda s: s | {"__replace_last_state": True})
             return step_log_fn(step_, p)
 
+        step_group_start_time = nowtz().timestamp()
         process: Process = Success(initial_state)
         process = _exec_steps(step_list, process, dblogstep)
-
         # Add instruction to replace state of last sub step before returning process _exec_steps higher in the call tree
-        return process.map(lambda s: s | {"__replace_last_state": True})
+        return process.map(
+            lambda s: s | {"__replace_last_state": True, "__last_step_started_at": step_group_start_time}
+        )
 
     # Make sure we return a form is a sub step has a form
     form = next((sub_step.form for sub_step in steps if sub_step.form), None) if extract_form else None
@@ -1454,6 +1457,8 @@ def _exec_steps(steps: StepList, starting_process: Process, dblogstep: StepLogFu
                     "Not executing Step as the workflow engine is Paused. Process will remain in state 'running'"
                 )
                 return process
+
+            process = process.map(lambda s: s | {"__last_step_started_at": nowtz().timestamp()})
             step_result_process = process.execute_step(step)
         except Exception as e:
             consolelogger.error("An exception occurred while executing the workflow step.")

orchestrator/workflows/modify_note.py

@@ -53,4 +53,4 @@ def store_subscription_note(subscription_id: UUIDstr, note: str) -> State:
 
 @workflow("Modify Note", initial_input_form=wrap_modify_initial_input_form(initial_input_form), target=Target.MODIFY)
 def modify_note() -> StepList:
-    return init >> store_process_subscription(Target.MODIFY) >> store_subscription_note >> done
+    return init >> store_process_subscription() >> store_subscription_note >> done

orchestrator/workflows/steps.py

@@ -23,6 +23,7 @@ from orchestrator.services.subscriptions import get_subscription
 from orchestrator.targets import Target
 from orchestrator.types import SubscriptionLifecycle
 from orchestrator.utils.json import to_serializable
+from orchestrator.websocket import sync_invalidate_subscription_cache
 from orchestrator.workflow import Step, step
 from pydantic_forms.types import State, UUIDstr
 
@@ -33,6 +34,7 @@ logger = structlog.get_logger(__name__)
 def resync(subscription: SubscriptionModel) -> State:
     """Transition a subscription to in sync."""
     subscription.insync = True
+    sync_invalidate_subscription_cache(subscription.subscription_id)
     return {"subscription": subscription}
 
 
@@ -93,6 +95,7 @@ def unsync(subscription_id: UUIDstr, __old_subscriptions__: dict | None = None)
     if not subscription.insync:
         raise ValueError("Subscription is already out of sync, cannot continue!")
     subscription.insync = False
+    sync_invalidate_subscription_cache(subscription.subscription_id)
 
     return {"subscription": subscription, "__old_subscriptions__": subscription_backup}
 
@@ -105,20 +108,23 @@ def unsync_unchecked(subscription_id: UUIDstr) -> State:
     return {"subscription": subscription}
 
 
-def store_process_subscription_relationship(
-    process_id: UUIDstr, subscription_id: UUIDstr, workflow_target: str
-) -> ProcessSubscriptionTable:
-    process_subscription = ProcessSubscriptionTable(
-        process_id=process_id, subscription_id=subscription_id, workflow_target=workflow_target
-    )
+def store_process_subscription_relationship(process_id: UUIDstr, subscription_id: UUIDstr) -> ProcessSubscriptionTable:
+    process_subscription = ProcessSubscriptionTable(process_id=process_id, subscription_id=subscription_id)
     db.session.add(process_subscription)
     return process_subscription
 
 
-def store_process_subscription(workflow_target: Target) -> Step:
+def store_process_subscription(workflow_target: Target | None = None) -> Step:
+    if workflow_target:
+        deprecation_warning = (
+            "Providing a workflow target to function store_process_subscription() is deprecated. "
+            "This information is already stored in the workflow table."
+        )
+        logger.warning(deprecation_warning)
+
     @step("Create Process Subscription relation")
     def _store_process_subscription(process_id: UUIDstr, subscription_id: UUIDstr) -> None:
-        store_process_subscription_relationship(process_id, subscription_id, workflow_target)
+        store_process_subscription_relationship(process_id, subscription_id)
 
     return _store_process_subscription
 

orchestrator/workflows/utils.py

@@ -265,7 +265,7 @@ def modify_workflow(
     def _modify_workflow(f: Callable[[], StepList]) -> Workflow:
         steplist = (
             init
-            >> store_process_subscription(Target.MODIFY)
+            >> store_process_subscription()
             >> unsync
             >> f()
             >> (additional_steps or StepList())
@@ -311,7 +311,7 @@ def terminate_workflow(
     def _terminate_workflow(f: Callable[[], StepList]) -> Workflow:
         steplist = (
             init
-            >> store_process_subscription(Target.TERMINATE)
+            >> store_process_subscription()
             >> unsync
             >> f()
             >> (additional_steps or StepList())
@@ -348,7 +348,7 @@ def validate_workflow(description: str) -> Callable[[Callable[[], StepList]], Wo
     """
 
     def _validate_workflow(f: Callable[[], StepList]) -> Workflow:
-        steplist = init >> store_process_subscription(Target.SYSTEM) >> unsync_unchecked >> f() >> resync >> done
+        steplist = init >> store_process_subscription() >> unsync_unchecked >> f() >> resync >> done
 
         return make_workflow(f, description, validate_initial_input_form_generator, Target.VALIDATE, steplist)
 

orchestrator_core-4.2.0.dist-info/METADATA

@@ -0,0 +1,167 @@
+Metadata-Version: 2.4
+Name: orchestrator-core
+Version: 4.2.0
+Summary: This is the orchestrator workflow engine.
+Author-email: SURF <automation-beheer@surf.nl>
+Requires-Python: >=3.11,<3.14
+Description-Content-Type: text/markdown
+License-Expression: Apache-2.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Web Environment
+Classifier: Framework :: AsyncIO
+Classifier: Framework :: FastAPI
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: System Administrators
+Classifier: Intended Audience :: Telecommunications Industry
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python
+Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Internet
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development
+Classifier: Typing :: Typed
+License-File: LICENSE
+Requires-Dist: alembic==1.16.1
+Requires-Dist: anyio>=3.7.0
+Requires-Dist: click==8.*
+Requires-Dist: deepmerge==2.0
+Requires-Dist: deprecated>=1.2.18
+Requires-Dist: fastapi~=0.115.2
+Requires-Dist: fastapi-etag==0.4.0
+Requires-Dist: itsdangerous>=2.2.0
+Requires-Dist: jinja2==3.1.6
+Requires-Dist: more-itertools~=10.7.0
+Requires-Dist: nwa-stdlib~=1.9.0
+Requires-Dist: oauth2-lib~=2.4.0
+Requires-Dist: orjson==3.10.18
+Requires-Dist: prometheus-client==0.22.1
+Requires-Dist: psycopg2-binary==2.9.10
+Requires-Dist: pydantic-forms>=1.4.0,<=2.1.0
+Requires-Dist: pydantic-settings~=2.9.1
+Requires-Dist: pydantic[email]~=2.8.2
+Requires-Dist: python-dateutil==2.8.2
+Requires-Dist: python-rapidjson>=1.18,<1.21
+Requires-Dist: pytz==2025.2
+Requires-Dist: redis==5.1.1
+Requires-Dist: schedule==1.1.0
+Requires-Dist: semver==3.0.4
+Requires-Dist: sentry-sdk[fastapi]~=2.29.1
+Requires-Dist: sqlalchemy==2.0.41
+Requires-Dist: sqlalchemy-utils==0.41.2
+Requires-Dist: strawberry-graphql>=0.246.2
+Requires-Dist: structlog>=25.4.0
+Requires-Dist: tabulate==0.9.0
+Requires-Dist: typer==0.15.4
+Requires-Dist: uvicorn[standard]~=0.34.0
+Requires-Dist: celery~=5.5.1 ; extra == "celery"
+Project-URL: Documentation, https://workfloworchestrator.org/orchestrator-core
+Project-URL: Homepage, https://workfloworchestrator.org/orchestrator-core
+Project-URL: Source, https://github.com/workfloworchestrator/orchestrator-core
+Provides-Extra: celery
+
+# Orchestrator-Core
+
+[![Downloads](https://pepy.tech/badge/orchestrator-core/month)](https://pepy.tech/project/orchestrator-core)
+[![codecov](https://codecov.io/gh/workfloworchestrator/orchestrator-core/branch/main/graph/badge.svg?token=5ANQFI2DHS)](https://codecov.io/gh/workfloworchestrator/orchestrator-core)
+[![pypi_version](https://img.shields.io/pypi/v/orchestrator-core?color=%2334D058&label=pypi%20package)](https://pypi.org/project/orchestrator-core)
+[![Supported python versions](https://img.shields.io/pypi/pyversions/orchestrator-core.svg?color=%2334D058)](https://pypi.org/project/orchestrator-core)
+![Discord](https://img.shields.io/discord/1295834294270558280?style=flat&logo=discord&label=discord&link=https%3A%2F%2Fdiscord.gg%2FKNgF6gE8)
+
+<p style="text-align: center"><em>Production ready Orchestration Framework to manage product lifecycle and workflows. Easy to use, built on top of FastAPI and Pydantic</em></p>
+
+## Documentation
+
+The documentation can be found at [workfloworchestrator.org](https://workfloworchestrator.org/orchestrator-core/).
+
+## Installation (quick start)
+
+Simplified steps to install and use the orchestrator-core.
+For more details, read the [Getting started](https://workfloworchestrator.org/orchestrator-core/getting-started/base/) documentation.
+
+### Step 1 - Install the package
+
+Create a virtualenv and install the orchestrator-core.
+
+```shell
+python -m venv .venv
+source .venv/bin/activate
+pip install orchestrator-core
+```
+
+### Step 2 - Setup the database
+
+Create a postgres database:
+
+```shell
+createuser -sP nwa
+createdb orchestrator-core -O nwa  # set password to 'nwa'
+```
+
+Configure the database URI in your local environment:
+
+```
+export DATABASE_URI=postgresql://nwa:nwa@localhost:5432/orchestrator-core
+```
+
+### Step 3 - Create main.py
+
+Create a `main.py` file.
+
+```python
+from orchestrator import OrchestratorCore
+from orchestrator.cli.main import app as core_cli
+from orchestrator.settings import AppSettings
+
+app = OrchestratorCore(base_settings=AppSettings())
+
+if __name__ == "__main__":
+    core_cli()
+```
+
+### Step 4 - Run the database migrations
+
+Initialize the migration environment and database tables.
+
+```shell
+python main.py db init
+python main.py db upgrade heads
+```
+
+### Step 5 - Run the app
+
+```shell
+export OAUTH2_ACTIVE=False
+uvicorn --reload --host 127.0.0.1 --port 8080 main:app
+```
+
+Visit the [ReDoc](http://127.0.0.1:8080/api/redoc) or [OpenAPI](http://127.0.0.1:8080/api/docs) page to view and interact with the API.
+
+## Contributing
+
+We use [uv](https://docs.astral.sh/uv/getting-started/installation/) to manage dependencies.
+
+To get started, follow these steps:
+
+```shell
+# in your postgres database
+createdb orchestrator-core-test -O nwa  # set password to 'nwa'
+
+# on your local machine
+git clone https://github.com/workfloworchestrator/orchestrator-core
+cd orchestrator-core
+export DATABASE_URI=postgresql://nwa:nwa@localhost:5432/orchestrator-core-test
+uv sync --all-extras --all-groups
+uv run pytest
+```
+
+For more details please read the [development docs](https://workfloworchestrator.org/orchestrator-core/contributing/development/).
+