orchestrator-core 3.1.0rc1__tar.gz → 3.1.2rc1__tar.gz

{orchestrator_core-3.1.0rc1 → orchestrator_core-3.1.2rc1}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.1.0rc1
+current_version = 3.1.1rc1
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(rc(?P<build>\d+))?

{orchestrator_core-3.1.0rc1 → orchestrator_core-3.1.2rc1}/.github/workflows/run-codspeed-tests.yml

@@ -11,7 +11,7 @@ jobs:
     name: Run benchmarks
     runs-on: ubuntu-latest
     container:
-      image: python:3.11
+      image: python:3.13
       options: --privileged
     services:
       postgres:

{orchestrator_core-3.1.0rc1 → orchestrator_core-3.1.2rc1}/.github/workflows/run-linting-tests.yml

@@ -5,9 +5,9 @@ name: Linting tests
 
 on:
   push:
-  pull_request:
-    branches: [master]
+    branches: [ main ]
   workflow_call:
+  pull_request:
 
 jobs:
   build:

{orchestrator_core-3.1.0rc1 → orchestrator_core-3.1.2rc1}/.github/workflows/run-unit-tests.yml

@@ -1,7 +1,9 @@
 name: Unit tests
 on:
   push:
+    branches: [ main ]
   workflow_call:
+  pull_request:
 
 jobs:
   container_job:

orchestrator_core-3.1.2rc1/PKG-INFO

@@ -0,0 +1,321 @@
+Metadata-Version: 2.4
+Name: orchestrator-core
+Version: 3.1.2rc1
+Summary: This is the orchestrator workflow engine.
+Author-email: SURF <automation-beheer@surf.nl>
+Requires-Python: >=3.11,<3.14
+Description-Content-Type: text/markdown
+License-Expression: Apache-2.0
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python
+Classifier: Topic :: Internet
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development
+Classifier: Typing :: Typed
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Web Environment
+Classifier: Framework :: AsyncIO
+Classifier: Framework :: FastAPI
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Telecommunications Industry
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
+Classifier: Topic :: Internet :: WWW/HTTP
+License-File: LICENSE
+Requires-Dist: alembic==1.15.1
+Requires-Dist: anyio>=3.7.0
+Requires-Dist: click==8.*
+Requires-Dist: deprecated
+Requires-Dist: deepmerge==2.0
+Requires-Dist: fastapi~=0.115.2
+Requires-Dist: fastapi-etag==0.4.0
+Requires-Dist: more-itertools~=10.6.0
+Requires-Dist: itsdangerous
+Requires-Dist: Jinja2==3.1.6
+Requires-Dist: orjson==3.10.15
+Requires-Dist: psycopg[binary]==3.2.6
+Requires-Dist: pydantic[email]~=2.10.6
+Requires-Dist: pydantic-settings~=2.8.0
+Requires-Dist: python-dateutil==2.8.2
+Requires-Dist: python-rapidjson>=1.18,<1.21
+Requires-Dist: pytz==2025.1
+Requires-Dist: redis==5.1.1
+Requires-Dist: schedule==1.1.0
+Requires-Dist: sentry-sdk[fastapi]~=2.22.0
+Requires-Dist: SQLAlchemy==2.0.39
+Requires-Dist: SQLAlchemy-Utils==0.41.2
+Requires-Dist: structlog
+Requires-Dist: typer==0.15.2
+Requires-Dist: uvicorn[standard]~=0.34.0
+Requires-Dist: nwa-stdlib~=1.9.0
+Requires-Dist: oauth2-lib~=2.4.0
+Requires-Dist: tabulate==0.9.0
+Requires-Dist: strawberry-graphql>=0.246.2
+Requires-Dist: pydantic-forms~=1.4.0
+Requires-Dist: celery~=5.4.0 ; extra == "celery"
+Requires-Dist: toml ; extra == "dev"
+Requires-Dist: bumpversion ; extra == "dev"
+Requires-Dist: mypy_extensions ; extra == "dev"
+Requires-Dist: pre-commit ; extra == "dev"
+Requires-Dist: pydocstyle ; extra == "dev"
+Requires-Dist: python-dotenv ; extra == "dev"
+Requires-Dist: watchdog ; extra == "dev"
+Requires-Dist: mkdocs ; extra == "doc"
+Requires-Dist: mkdocs-material[imaging] ; extra == "doc"
+Requires-Dist: mkdocs-render-swagger-plugin ; extra == "doc"
+Requires-Dist: mkdocs-include-markdown-plugin ; extra == "doc"
+Requires-Dist: mkdocstrings[python] ; extra == "doc"
+Requires-Dist: mkdocs-open-in-new-tab ; extra == "doc"
+Requires-Dist: mkdocs-macros-plugin ; extra == "doc"
+Requires-Dist: mkdocs-embed-external-markdown ; extra == "doc"
+Requires-Dist: apache-license-check ; extra == "test"
+Requires-Dist: black ; extra == "test"
+Requires-Dist: blinker ; extra == "test"
+Requires-Dist: deepdiff ; extra == "test"
+Requires-Dist: dirty-equals ; extra == "test"
+Requires-Dist: jsonref ; extra == "test"
+Requires-Dist: mypy==1.9 ; extra == "test"
+Requires-Dist: pyinstrument ; extra == "test"
+Requires-Dist: pytest==8.3.5 ; extra == "test"
+Requires-Dist: pytest-asyncio==0.21.2 ; extra == "test"
+Requires-Dist: pytest-codspeed ; extra == "test"
+Requires-Dist: pytest-cov ; extra == "test"
+Requires-Dist: pytest-httpx ; extra == "test"
+Requires-Dist: pytest-xdist ; extra == "test"
+Requires-Dist: requests-mock ; extra == "test"
+Requires-Dist: ruff ; extra == "test"
+Requires-Dist: sqlalchemy[mypy] ; extra == "test"
+Requires-Dist: urllib3-mock ; extra == "test"
+Requires-Dist: types-Deprecated ; extra == "test"
+Requires-Dist: types-Jinja2 ; extra == "test"
+Requires-Dist: types-aiofiles ; extra == "test"
+Requires-Dist: types-certifi ; extra == "test"
+Requires-Dist: types-click ; extra == "test"
+Requires-Dist: types-itsdangerous ; extra == "test"
+Requires-Dist: types-orjson ; extra == "test"
+Requires-Dist: types-python-dateutil ; extra == "test"
+Requires-Dist: types-pytz ; extra == "test"
+Requires-Dist: types-redis ; extra == "test"
+Requires-Dist: types-requests ; extra == "test"
+Requires-Dist: types-setuptools ; extra == "test"
+Requires-Dist: types-tabulate ; extra == "test"
+Requires-Dist: types-toml ; extra == "test"
+Requires-Dist: types-ujson ; extra == "test"
+Requires-Dist: types-PyYAML ; extra == "test"
+Project-URL: documentation, https://workfloworchestrator.org/orchestrator-core/
+Project-URL: releasenotes, https://github.com/workfloworchestrator/orchestrator-core/releases
+Project-URL: source, https://github.com/workfloworchestrator/orchestrator-core
+Provides-Extra: celery
+Provides-Extra: dev
+Provides-Extra: doc
+Provides-Extra: test
+
+# Orchestrator-Core
+[![Downloads](https://pepy.tech/badge/orchestrator-core/month)](https://pepy.tech/project/orchestrator-core)
+[![codecov](https://codecov.io/gh/workfloworchestrator/orchestrator-core/branch/main/graph/badge.svg?token=5ANQFI2DHS)](https://codecov.io/gh/workfloworchestrator/orchestrator-core)
+[![pypi_version](https://img.shields.io/pypi/v/orchestrator-core?color=%2334D058&label=pypi%20package)](https://pypi.org/project/orchestrator-core)
+[![Supported python versions](https://img.shields.io/pypi/pyversions/orchestrator-core.svg?color=%2334D058)](https://pypi.org/project/orchestrator-core)
+![Discord](https://img.shields.io/discord/1295834294270558280?style=flat&logo=discord&label=discord&link=https%3A%2F%2Fdiscord.gg%2FKNgF6gE8)
+
+<p style="text-align: center"><em>Production ready Orchestration Framework to manage product lifecycle and workflows. Easy to use, Built on top of FastAPI</em></p>
+
+
+## Documentation
+Can be found [here](https://workfloworchestrator.org/orchestrator-core/)
+
+## Usage
+This project can be installed as follows:
+
+#### Step 1:
+Install the core.
+```shell
+pip install orchestrator-core
+```
+
+#### Step 2:
+Create a postgres database:
+```shell
+createuser -sP nwa
+createdb orchestrator-core -O nwa
+```
+
+#### Step 3 (optional):
+When using multiple workers, you will need a redis server for live updates with websockets.
+
+By default it will use memory which works with only one worker.
+```shell
+export WEBSOCKET_BROADCASTER_URL="memory://"
+```
+
+For the redis connection you need to set the env variable with the connection url.
+```shell
+export WEBSOCKET_BROADCASTER_URL="redis://localhost:6379"
+```
+
+
+Websockets can also be turned off with:
+```shell
+export ENABLE_WEBSOCKETS=False
+```
+
+If you want to use pickle for CACHE serialization you will need to set the `CACHE_HMAC_SECRET`:
+```shell
+export CACHE_HMAC_SECRET="SOMESECRET"
+```
+**NOTE**: The key can be any length. However, the recommended size is 1024 bits.
+
+#### Step 4:
+Create a `main.py` file.
+
+```python
+from orchestrator import OrchestratorCore
+from orchestrator.cli.main import app as core_cli
+from orchestrator.settings import AppSettings
+
+app = OrchestratorCore(base_settings=AppSettings())
+
+if __name__ == "__main__":
+    core_cli()
+```
+
+#### Step 5 (Optional):
+OrchestratorCore comes with a graphql interface that can to be registered after you create your OrchestratorApp.
+If you add it after registering your `SUBSCRIPTION_MODEL_REGISTRY` it will automatically create graphql types for them.
+More info can be found in `docs/architecture/application/graphql.md`
+
+example:
+```python
+from orchestrator import OrchestratorCore
+from orchestrator.settings import AppSettings
+
+app = OrchestratorCore(base_settings=AppSettings())
+# register SUBSCRIPTION_MODEL_REGISTRY
+app.register_graphql()
+```
+
+#### Step 6:
+Initialize the migration environment.
+```shell
+PYTHONPATH=. python main.py db init
+PYTHONPATH=. python main.py db upgrade heads
+```
+
+### Step 7:
+Profit :)
+
+Authentication and authorization are default enabled, to disable set `OAUTH2_ACTIVE` and `OAUTH2_AUTHORIZATION_ACTIVE` to `False`.
+
+```shell
+uvicorn --reload --host 127.0.0.1 --port 8080 main:app
+```
+
+Visit [http://127.0.0.1:8080/api/redoc](http://127.0.0.1:8080/api/redoc) to view the api documentation.
+
+
+## Setting up a development environment
+
+To add features to the repository follow the following procedure to setup a working development environment.
+
+### Installation (Development standalone)
+Install the project and its dependencies to develop on the code.
+
+#### Step 1 - install flit:
+
+```shell
+python3 -m venv venv
+source venv/bin/activate
+pip install flit
+```
+
+#### Step 2 - install the development code:
+
+!!! danger
+    Make sure to use the flit binary that is installed in your environment. You can check the correct
+    path by running
+    ```shell
+    which flit
+    ```
+
+To be sure that the packages will be installed against the correct venv you can also prepend the python interpreter
+that you want to use:
+
+```shell
+flit install --deps develop --symlink --python venv/bin/python
+```
+
+
+### Running tests
+Run the unit-test suite to verify a correct setup.
+
+#### Step 1 - Create a database
+
+```shell
+createuser -sP nwa
+createdb orchestrator-core-test -O nwa
+```
+
+#### Step 2 - Run tests
+```shell
+pytest test/unit_tests
+```
+or with xdist:
+
+```shell
+pytest -n auto test/unit_tests
+```
+
+If you do not encounter any failures in the test, you should be able to develop features in the orchestrator-core.
+
+### Installation (Development symlinked into orchestrator SURF)
+
+If you are working on a project that already uses the `orchestrator-core` and you want to test your new core features
+against it, you can use some `flit` magic to symlink the dev version of the core to your project. It will
+automatically replace the pypi dep with a symlink to the development version
+of the core and update/downgrade all required packages in your own orchestrator project.
+
+#### Step 1 - install flit:
+
+```shell
+python - m venv venv
+source venv/bin/activate
+pip install flit
+```
+
+### Step 2 - symlink the core to your own project
+
+```shell
+flit install --deps develop --symlink --python /path/to/a/orchestrator-project/venv/bin/python
+```
+
+So if you have the core and your own orchestrator project repo in the same folder and the main project folder is
+`orchestrator` and you want to use relative links, this will be last step:
+
+```shell
+flit install --deps develop --symlink --python ../orchestrator/venv/bin/python
+```
+
+# Increasing the version number for a (pre) release.
+
+When your PR is accepted you will get a version number.
+
+You can do the necessary change with a clean, e.g. every change committed, branch:
+
+```shell
+bumpversion patch --new-version 0.4.1-rc3
+```
+
+### Changing the Core database schema
+When you would like to change the core database schema, execute the following steps.
+
+- Create the new model `orchestrator/database/models.py`
+- `cd orchestrator/migrations`
+- `alembic revision --autogenerate -m "Name of the migratioin"`
+

{orchestrator_core-3.1.0rc1 → orchestrator_core-3.1.2rc1}/docs/reference-docs/app/scaling.md

@@ -108,6 +108,112 @@ The code above sets our local Celery instance (which initializes the workflows)
 going to be used by the orchestrator-core. Without this code, the orchestrator-core would be only aware of a limited
 set of workflows that are part of orchestrator-core itself.
 
+### An example implementation
+
+When using Celery and Websockets you can use the following example and change it to your needs.
+
+```python
+"""This module contains functions and classes necessary for celery worker processes.
+
+When the orchestrator-core's thread process executor is specified as "celery", the `OrchestratorCore` FastAPI
+application registers celery-specific task functions and `start_process` and `resume_process` now defer to the
+celery task queue.
+
+Celery's task queue enables features like nightly validations by providing a task queue and workers to execute
+workflows that are all started in parallel, which would crash a single-threaded orchestrator-core.
+
+The application flow looks like this when "celery" is the executor (and websockets are enabled):
+
+- FastAPI application validates form input, and places a task on celery queue (create new process).
+  - If websockets are enabled, a connection should exist already b/t the client and backend.
+- FastAPI application begins watching redis pubsub channel for process updates from celery.
+- Celery worker picks up task from queue and begins executing.
+- On each step completion, it publishes state information to redis pubsub channel.
+- FastAPI application grabs this information and publishes it to the client websocket connection.
+
+A celery worker container will start by calling this module instead of `main.py` like so:
+```sh
+celery -A esnetorch.celery_worker worker -E -l INFO -Q new_tasks,resume_tasks,new_workflows,resume_workflows
+```
+
+* `-A` points to this module where the worker class is defined
+* `-E` sends task-related events (capturable and monitorable)
+* `-l` is the short flag for --loglevel
+* `-Q` specifies the queues which the worker should watch for new tasks
+
+See https://workfloworchestrator.org/orchestrator-core/reference-docs/app/scaling for more information.
+"""
+
+from uuid import UUID
+
+from celery import Celery
+from celery.signals import worker_shutting_down
+from orchestrator.db import init_database
+from orchestrator.domain import SUBSCRIPTION_MODEL_REGISTRY
+from orchestrator.services.tasks import initialise_celery
+from orchestrator.types import BroadcastFunc
+from orchestrator.websocket import broadcast_process_update_to_websocket, init_websocket_manager
+from orchestrator.websocket.websocket_manager import WebSocketManager
+from orchestrator.workflows import ALL_WORKFLOWS
+from structlog import get_logger
+
+from esnetorch.settings import app_settings
+
+
+logger = get_logger(__name__)
+
+
+def process_broadcast_fn(process_id: UUID) -> None:
+    # Catch all exceptions as broadcasting failure is noncritical to workflow completion
+    try:
+        broadcast_process_update_to_websocket(process_id)
+    except Exception as e:
+        logger.exception(e)
+
+
+class OrchestratorWorker(Celery):
+    websocket_manager: WebSocketManager
+    process_broadcast_fn: BroadcastFunc
+
+    def on_init(self) -> None:
+        init_database(app_settings)
+
+        # Prepare the wrapped_websocket_manager
+        # Note: cannot prepare the redis connections here as broadcasting is async
+        self.websocket_manager = init_websocket_manager(app_settings)
+        self.process_broadcast_fn = process_broadcast_fn
+
+        # Load the products and load the workflows
+        import esnetorch.products  # noqa: F401  Side-effects
+        import esnetorch.workflows  # noqa: F401  Side-effects
+
+        logger.info(
+            "Loaded the workflows and products",
+            workflows=len(ALL_WORKFLOWS.values()),
+            products=len(SUBSCRIPTION_MODEL_REGISTRY.values()),
+        )
+
+    def close(self) -> None:
+        super().close()
+
+
+celery = OrchestratorWorker(
+    f"{app_settings.SERVICE_NAME}-worker", broker=str(app_settings.CACHE_URI), include=["orchestrator.services.tasks"]
+)
+
+if app_settings.TESTING:
+    celery.conf.update(backend=str(app_settings.CACHE_URI), task_ignore_result=False)
+else:
+    celery.conf.update(task_ignore_result=True)
+
+celery.conf.update(
+    result_expires=3600,
+    worker_prefetch_multiplier=1,
+    worker_send_task_event=True,
+    task_send_sent_event=True,
+)
+```
+
 ### Running locally
 
 If you want to test your application locally you have to start both the orchestrator-api and one or more workers.

{orchestrator_core-3.1.0rc1 → orchestrator_core-3.1.2rc1}/orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "3.1.0rc1"
+__version__ = "3.1.2rc1"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings

{orchestrator_core-3.1.0rc1 → orchestrator_core-3.1.2rc1}/orchestrator/db/models.py

@@ -92,7 +92,7 @@ class InputStateTable(BaseModel):
 
     input_state_id = mapped_column(UUIDType, primary_key=True, server_default=text("uuid_generate_v4()"), index=True)
     process_id = mapped_column("pid", UUIDType, ForeignKey("processes.pid"), nullable=False)
-    input_state = mapped_column(pg.JSONB(), nullable=False)  # type: ignore
+    input_state = mapped_column(pg.JSONB(), nullable=False)
     input_time = mapped_column(UtcTimestamp, server_default=text("current_timestamp()"), nullable=False)
     input_type = mapped_column(Enum(InputType), nullable=False)
 
@@ -137,7 +137,7 @@ class ProcessStepTable(BaseModel):
     )
     name = mapped_column(String(), nullable=False)
     status = mapped_column(String(50), nullable=False)
-    state = mapped_column(pg.JSONB(), nullable=False)  # type: ignore
+    state = mapped_column(pg.JSONB(), nullable=False)
     created_by = mapped_column(String(255), nullable=True)
     executed_at = mapped_column(UtcTimestamp, server_default=text("statement_timestamp()"), nullable=False)
     commit_hash = mapped_column(String(40), nullable=True, default=GIT_COMMIT_HASH)
@@ -642,7 +642,7 @@ class SubscriptionMetadataTable(BaseModel):
         primary_key=True,
         index=True,
     )
-    metadata_ = mapped_column("metadata", pg.JSONB(), nullable=False)  # type: ignore
+    metadata_ = mapped_column("metadata", pg.JSONB(), nullable=False)
 
     @staticmethod
     def find_by_subscription_id(subscription_id: str) -> SubscriptionMetadataTable | None:
@@ -651,6 +651,8 @@ class SubscriptionMetadataTable(BaseModel):
 
 class SubscriptionSearchView(BaseModel):
     __tablename__ = "subscriptions_search"
+    __table_args__ = {"info": {"materialized_view": True}}
+
     subscription_id = mapped_column(
         UUIDType, ForeignKey("subscriptions.subscription_id"), nullable=False, index=True, primary_key=True
     )

{orchestrator_core-3.1.0rc1 → orchestrator_core-3.1.2rc1}/orchestrator/migrations/env.py

@@ -41,6 +41,14 @@ target_metadata = BaseModel.metadata
 # ... etc.
 
 
+def include_object(object, name, type_, reflected, compare_to):  # type: ignore
+    """Determines if an object should be included."""
+
+    if type_ == "table" and object.info.get("materialized_view", False):
+        return False
+    return True
+
+
 def run_migrations_offline() -> None:
     """Run migrations in 'offline' mode.
 
@@ -55,7 +63,11 @@ def run_migrations_offline() -> None:
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        include_object=include_object,
     )
 
     with context.begin_transaction():
@@ -90,6 +102,7 @@ def run_migrations_online() -> None:
         target_metadata=target_metadata,
         process_revision_directives=process_revision_directives,
         compare_type=True,
+        include_object=include_object,
     )
 
     try:

{orchestrator_core-3.1.0rc1 → orchestrator_core-3.1.2rc1}/orchestrator/migrations/versions/schema/2020-10-19_c112305b07d3_initial_schema_migration.py

@@ -157,7 +157,7 @@ def upgrade() -> None:
         sa.Column("pid", sqlalchemy_utils.types.uuid.UUIDType(), nullable=False),
         sa.Column("name", sa.String(), nullable=False),
         sa.Column("status", sa.String(length=50), nullable=False),
-        sa.Column("state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),  # type: ignore
+        sa.Column("state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
         sa.Column("created_by", sa.String(length=255), nullable=True),
         sa.Column(
             "executed_at",

{orchestrator_core-3.1.0rc1 → orchestrator_core-3.1.2rc1}/orchestrator/migrations/versions/schema/2023-05-25_b1970225392d_add_subscription_metadata_workflow.py

@@ -29,7 +29,7 @@ def upgrade() -> None:
             nullable=False,
             index=True,
         ),
-        sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False),  # type: ignore
+        sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
         sa.ForeignKeyConstraint(["subscription_id"], ["subscriptions.subscription_id"], ondelete="CASCADE"),
     )
 

{orchestrator_core-3.1.0rc1 → orchestrator_core-3.1.2rc1}/orchestrator/migrations/versions/schema/2025-02-12_bac6be6f2b4f_added_input_state_table.py

@@ -31,7 +31,7 @@ def upgrade() -> None:
             nullable=False,
         ),
         sa.Column("pid", sqlalchemy_utils.types.uuid.UUIDType(), nullable=False),
-        sa.Column("input_state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),  # type: ignore
+        sa.Column("input_state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
         sa.Column(
             "input_time",
             db.models.UtcTimestamp(timezone=True),