orchestrator-core 3.2.3__tar.gz → 4.0.0__tar.gz

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/.bumpversion.cfg

@@ -1,9 +1,9 @@
 [bumpversion]
-current_version = 3.2.3
+current_version = 4.0.0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(rc(?P<build>\d+))?
-serialize = 
+serialize =
 	{major}.{minor}.{patch}rc{build}
 	{major}.{minor}.{patch}
 

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/NOTICE

@@ -1,6 +1,7 @@
 The Orchestrator-Core
-Copyright [2018-2022] SURF, ESnet
+Copyright [2018-2025] SURF, ESnet, GÉANT
 
 This product includes software developed at
 SURF NL (https://surf.nl)
 ESnet (https://es.net)
+GÉANT Vereniging (https://geant.org)

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 3.2.3
+Version: 4.0.0
 Summary: This is the orchestrator workflow engine.
 Requires-Python: >=3.11,<3.14
 Classifier: Intended Audience :: Information Technology
@@ -35,30 +35,31 @@ Requires-Dist: deprecated
 Requires-Dist: deepmerge==2.0
 Requires-Dist: fastapi~=0.115.2
 Requires-Dist: fastapi-etag==0.4.0
-Requires-Dist: more-itertools~=10.6.0
+Requires-Dist: more-itertools~=10.7.0
 Requires-Dist: itsdangerous
 Requires-Dist: Jinja2==3.1.6
-Requires-Dist: orjson==3.10.16
+Requires-Dist: orjson==3.10.18
+Requires-Dist: prometheus-client==0.21.1
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: pydantic[email]~=2.8.2
-Requires-Dist: pydantic-settings~=2.8.0
+Requires-Dist: pydantic-settings~=2.9.1
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: python-rapidjson>=1.18,<1.21
 Requires-Dist: pytz==2025.2
 Requires-Dist: redis==5.1.1
 Requires-Dist: schedule==1.1.0
 Requires-Dist: semver==3.0.4
-Requires-Dist: sentry-sdk[fastapi]~=2.25.1
-Requires-Dist: SQLAlchemy==2.0.40
+Requires-Dist: sentry-sdk[fastapi]~=2.28.0
+Requires-Dist: SQLAlchemy==2.0.41
 Requires-Dist: SQLAlchemy-Utils==0.41.2
 Requires-Dist: structlog
-Requires-Dist: typer==0.15.2
+Requires-Dist: typer==0.15.4
 Requires-Dist: uvicorn[standard]~=0.34.0
 Requires-Dist: nwa-stdlib~=1.9.0
 Requires-Dist: oauth2-lib~=2.4.0
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: strawberry-graphql>=0.246.2
-Requires-Dist: pydantic-forms>=1.4.0, <=2.0.0
+Requires-Dist: pydantic-forms>=1.4.0, <=2.1.0
 Requires-Dist: celery~=5.5.1 ; extra == "celery"
 Requires-Dist: toml ; extra == "dev"
 Requires-Dist: bumpversion ; extra == "dev"

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/docs/architecture/application/workflow.md

@@ -204,14 +204,20 @@ Validate workflows run integrity checks on an existing subscription. Checking th
 ```python
     params = dict(
         name="validate_node_enrollment",
-        target="SYSTEM",
+        target="VALIDATE",
+        is_task=True,
         description="Validate Node Enrollment before production",
         tag="NodeEnrollment",
         search_phrase="Node Enrollment%",
     )
 ```
 
-It uses a `target` of `SYSTEM` - similar to how tasks are defined. That target is more of a free form sort of thing. Same thing with the `name` - that's the name of the actual workflow, and the `tag` is shared by of this set of workflows.
+It uses a `target` of `VALIDATE`. Unlike system tasks, which use the `target` of `SYSTEM` designation, validate
+workflows explicitly use `target="VALIDATE"` to distinguish themselves. This distinction reflects their different
+purposes.
+The `is_task` parameter is set to `True` to indicate that this workflow is a task. Tasks are workflows that are not
+directly associated with a subscription and are typically used for background processing or system maintenance.
+Both `SYSTEM` and `VALIDATE` workflows are considered tasks, but they serve different purposes.
 
 Generally the steps raise assertions if a check fails, otherwise return OK to the state:
 

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/docs/getting-started/base.md

@@ -15,29 +15,47 @@ Flask and Django, you install the core library, initialise it with configuration
 
 
 ### Step 1 - Install the package:
-Install the core.
+
+Create a virtualenv and install the core.
+
 <div class="termy">
-``` console
-$ pip install orchestrator-core
----> 100%
-Successfully installed orchestrator-core
+
+```shell
+python -m venv .venv
+source .venv/bin/activate
+pip install orchestrator-core
 ```
+
 </div>
 
 ### Step 2 - Setup the database:
+
 Create a postgres database:
 
 <div class="termy">
-``` shell
-$ createuser -sP nwa
-$ createdb orchestrator-core -O nwa
+
+```shell
+createuser -sP nwa
+createdb orchestrator-core -O nwa
 ```
+
 </div>
 
+Choose a password and remember it for later steps.
+
+As an example, you can run these docker commands in separate shells to start a temporary postgres instance:
+
+```shell
+docker run --rm --name temp-orch-db -e POSTGRES_PASSWORD=rootpassword -p 5432:5432 postgres:15
+
+docker exec -it temp-orch-db su - postgres -c 'createuser -sP nwa && createdb orchestrator-core -O nwa'
+```
+
 ### Step 3 - Create the main.py:
+
 Create a `main.py` file.
 
-``` python
+```python
 from orchestrator import OrchestratorCore
 from orchestrator.cli.main import app as core_cli
 from orchestrator.settings import AppSettings
@@ -48,37 +66,34 @@ if __name__ == "__main__":
     core_cli()
 ```
 
-
 ### Step 4 - Run the database migrations:
 
-Initialize the migration environment.
+Initialize the migration environment and database tables.
+
 <div class="termy">
-``` console
-$ PYTHONPATH=. python main.py db init
-$ PYTHONPATH=. python main.py db upgrade heads
+
+```shell
+export DATABASE_URI=postgresql://nwa:PASSWORD_FROM_STEP_2@localhost:5432/orchestrator-core
+
+python main.py db init
+python main.py db upgrade heads
 ```
+
 </div>
 
 ### Step 5 - Run the app
 
 <div class="termy">
 
-``` shell
-$ uvicorn --reload --host 127.0.0.1 --port 8080 main:app
-INFO:     Uvicorn running on http://127.0.0.1:8080 (Press CTRL+C to quit)
-INFO:     Started reloader process [62967] using watchgod
-ujson module not found, using json
-msgpack not installed, MsgPackSerializer unavailable
-2021-09-28 09:42:14 [warning  ] Database object configured, all methods referencing `db` should work. [orchestrator.db]
-INFO:     Started server process [62971]
-2021-09-28 09:42:14 [info     ] Started server process [62971] [uvicorn.error]
-INFO:     Waiting for application startup.
-2021-09-28 09:42:14 [info     ] Waiting for application startup. [uvicorn.error]
-INFO:     Application startup complete.
-2021-09-28 09:42:14 [info     ] Application startup complete.  [uvicorn.error]
+```shell
+export DATABASE_URI=postgresql://nwa:PASSWORD_FROM_STEP_2@localhost:5432/orchestrator-core
+export OAUTH2_ACTIVE=False
+
+uvicorn --reload --host 127.0.0.1 --port 8080 main:app
 ```
+
 </div>
 
 ### Step 6 - Profit :boom: :grin:
 
-Visit [the app](http://127.0.0.1:8080/api/redoc) to view the api documentation.
+Visit the [ReDoc](http://127.0.0.1:8080/api/redoc) or [OpenAPI](http://127.0.0.1:8080/api/docs) to view and interact with the API.

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/docs/getting-started/orchestration-ui.md

@@ -18,15 +18,34 @@ Before running an installation of the Workflow Orchestrator UI make sure to have
 
 ### Installation
 
--   Clone the example ui repository
-    `git clone https://github.com/workfloworchestrator/example-orchestrator-ui`
--   Install the npm packages
-    `npm i`
--   Set the correct env variables. The defaults from .env.example will work out of the box with the example orchestrator backend
-    `cp .env.example .env`
--   Run the application
-    `npm run dev`
--   The orchestrator ui now runs on http://localhost:3000
+Clone the example UI repository:
+
+```
+mkdir orchestrator-ui
+cd orchestrator-ui
+git clone https://github.com/workfloworchestrator/example-orchestrator-ui .
+```
+
+Install the npm packages:
+
+```
+npm i
+```
+
+Set the correct env variables.
+To run the UI against the dockerized example orchestrator setup, it's recommended to use the `orchestrator-ui.env` env-file.
+
+```
+wget -O .env https://raw.githubusercontent.com/workfloworchestrator/example-orchestrator/refs/heads/master/docker/orchestrator-ui/orchestrator-ui.env
+```
+
+Run the application:
+
+```
+npm run dev
+```
+
+The Orchestrator UI now runs on http://localhost:3000
 
 
 

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/docs/getting-started/prepare-source-folder.md

@@ -22,11 +22,10 @@ valid Git repository.
 
 ### Migrations folder and local Alembic head
 
-The `migrations` folder is created when running `python main.py db ini`.  This
-folder is used to store the local Alembic database migrations. The
-orchestrator-core package also contains Alembic database migrations, the Alembic
-head in the core is labeled `schema`.  The local Alembic head should be labeled
-`data`.
+The `migrations` folder is created when running `python main.py db init`.
+This folder is used to store the local Alembic database migrations.
+The orchestrator-core package also contains Alembic database migrations, the Alembic head in the core is labeled `schema`.
+The local Alembic head should be labeled `data`.
 
 When you already have product(s), workflow(s) or product template(s) defined,
 the local Alembic head can be created by running either `python main.py db

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/docs/migration-guide/2.0.md

@@ -1,7 +1,3 @@
----
-hide:
-  - navigation
----
 # 2.0 Migration Guide
 
 In this document we'll help you migrate your orchestrator application from orchestrator-core 1.3 to 2.0.

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/docs/migration-guide/3.0.md

@@ -1,7 +1,3 @@
----
-hide:
-  - navigation
----
 # 3.0 Migration Guide
 
 In this document we describe the steps that should be taken to migrate from `orchestrator-core` v2 to v3.

orchestrator_core-4.0.0/docs/migration-guide/4.0.md

@@ -0,0 +1,69 @@
+# 4.0 Migration Guide
+
+In this document we describe the steps that should be taken to migrate from `orchestrator-core` v3 to v4.
+
+## About 4.0
+
+### Removed caching of domain models
+
+In this release we have removed the caching of domain models. Domain models will always be loaded from the database.
+
+### Added Target.VALIDATE
+
+In this release, a new workflow target, `VALIDATE`, has been added
+for [validation workflows](../architecture/application/workflow.md#validate).
+Previously, the `SYSTEM` target was used for validation workflows, which implied that they were expected to run in a
+system context. However, this was not appropriate for all validation workflows.
+To address this, the new `VALIDATE` target has been introduced specifically for validation workflows. The `SYSTEM`
+target is now reserved exclusively for system workflows.
+
+The change of the `SYSTEM` target to `VALIDATE` is a breaking change, as it will break any workflows that are using the
+`SYSTEM` target for validation workflows. You will need to update your workflows to use the `VALIDATE` target instead.
+
+In the Steps section below we describe how to update your workflows to use the new `VALIDATE` target.
+
+## Steps
+
+To use 4.0.0, all workflows must have run to completion. The `cache_domain_models` step no longer is part of the codebase
+therefore `in flight` workflows will fail.
+
+After running the migration `(2025-05-08_cdf8758831d4_add_is_task_to_workflow.py)`, the workflow table should look like this:
+
+```sql
+| workflow_id                          | name                       | target | is_task | description                                                             | created_at                        | deleted_at |
+|--------------------------------------|----------------------------|--------|---------|-------------------------------------------------------------------------|-----------------------------------|------------|
+| ded79954-f16e-422b-a204-7770a59757e8 | modify_note                | MODIFY | FALSE   | Modify Note                                                             | 2025-05-01 09:57:28.033504 +00:00 | <null>     |
+| ca6a76ff-dd4e-4f66-9fb0-cee1878f0d20 | task_clean_up_tasks        | SYSTEM | FALSE   | Clean up old tasks                                                      | 2025-05-01 09:57:28.033504 +00:00 | <null>     |
+| 40058c3d-0c95-47f4-a75f-93719299c5be | task_resume_workflows      | SYSTEM | FALSE   | Resume all workflows that are stuck on tasks with the status 'waiting'  | 2025-05-01 09:57:28.033504 +00:00 | <null>     |
+| 33b5520e-85d4-4ca1-8713-d26f7de5b7a5 | task_validate_products     | SYSTEM | FALSE   | Validate products                                                       | 2025-05-01 09:57:28.033504 +00:00 | <null>     |
+| 94d4889e-6bb6-4724-a9d2-f21696fe6f43 | task_validate_product_type | SYSTEM | FALSE   | Validate all subscriptions of Product Type                              | 2025-05-01 09:57:28.033504 +00:00 | <null>     |
+| 0c4f3b8d-2a1e-4b5f-9a7c-6d8e0f1b2c3d | validate_some_thing        | SYSTEM | FALSE   | Validate The thing                                                      | 2025-05-01 09:57:28.033504 +00:00 | <null>     |
+| f4b0a2c1-5d3e-4c8f-9b6d-7a2e5f3b8c4e | validate_another_thing     | SYSTEM | FALSE   | Validate Another thing                                                  | 2025-05-01 09:57:28.033504 +00:00 | <null>     |
+```
+
+The `target` in this table is no longer valid for the `validate_some_thing` and `validate_another_thing`
+workflows. You will need to update the `target` to `VALIDATE` for these workflows. You will also need to update the
+is_task column to `TRUE` for all targets that are `SYSTEM` or `VALIDATE`. This is because the `is_task` column is used to
+determine if a workflow is a task or not. If the `is_task` column is set to `FALSE`, the workflow will not be run as a task.
+Tasks are `SYSTEM` or `VALIDATE` workflows that are run in the context of a system.
+
+Example on how to update the `target` and `is_task` for all workflows that start with `validate_`:
+
+```sql
+UPDATE workflow
+SET target = 'VALIDATE', is_task = TRUE
+WHERE name LIKE 'validate_%';
+```
+
+Example on how to update the `target` and `is_task` for all workflows that are `SYSTEM` or `VALIDATE`:
+
+```sql
+UPDATE workflow
+SET is_task = TRUE
+WHERE target IN ('SYSTEM', 'VALIDATE');
+```
+
+This will update the `target` for all workflows that are `SYSTEM` or `VALIDATE` and set the `is_task` column to `TRUE`.
+
+This is a breaking change, so you will need to test your workflows after making this change to ensure that they are
+working as expected.

orchestrator_core-4.0.0/docs/reference-docs/metrics.md

@@ -0,0 +1,45 @@
+# Collecting Metrics
+
+The ``orchestrator-core`` is capable of exporting metrics on an API endpoint that is compatible with
+[Prometheus](https://prometheus.io). Prometheus is a time-series database that can be used to collect metrics over time,
+to give insight in the usage and performance of your orchestrator, the running processes, and its subscriptions.
+
+By default, ``orchestrator-core`` exports metrics for: subscriptions, processes, and the workflow engine. These can be
+enabled by enabling the corresponding app setting ``ENABLE_PROMETHEUS_METRICS_ENDPOINT``. An API response on
+``/api/metrics`` with the default metrics enabled looks as follows:
+
+```
+# HELP wfo_subscriptions_count Number of subscriptions per product, lifecycle state, customer, and in sync state.
+# TYPE wfo_subscriptions_count gauge
+wfo_subscriptions_count{customer_id="00000000-0000-0000-0000-000000000000",insync="True",lifecycle_state="active",product_name="Router"} 53.0
+wfo_subscriptions_count{customer_id="00000000-0000-0000-0000-000000000000",insync="True",lifecycle_state="active",product_name="IP trunk"} 52.0
+wfo_subscriptions_count{customer_id="00000000-0000-0000-0000-000000000000",insync="True",lifecycle_state="active",product_name="Site"} 36.0
+wfo_subscriptions_count{customer_id="00000000-0000-0000-0000-000000000000",insync="True",lifecycle_state="terminated",product_name="Router"} 22.0
+# HELP wfo_process_count Number of processes per status, creator, task, product, workflow, customer, and target.
+# TYPE wfo_process_count gauge
+wfo_process_count{created_by="SYSTEM",customer_id="00000000-0000-0000-0000-000000000000",is_task="True",last_status="completed",product_name="IP trunk",workflow_name="validate_iptrunk",workflow_target="SYSTEM"} 5755.0
+wfo_process_count{created_by="SYSTEM",customer_id="00000000-0000-0000-0000-000000000000",is_task="True",last_status="completed",product_name="Router",workflow_name="validate_router",workflow_target="SYSTEM"} 4066.0
+wfo_process_count{created_by="SYSTEM",customer_id="12345678-1234-abcd-deff-123456789012",is_task="True",last_status="completed",product_name="Edge Port",workflow_name="validate_edge_port",workflow_target="SYSTEM"} 133.0
+# HELP wfo_process_seconds_total_count Total time spent on processes in seconds.
+# TYPE wfo_process_seconds_total_count gauge
+wfo_process_seconds_total_count{created_by="SYSTEM",customer_id="00000000-0000-0000-0000-000000000000",is_task="True",last_status="completed",product_name="IP trunk",workflow_name="validate_iptrunk",workflow_target="SYSTEM"} 3.11e+06
+wfo_process_seconds_total_count{created_by="SYSTEM",customer_id="00000000-0000-0000-0000-000000000000",is_task="True",last_status="completed",product_name="Router",workflow_name="validate_router",workflow_target="SYSTEM"} 6.72e+06
+wfo_process_seconds_total_count{created_by="SYSTEM",customer_id="12345678-1234-abcd-deff-123456789012",is_task="True",last_status="completed",product_name="Edge Port",workflow_name="validate_edge_port",workflow_target="SYSTEM"} 6514.921
+# HELP wfo_engine_status Current workflow engine status.
+# TYPE wfo_engine_status gauge
+wfo_engine_status{wfo_engine_status="PAUSED"} 0.0
+wfo_engine_status{wfo_engine_status="PAUSING"} 0.0
+wfo_engine_status{wfo_engine_status="RUNNING"} 1.0
+# HELP wfo_active_process_count Number of currently running processes in the workflow engine.
+# TYPE wfo_active_process_count gauge
+wfo_active_process_count 5.0
+```
+
+## Adding custom metrics
+
+It's possible to add more metric collectors to your orchestrator, if there are organisation-specific metrics you want
+to keep track of. This is done by implementing extra metrics from the ``prometheus_client`` library, documentation on
+how to achieve this is available [here](https://prometheus.github.io/client_python/).
+
+When your new collector is implemented, register it in the orchestrator metrics registry when initialising your
+orchestrator with ``ORCHESTRATOR_METRICS_REGISTRY.register(MyNewCollector())``.

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/mkdocs.yml

@@ -150,7 +150,7 @@ nav:
           - Prerequisites: getting-started/versions.md
           - Base Application:
                 - Preparing source folder: getting-started/prepare-source-folder.md
-                - Python Virtualenv: getting-started/base.md
+                - Base application: getting-started/base.md
           - Docker: getting-started/docker.md
           - Orchestrator UI: getting-started/orchestration-ui.md
     - Reference Documentation:
@@ -192,6 +192,7 @@ nav:
           - Migration guides:
                 - 2.x: migration-guide/2.0.md
                 - 3.x: migration-guide/3.0.md
+                - 4.x: migration-guide/4.0.md
 
     - Workshops:
           #    - Beginner:

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "3.2.3"
+__version__ = "4.0.0"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/orchestrator/api/api_v1/endpoints/settings.py

@@ -22,9 +22,10 @@ from sqlalchemy.exc import SQLAlchemyError
 from oauth2_lib.fastapi import OIDCUserModel
 from orchestrator.api.error_handling import raise_status
 from orchestrator.db import EngineSettingsTable
-from orchestrator.schemas import EngineSettingsBaseSchema, EngineSettingsSchema, GlobalStatusEnum, WorkerStatus
+from orchestrator.schemas import EngineSettingsBaseSchema, EngineSettingsSchema, WorkerStatus
 from orchestrator.security import authenticate
 from orchestrator.services import processes, settings
+from orchestrator.services.settings import generate_engine_global_status
 from orchestrator.settings import ExecutorType, app_settings
 from orchestrator.utils.json import json_dumps
 from orchestrator.utils.redis import delete_keys_matching_pattern
@@ -165,17 +166,6 @@ def generate_engine_status_response(
         Engine StatusEnum
 
     """
-
-    if engine_settings.global_lock and engine_settings.running_processes > 0:
-        result = EngineSettingsSchema.model_validate(engine_settings)
-        result.global_status = GlobalStatusEnum.PAUSING
-        return result
-
-    if engine_settings.global_lock and engine_settings.running_processes == 0:
-        result = EngineSettingsSchema.model_validate(engine_settings)
-        result.global_status = GlobalStatusEnum.PAUSED
-        return result
-
     result = EngineSettingsSchema.model_validate(engine_settings)
-    result.global_status = GlobalStatusEnum.RUNNING
+    result.global_status = generate_engine_global_status(engine_settings)
     return result

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/orchestrator/api/api_v1/endpoints/subscription_customer_descriptions.py

@@ -28,7 +28,6 @@ from orchestrator.domain.customer_description import (
 from orchestrator.schemas import SubscriptionDescriptionBaseSchema, SubscriptionDescriptionSchema
 from orchestrator.schemas.subscription_descriptions import UpdateSubscriptionDescriptionSchema
 from orchestrator.utils.errors import StaleDataError
-from orchestrator.utils.redis import delete_from_redis
 
 router = APIRouter()
 
@@ -55,7 +54,6 @@ def delete_subscription_customer_descriptions(_id: UUID) -> None:
     description = db.session.get(SubscriptionCustomerDescriptionTable, _id)
     if description:
         delete(SubscriptionCustomerDescriptionTable, _id)
-        delete_from_redis(description.subscription_id)
 
 
 @router.get("/{_id}", response_model=SubscriptionDescriptionSchema)

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/orchestrator/api/api_v1/endpoints/subscriptions.py

@@ -163,6 +163,7 @@ def subscriptions_search(
 @router.get(
     "/workflows/{subscription_id}",
     response_model=SubscriptionWorkflowListsSchema,
+    response_model_by_alias=True,
     response_model_exclude_none=True,
     deprecated=True,
     description="This endpoint is deprecated and will be removed in a future release. Please use the GraphQL query",

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/orchestrator/app.py

@@ -25,6 +25,7 @@ import structlog
 import typer
 from fastapi.applications import FastAPI
 from fastapi_etag.dependency import add_exception_handler
+from prometheus_client import make_asgi_app
 from sentry_sdk.integrations import Integration
 from sentry_sdk.integrations.asyncio import AsyncioIntegration
 from sentry_sdk.integrations.fastapi import FastApiIntegration
@@ -54,6 +55,7 @@ from orchestrator.graphql.schema import ContextGetterFactory
 from orchestrator.graphql.schemas.subscription import SubscriptionInterface
 from orchestrator.graphql.types import ScalarOverrideType, StrawberryModelType
 from orchestrator.log_config import LOGGER_OVERRIDES
+from orchestrator.metrics import ORCHESTRATOR_METRICS_REGISTRY, initialize_default_metrics
 from orchestrator.services.process_broadcast_thread import ProcessDataBroadcastThread
 from orchestrator.settings import AppSettings, ExecutorType, app_settings
 from orchestrator.version import GIT_COMMIT_HASH
@@ -143,9 +145,14 @@ class OrchestratorCore(FastAPI):
         self.add_exception_handler(ProblemDetailException, problem_detail_handler)  # type: ignore[arg-type]
         add_exception_handler(self)
 
+        if base_settings.ENABLE_PROMETHEUS_METRICS_ENDPOINT:
+            initialize_default_metrics()
+            metrics_app = make_asgi_app(registry=ORCHESTRATOR_METRICS_REGISTRY)
+            self.mount("/api/metrics", metrics_app)
+
         @self.router.get("/", response_model=str, response_class=JSONResponse, include_in_schema=False)
         def _index() -> str:
-            return "Orchestrator orchestrator"
+            return "Orchestrator Core"
 
     def add_sentry(
         self,
@@ -154,6 +161,7 @@ class OrchestratorCore(FastAPI):
         server_name: str,
         environment: str,
         release: str | None = GIT_COMMIT_HASH,
+        **sentry_kwargs: Any,
     ) -> None:
         logger.info("Adding Sentry middleware to app", app=self.title)
         if self.base_settings.EXECUTOR == ExecutorType.WORKER:
@@ -173,6 +181,7 @@ class OrchestratorCore(FastAPI):
             integrations=sentry_integrations,
             propagate_traces=True,
             profiles_sample_rate=trace_sample_rate,
+            **sentry_kwargs,
         )
 
     @staticmethod

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/orchestrator/cli/generator/templates/new_product_migration.j2

@@ -63,24 +63,28 @@ new_workflows = [
     {
         "name": "create_{{ product.variable }}",
         "target": Target.CREATE,
+        "is_task": False,
         "description": "Create {{ product.name }}",
         "product_type": "{{ product.type }}",
     },
     {
         "name": "modify_{{ product.variable }}",
         "target": Target.MODIFY,
+        "is_task": False,
         "description": "Modify {{ product.name }}",
         "product_type": "{{ product.type }}",
     },
     {
         "name": "terminate_{{ product.variable }}",
         "target": Target.TERMINATE,
+        "is_task": False,
         "description": "Terminate {{ product.name }}",
         "product_type": "{{ product.type }}",
     },
     {
         "name": "validate_{{ product.variable }}",
-        "target": Target.SYSTEM,
+        "target": Target.VALIDATE,
+        "is_task": True,
         "description": "Validate {{ product.name }}",
         "product_type": "{{ product.type }}",
     },

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/orchestrator/cli/migrate_tasks.py

@@ -150,22 +150,22 @@ def create_tasks_migration_wizard() -> tuple[list[dict], list[dict]]:
         - list of task items to add in the migration
         - list of task items to delete in the migration
     """
-    database_tasks = {
-        task.name: task for task in list(db.session.scalars(select(WorkflowTable))) if task.target == Target.SYSTEM
-    }
+    database_tasks = {task.name: task for task in list(db.session.scalars(select(WorkflowTable))) if task.is_task}
     registered_wf_instances = {
         task: cast(Workflow, get_workflow(task)) for task in orchestrator.workflows.ALL_WORKFLOWS.keys()
     }
 
+    is_task = [Target.SYSTEM, Target.VALIDATE]
+
     registered_tasks = dict(
         filter(
-            lambda task: task[1].target == Target.SYSTEM and task[0] in database_tasks.keys(),
+            lambda task: task[1].target in is_task and task[0] in database_tasks.keys(),
             registered_wf_instances.items(),
         )
     )
     available_tasks = dict(
         filter(
-            lambda task: task[1].target == Target.SYSTEM and task[0] not in database_tasks.keys(),
+            lambda task: task[1].target in is_task and task[0] not in database_tasks.keys(),
             registered_wf_instances.items(),
         )
     )

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/orchestrator/cli/migrate_workflows.py

@@ -11,7 +11,6 @@ import orchestrator.workflows
 from orchestrator.cli.helpers.input_helpers import _enumerate_menu_keys, _prompt_user_menu, get_user_input
 from orchestrator.cli.helpers.print_helpers import COLOR, noqa_print, print_fmt, str_fmt
 from orchestrator.db import ProductTable, WorkflowTable, db
-from orchestrator.targets import Target
 from orchestrator.workflows import LazyWorkflowInstance, get_workflow
 
 # Workflows are registered via migrations with product type. For every product with the given product_type, there will be an entry in products_workflows.
@@ -183,7 +182,7 @@ def create_workflows_migration_wizard() -> tuple[list[dict], list[dict]]:
     """
     database_workflows = list(db.session.scalars(select(WorkflowTable)))
     registered_workflows = orchestrator.workflows.ALL_WORKFLOWS
-    system_workflow_names = {wf.name for wf in database_workflows if wf.target == Target.SYSTEM}
+    system_workflow_names = {wf.name for wf in database_workflows if wf.is_task}
     registered_non_system_workflows = {k: v for k, v in registered_workflows.items() if k not in system_workflow_names}
 
     unregistered_workflows = [wf for wf in database_workflows if wf.name not in registered_workflows.keys()]

{orchestrator_core-3.2.3 → orchestrator_core-4.0.0}/orchestrator/db/models.py

@@ -400,7 +400,7 @@ class WorkflowTable(BaseModel):
     workflow_id = mapped_column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True)
     name = mapped_column(String(), nullable=False, unique=True)
     target = mapped_column(String(), nullable=False)
-    description = mapped_column(Text(), nullable=True)
+    description = mapped_column(Text(), nullable=False)
     created_at = mapped_column(UtcTimestamp, nullable=False, server_default=text("current_timestamp()"))
     deleted_at = mapped_column(UtcTimestamp, deferred=True)
 
@@ -412,6 +412,8 @@ class WorkflowTable(BaseModel):
     )
     processes = relationship("ProcessTable", cascade="all, delete-orphan", back_populates="workflow")
 
+    is_task = mapped_column(Boolean, nullable=False, server_default=text("false"))
+
     @staticmethod
     def select() -> Select:
         return (