orchestrator-core 4.0.0rc2__tar.gz → 4.0.0rc4__tar.gz

{orchestrator_core-4.0.0rc2 → orchestrator_core-4.0.0rc4}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.0.0rc2
+current_version = 4.0.0rc4
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(rc(?P<build>\d+))?

{orchestrator_core-4.0.0rc2 → orchestrator_core-4.0.0rc4}/NOTICE

@@ -1,6 +1,7 @@
 The Orchestrator-Core
-Copyright [2018-2022] SURF, ESnet
+Copyright [2018-2025] SURF, ESnet, GÉANT
 
 This product includes software developed at
 SURF NL (https://surf.nl)
 ESnet (https://es.net)
+GÉANT Vereniging (https://geant.org)

{orchestrator_core-4.0.0rc2 → orchestrator_core-4.0.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 4.0.0rc2
+Version: 4.0.0rc4
 Summary: This is the orchestrator workflow engine.
 Requires-Python: >=3.11,<3.14
 Classifier: Intended Audience :: Information Technology
@@ -39,6 +39,7 @@ Requires-Dist: more-itertools~=10.7.0
 Requires-Dist: itsdangerous
 Requires-Dist: Jinja2==3.1.6
 Requires-Dist: orjson==3.10.18
+Requires-Dist: prometheus-client==0.21.1
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: pydantic[email]~=2.8.2
 Requires-Dist: pydantic-settings~=2.9.1

orchestrator_core-4.0.0rc4/docs/reference-docs/metrics.md

@@ -0,0 +1,45 @@
+# Collecting Metrics
+
+The ``orchestrator-core`` is capable of exporting metrics on an API endpoint that is compatible with
+[Prometheus](https://prometheus.io). Prometheus is a time-series database that can be used to collect metrics over time,
+to give insight in the usage and performance of your orchestrator, the running processes, and its subscriptions.
+
+By default, ``orchestrator-core`` exports metrics for: subscriptions, processes, and the workflow engine. These can be
+enabled by enabling the corresponding app setting ``ENABLE_PROMETHEUS_METRICS_ENDPOINT``. An API response on
+``/api/metrics`` with the default metrics enabled looks as follows:
+
+```
+# HELP wfo_subscriptions_count Number of subscriptions per product, lifecycle state, customer, and in sync state.
+# TYPE wfo_subscriptions_count gauge
+wfo_subscriptions_count{customer_id="00000000-0000-0000-0000-000000000000",insync="True",lifecycle_state="active",product_name="Router"} 53.0
+wfo_subscriptions_count{customer_id="00000000-0000-0000-0000-000000000000",insync="True",lifecycle_state="active",product_name="IP trunk"} 52.0
+wfo_subscriptions_count{customer_id="00000000-0000-0000-0000-000000000000",insync="True",lifecycle_state="active",product_name="Site"} 36.0
+wfo_subscriptions_count{customer_id="00000000-0000-0000-0000-000000000000",insync="True",lifecycle_state="terminated",product_name="Router"} 22.0
+# HELP wfo_process_count Number of processes per status, creator, task, product, workflow, customer, and target.
+# TYPE wfo_process_count gauge
+wfo_process_count{created_by="SYSTEM",customer_id="00000000-0000-0000-0000-000000000000",is_task="True",last_status="completed",product_name="IP trunk",workflow_name="validate_iptrunk",workflow_target="SYSTEM"} 5755.0
+wfo_process_count{created_by="SYSTEM",customer_id="00000000-0000-0000-0000-000000000000",is_task="True",last_status="completed",product_name="Router",workflow_name="validate_router",workflow_target="SYSTEM"} 4066.0
+wfo_process_count{created_by="SYSTEM",customer_id="12345678-1234-abcd-deff-123456789012",is_task="True",last_status="completed",product_name="Edge Port",workflow_name="validate_edge_port",workflow_target="SYSTEM"} 133.0
+# HELP wfo_process_seconds_total_count Total time spent on processes in seconds.
+# TYPE wfo_process_seconds_total_count gauge
+wfo_process_seconds_total_count{created_by="SYSTEM",customer_id="00000000-0000-0000-0000-000000000000",is_task="True",last_status="completed",product_name="IP trunk",workflow_name="validate_iptrunk",workflow_target="SYSTEM"} 3.11e+06
+wfo_process_seconds_total_count{created_by="SYSTEM",customer_id="00000000-0000-0000-0000-000000000000",is_task="True",last_status="completed",product_name="Router",workflow_name="validate_router",workflow_target="SYSTEM"} 6.72e+06
+wfo_process_seconds_total_count{created_by="SYSTEM",customer_id="12345678-1234-abcd-deff-123456789012",is_task="True",last_status="completed",product_name="Edge Port",workflow_name="validate_edge_port",workflow_target="SYSTEM"} 6514.921
+# HELP wfo_engine_status Current workflow engine status.
+# TYPE wfo_engine_status gauge
+wfo_engine_status{wfo_engine_status="PAUSED"} 0.0
+wfo_engine_status{wfo_engine_status="PAUSING"} 0.0
+wfo_engine_status{wfo_engine_status="RUNNING"} 1.0
+# HELP wfo_active_process_count Number of currently running processes in the workflow engine.
+# TYPE wfo_active_process_count gauge
+wfo_active_process_count 5.0
+```
+
+## Adding custom metrics
+
+It's possible to add more metric collectors to your orchestrator, if there are organisation-specific metrics you want
+to keep track of. This is done by implementing extra metrics from the ``prometheus_client`` library, documentation on
+how to achieve this is available [here](https://prometheus.github.io/client_python/).
+
+When your new collector is implemented, register it in the orchestrator metrics registry when initialising your
+orchestrator with ``ORCHESTRATOR_METRICS_REGISTRY.register(MyNewCollector())``.

{orchestrator_core-4.0.0rc2 → orchestrator_core-4.0.0rc4}/orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "4.0.0rc2"
+__version__ = "4.0.0rc4"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings

{orchestrator_core-4.0.0rc2 → orchestrator_core-4.0.0rc4}/orchestrator/api/api_v1/endpoints/settings.py

@@ -22,9 +22,10 @@ from sqlalchemy.exc import SQLAlchemyError
 from oauth2_lib.fastapi import OIDCUserModel
 from orchestrator.api.error_handling import raise_status
 from orchestrator.db import EngineSettingsTable
-from orchestrator.schemas import EngineSettingsBaseSchema, EngineSettingsSchema, GlobalStatusEnum, WorkerStatus
+from orchestrator.schemas import EngineSettingsBaseSchema, EngineSettingsSchema, WorkerStatus
 from orchestrator.security import authenticate
 from orchestrator.services import processes, settings
+from orchestrator.services.settings import generate_engine_global_status
 from orchestrator.settings import ExecutorType, app_settings
 from orchestrator.utils.json import json_dumps
 from orchestrator.utils.redis import delete_keys_matching_pattern
@@ -165,17 +166,6 @@ def generate_engine_status_response(
         Engine StatusEnum
 
     """
-
-    if engine_settings.global_lock and engine_settings.running_processes > 0:
-        result = EngineSettingsSchema.model_validate(engine_settings)
-        result.global_status = GlobalStatusEnum.PAUSING
-        return result
-
-    if engine_settings.global_lock and engine_settings.running_processes == 0:
-        result = EngineSettingsSchema.model_validate(engine_settings)
-        result.global_status = GlobalStatusEnum.PAUSED
-        return result
-
     result = EngineSettingsSchema.model_validate(engine_settings)
-    result.global_status = GlobalStatusEnum.RUNNING
+    result.global_status = generate_engine_global_status(engine_settings)
     return result

{orchestrator_core-4.0.0rc2 → orchestrator_core-4.0.0rc4}/orchestrator/app.py

@@ -25,6 +25,7 @@ import structlog
 import typer
 from fastapi.applications import FastAPI
 from fastapi_etag.dependency import add_exception_handler
+from prometheus_client import make_asgi_app
 from sentry_sdk.integrations import Integration
 from sentry_sdk.integrations.asyncio import AsyncioIntegration
 from sentry_sdk.integrations.fastapi import FastApiIntegration
@@ -54,6 +55,7 @@ from orchestrator.graphql.schema import ContextGetterFactory
 from orchestrator.graphql.schemas.subscription import SubscriptionInterface
 from orchestrator.graphql.types import ScalarOverrideType, StrawberryModelType
 from orchestrator.log_config import LOGGER_OVERRIDES
+from orchestrator.metrics import ORCHESTRATOR_METRICS_REGISTRY, initialize_default_metrics
 from orchestrator.services.process_broadcast_thread import ProcessDataBroadcastThread
 from orchestrator.settings import AppSettings, ExecutorType, app_settings
 from orchestrator.version import GIT_COMMIT_HASH
@@ -143,9 +145,14 @@ class OrchestratorCore(FastAPI):
         self.add_exception_handler(ProblemDetailException, problem_detail_handler)  # type: ignore[arg-type]
         add_exception_handler(self)
 
+        if base_settings.ENABLE_PROMETHEUS_METRICS_ENDPOINT:
+            initialize_default_metrics()
+            metrics_app = make_asgi_app(registry=ORCHESTRATOR_METRICS_REGISTRY)
+            self.mount("/api/metrics", metrics_app)
+
         @self.router.get("/", response_model=str, response_class=JSONResponse, include_in_schema=False)
         def _index() -> str:
-            return "Orchestrator orchestrator"
+            return "Orchestrator Core"
 
     def add_sentry(
         self,
@@ -154,6 +161,7 @@ class OrchestratorCore(FastAPI):
         server_name: str,
         environment: str,
         release: str | None = GIT_COMMIT_HASH,
+        **sentry_kwargs: Any,
     ) -> None:
         logger.info("Adding Sentry middleware to app", app=self.title)
         if self.base_settings.EXECUTOR == ExecutorType.WORKER:
@@ -173,6 +181,7 @@ class OrchestratorCore(FastAPI):
             integrations=sentry_integrations,
             propagate_traces=True,
             profiles_sample_rate=trace_sample_rate,
+            **sentry_kwargs,
         )
 
     @staticmethod

orchestrator_core-4.0.0rc4/orchestrator/metrics/__init__.py

@@ -0,0 +1,3 @@
+from orchestrator.metrics.init import ORCHESTRATOR_METRICS_REGISTRY, initialize_default_metrics
+
+__all__ = ["initialize_default_metrics", "ORCHESTRATOR_METRICS_REGISTRY"]

orchestrator_core-4.0.0rc4/orchestrator/metrics/engine.py

@@ -0,0 +1,49 @@
+from typing import Iterable
+
+from prometheus_client import Metric
+from prometheus_client.metrics_core import GaugeMetricFamily, StateSetMetricFamily
+from prometheus_client.registry import Collector
+
+from orchestrator.schemas.engine_settings import GlobalStatusEnum
+from orchestrator.services import settings
+
+
+def _get_engine_status() -> tuple[GlobalStatusEnum, int]:
+    """Query for getting the current status of the workflow engine.
+
+    This includes the engine status, and the amount of currently running processes.
+    """
+    engine_settings = settings.get_engine_settings()
+    engine_status = settings.generate_engine_global_status(engine_settings)
+
+    return engine_status, int(engine_settings.running_processes)
+
+
+class WorkflowEngineCollector(Collector):
+    """Initialize a Prometheus enum and a gauge.
+
+    The enum of the current workflow engine status takes three values:
+        - RUNNING
+        - PAUSING
+        - PAUSED
+
+    This metric also exports the amount of currently running processes.
+    """
+
+    def collect(self) -> Iterable[Metric]:
+        current_engine_status, running_process_count = _get_engine_status()
+
+        engine_status = StateSetMetricFamily(
+            "wfo_engine_status",
+            documentation="Current workflow engine status.",
+            value={status: status == current_engine_status for status in GlobalStatusEnum.values()},
+        )
+
+        engine_process_count = GaugeMetricFamily(
+            "wfo_active_process_count",
+            unit="count",
+            value=running_process_count,
+            documentation="Number of currently running processes in the workflow engine.",
+        )
+
+        return [engine_status, engine_process_count]

orchestrator_core-4.0.0rc4/orchestrator/metrics/init.py

@@ -0,0 +1,14 @@
+from prometheus_client import CollectorRegistry
+
+from orchestrator.metrics.engine import WorkflowEngineCollector
+from orchestrator.metrics.processes import ProcessCollector
+from orchestrator.metrics.subscriptions import SubscriptionCollector
+
+ORCHESTRATOR_METRICS_REGISTRY = CollectorRegistry(auto_describe=True)
+
+
+def initialize_default_metrics() -> None:
+    """Register default Prometheus collectors."""
+    ORCHESTRATOR_METRICS_REGISTRY.register(SubscriptionCollector())
+    ORCHESTRATOR_METRICS_REGISTRY.register(ProcessCollector())
+    ORCHESTRATOR_METRICS_REGISTRY.register(WorkflowEngineCollector())

orchestrator_core-4.0.0rc4/orchestrator/metrics/processes.py

@@ -0,0 +1,147 @@
+from typing import Iterable
+
+from prometheus_client.metrics_core import GaugeMetricFamily, Metric
+from prometheus_client.registry import Collector
+from pydantic import BaseModel
+from sqlalchemy import desc, func
+
+from orchestrator.db import ProcessTable, ProductTable, SubscriptionTable, WorkflowTable, db
+from orchestrator.db.models import ProcessSubscriptionTable
+from orchestrator.targets import Target
+from orchestrator.workflow import ProcessStatus
+from pydantic_forms.types import UUIDstr
+
+
+class ProcessTableQueryResult(BaseModel):
+    workflow_name: str
+    customer_id: UUIDstr
+    workflow_target: Target
+    last_status: ProcessStatus
+    created_by: str
+    is_task: bool
+    product_name: str
+    total_runtime: float
+    process_count: int
+
+
+def _get_processes() -> list[ProcessTableQueryResult]:
+    """Query for getting all processes.
+
+    Equivalent to the following SQL statement:
+    ```sql
+    SELECT
+      workflows."name"                                                                          AS workflow_name
+      , subscriptions.customer_id
+      , workflows.target                                                                        AS workflow_target
+      , processes.last_status
+      , processes.created_by
+      , processes.is_task
+      , products."name"                                                                         AS product_name
+      , Coalesce(Sum(Extract(EPOCH FROM processes.last_modified_at - processes.started_at)), 0) AS total_runtime
+      , Count(workflows."name")                                                                 AS process_count
+    FROM
+      processes
+    JOIN workflows
+      ON processes.workflow_id = workflows.workflow_id
+    JOIN processes_subscriptions
+      ON processes.pid = processes_subscriptions.pid
+    JOIN subscriptions
+      ON processes_subscriptions.subscription_id = subscriptions.subscription_id
+    JOIN products
+      ON subscriptions.product_id = products.product_id
+    GROUP BY
+      workflows."name"
+      , subscriptions.customer_id
+      , workflows.target
+      , processes.last_status
+      , processes.created_by
+      , processes.is_task
+      , products."name"
+    ;
+    ```
+    """
+    process_count = func.count(WorkflowTable.name).label("process_count")
+    total_process_time = func.coalesce(
+        func.sum(func.extract("epoch", (ProcessTable.last_modified_at - ProcessTable.started_at))), 0
+    ).label("total_runtime")
+    return (
+        db.session.query(
+            ProcessTable.last_status,
+            ProcessTable.created_by,
+            ProcessTable.is_task,
+            ProductTable.name.label("product_name"),
+            WorkflowTable.name.label("workflow_name"),
+            SubscriptionTable.customer_id,
+            WorkflowTable.target.label("workflow_target"),
+            process_count,
+            total_process_time,
+        )
+        .join(WorkflowTable, WorkflowTable.workflow_id == ProcessTable.workflow_id)
+        .join(ProcessSubscriptionTable, ProcessSubscriptionTable.process_id == ProcessTable.process_id)
+        .join(SubscriptionTable, SubscriptionTable.subscription_id == ProcessSubscriptionTable.subscription_id)
+        .join(ProductTable, ProductTable.product_id == SubscriptionTable.product_id)
+        .group_by(
+            ProcessTable.last_status,
+            ProcessTable.created_by,
+            ProcessTable.is_task,
+            ProductTable.name,
+            WorkflowTable.name,
+            SubscriptionTable.customer_id,
+            WorkflowTable.target,
+        )
+        .order_by(desc(process_count))
+    ).all()
+
+
+class ProcessCollector(Collector):
+    """Collector that contains two Prometheus gauges with process counts and total runtime.
+
+    These gauges contain the amount of processes, and the total runtime in seconds, per every combination of the labels
+    that are defined:
+        - Process last status
+        - Process created by
+        - Process is task
+        - Product name
+        - Workflow name
+        - Customer ID
+        - Workflow target
+    """
+
+    def collect(self) -> Iterable[Metric]:
+        label_names = [
+            "last_status",
+            "created_by",
+            "is_task",
+            "product_name",
+            "workflow_name",
+            "customer_id",
+            "workflow_target",
+        ]
+        process_counts = GaugeMetricFamily(
+            "wfo_process_count",
+            labels=label_names,
+            unit="count",
+            documentation="Number of processes per status, creator, task, product, workflow, customer, and target.",
+        )
+        process_seconds_total = GaugeMetricFamily(
+            "wfo_process_seconds_total",
+            labels=label_names,
+            unit="count",
+            documentation="Total time spent on processes in seconds.",
+        )
+
+        for row in _get_processes():
+            label_values = [
+                row.last_status,
+                str(row.created_by),
+                str(row.is_task),
+                row.product_name,
+                row.workflow_name,
+                row.customer_id,
+                row.workflow_target,
+            ]
+
+            process_counts.add_metric(label_values, row.process_count)
+            process_seconds_total.add_metric(label_values, row.total_runtime)
+
+        return [process_counts, process_seconds_total]

orchestrator_core-4.0.0rc4/orchestrator/metrics/subscriptions.py

@@ -0,0 +1,93 @@
+from typing import Iterable
+
+from prometheus_client import Metric
+from prometheus_client.metrics_core import GaugeMetricFamily
+from prometheus_client.registry import Collector
+from pydantic import BaseModel
+from sqlalchemy import desc, func
+
+from orchestrator.db import ProductTable, SubscriptionTable, db
+from orchestrator.types import SubscriptionLifecycle
+from pydantic_forms.types import UUIDstr
+
+
+class SubscriptionTableQueryResult(BaseModel):
+    lifecycle_state: SubscriptionLifecycle
+    customer_id: UUIDstr
+    insync: bool
+    product_name: str
+    subscription_count: int
+
+
+def _get_subscriptions() -> list[SubscriptionTableQueryResult]:
+    """Query for getting all subscriptions.
+
+    Equivalent to the following SQL statement:
+    ```sql
+    SELECT
+        subscriptions.status                   AS lifecycle_state
+        , subscriptions.customer_id
+        , subscriptions.insync
+        , products."name"                      AS product_name
+        , Count(subscriptions.subscription_id) AS subscription_count
+    FROM
+      subscriptions
+      JOIN products
+        ON subscriptions.product_id = products.product_id
+    GROUP BY  subscriptions.status
+              , subscriptions.customer_id
+              , insync
+              , products."name"
+    ;
+    ```
+    """
+    subscription_count = func.count(SubscriptionTable.subscription_id).label("subscription_count")
+    return (
+        db.session.query(
+            SubscriptionTable.status.label("lifecycle_state"),
+            SubscriptionTable.customer_id,
+            SubscriptionTable.insync,
+            ProductTable.name.label("product_name"),
+            subscription_count,
+        )
+        .outerjoin(ProductTable, ProductTable.product_id == SubscriptionTable.product_id)
+        .group_by(
+            SubscriptionTable.status,
+            SubscriptionTable.customer_id,
+            SubscriptionTable.insync,
+            ProductTable.name,
+        )
+        .order_by(desc(subscription_count))
+    ).all()
+
+
+class SubscriptionCollector(Collector):
+    """Collector for Subscriptions stored in the subscription database.
+
+    This collector contains one gauge that contains the amount of subscriptions, per every combination of the labels
+    that are defined:
+        - Product name
+        - Subscription lifecycle
+        - Customer ID
+        - `insync` state
+    """
+
+    def collect(self) -> Iterable[Metric]:
+        subscriptions = GaugeMetricFamily(
+            name="wfo_subscriptions_count",
+            labels=[
+                "product_name",
+                "lifecycle_state",
+                "customer_id",
+                "insync",
+            ],
+            unit="count",
+            documentation="Number of subscriptions per product, lifecycle state, customer, and in sync state.",
+        )
+
+        for row in _get_subscriptions():
+            subscriptions.add_metric(
+                [row.product_name, row.lifecycle_state, row.customer_id, str(row.insync)], row.subscription_count
+            )
+
+        return [subscriptions]

{orchestrator_core-4.0.0rc2 → orchestrator_core-4.0.0rc4}/orchestrator/services/settings.py

@@ -19,7 +19,7 @@ from sqlalchemy import select, text
 from sqlalchemy.exc import SQLAlchemyError
 
 from orchestrator.db import EngineSettingsTable, db
-from orchestrator.schemas.engine_settings import EngineSettingsSchema
+from orchestrator.schemas.engine_settings import EngineSettingsSchema, GlobalStatusEnum
 from orchestrator.settings import app_settings
 
 logger = structlog.get_logger(__name__)
@@ -35,6 +35,15 @@ def get_engine_settings_for_update() -> EngineSettingsTable:
     return db.session.execute(select(EngineSettingsTable).with_for_update()).scalar_one()
 
 
+def generate_engine_global_status(engine_settings: EngineSettingsTable) -> GlobalStatusEnum:
+    """Returns the global status of the engine."""
+    if engine_settings.global_lock and engine_settings.running_processes > 0:
+        return GlobalStatusEnum.PAUSING
+    if engine_settings.global_lock and engine_settings.running_processes == 0:
+        return GlobalStatusEnum.PAUSED
+    return GlobalStatusEnum.RUNNING
+
+
 def post_update_to_slack(engine_status: EngineSettingsSchema, user: str) -> None:
     """Post engine settings update to slack.
 

{orchestrator_core-4.0.0rc2 → orchestrator_core-4.0.0rc4}/orchestrator/settings.py

@@ -84,6 +84,7 @@ class AppSettings(BaseSettings):
     ENABLE_GRAPHQL_DEPRECATION_CHECKER: bool = True
     ENABLE_GRAPHQL_PROFILING_EXTENSION: bool = False
     ENABLE_GRAPHQL_STATS_EXTENSION: bool = False
+    ENABLE_PROMETHEUS_METRICS_ENDPOINT: bool = False
     VALIDATE_OUT_OF_SYNC_SUBSCRIPTIONS: bool = False
     FILTER_BY_MODE: Literal["partial", "exact"] = "exact"
 

{orchestrator_core-4.0.0rc2 → orchestrator_core-4.0.0rc4}/pyproject.toml

@@ -47,6 +47,7 @@ dependencies = [
     "itsdangerous",
     "Jinja2==3.1.6",
     "orjson==3.10.18",
+    "prometheus-client==0.21.1",
     "psycopg2-binary==2.9.10",
     "pydantic[email]~=2.8.2",
     "pydantic-settings~=2.9.1",

{orchestrator_core-4.0.0rc2 → orchestrator_core-4.0.0rc4}/test/unit_tests/conftest.py

@@ -274,6 +274,7 @@ def fastapi_app(database, db_uri):
     oauth2lib_settings.OAUTH2_ACTIVE = False
     oauth2lib_settings.ENVIRONMENT_IGNORE_MUTATION_DISABLED = ["local", "TESTING"]
     app_settings.DATABASE_URI = db_uri
+    app_settings.ENABLE_PROMETHEUS_METRICS_ENDPOINT = True
     app = OrchestratorCore(base_settings=app_settings)
     # Start ProcessDataBroadcastThread to test websocket_manager with memory backend
     app.broadcast_thread.start()

orchestrator_core-4.0.0rc4/test/unit_tests/metrics/conftest.py

@@ -0,0 +1,27 @@
+from http import HTTPStatus
+
+import pytest
+
+EMPTY_METRICS = """# HELP wfo_subscriptions_count Number of subscriptions per product, lifecycle state, customer, and in sync state.
+# TYPE wfo_subscriptions_count gauge
+# HELP wfo_process_count Number of processes per status, creator, task, product, workflow, customer, and target.
+# TYPE wfo_process_count gauge
+# HELP wfo_process_seconds_total_count Total time spent on processes in seconds.
+# TYPE wfo_process_seconds_total_count gauge
+# HELP wfo_engine_status Current workflow engine status.
+# TYPE wfo_engine_status gauge
+wfo_engine_status{wfo_engine_status="PAUSED"} 0.0
+wfo_engine_status{wfo_engine_status="PAUSING"} 0.0
+wfo_engine_status{wfo_engine_status="RUNNING"} 1.0
+# HELP wfo_active_process_count Number of currently running processes in the workflow engine.
+# TYPE wfo_active_process_count gauge
+wfo_active_process_count 0.0
+"""
+
+
+@pytest.fixture(autouse=True, scope="function")
+def assume_empty_metrics_at_start(test_client) -> None:
+    """Assert that at the start of every unit test in this module, the metrics endpoint only contains empty data."""
+    response = test_client.get("/api/metrics")
+    assert HTTPStatus.OK == response.status_code
+    assert response.text == EMPTY_METRICS

orchestrator_core-4.0.0rc4/test/unit_tests/metrics/test_engine_metrics.py

@@ -0,0 +1,18 @@
+from test.unit_tests.metrics.conftest import EMPTY_METRICS
+
+
+def test_engine_metrics_success(test_client) -> None:
+    test_client.put("/api/settings/status", json={"global_lock": True})
+    response = test_client.get("api/metrics")
+    expected_metric_lines = [
+        "# HELP wfo_engine_status Current workflow engine status.",
+        "# TYPE wfo_engine_status gauge",
+        'wfo_engine_status{wfo_engine_status="PAUSED"} 1.0',
+        'wfo_engine_status{wfo_engine_status="PAUSING"} 0.0',
+        'wfo_engine_status{wfo_engine_status="RUNNING"} 0.0',
+    ]
+    assert all(line in response.text for line in expected_metric_lines)
+
+    test_client.put("/api/settings/status", json={"global_lock": False})
+    response = test_client.get("/api/metrics")
+    assert response.text == EMPTY_METRICS

orchestrator_core-4.0.0rc4/test/unit_tests/metrics/test_process_metrics.py

@@ -0,0 +1,27 @@
+from test.unit_tests.conftest import CUSTOMER_ID
+
+
+def test_process_metrics_success(test_client, mocked_processes) -> None:
+    response = test_client.get("api/metrics")
+    expected_metric_lines = [
+        "# HELP wfo_process_count Number of processes per status, creator, task, product, workflow, customer, and target.",
+        "# TYPE wfo_process_count gauge",
+        f'wfo_process_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="False",last_status="failed",product_name="Product 2",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 1.0',
+        f'wfo_process_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="False",last_status="resumed",product_name="Product 1",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 1.0',
+        f'wfo_process_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="False",last_status="completed",product_name="Product 2",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 1.0',
+        f'wfo_process_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="False",last_status="suspended",product_name="Product 1",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 1.0',
+        f'wfo_process_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="False",last_status="completed",product_name="Product 1",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 1.0',
+        f'wfo_process_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="True",last_status="suspended",product_name="Product 1",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 1.0',
+        f'wfo_process_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="True",last_status="completed",product_name="Product 2",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 1.0',
+        "# HELP wfo_process_seconds_total_count Total time spent on processes in seconds.",
+        "# TYPE wfo_process_seconds_total_count gauge",
+        f'wfo_process_seconds_total_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="False",last_status="failed",product_name="Product 2",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 600.0',
+        f'wfo_process_seconds_total_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="False",last_status="resumed",product_name="Product 1",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 600.0',
+        f'wfo_process_seconds_total_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="False",last_status="completed",product_name="Product 2",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 600.0',
+        f'wfo_process_seconds_total_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="False",last_status="suspended",product_name="Product 1",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 600.0',
+        f'wfo_process_seconds_total_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="False",last_status="completed",product_name="Product 1",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 600.0',
+        f'wfo_process_seconds_total_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="True",last_status="suspended",product_name="Product 1",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 600.0',
+        f'wfo_process_seconds_total_count{{created_by="None",customer_id="{CUSTOMER_ID}",is_task="True",last_status="completed",product_name="Product 2",workflow_name="workflow_for_testing_processes_py",workflow_target="SYSTEM"}} 600.0',
+    ]
+
+    assert all(line in response.text for line in expected_metric_lines)