orchestrator-core 4.0.3__tar.gz → 4.1.0rc1__tar.gz

{orchestrator_core-4.0.3 → orchestrator_core-4.1.0rc1}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.0.3
+current_version = 4.1.0rc1
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(rc(?P<build>\d+))?

{orchestrator_core-4.0.3 → orchestrator_core-4.1.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 4.0.3
+Version: 4.1.0rc1
 Summary: This is the orchestrator workflow engine.
 Requires-Python: >=3.11,<3.14
 Classifier: Intended Audience :: Information Technology
@@ -28,7 +28,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
 Classifier: Topic :: Internet :: WWW/HTTP
 License-File: LICENSE
-Requires-Dist: alembic==1.15.2
+Requires-Dist: alembic==1.16.1
 Requires-Dist: anyio>=3.7.0
 Requires-Dist: click==8.*
 Requires-Dist: deprecated
@@ -39,7 +39,7 @@ Requires-Dist: more-itertools~=10.7.0
 Requires-Dist: itsdangerous
 Requires-Dist: Jinja2==3.1.6
 Requires-Dist: orjson==3.10.18
-Requires-Dist: prometheus-client==0.21.1
+Requires-Dist: prometheus-client==0.22.0
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: pydantic[email]~=2.8.2
 Requires-Dist: pydantic-settings~=2.9.1
@@ -49,7 +49,7 @@ Requires-Dist: pytz==2025.2
 Requires-Dist: redis==5.1.1
 Requires-Dist: schedule==1.1.0
 Requires-Dist: semver==3.0.4
-Requires-Dist: sentry-sdk[fastapi]~=2.28.0
+Requires-Dist: sentry-sdk[fastapi]~=2.29.1
 Requires-Dist: SQLAlchemy==2.0.41
 Requires-Dist: SQLAlchemy-Utils==0.41.2
 Requires-Dist: structlog

orchestrator_core-4.1.0rc1/docs/reference-docs/app/settings_overview.md

@@ -0,0 +1,89 @@
+# Settings overview in Orchestrator
+
+You can use the `api/settings/overview` endpoint to get an overview of the settings that are used in the application.
+This endpoint provides a JSON response that contains the settings that are defined in the application. The settings are
+grouped by their names and sensitive values are masked for security reasons.
+Per default, the application settings are used to configure the application. The settings are defined in the
+`orchestrator.settings.py` module and can be used to configure the application.
+
+An example of the settings is shown below:
+
+```python
+from orchestrator.settings import BaseSettings
+
+
+class AppSettings(BaseSettings):
+    TESTING: bool = True
+    SESSION_SECRET: OrchSecretStr = "".join(secrets.choice(string.ascii_letters) for i in range(16))  # type: ignore
+    CORS_ORIGINS: str = "*"
+    ...
+    EXPOSE_SETTINGS: bool = False
+    EXPOSE_OAUTH_SETTINGS: bool = False
+
+
+app_settings = AppSettings()
+
+if app_settings.EXPOSE_SETTINGS:
+    expose_settings("app_settings", app_settings)  # type: ignore
+
+if app_settings.EXPOSE_OAUTH_SETTINGS:
+    expose_settings("oauth2lib_settings", oauth2lib_settings)  # type: ignore
+```
+
+What you see above is the default settings for the application. The settings are defined in the
+`orchestrator.settings.py` module and can be used to configure the application.
+The `EXPOSE_SETTINGS` and `EXPOSE_OAUTH_SETTINGS` flags are used to control whether the settings should be exposed via
+the `api/settings/overview` endpoint, the result looks like this:
+
+```json
+[
+  {
+    "name": "app_settings",
+    "variables": [
+      {
+        "env_name": "TESTING",
+        "env_value": false
+      },
+      {
+        "env_name": "SESSION_SECRET",
+        "env_value": "**********"
+      },
+      {
+        "env_name": "CORS_ORIGINS",
+        "env_value": "*"
+      }
+    ]
+  }
+]
+```
+
+The `app_settings` in the example above is a name of the settings class that is registered to be exposed.
+
+## Exposing your settings
+
+In order to expose your settings, you need to register them using the `expose_settings()` function. This function takes
+two arguments: the name of the settings class and the instance of the settings class.
+
+```python
+from orchestrator.settings import expose_settings, BaseSettings
+
+
+class MySettings(BaseSettings):
+    debug: bool = True
+
+
+my_settings = MySettings()
+
+expose_settings("my_settings", my_settings)
+```
+
+## Masking Secrets
+
+The following rules apply when exposing settings:
+
+### Rules for Masking Secrets
+
+- Keys containing `"password"` or `"secret"` in their names are masked.
+- `SecretStr` from `from pydantic import SecretStr` are masked.
+- `SecretStr` from `from orchestrator.utils.expose_settings import SecretStr` are masked.
+- `PostgresDsn` from `from pydantic import PostgresDsn` are masked.

{orchestrator_core-4.0.3 → orchestrator_core-4.1.0rc1}/docs/reference-docs/cli.md

@@ -35,7 +35,6 @@ Interact with the application database. By default, does nothing, specify `main.
         - migrate_tasks
         - revision
         - upgrade
-
       heading_level: 3
 
 ### migrate-domain-models

{orchestrator_core-4.0.3 → orchestrator_core-4.1.0rc1}/mkdocs.yml

@@ -182,6 +182,7 @@ nav:
                 - App.py: reference-docs/app/app.md
                 - Python Version: reference-docs/python.md
                 - Scaling: reference-docs/app/scaling.md
+                - Settings: reference-docs/app/settings_overview.md
           # - Tasks:  reference-docs/tasks.md
           # - Tests: reference-docs/tests.md
           - Workflows:

{orchestrator_core-4.0.3 → orchestrator_core-4.1.0rc1}/orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "4.0.3"
+__version__ = "4.1.0rc1"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings

{orchestrator_core-4.0.3 → orchestrator_core-4.1.0rc1}/orchestrator/api/api_v1/endpoints/settings.py

@@ -22,11 +22,17 @@ from sqlalchemy.exc import SQLAlchemyError
 from oauth2_lib.fastapi import OIDCUserModel
 from orchestrator.api.error_handling import raise_status
 from orchestrator.db import EngineSettingsTable
-from orchestrator.schemas import EngineSettingsBaseSchema, EngineSettingsSchema, WorkerStatus
+from orchestrator.schemas import (
+    EngineSettingsBaseSchema,
+    EngineSettingsSchema,
+    WorkerStatus,
+)
 from orchestrator.security import authenticate
 from orchestrator.services import processes, settings
 from orchestrator.services.settings import generate_engine_global_status
+from orchestrator.services.settings_env_variables import get_all_exposed_settings
 from orchestrator.settings import ExecutorType, app_settings
+from orchestrator.utils.expose_settings import SettingsExposedSchema
 from orchestrator.utils.json import json_dumps
 from orchestrator.utils.redis import delete_keys_matching_pattern
 from orchestrator.utils.redis_client import create_redis_asyncio_client
@@ -169,3 +175,8 @@ def generate_engine_status_response(
     result = EngineSettingsSchema.model_validate(engine_settings)
     result.global_status = generate_engine_global_status(engine_settings)
     return result
+
+
+@router.get("/overview", response_model=list[SettingsExposedSchema])
+def get_exposed_settings() -> list[SettingsExposedSchema]:
+    return get_all_exposed_settings()

{orchestrator_core-4.0.3 → orchestrator_core-4.1.0rc1}/orchestrator/migrations/helpers.py

@@ -35,14 +35,20 @@ def has_table_column(table_name: str, column_name: str, conn: sa.engine.Connecti
     :param conn: SQLAlchemy database Connection
     :return: True if the column exists, False otherwise
     """
-    inspector = sa.inspect(conn.engine)
-    try:
-        columns = inspector.get_columns(table_name)
-        return any(col["name"] == column_name for col in columns)
-    except sa.exc.NoSuchTableError:
-        # On some migrations the table might not exist yet, so we catch the exception
-        logger.warning(f"Table {table_name} does not exist.")
-        return False
+    result = conn.execute(
+        sa.text(
+            """
+            SELECT column_name
+            FROM information_schema.columns
+            WHERE table_name = :table_name and column_name = :column_name
+            """
+        ),
+        {
+            "table_name": table_name,
+            "column_name": column_name,
+        },
+    )
+    return result.first() is not None
 
 
 def get_resource_type_id_by_name(conn: sa.engine.Connection, name: str) -> UUID:

orchestrator_core-4.1.0rc1/orchestrator/services/settings_env_variables.py

@@ -0,0 +1,68 @@
+# Copyright 2019-2025 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, Type
+
+from pydantic import SecretStr as PydanticSecretStr
+from pydantic_core import MultiHostUrl
+from pydantic_settings import BaseSettings
+
+from orchestrator.utils.expose_settings import SecretStr as OrchSecretStr
+from orchestrator.utils.expose_settings import SettingsEnvVariablesSchema, SettingsExposedSchema
+
+EXPOSED_ENV_SETTINGS_REGISTRY: Dict[str, Type[BaseSettings]] = {}
+MASK = "**********"
+
+
+def expose_settings(settings_name: str, base_settings: Type[BaseSettings]) -> Type[BaseSettings]:
+    """Decorator to register settings classes."""
+    EXPOSED_ENV_SETTINGS_REGISTRY[settings_name] = base_settings
+    return base_settings
+
+
+def mask_value(key: str, value: Any) -> Any:
+    key_lower = key.lower()
+
+    if "secret" in key_lower or "password" in key_lower:
+        # Mask sensitive information
+        return MASK
+
+    if isinstance(value, PydanticSecretStr):
+        # Need to convert SecretStr to str for serialization
+        return str(value)
+
+    if isinstance(value, OrchSecretStr):
+        return MASK
+
+    # PostgresDsn is just MultiHostUrl with extra metadata (annotations)
+    if isinstance(value, MultiHostUrl):
+        # Convert PostgresDsn to str for serialization
+        return MASK
+
+    return value
+
+
+def get_all_exposed_settings() -> list[SettingsExposedSchema]:
+    """Return all registered settings as dicts."""
+
+    def _get_settings_env_variables(base_settings: Type[BaseSettings]) -> list[SettingsEnvVariablesSchema]:
+        """Get environment variables from settings."""
+        return [
+            SettingsEnvVariablesSchema(env_name=key, env_value=mask_value(key, value))
+            for key, value in base_settings.model_dump().items()  # type: ignore
+        ]
+
+    return [
+        SettingsExposedSchema(name=name, variables=_get_settings_env_variables(base_settings))
+        for name, base_settings in EXPOSED_ENV_SETTINGS_REGISTRY.items()
+    ]

{orchestrator_core-4.0.3 → orchestrator_core-4.1.0rc1}/orchestrator/settings.py

@@ -20,6 +20,8 @@ from pydantic import Field, NonNegativeInt, PostgresDsn, RedisDsn
 from pydantic_settings import BaseSettings
 
 from oauth2_lib.settings import oauth2lib_settings
+from orchestrator.services.settings_env_variables import expose_settings
+from orchestrator.utils.expose_settings import SecretStr as OrchSecretStr
 from pydantic_forms.types import strEnum
 
 
@@ -30,7 +32,7 @@ class ExecutorType(strEnum):
 
 class AppSettings(BaseSettings):
     TESTING: bool = True
-    SESSION_SECRET: str = "".join(secrets.choice(string.ascii_letters) for i in range(16))  # noqa: S311
+    SESSION_SECRET: OrchSecretStr = "".join(secrets.choice(string.ascii_letters) for i in range(16))  # type: ignore
     CORS_ORIGINS: str = "*"
     CORS_ALLOW_METHODS: list[str] = ["GET", "PUT", "PATCH", "POST", "DELETE", "OPTIONS", "HEAD"]
     CORS_ALLOW_HEADERS: list[str] = ["If-None-Match", "Authorization", "If-Match", "Content-Type"]
@@ -55,7 +57,7 @@ class AppSettings(BaseSettings):
     MAIL_PORT: int = 25
     MAIL_STARTTLS: bool = False
     CACHE_URI: RedisDsn = "redis://localhost:6379/0"  # type: ignore
-    CACHE_HMAC_SECRET: str | None = None  # HMAC signing key, used when pickling results in the cache
+    CACHE_HMAC_SECRET: OrchSecretStr | None = None  # HMAC signing key, used when pickling results in the cache
     REDIS_RETRY_COUNT: NonNegativeInt = Field(
         2, description="Number of retries for redis connection errors/timeouts, 0 to disable"
     )  # More info: https://redis-py.readthedocs.io/en/stable/retry.html
@@ -87,6 +89,8 @@ class AppSettings(BaseSettings):
     ENABLE_PROMETHEUS_METRICS_ENDPOINT: bool = False
     VALIDATE_OUT_OF_SYNC_SUBSCRIPTIONS: bool = False
     FILTER_BY_MODE: Literal["partial", "exact"] = "exact"
+    EXPOSE_SETTINGS: bool = False
+    EXPOSE_OAUTH_SETTINGS: bool = False
 
 
 app_settings = AppSettings()
@@ -94,3 +98,8 @@ app_settings = AppSettings()
 # Set oauth2lib_settings variables to the same (default) value of settings
 oauth2lib_settings.SERVICE_NAME = app_settings.SERVICE_NAME
 oauth2lib_settings.ENVIRONMENT = app_settings.ENVIRONMENT
+
+if app_settings.EXPOSE_SETTINGS:
+    expose_settings("app_settings", app_settings)  # type: ignore
+if app_settings.EXPOSE_OAUTH_SETTINGS:
+    expose_settings("oauth2lib_settings", oauth2lib_settings)  # type: ignore

orchestrator_core-4.1.0rc1/orchestrator/utils/expose_settings.py

@@ -0,0 +1,45 @@
+# Copyright 2019-2025 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility module for exposing settings in a structured format.
+
+Unfortunately, this module needs to be imported from the utils and cannot be added to the schemas folder.
+This is due to circular import issues with the combination of schemas/settings.
+"""
+
+from typing import Any
+
+from pydantic import BaseModel
+from pydantic_core import core_schema
+
+
+class SecretStr(str):
+    """A string that is treated as a secret, for example, passwords or API keys.
+
+    This class is used to indicate that the string should not be logged or displayed in plaintext.
+    """
+
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source_type, handler):  # type: ignore
+        # This method is used to define how the SecretStr type should be handled by Pydantic.
+        # With this implementation, it will fail validation.
+        return core_schema.no_info_plain_validator_function(cls)
+
+
+class SettingsEnvVariablesSchema(BaseModel):
+    env_name: str
+    env_value: Any
+
+
+class SettingsExposedSchema(BaseModel):
+    name: str
+    variables: list[SettingsEnvVariablesSchema]

{orchestrator_core-4.0.3 → orchestrator_core-4.1.0rc1}/pyproject.toml

@@ -36,7 +36,7 @@ classifiers = [
     "Topic :: Internet :: WWW/HTTP",
 ]
 dependencies = [
-    "alembic==1.15.2",
+    "alembic==1.16.1",
     "anyio>=3.7.0",
     "click==8.*",
     "deprecated",
@@ -47,7 +47,7 @@ dependencies = [
     "itsdangerous",
     "Jinja2==3.1.6",
     "orjson==3.10.18",
-    "prometheus-client==0.21.1",
+    "prometheus-client==0.22.0",
     "psycopg2-binary==2.9.10",
     "pydantic[email]~=2.8.2",
     "pydantic-settings~=2.9.1",
@@ -57,7 +57,7 @@ dependencies = [
     "redis==5.1.1",
     "schedule==1.1.0",
     "semver==3.0.4",
-    "sentry-sdk[fastapi]~=2.28.0",
+    "sentry-sdk[fastapi]~=2.29.1",
     "SQLAlchemy==2.0.41",
     "SQLAlchemy-Utils==0.41.2",
     "structlog",

{orchestrator_core-4.0.3 → orchestrator_core-4.1.0rc1}/test/unit_tests/api/test_settings.py

@@ -2,10 +2,13 @@ from http import HTTPStatus
 from unittest import mock
 from unittest.mock import Mock
 
+from pydantic import SecretStr
+from pydantic_settings import BaseSettings
 from sqlalchemy.exc import SQLAlchemyError
 
 from orchestrator.db import db
 from orchestrator.services.settings import get_engine_settings
+from orchestrator.services.settings_env_variables import expose_settings, get_all_exposed_settings
 
 
 def test_get_engine_status(test_client):
@@ -74,3 +77,22 @@ def test_reset_search_index_error(test_client, generic_subscription_1, generic_s
         ex.attach_mock(session_execute_mock, "execute")
         response = test_client.post("/api/settings/search-index/reset")
         assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
+
+
+def test_get_exposed_settings(test_client):
+    class MySettings(BaseSettings):
+        db_password: SecretStr = "test_password"  # noqa: S105
+
+    my_settings = MySettings()
+    expose_settings("my_settings", my_settings)
+    assert len(get_all_exposed_settings()) == 1
+
+    response = test_client.get("/api/settings/overview")
+    assert response.status_code == HTTPStatus.OK
+
+    exposed_settings = response.json()
+
+    # Find the env_name db_password and ensure it is masked is **********
+    session_secret = next((var for var in exposed_settings[0]["variables"] if var["env_name"] == "db_password"), None)
+    assert session_secret is not None
+    assert session_secret["env_value"] == "**********"

orchestrator_core-4.1.0rc1/test/unit_tests/db/test_migration_does_column_exist.py

@@ -0,0 +1,10 @@
+from orchestrator.db import db
+from orchestrator.migrations.helpers import has_table_column
+
+
+def test_select_from_table():
+    # Testing if Table Workflows exist with column is_task
+    # it should because the db.session depends on the session where all migrations are already run
+    session = db.session
+    result = has_table_column(table_name="workflows", column_name="is_task", conn=session)
+    assert result is True, "Column 'is_task' does not exist in 'workflows' table"

orchestrator_core-4.1.0rc1/test/unit_tests/services/test_expose_settings.py

@@ -0,0 +1,35 @@
+from pydantic import PostgresDsn
+from pydantic import SecretStr as PydanticSecretStr
+from pydantic_settings import BaseSettings
+
+from orchestrator.services.settings_env_variables import MASK, expose_settings, get_all_exposed_settings
+from orchestrator.utils.expose_settings import SecretStr as OrchSecretStr
+
+
+def test_expose_settings():
+
+    class MySettings(BaseSettings):
+        api_key: OrchSecretStr = "test_api_key"
+        db_password: PydanticSecretStr = "test_password"  # noqa: S105
+        debug_mode: bool = True
+        secret_test: str = "test_secret"  # noqa: S105
+        uri: PostgresDsn = "postgresql://user:password@localhost/dbname"
+
+    my_settings = MySettings()
+    expose_settings("my_settings", my_settings)
+
+    exposed_settings = get_all_exposed_settings()
+
+    assert len(exposed_settings) == 1
+    my_settings_index = 0
+
+    assert exposed_settings[my_settings_index].name == "my_settings"
+
+    assert len(exposed_settings[my_settings_index].variables) == 5
+
+    # Assert that sensitive values are masked
+    assert exposed_settings[my_settings_index].variables[0].env_value == MASK  # api_key
+    assert exposed_settings[my_settings_index].variables[1].env_value == MASK  # db_password
+    assert exposed_settings[my_settings_index].variables[2].env_value is True  # debug_mode
+    assert exposed_settings[my_settings_index].variables[3].env_value == MASK  # secret_test
+    assert exposed_settings[my_settings_index].variables[4].env_value == MASK  # uri