orchestrator-core 3.1.2rc4__py3-none-any.whl → 3.2.0__py3-none-any.whl

orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "3.1.2rc4"
+__version__ = "3.2.0"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings

orchestrator/api/api_v1/endpoints/processes.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2025 SURF, GÉANT.
+# Copyright 2019-2025 SURF, GÉANT, ESnet.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -40,13 +40,7 @@ from orchestrator.db.filters import Filter
 from orchestrator.db.filters.process import filter_processes
 from orchestrator.db.sorting import Sort, SortOrder
 from orchestrator.db.sorting.process import sort_processes
-from orchestrator.schemas import (
-    ProcessIdSchema,
-    ProcessResumeAllSchema,
-    ProcessSchema,
-    ProcessStatusCounts,
-    Reporter,
-)
+from orchestrator.schemas import ProcessIdSchema, ProcessResumeAllSchema, ProcessSchema, ProcessStatusCounts, Reporter
 from orchestrator.security import authenticate
 from orchestrator.services.process_broadcast_thread import api_broadcast_process_data
 from orchestrator.services.processes import (
@@ -139,9 +133,12 @@ async def new_process(
     request: Request,
     json_data: list[dict[str, Any]] | None = Body(...),
     user: str = Depends(user_name),
+    user_model: OIDCUserModel | None = Depends(authenticate),
 ) -> dict[str, UUID]:
     broadcast_func = api_broadcast_process_data(request)
-    process_id = start_process(workflow_key, user_inputs=json_data, user=user, broadcast_func=broadcast_func)
+    process_id = start_process(
+        workflow_key, user_inputs=json_data, user_model=user_model, user=user, broadcast_func=broadcast_func
+    )
 
     return {"id": process_id}
 

orchestrator/cli/generator/generator/workflow.py

@@ -10,12 +10,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 from collections.abc import Callable
 from functools import partial, wraps
+from importlib import metadata
 from pathlib import Path
 from typing import Any
 
+import semver
 import structlog
 from jinja2 import Environment
 
@@ -123,7 +124,17 @@ def get_product_workflow_path(config: dict, workflow_type: str) -> Path:
     return product_workflow_folder(config) / Path(f"{workflow_type}_{file_name}").with_suffix(".py")
 
 
+def eval_pydantic_forms_version() -> bool:
+    updated_version = semver.Version.parse("2.0.0")
+
+    installed_version = metadata.version("pydantic-forms")
+    installed_semver = semver.Version.parse(installed_version)
+
+    return installed_semver >= updated_version
+
+
 def render_template(environment: Environment, config: dict, template: str, workflow: str = "") -> str:
+    use_updated_readonly_field = eval_pydantic_forms_version()
     product_block = root_product_block(config)
     types_to_import = get_name_spaced_types_to_import(product_block["fields"])
     fields = get_input_fields(product_block)
@@ -152,6 +163,7 @@ def render_template(environment: Environment, config: dict, template: str, workf
         product_types_module=get_product_types_module(),
         workflows_module=get_workflows_module(),
         workflow_validations=workflow_validations if workflow else [],
+        use_updated_readonly_field=use_updated_readonly_field,
     )
 
 

orchestrator/cli/generator/templates/modify_product.j2

@@ -7,7 +7,12 @@ from typing import Annotated
 import structlog
 from pydantic import AfterValidator, ConfigDict, model_validator
 from pydantic_forms.types import FormGenerator, State, UUIDstr
+
+{%- if use_updated_readonly_field is true  %}
+from pydantic_forms.validators import read_only_field
+{%- else %}
 from pydantic_forms.validators import ReadOnlyField
+{%- endif %}
 
 from orchestrator.forms import FormPage
 from orchestrator.forms.validators import CustomerId, Divider
@@ -52,7 +57,11 @@ def initial_input_form_generator(subscription_id: UUIDstr) -> FormGenerator:
         divider_1: Divider
 
         {% for field in fields if not field.modifiable is defined -%}
+        {% if use_updated_readonly_field is true  -%}
+        {{ field.name }}: read_only_field({{ product_block.name }}.{{ field.name }})
+        {% else -%}
         {{ field.name }}: ReadOnlyField({{ product_block.name }}.{{ field.name }})
+        {% endif -%}
         {% endfor -%}
 
         {% for field in fields if field.modifiable is defined -%}

orchestrator/db/__init__.py

@@ -19,6 +19,7 @@ from orchestrator.db.database import Database, transactional
 from orchestrator.db.models import (  # noqa: F401
     EngineSettingsTable,
     FixedInputTable,
+    InputStateTable,
     ProcessStepTable,
     ProcessSubscriptionTable,
     ProcessTable,
@@ -85,6 +86,7 @@ __all__ = [
     "SubscriptionMetadataTable",
     "ResourceTypeTable",
     "FixedInputTable",
+    "InputStateTable",
     "EngineSettingsTable",
     "WorkflowTable",
     "SubscriptionCustomerDescriptionTable",

orchestrator/db/loaders.py

@@ -1,5 +1,5 @@
 from functools import reduce
-from typing import Any, Callable, Iterator, NamedTuple, cast
+from typing import Any, Callable, Iterable, Iterator, NamedTuple, cast
 
 import structlog
 from sqlalchemy import inspect
@@ -131,12 +131,15 @@ def init_model_loaders() -> None:
         _MODEL_LOADERS[model] = dict(_inspect_model(model))
 
 
-def lookup_attr_loaders(model: type[DbBaseModel], attr: str) -> list[AttrLoader]:
+def _lookup_attr_loaders(model: type[DbBaseModel], attr: str) -> list[AttrLoader]:
     """Return loader(s) for an attribute on the given model."""
+    if not _MODEL_LOADERS:
+        # Ensure loaders are always initialized
+        init_model_loaders()
     return _MODEL_LOADERS.get(model, {}).get(attr, [])
 
 
-def join_attr_loaders(loaders: list[AttrLoader]) -> Load | None:
+def _join_attr_loaders(loaders: list[AttrLoader]) -> Load | None:
     """Given 1 or more attribute loaders, instantiate and chain them together."""
     if not loaders:
         return None
@@ -151,3 +154,48 @@ def join_attr_loaders(loaders: list[AttrLoader]) -> Load | None:
         return getattr(final_loader, next.loader_fn.__name__)(next.attr)
 
     return reduce(chain_loader_func, other_loaders, loader_fn)
+
+
+def _split_path(query_path: str) -> Iterable[str]:
+    yield from (field for field in query_path.split("."))
+
+
+def get_query_loaders_for_model_paths(root_model: type[DbBaseModel], model_paths: list[str]) -> list[Load]:
+    """Get sqlalchemy query loaders to use for the model based on the paths."""
+    # Sort by length to find the longest match first
+    model_paths.sort(key=lambda x: x.count("."), reverse=True)
+
+    def get_loader_for_path(model_path: str) -> tuple[str, Load | None]:
+        next_model = root_model
+
+        matched_fields: list[str] = []
+        path_loaders: list[AttrLoader] = []
+
+        for field in _split_path(model_path):
+            if not (attr_loaders := _lookup_attr_loaders(next_model, field)):
+                break
+
+            matched_fields.append(field)
+            path_loaders.extend(attr_loaders)
+            next_model = attr_loaders[-1].next_model
+
+        return ".".join(matched_fields), _join_attr_loaders(path_loaders)
+
+    query_loaders: dict[str, Load] = {}
+
+    for path in model_paths:
+        matched_path, loader = get_loader_for_path(path)
+        if not matched_path or not loader or matched_path in query_loaders:
+            continue
+        if any(known_path.startswith(f"{matched_path}.") for known_path in query_loaders):
+            continue
+        query_loaders[matched_path] = loader
+
+    loaders = list(query_loaders.values())
+    logger.debug(
+        "Generated query loaders for paths",
+        root_model=root_model,
+        model_paths=model_paths,
+        query_loaders=[str(i.path) for i in loaders],
+    )
+    return loaders

orchestrator/db/models.py

@@ -15,6 +15,7 @@ from __future__ import annotations
 
 import enum
 from datetime import datetime, timezone
+from uuid import UUID
 
 import sqlalchemy
 import structlog
@@ -43,6 +44,7 @@ from sqlalchemy.exc import DontWrapMixin
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.ext.orderinglist import ordering_list
 from sqlalchemy.orm import Mapped, deferred, mapped_column, object_session, relationship, undefer
+from sqlalchemy.sql.functions import GenericFunction
 from sqlalchemy_utils import TSVectorType, UUIDType
 
 from orchestrator.config.assignee import Assignee
@@ -667,3 +669,14 @@ class EngineSettingsTable(BaseModel):
     global_lock = mapped_column(Boolean(), default=False, nullable=False, primary_key=True)
     running_processes = mapped_column(Integer(), default=0, nullable=False)
     __table_args__: tuple = (CheckConstraint(running_processes >= 0, name="check_running_processes_positive"), {})
+
+
+class SubscriptionInstanceAsJsonFunction(GenericFunction):
+    # Added in migration 42b3d076a85b
+    name = "subscription_instance_as_json"
+
+    type = pg.JSONB()
+    inherit_cache = True
+
+    def __init__(self, sub_inst_id: UUID):
+        super().__init__(sub_inst_id)

orchestrator/db/queries/subscription.py

@@ -0,0 +1,85 @@
+# Copyright 2019-2025 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from uuid import UUID
+
+from sqlalchemy import UUID as SA_UUID
+from sqlalchemy import cast as sa_cast
+from sqlalchemy import select
+from sqlalchemy.orm import raiseload
+
+from orchestrator.db import SubscriptionInstanceRelationTable, SubscriptionInstanceTable, db
+
+
+def _eagerload_subscription_instances(
+    subscription_id: UUID | str, instance_attributes: list[str]
+) -> list[SubscriptionInstanceTable]:
+    """Given a subscription id, recursively query all depends_on subscription instances with the instance_attributes eagerloaded.
+
+    Note: accessing instance attributes on the result that were not explicitly loaded will
+    trigger a sqlalchemy error.
+    """
+    from orchestrator.db.loaders import get_query_loaders_for_model_paths
+
+    # CTE to recursively get all subscription instance ids the subscription depends on
+    instance_ids_cte = (
+        select(
+            sa_cast(None, SA_UUID(as_uuid=True)).label("in_use_by_id"),
+            SubscriptionInstanceTable.subscription_instance_id.label("depends_on_id"),
+        )
+        .where(SubscriptionInstanceTable.subscription_id == subscription_id)
+        .cte(name="recursive_instance_ids", recursive=True)
+    )
+
+    cte_alias = instance_ids_cte.alias()
+    rel_alias = select(SubscriptionInstanceRelationTable).alias()
+
+    instance_ids = instance_ids_cte.union(
+        select(rel_alias.c.in_use_by_id, rel_alias.c.depends_on_id).where(
+            rel_alias.c.in_use_by_id == cte_alias.c.depends_on_id
+        )
+    )
+
+    select_all_instance_ids = select(instance_ids.c.depends_on_id).subquery()
+
+    # Eagerload specified instance attributes
+    query_loaders = get_query_loaders_for_model_paths(SubscriptionInstanceTable, instance_attributes)
+    # Prevent unwanted lazyloading of all other attributes
+    query_loaders += [raiseload("*")]  # type: ignore[list-item]  # todo fix this type
+    stmt = (
+        select(SubscriptionInstanceTable)
+        .where(SubscriptionInstanceTable.subscription_instance_id.in_(select(select_all_instance_ids)))
+        .options(*query_loaders)
+    )
+
+    return db.session.scalars(stmt).all()  # type: ignore[return-value]  # todo fix this type
+
+
+def eagerload_all_subscription_instances(subscription_id: UUID | str) -> list[SubscriptionInstanceTable]:
+    """Recursively find the subscription's depends_on instances and resolve relations for SubscriptionModel.from_subscription_id()."""
+    instance_attributes = [
+        "subscription.product",
+        "product_block",
+        "values.resource_type",
+        "depends_on",
+        "in_use_by",
+    ]
+    return _eagerload_subscription_instances(subscription_id, instance_attributes)
+
+
+def eagerload_all_subscription_instances_only_inuseby(subscription_id: UUID | str) -> list[SubscriptionInstanceTable]:
+    """Recursively find the subscription's depends_on instances and resolve their in_use_by relations."""
+    instance_attributes = [
+        "in_use_by",
+    ]
+    return _eagerload_subscription_instances(subscription_id, instance_attributes)

orchestrator/db/queries/subscription_instance.py

@@ -0,0 +1,28 @@
+# Copyright 2019-2025 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from uuid import UUID
+
+from sqlalchemy import select
+
+from orchestrator.db import db
+from orchestrator.db.models import SubscriptionInstanceAsJsonFunction
+
+
+def get_subscription_instance_dict(subscription_instance_id: UUID) -> dict:
+    """Query the subscription instance as aggregated JSONB and returns it as a dict.
+
+    Note: all values are returned as lists and have to be transformed by the caller.
+    It was attempted to do this in the DB query but this gave worse performance.
+    """
+    return db.session.execute(select(SubscriptionInstanceAsJsonFunction(subscription_instance_id))).scalar_one()