orchestrator-core 3.1.2rc3__py3-none-any.whl → 3.2.0__py3-none-any.whl

orchestrator/services/products.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/services/subscriptions.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -22,6 +22,7 @@ from uuid import UUID
 
 import more_itertools
 import structlog
+from more_itertools import first
 from sqlalchemy import Text, cast, not_, select
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import Query, aliased, joinedload
@@ -41,7 +42,11 @@ from orchestrator.db.models import (
     SubscriptionInstanceRelationTable,
     SubscriptionMetadataTable,
 )
+from orchestrator.db.queries.subscription import (
+    eagerload_all_subscription_instances_only_inuseby,
+)
 from orchestrator.domain.base import SubscriptionModel
+from orchestrator.domain.context_cache import cache_subscription_models
 from orchestrator.targets import Target
 from orchestrator.types import SubscriptionLifecycle
 from orchestrator.utils.datetime import nowtz
@@ -594,13 +599,15 @@ def convert_to_in_use_by_relation(obj: Any) -> dict[str, str]:
 
 def build_extended_domain_model(subscription_model: SubscriptionModel) -> dict:
     """Create a subscription dict from the SubscriptionModel with additional keys."""
-    stmt = select(SubscriptionCustomerDescriptionTable).filter(
+    from orchestrator.settings import app_settings
+
+    stmt = select(SubscriptionCustomerDescriptionTable).where(
         SubscriptionCustomerDescriptionTable.subscription_id == subscription_model.subscription_id
     )
     customer_descriptions = list(db.session.scalars(stmt))
 
-    subscription = subscription_model.model_dump()
-    paths = product_block_paths(subscription)
+    with cache_subscription_models():
+        subscription = subscription_model.model_dump()
 
     def inject_in_use_by_ids(path_to_block: str) -> None:
         if not (in_use_by_subs := getattr_in(subscription_model, f"{path_to_block}.in_use_by")):
@@ -611,15 +618,38 @@ def build_extended_domain_model(subscription_model: SubscriptionModel) -> dict:
         update_in(subscription, f"{path_to_block}.in_use_by_ids", in_use_by_ids)
         update_in(subscription, f"{path_to_block}.in_use_by_relations", in_use_by_relations)
 
-    # find all product blocks, check if they have in_use_by and inject the in_use_by_ids into the subscription dict.
-    for path in paths:
-        inject_in_use_by_ids(path)
+    if app_settings.ENABLE_SUBSCRIPTION_MODEL_OPTIMIZATIONS:
+        # TODO #900 remove toggle and make this path the default
+        # query all subscription instances and inject the in_use_by_ids/in_use_by_relations into the subscription dict.
+        instance_to_in_use_by = {
+            instance.subscription_instance_id: instance.in_use_by
+            for instance in eagerload_all_subscription_instances_only_inuseby(subscription_model.subscription_id)
+        }
+        inject_in_use_by_ids_v2(subscription, instance_to_in_use_by)
+    else:
+        # find all product blocks, check if they have in_use_by and inject the in_use_by_ids into the subscription dict.
+        for path in product_block_paths(subscription):
+            inject_in_use_by_ids(path)
 
     subscription["customer_descriptions"] = customer_descriptions
 
     return subscription
 
 
+def inject_in_use_by_ids_v2(dikt: dict, instance_to_in_use_by: dict[UUID, Sequence[SubscriptionInstanceTable]]) -> None:
+    for value in dikt.values():
+        if isinstance(value, dict):
+            inject_in_use_by_ids_v2(value, instance_to_in_use_by)
+        elif isinstance(value, list) and isinstance(first(value, None), dict):
+            for item in value:
+                inject_in_use_by_ids_v2(item, instance_to_in_use_by)
+
+    if subscription_instance_id := dikt.get("subscription_instance_id"):
+        in_use_by_subs = instance_to_in_use_by[subscription_instance_id]
+        dikt["in_use_by_ids"] = [i.subscription_instance_id for i in in_use_by_subs]
+        dikt["in_use_by_relations"] = [convert_to_in_use_by_relation(instance) for instance in in_use_by_subs]
+
+
 def format_special_types(subscription: dict) -> dict:
     """Modifies the subscription dict in-place, formatting special types to string.
 

orchestrator/services/tasks.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/settings.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -13,7 +13,6 @@
 
 import secrets
 import string
-import warnings
 from pathlib import Path
 from typing import Literal
 
@@ -24,10 +23,6 @@ from oauth2_lib.settings import oauth2lib_settings
 from pydantic_forms.types import strEnum
 
 
-class OrchestratorDeprecationWarning(DeprecationWarning):
-    pass
-
-
 class ExecutorType(strEnum):
     WORKER = "celery"
     THREADPOOL = "threadpool"
@@ -54,7 +49,7 @@ class AppSettings(BaseSettings):
     EXECUTOR: str = ExecutorType.THREADPOOL
     WORKFLOWS_SWAGGER_HOST: str = "localhost"
     WORKFLOWS_GUI_URI: str = "http://localhost:3000"
-    DATABASE_URI: PostgresDsn = "postgresql+psycopg://nwa:nwa@localhost/orchestrator-core"  # type: ignore
+    DATABASE_URI: PostgresDsn = "postgresql://nwa:nwa@localhost/orchestrator-core"  # type: ignore
     MAX_WORKERS: int = 5
     MAIL_SERVER: str = "localhost"
     MAIL_PORT: int = 25
@@ -92,22 +87,9 @@ class AppSettings(BaseSettings):
     ENABLE_GRAPHQL_STATS_EXTENSION: bool = False
     VALIDATE_OUT_OF_SYNC_SUBSCRIPTIONS: bool = False
     FILTER_BY_MODE: Literal["partial", "exact"] = "exact"
-
-    def __init__(self) -> None:
-        super(AppSettings, self).__init__()
-        self.DATABASE_URI = PostgresDsn(convert_database_uri(str(self.DATABASE_URI)))
-
-
-def convert_database_uri(db_uri: str) -> str:
-    if db_uri.startswith(("postgresql://", "postgresql+psycopg2://")):
-        db_uri = "postgresql+psycopg" + db_uri[db_uri.find("://") :]
-        warnings.filterwarnings("always", category=OrchestratorDeprecationWarning)
-        warnings.warn(
-            "DATABASE_URI converted to postgresql+psycopg:// format, please update your enviroment variable",
-            OrchestratorDeprecationWarning,
-            stacklevel=2,
-        )
-    return db_uri
+    ENABLE_SUBSCRIPTION_MODEL_OPTIMIZATIONS: bool = (
+        True  # True=ignore cache + optimized DB queries; False=use cache + unoptimized DB queries. Remove in #900
+    )
 
 
 app_settings = AppSettings()

orchestrator/targets.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/types.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/utils/errors.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/utils/functional.py

@@ -241,3 +241,12 @@ def to_ranges(i: Iterable[int]) -> Iterable[range]:
     for _, g in itertools.groupby(enumerate(i), lambda t: t[1] - t[0]):
         group = list(g)
         yield range(group[0][1], group[-1][1] + 1)
+
+
+K = TypeVar("K")
+V = TypeVar("V")
+
+
+def group_by_key(items: Iterable[tuple[K, V]]) -> dict[K, list[V]]:
+    groups = itertools.groupby(items, key=lambda item: item[0])
+    return {key: [item[1] for item in group] for key, group in groups}

orchestrator/utils/redis.py

@@ -55,6 +55,12 @@ def to_redis(subscription: dict[str, Any]) -> str | None:
 
 def from_redis(subscription_id: UUID) -> tuple[PY_JSON_TYPES, str] | None:
     log = logger.bind(subscription_id=subscription_id)
+
+    if app_settings.ENABLE_SUBSCRIPTION_MODEL_OPTIMIZATIONS:
+        # TODO #900 remove toggle and remove usage of this function in get_subscription_dict
+        log.warning("Using SubscriptionModel optimization, not loading subscription from cache")
+        return None
+
     if caching_models_enabled():
         log.debug("Try to retrieve subscription from cache")
         obj = cache.get(f"orchestrator:domain:{subscription_id}")

orchestrator/utils/state.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF, ESnet
+# Copyright 2019-2020 SURF, ESnet, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/websocket/websocket_manager.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/workflow.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2025 SURF, GÉANT, ESnet.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -39,6 +39,7 @@ from structlog.contextvars import bound_contextvars
 from structlog.stdlib import BoundLogger
 
 from nwastdlib import const, identity
+from oauth2_lib.fastapi import OIDCUserModel
 from orchestrator.config.assignee import Assignee
 from orchestrator.db import db, transactional
 from orchestrator.services.settings import get_engine_settings
@@ -89,6 +90,7 @@ class Workflow(Protocol):
     __qualname__: str
     name: str
     description: str
+    authorize_callback: Callable[[OIDCUserModel | None], bool]
     initial_input_form: InputFormGenerator | None = None
     target: Target
     steps: StepList
@@ -178,12 +180,18 @@ def _handle_simple_input_form_generator(f: StateInputStepFunc) -> StateInputForm
     return form_generator
 
 
+def allow(_: OIDCUserModel | None) -> bool:
+    """Default function to return True in absence of user-defined authorize function."""
+    return True
+
+
 def make_workflow(
     f: Callable,
     description: str,
     initial_input_form: InputStepFunc | None,
     target: Target,
     steps: StepList,
+    authorize_callback: Callable[[OIDCUserModel | None], bool] | None = None,
 ) -> Workflow:
     @functools.wraps(f)
     def wrapping_function() -> NoReturn:
@@ -193,6 +201,7 @@ def make_workflow(
 
     wrapping_function.name = f.__name__  # default, will be changed by LazyWorkflowInstance
     wrapping_function.description = description
+    wrapping_function.authorize_callback = allow if authorize_callback is None else authorize_callback
 
     if initial_input_form is None:
         # We always need a form to prevent starting a workflow when no input is needed.
@@ -214,7 +223,11 @@ def step(name: str) -> Callable[[StepFunc], Step]:
     def decorator(func: StepFunc) -> Step:
         @functools.wraps(func)
         def wrapper(state: State) -> Process:
-            with bound_contextvars(func=func.__qualname__):
+            with bound_contextvars(
+                func=func.__qualname__,
+                workflow_name=state.get("workflow_name"),
+                process_id=state.get("process_id"),
+            ):
                 step_in_inject_args = inject_args(func)
                 try:
                     with transactional(db, logger):
@@ -239,7 +252,11 @@ def retrystep(name: str) -> Callable[[StepFunc], Step]:
     def decorator(func: StepFunc) -> Step:
         @functools.wraps(func)
         def wrapper(state: State) -> Process:
-            with bound_contextvars(func=func.__qualname__):
+            with bound_contextvars(
+                func=func.__qualname__,
+                workflow_name=state.get("workflow_name"),
+                process_id=state.get("process_id"),
+            ):
                 step_in_inject_args = inject_args(func)
                 try:
                     with transactional(db, logger):
@@ -459,7 +476,10 @@ def focussteps(key: str) -> Callable[[Step | StepList], StepList]:
 
 
 def workflow(
-    description: str, initial_input_form: InputStepFunc | None = None, target: Target = Target.SYSTEM
+    description: str,
+    initial_input_form: InputStepFunc | None = None,
+    target: Target = Target.SYSTEM,
+    authorize_callback: Callable[[OIDCUserModel | None], bool] | None = None,
 ) -> Callable[[Callable[[], StepList]], Workflow]:
     """Transform an initial_input_form and a step list into a workflow.
 
@@ -479,7 +499,9 @@ def workflow(
         initial_input_form_in_form_inject_args = form_inject_args(initial_input_form)
 
     def _workflow(f: Callable[[], StepList]) -> Workflow:
-        return make_workflow(f, description, initial_input_form_in_form_inject_args, target, f())
+        return make_workflow(
+            f, description, initial_input_form_in_form_inject_args, target, f(), authorize_callback=authorize_callback
+        )
 
     return _workflow
 
@@ -491,13 +513,14 @@ class ProcessStat:
     state: Process
     log: StepList
     current_user: str
+    user_model: OIDCUserModel | None = None
 
     def update(self, **vs: Any) -> ProcessStat:
         """Update ProcessStat.
 
         >>> pstat = ProcessStat('', None, {}, [], "")
         >>> pstat.update(state={"a": "b"})
-        ProcessStat(process_id='', workflow=None, state={'a': 'b'}, log=[], current_user='')
+        ProcessStat(process_id='', workflow=None, state={'a': 'b'}, log=[], current_user='', user_model=None)
         """
         return ProcessStat(**{**asdict(self), **vs})
 

orchestrator/workflows/modify_note.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/workflows/steps.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/workflows/tasks/cleanup_tasks_log.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/workflows/tasks/resume_workflows.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/workflows/tasks/validate_product_type.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2024 SURF.
+# Copyright 2019-2024 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/workflows/tasks/validate_products.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2020 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/workflows/utils.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2025 SURF, GÉANT, ESnet.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -20,6 +20,7 @@ from more_itertools import first_true
 from pydantic import field_validator, model_validator
 from sqlalchemy import select
 
+from oauth2_lib.fastapi import OIDCUserModel
 from orchestrator.db import ProductTable, SubscriptionTable, db
 from orchestrator.forms.validators import ProductId
 from orchestrator.services import subscriptions
@@ -30,7 +31,7 @@ from orchestrator.utils.errors import StaleDataError
 from orchestrator.utils.redis import caching_models_enabled
 from orchestrator.utils.state import form_inject_args
 from orchestrator.utils.validate_data_version import validate_data_version
-from orchestrator.workflow import StepList, Workflow, conditional, done, init, make_workflow, step
+from orchestrator.workflow import Step, StepList, Workflow, begin, conditional, done, init, make_workflow, step
 from orchestrator.workflows.steps import (
     cache_domain_models,
     refresh_subscription_search_index,
@@ -205,6 +206,7 @@ def create_workflow(
     initial_input_form: InputStepFunc | None = None,
     status: SubscriptionLifecycle = SubscriptionLifecycle.ACTIVE,
     additional_steps: StepList | None = None,
+    authorize_callback: Callable[[OIDCUserModel | None], bool] | None = None,
 ) -> Callable[[Callable[[], StepList]], Workflow]:
     """Transform an initial_input_form and a step list into a workflow with a target=Target.CREATE.
 
@@ -231,7 +233,14 @@ def create_workflow(
             >> done
         )
 
-        return make_workflow(f, description, create_initial_input_form_generator, Target.CREATE, steplist)
+        return make_workflow(
+            f,
+            description,
+            create_initial_input_form_generator,
+            Target.CREATE,
+            steplist,
+            authorize_callback=authorize_callback,
+        )
 
     return _create_workflow
 
@@ -240,6 +249,7 @@ def modify_workflow(
     description: str,
     initial_input_form: InputStepFunc | None = None,
     additional_steps: StepList | None = None,
+    authorize_callback: Callable[[OIDCUserModel | None], bool] | None = None,
 ) -> Callable[[Callable[[], StepList]], Workflow]:
     """Transform an initial_input_form and a step list into a workflow.
 
@@ -269,7 +279,14 @@ def modify_workflow(
             >> done
         )
 
-        return make_workflow(f, description, wrapped_modify_initial_input_form_generator, Target.MODIFY, steplist)
+        return make_workflow(
+            f,
+            description,
+            wrapped_modify_initial_input_form_generator,
+            Target.MODIFY,
+            steplist,
+            authorize_callback=authorize_callback,
+        )
 
     return _modify_workflow
 
@@ -278,6 +295,7 @@ def terminate_workflow(
     description: str,
     initial_input_form: InputStepFunc | None = None,
     additional_steps: StepList | None = None,
+    authorize_callback: Callable[[OIDCUserModel | None], bool] | None = None,
 ) -> Callable[[Callable[[], StepList]], Workflow]:
     """Transform an initial_input_form and a step list into a workflow.
 
@@ -308,7 +326,14 @@ def terminate_workflow(
             >> done
         )
 
-        return make_workflow(f, description, wrapped_terminate_initial_input_form_generator, Target.TERMINATE, steplist)
+        return make_workflow(
+            f,
+            description,
+            wrapped_terminate_initial_input_form_generator,
+            Target.TERMINATE,
+            steplist,
+            authorize_callback=authorize_callback,
+        )
 
     return _terminate_workflow
 
@@ -344,6 +369,16 @@ def validate_workflow(description: str) -> Callable[[Callable[[], StepList]], Wo
     return _validate_workflow
 
 
+def ensure_provisioning_status(modify_steps: Step | StepList) -> StepList:
+    """Decorator to ensure subscription modifications are executed only during Provisioning status."""
+    return (
+        begin
+        >> set_status(SubscriptionLifecycle.PROVISIONING)
+        >> modify_steps
+        >> set_status(SubscriptionLifecycle.ACTIVE)
+    )
+
+
 @step("Equalize workflow step count")
 def obsolete_step() -> None:
     """Equalize workflow step counts.

{orchestrator_core-3.1.2rc3.dist-info → orchestrator_core-3.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-core
-Version: 3.1.2rc3
+Version: 3.2.0
 Summary: This is the orchestrator workflow engine.
 Requires-Python: >=3.11,<3.14
 Classifier: Intended Audience :: Information Technology
@@ -28,7 +28,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
 Classifier: Topic :: Internet :: WWW/HTTP
 License-File: LICENSE
-Requires-Dist: alembic==1.15.1
+Requires-Dist: alembic==1.15.2
 Requires-Dist: anyio>=3.7.0
 Requires-Dist: click==8.*
 Requires-Dist: deprecated
@@ -38,17 +38,18 @@ Requires-Dist: fastapi-etag==0.4.0
 Requires-Dist: more-itertools~=10.6.0
 Requires-Dist: itsdangerous
 Requires-Dist: Jinja2==3.1.6
-Requires-Dist: orjson==3.10.15
-Requires-Dist: psycopg[binary]==3.2.6
+Requires-Dist: orjson==3.10.16
+Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: pydantic[email]~=2.8.2
 Requires-Dist: pydantic-settings~=2.8.0
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: python-rapidjson>=1.18,<1.21
-Requires-Dist: pytz==2025.1
+Requires-Dist: pytz==2025.2
 Requires-Dist: redis==5.1.1
 Requires-Dist: schedule==1.1.0
-Requires-Dist: sentry-sdk[fastapi]~=2.22.0
-Requires-Dist: SQLAlchemy==2.0.39
+Requires-Dist: semver==3.0.4
+Requires-Dist: sentry-sdk[fastapi]~=2.25.1
+Requires-Dist: SQLAlchemy==2.0.40
 Requires-Dist: SQLAlchemy-Utils==0.41.2
 Requires-Dist: structlog
 Requires-Dist: typer==0.15.2
@@ -57,8 +58,8 @@ Requires-Dist: nwa-stdlib~=1.9.0
 Requires-Dist: oauth2-lib~=2.4.0
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: strawberry-graphql>=0.246.2
-Requires-Dist: pydantic-forms~=1.4.0
-Requires-Dist: celery~=5.4.0 ; extra == "celery"
+Requires-Dist: pydantic-forms>=1.4.0, <=2.0.0
+Requires-Dist: celery~=5.5.1 ; extra == "celery"
 Requires-Dist: toml ; extra == "dev"
 Requires-Dist: bumpversion ; extra == "dev"
 Requires-Dist: mypy_extensions ; extra == "dev"