orchestrator-core 3.1.2rc1__py3-none-any.whl → 3.1.2rc4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- orchestrator/__init__.py +2 -2
- orchestrator/api/api_v1/api.py +1 -1
- orchestrator/api/api_v1/endpoints/processes.py +24 -1
- orchestrator/api/api_v1/endpoints/settings.py +1 -1
- orchestrator/api/api_v1/endpoints/subscriptions.py +1 -1
- orchestrator/app.py +1 -1
- orchestrator/cli/database.py +1 -1
- orchestrator/cli/generator/generator/migration.py +2 -5
- orchestrator/cli/migrate_tasks.py +13 -0
- orchestrator/config/assignee.py +1 -1
- orchestrator/db/models.py +1 -1
- orchestrator/devtools/populator.py +1 -1
- orchestrator/domain/__init__.py +2 -3
- orchestrator/domain/base.py +74 -5
- orchestrator/domain/lifecycle.py +1 -1
- orchestrator/graphql/schema.py +1 -1
- orchestrator/graphql/types.py +1 -1
- orchestrator/graphql/utils/get_subscription_product_blocks.py +13 -0
- orchestrator/migrations/env.py +1 -1
- orchestrator/migrations/helpers.py +6 -6
- orchestrator/schemas/engine_settings.py +1 -1
- orchestrator/schemas/subscription.py +1 -1
- orchestrator/security.py +1 -1
- orchestrator/services/celery.py +1 -1
- orchestrator/services/processes.py +72 -10
- orchestrator/services/products.py +1 -1
- orchestrator/services/subscriptions.py +1 -1
- orchestrator/services/tasks.py +1 -1
- orchestrator/settings.py +2 -23
- orchestrator/targets.py +1 -1
- orchestrator/types.py +1 -1
- orchestrator/utils/errors.py +1 -1
- orchestrator/utils/state.py +1 -1
- orchestrator/websocket/websocket_manager.py +1 -1
- orchestrator/workflow.py +2 -1
- orchestrator/workflows/modify_note.py +1 -1
- orchestrator/workflows/steps.py +1 -1
- orchestrator/workflows/tasks/cleanup_tasks_log.py +1 -1
- orchestrator/workflows/tasks/resume_workflows.py +1 -1
- orchestrator/workflows/tasks/validate_product_type.py +1 -1
- orchestrator/workflows/tasks/validate_products.py +1 -1
- orchestrator/workflows/utils.py +1 -1
- orchestrator_core-3.1.2rc4.dist-info/METADATA +116 -0
- {orchestrator_core-3.1.2rc1.dist-info → orchestrator_core-3.1.2rc4.dist-info}/RECORD +46 -46
- {orchestrator_core-3.1.2rc1.dist-info → orchestrator_core-3.1.2rc4.dist-info}/WHEEL +1 -1
- orchestrator_core-3.1.2rc1.dist-info/METADATA +0 -321
- {orchestrator_core-3.1.2rc1.dist-info → orchestrator_core-3.1.2rc4.dist-info}/licenses/LICENSE +0 -0
orchestrator/__init__.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-
|
|
1
|
+
# Copyright 2019-2025 SURF, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
"""This is the orchestrator workflow engine."""
|
|
15
15
|
|
|
16
|
-
__version__ = "3.1.
|
|
16
|
+
__version__ = "3.1.2rc4"
|
|
17
17
|
|
|
18
18
|
from orchestrator.app import OrchestratorCore
|
|
19
19
|
from orchestrator.settings import app_settings
|
orchestrator/api/api_v1/api.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-
|
|
1
|
+
# Copyright 2019-2025 SURF, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -58,6 +58,7 @@ from orchestrator.services.processes import (
|
|
|
58
58
|
load_process,
|
|
59
59
|
resume_process,
|
|
60
60
|
start_process,
|
|
61
|
+
update_awaiting_process_progress,
|
|
61
62
|
)
|
|
62
63
|
from orchestrator.services.settings import get_engine_settings
|
|
63
64
|
from orchestrator.settings import app_settings
|
|
@@ -197,6 +198,28 @@ def continue_awaiting_process_endpoint(
|
|
|
197
198
|
raise_status(HTTPStatus.NOT_FOUND, str(e))
|
|
198
199
|
|
|
199
200
|
|
|
201
|
+
@router.post(
|
|
202
|
+
"/{process_id}/callback/{token}/progress",
|
|
203
|
+
response_model=None,
|
|
204
|
+
status_code=HTTPStatus.OK,
|
|
205
|
+
dependencies=[Depends(check_global_lock, use_cache=False)],
|
|
206
|
+
)
|
|
207
|
+
def update_progress_on_awaiting_process_endpoint(
|
|
208
|
+
process_id: UUID,
|
|
209
|
+
token: str,
|
|
210
|
+
data: str | State = Body(...),
|
|
211
|
+
) -> None:
|
|
212
|
+
process = _get_process(process_id)
|
|
213
|
+
|
|
214
|
+
if process.last_status != ProcessStatus.AWAITING_CALLBACK:
|
|
215
|
+
raise_status(HTTPStatus.CONFLICT, "This process is not in an awaiting state.")
|
|
216
|
+
|
|
217
|
+
try:
|
|
218
|
+
update_awaiting_process_progress(process, token=token, data=data)
|
|
219
|
+
except AssertionError as exc:
|
|
220
|
+
raise_status(HTTPStatus.NOT_FOUND, str(exc))
|
|
221
|
+
|
|
222
|
+
|
|
200
223
|
@router.put(
|
|
201
224
|
"/resume-all", response_model=ProcessResumeAllSchema, dependencies=[Depends(check_global_lock, use_cache=False)]
|
|
202
225
|
)
|
orchestrator/app.py
CHANGED
|
@@ -5,7 +5,7 @@ This module contains the main `OrchestratorCore` class for the `FastAPI` backend
|
|
|
5
5
|
provides the ability to run the CLI.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
# Copyright 2019-2020 SURF, ESnet
|
|
8
|
+
# Copyright 2019-2020 SURF, ESnet, GÉANT.
|
|
9
9
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
10
10
|
# you may not use this file except in compliance with the License.
|
|
11
11
|
# You may obtain a copy of the License at
|
orchestrator/cli/database.py
CHANGED
|
@@ -31,16 +31,13 @@ from orchestrator.cli.generator.generator.helpers import (
|
|
|
31
31
|
sort_product_blocks_by_dependencies,
|
|
32
32
|
)
|
|
33
33
|
from orchestrator.cli.generator.generator.settings import product_generator_settings as settings
|
|
34
|
-
from orchestrator.settings import convert_database_uri
|
|
35
34
|
|
|
36
35
|
logger = structlog.getLogger(__name__)
|
|
37
36
|
|
|
38
37
|
|
|
39
38
|
def create_migration_file(message: str, head: str) -> Path | None:
|
|
40
|
-
if environ.get("DATABASE_URI"):
|
|
41
|
-
environ.update({"DATABASE_URI":
|
|
42
|
-
else:
|
|
43
|
-
environ.update({"DATABASE_URI": "postgresql+psycopg://nwa:nwa@localhost/orchestrator-core"})
|
|
39
|
+
if not environ.get("DATABASE_URI"):
|
|
40
|
+
environ.update({"DATABASE_URI": "postgresql://nwa:nwa@localhost/orchestrator-core"})
|
|
44
41
|
if not environ.get("PYTHONPATH"):
|
|
45
42
|
environ.update({"PYTHONPATH": "."})
|
|
46
43
|
logger.info(
|
|
@@ -1,3 +1,16 @@
|
|
|
1
|
+
# Copyright 2019-2025 SURF, GÉANT.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
#
|
|
6
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
7
|
+
#
|
|
8
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
9
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
10
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
11
|
+
# See the License for the specific language governing permissions and
|
|
12
|
+
# limitations under the License.
|
|
13
|
+
|
|
1
14
|
import itertools
|
|
2
15
|
import operator
|
|
3
16
|
from collections.abc import Iterable
|
orchestrator/config/assignee.py
CHANGED
orchestrator/db/models.py
CHANGED
orchestrator/domain/__init__.py
CHANGED
|
@@ -11,11 +11,10 @@
|
|
|
11
11
|
# See the License for the specific language governing permissions and
|
|
12
12
|
# limitations under the License.
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
from orchestrator.domain.base import SubscriptionModel
|
|
14
|
+
from orchestrator.domain.base import SubscriptionModel, SubscriptionModelRegistry
|
|
16
15
|
from orchestrator.utils.docs import make_product_type_index_doc
|
|
17
16
|
|
|
18
|
-
SUBSCRIPTION_MODEL_REGISTRY:
|
|
17
|
+
SUBSCRIPTION_MODEL_REGISTRY: SubscriptionModelRegistry = SubscriptionModelRegistry()
|
|
19
18
|
|
|
20
19
|
__doc__ = make_product_type_index_doc(SUBSCRIPTION_MODEL_REGISTRY)
|
|
21
20
|
|
orchestrator/domain/base.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-
|
|
1
|
+
# Copyright 2019-2025 SURF, ESnet, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -12,14 +12,16 @@
|
|
|
12
12
|
# limitations under the License.
|
|
13
13
|
import itertools
|
|
14
14
|
from collections import defaultdict
|
|
15
|
-
from collections.abc import Callable, Iterable
|
|
16
15
|
from datetime import datetime
|
|
17
|
-
from inspect import get_annotations
|
|
16
|
+
from inspect import get_annotations, isclass
|
|
18
17
|
from itertools import groupby, zip_longest
|
|
19
18
|
from operator import attrgetter
|
|
20
19
|
from typing import (
|
|
21
20
|
Any,
|
|
21
|
+
Callable,
|
|
22
22
|
ClassVar,
|
|
23
|
+
Iterable,
|
|
24
|
+
Mapping,
|
|
23
25
|
Optional,
|
|
24
26
|
TypeVar,
|
|
25
27
|
Union,
|
|
@@ -596,7 +598,9 @@ class ProductBlockModel(DomainModel):
|
|
|
596
598
|
product_blocks_in_model = cls._get_depends_on_product_block_types()
|
|
597
599
|
product_blocks_types_in_model = get_depends_on_product_block_type_list(product_blocks_in_model)
|
|
598
600
|
|
|
599
|
-
product_blocks_in_model = set(
|
|
601
|
+
product_blocks_in_model = set(
|
|
602
|
+
flatten(map(attrgetter("__names__"), product_blocks_types_in_model))
|
|
603
|
+
) # type: ignore
|
|
600
604
|
|
|
601
605
|
missing_product_blocks_in_db = product_blocks_in_model - product_blocks_in_db # type: ignore
|
|
602
606
|
missing_product_blocks_in_model = product_blocks_in_db - product_blocks_in_model # type: ignore
|
|
@@ -1051,7 +1055,9 @@ class SubscriptionModel(DomainModel):
|
|
|
1051
1055
|
product_blocks_in_model = cls._get_depends_on_product_block_types()
|
|
1052
1056
|
product_blocks_types_in_model = get_depends_on_product_block_type_list(product_blocks_in_model)
|
|
1053
1057
|
|
|
1054
|
-
product_blocks_in_model = set(
|
|
1058
|
+
product_blocks_in_model = set(
|
|
1059
|
+
flatten(map(attrgetter("__names__"), product_blocks_types_in_model))
|
|
1060
|
+
) # type: ignore
|
|
1055
1061
|
|
|
1056
1062
|
missing_product_blocks_in_db = product_blocks_in_model - product_blocks_in_db # type: ignore
|
|
1057
1063
|
missing_product_blocks_in_model = product_blocks_in_db - product_blocks_in_model # type: ignore
|
|
@@ -1402,6 +1408,69 @@ class SubscriptionModel(DomainModel):
|
|
|
1402
1408
|
return self._db_model
|
|
1403
1409
|
|
|
1404
1410
|
|
|
1411
|
+
def validate_base_model(
|
|
1412
|
+
name: str, cls: type[Any], base_model: type[BaseModel] = DomainModel, errors: list[str] | None = None
|
|
1413
|
+
) -> None:
|
|
1414
|
+
"""Validates that the given class is not Pydantic BaseModel or its direct subclass."""
|
|
1415
|
+
# Instantiate errors list if not provided and avoid mutating default
|
|
1416
|
+
if errors is None:
|
|
1417
|
+
errors = []
|
|
1418
|
+
# Return early when the node is not a class as there is nothing to be done
|
|
1419
|
+
if not isclass(cls):
|
|
1420
|
+
return
|
|
1421
|
+
# Validate each field in the ProductBlockModel's field dictionaries
|
|
1422
|
+
if issubclass(cls, ProductBlockModel) or issubclass(cls, SubscriptionModel):
|
|
1423
|
+
for name, clz in cls._product_block_fields_.items():
|
|
1424
|
+
validate_base_model(name, clz, ProductBlockModel, errors)
|
|
1425
|
+
for name, clz in cls._non_product_block_fields_.items():
|
|
1426
|
+
validate_base_model(name, clz, SubscriptionModel, errors)
|
|
1427
|
+
# Generate error if node is Pydantic BaseModel or direct subclass
|
|
1428
|
+
if issubclass(cls, BaseModel):
|
|
1429
|
+
err_msg: str = (
|
|
1430
|
+
f"If this field was intended to be a {base_model.__name__}, define {name}:{cls.__name__} with "
|
|
1431
|
+
f"{base_model.__name__} as its superclass instead. e.g., class {cls.__name__}({base_model.__name__}):"
|
|
1432
|
+
)
|
|
1433
|
+
if cls is BaseModel:
|
|
1434
|
+
errors.append(f"Field {name}: {cls.__name__} can not be {BaseModel.__name__}. " + err_msg)
|
|
1435
|
+
if len(cls.__mro__) > 1 and cls.__mro__[1] is BaseModel:
|
|
1436
|
+
errors.append(
|
|
1437
|
+
f"Field {name}: {cls.__name__} can not be a direct subclass of {BaseModel.__name__}. " + err_msg
|
|
1438
|
+
)
|
|
1439
|
+
# Format all errors as one per line and raise a TypeError when they exist
|
|
1440
|
+
if errors:
|
|
1441
|
+
raise TypeError("\n".join(errors))
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
class SubscriptionModelRegistry(dict[str, type[SubscriptionModel]]):
|
|
1445
|
+
"""A registry for all subscription models."""
|
|
1446
|
+
|
|
1447
|
+
def __setitem__(self, __key: str, __value: type[SubscriptionModel]) -> None:
|
|
1448
|
+
"""Set value for key in while validating against Pydantic BaseModel."""
|
|
1449
|
+
validate_base_model(__key, __value)
|
|
1450
|
+
super().__setitem__(__key, __value)
|
|
1451
|
+
|
|
1452
|
+
def update(
|
|
1453
|
+
self,
|
|
1454
|
+
m: Any = None,
|
|
1455
|
+
/,
|
|
1456
|
+
**kwargs: type[SubscriptionModel],
|
|
1457
|
+
) -> None:
|
|
1458
|
+
"""Update dictionary with mapping and/or kwargs using `__setitem__`."""
|
|
1459
|
+
if m:
|
|
1460
|
+
if isinstance(m, Mapping):
|
|
1461
|
+
for key, value in m.items():
|
|
1462
|
+
self[key] = value
|
|
1463
|
+
elif isinstance(m, Iterable):
|
|
1464
|
+
for index, item in enumerate(m):
|
|
1465
|
+
try:
|
|
1466
|
+
key, value = item
|
|
1467
|
+
except ValueError:
|
|
1468
|
+
raise TypeError(f"dictionary update sequence element #{index} is not an iterable of length 2")
|
|
1469
|
+
self[key] = value
|
|
1470
|
+
for key, value in kwargs.items():
|
|
1471
|
+
self[key] = value
|
|
1472
|
+
|
|
1473
|
+
|
|
1405
1474
|
def _validate_lifecycle_change_for_product_block(
|
|
1406
1475
|
used_by: SubscriptionInstanceTable,
|
|
1407
1476
|
product_block_model: ProductBlockModel,
|
orchestrator/domain/lifecycle.py
CHANGED
orchestrator/graphql/schema.py
CHANGED
orchestrator/graphql/types.py
CHANGED
|
@@ -1,3 +1,16 @@
|
|
|
1
|
+
# Copyright 2022-2023 SURF, GÉANT.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
#
|
|
6
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
7
|
+
#
|
|
8
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
9
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
10
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
11
|
+
# See the License for the specific language governing permissions and
|
|
12
|
+
# limitations under the License.
|
|
13
|
+
|
|
1
14
|
from collections.abc import Generator
|
|
2
15
|
from itertools import count
|
|
3
16
|
from typing import TYPE_CHECKING, Annotated, Any
|
orchestrator/migrations/env.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-2020 SURF.
|
|
1
|
+
# Copyright 2019-2020 SURF, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -880,10 +880,10 @@ def delete_product(conn: sa.engine.Connection, name: str) -> None:
|
|
|
880
880
|
RETURNING product_id
|
|
881
881
|
),
|
|
882
882
|
deleted_p_pb AS (
|
|
883
|
-
DELETE FROM product_product_blocks WHERE product_id
|
|
883
|
+
DELETE FROM product_product_blocks WHERE product_id IN (SELECT product_id FROM deleted_p)
|
|
884
884
|
),
|
|
885
885
|
deleted_pb_rt AS (
|
|
886
|
-
DELETE FROM products_workflows WHERE product_id
|
|
886
|
+
DELETE FROM products_workflows WHERE product_id IN (SELECT product_id FROM deleted_p)
|
|
887
887
|
)
|
|
888
888
|
SELECT * from deleted_p;
|
|
889
889
|
"""
|
|
@@ -911,10 +911,10 @@ def delete_product_block(conn: sa.engine.Connection, name: str) -> None:
|
|
|
911
911
|
RETURNING product_block_id
|
|
912
912
|
),
|
|
913
913
|
deleted_p_pb AS (
|
|
914
|
-
DELETE FROM product_product_blocks WHERE product_block_id
|
|
914
|
+
DELETE FROM product_product_blocks WHERE product_block_id IN (SELECT product_block_id FROM deleted_pb)
|
|
915
915
|
),
|
|
916
916
|
deleted_pb_rt AS (
|
|
917
|
-
DELETE FROM product_block_resource_types WHERE product_block_id
|
|
917
|
+
DELETE FROM product_block_resource_types WHERE product_block_id IN (SELECT product_block_id FROM deleted_pb)
|
|
918
918
|
)
|
|
919
919
|
SELECT * from deleted_pb;
|
|
920
920
|
"""
|
|
@@ -968,7 +968,7 @@ def delete_resource_type(conn: sa.engine.Connection, resource_type: str) -> None
|
|
|
968
968
|
RETURNING resource_type_id
|
|
969
969
|
),
|
|
970
970
|
deleted_pb_rt AS (
|
|
971
|
-
DELETE FROM product_block_resource_types WHERE resource_type_id
|
|
971
|
+
DELETE FROM product_block_resource_types WHERE resource_type_id IN (SELECT resource_type_id FROM deleted_pb)
|
|
972
972
|
)
|
|
973
973
|
SELECT * from deleted_pb;
|
|
974
974
|
"""
|
orchestrator/security.py
CHANGED
orchestrator/services/celery.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-2020 SURF.
|
|
1
|
+
# Copyright 2019-2020 SURF, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -43,9 +43,10 @@ from orchestrator.targets import Target
|
|
|
43
43
|
from orchestrator.types import BroadcastFunc
|
|
44
44
|
from orchestrator.utils.datetime import nowtz
|
|
45
45
|
from orchestrator.utils.errors import error_state_to_dict
|
|
46
|
-
from orchestrator.websocket import broadcast_invalidate_status_counts
|
|
46
|
+
from orchestrator.websocket import broadcast_invalidate_status_counts, broadcast_process_update_to_websocket
|
|
47
47
|
from orchestrator.workflow import (
|
|
48
48
|
CALLBACK_TOKEN_KEY,
|
|
49
|
+
DEFAULT_CALLBACK_PROGRESS_KEY,
|
|
49
50
|
Failed,
|
|
50
51
|
ProcessStat,
|
|
51
52
|
ProcessStatus,
|
|
@@ -566,6 +567,39 @@ def resume_process(
|
|
|
566
567
|
return resume_func(process, user_inputs=user_inputs, user=user, broadcast_func=broadcast_func)
|
|
567
568
|
|
|
568
569
|
|
|
570
|
+
def ensure_correct_callback_token(pstat: ProcessStat, *, token: str) -> None:
|
|
571
|
+
"""Ensure that a callback token matches the expected value in state.
|
|
572
|
+
|
|
573
|
+
Args:
|
|
574
|
+
pstat: ProcessStat of process.
|
|
575
|
+
token: The token which was generated for the process.
|
|
576
|
+
|
|
577
|
+
Raises:
|
|
578
|
+
AssertionError: if the supplied token does not match the generated process token.
|
|
579
|
+
|
|
580
|
+
"""
|
|
581
|
+
state = pstat.state.unwrap()
|
|
582
|
+
|
|
583
|
+
# Check if the token matches
|
|
584
|
+
token_from_state = state.get(CALLBACK_TOKEN_KEY)
|
|
585
|
+
if token != token_from_state:
|
|
586
|
+
raise AssertionError("Invalid token")
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def replace_current_step_state(process: ProcessTable, *, new_state: State) -> None:
|
|
590
|
+
"""Replace the state of the current step in a process.
|
|
591
|
+
|
|
592
|
+
Args:
|
|
593
|
+
process: Process from database
|
|
594
|
+
new_state: The new state
|
|
595
|
+
|
|
596
|
+
"""
|
|
597
|
+
current_step = process.steps[-1]
|
|
598
|
+
current_step.state = new_state
|
|
599
|
+
db.session.add(current_step)
|
|
600
|
+
db.session.commit()
|
|
601
|
+
|
|
602
|
+
|
|
569
603
|
def continue_awaiting_process(
|
|
570
604
|
process: ProcessTable,
|
|
571
605
|
*,
|
|
@@ -589,10 +623,7 @@ def continue_awaiting_process(
|
|
|
589
623
|
pstat = load_process(process)
|
|
590
624
|
state = pstat.state.unwrap()
|
|
591
625
|
|
|
592
|
-
|
|
593
|
-
token_from_state = state.get(CALLBACK_TOKEN_KEY)
|
|
594
|
-
if token != token_from_state:
|
|
595
|
-
raise AssertionError("Invalid token")
|
|
626
|
+
ensure_correct_callback_token(pstat, token=token)
|
|
596
627
|
|
|
597
628
|
# We need to pass the callback data to the worker executor. Currently, this is not supported.
|
|
598
629
|
# Therefore, we update the step state in the db and kick-off resume_workflow
|
|
@@ -600,16 +631,47 @@ def continue_awaiting_process(
|
|
|
600
631
|
result_key = state.get("__callback_result_key", "callback_result")
|
|
601
632
|
state = {**state, result_key: input_data}
|
|
602
633
|
|
|
603
|
-
|
|
604
|
-
current_step.state = state
|
|
605
|
-
db.session.add(current_step)
|
|
606
|
-
db.session.commit()
|
|
634
|
+
replace_current_step_state(process, new_state=state)
|
|
607
635
|
|
|
608
636
|
# Continue the workflow
|
|
609
637
|
resume_func = get_execution_context()["resume"]
|
|
610
638
|
return resume_func(process, broadcast_func=broadcast_func)
|
|
611
639
|
|
|
612
640
|
|
|
641
|
+
def update_awaiting_process_progress(
|
|
642
|
+
process: ProcessTable,
|
|
643
|
+
*,
|
|
644
|
+
token: str,
|
|
645
|
+
data: str | State,
|
|
646
|
+
) -> UUID:
|
|
647
|
+
"""Update progress for a process awaiting data from a callback.
|
|
648
|
+
|
|
649
|
+
Args:
|
|
650
|
+
process: Process from database
|
|
651
|
+
token: The token which was generated for the process. This must match.
|
|
652
|
+
data: Progress data posted to the callback
|
|
653
|
+
|
|
654
|
+
Returns:
|
|
655
|
+
process id
|
|
656
|
+
|
|
657
|
+
Raises:
|
|
658
|
+
AssertionError: if the supplied token does not match the generated process token.
|
|
659
|
+
|
|
660
|
+
"""
|
|
661
|
+
pstat = load_process(process)
|
|
662
|
+
|
|
663
|
+
ensure_correct_callback_token(pstat, token=token)
|
|
664
|
+
|
|
665
|
+
state = pstat.state.unwrap()
|
|
666
|
+
progress_key = state.get(DEFAULT_CALLBACK_PROGRESS_KEY, "callback_progress")
|
|
667
|
+
state = {**state, progress_key: data} | {"__remove_keys": [progress_key]}
|
|
668
|
+
|
|
669
|
+
replace_current_step_state(process, new_state=state)
|
|
670
|
+
broadcast_process_update_to_websocket(process.process_id)
|
|
671
|
+
|
|
672
|
+
return process.process_id
|
|
673
|
+
|
|
674
|
+
|
|
613
675
|
async def _async_resume_processes(
|
|
614
676
|
processes: Sequence[ProcessTable],
|
|
615
677
|
user_name: str,
|
orchestrator/services/tasks.py
CHANGED
orchestrator/settings.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-2020 SURF.
|
|
1
|
+
# Copyright 2019-2020 SURF, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -13,7 +13,6 @@
|
|
|
13
13
|
|
|
14
14
|
import secrets
|
|
15
15
|
import string
|
|
16
|
-
import warnings
|
|
17
16
|
from pathlib import Path
|
|
18
17
|
from typing import Literal
|
|
19
18
|
|
|
@@ -24,10 +23,6 @@ from oauth2_lib.settings import oauth2lib_settings
|
|
|
24
23
|
from pydantic_forms.types import strEnum
|
|
25
24
|
|
|
26
25
|
|
|
27
|
-
class OrchestratorDeprecationWarning(DeprecationWarning):
|
|
28
|
-
pass
|
|
29
|
-
|
|
30
|
-
|
|
31
26
|
class ExecutorType(strEnum):
|
|
32
27
|
WORKER = "celery"
|
|
33
28
|
THREADPOOL = "threadpool"
|
|
@@ -54,7 +49,7 @@ class AppSettings(BaseSettings):
|
|
|
54
49
|
EXECUTOR: str = ExecutorType.THREADPOOL
|
|
55
50
|
WORKFLOWS_SWAGGER_HOST: str = "localhost"
|
|
56
51
|
WORKFLOWS_GUI_URI: str = "http://localhost:3000"
|
|
57
|
-
DATABASE_URI: PostgresDsn = "postgresql
|
|
52
|
+
DATABASE_URI: PostgresDsn = "postgresql://nwa:nwa@localhost/orchestrator-core" # type: ignore
|
|
58
53
|
MAX_WORKERS: int = 5
|
|
59
54
|
MAIL_SERVER: str = "localhost"
|
|
60
55
|
MAIL_PORT: int = 25
|
|
@@ -93,22 +88,6 @@ class AppSettings(BaseSettings):
|
|
|
93
88
|
VALIDATE_OUT_OF_SYNC_SUBSCRIPTIONS: bool = False
|
|
94
89
|
FILTER_BY_MODE: Literal["partial", "exact"] = "exact"
|
|
95
90
|
|
|
96
|
-
def __init__(self) -> None:
|
|
97
|
-
super(AppSettings, self).__init__()
|
|
98
|
-
self.DATABASE_URI = PostgresDsn(convert_database_uri(str(self.DATABASE_URI)))
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
def convert_database_uri(db_uri: str) -> str:
|
|
102
|
-
if db_uri.startswith(("postgresql://", "postgresql+psycopg2://")):
|
|
103
|
-
db_uri = "postgresql+psycopg" + db_uri[db_uri.find("://") :]
|
|
104
|
-
warnings.filterwarnings("always", category=OrchestratorDeprecationWarning)
|
|
105
|
-
warnings.warn(
|
|
106
|
-
"DATABASE_URI converted to postgresql+psycopg:// format, please update your enviroment variable",
|
|
107
|
-
OrchestratorDeprecationWarning,
|
|
108
|
-
stacklevel=2,
|
|
109
|
-
)
|
|
110
|
-
return db_uri
|
|
111
|
-
|
|
112
91
|
|
|
113
92
|
app_settings = AppSettings()
|
|
114
93
|
|
orchestrator/targets.py
CHANGED
orchestrator/types.py
CHANGED
orchestrator/utils/errors.py
CHANGED
orchestrator/utils/state.py
CHANGED
orchestrator/workflow.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-
|
|
1
|
+
# Copyright 2019-2025 SURF, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -69,6 +69,7 @@ step_log_fn_var: contextvars.ContextVar[StepLogFuncInternal] = contextvars.Conte
|
|
|
69
69
|
|
|
70
70
|
DEFAULT_CALLBACK_ROUTE_KEY = "callback_route"
|
|
71
71
|
CALLBACK_TOKEN_KEY = "__callback_token" # noqa: S105
|
|
72
|
+
DEFAULT_CALLBACK_PROGRESS_KEY = "callback_progress" # noqa: S105
|
|
72
73
|
|
|
73
74
|
|
|
74
75
|
@runtime_checkable
|