orchestrator-core 3.1.2rc1__py3-none-any.whl → 3.1.2rc4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. orchestrator/__init__.py +2 -2
  2. orchestrator/api/api_v1/api.py +1 -1
  3. orchestrator/api/api_v1/endpoints/processes.py +24 -1
  4. orchestrator/api/api_v1/endpoints/settings.py +1 -1
  5. orchestrator/api/api_v1/endpoints/subscriptions.py +1 -1
  6. orchestrator/app.py +1 -1
  7. orchestrator/cli/database.py +1 -1
  8. orchestrator/cli/generator/generator/migration.py +2 -5
  9. orchestrator/cli/migrate_tasks.py +13 -0
  10. orchestrator/config/assignee.py +1 -1
  11. orchestrator/db/models.py +1 -1
  12. orchestrator/devtools/populator.py +1 -1
  13. orchestrator/domain/__init__.py +2 -3
  14. orchestrator/domain/base.py +74 -5
  15. orchestrator/domain/lifecycle.py +1 -1
  16. orchestrator/graphql/schema.py +1 -1
  17. orchestrator/graphql/types.py +1 -1
  18. orchestrator/graphql/utils/get_subscription_product_blocks.py +13 -0
  19. orchestrator/migrations/env.py +1 -1
  20. orchestrator/migrations/helpers.py +6 -6
  21. orchestrator/schemas/engine_settings.py +1 -1
  22. orchestrator/schemas/subscription.py +1 -1
  23. orchestrator/security.py +1 -1
  24. orchestrator/services/celery.py +1 -1
  25. orchestrator/services/processes.py +72 -10
  26. orchestrator/services/products.py +1 -1
  27. orchestrator/services/subscriptions.py +1 -1
  28. orchestrator/services/tasks.py +1 -1
  29. orchestrator/settings.py +2 -23
  30. orchestrator/targets.py +1 -1
  31. orchestrator/types.py +1 -1
  32. orchestrator/utils/errors.py +1 -1
  33. orchestrator/utils/state.py +1 -1
  34. orchestrator/websocket/websocket_manager.py +1 -1
  35. orchestrator/workflow.py +2 -1
  36. orchestrator/workflows/modify_note.py +1 -1
  37. orchestrator/workflows/steps.py +1 -1
  38. orchestrator/workflows/tasks/cleanup_tasks_log.py +1 -1
  39. orchestrator/workflows/tasks/resume_workflows.py +1 -1
  40. orchestrator/workflows/tasks/validate_product_type.py +1 -1
  41. orchestrator/workflows/tasks/validate_products.py +1 -1
  42. orchestrator/workflows/utils.py +1 -1
  43. orchestrator_core-3.1.2rc4.dist-info/METADATA +116 -0
  44. {orchestrator_core-3.1.2rc1.dist-info → orchestrator_core-3.1.2rc4.dist-info}/RECORD +46 -46
  45. {orchestrator_core-3.1.2rc1.dist-info → orchestrator_core-3.1.2rc4.dist-info}/WHEEL +1 -1
  46. orchestrator_core-3.1.2rc1.dist-info/METADATA +0 -321
  47. {orchestrator_core-3.1.2rc1.dist-info → orchestrator_core-3.1.2rc4.dist-info}/licenses/LICENSE +0 -0
orchestrator/__init__.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2025 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -13,7 +13,7 @@
13
13
 
14
14
  """This is the orchestrator workflow engine."""
15
15
 
16
- __version__ = "3.1.2rc1"
16
+ __version__ = "3.1.2rc4"
17
17
 
18
18
  from orchestrator.app import OrchestratorCore
19
19
  from orchestrator.settings import app_settings
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2025 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -58,6 +58,7 @@ from orchestrator.services.processes import (
58
58
  load_process,
59
59
  resume_process,
60
60
  start_process,
61
+ update_awaiting_process_progress,
61
62
  )
62
63
  from orchestrator.services.settings import get_engine_settings
63
64
  from orchestrator.settings import app_settings
@@ -197,6 +198,28 @@ def continue_awaiting_process_endpoint(
197
198
  raise_status(HTTPStatus.NOT_FOUND, str(e))
198
199
 
199
200
 
201
+ @router.post(
202
+ "/{process_id}/callback/{token}/progress",
203
+ response_model=None,
204
+ status_code=HTTPStatus.OK,
205
+ dependencies=[Depends(check_global_lock, use_cache=False)],
206
+ )
207
+ def update_progress_on_awaiting_process_endpoint(
208
+ process_id: UUID,
209
+ token: str,
210
+ data: str | State = Body(...),
211
+ ) -> None:
212
+ process = _get_process(process_id)
213
+
214
+ if process.last_status != ProcessStatus.AWAITING_CALLBACK:
215
+ raise_status(HTTPStatus.CONFLICT, "This process is not in an awaiting state.")
216
+
217
+ try:
218
+ update_awaiting_process_progress(process, token=token, data=data)
219
+ except AssertionError as exc:
220
+ raise_status(HTTPStatus.NOT_FOUND, str(exc))
221
+
222
+
200
223
  @router.put(
201
224
  "/resume-all", response_model=ProcessResumeAllSchema, dependencies=[Depends(check_global_lock, use_cache=False)]
202
225
  )
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
orchestrator/app.py CHANGED
@@ -5,7 +5,7 @@ This module contains the main `OrchestratorCore` class for the `FastAPI` backend
5
5
  provides the ability to run the CLI.
6
6
  """
7
7
 
8
- # Copyright 2019-2020 SURF, ESnet
8
+ # Copyright 2019-2020 SURF, ESnet, GÉANT.
9
9
  # Licensed under the Apache License, Version 2.0 (the "License");
10
10
  # you may not use this file except in compliance with the License.
11
11
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF, ESnet
1
+ # Copyright 2019-2025 SURF, ESnet, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -31,16 +31,13 @@ from orchestrator.cli.generator.generator.helpers import (
31
31
  sort_product_blocks_by_dependencies,
32
32
  )
33
33
  from orchestrator.cli.generator.generator.settings import product_generator_settings as settings
34
- from orchestrator.settings import convert_database_uri
35
34
 
36
35
  logger = structlog.getLogger(__name__)
37
36
 
38
37
 
39
38
  def create_migration_file(message: str, head: str) -> Path | None:
40
- if environ.get("DATABASE_URI"):
41
- environ.update({"DATABASE_URI": convert_database_uri(environ["DATABASE_URI"])})
42
- else:
43
- environ.update({"DATABASE_URI": "postgresql+psycopg://nwa:nwa@localhost/orchestrator-core"})
39
+ if not environ.get("DATABASE_URI"):
40
+ environ.update({"DATABASE_URI": "postgresql://nwa:nwa@localhost/orchestrator-core"})
44
41
  if not environ.get("PYTHONPATH"):
45
42
  environ.update({"PYTHONPATH": "."})
46
43
  logger.info(
@@ -1,3 +1,16 @@
1
+ # Copyright 2019-2025 SURF, GÉANT.
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ #
6
+ # http://www.apache.org/licenses/LICENSE-2.0
7
+ #
8
+ # Unless required by applicable law or agreed to in writing, software
9
+ # distributed under the License is distributed on an "AS IS" BASIS,
10
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11
+ # See the License for the specific language governing permissions and
12
+ # limitations under the License.
13
+
1
14
  import itertools
2
15
  import operator
3
16
  from collections.abc import Iterable
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2025 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
orchestrator/db/models.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -11,11 +11,10 @@
11
11
  # See the License for the specific language governing permissions and
12
12
  # limitations under the License.
13
13
 
14
-
15
- from orchestrator.domain.base import SubscriptionModel
14
+ from orchestrator.domain.base import SubscriptionModel, SubscriptionModelRegistry
16
15
  from orchestrator.utils.docs import make_product_type_index_doc
17
16
 
18
- SUBSCRIPTION_MODEL_REGISTRY: dict[str, type[SubscriptionModel]] = {}
17
+ SUBSCRIPTION_MODEL_REGISTRY: SubscriptionModelRegistry = SubscriptionModelRegistry()
19
18
 
20
19
  __doc__ = make_product_type_index_doc(SUBSCRIPTION_MODEL_REGISTRY)
21
20
 
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF, ESnet.
1
+ # Copyright 2019-2025 SURF, ESnet, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -12,14 +12,16 @@
12
12
  # limitations under the License.
13
13
  import itertools
14
14
  from collections import defaultdict
15
- from collections.abc import Callable, Iterable
16
15
  from datetime import datetime
17
- from inspect import get_annotations
16
+ from inspect import get_annotations, isclass
18
17
  from itertools import groupby, zip_longest
19
18
  from operator import attrgetter
20
19
  from typing import (
21
20
  Any,
21
+ Callable,
22
22
  ClassVar,
23
+ Iterable,
24
+ Mapping,
23
25
  Optional,
24
26
  TypeVar,
25
27
  Union,
@@ -596,7 +598,9 @@ class ProductBlockModel(DomainModel):
596
598
  product_blocks_in_model = cls._get_depends_on_product_block_types()
597
599
  product_blocks_types_in_model = get_depends_on_product_block_type_list(product_blocks_in_model)
598
600
 
599
- product_blocks_in_model = set(flatten(map(attrgetter("__names__"), product_blocks_types_in_model))) # type: ignore
601
+ product_blocks_in_model = set(
602
+ flatten(map(attrgetter("__names__"), product_blocks_types_in_model))
603
+ ) # type: ignore
600
604
 
601
605
  missing_product_blocks_in_db = product_blocks_in_model - product_blocks_in_db # type: ignore
602
606
  missing_product_blocks_in_model = product_blocks_in_db - product_blocks_in_model # type: ignore
@@ -1051,7 +1055,9 @@ class SubscriptionModel(DomainModel):
1051
1055
  product_blocks_in_model = cls._get_depends_on_product_block_types()
1052
1056
  product_blocks_types_in_model = get_depends_on_product_block_type_list(product_blocks_in_model)
1053
1057
 
1054
- product_blocks_in_model = set(flatten(map(attrgetter("__names__"), product_blocks_types_in_model))) # type: ignore
1058
+ product_blocks_in_model = set(
1059
+ flatten(map(attrgetter("__names__"), product_blocks_types_in_model))
1060
+ ) # type: ignore
1055
1061
 
1056
1062
  missing_product_blocks_in_db = product_blocks_in_model - product_blocks_in_db # type: ignore
1057
1063
  missing_product_blocks_in_model = product_blocks_in_db - product_blocks_in_model # type: ignore
@@ -1402,6 +1408,69 @@ class SubscriptionModel(DomainModel):
1402
1408
  return self._db_model
1403
1409
 
1404
1410
 
1411
+ def validate_base_model(
1412
+ name: str, cls: type[Any], base_model: type[BaseModel] = DomainModel, errors: list[str] | None = None
1413
+ ) -> None:
1414
+ """Validates that the given class is not Pydantic BaseModel or its direct subclass."""
1415
+ # Instantiate errors list if not provided and avoid mutating default
1416
+ if errors is None:
1417
+ errors = []
1418
+ # Return early when the node is not a class as there is nothing to be done
1419
+ if not isclass(cls):
1420
+ return
1421
+ # Validate each field in the ProductBlockModel's field dictionaries
1422
+ if issubclass(cls, ProductBlockModel) or issubclass(cls, SubscriptionModel):
1423
+ for name, clz in cls._product_block_fields_.items():
1424
+ validate_base_model(name, clz, ProductBlockModel, errors)
1425
+ for name, clz in cls._non_product_block_fields_.items():
1426
+ validate_base_model(name, clz, SubscriptionModel, errors)
1427
+ # Generate error if node is Pydantic BaseModel or direct subclass
1428
+ if issubclass(cls, BaseModel):
1429
+ err_msg: str = (
1430
+ f"If this field was intended to be a {base_model.__name__}, define {name}:{cls.__name__} with "
1431
+ f"{base_model.__name__} as its superclass instead. e.g., class {cls.__name__}({base_model.__name__}):"
1432
+ )
1433
+ if cls is BaseModel:
1434
+ errors.append(f"Field {name}: {cls.__name__} can not be {BaseModel.__name__}. " + err_msg)
1435
+ if len(cls.__mro__) > 1 and cls.__mro__[1] is BaseModel:
1436
+ errors.append(
1437
+ f"Field {name}: {cls.__name__} can not be a direct subclass of {BaseModel.__name__}. " + err_msg
1438
+ )
1439
+ # Format all errors as one per line and raise a TypeError when they exist
1440
+ if errors:
1441
+ raise TypeError("\n".join(errors))
1442
+
1443
+
1444
+ class SubscriptionModelRegistry(dict[str, type[SubscriptionModel]]):
1445
+ """A registry for all subscription models."""
1446
+
1447
+ def __setitem__(self, __key: str, __value: type[SubscriptionModel]) -> None:
1448
+ """Set value for key in while validating against Pydantic BaseModel."""
1449
+ validate_base_model(__key, __value)
1450
+ super().__setitem__(__key, __value)
1451
+
1452
+ def update(
1453
+ self,
1454
+ m: Any = None,
1455
+ /,
1456
+ **kwargs: type[SubscriptionModel],
1457
+ ) -> None:
1458
+ """Update dictionary with mapping and/or kwargs using `__setitem__`."""
1459
+ if m:
1460
+ if isinstance(m, Mapping):
1461
+ for key, value in m.items():
1462
+ self[key] = value
1463
+ elif isinstance(m, Iterable):
1464
+ for index, item in enumerate(m):
1465
+ try:
1466
+ key, value = item
1467
+ except ValueError:
1468
+ raise TypeError(f"dictionary update sequence element #{index} is not an iterable of length 2")
1469
+ self[key] = value
1470
+ for key, value in kwargs.items():
1471
+ self[key] = value
1472
+
1473
+
1405
1474
  def _validate_lifecycle_change_for_product_block(
1406
1475
  used_by: SubscriptionInstanceTable,
1407
1476
  product_block_model: ProductBlockModel,
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2022-2023 SURF.
1
+ # Copyright 2022-2023 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2022-2023 SURF.
1
+ # Copyright 2022-2023 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,3 +1,16 @@
1
+ # Copyright 2022-2023 SURF, GÉANT.
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ #
6
+ # http://www.apache.org/licenses/LICENSE-2.0
7
+ #
8
+ # Unless required by applicable law or agreed to in writing, software
9
+ # distributed under the License is distributed on an "AS IS" BASIS,
10
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11
+ # See the License for the specific language governing permissions and
12
+ # limitations under the License.
13
+
1
14
  from collections.abc import Generator
2
15
  from itertools import count
3
16
  from typing import TYPE_CHECKING, Annotated, Any
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -880,10 +880,10 @@ def delete_product(conn: sa.engine.Connection, name: str) -> None:
880
880
  RETURNING product_id
881
881
  ),
882
882
  deleted_p_pb AS (
883
- DELETE FROM product_product_blocks WHERE product_id = ANY(SELECT product_id FROM deleted_p)
883
+ DELETE FROM product_product_blocks WHERE product_id IN (SELECT product_id FROM deleted_p)
884
884
  ),
885
885
  deleted_pb_rt AS (
886
- DELETE FROM products_workflows WHERE product_id = ANY(SELECT product_id FROM deleted_p)
886
+ DELETE FROM products_workflows WHERE product_id IN (SELECT product_id FROM deleted_p)
887
887
  )
888
888
  SELECT * from deleted_p;
889
889
  """
@@ -911,10 +911,10 @@ def delete_product_block(conn: sa.engine.Connection, name: str) -> None:
911
911
  RETURNING product_block_id
912
912
  ),
913
913
  deleted_p_pb AS (
914
- DELETE FROM product_product_blocks WHERE product_block_id =ANY(SELECT product_block_id FROM deleted_pb)
914
+ DELETE FROM product_product_blocks WHERE product_block_id IN (SELECT product_block_id FROM deleted_pb)
915
915
  ),
916
916
  deleted_pb_rt AS (
917
- DELETE FROM product_block_resource_types WHERE product_block_id =ANY(SELECT product_block_id FROM deleted_pb)
917
+ DELETE FROM product_block_resource_types WHERE product_block_id IN (SELECT product_block_id FROM deleted_pb)
918
918
  )
919
919
  SELECT * from deleted_pb;
920
920
  """
@@ -968,7 +968,7 @@ def delete_resource_type(conn: sa.engine.Connection, resource_type: str) -> None
968
968
  RETURNING resource_type_id
969
969
  ),
970
970
  deleted_pb_rt AS (
971
- DELETE FROM product_block_resource_types WHERE resource_type_id =ANY(SELECT resource_type_id FROM deleted_pb)
971
+ DELETE FROM product_block_resource_types WHERE resource_type_id IN (SELECT resource_type_id FROM deleted_pb)
972
972
  )
973
973
  SELECT * from deleted_pb;
974
974
  """
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2025 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
orchestrator/security.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -43,9 +43,10 @@ from orchestrator.targets import Target
43
43
  from orchestrator.types import BroadcastFunc
44
44
  from orchestrator.utils.datetime import nowtz
45
45
  from orchestrator.utils.errors import error_state_to_dict
46
- from orchestrator.websocket import broadcast_invalidate_status_counts
46
+ from orchestrator.websocket import broadcast_invalidate_status_counts, broadcast_process_update_to_websocket
47
47
  from orchestrator.workflow import (
48
48
  CALLBACK_TOKEN_KEY,
49
+ DEFAULT_CALLBACK_PROGRESS_KEY,
49
50
  Failed,
50
51
  ProcessStat,
51
52
  ProcessStatus,
@@ -566,6 +567,39 @@ def resume_process(
566
567
  return resume_func(process, user_inputs=user_inputs, user=user, broadcast_func=broadcast_func)
567
568
 
568
569
 
570
+ def ensure_correct_callback_token(pstat: ProcessStat, *, token: str) -> None:
571
+ """Ensure that a callback token matches the expected value in state.
572
+
573
+ Args:
574
+ pstat: ProcessStat of process.
575
+ token: The token which was generated for the process.
576
+
577
+ Raises:
578
+ AssertionError: if the supplied token does not match the generated process token.
579
+
580
+ """
581
+ state = pstat.state.unwrap()
582
+
583
+ # Check if the token matches
584
+ token_from_state = state.get(CALLBACK_TOKEN_KEY)
585
+ if token != token_from_state:
586
+ raise AssertionError("Invalid token")
587
+
588
+
589
+ def replace_current_step_state(process: ProcessTable, *, new_state: State) -> None:
590
+ """Replace the state of the current step in a process.
591
+
592
+ Args:
593
+ process: Process from database
594
+ new_state: The new state
595
+
596
+ """
597
+ current_step = process.steps[-1]
598
+ current_step.state = new_state
599
+ db.session.add(current_step)
600
+ db.session.commit()
601
+
602
+
569
603
  def continue_awaiting_process(
570
604
  process: ProcessTable,
571
605
  *,
@@ -589,10 +623,7 @@ def continue_awaiting_process(
589
623
  pstat = load_process(process)
590
624
  state = pstat.state.unwrap()
591
625
 
592
- # Check if the token matches
593
- token_from_state = state.get(CALLBACK_TOKEN_KEY)
594
- if token != token_from_state:
595
- raise AssertionError("Invalid token")
626
+ ensure_correct_callback_token(pstat, token=token)
596
627
 
597
628
  # We need to pass the callback data to the worker executor. Currently, this is not supported.
598
629
  # Therefore, we update the step state in the db and kick-off resume_workflow
@@ -600,16 +631,47 @@ def continue_awaiting_process(
600
631
  result_key = state.get("__callback_result_key", "callback_result")
601
632
  state = {**state, result_key: input_data}
602
633
 
603
- current_step = process.steps[-1]
604
- current_step.state = state
605
- db.session.add(current_step)
606
- db.session.commit()
634
+ replace_current_step_state(process, new_state=state)
607
635
 
608
636
  # Continue the workflow
609
637
  resume_func = get_execution_context()["resume"]
610
638
  return resume_func(process, broadcast_func=broadcast_func)
611
639
 
612
640
 
641
+ def update_awaiting_process_progress(
642
+ process: ProcessTable,
643
+ *,
644
+ token: str,
645
+ data: str | State,
646
+ ) -> UUID:
647
+ """Update progress for a process awaiting data from a callback.
648
+
649
+ Args:
650
+ process: Process from database
651
+ token: The token which was generated for the process. This must match.
652
+ data: Progress data posted to the callback
653
+
654
+ Returns:
655
+ process id
656
+
657
+ Raises:
658
+ AssertionError: if the supplied token does not match the generated process token.
659
+
660
+ """
661
+ pstat = load_process(process)
662
+
663
+ ensure_correct_callback_token(pstat, token=token)
664
+
665
+ state = pstat.state.unwrap()
666
+ progress_key = state.get(DEFAULT_CALLBACK_PROGRESS_KEY, "callback_progress")
667
+ state = {**state, progress_key: data} | {"__remove_keys": [progress_key]}
668
+
669
+ replace_current_step_state(process, new_state=state)
670
+ broadcast_process_update_to_websocket(process.process_id)
671
+
672
+ return process.process_id
673
+
674
+
613
675
  async def _async_resume_processes(
614
676
  processes: Sequence[ProcessTable],
615
677
  user_name: str,
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
orchestrator/settings.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -13,7 +13,6 @@
13
13
 
14
14
  import secrets
15
15
  import string
16
- import warnings
17
16
  from pathlib import Path
18
17
  from typing import Literal
19
18
 
@@ -24,10 +23,6 @@ from oauth2_lib.settings import oauth2lib_settings
24
23
  from pydantic_forms.types import strEnum
25
24
 
26
25
 
27
- class OrchestratorDeprecationWarning(DeprecationWarning):
28
- pass
29
-
30
-
31
26
  class ExecutorType(strEnum):
32
27
  WORKER = "celery"
33
28
  THREADPOOL = "threadpool"
@@ -54,7 +49,7 @@ class AppSettings(BaseSettings):
54
49
  EXECUTOR: str = ExecutorType.THREADPOOL
55
50
  WORKFLOWS_SWAGGER_HOST: str = "localhost"
56
51
  WORKFLOWS_GUI_URI: str = "http://localhost:3000"
57
- DATABASE_URI: PostgresDsn = "postgresql+psycopg://nwa:nwa@localhost/orchestrator-core" # type: ignore
52
+ DATABASE_URI: PostgresDsn = "postgresql://nwa:nwa@localhost/orchestrator-core" # type: ignore
58
53
  MAX_WORKERS: int = 5
59
54
  MAIL_SERVER: str = "localhost"
60
55
  MAIL_PORT: int = 25
@@ -93,22 +88,6 @@ class AppSettings(BaseSettings):
93
88
  VALIDATE_OUT_OF_SYNC_SUBSCRIPTIONS: bool = False
94
89
  FILTER_BY_MODE: Literal["partial", "exact"] = "exact"
95
90
 
96
- def __init__(self) -> None:
97
- super(AppSettings, self).__init__()
98
- self.DATABASE_URI = PostgresDsn(convert_database_uri(str(self.DATABASE_URI)))
99
-
100
-
101
- def convert_database_uri(db_uri: str) -> str:
102
- if db_uri.startswith(("postgresql://", "postgresql+psycopg2://")):
103
- db_uri = "postgresql+psycopg" + db_uri[db_uri.find("://") :]
104
- warnings.filterwarnings("always", category=OrchestratorDeprecationWarning)
105
- warnings.warn(
106
- "DATABASE_URI converted to postgresql+psycopg:// format, please update your enviroment variable",
107
- OrchestratorDeprecationWarning,
108
- stacklevel=2,
109
- )
110
- return db_uri
111
-
112
91
 
113
92
  app_settings = AppSettings()
114
93
 
orchestrator/targets.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
orchestrator/types.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF, ESnet
1
+ # Copyright 2019-2020 SURF, ESnet, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2020 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
orchestrator/workflow.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2019-2020 SURF.
1
+ # Copyright 2019-2025 SURF, GÉANT.
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -69,6 +69,7 @@ step_log_fn_var: contextvars.ContextVar[StepLogFuncInternal] = contextvars.Conte
69
69
 
70
70
  DEFAULT_CALLBACK_ROUTE_KEY = "callback_route"
71
71
  CALLBACK_TOKEN_KEY = "__callback_token" # noqa: S105
72
+ DEFAULT_CALLBACK_PROGRESS_KEY = "callback_progress" # noqa: S105
72
73
 
73
74
 
74
75
  @runtime_checkable