orchestrator-core 3.1.0rc1__py3-none-any.whl → 3.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- orchestrator/__init__.py +2 -2
- orchestrator/api/api_v1/api.py +1 -1
- orchestrator/api/api_v1/endpoints/processes.py +29 -9
- orchestrator/api/api_v1/endpoints/settings.py +1 -1
- orchestrator/api/api_v1/endpoints/subscriptions.py +1 -1
- orchestrator/app.py +1 -1
- orchestrator/cli/database.py +1 -1
- orchestrator/cli/generator/generator/migration.py +2 -5
- orchestrator/cli/migrate_tasks.py +13 -0
- orchestrator/config/assignee.py +1 -1
- orchestrator/db/__init__.py +2 -0
- orchestrator/db/models.py +6 -4
- orchestrator/devtools/populator.py +1 -1
- orchestrator/domain/__init__.py +2 -3
- orchestrator/domain/base.py +74 -5
- orchestrator/domain/lifecycle.py +1 -1
- orchestrator/graphql/schema.py +1 -1
- orchestrator/graphql/types.py +1 -1
- orchestrator/graphql/utils/get_subscription_product_blocks.py +13 -0
- orchestrator/migrations/env.py +15 -2
- orchestrator/migrations/helpers.py +6 -6
- orchestrator/migrations/versions/schema/2020-10-19_c112305b07d3_initial_schema_migration.py +1 -1
- orchestrator/migrations/versions/schema/2023-05-25_b1970225392d_add_subscription_metadata_workflow.py +1 -1
- orchestrator/migrations/versions/schema/2025-02-12_bac6be6f2b4f_added_input_state_table.py +1 -1
- orchestrator/schemas/engine_settings.py +1 -1
- orchestrator/schemas/subscription.py +1 -1
- orchestrator/security.py +1 -1
- orchestrator/services/celery.py +1 -1
- orchestrator/services/processes.py +99 -18
- orchestrator/services/products.py +1 -1
- orchestrator/services/subscriptions.py +1 -1
- orchestrator/services/tasks.py +1 -1
- orchestrator/settings.py +2 -23
- orchestrator/targets.py +1 -1
- orchestrator/types.py +1 -1
- orchestrator/utils/errors.py +1 -1
- orchestrator/utils/state.py +74 -54
- orchestrator/websocket/websocket_manager.py +1 -1
- orchestrator/workflow.py +20 -4
- orchestrator/workflows/modify_note.py +1 -1
- orchestrator/workflows/steps.py +1 -1
- orchestrator/workflows/tasks/cleanup_tasks_log.py +1 -1
- orchestrator/workflows/tasks/resume_workflows.py +1 -1
- orchestrator/workflows/tasks/validate_product_type.py +1 -1
- orchestrator/workflows/tasks/validate_products.py +1 -1
- orchestrator/workflows/utils.py +40 -5
- {orchestrator_core-3.1.0rc1.dist-info → orchestrator_core-3.1.2.dist-info}/METADATA +13 -13
- {orchestrator_core-3.1.0rc1.dist-info → orchestrator_core-3.1.2.dist-info}/RECORD +50 -50
- {orchestrator_core-3.1.0rc1.dist-info → orchestrator_core-3.1.2.dist-info}/WHEEL +1 -1
- {orchestrator_core-3.1.0rc1.dist-info → orchestrator_core-3.1.2.dist-info}/licenses/LICENSE +0 -0
orchestrator/__init__.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-
|
|
1
|
+
# Copyright 2019-2025 SURF, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
"""This is the orchestrator workflow engine."""
|
|
15
15
|
|
|
16
|
-
__version__ = "3.1.
|
|
16
|
+
__version__ = "3.1.2"
|
|
17
17
|
|
|
18
18
|
from orchestrator.app import OrchestratorCore
|
|
19
19
|
from orchestrator.settings import app_settings
|
orchestrator/api/api_v1/api.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-
|
|
1
|
+
# Copyright 2019-2025 SURF, GÉANT, ESnet.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -40,13 +40,7 @@ from orchestrator.db.filters import Filter
|
|
|
40
40
|
from orchestrator.db.filters.process import filter_processes
|
|
41
41
|
from orchestrator.db.sorting import Sort, SortOrder
|
|
42
42
|
from orchestrator.db.sorting.process import sort_processes
|
|
43
|
-
from orchestrator.schemas import
|
|
44
|
-
ProcessIdSchema,
|
|
45
|
-
ProcessResumeAllSchema,
|
|
46
|
-
ProcessSchema,
|
|
47
|
-
ProcessStatusCounts,
|
|
48
|
-
Reporter,
|
|
49
|
-
)
|
|
43
|
+
from orchestrator.schemas import ProcessIdSchema, ProcessResumeAllSchema, ProcessSchema, ProcessStatusCounts, Reporter
|
|
50
44
|
from orchestrator.security import authenticate
|
|
51
45
|
from orchestrator.services.process_broadcast_thread import api_broadcast_process_data
|
|
52
46
|
from orchestrator.services.processes import (
|
|
@@ -58,6 +52,7 @@ from orchestrator.services.processes import (
|
|
|
58
52
|
load_process,
|
|
59
53
|
resume_process,
|
|
60
54
|
start_process,
|
|
55
|
+
update_awaiting_process_progress,
|
|
61
56
|
)
|
|
62
57
|
from orchestrator.services.settings import get_engine_settings
|
|
63
58
|
from orchestrator.settings import app_settings
|
|
@@ -138,9 +133,12 @@ async def new_process(
|
|
|
138
133
|
request: Request,
|
|
139
134
|
json_data: list[dict[str, Any]] | None = Body(...),
|
|
140
135
|
user: str = Depends(user_name),
|
|
136
|
+
user_model: OIDCUserModel | None = Depends(authenticate),
|
|
141
137
|
) -> dict[str, UUID]:
|
|
142
138
|
broadcast_func = api_broadcast_process_data(request)
|
|
143
|
-
process_id = start_process(
|
|
139
|
+
process_id = start_process(
|
|
140
|
+
workflow_key, user_inputs=json_data, user_model=user_model, user=user, broadcast_func=broadcast_func
|
|
141
|
+
)
|
|
144
142
|
|
|
145
143
|
return {"id": process_id}
|
|
146
144
|
|
|
@@ -197,6 +195,28 @@ def continue_awaiting_process_endpoint(
|
|
|
197
195
|
raise_status(HTTPStatus.NOT_FOUND, str(e))
|
|
198
196
|
|
|
199
197
|
|
|
198
|
+
@router.post(
|
|
199
|
+
"/{process_id}/callback/{token}/progress",
|
|
200
|
+
response_model=None,
|
|
201
|
+
status_code=HTTPStatus.OK,
|
|
202
|
+
dependencies=[Depends(check_global_lock, use_cache=False)],
|
|
203
|
+
)
|
|
204
|
+
def update_progress_on_awaiting_process_endpoint(
|
|
205
|
+
process_id: UUID,
|
|
206
|
+
token: str,
|
|
207
|
+
data: str | State = Body(...),
|
|
208
|
+
) -> None:
|
|
209
|
+
process = _get_process(process_id)
|
|
210
|
+
|
|
211
|
+
if process.last_status != ProcessStatus.AWAITING_CALLBACK:
|
|
212
|
+
raise_status(HTTPStatus.CONFLICT, "This process is not in an awaiting state.")
|
|
213
|
+
|
|
214
|
+
try:
|
|
215
|
+
update_awaiting_process_progress(process, token=token, data=data)
|
|
216
|
+
except AssertionError as exc:
|
|
217
|
+
raise_status(HTTPStatus.NOT_FOUND, str(exc))
|
|
218
|
+
|
|
219
|
+
|
|
200
220
|
@router.put(
|
|
201
221
|
"/resume-all", response_model=ProcessResumeAllSchema, dependencies=[Depends(check_global_lock, use_cache=False)]
|
|
202
222
|
)
|
orchestrator/app.py
CHANGED
|
@@ -5,7 +5,7 @@ This module contains the main `OrchestratorCore` class for the `FastAPI` backend
|
|
|
5
5
|
provides the ability to run the CLI.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
# Copyright 2019-2020 SURF, ESnet
|
|
8
|
+
# Copyright 2019-2020 SURF, ESnet, GÉANT.
|
|
9
9
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
10
10
|
# you may not use this file except in compliance with the License.
|
|
11
11
|
# You may obtain a copy of the License at
|
orchestrator/cli/database.py
CHANGED
|
@@ -31,16 +31,13 @@ from orchestrator.cli.generator.generator.helpers import (
|
|
|
31
31
|
sort_product_blocks_by_dependencies,
|
|
32
32
|
)
|
|
33
33
|
from orchestrator.cli.generator.generator.settings import product_generator_settings as settings
|
|
34
|
-
from orchestrator.settings import convert_database_uri
|
|
35
34
|
|
|
36
35
|
logger = structlog.getLogger(__name__)
|
|
37
36
|
|
|
38
37
|
|
|
39
38
|
def create_migration_file(message: str, head: str) -> Path | None:
|
|
40
|
-
if environ.get("DATABASE_URI"):
|
|
41
|
-
environ.update({"DATABASE_URI":
|
|
42
|
-
else:
|
|
43
|
-
environ.update({"DATABASE_URI": "postgresql+psycopg://nwa:nwa@localhost/orchestrator-core"})
|
|
39
|
+
if not environ.get("DATABASE_URI"):
|
|
40
|
+
environ.update({"DATABASE_URI": "postgresql://nwa:nwa@localhost/orchestrator-core"})
|
|
44
41
|
if not environ.get("PYTHONPATH"):
|
|
45
42
|
environ.update({"PYTHONPATH": "."})
|
|
46
43
|
logger.info(
|
|
@@ -1,3 +1,16 @@
|
|
|
1
|
+
# Copyright 2019-2025 SURF, GÉANT.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
#
|
|
6
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
7
|
+
#
|
|
8
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
9
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
10
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
11
|
+
# See the License for the specific language governing permissions and
|
|
12
|
+
# limitations under the License.
|
|
13
|
+
|
|
1
14
|
import itertools
|
|
2
15
|
import operator
|
|
3
16
|
from collections.abc import Iterable
|
orchestrator/config/assignee.py
CHANGED
orchestrator/db/__init__.py
CHANGED
|
@@ -19,6 +19,7 @@ from orchestrator.db.database import Database, transactional
|
|
|
19
19
|
from orchestrator.db.models import ( # noqa: F401
|
|
20
20
|
EngineSettingsTable,
|
|
21
21
|
FixedInputTable,
|
|
22
|
+
InputStateTable,
|
|
22
23
|
ProcessStepTable,
|
|
23
24
|
ProcessSubscriptionTable,
|
|
24
25
|
ProcessTable,
|
|
@@ -85,6 +86,7 @@ __all__ = [
|
|
|
85
86
|
"SubscriptionMetadataTable",
|
|
86
87
|
"ResourceTypeTable",
|
|
87
88
|
"FixedInputTable",
|
|
89
|
+
"InputStateTable",
|
|
88
90
|
"EngineSettingsTable",
|
|
89
91
|
"WorkflowTable",
|
|
90
92
|
"SubscriptionCustomerDescriptionTable",
|
orchestrator/db/models.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-2020 SURF.
|
|
1
|
+
# Copyright 2019-2020 SURF, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -92,7 +92,7 @@ class InputStateTable(BaseModel):
|
|
|
92
92
|
|
|
93
93
|
input_state_id = mapped_column(UUIDType, primary_key=True, server_default=text("uuid_generate_v4()"), index=True)
|
|
94
94
|
process_id = mapped_column("pid", UUIDType, ForeignKey("processes.pid"), nullable=False)
|
|
95
|
-
input_state = mapped_column(pg.JSONB(), nullable=False)
|
|
95
|
+
input_state = mapped_column(pg.JSONB(), nullable=False)
|
|
96
96
|
input_time = mapped_column(UtcTimestamp, server_default=text("current_timestamp()"), nullable=False)
|
|
97
97
|
input_type = mapped_column(Enum(InputType), nullable=False)
|
|
98
98
|
|
|
@@ -137,7 +137,7 @@ class ProcessStepTable(BaseModel):
|
|
|
137
137
|
)
|
|
138
138
|
name = mapped_column(String(), nullable=False)
|
|
139
139
|
status = mapped_column(String(50), nullable=False)
|
|
140
|
-
state = mapped_column(pg.JSONB(), nullable=False)
|
|
140
|
+
state = mapped_column(pg.JSONB(), nullable=False)
|
|
141
141
|
created_by = mapped_column(String(255), nullable=True)
|
|
142
142
|
executed_at = mapped_column(UtcTimestamp, server_default=text("statement_timestamp()"), nullable=False)
|
|
143
143
|
commit_hash = mapped_column(String(40), nullable=True, default=GIT_COMMIT_HASH)
|
|
@@ -642,7 +642,7 @@ class SubscriptionMetadataTable(BaseModel):
|
|
|
642
642
|
primary_key=True,
|
|
643
643
|
index=True,
|
|
644
644
|
)
|
|
645
|
-
metadata_ = mapped_column("metadata", pg.JSONB(), nullable=False)
|
|
645
|
+
metadata_ = mapped_column("metadata", pg.JSONB(), nullable=False)
|
|
646
646
|
|
|
647
647
|
@staticmethod
|
|
648
648
|
def find_by_subscription_id(subscription_id: str) -> SubscriptionMetadataTable | None:
|
|
@@ -651,6 +651,8 @@ class SubscriptionMetadataTable(BaseModel):
|
|
|
651
651
|
|
|
652
652
|
class SubscriptionSearchView(BaseModel):
|
|
653
653
|
__tablename__ = "subscriptions_search"
|
|
654
|
+
__table_args__ = {"info": {"materialized_view": True}}
|
|
655
|
+
|
|
654
656
|
subscription_id = mapped_column(
|
|
655
657
|
UUIDType, ForeignKey("subscriptions.subscription_id"), nullable=False, index=True, primary_key=True
|
|
656
658
|
)
|
orchestrator/domain/__init__.py
CHANGED
|
@@ -11,11 +11,10 @@
|
|
|
11
11
|
# See the License for the specific language governing permissions and
|
|
12
12
|
# limitations under the License.
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
from orchestrator.domain.base import SubscriptionModel
|
|
14
|
+
from orchestrator.domain.base import SubscriptionModel, SubscriptionModelRegistry
|
|
16
15
|
from orchestrator.utils.docs import make_product_type_index_doc
|
|
17
16
|
|
|
18
|
-
SUBSCRIPTION_MODEL_REGISTRY:
|
|
17
|
+
SUBSCRIPTION_MODEL_REGISTRY: SubscriptionModelRegistry = SubscriptionModelRegistry()
|
|
19
18
|
|
|
20
19
|
__doc__ = make_product_type_index_doc(SUBSCRIPTION_MODEL_REGISTRY)
|
|
21
20
|
|
orchestrator/domain/base.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-
|
|
1
|
+
# Copyright 2019-2025 SURF, ESnet, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -12,14 +12,16 @@
|
|
|
12
12
|
# limitations under the License.
|
|
13
13
|
import itertools
|
|
14
14
|
from collections import defaultdict
|
|
15
|
-
from collections.abc import Callable, Iterable
|
|
16
15
|
from datetime import datetime
|
|
17
|
-
from inspect import get_annotations
|
|
16
|
+
from inspect import get_annotations, isclass
|
|
18
17
|
from itertools import groupby, zip_longest
|
|
19
18
|
from operator import attrgetter
|
|
20
19
|
from typing import (
|
|
21
20
|
Any,
|
|
21
|
+
Callable,
|
|
22
22
|
ClassVar,
|
|
23
|
+
Iterable,
|
|
24
|
+
Mapping,
|
|
23
25
|
Optional,
|
|
24
26
|
TypeVar,
|
|
25
27
|
Union,
|
|
@@ -596,7 +598,9 @@ class ProductBlockModel(DomainModel):
|
|
|
596
598
|
product_blocks_in_model = cls._get_depends_on_product_block_types()
|
|
597
599
|
product_blocks_types_in_model = get_depends_on_product_block_type_list(product_blocks_in_model)
|
|
598
600
|
|
|
599
|
-
product_blocks_in_model = set(
|
|
601
|
+
product_blocks_in_model = set(
|
|
602
|
+
flatten(map(attrgetter("__names__"), product_blocks_types_in_model))
|
|
603
|
+
) # type: ignore
|
|
600
604
|
|
|
601
605
|
missing_product_blocks_in_db = product_blocks_in_model - product_blocks_in_db # type: ignore
|
|
602
606
|
missing_product_blocks_in_model = product_blocks_in_db - product_blocks_in_model # type: ignore
|
|
@@ -1051,7 +1055,9 @@ class SubscriptionModel(DomainModel):
|
|
|
1051
1055
|
product_blocks_in_model = cls._get_depends_on_product_block_types()
|
|
1052
1056
|
product_blocks_types_in_model = get_depends_on_product_block_type_list(product_blocks_in_model)
|
|
1053
1057
|
|
|
1054
|
-
product_blocks_in_model = set(
|
|
1058
|
+
product_blocks_in_model = set(
|
|
1059
|
+
flatten(map(attrgetter("__names__"), product_blocks_types_in_model))
|
|
1060
|
+
) # type: ignore
|
|
1055
1061
|
|
|
1056
1062
|
missing_product_blocks_in_db = product_blocks_in_model - product_blocks_in_db # type: ignore
|
|
1057
1063
|
missing_product_blocks_in_model = product_blocks_in_db - product_blocks_in_model # type: ignore
|
|
@@ -1402,6 +1408,69 @@ class SubscriptionModel(DomainModel):
|
|
|
1402
1408
|
return self._db_model
|
|
1403
1409
|
|
|
1404
1410
|
|
|
1411
|
+
def validate_base_model(
|
|
1412
|
+
name: str, cls: type[Any], base_model: type[BaseModel] = DomainModel, errors: list[str] | None = None
|
|
1413
|
+
) -> None:
|
|
1414
|
+
"""Validates that the given class is not Pydantic BaseModel or its direct subclass."""
|
|
1415
|
+
# Instantiate errors list if not provided and avoid mutating default
|
|
1416
|
+
if errors is None:
|
|
1417
|
+
errors = []
|
|
1418
|
+
# Return early when the node is not a class as there is nothing to be done
|
|
1419
|
+
if not isclass(cls):
|
|
1420
|
+
return
|
|
1421
|
+
# Validate each field in the ProductBlockModel's field dictionaries
|
|
1422
|
+
if issubclass(cls, ProductBlockModel) or issubclass(cls, SubscriptionModel):
|
|
1423
|
+
for name, clz in cls._product_block_fields_.items():
|
|
1424
|
+
validate_base_model(name, clz, ProductBlockModel, errors)
|
|
1425
|
+
for name, clz in cls._non_product_block_fields_.items():
|
|
1426
|
+
validate_base_model(name, clz, SubscriptionModel, errors)
|
|
1427
|
+
# Generate error if node is Pydantic BaseModel or direct subclass
|
|
1428
|
+
if issubclass(cls, BaseModel):
|
|
1429
|
+
err_msg: str = (
|
|
1430
|
+
f"If this field was intended to be a {base_model.__name__}, define {name}:{cls.__name__} with "
|
|
1431
|
+
f"{base_model.__name__} as its superclass instead. e.g., class {cls.__name__}({base_model.__name__}):"
|
|
1432
|
+
)
|
|
1433
|
+
if cls is BaseModel:
|
|
1434
|
+
errors.append(f"Field {name}: {cls.__name__} can not be {BaseModel.__name__}. " + err_msg)
|
|
1435
|
+
if len(cls.__mro__) > 1 and cls.__mro__[1] is BaseModel:
|
|
1436
|
+
errors.append(
|
|
1437
|
+
f"Field {name}: {cls.__name__} can not be a direct subclass of {BaseModel.__name__}. " + err_msg
|
|
1438
|
+
)
|
|
1439
|
+
# Format all errors as one per line and raise a TypeError when they exist
|
|
1440
|
+
if errors:
|
|
1441
|
+
raise TypeError("\n".join(errors))
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
class SubscriptionModelRegistry(dict[str, type[SubscriptionModel]]):
|
|
1445
|
+
"""A registry for all subscription models."""
|
|
1446
|
+
|
|
1447
|
+
def __setitem__(self, __key: str, __value: type[SubscriptionModel]) -> None:
|
|
1448
|
+
"""Set value for key in while validating against Pydantic BaseModel."""
|
|
1449
|
+
validate_base_model(__key, __value)
|
|
1450
|
+
super().__setitem__(__key, __value)
|
|
1451
|
+
|
|
1452
|
+
def update(
|
|
1453
|
+
self,
|
|
1454
|
+
m: Any = None,
|
|
1455
|
+
/,
|
|
1456
|
+
**kwargs: type[SubscriptionModel],
|
|
1457
|
+
) -> None:
|
|
1458
|
+
"""Update dictionary with mapping and/or kwargs using `__setitem__`."""
|
|
1459
|
+
if m:
|
|
1460
|
+
if isinstance(m, Mapping):
|
|
1461
|
+
for key, value in m.items():
|
|
1462
|
+
self[key] = value
|
|
1463
|
+
elif isinstance(m, Iterable):
|
|
1464
|
+
for index, item in enumerate(m):
|
|
1465
|
+
try:
|
|
1466
|
+
key, value = item
|
|
1467
|
+
except ValueError:
|
|
1468
|
+
raise TypeError(f"dictionary update sequence element #{index} is not an iterable of length 2")
|
|
1469
|
+
self[key] = value
|
|
1470
|
+
for key, value in kwargs.items():
|
|
1471
|
+
self[key] = value
|
|
1472
|
+
|
|
1473
|
+
|
|
1405
1474
|
def _validate_lifecycle_change_for_product_block(
|
|
1406
1475
|
used_by: SubscriptionInstanceTable,
|
|
1407
1476
|
product_block_model: ProductBlockModel,
|
orchestrator/domain/lifecycle.py
CHANGED
orchestrator/graphql/schema.py
CHANGED
orchestrator/graphql/types.py
CHANGED
|
@@ -1,3 +1,16 @@
|
|
|
1
|
+
# Copyright 2022-2023 SURF, GÉANT.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
#
|
|
6
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
7
|
+
#
|
|
8
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
9
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
10
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
11
|
+
# See the License for the specific language governing permissions and
|
|
12
|
+
# limitations under the License.
|
|
13
|
+
|
|
1
14
|
from collections.abc import Generator
|
|
2
15
|
from itertools import count
|
|
3
16
|
from typing import TYPE_CHECKING, Annotated, Any
|
orchestrator/migrations/env.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-2020 SURF.
|
|
1
|
+
# Copyright 2019-2020 SURF, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -41,6 +41,14 @@ target_metadata = BaseModel.metadata
|
|
|
41
41
|
# ... etc.
|
|
42
42
|
|
|
43
43
|
|
|
44
|
+
def include_object(object, name, type_, reflected, compare_to): # type: ignore
|
|
45
|
+
"""Determines if an object should be included."""
|
|
46
|
+
|
|
47
|
+
if type_ == "table" and object.info.get("materialized_view", False):
|
|
48
|
+
return False
|
|
49
|
+
return True
|
|
50
|
+
|
|
51
|
+
|
|
44
52
|
def run_migrations_offline() -> None:
|
|
45
53
|
"""Run migrations in 'offline' mode.
|
|
46
54
|
|
|
@@ -55,7 +63,11 @@ def run_migrations_offline() -> None:
|
|
|
55
63
|
"""
|
|
56
64
|
url = config.get_main_option("sqlalchemy.url")
|
|
57
65
|
context.configure(
|
|
58
|
-
url=url,
|
|
66
|
+
url=url,
|
|
67
|
+
target_metadata=target_metadata,
|
|
68
|
+
literal_binds=True,
|
|
69
|
+
dialect_opts={"paramstyle": "named"},
|
|
70
|
+
include_object=include_object,
|
|
59
71
|
)
|
|
60
72
|
|
|
61
73
|
with context.begin_transaction():
|
|
@@ -90,6 +102,7 @@ def run_migrations_online() -> None:
|
|
|
90
102
|
target_metadata=target_metadata,
|
|
91
103
|
process_revision_directives=process_revision_directives,
|
|
92
104
|
compare_type=True,
|
|
105
|
+
include_object=include_object,
|
|
93
106
|
)
|
|
94
107
|
|
|
95
108
|
try:
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-2020 SURF.
|
|
1
|
+
# Copyright 2019-2020 SURF, GÉANT.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -880,10 +880,10 @@ def delete_product(conn: sa.engine.Connection, name: str) -> None:
|
|
|
880
880
|
RETURNING product_id
|
|
881
881
|
),
|
|
882
882
|
deleted_p_pb AS (
|
|
883
|
-
DELETE FROM product_product_blocks WHERE product_id
|
|
883
|
+
DELETE FROM product_product_blocks WHERE product_id IN (SELECT product_id FROM deleted_p)
|
|
884
884
|
),
|
|
885
885
|
deleted_pb_rt AS (
|
|
886
|
-
DELETE FROM products_workflows WHERE product_id
|
|
886
|
+
DELETE FROM products_workflows WHERE product_id IN (SELECT product_id FROM deleted_p)
|
|
887
887
|
)
|
|
888
888
|
SELECT * from deleted_p;
|
|
889
889
|
"""
|
|
@@ -911,10 +911,10 @@ def delete_product_block(conn: sa.engine.Connection, name: str) -> None:
|
|
|
911
911
|
RETURNING product_block_id
|
|
912
912
|
),
|
|
913
913
|
deleted_p_pb AS (
|
|
914
|
-
DELETE FROM product_product_blocks WHERE product_block_id
|
|
914
|
+
DELETE FROM product_product_blocks WHERE product_block_id IN (SELECT product_block_id FROM deleted_pb)
|
|
915
915
|
),
|
|
916
916
|
deleted_pb_rt AS (
|
|
917
|
-
DELETE FROM product_block_resource_types WHERE product_block_id
|
|
917
|
+
DELETE FROM product_block_resource_types WHERE product_block_id IN (SELECT product_block_id FROM deleted_pb)
|
|
918
918
|
)
|
|
919
919
|
SELECT * from deleted_pb;
|
|
920
920
|
"""
|
|
@@ -968,7 +968,7 @@ def delete_resource_type(conn: sa.engine.Connection, resource_type: str) -> None
|
|
|
968
968
|
RETURNING resource_type_id
|
|
969
969
|
),
|
|
970
970
|
deleted_pb_rt AS (
|
|
971
|
-
DELETE FROM product_block_resource_types WHERE resource_type_id
|
|
971
|
+
DELETE FROM product_block_resource_types WHERE resource_type_id IN (SELECT resource_type_id FROM deleted_pb)
|
|
972
972
|
)
|
|
973
973
|
SELECT * from deleted_pb;
|
|
974
974
|
"""
|
|
@@ -157,7 +157,7 @@ def upgrade() -> None:
|
|
|
157
157
|
sa.Column("pid", sqlalchemy_utils.types.uuid.UUIDType(), nullable=False),
|
|
158
158
|
sa.Column("name", sa.String(), nullable=False),
|
|
159
159
|
sa.Column("status", sa.String(length=50), nullable=False),
|
|
160
|
-
sa.Column("state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
|
160
|
+
sa.Column("state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
|
161
161
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
|
162
162
|
sa.Column(
|
|
163
163
|
"executed_at",
|
|
@@ -29,7 +29,7 @@ def upgrade() -> None:
|
|
|
29
29
|
nullable=False,
|
|
30
30
|
index=True,
|
|
31
31
|
),
|
|
32
|
-
sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
|
32
|
+
sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
|
33
33
|
sa.ForeignKeyConstraint(["subscription_id"], ["subscriptions.subscription_id"], ondelete="CASCADE"),
|
|
34
34
|
)
|
|
35
35
|
|
|
@@ -31,7 +31,7 @@ def upgrade() -> None:
|
|
|
31
31
|
nullable=False,
|
|
32
32
|
),
|
|
33
33
|
sa.Column("pid", sqlalchemy_utils.types.uuid.UUIDType(), nullable=False),
|
|
34
|
-
sa.Column("input_state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
|
34
|
+
sa.Column("input_state", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
|
35
35
|
sa.Column(
|
|
36
36
|
"input_time",
|
|
37
37
|
db.models.UtcTimestamp(timezone=True),
|
orchestrator/security.py
CHANGED
orchestrator/services/celery.py
CHANGED