cognite-neat 1.0.10__py3-none-any.whl → 1.0.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/__init__.py +2 -2
- cognite/neat/_config.py +14 -2
- cognite/neat/_data_model/_analysis.py +299 -150
- cognite/neat/_data_model/_snapshot.py +133 -0
- cognite/neat/_data_model/deployer/data_classes.py +1 -15
- cognite/neat/_data_model/deployer/deployer.py +2 -27
- cognite/neat/_data_model/models/dms/_references.py +6 -0
- cognite/neat/_data_model/models/dms/_views.py +6 -0
- cognite/neat/_data_model/validation/dms/_ai_readiness.py +44 -36
- cognite/neat/_data_model/validation/dms/_base.py +4 -329
- cognite/neat/_data_model/validation/dms/_connections.py +131 -97
- cognite/neat/_data_model/validation/dms/_consistency.py +12 -11
- cognite/neat/_data_model/validation/dms/_containers.py +57 -38
- cognite/neat/_data_model/validation/dms/_limits.py +43 -118
- cognite/neat/_data_model/validation/dms/_orchestrator.py +30 -186
- cognite/neat/_data_model/validation/dms/_views.py +29 -4
- cognite/neat/_session/_physical.py +42 -18
- cognite/neat/_session/_session.py +1 -1
- cognite/neat/_store/_store.py +21 -1
- cognite/neat/_version.py +1 -1
- {cognite_neat-1.0.10.dist-info → cognite_neat-1.0.12.dist-info}/METADATA +1 -1
- {cognite_neat-1.0.10.dist-info → cognite_neat-1.0.12.dist-info}/RECORD +23 -22
- {cognite_neat-1.0.10.dist-info → cognite_neat-1.0.12.dist-info}/WHEEL +1 -1
|
@@ -5,17 +5,13 @@ from typing import Literal
|
|
|
5
5
|
from cognite.neat._data_model.models.dms._container import ContainerRequest
|
|
6
6
|
from cognite.neat._data_model.models.dms._data_types import EnumProperty, ListablePropertyTypeDefinition
|
|
7
7
|
from cognite.neat._data_model.models.dms._indexes import BtreeIndex, InvertedIndex
|
|
8
|
-
from cognite.neat._data_model.models.dms._limits import SchemaLimits
|
|
9
8
|
from cognite.neat._data_model.models.dms._view_property import (
|
|
10
9
|
ViewCorePropertyRequest,
|
|
11
10
|
)
|
|
12
11
|
from cognite.neat._data_model.validation.dms._base import (
|
|
13
|
-
CDFResources,
|
|
14
12
|
DataModelValidator,
|
|
15
|
-
LocalResources,
|
|
16
13
|
)
|
|
17
14
|
from cognite.neat._issues import ConsistencyError
|
|
18
|
-
from cognite.neat._utils.useful_types import ModusOperandi
|
|
19
15
|
|
|
20
16
|
BASE_CODE = "NEAT-DMS-LIMITS"
|
|
21
17
|
|
|
@@ -40,29 +36,31 @@ class DataModelViewCountIsOutOfLimits(DataModelValidator):
|
|
|
40
36
|
code = f"{BASE_CODE}-DATA-MODEL-001"
|
|
41
37
|
issue_type = ConsistencyError
|
|
42
38
|
|
|
43
|
-
def __init__(
|
|
44
|
-
self,
|
|
45
|
-
local_resources: LocalResources,
|
|
46
|
-
cdf_resources: CDFResources,
|
|
47
|
-
limits: SchemaLimits,
|
|
48
|
-
modus_operandi: ModusOperandi = "additive",
|
|
49
|
-
) -> None:
|
|
50
|
-
super().__init__(local_resources, cdf_resources, modus_operandi)
|
|
51
|
-
self.limits = limits
|
|
52
|
-
|
|
53
39
|
def run(self) -> list[ConsistencyError]:
|
|
54
|
-
|
|
55
|
-
|
|
40
|
+
errors: list[ConsistencyError] = []
|
|
41
|
+
|
|
42
|
+
if self.validation_resources.merged_data_model.views is None:
|
|
43
|
+
errors.append(
|
|
44
|
+
ConsistencyError(
|
|
45
|
+
message="The data model does not have any views. This means it is not a data model.",
|
|
46
|
+
code=self.code,
|
|
47
|
+
)
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
elif (
|
|
51
|
+
len(self.validation_resources.merged_data_model.views) > self.validation_resources.limits.data_models.views
|
|
52
|
+
):
|
|
53
|
+
errors.append(
|
|
56
54
|
ConsistencyError(
|
|
57
55
|
message=(
|
|
58
|
-
f"The data model references {len(self.
|
|
56
|
+
f"The data model references {len(self.validation_resources.merged_data_model.views)} views, "
|
|
59
57
|
"which exceeds the limit of "
|
|
60
|
-
f"{self.limits.data_models.views} views per data model."
|
|
58
|
+
f"{self.validation_resources.limits.data_models.views} views per data model."
|
|
61
59
|
),
|
|
62
60
|
code=self.code,
|
|
63
61
|
)
|
|
64
|
-
|
|
65
|
-
return
|
|
62
|
+
)
|
|
63
|
+
return errors
|
|
66
64
|
|
|
67
65
|
|
|
68
66
|
### View level limits
|
|
@@ -85,38 +83,23 @@ class ViewPropertyCountIsOutOfLimits(DataModelValidator):
|
|
|
85
83
|
code = f"{BASE_CODE}-VIEW-001"
|
|
86
84
|
issue_type = ConsistencyError
|
|
87
85
|
|
|
88
|
-
def __init__(
|
|
89
|
-
self,
|
|
90
|
-
local_resources: LocalResources,
|
|
91
|
-
cdf_resources: CDFResources,
|
|
92
|
-
limits: SchemaLimits,
|
|
93
|
-
modus_operandi: ModusOperandi = "additive",
|
|
94
|
-
) -> None:
|
|
95
|
-
super().__init__(local_resources, cdf_resources, modus_operandi)
|
|
96
|
-
self.limits = limits
|
|
97
|
-
|
|
98
86
|
def run(self) -> list[ConsistencyError]:
|
|
99
87
|
errors: list[ConsistencyError] = []
|
|
100
|
-
merged_views = self.merged_views
|
|
101
|
-
|
|
102
|
-
for view_ref in self.local_resources.views_by_reference.keys():
|
|
103
|
-
view = merged_views.get(view_ref)
|
|
104
|
-
if not view:
|
|
105
|
-
raise RuntimeError(f"View {view_ref!s} not found in merged views. This is a bug!")
|
|
106
88
|
|
|
107
|
-
|
|
89
|
+
for view_ref, properties in self.validation_resources.properties_by_view.items():
|
|
90
|
+
if properties and len(properties) > self.validation_resources.limits.views.properties:
|
|
108
91
|
errors.append(
|
|
109
92
|
ConsistencyError(
|
|
110
93
|
message=(
|
|
111
|
-
f"View {
|
|
94
|
+
f"View {view_ref!s} has {len(properties)} properties,"
|
|
112
95
|
" which exceeds the limit of "
|
|
113
|
-
f"{self.limits.views.properties} properties per view."
|
|
96
|
+
f"{self.validation_resources.limits.views.properties} properties per view."
|
|
114
97
|
),
|
|
115
98
|
code=self.code,
|
|
116
99
|
)
|
|
117
100
|
)
|
|
118
101
|
|
|
119
|
-
elif not
|
|
102
|
+
elif not properties:
|
|
120
103
|
errors.append(
|
|
121
104
|
ConsistencyError(
|
|
122
105
|
message=(
|
|
@@ -149,41 +132,26 @@ class ViewContainerCountIsOutOfLimits(DataModelValidator):
|
|
|
149
132
|
code = f"{BASE_CODE}-VIEW-002"
|
|
150
133
|
issue_type = ConsistencyError
|
|
151
134
|
|
|
152
|
-
def __init__(
|
|
153
|
-
self,
|
|
154
|
-
local_resources: LocalResources,
|
|
155
|
-
cdf_resources: CDFResources,
|
|
156
|
-
limits: SchemaLimits,
|
|
157
|
-
modus_operandi: ModusOperandi = "additive",
|
|
158
|
-
) -> None:
|
|
159
|
-
super().__init__(local_resources, cdf_resources, modus_operandi)
|
|
160
|
-
self.limits = limits
|
|
161
|
-
|
|
162
135
|
def run(self) -> list[ConsistencyError]:
|
|
163
136
|
errors: list[ConsistencyError] = []
|
|
164
|
-
merged_views = self.merged_views
|
|
165
137
|
|
|
166
138
|
# Single loop over all views
|
|
167
|
-
for view_ref in self.
|
|
168
|
-
|
|
169
|
-
if not view:
|
|
170
|
-
raise RuntimeError(f"View {view_ref!s} not found in merged views. This is a bug!")
|
|
171
|
-
|
|
172
|
-
if view.properties:
|
|
139
|
+
for view_ref, properties in self.validation_resources.properties_by_view.items():
|
|
140
|
+
if properties:
|
|
173
141
|
count = len(
|
|
174
142
|
{
|
|
175
143
|
prop.container
|
|
176
|
-
for prop in
|
|
144
|
+
for prop in properties.values()
|
|
177
145
|
if (isinstance(prop, ViewCorePropertyRequest) and prop.container)
|
|
178
146
|
}
|
|
179
147
|
)
|
|
180
|
-
if count > self.limits.views.containers:
|
|
148
|
+
if count > self.validation_resources.limits.views.containers:
|
|
181
149
|
errors.append(
|
|
182
150
|
ConsistencyError(
|
|
183
151
|
message=(
|
|
184
152
|
f"View {view_ref!s} references "
|
|
185
153
|
f"{count} containers, which exceeds the limit of "
|
|
186
|
-
f"{self.limits.views.containers} containers per view."
|
|
154
|
+
f"{self.validation_resources.limits.views.containers} containers per view."
|
|
187
155
|
),
|
|
188
156
|
code=self.code,
|
|
189
157
|
)
|
|
@@ -209,33 +177,18 @@ class ViewImplementsCountIsOutOfLimits(DataModelValidator):
|
|
|
209
177
|
code = f"{BASE_CODE}-VIEW-003"
|
|
210
178
|
issue_type = ConsistencyError
|
|
211
179
|
|
|
212
|
-
def __init__(
|
|
213
|
-
self,
|
|
214
|
-
local_resources: LocalResources,
|
|
215
|
-
cdf_resources: CDFResources,
|
|
216
|
-
limits: SchemaLimits,
|
|
217
|
-
modus_operandi: ModusOperandi = "additive",
|
|
218
|
-
) -> None:
|
|
219
|
-
super().__init__(local_resources, cdf_resources, modus_operandi)
|
|
220
|
-
self.limits = limits
|
|
221
|
-
|
|
222
180
|
def run(self) -> list[ConsistencyError]:
|
|
223
181
|
errors: list[ConsistencyError] = []
|
|
224
|
-
merged_views = self.merged_views
|
|
225
182
|
|
|
226
183
|
# Single loop over all views
|
|
227
|
-
for view_ref in self.
|
|
228
|
-
|
|
229
|
-
if not view:
|
|
230
|
-
raise RuntimeError(f"View {view_ref!s} not found in merged views. This is a bug!")
|
|
231
|
-
|
|
232
|
-
if view.implements and len(view.implements) > self.limits.views.implements:
|
|
184
|
+
for view_ref, ancestors in self.validation_resources.ancestors_by_view.items():
|
|
185
|
+
if ancestors and len(ancestors) > self.validation_resources.limits.views.implements:
|
|
233
186
|
errors.append(
|
|
234
187
|
ConsistencyError(
|
|
235
188
|
message=(
|
|
236
|
-
f"View {view_ref!s} implements {len(
|
|
189
|
+
f"View {view_ref!s} implements {len(ancestors)} views,"
|
|
237
190
|
" which exceeds the limit of"
|
|
238
|
-
f" {self.limits.views.implements} implemented views per view."
|
|
191
|
+
f" {self.validation_resources.limits.views.implements} implemented views per view."
|
|
239
192
|
),
|
|
240
193
|
code=self.code,
|
|
241
194
|
)
|
|
@@ -264,33 +217,20 @@ class ContainerPropertyCountIsOutOfLimits(DataModelValidator):
|
|
|
264
217
|
code = f"{BASE_CODE}-CONTAINER-001"
|
|
265
218
|
issue_type = ConsistencyError
|
|
266
219
|
|
|
267
|
-
def __init__(
|
|
268
|
-
self,
|
|
269
|
-
local_resources: LocalResources,
|
|
270
|
-
cdf_resources: CDFResources,
|
|
271
|
-
limits: SchemaLimits,
|
|
272
|
-
modus_operandi: ModusOperandi = "additive",
|
|
273
|
-
) -> None:
|
|
274
|
-
super().__init__(local_resources, cdf_resources, modus_operandi)
|
|
275
|
-
self.limits = limits
|
|
276
|
-
|
|
277
220
|
def run(self) -> list[ConsistencyError]:
|
|
278
221
|
errors: list[ConsistencyError] = []
|
|
279
|
-
merged_containers = self.merged_containers
|
|
280
|
-
|
|
281
222
|
# Single loop over all containers
|
|
282
|
-
for container_ref in self.
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
if container.properties and len(container.properties) > self.limits.containers.properties():
|
|
223
|
+
for container_ref, container in self.validation_resources.merged.containers.items():
|
|
224
|
+
if (
|
|
225
|
+
container.properties
|
|
226
|
+
and len(container.properties) > self.validation_resources.limits.containers.properties()
|
|
227
|
+
):
|
|
288
228
|
errors.append(
|
|
289
229
|
ConsistencyError(
|
|
290
230
|
message=(
|
|
291
|
-
f"Container {
|
|
231
|
+
f"Container {container_ref!s} has {len(container.properties)} properties, "
|
|
292
232
|
"which exceeds the limit of "
|
|
293
|
-
f"{self.limits.containers.properties()} properties per container."
|
|
233
|
+
f"{self.validation_resources.limits.containers.properties()} properties per container."
|
|
294
234
|
),
|
|
295
235
|
fix="Define at least one property for container",
|
|
296
236
|
code=self.code,
|
|
@@ -299,7 +239,7 @@ class ContainerPropertyCountIsOutOfLimits(DataModelValidator):
|
|
|
299
239
|
elif not container.properties:
|
|
300
240
|
errors.append(
|
|
301
241
|
ConsistencyError(
|
|
302
|
-
message=(f"Container {
|
|
242
|
+
message=(f"Container {container_ref!s} does not have any properties defined."),
|
|
303
243
|
fix="Define at least one property for container",
|
|
304
244
|
code=self.code,
|
|
305
245
|
)
|
|
@@ -333,26 +273,11 @@ class ContainerPropertyListSizeIsOutOfLimits(DataModelValidator):
|
|
|
333
273
|
code = f"{BASE_CODE}-CONTAINER-002"
|
|
334
274
|
issue_type = ConsistencyError
|
|
335
275
|
|
|
336
|
-
def __init__(
|
|
337
|
-
self,
|
|
338
|
-
local_resources: LocalResources,
|
|
339
|
-
cdf_resources: CDFResources,
|
|
340
|
-
limits: SchemaLimits,
|
|
341
|
-
modus_operandi: ModusOperandi = "additive",
|
|
342
|
-
) -> None:
|
|
343
|
-
super().__init__(local_resources, cdf_resources, modus_operandi)
|
|
344
|
-
self.limits = limits
|
|
345
|
-
|
|
346
276
|
def run(self) -> list[ConsistencyError]:
|
|
347
277
|
errors: list[ConsistencyError] = []
|
|
348
|
-
merged_containers = self.merged_containers
|
|
349
278
|
|
|
350
279
|
# Single loop over all containers
|
|
351
|
-
for container_ref in self.
|
|
352
|
-
container = merged_containers.get(container_ref)
|
|
353
|
-
if not container:
|
|
354
|
-
raise RuntimeError(f"Container {container_ref!s} not found in merged containers. This is a bug!")
|
|
355
|
-
|
|
280
|
+
for container_ref, container in self.validation_resources.merged.containers.items():
|
|
356
281
|
properties_by_index_type = self.container_property_by_index_type(container)
|
|
357
282
|
|
|
358
283
|
for property_id, property_ in container.properties.items():
|
|
@@ -367,13 +292,13 @@ class ContainerPropertyListSizeIsOutOfLimits(DataModelValidator):
|
|
|
367
292
|
continue
|
|
368
293
|
|
|
369
294
|
has_btree_index = property_id in properties_by_index_type[BtreeIndex.model_fields["index_type"].default]
|
|
370
|
-
limit = self.limits.containers.properties.listable(type_, has_btree_index)
|
|
295
|
+
limit = self.validation_resources.limits.containers.properties.listable(type_, has_btree_index)
|
|
371
296
|
|
|
372
297
|
if type_.max_list_size > limit:
|
|
373
298
|
errors.append(
|
|
374
299
|
ConsistencyError(
|
|
375
300
|
message=(
|
|
376
|
-
f"Container {
|
|
301
|
+
f"Container {container_ref!s} has property {property_id} with list size "
|
|
377
302
|
f"{type_.max_list_size}, which exceeds the limit of {limit} "
|
|
378
303
|
f"for data type {type_.__class__.__name__}."
|
|
379
304
|
),
|
|
@@ -1,56 +1,15 @@
|
|
|
1
1
|
from collections.abc import Callable
|
|
2
|
-
from
|
|
2
|
+
from datetime import datetime, timezone
|
|
3
3
|
|
|
4
|
-
from cognite.neat.
|
|
5
|
-
from cognite.neat._data_model._analysis import DataModelAnalysis
|
|
4
|
+
from cognite.neat._data_model._analysis import ValidationResources
|
|
6
5
|
from cognite.neat._data_model._shared import OnSuccessIssuesChecker
|
|
7
|
-
from cognite.neat._data_model.
|
|
6
|
+
from cognite.neat._data_model._snapshot import SchemaSnapshot
|
|
8
7
|
from cognite.neat._data_model.models.dms._limits import SchemaLimits
|
|
9
|
-
from cognite.neat._data_model.models.dms._references import ContainerReference, DataModelReference, ViewReference
|
|
10
8
|
from cognite.neat._data_model.models.dms._schema import RequestSchema
|
|
11
|
-
from cognite.neat.
|
|
12
|
-
from cognite.neat._data_model.models.dms._views import ViewRequest
|
|
13
|
-
from cognite.neat._data_model.validation.dms._limits import (
|
|
14
|
-
ContainerPropertyCountIsOutOfLimits,
|
|
15
|
-
ContainerPropertyListSizeIsOutOfLimits,
|
|
16
|
-
DataModelViewCountIsOutOfLimits,
|
|
17
|
-
ViewContainerCountIsOutOfLimits,
|
|
18
|
-
ViewImplementsCountIsOutOfLimits,
|
|
19
|
-
ViewPropertyCountIsOutOfLimits,
|
|
20
|
-
)
|
|
9
|
+
from cognite.neat._utils.auxiliary import get_concrete_subclasses
|
|
21
10
|
from cognite.neat._utils.useful_types import ModusOperandi
|
|
22
11
|
|
|
23
|
-
from .
|
|
24
|
-
DataModelMissingDescription,
|
|
25
|
-
DataModelMissingName,
|
|
26
|
-
EnumerationMissingDescription,
|
|
27
|
-
EnumerationMissingName,
|
|
28
|
-
ViewMissingDescription,
|
|
29
|
-
ViewMissingName,
|
|
30
|
-
ViewPropertyMissingDescription,
|
|
31
|
-
ViewPropertyMissingName,
|
|
32
|
-
)
|
|
33
|
-
from ._base import CDFResources, DataModelValidator, LocalResources
|
|
34
|
-
from ._connections import (
|
|
35
|
-
ConnectionValueTypeUndefined,
|
|
36
|
-
ConnectionValueTypeUnexisting,
|
|
37
|
-
ReverseConnectionContainerMissing,
|
|
38
|
-
ReverseConnectionContainerPropertyMissing,
|
|
39
|
-
ReverseConnectionContainerPropertyWrongType,
|
|
40
|
-
ReverseConnectionPointsToAncestor,
|
|
41
|
-
ReverseConnectionSourcePropertyMissing,
|
|
42
|
-
ReverseConnectionSourcePropertyWrongType,
|
|
43
|
-
ReverseConnectionSourceViewMissing,
|
|
44
|
-
ReverseConnectionTargetMismatch,
|
|
45
|
-
ReverseConnectionTargetMissing,
|
|
46
|
-
)
|
|
47
|
-
from ._consistency import ViewSpaceVersionInconsistentWithDataModel
|
|
48
|
-
from ._containers import (
|
|
49
|
-
ExternalContainerDoesNotExist,
|
|
50
|
-
ExternalContainerPropertyDoesNotExist,
|
|
51
|
-
RequiredContainerDoesNotExist,
|
|
52
|
-
)
|
|
53
|
-
from ._views import ImplementedViewNotExisting, ViewToContainerMappingNotPossible
|
|
12
|
+
from ._base import DataModelValidator
|
|
54
13
|
|
|
55
14
|
|
|
56
15
|
class DmsDataModelValidation(OnSuccessIssuesChecker):
|
|
@@ -58,104 +17,26 @@ class DmsDataModelValidation(OnSuccessIssuesChecker):
|
|
|
58
17
|
|
|
59
18
|
def __init__(
|
|
60
19
|
self,
|
|
61
|
-
|
|
20
|
+
cdf_snapshot: SchemaSnapshot,
|
|
21
|
+
limits: SchemaLimits,
|
|
62
22
|
modus_operandi: ModusOperandi = "additive",
|
|
63
23
|
can_run_validator: Callable[[str, type], bool] | None = None,
|
|
64
24
|
) -> None:
|
|
65
25
|
super().__init__()
|
|
66
|
-
self.
|
|
67
|
-
self.
|
|
26
|
+
self._cdf_snapshot = cdf_snapshot
|
|
27
|
+
self._limits = limits
|
|
68
28
|
self._modus_operandi = modus_operandi
|
|
29
|
+
self._can_run_validator = can_run_validator or (lambda code, issue_type: True) # type: ignore
|
|
69
30
|
self._has_run = False
|
|
70
31
|
|
|
71
|
-
def
|
|
72
|
-
"""Gather local and CDF resources needed for validation."""
|
|
73
|
-
|
|
74
|
-
analysis = DataModelAnalysis(data_model)
|
|
75
|
-
|
|
76
|
-
local_views_by_reference = analysis.view_by_reference(include_inherited_properties=True)
|
|
77
|
-
local_ancestors_by_view_reference = analysis.ancestors_by_view(list(local_views_by_reference.values()))
|
|
78
|
-
local_reverse_to_direct_mapping = analysis.reverse_to_direct_mapping
|
|
79
|
-
local_containers_by_reference = analysis.container_by_reference
|
|
80
|
-
local_data_model_views = set(data_model.data_model.views) if data_model.data_model.views else set()
|
|
81
|
-
|
|
82
|
-
local_resources = LocalResources(
|
|
83
|
-
data_model_description=data_model.data_model.description,
|
|
84
|
-
data_model_name=data_model.data_model.name,
|
|
85
|
-
data_model_reference=data_model.data_model.as_reference(),
|
|
86
|
-
views_by_reference=local_views_by_reference,
|
|
87
|
-
properties_by_view=analysis.properties_by_view,
|
|
88
|
-
ancestors_by_view_reference=local_ancestors_by_view_reference,
|
|
89
|
-
reverse_to_direct_mapping=local_reverse_to_direct_mapping,
|
|
90
|
-
containers_by_reference=local_containers_by_reference,
|
|
91
|
-
connection_end_node_types=analysis.connection_end_node_types,
|
|
92
|
-
data_model_views=local_data_model_views,
|
|
93
|
-
)
|
|
94
|
-
|
|
95
|
-
cdf_views_by_reference = self._cdf_view_by_reference(
|
|
96
|
-
list(analysis.referenced_views(include_connection_end_node_types=True)),
|
|
97
|
-
include_inherited_properties=True,
|
|
98
|
-
)
|
|
99
|
-
cdf_ancestors_by_view_reference = analysis.ancestors_by_view(list(cdf_views_by_reference.values()))
|
|
100
|
-
cdf_containers_by_reference = self._cdf_container_by_reference(
|
|
101
|
-
list(self._referenced_containers(local_views_by_reference, cdf_views_by_reference))
|
|
102
|
-
)
|
|
103
|
-
cdf_data_model_views = self._cdf_data_model_views(data_model.data_model.as_reference())
|
|
104
|
-
|
|
105
|
-
cdf_resources = CDFResources(
|
|
106
|
-
views_by_reference=cdf_views_by_reference,
|
|
107
|
-
ancestors_by_view_reference=cdf_ancestors_by_view_reference,
|
|
108
|
-
containers_by_reference=cdf_containers_by_reference,
|
|
109
|
-
data_model_views=cdf_data_model_views,
|
|
110
|
-
)
|
|
111
|
-
|
|
112
|
-
return local_resources, cdf_resources, self._cdf_limits()
|
|
113
|
-
|
|
114
|
-
def run(self, data_model: RequestSchema) -> None:
|
|
32
|
+
def run(self, request_schema: RequestSchema) -> None:
|
|
115
33
|
"""Run quality assessment on the DMS data model."""
|
|
116
34
|
|
|
117
|
-
|
|
118
|
-
local_resources, cdf_resources, cdf_limits = self._gather_resources(data_model)
|
|
35
|
+
validation_resources = self._gather_validation_resources(request_schema)
|
|
119
36
|
|
|
120
37
|
# Initialize all validators
|
|
121
38
|
validators: list[DataModelValidator] = [
|
|
122
|
-
|
|
123
|
-
DataModelViewCountIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
|
|
124
|
-
ViewPropertyCountIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
|
|
125
|
-
ViewImplementsCountIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
|
|
126
|
-
ViewContainerCountIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
|
|
127
|
-
ContainerPropertyCountIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
|
|
128
|
-
ContainerPropertyListSizeIsOutOfLimits(local_resources, cdf_resources, cdf_limits, self._modus_operandi),
|
|
129
|
-
# Views
|
|
130
|
-
ViewToContainerMappingNotPossible(local_resources, cdf_resources, self._modus_operandi),
|
|
131
|
-
ImplementedViewNotExisting(local_resources, cdf_resources, self._modus_operandi),
|
|
132
|
-
# Containers
|
|
133
|
-
ExternalContainerDoesNotExist(local_resources, cdf_resources, self._modus_operandi),
|
|
134
|
-
ExternalContainerPropertyDoesNotExist(local_resources, cdf_resources, self._modus_operandi),
|
|
135
|
-
RequiredContainerDoesNotExist(local_resources, cdf_resources, self._modus_operandi),
|
|
136
|
-
# Consistency
|
|
137
|
-
ViewSpaceVersionInconsistentWithDataModel(local_resources, cdf_resources, self._modus_operandi),
|
|
138
|
-
# Connections
|
|
139
|
-
ConnectionValueTypeUnexisting(local_resources, cdf_resources, self._modus_operandi),
|
|
140
|
-
ConnectionValueTypeUndefined(local_resources, cdf_resources, self._modus_operandi),
|
|
141
|
-
ReverseConnectionContainerMissing(local_resources, cdf_resources, self._modus_operandi),
|
|
142
|
-
ReverseConnectionContainerPropertyMissing(local_resources, cdf_resources, self._modus_operandi),
|
|
143
|
-
ReverseConnectionContainerPropertyWrongType(local_resources, cdf_resources, self._modus_operandi),
|
|
144
|
-
ReverseConnectionSourceViewMissing(local_resources, cdf_resources, self._modus_operandi),
|
|
145
|
-
ReverseConnectionSourcePropertyMissing(local_resources, cdf_resources, self._modus_operandi),
|
|
146
|
-
ReverseConnectionSourcePropertyWrongType(local_resources, cdf_resources, self._modus_operandi),
|
|
147
|
-
ReverseConnectionPointsToAncestor(local_resources, cdf_resources, self._modus_operandi),
|
|
148
|
-
ReverseConnectionTargetMismatch(local_resources, cdf_resources, self._modus_operandi),
|
|
149
|
-
ReverseConnectionTargetMissing(local_resources, cdf_resources, self._modus_operandi),
|
|
150
|
-
# AI Readiness
|
|
151
|
-
DataModelMissingName(local_resources, cdf_resources, self._modus_operandi),
|
|
152
|
-
DataModelMissingDescription(local_resources, cdf_resources, self._modus_operandi),
|
|
153
|
-
ViewMissingName(local_resources, cdf_resources, self._modus_operandi),
|
|
154
|
-
ViewMissingDescription(local_resources, cdf_resources, self._modus_operandi),
|
|
155
|
-
ViewPropertyMissingName(local_resources, cdf_resources, self._modus_operandi),
|
|
156
|
-
ViewPropertyMissingDescription(local_resources, cdf_resources, self._modus_operandi),
|
|
157
|
-
EnumerationMissingName(local_resources, cdf_resources, self._modus_operandi),
|
|
158
|
-
EnumerationMissingDescription(local_resources, cdf_resources, self._modus_operandi),
|
|
39
|
+
validator(validation_resources) for validator in get_concrete_subclasses(DataModelValidator)
|
|
159
40
|
]
|
|
160
41
|
|
|
161
42
|
# Run validators
|
|
@@ -165,58 +46,21 @@ class DmsDataModelValidation(OnSuccessIssuesChecker):
|
|
|
165
46
|
|
|
166
47
|
self._has_run = True
|
|
167
48
|
|
|
168
|
-
def
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
self, views: list[ViewReference], include_inherited_properties: bool = True
|
|
180
|
-
) -> dict[ViewReference, ViewRequest]:
|
|
181
|
-
"""Fetch view definition from CDF."""
|
|
182
|
-
|
|
183
|
-
if not self._client:
|
|
184
|
-
return {}
|
|
185
|
-
return {
|
|
186
|
-
response.as_reference(): response.as_request()
|
|
187
|
-
for response in self._client.views.retrieve(
|
|
188
|
-
views, include_inherited_properties=include_inherited_properties
|
|
189
|
-
)
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
def _cdf_container_by_reference(
|
|
193
|
-
self, containers: list[ContainerReference]
|
|
194
|
-
) -> dict[ContainerReference, ContainerRequest]:
|
|
195
|
-
"""Fetch container definition from CDF."""
|
|
196
|
-
|
|
197
|
-
if not self._client:
|
|
198
|
-
return {}
|
|
199
|
-
return {
|
|
200
|
-
response.as_reference(): response.as_request() for response in self._client.containers.retrieve(containers)
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
def _cdf_limits(self) -> SchemaLimits:
|
|
204
|
-
"""Fetch DMS statistics from CDF."""
|
|
205
|
-
|
|
206
|
-
if not self._client:
|
|
207
|
-
return SchemaLimits()
|
|
208
|
-
return SchemaLimits.from_api_response(self._client.statistics.project())
|
|
49
|
+
def _gather_validation_resources(self, request_schema: RequestSchema) -> ValidationResources:
|
|
50
|
+
# we do not want to modify the original request schema during validation
|
|
51
|
+
copy = request_schema.model_copy(deep=True)
|
|
52
|
+
local = SchemaSnapshot(
|
|
53
|
+
data_model={request_schema.data_model.as_reference(): copy.data_model},
|
|
54
|
+
views={view.as_reference(): view for view in copy.views},
|
|
55
|
+
containers={container.as_reference(): container for container in copy.containers},
|
|
56
|
+
spaces={space.as_reference(): space for space in copy.spaces},
|
|
57
|
+
node_types={node_type: node_type for node_type in copy.node_types},
|
|
58
|
+
timestamp=datetime.now(timezone.utc),
|
|
59
|
+
)
|
|
209
60
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
return {
|
|
217
|
-
property_.container
|
|
218
|
-
for view in chain(local_views_by_reference.values(), cdf_views_by_reference.values())
|
|
219
|
-
if view.properties
|
|
220
|
-
for property_ in view.properties.values()
|
|
221
|
-
if isinstance(property_, ViewCorePropertyRequest)
|
|
222
|
-
}
|
|
61
|
+
return ValidationResources(
|
|
62
|
+
cdf=self._cdf_snapshot,
|
|
63
|
+
local=local,
|
|
64
|
+
limits=self._limits,
|
|
65
|
+
modus_operandi=self._modus_operandi,
|
|
66
|
+
)
|
|
@@ -29,7 +29,21 @@ class ViewToContainerMappingNotPossible(DataModelValidator):
|
|
|
29
29
|
def run(self) -> list[ConsistencyError]:
|
|
30
30
|
errors: list[ConsistencyError] = []
|
|
31
31
|
|
|
32
|
-
|
|
32
|
+
if not self.validation_resources.merged_data_model.views:
|
|
33
|
+
return errors
|
|
34
|
+
|
|
35
|
+
for view_ref in self.validation_resources.merged_data_model.views:
|
|
36
|
+
view = self.validation_resources.select_view(view_ref)
|
|
37
|
+
|
|
38
|
+
if not view:
|
|
39
|
+
raise RuntimeError(
|
|
40
|
+
f"ViewToContainerMappingNotPossible.run: View {view_ref!s} "
|
|
41
|
+
"not found in local resources. This is a bug in NEAT."
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
if view.properties is None:
|
|
45
|
+
continue
|
|
46
|
+
|
|
33
47
|
for property_ref, property_ in view.properties.items():
|
|
34
48
|
if not isinstance(property_, ViewCorePropertyRequest):
|
|
35
49
|
continue
|
|
@@ -37,7 +51,7 @@ class ViewToContainerMappingNotPossible(DataModelValidator):
|
|
|
37
51
|
container_ref = property_.container
|
|
38
52
|
container_property = property_.container_property_identifier
|
|
39
53
|
|
|
40
|
-
container = self.
|
|
54
|
+
container = self.validation_resources.select_container(container_ref, container_property)
|
|
41
55
|
|
|
42
56
|
if not container:
|
|
43
57
|
errors.append(
|
|
@@ -87,11 +101,22 @@ class ImplementedViewNotExisting(DataModelValidator):
|
|
|
87
101
|
def run(self) -> list[ConsistencyError]:
|
|
88
102
|
errors: list[ConsistencyError] = []
|
|
89
103
|
|
|
90
|
-
|
|
104
|
+
if not self.validation_resources.merged_data_model.views:
|
|
105
|
+
return errors
|
|
106
|
+
|
|
107
|
+
for view_ref in self.validation_resources.merged_data_model.views:
|
|
108
|
+
view = self.validation_resources.select_view(view_ref)
|
|
109
|
+
|
|
110
|
+
if not view:
|
|
111
|
+
raise RuntimeError(
|
|
112
|
+
f"ImplementedViewNotExisting.run: View {view_ref!s} "
|
|
113
|
+
"not found in local resources. This is a bug in NEAT."
|
|
114
|
+
)
|
|
115
|
+
|
|
91
116
|
if view.implements is None:
|
|
92
117
|
continue
|
|
93
118
|
for implement in view.implements:
|
|
94
|
-
if implement
|
|
119
|
+
if self.validation_resources.select_view(implement) is None:
|
|
95
120
|
errors.append(
|
|
96
121
|
ConsistencyError(
|
|
97
122
|
message=f"View {view_ref!s} implements {implement!s} which is not defined.",
|