cognite-neat 0.99.0__py3-none-any.whl → 0.100.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/_api/data_modeling_loaders.py +390 -116
- cognite/neat/_client/_api/schema.py +63 -2
- cognite/neat/_client/data_classes/data_modeling.py +4 -0
- cognite/neat/_client/data_classes/schema.py +2 -348
- cognite/neat/_constants.py +27 -4
- cognite/neat/_graph/extractors/_base.py +7 -0
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +28 -18
- cognite/neat/_graph/loaders/_rdf2dms.py +52 -13
- cognite/neat/_graph/transformers/__init__.py +3 -3
- cognite/neat/_graph/transformers/_classic_cdf.py +135 -56
- cognite/neat/_issues/_base.py +26 -17
- cognite/neat/_issues/errors/__init__.py +4 -2
- cognite/neat/_issues/errors/_external.py +7 -0
- cognite/neat/_issues/errors/_properties.py +2 -7
- cognite/neat/_issues/errors/_resources.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -2
- cognite/neat/_issues/warnings/_external.py +9 -1
- cognite/neat/_issues/warnings/_resources.py +41 -2
- cognite/neat/_issues/warnings/user_modeling.py +4 -4
- cognite/neat/_rules/_constants.py +2 -6
- cognite/neat/_rules/analysis/_base.py +15 -5
- cognite/neat/_rules/analysis/_dms.py +20 -0
- cognite/neat/_rules/analysis/_information.py +22 -0
- cognite/neat/_rules/exporters/_base.py +3 -5
- cognite/neat/_rules/exporters/_rules2dms.py +190 -200
- cognite/neat/_rules/importers/__init__.py +1 -3
- cognite/neat/_rules/importers/_base.py +1 -1
- cognite/neat/_rules/importers/_dms2rules.py +3 -25
- cognite/neat/_rules/importers/_rdf/__init__.py +5 -0
- cognite/neat/_rules/importers/_rdf/_base.py +34 -11
- cognite/neat/_rules/importers/_rdf/_imf2rules.py +91 -0
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +40 -7
- cognite/neat/_rules/importers/_rdf/_owl2rules.py +80 -0
- cognite/neat/_rules/importers/_rdf/_shared.py +138 -441
- cognite/neat/_rules/models/_base_rules.py +19 -0
- cognite/neat/_rules/models/_types.py +5 -0
- cognite/neat/_rules/models/dms/__init__.py +2 -0
- cognite/neat/_rules/models/dms/_exporter.py +247 -123
- cognite/neat/_rules/models/dms/_rules.py +7 -49
- cognite/neat/_rules/models/dms/_rules_input.py +8 -3
- cognite/neat/_rules/models/dms/_validation.py +421 -123
- cognite/neat/_rules/models/entities/_multi_value.py +3 -0
- cognite/neat/_rules/models/information/__init__.py +2 -0
- cognite/neat/_rules/models/information/_rules.py +17 -61
- cognite/neat/_rules/models/information/_rules_input.py +11 -2
- cognite/neat/_rules/models/information/_validation.py +107 -11
- cognite/neat/_rules/models/mapping/_classic2core.py +1 -1
- cognite/neat/_rules/models/mapping/_classic2core.yaml +8 -4
- cognite/neat/_rules/transformers/__init__.py +2 -1
- cognite/neat/_rules/transformers/_converters.py +163 -61
- cognite/neat/_rules/transformers/_mapping.py +132 -2
- cognite/neat/_rules/transformers/_pipelines.py +1 -1
- cognite/neat/_rules/transformers/_verification.py +29 -4
- cognite/neat/_session/_base.py +46 -60
- cognite/neat/_session/_mapping.py +105 -5
- cognite/neat/_session/_prepare.py +49 -14
- cognite/neat/_session/_read.py +50 -4
- cognite/neat/_session/_set.py +1 -0
- cognite/neat/_session/_to.py +38 -12
- cognite/neat/_session/_wizard.py +5 -0
- cognite/neat/_session/engine/_interface.py +3 -2
- cognite/neat/_session/exceptions.py +4 -0
- cognite/neat/_store/_base.py +79 -19
- cognite/neat/_utils/collection_.py +22 -0
- cognite/neat/_utils/rdf_.py +30 -4
- cognite/neat/_version.py +2 -2
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +3 -91
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +2 -16
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +3 -5
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/RECORD +74 -82
- cognite/neat/_rules/importers/_rdf/_imf2rules/__init__.py +0 -3
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +0 -86
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +0 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +0 -130
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2rules.py +0 -154
- cognite/neat/_rules/importers/_rdf/_owl2rules/__init__.py +0 -3
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +0 -58
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +0 -65
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +0 -59
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2rules.py +0 -39
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,37 +1,51 @@
|
|
|
1
|
-
|
|
2
|
-
from
|
|
1
|
+
import warnings
|
|
2
|
+
from collections import Counter, defaultdict
|
|
3
|
+
from functools import lru_cache
|
|
4
|
+
from typing import ClassVar
|
|
3
5
|
|
|
4
6
|
from cognite.client import data_modeling as dm
|
|
7
|
+
from cognite.client.data_classes.data_modeling import ContainerList, ViewId, ViewList
|
|
8
|
+
from cognite.client.data_classes.data_modeling.views import (
|
|
9
|
+
ReverseDirectRelation,
|
|
10
|
+
ReverseDirectRelationApply,
|
|
11
|
+
ViewProperty,
|
|
12
|
+
ViewPropertyApply,
|
|
13
|
+
)
|
|
5
14
|
|
|
15
|
+
from cognite.neat._client import NeatClient
|
|
16
|
+
from cognite.neat._client.data_classes.data_modeling import ViewApplyDict
|
|
6
17
|
from cognite.neat._client.data_classes.schema import DMSSchema
|
|
7
|
-
from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_LIMIT
|
|
8
|
-
from cognite.neat._issues import IssueList, NeatError,
|
|
18
|
+
from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_LIMIT, DMS_VIEW_CONTAINER_SIZE_LIMIT
|
|
19
|
+
from cognite.neat._issues import IssueList, NeatError, NeatIssueList
|
|
9
20
|
from cognite.neat._issues.errors import (
|
|
21
|
+
CDFMissingClientError,
|
|
10
22
|
PropertyDefinitionDuplicatedError,
|
|
11
|
-
|
|
23
|
+
PropertyMappingDuplicatedError,
|
|
24
|
+
PropertyNotFoundError,
|
|
25
|
+
ResourceDuplicatedError,
|
|
26
|
+
ResourceNotFoundError,
|
|
27
|
+
ReversedConnectionNotFeasibleError,
|
|
12
28
|
)
|
|
13
|
-
from cognite.neat._issues.errors._properties import ReversedConnectionNotFeasibleError
|
|
14
29
|
from cognite.neat._issues.warnings import (
|
|
15
30
|
NotSupportedHasDataFilterLimitWarning,
|
|
16
31
|
NotSupportedViewContainerLimitWarning,
|
|
17
32
|
UndefinedViewWarning,
|
|
18
33
|
)
|
|
19
34
|
from cognite.neat._issues.warnings.user_modeling import (
|
|
35
|
+
ContainerPropertyLimitWarning,
|
|
36
|
+
DirectRelationMissingSourceWarning,
|
|
20
37
|
NotNeatSupportedFilterWarning,
|
|
21
|
-
ViewPropertyLimitWarning,
|
|
22
38
|
)
|
|
23
|
-
from cognite.neat._rules.analysis import DMSAnalysis
|
|
24
39
|
from cognite.neat._rules.models.data_types import DataType
|
|
25
40
|
from cognite.neat._rules.models.entities import ContainerEntity, RawFilter
|
|
26
41
|
from cognite.neat._rules.models.entities._single_value import (
|
|
27
|
-
ReverseConnectionEntity,
|
|
28
42
|
ViewEntity,
|
|
29
43
|
)
|
|
30
44
|
|
|
31
45
|
from ._rules import DMSProperty, DMSRules
|
|
32
46
|
|
|
33
47
|
|
|
34
|
-
class
|
|
48
|
+
class DMSValidation:
|
|
35
49
|
"""This class does all the validation of the DMS rules that have dependencies between
|
|
36
50
|
components."""
|
|
37
51
|
|
|
@@ -39,33 +53,160 @@ class DMSPostValidation:
|
|
|
39
53
|
# For example, changing the filter is allowed, but changing the properties is not.
|
|
40
54
|
changeable_view_attributes: ClassVar[set[str]] = {"filter"}
|
|
41
55
|
|
|
42
|
-
def __init__(self, rules: DMSRules):
|
|
43
|
-
self.
|
|
44
|
-
self.
|
|
45
|
-
self.
|
|
46
|
-
self.
|
|
47
|
-
self.
|
|
48
|
-
self.
|
|
49
|
-
|
|
56
|
+
def __init__(self, rules: DMSRules, client: NeatClient | None = None) -> None:
|
|
57
|
+
self._rules = rules
|
|
58
|
+
self._client = client
|
|
59
|
+
self._metadata = rules.metadata
|
|
60
|
+
self._properties = rules.properties
|
|
61
|
+
self._containers = rules.containers
|
|
62
|
+
self._views = rules.views
|
|
63
|
+
|
|
64
|
+
def imported_views_and_containers_ids(
|
|
65
|
+
self, include_views_with_no_properties: bool = True
|
|
66
|
+
) -> tuple[set[ViewEntity], set[ContainerEntity]]:
|
|
67
|
+
existing_views = {view.view for view in self._views}
|
|
68
|
+
imported_views: set[ViewEntity] = set()
|
|
69
|
+
for view in self._views:
|
|
70
|
+
for parent in view.implements or []:
|
|
71
|
+
if parent not in existing_views:
|
|
72
|
+
imported_views.add(parent)
|
|
73
|
+
existing_containers = {container.container for container in self._containers or []}
|
|
74
|
+
imported_containers: set[ContainerEntity] = set()
|
|
75
|
+
view_with_properties: set[ViewEntity] = set()
|
|
76
|
+
for prop in self._properties:
|
|
77
|
+
if prop.container and prop.container not in existing_containers:
|
|
78
|
+
imported_containers.add(prop.container)
|
|
79
|
+
if prop.view not in existing_views:
|
|
80
|
+
imported_views.add(prop.view)
|
|
81
|
+
view_with_properties.add(prop.view)
|
|
82
|
+
|
|
83
|
+
if include_views_with_no_properties:
|
|
84
|
+
extra_views = existing_views - view_with_properties
|
|
85
|
+
imported_views.update({view for view in extra_views})
|
|
86
|
+
|
|
87
|
+
return imported_views, imported_containers
|
|
50
88
|
|
|
51
89
|
def validate(self) -> NeatIssueList:
|
|
52
|
-
self.
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
self.
|
|
90
|
+
imported_views, imported_containers = self.imported_views_and_containers_ids(
|
|
91
|
+
include_views_with_no_properties=False
|
|
92
|
+
)
|
|
93
|
+
if (imported_views or imported_containers) and self._client is None:
|
|
94
|
+
raise CDFMissingClientError(
|
|
95
|
+
f"{self._rules.metadata.as_data_model_id()} has imported views and/or container: "
|
|
96
|
+
f"{imported_views}, {imported_containers}."
|
|
97
|
+
)
|
|
98
|
+
referenced_views = ViewList([])
|
|
99
|
+
referenced_containers = ContainerList([])
|
|
100
|
+
if self._client:
|
|
101
|
+
referenced_views = self._client.loaders.views.retrieve(
|
|
102
|
+
list(imported_views), include_connected=True, include_ancestor=True
|
|
103
|
+
)
|
|
104
|
+
referenced_containers = self._client.loaders.containers.retrieve(
|
|
105
|
+
list(imported_containers), include_connected=True
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
# Setup data structures for validation
|
|
109
|
+
dms_schema = self._rules.as_schema()
|
|
110
|
+
ref_view_by_id = {view.as_id(): view for view in referenced_views}
|
|
111
|
+
ref_container_by_id = {container.as_id(): container for container in referenced_containers}
|
|
112
|
+
# All containers and views are the Containers/Views in the DMSRules + the referenced ones
|
|
113
|
+
all_containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container] = {
|
|
114
|
+
**dict(dms_schema.containers.items()),
|
|
115
|
+
**ref_container_by_id,
|
|
116
|
+
}
|
|
117
|
+
all_views_by_id: dict[dm.ViewId, dm.ViewApply | dm.View] = {**dict(dms_schema.views.items()), **ref_view_by_id}
|
|
118
|
+
properties_by_ids = self._as_properties_by_ids(dms_schema, ref_view_by_id)
|
|
119
|
+
view_properties_by_id = self._as_view_properties_by_id(properties_by_ids)
|
|
120
|
+
parents_view_ids_by_child_id = self._parent_view_ids_by_child_id(all_views_by_id)
|
|
56
121
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
122
|
+
issue_list = IssueList()
|
|
123
|
+
# Neat DMS classes Validation
|
|
124
|
+
# These are errors that can only happen due to the format of the Neat DMS classes
|
|
125
|
+
issue_list.extend(self._validate_raw_filter())
|
|
126
|
+
issue_list.extend(self._consistent_container_properties())
|
|
127
|
+
issue_list.extend(self._validate_value_type_existence())
|
|
128
|
+
issue_list.extend(
|
|
129
|
+
self._validate_property_referenced_views_and_containers_exists(all_views_by_id, all_containers_by_id)
|
|
130
|
+
)
|
|
61
131
|
|
|
62
|
-
|
|
132
|
+
# SDK classes validation
|
|
133
|
+
issue_list.extend(self._containers_are_proper_size(dms_schema))
|
|
134
|
+
issue_list.extend(
|
|
135
|
+
self._validate_reverse_connections(properties_by_ids, all_containers_by_id, parents_view_ids_by_child_id)
|
|
136
|
+
)
|
|
137
|
+
issue_list.extend(self._validate_schema(dms_schema, all_views_by_id, all_containers_by_id))
|
|
138
|
+
issue_list.extend(self._validate_referenced_container_limits(dms_schema.views, view_properties_by_id))
|
|
139
|
+
return issue_list
|
|
140
|
+
|
|
141
|
+
@staticmethod
|
|
142
|
+
def _as_properties_by_ids(
|
|
143
|
+
dms_schema: DMSSchema, ref_view_by_id: dict[dm.ViewId, dm.View]
|
|
144
|
+
) -> dict[tuple[ViewId, str], ViewPropertyApply | ViewProperty]:
|
|
145
|
+
# Priority DMS schema properties.
|
|
146
|
+
# No need to do long lookups in ref_views as these already contain all ancestor properties.
|
|
147
|
+
properties_by_id: dict[tuple[ViewId, str], ViewPropertyApply | ViewProperty] = {}
|
|
148
|
+
for view in dms_schema.views.values():
|
|
149
|
+
view_id = view.as_id()
|
|
150
|
+
for prop_id, prop in (view.properties or {}).items():
|
|
151
|
+
properties_by_id[(view_id, prop_id)] = prop
|
|
152
|
+
if view.implements:
|
|
153
|
+
to_check = view.implements.copy()
|
|
154
|
+
while to_check:
|
|
155
|
+
parent_id = to_check.pop()
|
|
156
|
+
if parent_id in dms_schema.views:
|
|
157
|
+
# Priority DMS Schema properties
|
|
158
|
+
parent_view = dms_schema.views[parent_id]
|
|
159
|
+
for prop_id, prop in (parent_view.properties or {}).items():
|
|
160
|
+
if (view_id, prop_id) not in properties_by_id:
|
|
161
|
+
properties_by_id[(view_id, prop_id)] = prop
|
|
162
|
+
to_check.extend(parent_view.implements or [])
|
|
163
|
+
elif parent_id in ref_view_by_id:
|
|
164
|
+
# SDK properties
|
|
165
|
+
parent_read_view = ref_view_by_id[parent_id]
|
|
166
|
+
for prop_id, read_prop in parent_read_view.properties.items():
|
|
167
|
+
if (view_id, prop_id) not in properties_by_id:
|
|
168
|
+
properties_by_id[(view_id, prop_id)] = read_prop
|
|
169
|
+
# Read format of views already includes all ancestor properties
|
|
170
|
+
# so no need to check further
|
|
171
|
+
else:
|
|
172
|
+
# Missing views are caught else where
|
|
173
|
+
continue
|
|
174
|
+
|
|
175
|
+
return properties_by_id
|
|
176
|
+
|
|
177
|
+
@staticmethod
|
|
178
|
+
def _as_view_properties_by_id(
|
|
179
|
+
properties_by_ids: dict[tuple[ViewId, str], ViewPropertyApply | ViewProperty],
|
|
180
|
+
) -> dict[ViewId, list[tuple[str, ViewProperty | ViewPropertyApply]]]:
|
|
181
|
+
view_properties_by_id: dict[dm.ViewId, list[tuple[str, ViewProperty | ViewPropertyApply]]] = defaultdict(list)
|
|
182
|
+
for (view_id, prop_id), prop in properties_by_ids.items():
|
|
183
|
+
view_properties_by_id[view_id].append((prop_id, prop))
|
|
184
|
+
return view_properties_by_id
|
|
185
|
+
|
|
186
|
+
@staticmethod
|
|
187
|
+
def _parent_view_ids_by_child_id(
|
|
188
|
+
all_views_by_id: dict[dm.ViewId, dm.ViewApply | dm.View],
|
|
189
|
+
) -> dict[ViewId, set[ViewId]]:
|
|
190
|
+
@lru_cache
|
|
191
|
+
def get_parents(child_view_id: ViewId) -> set[ViewId]:
|
|
192
|
+
child_view = all_views_by_id[child_view_id]
|
|
193
|
+
parents = set(child_view.implements or [])
|
|
194
|
+
for parent_id in child_view.implements or []:
|
|
195
|
+
parents.update(get_parents(parent_id))
|
|
196
|
+
return parents
|
|
197
|
+
|
|
198
|
+
parents_by_view: dict[dm.ViewId, set[dm.ViewId]] = {}
|
|
199
|
+
for view_id in all_views_by_id:
|
|
200
|
+
parents_by_view[view_id] = get_parents(view_id)
|
|
201
|
+
return parents_by_view
|
|
202
|
+
|
|
203
|
+
def _consistent_container_properties(self) -> IssueList:
|
|
63
204
|
container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, DMSProperty]]] = defaultdict(list)
|
|
64
|
-
for prop_no, prop in enumerate(self.
|
|
205
|
+
for prop_no, prop in enumerate(self._properties):
|
|
65
206
|
if prop.container and prop.container_property:
|
|
66
207
|
container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
|
|
67
208
|
|
|
68
|
-
errors
|
|
209
|
+
errors = IssueList()
|
|
69
210
|
for (container, prop_name), properties in container_properties_by_id.items():
|
|
70
211
|
if len(properties) == 1:
|
|
71
212
|
continue
|
|
@@ -152,135 +293,292 @@ class DMSPostValidation:
|
|
|
152
293
|
)
|
|
153
294
|
)
|
|
154
295
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
def _referenced_views_and_containers_are_existing_and_proper_size(self) -> None:
|
|
158
|
-
# TODO: Split this method and keep only validation that should be independent of
|
|
159
|
-
# whether view and/or container exist in the pydantic model instance
|
|
160
|
-
# other validation should be done through NeatSession.verify()
|
|
161
|
-
defined_views = {view.view.as_id() for view in self.views}
|
|
296
|
+
return errors
|
|
162
297
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
errors.append(
|
|
169
|
-
ResourceNotDefinedError(
|
|
170
|
-
identifier=view_id,
|
|
171
|
-
resource_type="view",
|
|
172
|
-
location="Views Sheet",
|
|
173
|
-
column_name="View",
|
|
174
|
-
row_number=prop_no,
|
|
175
|
-
sheet_name="Properties",
|
|
176
|
-
),
|
|
177
|
-
)
|
|
178
|
-
else:
|
|
179
|
-
property_count_by_view[view_id] += 1
|
|
180
|
-
for view_id, count in property_count_by_view.items():
|
|
298
|
+
@staticmethod
|
|
299
|
+
def _containers_are_proper_size(dms_schema: DMSSchema) -> IssueList:
|
|
300
|
+
errors = IssueList()
|
|
301
|
+
for container_id, container in dms_schema.containers.items():
|
|
302
|
+
count = len(container.properties or {})
|
|
181
303
|
if count > DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
|
|
182
|
-
errors.append(
|
|
304
|
+
errors.append(ContainerPropertyLimitWarning(container_id, count))
|
|
183
305
|
|
|
184
|
-
|
|
306
|
+
return errors
|
|
185
307
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
308
|
+
@staticmethod
|
|
309
|
+
def _validate_referenced_container_limits(
|
|
310
|
+
views: ViewApplyDict, view_properties_by_id: dict[dm.ViewId, list[tuple[str, ViewProperty | ViewPropertyApply]]]
|
|
311
|
+
) -> IssueList:
|
|
312
|
+
issue_list = IssueList()
|
|
313
|
+
for view_id, view in views.items():
|
|
314
|
+
view_properties = view_properties_by_id.get(view_id, [])
|
|
315
|
+
mapped_containers = {
|
|
316
|
+
prop.container
|
|
317
|
+
for _, prop in view_properties
|
|
318
|
+
if isinstance(prop, dm.MappedPropertyApply | dm.MappedProperty)
|
|
319
|
+
}
|
|
189
320
|
|
|
190
|
-
if mapped_containers and len(mapped_containers) >
|
|
191
|
-
|
|
321
|
+
if mapped_containers and len(mapped_containers) > DMS_VIEW_CONTAINER_SIZE_LIMIT:
|
|
322
|
+
issue_list.append(
|
|
192
323
|
NotSupportedViewContainerLimitWarning(
|
|
193
324
|
view_id,
|
|
194
325
|
len(mapped_containers),
|
|
195
326
|
)
|
|
196
327
|
)
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
NotSupportedHasDataFilterLimitWarning(
|
|
204
|
-
view_id,
|
|
205
|
-
len(view.filter.dump()["hasData"]),
|
|
206
|
-
)
|
|
328
|
+
|
|
329
|
+
if view.filter and isinstance(view.filter, dm.filters.HasData) and len(view.filter.dump()["hasData"]) > 10:
|
|
330
|
+
issue_list.append(
|
|
331
|
+
NotSupportedHasDataFilterLimitWarning(
|
|
332
|
+
view_id,
|
|
333
|
+
len(view.filter.dump()["hasData"]),
|
|
207
334
|
)
|
|
335
|
+
)
|
|
336
|
+
return issue_list
|
|
208
337
|
|
|
209
|
-
def _validate_raw_filter(self) ->
|
|
210
|
-
|
|
338
|
+
def _validate_raw_filter(self) -> IssueList:
|
|
339
|
+
issue_list = IssueList()
|
|
340
|
+
for view in self._views:
|
|
211
341
|
if view.filter_ and isinstance(view.filter_, RawFilter):
|
|
212
|
-
|
|
342
|
+
issue_list.append(
|
|
213
343
|
NotNeatSupportedFilterWarning(view.view.as_id()),
|
|
214
344
|
)
|
|
345
|
+
return issue_list
|
|
215
346
|
|
|
216
|
-
def _validate_value_type_existence(self) ->
|
|
217
|
-
views = {prop_.view for prop_ in self.
|
|
218
|
-
|
|
219
|
-
for prop_ in self.
|
|
347
|
+
def _validate_value_type_existence(self) -> IssueList:
|
|
348
|
+
views = {prop_.view for prop_ in self._properties}.union({view_.view for view_ in self._views})
|
|
349
|
+
issue_list = IssueList()
|
|
350
|
+
for prop_ in self._properties:
|
|
220
351
|
if isinstance(prop_.value_type, ViewEntity) and prop_.value_type not in views:
|
|
221
|
-
|
|
352
|
+
issue_list.append(
|
|
222
353
|
UndefinedViewWarning(
|
|
223
354
|
str(prop_.view),
|
|
224
355
|
str(prop_.value_type),
|
|
225
356
|
prop_.view_property,
|
|
226
357
|
)
|
|
227
358
|
)
|
|
359
|
+
return issue_list
|
|
228
360
|
|
|
229
|
-
def
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
361
|
+
def _validate_property_referenced_views_and_containers_exists(
|
|
362
|
+
self,
|
|
363
|
+
view_by_id: dict[dm.ViewId, dm.ViewApply | dm.View],
|
|
364
|
+
containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container],
|
|
365
|
+
) -> IssueList:
|
|
366
|
+
issue_list = IssueList()
|
|
367
|
+
for prop in self._properties:
|
|
368
|
+
if prop.container:
|
|
369
|
+
container_id = prop.container.as_id()
|
|
370
|
+
if container_id not in containers_by_id:
|
|
371
|
+
issue_list.append(
|
|
372
|
+
ResourceNotFoundError(
|
|
373
|
+
container_id,
|
|
374
|
+
"container",
|
|
375
|
+
prop.view,
|
|
376
|
+
"view",
|
|
377
|
+
)
|
|
378
|
+
)
|
|
379
|
+
elif (
|
|
380
|
+
prop.container_property and prop.container_property not in containers_by_id[container_id].properties
|
|
381
|
+
):
|
|
382
|
+
issue_list.append(
|
|
383
|
+
PropertyNotFoundError(
|
|
384
|
+
prop.container,
|
|
385
|
+
"container property",
|
|
386
|
+
prop.container_property,
|
|
387
|
+
dm.PropertyId(prop.view.as_id(), prop.view_property),
|
|
388
|
+
"view property",
|
|
389
|
+
)
|
|
390
|
+
)
|
|
233
391
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
392
|
+
if prop.view.as_id() not in view_by_id:
|
|
393
|
+
issue_list.append(
|
|
394
|
+
ResourceNotFoundError(
|
|
395
|
+
prop.view,
|
|
396
|
+
"view",
|
|
397
|
+
prop.view_property,
|
|
398
|
+
"property",
|
|
399
|
+
)
|
|
400
|
+
)
|
|
239
401
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
402
|
+
return issue_list
|
|
403
|
+
|
|
404
|
+
def _validate_reverse_connections(
|
|
405
|
+
self,
|
|
406
|
+
view_property_by_property_id: dict[tuple[dm.ViewId, str], ViewPropertyApply | ViewProperty],
|
|
407
|
+
containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container],
|
|
408
|
+
parents_by_view: dict[dm.ViewId, set[dm.ViewId]],
|
|
409
|
+
) -> IssueList:
|
|
410
|
+
issue_list = IssueList()
|
|
411
|
+
# do not check for reverse connections in Cognite models
|
|
412
|
+
if self._metadata.as_data_model_id() in COGNITE_MODELS:
|
|
413
|
+
return issue_list
|
|
245
414
|
|
|
246
|
-
for
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
415
|
+
for (view_id, prop_id), prop_ in view_property_by_property_id.items():
|
|
416
|
+
if not isinstance(prop_, ReverseDirectRelationApply | ReverseDirectRelation):
|
|
417
|
+
continue
|
|
418
|
+
target_id = prop_.through.source, prop_.through.property
|
|
419
|
+
if target_id not in view_property_by_property_id:
|
|
420
|
+
issue_list.append(
|
|
250
421
|
ReversedConnectionNotFeasibleError(
|
|
251
|
-
|
|
422
|
+
view_id,
|
|
252
423
|
"reversed connection",
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
str(prop_.value_type),
|
|
256
|
-
cast(ReverseConnectionEntity, prop_.connection).property_,
|
|
424
|
+
prop_id,
|
|
425
|
+
f"The {prop_.through.source} {prop_.through.property} does not exist",
|
|
257
426
|
)
|
|
258
427
|
)
|
|
428
|
+
continue
|
|
429
|
+
if isinstance(target_id[0], dm.ContainerId):
|
|
430
|
+
# Todo: How to handle this case? Should not happen if you created the model with Neat
|
|
431
|
+
continue
|
|
259
432
|
|
|
260
|
-
|
|
261
|
-
|
|
433
|
+
target_property = view_property_by_property_id[(target_id[0], target_id[1])]
|
|
434
|
+
# Validate that the target is a direct relation pointing to the view_id
|
|
435
|
+
is_direct_relation = False
|
|
436
|
+
if isinstance(target_property, dm.MappedProperty) and isinstance(target_property.type, dm.DirectRelation):
|
|
437
|
+
is_direct_relation = True
|
|
438
|
+
elif isinstance(target_property, dm.MappedPropertyApply):
|
|
439
|
+
container = containers_by_id[target_property.container]
|
|
440
|
+
if target_property.container_property_identifier in container.properties:
|
|
441
|
+
container_property = container.properties[target_property.container_property_identifier]
|
|
442
|
+
if isinstance(container_property.type, dm.DirectRelation):
|
|
443
|
+
is_direct_relation = True
|
|
444
|
+
if not is_direct_relation:
|
|
445
|
+
issue_list.append(
|
|
262
446
|
ReversedConnectionNotFeasibleError(
|
|
263
|
-
|
|
264
|
-
"
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
str(prop_.value_type),
|
|
268
|
-
cast(ReverseConnectionEntity, prop_.connection).property_,
|
|
447
|
+
view_id,
|
|
448
|
+
"reversed connection",
|
|
449
|
+
prop_id,
|
|
450
|
+
f"{prop_.through.source} {prop_.through.property} is not a direct relation",
|
|
269
451
|
)
|
|
270
452
|
)
|
|
271
|
-
|
|
272
|
-
else:
|
|
273
453
|
continue
|
|
454
|
+
if not (
|
|
455
|
+
isinstance(target_property, dm.MappedPropertyApply | dm.MappedProperty)
|
|
456
|
+
# The direct relation is pointing to the view_id or one of its parents
|
|
457
|
+
and (target_property.source == view_id or target_property.source in parents_by_view[view_id])
|
|
458
|
+
):
|
|
459
|
+
issue_list.append(
|
|
460
|
+
ReversedConnectionNotFeasibleError(
|
|
461
|
+
view_id,
|
|
462
|
+
"reversed connection",
|
|
463
|
+
prop_id,
|
|
464
|
+
f"{prop_.through.source} {prop_.through.property} is not pointing to {view_id}",
|
|
465
|
+
)
|
|
466
|
+
)
|
|
467
|
+
return issue_list
|
|
274
468
|
|
|
275
469
|
@staticmethod
|
|
276
|
-
def
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
470
|
+
def _validate_schema(
|
|
471
|
+
schema: DMSSchema,
|
|
472
|
+
view_by_id: dict[dm.ViewId, dm.ViewApply | dm.View],
|
|
473
|
+
containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container],
|
|
474
|
+
) -> IssueList:
|
|
475
|
+
errors: set[NeatError] = set()
|
|
476
|
+
defined_spaces = schema.spaces.copy()
|
|
477
|
+
|
|
478
|
+
for container_id, container in schema.containers.items():
|
|
479
|
+
if container.space not in defined_spaces:
|
|
480
|
+
errors.add(ResourceNotFoundError(container.space, "space", container_id, "container"))
|
|
481
|
+
for constraint in container.constraints.values():
|
|
482
|
+
if isinstance(constraint, dm.RequiresConstraint) and constraint.require not in containers_by_id:
|
|
483
|
+
errors.add(ResourceNotFoundError(constraint.require, "container", container_id, "container"))
|
|
484
|
+
|
|
485
|
+
for view_id, view in schema.views.items():
|
|
486
|
+
if view.space not in defined_spaces:
|
|
487
|
+
errors.add(ResourceNotFoundError(view.space, "space", view_id, "view"))
|
|
488
|
+
|
|
489
|
+
for parent in view.implements or []:
|
|
490
|
+
if parent not in view_by_id:
|
|
491
|
+
errors.add(PropertyNotFoundError(parent, "view", "implements", view_id, "view"))
|
|
492
|
+
|
|
493
|
+
for prop_name, prop in (view.properties or {}).items():
|
|
494
|
+
if isinstance(prop, dm.MappedPropertyApply):
|
|
495
|
+
ref_container = containers_by_id.get(prop.container)
|
|
496
|
+
if ref_container is None:
|
|
497
|
+
errors.add(ResourceNotFoundError(prop.container, "container", view_id, "view"))
|
|
498
|
+
elif prop.container_property_identifier not in ref_container.properties:
|
|
499
|
+
errors.add(
|
|
500
|
+
PropertyNotFoundError(
|
|
501
|
+
prop.container,
|
|
502
|
+
"container",
|
|
503
|
+
prop.container_property_identifier,
|
|
504
|
+
view_id,
|
|
505
|
+
"view",
|
|
506
|
+
)
|
|
507
|
+
)
|
|
508
|
+
else:
|
|
509
|
+
container_property = ref_container.properties[prop.container_property_identifier]
|
|
510
|
+
|
|
511
|
+
if isinstance(container_property.type, dm.DirectRelation) and prop.source is None:
|
|
512
|
+
warnings.warn(
|
|
513
|
+
DirectRelationMissingSourceWarning(view_id, prop_name),
|
|
514
|
+
stacklevel=2,
|
|
515
|
+
)
|
|
516
|
+
|
|
517
|
+
if (
|
|
518
|
+
isinstance(prop, dm.EdgeConnectionApply | ReverseDirectRelationApply)
|
|
519
|
+
and prop.source not in view_by_id
|
|
520
|
+
):
|
|
521
|
+
errors.add(PropertyNotFoundError(prop.source, "view", prop_name, view_id, "view"))
|
|
522
|
+
|
|
523
|
+
if (
|
|
524
|
+
isinstance(prop, dm.EdgeConnectionApply)
|
|
525
|
+
and prop.edge_source is not None
|
|
526
|
+
and prop.edge_source not in view_by_id
|
|
527
|
+
):
|
|
528
|
+
errors.add(PropertyNotFoundError(prop.edge_source, "view", prop_name, view_id, "view"))
|
|
529
|
+
|
|
530
|
+
# This allows for multiple view properties to be mapped to the same container property,
|
|
531
|
+
# as long as they have different external_id, otherwise this will lead to raising
|
|
532
|
+
# error ContainerPropertyUsedMultipleTimesError
|
|
533
|
+
property_count = Counter(
|
|
534
|
+
(prop.container, prop.container_property_identifier, view_property_identifier)
|
|
535
|
+
for view_property_identifier, prop in (view.properties or {}).items()
|
|
536
|
+
if isinstance(prop, dm.MappedPropertyApply)
|
|
537
|
+
)
|
|
538
|
+
|
|
539
|
+
for (
|
|
540
|
+
container_id,
|
|
541
|
+
container_property_identifier,
|
|
542
|
+
_,
|
|
543
|
+
), count in property_count.items():
|
|
544
|
+
if count > 1:
|
|
545
|
+
view_properties = [
|
|
546
|
+
prop_name
|
|
547
|
+
for prop_name, prop in (view.properties or {}).items()
|
|
548
|
+
if isinstance(prop, dm.MappedPropertyApply)
|
|
549
|
+
and (prop.container, prop.container_property_identifier)
|
|
550
|
+
== (container_id, container_property_identifier)
|
|
551
|
+
]
|
|
552
|
+
errors.add(
|
|
553
|
+
PropertyMappingDuplicatedError(
|
|
554
|
+
container_id,
|
|
555
|
+
"container",
|
|
556
|
+
container_property_identifier,
|
|
557
|
+
frozenset({dm.PropertyId(view_id, prop_name) for prop_name in view_properties}),
|
|
558
|
+
"view property",
|
|
559
|
+
)
|
|
560
|
+
)
|
|
561
|
+
|
|
562
|
+
if schema.data_model:
|
|
563
|
+
model = schema.data_model
|
|
564
|
+
if model.space not in defined_spaces:
|
|
565
|
+
errors.add(ResourceNotFoundError(model.space, "space", model.as_id(), "data model"))
|
|
566
|
+
|
|
567
|
+
view_counts: dict[dm.ViewId, int] = defaultdict(int)
|
|
568
|
+
for view_id_or_class in model.views or []:
|
|
569
|
+
view_id = view_id_or_class if isinstance(view_id_or_class, dm.ViewId) else view_id_or_class.as_id()
|
|
570
|
+
if view_id not in view_by_id:
|
|
571
|
+
errors.add(ResourceNotFoundError(view_id, "view", model.as_id(), "data model"))
|
|
572
|
+
view_counts[view_id] += 1
|
|
573
|
+
|
|
574
|
+
for view_id, count in view_counts.items():
|
|
575
|
+
if count > 1:
|
|
576
|
+
errors.add(
|
|
577
|
+
ResourceDuplicatedError(
|
|
578
|
+
view_id,
|
|
579
|
+
"view",
|
|
580
|
+
repr(model.as_id()),
|
|
581
|
+
)
|
|
582
|
+
)
|
|
583
|
+
|
|
584
|
+
return IssueList(list(errors))
|
|
@@ -78,3 +78,6 @@ class MultiValueTypeInfo(BaseModel):
|
|
|
78
78
|
def is_mixed_type(self) -> bool:
|
|
79
79
|
"""Will signalize to DMS converter to fall back to string"""
|
|
80
80
|
return not self.is_multi_object_type() and not self.is_multi_data_type()
|
|
81
|
+
|
|
82
|
+
def __hash__(self) -> int:
|
|
83
|
+
return hash(str(self))
|
|
@@ -5,6 +5,7 @@ from ._rules_input import (
|
|
|
5
5
|
InformationInputProperty,
|
|
6
6
|
InformationInputRules,
|
|
7
7
|
)
|
|
8
|
+
from ._validation import InformationValidation
|
|
8
9
|
|
|
9
10
|
__all__ = [
|
|
10
11
|
"InformationRules",
|
|
@@ -15,4 +16,5 @@ __all__ = [
|
|
|
15
16
|
"InformationInputMetadata",
|
|
16
17
|
"InformationInputClass",
|
|
17
18
|
"InformationInputProperty",
|
|
19
|
+
"InformationValidation",
|
|
18
20
|
]
|