cognite-neat 0.99.0__py3-none-any.whl → 0.100.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (84) hide show
  1. cognite/neat/_client/_api/data_modeling_loaders.py +390 -116
  2. cognite/neat/_client/_api/schema.py +63 -2
  3. cognite/neat/_client/data_classes/data_modeling.py +4 -0
  4. cognite/neat/_client/data_classes/schema.py +2 -348
  5. cognite/neat/_constants.py +27 -4
  6. cognite/neat/_graph/extractors/_base.py +7 -0
  7. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +28 -18
  8. cognite/neat/_graph/loaders/_rdf2dms.py +52 -13
  9. cognite/neat/_graph/transformers/__init__.py +3 -3
  10. cognite/neat/_graph/transformers/_classic_cdf.py +135 -56
  11. cognite/neat/_issues/_base.py +26 -17
  12. cognite/neat/_issues/errors/__init__.py +4 -2
  13. cognite/neat/_issues/errors/_external.py +7 -0
  14. cognite/neat/_issues/errors/_properties.py +2 -7
  15. cognite/neat/_issues/errors/_resources.py +1 -1
  16. cognite/neat/_issues/warnings/__init__.py +6 -2
  17. cognite/neat/_issues/warnings/_external.py +9 -1
  18. cognite/neat/_issues/warnings/_resources.py +41 -2
  19. cognite/neat/_issues/warnings/user_modeling.py +4 -4
  20. cognite/neat/_rules/_constants.py +2 -6
  21. cognite/neat/_rules/analysis/_base.py +15 -5
  22. cognite/neat/_rules/analysis/_dms.py +20 -0
  23. cognite/neat/_rules/analysis/_information.py +22 -0
  24. cognite/neat/_rules/exporters/_base.py +3 -5
  25. cognite/neat/_rules/exporters/_rules2dms.py +190 -200
  26. cognite/neat/_rules/importers/__init__.py +1 -3
  27. cognite/neat/_rules/importers/_base.py +1 -1
  28. cognite/neat/_rules/importers/_dms2rules.py +3 -25
  29. cognite/neat/_rules/importers/_rdf/__init__.py +5 -0
  30. cognite/neat/_rules/importers/_rdf/_base.py +34 -11
  31. cognite/neat/_rules/importers/_rdf/_imf2rules.py +91 -0
  32. cognite/neat/_rules/importers/_rdf/_inference2rules.py +40 -7
  33. cognite/neat/_rules/importers/_rdf/_owl2rules.py +80 -0
  34. cognite/neat/_rules/importers/_rdf/_shared.py +138 -441
  35. cognite/neat/_rules/models/_base_rules.py +19 -0
  36. cognite/neat/_rules/models/_types.py +5 -0
  37. cognite/neat/_rules/models/dms/__init__.py +2 -0
  38. cognite/neat/_rules/models/dms/_exporter.py +247 -123
  39. cognite/neat/_rules/models/dms/_rules.py +7 -49
  40. cognite/neat/_rules/models/dms/_rules_input.py +8 -3
  41. cognite/neat/_rules/models/dms/_validation.py +421 -123
  42. cognite/neat/_rules/models/entities/_multi_value.py +3 -0
  43. cognite/neat/_rules/models/information/__init__.py +2 -0
  44. cognite/neat/_rules/models/information/_rules.py +17 -61
  45. cognite/neat/_rules/models/information/_rules_input.py +11 -2
  46. cognite/neat/_rules/models/information/_validation.py +107 -11
  47. cognite/neat/_rules/models/mapping/_classic2core.py +1 -1
  48. cognite/neat/_rules/models/mapping/_classic2core.yaml +8 -4
  49. cognite/neat/_rules/transformers/__init__.py +2 -1
  50. cognite/neat/_rules/transformers/_converters.py +163 -61
  51. cognite/neat/_rules/transformers/_mapping.py +132 -2
  52. cognite/neat/_rules/transformers/_pipelines.py +1 -1
  53. cognite/neat/_rules/transformers/_verification.py +29 -4
  54. cognite/neat/_session/_base.py +46 -60
  55. cognite/neat/_session/_mapping.py +105 -5
  56. cognite/neat/_session/_prepare.py +49 -14
  57. cognite/neat/_session/_read.py +50 -4
  58. cognite/neat/_session/_set.py +1 -0
  59. cognite/neat/_session/_to.py +38 -12
  60. cognite/neat/_session/_wizard.py +5 -0
  61. cognite/neat/_session/engine/_interface.py +3 -2
  62. cognite/neat/_session/exceptions.py +4 -0
  63. cognite/neat/_store/_base.py +79 -19
  64. cognite/neat/_utils/collection_.py +22 -0
  65. cognite/neat/_utils/rdf_.py +30 -4
  66. cognite/neat/_version.py +2 -2
  67. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +3 -91
  68. cognite/neat/_workflows/steps/lib/current/rules_importer.py +2 -16
  69. cognite/neat/_workflows/steps/lib/current/rules_validator.py +3 -5
  70. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/METADATA +1 -1
  71. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/RECORD +74 -82
  72. cognite/neat/_rules/importers/_rdf/_imf2rules/__init__.py +0 -3
  73. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +0 -86
  74. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +0 -29
  75. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +0 -130
  76. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2rules.py +0 -154
  77. cognite/neat/_rules/importers/_rdf/_owl2rules/__init__.py +0 -3
  78. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +0 -58
  79. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +0 -65
  80. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +0 -59
  81. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2rules.py +0 -39
  82. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/LICENSE +0 -0
  83. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/WHEEL +0 -0
  84. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/entry_points.txt +0 -0
@@ -1,37 +1,51 @@
1
- from collections import defaultdict
2
- from typing import Any, ClassVar, cast
1
+ import warnings
2
+ from collections import Counter, defaultdict
3
+ from functools import lru_cache
4
+ from typing import ClassVar
3
5
 
4
6
  from cognite.client import data_modeling as dm
7
+ from cognite.client.data_classes.data_modeling import ContainerList, ViewId, ViewList
8
+ from cognite.client.data_classes.data_modeling.views import (
9
+ ReverseDirectRelation,
10
+ ReverseDirectRelationApply,
11
+ ViewProperty,
12
+ ViewPropertyApply,
13
+ )
5
14
 
15
+ from cognite.neat._client import NeatClient
16
+ from cognite.neat._client.data_classes.data_modeling import ViewApplyDict
6
17
  from cognite.neat._client.data_classes.schema import DMSSchema
7
- from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_LIMIT
8
- from cognite.neat._issues import IssueList, NeatError, NeatIssue, NeatIssueList
18
+ from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_LIMIT, DMS_VIEW_CONTAINER_SIZE_LIMIT
19
+ from cognite.neat._issues import IssueList, NeatError, NeatIssueList
9
20
  from cognite.neat._issues.errors import (
21
+ CDFMissingClientError,
10
22
  PropertyDefinitionDuplicatedError,
11
- ResourceNotDefinedError,
23
+ PropertyMappingDuplicatedError,
24
+ PropertyNotFoundError,
25
+ ResourceDuplicatedError,
26
+ ResourceNotFoundError,
27
+ ReversedConnectionNotFeasibleError,
12
28
  )
13
- from cognite.neat._issues.errors._properties import ReversedConnectionNotFeasibleError
14
29
  from cognite.neat._issues.warnings import (
15
30
  NotSupportedHasDataFilterLimitWarning,
16
31
  NotSupportedViewContainerLimitWarning,
17
32
  UndefinedViewWarning,
18
33
  )
19
34
  from cognite.neat._issues.warnings.user_modeling import (
35
+ ContainerPropertyLimitWarning,
36
+ DirectRelationMissingSourceWarning,
20
37
  NotNeatSupportedFilterWarning,
21
- ViewPropertyLimitWarning,
22
38
  )
23
- from cognite.neat._rules.analysis import DMSAnalysis
24
39
  from cognite.neat._rules.models.data_types import DataType
25
40
  from cognite.neat._rules.models.entities import ContainerEntity, RawFilter
26
41
  from cognite.neat._rules.models.entities._single_value import (
27
- ReverseConnectionEntity,
28
42
  ViewEntity,
29
43
  )
30
44
 
31
45
  from ._rules import DMSProperty, DMSRules
32
46
 
33
47
 
34
- class DMSPostValidation:
48
+ class DMSValidation:
35
49
  """This class does all the validation of the DMS rules that have dependencies between
36
50
  components."""
37
51
 
@@ -39,33 +53,160 @@ class DMSPostValidation:
39
53
  # For example, changing the filter is allowed, but changing the properties is not.
40
54
  changeable_view_attributes: ClassVar[set[str]] = {"filter"}
41
55
 
42
- def __init__(self, rules: DMSRules):
43
- self.rules = rules
44
- self.metadata = rules.metadata
45
- self.properties = rules.properties
46
- self.containers = rules.containers
47
- self.views = rules.views
48
- self.issue_list = IssueList()
49
- self.probe = DMSAnalysis(rules)
56
+ def __init__(self, rules: DMSRules, client: NeatClient | None = None) -> None:
57
+ self._rules = rules
58
+ self._client = client
59
+ self._metadata = rules.metadata
60
+ self._properties = rules.properties
61
+ self._containers = rules.containers
62
+ self._views = rules.views
63
+
64
+ def imported_views_and_containers_ids(
65
+ self, include_views_with_no_properties: bool = True
66
+ ) -> tuple[set[ViewEntity], set[ContainerEntity]]:
67
+ existing_views = {view.view for view in self._views}
68
+ imported_views: set[ViewEntity] = set()
69
+ for view in self._views:
70
+ for parent in view.implements or []:
71
+ if parent not in existing_views:
72
+ imported_views.add(parent)
73
+ existing_containers = {container.container for container in self._containers or []}
74
+ imported_containers: set[ContainerEntity] = set()
75
+ view_with_properties: set[ViewEntity] = set()
76
+ for prop in self._properties:
77
+ if prop.container and prop.container not in existing_containers:
78
+ imported_containers.add(prop.container)
79
+ if prop.view not in existing_views:
80
+ imported_views.add(prop.view)
81
+ view_with_properties.add(prop.view)
82
+
83
+ if include_views_with_no_properties:
84
+ extra_views = existing_views - view_with_properties
85
+ imported_views.update({view for view in extra_views})
86
+
87
+ return imported_views, imported_containers
50
88
 
51
89
  def validate(self) -> NeatIssueList:
52
- self._validate_raw_filter()
53
- self._consistent_container_properties()
54
- self._validate_value_type_existence()
55
- self._validate_reverse_connections()
90
+ imported_views, imported_containers = self.imported_views_and_containers_ids(
91
+ include_views_with_no_properties=False
92
+ )
93
+ if (imported_views or imported_containers) and self._client is None:
94
+ raise CDFMissingClientError(
95
+ f"{self._rules.metadata.as_data_model_id()} has imported views and/or container: "
96
+ f"{imported_views}, {imported_containers}."
97
+ )
98
+ referenced_views = ViewList([])
99
+ referenced_containers = ContainerList([])
100
+ if self._client:
101
+ referenced_views = self._client.loaders.views.retrieve(
102
+ list(imported_views), include_connected=True, include_ancestor=True
103
+ )
104
+ referenced_containers = self._client.loaders.containers.retrieve(
105
+ list(imported_containers), include_connected=True
106
+ )
107
+
108
+ # Setup data structures for validation
109
+ dms_schema = self._rules.as_schema()
110
+ ref_view_by_id = {view.as_id(): view for view in referenced_views}
111
+ ref_container_by_id = {container.as_id(): container for container in referenced_containers}
112
+ # All containers and views are the Containers/Views in the DMSRules + the referenced ones
113
+ all_containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container] = {
114
+ **dict(dms_schema.containers.items()),
115
+ **ref_container_by_id,
116
+ }
117
+ all_views_by_id: dict[dm.ViewId, dm.ViewApply | dm.View] = {**dict(dms_schema.views.items()), **ref_view_by_id}
118
+ properties_by_ids = self._as_properties_by_ids(dms_schema, ref_view_by_id)
119
+ view_properties_by_id = self._as_view_properties_by_id(properties_by_ids)
120
+ parents_view_ids_by_child_id = self._parent_view_ids_by_child_id(all_views_by_id)
56
121
 
57
- self._referenced_views_and_containers_are_existing_and_proper_size()
58
- dms_schema = self.rules.as_schema()
59
- self._validate_performance(dms_schema)
60
- return self.issue_list
122
+ issue_list = IssueList()
123
+ # Neat DMS classes Validation
124
+ # These are errors that can only happen due to the format of the Neat DMS classes
125
+ issue_list.extend(self._validate_raw_filter())
126
+ issue_list.extend(self._consistent_container_properties())
127
+ issue_list.extend(self._validate_value_type_existence())
128
+ issue_list.extend(
129
+ self._validate_property_referenced_views_and_containers_exists(all_views_by_id, all_containers_by_id)
130
+ )
61
131
 
62
- def _consistent_container_properties(self) -> None:
132
+ # SDK classes validation
133
+ issue_list.extend(self._containers_are_proper_size(dms_schema))
134
+ issue_list.extend(
135
+ self._validate_reverse_connections(properties_by_ids, all_containers_by_id, parents_view_ids_by_child_id)
136
+ )
137
+ issue_list.extend(self._validate_schema(dms_schema, all_views_by_id, all_containers_by_id))
138
+ issue_list.extend(self._validate_referenced_container_limits(dms_schema.views, view_properties_by_id))
139
+ return issue_list
140
+
141
+ @staticmethod
142
+ def _as_properties_by_ids(
143
+ dms_schema: DMSSchema, ref_view_by_id: dict[dm.ViewId, dm.View]
144
+ ) -> dict[tuple[ViewId, str], ViewPropertyApply | ViewProperty]:
145
+ # Priority DMS schema properties.
146
+ # No need to do long lookups in ref_views as these already contain all ancestor properties.
147
+ properties_by_id: dict[tuple[ViewId, str], ViewPropertyApply | ViewProperty] = {}
148
+ for view in dms_schema.views.values():
149
+ view_id = view.as_id()
150
+ for prop_id, prop in (view.properties or {}).items():
151
+ properties_by_id[(view_id, prop_id)] = prop
152
+ if view.implements:
153
+ to_check = view.implements.copy()
154
+ while to_check:
155
+ parent_id = to_check.pop()
156
+ if parent_id in dms_schema.views:
157
+ # Priority DMS Schema properties
158
+ parent_view = dms_schema.views[parent_id]
159
+ for prop_id, prop in (parent_view.properties or {}).items():
160
+ if (view_id, prop_id) not in properties_by_id:
161
+ properties_by_id[(view_id, prop_id)] = prop
162
+ to_check.extend(parent_view.implements or [])
163
+ elif parent_id in ref_view_by_id:
164
+ # SDK properties
165
+ parent_read_view = ref_view_by_id[parent_id]
166
+ for prop_id, read_prop in parent_read_view.properties.items():
167
+ if (view_id, prop_id) not in properties_by_id:
168
+ properties_by_id[(view_id, prop_id)] = read_prop
169
+ # Read format of views already includes all ancestor properties
170
+ # so no need to check further
171
+ else:
172
+ # Missing views are caught else where
173
+ continue
174
+
175
+ return properties_by_id
176
+
177
+ @staticmethod
178
+ def _as_view_properties_by_id(
179
+ properties_by_ids: dict[tuple[ViewId, str], ViewPropertyApply | ViewProperty],
180
+ ) -> dict[ViewId, list[tuple[str, ViewProperty | ViewPropertyApply]]]:
181
+ view_properties_by_id: dict[dm.ViewId, list[tuple[str, ViewProperty | ViewPropertyApply]]] = defaultdict(list)
182
+ for (view_id, prop_id), prop in properties_by_ids.items():
183
+ view_properties_by_id[view_id].append((prop_id, prop))
184
+ return view_properties_by_id
185
+
186
+ @staticmethod
187
+ def _parent_view_ids_by_child_id(
188
+ all_views_by_id: dict[dm.ViewId, dm.ViewApply | dm.View],
189
+ ) -> dict[ViewId, set[ViewId]]:
190
+ @lru_cache
191
+ def get_parents(child_view_id: ViewId) -> set[ViewId]:
192
+ child_view = all_views_by_id[child_view_id]
193
+ parents = set(child_view.implements or [])
194
+ for parent_id in child_view.implements or []:
195
+ parents.update(get_parents(parent_id))
196
+ return parents
197
+
198
+ parents_by_view: dict[dm.ViewId, set[dm.ViewId]] = {}
199
+ for view_id in all_views_by_id:
200
+ parents_by_view[view_id] = get_parents(view_id)
201
+ return parents_by_view
202
+
203
+ def _consistent_container_properties(self) -> IssueList:
63
204
  container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, DMSProperty]]] = defaultdict(list)
64
- for prop_no, prop in enumerate(self.properties):
205
+ for prop_no, prop in enumerate(self._properties):
65
206
  if prop.container and prop.container_property:
66
207
  container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
67
208
 
68
- errors: list[NeatError] = []
209
+ errors = IssueList()
69
210
  for (container, prop_name), properties in container_properties_by_id.items():
70
211
  if len(properties) == 1:
71
212
  continue
@@ -152,135 +293,292 @@ class DMSPostValidation:
152
293
  )
153
294
  )
154
295
 
155
- self.issue_list.extend(errors)
156
-
157
- def _referenced_views_and_containers_are_existing_and_proper_size(self) -> None:
158
- # TODO: Split this method and keep only validation that should be independent of
159
- # whether view and/or container exist in the pydantic model instance
160
- # other validation should be done through NeatSession.verify()
161
- defined_views = {view.view.as_id() for view in self.views}
296
+ return errors
162
297
 
163
- property_count_by_view: dict[dm.ViewId, int] = defaultdict(int)
164
- errors: list[NeatIssue] = []
165
- for prop_no, prop in enumerate(self.properties):
166
- view_id = prop.view.as_id()
167
- if view_id not in defined_views:
168
- errors.append(
169
- ResourceNotDefinedError(
170
- identifier=view_id,
171
- resource_type="view",
172
- location="Views Sheet",
173
- column_name="View",
174
- row_number=prop_no,
175
- sheet_name="Properties",
176
- ),
177
- )
178
- else:
179
- property_count_by_view[view_id] += 1
180
- for view_id, count in property_count_by_view.items():
298
+ @staticmethod
299
+ def _containers_are_proper_size(dms_schema: DMSSchema) -> IssueList:
300
+ errors = IssueList()
301
+ for container_id, container in dms_schema.containers.items():
302
+ count = len(container.properties or {})
181
303
  if count > DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
182
- errors.append(ViewPropertyLimitWarning(view_id, count))
304
+ errors.append(ContainerPropertyLimitWarning(container_id, count))
183
305
 
184
- self.issue_list.extend(errors)
306
+ return errors
185
307
 
186
- def _validate_performance(self, dms_schema: DMSSchema) -> None:
187
- for view_id, view in dms_schema.views.items():
188
- mapped_containers = dms_schema._get_mapped_container_from_view(view_id)
308
+ @staticmethod
309
+ def _validate_referenced_container_limits(
310
+ views: ViewApplyDict, view_properties_by_id: dict[dm.ViewId, list[tuple[str, ViewProperty | ViewPropertyApply]]]
311
+ ) -> IssueList:
312
+ issue_list = IssueList()
313
+ for view_id, view in views.items():
314
+ view_properties = view_properties_by_id.get(view_id, [])
315
+ mapped_containers = {
316
+ prop.container
317
+ for _, prop in view_properties
318
+ if isinstance(prop, dm.MappedPropertyApply | dm.MappedProperty)
319
+ }
189
320
 
190
- if mapped_containers and len(mapped_containers) > 10:
191
- self.issue_list.append(
321
+ if mapped_containers and len(mapped_containers) > DMS_VIEW_CONTAINER_SIZE_LIMIT:
322
+ issue_list.append(
192
323
  NotSupportedViewContainerLimitWarning(
193
324
  view_id,
194
325
  len(mapped_containers),
195
326
  )
196
327
  )
197
- if (
198
- view.filter
199
- and isinstance(view.filter, dm.filters.HasData)
200
- and len(view.filter.dump()["hasData"]) > 10
201
- ):
202
- self.issue_list.append(
203
- NotSupportedHasDataFilterLimitWarning(
204
- view_id,
205
- len(view.filter.dump()["hasData"]),
206
- )
328
+
329
+ if view.filter and isinstance(view.filter, dm.filters.HasData) and len(view.filter.dump()["hasData"]) > 10:
330
+ issue_list.append(
331
+ NotSupportedHasDataFilterLimitWarning(
332
+ view_id,
333
+ len(view.filter.dump()["hasData"]),
207
334
  )
335
+ )
336
+ return issue_list
208
337
 
209
- def _validate_raw_filter(self) -> None:
210
- for view in self.views:
338
+ def _validate_raw_filter(self) -> IssueList:
339
+ issue_list = IssueList()
340
+ for view in self._views:
211
341
  if view.filter_ and isinstance(view.filter_, RawFilter):
212
- self.issue_list.append(
342
+ issue_list.append(
213
343
  NotNeatSupportedFilterWarning(view.view.as_id()),
214
344
  )
345
+ return issue_list
215
346
 
216
- def _validate_value_type_existence(self) -> None:
217
- views = {prop_.view for prop_ in self.properties}.union({view_.view for view_ in self.views})
218
-
219
- for prop_ in self.properties:
347
+ def _validate_value_type_existence(self) -> IssueList:
348
+ views = {prop_.view for prop_ in self._properties}.union({view_.view for view_ in self._views})
349
+ issue_list = IssueList()
350
+ for prop_ in self._properties:
220
351
  if isinstance(prop_.value_type, ViewEntity) and prop_.value_type not in views:
221
- self.issue_list.append(
352
+ issue_list.append(
222
353
  UndefinedViewWarning(
223
354
  str(prop_.view),
224
355
  str(prop_.value_type),
225
356
  prop_.view_property,
226
357
  )
227
358
  )
359
+ return issue_list
228
360
 
229
- def _validate_reverse_connections(self) -> None:
230
- # do not check for reverse connections in Cognite models
231
- if self.metadata.as_data_model_id() in COGNITE_MODELS:
232
- return None
361
+ def _validate_property_referenced_views_and_containers_exists(
362
+ self,
363
+ view_by_id: dict[dm.ViewId, dm.ViewApply | dm.View],
364
+ containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container],
365
+ ) -> IssueList:
366
+ issue_list = IssueList()
367
+ for prop in self._properties:
368
+ if prop.container:
369
+ container_id = prop.container.as_id()
370
+ if container_id not in containers_by_id:
371
+ issue_list.append(
372
+ ResourceNotFoundError(
373
+ container_id,
374
+ "container",
375
+ prop.view,
376
+ "view",
377
+ )
378
+ )
379
+ elif (
380
+ prop.container_property and prop.container_property not in containers_by_id[container_id].properties
381
+ ):
382
+ issue_list.append(
383
+ PropertyNotFoundError(
384
+ prop.container,
385
+ "container property",
386
+ prop.container_property,
387
+ dm.PropertyId(prop.view.as_id(), prop.view_property),
388
+ "view property",
389
+ )
390
+ )
233
391
 
234
- properties_by_ids = {
235
- f"{prop_.view!s}.{prop_.view_property}": prop_
236
- for properties in self.probe.classes_with_properties(True, True).values()
237
- for prop_ in properties
238
- }
392
+ if prop.view.as_id() not in view_by_id:
393
+ issue_list.append(
394
+ ResourceNotFoundError(
395
+ prop.view,
396
+ "view",
397
+ prop.view_property,
398
+ "property",
399
+ )
400
+ )
239
401
 
240
- reversed_by_ids = {
241
- id_: prop_
242
- for id_, prop_ in properties_by_ids.items()
243
- if prop_.connection and isinstance(prop_.connection, ReverseConnectionEntity)
244
- }
402
+ return issue_list
403
+
404
+ def _validate_reverse_connections(
405
+ self,
406
+ view_property_by_property_id: dict[tuple[dm.ViewId, str], ViewPropertyApply | ViewProperty],
407
+ containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container],
408
+ parents_by_view: dict[dm.ViewId, set[dm.ViewId]],
409
+ ) -> IssueList:
410
+ issue_list = IssueList()
411
+ # do not check for reverse connections in Cognite models
412
+ if self._metadata.as_data_model_id() in COGNITE_MODELS:
413
+ return issue_list
245
414
 
246
- for id_, prop_ in reversed_by_ids.items():
247
- source_id = f"{prop_.value_type!s}." f"{cast(ReverseConnectionEntity, prop_.connection).property_}"
248
- if source_id not in properties_by_ids:
249
- self.issue_list.append(
415
+ for (view_id, prop_id), prop_ in view_property_by_property_id.items():
416
+ if not isinstance(prop_, ReverseDirectRelationApply | ReverseDirectRelation):
417
+ continue
418
+ target_id = prop_.through.source, prop_.through.property
419
+ if target_id not in view_property_by_property_id:
420
+ issue_list.append(
250
421
  ReversedConnectionNotFeasibleError(
251
- id_,
422
+ view_id,
252
423
  "reversed connection",
253
- prop_.view_property,
254
- str(prop_.view),
255
- str(prop_.value_type),
256
- cast(ReverseConnectionEntity, prop_.connection).property_,
424
+ prop_id,
425
+ f"The {prop_.through.source} {prop_.through.property} does not exist",
257
426
  )
258
427
  )
428
+ continue
429
+ if isinstance(target_id[0], dm.ContainerId):
430
+ # Todo: How to handle this case? Should not happen if you created the model with Neat
431
+ continue
259
432
 
260
- elif source_id in properties_by_ids and properties_by_ids[source_id].value_type != prop_.view:
261
- self.issue_list.append(
433
+ target_property = view_property_by_property_id[(target_id[0], target_id[1])]
434
+ # Validate that the target is a direct relation pointing to the view_id
435
+ is_direct_relation = False
436
+ if isinstance(target_property, dm.MappedProperty) and isinstance(target_property.type, dm.DirectRelation):
437
+ is_direct_relation = True
438
+ elif isinstance(target_property, dm.MappedPropertyApply):
439
+ container = containers_by_id[target_property.container]
440
+ if target_property.container_property_identifier in container.properties:
441
+ container_property = container.properties[target_property.container_property_identifier]
442
+ if isinstance(container_property.type, dm.DirectRelation):
443
+ is_direct_relation = True
444
+ if not is_direct_relation:
445
+ issue_list.append(
262
446
  ReversedConnectionNotFeasibleError(
263
- id_,
264
- "view property",
265
- prop_.view_property,
266
- str(prop_.view),
267
- str(prop_.value_type),
268
- cast(ReverseConnectionEntity, prop_.connection).property_,
447
+ view_id,
448
+ "reversed connection",
449
+ prop_id,
450
+ f"{prop_.through.source} {prop_.through.property} is not a direct relation",
269
451
  )
270
452
  )
271
-
272
- else:
273
453
  continue
454
+ if not (
455
+ isinstance(target_property, dm.MappedPropertyApply | dm.MappedProperty)
456
+ # The direct relation is pointing to the view_id or one of its parents
457
+ and (target_property.source == view_id or target_property.source in parents_by_view[view_id])
458
+ ):
459
+ issue_list.append(
460
+ ReversedConnectionNotFeasibleError(
461
+ view_id,
462
+ "reversed connection",
463
+ prop_id,
464
+ f"{prop_.through.source} {prop_.through.property} is not pointing to {view_id}",
465
+ )
466
+ )
467
+ return issue_list
274
468
 
275
469
  @staticmethod
276
- def _changed_attributes_and_properties(
277
- new_dumped: dict[str, Any], existing_dumped: dict[str, Any]
278
- ) -> tuple[list[str], list[str]]:
279
- """Helper method to find the changed attributes and properties between two containers or views."""
280
- new_attributes = {key: value for key, value in new_dumped.items() if key != "properties"}
281
- existing_attributes = {key: value for key, value in existing_dumped.items() if key != "properties"}
282
- changed_attributes = [key for key in new_attributes if new_attributes[key] != existing_attributes.get(key)]
283
- new_properties = new_dumped.get("properties", {})
284
- existing_properties = existing_dumped.get("properties", {})
285
- changed_properties = [prop for prop in new_properties if new_properties[prop] != existing_properties.get(prop)]
286
- return changed_attributes, changed_properties
470
+ def _validate_schema(
471
+ schema: DMSSchema,
472
+ view_by_id: dict[dm.ViewId, dm.ViewApply | dm.View],
473
+ containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container],
474
+ ) -> IssueList:
475
+ errors: set[NeatError] = set()
476
+ defined_spaces = schema.spaces.copy()
477
+
478
+ for container_id, container in schema.containers.items():
479
+ if container.space not in defined_spaces:
480
+ errors.add(ResourceNotFoundError(container.space, "space", container_id, "container"))
481
+ for constraint in container.constraints.values():
482
+ if isinstance(constraint, dm.RequiresConstraint) and constraint.require not in containers_by_id:
483
+ errors.add(ResourceNotFoundError(constraint.require, "container", container_id, "container"))
484
+
485
+ for view_id, view in schema.views.items():
486
+ if view.space not in defined_spaces:
487
+ errors.add(ResourceNotFoundError(view.space, "space", view_id, "view"))
488
+
489
+ for parent in view.implements or []:
490
+ if parent not in view_by_id:
491
+ errors.add(PropertyNotFoundError(parent, "view", "implements", view_id, "view"))
492
+
493
+ for prop_name, prop in (view.properties or {}).items():
494
+ if isinstance(prop, dm.MappedPropertyApply):
495
+ ref_container = containers_by_id.get(prop.container)
496
+ if ref_container is None:
497
+ errors.add(ResourceNotFoundError(prop.container, "container", view_id, "view"))
498
+ elif prop.container_property_identifier not in ref_container.properties:
499
+ errors.add(
500
+ PropertyNotFoundError(
501
+ prop.container,
502
+ "container",
503
+ prop.container_property_identifier,
504
+ view_id,
505
+ "view",
506
+ )
507
+ )
508
+ else:
509
+ container_property = ref_container.properties[prop.container_property_identifier]
510
+
511
+ if isinstance(container_property.type, dm.DirectRelation) and prop.source is None:
512
+ warnings.warn(
513
+ DirectRelationMissingSourceWarning(view_id, prop_name),
514
+ stacklevel=2,
515
+ )
516
+
517
+ if (
518
+ isinstance(prop, dm.EdgeConnectionApply | ReverseDirectRelationApply)
519
+ and prop.source not in view_by_id
520
+ ):
521
+ errors.add(PropertyNotFoundError(prop.source, "view", prop_name, view_id, "view"))
522
+
523
+ if (
524
+ isinstance(prop, dm.EdgeConnectionApply)
525
+ and prop.edge_source is not None
526
+ and prop.edge_source not in view_by_id
527
+ ):
528
+ errors.add(PropertyNotFoundError(prop.edge_source, "view", prop_name, view_id, "view"))
529
+
530
+ # This allows for multiple view properties to be mapped to the same container property,
531
+ # as long as they have different external_id, otherwise this will lead to raising
532
+ # error ContainerPropertyUsedMultipleTimesError
533
+ property_count = Counter(
534
+ (prop.container, prop.container_property_identifier, view_property_identifier)
535
+ for view_property_identifier, prop in (view.properties or {}).items()
536
+ if isinstance(prop, dm.MappedPropertyApply)
537
+ )
538
+
539
+ for (
540
+ container_id,
541
+ container_property_identifier,
542
+ _,
543
+ ), count in property_count.items():
544
+ if count > 1:
545
+ view_properties = [
546
+ prop_name
547
+ for prop_name, prop in (view.properties or {}).items()
548
+ if isinstance(prop, dm.MappedPropertyApply)
549
+ and (prop.container, prop.container_property_identifier)
550
+ == (container_id, container_property_identifier)
551
+ ]
552
+ errors.add(
553
+ PropertyMappingDuplicatedError(
554
+ container_id,
555
+ "container",
556
+ container_property_identifier,
557
+ frozenset({dm.PropertyId(view_id, prop_name) for prop_name in view_properties}),
558
+ "view property",
559
+ )
560
+ )
561
+
562
+ if schema.data_model:
563
+ model = schema.data_model
564
+ if model.space not in defined_spaces:
565
+ errors.add(ResourceNotFoundError(model.space, "space", model.as_id(), "data model"))
566
+
567
+ view_counts: dict[dm.ViewId, int] = defaultdict(int)
568
+ for view_id_or_class in model.views or []:
569
+ view_id = view_id_or_class if isinstance(view_id_or_class, dm.ViewId) else view_id_or_class.as_id()
570
+ if view_id not in view_by_id:
571
+ errors.add(ResourceNotFoundError(view_id, "view", model.as_id(), "data model"))
572
+ view_counts[view_id] += 1
573
+
574
+ for view_id, count in view_counts.items():
575
+ if count > 1:
576
+ errors.add(
577
+ ResourceDuplicatedError(
578
+ view_id,
579
+ "view",
580
+ repr(model.as_id()),
581
+ )
582
+ )
583
+
584
+ return IssueList(list(errors))
@@ -78,3 +78,6 @@ class MultiValueTypeInfo(BaseModel):
78
78
  def is_mixed_type(self) -> bool:
79
79
  """Will signalize to DMS converter to fall back to string"""
80
80
  return not self.is_multi_object_type() and not self.is_multi_data_type()
81
+
82
+ def __hash__(self) -> int:
83
+ return hash(str(self))
@@ -5,6 +5,7 @@ from ._rules_input import (
5
5
  InformationInputProperty,
6
6
  InformationInputRules,
7
7
  )
8
+ from ._validation import InformationValidation
8
9
 
9
10
  __all__ = [
10
11
  "InformationRules",
@@ -15,4 +16,5 @@ __all__ = [
15
16
  "InformationInputMetadata",
16
17
  "InformationInputClass",
17
18
  "InformationInputProperty",
19
+ "InformationValidation",
18
20
  ]