cognite-neat 0.99.1__py3-none-any.whl → 0.100.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/_api/data_modeling_loaders.py +403 -182
- cognite/neat/_client/data_classes/data_modeling.py +4 -0
- cognite/neat/_graph/extractors/_base.py +7 -0
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -13
- cognite/neat/_graph/loaders/_rdf2dms.py +50 -11
- cognite/neat/_graph/transformers/__init__.py +3 -3
- cognite/neat/_graph/transformers/_classic_cdf.py +120 -52
- cognite/neat/_issues/warnings/__init__.py +2 -0
- cognite/neat/_issues/warnings/_resources.py +15 -0
- cognite/neat/_rules/analysis/_base.py +15 -5
- cognite/neat/_rules/analysis/_dms.py +20 -0
- cognite/neat/_rules/analysis/_information.py +22 -0
- cognite/neat/_rules/exporters/_base.py +3 -5
- cognite/neat/_rules/exporters/_rules2dms.py +192 -200
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +22 -5
- cognite/neat/_rules/models/_base_rules.py +19 -0
- cognite/neat/_rules/models/_types.py +5 -0
- cognite/neat/_rules/models/dms/_exporter.py +215 -93
- cognite/neat/_rules/models/dms/_rules.py +4 -4
- cognite/neat/_rules/models/dms/_rules_input.py +8 -3
- cognite/neat/_rules/models/dms/_validation.py +42 -11
- cognite/neat/_rules/models/entities/_multi_value.py +3 -0
- cognite/neat/_rules/models/information/_rules.py +17 -2
- cognite/neat/_rules/models/information/_rules_input.py +11 -2
- cognite/neat/_rules/models/information/_validation.py +99 -3
- cognite/neat/_rules/models/mapping/_classic2core.yaml +1 -1
- cognite/neat/_rules/transformers/__init__.py +2 -1
- cognite/neat/_rules/transformers/_converters.py +163 -61
- cognite/neat/_rules/transformers/_mapping.py +132 -2
- cognite/neat/_session/_base.py +42 -31
- cognite/neat/_session/_mapping.py +105 -5
- cognite/neat/_session/_prepare.py +43 -9
- cognite/neat/_session/_read.py +50 -4
- cognite/neat/_session/_set.py +1 -0
- cognite/neat/_session/_to.py +36 -13
- cognite/neat/_session/_wizard.py +5 -0
- cognite/neat/_session/engine/_interface.py +3 -2
- cognite/neat/_store/_base.py +79 -19
- cognite/neat/_utils/collection_.py +22 -0
- cognite/neat/_utils/rdf_.py +24 -0
- cognite/neat/_version.py +2 -2
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +3 -3
- {cognite_neat-0.99.1.dist-info → cognite_neat-0.100.1.dist-info}/METADATA +1 -1
- {cognite_neat-0.99.1.dist-info → cognite_neat-0.100.1.dist-info}/RECORD +47 -47
- {cognite_neat-0.99.1.dist-info → cognite_neat-0.100.1.dist-info}/LICENSE +0 -0
- {cognite_neat-0.99.1.dist-info → cognite_neat-0.100.1.dist-info}/WHEEL +0 -0
- {cognite_neat-0.99.1.dist-info → cognite_neat-0.100.1.dist-info}/entry_points.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import warnings
|
|
2
2
|
from collections import defaultdict
|
|
3
3
|
from collections.abc import Collection, Hashable, Sequence
|
|
4
|
-
from typing import Any
|
|
4
|
+
from typing import Any, cast
|
|
5
5
|
|
|
6
6
|
from cognite.client.data_classes import data_modeling as dm
|
|
7
7
|
from cognite.client.data_classes.data_modeling.containers import BTreeIndex
|
|
@@ -21,7 +21,7 @@ from cognite.neat._client.data_classes.data_modeling import (
|
|
|
21
21
|
)
|
|
22
22
|
from cognite.neat._client.data_classes.schema import DMSSchema
|
|
23
23
|
from cognite.neat._constants import COGNITE_SPACES
|
|
24
|
-
from cognite.neat._issues.errors import NeatTypeError, ResourceNotFoundError
|
|
24
|
+
from cognite.neat._issues.errors import NeatTypeError, NeatValueError, ResourceNotFoundError
|
|
25
25
|
from cognite.neat._issues.warnings import NotSupportedWarning, PropertyNotFoundWarning
|
|
26
26
|
from cognite.neat._issues.warnings.user_modeling import (
|
|
27
27
|
EmptyContainerWarning,
|
|
@@ -140,13 +140,20 @@ class _DMSExporter:
|
|
|
140
140
|
if not (self.remove_cdf_spaces and dms_view.view.space in COGNITE_SPACES)
|
|
141
141
|
]
|
|
142
142
|
)
|
|
143
|
+
view_by_id = {dms_view.view: dms_view for dms_view in input_views}
|
|
144
|
+
|
|
145
|
+
edge_types_by_view_property_id = self._edge_types_by_view_property_id(
|
|
146
|
+
view_properties_with_ancestors_by_id, view_by_id
|
|
147
|
+
)
|
|
143
148
|
|
|
144
149
|
for view_id, view in views.items():
|
|
145
150
|
view.properties = {}
|
|
146
151
|
if not (view_properties := view_properties_by_id.get(view_id)):
|
|
147
152
|
continue
|
|
148
153
|
for prop in view_properties:
|
|
149
|
-
view_property = self._create_view_property(
|
|
154
|
+
view_property = self._create_view_property(
|
|
155
|
+
prop, view_properties_with_ancestors_by_id, edge_types_by_view_property_id
|
|
156
|
+
)
|
|
150
157
|
if view_property is not None:
|
|
151
158
|
view.properties[prop.view_property] = view_property
|
|
152
159
|
|
|
@@ -157,18 +164,112 @@ class _DMSExporter:
|
|
|
157
164
|
if isinstance(prop.connection, EdgeEntity) and prop.connection.edge_type is not None:
|
|
158
165
|
return prop.connection.edge_type.as_reference()
|
|
159
166
|
elif isinstance(prop.value_type, ViewEntity):
|
|
160
|
-
return cls.
|
|
167
|
+
return cls._default_edge_type_from_view_id(prop.view.as_id(), prop.view_property)
|
|
161
168
|
else:
|
|
162
169
|
raise NeatTypeError(f"Invalid valueType {prop.value_type!r}")
|
|
163
170
|
|
|
164
171
|
@staticmethod
|
|
165
|
-
def
|
|
172
|
+
def _default_edge_type_from_view_id(view_id: dm.ViewId, property_: str) -> dm.DirectRelationReference:
|
|
166
173
|
return dm.DirectRelationReference(
|
|
167
174
|
space=view_id.space,
|
|
168
175
|
# This is the same convention as used when converting GraphQL to DMS
|
|
169
176
|
external_id=f"{view_id.external_id}.{property_}",
|
|
170
177
|
)
|
|
171
178
|
|
|
179
|
+
@classmethod
|
|
180
|
+
def _edge_types_by_view_property_id(
|
|
181
|
+
cls,
|
|
182
|
+
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[DMSProperty]],
|
|
183
|
+
view_by_id: dict[ViewEntity, DMSView],
|
|
184
|
+
) -> dict[tuple[ViewEntity, str], dm.DirectRelationReference]:
|
|
185
|
+
edge_connection_property_by_view_property_id: dict[tuple[ViewEntity, str], DMSProperty] = {}
|
|
186
|
+
for properties in view_properties_with_ancestors_by_id.values():
|
|
187
|
+
for prop in properties:
|
|
188
|
+
if isinstance(prop.connection, EdgeEntity):
|
|
189
|
+
view_property_id = (prop.view, prop.view_property)
|
|
190
|
+
edge_connection_property_by_view_property_id[view_property_id] = prop
|
|
191
|
+
|
|
192
|
+
edge_types_by_view_property_id: dict[tuple[ViewEntity, str], dm.DirectRelationReference] = {}
|
|
193
|
+
|
|
194
|
+
outwards_type_by_view_value_type: dict[tuple[ViewEntity, ViewEntity], list[dm.DirectRelationReference]] = (
|
|
195
|
+
defaultdict(list)
|
|
196
|
+
)
|
|
197
|
+
# First set the edge types for outwards connections.
|
|
198
|
+
for (view_id, _), prop in edge_connection_property_by_view_property_id.items():
|
|
199
|
+
# We have already filtered out all non-EdgeEntity connections
|
|
200
|
+
connection = cast(EdgeEntity, prop.connection)
|
|
201
|
+
if connection.direction == "inwards":
|
|
202
|
+
continue
|
|
203
|
+
view = view_by_id[view_id]
|
|
204
|
+
|
|
205
|
+
edge_type = cls._get_edge_type_outwards_connection(
|
|
206
|
+
view, prop, view_by_id, edge_connection_property_by_view_property_id
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
edge_types_by_view_property_id[(prop.view, prop.view_property)] = edge_type
|
|
210
|
+
|
|
211
|
+
if isinstance(prop.value_type, ViewEntity):
|
|
212
|
+
outwards_type_by_view_value_type[(prop.value_type, prop.view)].append(edge_type)
|
|
213
|
+
|
|
214
|
+
# Then inwards connections = outwards connections
|
|
215
|
+
for (view_id, prop_id), prop in edge_connection_property_by_view_property_id.items():
|
|
216
|
+
# We have already filtered out all non-EdgeEntity connections
|
|
217
|
+
connection = cast(EdgeEntity, prop.connection)
|
|
218
|
+
|
|
219
|
+
if connection.direction == "inwards" and isinstance(prop.value_type, ViewEntity):
|
|
220
|
+
edge_type_candidates = outwards_type_by_view_value_type.get((prop.view, prop.value_type), [])
|
|
221
|
+
if len(edge_type_candidates) == 0:
|
|
222
|
+
# Warning in validation, should not have an inwards connection without an outwards connection
|
|
223
|
+
edge_type = cls._default_edge_type_from_view_id(prop.view.as_id(), prop_id)
|
|
224
|
+
elif len(edge_type_candidates) == 1:
|
|
225
|
+
edge_type = edge_type_candidates[0]
|
|
226
|
+
else:
|
|
227
|
+
raise NeatValueError(
|
|
228
|
+
f"Cannot infer edge type for {view_id}.{prop_id}, multiple candidates: {edge_type_candidates}."
|
|
229
|
+
"Please specify edge type explicitly, i.e., edge(type=<YOUR_TYPE>)."
|
|
230
|
+
)
|
|
231
|
+
view_property_id = (prop.view, prop.view_property)
|
|
232
|
+
edge_types_by_view_property_id[view_property_id] = edge_type
|
|
233
|
+
|
|
234
|
+
return edge_types_by_view_property_id
|
|
235
|
+
|
|
236
|
+
@classmethod
|
|
237
|
+
def _get_edge_type_outwards_connection(
|
|
238
|
+
cls,
|
|
239
|
+
view: DMSView,
|
|
240
|
+
prop: DMSProperty,
|
|
241
|
+
view_by_id: dict[ViewEntity, DMSView],
|
|
242
|
+
edge_connection_by_view_property_id: dict[tuple[ViewEntity, str], DMSProperty],
|
|
243
|
+
) -> dm.DirectRelationReference:
|
|
244
|
+
connection = cast(EdgeEntity, prop.connection)
|
|
245
|
+
if connection.edge_type is not None:
|
|
246
|
+
# Explicitly set edge type
|
|
247
|
+
return connection.edge_type.as_reference()
|
|
248
|
+
elif view.implements:
|
|
249
|
+
# Try to look for same property in parent views
|
|
250
|
+
candidates = []
|
|
251
|
+
for parent_id in view.implements:
|
|
252
|
+
if parent_view := view_by_id.get(parent_id):
|
|
253
|
+
parent_prop = edge_connection_by_view_property_id.get((parent_view.view, prop.view_property))
|
|
254
|
+
if parent_prop and isinstance(parent_prop.connection, EdgeEntity):
|
|
255
|
+
parent_edge_type = cls._get_edge_type_outwards_connection(
|
|
256
|
+
parent_view, parent_prop, view_by_id, edge_connection_by_view_property_id
|
|
257
|
+
)
|
|
258
|
+
candidates.append(parent_edge_type)
|
|
259
|
+
if len(candidates) == 0:
|
|
260
|
+
return cls._default_edge_type_from_view_id(prop.view.as_id(), prop.view_property)
|
|
261
|
+
elif len(candidates) == 1:
|
|
262
|
+
return candidates[0]
|
|
263
|
+
else:
|
|
264
|
+
raise NeatValueError(
|
|
265
|
+
f"Cannot infer edge type for {prop.view.as_id()!r}.{prop.view_property}, "
|
|
266
|
+
f"multiple candidates: {candidates}. "
|
|
267
|
+
"Please specify edge type explicitly, i.e., edge(type=<YOUR_TYPE>)."
|
|
268
|
+
)
|
|
269
|
+
else:
|
|
270
|
+
# No parent view, use the default
|
|
271
|
+
return cls._default_edge_type_from_view_id(prop.view.as_id(), prop.view_property)
|
|
272
|
+
|
|
172
273
|
def _create_containers(
|
|
173
274
|
self,
|
|
174
275
|
container_properties_by_id: dict[dm.ContainerId, list[DMSProperty]],
|
|
@@ -373,106 +474,127 @@ class _DMSExporter:
|
|
|
373
474
|
|
|
374
475
|
@classmethod
|
|
375
476
|
def _create_view_property(
|
|
376
|
-
cls,
|
|
477
|
+
cls,
|
|
478
|
+
prop: DMSProperty,
|
|
479
|
+
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[DMSProperty]],
|
|
480
|
+
edge_types_by_view_property_id: dict[tuple[ViewEntity, str], dm.DirectRelationReference],
|
|
377
481
|
) -> ViewPropertyApply | None:
|
|
378
482
|
if prop.container and prop.container_property:
|
|
379
|
-
|
|
380
|
-
extra_args: dict[str, Any] = {}
|
|
381
|
-
if prop.connection == "direct":
|
|
382
|
-
if isinstance(prop.value_type, ViewEntity):
|
|
383
|
-
extra_args["source"] = prop.value_type.as_id()
|
|
384
|
-
elif isinstance(prop.value_type, DMSUnknownEntity):
|
|
385
|
-
extra_args["source"] = None
|
|
386
|
-
else:
|
|
387
|
-
# Should have been validated.
|
|
388
|
-
raise ValueError(
|
|
389
|
-
"If this error occurs it is a bug in NEAT, please report"
|
|
390
|
-
f"Debug Info, Invalid valueType direct: {prop.model_dump_json()}"
|
|
391
|
-
)
|
|
392
|
-
elif prop.connection is not None:
|
|
393
|
-
# Should have been validated.
|
|
394
|
-
raise ValueError(
|
|
395
|
-
"If this error occurs it is a bug in NEAT, please report"
|
|
396
|
-
f"Debug Info, Invalid connection: {prop.model_dump_json()}"
|
|
397
|
-
)
|
|
398
|
-
return dm.MappedPropertyApply(
|
|
399
|
-
container=prop.container.as_id(),
|
|
400
|
-
container_property_identifier=container_prop_identifier,
|
|
401
|
-
name=prop.name,
|
|
402
|
-
description=prop.description,
|
|
403
|
-
**extra_args,
|
|
404
|
-
)
|
|
483
|
+
return cls._create_mapped_property(prop)
|
|
405
484
|
elif isinstance(prop.connection, EdgeEntity):
|
|
406
|
-
|
|
407
|
-
source_view_id = prop.value_type.as_id()
|
|
408
|
-
else:
|
|
409
|
-
# Should have been validated.
|
|
410
|
-
raise ValueError(
|
|
411
|
-
"If this error occurs it is a bug in NEAT, please report"
|
|
412
|
-
f"Debug Info, Invalid valueType edge: {prop.model_dump_json()}"
|
|
413
|
-
)
|
|
414
|
-
edge_source: dm.ViewId | None = None
|
|
415
|
-
if prop.connection.properties is not None:
|
|
416
|
-
edge_source = prop.connection.properties.as_id()
|
|
417
|
-
edge_cls: type[dm.EdgeConnectionApply] = dm.MultiEdgeConnectionApply
|
|
418
|
-
# If is_list is not set, we default to a MultiEdgeConnection
|
|
419
|
-
if prop.is_list is False:
|
|
420
|
-
edge_cls = SingleEdgeConnectionApply
|
|
421
|
-
|
|
422
|
-
return edge_cls(
|
|
423
|
-
type=cls._create_edge_type_from_prop(prop),
|
|
424
|
-
source=source_view_id,
|
|
425
|
-
direction=prop.connection.direction,
|
|
426
|
-
name=prop.name,
|
|
427
|
-
description=prop.description,
|
|
428
|
-
edge_source=edge_source,
|
|
429
|
-
)
|
|
485
|
+
return cls._create_edge_property(prop, edge_types_by_view_property_id)
|
|
430
486
|
elif isinstance(prop.connection, ReverseConnectionEntity):
|
|
431
|
-
|
|
487
|
+
return cls._create_reverse_direct_relation(prop, view_properties_with_ancestors_by_id)
|
|
488
|
+
elif prop.view and prop.view_property and prop.connection:
|
|
489
|
+
warnings.warn(
|
|
490
|
+
NotSupportedWarning(f"{prop.connection} in {prop.view.as_id()!r}.{prop.view_property}"), stacklevel=2
|
|
491
|
+
)
|
|
492
|
+
return None
|
|
493
|
+
|
|
494
|
+
@classmethod
|
|
495
|
+
def _create_mapped_property(cls, prop: DMSProperty) -> dm.MappedPropertyApply:
|
|
496
|
+
container = cast(ContainerEntity, prop.container)
|
|
497
|
+
container_prop_identifier = cast(str, prop.container_property)
|
|
498
|
+
extra_args: dict[str, Any] = {}
|
|
499
|
+
if prop.connection == "direct":
|
|
432
500
|
if isinstance(prop.value_type, ViewEntity):
|
|
433
|
-
|
|
501
|
+
extra_args["source"] = prop.value_type.as_id()
|
|
502
|
+
elif isinstance(prop.value_type, DMSUnknownEntity):
|
|
503
|
+
extra_args["source"] = None
|
|
434
504
|
else:
|
|
435
505
|
# Should have been validated.
|
|
436
506
|
raise ValueError(
|
|
437
507
|
"If this error occurs it is a bug in NEAT, please report"
|
|
438
|
-
f"Debug Info, Invalid valueType
|
|
508
|
+
f"Debug Info, Invalid valueType direct: {prop.model_dump_json()}"
|
|
439
509
|
)
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
),
|
|
446
|
-
None,
|
|
510
|
+
elif prop.connection is not None:
|
|
511
|
+
# Should have been validated.
|
|
512
|
+
raise ValueError(
|
|
513
|
+
"If this error occurs it is a bug in NEAT, please report"
|
|
514
|
+
f"Debug Info, Invalid connection: {prop.model_dump_json()}"
|
|
447
515
|
)
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
"view property",
|
|
456
|
-
),
|
|
457
|
-
stacklevel=2,
|
|
458
|
-
)
|
|
516
|
+
return dm.MappedPropertyApply(
|
|
517
|
+
container=container.as_id(),
|
|
518
|
+
container_property_identifier=container_prop_identifier,
|
|
519
|
+
name=prop.name,
|
|
520
|
+
description=prop.description,
|
|
521
|
+
**extra_args,
|
|
522
|
+
)
|
|
459
523
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
524
|
+
@classmethod
|
|
525
|
+
def _create_edge_property(
|
|
526
|
+
cls, prop: DMSProperty, edge_types_by_view_property_id: dict[tuple[ViewEntity, str], dm.DirectRelationReference]
|
|
527
|
+
) -> dm.EdgeConnectionApply:
|
|
528
|
+
connection = cast(EdgeEntity, prop.connection)
|
|
529
|
+
if isinstance(prop.value_type, ViewEntity):
|
|
530
|
+
source_view_id = prop.value_type.as_id()
|
|
531
|
+
else:
|
|
532
|
+
# Should have been validated.
|
|
533
|
+
raise ValueError(
|
|
534
|
+
"If this error occurs it is a bug in NEAT, please report"
|
|
535
|
+
f"Debug Info, Invalid valueType edge: {prop.model_dump_json()}"
|
|
536
|
+
)
|
|
537
|
+
edge_source: dm.ViewId | None = None
|
|
538
|
+
if connection.properties is not None:
|
|
539
|
+
edge_source = connection.properties.as_id()
|
|
540
|
+
edge_cls: type[dm.EdgeConnectionApply] = dm.MultiEdgeConnectionApply
|
|
541
|
+
# If is_list is not set, we default to a MultiEdgeConnection
|
|
542
|
+
if prop.is_list is False:
|
|
543
|
+
edge_cls = SingleEdgeConnectionApply
|
|
544
|
+
|
|
545
|
+
return edge_cls(
|
|
546
|
+
type=edge_types_by_view_property_id[(prop.view, prop.view_property)],
|
|
547
|
+
source=source_view_id,
|
|
548
|
+
direction=connection.direction,
|
|
549
|
+
name=prop.name,
|
|
550
|
+
description=prop.description,
|
|
551
|
+
edge_source=edge_source,
|
|
552
|
+
)
|
|
473
553
|
|
|
474
|
-
|
|
554
|
+
@classmethod
|
|
555
|
+
def _create_reverse_direct_relation(
|
|
556
|
+
cls, prop: DMSProperty, view_properties_with_ancestors_by_id: dict[dm.ViewId, list[DMSProperty]]
|
|
557
|
+
) -> dm.MultiReverseDirectRelationApply | SingleReverseDirectRelationApply | None:
|
|
558
|
+
connection = cast(ReverseConnectionEntity, prop.connection)
|
|
559
|
+
reverse_prop_id = connection.property_
|
|
560
|
+
if isinstance(prop.value_type, ViewEntity):
|
|
561
|
+
source_view_id = prop.value_type.as_id()
|
|
562
|
+
else:
|
|
563
|
+
# Should have been validated.
|
|
564
|
+
raise ValueError(
|
|
565
|
+
"If this error occurs it is a bug in NEAT, please report"
|
|
566
|
+
f"Debug Info, Invalid valueType reverse connection: {prop.model_dump_json()}"
|
|
567
|
+
)
|
|
568
|
+
reverse_prop = next(
|
|
569
|
+
(
|
|
570
|
+
prop
|
|
571
|
+
for prop in view_properties_with_ancestors_by_id.get(source_view_id, [])
|
|
572
|
+
if prop.view_property == reverse_prop_id
|
|
573
|
+
),
|
|
574
|
+
None,
|
|
575
|
+
)
|
|
576
|
+
if reverse_prop is None:
|
|
475
577
|
warnings.warn(
|
|
476
|
-
|
|
578
|
+
PropertyNotFoundWarning(
|
|
579
|
+
source_view_id,
|
|
580
|
+
"view",
|
|
581
|
+
reverse_prop_id or "MISSING",
|
|
582
|
+
dm.PropertyId(prop.view.as_id(), prop.view_property),
|
|
583
|
+
"view property",
|
|
584
|
+
),
|
|
585
|
+
stacklevel=2,
|
|
477
586
|
)
|
|
478
|
-
|
|
587
|
+
|
|
588
|
+
if reverse_prop and reverse_prop.connection == "direct":
|
|
589
|
+
args: dict[str, Any] = dict(
|
|
590
|
+
source=source_view_id,
|
|
591
|
+
through=dm.PropertyId(source=source_view_id, property=reverse_prop_id),
|
|
592
|
+
name=prop.name,
|
|
593
|
+
description=prop.description,
|
|
594
|
+
)
|
|
595
|
+
if prop.is_list in [True, None]:
|
|
596
|
+
return dm.MultiReverseDirectRelationApply(**args)
|
|
597
|
+
else:
|
|
598
|
+
return SingleReverseDirectRelationApply(**args)
|
|
599
|
+
else:
|
|
600
|
+
return None
|
|
@@ -6,7 +6,6 @@ import pandas as pd
|
|
|
6
6
|
from cognite.client import data_modeling as dm
|
|
7
7
|
from pydantic import Field, field_serializer, field_validator
|
|
8
8
|
from pydantic_core.core_schema import SerializationInfo, ValidationInfo
|
|
9
|
-
from rdflib import URIRef
|
|
10
9
|
|
|
11
10
|
from cognite.neat._client.data_classes.schema import DMSSchema
|
|
12
11
|
from cognite.neat._constants import COGNITE_SPACES
|
|
@@ -30,6 +29,7 @@ from cognite.neat._rules.models._types import (
|
|
|
30
29
|
ContainerEntityType,
|
|
31
30
|
DmsPropertyType,
|
|
32
31
|
StrListType,
|
|
32
|
+
URIRefType,
|
|
33
33
|
ViewEntityType,
|
|
34
34
|
)
|
|
35
35
|
from cognite.neat._rules.models.data_types import DataType
|
|
@@ -54,7 +54,7 @@ _DEFAULT_VERSION = "1"
|
|
|
54
54
|
class DMSMetadata(BaseMetadata):
|
|
55
55
|
role: ClassVar[RoleTypes] = RoleTypes.dms
|
|
56
56
|
aspect: ClassVar[DataModelAspect] = DataModelAspect.physical
|
|
57
|
-
logical:
|
|
57
|
+
logical: URIRefType | None = None
|
|
58
58
|
|
|
59
59
|
def as_space(self) -> dm.SpaceApply:
|
|
60
60
|
return dm.SpaceApply(
|
|
@@ -108,7 +108,7 @@ class DMSProperty(SheetRow):
|
|
|
108
108
|
container_property: DmsPropertyType | None = Field(None, alias="Container Property")
|
|
109
109
|
index: StrListType | None = Field(None, alias="Index")
|
|
110
110
|
constraint: StrListType | None = Field(None, alias="Constraint")
|
|
111
|
-
logical:
|
|
111
|
+
logical: URIRefType | None = Field(
|
|
112
112
|
None,
|
|
113
113
|
alias="Logical",
|
|
114
114
|
description="Used to make connection between physical and logical data model aspect",
|
|
@@ -246,7 +246,7 @@ class DMSView(SheetRow):
|
|
|
246
246
|
implements: ViewEntityList | None = Field(None, alias="Implements")
|
|
247
247
|
filter_: HasDataFilter | NodeTypeFilter | RawFilter | None = Field(None, alias="Filter")
|
|
248
248
|
in_model: bool = Field(True, alias="In Model")
|
|
249
|
-
logical:
|
|
249
|
+
logical: URIRefType | None = Field(
|
|
250
250
|
None,
|
|
251
251
|
alias="Logical",
|
|
252
252
|
description="Used to make connection between physical and logical data model aspect",
|
|
@@ -35,7 +35,7 @@ class DMSInputMetadata(InputComponent[DMSMetadata]):
|
|
|
35
35
|
description: str | None = None
|
|
36
36
|
created: datetime | str | None = None
|
|
37
37
|
updated: datetime | str | None = None
|
|
38
|
-
logical: str | None = None
|
|
38
|
+
logical: str | URIRef | None = None
|
|
39
39
|
|
|
40
40
|
@classmethod
|
|
41
41
|
def _get_verified_cls(cls) -> type[DMSMetadata]:
|
|
@@ -108,7 +108,8 @@ class DMSInputProperty(InputComponent[DMSProperty]):
|
|
|
108
108
|
container_property: str | None = None
|
|
109
109
|
index: str | list[str] | None = None
|
|
110
110
|
constraint: str | list[str] | None = None
|
|
111
|
-
|
|
111
|
+
neatId: str | URIRef | None = None
|
|
112
|
+
logical: str | URIRef | None = None
|
|
112
113
|
|
|
113
114
|
@classmethod
|
|
114
115
|
def _get_verified_cls(cls) -> type[DMSProperty]:
|
|
@@ -139,6 +140,7 @@ class DMSInputContainer(InputComponent[DMSContainer]):
|
|
|
139
140
|
name: str | None = None
|
|
140
141
|
description: str | None = None
|
|
141
142
|
constraint: str | None = None
|
|
143
|
+
neatId: str | URIRef | None = None
|
|
142
144
|
used_for: Literal["node", "edge", "all"] | None = None
|
|
143
145
|
|
|
144
146
|
@classmethod
|
|
@@ -183,7 +185,8 @@ class DMSInputView(InputComponent[DMSView]):
|
|
|
183
185
|
implements: str | None = None
|
|
184
186
|
filter_: Literal["hasData", "nodeType", "rawFilter"] | str | None = None
|
|
185
187
|
in_model: bool = True
|
|
186
|
-
|
|
188
|
+
neatId: str | URIRef | None = None
|
|
189
|
+
logical: str | URIRef | None = None
|
|
187
190
|
|
|
188
191
|
@classmethod
|
|
189
192
|
def _get_verified_cls(cls) -> type[DMSView]:
|
|
@@ -231,6 +234,7 @@ class DMSInputNode(InputComponent[DMSNode]):
|
|
|
231
234
|
usage: Literal["type", "collocation"]
|
|
232
235
|
name: str | None = None
|
|
233
236
|
description: str | None = None
|
|
237
|
+
neatId: str | URIRef | None = None
|
|
234
238
|
|
|
235
239
|
@classmethod
|
|
236
240
|
def _get_verified_cls(cls) -> type[DMSNode]:
|
|
@@ -252,6 +256,7 @@ class DMSInputEnum(InputComponent[DMSEnum]):
|
|
|
252
256
|
value: str
|
|
253
257
|
name: str | None = None
|
|
254
258
|
description: str | None = None
|
|
259
|
+
neatId: str | URIRef | None = None
|
|
255
260
|
|
|
256
261
|
@classmethod
|
|
257
262
|
def _get_verified_cls(cls) -> type[DMSEnum]:
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import warnings
|
|
2
2
|
from collections import Counter, defaultdict
|
|
3
|
+
from functools import lru_cache
|
|
3
4
|
from typing import ClassVar
|
|
4
5
|
|
|
5
6
|
from cognite.client import data_modeling as dm
|
|
@@ -108,15 +109,15 @@ class DMSValidation:
|
|
|
108
109
|
dms_schema = self._rules.as_schema()
|
|
109
110
|
ref_view_by_id = {view.as_id(): view for view in referenced_views}
|
|
110
111
|
ref_container_by_id = {container.as_id(): container for container in referenced_containers}
|
|
112
|
+
# All containers and views are the Containers/Views in the DMSRules + the referenced ones
|
|
111
113
|
all_containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container] = {
|
|
112
114
|
**dict(dms_schema.containers.items()),
|
|
113
115
|
**ref_container_by_id,
|
|
114
116
|
}
|
|
115
117
|
all_views_by_id: dict[dm.ViewId, dm.ViewApply | dm.View] = {**dict(dms_schema.views.items()), **ref_view_by_id}
|
|
116
118
|
properties_by_ids = self._as_properties_by_ids(dms_schema, ref_view_by_id)
|
|
117
|
-
view_properties_by_id
|
|
118
|
-
|
|
119
|
-
view_properties_by_id[view_id].append((prop_id, prop))
|
|
119
|
+
view_properties_by_id = self._as_view_properties_by_id(properties_by_ids)
|
|
120
|
+
parents_view_ids_by_child_id = self._parent_view_ids_by_child_id(all_views_by_id)
|
|
120
121
|
|
|
121
122
|
issue_list = IssueList()
|
|
122
123
|
# Neat DMS classes Validation
|
|
@@ -130,7 +131,9 @@ class DMSValidation:
|
|
|
130
131
|
|
|
131
132
|
# SDK classes validation
|
|
132
133
|
issue_list.extend(self._containers_are_proper_size(dms_schema))
|
|
133
|
-
issue_list.extend(
|
|
134
|
+
issue_list.extend(
|
|
135
|
+
self._validate_reverse_connections(properties_by_ids, all_containers_by_id, parents_view_ids_by_child_id)
|
|
136
|
+
)
|
|
134
137
|
issue_list.extend(self._validate_schema(dms_schema, all_views_by_id, all_containers_by_id))
|
|
135
138
|
issue_list.extend(self._validate_referenced_container_limits(dms_schema.views, view_properties_by_id))
|
|
136
139
|
return issue_list
|
|
@@ -171,6 +174,32 @@ class DMSValidation:
|
|
|
171
174
|
|
|
172
175
|
return properties_by_id
|
|
173
176
|
|
|
177
|
+
@staticmethod
|
|
178
|
+
def _as_view_properties_by_id(
|
|
179
|
+
properties_by_ids: dict[tuple[ViewId, str], ViewPropertyApply | ViewProperty],
|
|
180
|
+
) -> dict[ViewId, list[tuple[str, ViewProperty | ViewPropertyApply]]]:
|
|
181
|
+
view_properties_by_id: dict[dm.ViewId, list[tuple[str, ViewProperty | ViewPropertyApply]]] = defaultdict(list)
|
|
182
|
+
for (view_id, prop_id), prop in properties_by_ids.items():
|
|
183
|
+
view_properties_by_id[view_id].append((prop_id, prop))
|
|
184
|
+
return view_properties_by_id
|
|
185
|
+
|
|
186
|
+
@staticmethod
|
|
187
|
+
def _parent_view_ids_by_child_id(
|
|
188
|
+
all_views_by_id: dict[dm.ViewId, dm.ViewApply | dm.View],
|
|
189
|
+
) -> dict[ViewId, set[ViewId]]:
|
|
190
|
+
@lru_cache
|
|
191
|
+
def get_parents(child_view_id: ViewId) -> set[ViewId]:
|
|
192
|
+
child_view = all_views_by_id[child_view_id]
|
|
193
|
+
parents = set(child_view.implements or [])
|
|
194
|
+
for parent_id in child_view.implements or []:
|
|
195
|
+
parents.update(get_parents(parent_id))
|
|
196
|
+
return parents
|
|
197
|
+
|
|
198
|
+
parents_by_view: dict[dm.ViewId, set[dm.ViewId]] = {}
|
|
199
|
+
for view_id in all_views_by_id:
|
|
200
|
+
parents_by_view[view_id] = get_parents(view_id)
|
|
201
|
+
return parents_by_view
|
|
202
|
+
|
|
174
203
|
def _consistent_container_properties(self) -> IssueList:
|
|
175
204
|
container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, DMSProperty]]] = defaultdict(list)
|
|
176
205
|
for prop_no, prop in enumerate(self._properties):
|
|
@@ -374,19 +403,20 @@ class DMSValidation:
|
|
|
374
403
|
|
|
375
404
|
def _validate_reverse_connections(
|
|
376
405
|
self,
|
|
377
|
-
|
|
406
|
+
view_property_by_property_id: dict[tuple[dm.ViewId, str], ViewPropertyApply | ViewProperty],
|
|
378
407
|
containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container],
|
|
408
|
+
parents_by_view: dict[dm.ViewId, set[dm.ViewId]],
|
|
379
409
|
) -> IssueList:
|
|
380
410
|
issue_list = IssueList()
|
|
381
411
|
# do not check for reverse connections in Cognite models
|
|
382
412
|
if self._metadata.as_data_model_id() in COGNITE_MODELS:
|
|
383
413
|
return issue_list
|
|
384
414
|
|
|
385
|
-
for (view_id, prop_id), prop_ in
|
|
415
|
+
for (view_id, prop_id), prop_ in view_property_by_property_id.items():
|
|
386
416
|
if not isinstance(prop_, ReverseDirectRelationApply | ReverseDirectRelation):
|
|
387
417
|
continue
|
|
388
|
-
|
|
389
|
-
if
|
|
418
|
+
target_id = prop_.through.source, prop_.through.property
|
|
419
|
+
if target_id not in view_property_by_property_id:
|
|
390
420
|
issue_list.append(
|
|
391
421
|
ReversedConnectionNotFeasibleError(
|
|
392
422
|
view_id,
|
|
@@ -396,11 +426,11 @@ class DMSValidation:
|
|
|
396
426
|
)
|
|
397
427
|
)
|
|
398
428
|
continue
|
|
399
|
-
if isinstance(
|
|
429
|
+
if isinstance(target_id[0], dm.ContainerId):
|
|
400
430
|
# Todo: How to handle this case? Should not happen if you created the model with Neat
|
|
401
431
|
continue
|
|
402
432
|
|
|
403
|
-
target_property =
|
|
433
|
+
target_property = view_property_by_property_id[(target_id[0], target_id[1])]
|
|
404
434
|
# Validate that the target is a direct relation pointing to the view_id
|
|
405
435
|
is_direct_relation = False
|
|
406
436
|
if isinstance(target_property, dm.MappedProperty) and isinstance(target_property.type, dm.DirectRelation):
|
|
@@ -423,7 +453,8 @@ class DMSValidation:
|
|
|
423
453
|
continue
|
|
424
454
|
if not (
|
|
425
455
|
isinstance(target_property, dm.MappedPropertyApply | dm.MappedProperty)
|
|
426
|
-
|
|
456
|
+
# The direct relation is pointing to the view_id or one of its parents
|
|
457
|
+
and (target_property.source == view_id or target_property.source in parents_by_view[view_id])
|
|
427
458
|
):
|
|
428
459
|
issue_list.append(
|
|
429
460
|
ReversedConnectionNotFeasibleError(
|
|
@@ -78,3 +78,6 @@ class MultiValueTypeInfo(BaseModel):
|
|
|
78
78
|
def is_mixed_type(self) -> bool:
|
|
79
79
|
"""Will signalize to DMS converter to fall back to string"""
|
|
80
80
|
return not self.is_multi_object_type() and not self.is_multi_data_type()
|
|
81
|
+
|
|
82
|
+
def __hash__(self) -> int:
|
|
83
|
+
return hash(str(self))
|
|
@@ -27,7 +27,10 @@ from cognite.neat._rules.models._types import (
|
|
|
27
27
|
ClassEntityType,
|
|
28
28
|
InformationPropertyType,
|
|
29
29
|
MultiValueTypeType,
|
|
30
|
+
URIRefType,
|
|
30
31
|
)
|
|
32
|
+
|
|
33
|
+
# NeatIdType,
|
|
31
34
|
from cognite.neat._rules.models.data_types import DataType
|
|
32
35
|
from cognite.neat._rules.models.entities import (
|
|
33
36
|
ClassEntity,
|
|
@@ -45,8 +48,8 @@ class InformationMetadata(BaseMetadata):
|
|
|
45
48
|
aspect: ClassVar[DataModelAspect] = DataModelAspect.logical
|
|
46
49
|
|
|
47
50
|
# Linking to Conceptual and Physical data model aspects
|
|
48
|
-
physical: URIRef | None = Field(None, description="Link to the
|
|
49
|
-
conceptual: URIRef | None = Field(None, description="Link to the
|
|
51
|
+
physical: URIRef | str | None = Field(None, description="Link to the physical data model aspect")
|
|
52
|
+
conceptual: URIRef | str | None = Field(None, description="Link to the conceptual data model aspect")
|
|
50
53
|
|
|
51
54
|
|
|
52
55
|
def _get_metadata(context: Any) -> InformationMetadata | None:
|
|
@@ -70,6 +73,12 @@ class InformationClass(SheetRow):
|
|
|
70
73
|
description: str | None = Field(alias="Description", default=None)
|
|
71
74
|
implements: ClassEntityList | None = Field(alias="Implements", default=None)
|
|
72
75
|
|
|
76
|
+
physical: URIRefType | None = Field(
|
|
77
|
+
None,
|
|
78
|
+
description="Link to the class representation in the physical data model aspect",
|
|
79
|
+
)
|
|
80
|
+
conceptual: URIRefType | None = Field(None, description="Link to the conceptual data model aspect")
|
|
81
|
+
|
|
73
82
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
74
83
|
return (self.class_,)
|
|
75
84
|
|
|
@@ -128,6 +137,12 @@ class InformationProperty(SheetRow):
|
|
|
128
137
|
description="Flag to indicate if the property is inherited, only use for internal purposes",
|
|
129
138
|
)
|
|
130
139
|
|
|
140
|
+
physical: URIRefType | None = Field(
|
|
141
|
+
None,
|
|
142
|
+
description="Link to the class representation in the physical data model aspect",
|
|
143
|
+
)
|
|
144
|
+
conceptual: URIRefType | None = Field(None, description="Link to the conceptual data model aspect")
|
|
145
|
+
|
|
131
146
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
132
147
|
return self.class_, self.property_
|
|
133
148
|
|