cognite-neat 0.97.2__py3-none-any.whl → 0.98.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_graph/loaders/__init__.py +1 -2
- cognite/neat/_graph/queries/_base.py +25 -4
- cognite/neat/_issues/warnings/_models.py +9 -0
- cognite/neat/_rules/_shared.py +3 -8
- cognite/neat/_rules/analysis/__init__.py +1 -2
- cognite/neat/_rules/analysis/_base.py +2 -23
- cognite/neat/_rules/analysis/_dms.py +4 -10
- cognite/neat/_rules/analysis/_information.py +2 -10
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_rules2excel.py +15 -72
- cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/_rules/importers/_base.py +3 -4
- cognite/neat/_rules/importers/_dms2rules.py +17 -40
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
- cognite/neat/_rules/importers/_rdf/_base.py +17 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +30 -18
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
- cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
- cognite/neat/_rules/models/__init__.py +2 -16
- cognite/neat/_rules/models/_base_rules.py +98 -52
- cognite/neat/_rules/models/dms/_exporter.py +7 -160
- cognite/neat/_rules/models/dms/_rules.py +18 -126
- cognite/neat/_rules/models/dms/_rules_input.py +20 -48
- cognite/neat/_rules/models/dms/_schema.py +11 -0
- cognite/neat/_rules/models/dms/_validation.py +9 -122
- cognite/neat/_rules/models/information/_rules.py +19 -114
- cognite/neat/_rules/models/information/_rules_input.py +32 -41
- cognite/neat/_rules/models/information/_validation.py +34 -102
- cognite/neat/_rules/transformers/__init__.py +1 -4
- cognite/neat/_rules/transformers/_converters.py +18 -195
- cognite/neat/_rules/transformers/_mapping.py +1 -5
- cognite/neat/_rules/transformers/_verification.py +0 -14
- cognite/neat/_session/_base.py +37 -13
- cognite/neat/_session/_collector.py +126 -0
- cognite/neat/_session/_inspect.py +5 -5
- cognite/neat/_session/_prepare.py +37 -11
- cognite/neat/_session/_read.py +62 -9
- cognite/neat/_session/_set.py +2 -2
- cognite/neat/_session/_show.py +11 -11
- cognite/neat/_session/_to.py +24 -11
- cognite/neat/_session/exceptions.py +20 -3
- cognite/neat/_store/_provenance.py +2 -2
- cognite/neat/_utils/auxiliary.py +19 -0
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/data_contracts.py +2 -10
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +6 -46
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +2 -7
- {cognite_neat-0.97.2.dist-info → cognite_neat-0.98.0.dist-info}/METADATA +2 -1
- {cognite_neat-0.97.2.dist-info → cognite_neat-0.98.0.dist-info}/RECORD +59 -65
- cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
- cognite/neat/_rules/analysis/_asset.py +0 -173
- cognite/neat/_rules/models/asset/__init__.py +0 -13
- cognite/neat/_rules/models/asset/_rules.py +0 -109
- cognite/neat/_rules/models/asset/_rules_input.py +0 -101
- cognite/neat/_rules/models/asset/_validation.py +0 -45
- cognite/neat/_rules/models/domain.py +0 -136
- {cognite_neat-0.97.2.dist-info → cognite_neat-0.98.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.97.2.dist-info → cognite_neat-0.98.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.97.2.dist-info → cognite_neat-0.98.0.dist-info}/entry_points.txt +0 -0
|
@@ -8,7 +8,6 @@ from typing import Literal, TypeVar, cast
|
|
|
8
8
|
|
|
9
9
|
from cognite.client.data_classes import data_modeling as dms
|
|
10
10
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
|
|
11
|
-
from rdflib import Namespace
|
|
12
11
|
|
|
13
12
|
from cognite.neat._constants import (
|
|
14
13
|
COGNITE_MODELS,
|
|
@@ -20,8 +19,6 @@ from cognite.neat._issues.warnings._models import (
|
|
|
20
19
|
EnterpriseModelNotBuildOnTopOfCDMWarning,
|
|
21
20
|
SolutionModelBuildOnTopOfCDMWarning,
|
|
22
21
|
)
|
|
23
|
-
from cognite.neat._issues.warnings.user_modeling import ParentInDifferentSpaceWarning
|
|
24
|
-
from cognite.neat._rules._constants import EntityTypes
|
|
25
22
|
from cognite.neat._rules._shared import (
|
|
26
23
|
InputRules,
|
|
27
24
|
JustRules,
|
|
@@ -31,13 +28,9 @@ from cognite.neat._rules._shared import (
|
|
|
31
28
|
)
|
|
32
29
|
from cognite.neat._rules.analysis import DMSAnalysis
|
|
33
30
|
from cognite.neat._rules.models import (
|
|
34
|
-
AssetRules,
|
|
35
31
|
DMSInputRules,
|
|
36
32
|
DMSRules,
|
|
37
|
-
DomainRules,
|
|
38
|
-
ExtensionCategory,
|
|
39
33
|
InformationRules,
|
|
40
|
-
SchemaCompleteness,
|
|
41
34
|
SheetList,
|
|
42
35
|
data_types,
|
|
43
36
|
)
|
|
@@ -45,15 +38,11 @@ from cognite.neat._rules.models.data_types import DataType, String
|
|
|
45
38
|
from cognite.neat._rules.models.dms import DMSMetadata, DMSProperty, DMSView
|
|
46
39
|
from cognite.neat._rules.models.dms._rules import DMSContainer
|
|
47
40
|
from cognite.neat._rules.models.entities import (
|
|
48
|
-
AssetEntity,
|
|
49
|
-
AssetFields,
|
|
50
41
|
ClassEntity,
|
|
51
42
|
ContainerEntity,
|
|
52
43
|
DMSUnknownEntity,
|
|
53
44
|
EdgeEntity,
|
|
54
45
|
MultiValueTypeInfo,
|
|
55
|
-
ReferenceEntity,
|
|
56
|
-
RelationshipEntity,
|
|
57
46
|
ReverseConnectionEntity,
|
|
58
47
|
UnknownEntity,
|
|
59
48
|
ViewEntity,
|
|
@@ -97,10 +86,8 @@ class ToCompliantEntities(RulesTransformer[InformationInputRules, InformationInp
|
|
|
97
86
|
return ReadRules(self._transform(self._to_rules(rules)), IssueList(), {})
|
|
98
87
|
|
|
99
88
|
def _transform(self, rules: InformationInputRules) -> InformationInputRules:
|
|
100
|
-
rules.metadata.prefix = self._fix_entity(rules.metadata.prefix)
|
|
101
89
|
rules.classes = self._fix_classes(rules.classes)
|
|
102
90
|
rules.properties = self._fix_properties(rules.properties)
|
|
103
|
-
|
|
104
91
|
rules.metadata.version += "_dms_compliant"
|
|
105
92
|
|
|
106
93
|
return rules
|
|
@@ -199,20 +186,6 @@ class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
|
|
|
199
186
|
return _InformationRulesConverter(rules).as_dms_rules(self.ignore_undefined_value_types)
|
|
200
187
|
|
|
201
188
|
|
|
202
|
-
class InformationToAsset(ConversionTransformer[InformationRules, AssetRules]):
|
|
203
|
-
"""Converts InformationRules to AssetRules."""
|
|
204
|
-
|
|
205
|
-
def _transform(self, rules: InformationRules) -> AssetRules:
|
|
206
|
-
return _InformationRulesConverter(rules).as_asset_architect_rules()
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
class AssetToInformation(ConversionTransformer[AssetRules, InformationRules]):
|
|
210
|
-
"""Converts AssetRules to InformationRules."""
|
|
211
|
-
|
|
212
|
-
def _transform(self, rules: AssetRules) -> InformationRules:
|
|
213
|
-
return InformationRules.model_validate(rules.model_dump())
|
|
214
|
-
|
|
215
|
-
|
|
216
189
|
class DMSToInformation(ConversionTransformer[DMSRules, InformationRules]):
|
|
217
190
|
"""Converts DMSRules to InformationRules."""
|
|
218
191
|
|
|
@@ -231,16 +204,8 @@ class ConvertToRules(ConversionTransformer[VerifiedRules, VerifiedRules]):
|
|
|
231
204
|
return rules
|
|
232
205
|
if isinstance(rules, InformationRules) and self._out_cls is DMSRules:
|
|
233
206
|
return InformationToDMS().transform(rules).rules
|
|
234
|
-
if isinstance(rules, InformationRules) and self._out_cls is AssetRules:
|
|
235
|
-
return InformationToAsset().transform(rules).rules
|
|
236
|
-
if isinstance(rules, AssetRules) and self._out_cls is InformationRules:
|
|
237
|
-
return AssetToInformation().transform(rules).rules
|
|
238
|
-
if isinstance(rules, AssetRules) and self._out_cls is DMSRules:
|
|
239
|
-
return InformationToDMS().transform(AssetToInformation().transform(rules)).rules
|
|
240
207
|
if isinstance(rules, DMSRules) and self._out_cls is InformationRules:
|
|
241
208
|
return DMSToInformation().transform(rules).rules
|
|
242
|
-
if isinstance(rules, DMSRules) and self._out_cls is AssetRules:
|
|
243
|
-
return InformationToAsset().transform(DMSToInformation().transform(rules)).rules
|
|
244
209
|
raise ValueError(f"Unsupported conversion from {type(rules)} to {self._out_cls}")
|
|
245
210
|
|
|
246
211
|
|
|
@@ -322,8 +287,6 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
322
287
|
dump = reference_rules.dump()
|
|
323
288
|
|
|
324
289
|
# Prepare new model metadata prior validation
|
|
325
|
-
dump["metadata"]["schema_"] = SchemaCompleteness.partial.value
|
|
326
|
-
dump["metadata"]["data_model_type"] = self.type_
|
|
327
290
|
dump["metadata"]["name"] = f"{self.org_name} {self.type_} data model"
|
|
328
291
|
dump["metadata"]["space"] = self.new_model_id.space
|
|
329
292
|
dump["metadata"]["external_id"] = self.new_model_id.external_id
|
|
@@ -365,12 +328,10 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
365
328
|
# Remove Cognite affix in view external_id / suffix.
|
|
366
329
|
for prop in solution_model.properties:
|
|
367
330
|
prop.view = self._remove_cognite_affix(prop.view)
|
|
368
|
-
prop.class_ = self._remove_cognite_affix(prop.class_)
|
|
369
331
|
if isinstance(prop.value_type, ViewEntity):
|
|
370
332
|
prop.value_type = self._remove_cognite_affix(prop.value_type)
|
|
371
333
|
for view in solution_model.views:
|
|
372
334
|
view.view = self._remove_cognite_affix(view.view)
|
|
373
|
-
view.class_ = self._remove_cognite_affix(view.class_)
|
|
374
335
|
|
|
375
336
|
if self.mode == "write":
|
|
376
337
|
_, new_containers, new_properties = self._get_new_components(solution_model)
|
|
@@ -385,9 +346,6 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
385
346
|
def _to_enterprise(self, reference_model: DMSRules) -> JustRules[DMSRules]:
|
|
386
347
|
dump = reference_model.dump()
|
|
387
348
|
|
|
388
|
-
# This is must prior model validation to avoid validation issues
|
|
389
|
-
dump["metadata"]["schema_"] = SchemaCompleteness.partial.value
|
|
390
|
-
|
|
391
349
|
# This will create reference model components in the enterprise model space
|
|
392
350
|
enterprise_model = DMSRules.model_validate(DMSInputRules.load(dump).dump())
|
|
393
351
|
|
|
@@ -398,9 +356,6 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
398
356
|
enterprise_model.metadata.external_id = self.new_model_id.external_id
|
|
399
357
|
enterprise_model.metadata.version = cast(str, self.new_model_id.version)
|
|
400
358
|
|
|
401
|
-
if reference_model.metadata.as_data_model_id() in COGNITE_MODELS:
|
|
402
|
-
enterprise_model.reference = reference_model
|
|
403
|
-
|
|
404
359
|
# Here we are creating enterprise specific components
|
|
405
360
|
enterprise_views, enterprise_containers, enterprise_properties = self._get_new_components(enterprise_model)
|
|
406
361
|
|
|
@@ -444,19 +399,16 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
444
399
|
view_entity.version = cast(str, self.new_model_id.version)
|
|
445
400
|
view_entity.prefix = self.new_model_id.space
|
|
446
401
|
container_entity = ContainerEntity(space=view_entity.prefix, externalId=view_entity.external_id)
|
|
447
|
-
class_entity = ClassEntity(prefix=view_entity.prefix, suffix=view_entity.suffix)
|
|
448
402
|
|
|
449
403
|
view = DMSView(
|
|
450
404
|
view=view_entity,
|
|
451
405
|
implements=[definition.view],
|
|
452
406
|
in_model=True,
|
|
453
|
-
class_=class_entity,
|
|
454
407
|
name=definition.name,
|
|
455
408
|
)
|
|
456
409
|
|
|
457
410
|
container = DMSContainer(
|
|
458
411
|
container=container_entity,
|
|
459
|
-
class_=class_entity,
|
|
460
412
|
)
|
|
461
413
|
|
|
462
414
|
property_ = DMSProperty(
|
|
@@ -468,8 +420,6 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
468
420
|
is_list=False,
|
|
469
421
|
container=container_entity,
|
|
470
422
|
container_property=f"{to_camel(view_entity.suffix)}{self.dummy_property}",
|
|
471
|
-
class_=class_entity,
|
|
472
|
-
property_=f"{to_camel(view_entity.suffix)}{self.dummy_property}",
|
|
473
423
|
)
|
|
474
424
|
|
|
475
425
|
new_properties.append(property_)
|
|
@@ -569,12 +519,14 @@ class ReduceCogniteModel(RulesTransformer[DMSRules, DMSRules]):
|
|
|
569
519
|
[view for view in new_model.views if view.view.as_id() not in exclude_views]
|
|
570
520
|
)
|
|
571
521
|
new_properties = SheetList[DMSProperty]()
|
|
522
|
+
|
|
572
523
|
for view in new_model.views:
|
|
573
524
|
for prop in properties_by_view[view.view]:
|
|
574
525
|
if self._is_asset_3D_property(prop):
|
|
575
526
|
# We filter out the 3D property of asset
|
|
576
527
|
continue
|
|
577
528
|
new_properties.append(prop)
|
|
529
|
+
|
|
578
530
|
new_model.properties = new_properties
|
|
579
531
|
|
|
580
532
|
return JustRules(new_model)
|
|
@@ -582,51 +534,14 @@ class ReduceCogniteModel(RulesTransformer[DMSRules, DMSRules]):
|
|
|
582
534
|
def _is_asset_3D_property(self, prop: DMSProperty) -> bool:
|
|
583
535
|
if "3D" not in self.drop_collection:
|
|
584
536
|
return False
|
|
585
|
-
return prop.view.as_id() == self._ASSET_VIEW and prop.
|
|
537
|
+
return prop.view.as_id() == self._ASSET_VIEW and prop.view_property == "object3D"
|
|
586
538
|
|
|
587
539
|
|
|
588
540
|
class _InformationRulesConverter:
|
|
589
541
|
def __init__(self, information: InformationRules):
|
|
590
542
|
self.rules = information
|
|
591
|
-
self.is_addition = (
|
|
592
|
-
self.rules.metadata.schema_ is SchemaCompleteness.extended
|
|
593
|
-
and self.rules.metadata.extension is ExtensionCategory.addition
|
|
594
|
-
)
|
|
595
|
-
self.is_reshape = (
|
|
596
|
-
self.rules.metadata.schema_ is SchemaCompleteness.extended
|
|
597
|
-
and self.rules.metadata.extension is ExtensionCategory.reshape
|
|
598
|
-
)
|
|
599
|
-
if self.rules.last:
|
|
600
|
-
self.last_classes = {class_.class_: class_ for class_ in self.rules.last.classes}
|
|
601
|
-
else:
|
|
602
|
-
self.last_classes = {}
|
|
603
543
|
self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
|
|
604
544
|
|
|
605
|
-
def as_domain_rules(self) -> DomainRules:
|
|
606
|
-
raise NotImplementedError("DomainRules not implemented yet")
|
|
607
|
-
|
|
608
|
-
def as_asset_architect_rules(self) -> "AssetRules":
|
|
609
|
-
from cognite.neat._rules.models.asset._rules import AssetClass, AssetMetadata, AssetProperty, AssetRules
|
|
610
|
-
|
|
611
|
-
classes: SheetList[AssetClass] = SheetList[AssetClass](
|
|
612
|
-
[AssetClass(**class_.model_dump()) for class_ in self.rules.classes]
|
|
613
|
-
)
|
|
614
|
-
properties: SheetList[AssetProperty] = SheetList[AssetProperty]()
|
|
615
|
-
for prop_ in self.rules.properties:
|
|
616
|
-
if prop_.type_ == EntityTypes.data_property:
|
|
617
|
-
properties.append(
|
|
618
|
-
AssetProperty(**prop_.model_dump(), implementation=[AssetEntity(property=AssetFields.metadata)])
|
|
619
|
-
)
|
|
620
|
-
elif prop_.type_ == EntityTypes.object_property:
|
|
621
|
-
properties.append(AssetProperty(**prop_.model_dump(), implementation=[RelationshipEntity()]))
|
|
622
|
-
|
|
623
|
-
return AssetRules(
|
|
624
|
-
metadata=AssetMetadata(**self.rules.metadata.model_dump()),
|
|
625
|
-
properties=properties,
|
|
626
|
-
classes=classes,
|
|
627
|
-
prefixes=self.rules.prefixes,
|
|
628
|
-
)
|
|
629
|
-
|
|
630
545
|
def as_dms_rules(self, ignore_undefined_value_types: bool = False) -> "DMSRules":
|
|
631
546
|
from cognite.neat._rules.models.dms._rules import (
|
|
632
547
|
DMSContainer,
|
|
@@ -652,31 +567,17 @@ class _InformationRulesConverter:
|
|
|
652
567
|
|
|
653
568
|
views: list[DMSView] = [
|
|
654
569
|
DMSView(
|
|
655
|
-
class_=cls_.class_,
|
|
656
570
|
name=cls_.name,
|
|
657
571
|
view=cls_.class_.as_view_entity(default_space, default_version),
|
|
658
572
|
description=cls_.description,
|
|
659
|
-
reference=cls_.reference,
|
|
660
573
|
implements=self._get_view_implements(cls_, info_metadata),
|
|
661
574
|
)
|
|
662
575
|
for cls_ in self.rules.classes
|
|
663
576
|
]
|
|
664
577
|
|
|
665
|
-
last_dms_rules = _InformationRulesConverter(self.rules.last).as_dms_rules() if self.rules.last else None
|
|
666
|
-
ref_dms_rules = (
|
|
667
|
-
_InformationRulesConverter(self.rules.reference).as_dms_rules() if self.rules.reference else None
|
|
668
|
-
)
|
|
669
|
-
|
|
670
578
|
class_by_entity = {cls_.class_: cls_ for cls_ in self.rules.classes}
|
|
671
|
-
if self.rules.last:
|
|
672
|
-
for cls_ in self.rules.last.classes:
|
|
673
|
-
if cls_.class_ not in class_by_entity:
|
|
674
|
-
class_by_entity[cls_.class_] = cls_
|
|
675
579
|
|
|
676
580
|
existing_containers: set[ContainerEntity] = set()
|
|
677
|
-
for rule_set in [last_dms_rules, ref_dms_rules]:
|
|
678
|
-
if rule_set:
|
|
679
|
-
existing_containers.update({c.container for c in rule_set.containers or []})
|
|
680
581
|
|
|
681
582
|
containers: list[DMSContainer] = []
|
|
682
583
|
for container_entity, class_entities in referenced_containers.items():
|
|
@@ -688,7 +589,6 @@ class _InformationRulesConverter:
|
|
|
688
589
|
most_used_class_entity = class_entities.most_common(1)[0][0]
|
|
689
590
|
class_ = class_by_entity[most_used_class_entity]
|
|
690
591
|
container = DMSContainer(
|
|
691
|
-
class_=class_.class_,
|
|
692
592
|
container=container_entity,
|
|
693
593
|
name=class_.name,
|
|
694
594
|
description=class_.description,
|
|
@@ -701,8 +601,6 @@ class _InformationRulesConverter:
|
|
|
701
601
|
properties=SheetList[DMSProperty]([prop for prop_set in properties_by_class.values() for prop in prop_set]),
|
|
702
602
|
views=SheetList[DMSView](views),
|
|
703
603
|
containers=SheetList[DMSContainer](containers),
|
|
704
|
-
last=last_dms_rules,
|
|
705
|
-
reference=ref_dms_rules,
|
|
706
604
|
)
|
|
707
605
|
|
|
708
606
|
@staticmethod
|
|
@@ -715,7 +613,7 @@ class _InformationRulesConverter:
|
|
|
715
613
|
constrains: list[ContainerEntity] = []
|
|
716
614
|
for entity in class_entities:
|
|
717
615
|
class_ = class_by_entity[entity]
|
|
718
|
-
for parent in class_.
|
|
616
|
+
for parent in class_.implements or []:
|
|
719
617
|
parent_entity = parent.as_container_entity(default_space)
|
|
720
618
|
if parent_entity in referenced_containers:
|
|
721
619
|
constrains.append(parent_entity)
|
|
@@ -727,14 +625,10 @@ class _InformationRulesConverter:
|
|
|
727
625
|
DMSMetadata,
|
|
728
626
|
)
|
|
729
627
|
|
|
730
|
-
space = cls._to_space(metadata.prefix)
|
|
731
|
-
|
|
732
628
|
return DMSMetadata(
|
|
733
|
-
|
|
734
|
-
space=space,
|
|
735
|
-
data_model_type=metadata.data_model_type,
|
|
629
|
+
space=metadata.space,
|
|
736
630
|
version=metadata.version,
|
|
737
|
-
external_id=metadata.
|
|
631
|
+
external_id=metadata.external_id,
|
|
738
632
|
creator=metadata.creator,
|
|
739
633
|
name=metadata.name,
|
|
740
634
|
created=metadata.created,
|
|
@@ -798,15 +692,12 @@ class _InformationRulesConverter:
|
|
|
798
692
|
container, container_property = self._get_container(prop, default_space)
|
|
799
693
|
|
|
800
694
|
return DMSProperty(
|
|
801
|
-
class_=prop.class_,
|
|
802
695
|
name=prop.name,
|
|
803
|
-
property_=prop.property_,
|
|
804
696
|
value_type=value_type,
|
|
805
697
|
nullable=nullable,
|
|
806
698
|
is_list=is_list,
|
|
807
699
|
connection=connection,
|
|
808
700
|
default=prop.default,
|
|
809
|
-
reference=prop.reference,
|
|
810
701
|
container=container,
|
|
811
702
|
container_property=container_property,
|
|
812
703
|
view=prop.class_.as_view_entity(default_space, default_version),
|
|
@@ -825,18 +716,7 @@ class _InformationRulesConverter:
|
|
|
825
716
|
return prefix
|
|
826
717
|
|
|
827
718
|
def _get_container(self, prop: InformationProperty, default_space: str) -> tuple[ContainerEntity, str]:
|
|
828
|
-
|
|
829
|
-
return (
|
|
830
|
-
prop.reference.as_container_entity(default_space),
|
|
831
|
-
prop.reference.property_ or prop.property_,
|
|
832
|
-
)
|
|
833
|
-
elif (self.is_addition or self.is_reshape) and prop.class_ in self.last_classes:
|
|
834
|
-
# We need to create a new container for the property, as we cannot change
|
|
835
|
-
# the existing container in the last schema
|
|
836
|
-
container_entity = prop.class_.as_container_entity(default_space)
|
|
837
|
-
container_entity.suffix = self._bump_suffix(container_entity.suffix)
|
|
838
|
-
else:
|
|
839
|
-
container_entity = prop.class_.as_container_entity(default_space)
|
|
719
|
+
container_entity = prop.class_.as_container_entity(default_space)
|
|
840
720
|
|
|
841
721
|
while self.property_count_by_container[container_entity] >= DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
|
|
842
722
|
container_entity.suffix = self._bump_suffix(container_entity.suffix)
|
|
@@ -845,27 +725,9 @@ class _InformationRulesConverter:
|
|
|
845
725
|
return container_entity, prop.property_
|
|
846
726
|
|
|
847
727
|
def _get_view_implements(self, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
implements = [
|
|
852
|
-
cls_.reference.as_view_entity(
|
|
853
|
-
self.rules.reference.metadata.prefix, self.rules.reference.metadata.version
|
|
854
|
-
)
|
|
855
|
-
]
|
|
856
|
-
else:
|
|
857
|
-
implements = [
|
|
858
|
-
cls_.reference.as_view_entity(metadata.prefix, metadata.version),
|
|
859
|
-
]
|
|
860
|
-
else:
|
|
861
|
-
implements = []
|
|
862
|
-
for parent in cls_.parent or []:
|
|
863
|
-
if self.rules.reference and parent.prefix == self.rules.reference.metadata.prefix:
|
|
864
|
-
view_entity = parent.as_view_entity(
|
|
865
|
-
self.rules.reference.metadata.prefix, self.rules.reference.metadata.version
|
|
866
|
-
)
|
|
867
|
-
else:
|
|
868
|
-
view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
|
|
728
|
+
implements = []
|
|
729
|
+
for parent in cls_.implements or []:
|
|
730
|
+
view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
|
|
869
731
|
implements.append(view_entity)
|
|
870
732
|
return implements
|
|
871
733
|
|
|
@@ -906,9 +768,6 @@ class _DMSRulesConverter:
|
|
|
906
768
|
def __init__(self, dms: DMSRules):
|
|
907
769
|
self.dms = dms
|
|
908
770
|
|
|
909
|
-
def as_domain_rules(self) -> "DomainRules":
|
|
910
|
-
raise NotImplementedError("DomainRules not implemented yet")
|
|
911
|
-
|
|
912
771
|
def as_information_rules(
|
|
913
772
|
self,
|
|
914
773
|
) -> "InformationRules":
|
|
@@ -925,16 +784,13 @@ class _DMSRulesConverter:
|
|
|
925
784
|
classes = [
|
|
926
785
|
InformationClass(
|
|
927
786
|
# we do not want a version in class as we use URI for the class
|
|
928
|
-
class_=ClassEntity(prefix=view.
|
|
787
|
+
class_=ClassEntity(prefix=view.view.prefix, suffix=view.view.suffix),
|
|
929
788
|
description=view.description,
|
|
930
|
-
|
|
789
|
+
implements=[
|
|
931
790
|
# we do not want a version in class as we use URI for the class
|
|
932
791
|
implemented_view.as_class(skip_version=True)
|
|
933
|
-
# We only want parents in the same namespace, parent in a different namespace is a reference
|
|
934
792
|
for implemented_view in view.implements or []
|
|
935
|
-
if implemented_view.prefix == view.class_.prefix
|
|
936
793
|
],
|
|
937
|
-
reference=self._get_class_reference(view),
|
|
938
794
|
)
|
|
939
795
|
for view in self.dms.views
|
|
940
796
|
]
|
|
@@ -957,13 +813,12 @@ class _DMSRulesConverter:
|
|
|
957
813
|
properties.append(
|
|
958
814
|
InformationProperty(
|
|
959
815
|
# Removing version
|
|
960
|
-
class_=ClassEntity(suffix=property_.
|
|
816
|
+
class_=ClassEntity(suffix=property_.view.suffix, prefix=property_.view.prefix),
|
|
961
817
|
property_=property_.view_property,
|
|
962
818
|
value_type=value_type,
|
|
963
819
|
description=property_.description,
|
|
964
|
-
min_count=0 if property_.nullable or property_.nullable is None else 1,
|
|
965
|
-
max_count=float("inf") if property_.is_list or property_.nullable is None else 1,
|
|
966
|
-
reference=self._get_property_reference(property_),
|
|
820
|
+
min_count=(0 if property_.nullable or property_.nullable is None else 1),
|
|
821
|
+
max_count=(float("inf") if property_.is_list or property_.nullable is None else 1),
|
|
967
822
|
)
|
|
968
823
|
)
|
|
969
824
|
|
|
@@ -971,51 +826,19 @@ class _DMSRulesConverter:
|
|
|
971
826
|
metadata=metadata,
|
|
972
827
|
properties=SheetList[InformationProperty](properties),
|
|
973
828
|
classes=SheetList[InformationClass](classes),
|
|
974
|
-
last=_DMSRulesConverter(self.dms.last).as_information_rules() if self.dms.last else None,
|
|
975
|
-
reference=_DMSRulesConverter(self.dms.reference).as_information_rules() if self.dms.reference else None,
|
|
976
829
|
)
|
|
977
830
|
|
|
978
831
|
@classmethod
|
|
979
832
|
def _convert_metadata_to_info(cls, metadata: DMSMetadata) -> "InformationMetadata":
|
|
980
833
|
from cognite.neat._rules.models.information._rules import InformationMetadata
|
|
981
834
|
|
|
982
|
-
prefix = metadata.space
|
|
983
835
|
return InformationMetadata(
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
extension=metadata.extension,
|
|
987
|
-
prefix=prefix,
|
|
988
|
-
namespace=Namespace(f"https://purl.orgl/neat/{prefix}/"),
|
|
836
|
+
space=metadata.space,
|
|
837
|
+
external_id=metadata.external_id,
|
|
989
838
|
version=metadata.version,
|
|
990
839
|
description=metadata.description,
|
|
991
|
-
name=metadata.name
|
|
840
|
+
name=metadata.name,
|
|
992
841
|
creator=metadata.creator,
|
|
993
842
|
created=metadata.created,
|
|
994
843
|
updated=metadata.updated,
|
|
995
844
|
)
|
|
996
|
-
|
|
997
|
-
@classmethod
|
|
998
|
-
def _get_class_reference(cls, view: DMSView) -> ReferenceEntity | None:
|
|
999
|
-
parents_other_namespace = [parent for parent in view.implements or [] if parent.prefix != view.class_.prefix]
|
|
1000
|
-
if len(parents_other_namespace) == 0:
|
|
1001
|
-
return None
|
|
1002
|
-
if len(parents_other_namespace) > 1:
|
|
1003
|
-
warnings.warn(
|
|
1004
|
-
ParentInDifferentSpaceWarning(view.view.as_id()),
|
|
1005
|
-
stacklevel=2,
|
|
1006
|
-
)
|
|
1007
|
-
other_parent = parents_other_namespace[0]
|
|
1008
|
-
|
|
1009
|
-
return ReferenceEntity(prefix=other_parent.prefix, suffix=other_parent.suffix)
|
|
1010
|
-
|
|
1011
|
-
@classmethod
|
|
1012
|
-
def _get_property_reference(cls, property_: DMSProperty) -> ReferenceEntity | None:
|
|
1013
|
-
has_container_other_namespace = property_.container and property_.container.prefix != property_.class_.prefix
|
|
1014
|
-
if not has_container_other_namespace:
|
|
1015
|
-
return None
|
|
1016
|
-
container = cast(ContainerEntity, property_.container)
|
|
1017
|
-
return ReferenceEntity(
|
|
1018
|
-
prefix=container.prefix,
|
|
1019
|
-
suffix=container.suffix,
|
|
1020
|
-
property=property_.container_property,
|
|
1021
|
-
)
|
|
@@ -5,7 +5,7 @@ from cognite.neat._rules._shared import JustRules, OutRules
|
|
|
5
5
|
from cognite.neat._rules.models import DMSRules, InformationRules
|
|
6
6
|
from cognite.neat._rules.models._base_rules import ClassRef
|
|
7
7
|
from cognite.neat._rules.models.dms import DMSProperty
|
|
8
|
-
from cognite.neat._rules.models.entities import ClassEntity
|
|
8
|
+
from cognite.neat._rules.models.entities import ClassEntity
|
|
9
9
|
from cognite.neat._rules.models.information import InformationClass
|
|
10
10
|
from cognite.neat._rules.models.mapping import RuleMapping
|
|
11
11
|
|
|
@@ -51,9 +51,6 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
51
51
|
|
|
52
52
|
def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
|
|
53
53
|
solution: DMSRules = self._to_rules(rules)
|
|
54
|
-
if solution.reference is not None:
|
|
55
|
-
raise ValueError("Reference already exists")
|
|
56
|
-
solution.reference = self.reference
|
|
57
54
|
view_by_external_id = {view.view.external_id: view for view in solution.views}
|
|
58
55
|
ref_view_by_external_id = {view.view.external_id: view for view in self.reference.views}
|
|
59
56
|
|
|
@@ -95,7 +92,6 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
95
92
|
if ref_prop.container and ref_prop.container_property:
|
|
96
93
|
prop.container = ref_prop.container
|
|
97
94
|
prop.container_property = ref_prop.container_property
|
|
98
|
-
prop.reference = ReferenceEntity.from_entity(ref_prop.view, ref_prop.view_property)
|
|
99
95
|
|
|
100
96
|
return JustRules(solution)
|
|
101
97
|
|
|
@@ -13,12 +13,8 @@ from cognite.neat._rules._shared import (
|
|
|
13
13
|
VerifiedRules,
|
|
14
14
|
)
|
|
15
15
|
from cognite.neat._rules.models import (
|
|
16
|
-
AssetInputRules,
|
|
17
|
-
AssetRules,
|
|
18
16
|
DMSInputRules,
|
|
19
17
|
DMSRules,
|
|
20
|
-
DomainInputRules,
|
|
21
|
-
DomainRules,
|
|
22
18
|
InformationInputRules,
|
|
23
19
|
InformationRules,
|
|
24
20
|
)
|
|
@@ -68,12 +64,6 @@ class VerifyInformationRules(VerificationTransformer[InformationInputRules, Info
|
|
|
68
64
|
_rules_cls = InformationRules
|
|
69
65
|
|
|
70
66
|
|
|
71
|
-
class VerifyAssetRules(VerificationTransformer[AssetInputRules, AssetRules]):
|
|
72
|
-
"""Class to verify Asset rules."""
|
|
73
|
-
|
|
74
|
-
_rules_cls = AssetRules
|
|
75
|
-
|
|
76
|
-
|
|
77
67
|
class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
|
|
78
68
|
"""Class to verify arbitrary rules"""
|
|
79
69
|
|
|
@@ -82,9 +72,5 @@ class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
|
|
|
82
72
|
return InformationRules
|
|
83
73
|
elif isinstance(in_, DMSInputRules):
|
|
84
74
|
return DMSRules
|
|
85
|
-
elif isinstance(in_, AssetInputRules):
|
|
86
|
-
return AssetRules
|
|
87
|
-
elif isinstance(in_, DomainInputRules):
|
|
88
|
-
return DomainRules
|
|
89
75
|
else:
|
|
90
76
|
raise NeatTypeError(f"Unsupported rules type: {type(in_)}")
|
cognite/neat/_session/_base.py
CHANGED
|
@@ -9,10 +9,7 @@ from cognite.neat._issues import IssueList, catch_issues
|
|
|
9
9
|
from cognite.neat._issues.errors import RegexViolationError
|
|
10
10
|
from cognite.neat._rules import importers
|
|
11
11
|
from cognite.neat._rules._shared import ReadRules, VerifiedRules
|
|
12
|
-
from cognite.neat._rules.importers._rdf._base import DEFAULT_NON_EXISTING_NODE_TYPE
|
|
13
12
|
from cognite.neat._rules.models import DMSRules
|
|
14
|
-
from cognite.neat._rules.models.data_types import AnyURI
|
|
15
|
-
from cognite.neat._rules.models.entities._single_value import UnknownEntity
|
|
16
13
|
from cognite.neat._rules.models.information._rules import InformationRules
|
|
17
14
|
from cognite.neat._rules.models.information._rules_input import InformationInputRules
|
|
18
15
|
from cognite.neat._rules.transformers import ConvertToRules, VerifyAnyRules
|
|
@@ -21,7 +18,9 @@ from cognite.neat._store._provenance import (
|
|
|
21
18
|
INSTANCES_ENTITY,
|
|
22
19
|
Change,
|
|
23
20
|
)
|
|
21
|
+
from cognite.neat._utils.auth import _CLIENT_NAME
|
|
24
22
|
|
|
23
|
+
from ._collector import _COLLECTOR, Collector
|
|
25
24
|
from ._inspect import InspectAPI
|
|
26
25
|
from ._prepare import PrepareAPI
|
|
27
26
|
from ._read import ReadAPI
|
|
@@ -30,10 +29,10 @@ from ._show import ShowAPI
|
|
|
30
29
|
from ._state import SessionState
|
|
31
30
|
from ._to import ToAPI
|
|
32
31
|
from .engine import load_neat_engine
|
|
33
|
-
from .exceptions import NeatSessionError,
|
|
32
|
+
from .exceptions import NeatSessionError, session_class_wrapper
|
|
34
33
|
|
|
35
34
|
|
|
36
|
-
@
|
|
35
|
+
@session_class_wrapper
|
|
37
36
|
class NeatSession:
|
|
38
37
|
def __init__(
|
|
39
38
|
self,
|
|
@@ -51,6 +50,10 @@ class NeatSession:
|
|
|
51
50
|
self.show = ShowAPI(self._state)
|
|
52
51
|
self.set = SetAPI(self._state, verbose)
|
|
53
52
|
self.inspect = InspectAPI(self._state)
|
|
53
|
+
self.opt = OptAPI()
|
|
54
|
+
self.opt._display()
|
|
55
|
+
if self._client is not None and self._client._config is not None:
|
|
56
|
+
self._client._config.client_name = _CLIENT_NAME
|
|
54
57
|
if load_engine != "skip" and (engine_version := load_neat_engine(client, load_engine)):
|
|
55
58
|
print(f"Neat Engine {engine_version} loaded.")
|
|
56
59
|
|
|
@@ -71,7 +74,7 @@ class NeatSession:
|
|
|
71
74
|
transformer.agent,
|
|
72
75
|
start,
|
|
73
76
|
end,
|
|
74
|
-
f"Verified data model {source_id} as {output.rules.
|
|
77
|
+
f"Verified data model {source_id} as {output.rules.metadata.identifier}",
|
|
75
78
|
self._state.data_model.provenance.source_entity(source_id)
|
|
76
79
|
or self._state.data_model.provenance.target_entity(source_id),
|
|
77
80
|
)
|
|
@@ -116,7 +119,7 @@ class NeatSession:
|
|
|
116
119
|
converter.agent,
|
|
117
120
|
start,
|
|
118
121
|
end,
|
|
119
|
-
f"Converted data model {source_id} to {converted_rules.
|
|
122
|
+
f"Converted data model {source_id} to {converted_rules.metadata.identifier}",
|
|
120
123
|
self._state.data_model.provenance.source_entity(source_id)
|
|
121
124
|
or self._state.data_model.provenance.target_entity(source_id),
|
|
122
125
|
)
|
|
@@ -141,31 +144,28 @@ class NeatSession:
|
|
|
141
144
|
"NeatInferredDataModel",
|
|
142
145
|
"v1",
|
|
143
146
|
),
|
|
144
|
-
non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
|
|
145
147
|
max_number_of_instance: int = 100,
|
|
146
148
|
) -> IssueList:
|
|
147
149
|
"""Data model inference from instances.
|
|
148
150
|
|
|
149
151
|
Args:
|
|
150
152
|
model_id: The ID of the inferred data model.
|
|
151
|
-
|
|
153
|
+
max_number_of_instance: The maximum number of instances to use for inference.
|
|
152
154
|
"""
|
|
153
|
-
|
|
154
155
|
model_id = dm.DataModelId.load(model_id)
|
|
155
156
|
|
|
156
157
|
start = datetime.now(timezone.utc)
|
|
157
158
|
importer = importers.InferenceImporter.from_graph_store(
|
|
158
159
|
store=self._state.instances.store,
|
|
159
|
-
non_existing_node_type=non_existing_node_type,
|
|
160
160
|
max_number_of_instance=max_number_of_instance,
|
|
161
161
|
)
|
|
162
162
|
inferred_rules: ReadRules = importer.to_rules()
|
|
163
163
|
end = datetime.now(timezone.utc)
|
|
164
164
|
|
|
165
165
|
if model_id.space:
|
|
166
|
-
cast(InformationInputRules, inferred_rules.rules).metadata.
|
|
166
|
+
cast(InformationInputRules, inferred_rules.rules).metadata.space = model_id.space
|
|
167
167
|
if model_id.external_id:
|
|
168
|
-
cast(InformationInputRules, inferred_rules.rules).metadata.
|
|
168
|
+
cast(InformationInputRules, inferred_rules.rules).metadata.external_id = model_id.external_id
|
|
169
169
|
|
|
170
170
|
if model_id.version:
|
|
171
171
|
cast(InformationInputRules, inferred_rules.rules).metadata.version = model_id.version
|
|
@@ -205,3 +205,27 @@ class NeatSession:
|
|
|
205
205
|
output.append(f"<H2>Instances</H2> {state.instances.store._repr_html_()}")
|
|
206
206
|
|
|
207
207
|
return "<br />".join(output)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
@session_class_wrapper
|
|
211
|
+
class OptAPI:
|
|
212
|
+
def __init__(self, collector: Collector | None = None) -> None:
|
|
213
|
+
self._collector = collector or _COLLECTOR
|
|
214
|
+
|
|
215
|
+
def _display(self) -> None:
|
|
216
|
+
if self._collector.opted_in or self._collector.opted_out:
|
|
217
|
+
return
|
|
218
|
+
print(
|
|
219
|
+
"For Neat to improve, we need to collect usage information. "
|
|
220
|
+
"You acknowledge and agree that neat may collect usage information."
|
|
221
|
+
"To remove this message run 'neat.opt.in_() "
|
|
222
|
+
"or to stop collecting usage information run 'neat.opt.out()'."
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
def in_(self) -> None:
|
|
226
|
+
self._collector.enable()
|
|
227
|
+
print("You have successfully opted in to data collection.")
|
|
228
|
+
|
|
229
|
+
def out(self) -> None:
|
|
230
|
+
self._collector.disable()
|
|
231
|
+
print("You have successfully opted out of data collection.")
|