cognite-neat 0.109.1__py3-none-any.whl → 0.109.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/_alpha.py +15 -0
- cognite/neat/_client/testing.py +1 -1
- cognite/neat/_issues/_base.py +33 -9
- cognite/neat/_issues/errors/__init__.py +2 -10
- cognite/neat/_issues/errors/_general.py +1 -1
- cognite/neat/_issues/errors/_wrapper.py +11 -0
- cognite/neat/_rules/exporters/_rules2excel.py +31 -1
- cognite/neat/_rules/models/_rdfpath.py +2 -0
- cognite/neat/_rules/models/_types.py +4 -2
- cognite/neat/_rules/models/dms/_rules.py +0 -36
- cognite/neat/_rules/models/entities/_constants.py +3 -0
- cognite/neat/_rules/models/entities/_single_value.py +6 -1
- cognite/neat/_rules/models/entities/_wrapped.py +3 -0
- cognite/neat/_rules/transformers/__init__.py +4 -0
- cognite/neat/_rules/transformers/_converters.py +221 -15
- cognite/neat/_session/_base.py +7 -0
- cognite/neat/_session/_collector.py +4 -1
- cognite/neat/_session/_create.py +46 -12
- cognite/neat/_session/_prepare.py +11 -3
- cognite/neat/_session/_read.py +14 -2
- cognite/neat/_session/_state.py +7 -3
- cognite/neat/_session/_to.py +20 -5
- cognite/neat/_session/exceptions.py +16 -6
- cognite/neat/_store/_provenance.py +1 -0
- cognite/neat/_store/_rules_store.py +192 -127
- cognite/neat/_utils/spreadsheet.py +10 -1
- cognite/neat/_utils/text.py +40 -9
- cognite/neat/_version.py +1 -1
- {cognite_neat-0.109.1.dist-info → cognite_neat-0.109.3.dist-info}/METADATA +1 -1
- {cognite_neat-0.109.1.dist-info → cognite_neat-0.109.3.dist-info}/RECORD +33 -32
- cognite/neat/_issues/errors/_workflow.py +0 -36
- {cognite_neat-0.109.1.dist-info → cognite_neat-0.109.3.dist-info}/LICENSE +0 -0
- {cognite_neat-0.109.1.dist-info → cognite_neat-0.109.3.dist-info}/WHEEL +0 -0
- {cognite_neat-0.109.1.dist-info → cognite_neat-0.109.3.dist-info}/entry_points.txt +0 -0
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import re
|
|
2
|
+
import urllib.parse
|
|
2
3
|
import warnings
|
|
3
4
|
from abc import ABC
|
|
4
5
|
from collections import Counter, defaultdict
|
|
5
6
|
from collections.abc import Collection, Mapping
|
|
6
7
|
from datetime import date, datetime
|
|
8
|
+
from functools import cached_property
|
|
7
9
|
from typing import ClassVar, Literal, TypeVar, cast, overload
|
|
8
10
|
|
|
9
11
|
from cognite.client.data_classes import data_modeling as dms
|
|
@@ -23,11 +25,12 @@ from cognite.neat._constants import (
|
|
|
23
25
|
from cognite.neat._issues.errors import NeatValueError
|
|
24
26
|
from cognite.neat._issues.warnings import NeatValueWarning
|
|
25
27
|
from cognite.neat._issues.warnings._models import (
|
|
26
|
-
EnterpriseModelNotBuildOnTopOfCDMWarning,
|
|
27
28
|
SolutionModelBuildOnTopOfCDMWarning,
|
|
28
29
|
)
|
|
30
|
+
from cognite.neat._rules._constants import PATTERNS, get_reserved_words
|
|
29
31
|
from cognite.neat._rules._shared import (
|
|
30
32
|
ReadInputRules,
|
|
33
|
+
ReadRules,
|
|
31
34
|
VerifiedRules,
|
|
32
35
|
)
|
|
33
36
|
from cognite.neat._rules.analysis import DMSAnalysis
|
|
@@ -35,6 +38,7 @@ from cognite.neat._rules.importers import DMSImporter
|
|
|
35
38
|
from cognite.neat._rules.models import (
|
|
36
39
|
DMSInputRules,
|
|
37
40
|
DMSRules,
|
|
41
|
+
InformationInputRules,
|
|
38
42
|
InformationRules,
|
|
39
43
|
SheetList,
|
|
40
44
|
data_types,
|
|
@@ -56,9 +60,9 @@ from cognite.neat._rules.models.entities import (
|
|
|
56
60
|
ViewEntity,
|
|
57
61
|
)
|
|
58
62
|
from cognite.neat._rules.models.information import InformationClass, InformationMetadata, InformationProperty
|
|
59
|
-
from cognite.neat._utils.text import to_camel
|
|
63
|
+
from cognite.neat._utils.text import NamingStandardization, to_camel
|
|
60
64
|
|
|
61
|
-
from ._base import T_VerifiedIn, T_VerifiedOut, VerifiedRulesTransformer
|
|
65
|
+
from ._base import RulesTransformer, T_VerifiedIn, T_VerifiedOut, VerifiedRulesTransformer
|
|
62
66
|
from ._verification import VerifyDMSRules
|
|
63
67
|
|
|
64
68
|
T_InputInRules = TypeVar("T_InputInRules", bound=ReadInputRules)
|
|
@@ -71,6 +75,105 @@ class ConversionTransformer(VerifiedRulesTransformer[T_VerifiedIn, T_VerifiedOut
|
|
|
71
75
|
...
|
|
72
76
|
|
|
73
77
|
|
|
78
|
+
class ToInformationCompliantEntities(
|
|
79
|
+
RulesTransformer[ReadRules[InformationInputRules], ReadRules[InformationInputRules]]
|
|
80
|
+
):
|
|
81
|
+
"""Converts input rules to rules that is compliant with the Information Model.
|
|
82
|
+
|
|
83
|
+
This is typically used with importers from arbitrary sources to ensure that classes and properties have valid
|
|
84
|
+
names.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
renaming: How to handle renaming of entities that are not compliant with the Information Model.
|
|
88
|
+
- "warning": Raises a warning and renames the entity.
|
|
89
|
+
- "skip": Renames the entity without raising a warning.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
def __init__(self, renaming: Literal["warning", "skip"] = "skip") -> None:
|
|
93
|
+
self._renaming = renaming
|
|
94
|
+
|
|
95
|
+
@property
|
|
96
|
+
def description(self) -> str:
|
|
97
|
+
return "Ensures that all entities are compliant with the Information Model."
|
|
98
|
+
|
|
99
|
+
def transform(self, rules: ReadRules[InformationInputRules]) -> ReadRules[InformationInputRules]:
|
|
100
|
+
if rules.rules is None:
|
|
101
|
+
return rules
|
|
102
|
+
# Doing dump to obtain a copy, and ensure that all entities are created. Input allows
|
|
103
|
+
# string for entities, the dump call will convert these to entities.
|
|
104
|
+
dumped = rules.rules.dump()
|
|
105
|
+
copy = InformationInputRules.load(dumped)
|
|
106
|
+
|
|
107
|
+
new_by_old_class_suffix: dict[str, str] = {}
|
|
108
|
+
for cls in copy.classes:
|
|
109
|
+
cls_entity = cast(ClassEntity, cls.class_) # Safe due to the dump above
|
|
110
|
+
if not PATTERNS.class_id_compliance.match(cls_entity.suffix):
|
|
111
|
+
new_suffix = self._fix_cls_suffix(cls_entity.suffix)
|
|
112
|
+
if self._renaming == "warning":
|
|
113
|
+
warnings.warn(
|
|
114
|
+
NeatValueWarning(f"Invalid class name {cls_entity.suffix!r}.Renaming to {new_suffix}"),
|
|
115
|
+
stacklevel=2,
|
|
116
|
+
)
|
|
117
|
+
cls.class_.suffix = new_suffix # type: ignore[union-attr]
|
|
118
|
+
|
|
119
|
+
for cls_ in copy.classes:
|
|
120
|
+
if cls_.implements:
|
|
121
|
+
for i, parent in enumerate(cls_.implements):
|
|
122
|
+
if isinstance(parent, ClassEntity) and parent.suffix in new_by_old_class_suffix:
|
|
123
|
+
cls_.implements[i].suffix = new_by_old_class_suffix[parent.suffix] # type: ignore[union-attr]
|
|
124
|
+
|
|
125
|
+
for prop in copy.properties:
|
|
126
|
+
if not PATTERNS.information_property_id_compliance.match(prop.property_):
|
|
127
|
+
new_property = self._fix_property(prop.property_)
|
|
128
|
+
if self._renaming == "warning":
|
|
129
|
+
warnings.warn(
|
|
130
|
+
NeatValueWarning(
|
|
131
|
+
f"Invalid property name {prop.class_.suffix}.{prop.property_!r}. Renaming to {new_property}" # type: ignore[union-attr]
|
|
132
|
+
),
|
|
133
|
+
stacklevel=2,
|
|
134
|
+
)
|
|
135
|
+
prop.property_ = new_property
|
|
136
|
+
|
|
137
|
+
if isinstance(prop.class_, ClassEntity) and prop.class_.suffix in new_by_old_class_suffix:
|
|
138
|
+
prop.class_.suffix = new_by_old_class_suffix[prop.class_.suffix]
|
|
139
|
+
|
|
140
|
+
if isinstance(prop.value_type, ClassEntity) and prop.value_type.suffix in new_by_old_class_suffix:
|
|
141
|
+
prop.value_type.suffix = new_by_old_class_suffix[prop.value_type.suffix]
|
|
142
|
+
|
|
143
|
+
if isinstance(prop.value_type, MultiValueTypeInfo):
|
|
144
|
+
for i, value_type in enumerate(prop.value_type.types):
|
|
145
|
+
if isinstance(value_type, ClassEntity) and value_type.suffix in new_by_old_class_suffix:
|
|
146
|
+
prop.value_type.types[i].suffix = new_by_old_class_suffix[value_type.suffix] # type: ignore[union-attr]
|
|
147
|
+
|
|
148
|
+
return ReadRules(rules=copy, read_context=rules.read_context)
|
|
149
|
+
|
|
150
|
+
@cached_property
|
|
151
|
+
def _reserved_class_words(self) -> set[str]:
|
|
152
|
+
return set(get_reserved_words("class"))
|
|
153
|
+
|
|
154
|
+
@cached_property
|
|
155
|
+
def _reserved_property_words(self) -> set[str]:
|
|
156
|
+
return set(get_reserved_words("property"))
|
|
157
|
+
|
|
158
|
+
def _fix_cls_suffix(self, suffix: str) -> str:
|
|
159
|
+
if suffix in self._reserved_class_words:
|
|
160
|
+
return f"My{suffix}"
|
|
161
|
+
suffix = urllib.parse.unquote(suffix)
|
|
162
|
+
suffix = NamingStandardization.standardize_class_str(suffix)
|
|
163
|
+
if len(suffix) > 252:
|
|
164
|
+
suffix = suffix[:252]
|
|
165
|
+
return suffix
|
|
166
|
+
|
|
167
|
+
def _fix_property(self, property_: str) -> str:
|
|
168
|
+
if property_ in self._reserved_property_words:
|
|
169
|
+
return f"my{property_}"
|
|
170
|
+
property_ = urllib.parse.unquote(property_)
|
|
171
|
+
property_ = NamingStandardization.standardize_property_str(property_)
|
|
172
|
+
if len(property_) > 252:
|
|
173
|
+
property_ = property_[:252]
|
|
174
|
+
return property_
|
|
175
|
+
|
|
176
|
+
|
|
74
177
|
class ToCompliantEntities(VerifiedRulesTransformer[InformationRules, InformationRules]): # type: ignore[misc]
|
|
75
178
|
"""Converts input rules to rules with compliant entity IDs that match regex patters used
|
|
76
179
|
by DMS schema components."""
|
|
@@ -246,6 +349,116 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
|
|
|
246
349
|
return entity
|
|
247
350
|
|
|
248
351
|
|
|
352
|
+
class StandardizeNaming(ConversionTransformer):
|
|
353
|
+
"""Sets views/classes/container names to PascalCase and properties to camelCase."""
|
|
354
|
+
|
|
355
|
+
@property
|
|
356
|
+
def description(self) -> str:
|
|
357
|
+
return "Sets views/classes/containers names to PascalCase and properties to camelCase."
|
|
358
|
+
|
|
359
|
+
@overload
|
|
360
|
+
def transform(self, rules: DMSRules) -> DMSRules: ...
|
|
361
|
+
|
|
362
|
+
@overload
|
|
363
|
+
def transform(self, rules: InformationRules) -> InformationRules: ...
|
|
364
|
+
|
|
365
|
+
def transform(self, rules: InformationRules | DMSRules) -> InformationRules | DMSRules:
|
|
366
|
+
output = rules.model_copy(deep=True)
|
|
367
|
+
if isinstance(output, InformationRules):
|
|
368
|
+
return self._standardize_information_rules(output)
|
|
369
|
+
elif isinstance(output, DMSRules):
|
|
370
|
+
return self._standardize_dms_rules(output)
|
|
371
|
+
raise NeatValueError(f"Unsupported rules type: {type(output)}")
|
|
372
|
+
|
|
373
|
+
def _standardize_information_rules(self, rules: InformationRules) -> InformationRules:
|
|
374
|
+
new_by_old_class_suffix: dict[str, str] = {}
|
|
375
|
+
for cls in rules.classes:
|
|
376
|
+
new_suffix = NamingStandardization.standardize_class_str(cls.class_.suffix)
|
|
377
|
+
new_by_old_class_suffix[cls.class_.suffix] = new_suffix
|
|
378
|
+
cls.class_.suffix = new_suffix
|
|
379
|
+
|
|
380
|
+
for cls in rules.classes:
|
|
381
|
+
if cls.implements:
|
|
382
|
+
for i, parent in enumerate(cls.implements):
|
|
383
|
+
if parent.suffix in new_by_old_class_suffix:
|
|
384
|
+
cls.implements[i].suffix = new_by_old_class_suffix[parent.suffix]
|
|
385
|
+
|
|
386
|
+
for prop in rules.properties:
|
|
387
|
+
prop.property_ = NamingStandardization.standardize_property_str(prop.property_)
|
|
388
|
+
if prop.class_.suffix in new_by_old_class_suffix:
|
|
389
|
+
prop.class_.suffix = new_by_old_class_suffix[prop.class_.suffix]
|
|
390
|
+
|
|
391
|
+
if isinstance(prop.value_type, ClassEntity) and prop.value_type.suffix in new_by_old_class_suffix:
|
|
392
|
+
prop.value_type.suffix = new_by_old_class_suffix[prop.value_type.suffix]
|
|
393
|
+
|
|
394
|
+
if isinstance(prop.value_type, MultiValueTypeInfo):
|
|
395
|
+
for i, value_type in enumerate(prop.value_type.types):
|
|
396
|
+
if isinstance(value_type, ClassEntity) and value_type.suffix in new_by_old_class_suffix:
|
|
397
|
+
prop.value_type.types[i].suffix = new_by_old_class_suffix[value_type.suffix] # type: ignore[union-attr]
|
|
398
|
+
|
|
399
|
+
return rules
|
|
400
|
+
|
|
401
|
+
def _standardize_dms_rules(self, rules: DMSRules) -> DMSRules:
|
|
402
|
+
new_by_old_view: dict[str, str] = {}
|
|
403
|
+
for view in rules.views:
|
|
404
|
+
new_suffix = NamingStandardization.standardize_class_str(view.view.suffix)
|
|
405
|
+
new_by_old_view[view.view.suffix] = new_suffix
|
|
406
|
+
view.view.suffix = new_suffix
|
|
407
|
+
new_by_old_container: dict[str, str] = {}
|
|
408
|
+
if rules.containers:
|
|
409
|
+
for container in rules.containers:
|
|
410
|
+
new_suffix = NamingStandardization.standardize_class_str(container.container.suffix)
|
|
411
|
+
new_by_old_container[container.container.suffix] = new_suffix
|
|
412
|
+
container.container.suffix = new_suffix
|
|
413
|
+
|
|
414
|
+
for view in rules.views:
|
|
415
|
+
if view.implements:
|
|
416
|
+
for i, parent in enumerate(view.implements):
|
|
417
|
+
if parent.suffix in new_by_old_view:
|
|
418
|
+
view.implements[i].suffix = new_by_old_view[parent.suffix]
|
|
419
|
+
if view.filter_ and isinstance(view.filter_, HasDataFilter) and view.filter_.inner:
|
|
420
|
+
for i, item in enumerate(view.filter_.inner):
|
|
421
|
+
if isinstance(item, ContainerEntity) and item.suffix in new_by_old_container:
|
|
422
|
+
view.filter_.inner[i].suffix = new_by_old_container[item.suffix]
|
|
423
|
+
if isinstance(item, ViewEntity) and item.suffix in new_by_old_view:
|
|
424
|
+
view.filter_.inner[i].suffix = new_by_old_view[item.suffix]
|
|
425
|
+
if rules.containers:
|
|
426
|
+
for container in rules.containers:
|
|
427
|
+
if container.constraint:
|
|
428
|
+
for i, constraint in enumerate(container.constraint):
|
|
429
|
+
if constraint.suffix in new_by_old_container:
|
|
430
|
+
container.constraint[i].suffix = new_by_old_container[constraint.suffix]
|
|
431
|
+
new_property_by_view_by_old_property: dict[ViewEntity, dict[str, str]] = defaultdict(dict)
|
|
432
|
+
for prop in rules.properties:
|
|
433
|
+
if prop.view.suffix in new_by_old_view:
|
|
434
|
+
prop.view.suffix = new_by_old_view[prop.view.suffix]
|
|
435
|
+
new_view_property = NamingStandardization.standardize_property_str(prop.view_property)
|
|
436
|
+
new_property_by_view_by_old_property[prop.view][prop.view_property] = new_view_property
|
|
437
|
+
prop.view_property = new_view_property
|
|
438
|
+
if isinstance(prop.value_type, ViewEntity) and prop.value_type.suffix in new_by_old_view:
|
|
439
|
+
prop.value_type.suffix = new_by_old_view[prop.value_type.suffix]
|
|
440
|
+
if (
|
|
441
|
+
isinstance(prop.connection, EdgeEntity)
|
|
442
|
+
and prop.connection.properties
|
|
443
|
+
and prop.connection.properties.suffix in new_by_old_view
|
|
444
|
+
):
|
|
445
|
+
prop.connection.properties.suffix = new_by_old_view[prop.connection.properties.suffix]
|
|
446
|
+
if isinstance(prop.container, ContainerEntity) and prop.container.suffix in new_by_old_container:
|
|
447
|
+
prop.container.suffix = new_by_old_container[prop.container.suffix]
|
|
448
|
+
if prop.container_property:
|
|
449
|
+
prop.container_property = NamingStandardization.standardize_property_str(prop.container_property)
|
|
450
|
+
for prop in rules.properties:
|
|
451
|
+
if (
|
|
452
|
+
isinstance(prop.connection, ReverseConnectionEntity)
|
|
453
|
+
and isinstance(prop.value_type, ViewEntity)
|
|
454
|
+
and prop.value_type in new_property_by_view_by_old_property
|
|
455
|
+
):
|
|
456
|
+
new_by_old_property = new_property_by_view_by_old_property[prop.value_type]
|
|
457
|
+
if prop.connection.property_ in new_by_old_property:
|
|
458
|
+
prop.connection.property_ = new_by_old_property[prop.connection.property_]
|
|
459
|
+
return rules
|
|
460
|
+
|
|
461
|
+
|
|
249
462
|
class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
|
|
250
463
|
"""Converts InformationRules to DMSRules."""
|
|
251
464
|
|
|
@@ -339,13 +552,6 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
339
552
|
self.move_connections = move_connections
|
|
340
553
|
|
|
341
554
|
def transform(self, rules: DMSRules) -> DMSRules:
|
|
342
|
-
reference_model_id = rules.metadata.as_data_model_id()
|
|
343
|
-
if reference_model_id not in COGNITE_MODELS:
|
|
344
|
-
warnings.warn(
|
|
345
|
-
EnterpriseModelNotBuildOnTopOfCDMWarning(reference_model_id=reference_model_id).as_message(),
|
|
346
|
-
stacklevel=2,
|
|
347
|
-
)
|
|
348
|
-
|
|
349
555
|
return self._to_enterprise(rules)
|
|
350
556
|
|
|
351
557
|
def _to_enterprise(self, reference_model: DMSRules) -> DMSRules:
|
|
@@ -565,12 +771,10 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
565
771
|
renaming: dict[ViewEntity, ViewEntity] = {}
|
|
566
772
|
new_views = SheetList[DMSView]()
|
|
567
773
|
read_view_by_new_view: dict[ViewEntity, ViewEntity] = {}
|
|
568
|
-
skipped_views: set[ViewEntity] = set()
|
|
569
774
|
for ref_view in reference.views:
|
|
570
775
|
if (self.skip_cognite_views and ref_view.view.space in COGNITE_SPACES) or (
|
|
571
776
|
self.exclude_views_in_other_spaces and ref_view.view.space != reference.metadata.space
|
|
572
777
|
):
|
|
573
|
-
skipped_views.add(ref_view.view)
|
|
574
778
|
continue
|
|
575
779
|
new_entity = ViewEntity(
|
|
576
780
|
# MyPy we validate that version is string in the constructor
|
|
@@ -603,15 +807,17 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
603
807
|
new_views.append(ref_view.model_copy(deep=True, update={"implements": None, "view": new_entity}))
|
|
604
808
|
|
|
605
809
|
new_properties = SheetList[DMSProperty]()
|
|
810
|
+
new_view_entities = {view.view for view in new_views}
|
|
606
811
|
for prop in reference.properties:
|
|
607
|
-
if prop.view in skipped_views:
|
|
608
|
-
continue
|
|
609
812
|
new_property = prop.model_copy(deep=True)
|
|
610
813
|
if new_property.value_type in renaming and isinstance(new_property.value_type, ViewEntity):
|
|
611
814
|
new_property.value_type = renaming[new_property.value_type]
|
|
612
815
|
if new_property.view in renaming:
|
|
613
816
|
new_property.view = renaming[new_property.view]
|
|
614
|
-
|
|
817
|
+
if new_property.view in new_view_entities and (
|
|
818
|
+
not isinstance(new_property.value_type, ViewEntity) or new_property.value_type in new_view_entities
|
|
819
|
+
):
|
|
820
|
+
new_properties.append(new_property)
|
|
615
821
|
return new_views, new_properties, read_view_by_new_view
|
|
616
822
|
|
|
617
823
|
def _create_containers_update_view_filter(
|
cognite/neat/_session/_base.py
CHANGED
|
@@ -16,6 +16,7 @@ from cognite.neat._rules.transformers import (
|
|
|
16
16
|
InformationToDMS,
|
|
17
17
|
MergeDMSRules,
|
|
18
18
|
MergeInformationRules,
|
|
19
|
+
ToInformationCompliantEntities,
|
|
19
20
|
VerifyInformationRules,
|
|
20
21
|
)
|
|
21
22
|
from cognite.neat._store._rules_store import RulesEntity
|
|
@@ -235,13 +236,19 @@ class NeatSession:
|
|
|
235
236
|
|
|
236
237
|
def action() -> tuple[InformationRules, DMSRules | None]:
|
|
237
238
|
unverified_information = importer.to_rules()
|
|
239
|
+
unverified_information = ToInformationCompliantEntities(renaming="warning").transform(
|
|
240
|
+
unverified_information
|
|
241
|
+
)
|
|
242
|
+
|
|
238
243
|
extra_info = VerifyInformationRules().transform(unverified_information)
|
|
239
244
|
if not last_entity:
|
|
240
245
|
return extra_info, None
|
|
241
246
|
merged_info = MergeInformationRules(extra_info).transform(last_entity.information)
|
|
242
247
|
if not last_entity.dms:
|
|
243
248
|
return merged_info, None
|
|
249
|
+
|
|
244
250
|
extra_dms = InformationToDMS(reserved_properties="warning").transform(extra_info)
|
|
251
|
+
|
|
245
252
|
merged_dms = MergeDMSRules(extra_dms).transform(last_entity.dms)
|
|
246
253
|
return merged_info, merged_dms
|
|
247
254
|
|
|
@@ -65,7 +65,10 @@ class Collector:
|
|
|
65
65
|
if kwargs:
|
|
66
66
|
for key, value in kwargs.items():
|
|
67
67
|
event_information[key] = self._serialize_value(value)[:500]
|
|
68
|
-
|
|
68
|
+
|
|
69
|
+
with suppress(RuntimeError):
|
|
70
|
+
# In case any thread issues, the tracking should not crash the program
|
|
71
|
+
self._track(command, event_information)
|
|
69
72
|
|
|
70
73
|
@staticmethod
|
|
71
74
|
def _serialize_value(value: Any) -> str:
|
cognite/neat/_session/_create.py
CHANGED
|
@@ -3,6 +3,7 @@ from typing import Literal
|
|
|
3
3
|
from cognite.client.data_classes.data_modeling import DataModelIdentifier
|
|
4
4
|
|
|
5
5
|
from cognite.neat._issues import IssueList
|
|
6
|
+
from cognite.neat._rules.models import DMSRules, InformationRules
|
|
6
7
|
from cognite.neat._rules.models.dms import DMSValidation
|
|
7
8
|
from cognite.neat._rules.transformers import (
|
|
8
9
|
IncludeReferenced,
|
|
@@ -38,22 +39,27 @@ class CreateAPI:
|
|
|
38
39
|
org_name: Organization name to use for the views in the enterprise data model.
|
|
39
40
|
dummy_property: The dummy property to use as placeholder for the views in the new data model.
|
|
40
41
|
|
|
42
|
+
What does this function do?
|
|
43
|
+
1. It creates a new view for each view in the current data model that implements the view it is based on.
|
|
44
|
+
2. If dummy_property is set, it will create a container with one property for each view and connect the
|
|
45
|
+
view to the container.
|
|
46
|
+
3. It will repeat all connection properties in the new views and update the ValueTypes to match the new
|
|
47
|
+
views.
|
|
48
|
+
|
|
41
49
|
!!! note "Enterprise Data Model Creation"
|
|
42
50
|
|
|
43
51
|
Always create an enterprise data model from a Cognite Data Model as this will
|
|
44
52
|
assure all the Cognite Data Fusion applications to run smoothly, such as
|
|
45
53
|
- Search
|
|
46
54
|
- Atlas AI
|
|
47
|
-
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
If you want to move the connections to the new data model, set the move_connections
|
|
52
|
-
to True. This will move the connections to the new data model and use new model
|
|
53
|
-
views as the source and target views.
|
|
55
|
+
- Infield
|
|
56
|
+
- Canvas
|
|
57
|
+
- Maintain
|
|
58
|
+
- Charts
|
|
54
59
|
|
|
55
60
|
"""
|
|
56
|
-
|
|
61
|
+
last_rules = self._get_last_rules()
|
|
62
|
+
issues = self._state.rule_transform(
|
|
57
63
|
ToEnterpriseModel(
|
|
58
64
|
new_model_id=data_model_id,
|
|
59
65
|
org_name=org_name,
|
|
@@ -61,6 +67,15 @@ class CreateAPI:
|
|
|
61
67
|
move_connections=True,
|
|
62
68
|
)
|
|
63
69
|
)
|
|
70
|
+
if last_rules and not issues.has_errors:
|
|
71
|
+
self._state.last_reference = last_rules
|
|
72
|
+
return issues
|
|
73
|
+
|
|
74
|
+
def _get_last_rules(self) -> InformationRules | DMSRules | None:
|
|
75
|
+
if not self._state.rule_store.provenance:
|
|
76
|
+
return None
|
|
77
|
+
last_entity = self._state.rule_store.provenance[-1].target_entity
|
|
78
|
+
return last_entity.dms or last_entity.information
|
|
64
79
|
|
|
65
80
|
def solution_model(
|
|
66
81
|
self,
|
|
@@ -76,6 +91,13 @@ class CreateAPI:
|
|
|
76
91
|
and the enterprise data model.
|
|
77
92
|
view_prefix: The prefix to use for the views in the enterprise data model.
|
|
78
93
|
|
|
94
|
+
What does this function do?
|
|
95
|
+
1. It will create two new views for each view in the current data model. The first view will be read-only and
|
|
96
|
+
prefixed with the 'view_prefix'. The second view will be writable and have one property that connects to the
|
|
97
|
+
read-only view named 'direct_property'.
|
|
98
|
+
2. It will repeat all connection properties in the new views and update the ValueTypes to match the new views.
|
|
99
|
+
3. Each writable view will have a container with the single property that connects to the read-only view.
|
|
100
|
+
|
|
79
101
|
!!! note "Solution Data Model Mode"
|
|
80
102
|
|
|
81
103
|
The read-only solution model will only be able to read from the existing containers
|
|
@@ -88,7 +110,8 @@ class CreateAPI:
|
|
|
88
110
|
the containers in the solution data model space.
|
|
89
111
|
|
|
90
112
|
"""
|
|
91
|
-
|
|
113
|
+
last_rules = self._get_last_rules()
|
|
114
|
+
issues = self._state.rule_transform(
|
|
92
115
|
ToSolutionModel(
|
|
93
116
|
new_model_id=data_model_id,
|
|
94
117
|
properties="connection",
|
|
@@ -96,24 +119,32 @@ class CreateAPI:
|
|
|
96
119
|
view_prefix=view_prefix,
|
|
97
120
|
)
|
|
98
121
|
)
|
|
122
|
+
if last_rules and not issues.has_errors:
|
|
123
|
+
self._state.last_reference = last_rules
|
|
124
|
+
return issues
|
|
99
125
|
|
|
100
126
|
def data_product_model(
|
|
101
127
|
self,
|
|
102
128
|
data_model_id: DataModelIdentifier,
|
|
103
129
|
include: Literal["same-space", "all"] = "same-space",
|
|
104
|
-
) ->
|
|
130
|
+
) -> IssueList:
|
|
105
131
|
"""Uses the current data model as a basis to create data product data model.
|
|
106
132
|
|
|
107
133
|
A data product model is a data model that ONLY maps to containers and do not use implements. This is
|
|
108
134
|
typically used for defining the data in a data product.
|
|
109
135
|
|
|
136
|
+
What does this function do?
|
|
137
|
+
1. It creates a new view for each view in the current data model. The new views uses the same filter
|
|
138
|
+
as the view it is based on.
|
|
139
|
+
2. It will repeat all connection properties in the new views and update the ValueTypes to match the new views.
|
|
140
|
+
|
|
110
141
|
Args:
|
|
111
142
|
data_model_id: The data product data model id that is being created.
|
|
112
143
|
include: The views to include in the data product data model. Can be either "same-space" or "all".
|
|
113
144
|
If you set same-space, only the properties of the views in the same space as the data model
|
|
114
145
|
will be included.
|
|
115
146
|
"""
|
|
116
|
-
|
|
147
|
+
last_rules = self._get_last_rules()
|
|
117
148
|
view_ids, container_ids = DMSValidation(
|
|
118
149
|
self._state.rule_store.last_verified_dms_rules
|
|
119
150
|
).imported_views_and_containers_ids()
|
|
@@ -130,4 +161,7 @@ class CreateAPI:
|
|
|
130
161
|
|
|
131
162
|
transformers.append(ToDataProductModel(new_model_id=data_model_id, include=include))
|
|
132
163
|
|
|
133
|
-
self._state.rule_transform(*transformers)
|
|
164
|
+
issues = self._state.rule_transform(*transformers)
|
|
165
|
+
if last_rules and not issues.has_errors:
|
|
166
|
+
self._state.last_reference = last_rules
|
|
167
|
+
return issues
|
|
@@ -3,6 +3,7 @@ from typing import Any
|
|
|
3
3
|
|
|
4
4
|
from rdflib import URIRef
|
|
5
5
|
|
|
6
|
+
from cognite.neat._alpha import AlphaFlags
|
|
6
7
|
from cognite.neat._graph.transformers import (
|
|
7
8
|
ConnectionToLiteral,
|
|
8
9
|
ConvertLiteral,
|
|
@@ -12,9 +13,7 @@ from cognite.neat._graph.transformers import (
|
|
|
12
13
|
from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
|
|
13
14
|
from cognite.neat._issues import IssueList
|
|
14
15
|
from cognite.neat._issues.errors import NeatValueError
|
|
15
|
-
from cognite.neat._rules.transformers import
|
|
16
|
-
PrefixEntities,
|
|
17
|
-
)
|
|
16
|
+
from cognite.neat._rules.transformers import PrefixEntities, StandardizeNaming
|
|
18
17
|
from cognite.neat._utils.text import humanize_collection
|
|
19
18
|
|
|
20
19
|
from ._state import SessionState
|
|
@@ -264,3 +263,12 @@ class DataModelPrepareAPI:
|
|
|
264
263
|
"""
|
|
265
264
|
|
|
266
265
|
return self._state.rule_transform(PrefixEntities(prefix)) # type: ignore[arg-type]
|
|
266
|
+
|
|
267
|
+
def standardize_naming(self) -> IssueList:
|
|
268
|
+
"""Standardize the naming of all views/classes/properties in the data model.
|
|
269
|
+
|
|
270
|
+
For classes/views/containers, the naming will be standardized to PascalCase.
|
|
271
|
+
For properties, the naming will be standardized to camelCase.
|
|
272
|
+
"""
|
|
273
|
+
AlphaFlags.standardize_naming.warn()
|
|
274
|
+
return self._state.rule_transform(StandardizeNaming())
|
cognite/neat/_session/_read.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import warnings
|
|
1
2
|
from typing import Any, Literal, cast
|
|
2
3
|
|
|
3
4
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
|
|
4
5
|
from cognite.client.utils.useful_types import SequenceNotStr
|
|
5
6
|
|
|
7
|
+
from cognite.neat._alpha import AlphaFlags
|
|
6
8
|
from cognite.neat._client import NeatClient
|
|
7
9
|
from cognite.neat._constants import (
|
|
8
10
|
CLASSIC_CDF_NAMESPACE,
|
|
@@ -286,16 +288,26 @@ class ExcelReadAPI(BaseReadAPI):
|
|
|
286
288
|
super().__init__(state, verbose)
|
|
287
289
|
self.examples = ExcelExampleAPI(state, verbose)
|
|
288
290
|
|
|
289
|
-
def __call__(self, io: Any) -> IssueList:
|
|
291
|
+
def __call__(self, io: Any, enable_manual_edit: bool = False) -> IssueList:
|
|
290
292
|
"""Reads a Neat Excel Rules sheet to the graph store. The rules sheet may stem from an Information architect,
|
|
291
293
|
or a DMS Architect.
|
|
292
294
|
|
|
293
295
|
Args:
|
|
294
296
|
io: file path to the Excel sheet
|
|
297
|
+
enable_manual_edit: If True, the user will be able to re-import rules which where edit outside NeatSession
|
|
298
|
+
|
|
299
|
+
!!! note "Manual Edit Warning"
|
|
300
|
+
This is an alpha feature and is subject to change without notice.
|
|
301
|
+
It is expected to have some limitations and may not work as expected in all cases.
|
|
295
302
|
"""
|
|
296
303
|
reader = NeatReader.create(io)
|
|
297
304
|
path = reader.materialize_path()
|
|
298
|
-
|
|
305
|
+
|
|
306
|
+
if enable_manual_edit:
|
|
307
|
+
warnings.filterwarnings("default")
|
|
308
|
+
AlphaFlags.manual_rules_edit.warn()
|
|
309
|
+
|
|
310
|
+
return self._state.rule_import(importers.ExcelImporter(path), enable_manual_edit)
|
|
299
311
|
|
|
300
312
|
|
|
301
313
|
@session_class_wrapper
|
cognite/neat/_session/_state.py
CHANGED
|
@@ -40,8 +40,12 @@ class SessionState:
|
|
|
40
40
|
self.instances.store.add_rules(last_entity.information)
|
|
41
41
|
return issues
|
|
42
42
|
|
|
43
|
-
def rule_import(self, importer: BaseImporter) -> IssueList:
|
|
44
|
-
issues = self.rule_store.import_rules(
|
|
43
|
+
def rule_import(self, importer: BaseImporter, enable_manual_edit: bool = False) -> IssueList:
|
|
44
|
+
issues = self.rule_store.import_rules(
|
|
45
|
+
importer,
|
|
46
|
+
client=self.client,
|
|
47
|
+
enable_manual_edit=enable_manual_edit,
|
|
48
|
+
)
|
|
45
49
|
if self.rule_store.empty:
|
|
46
50
|
result = "failed"
|
|
47
51
|
else:
|
|
@@ -74,7 +78,7 @@ class InstancesState:
|
|
|
74
78
|
self.outcome = UploadResultList()
|
|
75
79
|
|
|
76
80
|
# Ensure that error handling is done in the constructor
|
|
77
|
-
self.store = _session_method_wrapper(self._create_store, "NeatSession")()
|
|
81
|
+
self.store: NeatGraphStore = _session_method_wrapper(self._create_store, "NeatSession")()
|
|
78
82
|
|
|
79
83
|
if self.storage_path:
|
|
80
84
|
print("Remember to close neat session .close() once you are done to avoid oxigraph lock.")
|
cognite/neat/_session/_to.py
CHANGED
|
@@ -6,6 +6,7 @@ from typing import Any, Literal, overload
|
|
|
6
6
|
|
|
7
7
|
from cognite.client import data_modeling as dm
|
|
8
8
|
|
|
9
|
+
from cognite.neat._alpha import AlphaFlags
|
|
9
10
|
from cognite.neat._constants import COGNITE_MODELS
|
|
10
11
|
from cognite.neat._graph import loaders
|
|
11
12
|
from cognite.neat._rules import exporters
|
|
@@ -35,6 +36,8 @@ class ToAPI:
|
|
|
35
36
|
self,
|
|
36
37
|
io: Any,
|
|
37
38
|
include_reference: bool = True,
|
|
39
|
+
include_properties: Literal["same-space", "all"] = "all",
|
|
40
|
+
add_empty_rows: bool = False,
|
|
38
41
|
) -> None:
|
|
39
42
|
"""Export the verified data model to Excel.
|
|
40
43
|
|
|
@@ -42,7 +45,10 @@ class ToAPI:
|
|
|
42
45
|
io: The file path or file-like object to write the Excel file to.
|
|
43
46
|
include_reference: If True, the reference data model will be included. Defaults to True.
|
|
44
47
|
Note that this only applies if you have created the data model using the
|
|
45
|
-
.
|
|
48
|
+
create.enterprise_model(...), create.solution_model(), or create.data_product_model() methods.
|
|
49
|
+
include_properties: The properties to include in the Excel file. Defaults to "all".
|
|
50
|
+
- "same-space": Only properties that are in the same space as the data model will be included.
|
|
51
|
+
add_empty_rows: If True, empty rows will be added between each component. Defaults to False.
|
|
46
52
|
|
|
47
53
|
Example:
|
|
48
54
|
Export information model to excel rules sheet
|
|
@@ -58,17 +64,17 @@ class ToAPI:
|
|
|
58
64
|
neat = NeatSession(client)
|
|
59
65
|
|
|
60
66
|
neat.read.cdf(("cdf_cdm", "CogniteCore", "v1"))
|
|
61
|
-
neat.
|
|
62
|
-
neat.prepare.data_model.to_enterprise(
|
|
67
|
+
neat.create.enterprise_model(
|
|
63
68
|
data_model_id=("sp_doctrino_space", "ExtensionCore", "v1"),
|
|
64
69
|
org_name="MyOrg",
|
|
65
|
-
move_connections=True
|
|
66
70
|
)
|
|
67
71
|
dms_rules_file_name = "dms_rules.xlsx"
|
|
68
72
|
neat.to.excel(dms_rules_file_name, include_reference=True)
|
|
69
73
|
```
|
|
70
74
|
"""
|
|
71
75
|
reference_rules_with_prefix: tuple[VerifiedRules, str] | None = None
|
|
76
|
+
include_properties = include_properties.strip().lower()
|
|
77
|
+
|
|
72
78
|
if include_reference and self._state.last_reference:
|
|
73
79
|
if (
|
|
74
80
|
isinstance(self._state.last_reference.metadata, DMSMetadata)
|
|
@@ -79,7 +85,16 @@ class ToAPI:
|
|
|
79
85
|
prefix = "Ref"
|
|
80
86
|
reference_rules_with_prefix = self._state.last_reference, prefix
|
|
81
87
|
|
|
82
|
-
|
|
88
|
+
if include_properties == "same-space":
|
|
89
|
+
warnings.filterwarnings("default")
|
|
90
|
+
AlphaFlags.same_space_properties_only_export.warn()
|
|
91
|
+
|
|
92
|
+
exporter = exporters.ExcelExporter(
|
|
93
|
+
styling="maximal",
|
|
94
|
+
reference_rules_with_prefix=reference_rules_with_prefix,
|
|
95
|
+
add_empty_rows=add_empty_rows,
|
|
96
|
+
include_properties=include_properties, # type: ignore
|
|
97
|
+
)
|
|
83
98
|
return self._state.rule_store.export_to_file(exporter, Path(io))
|
|
84
99
|
|
|
85
100
|
def session(self, io: Any) -> None:
|