cognite-neat 0.109.1__py3-none-any.whl → 0.109.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. cognite/neat/_alpha.py +15 -0
  2. cognite/neat/_client/testing.py +1 -1
  3. cognite/neat/_issues/_base.py +33 -9
  4. cognite/neat/_issues/errors/__init__.py +2 -10
  5. cognite/neat/_issues/errors/_general.py +1 -1
  6. cognite/neat/_issues/errors/_wrapper.py +11 -0
  7. cognite/neat/_rules/exporters/_rules2excel.py +31 -1
  8. cognite/neat/_rules/models/_rdfpath.py +2 -0
  9. cognite/neat/_rules/models/_types.py +4 -2
  10. cognite/neat/_rules/models/dms/_rules.py +0 -36
  11. cognite/neat/_rules/models/entities/_constants.py +3 -0
  12. cognite/neat/_rules/models/entities/_single_value.py +6 -1
  13. cognite/neat/_rules/models/entities/_wrapped.py +3 -0
  14. cognite/neat/_rules/transformers/__init__.py +4 -0
  15. cognite/neat/_rules/transformers/_converters.py +221 -15
  16. cognite/neat/_session/_base.py +7 -0
  17. cognite/neat/_session/_collector.py +4 -1
  18. cognite/neat/_session/_create.py +46 -12
  19. cognite/neat/_session/_prepare.py +11 -3
  20. cognite/neat/_session/_read.py +14 -2
  21. cognite/neat/_session/_state.py +7 -3
  22. cognite/neat/_session/_to.py +20 -5
  23. cognite/neat/_session/exceptions.py +16 -6
  24. cognite/neat/_store/_provenance.py +1 -0
  25. cognite/neat/_store/_rules_store.py +192 -127
  26. cognite/neat/_utils/spreadsheet.py +10 -1
  27. cognite/neat/_utils/text.py +40 -9
  28. cognite/neat/_version.py +1 -1
  29. {cognite_neat-0.109.1.dist-info → cognite_neat-0.109.3.dist-info}/METADATA +1 -1
  30. {cognite_neat-0.109.1.dist-info → cognite_neat-0.109.3.dist-info}/RECORD +33 -32
  31. cognite/neat/_issues/errors/_workflow.py +0 -36
  32. {cognite_neat-0.109.1.dist-info → cognite_neat-0.109.3.dist-info}/LICENSE +0 -0
  33. {cognite_neat-0.109.1.dist-info → cognite_neat-0.109.3.dist-info}/WHEEL +0 -0
  34. {cognite_neat-0.109.1.dist-info → cognite_neat-0.109.3.dist-info}/entry_points.txt +0 -0
@@ -1,9 +1,11 @@
1
1
  import re
2
+ import urllib.parse
2
3
  import warnings
3
4
  from abc import ABC
4
5
  from collections import Counter, defaultdict
5
6
  from collections.abc import Collection, Mapping
6
7
  from datetime import date, datetime
8
+ from functools import cached_property
7
9
  from typing import ClassVar, Literal, TypeVar, cast, overload
8
10
 
9
11
  from cognite.client.data_classes import data_modeling as dms
@@ -23,11 +25,12 @@ from cognite.neat._constants import (
23
25
  from cognite.neat._issues.errors import NeatValueError
24
26
  from cognite.neat._issues.warnings import NeatValueWarning
25
27
  from cognite.neat._issues.warnings._models import (
26
- EnterpriseModelNotBuildOnTopOfCDMWarning,
27
28
  SolutionModelBuildOnTopOfCDMWarning,
28
29
  )
30
+ from cognite.neat._rules._constants import PATTERNS, get_reserved_words
29
31
  from cognite.neat._rules._shared import (
30
32
  ReadInputRules,
33
+ ReadRules,
31
34
  VerifiedRules,
32
35
  )
33
36
  from cognite.neat._rules.analysis import DMSAnalysis
@@ -35,6 +38,7 @@ from cognite.neat._rules.importers import DMSImporter
35
38
  from cognite.neat._rules.models import (
36
39
  DMSInputRules,
37
40
  DMSRules,
41
+ InformationInputRules,
38
42
  InformationRules,
39
43
  SheetList,
40
44
  data_types,
@@ -56,9 +60,9 @@ from cognite.neat._rules.models.entities import (
56
60
  ViewEntity,
57
61
  )
58
62
  from cognite.neat._rules.models.information import InformationClass, InformationMetadata, InformationProperty
59
- from cognite.neat._utils.text import to_camel
63
+ from cognite.neat._utils.text import NamingStandardization, to_camel
60
64
 
61
- from ._base import T_VerifiedIn, T_VerifiedOut, VerifiedRulesTransformer
65
+ from ._base import RulesTransformer, T_VerifiedIn, T_VerifiedOut, VerifiedRulesTransformer
62
66
  from ._verification import VerifyDMSRules
63
67
 
64
68
  T_InputInRules = TypeVar("T_InputInRules", bound=ReadInputRules)
@@ -71,6 +75,105 @@ class ConversionTransformer(VerifiedRulesTransformer[T_VerifiedIn, T_VerifiedOut
71
75
  ...
72
76
 
73
77
 
78
+ class ToInformationCompliantEntities(
79
+ RulesTransformer[ReadRules[InformationInputRules], ReadRules[InformationInputRules]]
80
+ ):
81
+ """Converts input rules to rules that is compliant with the Information Model.
82
+
83
+ This is typically used with importers from arbitrary sources to ensure that classes and properties have valid
84
+ names.
85
+
86
+ Args:
87
+ renaming: How to handle renaming of entities that are not compliant with the Information Model.
88
+ - "warning": Raises a warning and renames the entity.
89
+ - "skip": Renames the entity without raising a warning.
90
+ """
91
+
92
+ def __init__(self, renaming: Literal["warning", "skip"] = "skip") -> None:
93
+ self._renaming = renaming
94
+
95
+ @property
96
+ def description(self) -> str:
97
+ return "Ensures that all entities are compliant with the Information Model."
98
+
99
+ def transform(self, rules: ReadRules[InformationInputRules]) -> ReadRules[InformationInputRules]:
100
+ if rules.rules is None:
101
+ return rules
102
+ # Doing dump to obtain a copy, and ensure that all entities are created. Input allows
103
+ # string for entities, the dump call will convert these to entities.
104
+ dumped = rules.rules.dump()
105
+ copy = InformationInputRules.load(dumped)
106
+
107
+ new_by_old_class_suffix: dict[str, str] = {}
108
+ for cls in copy.classes:
109
+ cls_entity = cast(ClassEntity, cls.class_) # Safe due to the dump above
110
+ if not PATTERNS.class_id_compliance.match(cls_entity.suffix):
111
+ new_suffix = self._fix_cls_suffix(cls_entity.suffix)
112
+ if self._renaming == "warning":
113
+ warnings.warn(
114
+ NeatValueWarning(f"Invalid class name {cls_entity.suffix!r}.Renaming to {new_suffix}"),
115
+ stacklevel=2,
116
+ )
117
+ cls.class_.suffix = new_suffix # type: ignore[union-attr]
118
+
119
+ for cls_ in copy.classes:
120
+ if cls_.implements:
121
+ for i, parent in enumerate(cls_.implements):
122
+ if isinstance(parent, ClassEntity) and parent.suffix in new_by_old_class_suffix:
123
+ cls_.implements[i].suffix = new_by_old_class_suffix[parent.suffix] # type: ignore[union-attr]
124
+
125
+ for prop in copy.properties:
126
+ if not PATTERNS.information_property_id_compliance.match(prop.property_):
127
+ new_property = self._fix_property(prop.property_)
128
+ if self._renaming == "warning":
129
+ warnings.warn(
130
+ NeatValueWarning(
131
+ f"Invalid property name {prop.class_.suffix}.{prop.property_!r}. Renaming to {new_property}" # type: ignore[union-attr]
132
+ ),
133
+ stacklevel=2,
134
+ )
135
+ prop.property_ = new_property
136
+
137
+ if isinstance(prop.class_, ClassEntity) and prop.class_.suffix in new_by_old_class_suffix:
138
+ prop.class_.suffix = new_by_old_class_suffix[prop.class_.suffix]
139
+
140
+ if isinstance(prop.value_type, ClassEntity) and prop.value_type.suffix in new_by_old_class_suffix:
141
+ prop.value_type.suffix = new_by_old_class_suffix[prop.value_type.suffix]
142
+
143
+ if isinstance(prop.value_type, MultiValueTypeInfo):
144
+ for i, value_type in enumerate(prop.value_type.types):
145
+ if isinstance(value_type, ClassEntity) and value_type.suffix in new_by_old_class_suffix:
146
+ prop.value_type.types[i].suffix = new_by_old_class_suffix[value_type.suffix] # type: ignore[union-attr]
147
+
148
+ return ReadRules(rules=copy, read_context=rules.read_context)
149
+
150
+ @cached_property
151
+ def _reserved_class_words(self) -> set[str]:
152
+ return set(get_reserved_words("class"))
153
+
154
+ @cached_property
155
+ def _reserved_property_words(self) -> set[str]:
156
+ return set(get_reserved_words("property"))
157
+
158
+ def _fix_cls_suffix(self, suffix: str) -> str:
159
+ if suffix in self._reserved_class_words:
160
+ return f"My{suffix}"
161
+ suffix = urllib.parse.unquote(suffix)
162
+ suffix = NamingStandardization.standardize_class_str(suffix)
163
+ if len(suffix) > 252:
164
+ suffix = suffix[:252]
165
+ return suffix
166
+
167
+ def _fix_property(self, property_: str) -> str:
168
+ if property_ in self._reserved_property_words:
169
+ return f"my{property_}"
170
+ property_ = urllib.parse.unquote(property_)
171
+ property_ = NamingStandardization.standardize_property_str(property_)
172
+ if len(property_) > 252:
173
+ property_ = property_[:252]
174
+ return property_
175
+
176
+
74
177
  class ToCompliantEntities(VerifiedRulesTransformer[InformationRules, InformationRules]): # type: ignore[misc]
75
178
  """Converts input rules to rules with compliant entity IDs that match regex patters used
76
179
  by DMS schema components."""
@@ -246,6 +349,116 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
246
349
  return entity
247
350
 
248
351
 
352
+ class StandardizeNaming(ConversionTransformer):
353
+ """Sets views/classes/container names to PascalCase and properties to camelCase."""
354
+
355
+ @property
356
+ def description(self) -> str:
357
+ return "Sets views/classes/containers names to PascalCase and properties to camelCase."
358
+
359
+ @overload
360
+ def transform(self, rules: DMSRules) -> DMSRules: ...
361
+
362
+ @overload
363
+ def transform(self, rules: InformationRules) -> InformationRules: ...
364
+
365
+ def transform(self, rules: InformationRules | DMSRules) -> InformationRules | DMSRules:
366
+ output = rules.model_copy(deep=True)
367
+ if isinstance(output, InformationRules):
368
+ return self._standardize_information_rules(output)
369
+ elif isinstance(output, DMSRules):
370
+ return self._standardize_dms_rules(output)
371
+ raise NeatValueError(f"Unsupported rules type: {type(output)}")
372
+
373
+ def _standardize_information_rules(self, rules: InformationRules) -> InformationRules:
374
+ new_by_old_class_suffix: dict[str, str] = {}
375
+ for cls in rules.classes:
376
+ new_suffix = NamingStandardization.standardize_class_str(cls.class_.suffix)
377
+ new_by_old_class_suffix[cls.class_.suffix] = new_suffix
378
+ cls.class_.suffix = new_suffix
379
+
380
+ for cls in rules.classes:
381
+ if cls.implements:
382
+ for i, parent in enumerate(cls.implements):
383
+ if parent.suffix in new_by_old_class_suffix:
384
+ cls.implements[i].suffix = new_by_old_class_suffix[parent.suffix]
385
+
386
+ for prop in rules.properties:
387
+ prop.property_ = NamingStandardization.standardize_property_str(prop.property_)
388
+ if prop.class_.suffix in new_by_old_class_suffix:
389
+ prop.class_.suffix = new_by_old_class_suffix[prop.class_.suffix]
390
+
391
+ if isinstance(prop.value_type, ClassEntity) and prop.value_type.suffix in new_by_old_class_suffix:
392
+ prop.value_type.suffix = new_by_old_class_suffix[prop.value_type.suffix]
393
+
394
+ if isinstance(prop.value_type, MultiValueTypeInfo):
395
+ for i, value_type in enumerate(prop.value_type.types):
396
+ if isinstance(value_type, ClassEntity) and value_type.suffix in new_by_old_class_suffix:
397
+ prop.value_type.types[i].suffix = new_by_old_class_suffix[value_type.suffix] # type: ignore[union-attr]
398
+
399
+ return rules
400
+
401
+ def _standardize_dms_rules(self, rules: DMSRules) -> DMSRules:
402
+ new_by_old_view: dict[str, str] = {}
403
+ for view in rules.views:
404
+ new_suffix = NamingStandardization.standardize_class_str(view.view.suffix)
405
+ new_by_old_view[view.view.suffix] = new_suffix
406
+ view.view.suffix = new_suffix
407
+ new_by_old_container: dict[str, str] = {}
408
+ if rules.containers:
409
+ for container in rules.containers:
410
+ new_suffix = NamingStandardization.standardize_class_str(container.container.suffix)
411
+ new_by_old_container[container.container.suffix] = new_suffix
412
+ container.container.suffix = new_suffix
413
+
414
+ for view in rules.views:
415
+ if view.implements:
416
+ for i, parent in enumerate(view.implements):
417
+ if parent.suffix in new_by_old_view:
418
+ view.implements[i].suffix = new_by_old_view[parent.suffix]
419
+ if view.filter_ and isinstance(view.filter_, HasDataFilter) and view.filter_.inner:
420
+ for i, item in enumerate(view.filter_.inner):
421
+ if isinstance(item, ContainerEntity) and item.suffix in new_by_old_container:
422
+ view.filter_.inner[i].suffix = new_by_old_container[item.suffix]
423
+ if isinstance(item, ViewEntity) and item.suffix in new_by_old_view:
424
+ view.filter_.inner[i].suffix = new_by_old_view[item.suffix]
425
+ if rules.containers:
426
+ for container in rules.containers:
427
+ if container.constraint:
428
+ for i, constraint in enumerate(container.constraint):
429
+ if constraint.suffix in new_by_old_container:
430
+ container.constraint[i].suffix = new_by_old_container[constraint.suffix]
431
+ new_property_by_view_by_old_property: dict[ViewEntity, dict[str, str]] = defaultdict(dict)
432
+ for prop in rules.properties:
433
+ if prop.view.suffix in new_by_old_view:
434
+ prop.view.suffix = new_by_old_view[prop.view.suffix]
435
+ new_view_property = NamingStandardization.standardize_property_str(prop.view_property)
436
+ new_property_by_view_by_old_property[prop.view][prop.view_property] = new_view_property
437
+ prop.view_property = new_view_property
438
+ if isinstance(prop.value_type, ViewEntity) and prop.value_type.suffix in new_by_old_view:
439
+ prop.value_type.suffix = new_by_old_view[prop.value_type.suffix]
440
+ if (
441
+ isinstance(prop.connection, EdgeEntity)
442
+ and prop.connection.properties
443
+ and prop.connection.properties.suffix in new_by_old_view
444
+ ):
445
+ prop.connection.properties.suffix = new_by_old_view[prop.connection.properties.suffix]
446
+ if isinstance(prop.container, ContainerEntity) and prop.container.suffix in new_by_old_container:
447
+ prop.container.suffix = new_by_old_container[prop.container.suffix]
448
+ if prop.container_property:
449
+ prop.container_property = NamingStandardization.standardize_property_str(prop.container_property)
450
+ for prop in rules.properties:
451
+ if (
452
+ isinstance(prop.connection, ReverseConnectionEntity)
453
+ and isinstance(prop.value_type, ViewEntity)
454
+ and prop.value_type in new_property_by_view_by_old_property
455
+ ):
456
+ new_by_old_property = new_property_by_view_by_old_property[prop.value_type]
457
+ if prop.connection.property_ in new_by_old_property:
458
+ prop.connection.property_ = new_by_old_property[prop.connection.property_]
459
+ return rules
460
+
461
+
249
462
  class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
250
463
  """Converts InformationRules to DMSRules."""
251
464
 
@@ -339,13 +552,6 @@ class ToEnterpriseModel(ToExtensionModel):
339
552
  self.move_connections = move_connections
340
553
 
341
554
  def transform(self, rules: DMSRules) -> DMSRules:
342
- reference_model_id = rules.metadata.as_data_model_id()
343
- if reference_model_id not in COGNITE_MODELS:
344
- warnings.warn(
345
- EnterpriseModelNotBuildOnTopOfCDMWarning(reference_model_id=reference_model_id).as_message(),
346
- stacklevel=2,
347
- )
348
-
349
555
  return self._to_enterprise(rules)
350
556
 
351
557
  def _to_enterprise(self, reference_model: DMSRules) -> DMSRules:
@@ -565,12 +771,10 @@ class ToSolutionModel(ToExtensionModel):
565
771
  renaming: dict[ViewEntity, ViewEntity] = {}
566
772
  new_views = SheetList[DMSView]()
567
773
  read_view_by_new_view: dict[ViewEntity, ViewEntity] = {}
568
- skipped_views: set[ViewEntity] = set()
569
774
  for ref_view in reference.views:
570
775
  if (self.skip_cognite_views and ref_view.view.space in COGNITE_SPACES) or (
571
776
  self.exclude_views_in_other_spaces and ref_view.view.space != reference.metadata.space
572
777
  ):
573
- skipped_views.add(ref_view.view)
574
778
  continue
575
779
  new_entity = ViewEntity(
576
780
  # MyPy we validate that version is string in the constructor
@@ -603,15 +807,17 @@ class ToSolutionModel(ToExtensionModel):
603
807
  new_views.append(ref_view.model_copy(deep=True, update={"implements": None, "view": new_entity}))
604
808
 
605
809
  new_properties = SheetList[DMSProperty]()
810
+ new_view_entities = {view.view for view in new_views}
606
811
  for prop in reference.properties:
607
- if prop.view in skipped_views:
608
- continue
609
812
  new_property = prop.model_copy(deep=True)
610
813
  if new_property.value_type in renaming and isinstance(new_property.value_type, ViewEntity):
611
814
  new_property.value_type = renaming[new_property.value_type]
612
815
  if new_property.view in renaming:
613
816
  new_property.view = renaming[new_property.view]
614
- new_properties.append(new_property)
817
+ if new_property.view in new_view_entities and (
818
+ not isinstance(new_property.value_type, ViewEntity) or new_property.value_type in new_view_entities
819
+ ):
820
+ new_properties.append(new_property)
615
821
  return new_views, new_properties, read_view_by_new_view
616
822
 
617
823
  def _create_containers_update_view_filter(
@@ -16,6 +16,7 @@ from cognite.neat._rules.transformers import (
16
16
  InformationToDMS,
17
17
  MergeDMSRules,
18
18
  MergeInformationRules,
19
+ ToInformationCompliantEntities,
19
20
  VerifyInformationRules,
20
21
  )
21
22
  from cognite.neat._store._rules_store import RulesEntity
@@ -235,13 +236,19 @@ class NeatSession:
235
236
 
236
237
  def action() -> tuple[InformationRules, DMSRules | None]:
237
238
  unverified_information = importer.to_rules()
239
+ unverified_information = ToInformationCompliantEntities(renaming="warning").transform(
240
+ unverified_information
241
+ )
242
+
238
243
  extra_info = VerifyInformationRules().transform(unverified_information)
239
244
  if not last_entity:
240
245
  return extra_info, None
241
246
  merged_info = MergeInformationRules(extra_info).transform(last_entity.information)
242
247
  if not last_entity.dms:
243
248
  return merged_info, None
249
+
244
250
  extra_dms = InformationToDMS(reserved_properties="warning").transform(extra_info)
251
+
245
252
  merged_dms = MergeDMSRules(extra_dms).transform(last_entity.dms)
246
253
  return merged_info, merged_dms
247
254
 
@@ -65,7 +65,10 @@ class Collector:
65
65
  if kwargs:
66
66
  for key, value in kwargs.items():
67
67
  event_information[key] = self._serialize_value(value)[:500]
68
- self._track(command, event_information)
68
+
69
+ with suppress(RuntimeError):
70
+ # In case any thread issues, the tracking should not crash the program
71
+ self._track(command, event_information)
69
72
 
70
73
  @staticmethod
71
74
  def _serialize_value(value: Any) -> str:
@@ -3,6 +3,7 @@ from typing import Literal
3
3
  from cognite.client.data_classes.data_modeling import DataModelIdentifier
4
4
 
5
5
  from cognite.neat._issues import IssueList
6
+ from cognite.neat._rules.models import DMSRules, InformationRules
6
7
  from cognite.neat._rules.models.dms import DMSValidation
7
8
  from cognite.neat._rules.transformers import (
8
9
  IncludeReferenced,
@@ -38,22 +39,27 @@ class CreateAPI:
38
39
  org_name: Organization name to use for the views in the enterprise data model.
39
40
  dummy_property: The dummy property to use as placeholder for the views in the new data model.
40
41
 
42
+ What does this function do?
43
+ 1. It creates a new view for each view in the current data model that implements the view it is based on.
44
+ 2. If dummy_property is set, it will create a container with one property for each view and connect the
45
+ view to the container.
46
+ 3. It will repeat all connection properties in the new views and update the ValueTypes to match the new
47
+ views.
48
+
41
49
  !!! note "Enterprise Data Model Creation"
42
50
 
43
51
  Always create an enterprise data model from a Cognite Data Model as this will
44
52
  assure all the Cognite Data Fusion applications to run smoothly, such as
45
53
  - Search
46
54
  - Atlas AI
47
- - ...
48
-
49
- !!! note "Move Connections"
50
-
51
- If you want to move the connections to the new data model, set the move_connections
52
- to True. This will move the connections to the new data model and use new model
53
- views as the source and target views.
55
+ - Infield
56
+ - Canvas
57
+ - Maintain
58
+ - Charts
54
59
 
55
60
  """
56
- return self._state.rule_transform(
61
+ last_rules = self._get_last_rules()
62
+ issues = self._state.rule_transform(
57
63
  ToEnterpriseModel(
58
64
  new_model_id=data_model_id,
59
65
  org_name=org_name,
@@ -61,6 +67,15 @@ class CreateAPI:
61
67
  move_connections=True,
62
68
  )
63
69
  )
70
+ if last_rules and not issues.has_errors:
71
+ self._state.last_reference = last_rules
72
+ return issues
73
+
74
+ def _get_last_rules(self) -> InformationRules | DMSRules | None:
75
+ if not self._state.rule_store.provenance:
76
+ return None
77
+ last_entity = self._state.rule_store.provenance[-1].target_entity
78
+ return last_entity.dms or last_entity.information
64
79
 
65
80
  def solution_model(
66
81
  self,
@@ -76,6 +91,13 @@ class CreateAPI:
76
91
  and the enterprise data model.
77
92
  view_prefix: The prefix to use for the views in the enterprise data model.
78
93
 
94
+ What does this function do?
95
+ 1. It will create two new views for each view in the current data model. The first view will be read-only and
96
+ prefixed with the 'view_prefix'. The second view will be writable and have one property that connects to the
97
+ read-only view named 'direct_property'.
98
+ 2. It will repeat all connection properties in the new views and update the ValueTypes to match the new views.
99
+ 3. Each writable view will have a container with the single property that connects to the read-only view.
100
+
79
101
  !!! note "Solution Data Model Mode"
80
102
 
81
103
  The read-only solution model will only be able to read from the existing containers
@@ -88,7 +110,8 @@ class CreateAPI:
88
110
  the containers in the solution data model space.
89
111
 
90
112
  """
91
- return self._state.rule_transform(
113
+ last_rules = self._get_last_rules()
114
+ issues = self._state.rule_transform(
92
115
  ToSolutionModel(
93
116
  new_model_id=data_model_id,
94
117
  properties="connection",
@@ -96,24 +119,32 @@ class CreateAPI:
96
119
  view_prefix=view_prefix,
97
120
  )
98
121
  )
122
+ if last_rules and not issues.has_errors:
123
+ self._state.last_reference = last_rules
124
+ return issues
99
125
 
100
126
  def data_product_model(
101
127
  self,
102
128
  data_model_id: DataModelIdentifier,
103
129
  include: Literal["same-space", "all"] = "same-space",
104
- ) -> None:
130
+ ) -> IssueList:
105
131
  """Uses the current data model as a basis to create data product data model.
106
132
 
107
133
  A data product model is a data model that ONLY maps to containers and do not use implements. This is
108
134
  typically used for defining the data in a data product.
109
135
 
136
+ What does this function do?
137
+ 1. It creates a new view for each view in the current data model. The new views uses the same filter
138
+ as the view it is based on.
139
+ 2. It will repeat all connection properties in the new views and update the ValueTypes to match the new views.
140
+
110
141
  Args:
111
142
  data_model_id: The data product data model id that is being created.
112
143
  include: The views to include in the data product data model. Can be either "same-space" or "all".
113
144
  If you set same-space, only the properties of the views in the same space as the data model
114
145
  will be included.
115
146
  """
116
-
147
+ last_rules = self._get_last_rules()
117
148
  view_ids, container_ids = DMSValidation(
118
149
  self._state.rule_store.last_verified_dms_rules
119
150
  ).imported_views_and_containers_ids()
@@ -130,4 +161,7 @@ class CreateAPI:
130
161
 
131
162
  transformers.append(ToDataProductModel(new_model_id=data_model_id, include=include))
132
163
 
133
- self._state.rule_transform(*transformers)
164
+ issues = self._state.rule_transform(*transformers)
165
+ if last_rules and not issues.has_errors:
166
+ self._state.last_reference = last_rules
167
+ return issues
@@ -3,6 +3,7 @@ from typing import Any
3
3
 
4
4
  from rdflib import URIRef
5
5
 
6
+ from cognite.neat._alpha import AlphaFlags
6
7
  from cognite.neat._graph.transformers import (
7
8
  ConnectionToLiteral,
8
9
  ConvertLiteral,
@@ -12,9 +13,7 @@ from cognite.neat._graph.transformers import (
12
13
  from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
13
14
  from cognite.neat._issues import IssueList
14
15
  from cognite.neat._issues.errors import NeatValueError
15
- from cognite.neat._rules.transformers import (
16
- PrefixEntities,
17
- )
16
+ from cognite.neat._rules.transformers import PrefixEntities, StandardizeNaming
18
17
  from cognite.neat._utils.text import humanize_collection
19
18
 
20
19
  from ._state import SessionState
@@ -264,3 +263,12 @@ class DataModelPrepareAPI:
264
263
  """
265
264
 
266
265
  return self._state.rule_transform(PrefixEntities(prefix)) # type: ignore[arg-type]
266
+
267
+ def standardize_naming(self) -> IssueList:
268
+ """Standardize the naming of all views/classes/properties in the data model.
269
+
270
+ For classes/views/containers, the naming will be standardized to PascalCase.
271
+ For properties, the naming will be standardized to camelCase.
272
+ """
273
+ AlphaFlags.standardize_naming.warn()
274
+ return self._state.rule_transform(StandardizeNaming())
@@ -1,8 +1,10 @@
1
+ import warnings
1
2
  from typing import Any, Literal, cast
2
3
 
3
4
  from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
4
5
  from cognite.client.utils.useful_types import SequenceNotStr
5
6
 
7
+ from cognite.neat._alpha import AlphaFlags
6
8
  from cognite.neat._client import NeatClient
7
9
  from cognite.neat._constants import (
8
10
  CLASSIC_CDF_NAMESPACE,
@@ -286,16 +288,26 @@ class ExcelReadAPI(BaseReadAPI):
286
288
  super().__init__(state, verbose)
287
289
  self.examples = ExcelExampleAPI(state, verbose)
288
290
 
289
- def __call__(self, io: Any) -> IssueList:
291
+ def __call__(self, io: Any, enable_manual_edit: bool = False) -> IssueList:
290
292
  """Reads a Neat Excel Rules sheet to the graph store. The rules sheet may stem from an Information architect,
291
293
  or a DMS Architect.
292
294
 
293
295
  Args:
294
296
  io: file path to the Excel sheet
297
+ enable_manual_edit: If True, the user will be able to re-import rules which where edit outside NeatSession
298
+
299
+ !!! note "Manual Edit Warning"
300
+ This is an alpha feature and is subject to change without notice.
301
+ It is expected to have some limitations and may not work as expected in all cases.
295
302
  """
296
303
  reader = NeatReader.create(io)
297
304
  path = reader.materialize_path()
298
- return self._state.rule_import(importers.ExcelImporter(path))
305
+
306
+ if enable_manual_edit:
307
+ warnings.filterwarnings("default")
308
+ AlphaFlags.manual_rules_edit.warn()
309
+
310
+ return self._state.rule_import(importers.ExcelImporter(path), enable_manual_edit)
299
311
 
300
312
 
301
313
  @session_class_wrapper
@@ -40,8 +40,12 @@ class SessionState:
40
40
  self.instances.store.add_rules(last_entity.information)
41
41
  return issues
42
42
 
43
- def rule_import(self, importer: BaseImporter) -> IssueList:
44
- issues = self.rule_store.import_rules(importer, client=self.client)
43
+ def rule_import(self, importer: BaseImporter, enable_manual_edit: bool = False) -> IssueList:
44
+ issues = self.rule_store.import_rules(
45
+ importer,
46
+ client=self.client,
47
+ enable_manual_edit=enable_manual_edit,
48
+ )
45
49
  if self.rule_store.empty:
46
50
  result = "failed"
47
51
  else:
@@ -74,7 +78,7 @@ class InstancesState:
74
78
  self.outcome = UploadResultList()
75
79
 
76
80
  # Ensure that error handling is done in the constructor
77
- self.store = _session_method_wrapper(self._create_store, "NeatSession")()
81
+ self.store: NeatGraphStore = _session_method_wrapper(self._create_store, "NeatSession")()
78
82
 
79
83
  if self.storage_path:
80
84
  print("Remember to close neat session .close() once you are done to avoid oxigraph lock.")
@@ -6,6 +6,7 @@ from typing import Any, Literal, overload
6
6
 
7
7
  from cognite.client import data_modeling as dm
8
8
 
9
+ from cognite.neat._alpha import AlphaFlags
9
10
  from cognite.neat._constants import COGNITE_MODELS
10
11
  from cognite.neat._graph import loaders
11
12
  from cognite.neat._rules import exporters
@@ -35,6 +36,8 @@ class ToAPI:
35
36
  self,
36
37
  io: Any,
37
38
  include_reference: bool = True,
39
+ include_properties: Literal["same-space", "all"] = "all",
40
+ add_empty_rows: bool = False,
38
41
  ) -> None:
39
42
  """Export the verified data model to Excel.
40
43
 
@@ -42,7 +45,10 @@ class ToAPI:
42
45
  io: The file path or file-like object to write the Excel file to.
43
46
  include_reference: If True, the reference data model will be included. Defaults to True.
44
47
  Note that this only applies if you have created the data model using the
45
- .to_enterprise(), .to_solution(), or .to_data_product() methods.
48
+ create.enterprise_model(...), create.solution_model(), or create.data_product_model() methods.
49
+ include_properties: The properties to include in the Excel file. Defaults to "all".
50
+ - "same-space": Only properties that are in the same space as the data model will be included.
51
+ add_empty_rows: If True, empty rows will be added between each component. Defaults to False.
46
52
 
47
53
  Example:
48
54
  Export information model to excel rules sheet
@@ -58,17 +64,17 @@ class ToAPI:
58
64
  neat = NeatSession(client)
59
65
 
60
66
  neat.read.cdf(("cdf_cdm", "CogniteCore", "v1"))
61
- neat.verify()
62
- neat.prepare.data_model.to_enterprise(
67
+ neat.create.enterprise_model(
63
68
  data_model_id=("sp_doctrino_space", "ExtensionCore", "v1"),
64
69
  org_name="MyOrg",
65
- move_connections=True
66
70
  )
67
71
  dms_rules_file_name = "dms_rules.xlsx"
68
72
  neat.to.excel(dms_rules_file_name, include_reference=True)
69
73
  ```
70
74
  """
71
75
  reference_rules_with_prefix: tuple[VerifiedRules, str] | None = None
76
+ include_properties = include_properties.strip().lower()
77
+
72
78
  if include_reference and self._state.last_reference:
73
79
  if (
74
80
  isinstance(self._state.last_reference.metadata, DMSMetadata)
@@ -79,7 +85,16 @@ class ToAPI:
79
85
  prefix = "Ref"
80
86
  reference_rules_with_prefix = self._state.last_reference, prefix
81
87
 
82
- exporter = exporters.ExcelExporter(styling="maximal", reference_rules_with_prefix=reference_rules_with_prefix)
88
+ if include_properties == "same-space":
89
+ warnings.filterwarnings("default")
90
+ AlphaFlags.same_space_properties_only_export.warn()
91
+
92
+ exporter = exporters.ExcelExporter(
93
+ styling="maximal",
94
+ reference_rules_with_prefix=reference_rules_with_prefix,
95
+ add_empty_rows=add_empty_rows,
96
+ include_properties=include_properties, # type: ignore
97
+ )
83
98
  return self._state.rule_store.export_to_file(exporter, Path(io))
84
99
 
85
100
  def session(self, io: Any) -> None: