cognite-neat 0.103.1__py3-none-any.whl → 0.105.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (175) hide show
  1. cognite/neat/_client/_api/data_modeling_loaders.py +83 -23
  2. cognite/neat/_client/_api/schema.py +2 -1
  3. cognite/neat/_client/data_classes/neat_sequence.py +261 -0
  4. cognite/neat/_client/data_classes/schema.py +5 -1
  5. cognite/neat/_client/testing.py +33 -0
  6. cognite/neat/_constants.py +56 -0
  7. cognite/neat/_graph/extractors/_classic_cdf/_base.py +6 -5
  8. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +225 -11
  9. cognite/neat/_graph/extractors/_mock_graph_generator.py +2 -2
  10. cognite/neat/_graph/loaders/_rdf2dms.py +13 -2
  11. cognite/neat/_graph/transformers/__init__.py +3 -1
  12. cognite/neat/_graph/transformers/_base.py +109 -1
  13. cognite/neat/_graph/transformers/_classic_cdf.py +6 -1
  14. cognite/neat/_graph/transformers/_prune_graph.py +103 -47
  15. cognite/neat/_graph/transformers/_rdfpath.py +41 -17
  16. cognite/neat/_graph/transformers/_value_type.py +188 -151
  17. cognite/neat/_issues/__init__.py +0 -2
  18. cognite/neat/_issues/_base.py +54 -43
  19. cognite/neat/_issues/warnings/__init__.py +4 -1
  20. cognite/neat/_issues/warnings/_general.py +7 -0
  21. cognite/neat/_issues/warnings/_resources.py +12 -1
  22. cognite/neat/_rules/_shared.py +18 -34
  23. cognite/neat/_rules/exporters/_base.py +28 -2
  24. cognite/neat/_rules/exporters/_rules2dms.py +39 -1
  25. cognite/neat/_rules/exporters/_rules2excel.py +13 -2
  26. cognite/neat/_rules/exporters/_rules2instance_template.py +4 -0
  27. cognite/neat/_rules/exporters/_rules2ontology.py +13 -1
  28. cognite/neat/_rules/exporters/_rules2yaml.py +4 -0
  29. cognite/neat/_rules/importers/_base.py +9 -0
  30. cognite/neat/_rules/importers/_dms2rules.py +80 -57
  31. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +5 -2
  32. cognite/neat/_rules/importers/_rdf/_base.py +10 -8
  33. cognite/neat/_rules/importers/_rdf/_imf2rules.py +4 -0
  34. cognite/neat/_rules/importers/_rdf/_inference2rules.py +7 -0
  35. cognite/neat/_rules/importers/_rdf/_owl2rules.py +4 -0
  36. cognite/neat/_rules/importers/_spreadsheet2rules.py +17 -8
  37. cognite/neat/_rules/importers/_yaml2rules.py +21 -7
  38. cognite/neat/_rules/models/_base_input.py +1 -1
  39. cognite/neat/_rules/models/_base_rules.py +9 -1
  40. cognite/neat/_rules/models/dms/_rules.py +4 -0
  41. cognite/neat/_rules/models/dms/_rules_input.py +9 -0
  42. cognite/neat/_rules/models/entities/_wrapped.py +10 -5
  43. cognite/neat/_rules/models/information/_rules.py +4 -0
  44. cognite/neat/_rules/models/information/_rules_input.py +9 -0
  45. cognite/neat/_rules/models/mapping/_classic2core.py +2 -5
  46. cognite/neat/_rules/models/mapping/_classic2core.yaml +239 -38
  47. cognite/neat/_rules/transformers/__init__.py +13 -6
  48. cognite/neat/_rules/transformers/_base.py +41 -65
  49. cognite/neat/_rules/transformers/_converters.py +404 -234
  50. cognite/neat/_rules/transformers/_mapping.py +93 -72
  51. cognite/neat/_rules/transformers/_verification.py +50 -38
  52. cognite/neat/_session/_base.py +32 -121
  53. cognite/neat/_session/_inspect.py +5 -3
  54. cognite/neat/_session/_mapping.py +17 -105
  55. cognite/neat/_session/_prepare.py +138 -268
  56. cognite/neat/_session/_read.py +39 -195
  57. cognite/neat/_session/_set.py +6 -30
  58. cognite/neat/_session/_show.py +40 -21
  59. cognite/neat/_session/_state.py +49 -107
  60. cognite/neat/_session/_to.py +44 -33
  61. cognite/neat/_shared.py +23 -2
  62. cognite/neat/_store/_provenance.py +3 -82
  63. cognite/neat/_store/_rules_store.py +368 -10
  64. cognite/neat/_store/exceptions.py +23 -0
  65. cognite/neat/_utils/graph_transformations_report.py +36 -0
  66. cognite/neat/_utils/rdf_.py +8 -0
  67. cognite/neat/_utils/reader/_base.py +27 -0
  68. cognite/neat/_utils/spreadsheet.py +5 -4
  69. cognite/neat/_version.py +1 -1
  70. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/METADATA +3 -2
  71. cognite_neat-0.105.0.dist-info/RECORD +179 -0
  72. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/WHEEL +1 -1
  73. cognite/neat/_app/api/__init__.py +0 -0
  74. cognite/neat/_app/api/asgi/metrics.py +0 -4
  75. cognite/neat/_app/api/configuration.py +0 -98
  76. cognite/neat/_app/api/context_manager/__init__.py +0 -3
  77. cognite/neat/_app/api/context_manager/manager.py +0 -16
  78. cognite/neat/_app/api/data_classes/__init__.py +0 -0
  79. cognite/neat/_app/api/data_classes/rest.py +0 -59
  80. cognite/neat/_app/api/explorer.py +0 -66
  81. cognite/neat/_app/api/routers/configuration.py +0 -25
  82. cognite/neat/_app/api/routers/crud.py +0 -102
  83. cognite/neat/_app/api/routers/metrics.py +0 -10
  84. cognite/neat/_app/api/routers/workflows.py +0 -224
  85. cognite/neat/_app/api/utils/__init__.py +0 -0
  86. cognite/neat/_app/api/utils/data_mapping.py +0 -17
  87. cognite/neat/_app/api/utils/logging.py +0 -26
  88. cognite/neat/_app/api/utils/query_templates.py +0 -92
  89. cognite/neat/_app/main.py +0 -17
  90. cognite/neat/_app/monitoring/__init__.py +0 -0
  91. cognite/neat/_app/monitoring/metrics.py +0 -69
  92. cognite/neat/_app/ui/index.html +0 -1
  93. cognite/neat/_app/ui/neat-app/.gitignore +0 -23
  94. cognite/neat/_app/ui/neat-app/README.md +0 -70
  95. cognite/neat/_app/ui/neat-app/build/asset-manifest.json +0 -14
  96. cognite/neat/_app/ui/neat-app/build/favicon.ico +0 -0
  97. cognite/neat/_app/ui/neat-app/build/img/architect-icon.svg +0 -116
  98. cognite/neat/_app/ui/neat-app/build/img/developer-icon.svg +0 -112
  99. cognite/neat/_app/ui/neat-app/build/img/sme-icon.svg +0 -34
  100. cognite/neat/_app/ui/neat-app/build/index.html +0 -1
  101. cognite/neat/_app/ui/neat-app/build/logo192.png +0 -0
  102. cognite/neat/_app/ui/neat-app/build/manifest.json +0 -25
  103. cognite/neat/_app/ui/neat-app/build/robots.txt +0 -3
  104. cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css +0 -2
  105. cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css.map +0 -1
  106. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js +0 -3
  107. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.LICENSE.txt +0 -88
  108. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.map +0 -1
  109. cognite/neat/_app/ui/neat-app/build/static/media/logo.8093b84df9ed36a174c629d6fe0b730d.svg +0 -1
  110. cognite/neat/_app/ui/neat-app/package-lock.json +0 -18306
  111. cognite/neat/_app/ui/neat-app/package.json +0 -62
  112. cognite/neat/_app/ui/neat-app/public/favicon.ico +0 -0
  113. cognite/neat/_app/ui/neat-app/public/img/architect-icon.svg +0 -116
  114. cognite/neat/_app/ui/neat-app/public/img/developer-icon.svg +0 -112
  115. cognite/neat/_app/ui/neat-app/public/img/sme-icon.svg +0 -34
  116. cognite/neat/_app/ui/neat-app/public/index.html +0 -43
  117. cognite/neat/_app/ui/neat-app/public/logo192.png +0 -0
  118. cognite/neat/_app/ui/neat-app/public/manifest.json +0 -25
  119. cognite/neat/_app/ui/neat-app/public/robots.txt +0 -3
  120. cognite/neat/_app/ui/neat-app/src/App.css +0 -38
  121. cognite/neat/_app/ui/neat-app/src/App.js +0 -17
  122. cognite/neat/_app/ui/neat-app/src/App.test.js +0 -8
  123. cognite/neat/_app/ui/neat-app/src/MainContainer.tsx +0 -70
  124. cognite/neat/_app/ui/neat-app/src/components/JsonViewer.tsx +0 -43
  125. cognite/neat/_app/ui/neat-app/src/components/LocalUploader.tsx +0 -124
  126. cognite/neat/_app/ui/neat-app/src/components/OverviewComponentEditorDialog.tsx +0 -63
  127. cognite/neat/_app/ui/neat-app/src/components/StepEditorDialog.tsx +0 -511
  128. cognite/neat/_app/ui/neat-app/src/components/TabPanel.tsx +0 -36
  129. cognite/neat/_app/ui/neat-app/src/components/Utils.tsx +0 -56
  130. cognite/neat/_app/ui/neat-app/src/components/WorkflowDeleteDialog.tsx +0 -60
  131. cognite/neat/_app/ui/neat-app/src/components/WorkflowExecutionReport.tsx +0 -112
  132. cognite/neat/_app/ui/neat-app/src/components/WorkflowImportExportDialog.tsx +0 -67
  133. cognite/neat/_app/ui/neat-app/src/components/WorkflowMetadataDialog.tsx +0 -79
  134. cognite/neat/_app/ui/neat-app/src/index.css +0 -13
  135. cognite/neat/_app/ui/neat-app/src/index.js +0 -13
  136. cognite/neat/_app/ui/neat-app/src/logo.svg +0 -1
  137. cognite/neat/_app/ui/neat-app/src/reportWebVitals.js +0 -13
  138. cognite/neat/_app/ui/neat-app/src/setupTests.js +0 -5
  139. cognite/neat/_app/ui/neat-app/src/types/WorkflowTypes.ts +0 -388
  140. cognite/neat/_app/ui/neat-app/src/views/AboutView.tsx +0 -61
  141. cognite/neat/_app/ui/neat-app/src/views/ConfigView.tsx +0 -184
  142. cognite/neat/_app/ui/neat-app/src/views/GlobalConfigView.tsx +0 -180
  143. cognite/neat/_app/ui/neat-app/src/views/WorkflowView.tsx +0 -570
  144. cognite/neat/_app/ui/neat-app/tsconfig.json +0 -27
  145. cognite/neat/_rules/transformers/_pipelines.py +0 -70
  146. cognite/neat/_workflows/__init__.py +0 -17
  147. cognite/neat/_workflows/base.py +0 -590
  148. cognite/neat/_workflows/cdf_store.py +0 -393
  149. cognite/neat/_workflows/examples/Export_DMS/workflow.yaml +0 -89
  150. cognite/neat/_workflows/examples/Export_Semantic_Data_Model/workflow.yaml +0 -66
  151. cognite/neat/_workflows/examples/Import_DMS/workflow.yaml +0 -65
  152. cognite/neat/_workflows/examples/Validate_Rules/workflow.yaml +0 -67
  153. cognite/neat/_workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
  154. cognite/neat/_workflows/manager.py +0 -292
  155. cognite/neat/_workflows/model.py +0 -203
  156. cognite/neat/_workflows/steps/__init__.py +0 -0
  157. cognite/neat/_workflows/steps/data_contracts.py +0 -109
  158. cognite/neat/_workflows/steps/lib/__init__.py +0 -0
  159. cognite/neat/_workflows/steps/lib/current/__init__.py +0 -6
  160. cognite/neat/_workflows/steps/lib/current/graph_extractor.py +0 -100
  161. cognite/neat/_workflows/steps/lib/current/graph_loader.py +0 -51
  162. cognite/neat/_workflows/steps/lib/current/graph_store.py +0 -48
  163. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +0 -537
  164. cognite/neat/_workflows/steps/lib/current/rules_importer.py +0 -398
  165. cognite/neat/_workflows/steps/lib/current/rules_validator.py +0 -106
  166. cognite/neat/_workflows/steps/lib/io/__init__.py +0 -1
  167. cognite/neat/_workflows/steps/lib/io/io_steps.py +0 -393
  168. cognite/neat/_workflows/steps/step_model.py +0 -79
  169. cognite/neat/_workflows/steps_registry.py +0 -218
  170. cognite/neat/_workflows/tasks.py +0 -18
  171. cognite/neat/_workflows/triggers.py +0 -169
  172. cognite/neat/_workflows/utils.py +0 -19
  173. cognite_neat-0.103.1.dist-info/RECORD +0 -275
  174. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/LICENSE +0 -0
  175. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/entry_points.txt +0 -0
@@ -1,19 +1,17 @@
1
1
  import warnings
2
2
  from abc import ABC
3
3
  from collections import defaultdict
4
- from functools import cached_property
5
4
  from typing import Any, ClassVar, Literal
6
5
 
7
6
  from cognite.client import data_modeling as dm
8
7
 
9
8
  from cognite.neat._client import NeatClient
10
9
  from cognite.neat._issues.errors import CDFMissingClientError, NeatValueError, ResourceNotFoundError
11
- from cognite.neat._issues.warnings import NeatValueWarning, PropertyOverwritingWarning
12
- from cognite.neat._rules._shared import JustRules, OutRules
10
+ from cognite.neat._issues.warnings import PropertyOverwritingWarning
13
11
  from cognite.neat._rules.models import DMSRules, SheetList
14
12
  from cognite.neat._rules.models.data_types import Enum
15
- from cognite.neat._rules.models.dms import DMSEnum, DMSProperty, DMSView
16
- from cognite.neat._rules.models.entities import ContainerEntity, ViewEntity
13
+ from cognite.neat._rules.models.dms import DMSContainer, DMSEnum, DMSProperty
14
+ from cognite.neat._rules.models.entities import ClassEntity, ContainerEntity, ViewEntity
17
15
 
18
16
  from ._base import RulesTransformer
19
17
 
@@ -55,8 +53,8 @@ class MapOneToOne(MapOntoTransformers):
55
53
  self.view_extension_mapping = view_extension_mapping
56
54
  self.default_extension = default_extension
57
55
 
58
- def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
59
- solution: DMSRules = self._to_rules(rules)
56
+ def transform(self, rules: DMSRules) -> DMSRules:
57
+ solution: DMSRules = rules
60
58
  view_by_external_id = {view.view.external_id: view for view in solution.views}
61
59
  ref_view_by_external_id = {view.view.external_id: view for view in self.reference.views}
62
60
 
@@ -99,17 +97,18 @@ class MapOneToOne(MapOntoTransformers):
99
97
  prop.container = ref_prop.container
100
98
  prop.container_property = ref_prop.container_property
101
99
 
102
- return JustRules(solution)
100
+ return solution
103
101
 
104
102
 
105
103
  class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
106
104
  """Maps properties and classes using the given mapping.
107
105
 
108
- **Note**: This transformer mutates the input rules.
109
-
110
106
  Args:
111
- mapping: The mapping to use.
112
-
107
+ mapping: The mapping to use represented as a DMSRules object.
108
+ data_type_conflict: How to handle data type conflicts. The default is "overwrite".
109
+ A data type conflicts occurs when the data type of a property in the mapping is different from the
110
+ data type of the property in the input rules. If "overwrite" the data type in the input rules is overwritten
111
+ with the data type in the mapping.
113
112
  """
114
113
 
115
114
  _mapping_fields: ClassVar[frozenset[str]] = frozenset(
@@ -120,70 +119,85 @@ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
120
119
  self.mapping = mapping
121
120
  self.data_type_conflict = data_type_conflict
122
121
 
123
- @cached_property
124
- def _view_by_entity_id(self) -> dict[str, DMSView]:
125
- return {view.view.external_id: view for view in self.mapping.views}
126
-
127
- @cached_property
128
- def _property_by_view_property(self) -> dict[tuple[str, str], DMSProperty]:
129
- return {(prop.view.external_id, prop.view_property): prop for prop in self.mapping.properties}
130
-
131
- def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
122
+ def transform(self, rules: DMSRules) -> DMSRules:
132
123
  if self.data_type_conflict != "overwrite":
133
124
  raise NeatValueError(f"Invalid data_type_conflict: {self.data_type_conflict}")
134
- input_rules = self._to_rules(rules)
125
+ input_rules = rules
135
126
  new_rules = input_rules.model_copy(deep=True)
136
- new_rules.metadata.version += "_mapped"
137
127
 
138
- for view in new_rules.views:
139
- if mapping_view := self._view_by_entity_id.get(view.view.external_id):
140
- view.implements = mapping_view.implements
141
-
142
- for prop in new_rules.properties:
143
- key = (prop.view.external_id, prop.view_property)
144
- if key not in self._property_by_view_property:
145
- continue
146
- mapping_prop = self._property_by_view_property[key]
147
- to_overwrite, conflicts = self._find_overwrites(prop, mapping_prop)
148
- if conflicts and self.data_type_conflict == "overwrite":
149
- warnings.warn(
150
- PropertyOverwritingWarning(prop.view.as_id(), "view", prop.view_property, tuple(conflicts)),
151
- stacklevel=2,
152
- )
153
- elif conflicts:
154
- raise NeatValueError(f"Conflicting properties for {prop.view}.{prop.view_property}: {conflicts}")
155
- for field_name, value in to_overwrite.items():
156
- setattr(prop, field_name, value)
157
- prop.container = mapping_prop.container
158
- prop.container_property = mapping_prop.container_property
159
-
160
- # Add missing views used as value types
161
- existing_views = {view.view for view in new_rules.views}
162
- new_value_types = {
163
- prop.value_type
164
- for prop in new_rules.properties
165
- if isinstance(prop.value_type, ViewEntity) and prop.value_type not in existing_views
166
- }
167
- for new_value_type in new_value_types:
168
- if mapping_view := self._view_by_entity_id.get(new_value_type.external_id):
169
- new_rules.views.append(mapping_view)
128
+ views_by_external_id = {view.view.external_id: view for view in new_rules.views}
129
+ new_views: set[ViewEntity] = set()
130
+ for mapping_view in self.mapping.views:
131
+ if existing_view := views_by_external_id.get(mapping_view.view.external_id):
132
+ existing_view.implements = mapping_view.implements
170
133
  else:
171
- warnings.warn(NeatValueWarning(f"View {new_value_type} not found in mapping"), stacklevel=2)
134
+ # We need to add all the views in the mapping that are not in the input rules.
135
+ # This is to ensure that all ValueTypes are present in the resulting rules.
136
+ # For example, if a property is a direct relation to an Equipment view, we need to add
137
+ # the Equipment view to the rules.
138
+ new_rules.views.append(mapping_view)
139
+ new_views.add(mapping_view.view)
172
140
 
173
- # Add missing enums
174
- existing_enum_collections = {item.collection for item in new_rules.enum or []}
175
- new_enums = {
176
- prop.value_type.collection
177
- for prop in new_rules.properties
178
- if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections
141
+ properties_by_view_property = {
142
+ (prop.view.external_id, prop.view_property): prop for prop in new_rules.properties
179
143
  }
180
- if new_enums:
181
- new_rules.enum = new_rules.enum or SheetList[DMSEnum]([])
182
- for item in self.mapping.enum or []:
183
- if item.collection in new_enums:
184
- new_rules.enum.append(item)
144
+ existing_enum_collections = {item.collection for item in new_rules.enum or []}
145
+ mapping_enums_by_collection: dict[ClassEntity, list[DMSEnum]] = defaultdict(list)
146
+ for item in self.mapping.enum or []:
147
+ mapping_enums_by_collection[item.collection].append(item)
148
+ existing_containers = {container.container for container in new_rules.containers or []}
149
+ mapping_containers_by_id = {container.container: container for container in self.mapping.containers or []}
150
+ for mapping_prop in self.mapping.properties:
151
+ if existing_prop := properties_by_view_property.get(
152
+ (mapping_prop.view.external_id, mapping_prop.view_property)
153
+ ):
154
+ to_overwrite, conflicts = self._find_overwrites(existing_prop, mapping_prop)
155
+ if conflicts and self.data_type_conflict == "overwrite":
156
+ warnings.warn(
157
+ PropertyOverwritingWarning(
158
+ existing_prop.view.as_id(), "view", existing_prop.view_property, tuple(conflicts)
159
+ ),
160
+ stacklevel=2,
161
+ )
162
+ elif conflicts:
163
+ raise NeatValueError(
164
+ f"Conflicting properties for {existing_prop.view}.{existing_prop.view_property}: {conflicts}"
165
+ )
166
+
167
+ for field_name, value in to_overwrite.items():
168
+ setattr(existing_prop, field_name, value)
169
+ existing_prop.container = mapping_prop.container
170
+ existing_prop.container_property = mapping_prop.container_property
171
+ elif isinstance(mapping_prop.value_type, ViewEntity):
172
+ # All connections must be included in the rules. This is to update the
173
+ # ValueTypes of the implemented views.
174
+ new_rules.properties.append(mapping_prop)
175
+ elif mapping_prop.view in new_views:
176
+ # All properties of new views are included. Main motivation is GUIDs properties
177
+ new_rules.properties.append(mapping_prop)
178
+ else:
179
+ # Skipping mapped properties that are not in the input rules.
180
+ continue
185
181
 
186
- return JustRules(new_rules)
182
+ if (
183
+ isinstance(mapping_prop.value_type, Enum)
184
+ and mapping_prop.value_type.collection not in existing_enum_collections
185
+ ):
186
+ if not new_rules.enum:
187
+ new_rules.enum = SheetList[DMSEnum]([])
188
+ new_rules.enum.extend(mapping_enums_by_collection[mapping_prop.value_type.collection])
189
+
190
+ if (
191
+ mapping_prop.container
192
+ and mapping_prop.container not in existing_containers
193
+ and (new_container := mapping_containers_by_id.get(mapping_prop.container))
194
+ ):
195
+ # Mapping can include new containers for GUID properties
196
+ if not new_rules.containers:
197
+ new_rules.containers = SheetList[DMSContainer]([])
198
+ new_rules.containers.append(new_container)
199
+
200
+ return new_rules
187
201
 
188
202
  def _find_overwrites(self, prop: DMSProperty, mapping_prop: DMSProperty) -> tuple[dict[str, Any], list[str]]:
189
203
  """Finds the properties that need to be overwritten and returns them.
@@ -212,6 +226,10 @@ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
212
226
  conflicts.append(mapping_prop.model_fields[field_name].alias or field_name)
213
227
  return to_overwrite, conflicts
214
228
 
229
+ @property
230
+ def description(self) -> str:
231
+ return f"Mapping to {self.mapping.metadata.as_data_model_id()!r}."
232
+
215
233
 
216
234
  class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
217
235
  """Looks up all view properties that map to the same container property,
@@ -221,10 +239,9 @@ class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
221
239
  def __init__(self, client: NeatClient | None = None) -> None:
222
240
  self._client = client
223
241
 
224
- def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
225
- input_rules = self._to_rules(rules)
242
+ def transform(self, rules: DMSRules) -> DMSRules:
243
+ input_rules = rules
226
244
  new_rules = input_rules.model_copy(deep=True)
227
- new_rules.metadata.version += "_as_parent_name"
228
245
 
229
246
  path_by_view = self._inheritance_path_by_view(new_rules)
230
247
  view_by_container_property = self._view_by_container_properties(new_rules)
@@ -240,7 +257,7 @@ class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
240
257
  ):
241
258
  prop.view_property = parent_name
242
259
 
243
- return JustRules(new_rules)
260
+ return new_rules
244
261
 
245
262
  # Todo: Move into Probe class. Note this means that the Probe class must take a NeatClient as an argument.
246
263
  def _inheritance_path_by_view(self, rules: DMSRules) -> dict[ViewEntity, list[ViewEntity]]:
@@ -338,3 +355,7 @@ class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
338
355
  _, prop_name = min(view_properties, key=lambda prop: len(path_by_view[prop[0]]))
339
356
  parent_name_by_container_property[(container, container_property)] = prop_name
340
357
  return parent_name_by_container_property
358
+
359
+ @property
360
+ def description(self) -> str:
361
+ return "Renaming property names to parent name"
@@ -1,14 +1,11 @@
1
1
  from abc import ABC
2
- from typing import Any, Literal
3
2
 
4
- from cognite.neat._issues import IssueList, MultiValueError, NeatError, NeatWarning, catch_issues
5
- from cognite.neat._issues.errors import NeatTypeError
3
+ from cognite.neat._client import NeatClient
4
+ from cognite.neat._issues import MultiValueError, catch_issues
5
+ from cognite.neat._issues.errors import NeatTypeError, NeatValueError
6
6
  from cognite.neat._rules._shared import (
7
- InputRules,
8
- MaybeRules,
9
- OutRules,
10
7
  ReadRules,
11
- T_InputRules,
8
+ T_ReadInputRules,
12
9
  T_VerifiedRules,
13
10
  VerifiedRules,
14
11
  )
@@ -24,78 +21,93 @@ from cognite.neat._rules.models.information import InformationValidation
24
21
  from ._base import RulesTransformer
25
22
 
26
23
 
27
- class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], ABC):
24
+ class VerificationTransformer(RulesTransformer[T_ReadInputRules, T_VerifiedRules], ABC):
28
25
  """Base class for all verification transformers."""
29
26
 
30
27
  _rules_cls: type[T_VerifiedRules]
31
28
  _validation_cls: type
32
29
 
33
- def __init__(self, errors: Literal["raise", "continue"], validate: bool = True) -> None:
34
- self.errors = errors
30
+ def __init__(self, validate: bool = True, client: NeatClient | None = None) -> None:
35
31
  self.validate = validate
32
+ self._client = client
36
33
 
37
- def transform(self, rules: T_InputRules | OutRules[T_InputRules]) -> MaybeRules[T_VerifiedRules]:
38
- issues = IssueList()
39
- in_: T_InputRules = self._to_rules(rules)
40
- error_args: dict[str, Any] = {}
41
- if isinstance(rules, ReadRules):
42
- error_args = rules.read_context
34
+ def transform(self, rules: T_ReadInputRules) -> T_VerifiedRules:
35
+ in_ = rules.rules
36
+ if in_ is None:
37
+ raise NeatValueError("Cannot verify rules. The reading of the rules failed.")
38
+ error_args = rules.read_context
43
39
  verified_rules: T_VerifiedRules | None = None
44
- with catch_issues(issues, NeatError, NeatWarning, error_args) as future:
45
- rules_cls = self._get_rules_cls(in_)
40
+ # We need to catch issues as we use the error args to provide extra context for the errors/warnings
41
+ # For example, which row in the spreadsheet the error occurred o
42
+ with catch_issues(error_args=error_args) as issues:
43
+ rules_cls = self._get_rules_cls(rules)
46
44
  dumped = in_.dump()
47
45
  verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
48
46
  if self.validate:
49
47
  validation_cls = self._get_validation_cls(verified_rules) # type: ignore[arg-type]
50
- validation_issues = validation_cls(verified_rules).validate()
51
- # We need to trigger warnings are raise exceptions such that they are caught by the context manager
52
- # and processed with the read context
53
- if validation_issues.warnings:
54
- validation_issues.trigger_warnings()
48
+ if issubclass(validation_cls, DMSValidation):
49
+ validation_issues = DMSValidation(verified_rules, self._client).validate() # type: ignore[arg-type]
50
+ elif issubclass(validation_cls, InformationValidation):
51
+ validation_issues = InformationValidation(verified_rules).validate() # type: ignore[arg-type]
52
+ else:
53
+ raise NeatValueError("Unsupported rule type")
54
+
55
+ # Need to trigger and raise such that the catch_issues can add the extra context
56
+ validation_issues.trigger_warnings()
55
57
  if validation_issues.has_errors:
56
- verified_rules = None
57
58
  raise MultiValueError(validation_issues.errors)
58
59
 
59
- if (future.result == "failure" or issues.has_errors or verified_rules is None) and self.errors == "raise":
60
- raise issues.as_errors()
61
- return MaybeRules[T_VerifiedRules](
62
- rules=verified_rules,
63
- issues=issues,
64
- )
60
+ # Raise issues which is expected to be handled outside of this method
61
+ issues.trigger_warnings()
62
+ if issues.has_errors:
63
+ raise MultiValueError(issues.errors)
64
+ if verified_rules is None:
65
+ raise NeatValueError("Rules were not verified")
66
+ return verified_rules
65
67
 
66
- def _get_rules_cls(self, in_: T_InputRules) -> type[T_VerifiedRules]:
68
+ def _get_rules_cls(self, in_: T_ReadInputRules) -> type[T_VerifiedRules]:
67
69
  return self._rules_cls
68
70
 
69
71
  def _get_validation_cls(self, rules: T_VerifiedRules) -> type:
70
72
  return self._validation_cls
71
73
 
74
+ @property
75
+ def description(self) -> str:
76
+ return "Verify rules"
72
77
 
73
- class VerifyDMSRules(VerificationTransformer[DMSInputRules, DMSRules]):
78
+
79
+ class VerifyDMSRules(VerificationTransformer[ReadRules[DMSInputRules], DMSRules]):
74
80
  """Class to verify DMS rules."""
75
81
 
76
82
  _rules_cls = DMSRules
77
83
  _validation_cls = DMSValidation
78
84
 
85
+ def transform(self, rules: ReadRules[DMSInputRules]) -> DMSRules:
86
+ return super().transform(rules)
87
+
79
88
 
80
- class VerifyInformationRules(VerificationTransformer[InformationInputRules, InformationRules]):
89
+ class VerifyInformationRules(VerificationTransformer[ReadRules[InformationInputRules], InformationRules]):
81
90
  """Class to verify Information rules."""
82
91
 
83
92
  _rules_cls = InformationRules
84
93
  _validation_cls = InformationValidation
85
94
 
95
+ def transform(self, rules: ReadRules[InformationInputRules]) -> InformationRules:
96
+ return super().transform(rules)
86
97
 
87
- class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
98
+
99
+ class VerifyAnyRules(VerificationTransformer[T_ReadInputRules, VerifiedRules]):
88
100
  """Class to verify arbitrary rules"""
89
101
 
90
- def _get_rules_cls(self, in_: InputRules) -> type[VerifiedRules]:
91
- if isinstance(in_, InformationInputRules):
102
+ def _get_rules_cls(self, in_: T_ReadInputRules) -> type[VerifiedRules]:
103
+ if isinstance(in_.rules, InformationInputRules):
92
104
  return InformationRules
93
- elif isinstance(in_, DMSInputRules):
105
+ elif isinstance(in_.rules, DMSInputRules):
94
106
  return DMSRules
95
107
  else:
96
108
  raise NeatTypeError(f"Unsupported rules type: {type(in_)}")
97
109
 
98
- def _get_validation_cls(self, rules: T_VerifiedRules) -> type:
110
+ def _get_validation_cls(self, rules: VerifiedRules) -> type:
99
111
  if isinstance(rules, InformationRules):
100
112
  return InformationValidation
101
113
  elif isinstance(rules, DMSRules):
@@ -1,26 +1,17 @@
1
- from datetime import datetime, timezone
2
- from typing import Literal, cast
1
+ from typing import Literal
3
2
 
4
3
  from cognite.client import CogniteClient
5
4
  from cognite.client import data_modeling as dm
6
5
 
7
6
  from cognite.neat import _version
8
7
  from cognite.neat._client import NeatClient
9
- from cognite.neat._issues import IssueList, catch_issues
8
+ from cognite.neat._issues import IssueList
10
9
  from cognite.neat._issues.errors import RegexViolationError
11
10
  from cognite.neat._rules import importers
12
- from cognite.neat._rules._shared import ReadRules, VerifiedRules
13
- from cognite.neat._rules.models import DMSRules
14
- from cognite.neat._rules.models.dms import DMSValidation
15
- from cognite.neat._rules.models.information import InformationValidation
11
+ from cognite.neat._rules.models._base_input import InputRules
16
12
  from cognite.neat._rules.models.information._rules import InformationRules
17
- from cognite.neat._rules.models.information._rules_input import InformationInputRules
18
13
  from cognite.neat._rules.transformers import ConvertToRules, InformationToDMS, VerifyAnyRules
19
14
  from cognite.neat._rules.transformers._converters import ConversionTransformer
20
- from cognite.neat._store._provenance import (
21
- INSTANCES_ENTITY,
22
- Change,
23
- )
24
15
 
25
16
  from ._collector import _COLLECTOR, Collector
26
17
  from ._drop import DropAPI
@@ -128,47 +119,16 @@ class NeatSession:
128
119
  neat.verify()
129
120
  ```
130
121
  """
131
- source_id, last_unverified_rule = self._state.data_model.last_unverified_rule
132
- transformer = VerifyAnyRules("continue", validate=False)
133
- start = datetime.now(timezone.utc)
134
- output = transformer.try_transform(last_unverified_rule)
135
-
136
- if output.rules:
137
- if isinstance(output.rules, DMSRules):
138
- issues = DMSValidation(output.rules, self._client).validate()
139
- elif isinstance(output.rules, InformationRules):
140
- issues = InformationValidation(output.rules).validate()
141
- else:
142
- raise NeatSessionError("Unsupported rule type")
143
- if issues.has_errors:
144
- # This is up for discussion, but I think we should not return rules that
145
- # only pass the verification but not the validation.
146
- output.rules = None
147
- output.issues.extend(issues)
148
-
149
- end = datetime.now(timezone.utc)
150
-
151
- if output.rules:
152
- change = Change.from_rules_activity(
153
- output.rules,
154
- transformer.agent,
155
- start,
156
- end,
157
- f"Verified data model {source_id} as {output.rules.metadata.identifier}",
158
- self._state.data_model.provenance.source_entity(source_id)
159
- or self._state.data_model.provenance.target_entity(source_id),
160
- )
161
-
162
- self._state.data_model.write(output.rules, change)
163
-
164
- if isinstance(output.rules, InformationRules):
165
- self._state.instances.store.add_rules(output.rules)
166
-
167
- output.issues.action = "verify"
168
- self._state.data_model.issue_lists.append(output.issues)
169
- if output.issues:
122
+ transformer = VerifyAnyRules(validate=True, client=self._client) # type: ignore[var-annotated]
123
+ issues = self._state.rule_transform(transformer)
124
+ if not issues.has_errors:
125
+ rules = self._state.rule_store.last_verified_rule
126
+ if isinstance(rules, InformationRules):
127
+ self._state.instances.store.add_rules(rules)
128
+
129
+ if issues:
170
130
  print("You can inspect the issues with the .inspect.issues(...) method.")
171
- return output.issues
131
+ return issues
172
132
 
173
133
  def convert(
174
134
  self, target: Literal["dms", "information"], mode: Literal["edge_properties"] | None = None
@@ -192,43 +152,17 @@ class NeatSession:
192
152
  neat.convert(target="information")
193
153
  ```
194
154
  """
195
- start = datetime.now(timezone.utc)
196
- issues = IssueList()
197
- converter: ConversionTransformer | None = None
198
- converted_rules: VerifiedRules | None = None
199
- with catch_issues(issues):
200
- if target == "dms":
201
- source_id, info_rules = self._state.data_model.last_verified_information_rules
202
- converter = InformationToDMS(mode=mode)
203
- converted_rules = converter.transform(info_rules).rules
204
- elif target == "information":
205
- source_id, dms_rules = self._state.data_model.last_verified_dms_rules
206
- converter = ConvertToRules(InformationRules)
207
- converted_rules = converter.transform(dms_rules).rules
208
- else:
209
- # Session errors are not caught by the catch_issues context manager
210
- raise NeatSessionError(f"Target {target} not supported.")
211
-
212
- end = datetime.now(timezone.utc)
213
- if issues:
214
- self._state.data_model.issue_lists.append(issues)
215
-
216
- if converted_rules is not None and converter is not None:
217
- # Provenance
218
- change = Change.from_rules_activity(
219
- converted_rules,
220
- converter.agent,
221
- start,
222
- end,
223
- f"Converted data model {source_id} to {converted_rules.metadata.identifier}",
224
- self._state.data_model.provenance.source_entity(source_id)
225
- or self._state.data_model.provenance.target_entity(source_id),
226
- )
227
-
228
- self._state.data_model.write(converted_rules, change)
229
-
230
- if self._verbose and not issues.has_errors:
231
- print(f"Rules converted to {target}")
155
+ converter: ConversionTransformer
156
+ if target == "dms":
157
+ converter = InformationToDMS(mode=mode)
158
+ elif target == "information":
159
+ converter = ConvertToRules(InformationRules)
160
+ else:
161
+ raise NeatSessionError(f"Target {target} not supported.")
162
+ issues = self._state.rule_transform(converter)
163
+
164
+ if self._verbose and not issues.has_errors:
165
+ print(f"Rules converted to {target}")
232
166
  else:
233
167
  print("Conversion failed.")
234
168
  if issues:
@@ -263,53 +197,30 @@ class NeatSession:
263
197
  ```
264
198
  """
265
199
  model_id = dm.DataModelId.load(model_id)
266
-
267
- start = datetime.now(timezone.utc)
268
200
  importer = importers.InferenceImporter.from_graph_store(
269
201
  store=self._state.instances.store,
270
202
  max_number_of_instance=max_number_of_instance,
203
+ data_model_id=model_id,
271
204
  )
272
- inferred_rules: ReadRules = importer.to_rules()
273
- end = datetime.now(timezone.utc)
274
-
275
- if model_id.space:
276
- cast(InformationInputRules, inferred_rules.rules).metadata.space = model_id.space
277
- if model_id.external_id:
278
- cast(InformationInputRules, inferred_rules.rules).metadata.external_id = model_id.external_id
279
-
280
- if model_id.version:
281
- cast(InformationInputRules, inferred_rules.rules).metadata.version = model_id.version
282
-
283
- # Provenance
284
- change = Change.from_rules_activity(
285
- inferred_rules,
286
- importer.agent,
287
- start,
288
- end,
289
- "Inferred data model",
290
- INSTANCES_ENTITY,
291
- )
292
-
293
- self._state.data_model.write(inferred_rules, change)
294
- return inferred_rules.issues
205
+ return self._state.rule_import(importer)
295
206
 
296
207
  def _repr_html_(self) -> str:
297
208
  state = self._state
298
209
  if (
299
210
  not state.instances.has_store
300
- and not state.data_model.has_unverified_rules
301
- and not state.data_model.has_verified_rules
211
+ and not state.rule_store.has_unverified_rules
212
+ and not state.rule_store.has_verified_rules
302
213
  ):
303
214
  return "<strong>Empty session</strong>. Get started by reading something with the <em>.read</em> attribute."
304
215
 
305
216
  output = []
306
217
 
307
- if state.data_model.has_unverified_rules and not state.data_model.has_verified_rules:
308
- rules: ReadRules = state.data_model.last_unverified_rule[1]
309
- output.append(f"<H2>Unverified Data Model</H2><br />{rules.rules._repr_html_()}") # type: ignore
218
+ if state.rule_store.has_unverified_rules and not state.rule_store.has_verified_rules:
219
+ rules: InputRules = state.rule_store.last_unverified_rule
220
+ output.append(f"<H2>Unverified Data Model</H2><br />{rules._repr_html_()}") # type: ignore
310
221
 
311
- if state.data_model.has_verified_rules:
312
- output.append(f"<H2>Verified Data Model</H2><br />{state.data_model.last_verified_rule[1]._repr_html_()}") # type: ignore
222
+ if state.rule_store.has_verified_rules:
223
+ output.append(f"<H2>Verified Data Model</H2><br />{state.rule_store.last_verified_rule._repr_html_()}") # type: ignore
313
224
 
314
225
  if state.instances.has_store:
315
226
  output.append(f"<H2>Instances</H2> {state.instances.store._repr_html_()}")
@@ -59,7 +59,9 @@ class InspectAPI:
59
59
  neat.inspect.properties
60
60
  ```
61
61
  """
62
- return self._state.data_model.last_verified_rule[1].properties.to_pandas()
62
+ df = self._state.rule_store.last_verified_rule.properties.to_pandas()
63
+ df.drop(columns=["neatId"], errors="ignore", inplace=True)
64
+ return df
63
65
 
64
66
 
65
67
  @session_class_wrapper
@@ -89,7 +91,7 @@ class InspectIssues:
89
91
  return_dataframe: bool = (False if IN_NOTEBOOK else True), # type: ignore[assignment]
90
92
  ) -> pd.DataFrame | None:
91
93
  """Returns the issues of the current data model."""
92
- issues = self._state.data_model.last_issues
94
+ issues = self._state.rule_store.last_issues
93
95
  if not issues:
94
96
  self._print("No issues found.")
95
97
 
@@ -141,7 +143,7 @@ class InspectOutcome:
141
143
  """
142
144
 
143
145
  def __init__(self, state: SessionState) -> None:
144
- self.data_model = InspectUploadOutcome(lambda: state.data_model.last_outcome)
146
+ self.data_model = InspectUploadOutcome(lambda: state.rule_store.last_outcome)
145
147
  self.instances = InspectUploadOutcome(lambda: state.instances.last_outcome)
146
148
 
147
149