cognite-neat 0.98.0__py3-none-any.whl → 0.99.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (72) hide show
  1. cognite/neat/_client/__init__.py +4 -0
  2. cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
  3. cognite/neat/_client/_api/schema.py +50 -0
  4. cognite/neat/_client/_api_client.py +17 -0
  5. cognite/neat/_client/data_classes/__init__.py +0 -0
  6. cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
  7. cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +21 -281
  8. cognite/neat/_graph/_shared.py +14 -15
  9. cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
  10. cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
  11. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
  12. cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
  13. cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
  14. cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
  15. cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
  16. cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
  17. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
  18. cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
  19. cognite/neat/_graph/extractors/_rdf_file.py +6 -7
  20. cognite/neat/_graph/queries/_base.py +17 -1
  21. cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
  22. cognite/neat/_graph/transformers/_prune_graph.py +1 -1
  23. cognite/neat/_graph/transformers/_rdfpath.py +1 -1
  24. cognite/neat/_issues/warnings/__init__.py +6 -0
  25. cognite/neat/_issues/warnings/_external.py +8 -0
  26. cognite/neat/_issues/warnings/_properties.py +16 -0
  27. cognite/neat/_rules/_constants.py +7 -6
  28. cognite/neat/_rules/analysis/_base.py +8 -4
  29. cognite/neat/_rules/exporters/_base.py +3 -4
  30. cognite/neat/_rules/exporters/_rules2dms.py +29 -40
  31. cognite/neat/_rules/importers/_dms2rules.py +4 -5
  32. cognite/neat/_rules/importers/_rdf/_inference2rules.py +25 -33
  33. cognite/neat/_rules/models/__init__.py +1 -1
  34. cognite/neat/_rules/models/_base_rules.py +22 -12
  35. cognite/neat/_rules/models/dms/__init__.py +2 -2
  36. cognite/neat/_rules/models/dms/_exporter.py +15 -20
  37. cognite/neat/_rules/models/dms/_rules.py +48 -3
  38. cognite/neat/_rules/models/dms/_rules_input.py +52 -8
  39. cognite/neat/_rules/models/dms/_validation.py +10 -5
  40. cognite/neat/_rules/models/entities/_single_value.py +32 -4
  41. cognite/neat/_rules/models/information/_rules.py +0 -8
  42. cognite/neat/_rules/models/mapping/__init__.py +2 -3
  43. cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
  44. cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
  45. cognite/neat/_rules/transformers/__init__.py +2 -2
  46. cognite/neat/_rules/transformers/_converters.py +110 -11
  47. cognite/neat/_rules/transformers/_mapping.py +105 -30
  48. cognite/neat/_rules/transformers/_verification.py +5 -2
  49. cognite/neat/_session/_base.py +49 -8
  50. cognite/neat/_session/_drop.py +35 -0
  51. cognite/neat/_session/_inspect.py +17 -5
  52. cognite/neat/_session/_mapping.py +39 -0
  53. cognite/neat/_session/_prepare.py +218 -23
  54. cognite/neat/_session/_read.py +49 -12
  55. cognite/neat/_session/_to.py +3 -3
  56. cognite/neat/_store/_base.py +27 -24
  57. cognite/neat/_utils/rdf_.py +28 -1
  58. cognite/neat/_version.py +1 -1
  59. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +8 -3
  60. cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
  61. cognite/neat/_workflows/steps/lib/current/rules_validator.py +3 -2
  62. {cognite_neat-0.98.0.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +3 -3
  63. {cognite_neat-0.98.0.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +67 -64
  64. cognite/neat/_rules/models/mapping/_base.py +0 -131
  65. cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
  66. cognite/neat/_utils/cdf/loaders/_base.py +0 -54
  67. cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
  68. cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
  69. /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
  70. {cognite_neat-0.98.0.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
  71. {cognite_neat-0.98.0.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
  72. {cognite_neat-0.98.0.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
4
4
  from collections import Counter, defaultdict
5
5
  from collections.abc import Collection, Mapping
6
6
  from datetime import date, datetime
7
- from typing import Literal, TypeVar, cast
7
+ from typing import Literal, TypeVar, cast, overload
8
8
 
9
9
  from cognite.client.data_classes import data_modeling as dms
10
10
  from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
@@ -42,8 +42,10 @@ from cognite.neat._rules.models.entities import (
42
42
  ContainerEntity,
43
43
  DMSUnknownEntity,
44
44
  EdgeEntity,
45
+ Entity,
45
46
  MultiValueTypeInfo,
46
47
  ReverseConnectionEntity,
48
+ T_Entity,
47
49
  UnknownEntity,
48
50
  ViewEntity,
49
51
  )
@@ -176,6 +178,74 @@ class ToCompliantEntities(RulesTransformer[InformationInputRules, InformationInp
176
178
  return fixed_definitions
177
179
 
178
180
 
181
+ class PrefixEntities(RulesTransformer[InputRules, InputRules]): # type: ignore[misc]
182
+ """Prefixes all entities with a given prefix."""
183
+
184
+ def __init__(self, prefix: str) -> None:
185
+ self._prefix = prefix
186
+
187
+ def transform(self, rules: InputRules | OutRules[InputRules]) -> ReadRules[InputRules]:
188
+ return ReadRules(self._transform(self._to_rules(rules)), IssueList(), {})
189
+
190
+ def _transform(self, rules: InputRules) -> InputRules:
191
+ rules.metadata.version += f"_prefixed_{self._prefix}"
192
+
193
+ if isinstance(rules, InformationInputRules):
194
+ # Todo Make Not mutate input class
195
+ prefixed_by_class: dict[str, str] = {}
196
+ for cls in rules.classes:
197
+ prefixed = str(self._with_prefix(cls.class_))
198
+ prefixed_by_class[str(cls.class_)] = prefixed
199
+ cls.class_ = prefixed
200
+ for prop in rules.properties:
201
+ prop.class_ = self._with_prefix(prop.class_)
202
+ if str(prop.value_type) in prefixed_by_class:
203
+ prop.value_type = prefixed_by_class[str(prop.value_type)]
204
+ return rules
205
+ elif isinstance(rules, DMSInputRules):
206
+ # Todo not mutate input class new_dms = copy.deepcopy(rules)
207
+ prefixed_by_view: dict[str, str] = {}
208
+ for view in rules.views:
209
+ prefixed = str(self._with_prefix(view.view))
210
+ prefixed_by_view[str(view.view)] = prefixed
211
+ view.view = prefixed
212
+ for dms_prop in rules.properties:
213
+ dms_prop.view = self._with_prefix(dms_prop.view)
214
+ if str(dms_prop.value_type) in prefixed_by_view:
215
+ dms_prop.value_type = prefixed_by_view[str(dms_prop.value_type)]
216
+ if rules.containers:
217
+ for container in rules.containers:
218
+ container.container = self._with_prefix(container.container)
219
+ return rules
220
+ raise NeatValueError(f"Unsupported rules type: {type(rules)}")
221
+
222
+ @overload
223
+ def _with_prefix(self, raw: str) -> str: ...
224
+
225
+ @overload
226
+ def _with_prefix(self, raw: T_Entity) -> T_Entity: ...
227
+
228
+ def _with_prefix(self, raw: str | T_Entity) -> str | T_Entity:
229
+ is_entity_format = not isinstance(raw, str)
230
+ entity = Entity.load(raw)
231
+ output: ClassEntity | ViewEntity | ContainerEntity
232
+ if isinstance(entity, ClassEntity):
233
+ output = ClassEntity(prefix=entity.prefix, suffix=f"{self._prefix}{entity.suffix}", version=entity.version)
234
+ elif isinstance(entity, ViewEntity):
235
+ output = ViewEntity(
236
+ space=entity.space, externalId=f"{self._prefix}{entity.external_id}", version=entity.version
237
+ )
238
+ elif isinstance(entity, ContainerEntity):
239
+ output = ContainerEntity(space=entity.space, externalId=f"{self._prefix}{entity.external_id}")
240
+ elif isinstance(entity, UnknownEntity | Entity):
241
+ return f"{self._prefix}{raw}"
242
+ else:
243
+ raise NeatValueError(f"Unsupported entity type: {type(entity)}")
244
+ if is_entity_format:
245
+ return cast(T_Entity, output)
246
+ return str(output)
247
+
248
+
179
249
  class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
180
250
  """Converts InformationRules to DMSRules."""
181
251
 
@@ -233,10 +303,11 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
233
303
  self,
234
304
  new_model_id: DataModelIdentifier,
235
305
  org_name: str = "My",
236
- type_: Literal["enterprise", "solution"] = "enterprise",
306
+ type_: Literal["enterprise", "solution", "data_product"] = "enterprise",
237
307
  mode: Literal["read", "write"] = "read",
238
308
  dummy_property: str = "GUID",
239
309
  move_connections: bool = False,
310
+ include: Literal["same-space", "all"] = "same-space",
240
311
  ):
241
312
  self.new_model_id = DataModelId.load(new_model_id)
242
313
  if not self.new_model_id.version:
@@ -247,6 +318,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
247
318
  self.type_ = type_
248
319
  self.dummy_property = dummy_property
249
320
  self.move_connections = move_connections
321
+ self.include = include
250
322
 
251
323
  def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
252
324
  # Copy to ensure immutability
@@ -274,6 +346,16 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
274
346
  )
275
347
 
276
348
  return self._to_enterprise(reference_model)
349
+ elif self.type_ == "data_product":
350
+ expanded = self._expand_properties(reference_model.model_copy(deep=True))
351
+ if self.include == "same-space":
352
+ expanded.properties = SheetList[DMSProperty](
353
+ [prop for prop in expanded.properties if prop.view.space == expanded.metadata.space]
354
+ )
355
+ expanded.views = SheetList[DMSView](
356
+ [view for view in expanded.views if view.view.space == expanded.metadata.space]
357
+ )
358
+ return self._to_solution(expanded, remove_views_in_other_space=False)
277
359
 
278
360
  else:
279
361
  raise NeatValueError(f"Unsupported data model type: {self.type_}")
@@ -281,7 +363,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
281
363
  def _has_views_in_multiple_space(self, rules: DMSRules) -> bool:
282
364
  return any(view.view.space != rules.metadata.space for view in rules.views)
283
365
 
284
- def _to_solution(self, reference_rules: DMSRules) -> JustRules[DMSRules]:
366
+ def _to_solution(self, reference_rules: DMSRules, remove_views_in_other_space: bool = True) -> JustRules[DMSRules]:
285
367
  """For creation of solution data model / rules specifically for mapping over existing containers."""
286
368
 
287
369
  dump = reference_rules.dump()
@@ -292,16 +374,11 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
292
374
  dump["metadata"]["external_id"] = self.new_model_id.external_id
293
375
  dump["metadata"]["version"] = self.new_model_id.version
294
376
 
295
- # dropping reference and last from the dump as they can cause validation
296
- # issues especially if reference is enterprise model build on top of CDM
297
- dump.pop("reference", None)
298
- dump.pop("last", None)
299
-
300
377
  # Set implement to NONE for all views
301
378
  for view in dump["views"]:
302
379
  view["implements"] = None
303
380
 
304
- if self._has_views_in_multiple_space(reference_rules):
381
+ if remove_views_in_other_space and self._has_views_in_multiple_space(reference_rules):
305
382
  views_to_remove = []
306
383
  for view in dump["views"]:
307
384
  if ":" in view["view"]:
@@ -312,7 +389,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
312
389
  solution_model = DMSRules.model_validate(DMSInputRules.load(dump).dump())
313
390
 
314
391
  # Dropping containers coming from reference model
315
- solution_model.containers = SheetList[DMSContainer]()
392
+ solution_model.containers = None
316
393
 
317
394
  # We want to map properties to existing containers allowing extension
318
395
  for prop in solution_model.properties:
@@ -338,7 +415,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
338
415
 
339
416
  # Here we add ONLY dummy properties of the solution model and
340
417
  # corresponding solution model space containers to hold them
341
- solution_model.containers.extend(new_containers)
418
+ solution_model.containers = new_containers
342
419
  solution_model.properties.extend(new_properties)
343
420
 
344
421
  return JustRules(solution_model)
@@ -377,6 +454,28 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
377
454
 
378
455
  return JustRules(enterprise_model)
379
456
 
457
+ @staticmethod
458
+ def _expand_properties(rules: DMSRules) -> DMSRules:
459
+ probe = DMSAnalysis(rules)
460
+ ancestor_properties_by_view = probe.classes_with_properties(
461
+ consider_inheritance=True, allow_different_namespace=True
462
+ )
463
+ property_ids_by_view = {
464
+ view: {prop.view_property for prop in properties}
465
+ for view, properties in probe.classes_with_properties(consider_inheritance=False).items()
466
+ }
467
+ for view, property_ids in property_ids_by_view.items():
468
+ ancestor_properties = ancestor_properties_by_view.get(view, [])
469
+ for prop in ancestor_properties:
470
+ if isinstance(prop.connection, ReverseConnectionEntity):
471
+ # If you try to add a reverse direct relation of a parent, it will fail as the ValueType of the
472
+ # original property will point to the parent view, and not the child.
473
+ continue
474
+ if prop.view_property not in property_ids:
475
+ rules.properties.append(prop)
476
+ property_ids.add(prop.view_property)
477
+ return rules
478
+
380
479
  def _remove_cognite_affix(self, entity: _T_Entity) -> _T_Entity:
381
480
  """This method removes `Cognite` affix from the entity."""
382
481
  new_suffix = entity.suffix.replace("Cognite", self.org_name or "")
@@ -1,13 +1,16 @@
1
+ import warnings
1
2
  from abc import ABC
2
3
  from collections import defaultdict
4
+ from functools import cached_property
5
+ from typing import Any, ClassVar, Literal
3
6
 
7
+ from cognite.neat._issues.errors import NeatValueError
8
+ from cognite.neat._issues.warnings import NeatValueWarning, PropertyOverwritingWarning
4
9
  from cognite.neat._rules._shared import JustRules, OutRules
5
- from cognite.neat._rules.models import DMSRules, InformationRules
6
- from cognite.neat._rules.models._base_rules import ClassRef
7
- from cognite.neat._rules.models.dms import DMSProperty
8
- from cognite.neat._rules.models.entities import ClassEntity
9
- from cognite.neat._rules.models.information import InformationClass
10
- from cognite.neat._rules.models.mapping import RuleMapping
10
+ from cognite.neat._rules.models import DMSRules, SheetList
11
+ from cognite.neat._rules.models.data_types import Enum
12
+ from cognite.neat._rules.models.dms import DMSEnum, DMSProperty, DMSView
13
+ from cognite.neat._rules.models.entities import ViewEntity
11
14
 
12
15
  from ._base import RulesTransformer
13
16
 
@@ -96,7 +99,7 @@ class MapOneToOne(MapOntoTransformers):
96
99
  return JustRules(solution)
97
100
 
98
101
 
99
- class RuleMapper(RulesTransformer[InformationRules, InformationRules]):
102
+ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
100
103
  """Maps properties and classes using the given mapping.
101
104
 
102
105
  **Note**: This transformer mutates the input rules.
@@ -106,30 +109,102 @@ class RuleMapper(RulesTransformer[InformationRules, InformationRules]):
106
109
 
107
110
  """
108
111
 
109
- def __init__(self, mapping: RuleMapping) -> None:
112
+ _mapping_fields: ClassVar[frozenset[str]] = frozenset(
113
+ ["connection", "value_type", "nullable", "immutable", "is_list", "default", "index", "constraint"]
114
+ )
115
+
116
+ def __init__(self, mapping: DMSRules, data_type_conflict: Literal["overwrite"] = "overwrite") -> None:
110
117
  self.mapping = mapping
118
+ self.data_type_conflict = data_type_conflict
119
+
120
+ @cached_property
121
+ def _view_by_entity_id(self) -> dict[str, DMSView]:
122
+ return {view.view.external_id: view for view in self.mapping.views}
111
123
 
112
- def transform(self, rules: InformationRules | OutRules[InformationRules]) -> JustRules[InformationRules]:
124
+ @cached_property
125
+ def _property_by_view_property(self) -> dict[tuple[str, str], DMSProperty]:
126
+ return {(prop.view.external_id, prop.view_property): prop for prop in self.mapping.properties}
127
+
128
+ def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
129
+ if self.data_type_conflict != "overwrite":
130
+ raise NeatValueError(f"Invalid data_type_conflict: {self.data_type_conflict}")
113
131
  input_rules = self._to_rules(rules)
132
+ new_rules = input_rules.model_copy(deep=True)
133
+ new_rules.metadata.version += "_mapped"
114
134
 
115
- destination_by_source = self.mapping.properties.as_destination_by_source()
116
- destination_cls_by_source = self.mapping.classes.as_destination_by_source()
117
- used_destination_classes: set[ClassEntity] = set()
118
- for prop in input_rules.properties:
119
- if destination_prop := destination_by_source.get(prop.as_reference()):
120
- prop.class_ = destination_prop.class_
121
- prop.property_ = destination_prop.property_
122
- used_destination_classes.add(destination_prop.class_)
123
- elif destination_cls := destination_cls_by_source.get(ClassRef(Class=prop.class_)):
124
- # If the property is not in the mapping, but the class is,
125
- # then we should map the class to the destination
126
- prop.class_ = destination_cls.class_
127
-
128
- for cls_ in input_rules.classes:
129
- if destination_cls := destination_cls_by_source.get(cls_.as_reference()):
130
- cls_.class_ = destination_cls.class_
131
- existing_classes = {cls_.class_ for cls_ in input_rules.classes}
132
- for new_cls in used_destination_classes - existing_classes:
133
- input_rules.classes.append(InformationClass(class_=new_cls))
134
-
135
- return JustRules(input_rules)
135
+ for view in new_rules.views:
136
+ if mapping_view := self._view_by_entity_id.get(view.view.external_id):
137
+ view.implements = mapping_view.implements
138
+
139
+ for prop in new_rules.properties:
140
+ key = (prop.view.external_id, prop.view_property)
141
+ if key not in self._property_by_view_property:
142
+ continue
143
+ mapping_prop = self._property_by_view_property[key]
144
+ to_overwrite, conflicts = self._find_overwrites(prop, mapping_prop)
145
+ if conflicts and self.data_type_conflict == "overwrite":
146
+ warnings.warn(
147
+ PropertyOverwritingWarning(prop.view.as_id(), "view", prop.view_property, tuple(conflicts)),
148
+ stacklevel=2,
149
+ )
150
+ elif conflicts:
151
+ raise NeatValueError(f"Conflicting properties for {prop.view}.{prop.view_property}: {conflicts}")
152
+ for field_name, value in to_overwrite.items():
153
+ setattr(prop, field_name, value)
154
+ prop.container = mapping_prop.container
155
+ prop.container_property = mapping_prop.container_property
156
+
157
+ # Add missing views used as value types
158
+ existing_views = {view.view for view in new_rules.views}
159
+ new_value_types = {
160
+ prop.value_type
161
+ for prop in new_rules.properties
162
+ if isinstance(prop.value_type, ViewEntity) and prop.value_type not in existing_views
163
+ }
164
+ for new_value_type in new_value_types:
165
+ if mapping_view := self._view_by_entity_id.get(new_value_type.external_id):
166
+ new_rules.views.append(mapping_view)
167
+ else:
168
+ warnings.warn(NeatValueWarning(f"View {new_value_type} not found in mapping"), stacklevel=2)
169
+
170
+ # Add missing enums
171
+ existing_enum_collections = {item.collection for item in new_rules.enum or []}
172
+ new_enums = {
173
+ prop.value_type.collection
174
+ for prop in new_rules.properties
175
+ if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections
176
+ }
177
+ if new_enums:
178
+ new_rules.enum = new_rules.enum or SheetList[DMSEnum]([])
179
+ for item in self.mapping.enum or []:
180
+ if item.collection in new_enums:
181
+ new_rules.enum.append(item)
182
+
183
+ return JustRules(new_rules)
184
+
185
+ def _find_overwrites(self, prop: DMSProperty, mapping_prop: DMSProperty) -> tuple[dict[str, Any], list[str]]:
186
+ """Finds the properties that need to be overwritten and returns them.
187
+
188
+ In addition, conflicting properties are returned. Note that overwriting properties that are
189
+ originally None is not considered a conflict. Thus, you can have properties to overwrite but no
190
+ conflicts.
191
+
192
+ Args:
193
+ prop: The property to compare.
194
+ mapping_prop: The property to compare against.
195
+
196
+ Returns:
197
+ A tuple with the properties to overwrite and the conflicting properties.
198
+
199
+ """
200
+ to_overwrite: dict[str, Any] = {}
201
+ conflicts: list[str] = []
202
+ for field_name in self._mapping_fields:
203
+ mapping_value = getattr(mapping_prop, field_name)
204
+ source_value = getattr(prop, field_name)
205
+ if mapping_value != source_value:
206
+ to_overwrite[field_name] = mapping_value
207
+ if source_value is not None:
208
+ # These are used for warnings so we use the alias to make it more readable for the user
209
+ conflicts.append(mapping_prop.model_fields[field_name].alias or field_name)
210
+ return to_overwrite, conflicts
@@ -27,8 +27,9 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
27
27
 
28
28
  _rules_cls: type[T_VerifiedRules]
29
29
 
30
- def __init__(self, errors: Literal["raise", "continue"]) -> None:
30
+ def __init__(self, errors: Literal["raise", "continue"], post_validate: bool = True) -> None:
31
31
  self.errors = errors
32
+ self.post_validate = post_validate
32
33
 
33
34
  def transform(self, rules: T_InputRules | OutRules[T_InputRules]) -> MaybeRules[T_VerifiedRules]:
34
35
  issues = IssueList()
@@ -39,7 +40,9 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
39
40
  verified_rules: T_VerifiedRules | None = None
40
41
  with catch_issues(issues, NeatError, NeatWarning, error_args) as future:
41
42
  rules_cls = self._get_rules_cls(in_)
42
- verified_rules = rules_cls.model_validate(in_.dump()) # type: ignore[assignment]
43
+ dumped = in_.dump()
44
+ dumped["post_validate"] = self.post_validate
45
+ verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
43
46
 
44
47
  if (future.result == "failure" or issues.has_errors or verified_rules is None) and self.errors == "raise":
45
48
  raise issues.as_errors()
@@ -5,11 +5,13 @@ from cognite.client import CogniteClient
5
5
  from cognite.client import data_modeling as dm
6
6
 
7
7
  from cognite.neat import _version
8
+ from cognite.neat._client import NeatClient
8
9
  from cognite.neat._issues import IssueList, catch_issues
9
10
  from cognite.neat._issues.errors import RegexViolationError
10
11
  from cognite.neat._rules import importers
11
12
  from cognite.neat._rules._shared import ReadRules, VerifiedRules
12
- from cognite.neat._rules.models import DMSRules
13
+ from cognite.neat._rules.importers import DMSImporter
14
+ from cognite.neat._rules.models import DMSInputRules, DMSRules, SheetList
13
15
  from cognite.neat._rules.models.information._rules import InformationRules
14
16
  from cognite.neat._rules.models.information._rules_input import InformationInputRules
15
17
  from cognite.neat._rules.transformers import ConvertToRules, VerifyAnyRules
@@ -18,10 +20,11 @@ from cognite.neat._store._provenance import (
18
20
  INSTANCES_ENTITY,
19
21
  Change,
20
22
  )
21
- from cognite.neat._utils.auth import _CLIENT_NAME
22
23
 
23
24
  from ._collector import _COLLECTOR, Collector
25
+ from ._drop import DropAPI
24
26
  from ._inspect import InspectAPI
27
+ from ._mapping import MappingAPI
25
28
  from ._prepare import PrepareAPI
26
29
  from ._read import ReadAPI
27
30
  from ._set import SetAPI
@@ -41,19 +44,19 @@ class NeatSession:
41
44
  verbose: bool = True,
42
45
  load_engine: Literal["newest", "cache", "skip"] = "cache",
43
46
  ) -> None:
44
- self._client = client
47
+ self._client = NeatClient(client) if client else None
45
48
  self._verbose = verbose
46
49
  self._state = SessionState(store_type=storage)
47
- self.read = ReadAPI(self._state, client, verbose)
48
- self.to = ToAPI(self._state, client, verbose)
49
- self.prepare = PrepareAPI(self._state, verbose)
50
+ self.read = ReadAPI(self._state, self._client, verbose)
51
+ self.to = ToAPI(self._state, self._client, verbose)
52
+ self.prepare = PrepareAPI(self._client, self._state, verbose)
50
53
  self.show = ShowAPI(self._state)
51
54
  self.set = SetAPI(self._state, verbose)
52
55
  self.inspect = InspectAPI(self._state)
56
+ self.mapping = MappingAPI(self._state)
57
+ self.drop = DropAPI(self._state)
53
58
  self.opt = OptAPI()
54
59
  self.opt._display()
55
- if self._client is not None and self._client._config is not None:
56
- self._client._config.client_name = _CLIENT_NAME
57
60
  if load_engine != "skip" and (engine_version := load_neat_engine(client, load_engine)):
58
61
  print(f"Neat Engine {engine_version} loaded.")
59
62
 
@@ -63,6 +66,28 @@ class NeatSession:
63
66
 
64
67
  def verify(self) -> IssueList:
65
68
  source_id, last_unverified_rule = self._state.data_model.last_unverified_rule
69
+
70
+ reference_rules: DMSInputRules | None = None
71
+ if isinstance(last_unverified_rule.rules, DMSInputRules):
72
+ dms_rules = last_unverified_rule.rules
73
+ views_ids, containers_ids = dms_rules.imported_views_and_containers_ids()
74
+ if views_ids or containers_ids:
75
+ if self._client is None:
76
+ raise NeatSessionError(
77
+ "No client provided. You are referencing unknown views and containers in your data model, "
78
+ "NEAT needs a client to lookup the definitions. "
79
+ "Please set the client in the session, NeatSession(client=client)."
80
+ )
81
+ schema = self._client.schema.retrieve(list(views_ids), list(containers_ids))
82
+
83
+ importer = DMSImporter(schema)
84
+ reference_rules = importer.to_rules().rules
85
+
86
+ if reference_rules is not None:
87
+ dms_rules.views.extend(reference_rules.views)
88
+ if dms_rules.containers:
89
+ dms_rules.containers.extend(reference_rules.containers or [])
90
+
66
91
  transformer = VerifyAnyRules("continue")
67
92
  start = datetime.now(timezone.utc)
68
93
  output = transformer.try_transform(last_unverified_rule)
@@ -78,6 +103,22 @@ class NeatSession:
78
103
  self._state.data_model.provenance.source_entity(source_id)
79
104
  or self._state.data_model.provenance.target_entity(source_id),
80
105
  )
106
+ if reference_rules is not None and isinstance(output.rules, DMSRules):
107
+ # Remove the referenced views and containers from the rules
108
+ ref_view_ids = set(reference_rules.as_view_entities())
109
+ if ref_view_ids:
110
+ output.rules.views = SheetList(
111
+ [view for view in output.rules.views if view.view not in ref_view_ids]
112
+ )
113
+ ref_container_ids = reference_rules.as_container_entities()
114
+ if output.rules.containers and ref_container_ids:
115
+ output.rules.containers = SheetList(
116
+ [
117
+ container
118
+ for container in output.rules.containers
119
+ if container.container not in ref_container_ids
120
+ ]
121
+ )
81
122
 
82
123
  self._state.data_model.write(output.rules, change)
83
124
 
@@ -0,0 +1,35 @@
1
+ from rdflib import URIRef
2
+
3
+ from ._state import SessionState
4
+ from .exceptions import session_class_wrapper
5
+
6
+ try:
7
+ from rich import print
8
+ except ImportError:
9
+ ...
10
+
11
+
12
+ @session_class_wrapper
13
+ class DropAPI:
14
+ def __init__(self, state: SessionState):
15
+ self._state = state
16
+
17
+ def instances(self, type: str | list[str]) -> None:
18
+ """Drop instances from the session.
19
+
20
+ Args:
21
+ type: The type of instances to drop.
22
+ """
23
+ type_list = type if isinstance(type, list) else [type]
24
+ uri_type_type = dict((v, k) for k, v in self._state.instances.store.queries.types.items())
25
+ selected_uri_by_type: dict[URIRef, str] = {}
26
+ for type_item in type_list:
27
+ if type_item not in uri_type_type:
28
+ print(f"Type {type_item} not found.")
29
+ selected_uri_by_type[uri_type_type[type_item]] = type_item
30
+
31
+ result = self._state.instances.store.queries.drop_types(list(selected_uri_by_type.keys()))
32
+
33
+ for type_uri, count in result.items():
34
+ print(f"Dropped {count} instances of type {selected_uri_by_type[type_uri]}")
35
+ return None
@@ -11,6 +11,13 @@ from cognite.neat._utils.upload import UploadResult, UploadResultCore, UploadRes
11
11
  from ._state import SessionState
12
12
  from .exceptions import session_class_wrapper
13
13
 
14
+ try:
15
+ from rich.markdown import Markdown as RichMarkdown
16
+
17
+ RICH_AVAILABLE = True
18
+ except ImportError:
19
+ RICH_AVAILABLE = False
20
+
14
21
 
15
22
  @session_class_wrapper
16
23
  class InspectAPI:
@@ -61,14 +68,19 @@ class InspectIssues:
61
68
  closest_match = set(difflib.get_close_matches(search, unique_types))
62
69
  issues = IssueList([issue for issue in issues if type(issue).__name__ in closest_match])
63
70
 
71
+ issue_str = "\n".join(
72
+ [f" * **{type(issue).__name__}**: {issue.as_message(include_type=False)}" for issue in issues]
73
+ )
74
+ markdown_str = f"### {len(issues)} issues found\n\n{issue_str}"
75
+
64
76
  if IN_NOTEBOOK:
65
77
  from IPython.display import Markdown, display
66
78
 
67
- issue_str = "\n".join(
68
- [f" * **{type(issue).__name__}**: {issue.as_message(include_type=False)}" for issue in issues]
69
- )
70
- message = f"### {len(issues)} issues found\n\n{issue_str}"
71
- display(Markdown(message))
79
+ display(Markdown(markdown_str))
80
+ elif RICH_AVAILABLE:
81
+ from rich import print
82
+
83
+ print(RichMarkdown(markdown_str))
72
84
 
73
85
  if return_dataframe:
74
86
  return issues.to_pandas()
@@ -0,0 +1,39 @@
1
+ from datetime import datetime, timezone
2
+
3
+ from cognite.neat._rules.models.mapping import load_classic_to_core_mapping
4
+ from cognite.neat._rules.transformers import RuleMapper
5
+ from cognite.neat._store._provenance import Change
6
+
7
+ from ._state import SessionState
8
+ from .exceptions import session_class_wrapper
9
+
10
+
11
+ @session_class_wrapper
12
+ class MappingAPI:
13
+ def __init__(self, state: SessionState):
14
+ self._state = state
15
+
16
+ def classic_to_core(self, org_name: str) -> None:
17
+ """Map classic types to core types.
18
+
19
+ Note this automatically creates an extended CogniteCore model.
20
+
21
+ """
22
+ source_id, rules = self._state.data_model.last_verified_dms_rules
23
+
24
+ start = datetime.now(timezone.utc)
25
+ transformer = RuleMapper(load_classic_to_core_mapping(org_name, rules.metadata.space, rules.metadata.version))
26
+ output = transformer.transform(rules)
27
+ end = datetime.now(timezone.utc)
28
+
29
+ change = Change.from_rules_activity(
30
+ output,
31
+ transformer.agent,
32
+ start,
33
+ end,
34
+ "Mapping classic to core",
35
+ self._state.data_model.provenance.source_entity(source_id)
36
+ or self._state.data_model.provenance.target_entity(source_id),
37
+ )
38
+
39
+ self._state.data_model.write(output.rules, change)