cognite-neat 0.107.0__py3-none-any.whl → 0.109.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (69) hide show
  1. cognite/neat/_constants.py +35 -1
  2. cognite/neat/_graph/_shared.py +4 -0
  3. cognite/neat/_graph/extractors/_classic_cdf/_base.py +115 -14
  4. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +87 -6
  5. cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +48 -12
  6. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +19 -1
  7. cognite/neat/_graph/extractors/_dms.py +162 -47
  8. cognite/neat/_graph/extractors/_dms_graph.py +54 -4
  9. cognite/neat/_graph/extractors/_mock_graph_generator.py +1 -1
  10. cognite/neat/_graph/extractors/_rdf_file.py +3 -2
  11. cognite/neat/_graph/loaders/__init__.py +1 -3
  12. cognite/neat/_graph/loaders/_rdf2dms.py +20 -10
  13. cognite/neat/_graph/queries/_base.py +144 -84
  14. cognite/neat/_graph/queries/_construct.py +1 -1
  15. cognite/neat/_graph/transformers/__init__.py +3 -1
  16. cognite/neat/_graph/transformers/_base.py +4 -4
  17. cognite/neat/_graph/transformers/_classic_cdf.py +13 -13
  18. cognite/neat/_graph/transformers/_prune_graph.py +3 -3
  19. cognite/neat/_graph/transformers/_rdfpath.py +3 -4
  20. cognite/neat/_graph/transformers/_value_type.py +71 -13
  21. cognite/neat/_issues/errors/__init__.py +2 -0
  22. cognite/neat/_issues/errors/_external.py +8 -0
  23. cognite/neat/_issues/errors/_resources.py +1 -1
  24. cognite/neat/_issues/warnings/__init__.py +0 -2
  25. cognite/neat/_issues/warnings/_models.py +1 -1
  26. cognite/neat/_issues/warnings/_properties.py +0 -8
  27. cognite/neat/_issues/warnings/_resources.py +1 -1
  28. cognite/neat/_rules/catalog/classic_model.xlsx +0 -0
  29. cognite/neat/_rules/exporters/_rules2instance_template.py +3 -3
  30. cognite/neat/_rules/exporters/_rules2yaml.py +1 -1
  31. cognite/neat/_rules/importers/__init__.py +3 -1
  32. cognite/neat/_rules/importers/_dtdl2rules/spec.py +1 -2
  33. cognite/neat/_rules/importers/_rdf/__init__.py +2 -2
  34. cognite/neat/_rules/importers/_rdf/_base.py +2 -2
  35. cognite/neat/_rules/importers/_rdf/_inference2rules.py +310 -26
  36. cognite/neat/_rules/models/_base_rules.py +22 -11
  37. cognite/neat/_rules/models/dms/_exporter.py +5 -4
  38. cognite/neat/_rules/models/dms/_rules.py +1 -8
  39. cognite/neat/_rules/models/dms/_rules_input.py +4 -0
  40. cognite/neat/_rules/models/information/_rules_input.py +5 -0
  41. cognite/neat/_rules/transformers/__init__.py +10 -3
  42. cognite/neat/_rules/transformers/_base.py +6 -1
  43. cognite/neat/_rules/transformers/_converters.py +530 -364
  44. cognite/neat/_rules/transformers/_mapping.py +4 -4
  45. cognite/neat/_session/_base.py +100 -47
  46. cognite/neat/_session/_create.py +133 -0
  47. cognite/neat/_session/_drop.py +60 -2
  48. cognite/neat/_session/_fix.py +28 -0
  49. cognite/neat/_session/_inspect.py +22 -7
  50. cognite/neat/_session/_mapping.py +8 -8
  51. cognite/neat/_session/_prepare.py +3 -247
  52. cognite/neat/_session/_read.py +138 -17
  53. cognite/neat/_session/_set.py +50 -1
  54. cognite/neat/_session/_show.py +16 -43
  55. cognite/neat/_session/_state.py +53 -52
  56. cognite/neat/_session/_to.py +11 -4
  57. cognite/neat/_session/_wizard.py +1 -1
  58. cognite/neat/_session/exceptions.py +8 -1
  59. cognite/neat/_store/_graph_store.py +301 -146
  60. cognite/neat/_store/_provenance.py +36 -20
  61. cognite/neat/_store/_rules_store.py +253 -267
  62. cognite/neat/_store/exceptions.py +40 -4
  63. cognite/neat/_utils/auth.py +5 -3
  64. cognite/neat/_version.py +1 -1
  65. {cognite_neat-0.107.0.dist-info → cognite_neat-0.109.0.dist-info}/METADATA +1 -1
  66. {cognite_neat-0.107.0.dist-info → cognite_neat-0.109.0.dist-info}/RECORD +69 -67
  67. {cognite_neat-0.107.0.dist-info → cognite_neat-0.109.0.dist-info}/LICENSE +0 -0
  68. {cognite_neat-0.107.0.dist-info → cognite_neat-0.109.0.dist-info}/WHEEL +0 -0
  69. {cognite_neat-0.107.0.dist-info → cognite_neat-0.109.0.dist-info}/entry_points.txt +0 -0
@@ -1,4 +1,3 @@
1
- import dataclasses
2
1
  import re
3
2
  import warnings
4
3
  from abc import ABC
@@ -9,13 +8,16 @@ from typing import ClassVar, Literal, TypeVar, cast, overload
9
8
 
10
9
  from cognite.client.data_classes import data_modeling as dms
11
10
  from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
11
+ from cognite.client.utils.useful_types import SequenceNotStr
12
12
  from rdflib import Namespace
13
13
 
14
14
  from cognite.neat._client import NeatClient
15
15
  from cognite.neat._client.data_classes.data_modeling import ContainerApplyDict, ViewApplyDict
16
16
  from cognite.neat._constants import (
17
17
  COGNITE_MODELS,
18
+ COGNITE_SPACES,
18
19
  DMS_CONTAINER_PROPERTY_SIZE_LIMIT,
20
+ DMS_RESERVED_PROPERTIES,
19
21
  get_default_prefixes_and_namespaces,
20
22
  )
21
23
  from cognite.neat._issues.errors import NeatValueError
@@ -26,8 +28,6 @@ from cognite.neat._issues.warnings._models import (
26
28
  )
27
29
  from cognite.neat._rules._shared import (
28
30
  ReadInputRules,
29
- ReadRules,
30
- T_InputRules,
31
31
  VerifiedRules,
32
32
  )
33
33
  from cognite.neat._rules.analysis import DMSAnalysis
@@ -41,46 +41,37 @@ from cognite.neat._rules.models import (
41
41
  )
42
42
  from cognite.neat._rules.models._rdfpath import Entity as RDFPathEntity
43
43
  from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
44
- from cognite.neat._rules.models.data_types import AnyURI, DataType, String
44
+ from cognite.neat._rules.models.data_types import AnyURI, DataType, Enum, File, String, Timeseries
45
45
  from cognite.neat._rules.models.dms import DMSMetadata, DMSProperty, DMSValidation, DMSView
46
- from cognite.neat._rules.models.dms._rules import DMSContainer
46
+ from cognite.neat._rules.models.dms._rules import DMSContainer, DMSEnum, DMSNode
47
47
  from cognite.neat._rules.models.entities import (
48
48
  ClassEntity,
49
49
  ContainerEntity,
50
50
  DMSUnknownEntity,
51
51
  EdgeEntity,
52
- Entity,
53
52
  HasDataFilter,
54
53
  MultiValueTypeInfo,
55
54
  ReverseConnectionEntity,
56
- T_Entity,
57
55
  UnknownEntity,
58
56
  ViewEntity,
59
57
  )
60
58
  from cognite.neat._rules.models.information import InformationClass, InformationMetadata, InformationProperty
61
- from cognite.neat._rules.models.information._rules_input import (
62
- InformationInputClass,
63
- InformationInputProperty,
64
- InformationInputRules,
65
- )
66
59
  from cognite.neat._utils.text import to_camel
67
60
 
68
- from ._base import RulesTransformer
61
+ from ._base import T_VerifiedIn, T_VerifiedOut, VerifiedRulesTransformer
69
62
  from ._verification import VerifyDMSRules
70
63
 
71
- T_VerifiedInRules = TypeVar("T_VerifiedInRules", bound=VerifiedRules)
72
- T_VerifiedOutRules = TypeVar("T_VerifiedOutRules", bound=VerifiedRules)
73
64
  T_InputInRules = TypeVar("T_InputInRules", bound=ReadInputRules)
74
65
  T_InputOutRules = TypeVar("T_InputOutRules", bound=ReadInputRules)
75
66
 
76
67
 
77
- class ConversionTransformer(RulesTransformer[T_VerifiedInRules, T_VerifiedOutRules], ABC):
68
+ class ConversionTransformer(VerifiedRulesTransformer[T_VerifiedIn, T_VerifiedOut], ABC):
78
69
  """Base class for all conversion transformers."""
79
70
 
80
71
  ...
81
72
 
82
73
 
83
- class ToCompliantEntities(RulesTransformer[ReadRules[InformationInputRules], ReadRules[InformationInputRules]]): # type: ignore[misc]
74
+ class ToCompliantEntities(VerifiedRulesTransformer[InformationRules, InformationRules]): # type: ignore[misc]
84
75
  """Converts input rules to rules with compliant entity IDs that match regex patters used
85
76
  by DMS schema components."""
86
77
 
@@ -88,13 +79,11 @@ class ToCompliantEntities(RulesTransformer[ReadRules[InformationInputRules], Rea
88
79
  def description(self) -> str:
89
80
  return "Ensures externalIDs are compliant with CDF"
90
81
 
91
- def transform(self, rules: ReadRules[InformationInputRules]) -> ReadRules[InformationInputRules]:
92
- if rules.rules is None:
93
- return rules
94
- copy: InformationInputRules = dataclasses.replace(rules.rules)
82
+ def transform(self, rules: InformationRules) -> InformationRules:
83
+ copy = rules.model_copy(deep=True)
95
84
  copy.classes = self._fix_classes(copy.classes)
96
85
  copy.properties = self._fix_properties(copy.properties)
97
- return ReadRules(copy, rules.read_context)
86
+ return copy
98
87
 
99
88
  @classmethod
100
89
  def _fix_entity(cls, entity: str) -> str:
@@ -111,16 +100,8 @@ class ToCompliantEntities(RulesTransformer[ReadRules[InformationInputRules], Rea
111
100
  return re.sub(r"[^a-zA-Z0-9]+", "_", entity)
112
101
 
113
102
  @classmethod
114
- def _fix_class(cls, class_: str | ClassEntity) -> str | ClassEntity:
115
- if isinstance(class_, str):
116
- if len(class_.split(":")) == 2:
117
- prefix, suffix = class_.split(":")
118
- class_ = f"{cls._fix_entity(prefix)}:{cls._fix_entity(suffix)}"
119
-
120
- else:
121
- class_ = cls._fix_entity(class_)
122
-
123
- elif isinstance(class_, ClassEntity) and type(class_.prefix) is str:
103
+ def _fix_class(cls, class_: ClassEntity) -> ClassEntity:
104
+ if isinstance(class_, ClassEntity) and type(class_.prefix) is str:
124
105
  class_ = ClassEntity(
125
106
  prefix=cls._fix_entity(class_.prefix),
126
107
  suffix=cls._fix_entity(class_.suffix),
@@ -130,28 +111,17 @@ class ToCompliantEntities(RulesTransformer[ReadRules[InformationInputRules], Rea
130
111
 
131
112
  @classmethod
132
113
  def _fix_value_type(
133
- cls, value_type: str | DataType | ClassEntity | MultiValueTypeInfo
134
- ) -> str | DataType | ClassEntity | MultiValueTypeInfo:
135
- fixed_value_type: str | DataType | ClassEntity | MultiValueTypeInfo
136
-
137
- if isinstance(value_type, str):
138
- # this is a multi value type but as string
139
- if " | " in value_type:
140
- value_types = value_type.split(" | ")
141
- fixed_value_type = " | ".join([cast(str, cls._fix_value_type(v)) for v in value_types])
142
- # this is value type specified with prefix:suffix string
143
- elif ":" in value_type:
144
- fixed_value_type = cls._fix_class(value_type)
145
-
146
- # this is value type specified as suffix only
147
- else:
148
- fixed_value_type = cls._fix_entity(value_type)
114
+ cls, value_type: DataType | ClassEntity | MultiValueTypeInfo
115
+ ) -> DataType | ClassEntity | MultiValueTypeInfo:
116
+ fixed_value_type: DataType | ClassEntity | MultiValueTypeInfo
149
117
 
150
- # value type specified as instances of DataType, ClassEntity or MultiValueTypeInfo
151
- elif isinstance(value_type, MultiValueTypeInfo):
118
+ # value type specified as MultiValueTypeInfo
119
+ if isinstance(value_type, MultiValueTypeInfo):
152
120
  fixed_value_type = MultiValueTypeInfo(
153
121
  types=[cast(DataType | ClassEntity, cls._fix_value_type(type_)) for type_ in value_type.types],
154
122
  )
123
+
124
+ # value type specified as ClassEntity instance
155
125
  elif isinstance(value_type, ClassEntity):
156
126
  fixed_value_type = cls._fix_class(value_type)
157
127
 
@@ -162,16 +132,16 @@ class ToCompliantEntities(RulesTransformer[ReadRules[InformationInputRules], Rea
162
132
  return fixed_value_type
163
133
 
164
134
  @classmethod
165
- def _fix_classes(cls, definitions: list[InformationInputClass]) -> list[InformationInputClass]:
166
- fixed_definitions = []
135
+ def _fix_classes(cls, definitions: SheetList[InformationClass]) -> SheetList[InformationClass]:
136
+ fixed_definitions = SheetList[InformationClass]()
167
137
  for definition in definitions:
168
138
  definition.class_ = cls._fix_class(definition.class_)
169
139
  fixed_definitions.append(definition)
170
140
  return fixed_definitions
171
141
 
172
142
  @classmethod
173
- def _fix_properties(cls, definitions: list[InformationInputProperty]) -> list[InformationInputProperty]:
174
- fixed_definitions = []
143
+ def _fix_properties(cls, definitions: SheetList[InformationProperty]) -> SheetList[InformationProperty]:
144
+ fixed_definitions = SheetList[InformationProperty]()
175
145
  for definition in definitions:
176
146
  definition.class_ = cls._fix_class(definition.class_)
177
147
  definition.property_ = cls._fix_entity(definition.property_)
@@ -180,83 +150,115 @@ class ToCompliantEntities(RulesTransformer[ReadRules[InformationInputRules], Rea
180
150
  return fixed_definitions
181
151
 
182
152
 
183
- class PrefixEntities(RulesTransformer[ReadRules[T_InputRules], ReadRules[T_InputRules]]): # type: ignore[type-var]
184
- """Prefixes all entities with a given prefix."""
153
+ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
154
+ """Prefixes all entities with a given prefix if they are in the same space as data model."""
185
155
 
186
156
  def __init__(self, prefix: str) -> None:
187
157
  self._prefix = prefix
188
158
 
189
159
  @property
190
160
  def description(self) -> str:
191
- return f"Prefixes all views with {self._prefix!r}"
161
+ return f"Prefixes all entities with {self._prefix!r} prefix if they are in the same space as data model."
192
162
 
193
- def transform(self, rules: ReadRules[T_InputRules]) -> ReadRules[T_InputRules]:
194
- in_ = rules.rules
195
- if in_ is None:
196
- return rules
197
- copy: T_InputRules = dataclasses.replace(in_)
198
- if isinstance(copy, InformationInputRules):
199
- prefixed_by_class: dict[str, str] = {}
163
+ @overload
164
+ def transform(self, rules: DMSRules) -> DMSRules: ...
165
+
166
+ @overload
167
+ def transform(self, rules: InformationRules) -> InformationRules: ...
168
+
169
+ def transform(self, rules: InformationRules | DMSRules) -> InformationRules | DMSRules:
170
+ copy: InformationRules | DMSRules = rules.model_copy(deep=True)
171
+
172
+ # Case: Prefix Information Rules
173
+ if isinstance(copy, InformationRules):
174
+ # prefix classes
200
175
  for cls in copy.classes:
201
- prefixed = str(self._with_prefix(cls.class_))
202
- prefixed_by_class[str(cls.class_)] = prefixed
203
- cls.class_ = prefixed
176
+ if cls.class_.prefix == copy.metadata.prefix:
177
+ cls.class_ = self._with_prefix(cls.class_)
178
+
179
+ if cls.implements:
180
+ # prefix parents
181
+ for i, parent_class in enumerate(cls.implements):
182
+ if parent_class.prefix == copy.metadata.prefix:
183
+ cls.implements[i] = self._with_prefix(parent_class)
184
+
204
185
  for prop in copy.properties:
205
- prop.class_ = self._with_prefix(prop.class_)
206
- if str(prop.value_type) in prefixed_by_class:
207
- prop.value_type = prefixed_by_class[str(prop.value_type)]
208
- return ReadRules(copy, rules.read_context) # type: ignore[arg-type]
209
- elif isinstance(copy, DMSInputRules):
210
- prefixed_by_view: dict[str, str] = {}
186
+ if prop.class_.prefix == copy.metadata.prefix:
187
+ prop.class_ = self._with_prefix(prop.class_)
188
+
189
+ # value type property is not multi and it is ClassEntity
190
+
191
+ if isinstance(prop.value_type, ClassEntity) and prop.value_type.prefix == copy.metadata.prefix:
192
+ prop.value_type = self._with_prefix(cast(ClassEntity, prop.value_type))
193
+ elif isinstance(prop.value_type, MultiValueTypeInfo):
194
+ for i, value_type in enumerate(prop.value_type.types):
195
+ if isinstance(value_type, ClassEntity) and value_type.prefix == copy.metadata.prefix:
196
+ prop.value_type.types[i] = self._with_prefix(cast(ClassEntity, value_type))
197
+ return copy
198
+
199
+ # Case: Prefix DMS Rules
200
+ elif isinstance(copy, DMSRules):
211
201
  for view in copy.views:
212
- prefixed = str(self._with_prefix(view.view))
213
- prefixed_by_view[str(view.view)] = prefixed
214
- view.view = prefixed
202
+ if view.view.space == copy.metadata.space:
203
+ view.view = self._with_prefix(view.view)
204
+
205
+ if view.implements:
206
+ for i, parent_view in enumerate(view.implements):
207
+ if parent_view.space == copy.metadata.space:
208
+ view.implements[i] = self._with_prefix(parent_view)
209
+
215
210
  for dms_prop in copy.properties:
216
- dms_prop.view = self._with_prefix(dms_prop.view)
217
- if str(dms_prop.value_type) in prefixed_by_view:
218
- dms_prop.value_type = prefixed_by_view[str(dms_prop.value_type)]
211
+ if dms_prop.view.space == copy.metadata.space:
212
+ dms_prop.view = self._with_prefix(dms_prop.view)
213
+
214
+ if isinstance(dms_prop.value_type, ViewEntity) and dms_prop.value_type.space == copy.metadata.space:
215
+ dms_prop.value_type = self._with_prefix(dms_prop.value_type)
216
+
217
+ if isinstance(dms_prop.container, ContainerEntity) and dms_prop.container.space == copy.metadata.space:
218
+ dms_prop.container = self._with_prefix(dms_prop.container)
219
+
219
220
  if copy.containers:
220
221
  for container in copy.containers:
221
- container.container = self._with_prefix(container.container)
222
- return ReadRules(copy, rules.read_context)
222
+ if container.container.space == copy.metadata.space:
223
+ container.container = self._with_prefix(container.container)
224
+ return copy
225
+
223
226
  raise NeatValueError(f"Unsupported rules type: {type(copy)}")
224
227
 
225
228
  @overload
226
- def _with_prefix(self, raw: str) -> str: ...
229
+ def _with_prefix(self, entity: ClassEntity) -> ClassEntity: ...
227
230
 
228
231
  @overload
229
- def _with_prefix(self, raw: T_Entity) -> T_Entity: ...
230
-
231
- def _with_prefix(self, raw: str | T_Entity) -> str | T_Entity:
232
- is_entity_format = not isinstance(raw, str)
233
- entity = Entity.load(raw)
234
- output: ClassEntity | ViewEntity | ContainerEntity
235
- if isinstance(entity, ClassEntity):
236
- output = ClassEntity(prefix=entity.prefix, suffix=f"{self._prefix}{entity.suffix}", version=entity.version)
237
- elif isinstance(entity, ViewEntity):
238
- output = ViewEntity(
239
- space=entity.space, externalId=f"{self._prefix}{entity.external_id}", version=entity.version
240
- )
241
- elif isinstance(entity, ContainerEntity):
242
- output = ContainerEntity(space=entity.space, externalId=f"{self._prefix}{entity.external_id}")
243
- elif isinstance(entity, UnknownEntity | Entity):
244
- return f"{self._prefix}{raw}"
232
+ def _with_prefix(self, entity: ViewEntity) -> ViewEntity: ...
233
+
234
+ @overload
235
+ def _with_prefix(self, entity: ContainerEntity) -> ContainerEntity: ...
236
+
237
+ def _with_prefix(
238
+ self, entity: ViewEntity | ContainerEntity | ClassEntity
239
+ ) -> ViewEntity | ContainerEntity | ClassEntity:
240
+ if isinstance(entity, ViewEntity | ContainerEntity | ClassEntity):
241
+ entity.suffix = f"{self._prefix}{entity.suffix}"
242
+
245
243
  else:
246
244
  raise NeatValueError(f"Unsupported entity type: {type(entity)}")
247
- if is_entity_format:
248
- return cast(T_Entity, output)
249
- return str(output)
245
+
246
+ return entity
250
247
 
251
248
 
252
249
  class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
253
250
  """Converts InformationRules to DMSRules."""
254
251
 
255
- def __init__(self, ignore_undefined_value_types: bool = False):
252
+ def __init__(
253
+ self, ignore_undefined_value_types: bool = False, reserved_properties: Literal["error", "warning"] = "error"
254
+ ):
256
255
  self.ignore_undefined_value_types = ignore_undefined_value_types
256
+ self.reserved_properties = reserved_properties
257
257
 
258
258
  def transform(self, rules: InformationRules) -> DMSRules:
259
- return _InformationRulesConverter(rules).as_dms_rules(self.ignore_undefined_value_types)
259
+ return _InformationRulesConverter(rules).as_dms_rules(
260
+ self.ignore_undefined_value_types, self.reserved_properties
261
+ )
260
262
 
261
263
 
262
264
  class DMSToInformation(ConversionTransformer[DMSRules, InformationRules]):
@@ -288,7 +290,7 @@ class ConvertToRules(ConversionTransformer[VerifiedRules, VerifiedRules]):
288
290
  _T_Entity = TypeVar("_T_Entity", bound=ClassEntity | ViewEntity)
289
291
 
290
292
 
291
- class SetIDDMSModel(RulesTransformer[DMSRules, DMSRules]):
293
+ class SetIDDMSModel(VerifiedRulesTransformer[DMSRules, DMSRules]):
292
294
  def __init__(self, new_id: DataModelId | tuple[str, str, str]):
293
295
  self.new_id = DataModelId.load(new_id)
294
296
 
@@ -308,77 +310,17 @@ class SetIDDMSModel(RulesTransformer[DMSRules, DMSRules]):
308
310
  return DMSRules.model_validate(DMSInputRules.load(dump).dump())
309
311
 
310
312
 
311
- class ToExtensionModel(RulesTransformer[DMSRules, DMSRules], ABC):
313
+ class ToExtensionModel(VerifiedRulesTransformer[DMSRules, DMSRules], ABC):
312
314
  type_: ClassVar[str]
313
315
 
314
- def __init__(self, new_model_id: DataModelIdentifier, org_name: str, dummy_property: str | None = None) -> None:
316
+ def __init__(self, new_model_id: DataModelIdentifier) -> None:
315
317
  self.new_model_id = DataModelId.load(new_model_id)
316
318
  if not self.new_model_id.version:
317
319
  raise NeatValueError("Version is required for the new model.")
318
- self.org_name = org_name
319
- self.dummy_property = dummy_property
320
320
 
321
321
  @property
322
322
  def description(self) -> str:
323
- return f"Prepared data model {self.new_model_id} to be {self.type_.replace('_', ' ')} data model."
324
-
325
- def _create_new_views(
326
- self, rules: DMSRules
327
- ) -> tuple[SheetList[DMSView], SheetList[DMSContainer], SheetList[DMSProperty]]:
328
- """Creates new views for the new model.
329
-
330
- If the dummy property is provided, it will also create a new container for each view
331
- with a single property that is the dummy property.
332
- """
333
- new_views = SheetList[DMSView]()
334
- new_containers = SheetList[DMSContainer]()
335
- new_properties = SheetList[DMSProperty]()
336
-
337
- for definition in rules.views:
338
- view_entity = self._remove_cognite_affix(definition.view)
339
- view_entity.version = cast(str, self.new_model_id.version)
340
- view_entity.prefix = self.new_model_id.space
341
-
342
- new_views.append(
343
- DMSView(
344
- view=view_entity,
345
- implements=[definition.view],
346
- in_model=True,
347
- name=definition.name,
348
- )
349
- )
350
- if self.dummy_property is None:
351
- continue
352
-
353
- container_entity = ContainerEntity(space=view_entity.prefix, externalId=view_entity.external_id)
354
-
355
- container = DMSContainer(container=container_entity)
356
-
357
- prefix = to_camel(view_entity.suffix)
358
- property_ = DMSProperty(
359
- view=view_entity,
360
- view_property=f"{prefix}{self.dummy_property}",
361
- value_type=String(),
362
- nullable=True,
363
- immutable=False,
364
- is_list=False,
365
- container=container_entity,
366
- container_property=f"{prefix}{self.dummy_property}",
367
- )
368
-
369
- new_properties.append(property_)
370
- new_containers.append(container)
371
-
372
- return new_views, new_containers, new_properties
373
-
374
- def _remove_cognite_affix(self, entity: _T_Entity) -> _T_Entity:
375
- """This method removes `Cognite` affix from the entity."""
376
- new_suffix = entity.suffix.replace("Cognite", self.org_name or "")
377
- if isinstance(entity, ViewEntity):
378
- return ViewEntity(space=entity.space, externalId=new_suffix, version=entity.version) # type: ignore[return-value]
379
- elif isinstance(entity, ClassEntity):
380
- return ClassEntity(prefix=entity.prefix, suffix=new_suffix, version=entity.version) # type: ignore[return-value]
381
- raise ValueError(f"Unsupported entity type: {type(entity)}")
323
+ return f"Create new data model {self.new_model_id} of type {self.type_.replace('_', ' ')} data model."
382
324
 
383
325
 
384
326
  class ToEnterpriseModel(ToExtensionModel):
@@ -391,7 +333,9 @@ class ToEnterpriseModel(ToExtensionModel):
391
333
  dummy_property: str = "GUID",
392
334
  move_connections: bool = False,
393
335
  ):
394
- super().__init__(new_model_id, org_name, dummy_property)
336
+ super().__init__(new_model_id)
337
+ self.dummy_property = dummy_property
338
+ self.org_name = org_name
395
339
  self.move_connections = move_connections
396
340
 
397
341
  def transform(self, rules: DMSRules) -> DMSRules:
@@ -405,13 +349,8 @@ class ToEnterpriseModel(ToExtensionModel):
405
349
  return self._to_enterprise(rules)
406
350
 
407
351
  def _to_enterprise(self, reference_model: DMSRules) -> DMSRules:
408
- dump = reference_model.dump()
409
-
410
- # This will create reference model components in the enterprise model space
411
- enterprise_model = DMSRules.model_validate(DMSInputRules.load(dump).dump())
352
+ enterprise_model = reference_model.model_copy(deep=True)
412
353
 
413
- # Post validation metadata update:
414
- enterprise_model.metadata.name = self.type_
415
354
  enterprise_model.metadata.name = f"{self.org_name} {self.type_} data model"
416
355
  enterprise_model.metadata.space = self.new_model_id.space
417
356
  enterprise_model.metadata.external_id = self.new_model_id.external_id
@@ -426,43 +365,91 @@ class ToEnterpriseModel(ToExtensionModel):
426
365
 
427
366
  if self.move_connections:
428
367
  # Move connections from reference model to new enterprise model
429
- enterprise_properties.extend(self._move_connections(enterprise_model))
368
+ enterprise_properties.extend(self._create_connection_properties(enterprise_model, enterprise_views))
430
369
 
431
370
  # ... however, we do not want to keep the reference containers and properties
371
+ # these we are getting for free through the implements.
432
372
  enterprise_model.containers = enterprise_containers
433
373
  enterprise_model.properties = enterprise_properties
434
374
 
435
375
  return enterprise_model
436
376
 
437
377
  @staticmethod
438
- def _move_connections(rules: DMSRules) -> SheetList[DMSProperty]:
439
- implements: dict[ViewEntity, list[ViewEntity]] = defaultdict(list)
378
+ def _create_connection_properties(rules: DMSRules, new_views: SheetList[DMSView]) -> SheetList[DMSProperty]:
379
+ """Creates a new connection property for each connection property in the reference model.
380
+
381
+ This is for example when you create an enterprise model from CogniteCore, you ensure that your
382
+ new Asset, Equipment, TimeSeries, Activity, and File views all point to each other.
383
+ """
384
+ # Note all new news have an implements attribute that points to the original view
385
+ previous_by_new_view = {view.implements[0]: view.view for view in new_views if view.implements}
386
+ connection_properties = SheetList[DMSProperty]()
387
+ for prop in rules.properties:
388
+ if (
389
+ isinstance(prop.value_type, ViewEntity)
390
+ and prop.view in previous_by_new_view
391
+ and prop.value_type in previous_by_new_view
392
+ ):
393
+ new_property = prop.model_copy(deep=True)
394
+ new_property.view = previous_by_new_view[prop.view]
395
+ new_property.value_type = previous_by_new_view[prop.value_type]
396
+ connection_properties.append(new_property)
397
+
398
+ return connection_properties
399
+
400
+ def _create_new_views(
401
+ self, rules: DMSRules
402
+ ) -> tuple[SheetList[DMSView], SheetList[DMSContainer], SheetList[DMSProperty]]:
403
+ """Creates new views for the new model.
404
+
405
+ If the dummy property is provided, it will also create a new container for each view
406
+ with a single property that is the dummy property.
407
+ """
408
+ new_views = SheetList[DMSView]()
409
+ new_containers = SheetList[DMSContainer]()
440
410
  new_properties = SheetList[DMSProperty]()
441
411
 
442
- for view in rules.views:
443
- if view.view.space == rules.metadata.space and view.implements:
444
- for implemented_view in view.implements:
445
- implements.setdefault(implemented_view, []).append(view.view)
446
-
447
- # currently only supporting single implementation of reference view in enterprise view
448
- # connections that do not have properties
449
- if all(len(v) == 1 for v in implements.values()):
450
- for prop_ in rules.properties:
451
- if (
452
- prop_.view.space != rules.metadata.space
453
- and prop_.connection
454
- and isinstance(prop_.value_type, ViewEntity)
455
- and implements.get(prop_.view)
456
- and implements.get(prop_.value_type)
457
- ):
458
- if isinstance(prop_.connection, EdgeEntity) and prop_.connection.properties:
459
- continue
460
- new_property = prop_.model_copy(deep=True)
461
- new_property.view = implements[prop_.view][0]
462
- new_property.value_type = implements[prop_.value_type][0]
463
- new_properties.append(new_property)
464
-
465
- return new_properties
412
+ for definition in rules.views:
413
+ view_entity = self._remove_cognite_affix(definition.view)
414
+ view_entity.version = cast(str, self.new_model_id.version)
415
+ view_entity.prefix = self.new_model_id.space
416
+ new_views.append(
417
+ DMSView(
418
+ view=view_entity,
419
+ implements=[definition.view],
420
+ in_model=True,
421
+ name=definition.name,
422
+ )
423
+ )
424
+
425
+ if self.dummy_property is None:
426
+ continue
427
+
428
+ container_entity = ContainerEntity(space=view_entity.prefix, externalId=view_entity.external_id)
429
+
430
+ container = DMSContainer(container=container_entity)
431
+
432
+ property_id = f"{to_camel(view_entity.suffix)}{self.dummy_property}"
433
+ property_ = DMSProperty(
434
+ view=view_entity,
435
+ view_property=property_id,
436
+ value_type=String(),
437
+ nullable=True,
438
+ immutable=False,
439
+ is_list=False,
440
+ container=container_entity,
441
+ container_property=property_id,
442
+ )
443
+
444
+ new_properties.append(property_)
445
+ new_containers.append(container)
446
+
447
+ return new_views, new_containers, new_properties
448
+
449
+ def _remove_cognite_affix(self, entity: ViewEntity) -> ViewEntity:
450
+ """This method removes `Cognite` affix from the entity."""
451
+ new_suffix = entity.suffix.replace("Cognite", self.org_name)
452
+ return ViewEntity(space=entity.space, externalId=new_suffix, version=entity.version)
466
453
 
467
454
 
468
455
  class ToSolutionModel(ToExtensionModel):
@@ -472,18 +459,10 @@ class ToSolutionModel(ToExtensionModel):
472
459
 
473
460
  Args:
474
461
  new_model_id: DataData model identifier for the new model.
475
- org_name: If the existing model is a Cognite Data Model, this will replace the "Cognite" affix.
476
- mode: The mode of the solution model. Either "read" or "write". A "write" model will create a new
477
- container for each view with a dummy property. Read mode will only inherit the view filter from the
478
- original model.
479
462
  dummy_property: Only applicable if mode='write'. The identifier of the dummy property in the newly created
480
463
  container.
481
464
  exclude_views_in_other_spaces: Whether to exclude views that are not in the same space as the existing model,
482
465
  when creating the solution model.
483
- filter_type: If mode="read", this is the type of filter to apply to the new views. The filter is used to
484
- ensure that the new views will return the same instance as the original views. The view filter is the
485
- simplest filter, but it has limitation in the fusion UI. The container filter is in essence a more
486
- verbose version of the view filter, and it has better support in the fusion UI. The default is "container".
487
466
 
488
467
  """
489
468
 
@@ -492,152 +471,71 @@ class ToSolutionModel(ToExtensionModel):
492
471
  def __init__(
493
472
  self,
494
473
  new_model_id: DataModelIdentifier,
495
- org_name: str = "My",
496
- mode: Literal["read", "write"] = "read",
474
+ properties: Literal["repeat", "connection"] = "connection",
497
475
  dummy_property: str | None = "GUID",
498
- exclude_views_in_other_spaces: bool = True,
476
+ direct_property: str = "enterprise",
477
+ view_prefix: str = "Enterprise",
499
478
  filter_type: Literal["container", "view"] = "container",
479
+ exclude_views_in_other_spaces: bool = False,
480
+ skip_cognite_views: bool = True,
500
481
  ):
501
- super().__init__(new_model_id, org_name, dummy_property if mode == "write" else None)
502
- self.mode = mode
503
- self.exclude_views_in_other_spaces = exclude_views_in_other_spaces
482
+ super().__init__(new_model_id)
483
+ self.properties = properties
484
+ self.dummy_property = dummy_property
485
+ self.direct_property = direct_property
486
+ self.view_prefix = view_prefix
504
487
  self.filter_type = filter_type
488
+ self.exclude_views_in_other_spaces = exclude_views_in_other_spaces
489
+ self.skip_cognite_views = skip_cognite_views
505
490
 
506
491
  def transform(self, rules: DMSRules) -> DMSRules:
507
492
  reference_model = rules
508
493
  reference_model_id = reference_model.metadata.as_data_model_id()
509
-
510
- # if model is solution then we need to get correct space for views and containers
511
- if self.mode not in ["read", "write"]:
512
- raise NeatValueError(f"Unsupported mode: {self.mode}")
513
-
514
494
  if reference_model_id in COGNITE_MODELS:
515
495
  warnings.warn(
516
496
  SolutionModelBuildOnTopOfCDMWarning(reference_model_id=reference_model_id),
517
497
  stacklevel=2,
518
498
  )
519
-
520
499
  return self._to_solution(reference_model)
521
500
 
522
- @staticmethod
523
- def _has_views_in_multiple_space(rules: DMSRules) -> bool:
524
- return any(view.view.space != rules.metadata.space for view in rules.views)
525
-
526
501
  def _to_solution(self, reference_rules: DMSRules) -> DMSRules:
527
502
  """For creation of solution data model / rules specifically for mapping over existing containers."""
503
+ reference_rules = self._expand_properties(reference_rules.model_copy(deep=True))
528
504
 
529
- dump = reference_rules.dump(entities_exclude_defaults=True)
530
-
531
- # Prepare new model metadata prior validation
532
- # Since we dropped the defaults, all entities will update the space and version
533
- # to the new model space and version
534
- dump["metadata"]["name"] = f"{self.org_name} {self.type_} data model"
535
- dump["metadata"]["space"] = self.new_model_id.space
536
- dump["metadata"]["external_id"] = self.new_model_id.external_id
537
- dump["metadata"]["version"] = self.new_model_id.version
538
-
539
- solution_model = DMSRules.model_validate(DMSInputRules.load(dump).dump())
540
-
541
- # This is not desirable for the containers, so we manually fix that here.
542
- # It is easier to change the space for all entities and then revert the containers, than
543
- # to change the space for all entities except the containers.
544
- for prop in solution_model.properties:
545
- if prop.container and prop.container.space == self.new_model_id.space:
546
- # If the container is in the new model space, we want to map it to the reference model space
547
- # This is reverting the .dump() -> .load() above.
548
- prop.container = ContainerEntity(
549
- space=reference_rules.metadata.space,
550
- externalId=prop.container.suffix,
505
+ new_views, new_properties, read_view_by_new_view = self._create_views(reference_rules)
506
+ new_containers, new_container_properties = self._create_containers_update_view_filter(
507
+ new_views, reference_rules, read_view_by_new_view
508
+ )
509
+ new_properties.extend(new_container_properties)
510
+
511
+ if self.properties == "connection":
512
+ # Ensure the Enterprise view and the new solution view are next to each other
513
+ new_properties.sort(
514
+ key=lambda prop: (
515
+ prop.view.external_id.removeprefix(self.view_prefix),
516
+ bool(prop.view.external_id.startswith(self.view_prefix)),
551
517
  )
552
-
553
- for view in solution_model.views:
554
- view.implements = None
555
-
556
- if self.exclude_views_in_other_spaces and self._has_views_in_multiple_space(reference_rules):
557
- solution_model.views = SheetList[DMSView](
558
- [view for view in solution_model.views if view.view.space == solution_model.metadata.space]
559
- )
560
-
561
- # Dropping containers coming from reference model
562
- solution_model.containers = None
563
-
564
- # If reference model on which we are mapping one of Cognite Data Models
565
- # since we want to affix these with the organization name
566
- if reference_rules.metadata.as_data_model_id() in COGNITE_MODELS:
567
- # Remove Cognite affix in view external_id / suffix.
568
- for prop in solution_model.properties:
569
- prop.view = self._remove_cognite_affix(prop.view)
570
- if isinstance(prop.value_type, ViewEntity):
571
- prop.value_type = self._remove_cognite_affix(prop.value_type)
572
- for view in solution_model.views:
573
- view.view = self._remove_cognite_affix(view.view)
574
- if view.implements:
575
- view.implements = [self._remove_cognite_affix(implemented) for implemented in view.implements]
576
-
577
- if self.mode == "write":
578
- _, new_containers, new_properties = self._create_new_views(solution_model)
579
- # Here we add ONLY dummy properties of the solution model and
580
- # corresponding solution model space containers to hold them
581
- solution_model.containers = new_containers
582
- solution_model.properties.extend(new_properties)
583
- elif self.mode == "read":
584
- # Inherit view filter from original model to ensure the same instances are returned
585
- # when querying the new view.
586
- ref_views_by_external_id = {
587
- view.view.external_id: view
588
- for view in reference_rules.views
589
- if view.view.space == reference_rules.metadata.space
590
- }
591
- ref_containers_by_ref_view = defaultdict(set)
592
- for prop in reference_rules.properties:
593
- if prop.container:
594
- ref_containers_by_ref_view[prop.view].add(prop.container)
595
- for view in solution_model.views:
596
- if ref_view := ref_views_by_external_id.get(view.view.external_id):
597
- if self.filter_type == "view":
598
- view.filter_ = HasDataFilter(inner=[ref_view.view])
599
- elif self.filter_type == "container" and (
600
- ref_containers := ref_containers_by_ref_view.get(ref_view.view)
601
- ):
602
- # Sorting to ensure deterministic order
603
- view.filter_ = HasDataFilter(inner=sorted(ref_containers))
604
-
605
- return solution_model
606
-
607
-
608
- class ToDataProductModel(ToSolutionModel):
609
- type_: ClassVar[str] = "data_product"
610
-
611
- def __init__(
612
- self,
613
- new_model_id: DataModelIdentifier,
614
- org_name: str = "My",
615
- include: Literal["same-space", "all"] = "same-space",
616
- ):
617
- super().__init__(new_model_id, org_name, mode="read", dummy_property=None, exclude_views_in_other_spaces=False)
618
- self.include = include
619
-
620
- def transform(self, rules: DMSRules) -> DMSRules:
621
- # Copy to ensure immutability
622
- expanded = self._expand_properties(rules.model_copy(deep=True))
623
- if self.include == "same-space":
624
- expanded.views = SheetList[DMSView](
625
- [view for view in expanded.views if view.view.space == expanded.metadata.space]
626
518
  )
627
- used_view_entities = {view.view for view in expanded.views}
628
- expanded.properties = SheetList[DMSProperty](
629
- [
630
- prop
631
- for prop in expanded.properties
632
- if prop.view.space == expanded.metadata.space
633
- and (
634
- (isinstance(prop.value_type, ViewEntity) and prop.value_type in used_view_entities)
635
- or not isinstance(prop.value_type, ViewEntity)
636
- )
637
- ]
638
- )
639
-
640
- return self._to_solution(expanded)
519
+ else:
520
+ new_properties.sort(key=lambda prop: (prop.view.external_id, prop.view_property))
521
+
522
+ metadata = reference_rules.metadata.model_copy(
523
+ deep=True,
524
+ update={
525
+ "space": self.new_model_id.space,
526
+ "external_id": self.new_model_id.external_id,
527
+ "version": self.new_model_id.version,
528
+ "name": f"{self.type_} data model",
529
+ },
530
+ )
531
+ return DMSRules(
532
+ metadata=metadata,
533
+ properties=new_properties,
534
+ views=new_views,
535
+ containers=new_containers or None,
536
+ enum=reference_rules.enum,
537
+ nodes=reference_rules.nodes,
538
+ )
641
539
 
642
540
  @staticmethod
643
541
  def _expand_properties(rules: DMSRules) -> DMSRules:
@@ -661,8 +559,169 @@ class ToDataProductModel(ToSolutionModel):
661
559
  property_ids.add(prop.view_property)
662
560
  return rules
663
561
 
562
+ def _create_views(
563
+ self, reference: DMSRules
564
+ ) -> tuple[SheetList[DMSView], SheetList[DMSProperty], dict[ViewEntity, ViewEntity]]:
565
+ renaming: dict[ViewEntity, ViewEntity] = {}
566
+ new_views = SheetList[DMSView]()
567
+ read_view_by_new_view: dict[ViewEntity, ViewEntity] = {}
568
+ skipped_views: set[ViewEntity] = set()
569
+ for ref_view in reference.views:
570
+ if (self.skip_cognite_views and ref_view.view.space in COGNITE_SPACES) or (
571
+ self.exclude_views_in_other_spaces and ref_view.view.space != reference.metadata.space
572
+ ):
573
+ skipped_views.add(ref_view.view)
574
+ continue
575
+ new_entity = ViewEntity(
576
+ # MyPy we validate that version is string in the constructor
577
+ space=self.new_model_id.space,
578
+ externalId=ref_view.view.external_id,
579
+ version=self.new_model_id.version, # type: ignore[arg-type]
580
+ )
581
+ if self.properties == "connection":
582
+ # Set suffix to existing view and introduce a new view.
583
+ # This will be used to point to the one view in the Enterprise model,
584
+ # while the new view will to be written to.
585
+ new_entity.suffix = f"{self.view_prefix}{ref_view.view.suffix}"
586
+ new_view = DMSView(
587
+ view=ViewEntity(
588
+ # MyPy we validate that version is string in the constructor
589
+ space=self.new_model_id.space,
590
+ externalId=ref_view.view.external_id,
591
+ version=self.new_model_id.version, # type: ignore[arg-type]
592
+ )
593
+ )
594
+ new_views.append(new_view)
595
+ read_view_by_new_view[new_view.view] = new_entity
596
+ elif self.properties == "repeat":
597
+ # This is a slight misuse of the read_view_by_new_view. For the repeat mode, we only
598
+ # care about the keys in this dictionary. But instead of creating a separate set, we
599
+ # do this.
600
+ read_view_by_new_view[new_entity] = new_entity
601
+
602
+ renaming[ref_view.view] = new_entity
603
+ new_views.append(ref_view.model_copy(deep=True, update={"implements": None, "view": new_entity}))
664
604
 
665
- class ReduceCogniteModel(RulesTransformer[DMSRules, DMSRules]):
605
+ new_properties = SheetList[DMSProperty]()
606
+ for prop in reference.properties:
607
+ if prop.view in skipped_views:
608
+ continue
609
+ new_property = prop.model_copy(deep=True)
610
+ if new_property.value_type in renaming and isinstance(new_property.value_type, ViewEntity):
611
+ new_property.value_type = renaming[new_property.value_type]
612
+ if new_property.view in renaming:
613
+ new_property.view = renaming[new_property.view]
614
+ new_properties.append(new_property)
615
+ return new_views, new_properties, read_view_by_new_view
616
+
617
+ def _create_containers_update_view_filter(
618
+ self, new_views: SheetList[DMSView], reference: DMSRules, read_view_by_new_view: dict[ViewEntity, ViewEntity]
619
+ ) -> tuple[SheetList[DMSContainer], SheetList[DMSProperty]]:
620
+ new_containers = SheetList[DMSContainer]()
621
+ container_properties: SheetList[DMSProperty] = SheetList[DMSProperty]()
622
+ ref_containers_by_ref_view: dict[ViewEntity, set[ContainerEntity]] = defaultdict(set)
623
+ ref_views_by_external_id = {
624
+ view.view.external_id: view for view in reference.views if view.view.space == reference.metadata.space
625
+ }
626
+ if self.filter_type == "container":
627
+ for prop in reference.properties:
628
+ if prop.container:
629
+ ref_containers_by_ref_view[prop.view].add(prop.container)
630
+ read_views = set(read_view_by_new_view.values())
631
+ for view in new_views:
632
+ if view.view in read_view_by_new_view:
633
+ read_view = read_view_by_new_view[view.view]
634
+ container_entity = ContainerEntity(space=self.new_model_id.space, externalId=view.view.external_id)
635
+ prefix = to_camel(view.view.suffix)
636
+ if self.properties == "repeat" and self.dummy_property:
637
+ property_ = DMSProperty(
638
+ view=view.view,
639
+ view_property=f"{prefix}{self.dummy_property}",
640
+ value_type=String(),
641
+ nullable=True,
642
+ immutable=False,
643
+ is_list=False,
644
+ container=container_entity,
645
+ container_property=f"{prefix}{self.dummy_property}",
646
+ )
647
+ new_containers.append(DMSContainer(container=container_entity))
648
+ container_properties.append(property_)
649
+ elif self.properties == "repeat" and self.dummy_property is None:
650
+ # For this case we set the filter. This is used by the DataProductModel.
651
+ # Inherit view filter from original model to ensure the same instances are returned
652
+ # when querying the new view.
653
+ if ref_view := ref_views_by_external_id.get(view.view.external_id):
654
+ self._set_view_filter(view, ref_containers_by_ref_view, ref_view)
655
+ elif self.properties == "connection" and self.direct_property:
656
+ property_ = DMSProperty(
657
+ view=view.view,
658
+ view_property=self.direct_property,
659
+ value_type=read_view,
660
+ nullable=True,
661
+ immutable=False,
662
+ is_list=False,
663
+ container=container_entity,
664
+ container_property=self.direct_property,
665
+ )
666
+ new_containers.append(DMSContainer(container=container_entity))
667
+ container_properties.append(property_)
668
+ else:
669
+ raise NeatValueError(f"Unsupported properties mode: {self.properties}")
670
+
671
+ if self.properties == "connection" and view.view in read_views:
672
+ # Need to ensure that the 'Enterprise' view always returns the same instances
673
+ # as the original view, no matter which properties are removed by the user.
674
+ if ref_view := ref_views_by_external_id.get(view.view.external_id.removeprefix(self.view_prefix)):
675
+ self._set_view_filter(view, ref_containers_by_ref_view, ref_view)
676
+ return new_containers, container_properties
677
+
678
+ def _set_view_filter(
679
+ self, view: DMSView, ref_containers_by_ref_view: dict[ViewEntity, set[ContainerEntity]], ref_view: DMSView
680
+ ) -> None:
681
+ if self.filter_type == "view":
682
+ view.filter_ = HasDataFilter(inner=[ref_view.view])
683
+ elif self.filter_type == "container" and (ref_containers := ref_containers_by_ref_view.get(ref_view.view)):
684
+ # Sorting to ensure deterministic order
685
+ view.filter_ = HasDataFilter(inner=sorted(ref_containers))
686
+
687
+
688
+ class ToDataProductModel(ToSolutionModel):
689
+ """
690
+
691
+ Args:
692
+ new_model_id: DataData model identifier for the new model.
693
+ include: The views to include in the data product data model. Can be either "same-space" or "all".
694
+ filter_type: This is the type of filter to apply to the new views. The filter is used to
695
+ ensure that the new views will return the same instance as the original views. The view filter is the
696
+ simplest filter, but it has limitation in the fusion UI. The container filter is in essence a more
697
+ verbose version of the view filter, and it has better support in the fusion UI. The default is "container".
698
+ """
699
+
700
+ type_: ClassVar[str] = "data_product"
701
+
702
+ def __init__(
703
+ self,
704
+ new_model_id: DataModelIdentifier,
705
+ include: Literal["same-space", "all"] = "same-space",
706
+ filter_type: Literal["container", "view"] = "container",
707
+ skip_cognite_views: bool = True,
708
+ ):
709
+ super().__init__(
710
+ new_model_id,
711
+ properties="repeat",
712
+ dummy_property=None,
713
+ filter_type=filter_type,
714
+ exclude_views_in_other_spaces=include == "same-space",
715
+ skip_cognite_views=skip_cognite_views,
716
+ )
717
+ self.include = include
718
+
719
+ def transform(self, rules: DMSRules) -> DMSRules:
720
+ # Overwrite this to avoid the warning.
721
+ return self._to_solution(rules)
722
+
723
+
724
+ class DropModelViews(VerifiedRulesTransformer[DMSRules, DMSRules]):
666
725
  _ASSET_VIEW = ViewId("cdf_cdm", "CogniteAsset", "v1")
667
726
  _VIEW_BY_COLLECTION: Mapping[Literal["3D", "Annotation", "BaseViews"], frozenset[ViewId]] = {
668
727
  "3D": frozenset(
@@ -701,51 +760,73 @@ class ReduceCogniteModel(RulesTransformer[DMSRules, DMSRules]):
701
760
  ),
702
761
  }
703
762
 
704
- def __init__(self, drop: Collection[Literal["3D", "Annotation", "BaseViews"] | str]):
705
- self.drop_collection = cast(
706
- list[Literal["3D", "Annotation", "BaseViews"]],
707
- [collection for collection in drop if collection in self._VIEW_BY_COLLECTION],
763
+ def __init__(
764
+ self,
765
+ view_external_id: str | SequenceNotStr[str] | None = None,
766
+ group: Literal["3D", "Annotation", "BaseViews"]
767
+ | Collection[Literal["3D", "Annotation", "BaseViews"]]
768
+ | None = None,
769
+ ):
770
+ self.drop_external_ids = (
771
+ {view_external_id} if isinstance(view_external_id, str) else set(view_external_id or [])
772
+ )
773
+ self.drop_collection = (
774
+ [group]
775
+ if isinstance(group, str)
776
+ else cast(
777
+ list[Literal["3D", "Annotation", "BaseViews"]],
778
+ [collection for collection in group or [] if collection in self._VIEW_BY_COLLECTION],
779
+ )
708
780
  )
709
- self.drop_external_ids = {external_id for external_id in drop if external_id not in self._VIEW_BY_COLLECTION}
710
781
 
711
782
  def transform(self, rules: DMSRules) -> DMSRules:
712
- verified = rules
713
- if verified.metadata.as_data_model_id() not in COGNITE_MODELS:
714
- raise NeatValueError(f"Can only reduce Cognite Data Models, not {verified.metadata.as_data_model_id()}")
715
-
716
- exclude_views = {view for collection in self.drop_collection for view in self._VIEW_BY_COLLECTION[collection]}
717
- exclude_views |= {view.view.as_id() for view in verified.views if view.view.suffix in self.drop_external_ids}
718
- new_model = verified.model_copy(deep=True)
783
+ exclude_views: set[ViewEntity] = {
784
+ view.view for view in rules.views if view.view.suffix in self.drop_external_ids
785
+ }
786
+ if rules.metadata.as_data_model_id() in COGNITE_MODELS:
787
+ exclude_views |= {
788
+ ViewEntity.from_id(view_id, "v1")
789
+ for collection in self.drop_collection
790
+ for view_id in self._VIEW_BY_COLLECTION[collection]
791
+ }
792
+ new_model = rules.model_copy(deep=True)
719
793
 
720
794
  properties_by_view = DMSAnalysis(new_model).classes_with_properties(consider_inheritance=True)
721
795
 
722
- new_model.views = SheetList[DMSView](
723
- [view for view in new_model.views if view.view.as_id() not in exclude_views]
724
- )
796
+ new_model.views = SheetList[DMSView]([view for view in new_model.views if view.view not in exclude_views])
725
797
  new_properties = SheetList[DMSProperty]()
726
-
798
+ mapped_containers: set[ContainerEntity] = set()
727
799
  for view in new_model.views:
728
800
  for prop in properties_by_view[view.view]:
729
- if self._is_asset_3D_property(prop):
801
+ if "3D" in self.drop_collection and self._is_asset_3D_property(prop):
730
802
  # We filter out the 3D property of asset
731
803
  continue
804
+ if isinstance(prop.value_type, ViewEntity) and prop.value_type in exclude_views:
805
+ continue
732
806
  new_properties.append(prop)
807
+ if prop.container:
808
+ mapped_containers.add(prop.container)
733
809
 
734
810
  new_model.properties = new_properties
811
+ new_model.containers = (
812
+ SheetList[DMSContainer](
813
+ [container for container in new_model.containers or [] if container.container in mapped_containers]
814
+ )
815
+ or None
816
+ )
735
817
 
736
818
  return new_model
737
819
 
738
- def _is_asset_3D_property(self, prop: DMSProperty) -> bool:
739
- if "3D" not in self.drop_collection:
740
- return False
741
- return prop.view.as_id() == self._ASSET_VIEW and prop.view_property == "object3D"
820
+ @classmethod
821
+ def _is_asset_3D_property(cls, prop: DMSProperty) -> bool:
822
+ return prop.view.as_id() == cls._ASSET_VIEW and prop.view_property == "object3D"
742
823
 
743
824
  @property
744
825
  def description(self) -> str:
745
826
  return f"Removed {len(self.drop_external_ids) + len(self.drop_collection)} views from data model"
746
827
 
747
828
 
748
- class IncludeReferenced(RulesTransformer[DMSRules, DMSRules]):
829
+ class IncludeReferenced(VerifiedRulesTransformer[DMSRules, DMSRules]):
749
830
  def __init__(self, client: NeatClient, include_properties: bool = False) -> None:
750
831
  self._client = client
751
832
  self.include_properties = include_properties
@@ -796,7 +877,7 @@ class IncludeReferenced(RulesTransformer[DMSRules, DMSRules]):
796
877
  return "Included referenced views and containers in the data model."
797
878
 
798
879
 
799
- class AddClassImplements(RulesTransformer[InformationRules, InformationRules]):
880
+ class AddClassImplements(VerifiedRulesTransformer[InformationRules, InformationRules]):
800
881
  def __init__(self, implements: str, suffix: str):
801
882
  self.implements = implements
802
883
  self.suffix = suffix
@@ -814,7 +895,7 @@ class AddClassImplements(RulesTransformer[InformationRules, InformationRules]):
814
895
  return f"Added implements property to classes with suffix {self.suffix}"
815
896
 
816
897
 
817
- class ClassicPrepareCore(RulesTransformer[InformationRules, InformationRules]):
898
+ class ClassicPrepareCore(VerifiedRulesTransformer[InformationRules, InformationRules]):
818
899
  """Update the classic data model with the following:
819
900
 
820
901
  This is a special purpose transformer that is only intended to be used with when reading
@@ -823,11 +904,21 @@ class ClassicPrepareCore(RulesTransformer[InformationRules, InformationRules]):
823
904
  - ClassicTimeseries.isString from boolean to string
824
905
  - Add class ClassicSourceSystem, and update all source properties from string to ClassicSourceSystem.
825
906
  - Rename externalId properties to classicExternalId
826
- - Renames the Relationship.sourceExternaId and Relationship.targetExternalId to startNode and endNode
907
+ - Renames the Relationship.sourceExternalId and Relationship.targetExternalId to startNode and endNode
908
+ - If reference_timeseries is True, the classicExternalId property of the TimeSeries class will change type
909
+ from string to timeseries.
910
+ - If reference_files is True, the classicExternalId property of the File class will change type from string to file.
827
911
  """
828
912
 
829
- def __init__(self, instance_namespace: Namespace) -> None:
913
+ def __init__(
914
+ self,
915
+ instance_namespace: Namespace,
916
+ reference_timeseries: bool = False,
917
+ reference_files: bool = False,
918
+ ) -> None:
830
919
  self.instance_namespace = instance_namespace
920
+ self.reference_timeseries = reference_timeseries
921
+ self.reference_files = reference_files
831
922
 
832
923
  @property
833
924
  def description(self) -> str:
@@ -851,6 +942,10 @@ class ClassicPrepareCore(RulesTransformer[InformationRules, InformationRules]):
851
942
  prop.value_type = ClassEntity(prefix=prefix, suffix="ClassicSourceSystem")
852
943
  elif prop.property_ == "externalId":
853
944
  prop.property_ = "classicExternalId"
945
+ if self.reference_timeseries and prop.class_.suffix == "ClassicTimeSeries":
946
+ prop.value_type = Timeseries()
947
+ elif self.reference_files and prop.class_.suffix == "ClassicFile":
948
+ prop.value_type = File()
854
949
  elif prop.property_ == "sourceExternalId" and prop.class_.suffix == "ClassicRelationship":
855
950
  prop.property_ = "startNode"
856
951
  elif prop.property_ == "targetExternalId" and prop.class_.suffix == "ClassicRelationship":
@@ -882,7 +977,7 @@ class ClassicPrepareCore(RulesTransformer[InformationRules, InformationRules]):
882
977
  return output
883
978
 
884
979
 
885
- class ChangeViewPrefix(RulesTransformer[DMSRules, DMSRules]):
980
+ class ChangeViewPrefix(VerifiedRulesTransformer[DMSRules, DMSRules]):
886
981
  def __init__(self, old: str, new: str) -> None:
887
982
  self.old = old
888
983
  self.new = new
@@ -907,6 +1002,68 @@ class ChangeViewPrefix(RulesTransformer[DMSRules, DMSRules]):
907
1002
  return output
908
1003
 
909
1004
 
1005
+ class MergeDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
1006
+ def __init__(self, extra: DMSRules) -> None:
1007
+ self.extra = extra
1008
+
1009
+ def transform(self, rules: DMSRules) -> DMSRules:
1010
+ output = rules.model_copy(deep=True)
1011
+ existing_views = {view.view for view in output.views}
1012
+ for view in self.extra.views:
1013
+ if view.view not in existing_views:
1014
+ output.views.append(view)
1015
+ existing_properties = {(prop.view, prop.view_property) for prop in output.properties}
1016
+ existing_containers = {container.container for container in output.containers or []}
1017
+ existing_enum_collections = {collection.collection for collection in output.enum or []}
1018
+ new_containers_by_entity = {container.container: container for container in self.extra.containers or []}
1019
+ new_enum_collections_by_entity = {collection.collection: collection for collection in self.extra.enum or []}
1020
+ for prop in self.extra.properties:
1021
+ if (prop.view, prop.view_property) in existing_properties:
1022
+ continue
1023
+ output.properties.append(prop)
1024
+ if prop.container and prop.container not in existing_containers:
1025
+ if output.containers is None:
1026
+ output.containers = SheetList[DMSContainer]()
1027
+ output.containers.append(new_containers_by_entity[prop.container])
1028
+ if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections:
1029
+ if output.enum is None:
1030
+ output.enum = SheetList[DMSEnum]()
1031
+ output.enum.append(new_enum_collections_by_entity[prop.value_type.collection])
1032
+
1033
+ existing_nodes = {node.node for node in output.nodes or []}
1034
+ for node in self.extra.nodes or []:
1035
+ if node.node not in existing_nodes:
1036
+ if output.nodes is None:
1037
+ output.nodes = SheetList[DMSNode]()
1038
+ output.nodes.append(node)
1039
+
1040
+ return output
1041
+
1042
+ @property
1043
+ def description(self) -> str:
1044
+ return f"Merged with {self.extra.metadata.as_data_model_id()}"
1045
+
1046
+
1047
+ class MergeInformationRules(VerifiedRulesTransformer[InformationRules, InformationRules]):
1048
+ def __init__(self, extra: InformationRules) -> None:
1049
+ self.extra = extra
1050
+
1051
+ def transform(self, rules: InformationRules) -> InformationRules:
1052
+ output = rules.model_copy(deep=True)
1053
+ existing_classes = {cls.class_ for cls in output.classes}
1054
+ for cls in self.extra.classes:
1055
+ if cls.class_ not in existing_classes:
1056
+ output.classes.append(cls)
1057
+ existing_properties = {(prop.class_, prop.property_) for prop in output.properties}
1058
+ for prop in self.extra.properties:
1059
+ if (prop.class_, prop.property_) not in existing_properties:
1060
+ output.properties.append(prop)
1061
+ for prefix, namespace in self.extra.prefixes.items():
1062
+ if prefix not in output.prefixes:
1063
+ output.prefixes[prefix] = namespace
1064
+ return output
1065
+
1066
+
910
1067
  class _InformationRulesConverter:
911
1068
  _start_or_end_node: ClassVar[frozenset[str]] = frozenset({"endNode", "end_node", "startNode", "start_node"})
912
1069
 
@@ -914,7 +1071,9 @@ class _InformationRulesConverter:
914
1071
  self.rules = information
915
1072
  self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
916
1073
 
917
- def as_dms_rules(self, ignore_undefined_value_types: bool = False) -> "DMSRules":
1074
+ def as_dms_rules(
1075
+ self, ignore_undefined_value_types: bool = False, reserved_properties: Literal["error", "warning"] = "error"
1076
+ ) -> "DMSRules":
918
1077
  from cognite.neat._rules.models.dms._rules import (
919
1078
  DMSContainer,
920
1079
  DMSProperty,
@@ -957,6 +1116,13 @@ class _InformationRulesConverter:
957
1116
  continue
958
1117
  if prop.class_ in edge_classes and prop.property_ in self._start_or_end_node:
959
1118
  continue
1119
+ if prop.property_ in DMS_RESERVED_PROPERTIES:
1120
+ msg = f"Property {prop.property_} is a reserved property in DMS."
1121
+ if reserved_properties == "error":
1122
+ raise NeatValueError(msg)
1123
+ warnings.warn(NeatValueWarning(f"{msg} Skipping..."), stacklevel=2)
1124
+ continue
1125
+
960
1126
  dms_property = self._as_dms_property(
961
1127
  prop,
962
1128
  default_space,