cognite-neat 0.88.3__py3-none-any.whl → 0.89.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (75) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/constants.py +3 -0
  3. cognite/neat/graph/extractors/_mock_graph_generator.py +2 -1
  4. cognite/neat/issues/_base.py +2 -1
  5. cognite/neat/issues/errors/__init__.py +2 -1
  6. cognite/neat/issues/errors/_general.py +7 -0
  7. cognite/neat/issues/warnings/_models.py +1 -1
  8. cognite/neat/issues/warnings/user_modeling.py +1 -1
  9. cognite/neat/rules/_shared.py +49 -6
  10. cognite/neat/rules/analysis/_base.py +1 -1
  11. cognite/neat/rules/exporters/_base.py +7 -18
  12. cognite/neat/rules/exporters/_rules2dms.py +8 -18
  13. cognite/neat/rules/exporters/_rules2excel.py +5 -12
  14. cognite/neat/rules/exporters/_rules2ontology.py +9 -19
  15. cognite/neat/rules/exporters/_rules2yaml.py +3 -6
  16. cognite/neat/rules/importers/_base.py +7 -52
  17. cognite/neat/rules/importers/_dms2rules.py +171 -115
  18. cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +26 -18
  19. cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +14 -30
  20. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +7 -3
  21. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +3 -3
  22. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +18 -11
  23. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2rules.py +9 -18
  24. cognite/neat/rules/importers/_rdf/_inference2rules.py +10 -33
  25. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +9 -20
  26. cognite/neat/rules/importers/_rdf/_shared.py +1 -1
  27. cognite/neat/rules/importers/_spreadsheet2rules.py +22 -86
  28. cognite/neat/rules/importers/_yaml2rules.py +14 -41
  29. cognite/neat/rules/models/__init__.py +21 -5
  30. cognite/neat/rules/models/_base_input.py +162 -0
  31. cognite/neat/rules/models/{_base.py → _base_rules.py} +1 -12
  32. cognite/neat/rules/models/asset/__init__.py +5 -2
  33. cognite/neat/rules/models/asset/_rules.py +2 -20
  34. cognite/neat/rules/models/asset/_rules_input.py +40 -115
  35. cognite/neat/rules/models/asset/_validation.py +1 -1
  36. cognite/neat/rules/models/data_types.py +150 -44
  37. cognite/neat/rules/models/dms/__init__.py +19 -7
  38. cognite/neat/rules/models/dms/_exporter.py +72 -26
  39. cognite/neat/rules/models/dms/_rules.py +42 -155
  40. cognite/neat/rules/models/dms/_rules_input.py +186 -254
  41. cognite/neat/rules/models/dms/_serializer.py +44 -3
  42. cognite/neat/rules/models/dms/_validation.py +3 -4
  43. cognite/neat/rules/models/domain.py +52 -1
  44. cognite/neat/rules/models/entities/__init__.py +63 -0
  45. cognite/neat/rules/models/entities/_constants.py +73 -0
  46. cognite/neat/rules/models/entities/_loaders.py +76 -0
  47. cognite/neat/rules/models/entities/_multi_value.py +67 -0
  48. cognite/neat/rules/models/{entities.py → entities/_single_value.py} +74 -232
  49. cognite/neat/rules/models/entities/_types.py +86 -0
  50. cognite/neat/rules/models/{wrapped_entities.py → entities/_wrapped.py} +1 -1
  51. cognite/neat/rules/models/information/__init__.py +10 -2
  52. cognite/neat/rules/models/information/_rules.py +3 -14
  53. cognite/neat/rules/models/information/_rules_input.py +57 -204
  54. cognite/neat/rules/models/information/_validation.py +1 -1
  55. cognite/neat/rules/transformers/__init__.py +21 -0
  56. cognite/neat/rules/transformers/_base.py +69 -3
  57. cognite/neat/rules/{models/information/_converter.py → transformers/_converters.py} +216 -20
  58. cognite/neat/rules/transformers/_map_onto.py +97 -0
  59. cognite/neat/rules/transformers/_pipelines.py +61 -0
  60. cognite/neat/rules/transformers/_verification.py +136 -0
  61. cognite/neat/store/_provenance.py +10 -1
  62. cognite/neat/utils/cdf/data_classes.py +20 -0
  63. cognite/neat/utils/regex_patterns.py +6 -0
  64. cognite/neat/workflows/steps/lib/current/rules_exporter.py +106 -37
  65. cognite/neat/workflows/steps/lib/current/rules_importer.py +24 -22
  66. {cognite_neat-0.88.3.dist-info → cognite_neat-0.89.0.dist-info}/METADATA +1 -1
  67. {cognite_neat-0.88.3.dist-info → cognite_neat-0.89.0.dist-info}/RECORD +71 -66
  68. cognite/neat/rules/models/_constants.py +0 -2
  69. cognite/neat/rules/models/_types/__init__.py +0 -19
  70. cognite/neat/rules/models/asset/_converter.py +0 -4
  71. cognite/neat/rules/models/dms/_converter.py +0 -143
  72. /cognite/neat/rules/models/{_types/_field.py → _types.py} +0 -0
  73. {cognite_neat-0.88.3.dist-info → cognite_neat-0.89.0.dist-info}/LICENSE +0 -0
  74. {cognite_neat-0.88.3.dist-info → cognite_neat-0.89.0.dist-info}/WHEEL +0 -0
  75. {cognite_neat-0.88.3.dist-info → cognite_neat-0.89.0.dist-info}/entry_points.txt +0 -0
@@ -1,14 +1,15 @@
1
1
  from collections import Counter
2
- from collections.abc import Sequence
2
+ from collections.abc import Collection, Sequence
3
3
  from datetime import datetime
4
4
  from pathlib import Path
5
- from typing import Any, Literal, cast, overload
5
+ from typing import Literal, cast
6
6
 
7
7
  from cognite.client import CogniteClient
8
8
  from cognite.client import data_modeling as dm
9
9
  from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
10
10
  from cognite.client.data_classes.data_modeling.containers import BTreeIndex, InvertedIndex
11
- from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
11
+ from cognite.client.data_classes.data_modeling.data_types import Enum as DMSEnum
12
+ from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType, PropertyTypeWithUnit
12
13
  from cognite.client.data_classes.data_modeling.views import (
13
14
  MultiEdgeConnectionApply,
14
15
  MultiReverseDirectRelationApply,
@@ -24,34 +25,37 @@ from cognite.neat.issues.warnings import (
24
25
  PropertyNotFoundWarning,
25
26
  PropertyTypeNotSupportedWarning,
26
27
  ResourceNotFoundWarning,
28
+ ResourcesDuplicatedWarning,
27
29
  )
28
- from cognite.neat.rules.importers._base import BaseImporter, VerifiedRules, _handle_issues
30
+ from cognite.neat.rules._shared import ReadRules
31
+ from cognite.neat.rules.importers._base import BaseImporter, _handle_issues
29
32
  from cognite.neat.rules.models import (
30
33
  DataModelType,
31
- DMSRules,
34
+ DMSInputRules,
32
35
  DMSSchema,
33
- ExtensionCategory,
34
- RoleTypes,
35
36
  SchemaCompleteness,
36
- SheetList,
37
37
  )
38
- from cognite.neat.rules.models.data_types import DataType
38
+ from cognite.neat.rules.models.data_types import DataType, Enum
39
39
  from cognite.neat.rules.models.dms import (
40
- DMSContainer,
41
- DMSMetadata,
42
- DMSProperty,
43
- DMSView,
40
+ DMSInputContainer,
41
+ DMSInputEnum,
42
+ DMSInputMetadata,
43
+ DMSInputNode,
44
+ DMSInputProperty,
45
+ DMSInputView,
44
46
  )
45
47
  from cognite.neat.rules.models.entities import (
46
48
  ClassEntity,
47
49
  ContainerEntity,
50
+ DMSNodeEntity,
48
51
  DMSUnknownEntity,
52
+ EdgeEntity,
53
+ ReverseConnectionEntity,
49
54
  ViewEntity,
50
- ViewPropertyEntity,
51
55
  )
52
56
 
53
57
 
54
- class DMSImporter(BaseImporter):
58
+ class DMSImporter(BaseImporter[DMSInputRules]):
55
59
  """Imports a Data Model from Cognite Data Fusion.
56
60
 
57
61
  Args:
@@ -66,8 +70,8 @@ class DMSImporter(BaseImporter):
66
70
  self,
67
71
  schema: DMSSchema,
68
72
  read_issues: Sequence[NeatIssue] | None = None,
69
- metadata: DMSMetadata | None = None,
70
- ref_metadata: DMSMetadata | None = None,
73
+ metadata: DMSInputMetadata | None = None,
74
+ ref_metadata: DMSInputMetadata | None = None,
71
75
  ):
72
76
  # Calling this root schema to distinguish it from
73
77
  # * User Schema
@@ -77,10 +81,10 @@ class DMSImporter(BaseImporter):
77
81
  self.ref_metadata = ref_metadata
78
82
  self.issue_list = IssueList(read_issues)
79
83
  self._all_containers_by_id = schema.containers.copy()
80
- self._all_view_ids = set(self.root_schema.views.keys())
81
- if self.root_schema.reference:
82
- self._all_containers_by_id.update(self.root_schema.reference.containers)
83
- self._all_view_ids.update(self.root_schema.reference.views.keys())
84
+ self._all_views_by_id = schema.views.copy()
85
+ if schema.reference:
86
+ self._all_containers_by_id.update(schema.reference.containers.items())
87
+ self._all_views_by_id.update(schema.reference.views.items())
84
88
 
85
89
  @classmethod
86
90
  def from_data_model_id(
@@ -162,8 +166,8 @@ class DMSImporter(BaseImporter):
162
166
  cls,
163
167
  model: dm.DataModel[dm.View] | dm.DataModelApply,
164
168
  has_reference: bool = False,
165
- ) -> DMSMetadata:
166
- description, creator = DMSMetadata._get_description_and_creator(model.description)
169
+ ) -> DMSInputMetadata:
170
+ description, creator = DMSInputMetadata._get_description_and_creator(model.description)
167
171
 
168
172
  if isinstance(model, dm.DataModel):
169
173
  created = ms_to_datetime(model.created_time)
@@ -172,17 +176,17 @@ class DMSImporter(BaseImporter):
172
176
  now = datetime.now().replace(microsecond=0)
173
177
  created = now
174
178
  updated = now
175
- return DMSMetadata(
176
- schema_=SchemaCompleteness.complete,
177
- data_model_type=DataModelType.solution if has_reference else DataModelType.enterprise,
178
- extension=ExtensionCategory.addition,
179
+ return DMSInputMetadata(
180
+ schema_="complete",
181
+ data_model_type="solution" if has_reference else "enterprise",
182
+ extension="addition",
179
183
  space=model.space,
180
184
  external_id=model.external_id,
181
185
  name=model.name or model.external_id,
182
186
  version=model.version or "0.1.0",
183
187
  updated=updated,
184
188
  created=created,
185
- creator=creator,
189
+ creator=",".join(creator),
186
190
  description=description,
187
191
  )
188
192
 
@@ -203,71 +207,52 @@ class DMSImporter(BaseImporter):
203
207
  schema = DMSSchema.from_zip(zip_file)
204
208
  return cls(schema, issue_list)
205
209
 
206
- @overload
207
- def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
208
-
209
- @overload
210
- def to_rules(
211
- self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
212
- ) -> tuple[VerifiedRules | None, IssueList]: ...
213
-
214
- def to_rules(
215
- self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
216
- ) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
210
+ def to_rules(self) -> ReadRules[DMSInputRules]:
217
211
  if self.issue_list.has_errors:
218
212
  # In case there were errors during the import, the to_rules method will return None
219
- return self._return_or_raise(self.issue_list, errors)
213
+ return ReadRules(None, self.issue_list, {})
220
214
 
221
215
  if not self.root_schema.data_model:
222
216
  self.issue_list.append(ResourceMissingIdentifierError("data model", type(self.root_schema).__name__))
223
- return self._return_or_raise(self.issue_list, errors)
217
+ return ReadRules(None, self.issue_list, {})
218
+
224
219
  model = self.root_schema.data_model
225
- with _handle_issues(
226
- self.issue_list,
227
- ) as future:
228
- schema_completeness = SchemaCompleteness.complete
229
- data_model_type = DataModelType.enterprise
230
- reference: DMSRules | None = None
231
- if (ref_schema := self.root_schema.reference) and (ref_model := ref_schema.data_model):
232
- # Reference should always be an enterprise model.
233
- reference = DMSRules(
234
- **self._create_rule_components(
235
- ref_model,
236
- ref_schema,
237
- self.ref_metadata
238
- or self._create_default_metadata(list(ref_schema.views.values()), is_ref=True),
239
- DataModelType.enterprise,
240
- )
241
- )
242
- data_model_type = DataModelType.solution
243
-
244
- user_rules = DMSRules(
245
- **self._create_rule_components(
246
- model,
247
- self.root_schema,
248
- self.metadata,
249
- data_model_type,
250
- schema_completeness,
251
- has_reference=reference is not None,
252
- ),
253
- reference=reference,
254
- )
255
220
 
256
- if future.result == "failure" or self.issue_list.has_errors:
257
- return self._return_or_raise(self.issue_list, errors)
221
+ schema_completeness = SchemaCompleteness.complete
222
+ data_model_type = DataModelType.enterprise
223
+ reference: DMSInputRules | None = None
224
+ if (ref_schema := self.root_schema.reference) and (ref_model := ref_schema.data_model):
225
+ # Reference should always be an enterprise model.
226
+ reference = self._create_rule_components(
227
+ ref_model,
228
+ ref_schema,
229
+ self.ref_metadata or self._create_default_metadata(list(ref_schema.views.values()), is_ref=True),
230
+ DataModelType.enterprise,
231
+ )
232
+ data_model_type = DataModelType.solution
233
+
234
+ user_rules = self._create_rule_components(
235
+ model,
236
+ self.root_schema,
237
+ self.metadata,
238
+ data_model_type,
239
+ schema_completeness,
240
+ has_reference=reference is not None,
241
+ )
242
+ user_rules.reference = reference
258
243
 
259
- return self._to_output(user_rules, self.issue_list, errors, role)
244
+ return ReadRules(user_rules, self.issue_list, {})
260
245
 
261
246
  def _create_rule_components(
262
247
  self,
263
248
  data_model: dm.DataModelApply,
264
249
  schema: DMSSchema,
265
- metadata: DMSMetadata | None = None,
250
+ metadata: DMSInputMetadata | None = None,
266
251
  data_model_type: DataModelType | None = None,
267
252
  schema_completeness: SchemaCompleteness | None = None,
268
253
  has_reference: bool = False,
269
- ) -> dict[str, Any]:
270
- properties = SheetList[DMSProperty]()
254
+ ) -> DMSInputRules:
255
+ properties: list[DMSInputProperty] = []
271
256
  for view_id, view in schema.views.items():
272
257
  view_entity = ViewEntity.from_id(view_id)
273
258
  class_entity = view_entity.as_class()
@@ -280,44 +265,47 @@ class DMSImporter(BaseImporter):
280
265
  view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
281
266
  }
282
267
 
283
- metadata = metadata or DMSMetadata.from_data_model(data_model, has_reference)
268
+ metadata = metadata or DMSInputMetadata.from_data_model(data_model, has_reference)
284
269
  if data_model_type is not None:
285
- metadata.data_model_type = data_model_type
270
+ metadata.data_model_type = str(data_model_type) # type: ignore[assignment]
286
271
  if schema_completeness is not None:
287
- metadata.schema_ = schema_completeness
288
- return dict(
272
+ metadata.schema_ = str(schema_completeness) # type: ignore[assignment]
273
+
274
+ enum = self._create_enum_collections(schema.containers.values())
275
+
276
+ return DMSInputRules(
289
277
  metadata=metadata,
290
278
  properties=properties,
291
- containers=SheetList[DMSContainer](
292
- data=[DMSContainer.from_container(container) for container in schema.containers.values()]
293
- ),
294
- views=SheetList[DMSView](
295
- data=[
296
- DMSView.from_view(view, in_model=view_id in data_model_view_ids)
297
- for view_id, view in schema.views.items()
298
- ]
299
- ),
279
+ containers=[DMSInputContainer.from_container(container) for container in schema.containers.values()],
280
+ views=[
281
+ DMSInputView.from_view(view, in_model=view_id in data_model_view_ids)
282
+ for view_id, view in schema.views.items()
283
+ ],
284
+ nodes=[DMSInputNode.from_node_type(node_type) for node_type in schema.node_types.values()],
285
+ enum=enum,
300
286
  )
301
287
 
302
288
  @classmethod
303
- def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply], is_ref: bool = False) -> DMSMetadata:
289
+ def _create_default_metadata(
290
+ cls, views: Sequence[dm.View | dm.ViewApply], is_ref: bool = False
291
+ ) -> DMSInputMetadata:
304
292
  now = datetime.now().replace(microsecond=0)
305
293
  space = Counter(view.space for view in views).most_common(1)[0][0]
306
- return DMSMetadata(
307
- schema_=SchemaCompleteness.complete,
308
- extension=ExtensionCategory.addition,
309
- data_model_type=DataModelType.enterprise if is_ref else DataModelType.solution,
294
+ return DMSInputMetadata(
295
+ schema_="complete",
296
+ extension="addition",
297
+ data_model_type="enterprise" if is_ref else "solution",
310
298
  space=space,
311
299
  external_id="Unknown",
312
300
  version="0.1.0",
313
- creator=["Unknown"],
301
+ creator="Unknown",
314
302
  created=now,
315
303
  updated=now,
316
304
  )
317
305
 
318
306
  def _create_dms_property(
319
307
  self, prop_id: str, prop: ViewPropertyApply, view_entity: ViewEntity, class_entity: ClassEntity
320
- ) -> DMSProperty | None:
308
+ ) -> DMSInputProperty | None:
321
309
  if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._all_containers_by_id:
322
310
  self.issue_list.append(
323
311
  ResourceNotFoundWarning[dm.ContainerId, dm.PropertyId](
@@ -353,20 +341,22 @@ class DMSImporter(BaseImporter):
353
341
  if value_type is None:
354
342
  return None
355
343
 
356
- return DMSProperty(
357
- class_=class_entity,
344
+ return DMSInputProperty(
345
+ class_=str(class_entity),
358
346
  property_=prop_id,
359
347
  description=prop.description,
360
348
  name=prop.name,
361
- connection=self._get_relation_type(prop),
362
- value_type=value_type,
349
+ connection=self._get_connection_type(prop_id, prop, view_entity.as_id()),
350
+ value_type=str(value_type),
363
351
  is_list=self._get_is_list(prop),
364
352
  nullable=self._get_nullable(prop),
365
353
  immutable=self._get_immutable(prop),
366
354
  default=self._get_default(prop),
367
- container=ContainerEntity.from_id(prop.container) if isinstance(prop, dm.MappedPropertyApply) else None,
355
+ container=str(ContainerEntity.from_id(prop.container))
356
+ if isinstance(prop, dm.MappedPropertyApply)
357
+ else None,
368
358
  container_property=prop.container_property_identifier if isinstance(prop, dm.MappedPropertyApply) else None,
369
- view=view_entity,
359
+ view=str(view_entity),
370
360
  view_property=prop_id,
371
361
  index=self._get_index(prop, prop_id),
372
362
  constraint=self._get_constraint(prop, prop_id),
@@ -376,13 +366,22 @@ class DMSImporter(BaseImporter):
376
366
  """This method assumes you have already checked that the container with property exists."""
377
367
  return self._all_containers_by_id[prop.container].properties[prop.container_property_identifier]
378
368
 
379
- def _get_relation_type(self, prop: ViewPropertyApply) -> Literal["edge", "reverse", "direct"] | None:
369
+ def _get_connection_type(
370
+ self, prop_id: str, prop: ViewPropertyApply, view_id: dm.ViewId
371
+ ) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
380
372
  if isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "outwards":
381
- return "edge"
373
+ properties = ViewEntity.from_id(prop.edge_source) if prop.edge_source is not None else None
374
+ return EdgeEntity(properties=properties, type=DMSNodeEntity.from_reference(prop.type), direction="outwards")
382
375
  elif isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "inwards":
383
- return "reverse"
376
+ if reverse_prop := self._find_reverse_edge(prop_id, prop, view_id):
377
+ return ReverseConnectionEntity(property=reverse_prop)
378
+ else:
379
+ properties = ViewEntity.from_id(prop.source) if prop.edge_source is not None else None
380
+ return EdgeEntity(
381
+ properties=properties, type=DMSNodeEntity.from_reference(prop.type), direction="inwards"
382
+ )
384
383
  elif isinstance(prop, SingleReverseDirectRelationApply | MultiReverseDirectRelationApply):
385
- return "reverse"
384
+ return ReverseConnectionEntity(property=prop.through.property)
386
385
  elif isinstance(prop, dm.MappedPropertyApply) and isinstance(
387
386
  self._container_prop_unsafe(prop).type, dm.DirectRelation
388
387
  ):
@@ -392,21 +391,26 @@ class DMSImporter(BaseImporter):
392
391
 
393
392
  def _get_value_type(
394
393
  self, prop: ViewPropertyApply, view_entity: ViewEntity, prop_id
395
- ) -> DataType | ViewEntity | ViewPropertyEntity | DMSUnknownEntity | None:
396
- if isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "outwards":
397
- return ViewEntity.from_id(prop.source)
398
- elif isinstance(prop, SingleReverseDirectRelationApply | MultiReverseDirectRelationApply):
399
- return ViewPropertyEntity.from_id(prop.through)
400
- elif isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "inwards":
394
+ ) -> DataType | ViewEntity | DMSUnknownEntity | None:
395
+ if isinstance(
396
+ prop,
397
+ SingleEdgeConnectionApply
398
+ | MultiEdgeConnectionApply
399
+ | SingleReverseDirectRelationApply
400
+ | MultiReverseDirectRelationApply,
401
+ ):
401
402
  return ViewEntity.from_id(prop.source)
402
403
  elif isinstance(prop, dm.MappedPropertyApply):
403
404
  container_prop = self._container_prop_unsafe(cast(dm.MappedPropertyApply, prop))
404
405
  if isinstance(container_prop.type, dm.DirectRelation):
405
- if prop.source is None or prop.source not in self._all_view_ids:
406
- # The warning is issued when the DMS Rules are created.
406
+ if prop.source is None or prop.source not in self._all_views_by_id:
407
407
  return DMSUnknownEntity()
408
408
  else:
409
409
  return ViewEntity.from_id(prop.source)
410
+ elif isinstance(container_prop.type, PropertyTypeWithUnit) and container_prop.type.unit:
411
+ return DataType.load(f"{container_prop.type._type}(unit={container_prop.type.unit.external_id})")
412
+ elif isinstance(container_prop.type, DMSEnum):
413
+ return Enum(collection=ClassEntity(suffix=prop_id), unknownValue=container_prop.type.unknown_value)
410
414
  else:
411
415
  return DataType.load(container_prop.type._type)
412
416
  else:
@@ -476,3 +480,55 @@ class DMSImporter(BaseImporter):
476
480
  )
477
481
  )
478
482
  return unique_constraints or None
483
+
484
+ def _find_reverse_edge(
485
+ self, prop_id: str, prop: SingleEdgeConnectionApply | MultiEdgeConnectionApply, view_id: dm.ViewId
486
+ ) -> str | None:
487
+ if prop.source not in self._all_views_by_id:
488
+ return None
489
+ view = self._all_views_by_id[prop.source]
490
+ candidates = []
491
+ for prop_name, reverse_prop in (view.properties or {}).items():
492
+ if isinstance(reverse_prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply):
493
+ if (
494
+ reverse_prop.type == prop.type
495
+ and reverse_prop.source == view_id
496
+ and reverse_prop.direction != prop.direction
497
+ ):
498
+ candidates.append(prop_name)
499
+ if len(candidates) == 0:
500
+ self.issue_list.append(
501
+ PropertyNotFoundWarning(
502
+ prop.source,
503
+ "view property",
504
+ f"reverse edge of {prop_id}",
505
+ dm.PropertyId(view_id, prop_id),
506
+ "view property",
507
+ )
508
+ )
509
+ return None
510
+ if len(candidates) > 1:
511
+ self.issue_list.append(
512
+ ResourcesDuplicatedWarning(
513
+ frozenset(dm.PropertyId(view.as_id(), candidate) for candidate in candidates),
514
+ "view property",
515
+ default_action="Multiple reverse edges found for "
516
+ f"{dm.PropertyId(view_id, prop_id)!r}. Will use {candidates[0]}",
517
+ )
518
+ )
519
+
520
+ return candidates[0]
521
+
522
+ @staticmethod
523
+ def _create_enum_collections(containers: Collection[dm.ContainerApply]) -> list[DMSInputEnum] | None:
524
+ enum_collections: list[DMSInputEnum] = []
525
+ for container in containers:
526
+ for prop_id, prop in container.properties.items():
527
+ if isinstance(prop.type, DMSEnum):
528
+ for identifier, value in prop.type.values.items():
529
+ enum_collections.append(
530
+ DMSInputEnum(
531
+ collection=prop_id, value=identifier, name=value.name, description=value.description
532
+ )
533
+ )
534
+ return enum_collections
@@ -1,7 +1,7 @@
1
1
  from collections import Counter
2
2
  from collections.abc import Callable, Sequence
3
3
 
4
- from cognite.neat.issues import IssueList, NeatIssue
4
+ from cognite.neat.issues import IssueList
5
5
  from cognite.neat.issues.errors import (
6
6
  PropertyTypeNotSupportedError,
7
7
  ResourceMissingIdentifierError,
@@ -27,17 +27,20 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
27
27
  )
28
28
  from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
29
29
  from cognite.neat.rules.models.entities import ClassEntity
30
- from cognite.neat.rules.models.information import InformationClass, InformationProperty
30
+ from cognite.neat.rules.models.information import (
31
+ InformationInputClass,
32
+ InformationInputProperty,
33
+ )
31
34
 
32
35
 
33
36
  class _DTDLConverter:
34
- def __init__(self, issues: list[NeatIssue] | None = None) -> None:
37
+ def __init__(self, issues: IssueList | None = None) -> None:
35
38
  self.issues = IssueList(issues or [])
36
- self.properties: list[InformationProperty] = []
37
- self.classes: list[InformationClass] = []
39
+ self.properties: list[InformationInputProperty] = []
40
+ self.classes: list[InformationInputClass] = []
38
41
  self._item_by_id: dict[DTMI, DTDLBase] = {}
39
42
 
40
- self._method_by_type: dict[type[DTDLBase], Callable[[DTDLBase, str | None], None]] = {
43
+ self._method_by_type = {
41
44
  Interface: self.convert_interface, # type: ignore[dict-item]
42
45
  Property: self.convert_property, # type: ignore[dict-item]
43
46
  PropertyV2: self.convert_property, # type: ignore[dict-item]
@@ -53,7 +56,11 @@ class _DTDLConverter:
53
56
  def get_most_common_prefix(self) -> str:
54
57
  if not self.classes:
55
58
  raise ValueError("No classes found")
56
- counted = Counter(cls_.class_.prefix for cls_ in self.classes if isinstance(cls_.class_.prefix, str))
59
+ counted = Counter(
60
+ class_.prefix
61
+ for class_ in (cls_.class_ for cls_ in self.classes)
62
+ if isinstance(class_, ClassEntity) and isinstance(class_.prefix, str)
63
+ )
57
64
  if not counted:
58
65
  raise ValueError("No prefixes found")
59
66
  return counted.most_common(1)[0][0]
@@ -75,7 +82,8 @@ class _DTDLConverter:
75
82
  self.convert_item(item)
76
83
 
77
84
  def convert_item(self, item: DTDLBase, parent: str | None = None) -> None:
78
- convert_method = self._method_by_type.get(type(item))
85
+ # Bug in mypy https://github.com/python/mypy/issues/17478
86
+ convert_method: Callable[[DTDLBase, str | None], None] | None = self._method_by_type.get(type(item)) # type: ignore[assignment]
79
87
  if convert_method is not None:
80
88
  convert_method(item, parent)
81
89
  else:
@@ -87,7 +95,7 @@ class _DTDLConverter:
87
95
  )
88
96
 
89
97
  def convert_interface(self, item: Interface, _: str | None) -> None:
90
- class_ = InformationClass(
98
+ class_ = InformationInputClass(
91
99
  class_=item.id_.as_class_id(),
92
100
  name=item.display_name,
93
101
  description=item.description,
@@ -107,9 +115,9 @@ class _DTDLConverter:
107
115
  )
108
116
  elif isinstance(sub_item_or_id, DTMI):
109
117
  sub_item = self._item_by_id[sub_item_or_id]
110
- self.convert_item(sub_item, class_.class_.versioned_id)
118
+ self.convert_item(sub_item, class_.class_str)
111
119
  else:
112
- self.convert_item(sub_item_or_id, class_.class_.versioned_id)
120
+ self.convert_item(sub_item_or_id, class_.class_str)
113
121
  # interface.schema objects are handled in the convert method
114
122
 
115
123
  def convert_property(
@@ -122,7 +130,7 @@ class _DTDLConverter:
122
130
  if value_type is None:
123
131
  return None
124
132
 
125
- prop = InformationProperty(
133
+ prop = InformationInputProperty(
126
134
  class_=ClassEntity.load(parent),
127
135
  property_=item.name,
128
136
  name=item.display_name,
@@ -168,7 +176,7 @@ class _DTDLConverter:
168
176
  value_type = self.schema_to_value_type(item.request.schema_, item)
169
177
  if value_type is None:
170
178
  return
171
- prop = InformationProperty(
179
+ prop = InformationInputProperty(
172
180
  class_=ClassEntity.load(parent),
173
181
  property_=item.name,
174
182
  name=item.display_name,
@@ -188,7 +196,7 @@ class _DTDLConverter:
188
196
  value_type = self.schema_to_value_type(item.schema_, item)
189
197
  if value_type is None:
190
198
  return
191
- prop = InformationProperty(
199
+ prop = InformationInputProperty(
192
200
  class_=ClassEntity.load(parent),
193
201
  property_=item.name,
194
202
  name=item.display_name,
@@ -218,7 +226,7 @@ class _DTDLConverter:
218
226
  )
219
227
  value_type = Json()
220
228
 
221
- prop = InformationProperty(
229
+ prop = InformationInputProperty(
222
230
  class_=ClassEntity.load(parent),
223
231
  property_=item.name,
224
232
  name=item.display_name,
@@ -243,7 +251,7 @@ class _DTDLConverter:
243
251
  )
244
252
  return None
245
253
 
246
- class_ = InformationClass(
254
+ class_ = InformationInputClass(
247
255
  class_=item.id_.as_class_id(),
248
256
  name=item.display_name,
249
257
  description=item.description,
@@ -255,7 +263,7 @@ class _DTDLConverter:
255
263
  value_type = self.schema_to_value_type(field_.schema_, item)
256
264
  if value_type is None:
257
265
  continue
258
- prop = InformationProperty(
266
+ prop = InformationInputProperty(
259
267
  class_=class_.class_,
260
268
  name=field_.name,
261
269
  description=field_.description,
@@ -297,7 +305,7 @@ class _DTDLConverter:
297
305
  else:
298
306
  if isinstance(input_type, Object):
299
307
  self.convert_object(input_type, None)
300
- return ClassEntity.load(input_type.id_.as_class_id())
308
+ return input_type.id_.as_class_id()
301
309
  else:
302
310
  self.issues.append(
303
311
  PropertyTypeNotSupportedWarning(
@@ -2,7 +2,6 @@ import json
2
2
  import zipfile
3
3
  from collections.abc import Iterable, Sequence
4
4
  from pathlib import Path
5
- from typing import Literal, overload
6
5
 
7
6
  from pydantic import ValidationError
8
7
 
@@ -14,16 +13,16 @@ from cognite.neat.issues.warnings import (
14
13
  FileTypeUnexpectedWarning,
15
14
  NeatValueWarning,
16
15
  )
17
- from cognite.neat.rules._shared import VerifiedRules
18
- from cognite.neat.rules.importers._base import BaseImporter, _handle_issues
16
+ from cognite.neat.rules._shared import ReadRules
17
+ from cognite.neat.rules.importers._base import BaseImporter
19
18
  from cognite.neat.rules.importers._dtdl2rules.dtdl_converter import _DTDLConverter
20
19
  from cognite.neat.rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
21
- from cognite.neat.rules.models import InformationRules, RoleTypes, SchemaCompleteness, SheetList
22
- from cognite.neat.rules.models.information import InformationClass, InformationProperty
20
+ from cognite.neat.rules.models import InformationInputRules, SchemaCompleteness
21
+ from cognite.neat.rules.models.information import InformationInputMetadata
23
22
  from cognite.neat.utils.text import humanize_collection, to_pascal
24
23
 
25
24
 
26
- class DTDLImporter(BaseImporter):
25
+ class DTDLImporter(BaseImporter[InformationInputRules]):
27
26
  """Importer from Azure Digital Twin - DTDL (Digital Twin Definition Language).
28
27
 
29
28
  This importer supports DTDL v2.0 and v3.0.
@@ -47,7 +46,7 @@ class DTDLImporter(BaseImporter):
47
46
  ) -> None:
48
47
  self._items = items
49
48
  self.title = title
50
- self._read_issues = read_issues
49
+ self._read_issues = IssueList(read_issues)
51
50
  self._schema_completeness = schema
52
51
 
53
52
  @classmethod
@@ -125,17 +124,7 @@ class DTDLImporter(BaseImporter):
125
124
  items.append(item)
126
125
  return cls(items, zip_file.stem, read_issues=issues)
127
126
 
128
- @overload
129
- def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
130
-
131
- @overload
132
- def to_rules(
133
- self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
134
- ) -> tuple[VerifiedRules | None, IssueList]: ...
135
-
136
- def to_rules(
137
- self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
138
- ) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
127
+ def to_rules(self) -> ReadRules[InformationInputRules]:
139
128
  converter = _DTDLConverter(self._read_issues)
140
129
 
141
130
  converter.convert(self._items)
@@ -152,16 +141,11 @@ class DTDLImporter(BaseImporter):
152
141
  ...
153
142
  else:
154
143
  metadata["prefix"] = most_common_prefix
155
- with _handle_issues(converter.issues) as future:
156
- rules = InformationRules(
157
- metadata=metadata,
158
- properties=SheetList[InformationProperty](data=converter.properties),
159
- classes=SheetList[InformationClass](data=converter.classes),
160
- )
161
- if future.result == "failure":
162
- if errors == "continue":
163
- return None, converter.issues
164
- else:
165
- raise converter.issues.as_errors()
166
144
 
167
- return self._to_output(rules, converter.issues, errors, role)
145
+ rules = InformationInputRules(
146
+ metadata=InformationInputMetadata.load(metadata),
147
+ properties=converter.properties,
148
+ classes=converter.classes,
149
+ )
150
+
151
+ return ReadRules(rules, converter.issues, {})