cognite-neat 0.75.8__py3-none-any.whl → 0.76.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (99) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/configuration.py +4 -9
  3. cognite/neat/app/api/routers/configuration.py +2 -1
  4. cognite/neat/app/api/routers/crud.py +5 -5
  5. cognite/neat/app/api/routers/data_exploration.py +3 -1
  6. cognite/neat/app/api/routers/rules.py +3 -3
  7. cognite/neat/app/api/routers/workflows.py +3 -3
  8. cognite/neat/app/ui/neat-app/build/asset-manifest.json +3 -3
  9. cognite/neat/app/ui/neat-app/build/index.html +1 -1
  10. cognite/neat/app/ui/neat-app/build/static/js/{main.4345d42f.js → main.ec7f72e2.js} +3 -3
  11. cognite/neat/app/ui/neat-app/build/static/js/{main.4345d42f.js.map → main.ec7f72e2.js.map} +1 -1
  12. cognite/neat/config.py +147 -12
  13. cognite/neat/constants.py +1 -0
  14. cognite/neat/graph/exceptions.py +1 -2
  15. cognite/neat/legacy/graph/exceptions.py +1 -2
  16. cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +1 -2
  17. cognite/neat/legacy/graph/loaders/_asset_loader.py +8 -13
  18. cognite/neat/legacy/graph/loaders/_base.py +2 -4
  19. cognite/neat/legacy/graph/loaders/_exceptions.py +1 -3
  20. cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +4 -8
  21. cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +2 -4
  22. cognite/neat/legacy/graph/loaders/rdf_to_dms.py +2 -4
  23. cognite/neat/legacy/graph/loaders/validator.py +1 -1
  24. cognite/neat/legacy/graph/transformations/transformer.py +1 -2
  25. cognite/neat/legacy/rules/exporters/_rules2dms.py +1 -2
  26. cognite/neat/legacy/rules/exporters/_validation.py +4 -8
  27. cognite/neat/legacy/rules/importers/_base.py +0 -4
  28. cognite/neat/legacy/rules/importers/_dms2rules.py +0 -2
  29. cognite/neat/legacy/rules/models/rdfpath.py +1 -2
  30. cognite/neat/legacy/workflows/examples/Export_DMS/workflow.yaml +89 -0
  31. cognite/neat/legacy/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +152 -0
  32. cognite/neat/legacy/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +139 -0
  33. cognite/neat/legacy/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +270 -0
  34. cognite/neat/legacy/workflows/examples/Import_DMS/workflow.yaml +65 -0
  35. cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml +116 -0
  36. cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml +67 -0
  37. cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml +64 -0
  38. cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml +95 -0
  39. cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml +111 -0
  40. cognite/neat/rules/exporters/_models.py +3 -0
  41. cognite/neat/rules/exporters/_rules2dms.py +46 -4
  42. cognite/neat/rules/exporters/_rules2excel.py +2 -11
  43. cognite/neat/rules/exporters/_validation.py +6 -8
  44. cognite/neat/rules/importers/_base.py +8 -4
  45. cognite/neat/rules/importers/_dms2rules.py +321 -129
  46. cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +2 -4
  47. cognite/neat/rules/importers/_dtdl2rules/spec.py +2 -4
  48. cognite/neat/rules/importers/_owl2rules/_owl2rules.py +2 -4
  49. cognite/neat/rules/importers/_spreadsheet2rules.py +18 -16
  50. cognite/neat/rules/importers/_yaml2rules.py +2 -4
  51. cognite/neat/rules/issues/base.py +3 -0
  52. cognite/neat/rules/issues/dms.py +144 -58
  53. cognite/neat/rules/issues/fileread.py +41 -0
  54. cognite/neat/rules/issues/formatters.py +3 -1
  55. cognite/neat/rules/issues/importing.py +155 -0
  56. cognite/neat/rules/issues/spreadsheet.py +12 -9
  57. cognite/neat/rules/models/entities.py +30 -8
  58. cognite/neat/rules/models/rdfpath.py +1 -2
  59. cognite/neat/rules/models/rules/_base.py +5 -6
  60. cognite/neat/rules/models/rules/_dms_architect_rules.py +494 -333
  61. cognite/neat/rules/models/rules/_dms_rules_write.py +43 -52
  62. cognite/neat/rules/models/rules/_dms_schema.py +112 -22
  63. cognite/neat/rules/models/rules/_domain_rules.py +5 -0
  64. cognite/neat/rules/models/rules/_information_rules.py +13 -6
  65. cognite/neat/rules/models/wrapped_entities.py +166 -0
  66. cognite/neat/utils/cdf_loaders/_data_modeling.py +3 -1
  67. cognite/neat/utils/cdf_loaders/_ingestion.py +2 -4
  68. cognite/neat/utils/spreadsheet.py +2 -4
  69. cognite/neat/utils/utils.py +2 -4
  70. cognite/neat/workflows/base.py +5 -5
  71. cognite/neat/workflows/manager.py +32 -22
  72. cognite/neat/workflows/model.py +3 -3
  73. cognite/neat/workflows/steps/lib/__init__.py +0 -7
  74. cognite/neat/workflows/steps/lib/current/__init__.py +6 -0
  75. cognite/neat/workflows/steps/lib/{rules_exporter.py → current/rules_exporter.py} +8 -8
  76. cognite/neat/workflows/steps/lib/{rules_importer.py → current/rules_importer.py} +4 -4
  77. cognite/neat/workflows/steps/lib/io/__init__.py +1 -0
  78. cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_contextualization.py +2 -2
  79. cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_extractor.py +9 -9
  80. cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_loader.py +9 -9
  81. cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_store.py +4 -4
  82. cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_transformer.py +2 -2
  83. cognite/neat/workflows/steps/lib/{v1 → legacy}/rules_exporter.py +15 -17
  84. cognite/neat/workflows/steps/lib/{v1 → legacy}/rules_importer.py +7 -7
  85. cognite/neat/workflows/steps/step_model.py +5 -9
  86. cognite/neat/workflows/steps_registry.py +20 -11
  87. {cognite_neat-0.75.8.dist-info → cognite_neat-0.76.0.dist-info}/METADATA +1 -1
  88. {cognite_neat-0.75.8.dist-info → cognite_neat-0.76.0.dist-info}/RECORD +98 -86
  89. cognite/neat/app/api/data_classes/configuration.py +0 -121
  90. /cognite/neat/app/ui/neat-app/build/static/js/{main.4345d42f.js.LICENSE.txt → main.ec7f72e2.js.LICENSE.txt} +0 -0
  91. /cognite/neat/workflows/steps/lib/{graph_extractor.py → current/graph_extractor.py} +0 -0
  92. /cognite/neat/workflows/steps/lib/{graph_loader.py → current/graph_loader.py} +0 -0
  93. /cognite/neat/workflows/steps/lib/{graph_store.py → current/graph_store.py} +0 -0
  94. /cognite/neat/workflows/steps/lib/{rules_validator.py → current/rules_validator.py} +0 -0
  95. /cognite/neat/workflows/steps/lib/{io_steps.py → io/io_steps.py} +0 -0
  96. /cognite/neat/workflows/steps/lib/{v1 → legacy}/__init__.py +0 -0
  97. {cognite_neat-0.75.8.dist-info → cognite_neat-0.76.0.dist-info}/LICENSE +0 -0
  98. {cognite_neat-0.75.8.dist-info → cognite_neat-0.76.0.dist-info}/WHEEL +0 -0
  99. {cognite_neat-0.75.8.dist-info → cognite_neat-0.76.0.dist-info}/entry_points.txt +0 -0
@@ -1,3 +1,6 @@
1
+ from collections import Counter
2
+ from collections.abc import Sequence
3
+ from datetime import datetime
1
4
  from pathlib import Path
2
5
  from typing import Literal, cast, overload
3
6
 
@@ -5,21 +8,29 @@ from cognite.client import CogniteClient
5
8
  from cognite.client import data_modeling as dm
6
9
  from cognite.client.data_classes.data_modeling import DataModelIdentifier
7
10
  from cognite.client.data_classes.data_modeling.containers import BTreeIndex, InvertedIndex
11
+ from cognite.client.data_classes.data_modeling.views import (
12
+ MultiEdgeConnectionApply,
13
+ MultiReverseDirectRelationApply,
14
+ SingleEdgeConnectionApply,
15
+ SingleReverseDirectRelationApply,
16
+ ViewPropertyApply,
17
+ )
8
18
  from cognite.client.utils import ms_to_datetime
9
19
 
10
20
  from cognite.neat.rules import issues
11
- from cognite.neat.rules.importers._base import BaseImporter, Rules
12
- from cognite.neat.rules.issues import IssueList
21
+ from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
22
+ from cognite.neat.rules.issues import IssueList, ValidationIssue
13
23
  from cognite.neat.rules.models.data_types import DataType
14
24
  from cognite.neat.rules.models.entities import (
15
25
  ClassEntity,
16
26
  ContainerEntity,
27
+ DataModelEntity,
17
28
  DMSUnknownEntity,
18
29
  ViewEntity,
19
30
  ViewPropertyEntity,
20
31
  )
21
32
  from cognite.neat.rules.models.rules import DMSRules, DMSSchema, RoleTypes
22
- from cognite.neat.rules.models.rules._base import ExtensionCategory, SchemaCompleteness
33
+ from cognite.neat.rules.models.rules._base import DataModelType, ExtensionCategory, SchemaCompleteness
23
34
  from cognite.neat.rules.models.rules._dms_architect_rules import (
24
35
  DMSContainer,
25
36
  DMSMetadata,
@@ -30,9 +41,16 @@ from cognite.neat.rules.models.rules._dms_architect_rules import (
30
41
 
31
42
 
32
43
  class DMSImporter(BaseImporter):
33
- def __init__(self, schema: DMSSchema, metadata: DMSMetadata | None = None):
44
+ def __init__(
45
+ self,
46
+ schema: DMSSchema,
47
+ read_issues: Sequence[ValidationIssue] | None = None,
48
+ metadata: DMSMetadata | None = None,
49
+ ):
34
50
  self.schema = schema
35
51
  self.metadata = metadata
52
+ self.issue_list = IssueList(read_issues)
53
+ self._container_by_id = {container.as_id(): container for container in schema.containers}
36
54
 
37
55
  @classmethod
38
56
  def from_data_model_id(cls, client: CogniteClient, data_model_id: DataModelIdentifier) -> "DMSImporter":
@@ -47,172 +65,346 @@ class DMSImporter(BaseImporter):
47
65
  """
48
66
  data_models = client.data_modeling.data_models.retrieve(data_model_id, inline_views=True)
49
67
  if len(data_models) == 0:
50
- raise ValueError(f"Data model {data_model_id} not found")
68
+ return cls(DMSSchema(), [issues.importing.NoDataModelError(f"Data model {data_model_id} not found")])
51
69
  data_model = data_models.latest_version()
52
- schema = DMSSchema.from_data_model(client, data_model)
53
- description, creator = DMSMetadata._get_description_and_creator(data_model.description)
70
+
71
+ try:
72
+ schema = DMSSchema.from_data_model(client, data_model)
73
+ except Exception as e:
74
+ return cls(DMSSchema(), [issues.importing.APIError(str(e))])
54
75
 
55
76
  created = ms_to_datetime(data_model.created_time)
56
77
  updated = ms_to_datetime(data_model.last_updated_time)
57
78
 
58
- metadata = DMSMetadata(
79
+ metadata = cls._create_metadata_from_model(data_model, created, updated)
80
+
81
+ return cls(schema, [], metadata)
82
+
83
+ @classmethod
84
+ def _create_metadata_from_model(
85
+ cls,
86
+ model: dm.DataModel[dm.View] | dm.DataModelApply,
87
+ created: datetime | None = None,
88
+ updated: datetime | None = None,
89
+ ) -> DMSMetadata:
90
+ description, creator = DMSMetadata._get_description_and_creator(model.description)
91
+ now = datetime.now().replace(microsecond=0)
92
+ return DMSMetadata(
59
93
  schema_=SchemaCompleteness.complete,
60
94
  extension=ExtensionCategory.addition,
61
- space=data_model.space,
62
- external_id=data_model.external_id,
63
- name=data_model.name or data_model.external_id,
64
- version=data_model.version or "0.1.0",
65
- updated=updated,
66
- created=created,
95
+ space=model.space,
96
+ external_id=model.external_id,
97
+ name=model.name or model.external_id,
98
+ version=model.version or "0.1.0",
99
+ updated=updated or now,
100
+ created=created or now,
67
101
  creator=creator,
68
102
  description=description,
69
- default_view_version=data_model.version or "0.1.0",
70
103
  )
71
- return cls(schema, metadata)
72
104
 
73
105
  @classmethod
74
106
  def from_directory(cls, directory: str | Path) -> "DMSImporter":
75
- return cls(DMSSchema.from_directory(directory))
107
+ issue_list = IssueList()
108
+ with _handle_issues(issue_list) as _:
109
+ schema = DMSSchema.from_directory(directory)
110
+ # If there were errors during the import, the to_rules
111
+ return cls(schema, issue_list)
76
112
 
77
113
  @classmethod
78
114
  def from_zip_file(cls, zip_file: str | Path) -> "DMSImporter":
79
115
  if Path(zip_file).suffix != ".zip":
80
- raise ValueError("File extension is not .zip")
81
- return cls(DMSSchema.from_zip(zip_file))
116
+ return cls(DMSSchema(), [issues.fileread.InvalidFileFormatError(Path(zip_file), [".zip"])])
117
+ issue_list = IssueList()
118
+ with _handle_issues(issue_list) as _:
119
+ schema = DMSSchema.from_zip(zip_file)
120
+ return cls(schema, issue_list)
82
121
 
83
122
  @overload
84
- def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
85
- ...
123
+ def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
86
124
 
87
125
  @overload
88
126
  def to_rules(
89
127
  self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
90
- ) -> tuple[Rules | None, IssueList]:
91
- ...
128
+ ) -> tuple[Rules | None, IssueList]: ...
92
129
 
93
130
  def to_rules(
94
131
  self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
95
132
  ) -> tuple[Rules | None, IssueList] | Rules:
96
- if role is RoleTypes.domain_expert:
97
- raise ValueError(f"Role {role} is not supported for DMSImporter")
98
- issue_list = IssueList()
99
- data_model = self.schema.data_models[0]
133
+ if self.issue_list.has_errors:
134
+ # In case there were errors during the import, the to_rules method will return None
135
+ return self._return_or_raise(self.issue_list, errors)
100
136
 
101
- container_by_id = {container.as_id(): container for container in self.schema.containers}
137
+ if len(self.schema.data_models) == 0:
138
+ self.issue_list.append(issues.importing.NoDataModelError("No data model found."))
139
+ return self._return_or_raise(self.issue_list, errors)
140
+
141
+ if len(self.schema.data_models) > 2:
142
+ # Creating a DataModelEntity to convert the data model id to a string.
143
+ self.issue_list.append(
144
+ issues.importing.MultipleDataModelsWarning(
145
+ [str(DataModelEntity.from_id(model.as_id())) for model in self.schema.data_models]
146
+ )
147
+ )
148
+
149
+ data_model = self.schema.data_models[0]
102
150
 
103
151
  properties = SheetList[DMSProperty]()
152
+ ref_properties = SheetList[DMSProperty]()
104
153
  for view in self.schema.views:
105
- class_entity = ClassEntity(prefix=view.space, suffix=view.external_id, version=view.version)
154
+ view_id = view.as_id()
155
+ view_entity = ViewEntity.from_id(view_id)
156
+ class_entity = view_entity.as_class()
106
157
  for prop_id, prop in (view.properties or {}).items():
107
- if isinstance(prop, dm.MappedPropertyApply):
108
- if prop.container not in container_by_id:
109
- raise ValueError(f"Container {prop.container} not found")
110
- container = container_by_id[prop.container]
111
- if prop.container_property_identifier not in container.properties:
112
- raise ValueError(
113
- f"Property {prop.container_property_identifier} not found "
114
- f"in container {container.external_id}"
115
- )
116
- container_prop = container.properties[prop.container_property_identifier]
117
-
118
- index: list[str] = []
119
- for index_name, index_obj in (container.indexes or {}).items():
120
- if isinstance(index_obj, BTreeIndex | InvertedIndex) and prop_id in index_obj.properties:
121
- index.append(index_name)
122
- unique_constraints: list[str] = []
123
- for constraint_name, constraint_obj in (container.constraints or {}).items():
124
- if isinstance(constraint_obj, dm.RequiresConstraint):
125
- # This is handled in the .from_container method of DMSContainer
126
- continue
127
- elif (
128
- isinstance(constraint_obj, dm.UniquenessConstraint) and prop_id in constraint_obj.properties
129
- ):
130
- unique_constraints.append(constraint_name)
131
- elif isinstance(constraint_obj, dm.UniquenessConstraint):
132
- # This does not apply to this property
133
- continue
134
- else:
135
- raise NotImplementedError(f"Constraint type {type(constraint_obj)} not implemented")
136
-
137
- if isinstance(container_prop.type, dm.DirectRelation):
138
- direct_value_type: str | ViewEntity | DataType | DMSUnknownEntity
139
- if prop.source is None:
140
- issue_list.append(
141
- issues.importing.UnknownValueTypeWarning(class_entity.versioned_id, prop_id)
142
- )
143
- direct_value_type = DMSUnknownEntity()
144
- else:
145
- direct_value_type = ViewEntity.from_id(prop.source)
146
-
147
- dms_property = DMSProperty(
148
- class_=class_entity,
149
- property_=prop_id,
150
- description=prop.description,
151
- name=prop.name,
152
- value_type=direct_value_type,
153
- relation="direct",
154
- nullable=container_prop.nullable,
155
- default=container_prop.default_value,
156
- is_list=False,
157
- container=ContainerEntity.from_id(container.as_id()),
158
- container_property=prop.container_property_identifier,
159
- view=ViewEntity.from_id(view.as_id()),
160
- view_property=prop_id,
161
- index=index or None,
162
- constraint=unique_constraints or None,
163
- )
158
+ dms_property = self._create_dms_property(prop_id, prop, view_entity, class_entity)
159
+ if dms_property is not None:
160
+ if view_id in self.schema.frozen_ids:
161
+ ref_properties.append(dms_property)
164
162
  else:
165
- dms_property = DMSProperty(
166
- class_=ClassEntity(prefix=view.space, suffix=view.external_id, version=view.version),
167
- property_=prop_id,
168
- description=prop.description,
169
- name=prop.name,
170
- value_type=cast(ViewPropertyEntity | DataType, container_prop.type._type),
171
- nullable=container_prop.nullable,
172
- is_list=container_prop.type.is_list,
173
- default=container_prop.default_value,
174
- container=ContainerEntity.from_id(container.as_id()),
175
- container_property=prop.container_property_identifier,
176
- view=ViewEntity.from_id(view.as_id()),
177
- view_property=prop_id,
178
- index=index or None,
179
- constraint=unique_constraints or None,
180
- )
181
- elif isinstance(prop, dm.MultiEdgeConnectionApply):
182
- view_entity = ViewEntity.from_id(prop.source)
183
- dms_property = DMSProperty(
184
- class_=ClassEntity(prefix=view.space, suffix=view.external_id, version=view.version),
185
- property_=prop_id,
186
- relation="multiedge",
187
- description=prop.description,
188
- name=prop.name,
189
- value_type=view_entity,
190
- view=ViewEntity.from_id(view.as_id()),
191
- view_property=prop_id,
192
- )
193
- else:
194
- raise NotImplementedError(f"Property type {type(prop)} not implemented")
195
-
196
- properties.append(dms_property)
163
+ properties.append(dms_property)
197
164
 
198
165
  data_model_view_ids: set[dm.ViewId] = {
199
166
  view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
200
167
  }
201
168
 
202
- dms_rules = DMSRules(
203
- metadata=self.metadata or DMSMetadata.from_data_model(data_model),
169
+ metadata = self.metadata or DMSMetadata.from_data_model(data_model)
170
+ metadata.data_model_type = self._infer_data_model_type(metadata.space)
171
+ if ref_properties:
172
+ metadata.schema_ = SchemaCompleteness.extended
173
+
174
+ with _handle_issues(
175
+ self.issue_list,
176
+ ) as future:
177
+ user_rules = DMSRules(
178
+ metadata=metadata,
179
+ properties=properties,
180
+ containers=SheetList[DMSContainer](
181
+ data=[
182
+ DMSContainer.from_container(container)
183
+ for container in self.schema.containers
184
+ if container.as_id() not in self.schema.frozen_ids
185
+ ]
186
+ ),
187
+ views=SheetList[DMSView](
188
+ data=[
189
+ DMSView.from_view(view, in_model=view.as_id() in data_model_view_ids)
190
+ for view in self.schema.views
191
+ if view.as_id() not in self.schema.frozen_ids
192
+ ]
193
+ ),
194
+ reference=self._create_reference_rules(ref_properties),
195
+ )
196
+
197
+ if future.result == "failure" or self.issue_list.has_errors:
198
+ return self._return_or_raise(self.issue_list, errors)
199
+
200
+ return self._to_output(user_rules, self.issue_list, errors, role)
201
+
202
+ def _create_reference_rules(self, properties: SheetList[DMSProperty]) -> DMSRules | None:
203
+ if not properties:
204
+ return None
205
+
206
+ if len(self.schema.data_models) == 2:
207
+ data_model = self.schema.data_models[1]
208
+ data_model_view_ids: set[dm.ViewId] = {
209
+ view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
210
+ }
211
+ metadata = self._create_metadata_from_model(data_model)
212
+ else:
213
+ data_model_view_ids = set()
214
+ now = datetime.now().replace(microsecond=0)
215
+ space = Counter(prop.view.space for prop in properties).most_common(1)[0][0]
216
+ metadata = DMSMetadata(
217
+ schema_=SchemaCompleteness.complete,
218
+ extension=ExtensionCategory.addition,
219
+ space=space,
220
+ external_id="Unknown",
221
+ version="0.1.0",
222
+ creator=["Unknown"],
223
+ created=now,
224
+ updated=now,
225
+ )
226
+
227
+ metadata.data_model_type = DataModelType.enterprise
228
+ return DMSRules(
229
+ metadata=metadata,
204
230
  properties=properties,
231
+ views=SheetList[DMSView](
232
+ data=[
233
+ DMSView.from_view(view, in_model=not data_model_view_ids or (view.as_id() in data_model_view_ids))
234
+ for view in self.schema.views
235
+ if view.as_id() in self.schema.frozen_ids
236
+ ]
237
+ ),
205
238
  containers=SheetList[DMSContainer](
206
- data=[DMSContainer.from_container(container) for container in self.schema.containers]
239
+ data=[
240
+ DMSContainer.from_container(container)
241
+ for container in self.schema.containers
242
+ if container.as_id() in self.schema.frozen_ids
243
+ ]
207
244
  ),
208
- views=SheetList[DMSView](data=[DMSView.from_view(view, data_model_view_ids) for view in self.schema.views]),
245
+ reference=None,
246
+ )
247
+
248
+ def _infer_data_model_type(self, space: str) -> DataModelType:
249
+ if self.schema.referenced_spaces() - {space}:
250
+ # If the data model has containers, views, node types in another space
251
+ # we assume it is a solution model.
252
+ return DataModelType.solution
253
+ else:
254
+ # All containers, views, node types are in the same space as the data model
255
+ return DataModelType.enterprise
256
+
257
+ def _create_dms_property(
258
+ self, prop_id: str, prop: ViewPropertyApply, view_entity: ViewEntity, class_entity: ClassEntity
259
+ ) -> DMSProperty | None:
260
+ if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._container_by_id:
261
+ self.issue_list.append(
262
+ issues.importing.MissingContainerWarning(
263
+ view_id=str(view_entity),
264
+ property_=prop_id,
265
+ container_id=str(ContainerEntity.from_id(prop.container)),
266
+ )
267
+ )
268
+ return None
269
+ if (
270
+ isinstance(prop, dm.MappedPropertyApply)
271
+ and prop.container_property_identifier not in self._container_by_id[prop.container].properties
272
+ ):
273
+ self.issue_list.append(
274
+ issues.importing.MissingContainerPropertyWarning(
275
+ view_id=str(view_entity),
276
+ property_=prop_id,
277
+ container_id=str(ContainerEntity.from_id(prop.container)),
278
+ )
279
+ )
280
+ return None
281
+ if not isinstance(
282
+ prop,
283
+ dm.MappedPropertyApply
284
+ | SingleEdgeConnectionApply
285
+ | MultiEdgeConnectionApply
286
+ | SingleReverseDirectRelationApply
287
+ | MultiReverseDirectRelationApply,
288
+ ):
289
+ self.issue_list.append(
290
+ issues.importing.UnknownPropertyTypeWarning(view_entity.versioned_id, prop_id, type(prop).__name__)
291
+ )
292
+ return None
293
+
294
+ value_type = self._get_value_type(prop, view_entity, prop_id)
295
+ if value_type is None:
296
+ return None
297
+
298
+ return DMSProperty(
299
+ class_=class_entity,
300
+ property_=prop_id,
301
+ description=prop.description,
302
+ name=prop.name,
303
+ connection=self._get_relation_type(prop),
304
+ value_type=value_type,
305
+ is_list=self._get_is_list(prop),
306
+ nullable=self._get_nullable(prop),
307
+ default=self._get_default(prop),
308
+ container=ContainerEntity.from_id(prop.container) if isinstance(prop, dm.MappedPropertyApply) else None,
309
+ container_property=prop.container_property_identifier if isinstance(prop, dm.MappedPropertyApply) else None,
310
+ view=view_entity,
311
+ view_property=prop_id,
312
+ index=self._get_index(prop, prop_id),
313
+ constraint=self._get_constraint(prop, prop_id),
209
314
  )
210
- output_rules: Rules
211
- if role is RoleTypes.information_architect:
212
- output_rules = dms_rules.as_information_architect_rules()
315
+
316
+ def _container_prop_unsafe(self, prop: dm.MappedPropertyApply) -> dm.ContainerProperty:
317
+ """This method assumes you have already checked that the container with property exists."""
318
+ return self._container_by_id[prop.container].properties[prop.container_property_identifier]
319
+
320
+ def _get_relation_type(self, prop: ViewPropertyApply) -> Literal["edge", "reverse", "direct"] | None:
321
+ if isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "outwards":
322
+ return "edge"
323
+ elif isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "inwards":
324
+ return "reverse"
325
+ elif isinstance(prop, SingleReverseDirectRelationApply | MultiReverseDirectRelationApply):
326
+ return "reverse"
327
+ elif isinstance(prop, dm.MappedPropertyApply) and isinstance(
328
+ self._container_prop_unsafe(prop).type, dm.DirectRelation
329
+ ):
330
+ return "direct"
213
331
  else:
214
- output_rules = dms_rules
215
- if errors == "raise":
216
- return output_rules
332
+ return None
333
+
334
+ def _get_value_type(
335
+ self, prop: ViewPropertyApply, view_entity: ViewEntity, prop_id
336
+ ) -> DataType | ViewEntity | ViewPropertyEntity | DMSUnknownEntity | None:
337
+ if isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "outwards":
338
+ return ViewEntity.from_id(prop.source)
339
+ elif isinstance(prop, SingleReverseDirectRelationApply | MultiReverseDirectRelationApply):
340
+ return ViewPropertyEntity.from_id(prop.through)
341
+ elif isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "inwards":
342
+ return ViewEntity.from_id(prop.source)
343
+ elif isinstance(prop, dm.MappedPropertyApply):
344
+ container_prop = self._container_prop_unsafe(cast(dm.MappedPropertyApply, prop))
345
+ if isinstance(container_prop.type, dm.DirectRelation):
346
+ if prop.source is None:
347
+ # The warning is issued when the DMS Rules are created.
348
+ return DMSUnknownEntity()
349
+ else:
350
+ return ViewEntity.from_id(prop.source)
351
+ else:
352
+ return DataType.load(container_prop.type._type)
353
+ else:
354
+ self.issue_list.append(issues.importing.FailedToInferValueTypeWarning(str(view_entity), prop_id))
355
+ return None
356
+
357
+ def _get_nullable(self, prop: ViewPropertyApply) -> bool | None:
358
+ if isinstance(prop, dm.MappedPropertyApply):
359
+ return self._container_prop_unsafe(prop).nullable
360
+ else:
361
+ return None
362
+
363
+ def _get_is_list(self, prop: ViewPropertyApply) -> bool | None:
364
+ if isinstance(prop, dm.MappedPropertyApply):
365
+ return self._container_prop_unsafe(prop).type.is_list
366
+ elif isinstance(prop, MultiEdgeConnectionApply | MultiReverseDirectRelationApply):
367
+ return True
368
+ elif isinstance(prop, SingleEdgeConnectionApply | SingleReverseDirectRelationApply):
369
+ return False
217
370
  else:
218
- return output_rules, issue_list
371
+ return None
372
+
373
+ def _get_default(self, prop: ViewPropertyApply) -> str | None:
374
+ if isinstance(prop, dm.MappedPropertyApply):
375
+ default = self._container_prop_unsafe(prop).default_value
376
+ if default is not None:
377
+ return str(default)
378
+ return None
379
+
380
+ def _get_index(self, prop: ViewPropertyApply, prop_id) -> list[str] | None:
381
+ if not isinstance(prop, dm.MappedPropertyApply):
382
+ return None
383
+ container = self._container_by_id[prop.container]
384
+ index: list[str] = []
385
+ for index_name, index_obj in (container.indexes or {}).items():
386
+ if isinstance(index_obj, BTreeIndex | InvertedIndex) and prop_id in index_obj.properties:
387
+ index.append(index_name)
388
+ return index or None
389
+
390
+ def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[str] | None:
391
+ if not isinstance(prop, dm.MappedPropertyApply):
392
+ return None
393
+ container = self._container_by_id[prop.container]
394
+ unique_constraints: list[str] = []
395
+ for constraint_name, constraint_obj in (container.constraints or {}).items():
396
+ if isinstance(constraint_obj, dm.RequiresConstraint):
397
+ # This is handled in the .from_container method of DMSContainer
398
+ continue
399
+ elif isinstance(constraint_obj, dm.UniquenessConstraint) and prop_id in constraint_obj.properties:
400
+ unique_constraints.append(constraint_name)
401
+ elif isinstance(constraint_obj, dm.UniquenessConstraint):
402
+ # This does not apply to this property
403
+ continue
404
+ else:
405
+ self.issue_list.append(
406
+ issues.importing.UnknownContainerConstraintWarning(
407
+ str(ContainerEntity.from_id(prop.container)), prop_id, type(constraint_obj).__name__
408
+ )
409
+ )
410
+ return unique_constraints or None
@@ -119,14 +119,12 @@ class DTDLImporter(BaseImporter):
119
119
  return cls(items, zip_file.stem, read_issues=issues)
120
120
 
121
121
  @overload
122
- def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
123
- ...
122
+ def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
124
123
 
125
124
  @overload
126
125
  def to_rules(
127
126
  self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
128
- ) -> tuple[Rules | None, IssueList]:
129
- ...
127
+ ) -> tuple[Rules | None, IssueList]: ...
130
128
 
131
129
  def to_rules(
132
130
  self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
@@ -72,8 +72,7 @@ class DTMI(BaseModel):
72
72
  exclude_none: bool = False,
73
73
  round_trip: bool = False,
74
74
  warnings: bool = True,
75
- ) -> str:
76
- ...
75
+ ) -> str: ...
77
76
 
78
77
 
79
78
  IRI: TypeAlias = str
@@ -124,8 +123,7 @@ class Unit(BaseModel, ABC):
124
123
  exclude_none: bool = False,
125
124
  round_trip: bool = False,
126
125
  warnings: bool = True,
127
- ) -> str:
128
- ...
126
+ ) -> str: ...
129
127
 
130
128
 
131
129
  class DTDLBase(BaseModel, ABC):
@@ -43,14 +43,12 @@ class OWLImporter(BaseImporter):
43
43
  self.make_compliant = make_compliant
44
44
 
45
45
  @overload
46
- def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
47
- ...
46
+ def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
48
47
 
49
48
  @overload
50
49
  def to_rules(
51
50
  self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
52
- ) -> tuple[Rules | None, IssueList]:
53
- ...
51
+ ) -> tuple[Rules | None, IssueList]: ...
54
52
 
55
53
  def to_rules(
56
54
  self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
@@ -22,7 +22,15 @@ from cognite.neat.utils.spreadsheet import SpreadsheetRead, read_individual_shee
22
22
  from ._base import BaseImporter, Rules, _handle_issues
23
23
 
24
24
  SOURCE_SHEET__TARGET_FIELD__HEADERS = [
25
- ("Properties", "Properties", "Class"),
25
+ (
26
+ "Properties",
27
+ "Properties",
28
+ {
29
+ RoleTypes.domain_expert: "Property",
30
+ RoleTypes.information_architect: "Property",
31
+ RoleTypes.dms_architect: "View Property",
32
+ },
33
+ ),
26
34
  ("Classes", "Classes", "Class"),
27
35
  ("Containers", "Containers", "Container"),
28
36
  ("Views", "Views", "View"),
@@ -131,11 +139,15 @@ class SpreadsheetReader:
131
139
  )
132
140
  return None, read_info_by_sheet
133
141
 
134
- for source_sheet_name, target_sheet_name, headers in SOURCE_SHEET__TARGET_FIELD__HEADERS:
142
+ for source_sheet_name, target_sheet_name, headers_input in SOURCE_SHEET__TARGET_FIELD__HEADERS:
135
143
  source_sheet_name = self.to_reference_sheet(source_sheet_name) if self._is_reference else source_sheet_name
136
144
 
137
145
  if source_sheet_name not in excel_file.sheet_names:
138
146
  continue
147
+ if isinstance(headers_input, dict):
148
+ headers = headers_input[metadata.role]
149
+ else:
150
+ headers = headers_input
139
151
 
140
152
  try:
141
153
  sheets[target_sheet_name], read_info_by_sheet[source_sheet_name] = read_individual_sheet(
@@ -153,16 +165,14 @@ class ExcelImporter(BaseImporter):
153
165
  self.filepath = filepath
154
166
 
155
167
  @overload
156
- def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
157
- ...
168
+ def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
158
169
 
159
170
  @overload
160
171
  def to_rules(
161
172
  self,
162
173
  errors: Literal["continue"] = "continue",
163
174
  role: RoleTypes | None = None,
164
- ) -> tuple[Rules | None, IssueList]:
165
- ...
175
+ ) -> tuple[Rules | None, IssueList]: ...
166
176
 
167
177
  def to_rules(
168
178
  self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
@@ -231,12 +241,6 @@ class ExcelImporter(BaseImporter):
231
241
  role=role,
232
242
  )
233
243
 
234
- @classmethod
235
- def _return_or_raise(cls, issue_list: IssueList, errors: Literal["raise", "continue"]) -> tuple[None, IssueList]:
236
- if errors == "raise":
237
- raise issue_list.as_errors()
238
- return None, issue_list
239
-
240
244
 
241
245
  class GoogleSheetImporter(BaseImporter):
242
246
  def __init__(self, sheet_id: str, skiprows: int = 1):
@@ -244,14 +248,12 @@ class GoogleSheetImporter(BaseImporter):
244
248
  self.skiprows = skiprows
245
249
 
246
250
  @overload
247
- def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
248
- ...
251
+ def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
249
252
 
250
253
  @overload
251
254
  def to_rules(
252
255
  self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
253
- ) -> tuple[Rules | None, IssueList]:
254
- ...
256
+ ) -> tuple[Rules | None, IssueList]: ...
255
257
 
256
258
  def to_rules(
257
259
  self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None