cognite-neat 0.88.3__py3-none-any.whl → 0.89.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/constants.py +3 -0
- cognite/neat/graph/extractors/_mock_graph_generator.py +2 -1
- cognite/neat/issues/_base.py +2 -1
- cognite/neat/issues/errors/__init__.py +2 -1
- cognite/neat/issues/errors/_general.py +7 -0
- cognite/neat/issues/warnings/_models.py +1 -1
- cognite/neat/issues/warnings/user_modeling.py +1 -1
- cognite/neat/rules/_shared.py +49 -6
- cognite/neat/rules/analysis/_base.py +1 -1
- cognite/neat/rules/exporters/_base.py +7 -18
- cognite/neat/rules/exporters/_rules2dms.py +8 -18
- cognite/neat/rules/exporters/_rules2excel.py +5 -12
- cognite/neat/rules/exporters/_rules2ontology.py +9 -19
- cognite/neat/rules/exporters/_rules2yaml.py +3 -6
- cognite/neat/rules/importers/_base.py +7 -52
- cognite/neat/rules/importers/_dms2rules.py +171 -115
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +26 -18
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +14 -30
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +7 -3
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +3 -3
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +18 -11
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2rules.py +9 -18
- cognite/neat/rules/importers/_rdf/_inference2rules.py +10 -33
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +9 -20
- cognite/neat/rules/importers/_rdf/_shared.py +1 -1
- cognite/neat/rules/importers/_spreadsheet2rules.py +22 -86
- cognite/neat/rules/importers/_yaml2rules.py +14 -41
- cognite/neat/rules/models/__init__.py +21 -5
- cognite/neat/rules/models/_base_input.py +162 -0
- cognite/neat/rules/models/{_base.py → _base_rules.py} +1 -12
- cognite/neat/rules/models/asset/__init__.py +5 -2
- cognite/neat/rules/models/asset/_rules.py +2 -20
- cognite/neat/rules/models/asset/_rules_input.py +40 -115
- cognite/neat/rules/models/asset/_validation.py +1 -1
- cognite/neat/rules/models/data_types.py +150 -44
- cognite/neat/rules/models/dms/__init__.py +19 -7
- cognite/neat/rules/models/dms/_exporter.py +72 -26
- cognite/neat/rules/models/dms/_rules.py +42 -155
- cognite/neat/rules/models/dms/_rules_input.py +186 -254
- cognite/neat/rules/models/dms/_serializer.py +44 -3
- cognite/neat/rules/models/dms/_validation.py +3 -4
- cognite/neat/rules/models/domain.py +52 -1
- cognite/neat/rules/models/entities/__init__.py +63 -0
- cognite/neat/rules/models/entities/_constants.py +73 -0
- cognite/neat/rules/models/entities/_loaders.py +76 -0
- cognite/neat/rules/models/entities/_multi_value.py +67 -0
- cognite/neat/rules/models/{entities.py → entities/_single_value.py} +74 -232
- cognite/neat/rules/models/entities/_types.py +86 -0
- cognite/neat/rules/models/{wrapped_entities.py → entities/_wrapped.py} +1 -1
- cognite/neat/rules/models/information/__init__.py +10 -2
- cognite/neat/rules/models/information/_rules.py +3 -14
- cognite/neat/rules/models/information/_rules_input.py +57 -204
- cognite/neat/rules/models/information/_validation.py +1 -1
- cognite/neat/rules/transformers/__init__.py +21 -0
- cognite/neat/rules/transformers/_base.py +69 -3
- cognite/neat/rules/{models/information/_converter.py → transformers/_converters.py} +216 -20
- cognite/neat/rules/transformers/_map_onto.py +97 -0
- cognite/neat/rules/transformers/_pipelines.py +61 -0
- cognite/neat/rules/transformers/_verification.py +136 -0
- cognite/neat/store/_provenance.py +10 -1
- cognite/neat/utils/cdf/data_classes.py +20 -0
- cognite/neat/utils/regex_patterns.py +6 -0
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +106 -37
- cognite/neat/workflows/steps/lib/current/rules_importer.py +24 -22
- {cognite_neat-0.88.3.dist-info → cognite_neat-0.89.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.88.3.dist-info → cognite_neat-0.89.0.dist-info}/RECORD +71 -66
- cognite/neat/rules/models/_constants.py +0 -2
- cognite/neat/rules/models/_types/__init__.py +0 -19
- cognite/neat/rules/models/asset/_converter.py +0 -4
- cognite/neat/rules/models/dms/_converter.py +0 -143
- /cognite/neat/rules/models/{_types/_field.py → _types.py} +0 -0
- {cognite_neat-0.88.3.dist-info → cognite_neat-0.89.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.88.3.dist-info → cognite_neat-0.89.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.88.3.dist-info → cognite_neat-0.89.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
from collections import Counter
|
|
2
|
-
from collections.abc import Sequence
|
|
2
|
+
from collections.abc import Collection, Sequence
|
|
3
3
|
from datetime import datetime
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import
|
|
5
|
+
from typing import Literal, cast
|
|
6
6
|
|
|
7
7
|
from cognite.client import CogniteClient
|
|
8
8
|
from cognite.client import data_modeling as dm
|
|
9
9
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
|
|
10
10
|
from cognite.client.data_classes.data_modeling.containers import BTreeIndex, InvertedIndex
|
|
11
|
-
from cognite.client.data_classes.data_modeling.data_types import
|
|
11
|
+
from cognite.client.data_classes.data_modeling.data_types import Enum as DMSEnum
|
|
12
|
+
from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType, PropertyTypeWithUnit
|
|
12
13
|
from cognite.client.data_classes.data_modeling.views import (
|
|
13
14
|
MultiEdgeConnectionApply,
|
|
14
15
|
MultiReverseDirectRelationApply,
|
|
@@ -24,34 +25,37 @@ from cognite.neat.issues.warnings import (
|
|
|
24
25
|
PropertyNotFoundWarning,
|
|
25
26
|
PropertyTypeNotSupportedWarning,
|
|
26
27
|
ResourceNotFoundWarning,
|
|
28
|
+
ResourcesDuplicatedWarning,
|
|
27
29
|
)
|
|
28
|
-
from cognite.neat.rules.
|
|
30
|
+
from cognite.neat.rules._shared import ReadRules
|
|
31
|
+
from cognite.neat.rules.importers._base import BaseImporter, _handle_issues
|
|
29
32
|
from cognite.neat.rules.models import (
|
|
30
33
|
DataModelType,
|
|
31
|
-
|
|
34
|
+
DMSInputRules,
|
|
32
35
|
DMSSchema,
|
|
33
|
-
ExtensionCategory,
|
|
34
|
-
RoleTypes,
|
|
35
36
|
SchemaCompleteness,
|
|
36
|
-
SheetList,
|
|
37
37
|
)
|
|
38
|
-
from cognite.neat.rules.models.data_types import DataType
|
|
38
|
+
from cognite.neat.rules.models.data_types import DataType, Enum
|
|
39
39
|
from cognite.neat.rules.models.dms import (
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
40
|
+
DMSInputContainer,
|
|
41
|
+
DMSInputEnum,
|
|
42
|
+
DMSInputMetadata,
|
|
43
|
+
DMSInputNode,
|
|
44
|
+
DMSInputProperty,
|
|
45
|
+
DMSInputView,
|
|
44
46
|
)
|
|
45
47
|
from cognite.neat.rules.models.entities import (
|
|
46
48
|
ClassEntity,
|
|
47
49
|
ContainerEntity,
|
|
50
|
+
DMSNodeEntity,
|
|
48
51
|
DMSUnknownEntity,
|
|
52
|
+
EdgeEntity,
|
|
53
|
+
ReverseConnectionEntity,
|
|
49
54
|
ViewEntity,
|
|
50
|
-
ViewPropertyEntity,
|
|
51
55
|
)
|
|
52
56
|
|
|
53
57
|
|
|
54
|
-
class DMSImporter(BaseImporter):
|
|
58
|
+
class DMSImporter(BaseImporter[DMSInputRules]):
|
|
55
59
|
"""Imports a Data Model from Cognite Data Fusion.
|
|
56
60
|
|
|
57
61
|
Args:
|
|
@@ -66,8 +70,8 @@ class DMSImporter(BaseImporter):
|
|
|
66
70
|
self,
|
|
67
71
|
schema: DMSSchema,
|
|
68
72
|
read_issues: Sequence[NeatIssue] | None = None,
|
|
69
|
-
metadata:
|
|
70
|
-
ref_metadata:
|
|
73
|
+
metadata: DMSInputMetadata | None = None,
|
|
74
|
+
ref_metadata: DMSInputMetadata | None = None,
|
|
71
75
|
):
|
|
72
76
|
# Calling this root schema to distinguish it from
|
|
73
77
|
# * User Schema
|
|
@@ -77,10 +81,10 @@ class DMSImporter(BaseImporter):
|
|
|
77
81
|
self.ref_metadata = ref_metadata
|
|
78
82
|
self.issue_list = IssueList(read_issues)
|
|
79
83
|
self._all_containers_by_id = schema.containers.copy()
|
|
80
|
-
self.
|
|
81
|
-
if
|
|
82
|
-
self._all_containers_by_id.update(
|
|
83
|
-
self.
|
|
84
|
+
self._all_views_by_id = schema.views.copy()
|
|
85
|
+
if schema.reference:
|
|
86
|
+
self._all_containers_by_id.update(schema.reference.containers.items())
|
|
87
|
+
self._all_views_by_id.update(schema.reference.views.items())
|
|
84
88
|
|
|
85
89
|
@classmethod
|
|
86
90
|
def from_data_model_id(
|
|
@@ -162,8 +166,8 @@ class DMSImporter(BaseImporter):
|
|
|
162
166
|
cls,
|
|
163
167
|
model: dm.DataModel[dm.View] | dm.DataModelApply,
|
|
164
168
|
has_reference: bool = False,
|
|
165
|
-
) ->
|
|
166
|
-
description, creator =
|
|
169
|
+
) -> DMSInputMetadata:
|
|
170
|
+
description, creator = DMSInputMetadata._get_description_and_creator(model.description)
|
|
167
171
|
|
|
168
172
|
if isinstance(model, dm.DataModel):
|
|
169
173
|
created = ms_to_datetime(model.created_time)
|
|
@@ -172,17 +176,17 @@ class DMSImporter(BaseImporter):
|
|
|
172
176
|
now = datetime.now().replace(microsecond=0)
|
|
173
177
|
created = now
|
|
174
178
|
updated = now
|
|
175
|
-
return
|
|
176
|
-
schema_=
|
|
177
|
-
data_model_type=
|
|
178
|
-
extension=
|
|
179
|
+
return DMSInputMetadata(
|
|
180
|
+
schema_="complete",
|
|
181
|
+
data_model_type="solution" if has_reference else "enterprise",
|
|
182
|
+
extension="addition",
|
|
179
183
|
space=model.space,
|
|
180
184
|
external_id=model.external_id,
|
|
181
185
|
name=model.name or model.external_id,
|
|
182
186
|
version=model.version or "0.1.0",
|
|
183
187
|
updated=updated,
|
|
184
188
|
created=created,
|
|
185
|
-
creator=creator,
|
|
189
|
+
creator=",".join(creator),
|
|
186
190
|
description=description,
|
|
187
191
|
)
|
|
188
192
|
|
|
@@ -203,71 +207,52 @@ class DMSImporter(BaseImporter):
|
|
|
203
207
|
schema = DMSSchema.from_zip(zip_file)
|
|
204
208
|
return cls(schema, issue_list)
|
|
205
209
|
|
|
206
|
-
|
|
207
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
|
|
208
|
-
|
|
209
|
-
@overload
|
|
210
|
-
def to_rules(
|
|
211
|
-
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
212
|
-
) -> tuple[VerifiedRules | None, IssueList]: ...
|
|
213
|
-
|
|
214
|
-
def to_rules(
|
|
215
|
-
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
216
|
-
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
210
|
+
def to_rules(self) -> ReadRules[DMSInputRules]:
|
|
217
211
|
if self.issue_list.has_errors:
|
|
218
212
|
# In case there were errors during the import, the to_rules method will return None
|
|
219
|
-
return
|
|
213
|
+
return ReadRules(None, self.issue_list, {})
|
|
220
214
|
|
|
221
215
|
if not self.root_schema.data_model:
|
|
222
216
|
self.issue_list.append(ResourceMissingIdentifierError("data model", type(self.root_schema).__name__))
|
|
223
|
-
return
|
|
217
|
+
return ReadRules(None, self.issue_list, {})
|
|
218
|
+
|
|
224
219
|
model = self.root_schema.data_model
|
|
225
|
-
with _handle_issues(
|
|
226
|
-
self.issue_list,
|
|
227
|
-
) as future:
|
|
228
|
-
schema_completeness = SchemaCompleteness.complete
|
|
229
|
-
data_model_type = DataModelType.enterprise
|
|
230
|
-
reference: DMSRules | None = None
|
|
231
|
-
if (ref_schema := self.root_schema.reference) and (ref_model := ref_schema.data_model):
|
|
232
|
-
# Reference should always be an enterprise model.
|
|
233
|
-
reference = DMSRules(
|
|
234
|
-
**self._create_rule_components(
|
|
235
|
-
ref_model,
|
|
236
|
-
ref_schema,
|
|
237
|
-
self.ref_metadata
|
|
238
|
-
or self._create_default_metadata(list(ref_schema.views.values()), is_ref=True),
|
|
239
|
-
DataModelType.enterprise,
|
|
240
|
-
)
|
|
241
|
-
)
|
|
242
|
-
data_model_type = DataModelType.solution
|
|
243
|
-
|
|
244
|
-
user_rules = DMSRules(
|
|
245
|
-
**self._create_rule_components(
|
|
246
|
-
model,
|
|
247
|
-
self.root_schema,
|
|
248
|
-
self.metadata,
|
|
249
|
-
data_model_type,
|
|
250
|
-
schema_completeness,
|
|
251
|
-
has_reference=reference is not None,
|
|
252
|
-
),
|
|
253
|
-
reference=reference,
|
|
254
|
-
)
|
|
255
220
|
|
|
256
|
-
|
|
257
|
-
|
|
221
|
+
schema_completeness = SchemaCompleteness.complete
|
|
222
|
+
data_model_type = DataModelType.enterprise
|
|
223
|
+
reference: DMSInputRules | None = None
|
|
224
|
+
if (ref_schema := self.root_schema.reference) and (ref_model := ref_schema.data_model):
|
|
225
|
+
# Reference should always be an enterprise model.
|
|
226
|
+
reference = self._create_rule_components(
|
|
227
|
+
ref_model,
|
|
228
|
+
ref_schema,
|
|
229
|
+
self.ref_metadata or self._create_default_metadata(list(ref_schema.views.values()), is_ref=True),
|
|
230
|
+
DataModelType.enterprise,
|
|
231
|
+
)
|
|
232
|
+
data_model_type = DataModelType.solution
|
|
233
|
+
|
|
234
|
+
user_rules = self._create_rule_components(
|
|
235
|
+
model,
|
|
236
|
+
self.root_schema,
|
|
237
|
+
self.metadata,
|
|
238
|
+
data_model_type,
|
|
239
|
+
schema_completeness,
|
|
240
|
+
has_reference=reference is not None,
|
|
241
|
+
)
|
|
242
|
+
user_rules.reference = reference
|
|
258
243
|
|
|
259
|
-
return
|
|
244
|
+
return ReadRules(user_rules, self.issue_list, {})
|
|
260
245
|
|
|
261
246
|
def _create_rule_components(
|
|
262
247
|
self,
|
|
263
248
|
data_model: dm.DataModelApply,
|
|
264
249
|
schema: DMSSchema,
|
|
265
|
-
metadata:
|
|
250
|
+
metadata: DMSInputMetadata | None = None,
|
|
266
251
|
data_model_type: DataModelType | None = None,
|
|
267
252
|
schema_completeness: SchemaCompleteness | None = None,
|
|
268
253
|
has_reference: bool = False,
|
|
269
|
-
) ->
|
|
270
|
-
properties =
|
|
254
|
+
) -> DMSInputRules:
|
|
255
|
+
properties: list[DMSInputProperty] = []
|
|
271
256
|
for view_id, view in schema.views.items():
|
|
272
257
|
view_entity = ViewEntity.from_id(view_id)
|
|
273
258
|
class_entity = view_entity.as_class()
|
|
@@ -280,44 +265,47 @@ class DMSImporter(BaseImporter):
|
|
|
280
265
|
view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
|
|
281
266
|
}
|
|
282
267
|
|
|
283
|
-
metadata = metadata or
|
|
268
|
+
metadata = metadata or DMSInputMetadata.from_data_model(data_model, has_reference)
|
|
284
269
|
if data_model_type is not None:
|
|
285
|
-
metadata.data_model_type = data_model_type
|
|
270
|
+
metadata.data_model_type = str(data_model_type) # type: ignore[assignment]
|
|
286
271
|
if schema_completeness is not None:
|
|
287
|
-
metadata.schema_ = schema_completeness
|
|
288
|
-
|
|
272
|
+
metadata.schema_ = str(schema_completeness) # type: ignore[assignment]
|
|
273
|
+
|
|
274
|
+
enum = self._create_enum_collections(schema.containers.values())
|
|
275
|
+
|
|
276
|
+
return DMSInputRules(
|
|
289
277
|
metadata=metadata,
|
|
290
278
|
properties=properties,
|
|
291
|
-
containers=
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
]
|
|
299
|
-
),
|
|
279
|
+
containers=[DMSInputContainer.from_container(container) for container in schema.containers.values()],
|
|
280
|
+
views=[
|
|
281
|
+
DMSInputView.from_view(view, in_model=view_id in data_model_view_ids)
|
|
282
|
+
for view_id, view in schema.views.items()
|
|
283
|
+
],
|
|
284
|
+
nodes=[DMSInputNode.from_node_type(node_type) for node_type in schema.node_types.values()],
|
|
285
|
+
enum=enum,
|
|
300
286
|
)
|
|
301
287
|
|
|
302
288
|
@classmethod
|
|
303
|
-
def _create_default_metadata(
|
|
289
|
+
def _create_default_metadata(
|
|
290
|
+
cls, views: Sequence[dm.View | dm.ViewApply], is_ref: bool = False
|
|
291
|
+
) -> DMSInputMetadata:
|
|
304
292
|
now = datetime.now().replace(microsecond=0)
|
|
305
293
|
space = Counter(view.space for view in views).most_common(1)[0][0]
|
|
306
|
-
return
|
|
307
|
-
schema_=
|
|
308
|
-
extension=
|
|
309
|
-
data_model_type=
|
|
294
|
+
return DMSInputMetadata(
|
|
295
|
+
schema_="complete",
|
|
296
|
+
extension="addition",
|
|
297
|
+
data_model_type="enterprise" if is_ref else "solution",
|
|
310
298
|
space=space,
|
|
311
299
|
external_id="Unknown",
|
|
312
300
|
version="0.1.0",
|
|
313
|
-
creator=
|
|
301
|
+
creator="Unknown",
|
|
314
302
|
created=now,
|
|
315
303
|
updated=now,
|
|
316
304
|
)
|
|
317
305
|
|
|
318
306
|
def _create_dms_property(
|
|
319
307
|
self, prop_id: str, prop: ViewPropertyApply, view_entity: ViewEntity, class_entity: ClassEntity
|
|
320
|
-
) ->
|
|
308
|
+
) -> DMSInputProperty | None:
|
|
321
309
|
if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._all_containers_by_id:
|
|
322
310
|
self.issue_list.append(
|
|
323
311
|
ResourceNotFoundWarning[dm.ContainerId, dm.PropertyId](
|
|
@@ -353,20 +341,22 @@ class DMSImporter(BaseImporter):
|
|
|
353
341
|
if value_type is None:
|
|
354
342
|
return None
|
|
355
343
|
|
|
356
|
-
return
|
|
357
|
-
class_=class_entity,
|
|
344
|
+
return DMSInputProperty(
|
|
345
|
+
class_=str(class_entity),
|
|
358
346
|
property_=prop_id,
|
|
359
347
|
description=prop.description,
|
|
360
348
|
name=prop.name,
|
|
361
|
-
connection=self.
|
|
362
|
-
value_type=value_type,
|
|
349
|
+
connection=self._get_connection_type(prop_id, prop, view_entity.as_id()),
|
|
350
|
+
value_type=str(value_type),
|
|
363
351
|
is_list=self._get_is_list(prop),
|
|
364
352
|
nullable=self._get_nullable(prop),
|
|
365
353
|
immutable=self._get_immutable(prop),
|
|
366
354
|
default=self._get_default(prop),
|
|
367
|
-
container=ContainerEntity.from_id(prop.container)
|
|
355
|
+
container=str(ContainerEntity.from_id(prop.container))
|
|
356
|
+
if isinstance(prop, dm.MappedPropertyApply)
|
|
357
|
+
else None,
|
|
368
358
|
container_property=prop.container_property_identifier if isinstance(prop, dm.MappedPropertyApply) else None,
|
|
369
|
-
view=view_entity,
|
|
359
|
+
view=str(view_entity),
|
|
370
360
|
view_property=prop_id,
|
|
371
361
|
index=self._get_index(prop, prop_id),
|
|
372
362
|
constraint=self._get_constraint(prop, prop_id),
|
|
@@ -376,13 +366,22 @@ class DMSImporter(BaseImporter):
|
|
|
376
366
|
"""This method assumes you have already checked that the container with property exists."""
|
|
377
367
|
return self._all_containers_by_id[prop.container].properties[prop.container_property_identifier]
|
|
378
368
|
|
|
379
|
-
def
|
|
369
|
+
def _get_connection_type(
|
|
370
|
+
self, prop_id: str, prop: ViewPropertyApply, view_id: dm.ViewId
|
|
371
|
+
) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
|
|
380
372
|
if isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "outwards":
|
|
381
|
-
|
|
373
|
+
properties = ViewEntity.from_id(prop.edge_source) if prop.edge_source is not None else None
|
|
374
|
+
return EdgeEntity(properties=properties, type=DMSNodeEntity.from_reference(prop.type), direction="outwards")
|
|
382
375
|
elif isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "inwards":
|
|
383
|
-
|
|
376
|
+
if reverse_prop := self._find_reverse_edge(prop_id, prop, view_id):
|
|
377
|
+
return ReverseConnectionEntity(property=reverse_prop)
|
|
378
|
+
else:
|
|
379
|
+
properties = ViewEntity.from_id(prop.source) if prop.edge_source is not None else None
|
|
380
|
+
return EdgeEntity(
|
|
381
|
+
properties=properties, type=DMSNodeEntity.from_reference(prop.type), direction="inwards"
|
|
382
|
+
)
|
|
384
383
|
elif isinstance(prop, SingleReverseDirectRelationApply | MultiReverseDirectRelationApply):
|
|
385
|
-
return
|
|
384
|
+
return ReverseConnectionEntity(property=prop.through.property)
|
|
386
385
|
elif isinstance(prop, dm.MappedPropertyApply) and isinstance(
|
|
387
386
|
self._container_prop_unsafe(prop).type, dm.DirectRelation
|
|
388
387
|
):
|
|
@@ -392,21 +391,26 @@ class DMSImporter(BaseImporter):
|
|
|
392
391
|
|
|
393
392
|
def _get_value_type(
|
|
394
393
|
self, prop: ViewPropertyApply, view_entity: ViewEntity, prop_id
|
|
395
|
-
) -> DataType | ViewEntity |
|
|
396
|
-
if isinstance(
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
394
|
+
) -> DataType | ViewEntity | DMSUnknownEntity | None:
|
|
395
|
+
if isinstance(
|
|
396
|
+
prop,
|
|
397
|
+
SingleEdgeConnectionApply
|
|
398
|
+
| MultiEdgeConnectionApply
|
|
399
|
+
| SingleReverseDirectRelationApply
|
|
400
|
+
| MultiReverseDirectRelationApply,
|
|
401
|
+
):
|
|
401
402
|
return ViewEntity.from_id(prop.source)
|
|
402
403
|
elif isinstance(prop, dm.MappedPropertyApply):
|
|
403
404
|
container_prop = self._container_prop_unsafe(cast(dm.MappedPropertyApply, prop))
|
|
404
405
|
if isinstance(container_prop.type, dm.DirectRelation):
|
|
405
|
-
if prop.source is None or prop.source not in self.
|
|
406
|
-
# The warning is issued when the DMS Rules are created.
|
|
406
|
+
if prop.source is None or prop.source not in self._all_views_by_id:
|
|
407
407
|
return DMSUnknownEntity()
|
|
408
408
|
else:
|
|
409
409
|
return ViewEntity.from_id(prop.source)
|
|
410
|
+
elif isinstance(container_prop.type, PropertyTypeWithUnit) and container_prop.type.unit:
|
|
411
|
+
return DataType.load(f"{container_prop.type._type}(unit={container_prop.type.unit.external_id})")
|
|
412
|
+
elif isinstance(container_prop.type, DMSEnum):
|
|
413
|
+
return Enum(collection=ClassEntity(suffix=prop_id), unknownValue=container_prop.type.unknown_value)
|
|
410
414
|
else:
|
|
411
415
|
return DataType.load(container_prop.type._type)
|
|
412
416
|
else:
|
|
@@ -476,3 +480,55 @@ class DMSImporter(BaseImporter):
|
|
|
476
480
|
)
|
|
477
481
|
)
|
|
478
482
|
return unique_constraints or None
|
|
483
|
+
|
|
484
|
+
def _find_reverse_edge(
|
|
485
|
+
self, prop_id: str, prop: SingleEdgeConnectionApply | MultiEdgeConnectionApply, view_id: dm.ViewId
|
|
486
|
+
) -> str | None:
|
|
487
|
+
if prop.source not in self._all_views_by_id:
|
|
488
|
+
return None
|
|
489
|
+
view = self._all_views_by_id[prop.source]
|
|
490
|
+
candidates = []
|
|
491
|
+
for prop_name, reverse_prop in (view.properties or {}).items():
|
|
492
|
+
if isinstance(reverse_prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply):
|
|
493
|
+
if (
|
|
494
|
+
reverse_prop.type == prop.type
|
|
495
|
+
and reverse_prop.source == view_id
|
|
496
|
+
and reverse_prop.direction != prop.direction
|
|
497
|
+
):
|
|
498
|
+
candidates.append(prop_name)
|
|
499
|
+
if len(candidates) == 0:
|
|
500
|
+
self.issue_list.append(
|
|
501
|
+
PropertyNotFoundWarning(
|
|
502
|
+
prop.source,
|
|
503
|
+
"view property",
|
|
504
|
+
f"reverse edge of {prop_id}",
|
|
505
|
+
dm.PropertyId(view_id, prop_id),
|
|
506
|
+
"view property",
|
|
507
|
+
)
|
|
508
|
+
)
|
|
509
|
+
return None
|
|
510
|
+
if len(candidates) > 1:
|
|
511
|
+
self.issue_list.append(
|
|
512
|
+
ResourcesDuplicatedWarning(
|
|
513
|
+
frozenset(dm.PropertyId(view.as_id(), candidate) for candidate in candidates),
|
|
514
|
+
"view property",
|
|
515
|
+
default_action="Multiple reverse edges found for "
|
|
516
|
+
f"{dm.PropertyId(view_id, prop_id)!r}. Will use {candidates[0]}",
|
|
517
|
+
)
|
|
518
|
+
)
|
|
519
|
+
|
|
520
|
+
return candidates[0]
|
|
521
|
+
|
|
522
|
+
@staticmethod
|
|
523
|
+
def _create_enum_collections(containers: Collection[dm.ContainerApply]) -> list[DMSInputEnum] | None:
|
|
524
|
+
enum_collections: list[DMSInputEnum] = []
|
|
525
|
+
for container in containers:
|
|
526
|
+
for prop_id, prop in container.properties.items():
|
|
527
|
+
if isinstance(prop.type, DMSEnum):
|
|
528
|
+
for identifier, value in prop.type.values.items():
|
|
529
|
+
enum_collections.append(
|
|
530
|
+
DMSInputEnum(
|
|
531
|
+
collection=prop_id, value=identifier, name=value.name, description=value.description
|
|
532
|
+
)
|
|
533
|
+
)
|
|
534
|
+
return enum_collections
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from collections import Counter
|
|
2
2
|
from collections.abc import Callable, Sequence
|
|
3
3
|
|
|
4
|
-
from cognite.neat.issues import IssueList
|
|
4
|
+
from cognite.neat.issues import IssueList
|
|
5
5
|
from cognite.neat.issues.errors import (
|
|
6
6
|
PropertyTypeNotSupportedError,
|
|
7
7
|
ResourceMissingIdentifierError,
|
|
@@ -27,17 +27,20 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
|
27
27
|
)
|
|
28
28
|
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
|
|
29
29
|
from cognite.neat.rules.models.entities import ClassEntity
|
|
30
|
-
from cognite.neat.rules.models.information import
|
|
30
|
+
from cognite.neat.rules.models.information import (
|
|
31
|
+
InformationInputClass,
|
|
32
|
+
InformationInputProperty,
|
|
33
|
+
)
|
|
31
34
|
|
|
32
35
|
|
|
33
36
|
class _DTDLConverter:
|
|
34
|
-
def __init__(self, issues:
|
|
37
|
+
def __init__(self, issues: IssueList | None = None) -> None:
|
|
35
38
|
self.issues = IssueList(issues or [])
|
|
36
|
-
self.properties: list[
|
|
37
|
-
self.classes: list[
|
|
39
|
+
self.properties: list[InformationInputProperty] = []
|
|
40
|
+
self.classes: list[InformationInputClass] = []
|
|
38
41
|
self._item_by_id: dict[DTMI, DTDLBase] = {}
|
|
39
42
|
|
|
40
|
-
self._method_by_type
|
|
43
|
+
self._method_by_type = {
|
|
41
44
|
Interface: self.convert_interface, # type: ignore[dict-item]
|
|
42
45
|
Property: self.convert_property, # type: ignore[dict-item]
|
|
43
46
|
PropertyV2: self.convert_property, # type: ignore[dict-item]
|
|
@@ -53,7 +56,11 @@ class _DTDLConverter:
|
|
|
53
56
|
def get_most_common_prefix(self) -> str:
|
|
54
57
|
if not self.classes:
|
|
55
58
|
raise ValueError("No classes found")
|
|
56
|
-
counted = Counter(
|
|
59
|
+
counted = Counter(
|
|
60
|
+
class_.prefix
|
|
61
|
+
for class_ in (cls_.class_ for cls_ in self.classes)
|
|
62
|
+
if isinstance(class_, ClassEntity) and isinstance(class_.prefix, str)
|
|
63
|
+
)
|
|
57
64
|
if not counted:
|
|
58
65
|
raise ValueError("No prefixes found")
|
|
59
66
|
return counted.most_common(1)[0][0]
|
|
@@ -75,7 +82,8 @@ class _DTDLConverter:
|
|
|
75
82
|
self.convert_item(item)
|
|
76
83
|
|
|
77
84
|
def convert_item(self, item: DTDLBase, parent: str | None = None) -> None:
|
|
78
|
-
|
|
85
|
+
# Bug in mypy https://github.com/python/mypy/issues/17478
|
|
86
|
+
convert_method: Callable[[DTDLBase, str | None], None] | None = self._method_by_type.get(type(item)) # type: ignore[assignment]
|
|
79
87
|
if convert_method is not None:
|
|
80
88
|
convert_method(item, parent)
|
|
81
89
|
else:
|
|
@@ -87,7 +95,7 @@ class _DTDLConverter:
|
|
|
87
95
|
)
|
|
88
96
|
|
|
89
97
|
def convert_interface(self, item: Interface, _: str | None) -> None:
|
|
90
|
-
class_ =
|
|
98
|
+
class_ = InformationInputClass(
|
|
91
99
|
class_=item.id_.as_class_id(),
|
|
92
100
|
name=item.display_name,
|
|
93
101
|
description=item.description,
|
|
@@ -107,9 +115,9 @@ class _DTDLConverter:
|
|
|
107
115
|
)
|
|
108
116
|
elif isinstance(sub_item_or_id, DTMI):
|
|
109
117
|
sub_item = self._item_by_id[sub_item_or_id]
|
|
110
|
-
self.convert_item(sub_item, class_.
|
|
118
|
+
self.convert_item(sub_item, class_.class_str)
|
|
111
119
|
else:
|
|
112
|
-
self.convert_item(sub_item_or_id, class_.
|
|
120
|
+
self.convert_item(sub_item_or_id, class_.class_str)
|
|
113
121
|
# interface.schema objects are handled in the convert method
|
|
114
122
|
|
|
115
123
|
def convert_property(
|
|
@@ -122,7 +130,7 @@ class _DTDLConverter:
|
|
|
122
130
|
if value_type is None:
|
|
123
131
|
return None
|
|
124
132
|
|
|
125
|
-
prop =
|
|
133
|
+
prop = InformationInputProperty(
|
|
126
134
|
class_=ClassEntity.load(parent),
|
|
127
135
|
property_=item.name,
|
|
128
136
|
name=item.display_name,
|
|
@@ -168,7 +176,7 @@ class _DTDLConverter:
|
|
|
168
176
|
value_type = self.schema_to_value_type(item.request.schema_, item)
|
|
169
177
|
if value_type is None:
|
|
170
178
|
return
|
|
171
|
-
prop =
|
|
179
|
+
prop = InformationInputProperty(
|
|
172
180
|
class_=ClassEntity.load(parent),
|
|
173
181
|
property_=item.name,
|
|
174
182
|
name=item.display_name,
|
|
@@ -188,7 +196,7 @@ class _DTDLConverter:
|
|
|
188
196
|
value_type = self.schema_to_value_type(item.schema_, item)
|
|
189
197
|
if value_type is None:
|
|
190
198
|
return
|
|
191
|
-
prop =
|
|
199
|
+
prop = InformationInputProperty(
|
|
192
200
|
class_=ClassEntity.load(parent),
|
|
193
201
|
property_=item.name,
|
|
194
202
|
name=item.display_name,
|
|
@@ -218,7 +226,7 @@ class _DTDLConverter:
|
|
|
218
226
|
)
|
|
219
227
|
value_type = Json()
|
|
220
228
|
|
|
221
|
-
prop =
|
|
229
|
+
prop = InformationInputProperty(
|
|
222
230
|
class_=ClassEntity.load(parent),
|
|
223
231
|
property_=item.name,
|
|
224
232
|
name=item.display_name,
|
|
@@ -243,7 +251,7 @@ class _DTDLConverter:
|
|
|
243
251
|
)
|
|
244
252
|
return None
|
|
245
253
|
|
|
246
|
-
class_ =
|
|
254
|
+
class_ = InformationInputClass(
|
|
247
255
|
class_=item.id_.as_class_id(),
|
|
248
256
|
name=item.display_name,
|
|
249
257
|
description=item.description,
|
|
@@ -255,7 +263,7 @@ class _DTDLConverter:
|
|
|
255
263
|
value_type = self.schema_to_value_type(field_.schema_, item)
|
|
256
264
|
if value_type is None:
|
|
257
265
|
continue
|
|
258
|
-
prop =
|
|
266
|
+
prop = InformationInputProperty(
|
|
259
267
|
class_=class_.class_,
|
|
260
268
|
name=field_.name,
|
|
261
269
|
description=field_.description,
|
|
@@ -297,7 +305,7 @@ class _DTDLConverter:
|
|
|
297
305
|
else:
|
|
298
306
|
if isinstance(input_type, Object):
|
|
299
307
|
self.convert_object(input_type, None)
|
|
300
|
-
return
|
|
308
|
+
return input_type.id_.as_class_id()
|
|
301
309
|
else:
|
|
302
310
|
self.issues.append(
|
|
303
311
|
PropertyTypeNotSupportedWarning(
|
|
@@ -2,7 +2,6 @@ import json
|
|
|
2
2
|
import zipfile
|
|
3
3
|
from collections.abc import Iterable, Sequence
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Literal, overload
|
|
6
5
|
|
|
7
6
|
from pydantic import ValidationError
|
|
8
7
|
|
|
@@ -14,16 +13,16 @@ from cognite.neat.issues.warnings import (
|
|
|
14
13
|
FileTypeUnexpectedWarning,
|
|
15
14
|
NeatValueWarning,
|
|
16
15
|
)
|
|
17
|
-
from cognite.neat.rules._shared import
|
|
18
|
-
from cognite.neat.rules.importers._base import BaseImporter
|
|
16
|
+
from cognite.neat.rules._shared import ReadRules
|
|
17
|
+
from cognite.neat.rules.importers._base import BaseImporter
|
|
19
18
|
from cognite.neat.rules.importers._dtdl2rules.dtdl_converter import _DTDLConverter
|
|
20
19
|
from cognite.neat.rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
|
|
21
|
-
from cognite.neat.rules.models import
|
|
22
|
-
from cognite.neat.rules.models.information import
|
|
20
|
+
from cognite.neat.rules.models import InformationInputRules, SchemaCompleteness
|
|
21
|
+
from cognite.neat.rules.models.information import InformationInputMetadata
|
|
23
22
|
from cognite.neat.utils.text import humanize_collection, to_pascal
|
|
24
23
|
|
|
25
24
|
|
|
26
|
-
class DTDLImporter(BaseImporter):
|
|
25
|
+
class DTDLImporter(BaseImporter[InformationInputRules]):
|
|
27
26
|
"""Importer from Azure Digital Twin - DTDL (Digital Twin Definition Language).
|
|
28
27
|
|
|
29
28
|
This importer supports DTDL v2.0 and v3.0.
|
|
@@ -47,7 +46,7 @@ class DTDLImporter(BaseImporter):
|
|
|
47
46
|
) -> None:
|
|
48
47
|
self._items = items
|
|
49
48
|
self.title = title
|
|
50
|
-
self._read_issues = read_issues
|
|
49
|
+
self._read_issues = IssueList(read_issues)
|
|
51
50
|
self._schema_completeness = schema
|
|
52
51
|
|
|
53
52
|
@classmethod
|
|
@@ -125,17 +124,7 @@ class DTDLImporter(BaseImporter):
|
|
|
125
124
|
items.append(item)
|
|
126
125
|
return cls(items, zip_file.stem, read_issues=issues)
|
|
127
126
|
|
|
128
|
-
|
|
129
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
|
|
130
|
-
|
|
131
|
-
@overload
|
|
132
|
-
def to_rules(
|
|
133
|
-
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
134
|
-
) -> tuple[VerifiedRules | None, IssueList]: ...
|
|
135
|
-
|
|
136
|
-
def to_rules(
|
|
137
|
-
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
138
|
-
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
127
|
+
def to_rules(self) -> ReadRules[InformationInputRules]:
|
|
139
128
|
converter = _DTDLConverter(self._read_issues)
|
|
140
129
|
|
|
141
130
|
converter.convert(self._items)
|
|
@@ -152,16 +141,11 @@ class DTDLImporter(BaseImporter):
|
|
|
152
141
|
...
|
|
153
142
|
else:
|
|
154
143
|
metadata["prefix"] = most_common_prefix
|
|
155
|
-
with _handle_issues(converter.issues) as future:
|
|
156
|
-
rules = InformationRules(
|
|
157
|
-
metadata=metadata,
|
|
158
|
-
properties=SheetList[InformationProperty](data=converter.properties),
|
|
159
|
-
classes=SheetList[InformationClass](data=converter.classes),
|
|
160
|
-
)
|
|
161
|
-
if future.result == "failure":
|
|
162
|
-
if errors == "continue":
|
|
163
|
-
return None, converter.issues
|
|
164
|
-
else:
|
|
165
|
-
raise converter.issues.as_errors()
|
|
166
144
|
|
|
167
|
-
|
|
145
|
+
rules = InformationInputRules(
|
|
146
|
+
metadata=InformationInputMetadata.load(metadata),
|
|
147
|
+
properties=converter.properties,
|
|
148
|
+
classes=converter.classes,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
return ReadRules(rules, converter.issues, {})
|