cognite-neat 0.121.1__py3-none-any.whl → 0.122.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/_data_model/_constants.py +23 -16
- cognite/neat/core/_data_model/_shared.py +33 -17
- cognite/neat/core/_data_model/analysis/__init__.py +2 -2
- cognite/neat/core/_data_model/analysis/_base.py +186 -183
- cognite/neat/core/_data_model/catalog/__init__.py +2 -2
- cognite/neat/core/_data_model/exporters/__init__.py +6 -6
- cognite/neat/core/_data_model/exporters/_base.py +10 -8
- cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
- cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
- cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +14 -14
- cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
- cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
- cognite/neat/core/_data_model/importers/__init__.py +8 -8
- cognite/neat/core/_data_model/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +50 -25
- cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/__init__.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/_base.py +15 -15
- cognite/neat/core/_data_model/importers/_rdf/{_imf2rules.py → _imf2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/{_inference2rules.py → _inference2rdata_model.py} +59 -59
- cognite/neat/core/_data_model/importers/_rdf/{_owl2rules.py → _owl2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +76 -19
- cognite/neat/core/_data_model/models/__init__.py +11 -9
- cognite/neat/core/_data_model/models/_base_unverified.py +12 -12
- cognite/neat/core/_data_model/models/_base_verified.py +9 -14
- cognite/neat/core/_data_model/models/_types.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
- cognite/neat/core/_data_model/models/conceptual/_validation.py +88 -78
- cognite/neat/core/_data_model/models/conceptual/_verified.py +54 -52
- cognite/neat/core/_data_model/models/data_types.py +2 -2
- cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
- cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
- cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
- cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
- cognite/neat/core/_data_model/models/entities/_types.py +9 -3
- cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
- cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +75 -55
- cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
- cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +17 -15
- cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
- cognite/neat/core/_data_model/transformers/__init__.py +29 -25
- cognite/neat/core/_data_model/transformers/_base.py +27 -20
- cognite/neat/core/_data_model/transformers/_converters.py +707 -622
- cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
- cognite/neat/core/_data_model/transformers/_verification.py +64 -55
- cognite/neat/core/_instances/extractors/_base.py +2 -2
- cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +9 -9
- cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
- cognite/neat/core/_instances/extractors/_mock_graph_generator.py +107 -103
- cognite/neat/core/_instances/loaders/_base.py +3 -3
- cognite/neat/core/_instances/loaders/_rdf2dms.py +22 -22
- cognite/neat/core/_instances/transformers/_base.py +7 -4
- cognite/neat/core/_instances/transformers/_rdfpath.py +1 -1
- cognite/neat/core/_instances/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/_factory.py +1 -1
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_resources.py +1 -1
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_issues/warnings/_properties.py +1 -1
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +2 -2
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/text.py +1 -1
- cognite/neat/session/_base.py +42 -36
- cognite/neat/session/_drop.py +2 -2
- cognite/neat/session/_experimental.py +1 -1
- cognite/neat/session/_inspect.py +13 -13
- cognite/neat/session/_mapping.py +15 -9
- cognite/neat/session/_read.py +39 -37
- cognite/neat/session/_set.py +6 -6
- cognite/neat/session/_show.py +24 -21
- cognite/neat/session/_state/README.md +1 -1
- cognite/neat/session/_state.py +27 -27
- cognite/neat/session/_subset.py +14 -11
- cognite/neat/session/_template.py +23 -21
- cognite/neat/session/_to.py +42 -42
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/METADATA +14 -7
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/RECORD +102 -100
- cognite/neat/core/_data_model/exporters/_validation.py +0 -14
- cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
- /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -27,14 +27,14 @@ from cognite.neat.core._constants import (
|
|
|
27
27
|
)
|
|
28
28
|
from cognite.neat.core._data_model.models.data_types import DataType, Double, Enum, Float
|
|
29
29
|
from cognite.neat.core._data_model.models.entities import (
|
|
30
|
-
|
|
30
|
+
ConceptEntity,
|
|
31
31
|
ContainerEntity,
|
|
32
32
|
DMSFilter,
|
|
33
33
|
DMSNodeEntity,
|
|
34
|
-
DMSUnknownEntity,
|
|
35
34
|
EdgeEntity,
|
|
36
35
|
HasDataFilter,
|
|
37
36
|
NodeTypeFilter,
|
|
37
|
+
PhysicalUnknownEntity,
|
|
38
38
|
ReverseConnectionEntity,
|
|
39
39
|
UnitEntity,
|
|
40
40
|
ViewEntity,
|
|
@@ -53,45 +53,57 @@ from cognite.neat.core._issues.warnings.user_modeling import (
|
|
|
53
53
|
HasDataFilterOnNoPropertiesViewWarning,
|
|
54
54
|
)
|
|
55
55
|
|
|
56
|
-
from .
|
|
56
|
+
from ._verified import (
|
|
57
|
+
PhysicalDataModel,
|
|
58
|
+
PhysicalEnum,
|
|
59
|
+
PhysicalMetadata,
|
|
60
|
+
PhysicalProperty,
|
|
61
|
+
PhysicalView,
|
|
62
|
+
)
|
|
57
63
|
|
|
58
64
|
|
|
59
65
|
class _DMSExporter:
|
|
60
|
-
"""The DMS Exporter is responsible for exporting the
|
|
66
|
+
"""The DMS Exporter is responsible for exporting the physical data model to a DMSSchema.
|
|
61
67
|
|
|
62
|
-
This kept in this location such that it can be used by the
|
|
68
|
+
This kept in this location such that it can be used by the physical data model to validate the schema.
|
|
63
69
|
(This module cannot have a dependency on the exporter module, as it would create a circular dependency.)
|
|
64
70
|
|
|
65
71
|
Args
|
|
66
72
|
include_pipeline (bool): If True, the pipeline will be included with the schema. Pipeline means the
|
|
67
73
|
raw tables and transformations necessary to populate the data model.
|
|
68
|
-
instance_space (str): The space to use for the instance. Defaults to None,`
|
|
74
|
+
instance_space (str): The space to use for the instance. Defaults to None,`
|
|
75
|
+
PhysicalDataModel.metadata.space` will be used
|
|
69
76
|
remove_cdf_spaces(bool): The
|
|
70
77
|
"""
|
|
71
78
|
|
|
72
|
-
def __init__(
|
|
79
|
+
def __init__(
|
|
80
|
+
self,
|
|
81
|
+
data_model: PhysicalDataModel,
|
|
82
|
+
instance_space: str | None = None,
|
|
83
|
+
remove_cdf_spaces: bool = False,
|
|
84
|
+
):
|
|
73
85
|
self.instance_space = instance_space
|
|
74
|
-
self.
|
|
86
|
+
self.data_model = data_model
|
|
75
87
|
self.remove_cdf_spaces = remove_cdf_spaces
|
|
76
88
|
|
|
77
89
|
def to_schema(self) -> DMSSchema:
|
|
78
|
-
|
|
79
|
-
container_properties_by_id, view_properties_by_id = self._gather_properties(list(self.
|
|
90
|
+
data_model = self.data_model
|
|
91
|
+
container_properties_by_id, view_properties_by_id = self._gather_properties(list(self.data_model.properties))
|
|
80
92
|
|
|
81
|
-
containers = self._create_containers(container_properties_by_id,
|
|
93
|
+
containers = self._create_containers(container_properties_by_id, data_model.enum) # type: ignore[arg-type]
|
|
82
94
|
|
|
83
95
|
view_properties_with_ancestors_by_id = self._gather_properties_with_ancestors(
|
|
84
|
-
view_properties_by_id,
|
|
96
|
+
view_properties_by_id, data_model.views
|
|
85
97
|
)
|
|
86
98
|
|
|
87
99
|
views = self._create_views(view_properties_by_id, view_properties_with_ancestors_by_id)
|
|
88
100
|
view_node_type_filters: set[dm.NodeId] = set()
|
|
89
|
-
for dms_view in
|
|
101
|
+
for dms_view in data_model.views:
|
|
90
102
|
if isinstance(dms_view.filter_, NodeTypeFilter):
|
|
91
103
|
view_node_type_filters.update(node.as_id() for node in dms_view.filter_.inner or [])
|
|
92
|
-
if
|
|
104
|
+
if data_model.nodes:
|
|
93
105
|
node_types = NodeApplyDict(
|
|
94
|
-
[node.as_node() for node in
|
|
106
|
+
[node.as_node() for node in data_model.nodes]
|
|
95
107
|
+ [dm.NodeApply(node.space, node.external_id) for node in view_node_type_filters]
|
|
96
108
|
)
|
|
97
109
|
else:
|
|
@@ -103,17 +115,17 @@ class _DMSExporter:
|
|
|
103
115
|
]
|
|
104
116
|
)
|
|
105
117
|
|
|
106
|
-
|
|
118
|
+
dms_data_model = data_model.metadata.as_data_model()
|
|
107
119
|
# Sorting to ensure deterministic order
|
|
108
|
-
|
|
109
|
-
[dms_view.view.as_id() for dms_view in
|
|
120
|
+
dms_data_model.views = sorted(
|
|
121
|
+
[dms_view.view.as_id() for dms_view in data_model.views if dms_view.in_model],
|
|
110
122
|
key=lambda x: x.as_tuple(), # type: ignore[union-attr]
|
|
111
123
|
)
|
|
112
|
-
spaces = self._create_spaces(
|
|
124
|
+
spaces = self._create_spaces(data_model.metadata, containers, views, dms_data_model)
|
|
113
125
|
|
|
114
126
|
return DMSSchema(
|
|
115
127
|
spaces=spaces,
|
|
116
|
-
data_model=
|
|
128
|
+
data_model=dms_data_model,
|
|
117
129
|
views=views,
|
|
118
130
|
containers=containers,
|
|
119
131
|
node_types=node_types,
|
|
@@ -121,7 +133,7 @@ class _DMSExporter:
|
|
|
121
133
|
|
|
122
134
|
def _create_spaces(
|
|
123
135
|
self,
|
|
124
|
-
metadata:
|
|
136
|
+
metadata: PhysicalMetadata,
|
|
125
137
|
containers: ContainerApplyDict,
|
|
126
138
|
views: ViewApplyDict,
|
|
127
139
|
data_model: dm.DataModelApply,
|
|
@@ -140,10 +152,10 @@ class _DMSExporter:
|
|
|
140
152
|
|
|
141
153
|
def _create_views(
|
|
142
154
|
self,
|
|
143
|
-
view_properties_by_id: dict[dm.ViewId, list[
|
|
144
|
-
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[
|
|
155
|
+
view_properties_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
156
|
+
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
145
157
|
) -> ViewApplyDict:
|
|
146
|
-
input_views = list(self.
|
|
158
|
+
input_views = list(self.data_model.views)
|
|
147
159
|
|
|
148
160
|
views = ViewApplyDict(
|
|
149
161
|
[
|
|
@@ -172,7 +184,7 @@ class _DMSExporter:
|
|
|
172
184
|
return views
|
|
173
185
|
|
|
174
186
|
@classmethod
|
|
175
|
-
def _create_edge_type_from_prop(cls, prop:
|
|
187
|
+
def _create_edge_type_from_prop(cls, prop: PhysicalProperty) -> dm.DirectRelationReference:
|
|
176
188
|
if isinstance(prop.connection, EdgeEntity) and prop.connection.edge_type is not None:
|
|
177
189
|
return prop.connection.edge_type.as_reference()
|
|
178
190
|
elif isinstance(prop.value_type, ViewEntity):
|
|
@@ -191,10 +203,10 @@ class _DMSExporter:
|
|
|
191
203
|
@classmethod
|
|
192
204
|
def _edge_types_by_view_property_id(
|
|
193
205
|
cls,
|
|
194
|
-
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[
|
|
195
|
-
view_by_id: dict[ViewEntity,
|
|
206
|
+
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
207
|
+
view_by_id: dict[ViewEntity, PhysicalView],
|
|
196
208
|
) -> dict[tuple[ViewEntity, str], dm.DirectRelationReference]:
|
|
197
|
-
edge_connection_property_by_view_property_id: dict[tuple[ViewEntity, str],
|
|
209
|
+
edge_connection_property_by_view_property_id: dict[tuple[ViewEntity, str], PhysicalProperty] = {}
|
|
198
210
|
for properties in view_properties_with_ancestors_by_id.values():
|
|
199
211
|
for prop in properties:
|
|
200
212
|
if isinstance(prop.connection, EdgeEntity):
|
|
@@ -248,10 +260,10 @@ class _DMSExporter:
|
|
|
248
260
|
@classmethod
|
|
249
261
|
def _get_edge_type_outwards_connection(
|
|
250
262
|
cls,
|
|
251
|
-
view:
|
|
252
|
-
prop:
|
|
253
|
-
view_by_id: dict[ViewEntity,
|
|
254
|
-
edge_connection_by_view_property_id: dict[tuple[ViewEntity, str],
|
|
263
|
+
view: PhysicalView,
|
|
264
|
+
prop: PhysicalProperty,
|
|
265
|
+
view_by_id: dict[ViewEntity, PhysicalView],
|
|
266
|
+
edge_connection_by_view_property_id: dict[tuple[ViewEntity, str], PhysicalProperty],
|
|
255
267
|
) -> dm.DirectRelationReference:
|
|
256
268
|
connection = cast(EdgeEntity, prop.connection)
|
|
257
269
|
if connection.edge_type is not None:
|
|
@@ -284,14 +296,14 @@ class _DMSExporter:
|
|
|
284
296
|
|
|
285
297
|
def _create_containers(
|
|
286
298
|
self,
|
|
287
|
-
container_properties_by_id: dict[dm.ContainerId, list[
|
|
288
|
-
enum: Collection[
|
|
299
|
+
container_properties_by_id: dict[dm.ContainerId, list[PhysicalProperty]],
|
|
300
|
+
enum: Collection[PhysicalEnum] | None,
|
|
289
301
|
) -> ContainerApplyDict:
|
|
290
|
-
enum_values_by_collection: dict[
|
|
302
|
+
enum_values_by_collection: dict[ConceptEntity, list[PhysicalEnum]] = defaultdict(list)
|
|
291
303
|
for enum_value in enum or []:
|
|
292
304
|
enum_values_by_collection[enum_value.collection].append(enum_value)
|
|
293
305
|
|
|
294
|
-
containers = list(self.
|
|
306
|
+
containers = list(self.data_model.containers or [])
|
|
295
307
|
|
|
296
308
|
containers = dm.ContainerApplyList(
|
|
297
309
|
[
|
|
@@ -390,10 +402,13 @@ class _DMSExporter:
|
|
|
390
402
|
|
|
391
403
|
@staticmethod
|
|
392
404
|
def _gather_properties(
|
|
393
|
-
properties: Sequence[
|
|
394
|
-
) -> tuple[
|
|
395
|
-
|
|
396
|
-
|
|
405
|
+
properties: Sequence[PhysicalProperty],
|
|
406
|
+
) -> tuple[
|
|
407
|
+
dict[dm.ContainerId, list[PhysicalProperty]],
|
|
408
|
+
dict[dm.ViewId, list[PhysicalProperty]],
|
|
409
|
+
]:
|
|
410
|
+
container_properties_by_id: dict[dm.ContainerId, list[PhysicalProperty]] = defaultdict(list)
|
|
411
|
+
view_properties_by_id: dict[dm.ViewId, list[PhysicalProperty]] = defaultdict(list)
|
|
397
412
|
for prop in properties:
|
|
398
413
|
view_id = prop.view.as_id()
|
|
399
414
|
view_properties_by_id[view_id].append(prop)
|
|
@@ -406,12 +421,12 @@ class _DMSExporter:
|
|
|
406
421
|
|
|
407
422
|
def _gather_properties_with_ancestors(
|
|
408
423
|
self,
|
|
409
|
-
view_properties_by_id: dict[dm.ViewId, list[
|
|
410
|
-
views: Sequence[
|
|
411
|
-
) -> dict[dm.ViewId, list[
|
|
424
|
+
view_properties_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
425
|
+
views: Sequence[PhysicalView],
|
|
426
|
+
) -> dict[dm.ViewId, list[PhysicalProperty]]:
|
|
412
427
|
all_view_properties_by_id = view_properties_by_id.copy()
|
|
413
428
|
|
|
414
|
-
view_properties_with_parents_by_id: dict[dm.ViewId, list[
|
|
429
|
+
view_properties_with_parents_by_id: dict[dm.ViewId, list[PhysicalProperty]] = defaultdict(list)
|
|
415
430
|
view_by_view_id = {view.view.as_id(): view for view in views}
|
|
416
431
|
for view in views:
|
|
417
432
|
view_id = view.view.as_id()
|
|
@@ -446,9 +461,9 @@ class _DMSExporter:
|
|
|
446
461
|
@classmethod
|
|
447
462
|
def _update_with_properties(
|
|
448
463
|
cls,
|
|
449
|
-
selected_properties: Sequence[
|
|
450
|
-
container_properties_by_id: dict[dm.ContainerId, list[
|
|
451
|
-
view_properties_by_id: dict[dm.ViewId, list[
|
|
464
|
+
selected_properties: Sequence[PhysicalProperty],
|
|
465
|
+
container_properties_by_id: dict[dm.ContainerId, list[PhysicalProperty]],
|
|
466
|
+
view_properties_by_id: dict[dm.ViewId, list[PhysicalProperty]] | None,
|
|
452
467
|
include_new_containers: bool = False,
|
|
453
468
|
) -> None:
|
|
454
469
|
view_properties_by_id = view_properties_by_id or {}
|
|
@@ -472,7 +487,7 @@ class _DMSExporter:
|
|
|
472
487
|
def _create_view_filter(
|
|
473
488
|
self,
|
|
474
489
|
view: dm.ViewApply,
|
|
475
|
-
dms_view:
|
|
490
|
+
dms_view: PhysicalView | None,
|
|
476
491
|
) -> DMSFilter | None:
|
|
477
492
|
selected_filter_name = (dms_view and dms_view.filter_ and dms_view.filter_.name) or ""
|
|
478
493
|
|
|
@@ -497,8 +512,8 @@ class _DMSExporter:
|
|
|
497
512
|
@classmethod
|
|
498
513
|
def _create_view_property(
|
|
499
514
|
cls,
|
|
500
|
-
prop:
|
|
501
|
-
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[
|
|
515
|
+
prop: PhysicalProperty,
|
|
516
|
+
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
502
517
|
edge_types_by_view_property_id: dict[tuple[ViewEntity, str], dm.DirectRelationReference],
|
|
503
518
|
) -> ViewPropertyApply | None:
|
|
504
519
|
if prop.container and prop.container_property:
|
|
@@ -509,19 +524,20 @@ class _DMSExporter:
|
|
|
509
524
|
return cls._create_reverse_direct_relation(prop, view_properties_with_ancestors_by_id)
|
|
510
525
|
elif prop.view and prop.view_property and prop.connection:
|
|
511
526
|
warnings.warn(
|
|
512
|
-
NotSupportedWarning(f"{prop.connection} in {prop.view.as_id()!r}.{prop.view_property}"),
|
|
527
|
+
NotSupportedWarning(f"{prop.connection} in {prop.view.as_id()!r}.{prop.view_property}"),
|
|
528
|
+
stacklevel=2,
|
|
513
529
|
)
|
|
514
530
|
return None
|
|
515
531
|
|
|
516
532
|
@classmethod
|
|
517
|
-
def _create_mapped_property(cls, prop:
|
|
533
|
+
def _create_mapped_property(cls, prop: PhysicalProperty) -> dm.MappedPropertyApply:
|
|
518
534
|
container = cast(ContainerEntity, prop.container)
|
|
519
535
|
container_prop_identifier = cast(str, prop.container_property)
|
|
520
536
|
extra_args: dict[str, Any] = {}
|
|
521
537
|
if prop.connection == "direct":
|
|
522
538
|
if isinstance(prop.value_type, ViewEntity):
|
|
523
539
|
extra_args["source"] = prop.value_type.as_id()
|
|
524
|
-
elif isinstance(prop.value_type,
|
|
540
|
+
elif isinstance(prop.value_type, PhysicalUnknownEntity):
|
|
525
541
|
extra_args["source"] = None
|
|
526
542
|
else:
|
|
527
543
|
# Should have been validated.
|
|
@@ -545,7 +561,9 @@ class _DMSExporter:
|
|
|
545
561
|
|
|
546
562
|
@classmethod
|
|
547
563
|
def _create_edge_property(
|
|
548
|
-
cls,
|
|
564
|
+
cls,
|
|
565
|
+
prop: PhysicalProperty,
|
|
566
|
+
edge_types_by_view_property_id: dict[tuple[ViewEntity, str], dm.DirectRelationReference],
|
|
549
567
|
) -> dm.EdgeConnectionApply:
|
|
550
568
|
connection = cast(EdgeEntity, prop.connection)
|
|
551
569
|
if isinstance(prop.value_type, ViewEntity):
|
|
@@ -575,7 +593,9 @@ class _DMSExporter:
|
|
|
575
593
|
|
|
576
594
|
@classmethod
|
|
577
595
|
def _create_reverse_direct_relation(
|
|
578
|
-
cls,
|
|
596
|
+
cls,
|
|
597
|
+
prop: PhysicalProperty,
|
|
598
|
+
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
579
599
|
) -> dm.MultiReverseDirectRelationApply | SingleReverseDirectRelationApply | None:
|
|
580
600
|
connection = cast(ReverseConnectionEntity, prop.connection)
|
|
581
601
|
reverse_prop_id = connection.property_
|
|
@@ -22,8 +22,8 @@ from cognite.neat.core._data_model.models.data_types import DataType
|
|
|
22
22
|
from cognite.neat.core._data_model.models.entities import (
|
|
23
23
|
ContainerEntity,
|
|
24
24
|
DMSNodeEntity,
|
|
25
|
-
DMSUnknownEntity,
|
|
26
25
|
EdgeEntity,
|
|
26
|
+
PhysicalUnknownEntity,
|
|
27
27
|
ReverseConnectionEntity,
|
|
28
28
|
ViewEntity,
|
|
29
29
|
load_connection,
|
|
@@ -33,7 +33,16 @@ from cognite.neat.core._data_model.models.entities._wrapped import DMSFilter
|
|
|
33
33
|
from cognite.neat.core._issues.warnings import DeprecatedWarning
|
|
34
34
|
from cognite.neat.core._utils.rdf_ import uri_display_name
|
|
35
35
|
|
|
36
|
-
from .
|
|
36
|
+
from ._verified import (
|
|
37
|
+
_DEFAULT_VERSION,
|
|
38
|
+
PhysicalContainer,
|
|
39
|
+
PhysicalDataModel,
|
|
40
|
+
PhysicalEnum,
|
|
41
|
+
PhysicalMetadata,
|
|
42
|
+
PhysicalNodeType,
|
|
43
|
+
PhysicalProperty,
|
|
44
|
+
PhysicalView,
|
|
45
|
+
)
|
|
37
46
|
|
|
38
47
|
if sys.version_info >= (3, 11):
|
|
39
48
|
from typing import Self
|
|
@@ -42,7 +51,7 @@ else:
|
|
|
42
51
|
|
|
43
52
|
|
|
44
53
|
@dataclass
|
|
45
|
-
class
|
|
54
|
+
class UnverifiedPhysicalMetadata(UnverifiedComponent[PhysicalMetadata]):
|
|
46
55
|
space: str
|
|
47
56
|
external_id: str
|
|
48
57
|
creator: str
|
|
@@ -51,12 +60,12 @@ class DMSInputMetadata(UnverifiedComponent[DMSMetadata]):
|
|
|
51
60
|
description: str | None = None
|
|
52
61
|
created: datetime | str | None = None
|
|
53
62
|
updated: datetime | str | None = None
|
|
54
|
-
|
|
63
|
+
conceptual: str | URIRef | None = None
|
|
55
64
|
source_id: str | URIRef | None = None
|
|
56
65
|
|
|
57
66
|
@classmethod
|
|
58
|
-
def _get_verified_cls(cls) -> type[
|
|
59
|
-
return
|
|
67
|
+
def _get_verified_cls(cls) -> type[PhysicalMetadata]:
|
|
68
|
+
return PhysicalMetadata
|
|
60
69
|
|
|
61
70
|
def dump(self) -> dict[str, Any]: # type: ignore[override]
|
|
62
71
|
output = super().dump()
|
|
@@ -67,7 +76,7 @@ class DMSInputMetadata(UnverifiedComponent[DMSMetadata]):
|
|
|
67
76
|
return output
|
|
68
77
|
|
|
69
78
|
@classmethod
|
|
70
|
-
def from_data_model(cls, data_model: dm.DataModelApply) -> "
|
|
79
|
+
def from_data_model(cls, data_model: dm.DataModelApply) -> "UnverifiedPhysicalMetadata":
|
|
71
80
|
description, creator = cls._get_description_and_creator(data_model.description)
|
|
72
81
|
return cls(
|
|
73
82
|
space=data_model.space,
|
|
@@ -113,10 +122,10 @@ class DMSInputMetadata(UnverifiedComponent[DMSMetadata]):
|
|
|
113
122
|
|
|
114
123
|
|
|
115
124
|
@dataclass
|
|
116
|
-
class
|
|
125
|
+
class UnverifiedPhysicalProperty(UnverifiedComponent[PhysicalProperty]):
|
|
117
126
|
view: str
|
|
118
127
|
view_property: str | None
|
|
119
|
-
value_type: str | DataType | ViewEntity |
|
|
128
|
+
value_type: str | DataType | ViewEntity | PhysicalUnknownEntity
|
|
120
129
|
name: str | None = None
|
|
121
130
|
description: str | None = None
|
|
122
131
|
connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | str | None = None
|
|
@@ -129,7 +138,7 @@ class DMSInputProperty(UnverifiedComponent[DMSProperty]):
|
|
|
129
138
|
index: str | list[str] | None = None
|
|
130
139
|
constraint: str | list[str] | None = None
|
|
131
140
|
neatId: str | URIRef | None = None
|
|
132
|
-
|
|
141
|
+
conceptual: str | URIRef | None = None
|
|
133
142
|
|
|
134
143
|
@property
|
|
135
144
|
def nullable(self) -> bool | None:
|
|
@@ -145,8 +154,8 @@ class DMSInputProperty(UnverifiedComponent[DMSProperty]):
|
|
|
145
154
|
)
|
|
146
155
|
|
|
147
156
|
@classmethod
|
|
148
|
-
def _get_verified_cls(cls) -> type[
|
|
149
|
-
return
|
|
157
|
+
def _get_verified_cls(cls) -> type[PhysicalProperty]:
|
|
158
|
+
return PhysicalProperty
|
|
150
159
|
|
|
151
160
|
def dump(self, default_space: str, default_version: str) -> dict[str, Any]: # type: ignore[override]
|
|
152
161
|
output = super().dump()
|
|
@@ -206,7 +215,7 @@ class DMSInputProperty(UnverifiedComponent[DMSProperty]):
|
|
|
206
215
|
|
|
207
216
|
|
|
208
217
|
@dataclass
|
|
209
|
-
class
|
|
218
|
+
class UnverifiedPhysicalContainer(UnverifiedComponent[PhysicalContainer]):
|
|
210
219
|
container: str
|
|
211
220
|
name: str | None = None
|
|
212
221
|
description: str | None = None
|
|
@@ -215,8 +224,8 @@ class DMSInputContainer(UnverifiedComponent[DMSContainer]):
|
|
|
215
224
|
used_for: Literal["node", "edge", "all"] | None = None
|
|
216
225
|
|
|
217
226
|
@classmethod
|
|
218
|
-
def _get_verified_cls(cls) -> type[
|
|
219
|
-
return
|
|
227
|
+
def _get_verified_cls(cls) -> type[PhysicalContainer]:
|
|
228
|
+
return PhysicalContainer
|
|
220
229
|
|
|
221
230
|
def dump(self, default_space: str) -> dict[str, Any]: # type: ignore[override]
|
|
222
231
|
output = super().dump()
|
|
@@ -232,7 +241,7 @@ class DMSInputContainer(UnverifiedComponent[DMSContainer]):
|
|
|
232
241
|
return ContainerEntity.load(self.container, strict=True, space=default_space)
|
|
233
242
|
|
|
234
243
|
@classmethod
|
|
235
|
-
def from_container(cls, container: dm.ContainerApply) -> "
|
|
244
|
+
def from_container(cls, container: dm.ContainerApply) -> "UnverifiedPhysicalContainer":
|
|
236
245
|
constraints: list[str] = []
|
|
237
246
|
for _, constraint_obj in (container.constraints or {}).items():
|
|
238
247
|
if isinstance(constraint_obj, dm.RequiresConstraint):
|
|
@@ -249,7 +258,7 @@ class DMSInputContainer(UnverifiedComponent[DMSContainer]):
|
|
|
249
258
|
|
|
250
259
|
|
|
251
260
|
@dataclass
|
|
252
|
-
class
|
|
261
|
+
class UnverifiedPhysicalView(UnverifiedComponent[PhysicalView]):
|
|
253
262
|
view: str
|
|
254
263
|
name: str | None = None
|
|
255
264
|
description: str | None = None
|
|
@@ -257,15 +266,15 @@ class DMSInputView(UnverifiedComponent[DMSView]):
|
|
|
257
266
|
filter_: Literal["hasData", "nodeType", "rawFilter"] | str | None = None
|
|
258
267
|
in_model: bool = True
|
|
259
268
|
neatId: str | URIRef | None = None
|
|
260
|
-
|
|
269
|
+
conceptual: str | URIRef | None = None
|
|
261
270
|
|
|
262
271
|
def __post_init__(self) -> None:
|
|
263
272
|
if self.in_model is None:
|
|
264
273
|
self.in_model = True
|
|
265
274
|
|
|
266
275
|
@classmethod
|
|
267
|
-
def _get_verified_cls(cls) -> type[
|
|
268
|
-
return
|
|
276
|
+
def _get_verified_cls(cls) -> type[PhysicalView]:
|
|
277
|
+
return PhysicalView
|
|
269
278
|
|
|
270
279
|
def dump(self, default_space: str, default_version: str) -> dict[str, Any]: # type: ignore[override]
|
|
271
280
|
output = super().dump()
|
|
@@ -292,7 +301,7 @@ class DMSInputView(UnverifiedComponent[DMSView]):
|
|
|
292
301
|
return self._load_implements(default_space, default_version) or []
|
|
293
302
|
|
|
294
303
|
@classmethod
|
|
295
|
-
def from_view(cls, view: dm.ViewApply, in_model: bool) -> "
|
|
304
|
+
def from_view(cls, view: dm.ViewApply, in_model: bool) -> "UnverifiedPhysicalView":
|
|
296
305
|
view_entity = ViewEntity.from_id(view.as_id())
|
|
297
306
|
|
|
298
307
|
return cls(
|
|
@@ -307,7 +316,7 @@ class DMSInputView(UnverifiedComponent[DMSView]):
|
|
|
307
316
|
|
|
308
317
|
|
|
309
318
|
@dataclass
|
|
310
|
-
class
|
|
319
|
+
class UnverifiedPhysicalNodeType(UnverifiedComponent[PhysicalNodeType]):
|
|
311
320
|
node: str
|
|
312
321
|
usage: Literal["type", "collocation"]
|
|
313
322
|
name: str | None = None
|
|
@@ -315,11 +324,11 @@ class DMSInputNode(UnverifiedComponent[DMSNode]):
|
|
|
315
324
|
neatId: str | URIRef | None = None
|
|
316
325
|
|
|
317
326
|
@classmethod
|
|
318
|
-
def _get_verified_cls(cls) -> type[
|
|
319
|
-
return
|
|
327
|
+
def _get_verified_cls(cls) -> type[PhysicalNodeType]:
|
|
328
|
+
return PhysicalNodeType
|
|
320
329
|
|
|
321
330
|
@classmethod
|
|
322
|
-
def from_node_type(cls, node_type: dm.NodeApply) -> "
|
|
331
|
+
def from_node_type(cls, node_type: dm.NodeApply) -> "UnverifiedPhysicalNodeType":
|
|
323
332
|
return cls(node=f"{node_type.space}:{node_type.external_id}", usage="type")
|
|
324
333
|
|
|
325
334
|
def dump(self, default_space: str, **_) -> dict[str, Any]: # type: ignore
|
|
@@ -329,7 +338,7 @@ class DMSInputNode(UnverifiedComponent[DMSNode]):
|
|
|
329
338
|
|
|
330
339
|
|
|
331
340
|
@dataclass
|
|
332
|
-
class
|
|
341
|
+
class UnverifiedPhysicalEnum(UnverifiedComponent[PhysicalEnum]):
|
|
333
342
|
collection: str
|
|
334
343
|
value: str
|
|
335
344
|
name: str | None = None
|
|
@@ -337,22 +346,22 @@ class DMSInputEnum(UnverifiedComponent[DMSEnum]):
|
|
|
337
346
|
neatId: str | URIRef | None = None
|
|
338
347
|
|
|
339
348
|
@classmethod
|
|
340
|
-
def _get_verified_cls(cls) -> type[
|
|
341
|
-
return
|
|
349
|
+
def _get_verified_cls(cls) -> type[PhysicalEnum]:
|
|
350
|
+
return PhysicalEnum
|
|
342
351
|
|
|
343
352
|
|
|
344
353
|
@dataclass
|
|
345
|
-
class
|
|
346
|
-
metadata:
|
|
347
|
-
properties: list[
|
|
348
|
-
views: list[
|
|
349
|
-
containers: list[
|
|
350
|
-
enum: list[
|
|
351
|
-
nodes: list[
|
|
354
|
+
class UnverifiedPhysicalDataModel(UnverifiedDataModel[PhysicalDataModel]):
|
|
355
|
+
metadata: UnverifiedPhysicalMetadata
|
|
356
|
+
properties: list[UnverifiedPhysicalProperty]
|
|
357
|
+
views: list[UnverifiedPhysicalView]
|
|
358
|
+
containers: list[UnverifiedPhysicalContainer] | None = None
|
|
359
|
+
enum: list[UnverifiedPhysicalEnum] | None = None
|
|
360
|
+
nodes: list[UnverifiedPhysicalNodeType] | None = None
|
|
352
361
|
|
|
353
362
|
@classmethod
|
|
354
|
-
def _get_verified_cls(cls) -> type[
|
|
355
|
-
return
|
|
363
|
+
def _get_verified_cls(cls) -> type[PhysicalDataModel]:
|
|
364
|
+
return PhysicalDataModel
|
|
356
365
|
|
|
357
366
|
def dump(self) -> dict[str, Any]:
|
|
358
367
|
default_space = self.metadata.space
|
|
@@ -369,7 +378,7 @@ class DMSInputRules(UnverifiedDataModel[DMSRules]):
|
|
|
369
378
|
|
|
370
379
|
@classmethod
|
|
371
380
|
def display_type_name(cls) -> str:
|
|
372
|
-
return "
|
|
381
|
+
return "UnverifiedPhysicalModel"
|
|
373
382
|
|
|
374
383
|
@property
|
|
375
384
|
def display_name(self) -> str:
|
|
@@ -378,7 +387,7 @@ class DMSInputRules(UnverifiedDataModel[DMSRules]):
|
|
|
378
387
|
def _repr_html_(self) -> str:
|
|
379
388
|
summary = {
|
|
380
389
|
"type": "Physical Data Model",
|
|
381
|
-
"intended for": "
|
|
390
|
+
"intended for": "Data Engineer",
|
|
382
391
|
"name": self.metadata.name,
|
|
383
392
|
"space": self.metadata.space,
|
|
384
393
|
"external_id": self.metadata.external_id,
|
|
@@ -52,25 +52,25 @@ from cognite.neat.core._issues.warnings.user_modeling import (
|
|
|
52
52
|
from cognite.neat.core._utils.spreadsheet import SpreadsheetRead
|
|
53
53
|
from cognite.neat.core._utils.text import humanize_collection
|
|
54
54
|
|
|
55
|
-
from .
|
|
55
|
+
from ._verified import PhysicalDataModel, PhysicalProperty
|
|
56
56
|
|
|
57
57
|
|
|
58
|
-
class
|
|
59
|
-
"""This class does all the validation of the
|
|
60
|
-
components."""
|
|
58
|
+
class PhysicalValidation:
|
|
59
|
+
"""This class does all the validation of the physical data model that
|
|
60
|
+
have dependencies between components."""
|
|
61
61
|
|
|
62
62
|
def __init__(
|
|
63
63
|
self,
|
|
64
|
-
|
|
64
|
+
data_model: PhysicalDataModel,
|
|
65
65
|
client: NeatClient | None = None,
|
|
66
66
|
read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None,
|
|
67
67
|
) -> None:
|
|
68
|
-
self.
|
|
68
|
+
self._data_model = data_model
|
|
69
69
|
self._client = client
|
|
70
|
-
self._metadata =
|
|
71
|
-
self._properties =
|
|
72
|
-
self._containers =
|
|
73
|
-
self._views =
|
|
70
|
+
self._metadata = data_model.metadata
|
|
71
|
+
self._properties = data_model.properties
|
|
72
|
+
self._containers = data_model.containers
|
|
73
|
+
self._views = data_model.views
|
|
74
74
|
self._read_info_by_spreadsheet = read_info_by_spreadsheet or {}
|
|
75
75
|
|
|
76
76
|
def imported_views_and_containers_ids(
|
|
@@ -109,7 +109,7 @@ class DMSValidation:
|
|
|
109
109
|
)
|
|
110
110
|
if (imported_views or imported_containers) and self._client is None:
|
|
111
111
|
raise CDFMissingClientError(
|
|
112
|
-
f"{self.
|
|
112
|
+
f"{self._data_model.metadata.as_data_model_id()} has imported views and/or container: "
|
|
113
113
|
f"{imported_views}, {imported_containers}."
|
|
114
114
|
)
|
|
115
115
|
referenced_views = ViewList([])
|
|
@@ -131,10 +131,10 @@ class DMSValidation:
|
|
|
131
131
|
raise CDFMissingResourcesError(containers=tuple(missing_containers), views=tuple(missing_views))
|
|
132
132
|
|
|
133
133
|
# Setup data structures for validation
|
|
134
|
-
dms_schema = self.
|
|
134
|
+
dms_schema = self._data_model.as_schema()
|
|
135
135
|
ref_view_by_id = {view.as_id(): view for view in referenced_views}
|
|
136
136
|
ref_container_by_id = {container.as_id(): container for container in referenced_containers}
|
|
137
|
-
# All containers and views are the Containers/Views in the
|
|
137
|
+
# All containers and views are the Containers/Views in the Physical DM + the referenced ones
|
|
138
138
|
all_containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container] = {
|
|
139
139
|
**dict(dms_schema.containers.items()),
|
|
140
140
|
**ref_container_by_id,
|
|
@@ -175,7 +175,7 @@ class DMSValidation:
|
|
|
175
175
|
def _same_space_views_and_data_model(self) -> IssueList:
|
|
176
176
|
issue_list = IssueList()
|
|
177
177
|
|
|
178
|
-
schema = self.
|
|
178
|
+
schema = self._data_model.as_schema(remove_cdf_spaces=True)
|
|
179
179
|
|
|
180
180
|
if schema.data_model and schema.views:
|
|
181
181
|
data_model_space = schema.data_model.space
|
|
@@ -321,7 +321,9 @@ class DMSValidation:
|
|
|
321
321
|
return parents_by_view
|
|
322
322
|
|
|
323
323
|
def _consistent_container_properties(self) -> IssueList:
|
|
324
|
-
container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int,
|
|
324
|
+
container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, PhysicalProperty]]] = defaultdict(
|
|
325
|
+
list
|
|
326
|
+
)
|
|
325
327
|
for prop_no, prop in enumerate(self._properties):
|
|
326
328
|
if prop.container and prop.container_property:
|
|
327
329
|
container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
|