cognite-neat 0.121.1__py3-none-any.whl → 0.121.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/_data_model/_constants.py +23 -16
- cognite/neat/core/_data_model/_shared.py +33 -17
- cognite/neat/core/_data_model/analysis/__init__.py +2 -2
- cognite/neat/core/_data_model/analysis/_base.py +186 -183
- cognite/neat/core/_data_model/catalog/__init__.py +1 -1
- cognite/neat/core/_data_model/exporters/__init__.py +5 -5
- cognite/neat/core/_data_model/exporters/_base.py +10 -8
- cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
- cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
- cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +4 -4
- cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
- cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
- cognite/neat/core/_data_model/importers/__init__.py +6 -6
- cognite/neat/core/_data_model/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +40 -20
- cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/_base.py +9 -9
- cognite/neat/core/_data_model/importers/_rdf/_imf2rules.py +15 -15
- cognite/neat/core/_data_model/importers/_rdf/_inference2rules.py +36 -36
- cognite/neat/core/_data_model/importers/_rdf/_owl2rules.py +12 -12
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +72 -12
- cognite/neat/core/_data_model/models/__init__.py +8 -8
- cognite/neat/core/_data_model/models/_base_unverified.py +1 -1
- cognite/neat/core/_data_model/models/_base_verified.py +3 -3
- cognite/neat/core/_data_model/models/_types.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
- cognite/neat/core/_data_model/models/conceptual/_validation.py +87 -77
- cognite/neat/core/_data_model/models/conceptual/_verified.py +53 -51
- cognite/neat/core/_data_model/models/data_types.py +2 -2
- cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
- cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
- cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
- cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
- cognite/neat/core/_data_model/models/entities/_types.py +9 -3
- cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
- cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +71 -52
- cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
- cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +13 -11
- cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
- cognite/neat/core/_data_model/transformers/__init__.py +27 -23
- cognite/neat/core/_data_model/transformers/_base.py +26 -19
- cognite/neat/core/_data_model/transformers/_converters.py +703 -618
- cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
- cognite/neat/core/_data_model/transformers/_verification.py +63 -54
- cognite/neat/core/_instances/extractors/_base.py +1 -1
- cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +8 -8
- cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
- cognite/neat/core/_instances/extractors/_mock_graph_generator.py +98 -95
- cognite/neat/core/_instances/loaders/_base.py +2 -2
- cognite/neat/core/_instances/loaders/_rdf2dms.py +6 -6
- cognite/neat/core/_instances/transformers/_base.py +7 -4
- cognite/neat/core/_instances/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +2 -2
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/text.py +1 -1
- cognite/neat/session/_base.py +22 -20
- cognite/neat/session/_drop.py +2 -2
- cognite/neat/session/_inspect.py +5 -5
- cognite/neat/session/_mapping.py +8 -6
- cognite/neat/session/_read.py +2 -2
- cognite/neat/session/_set.py +3 -3
- cognite/neat/session/_show.py +11 -11
- cognite/neat/session/_state.py +13 -13
- cognite/neat/session/_subset.py +12 -9
- cognite/neat/session/_template.py +13 -13
- cognite/neat/session/_to.py +17 -17
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/RECORD +95 -93
- cognite/neat/core/_data_model/exporters/_validation.py +0 -14
- cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
- /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -6,7 +6,13 @@ from pydantic import (
|
|
|
6
6
|
PlainSerializer,
|
|
7
7
|
)
|
|
8
8
|
|
|
9
|
-
from ._single_value import
|
|
9
|
+
from ._single_value import (
|
|
10
|
+
AssetEntity,
|
|
11
|
+
ConceptEntity,
|
|
12
|
+
ContainerEntity,
|
|
13
|
+
RelationshipEntity,
|
|
14
|
+
ViewEntity,
|
|
15
|
+
)
|
|
10
16
|
|
|
11
17
|
|
|
12
18
|
def _split_str(v: Any) -> list[str]:
|
|
@@ -15,7 +21,7 @@ def _split_str(v: Any) -> list[str]:
|
|
|
15
21
|
return v
|
|
16
22
|
|
|
17
23
|
|
|
18
|
-
def _join_str(v: list[
|
|
24
|
+
def _join_str(v: list[ConceptEntity]) -> str | None:
|
|
19
25
|
return ",".join([entry.id for entry in v]) if v else None
|
|
20
26
|
|
|
21
27
|
|
|
@@ -39,7 +45,7 @@ def _generate_cdf_resource_list(v: Any) -> list[AssetEntity | RelationshipEntity
|
|
|
39
45
|
|
|
40
46
|
|
|
41
47
|
ClassEntityList = Annotated[
|
|
42
|
-
list[
|
|
48
|
+
list[ConceptEntity],
|
|
43
49
|
BeforeValidator(_split_str),
|
|
44
50
|
]
|
|
45
51
|
|
|
@@ -8,14 +8,14 @@ from cognite.client import data_modeling as dm
|
|
|
8
8
|
from cognite.client.data_classes.data_modeling import ContainerId, NodeId
|
|
9
9
|
from pydantic import BaseModel, model_serializer, model_validator
|
|
10
10
|
|
|
11
|
-
from ._single_value import ContainerEntity, DMSNodeEntity,
|
|
11
|
+
from ._single_value import ConceptualEntity, ContainerEntity, DMSNodeEntity, ViewEntity
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
@total_ordering
|
|
15
15
|
class WrappedEntity(BaseModel, ABC):
|
|
16
16
|
name: ClassVar[str]
|
|
17
|
-
_inner_cls: ClassVar[type[
|
|
18
|
-
inner: list[
|
|
17
|
+
_inner_cls: ClassVar[type[ConceptualEntity]]
|
|
18
|
+
inner: list[ConceptualEntity] | None
|
|
19
19
|
|
|
20
20
|
@classmethod
|
|
21
21
|
def load(cls: "type[T_WrappedEntity]", data: Any) -> "T_WrappedEntity":
|
|
@@ -3,8 +3,11 @@ from pathlib import Path
|
|
|
3
3
|
|
|
4
4
|
import yaml
|
|
5
5
|
|
|
6
|
-
from cognite.neat.core._data_model._shared import
|
|
7
|
-
from cognite.neat.core._data_model.models.
|
|
6
|
+
from cognite.neat.core._data_model._shared import ImportedDataModel
|
|
7
|
+
from cognite.neat.core._data_model.models.physical import (
|
|
8
|
+
PhysicalDataModel,
|
|
9
|
+
UnverifiedPhysicalDataModel,
|
|
10
|
+
)
|
|
8
11
|
from cognite.neat.core._issues.errors import NeatValueError
|
|
9
12
|
|
|
10
13
|
_CLASSIC_TO_CORE_MAPPING = Path(__file__).resolve().parent / "_classic2core.yaml"
|
|
@@ -15,9 +18,9 @@ def _read_source_file() -> str:
|
|
|
15
18
|
return _CLASSIC_TO_CORE_MAPPING.read_text()
|
|
16
19
|
|
|
17
20
|
|
|
18
|
-
def load_classic_to_core_mapping(org_name: str | None, source_space: str, source_version: str) ->
|
|
19
|
-
from cognite.neat.core._data_model.importers import
|
|
20
|
-
from cognite.neat.core._data_model.transformers import
|
|
21
|
+
def load_classic_to_core_mapping(org_name: str | None, source_space: str, source_version: str) -> PhysicalDataModel:
|
|
22
|
+
from cognite.neat.core._data_model.importers import DictImporter
|
|
23
|
+
from cognite.neat.core._data_model.transformers import VerifyPhysicalDataModel
|
|
21
24
|
|
|
22
25
|
raw_str = _read_source_file()
|
|
23
26
|
if org_name is not None:
|
|
@@ -27,10 +30,10 @@ def load_classic_to_core_mapping(org_name: str | None, source_space: str, source
|
|
|
27
30
|
loaded["metadata"]["space"] = source_space
|
|
28
31
|
loaded["metadata"]["version"] = source_version
|
|
29
32
|
|
|
30
|
-
read:
|
|
31
|
-
if not isinstance(read.
|
|
32
|
-
raise NeatValueError(f"Expected DMS rules, but got {type(read.
|
|
33
|
+
read: ImportedDataModel[UnverifiedPhysicalDataModel] = DictImporter(loaded).to_data_model()
|
|
34
|
+
if not isinstance(read.unverified_data_model, UnverifiedPhysicalDataModel):
|
|
35
|
+
raise NeatValueError(f"Expected DMS rules, but got {type(read.unverified_data_model).__name__}")
|
|
33
36
|
|
|
34
|
-
verified =
|
|
37
|
+
verified = VerifyPhysicalDataModel(validate=False).transform(read)
|
|
35
38
|
|
|
36
39
|
return verified
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from cognite.neat.core._client.data_classes.schema import DMSSchema
|
|
2
|
+
|
|
3
|
+
from ._unverified import (
|
|
4
|
+
UnverifiedPhysicalContainer,
|
|
5
|
+
UnverifiedPhysicalDataModel,
|
|
6
|
+
UnverifiedPhysicalEnum,
|
|
7
|
+
UnverifiedPhysicalMetadata,
|
|
8
|
+
UnverifiedPhysicalNodeType,
|
|
9
|
+
UnverifiedPhysicalProperty,
|
|
10
|
+
UnverifiedPhysicalView,
|
|
11
|
+
)
|
|
12
|
+
from ._validation import PhysicalValidation
|
|
13
|
+
from ._verified import (
|
|
14
|
+
PhysicalContainer,
|
|
15
|
+
PhysicalDataModel,
|
|
16
|
+
PhysicalEnum,
|
|
17
|
+
PhysicalMetadata,
|
|
18
|
+
PhysicalNodeType,
|
|
19
|
+
PhysicalProperty,
|
|
20
|
+
PhysicalView,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"DMSSchema",
|
|
25
|
+
"PhysicalContainer",
|
|
26
|
+
"PhysicalDataModel",
|
|
27
|
+
"PhysicalEnum",
|
|
28
|
+
"PhysicalMetadata",
|
|
29
|
+
"PhysicalNodeType",
|
|
30
|
+
"PhysicalProperty",
|
|
31
|
+
"PhysicalValidation",
|
|
32
|
+
"PhysicalView",
|
|
33
|
+
"UnverifiedPhysicalContainer",
|
|
34
|
+
"UnverifiedPhysicalDataModel",
|
|
35
|
+
"UnverifiedPhysicalEnum",
|
|
36
|
+
"UnverifiedPhysicalMetadata",
|
|
37
|
+
"UnverifiedPhysicalNodeType",
|
|
38
|
+
"UnverifiedPhysicalProperty",
|
|
39
|
+
"UnverifiedPhysicalView",
|
|
40
|
+
]
|
|
@@ -27,14 +27,14 @@ from cognite.neat.core._constants import (
|
|
|
27
27
|
)
|
|
28
28
|
from cognite.neat.core._data_model.models.data_types import DataType, Double, Enum, Float
|
|
29
29
|
from cognite.neat.core._data_model.models.entities import (
|
|
30
|
-
|
|
30
|
+
ConceptEntity,
|
|
31
31
|
ContainerEntity,
|
|
32
32
|
DMSFilter,
|
|
33
33
|
DMSNodeEntity,
|
|
34
|
-
DMSUnknownEntity,
|
|
35
34
|
EdgeEntity,
|
|
36
35
|
HasDataFilter,
|
|
37
36
|
NodeTypeFilter,
|
|
37
|
+
PhysicalUnknownEntity,
|
|
38
38
|
ReverseConnectionEntity,
|
|
39
39
|
UnitEntity,
|
|
40
40
|
ViewEntity,
|
|
@@ -53,7 +53,13 @@ from cognite.neat.core._issues.warnings.user_modeling import (
|
|
|
53
53
|
HasDataFilterOnNoPropertiesViewWarning,
|
|
54
54
|
)
|
|
55
55
|
|
|
56
|
-
from .
|
|
56
|
+
from ._verified import (
|
|
57
|
+
PhysicalDataModel,
|
|
58
|
+
PhysicalEnum,
|
|
59
|
+
PhysicalMetadata,
|
|
60
|
+
PhysicalProperty,
|
|
61
|
+
PhysicalView,
|
|
62
|
+
)
|
|
57
63
|
|
|
58
64
|
|
|
59
65
|
class _DMSExporter:
|
|
@@ -69,29 +75,34 @@ class _DMSExporter:
|
|
|
69
75
|
remove_cdf_spaces(bool): The
|
|
70
76
|
"""
|
|
71
77
|
|
|
72
|
-
def __init__(
|
|
78
|
+
def __init__(
|
|
79
|
+
self,
|
|
80
|
+
data_model: PhysicalDataModel,
|
|
81
|
+
instance_space: str | None = None,
|
|
82
|
+
remove_cdf_spaces: bool = False,
|
|
83
|
+
):
|
|
73
84
|
self.instance_space = instance_space
|
|
74
|
-
self.
|
|
85
|
+
self.data_model = data_model
|
|
75
86
|
self.remove_cdf_spaces = remove_cdf_spaces
|
|
76
87
|
|
|
77
88
|
def to_schema(self) -> DMSSchema:
|
|
78
|
-
|
|
79
|
-
container_properties_by_id, view_properties_by_id = self._gather_properties(list(self.
|
|
89
|
+
data_model = self.data_model
|
|
90
|
+
container_properties_by_id, view_properties_by_id = self._gather_properties(list(self.data_model.properties))
|
|
80
91
|
|
|
81
|
-
containers = self._create_containers(container_properties_by_id,
|
|
92
|
+
containers = self._create_containers(container_properties_by_id, data_model.enum) # type: ignore[arg-type]
|
|
82
93
|
|
|
83
94
|
view_properties_with_ancestors_by_id = self._gather_properties_with_ancestors(
|
|
84
|
-
view_properties_by_id,
|
|
95
|
+
view_properties_by_id, data_model.views
|
|
85
96
|
)
|
|
86
97
|
|
|
87
98
|
views = self._create_views(view_properties_by_id, view_properties_with_ancestors_by_id)
|
|
88
99
|
view_node_type_filters: set[dm.NodeId] = set()
|
|
89
|
-
for dms_view in
|
|
100
|
+
for dms_view in data_model.views:
|
|
90
101
|
if isinstance(dms_view.filter_, NodeTypeFilter):
|
|
91
102
|
view_node_type_filters.update(node.as_id() for node in dms_view.filter_.inner or [])
|
|
92
|
-
if
|
|
103
|
+
if data_model.nodes:
|
|
93
104
|
node_types = NodeApplyDict(
|
|
94
|
-
[node.as_node() for node in
|
|
105
|
+
[node.as_node() for node in data_model.nodes]
|
|
95
106
|
+ [dm.NodeApply(node.space, node.external_id) for node in view_node_type_filters]
|
|
96
107
|
)
|
|
97
108
|
else:
|
|
@@ -103,17 +114,17 @@ class _DMSExporter:
|
|
|
103
114
|
]
|
|
104
115
|
)
|
|
105
116
|
|
|
106
|
-
|
|
117
|
+
dms_data_model = data_model.metadata.as_data_model()
|
|
107
118
|
# Sorting to ensure deterministic order
|
|
108
|
-
|
|
109
|
-
[dms_view.view.as_id() for dms_view in
|
|
119
|
+
dms_data_model.views = sorted(
|
|
120
|
+
[dms_view.view.as_id() for dms_view in data_model.views if dms_view.in_model],
|
|
110
121
|
key=lambda x: x.as_tuple(), # type: ignore[union-attr]
|
|
111
122
|
)
|
|
112
|
-
spaces = self._create_spaces(
|
|
123
|
+
spaces = self._create_spaces(data_model.metadata, containers, views, dms_data_model)
|
|
113
124
|
|
|
114
125
|
return DMSSchema(
|
|
115
126
|
spaces=spaces,
|
|
116
|
-
data_model=
|
|
127
|
+
data_model=dms_data_model,
|
|
117
128
|
views=views,
|
|
118
129
|
containers=containers,
|
|
119
130
|
node_types=node_types,
|
|
@@ -121,7 +132,7 @@ class _DMSExporter:
|
|
|
121
132
|
|
|
122
133
|
def _create_spaces(
|
|
123
134
|
self,
|
|
124
|
-
metadata:
|
|
135
|
+
metadata: PhysicalMetadata,
|
|
125
136
|
containers: ContainerApplyDict,
|
|
126
137
|
views: ViewApplyDict,
|
|
127
138
|
data_model: dm.DataModelApply,
|
|
@@ -140,10 +151,10 @@ class _DMSExporter:
|
|
|
140
151
|
|
|
141
152
|
def _create_views(
|
|
142
153
|
self,
|
|
143
|
-
view_properties_by_id: dict[dm.ViewId, list[
|
|
144
|
-
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[
|
|
154
|
+
view_properties_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
155
|
+
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
145
156
|
) -> ViewApplyDict:
|
|
146
|
-
input_views = list(self.
|
|
157
|
+
input_views = list(self.data_model.views)
|
|
147
158
|
|
|
148
159
|
views = ViewApplyDict(
|
|
149
160
|
[
|
|
@@ -172,7 +183,7 @@ class _DMSExporter:
|
|
|
172
183
|
return views
|
|
173
184
|
|
|
174
185
|
@classmethod
|
|
175
|
-
def _create_edge_type_from_prop(cls, prop:
|
|
186
|
+
def _create_edge_type_from_prop(cls, prop: PhysicalProperty) -> dm.DirectRelationReference:
|
|
176
187
|
if isinstance(prop.connection, EdgeEntity) and prop.connection.edge_type is not None:
|
|
177
188
|
return prop.connection.edge_type.as_reference()
|
|
178
189
|
elif isinstance(prop.value_type, ViewEntity):
|
|
@@ -191,10 +202,10 @@ class _DMSExporter:
|
|
|
191
202
|
@classmethod
|
|
192
203
|
def _edge_types_by_view_property_id(
|
|
193
204
|
cls,
|
|
194
|
-
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[
|
|
195
|
-
view_by_id: dict[ViewEntity,
|
|
205
|
+
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
206
|
+
view_by_id: dict[ViewEntity, PhysicalView],
|
|
196
207
|
) -> dict[tuple[ViewEntity, str], dm.DirectRelationReference]:
|
|
197
|
-
edge_connection_property_by_view_property_id: dict[tuple[ViewEntity, str],
|
|
208
|
+
edge_connection_property_by_view_property_id: dict[tuple[ViewEntity, str], PhysicalProperty] = {}
|
|
198
209
|
for properties in view_properties_with_ancestors_by_id.values():
|
|
199
210
|
for prop in properties:
|
|
200
211
|
if isinstance(prop.connection, EdgeEntity):
|
|
@@ -248,10 +259,10 @@ class _DMSExporter:
|
|
|
248
259
|
@classmethod
|
|
249
260
|
def _get_edge_type_outwards_connection(
|
|
250
261
|
cls,
|
|
251
|
-
view:
|
|
252
|
-
prop:
|
|
253
|
-
view_by_id: dict[ViewEntity,
|
|
254
|
-
edge_connection_by_view_property_id: dict[tuple[ViewEntity, str],
|
|
262
|
+
view: PhysicalView,
|
|
263
|
+
prop: PhysicalProperty,
|
|
264
|
+
view_by_id: dict[ViewEntity, PhysicalView],
|
|
265
|
+
edge_connection_by_view_property_id: dict[tuple[ViewEntity, str], PhysicalProperty],
|
|
255
266
|
) -> dm.DirectRelationReference:
|
|
256
267
|
connection = cast(EdgeEntity, prop.connection)
|
|
257
268
|
if connection.edge_type is not None:
|
|
@@ -284,14 +295,14 @@ class _DMSExporter:
|
|
|
284
295
|
|
|
285
296
|
def _create_containers(
|
|
286
297
|
self,
|
|
287
|
-
container_properties_by_id: dict[dm.ContainerId, list[
|
|
288
|
-
enum: Collection[
|
|
298
|
+
container_properties_by_id: dict[dm.ContainerId, list[PhysicalProperty]],
|
|
299
|
+
enum: Collection[PhysicalEnum] | None,
|
|
289
300
|
) -> ContainerApplyDict:
|
|
290
|
-
enum_values_by_collection: dict[
|
|
301
|
+
enum_values_by_collection: dict[ConceptEntity, list[PhysicalEnum]] = defaultdict(list)
|
|
291
302
|
for enum_value in enum or []:
|
|
292
303
|
enum_values_by_collection[enum_value.collection].append(enum_value)
|
|
293
304
|
|
|
294
|
-
containers = list(self.
|
|
305
|
+
containers = list(self.data_model.containers or [])
|
|
295
306
|
|
|
296
307
|
containers = dm.ContainerApplyList(
|
|
297
308
|
[
|
|
@@ -390,10 +401,13 @@ class _DMSExporter:
|
|
|
390
401
|
|
|
391
402
|
@staticmethod
|
|
392
403
|
def _gather_properties(
|
|
393
|
-
properties: Sequence[
|
|
394
|
-
) -> tuple[
|
|
395
|
-
|
|
396
|
-
|
|
404
|
+
properties: Sequence[PhysicalProperty],
|
|
405
|
+
) -> tuple[
|
|
406
|
+
dict[dm.ContainerId, list[PhysicalProperty]],
|
|
407
|
+
dict[dm.ViewId, list[PhysicalProperty]],
|
|
408
|
+
]:
|
|
409
|
+
container_properties_by_id: dict[dm.ContainerId, list[PhysicalProperty]] = defaultdict(list)
|
|
410
|
+
view_properties_by_id: dict[dm.ViewId, list[PhysicalProperty]] = defaultdict(list)
|
|
397
411
|
for prop in properties:
|
|
398
412
|
view_id = prop.view.as_id()
|
|
399
413
|
view_properties_by_id[view_id].append(prop)
|
|
@@ -406,12 +420,12 @@ class _DMSExporter:
|
|
|
406
420
|
|
|
407
421
|
def _gather_properties_with_ancestors(
|
|
408
422
|
self,
|
|
409
|
-
view_properties_by_id: dict[dm.ViewId, list[
|
|
410
|
-
views: Sequence[
|
|
411
|
-
) -> dict[dm.ViewId, list[
|
|
423
|
+
view_properties_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
424
|
+
views: Sequence[PhysicalView],
|
|
425
|
+
) -> dict[dm.ViewId, list[PhysicalProperty]]:
|
|
412
426
|
all_view_properties_by_id = view_properties_by_id.copy()
|
|
413
427
|
|
|
414
|
-
view_properties_with_parents_by_id: dict[dm.ViewId, list[
|
|
428
|
+
view_properties_with_parents_by_id: dict[dm.ViewId, list[PhysicalProperty]] = defaultdict(list)
|
|
415
429
|
view_by_view_id = {view.view.as_id(): view for view in views}
|
|
416
430
|
for view in views:
|
|
417
431
|
view_id = view.view.as_id()
|
|
@@ -446,9 +460,9 @@ class _DMSExporter:
|
|
|
446
460
|
@classmethod
|
|
447
461
|
def _update_with_properties(
|
|
448
462
|
cls,
|
|
449
|
-
selected_properties: Sequence[
|
|
450
|
-
container_properties_by_id: dict[dm.ContainerId, list[
|
|
451
|
-
view_properties_by_id: dict[dm.ViewId, list[
|
|
463
|
+
selected_properties: Sequence[PhysicalProperty],
|
|
464
|
+
container_properties_by_id: dict[dm.ContainerId, list[PhysicalProperty]],
|
|
465
|
+
view_properties_by_id: dict[dm.ViewId, list[PhysicalProperty]] | None,
|
|
452
466
|
include_new_containers: bool = False,
|
|
453
467
|
) -> None:
|
|
454
468
|
view_properties_by_id = view_properties_by_id or {}
|
|
@@ -472,7 +486,7 @@ class _DMSExporter:
|
|
|
472
486
|
def _create_view_filter(
|
|
473
487
|
self,
|
|
474
488
|
view: dm.ViewApply,
|
|
475
|
-
dms_view:
|
|
489
|
+
dms_view: PhysicalView | None,
|
|
476
490
|
) -> DMSFilter | None:
|
|
477
491
|
selected_filter_name = (dms_view and dms_view.filter_ and dms_view.filter_.name) or ""
|
|
478
492
|
|
|
@@ -497,8 +511,8 @@ class _DMSExporter:
|
|
|
497
511
|
@classmethod
|
|
498
512
|
def _create_view_property(
|
|
499
513
|
cls,
|
|
500
|
-
prop:
|
|
501
|
-
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[
|
|
514
|
+
prop: PhysicalProperty,
|
|
515
|
+
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
502
516
|
edge_types_by_view_property_id: dict[tuple[ViewEntity, str], dm.DirectRelationReference],
|
|
503
517
|
) -> ViewPropertyApply | None:
|
|
504
518
|
if prop.container and prop.container_property:
|
|
@@ -509,19 +523,20 @@ class _DMSExporter:
|
|
|
509
523
|
return cls._create_reverse_direct_relation(prop, view_properties_with_ancestors_by_id)
|
|
510
524
|
elif prop.view and prop.view_property and prop.connection:
|
|
511
525
|
warnings.warn(
|
|
512
|
-
NotSupportedWarning(f"{prop.connection} in {prop.view.as_id()!r}.{prop.view_property}"),
|
|
526
|
+
NotSupportedWarning(f"{prop.connection} in {prop.view.as_id()!r}.{prop.view_property}"),
|
|
527
|
+
stacklevel=2,
|
|
513
528
|
)
|
|
514
529
|
return None
|
|
515
530
|
|
|
516
531
|
@classmethod
|
|
517
|
-
def _create_mapped_property(cls, prop:
|
|
532
|
+
def _create_mapped_property(cls, prop: PhysicalProperty) -> dm.MappedPropertyApply:
|
|
518
533
|
container = cast(ContainerEntity, prop.container)
|
|
519
534
|
container_prop_identifier = cast(str, prop.container_property)
|
|
520
535
|
extra_args: dict[str, Any] = {}
|
|
521
536
|
if prop.connection == "direct":
|
|
522
537
|
if isinstance(prop.value_type, ViewEntity):
|
|
523
538
|
extra_args["source"] = prop.value_type.as_id()
|
|
524
|
-
elif isinstance(prop.value_type,
|
|
539
|
+
elif isinstance(prop.value_type, PhysicalUnknownEntity):
|
|
525
540
|
extra_args["source"] = None
|
|
526
541
|
else:
|
|
527
542
|
# Should have been validated.
|
|
@@ -545,7 +560,9 @@ class _DMSExporter:
|
|
|
545
560
|
|
|
546
561
|
@classmethod
|
|
547
562
|
def _create_edge_property(
|
|
548
|
-
cls,
|
|
563
|
+
cls,
|
|
564
|
+
prop: PhysicalProperty,
|
|
565
|
+
edge_types_by_view_property_id: dict[tuple[ViewEntity, str], dm.DirectRelationReference],
|
|
549
566
|
) -> dm.EdgeConnectionApply:
|
|
550
567
|
connection = cast(EdgeEntity, prop.connection)
|
|
551
568
|
if isinstance(prop.value_type, ViewEntity):
|
|
@@ -575,7 +592,9 @@ class _DMSExporter:
|
|
|
575
592
|
|
|
576
593
|
@classmethod
|
|
577
594
|
def _create_reverse_direct_relation(
|
|
578
|
-
cls,
|
|
595
|
+
cls,
|
|
596
|
+
prop: PhysicalProperty,
|
|
597
|
+
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[PhysicalProperty]],
|
|
579
598
|
) -> dm.MultiReverseDirectRelationApply | SingleReverseDirectRelationApply | None:
|
|
580
599
|
connection = cast(ReverseConnectionEntity, prop.connection)
|
|
581
600
|
reverse_prop_id = connection.property_
|
|
@@ -22,8 +22,8 @@ from cognite.neat.core._data_model.models.data_types import DataType
|
|
|
22
22
|
from cognite.neat.core._data_model.models.entities import (
|
|
23
23
|
ContainerEntity,
|
|
24
24
|
DMSNodeEntity,
|
|
25
|
-
DMSUnknownEntity,
|
|
26
25
|
EdgeEntity,
|
|
26
|
+
PhysicalUnknownEntity,
|
|
27
27
|
ReverseConnectionEntity,
|
|
28
28
|
ViewEntity,
|
|
29
29
|
load_connection,
|
|
@@ -33,7 +33,16 @@ from cognite.neat.core._data_model.models.entities._wrapped import DMSFilter
|
|
|
33
33
|
from cognite.neat.core._issues.warnings import DeprecatedWarning
|
|
34
34
|
from cognite.neat.core._utils.rdf_ import uri_display_name
|
|
35
35
|
|
|
36
|
-
from .
|
|
36
|
+
from ._verified import (
|
|
37
|
+
_DEFAULT_VERSION,
|
|
38
|
+
PhysicalContainer,
|
|
39
|
+
PhysicalDataModel,
|
|
40
|
+
PhysicalEnum,
|
|
41
|
+
PhysicalMetadata,
|
|
42
|
+
PhysicalNodeType,
|
|
43
|
+
PhysicalProperty,
|
|
44
|
+
PhysicalView,
|
|
45
|
+
)
|
|
37
46
|
|
|
38
47
|
if sys.version_info >= (3, 11):
|
|
39
48
|
from typing import Self
|
|
@@ -42,7 +51,7 @@ else:
|
|
|
42
51
|
|
|
43
52
|
|
|
44
53
|
@dataclass
|
|
45
|
-
class
|
|
54
|
+
class UnverifiedPhysicalMetadata(UnverifiedComponent[PhysicalMetadata]):
|
|
46
55
|
space: str
|
|
47
56
|
external_id: str
|
|
48
57
|
creator: str
|
|
@@ -51,12 +60,12 @@ class DMSInputMetadata(UnverifiedComponent[DMSMetadata]):
|
|
|
51
60
|
description: str | None = None
|
|
52
61
|
created: datetime | str | None = None
|
|
53
62
|
updated: datetime | str | None = None
|
|
54
|
-
|
|
63
|
+
conceptual: str | URIRef | None = None
|
|
55
64
|
source_id: str | URIRef | None = None
|
|
56
65
|
|
|
57
66
|
@classmethod
|
|
58
|
-
def _get_verified_cls(cls) -> type[
|
|
59
|
-
return
|
|
67
|
+
def _get_verified_cls(cls) -> type[PhysicalMetadata]:
|
|
68
|
+
return PhysicalMetadata
|
|
60
69
|
|
|
61
70
|
def dump(self) -> dict[str, Any]: # type: ignore[override]
|
|
62
71
|
output = super().dump()
|
|
@@ -67,7 +76,7 @@ class DMSInputMetadata(UnverifiedComponent[DMSMetadata]):
|
|
|
67
76
|
return output
|
|
68
77
|
|
|
69
78
|
@classmethod
|
|
70
|
-
def from_data_model(cls, data_model: dm.DataModelApply) -> "
|
|
79
|
+
def from_data_model(cls, data_model: dm.DataModelApply) -> "UnverifiedPhysicalMetadata":
|
|
71
80
|
description, creator = cls._get_description_and_creator(data_model.description)
|
|
72
81
|
return cls(
|
|
73
82
|
space=data_model.space,
|
|
@@ -113,10 +122,10 @@ class DMSInputMetadata(UnverifiedComponent[DMSMetadata]):
|
|
|
113
122
|
|
|
114
123
|
|
|
115
124
|
@dataclass
|
|
116
|
-
class
|
|
125
|
+
class UnverifiedPhysicalProperty(UnverifiedComponent[PhysicalProperty]):
|
|
117
126
|
view: str
|
|
118
127
|
view_property: str | None
|
|
119
|
-
value_type: str | DataType | ViewEntity |
|
|
128
|
+
value_type: str | DataType | ViewEntity | PhysicalUnknownEntity
|
|
120
129
|
name: str | None = None
|
|
121
130
|
description: str | None = None
|
|
122
131
|
connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | str | None = None
|
|
@@ -129,7 +138,7 @@ class DMSInputProperty(UnverifiedComponent[DMSProperty]):
|
|
|
129
138
|
index: str | list[str] | None = None
|
|
130
139
|
constraint: str | list[str] | None = None
|
|
131
140
|
neatId: str | URIRef | None = None
|
|
132
|
-
|
|
141
|
+
conceptual: str | URIRef | None = None
|
|
133
142
|
|
|
134
143
|
@property
|
|
135
144
|
def nullable(self) -> bool | None:
|
|
@@ -145,8 +154,8 @@ class DMSInputProperty(UnverifiedComponent[DMSProperty]):
|
|
|
145
154
|
)
|
|
146
155
|
|
|
147
156
|
@classmethod
|
|
148
|
-
def _get_verified_cls(cls) -> type[
|
|
149
|
-
return
|
|
157
|
+
def _get_verified_cls(cls) -> type[PhysicalProperty]:
|
|
158
|
+
return PhysicalProperty
|
|
150
159
|
|
|
151
160
|
def dump(self, default_space: str, default_version: str) -> dict[str, Any]: # type: ignore[override]
|
|
152
161
|
output = super().dump()
|
|
@@ -206,7 +215,7 @@ class DMSInputProperty(UnverifiedComponent[DMSProperty]):
|
|
|
206
215
|
|
|
207
216
|
|
|
208
217
|
@dataclass
|
|
209
|
-
class
|
|
218
|
+
class UnverifiedPhysicalContainer(UnverifiedComponent[PhysicalContainer]):
|
|
210
219
|
container: str
|
|
211
220
|
name: str | None = None
|
|
212
221
|
description: str | None = None
|
|
@@ -215,8 +224,8 @@ class DMSInputContainer(UnverifiedComponent[DMSContainer]):
|
|
|
215
224
|
used_for: Literal["node", "edge", "all"] | None = None
|
|
216
225
|
|
|
217
226
|
@classmethod
|
|
218
|
-
def _get_verified_cls(cls) -> type[
|
|
219
|
-
return
|
|
227
|
+
def _get_verified_cls(cls) -> type[PhysicalContainer]:
|
|
228
|
+
return PhysicalContainer
|
|
220
229
|
|
|
221
230
|
def dump(self, default_space: str) -> dict[str, Any]: # type: ignore[override]
|
|
222
231
|
output = super().dump()
|
|
@@ -232,7 +241,7 @@ class DMSInputContainer(UnverifiedComponent[DMSContainer]):
|
|
|
232
241
|
return ContainerEntity.load(self.container, strict=True, space=default_space)
|
|
233
242
|
|
|
234
243
|
@classmethod
|
|
235
|
-
def from_container(cls, container: dm.ContainerApply) -> "
|
|
244
|
+
def from_container(cls, container: dm.ContainerApply) -> "UnverifiedPhysicalContainer":
|
|
236
245
|
constraints: list[str] = []
|
|
237
246
|
for _, constraint_obj in (container.constraints or {}).items():
|
|
238
247
|
if isinstance(constraint_obj, dm.RequiresConstraint):
|
|
@@ -249,7 +258,7 @@ class DMSInputContainer(UnverifiedComponent[DMSContainer]):
|
|
|
249
258
|
|
|
250
259
|
|
|
251
260
|
@dataclass
|
|
252
|
-
class
|
|
261
|
+
class UnverifiedPhysicalView(UnverifiedComponent[PhysicalView]):
|
|
253
262
|
view: str
|
|
254
263
|
name: str | None = None
|
|
255
264
|
description: str | None = None
|
|
@@ -257,15 +266,15 @@ class DMSInputView(UnverifiedComponent[DMSView]):
|
|
|
257
266
|
filter_: Literal["hasData", "nodeType", "rawFilter"] | str | None = None
|
|
258
267
|
in_model: bool = True
|
|
259
268
|
neatId: str | URIRef | None = None
|
|
260
|
-
|
|
269
|
+
conceptual: str | URIRef | None = None
|
|
261
270
|
|
|
262
271
|
def __post_init__(self) -> None:
|
|
263
272
|
if self.in_model is None:
|
|
264
273
|
self.in_model = True
|
|
265
274
|
|
|
266
275
|
@classmethod
|
|
267
|
-
def _get_verified_cls(cls) -> type[
|
|
268
|
-
return
|
|
276
|
+
def _get_verified_cls(cls) -> type[PhysicalView]:
|
|
277
|
+
return PhysicalView
|
|
269
278
|
|
|
270
279
|
def dump(self, default_space: str, default_version: str) -> dict[str, Any]: # type: ignore[override]
|
|
271
280
|
output = super().dump()
|
|
@@ -292,7 +301,7 @@ class DMSInputView(UnverifiedComponent[DMSView]):
|
|
|
292
301
|
return self._load_implements(default_space, default_version) or []
|
|
293
302
|
|
|
294
303
|
@classmethod
|
|
295
|
-
def from_view(cls, view: dm.ViewApply, in_model: bool) -> "
|
|
304
|
+
def from_view(cls, view: dm.ViewApply, in_model: bool) -> "UnverifiedPhysicalView":
|
|
296
305
|
view_entity = ViewEntity.from_id(view.as_id())
|
|
297
306
|
|
|
298
307
|
return cls(
|
|
@@ -307,7 +316,7 @@ class DMSInputView(UnverifiedComponent[DMSView]):
|
|
|
307
316
|
|
|
308
317
|
|
|
309
318
|
@dataclass
|
|
310
|
-
class
|
|
319
|
+
class UnverifiedPhysicalNodeType(UnverifiedComponent[PhysicalNodeType]):
|
|
311
320
|
node: str
|
|
312
321
|
usage: Literal["type", "collocation"]
|
|
313
322
|
name: str | None = None
|
|
@@ -315,11 +324,11 @@ class DMSInputNode(UnverifiedComponent[DMSNode]):
|
|
|
315
324
|
neatId: str | URIRef | None = None
|
|
316
325
|
|
|
317
326
|
@classmethod
|
|
318
|
-
def _get_verified_cls(cls) -> type[
|
|
319
|
-
return
|
|
327
|
+
def _get_verified_cls(cls) -> type[PhysicalNodeType]:
|
|
328
|
+
return PhysicalNodeType
|
|
320
329
|
|
|
321
330
|
@classmethod
|
|
322
|
-
def from_node_type(cls, node_type: dm.NodeApply) -> "
|
|
331
|
+
def from_node_type(cls, node_type: dm.NodeApply) -> "UnverifiedPhysicalNodeType":
|
|
323
332
|
return cls(node=f"{node_type.space}:{node_type.external_id}", usage="type")
|
|
324
333
|
|
|
325
334
|
def dump(self, default_space: str, **_) -> dict[str, Any]: # type: ignore
|
|
@@ -329,7 +338,7 @@ class DMSInputNode(UnverifiedComponent[DMSNode]):
|
|
|
329
338
|
|
|
330
339
|
|
|
331
340
|
@dataclass
|
|
332
|
-
class
|
|
341
|
+
class UnverifiedPhysicalEnum(UnverifiedComponent[PhysicalEnum]):
|
|
333
342
|
collection: str
|
|
334
343
|
value: str
|
|
335
344
|
name: str | None = None
|
|
@@ -337,22 +346,22 @@ class DMSInputEnum(UnverifiedComponent[DMSEnum]):
|
|
|
337
346
|
neatId: str | URIRef | None = None
|
|
338
347
|
|
|
339
348
|
@classmethod
|
|
340
|
-
def _get_verified_cls(cls) -> type[
|
|
341
|
-
return
|
|
349
|
+
def _get_verified_cls(cls) -> type[PhysicalEnum]:
|
|
350
|
+
return PhysicalEnum
|
|
342
351
|
|
|
343
352
|
|
|
344
353
|
@dataclass
|
|
345
|
-
class
|
|
346
|
-
metadata:
|
|
347
|
-
properties: list[
|
|
348
|
-
views: list[
|
|
349
|
-
containers: list[
|
|
350
|
-
enum: list[
|
|
351
|
-
nodes: list[
|
|
354
|
+
class UnverifiedPhysicalDataModel(UnverifiedDataModel[PhysicalDataModel]):
|
|
355
|
+
metadata: UnverifiedPhysicalMetadata
|
|
356
|
+
properties: list[UnverifiedPhysicalProperty]
|
|
357
|
+
views: list[UnverifiedPhysicalView]
|
|
358
|
+
containers: list[UnverifiedPhysicalContainer] | None = None
|
|
359
|
+
enum: list[UnverifiedPhysicalEnum] | None = None
|
|
360
|
+
nodes: list[UnverifiedPhysicalNodeType] | None = None
|
|
352
361
|
|
|
353
362
|
@classmethod
|
|
354
|
-
def _get_verified_cls(cls) -> type[
|
|
355
|
-
return
|
|
363
|
+
def _get_verified_cls(cls) -> type[PhysicalDataModel]:
|
|
364
|
+
return PhysicalDataModel
|
|
356
365
|
|
|
357
366
|
def dump(self) -> dict[str, Any]:
|
|
358
367
|
default_space = self.metadata.space
|
|
@@ -369,7 +378,7 @@ class DMSInputRules(UnverifiedDataModel[DMSRules]):
|
|
|
369
378
|
|
|
370
379
|
@classmethod
|
|
371
380
|
def display_type_name(cls) -> str:
|
|
372
|
-
return "
|
|
381
|
+
return "UnverifiedPhysicalModel"
|
|
373
382
|
|
|
374
383
|
@property
|
|
375
384
|
def display_name(self) -> str:
|
|
@@ -378,7 +387,7 @@ class DMSInputRules(UnverifiedDataModel[DMSRules]):
|
|
|
378
387
|
def _repr_html_(self) -> str:
|
|
379
388
|
summary = {
|
|
380
389
|
"type": "Physical Data Model",
|
|
381
|
-
"intended for": "
|
|
390
|
+
"intended for": "Data Engineer",
|
|
382
391
|
"name": self.metadata.name,
|
|
383
392
|
"space": self.metadata.space,
|
|
384
393
|
"external_id": self.metadata.external_id,
|