cognite-neat 0.121.1__py3-none-any.whl → 0.122.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/_data_model/_constants.py +23 -16
- cognite/neat/core/_data_model/_shared.py +33 -17
- cognite/neat/core/_data_model/analysis/__init__.py +2 -2
- cognite/neat/core/_data_model/analysis/_base.py +186 -183
- cognite/neat/core/_data_model/catalog/__init__.py +2 -2
- cognite/neat/core/_data_model/exporters/__init__.py +6 -6
- cognite/neat/core/_data_model/exporters/_base.py +10 -8
- cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
- cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
- cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +14 -14
- cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
- cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
- cognite/neat/core/_data_model/importers/__init__.py +8 -8
- cognite/neat/core/_data_model/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +50 -25
- cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/__init__.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/_base.py +15 -15
- cognite/neat/core/_data_model/importers/_rdf/{_imf2rules.py → _imf2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/{_inference2rules.py → _inference2rdata_model.py} +59 -59
- cognite/neat/core/_data_model/importers/_rdf/{_owl2rules.py → _owl2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +76 -19
- cognite/neat/core/_data_model/models/__init__.py +11 -9
- cognite/neat/core/_data_model/models/_base_unverified.py +12 -12
- cognite/neat/core/_data_model/models/_base_verified.py +9 -14
- cognite/neat/core/_data_model/models/_types.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
- cognite/neat/core/_data_model/models/conceptual/_validation.py +88 -78
- cognite/neat/core/_data_model/models/conceptual/_verified.py +54 -52
- cognite/neat/core/_data_model/models/data_types.py +2 -2
- cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
- cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
- cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
- cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
- cognite/neat/core/_data_model/models/entities/_types.py +9 -3
- cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
- cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +75 -55
- cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
- cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +17 -15
- cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
- cognite/neat/core/_data_model/transformers/__init__.py +29 -25
- cognite/neat/core/_data_model/transformers/_base.py +27 -20
- cognite/neat/core/_data_model/transformers/_converters.py +707 -622
- cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
- cognite/neat/core/_data_model/transformers/_verification.py +64 -55
- cognite/neat/core/_instances/extractors/_base.py +2 -2
- cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +9 -9
- cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
- cognite/neat/core/_instances/extractors/_mock_graph_generator.py +107 -103
- cognite/neat/core/_instances/loaders/_base.py +3 -3
- cognite/neat/core/_instances/loaders/_rdf2dms.py +22 -22
- cognite/neat/core/_instances/transformers/_base.py +7 -4
- cognite/neat/core/_instances/transformers/_rdfpath.py +1 -1
- cognite/neat/core/_instances/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/_factory.py +1 -1
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_resources.py +1 -1
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_issues/warnings/_properties.py +1 -1
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +2 -2
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/text.py +1 -1
- cognite/neat/session/_base.py +42 -36
- cognite/neat/session/_drop.py +2 -2
- cognite/neat/session/_experimental.py +1 -1
- cognite/neat/session/_inspect.py +13 -13
- cognite/neat/session/_mapping.py +15 -9
- cognite/neat/session/_read.py +39 -37
- cognite/neat/session/_set.py +6 -6
- cognite/neat/session/_show.py +24 -21
- cognite/neat/session/_state/README.md +1 -1
- cognite/neat/session/_state.py +27 -27
- cognite/neat/session/_subset.py +14 -11
- cognite/neat/session/_template.py +23 -21
- cognite/neat/session/_to.py +42 -42
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/METADATA +14 -7
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/RECORD +102 -100
- cognite/neat/core/_data_model/exporters/_validation.py +0 -14
- cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
- /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/licenses/LICENSE +0 -0
cognite/neat/session/_base.py
CHANGED
|
@@ -7,21 +7,21 @@ from cognite.client import data_modeling as dm
|
|
|
7
7
|
from cognite.neat import _version
|
|
8
8
|
from cognite.neat.core._client import NeatClient
|
|
9
9
|
from cognite.neat.core._data_model import importers
|
|
10
|
-
from cognite.neat.core._data_model.models import
|
|
10
|
+
from cognite.neat.core._data_model.models import PhysicalDataModel
|
|
11
11
|
from cognite.neat.core._data_model.models.conceptual._verified import (
|
|
12
12
|
ConceptualDataModel,
|
|
13
13
|
)
|
|
14
14
|
from cognite.neat.core._data_model.transformers import (
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
ConceptualToPhysical,
|
|
16
|
+
MergeConceptualDataModels,
|
|
17
|
+
MergePhysicalDataModels,
|
|
18
18
|
ToDMSCompliantEntities,
|
|
19
|
-
|
|
19
|
+
VerifyConceptualDataModel,
|
|
20
20
|
)
|
|
21
21
|
from cognite.neat.core._issues import IssueList
|
|
22
22
|
from cognite.neat.core._issues.errors import RegexViolationError
|
|
23
23
|
from cognite.neat.core._issues.errors._general import NeatImportError
|
|
24
|
-
from cognite.neat.core._store.
|
|
24
|
+
from cognite.neat.core._store._data_model import DataModelEntity
|
|
25
25
|
from cognite.neat.core._utils.auxiliary import local_import
|
|
26
26
|
|
|
27
27
|
from ._collector import _COLLECTOR, Collector
|
|
@@ -162,20 +162,22 @@ class NeatSession:
|
|
|
162
162
|
reserved_properties: What to do with reserved properties. Can be "error" or "warning".
|
|
163
163
|
|
|
164
164
|
Example:
|
|
165
|
-
Convert to
|
|
165
|
+
Convert to Physical Data Model
|
|
166
166
|
```python
|
|
167
167
|
neat.convert()
|
|
168
168
|
```
|
|
169
169
|
"""
|
|
170
170
|
self._state._raise_exception_if_condition_not_met(
|
|
171
|
-
"Convert to physical",
|
|
171
|
+
"Convert to physical",
|
|
172
|
+
has_physical_data_model=False,
|
|
173
|
+
has_conceptual_data_model=True,
|
|
172
174
|
)
|
|
173
|
-
converter =
|
|
175
|
+
converter = ConceptualToPhysical(reserved_properties=reserved_properties, client=self._state.client)
|
|
174
176
|
|
|
175
177
|
issues = self._state.rule_transform(converter)
|
|
176
178
|
|
|
177
179
|
if self._verbose and not issues.has_errors:
|
|
178
|
-
print("
|
|
180
|
+
print("Conceptual data model converted to physical data model.")
|
|
179
181
|
else:
|
|
180
182
|
print("Conversion failed.")
|
|
181
183
|
if issues:
|
|
@@ -231,50 +233,54 @@ class NeatSession:
|
|
|
231
233
|
),
|
|
232
234
|
) -> IssueList:
|
|
233
235
|
"""Infer data model from instances."""
|
|
234
|
-
last_entity:
|
|
235
|
-
if self._state.
|
|
236
|
-
last_entity = self._state.
|
|
237
|
-
|
|
238
|
-
# Note that this importer behaves as a transformer in the
|
|
239
|
-
#
|
|
236
|
+
last_entity: DataModelEntity | None = None
|
|
237
|
+
if self._state.data_model_store.provenance:
|
|
238
|
+
last_entity = self._state.data_model_store.provenance[-1].target_entity
|
|
239
|
+
|
|
240
|
+
# Note that this importer behaves as a transformer in the data model store when there
|
|
241
|
+
# is an existing data model.
|
|
242
|
+
# We are essentially transforming the last entity's conceptual data model
|
|
243
|
+
# into a new conceptual data model.
|
|
240
244
|
importer = importers.SubclassInferenceImporter(
|
|
241
245
|
issue_list=IssueList(),
|
|
242
246
|
graph=self._state.instances.store.graph(),
|
|
243
|
-
|
|
244
|
-
data_model_id=dm.DataModelId.load(model_id) if last_entity is None else None,
|
|
247
|
+
data_model=last_entity.conceptual if last_entity is not None else None,
|
|
248
|
+
data_model_id=(dm.DataModelId.load(model_id) if last_entity is None else None),
|
|
245
249
|
)
|
|
246
250
|
|
|
247
|
-
def action() -> tuple[ConceptualDataModel,
|
|
248
|
-
|
|
249
|
-
|
|
251
|
+
def action() -> tuple[ConceptualDataModel, PhysicalDataModel | None]:
|
|
252
|
+
unverified_conceptual = importer.to_data_model()
|
|
253
|
+
unverified_conceptual = ToDMSCompliantEntities(rename_warning="raise").transform(unverified_conceptual)
|
|
250
254
|
|
|
251
|
-
|
|
255
|
+
extra_conceptual = VerifyConceptualDataModel().transform(unverified_conceptual)
|
|
252
256
|
if not last_entity:
|
|
253
|
-
return
|
|
254
|
-
|
|
255
|
-
if not last_entity.
|
|
256
|
-
return
|
|
257
|
+
return extra_conceptual, None
|
|
258
|
+
merged_conceptual = MergeConceptualDataModels(extra_conceptual).transform(last_entity.conceptual)
|
|
259
|
+
if not last_entity.physical:
|
|
260
|
+
return merged_conceptual, None
|
|
257
261
|
|
|
258
|
-
|
|
262
|
+
extra_physical = ConceptualToPhysical(reserved_properties="warning", client=self._state.client).transform(
|
|
263
|
+
extra_conceptual
|
|
264
|
+
)
|
|
259
265
|
|
|
260
|
-
|
|
261
|
-
return
|
|
266
|
+
merged_physical = MergePhysicalDataModels(extra_physical).transform(last_entity.physical)
|
|
267
|
+
return merged_conceptual, merged_physical
|
|
262
268
|
|
|
263
|
-
return self._state.
|
|
269
|
+
return self._state.data_model_store.do_activity(action, importer)
|
|
264
270
|
|
|
265
271
|
def _repr_html_(self) -> str:
|
|
266
272
|
state = self._state
|
|
267
|
-
if state.instances.empty and state.
|
|
273
|
+
if state.instances.empty and state.data_model_store.empty:
|
|
268
274
|
return "<strong>Empty session</strong>. Get started by reading something with the <em>.read</em> attribute."
|
|
269
275
|
|
|
270
276
|
output = []
|
|
271
277
|
|
|
272
|
-
if state.
|
|
273
|
-
last_entity = state.
|
|
274
|
-
if last_entity.
|
|
275
|
-
html = last_entity.
|
|
278
|
+
if state.data_model_store.provenance:
|
|
279
|
+
last_entity = state.data_model_store.provenance[-1].target_entity
|
|
280
|
+
if last_entity.physical:
|
|
281
|
+
html = last_entity.physical._repr_html_()
|
|
276
282
|
else:
|
|
277
|
-
html = last_entity.
|
|
283
|
+
html = last_entity.conceptual._repr_html_()
|
|
278
284
|
output.append(f"<H2>Data Model</H2><br />{html}") # type: ignore
|
|
279
285
|
|
|
280
286
|
if not state.instances.empty:
|
cognite/neat/session/_drop.py
CHANGED
|
@@ -78,7 +78,7 @@ class DropDataModelAPI:
|
|
|
78
78
|
"""
|
|
79
79
|
if sum([view_external_id is not None, group is not None]) != 1:
|
|
80
80
|
raise NeatSessionError("Only one of view_external_id or group can be specified.")
|
|
81
|
-
last_dms = self._state.
|
|
81
|
+
last_dms = self._state.data_model_store.last_verified_physical_data_model
|
|
82
82
|
if group is not None and last_dms.metadata.as_data_model_id() not in COGNITE_MODELS:
|
|
83
83
|
raise NeatSessionError("Group can only be specified for CogniteCore models.")
|
|
84
84
|
if view_external_id is not None:
|
|
@@ -98,6 +98,6 @@ class DropDataModelAPI:
|
|
|
98
98
|
)
|
|
99
99
|
before = len(last_dms.views)
|
|
100
100
|
issues = self._state.rule_transform(DropModelViews(view_external_id, group))
|
|
101
|
-
after = len(self._state.
|
|
101
|
+
after = len(self._state.data_model_store.last_verified_physical_data_model.views)
|
|
102
102
|
print(f"Dropped {before - after} views.")
|
|
103
103
|
return issues
|
|
@@ -10,7 +10,7 @@ class ExperimentalFeatureWarning(UserWarning):
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class ExperimentalFlags:
|
|
13
|
-
|
|
13
|
+
manual_data_model_edit = ExperimentalFeatureWarning("enable_manual_edit")
|
|
14
14
|
same_space_properties_only_export = ExperimentalFeatureWarning("same-space-properties-only")
|
|
15
15
|
standardize_naming = ExperimentalFeatureWarning("standardize_naming")
|
|
16
16
|
standardize_space_and_version = ExperimentalFeatureWarning("standardize_space_and_version")
|
cognite/neat/session/_inspect.py
CHANGED
|
@@ -62,23 +62,23 @@ class InspectAPI:
|
|
|
62
62
|
neat.inspect.properties
|
|
63
63
|
```
|
|
64
64
|
"""
|
|
65
|
-
if self._state.
|
|
65
|
+
if self._state.data_model_store.empty:
|
|
66
66
|
return pd.DataFrame()
|
|
67
|
-
last_entity = self._state.
|
|
68
|
-
if last_entity.
|
|
69
|
-
df = last_entity.
|
|
67
|
+
last_entity = self._state.data_model_store.provenance[-1].target_entity
|
|
68
|
+
if last_entity.physical:
|
|
69
|
+
df = last_entity.physical.properties.to_pandas()
|
|
70
70
|
else:
|
|
71
|
-
df = last_entity.
|
|
71
|
+
df = last_entity.conceptual.properties.to_pandas()
|
|
72
72
|
df.drop(columns=["neatId"], errors="ignore", inplace=True)
|
|
73
73
|
return df
|
|
74
74
|
|
|
75
75
|
def views(self) -> pd.DataFrame:
|
|
76
|
-
if self._state.
|
|
76
|
+
if self._state.data_model_store.empty:
|
|
77
77
|
return pd.DataFrame()
|
|
78
|
-
last_entity = self._state.
|
|
79
|
-
if last_entity.
|
|
78
|
+
last_entity = self._state.data_model_store.provenance[-1].target_entity
|
|
79
|
+
if last_entity.physical is None:
|
|
80
80
|
return pd.DataFrame()
|
|
81
|
-
df = last_entity.
|
|
81
|
+
df = last_entity.physical.views.to_pandas()
|
|
82
82
|
df.drop(columns=["neatId"], errors="ignore", inplace=True)
|
|
83
83
|
return df
|
|
84
84
|
|
|
@@ -95,7 +95,7 @@ class InspectIssues:
|
|
|
95
95
|
def __call__(
|
|
96
96
|
self,
|
|
97
97
|
search: str | None = None,
|
|
98
|
-
include: Literal["all", "errors", "warning"] | Set[Literal["all", "errors", "warning"]] = "all",
|
|
98
|
+
include: (Literal["all", "errors", "warning"] | Set[Literal["all", "errors", "warning"]]) = "all",
|
|
99
99
|
return_dataframe: Literal[True] = (False if IN_NOTEBOOK else True), # type: ignore[assignment]
|
|
100
100
|
) -> pd.DataFrame: ...
|
|
101
101
|
|
|
@@ -103,7 +103,7 @@ class InspectIssues:
|
|
|
103
103
|
def __call__(
|
|
104
104
|
self,
|
|
105
105
|
search: str | None = None,
|
|
106
|
-
include: Literal["all", "errors", "warning"] | Set[Literal["all", "errors", "warning"]] = "all",
|
|
106
|
+
include: (Literal["all", "errors", "warning"] | Set[Literal["all", "errors", "warning"]]) = "all",
|
|
107
107
|
return_dataframe: Literal[False] = (False if IN_NOTEBOOK else True), # type: ignore[assignment]
|
|
108
108
|
) -> None: ...
|
|
109
109
|
|
|
@@ -114,7 +114,7 @@ class InspectIssues:
|
|
|
114
114
|
return_dataframe: bool = (False if IN_NOTEBOOK else True), # type: ignore[assignment]
|
|
115
115
|
) -> pd.DataFrame | None:
|
|
116
116
|
"""Returns the issues of the current data model."""
|
|
117
|
-
issues = self._state.
|
|
117
|
+
issues = self._state.data_model_store.last_issues
|
|
118
118
|
if issues is None and self._state.instances.store.provenance:
|
|
119
119
|
last_change = self._state.instances.store.provenance[-1]
|
|
120
120
|
issues = last_change.target_entity.issues
|
|
@@ -180,7 +180,7 @@ class InspectOutcome:
|
|
|
180
180
|
"""
|
|
181
181
|
|
|
182
182
|
def __init__(self, state: SessionState) -> None:
|
|
183
|
-
self.data_model = InspectUploadOutcome(lambda: state.
|
|
183
|
+
self.data_model = InspectUploadOutcome(lambda: state.data_model_store.last_outcome)
|
|
184
184
|
self.instances = InspectUploadOutcome(lambda: state.instances.last_outcome)
|
|
185
185
|
|
|
186
186
|
|
cognite/neat/session/_mapping.py
CHANGED
|
@@ -3,8 +3,8 @@ from cognite.neat.core._data_model.transformers import (
|
|
|
3
3
|
AsParentPropertyId,
|
|
4
4
|
ChangeViewPrefix,
|
|
5
5
|
IncludeReferenced,
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
PhysicalDataModelMapper,
|
|
7
|
+
VerifiedDataModelTransformer,
|
|
8
8
|
)
|
|
9
9
|
from cognite.neat.core._issues import IssueList
|
|
10
10
|
|
|
@@ -42,21 +42,27 @@ class DataModelMappingAPI:
|
|
|
42
42
|
neat.mapping.classic_to_core(company_prefix="WindFarmX", use_parent_property_name=True)
|
|
43
43
|
```
|
|
44
44
|
"""
|
|
45
|
-
if self._state.
|
|
46
|
-
raise NeatSessionError("No
|
|
47
|
-
last_entity = self._state.
|
|
48
|
-
if last_entity.
|
|
45
|
+
if self._state.data_model_store.empty:
|
|
46
|
+
raise NeatSessionError("No data model to map")
|
|
47
|
+
last_entity = self._state.data_model_store.provenance[-1].target_entity
|
|
48
|
+
if last_entity.physical is None:
|
|
49
49
|
raise NeatSessionError("Data model not converted to DMS. Try running `neat.convert('dms')` first.")
|
|
50
|
-
|
|
50
|
+
data_model = last_entity.physical
|
|
51
51
|
if self._state.client is None:
|
|
52
52
|
raise NeatSessionError("Client is required to map classic to core")
|
|
53
53
|
|
|
54
|
-
transformers: list[
|
|
54
|
+
transformers: list[VerifiedDataModelTransformer] = []
|
|
55
55
|
if company_prefix:
|
|
56
56
|
transformers.append(ChangeViewPrefix("Classic", company_prefix))
|
|
57
57
|
transformers.extend(
|
|
58
58
|
[
|
|
59
|
-
|
|
59
|
+
PhysicalDataModelMapper(
|
|
60
|
+
load_classic_to_core_mapping(
|
|
61
|
+
company_prefix,
|
|
62
|
+
data_model.metadata.space,
|
|
63
|
+
data_model.metadata.version,
|
|
64
|
+
)
|
|
65
|
+
),
|
|
60
66
|
IncludeReferenced(self._state.client),
|
|
61
67
|
]
|
|
62
68
|
)
|
cognite/neat/session/_read.py
CHANGED
|
@@ -15,7 +15,7 @@ from cognite.neat.core._data_model.models.entities._single_value import ViewEnti
|
|
|
15
15
|
from cognite.neat.core._data_model.transformers import ClassicPrepareCore
|
|
16
16
|
from cognite.neat.core._data_model.transformers._converters import (
|
|
17
17
|
ToEnterpriseModel,
|
|
18
|
-
|
|
18
|
+
_SubsetEditableCDMPhysicalDataModel,
|
|
19
19
|
)
|
|
20
20
|
from cognite.neat.core._instances import examples as instances_examples
|
|
21
21
|
from cognite.neat.core._instances import extractors
|
|
@@ -113,7 +113,7 @@ class CDFReadAPI(BaseReadAPI):
|
|
|
113
113
|
|
|
114
114
|
self._state._raise_exception_if_condition_not_met(
|
|
115
115
|
"Read data model from CDF",
|
|
116
|
-
|
|
116
|
+
empty_data_model_store_required=True,
|
|
117
117
|
client_required=True,
|
|
118
118
|
)
|
|
119
119
|
|
|
@@ -148,7 +148,7 @@ class CDFReadAPI(BaseReadAPI):
|
|
|
148
148
|
|
|
149
149
|
self._state._raise_exception_if_condition_not_met(
|
|
150
150
|
"Subset Core Data Model",
|
|
151
|
-
|
|
151
|
+
empty_data_model_store_required=True,
|
|
152
152
|
client_required=True,
|
|
153
153
|
)
|
|
154
154
|
|
|
@@ -164,7 +164,7 @@ class CDFReadAPI(BaseReadAPI):
|
|
|
164
164
|
if issues.has_errors:
|
|
165
165
|
return issues
|
|
166
166
|
|
|
167
|
-
|
|
167
|
+
cdm_data_model = self._state.data_model_store.last_verified_data_model
|
|
168
168
|
|
|
169
169
|
issues.extend(
|
|
170
170
|
self._state.rule_transform(
|
|
@@ -182,7 +182,7 @@ class CDFReadAPI(BaseReadAPI):
|
|
|
182
182
|
|
|
183
183
|
issues.extend(
|
|
184
184
|
self._state.rule_transform(
|
|
185
|
-
|
|
185
|
+
_SubsetEditableCDMPhysicalDataModel(
|
|
186
186
|
views={
|
|
187
187
|
ViewEntity(
|
|
188
188
|
space=cdm_v1.space,
|
|
@@ -195,8 +195,8 @@ class CDFReadAPI(BaseReadAPI):
|
|
|
195
195
|
)
|
|
196
196
|
)
|
|
197
197
|
|
|
198
|
-
if
|
|
199
|
-
self._state.last_reference =
|
|
198
|
+
if cdm_data_model and not issues.has_errors:
|
|
199
|
+
self._state.last_reference = cdm_data_model
|
|
200
200
|
|
|
201
201
|
return issues
|
|
202
202
|
|
|
@@ -220,7 +220,7 @@ class CDFReadAPI(BaseReadAPI):
|
|
|
220
220
|
"""
|
|
221
221
|
self._state._raise_exception_if_condition_not_met(
|
|
222
222
|
"Read DMS Graph",
|
|
223
|
-
|
|
223
|
+
empty_data_model_store_required=True,
|
|
224
224
|
empty_instances_store_required=True,
|
|
225
225
|
client_required=True,
|
|
226
226
|
)
|
|
@@ -350,7 +350,7 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
350
350
|
"""
|
|
351
351
|
self._state._raise_exception_if_condition_not_met(
|
|
352
352
|
"Read classic graph",
|
|
353
|
-
|
|
353
|
+
empty_data_model_store_required=True,
|
|
354
354
|
empty_instances_store_required=True,
|
|
355
355
|
client_required=True,
|
|
356
356
|
)
|
|
@@ -399,7 +399,7 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
399
399
|
# The above transformations creates a new type, so we need to update
|
|
400
400
|
self._state.instances.neat_prefix_by_type_uri.update({namespace["ClassicSourceSystem"]: "ClassicSourceSystem_"})
|
|
401
401
|
# Updating the information model.
|
|
402
|
-
prepare_issues = self._state.
|
|
402
|
+
prepare_issues = self._state.data_model_store.transform(
|
|
403
403
|
ClassicPrepareCore(namespace, reference_timeseries, reference_files)
|
|
404
404
|
)
|
|
405
405
|
|
|
@@ -432,7 +432,7 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
432
432
|
namespace = CLASSIC_CDF_NAMESPACE
|
|
433
433
|
self._state._raise_exception_if_condition_not_met(
|
|
434
434
|
"Read time series",
|
|
435
|
-
|
|
435
|
+
empty_data_model_store_required=True,
|
|
436
436
|
empty_instances_store_required=True,
|
|
437
437
|
client_required=True,
|
|
438
438
|
)
|
|
@@ -491,7 +491,7 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
491
491
|
namespace = CLASSIC_CDF_NAMESPACE
|
|
492
492
|
self._state._raise_exception_if_condition_not_met(
|
|
493
493
|
"Read time series",
|
|
494
|
-
|
|
494
|
+
empty_data_model_store_required=True,
|
|
495
495
|
empty_instances_store_required=True,
|
|
496
496
|
client_required=True,
|
|
497
497
|
)
|
|
@@ -528,15 +528,15 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
528
528
|
|
|
529
529
|
@session_class_wrapper
|
|
530
530
|
class ExcelReadAPI(BaseReadAPI):
|
|
531
|
-
"""Reads a Neat Excel
|
|
532
|
-
or
|
|
531
|
+
"""Reads a Neat Excel Data Model to the data model store.
|
|
532
|
+
The data model spreadsheets may contain conceptual or physical data model definitions.
|
|
533
533
|
|
|
534
534
|
Args:
|
|
535
535
|
io: file path to the Excel sheet
|
|
536
536
|
|
|
537
537
|
Example:
|
|
538
538
|
```python
|
|
539
|
-
neat.read.excel("
|
|
539
|
+
neat.read.excel("conceptual_or_physical_data_model.xlsx")
|
|
540
540
|
```
|
|
541
541
|
"""
|
|
542
542
|
|
|
@@ -544,27 +544,28 @@ class ExcelReadAPI(BaseReadAPI):
|
|
|
544
544
|
super().__init__(state, verbose)
|
|
545
545
|
|
|
546
546
|
def __call__(self, io: Any, enable_manual_edit: bool = False) -> IssueList:
|
|
547
|
-
"""Reads a Neat Excel
|
|
548
|
-
or
|
|
547
|
+
"""Reads a Neat Excel Data Model to the data model store.
|
|
548
|
+
The data model spreadsheets may contain conceptual or physical data model definitions.
|
|
549
549
|
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
550
|
+
Args:
|
|
551
|
+
io: file path to the Excel sheet
|
|
552
|
+
enable_manual_edit: If True, the user will be able to re-import data model
|
|
553
|
+
which where edit outside NeatSession
|
|
553
554
|
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
555
|
+
!!! note "Manual Edit Warning"
|
|
556
|
+
This is an alpha feature and is subject to change without notice.
|
|
557
|
+
It is expected to have some limitations and may not work as expected in all cases.
|
|
557
558
|
"""
|
|
558
559
|
reader = NeatReader.create(io)
|
|
559
560
|
path = reader.materialize_path()
|
|
560
561
|
|
|
561
562
|
if enable_manual_edit:
|
|
562
563
|
warnings.filterwarnings("default")
|
|
563
|
-
ExperimentalFlags.
|
|
564
|
+
ExperimentalFlags.manual_data_model_edit.warn()
|
|
564
565
|
else:
|
|
565
566
|
self._state._raise_exception_if_condition_not_met(
|
|
566
|
-
"Read Excel
|
|
567
|
-
|
|
567
|
+
"Read Excel Data Model",
|
|
568
|
+
empty_data_model_store_required=True,
|
|
568
569
|
)
|
|
569
570
|
|
|
570
571
|
return self._state.rule_import(importers.ExcelImporter(path), enable_manual_edit)
|
|
@@ -573,7 +574,8 @@ class ExcelReadAPI(BaseReadAPI):
|
|
|
573
574
|
@session_class_wrapper
|
|
574
575
|
class YamlReadAPI(BaseReadAPI):
|
|
575
576
|
def __call__(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> IssueList:
|
|
576
|
-
"""Reads a yaml with either neat
|
|
577
|
+
"""Reads a yaml with either neat data mode, or several toolkit yaml files to
|
|
578
|
+
import Data Model(s) into NeatSession.
|
|
577
579
|
|
|
578
580
|
Args:
|
|
579
581
|
io: File path to the Yaml file in the case of "neat" yaml, or path to a zip folder or directory with several
|
|
@@ -587,13 +589,13 @@ class YamlReadAPI(BaseReadAPI):
|
|
|
587
589
|
"""
|
|
588
590
|
self._state._raise_exception_if_condition_not_met(
|
|
589
591
|
"Read YAML data model",
|
|
590
|
-
|
|
592
|
+
empty_data_model_store_required=True,
|
|
591
593
|
)
|
|
592
594
|
reader = NeatReader.create(io)
|
|
593
595
|
path = reader.materialize_path()
|
|
594
596
|
importer: BaseImporter
|
|
595
597
|
if format == "neat":
|
|
596
|
-
importer = importers.
|
|
598
|
+
importer = importers.DictImporter.from_yaml_file(path, source_name=f"{reader!s}")
|
|
597
599
|
elif format == "toolkit":
|
|
598
600
|
dms_importer = importers.DMSImporter.from_path(path, self._state.client)
|
|
599
601
|
if dms_importer.issue_list.has_warning_type(MissingCogniteClientWarning):
|
|
@@ -695,7 +697,7 @@ class XMLReadAPI(BaseReadAPI):
|
|
|
695
697
|
|
|
696
698
|
self._state._raise_exception_if_condition_not_met(
|
|
697
699
|
"Read DEXPI file",
|
|
698
|
-
|
|
700
|
+
empty_data_model_store_required=True,
|
|
699
701
|
empty_instances_store_required=True,
|
|
700
702
|
)
|
|
701
703
|
|
|
@@ -757,7 +759,7 @@ class XMLReadAPI(BaseReadAPI):
|
|
|
757
759
|
|
|
758
760
|
self._state._raise_exception_if_condition_not_met(
|
|
759
761
|
"Read AML file",
|
|
760
|
-
|
|
762
|
+
empty_data_model_store_required=True,
|
|
761
763
|
empty_instances_store_required=True,
|
|
762
764
|
)
|
|
763
765
|
|
|
@@ -817,7 +819,7 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
817
819
|
|
|
818
820
|
self._state._raise_exception_if_condition_not_met(
|
|
819
821
|
"Read Ontology file",
|
|
820
|
-
|
|
822
|
+
empty_data_model_store_required=True,
|
|
821
823
|
)
|
|
822
824
|
|
|
823
825
|
reader = NeatReader.create(io)
|
|
@@ -840,7 +842,7 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
840
842
|
|
|
841
843
|
self._state._raise_exception_if_condition_not_met(
|
|
842
844
|
"Read IMF file",
|
|
843
|
-
|
|
845
|
+
empty_data_model_store_required=True,
|
|
844
846
|
)
|
|
845
847
|
|
|
846
848
|
reader = NeatReader.create(io)
|
|
@@ -850,7 +852,7 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
850
852
|
def instances(self, io: Any) -> IssueList:
|
|
851
853
|
self._state._raise_exception_if_condition_not_met(
|
|
852
854
|
"Read RDF Instances",
|
|
853
|
-
|
|
855
|
+
empty_data_model_store_required=True,
|
|
854
856
|
)
|
|
855
857
|
reader = NeatReader.create(io)
|
|
856
858
|
self._state.instances.store.write(extractors.RdfFileExtractor(reader.materialize_path()))
|
|
@@ -898,7 +900,7 @@ class Examples:
|
|
|
898
900
|
self._state._raise_exception_if_condition_not_met(
|
|
899
901
|
"Read Nordic44 graph example",
|
|
900
902
|
empty_instances_store_required=True,
|
|
901
|
-
|
|
903
|
+
empty_data_model_store_required=True,
|
|
902
904
|
)
|
|
903
905
|
|
|
904
906
|
self._state.instances.store.write(extractors.RdfFileExtractor(instances_examples.nordic44_knowledge_graph))
|
|
@@ -909,7 +911,7 @@ class Examples:
|
|
|
909
911
|
|
|
910
912
|
self._state._raise_exception_if_condition_not_met(
|
|
911
913
|
"Read Pump Data Model example",
|
|
912
|
-
|
|
914
|
+
empty_data_model_store_required=True,
|
|
913
915
|
)
|
|
914
916
|
|
|
915
917
|
importer: importers.ExcelImporter = importers.ExcelImporter(catalog.hello_world_pump)
|
|
@@ -920,7 +922,7 @@ class Examples:
|
|
|
920
922
|
|
|
921
923
|
self._state._raise_exception_if_condition_not_met(
|
|
922
924
|
"Read Core Data Model example",
|
|
923
|
-
|
|
925
|
+
empty_data_model_store_required=True,
|
|
924
926
|
client_required=True,
|
|
925
927
|
)
|
|
926
928
|
|
cognite/neat/session/_set.py
CHANGED
|
@@ -3,7 +3,7 @@ from cognite.client import data_modeling as dm
|
|
|
3
3
|
|
|
4
4
|
from cognite.neat.core._client import NeatClient
|
|
5
5
|
from cognite.neat.core._constants import COGNITE_MODELS
|
|
6
|
-
from cognite.neat.core._data_model.models import
|
|
6
|
+
from cognite.neat.core._data_model.models import PhysicalDataModel
|
|
7
7
|
from cognite.neat.core._data_model.transformers import SetIDDMSModel
|
|
8
8
|
from cognite.neat.core._instances.transformers import SetType
|
|
9
9
|
from cognite.neat.core._issues import IssueList
|
|
@@ -39,11 +39,11 @@ class SetAPI:
|
|
|
39
39
|
neat.set.data_model_id(("my_data_model_space", "MyDataModel", "v1"), name="My Data Model")
|
|
40
40
|
```
|
|
41
41
|
"""
|
|
42
|
-
if self._state.
|
|
43
|
-
raise NeatSessionError("No
|
|
44
|
-
|
|
45
|
-
if isinstance(
|
|
46
|
-
if
|
|
42
|
+
if self._state.data_model_store.empty:
|
|
43
|
+
raise NeatSessionError("No data model to set the data model ID.")
|
|
44
|
+
data_model = self._state.data_model_store.provenance[-1].target_entity.physical
|
|
45
|
+
if isinstance(data_model, PhysicalDataModel):
|
|
46
|
+
if data_model.metadata.as_data_model_id() in COGNITE_MODELS:
|
|
47
47
|
raise NeatSessionError(
|
|
48
48
|
"Cannot change the data model ID of a Cognite Data Model in NeatSession"
|
|
49
49
|
" due to temporarily issue with the reverse direct relation interpretation"
|
cognite/neat/session/_show.py
CHANGED
|
@@ -7,7 +7,7 @@ from IPython.display import HTML, display
|
|
|
7
7
|
from pyvis.network import Network as PyVisNetwork # type: ignore
|
|
8
8
|
|
|
9
9
|
from cognite.neat.core._constants import IN_NOTEBOOK, IN_PYODIDE
|
|
10
|
-
from cognite.neat.core._data_model.analysis._base import
|
|
10
|
+
from cognite.neat.core._data_model.analysis._base import DataModelAnalysis
|
|
11
11
|
from cognite.neat.core._utils.io_ import to_directory_compatible
|
|
12
12
|
from cognite.neat.core._utils.rdf_ import remove_namespace_from_uri, uri_display_name
|
|
13
13
|
from cognite.neat.session.exceptions import NeatSessionError
|
|
@@ -95,19 +95,19 @@ class ShowDataModelAPI(ShowBaseAPI):
|
|
|
95
95
|
self.implements = ShowDataModelImplementsAPI(self._state)
|
|
96
96
|
|
|
97
97
|
def __call__(self) -> Any:
|
|
98
|
-
if self._state.
|
|
98
|
+
if self._state.data_model_store.empty:
|
|
99
99
|
raise NeatSessionError("No data model available. Try using [bold].read[/bold] to read a data model.")
|
|
100
100
|
|
|
101
|
-
last_target = self._state.
|
|
102
|
-
|
|
103
|
-
analysis =
|
|
101
|
+
last_target = self._state.data_model_store.provenance[-1].target_entity
|
|
102
|
+
data_model = last_target.physical or last_target.conceptual
|
|
103
|
+
analysis = DataModelAnalysis(physical=last_target.physical, conceptual=last_target.conceptual)
|
|
104
104
|
|
|
105
|
-
if last_target.
|
|
106
|
-
di_graph = analysis.
|
|
105
|
+
if last_target.physical is not None:
|
|
106
|
+
di_graph = analysis._physical_di_graph(format="data-model")
|
|
107
107
|
else:
|
|
108
|
-
di_graph = analysis.
|
|
108
|
+
di_graph = analysis._conceptual_di_graph(format="data-model")
|
|
109
109
|
|
|
110
|
-
identifier = to_directory_compatible(str(
|
|
110
|
+
identifier = to_directory_compatible(str(data_model.metadata.identifier))
|
|
111
111
|
name = f"{identifier}.html"
|
|
112
112
|
return self._generate_visualization(di_graph, name)
|
|
113
113
|
|
|
@@ -119,18 +119,18 @@ class ShowDataModelImplementsAPI(ShowBaseAPI):
|
|
|
119
119
|
self._state = state
|
|
120
120
|
|
|
121
121
|
def __call__(self) -> Any:
|
|
122
|
-
if self._state.
|
|
122
|
+
if self._state.data_model_store.empty:
|
|
123
123
|
raise NeatSessionError("No data model available. Try using [bold].read[/bold] to read a data model.")
|
|
124
124
|
|
|
125
|
-
last_target = self._state.
|
|
126
|
-
|
|
127
|
-
analysis =
|
|
125
|
+
last_target = self._state.data_model_store.provenance[-1].target_entity
|
|
126
|
+
data_model = last_target.physical or last_target.conceptual
|
|
127
|
+
analysis = DataModelAnalysis(physical=last_target.physical, conceptual=last_target.conceptual)
|
|
128
128
|
|
|
129
|
-
if last_target.
|
|
130
|
-
di_graph = analysis.
|
|
129
|
+
if last_target.physical is not None:
|
|
130
|
+
di_graph = analysis._physical_di_graph(format="implements")
|
|
131
131
|
else:
|
|
132
|
-
di_graph = analysis.
|
|
133
|
-
identifier = to_directory_compatible(str(
|
|
132
|
+
di_graph = analysis._conceptual_di_graph(format="implements")
|
|
133
|
+
identifier = to_directory_compatible(str(data_model.metadata.identifier))
|
|
134
134
|
name = f"{identifier}_implements.html"
|
|
135
135
|
return self._generate_visualization(di_graph, name)
|
|
136
136
|
|
|
@@ -144,18 +144,18 @@ class ShowDataModelProvenanceAPI(ShowBaseAPI):
|
|
|
144
144
|
self._state = state
|
|
145
145
|
|
|
146
146
|
def __call__(self) -> Any:
|
|
147
|
-
if not self._state.
|
|
147
|
+
if not self._state.data_model_store.provenance:
|
|
148
148
|
raise NeatSessionError("No data model available. Try using [bold].read[/bold] to load data model.")
|
|
149
149
|
|
|
150
150
|
di_graph = self._generate_dm_provenance_di_graph_and_types()
|
|
151
|
-
unique_hash = self._state.
|
|
151
|
+
unique_hash = self._state.data_model_store.calculate_provenance_hash(shorten=True)
|
|
152
152
|
return self._generate_visualization(di_graph, name=f"data_model_provenance_{unique_hash}.html")
|
|
153
153
|
|
|
154
154
|
def _generate_dm_provenance_di_graph_and_types(self) -> nx.DiGraph:
|
|
155
155
|
di_graph = nx.DiGraph()
|
|
156
156
|
hex_colored_types = _generate_hex_color_per_type(["Agent", "Entity", "Activity", "Export", "Pruned"])
|
|
157
157
|
|
|
158
|
-
for change in self._state.
|
|
158
|
+
for change in self._state.data_model_store.provenance:
|
|
159
159
|
source = uri_display_name(change.source_entity.id_)
|
|
160
160
|
target = uri_display_name(change.target_entity.id_)
|
|
161
161
|
agent = uri_display_name(change.agent.id_)
|
|
@@ -187,7 +187,10 @@ class ShowDataModelProvenanceAPI(ShowBaseAPI):
|
|
|
187
187
|
di_graph.add_edge(source, agent, label="used", color="grey")
|
|
188
188
|
di_graph.add_edge(agent, target, label="generated", color="grey")
|
|
189
189
|
|
|
190
|
-
for
|
|
190
|
+
for (
|
|
191
|
+
source_id,
|
|
192
|
+
exports,
|
|
193
|
+
) in self._state.data_model_store.exports_by_source_entity_id.items():
|
|
191
194
|
source_shorten = uri_display_name(source_id)
|
|
192
195
|
for export in exports:
|
|
193
196
|
export_id = uri_display_name(export.target_entity.id_)
|