cognite-neat 0.121.1__py3-none-any.whl → 0.121.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/_data_model/_constants.py +23 -16
- cognite/neat/core/_data_model/_shared.py +33 -17
- cognite/neat/core/_data_model/analysis/__init__.py +2 -2
- cognite/neat/core/_data_model/analysis/_base.py +186 -183
- cognite/neat/core/_data_model/catalog/__init__.py +1 -1
- cognite/neat/core/_data_model/exporters/__init__.py +5 -5
- cognite/neat/core/_data_model/exporters/_base.py +10 -8
- cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
- cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
- cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +4 -4
- cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
- cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
- cognite/neat/core/_data_model/importers/__init__.py +6 -6
- cognite/neat/core/_data_model/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +40 -20
- cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/_base.py +9 -9
- cognite/neat/core/_data_model/importers/_rdf/_imf2rules.py +15 -15
- cognite/neat/core/_data_model/importers/_rdf/_inference2rules.py +36 -36
- cognite/neat/core/_data_model/importers/_rdf/_owl2rules.py +12 -12
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +72 -12
- cognite/neat/core/_data_model/models/__init__.py +8 -8
- cognite/neat/core/_data_model/models/_base_unverified.py +1 -1
- cognite/neat/core/_data_model/models/_base_verified.py +3 -3
- cognite/neat/core/_data_model/models/_types.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
- cognite/neat/core/_data_model/models/conceptual/_validation.py +87 -77
- cognite/neat/core/_data_model/models/conceptual/_verified.py +53 -51
- cognite/neat/core/_data_model/models/data_types.py +2 -2
- cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
- cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
- cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
- cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
- cognite/neat/core/_data_model/models/entities/_types.py +9 -3
- cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
- cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +71 -52
- cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
- cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +13 -11
- cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
- cognite/neat/core/_data_model/transformers/__init__.py +27 -23
- cognite/neat/core/_data_model/transformers/_base.py +26 -19
- cognite/neat/core/_data_model/transformers/_converters.py +703 -618
- cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
- cognite/neat/core/_data_model/transformers/_verification.py +63 -54
- cognite/neat/core/_instances/extractors/_base.py +1 -1
- cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +8 -8
- cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
- cognite/neat/core/_instances/extractors/_mock_graph_generator.py +98 -95
- cognite/neat/core/_instances/loaders/_base.py +2 -2
- cognite/neat/core/_instances/loaders/_rdf2dms.py +6 -6
- cognite/neat/core/_instances/transformers/_base.py +7 -4
- cognite/neat/core/_instances/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +2 -2
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/text.py +1 -1
- cognite/neat/session/_base.py +22 -20
- cognite/neat/session/_drop.py +2 -2
- cognite/neat/session/_inspect.py +5 -5
- cognite/neat/session/_mapping.py +8 -6
- cognite/neat/session/_read.py +2 -2
- cognite/neat/session/_set.py +3 -3
- cognite/neat/session/_show.py +11 -11
- cognite/neat/session/_state.py +13 -13
- cognite/neat/session/_subset.py +12 -9
- cognite/neat/session/_template.py +13 -13
- cognite/neat/session/_to.py +17 -17
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/RECORD +95 -93
- cognite/neat/core/_data_model/exporters/_validation.py +0 -14
- cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
- /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
cognite/neat/session/_base.py
CHANGED
|
@@ -7,21 +7,21 @@ from cognite.client import data_modeling as dm
|
|
|
7
7
|
from cognite.neat import _version
|
|
8
8
|
from cognite.neat.core._client import NeatClient
|
|
9
9
|
from cognite.neat.core._data_model import importers
|
|
10
|
-
from cognite.neat.core._data_model.models import
|
|
10
|
+
from cognite.neat.core._data_model.models import PhysicalDataModel
|
|
11
11
|
from cognite.neat.core._data_model.models.conceptual._verified import (
|
|
12
12
|
ConceptualDataModel,
|
|
13
13
|
)
|
|
14
14
|
from cognite.neat.core._data_model.transformers import (
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
ConceptualToPhysical,
|
|
16
|
+
MergeConceptualDataModels,
|
|
17
|
+
MergePhysicalDataModels,
|
|
18
18
|
ToDMSCompliantEntities,
|
|
19
|
-
|
|
19
|
+
VerifyConceptualDataModel,
|
|
20
20
|
)
|
|
21
21
|
from cognite.neat.core._issues import IssueList
|
|
22
22
|
from cognite.neat.core._issues.errors import RegexViolationError
|
|
23
23
|
from cognite.neat.core._issues.errors._general import NeatImportError
|
|
24
|
-
from cognite.neat.core._store.
|
|
24
|
+
from cognite.neat.core._store._data_model import DataModelEntity
|
|
25
25
|
from cognite.neat.core._utils.auxiliary import local_import
|
|
26
26
|
|
|
27
27
|
from ._collector import _COLLECTOR, Collector
|
|
@@ -170,7 +170,7 @@ class NeatSession:
|
|
|
170
170
|
self._state._raise_exception_if_condition_not_met(
|
|
171
171
|
"Convert to physical", has_dms_rules=False, has_information_rules=True
|
|
172
172
|
)
|
|
173
|
-
converter =
|
|
173
|
+
converter = ConceptualToPhysical(reserved_properties=reserved_properties, client=self._state.client)
|
|
174
174
|
|
|
175
175
|
issues = self._state.rule_transform(converter)
|
|
176
176
|
|
|
@@ -231,7 +231,7 @@ class NeatSession:
|
|
|
231
231
|
),
|
|
232
232
|
) -> IssueList:
|
|
233
233
|
"""Infer data model from instances."""
|
|
234
|
-
last_entity:
|
|
234
|
+
last_entity: DataModelEntity | None = None
|
|
235
235
|
if self._state.rule_store.provenance:
|
|
236
236
|
last_entity = self._state.rule_store.provenance[-1].target_entity
|
|
237
237
|
|
|
@@ -240,24 +240,26 @@ class NeatSession:
|
|
|
240
240
|
importer = importers.SubclassInferenceImporter(
|
|
241
241
|
issue_list=IssueList(),
|
|
242
242
|
graph=self._state.instances.store.graph(),
|
|
243
|
-
rules=last_entity.
|
|
244
|
-
data_model_id=dm.DataModelId.load(model_id) if last_entity is None else None,
|
|
243
|
+
rules=last_entity.conceptual if last_entity is not None else None,
|
|
244
|
+
data_model_id=(dm.DataModelId.load(model_id) if last_entity is None else None),
|
|
245
245
|
)
|
|
246
246
|
|
|
247
|
-
def action() -> tuple[ConceptualDataModel,
|
|
248
|
-
unverified_information = importer.
|
|
247
|
+
def action() -> tuple[ConceptualDataModel, PhysicalDataModel | None]:
|
|
248
|
+
unverified_information = importer.to_data_model()
|
|
249
249
|
unverified_information = ToDMSCompliantEntities(rename_warning="raise").transform(unverified_information)
|
|
250
250
|
|
|
251
|
-
extra_info =
|
|
251
|
+
extra_info = VerifyConceptualDataModel().transform(unverified_information)
|
|
252
252
|
if not last_entity:
|
|
253
253
|
return extra_info, None
|
|
254
|
-
merged_info =
|
|
255
|
-
if not last_entity.
|
|
254
|
+
merged_info = MergeConceptualDataModels(extra_info).transform(last_entity.conceptual)
|
|
255
|
+
if not last_entity.physical:
|
|
256
256
|
return merged_info, None
|
|
257
257
|
|
|
258
|
-
extra_dms =
|
|
258
|
+
extra_dms = ConceptualToPhysical(reserved_properties="warning", client=self._state.client).transform(
|
|
259
|
+
extra_info
|
|
260
|
+
)
|
|
259
261
|
|
|
260
|
-
merged_dms =
|
|
262
|
+
merged_dms = MergePhysicalDataModels(extra_dms).transform(last_entity.physical)
|
|
261
263
|
return merged_info, merged_dms
|
|
262
264
|
|
|
263
265
|
return self._state.rule_store.do_activity(action, importer)
|
|
@@ -271,10 +273,10 @@ class NeatSession:
|
|
|
271
273
|
|
|
272
274
|
if state.rule_store.provenance:
|
|
273
275
|
last_entity = state.rule_store.provenance[-1].target_entity
|
|
274
|
-
if last_entity.
|
|
275
|
-
html = last_entity.
|
|
276
|
+
if last_entity.physical:
|
|
277
|
+
html = last_entity.physical._repr_html_()
|
|
276
278
|
else:
|
|
277
|
-
html = last_entity.
|
|
279
|
+
html = last_entity.conceptual._repr_html_()
|
|
278
280
|
output.append(f"<H2>Data Model</H2><br />{html}") # type: ignore
|
|
279
281
|
|
|
280
282
|
if not state.instances.empty:
|
cognite/neat/session/_drop.py
CHANGED
|
@@ -78,7 +78,7 @@ class DropDataModelAPI:
|
|
|
78
78
|
"""
|
|
79
79
|
if sum([view_external_id is not None, group is not None]) != 1:
|
|
80
80
|
raise NeatSessionError("Only one of view_external_id or group can be specified.")
|
|
81
|
-
last_dms = self._state.rule_store.
|
|
81
|
+
last_dms = self._state.rule_store.last_verified_physical_data_model
|
|
82
82
|
if group is not None and last_dms.metadata.as_data_model_id() not in COGNITE_MODELS:
|
|
83
83
|
raise NeatSessionError("Group can only be specified for CogniteCore models.")
|
|
84
84
|
if view_external_id is not None:
|
|
@@ -98,6 +98,6 @@ class DropDataModelAPI:
|
|
|
98
98
|
)
|
|
99
99
|
before = len(last_dms.views)
|
|
100
100
|
issues = self._state.rule_transform(DropModelViews(view_external_id, group))
|
|
101
|
-
after = len(self._state.rule_store.
|
|
101
|
+
after = len(self._state.rule_store.last_verified_physical_data_model.views)
|
|
102
102
|
print(f"Dropped {before - after} views.")
|
|
103
103
|
return issues
|
cognite/neat/session/_inspect.py
CHANGED
|
@@ -65,10 +65,10 @@ class InspectAPI:
|
|
|
65
65
|
if self._state.rule_store.empty:
|
|
66
66
|
return pd.DataFrame()
|
|
67
67
|
last_entity = self._state.rule_store.provenance[-1].target_entity
|
|
68
|
-
if last_entity.
|
|
69
|
-
df = last_entity.
|
|
68
|
+
if last_entity.physical:
|
|
69
|
+
df = last_entity.physical.properties.to_pandas()
|
|
70
70
|
else:
|
|
71
|
-
df = last_entity.
|
|
71
|
+
df = last_entity.conceptual.properties.to_pandas()
|
|
72
72
|
df.drop(columns=["neatId"], errors="ignore", inplace=True)
|
|
73
73
|
return df
|
|
74
74
|
|
|
@@ -76,9 +76,9 @@ class InspectAPI:
|
|
|
76
76
|
if self._state.rule_store.empty:
|
|
77
77
|
return pd.DataFrame()
|
|
78
78
|
last_entity = self._state.rule_store.provenance[-1].target_entity
|
|
79
|
-
if last_entity.
|
|
79
|
+
if last_entity.physical is None:
|
|
80
80
|
return pd.DataFrame()
|
|
81
|
-
df = last_entity.
|
|
81
|
+
df = last_entity.physical.views.to_pandas()
|
|
82
82
|
df.drop(columns=["neatId"], errors="ignore", inplace=True)
|
|
83
83
|
return df
|
|
84
84
|
|
cognite/neat/session/_mapping.py
CHANGED
|
@@ -3,8 +3,8 @@ from cognite.neat.core._data_model.transformers import (
|
|
|
3
3
|
AsParentPropertyId,
|
|
4
4
|
ChangeViewPrefix,
|
|
5
5
|
IncludeReferenced,
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
PhysicalDataModelMapper,
|
|
7
|
+
VerifiedDataModelTransformer,
|
|
8
8
|
)
|
|
9
9
|
from cognite.neat.core._issues import IssueList
|
|
10
10
|
|
|
@@ -45,18 +45,20 @@ class DataModelMappingAPI:
|
|
|
45
45
|
if self._state.rule_store.empty:
|
|
46
46
|
raise NeatSessionError("No rules to map")
|
|
47
47
|
last_entity = self._state.rule_store.provenance[-1].target_entity
|
|
48
|
-
if last_entity.
|
|
48
|
+
if last_entity.physical is None:
|
|
49
49
|
raise NeatSessionError("Data model not converted to DMS. Try running `neat.convert('dms')` first.")
|
|
50
|
-
rules = last_entity.
|
|
50
|
+
rules = last_entity.physical
|
|
51
51
|
if self._state.client is None:
|
|
52
52
|
raise NeatSessionError("Client is required to map classic to core")
|
|
53
53
|
|
|
54
|
-
transformers: list[
|
|
54
|
+
transformers: list[VerifiedDataModelTransformer] = []
|
|
55
55
|
if company_prefix:
|
|
56
56
|
transformers.append(ChangeViewPrefix("Classic", company_prefix))
|
|
57
57
|
transformers.extend(
|
|
58
58
|
[
|
|
59
|
-
|
|
59
|
+
PhysicalDataModelMapper(
|
|
60
|
+
load_classic_to_core_mapping(company_prefix, rules.metadata.space, rules.metadata.version)
|
|
61
|
+
),
|
|
60
62
|
IncludeReferenced(self._state.client),
|
|
61
63
|
]
|
|
62
64
|
)
|
cognite/neat/session/_read.py
CHANGED
|
@@ -164,7 +164,7 @@ class CDFReadAPI(BaseReadAPI):
|
|
|
164
164
|
if issues.has_errors:
|
|
165
165
|
return issues
|
|
166
166
|
|
|
167
|
-
cdm_rules = self._state.rule_store.
|
|
167
|
+
cdm_rules = self._state.rule_store.last_verified_data_model
|
|
168
168
|
|
|
169
169
|
issues.extend(
|
|
170
170
|
self._state.rule_transform(
|
|
@@ -593,7 +593,7 @@ class YamlReadAPI(BaseReadAPI):
|
|
|
593
593
|
path = reader.materialize_path()
|
|
594
594
|
importer: BaseImporter
|
|
595
595
|
if format == "neat":
|
|
596
|
-
importer = importers.
|
|
596
|
+
importer = importers.DictImporter.from_yaml_file(path, source_name=f"{reader!s}")
|
|
597
597
|
elif format == "toolkit":
|
|
598
598
|
dms_importer = importers.DMSImporter.from_path(path, self._state.client)
|
|
599
599
|
if dms_importer.issue_list.has_warning_type(MissingCogniteClientWarning):
|
cognite/neat/session/_set.py
CHANGED
|
@@ -3,7 +3,7 @@ from cognite.client import data_modeling as dm
|
|
|
3
3
|
|
|
4
4
|
from cognite.neat.core._client import NeatClient
|
|
5
5
|
from cognite.neat.core._constants import COGNITE_MODELS
|
|
6
|
-
from cognite.neat.core._data_model.models import
|
|
6
|
+
from cognite.neat.core._data_model.models import PhysicalDataModel
|
|
7
7
|
from cognite.neat.core._data_model.transformers import SetIDDMSModel
|
|
8
8
|
from cognite.neat.core._instances.transformers import SetType
|
|
9
9
|
from cognite.neat.core._issues import IssueList
|
|
@@ -41,8 +41,8 @@ class SetAPI:
|
|
|
41
41
|
"""
|
|
42
42
|
if self._state.rule_store.empty:
|
|
43
43
|
raise NeatSessionError("No rules to set the data model ID.")
|
|
44
|
-
rules = self._state.rule_store.provenance[-1].target_entity.
|
|
45
|
-
if isinstance(rules,
|
|
44
|
+
rules = self._state.rule_store.provenance[-1].target_entity.physical
|
|
45
|
+
if isinstance(rules, PhysicalDataModel):
|
|
46
46
|
if rules.metadata.as_data_model_id() in COGNITE_MODELS:
|
|
47
47
|
raise NeatSessionError(
|
|
48
48
|
"Cannot change the data model ID of a Cognite Data Model in NeatSession"
|
cognite/neat/session/_show.py
CHANGED
|
@@ -7,7 +7,7 @@ from IPython.display import HTML, display
|
|
|
7
7
|
from pyvis.network import Network as PyVisNetwork # type: ignore
|
|
8
8
|
|
|
9
9
|
from cognite.neat.core._constants import IN_NOTEBOOK, IN_PYODIDE
|
|
10
|
-
from cognite.neat.core._data_model.analysis._base import
|
|
10
|
+
from cognite.neat.core._data_model.analysis._base import DataModelAnalysis
|
|
11
11
|
from cognite.neat.core._utils.io_ import to_directory_compatible
|
|
12
12
|
from cognite.neat.core._utils.rdf_ import remove_namespace_from_uri, uri_display_name
|
|
13
13
|
from cognite.neat.session.exceptions import NeatSessionError
|
|
@@ -99,13 +99,13 @@ class ShowDataModelAPI(ShowBaseAPI):
|
|
|
99
99
|
raise NeatSessionError("No data model available. Try using [bold].read[/bold] to read a data model.")
|
|
100
100
|
|
|
101
101
|
last_target = self._state.rule_store.provenance[-1].target_entity
|
|
102
|
-
rules = last_target.
|
|
103
|
-
analysis =
|
|
102
|
+
rules = last_target.physical or last_target.conceptual
|
|
103
|
+
analysis = DataModelAnalysis(physical=last_target.physical, conceptual=last_target.conceptual)
|
|
104
104
|
|
|
105
|
-
if last_target.
|
|
106
|
-
di_graph = analysis.
|
|
105
|
+
if last_target.physical is not None:
|
|
106
|
+
di_graph = analysis._physical_di_graph(format="data-model")
|
|
107
107
|
else:
|
|
108
|
-
di_graph = analysis.
|
|
108
|
+
di_graph = analysis._conceptual_di_graph(format="data-model")
|
|
109
109
|
|
|
110
110
|
identifier = to_directory_compatible(str(rules.metadata.identifier))
|
|
111
111
|
name = f"{identifier}.html"
|
|
@@ -123,13 +123,13 @@ class ShowDataModelImplementsAPI(ShowBaseAPI):
|
|
|
123
123
|
raise NeatSessionError("No data model available. Try using [bold].read[/bold] to read a data model.")
|
|
124
124
|
|
|
125
125
|
last_target = self._state.rule_store.provenance[-1].target_entity
|
|
126
|
-
rules = last_target.
|
|
127
|
-
analysis =
|
|
126
|
+
rules = last_target.physical or last_target.conceptual
|
|
127
|
+
analysis = DataModelAnalysis(physical=last_target.physical, conceptual=last_target.conceptual)
|
|
128
128
|
|
|
129
|
-
if last_target.
|
|
130
|
-
di_graph = analysis.
|
|
129
|
+
if last_target.physical is not None:
|
|
130
|
+
di_graph = analysis._physical_di_graph(format="implements")
|
|
131
131
|
else:
|
|
132
|
-
di_graph = analysis.
|
|
132
|
+
di_graph = analysis._conceptual_di_graph(format="implements")
|
|
133
133
|
identifier = to_directory_compatible(str(rules.metadata.identifier))
|
|
134
134
|
name = f"{identifier}_implements.html"
|
|
135
135
|
return self._generate_visualization(di_graph, name)
|
cognite/neat/session/_state.py
CHANGED
|
@@ -5,13 +5,13 @@ from rdflib import URIRef
|
|
|
5
5
|
|
|
6
6
|
from cognite.neat.core._client import NeatClient
|
|
7
7
|
from cognite.neat.core._data_model.importers import BaseImporter, InferenceImporter
|
|
8
|
-
from cognite.neat.core._data_model.models import ConceptualDataModel,
|
|
8
|
+
from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
|
|
9
9
|
from cognite.neat.core._data_model.transformers import (
|
|
10
|
-
|
|
10
|
+
VerifiedDataModelTransformer,
|
|
11
11
|
)
|
|
12
12
|
from cognite.neat.core._instances.extractors import KnowledgeGraphExtractor
|
|
13
13
|
from cognite.neat.core._issues import IssueList
|
|
14
|
-
from cognite.neat.core._store import
|
|
14
|
+
from cognite.neat.core._store import NeatDataModelStore, NeatInstanceStore
|
|
15
15
|
from cognite.neat.core._utils.upload import UploadResultList
|
|
16
16
|
|
|
17
17
|
from .exceptions import NeatSessionError, _session_method_wrapper
|
|
@@ -25,12 +25,12 @@ class SessionState:
|
|
|
25
25
|
client: NeatClient | None = None,
|
|
26
26
|
) -> None:
|
|
27
27
|
self.instances = InstancesState(store_type, storage_path=storage_path)
|
|
28
|
-
self.rule_store =
|
|
29
|
-
self.last_reference:
|
|
28
|
+
self.rule_store = NeatDataModelStore()
|
|
29
|
+
self.last_reference: PhysicalDataModel | ConceptualDataModel | None = None
|
|
30
30
|
self.client = client
|
|
31
31
|
self.quoted_source_identifiers = False
|
|
32
32
|
|
|
33
|
-
def rule_transform(self, *transformer:
|
|
33
|
+
def rule_transform(self, *transformer: VerifiedDataModelTransformer) -> IssueList:
|
|
34
34
|
if not transformer:
|
|
35
35
|
raise NeatSessionError("No transformers provided.")
|
|
36
36
|
start = self.rule_store.provenance[-1].target_entity.display_name
|
|
@@ -41,7 +41,7 @@ class SessionState:
|
|
|
41
41
|
return issues
|
|
42
42
|
|
|
43
43
|
def rule_import(self, importer: BaseImporter, enable_manual_edit: bool = False) -> IssueList:
|
|
44
|
-
issues = self.rule_store.
|
|
44
|
+
issues = self.rule_store.import_data_model(
|
|
45
45
|
importer,
|
|
46
46
|
client=self.client,
|
|
47
47
|
enable_manual_edit=enable_manual_edit,
|
|
@@ -81,10 +81,10 @@ class SessionState:
|
|
|
81
81
|
if client_required and not self.client:
|
|
82
82
|
condition.add(f"{activity} expects a client in NEAT session")
|
|
83
83
|
suggestion.add("Please provide a client")
|
|
84
|
-
if has_information_rules is True and self.rule_store.
|
|
84
|
+
if has_information_rules is True and self.rule_store.try_get_last_conceptual_data_model is None:
|
|
85
85
|
condition.add(f"{activity} expects information rules in NEAT session")
|
|
86
86
|
suggestion.add("Read in information rules to neat session")
|
|
87
|
-
if has_dms_rules is False and self.rule_store.
|
|
87
|
+
if has_dms_rules is False and self.rule_store.try_get_last_physical_data_model is not None:
|
|
88
88
|
condition.add(f"{activity} expects no DMS data model in NEAT session")
|
|
89
89
|
suggestion.add("You already have a DMS data model in the session")
|
|
90
90
|
try_again = False
|
|
@@ -121,18 +121,18 @@ class InstancesState:
|
|
|
121
121
|
self.neat_prefix_by_type_uri: dict[URIRef, str] = {}
|
|
122
122
|
|
|
123
123
|
# Ensure that error handling is done in the constructor
|
|
124
|
-
self.store:
|
|
124
|
+
self.store: NeatInstanceStore = _session_method_wrapper(self._create_store, "NeatSession")()
|
|
125
125
|
|
|
126
126
|
if self.storage_path:
|
|
127
127
|
print("Remember to close neat session .close() once you are done to avoid oxigraph lock.")
|
|
128
128
|
|
|
129
|
-
def _create_store(self) ->
|
|
129
|
+
def _create_store(self) -> NeatInstanceStore:
|
|
130
130
|
if self.store_type == "oxigraph":
|
|
131
131
|
if self.storage_path:
|
|
132
132
|
self.storage_path.mkdir(parents=True, exist_ok=True)
|
|
133
|
-
return
|
|
133
|
+
return NeatInstanceStore.from_oxi_local_store(storage_dir=self.storage_path)
|
|
134
134
|
else:
|
|
135
|
-
return
|
|
135
|
+
return NeatInstanceStore.from_memory_store()
|
|
136
136
|
|
|
137
137
|
@property
|
|
138
138
|
def empty(self) -> bool:
|
cognite/neat/session/_subset.py
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import warnings
|
|
2
2
|
|
|
3
3
|
from cognite.neat.core._data_model.models.entities._single_value import (
|
|
4
|
-
|
|
4
|
+
ConceptEntity,
|
|
5
5
|
ViewEntity,
|
|
6
6
|
)
|
|
7
|
-
from cognite.neat.core._data_model.transformers import
|
|
7
|
+
from cognite.neat.core._data_model.transformers import (
|
|
8
|
+
SubsetConceptualDataModel,
|
|
9
|
+
SubsetPhysicalDataModel,
|
|
10
|
+
)
|
|
8
11
|
from cognite.neat.core._issues._base import IssueList
|
|
9
12
|
from cognite.neat.session._experimental import ExperimentalFlags
|
|
10
13
|
|
|
@@ -52,8 +55,8 @@ class SubsetAPI:
|
|
|
52
55
|
warnings.filterwarnings("default")
|
|
53
56
|
ExperimentalFlags.data_model_subsetting.warn()
|
|
54
57
|
|
|
55
|
-
dms = self._state.rule_store.provenance[-1].target_entity.
|
|
56
|
-
information = self._state.rule_store.provenance[-1].target_entity.
|
|
58
|
+
dms = self._state.rule_store.provenance[-1].target_entity.physical
|
|
59
|
+
information = self._state.rule_store.provenance[-1].target_entity.conceptual
|
|
57
60
|
|
|
58
61
|
if dms:
|
|
59
62
|
views = {
|
|
@@ -65,16 +68,16 @@ class SubsetAPI:
|
|
|
65
68
|
for concept in concepts
|
|
66
69
|
}
|
|
67
70
|
|
|
68
|
-
issues = self._state.rule_transform(
|
|
71
|
+
issues = self._state.rule_transform(SubsetPhysicalDataModel(views=views))
|
|
69
72
|
if not issues:
|
|
70
|
-
after = len(self._state.rule_store.
|
|
73
|
+
after = len(self._state.rule_store.last_verified_physical_data_model.views)
|
|
71
74
|
|
|
72
75
|
elif information:
|
|
73
|
-
classes = {
|
|
76
|
+
classes = {ConceptEntity(prefix=information.metadata.space, suffix=concept) for concept in concepts}
|
|
74
77
|
|
|
75
|
-
issues = self._state.rule_transform(
|
|
78
|
+
issues = self._state.rule_transform(SubsetConceptualDataModel(concepts=classes))
|
|
76
79
|
if not issues:
|
|
77
|
-
after = len(self._state.rule_store.
|
|
80
|
+
after = len(self._state.rule_store.last_verified_conceptual_data_model.concepts)
|
|
78
81
|
|
|
79
82
|
else:
|
|
80
83
|
raise NeatSessionError("Something went terrible wrong. Please contact the neat team.")
|
|
@@ -4,18 +4,18 @@ from typing import Any, Literal
|
|
|
4
4
|
from cognite.client.data_classes.data_modeling import DataModelIdentifier
|
|
5
5
|
|
|
6
6
|
from cognite.neat.core._constants import BASE_MODEL
|
|
7
|
-
from cognite.neat.core._data_model._shared import
|
|
7
|
+
from cognite.neat.core._data_model._shared import ImportedDataModel
|
|
8
8
|
from cognite.neat.core._data_model.exporters import ExcelExporter
|
|
9
9
|
from cognite.neat.core._data_model.importers import ExcelImporter
|
|
10
10
|
from cognite.neat.core._data_model.models import UnverifiedConceptualDataModel
|
|
11
11
|
from cognite.neat.core._data_model.models._base_verified import RoleTypes
|
|
12
|
-
from cognite.neat.core._data_model.models.
|
|
12
|
+
from cognite.neat.core._data_model.models.physical import PhysicalValidation
|
|
13
13
|
from cognite.neat.core._data_model.transformers import (
|
|
14
14
|
AddCogniteProperties,
|
|
15
15
|
IncludeReferenced,
|
|
16
16
|
ToDataProductModel,
|
|
17
17
|
ToEnterpriseModel,
|
|
18
|
-
|
|
18
|
+
VerifiedDataModelTransformer,
|
|
19
19
|
)
|
|
20
20
|
from cognite.neat.core._issues import IssueList, catch_issues
|
|
21
21
|
from cognite.neat.core._utils.reader import NeatReader, PathReader
|
|
@@ -69,7 +69,7 @@ class TemplateAPI:
|
|
|
69
69
|
- Charts
|
|
70
70
|
|
|
71
71
|
"""
|
|
72
|
-
last_rules = self._state.rule_store.
|
|
72
|
+
last_rules = self._state.rule_store.last_verified_data_model
|
|
73
73
|
issues = self._state.rule_transform(
|
|
74
74
|
ToEnterpriseModel(
|
|
75
75
|
new_model_id=data_model_id,
|
|
@@ -105,11 +105,11 @@ class TemplateAPI:
|
|
|
105
105
|
If you set same-space, only the properties of the views in the same space as the data model
|
|
106
106
|
will be included.
|
|
107
107
|
"""
|
|
108
|
-
last_rules = self._state.rule_store.
|
|
109
|
-
view_ids, container_ids =
|
|
110
|
-
self._state.rule_store.
|
|
108
|
+
last_rules = self._state.rule_store.last_verified_data_model
|
|
109
|
+
view_ids, container_ids = PhysicalValidation(
|
|
110
|
+
self._state.rule_store.last_verified_physical_data_model
|
|
111
111
|
).imported_views_and_containers_ids()
|
|
112
|
-
transformers: list[
|
|
112
|
+
transformers: list[VerifiedDataModelTransformer] = []
|
|
113
113
|
client = self._state.client
|
|
114
114
|
if (view_ids or container_ids) and client is None:
|
|
115
115
|
raise NeatSessionError(
|
|
@@ -185,17 +185,17 @@ class TemplateAPI:
|
|
|
185
185
|
output_path = Path(output)
|
|
186
186
|
|
|
187
187
|
with catch_issues() as issues:
|
|
188
|
-
read:
|
|
189
|
-
if read.
|
|
188
|
+
read: ImportedDataModel[UnverifiedConceptualDataModel] = ExcelImporter(path).to_data_model()
|
|
189
|
+
if read.unverified_data_model is not None:
|
|
190
190
|
# If rules are None there will be issues that are already caught.
|
|
191
|
-
if not isinstance(read.
|
|
191
|
+
if not isinstance(read.unverified_data_model, UnverifiedConceptualDataModel):
|
|
192
192
|
raise NeatSessionError(f"The input {reader.name} must contain an InformationInputRules object. ")
|
|
193
193
|
if self._state.client is None:
|
|
194
194
|
raise NeatSessionError("Client must be set in the session to run the extension.")
|
|
195
195
|
modified = AddCogniteProperties(self._state.client, dummy_property).transform(read)
|
|
196
|
-
if modified.
|
|
196
|
+
if modified.unverified_data_model is not None:
|
|
197
197
|
# If rules are None there will be issues that are already caught.
|
|
198
|
-
info = modified.
|
|
198
|
+
info = modified.unverified_data_model.as_verified_data_model()
|
|
199
199
|
|
|
200
200
|
ExcelExporter(styling="maximal").export_to_file(info, output_path)
|
|
201
201
|
issues.action = "Created extension template"
|
cognite/neat/session/_to.py
CHANGED
|
@@ -10,10 +10,10 @@ from cognite.neat.core._client._api_client import NeatClient
|
|
|
10
10
|
from cognite.neat.core._constants import COGNITE_MODELS
|
|
11
11
|
from cognite.neat.core._data_model import exporters
|
|
12
12
|
from cognite.neat.core._data_model._constants import PATTERNS
|
|
13
|
-
from cognite.neat.core._data_model._shared import
|
|
13
|
+
from cognite.neat.core._data_model._shared import VerifiedDataModel
|
|
14
14
|
from cognite.neat.core._data_model.importers import DMSImporter
|
|
15
|
-
from cognite.neat.core._data_model.models import ConceptualDataModel,
|
|
16
|
-
from cognite.neat.core._data_model.models.
|
|
15
|
+
from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
|
|
16
|
+
from cognite.neat.core._data_model.models.physical import PhysicalMetadata
|
|
17
17
|
from cognite.neat.core._instances import loaders
|
|
18
18
|
from cognite.neat.core._issues import IssueList, NeatIssue, catch_issues
|
|
19
19
|
from cognite.neat.core._utils.upload import UploadResultList
|
|
@@ -113,12 +113,12 @@ class ToAPI:
|
|
|
113
113
|
dms_rules_file_name = "dms_rules.xlsx"
|
|
114
114
|
neat.to.excel(dms_rules_file_name, include_reference=("cdf_cdm", "CogniteCore", "v1"))
|
|
115
115
|
"""
|
|
116
|
-
|
|
116
|
+
reference_data_model_with_prefix: tuple[VerifiedDataModel, str] | None = None
|
|
117
117
|
include_properties = include_properties.strip().lower()
|
|
118
118
|
|
|
119
119
|
if include_reference is not False:
|
|
120
120
|
if include_reference is True and self._state.last_reference is not None:
|
|
121
|
-
ref_rules: ConceptualDataModel |
|
|
121
|
+
ref_rules: ConceptualDataModel | PhysicalDataModel | None = self._state.last_reference
|
|
122
122
|
elif include_reference is True:
|
|
123
123
|
ref_rules = None
|
|
124
124
|
else:
|
|
@@ -126,20 +126,20 @@ class ToAPI:
|
|
|
126
126
|
raise NeatSessionError("No client provided!")
|
|
127
127
|
ref_rules = None
|
|
128
128
|
with catch_issues() as issues:
|
|
129
|
-
ref_read = DMSImporter.from_data_model_id(self._state.client, include_reference).
|
|
130
|
-
if ref_read.
|
|
131
|
-
ref_rules = ref_read.
|
|
129
|
+
ref_read = DMSImporter.from_data_model_id(self._state.client, include_reference).to_data_model()
|
|
130
|
+
if ref_read.unverified_data_model is not None:
|
|
131
|
+
ref_rules = ref_read.unverified_data_model.as_verified_data_model()
|
|
132
132
|
if ref_rules is None or issues.has_errors:
|
|
133
133
|
issues.action = f"Read {include_reference}"
|
|
134
134
|
return issues
|
|
135
135
|
if ref_rules is not None:
|
|
136
136
|
prefix = "Ref"
|
|
137
137
|
if (
|
|
138
|
-
isinstance(ref_rules.metadata,
|
|
138
|
+
isinstance(ref_rules.metadata, PhysicalMetadata)
|
|
139
139
|
and ref_rules.metadata.as_data_model_id() in COGNITE_MODELS
|
|
140
140
|
):
|
|
141
141
|
prefix = "CDM"
|
|
142
|
-
|
|
142
|
+
reference_data_model_with_prefix = ref_rules, prefix
|
|
143
143
|
|
|
144
144
|
if include_properties == "same-space":
|
|
145
145
|
warnings.filterwarnings("default")
|
|
@@ -147,7 +147,7 @@ class ToAPI:
|
|
|
147
147
|
|
|
148
148
|
exporter = exporters.ExcelExporter(
|
|
149
149
|
styling="maximal",
|
|
150
|
-
|
|
150
|
+
reference_data_model_with_prefix=reference_data_model_with_prefix,
|
|
151
151
|
add_empty_rows=add_empty_rows,
|
|
152
152
|
include_properties=include_properties, # type: ignore
|
|
153
153
|
)
|
|
@@ -310,7 +310,7 @@ class CDFToAPI:
|
|
|
310
310
|
)
|
|
311
311
|
|
|
312
312
|
client = cast(NeatClient, self._state.client)
|
|
313
|
-
dms_rules = self._state.rule_store.
|
|
313
|
+
dms_rules = self._state.rule_store.last_verified_physical_data_model
|
|
314
314
|
instance_space = instance_space or f"{dms_rules.metadata.space}_instances"
|
|
315
315
|
|
|
316
316
|
if instance_space and instance_space == dms_rules.metadata.space:
|
|
@@ -322,8 +322,8 @@ class CDFToAPI:
|
|
|
322
322
|
client.data_modeling.spaces.apply(dm.SpaceApply(space=instance_space))
|
|
323
323
|
|
|
324
324
|
loader = loaders.DMSLoader(
|
|
325
|
-
self._state.rule_store.
|
|
326
|
-
self._state.rule_store.
|
|
325
|
+
self._state.rule_store.last_verified_physical_data_model,
|
|
326
|
+
self._state.rule_store.last_verified_conceptual_data_model,
|
|
327
327
|
self._state.instances.store,
|
|
328
328
|
instance_space=instance_space,
|
|
329
329
|
client=client,
|
|
@@ -418,7 +418,7 @@ class ToPythonAPI:
|
|
|
418
418
|
instances = neat.to._python.instances(space_from_property="dataSetId")
|
|
419
419
|
```
|
|
420
420
|
"""
|
|
421
|
-
dms_rules = self._state.rule_store.
|
|
421
|
+
dms_rules = self._state.rule_store.last_verified_physical_data_model
|
|
422
422
|
instance_space = instance_space or f"{dms_rules.metadata.space}_instances"
|
|
423
423
|
|
|
424
424
|
if instance_space and instance_space == dms_rules.metadata.space:
|
|
@@ -427,8 +427,8 @@ class ToPythonAPI:
|
|
|
427
427
|
raise NeatSessionError(f"Please provide a valid space name. {PATTERNS.space_compliance.pattern}")
|
|
428
428
|
|
|
429
429
|
loader = loaders.DMSLoader(
|
|
430
|
-
self._state.rule_store.
|
|
431
|
-
self._state.rule_store.
|
|
430
|
+
self._state.rule_store.last_verified_physical_data_model,
|
|
431
|
+
self._state.rule_store.last_verified_conceptual_data_model,
|
|
432
432
|
self._state.instances.store,
|
|
433
433
|
instance_space=instance_space,
|
|
434
434
|
space_property=space_from_property,
|