cognite-neat 0.121.0__py3-none-any.whl → 0.121.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/{_rules → _data_model}/_constants.py +25 -18
- cognite/neat/core/_data_model/_shared.py +59 -0
- cognite/neat/core/_data_model/analysis/__init__.py +3 -0
- cognite/neat/core/{_rules → _data_model}/analysis/_base.py +202 -195
- cognite/neat/core/{_rules → _data_model}/catalog/__init__.py +1 -1
- cognite/neat/core/{_rules → _data_model}/exporters/__init__.py +5 -5
- cognite/neat/core/{_rules → _data_model}/exporters/_base.py +10 -8
- cognite/neat/core/{_rules/exporters/_rules2dms.py → _data_model/exporters/_data_model2dms.py} +22 -18
- cognite/neat/core/{_rules/exporters/_rules2excel.py → _data_model/exporters/_data_model2excel.py} +61 -56
- cognite/neat/core/{_rules/exporters/_rules2instance_template.py → _data_model/exporters/_data_model2instance_template.py} +11 -9
- cognite/neat/core/{_rules/exporters/_rules2ontology.py → _data_model/exporters/_data_model2ontology.py} +64 -61
- cognite/neat/core/{_rules/exporters/_rules2yaml.py → _data_model/exporters/_data_model2yaml.py} +21 -18
- cognite/neat/core/{_rules → _data_model}/importers/__init__.py +6 -8
- cognite/neat/core/{_rules → _data_model}/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/{_rules/importers/_yaml2rules.py → _data_model/importers/_dict2data_model.py} +41 -21
- cognite/neat/core/{_rules/importers/_dms2rules.py → _data_model/importers/_dms2data_model.py} +79 -66
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_converter.py +41 -41
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_importer.py +16 -16
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_base.py +18 -16
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_imf2rules.py +17 -17
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_inference2rules.py +50 -50
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_owl2rules.py +14 -14
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/{_rules/importers/_spreadsheet2rules.py → _data_model/importers/_spreadsheet2data_model.py} +69 -38
- cognite/neat/core/_data_model/models/__init__.py +36 -0
- cognite/neat/core/{_rules/models/_base_input.py → _data_model/models/_base_unverified.py} +12 -12
- cognite/neat/core/{_rules/models/_base_rules.py → _data_model/models/_base_verified.py} +13 -13
- cognite/neat/core/{_rules → _data_model}/models/_types.py +13 -13
- cognite/neat/core/_data_model/models/conceptual/__init__.py +25 -0
- cognite/neat/core/{_rules/models/information/_rules_input.py → _data_model/models/conceptual/_unverified.py} +46 -43
- cognite/neat/core/{_rules/models/information → _data_model/models/conceptual}/_validation.py +93 -79
- cognite/neat/core/{_rules/models/information/_rules.py → _data_model/models/conceptual/_verified.py} +83 -83
- cognite/neat/core/{_rules → _data_model}/models/data_types.py +4 -4
- cognite/neat/core/{_rules → _data_model}/models/entities/__init__.py +8 -8
- cognite/neat/core/{_rules → _data_model}/models/entities/_loaders.py +12 -11
- cognite/neat/core/{_rules → _data_model}/models/entities/_multi_value.py +7 -7
- cognite/neat/core/{_rules → _data_model}/models/entities/_single_value.py +45 -39
- cognite/neat/core/{_rules → _data_model}/models/entities/_types.py +9 -3
- cognite/neat/core/{_rules → _data_model}/models/entities/_wrapped.py +3 -3
- cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_exporter.py +83 -64
- cognite/neat/core/{_rules/models/dms/_rules_input.py → _data_model/models/physical/_unverified.py} +56 -44
- cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_validation.py +20 -17
- cognite/neat/core/{_rules/models/dms/_rules.py → _data_model/models/physical/_verified.py} +79 -71
- cognite/neat/core/{_rules → _data_model}/transformers/__init__.py +27 -23
- cognite/neat/core/{_rules → _data_model}/transformers/_base.py +29 -19
- cognite/neat/core/{_rules → _data_model}/transformers/_converters.py +758 -659
- cognite/neat/core/{_rules → _data_model}/transformers/_mapping.py +79 -60
- cognite/neat/core/_data_model/transformers/_verification.py +120 -0
- cognite/neat/core/{_graph → _instances}/extractors/_base.py +2 -2
- cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_base.py +1 -1
- cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_classic.py +17 -11
- cognite/neat/core/{_graph → _instances}/extractors/_dms_graph.py +47 -39
- cognite/neat/core/{_graph → _instances}/extractors/_mock_graph_generator.py +102 -99
- cognite/neat/core/{_graph → _instances}/extractors/_rdf_file.py +2 -2
- cognite/neat/core/{_graph → _instances}/loaders/_base.py +2 -2
- cognite/neat/core/{_graph → _instances}/loaders/_rdf2dms.py +16 -14
- cognite/neat/core/{_graph → _instances}/transformers/_base.py +7 -4
- cognite/neat/core/{_graph → _instances}/transformers/_classic_cdf.py +1 -1
- cognite/neat/core/{_graph → _instances}/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/__init__.py +2 -0
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_issues/warnings/_properties.py +7 -0
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +128 -121
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +7 -8
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +4 -4
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/spreadsheet.py +1 -1
- cognite/neat/core/_utils/text.py +2 -2
- cognite/neat/session/_base.py +29 -25
- cognite/neat/session/_drop.py +3 -3
- cognite/neat/session/_fix.py +2 -2
- cognite/neat/session/_inspect.py +5 -5
- cognite/neat/session/_mapping.py +11 -9
- cognite/neat/session/_prepare.py +4 -4
- cognite/neat/session/_read.py +15 -15
- cognite/neat/session/_set.py +5 -5
- cognite/neat/session/_show.py +11 -11
- cognite/neat/session/_state.py +17 -17
- cognite/neat/session/_subset.py +14 -11
- cognite/neat/session/_template.py +19 -19
- cognite/neat/session/_to.py +21 -21
- cognite/neat/session/_wizard.py +1 -1
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
- cognite_neat-0.121.2.dist-info/RECORD +189 -0
- cognite/neat/core/_rules/_shared.py +0 -43
- cognite/neat/core/_rules/analysis/__init__.py +0 -3
- cognite/neat/core/_rules/exporters/_validation.py +0 -14
- cognite/neat/core/_rules/models/__init__.py +0 -34
- cognite/neat/core/_rules/models/dms/__init__.py +0 -32
- cognite/neat/core/_rules/models/information/__init__.py +0 -20
- cognite/neat/core/_rules/transformers/_verification.py +0 -111
- cognite_neat-0.121.0.dist-info/RECORD +0 -187
- /cognite/neat/core/{_graph → _data_model}/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/catalog/classic_model.xlsx +0 -0
- /cognite/neat/core/{_rules/catalog/info-rules-imf.xlsx → _data_model/catalog/conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/{_rules → _data_model}/catalog/hello_world_pump.xlsx +0 -0
- /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/_unit_lookup.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/importers/_rdf/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/entities/_constants.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/mapping/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.yaml +0 -0
- /cognite/neat/core/{_graph/extractors/_classic_cdf → _instances}/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_shared.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/base.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/log.py +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44.xml +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/__init__.py +0 -0
- /cognite/neat/core/{_rules → _instances/extractors/_classic_cdf}/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_assets.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_data_sets.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_events.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_files.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_labels.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_relationships.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_sequences.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_timeseries.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_dict.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_dms.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_raw.py +0 -0
- /cognite/neat/core/{_graph → _instances}/loaders/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_base.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_queries.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_select.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_update.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/_prune_graph.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/_rdfpath.py +0 -0
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -3,6 +3,6 @@
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
|
|
5
5
|
_CATALOG = Path(__file__).parent
|
|
6
|
-
imf_attributes = _CATALOG / "
|
|
6
|
+
imf_attributes = _CATALOG / "conceptual-imf-data-model.xlsx"
|
|
7
7
|
hello_world_pump = _CATALOG / "hello_world_pump.xlsx"
|
|
8
8
|
classic_model = _CATALOG / "classic_model.xlsx"
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
from ._base import BaseExporter, CDFExporter
|
|
2
|
-
from .
|
|
3
|
-
from .
|
|
4
|
-
from .
|
|
5
|
-
from .
|
|
6
|
-
from .
|
|
2
|
+
from ._data_model2dms import DMSExporter
|
|
3
|
+
from ._data_model2excel import ExcelExporter
|
|
4
|
+
from ._data_model2instance_template import InstanceTemplateExporter
|
|
5
|
+
from ._data_model2ontology import GraphExporter, OWLExporter, SemanticDataModelExporter, SHACLExporter
|
|
6
|
+
from ._data_model2yaml import YAMLExporter
|
|
7
7
|
|
|
8
8
|
__all__ = [
|
|
9
9
|
"BaseExporter",
|
|
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Generic, TypeVar, Union, get_args, get_origin
|
|
|
7
7
|
|
|
8
8
|
from cognite.neat.core._client import NeatClient
|
|
9
9
|
from cognite.neat.core._constants import DEFAULT_NAMESPACE
|
|
10
|
-
from cognite.neat.core.
|
|
10
|
+
from cognite.neat.core._data_model._shared import T_VerifiedDataModel
|
|
11
11
|
from cognite.neat.core._utils.auxiliary import class_html_doc
|
|
12
12
|
from cognite.neat.core._utils.upload import UploadResult, UploadResultList
|
|
13
13
|
|
|
@@ -17,16 +17,16 @@ if TYPE_CHECKING:
|
|
|
17
17
|
T_Export = TypeVar("T_Export")
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
class BaseExporter(ABC, Generic[
|
|
20
|
+
class BaseExporter(ABC, Generic[T_VerifiedDataModel, T_Export]):
|
|
21
21
|
_new_line = "\n"
|
|
22
22
|
_encoding = "utf-8"
|
|
23
23
|
|
|
24
24
|
@abstractmethod
|
|
25
|
-
def export_to_file(self,
|
|
25
|
+
def export_to_file(self, data_model: T_VerifiedDataModel, filepath: Path) -> None:
|
|
26
26
|
raise NotImplementedError
|
|
27
27
|
|
|
28
28
|
@abstractmethod
|
|
29
|
-
def export(self,
|
|
29
|
+
def export(self, data_model: T_VerifiedDataModel) -> T_Export:
|
|
30
30
|
raise NotImplementedError
|
|
31
31
|
|
|
32
32
|
@classmethod
|
|
@@ -54,12 +54,14 @@ class BaseExporter(ABC, Generic[T_VerifiedRules, T_Export]):
|
|
|
54
54
|
return (source_type,)
|
|
55
55
|
|
|
56
56
|
|
|
57
|
-
class CDFExporter(BaseExporter[
|
|
57
|
+
class CDFExporter(BaseExporter[T_VerifiedDataModel, T_Export], ABC):
|
|
58
58
|
@abstractmethod
|
|
59
59
|
def export_to_cdf_iterable(
|
|
60
|
-
self,
|
|
60
|
+
self, data_model: T_VerifiedDataModel, client: NeatClient, dry_run: bool = False
|
|
61
61
|
) -> Iterable[UploadResult]:
|
|
62
62
|
raise NotImplementedError
|
|
63
63
|
|
|
64
|
-
def export_to_cdf(
|
|
65
|
-
|
|
64
|
+
def export_to_cdf(
|
|
65
|
+
self, data_model: T_VerifiedDataModel, client: NeatClient, dry_run: bool = False
|
|
66
|
+
) -> UploadResultList:
|
|
67
|
+
return UploadResultList(self.export_to_cdf_iterable(data_model, client, dry_run))
|
cognite/neat/core/{_rules/exporters/_rules2dms.py → _data_model/exporters/_data_model2dms.py}
RENAMED
|
@@ -28,12 +28,12 @@ from cognite.neat.core._client.data_classes.data_modeling import (
|
|
|
28
28
|
ViewApplyDict,
|
|
29
29
|
)
|
|
30
30
|
from cognite.neat.core._client.data_classes.schema import DMSSchema
|
|
31
|
+
from cognite.neat.core._data_model.models.physical import PhysicalDataModel
|
|
31
32
|
from cognite.neat.core._issues import IssueList
|
|
32
33
|
from cognite.neat.core._issues.warnings import (
|
|
33
34
|
PrincipleOneModelOneSpaceWarning,
|
|
34
35
|
ResourceRetrievalWarning,
|
|
35
36
|
)
|
|
36
|
-
from cognite.neat.core._rules.models.dms import DMSRules
|
|
37
37
|
from cognite.neat.core._shared import T_ID
|
|
38
38
|
from cognite.neat.core._utils.upload import UploadResult
|
|
39
39
|
|
|
@@ -74,8 +74,8 @@ class ItemCategorized(Generic[T_ID, T_WriteClass]):
|
|
|
74
74
|
yield from (self.as_id(item) for item in self.to_create + self.to_update + self.to_delete + self.unchanged)
|
|
75
75
|
|
|
76
76
|
|
|
77
|
-
class DMSExporter(CDFExporter[
|
|
78
|
-
"""Export
|
|
77
|
+
class DMSExporter(CDFExporter[PhysicalDataModel, DMSSchema]):
|
|
78
|
+
"""Export data model to Cognite Data Fusion's Data Model Storage (DMS) service.
|
|
79
79
|
|
|
80
80
|
Args:
|
|
81
81
|
export_components (frozenset[Literal["all", "spaces", "data_models", "views", "containers"]], optional):
|
|
@@ -120,31 +120,31 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
120
120
|
def description(self) -> str:
|
|
121
121
|
return "Export verified DMS Model to CDF."
|
|
122
122
|
|
|
123
|
-
def export_to_file(self,
|
|
124
|
-
"""Export the
|
|
123
|
+
def export_to_file(self, data_model: PhysicalDataModel, filepath: Path) -> None:
|
|
124
|
+
"""Export the data_model to a file(s).
|
|
125
125
|
|
|
126
126
|
If the file is a directory, the components will be exported to separate files, otherwise they will be
|
|
127
127
|
exported to a zip file.
|
|
128
128
|
|
|
129
129
|
Args:
|
|
130
130
|
filepath: Directory or zip file path to export to.
|
|
131
|
-
|
|
131
|
+
data_model:
|
|
132
132
|
"""
|
|
133
133
|
if filepath.is_dir():
|
|
134
|
-
self._export_to_directory(filepath,
|
|
134
|
+
self._export_to_directory(filepath, data_model)
|
|
135
135
|
else:
|
|
136
|
-
self._export_to_zip_file(filepath,
|
|
136
|
+
self._export_to_zip_file(filepath, data_model)
|
|
137
137
|
|
|
138
|
-
def _export_to_directory(self, directory: Path,
|
|
139
|
-
schema = self.export(
|
|
138
|
+
def _export_to_directory(self, directory: Path, data_model: PhysicalDataModel) -> None:
|
|
139
|
+
schema = self.export(data_model)
|
|
140
140
|
exclude = self._create_exclude_set()
|
|
141
141
|
schema.to_directory(directory, exclude=exclude, new_line=self._new_line, encoding=self._encoding)
|
|
142
142
|
|
|
143
|
-
def _export_to_zip_file(self, filepath: Path,
|
|
143
|
+
def _export_to_zip_file(self, filepath: Path, data_model: PhysicalDataModel) -> None:
|
|
144
144
|
if filepath.suffix not in {".zip"}:
|
|
145
145
|
warnings.warn("File extension is not .zip, adding it to the file name", stacklevel=2)
|
|
146
146
|
filepath = filepath.with_suffix(".zip")
|
|
147
|
-
schema = self.export(
|
|
147
|
+
schema = self.export(data_model)
|
|
148
148
|
exclude = self._create_exclude_set()
|
|
149
149
|
schema.to_zip(filepath, exclude=exclude)
|
|
150
150
|
|
|
@@ -155,14 +155,18 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
155
155
|
exclude = {"spaces", "data_models", "views", "containers", "node_types"} - set(self.export_components)
|
|
156
156
|
return exclude
|
|
157
157
|
|
|
158
|
-
def export(self,
|
|
158
|
+
def export(self, data_model: PhysicalDataModel) -> DMSSchema:
|
|
159
159
|
# We do not want to include CogniteCore/CogniteProcess Industries in the schema
|
|
160
|
-
return
|
|
160
|
+
return data_model.as_schema(instance_space=self.instance_space, remove_cdf_spaces=self.remove_cdf_spaces)
|
|
161
161
|
|
|
162
162
|
def delete_from_cdf(
|
|
163
|
-
self,
|
|
163
|
+
self,
|
|
164
|
+
data_model: PhysicalDataModel,
|
|
165
|
+
client: NeatClient,
|
|
166
|
+
dry_run: bool = False,
|
|
167
|
+
skip_space: bool = False,
|
|
164
168
|
) -> Iterable[UploadResult]:
|
|
165
|
-
schema = self.export(
|
|
169
|
+
schema = self.export(data_model)
|
|
166
170
|
|
|
167
171
|
# we need to reverse order in which we are picking up the items to delete
|
|
168
172
|
# as they are sorted in the order of creation and we need to delete them in reverse order
|
|
@@ -202,9 +206,9 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
202
206
|
yield result
|
|
203
207
|
|
|
204
208
|
def export_to_cdf_iterable(
|
|
205
|
-
self,
|
|
209
|
+
self, data_model: PhysicalDataModel, client: NeatClient, dry_run: bool = False
|
|
206
210
|
) -> Iterable[UploadResult]:
|
|
207
|
-
schema = self.export(
|
|
211
|
+
schema = self.export(data_model)
|
|
208
212
|
|
|
209
213
|
# The CDF UI does not deal well with a child view overwriting a parent property with the same name
|
|
210
214
|
# This is a workaround to remove the duplicated properties
|
cognite/neat/core/{_rules/exporters/_rules2excel.py → _data_model/exporters/_data_model2excel.py}
RENAMED
|
@@ -15,17 +15,22 @@ from openpyxl.worksheet.worksheet import Worksheet
|
|
|
15
15
|
from rdflib import Namespace
|
|
16
16
|
|
|
17
17
|
from cognite.neat.core._constants import BASE_MODEL, get_base_concepts
|
|
18
|
-
from cognite.neat.core.
|
|
19
|
-
from cognite.neat.core.
|
|
20
|
-
from cognite.neat.core.
|
|
18
|
+
from cognite.neat.core._data_model._constants import get_internal_properties
|
|
19
|
+
from cognite.neat.core._data_model._shared import VerifiedDataModel
|
|
20
|
+
from cognite.neat.core._data_model.models import (
|
|
21
21
|
SheetRow,
|
|
22
22
|
)
|
|
23
|
-
from cognite.neat.core.
|
|
24
|
-
|
|
23
|
+
from cognite.neat.core._data_model.models._base_verified import (
|
|
24
|
+
BaseVerifiedMetadata,
|
|
25
|
+
RoleTypes,
|
|
26
|
+
)
|
|
27
|
+
from cognite.neat.core._data_model.models.conceptual._verified import (
|
|
28
|
+
ConceptualDataModel,
|
|
29
|
+
)
|
|
30
|
+
from cognite.neat.core._data_model.models.data_types import (
|
|
25
31
|
_DATA_TYPE_BY_DMS_TYPE,
|
|
26
32
|
)
|
|
27
|
-
from cognite.neat.core.
|
|
28
|
-
from cognite.neat.core._rules.models.information._rules import InformationRules
|
|
33
|
+
from cognite.neat.core._data_model.models.physical._verified import PhysicalDataModel
|
|
29
34
|
from cognite.neat.core._utils.spreadsheet import (
|
|
30
35
|
find_column_with_value,
|
|
31
36
|
generate_data_validation,
|
|
@@ -36,16 +41,16 @@ from ._base import BaseExporter
|
|
|
36
41
|
MAX_COLUMN_WIDTH = 70.0
|
|
37
42
|
|
|
38
43
|
|
|
39
|
-
class ExcelExporter(BaseExporter[
|
|
40
|
-
"""Export
|
|
44
|
+
class ExcelExporter(BaseExporter[VerifiedDataModel, Workbook]):
|
|
45
|
+
"""Export data_model to Excel.
|
|
41
46
|
|
|
42
47
|
Args:
|
|
43
48
|
styling: The styling to use for the Excel file. Defaults to "default". See below for details
|
|
44
49
|
on the different styles.
|
|
45
50
|
new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
|
|
46
|
-
|
|
51
|
+
data_model have 'is_reference' set. If provided, the model ID will be used to automatically create the
|
|
47
52
|
new metadata sheet in the Excel file. The model id is expected to be a tuple of (prefix, title)
|
|
48
|
-
(space, external_id) for
|
|
53
|
+
(space, external_id) for Conceptual and Physical Data Model respectively.
|
|
49
54
|
|
|
50
55
|
sheet_prefix: The prefix to use for the sheet names in the Excel file. Defaults to an empty string.
|
|
51
56
|
|
|
@@ -63,14 +68,14 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
63
68
|
_helper_sheet_name: str = "_helper"
|
|
64
69
|
_main_header_by_sheet_name: ClassVar[dict[str, str]] = {
|
|
65
70
|
"Properties": "Definition of Properties",
|
|
66
|
-
"
|
|
71
|
+
"Concepts": "Definition of Concepts",
|
|
67
72
|
"Views": "Definition of Views",
|
|
68
73
|
"Containers": "Definition of Containers",
|
|
69
74
|
"Nodes": "Definition of Nodes",
|
|
70
75
|
"Enum": "Definition of Enum Collections",
|
|
71
76
|
}
|
|
72
77
|
_helper_sheet_column_indexes_by_names: ClassVar[dict[str, int]] = {
|
|
73
|
-
"
|
|
78
|
+
"Concept": 1,
|
|
74
79
|
"View": 1,
|
|
75
80
|
"Implements": 2,
|
|
76
81
|
"Value Type": 3,
|
|
@@ -87,7 +92,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
87
92
|
styling: Style = "default",
|
|
88
93
|
new_model_id: tuple[str, str] | None = None,
|
|
89
94
|
sheet_prefix: str | None = None,
|
|
90
|
-
|
|
95
|
+
reference_data_model_with_prefix: tuple[VerifiedDataModel, str] | None = None,
|
|
91
96
|
add_empty_rows: bool = False,
|
|
92
97
|
hide_internal_columns: bool = True,
|
|
93
98
|
include_properties: Literal["same-space", "all"] = "all",
|
|
@@ -101,7 +106,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
101
106
|
self.styling = styling
|
|
102
107
|
self._styling_level = self.style_options.index(styling)
|
|
103
108
|
self.new_model_id = new_model_id
|
|
104
|
-
self.
|
|
109
|
+
self.reference_data_model_with_prefix = reference_data_model_with_prefix
|
|
105
110
|
self.add_empty_rows = add_empty_rows
|
|
106
111
|
self.hide_internal_columns = hide_internal_columns
|
|
107
112
|
self.include_properties = include_properties
|
|
@@ -113,9 +118,9 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
113
118
|
def description(self) -> str:
|
|
114
119
|
return "Export verified model to Excel."
|
|
115
120
|
|
|
116
|
-
def export_to_file(self,
|
|
117
|
-
"""Exports transformation
|
|
118
|
-
data = self.export(
|
|
121
|
+
def export_to_file(self, data_model: VerifiedDataModel, filepath: Path) -> None:
|
|
122
|
+
"""Exports transformation data_model to excel file."""
|
|
123
|
+
data = self.export(data_model)
|
|
119
124
|
try:
|
|
120
125
|
data.save(filepath)
|
|
121
126
|
finally:
|
|
@@ -140,14 +145,14 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
140
145
|
# Remove default sheet named "Sheet"
|
|
141
146
|
workbook.remove(workbook["Sheet"])
|
|
142
147
|
|
|
143
|
-
|
|
148
|
+
data_model_model = PhysicalDataModel if role == RoleTypes.dms else ConceptualDataModel
|
|
144
149
|
|
|
145
|
-
headers_by_sheet =
|
|
150
|
+
headers_by_sheet = data_model_model.headers_by_sheet(by_alias=True)
|
|
146
151
|
headers_by_sheet.pop("Metadata")
|
|
147
152
|
|
|
148
153
|
self._write_metadata_sheet(
|
|
149
154
|
workbook,
|
|
150
|
-
cast(
|
|
155
|
+
cast(BaseVerifiedMetadata, data_model_model.model_fields["metadata"].annotation).default().model_dump(),
|
|
151
156
|
)
|
|
152
157
|
|
|
153
158
|
for sheet_name, headers in headers_by_sheet.items():
|
|
@@ -170,23 +175,23 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
170
175
|
|
|
171
176
|
return workbook
|
|
172
177
|
|
|
173
|
-
def export(self,
|
|
178
|
+
def export(self, data_model: VerifiedDataModel) -> Workbook:
|
|
174
179
|
workbook = Workbook()
|
|
175
180
|
# Remove default sheet named "Sheet"
|
|
176
181
|
workbook.remove(workbook["Sheet"])
|
|
177
182
|
|
|
178
|
-
|
|
183
|
+
dumped_user_data_model: dict[str, Any] = data_model.dump(by_alias=True)
|
|
179
184
|
|
|
180
|
-
self._write_metadata_sheet(workbook,
|
|
181
|
-
self._write_sheets(workbook,
|
|
182
|
-
if self.
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
self._write_sheets(workbook,
|
|
186
|
-
self._write_metadata_sheet(workbook,
|
|
185
|
+
self._write_metadata_sheet(workbook, dumped_user_data_model["Metadata"], sheet_prefix=self.sheet_prefix)
|
|
186
|
+
self._write_sheets(workbook, dumped_user_data_model, data_model, sheet_prefix=self.sheet_prefix)
|
|
187
|
+
if self.reference_data_model_with_prefix:
|
|
188
|
+
reference_data_model, prefix = self.reference_data_model_with_prefix
|
|
189
|
+
dumped_reference_data_model = reference_data_model.dump(entities_exclude_defaults=False, by_alias=True)
|
|
190
|
+
self._write_sheets(workbook, dumped_reference_data_model, reference_data_model, sheet_prefix=prefix)
|
|
191
|
+
self._write_metadata_sheet(workbook, dumped_reference_data_model["Metadata"], sheet_prefix=prefix)
|
|
187
192
|
|
|
188
|
-
if isinstance(
|
|
189
|
-
self._write_prefixes_sheet(workbook,
|
|
193
|
+
if isinstance(data_model, ConceptualDataModel) and data_model.prefixes:
|
|
194
|
+
self._write_prefixes_sheet(workbook, data_model.prefixes)
|
|
190
195
|
|
|
191
196
|
if self._styling_level > 0:
|
|
192
197
|
self._adjust_column_widths(workbook)
|
|
@@ -194,9 +199,9 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
194
199
|
if self.hide_internal_columns:
|
|
195
200
|
self._hide_internal_columns(workbook)
|
|
196
201
|
|
|
197
|
-
# Only add drop downs if the
|
|
202
|
+
# Only add drop downs if the data_model are Physical Data Model
|
|
198
203
|
if self.add_drop_downs:
|
|
199
|
-
self._add_drop_downs(
|
|
204
|
+
self._add_drop_downs(data_model.metadata.role, workbook)
|
|
200
205
|
|
|
201
206
|
return workbook
|
|
202
207
|
|
|
@@ -222,7 +227,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
222
227
|
workbook: Workbook,
|
|
223
228
|
) -> None:
|
|
224
229
|
"""Adds drop down menus to specific columns for fast and accurate data entry
|
|
225
|
-
for both
|
|
230
|
+
for both Conceptual and Physical Data Model.
|
|
226
231
|
|
|
227
232
|
Args:
|
|
228
233
|
role: The role for which the drop downs are created. Can be either "dms" or "information".
|
|
@@ -236,8 +241,8 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
236
241
|
|
|
237
242
|
!!! note "Why 100*100 rows?"
|
|
238
243
|
This is due to the fact that we need to add drop down menus for all properties
|
|
239
|
-
in the sheet. The maximum number of properties per view/
|
|
240
|
-
maximum number of views/
|
|
244
|
+
in the sheet. The maximum number of properties per view/concept is 100. So considering
|
|
245
|
+
maximum number of views/concepts is 100, we need to add 100*100 rows.
|
|
241
246
|
"""
|
|
242
247
|
|
|
243
248
|
self._make_helper_sheet(role, workbook, 100)
|
|
@@ -249,7 +254,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
249
254
|
|
|
250
255
|
self._add_data_validation(
|
|
251
256
|
workbook,
|
|
252
|
-
sheet_name="Views" if role == RoleTypes.dms else "
|
|
257
|
+
sheet_name="Views" if role == RoleTypes.dms else "Concepts",
|
|
253
258
|
column_name="Implements",
|
|
254
259
|
data_validator_name="implements",
|
|
255
260
|
data_validators=data_validators,
|
|
@@ -270,8 +275,8 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
270
275
|
self._add_data_validation(
|
|
271
276
|
workbook,
|
|
272
277
|
sheet_name="Properties",
|
|
273
|
-
column_name="View" if role == RoleTypes.dms else "
|
|
274
|
-
data_validator_name="
|
|
278
|
+
column_name="View" if role == RoleTypes.dms else "Concept",
|
|
279
|
+
data_validator_name="views_or_concepts",
|
|
275
280
|
data_validators=data_validators,
|
|
276
281
|
validation_range=100,
|
|
277
282
|
total_rows=100 * 100,
|
|
@@ -359,26 +364,26 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
359
364
|
)
|
|
360
365
|
concept_counter += 1
|
|
361
366
|
|
|
362
|
-
|
|
363
|
-
|
|
367
|
+
views_or_concepts_sheet = "Views" if role == RoleTypes.dms else "Concepts"
|
|
368
|
+
view_or_concept_column = "View" if role == RoleTypes.dms else "Concept"
|
|
364
369
|
|
|
365
370
|
for i in range(no_rows):
|
|
366
371
|
workbook[self._helper_sheet_name].cell(
|
|
367
372
|
row=i + 1,
|
|
368
|
-
column=self._helper_sheet_column_indexes_by_names[
|
|
369
|
-
value=f'=IF(ISBLANK({
|
|
373
|
+
column=self._helper_sheet_column_indexes_by_names[view_or_concept_column],
|
|
374
|
+
value=f'=IF(ISBLANK({views_or_concepts_sheet}!A{i + 3}), "", {views_or_concepts_sheet}!A{i + 3})',
|
|
370
375
|
)
|
|
371
376
|
|
|
372
377
|
workbook[self._helper_sheet_name].cell(
|
|
373
378
|
row=concept_counter + i + 1,
|
|
374
379
|
column=self._helper_sheet_column_indexes_by_names["Implements"],
|
|
375
|
-
value=f'=IF(ISBLANK({
|
|
380
|
+
value=f'=IF(ISBLANK({views_or_concepts_sheet}!A{i + 3}), "", {views_or_concepts_sheet}!A{i + 3})',
|
|
376
381
|
)
|
|
377
382
|
|
|
378
383
|
workbook[self._helper_sheet_name].cell(
|
|
379
384
|
row=value_type_counter + i + 1,
|
|
380
385
|
column=self._helper_sheet_column_indexes_by_names["Value Type"],
|
|
381
|
-
value=f'=IF(ISBLANK({
|
|
386
|
+
value=f'=IF(ISBLANK({views_or_concepts_sheet}!A{i + 3}), "", {views_or_concepts_sheet}!A{i + 3})',
|
|
382
387
|
)
|
|
383
388
|
|
|
384
389
|
if role == RoleTypes.dms:
|
|
@@ -502,11 +507,11 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
502
507
|
def _write_sheets(
|
|
503
508
|
self,
|
|
504
509
|
workbook: Workbook,
|
|
505
|
-
|
|
506
|
-
|
|
510
|
+
dumped_data_model: dict[str, Any],
|
|
511
|
+
data_model: VerifiedDataModel,
|
|
507
512
|
sheet_prefix: str = "",
|
|
508
513
|
) -> None:
|
|
509
|
-
for sheet_name, headers in
|
|
514
|
+
for sheet_name, headers in data_model.headers_by_sheet(by_alias=True).items():
|
|
510
515
|
if sheet_name in ("Metadata", "Prefixes", "Reference", "Last"):
|
|
511
516
|
continue
|
|
512
517
|
|
|
@@ -514,17 +519,17 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
514
519
|
|
|
515
520
|
fill_colors = itertools.cycle(["CADCFC", "FFFFFF"])
|
|
516
521
|
fill_color = next(fill_colors)
|
|
517
|
-
|
|
522
|
+
last_concept: str | None = None
|
|
518
523
|
item: dict[str, Any]
|
|
519
|
-
for item in
|
|
524
|
+
for item in dumped_data_model.get(sheet_name) or []:
|
|
520
525
|
if "Neat ID" in item:
|
|
521
526
|
# Move the Neat ID to the end of the columns
|
|
522
527
|
item["Neat ID"] = item.pop("Neat ID")
|
|
523
528
|
row = list(item.values())
|
|
524
|
-
|
|
529
|
+
concept = row[0]
|
|
525
530
|
|
|
526
531
|
is_properties = sheet_name == "Properties"
|
|
527
|
-
is_new_class =
|
|
532
|
+
is_new_class = concept != last_concept and last_concept is not None
|
|
528
533
|
if self._styling_level > 2 and is_new_class and is_properties:
|
|
529
534
|
if self.add_empty_rows:
|
|
530
535
|
sheet.append([""] * len(headers))
|
|
@@ -535,8 +540,8 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
535
540
|
fill_color = next(fill_colors)
|
|
536
541
|
|
|
537
542
|
if is_properties and self.include_properties == "same-space":
|
|
538
|
-
space =
|
|
539
|
-
if space !=
|
|
543
|
+
space = concept.split(":")[0] if ":" in concept else data_model.metadata.space
|
|
544
|
+
if space != data_model.metadata.space:
|
|
540
545
|
continue
|
|
541
546
|
|
|
542
547
|
sheet.append(row)
|
|
@@ -545,7 +550,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
545
550
|
cell.fill = PatternFill(fgColor=fill_color, patternType="solid")
|
|
546
551
|
side = Side(style="thin", color="000000")
|
|
547
552
|
cell.border = Border(left=side, right=side, top=side, bottom=side)
|
|
548
|
-
|
|
553
|
+
last_concept = concept
|
|
549
554
|
|
|
550
555
|
self._style_sheet_header(sheet, headers)
|
|
551
556
|
|
|
@@ -8,15 +8,17 @@ from openpyxl.styles import Alignment, Border, Font, NamedStyle, PatternFill, Si
|
|
|
8
8
|
from openpyxl.utils import get_column_letter
|
|
9
9
|
from openpyxl.worksheet.datavalidation import DataValidation
|
|
10
10
|
|
|
11
|
-
from cognite.neat.core.
|
|
12
|
-
from cognite.neat.core.
|
|
13
|
-
from cognite.neat.core.
|
|
14
|
-
|
|
11
|
+
from cognite.neat.core._data_model._constants import EntityTypes
|
|
12
|
+
from cognite.neat.core._data_model.analysis import DataModelAnalysis
|
|
13
|
+
from cognite.neat.core._data_model.models.conceptual._verified import (
|
|
14
|
+
ConceptualDataModel,
|
|
15
|
+
)
|
|
16
|
+
from cognite.neat.core._data_model.models.entities._single_value import ConceptEntity
|
|
15
17
|
|
|
16
18
|
from ._base import BaseExporter
|
|
17
19
|
|
|
18
20
|
|
|
19
|
-
class InstanceTemplateExporter(BaseExporter[
|
|
21
|
+
class InstanceTemplateExporter(BaseExporter[ConceptualDataModel, Workbook]):
|
|
20
22
|
"""
|
|
21
23
|
Converts Information Rules to a templated spreadsheet meant for capturing
|
|
22
24
|
instances based on class definitions in the rules.
|
|
@@ -48,14 +50,14 @@ class InstanceTemplateExporter(BaseExporter[InformationRules, Workbook]):
|
|
|
48
50
|
|
|
49
51
|
def export(
|
|
50
52
|
self,
|
|
51
|
-
rules:
|
|
53
|
+
rules: ConceptualDataModel,
|
|
52
54
|
) -> Workbook:
|
|
53
55
|
workbook = Workbook()
|
|
54
56
|
|
|
55
57
|
# Remove default sheet named "Sheet"
|
|
56
58
|
workbook.remove(workbook["Sheet"])
|
|
57
59
|
|
|
58
|
-
for class_, properties in
|
|
60
|
+
for class_, properties in DataModelAnalysis(rules).properties_by_id_by_concept().items():
|
|
59
61
|
workbook.create_sheet(title=class_.suffix)
|
|
60
62
|
|
|
61
63
|
# Add header rows
|
|
@@ -74,7 +76,7 @@ class InstanceTemplateExporter(BaseExporter[InformationRules, Workbook]):
|
|
|
74
76
|
class_.suffix,
|
|
75
77
|
get_column_letter(i + 2),
|
|
76
78
|
self.no_rows,
|
|
77
|
-
cast(
|
|
79
|
+
cast(ConceptEntity, property_.value_type).suffix,
|
|
78
80
|
"A",
|
|
79
81
|
)
|
|
80
82
|
|
|
@@ -83,7 +85,7 @@ class InstanceTemplateExporter(BaseExporter[InformationRules, Workbook]):
|
|
|
83
85
|
|
|
84
86
|
return workbook
|
|
85
87
|
|
|
86
|
-
def export_to_file(self, rules:
|
|
88
|
+
def export_to_file(self, rules: ConceptualDataModel, filepath: Path) -> None:
|
|
87
89
|
"""Exports graph capturing sheet to excel file."""
|
|
88
90
|
data = self.export(rules)
|
|
89
91
|
try:
|