cognite-neat 0.87.6__py3-none-any.whl → 0.88.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/data_classes/rest.py +0 -19
- cognite/neat/app/api/explorer.py +6 -4
- cognite/neat/app/api/routers/crud.py +11 -21
- cognite/neat/app/api/routers/workflows.py +24 -94
- cognite/neat/graph/stores/_base.py +5 -0
- cognite/neat/rules/importers/_inference2rules.py +31 -35
- cognite/neat/workflows/steps/data_contracts.py +17 -43
- cognite/neat/workflows/steps/lib/current/graph_extractor.py +28 -24
- cognite/neat/workflows/steps/lib/current/graph_loader.py +4 -21
- cognite/neat/workflows/steps/lib/current/graph_store.py +18 -134
- cognite/neat/workflows/steps_registry.py +5 -7
- {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/RECORD +17 -125
- cognite/neat/app/api/routers/core.py +0 -91
- cognite/neat/app/api/routers/data_exploration.py +0 -336
- cognite/neat/app/api/routers/rules.py +0 -203
- cognite/neat/legacy/__init__.py +0 -0
- cognite/neat/legacy/graph/__init__.py +0 -3
- cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -20182
- cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44.xml +0 -20163
- cognite/neat/legacy/graph/examples/__init__.py +0 -10
- cognite/neat/legacy/graph/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
- cognite/neat/legacy/graph/exceptions.py +0 -90
- cognite/neat/legacy/graph/extractors/__init__.py +0 -6
- cognite/neat/legacy/graph/extractors/_base.py +0 -14
- cognite/neat/legacy/graph/extractors/_dexpi.py +0 -44
- cognite/neat/legacy/graph/extractors/_graph_capturing_sheet.py +0 -403
- cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +0 -361
- cognite/neat/legacy/graph/loaders/__init__.py +0 -23
- cognite/neat/legacy/graph/loaders/_asset_loader.py +0 -511
- cognite/neat/legacy/graph/loaders/_base.py +0 -67
- cognite/neat/legacy/graph/loaders/_exceptions.py +0 -85
- cognite/neat/legacy/graph/loaders/core/__init__.py +0 -0
- cognite/neat/legacy/graph/loaders/core/labels.py +0 -58
- cognite/neat/legacy/graph/loaders/core/models.py +0 -136
- cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +0 -1046
- cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +0 -559
- cognite/neat/legacy/graph/loaders/rdf_to_dms.py +0 -309
- cognite/neat/legacy/graph/loaders/validator.py +0 -87
- cognite/neat/legacy/graph/models.py +0 -6
- cognite/neat/legacy/graph/stores/__init__.py +0 -13
- cognite/neat/legacy/graph/stores/_base.py +0 -400
- cognite/neat/legacy/graph/stores/_graphdb_store.py +0 -52
- cognite/neat/legacy/graph/stores/_memory_store.py +0 -43
- cognite/neat/legacy/graph/stores/_oxigraph_store.py +0 -151
- cognite/neat/legacy/graph/stores/_oxrdflib.py +0 -247
- cognite/neat/legacy/graph/stores/_rdf_to_graph.py +0 -42
- cognite/neat/legacy/graph/transformations/__init__.py +0 -0
- cognite/neat/legacy/graph/transformations/entity_matcher.py +0 -101
- cognite/neat/legacy/graph/transformations/query_generator/__init__.py +0 -3
- cognite/neat/legacy/graph/transformations/query_generator/sparql.py +0 -575
- cognite/neat/legacy/graph/transformations/transformer.py +0 -322
- cognite/neat/legacy/rules/__init__.py +0 -0
- cognite/neat/legacy/rules/analysis.py +0 -231
- cognite/neat/legacy/rules/examples/Rules-Nordic44-to-graphql.xlsx +0 -0
- cognite/neat/legacy/rules/examples/Rules-Nordic44.xlsx +0 -0
- cognite/neat/legacy/rules/examples/__init__.py +0 -18
- cognite/neat/legacy/rules/examples/power-grid-containers.yaml +0 -124
- cognite/neat/legacy/rules/examples/power-grid-example.xlsx +0 -0
- cognite/neat/legacy/rules/examples/power-grid-model.yaml +0 -224
- cognite/neat/legacy/rules/examples/rules-template.xlsx +0 -0
- cognite/neat/legacy/rules/examples/sheet2cdf-transformation-rules.xlsx +0 -0
- cognite/neat/legacy/rules/examples/skos-rules.xlsx +0 -0
- cognite/neat/legacy/rules/examples/source-to-solution-mapping-rules.xlsx +0 -0
- cognite/neat/legacy/rules/examples/wind-energy.owl +0 -1511
- cognite/neat/legacy/rules/exceptions.py +0 -2972
- cognite/neat/legacy/rules/exporters/__init__.py +0 -20
- cognite/neat/legacy/rules/exporters/_base.py +0 -45
- cognite/neat/legacy/rules/exporters/_core/__init__.py +0 -5
- cognite/neat/legacy/rules/exporters/_core/rules2labels.py +0 -24
- cognite/neat/legacy/rules/exporters/_rules2dms.py +0 -885
- cognite/neat/legacy/rules/exporters/_rules2excel.py +0 -213
- cognite/neat/legacy/rules/exporters/_rules2graphql.py +0 -183
- cognite/neat/legacy/rules/exporters/_rules2ontology.py +0 -524
- cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +0 -748
- cognite/neat/legacy/rules/exporters/_rules2rules.py +0 -105
- cognite/neat/legacy/rules/exporters/_rules2triples.py +0 -38
- cognite/neat/legacy/rules/exporters/_validation.py +0 -146
- cognite/neat/legacy/rules/importers/__init__.py +0 -22
- cognite/neat/legacy/rules/importers/_base.py +0 -66
- cognite/neat/legacy/rules/importers/_dict2rules.py +0 -158
- cognite/neat/legacy/rules/importers/_dms2rules.py +0 -194
- cognite/neat/legacy/rules/importers/_graph2rules.py +0 -308
- cognite/neat/legacy/rules/importers/_json2rules.py +0 -39
- cognite/neat/legacy/rules/importers/_owl2rules/__init__.py +0 -3
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +0 -239
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +0 -260
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +0 -217
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2rules.py +0 -290
- cognite/neat/legacy/rules/importers/_spreadsheet2rules.py +0 -45
- cognite/neat/legacy/rules/importers/_xsd2rules.py +0 -20
- cognite/neat/legacy/rules/importers/_yaml2rules.py +0 -39
- cognite/neat/legacy/rules/models/__init__.py +0 -5
- cognite/neat/legacy/rules/models/_base.py +0 -151
- cognite/neat/legacy/rules/models/raw_rules.py +0 -316
- cognite/neat/legacy/rules/models/rdfpath.py +0 -237
- cognite/neat/legacy/rules/models/rules.py +0 -1289
- cognite/neat/legacy/rules/models/tables.py +0 -9
- cognite/neat/legacy/rules/models/value_types.py +0 -118
- cognite/neat/legacy/workflows/examples/Export_DMS/workflow.yaml +0 -89
- cognite/neat/legacy/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
- cognite/neat/legacy/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
- cognite/neat/legacy/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
- cognite/neat/legacy/workflows/examples/Import_DMS/workflow.yaml +0 -65
- cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
- cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml +0 -67
- cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
- cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml +0 -95
- cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml +0 -111
- cognite/neat/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
- cognite/neat/workflows/migration/__init__.py +0 -0
- cognite/neat/workflows/migration/steps.py +0 -91
- cognite/neat/workflows/migration/wf_manifests.py +0 -33
- cognite/neat/workflows/steps/lib/legacy/__init__.py +0 -7
- cognite/neat/workflows/steps/lib/legacy/graph_contextualization.py +0 -82
- cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +0 -746
- cognite/neat/workflows/steps/lib/legacy/graph_loader.py +0 -606
- cognite/neat/workflows/steps/lib/legacy/graph_store.py +0 -307
- cognite/neat/workflows/steps/lib/legacy/graph_transformer.py +0 -58
- cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +0 -511
- cognite/neat/workflows/steps/lib/legacy/rules_importer.py +0 -612
- {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,885 +0,0 @@
|
|
|
1
|
-
"""Exports rules to CDF Data Model Storage (DMS) through cognite-sdk."""
|
|
2
|
-
|
|
3
|
-
import dataclasses
|
|
4
|
-
import sys
|
|
5
|
-
import warnings
|
|
6
|
-
from dataclasses import dataclass
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
from typing import ClassVar, Literal, cast, no_type_check
|
|
9
|
-
|
|
10
|
-
import yaml
|
|
11
|
-
from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
|
|
12
|
-
|
|
13
|
-
from cognite.neat.legacy.rules.models.value_types import ValueTypeMapping
|
|
14
|
-
|
|
15
|
-
if sys.version_info >= (3, 11):
|
|
16
|
-
from typing import Self
|
|
17
|
-
else:
|
|
18
|
-
from typing_extensions import Self
|
|
19
|
-
|
|
20
|
-
from cognite.client import CogniteClient
|
|
21
|
-
from cognite.client import data_modeling as dm
|
|
22
|
-
from cognite.client.data_classes.data_modeling import (
|
|
23
|
-
ContainerApply,
|
|
24
|
-
ContainerApplyList,
|
|
25
|
-
ContainerId,
|
|
26
|
-
ContainerProperty,
|
|
27
|
-
DataModelApply,
|
|
28
|
-
DataModelId,
|
|
29
|
-
DirectRelation,
|
|
30
|
-
DirectRelationReference,
|
|
31
|
-
MappedPropertyApply,
|
|
32
|
-
PropertyType,
|
|
33
|
-
SpaceApply,
|
|
34
|
-
ViewApply,
|
|
35
|
-
ViewId,
|
|
36
|
-
)
|
|
37
|
-
from cognite.client.data_classes.data_modeling.views import (
|
|
38
|
-
ConnectionDefinitionApply,
|
|
39
|
-
SingleHopConnectionDefinitionApply,
|
|
40
|
-
)
|
|
41
|
-
from cognite.client.exceptions import CogniteAPIError
|
|
42
|
-
from pydantic import BaseModel, ConfigDict, field_validator
|
|
43
|
-
|
|
44
|
-
from cognite.neat.legacy.rules import exceptions
|
|
45
|
-
from cognite.neat.legacy.rules.exporters._base import BaseExporter
|
|
46
|
-
from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant
|
|
47
|
-
from cognite.neat.legacy.rules.models._base import ContainerEntity, EntityTypes, ParentClass
|
|
48
|
-
from cognite.neat.legacy.rules.models.rules import Class, Property, Rules
|
|
49
|
-
from cognite.neat.utils.auxiliary import generate_exception_report
|
|
50
|
-
|
|
51
|
-
if sys.version_info < (3, 11):
|
|
52
|
-
from exceptiongroup import ExceptionGroup
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
@dataclass
|
|
56
|
-
class EmptyPropertyType(PropertyType):
|
|
57
|
-
_type = "No property"
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
@dataclass
|
|
61
|
-
class DMSSchema:
|
|
62
|
-
data_model: DataModelApply
|
|
63
|
-
containers: ContainerApplyList
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
class DMSExporter(BaseExporter[DMSSchema]):
|
|
67
|
-
"""Class for exporting transformation rules object to CDF Data Model Storage (DMS).
|
|
68
|
-
|
|
69
|
-
Args:
|
|
70
|
-
rules: Transformation rules object.
|
|
71
|
-
data_model_id: The id of the data model to be created.
|
|
72
|
-
container_policy: How to create/reuse existing containers.
|
|
73
|
-
existing_model: In the case of updating an existing model, this is the existing model.
|
|
74
|
-
report: Report. This is used when the exporter object is created from RawRules
|
|
75
|
-
|
|
76
|
-
!!! note "Container policy"
|
|
77
|
-
Here is more information about the different container policies:
|
|
78
|
-
- `create`: assumes no containers exist in CDF, will attempt to create them
|
|
79
|
-
- `reuse`: assumes containers exists in CDF, will attempt to only re-use them
|
|
80
|
-
- `extend`: will re-use existing, extend and/or create only missing containers
|
|
81
|
-
- `optimize`: create containers of size's prescribed by NEAT's optimization algorithm
|
|
82
|
-
"""
|
|
83
|
-
|
|
84
|
-
def __init__(
|
|
85
|
-
self,
|
|
86
|
-
rules: Rules,
|
|
87
|
-
data_model_id: dm.DataModelId | None = None,
|
|
88
|
-
container_policy: Literal["create", "reuse", "extend", "optimize"] = "create",
|
|
89
|
-
existing_model: dm.DataModel[dm.View] | None = None,
|
|
90
|
-
report: str | None = None,
|
|
91
|
-
):
|
|
92
|
-
super().__init__(rules, report)
|
|
93
|
-
self.data_model_id = data_model_id
|
|
94
|
-
self.container_policy = container_policy
|
|
95
|
-
if container_policy == "extend" and existing_model is None:
|
|
96
|
-
raise ValueError("Container policy is extend-existing, but no existing model is provided")
|
|
97
|
-
if container_policy != "create":
|
|
98
|
-
raise NotImplementedError("Only one-to-one-view container policy is currently supported")
|
|
99
|
-
self.existing_model = existing_model
|
|
100
|
-
|
|
101
|
-
def _export_to_file(self, filepath: Path) -> None:
|
|
102
|
-
if filepath.suffix not in {".yaml", ".yml"}:
|
|
103
|
-
warnings.warn("File extension is not .yaml, adding it to the file name", stacklevel=2)
|
|
104
|
-
filepath = filepath.with_suffix(".yaml")
|
|
105
|
-
schema = self.export()
|
|
106
|
-
filepath.write_text(
|
|
107
|
-
yaml.safe_dump(
|
|
108
|
-
{
|
|
109
|
-
"data_models": [schema.data_model.dump(camel_case=True)],
|
|
110
|
-
"containers": schema.containers.dump(camel_case=True),
|
|
111
|
-
}
|
|
112
|
-
)
|
|
113
|
-
)
|
|
114
|
-
|
|
115
|
-
def export(self) -> DMSSchema:
|
|
116
|
-
model = DMSSchemaComponents.from_rules(self.rules, self.data_model_id)
|
|
117
|
-
return DMSSchema(
|
|
118
|
-
data_model=DataModelApply(
|
|
119
|
-
space=model.space,
|
|
120
|
-
external_id=model.external_id,
|
|
121
|
-
version=model.version,
|
|
122
|
-
description=model.description,
|
|
123
|
-
name=model.name,
|
|
124
|
-
views=list(model.views.values()),
|
|
125
|
-
),
|
|
126
|
-
containers=ContainerApplyList(model.containers.values()),
|
|
127
|
-
)
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
class DMSSchemaComponents(BaseModel):
|
|
131
|
-
"""
|
|
132
|
-
DMS Schema Components pydantic class used to create space(s), containers, views and data model in CDF.
|
|
133
|
-
|
|
134
|
-
This can be used to create a data model in CDF from rules.
|
|
135
|
-
|
|
136
|
-
Args:
|
|
137
|
-
space: Name of the space to place the resulting data model.
|
|
138
|
-
external_id: External id of the data model.
|
|
139
|
-
version: Version of the data model.
|
|
140
|
-
description: Description of the data model.
|
|
141
|
-
name: Name of the data model.
|
|
142
|
-
containers: Containers connected to the data model.
|
|
143
|
-
views: Views connected to the data model.
|
|
144
|
-
|
|
145
|
-
"""
|
|
146
|
-
|
|
147
|
-
space: str
|
|
148
|
-
external_id: str
|
|
149
|
-
version: str
|
|
150
|
-
description: str | None = None
|
|
151
|
-
name: str | None = None
|
|
152
|
-
containers: dict[str, ContainerApply]
|
|
153
|
-
views: dict[str, ViewApply]
|
|
154
|
-
|
|
155
|
-
model_config: ClassVar[ConfigDict] = ConfigDict(
|
|
156
|
-
populate_by_name=True, str_strip_whitespace=True, arbitrary_types_allowed=True, strict=False, extra="allow"
|
|
157
|
-
)
|
|
158
|
-
|
|
159
|
-
@field_validator("views", mode="after")
|
|
160
|
-
def remove_empty_views(cls, value):
|
|
161
|
-
"""This validator removes views that do not have any properties or implements."""
|
|
162
|
-
for view_id, view in list(value.items()):
|
|
163
|
-
if not view.properties and not view.implements:
|
|
164
|
-
del value[view_id]
|
|
165
|
-
|
|
166
|
-
return value
|
|
167
|
-
|
|
168
|
-
@property
|
|
169
|
-
def spaces(self):
|
|
170
|
-
return set(
|
|
171
|
-
[container.space for container in self.containers.values()]
|
|
172
|
-
+ [self.space]
|
|
173
|
-
+ [view.space for view in self.views.values()]
|
|
174
|
-
)
|
|
175
|
-
|
|
176
|
-
@classmethod
|
|
177
|
-
def from_rules(
|
|
178
|
-
cls, rules: Rules, data_model_id: dm.DataModelId | None = None, set_expected_source: bool = True
|
|
179
|
-
) -> Self:
|
|
180
|
-
"""Generates a DataModel class instance from a Rules instance.
|
|
181
|
-
|
|
182
|
-
Args:
|
|
183
|
-
rules: instance of Rules.
|
|
184
|
-
data_model_id: The id of the data model to be created.
|
|
185
|
-
set_expected_source: Whether to set the expected source on views with direct relation properties.
|
|
186
|
-
|
|
187
|
-
Returns:
|
|
188
|
-
Instance of DataModel.
|
|
189
|
-
"""
|
|
190
|
-
if data_model_id and data_model_id.space:
|
|
191
|
-
# update space in rules to match space
|
|
192
|
-
rules.update_space(data_model_id.space)
|
|
193
|
-
|
|
194
|
-
if data_model_id and data_model_id.version:
|
|
195
|
-
# update version in rules to match version
|
|
196
|
-
rules.update_version(data_model_id.version)
|
|
197
|
-
|
|
198
|
-
if data_model_id and data_model_id.external_id:
|
|
199
|
-
# update data model name to match external_id
|
|
200
|
-
rules.metadata.suffix = data_model_id.external_id
|
|
201
|
-
|
|
202
|
-
names_compliant, name_warnings = are_entity_names_dms_compliant(rules, return_report=True)
|
|
203
|
-
if not names_compliant:
|
|
204
|
-
raise exceptions.EntitiesContainNonDMSCompliantCharacters(report=generate_exception_report(name_warnings))
|
|
205
|
-
|
|
206
|
-
if rules.metadata.external_id is None:
|
|
207
|
-
raise exceptions.DataModelIdMissing(prefix=rules.metadata.space)
|
|
208
|
-
|
|
209
|
-
return cls(
|
|
210
|
-
space=rules.metadata.space,
|
|
211
|
-
external_id=rules.metadata.external_id,
|
|
212
|
-
version=rules.metadata.version,
|
|
213
|
-
description=rules.metadata.description,
|
|
214
|
-
name=rules.metadata.name,
|
|
215
|
-
containers=cls.containers_from_rules(rules),
|
|
216
|
-
views=cls.views_from_rules(rules),
|
|
217
|
-
)
|
|
218
|
-
|
|
219
|
-
@classmethod
|
|
220
|
-
def containers_from_rules(cls, rules: Rules) -> dict[str, ContainerApply]:
|
|
221
|
-
"""Create a dictionary of ContainerApply instances from a Rules instance.
|
|
222
|
-
|
|
223
|
-
Args:
|
|
224
|
-
rules: instance of Rules.`
|
|
225
|
-
|
|
226
|
-
Returns:
|
|
227
|
-
Dictionary of ContainerApply instances.
|
|
228
|
-
"""
|
|
229
|
-
|
|
230
|
-
containers: dict[str, ContainerApply] = {}
|
|
231
|
-
errors: list = []
|
|
232
|
-
|
|
233
|
-
for row, property_ in rules.properties.items():
|
|
234
|
-
if property_.container:
|
|
235
|
-
container_property_id: str = cast(str, property_.container_property)
|
|
236
|
-
container_id = property_.container.id
|
|
237
|
-
|
|
238
|
-
api_container = cls.as_api_container(property_.container)
|
|
239
|
-
api_container_property = cls.as_api_container_property(property_)
|
|
240
|
-
|
|
241
|
-
# scenario: adding new container to the data model for the first time
|
|
242
|
-
if not isinstance(api_container_property.type, EmptyPropertyType) and container_id not in containers:
|
|
243
|
-
containers[container_id] = api_container
|
|
244
|
-
containers[container_id].properties[container_property_id] = api_container_property
|
|
245
|
-
|
|
246
|
-
# scenario: adding new property to an existing container
|
|
247
|
-
elif (
|
|
248
|
-
not isinstance(api_container_property.type, EmptyPropertyType)
|
|
249
|
-
and container_property_id not in containers[container_id].properties
|
|
250
|
-
):
|
|
251
|
-
containers[container_id].properties[container_property_id] = api_container_property
|
|
252
|
-
|
|
253
|
-
# scenario: property is redefined checking for potential sub-scenarios
|
|
254
|
-
elif (
|
|
255
|
-
not isinstance(api_container_property.type, EmptyPropertyType)
|
|
256
|
-
and container_property_id in containers[container_id].properties
|
|
257
|
-
):
|
|
258
|
-
existing_property = containers[container_id].properties[container_property_id]
|
|
259
|
-
|
|
260
|
-
# scenario: property is redefined with a different value type -> raise error
|
|
261
|
-
if not isinstance(existing_property.type, type(api_container_property.type)):
|
|
262
|
-
errors.append(
|
|
263
|
-
exceptions.ContainerPropertyValueTypeRedefinition(
|
|
264
|
-
container_id=container_id,
|
|
265
|
-
property_id=container_property_id,
|
|
266
|
-
current_value_type=str(existing_property.type),
|
|
267
|
-
redefined_value_type=str(api_container_property.type),
|
|
268
|
-
loc=f"[Properties/Type/{row}]",
|
|
269
|
-
)
|
|
270
|
-
)
|
|
271
|
-
|
|
272
|
-
# scenario: default value is redefined -> set default value to None
|
|
273
|
-
if (
|
|
274
|
-
not isinstance(existing_property.type, DirectRelation)
|
|
275
|
-
and not isinstance(api_container_property.type, DirectRelation)
|
|
276
|
-
and existing_property.default_value != api_container_property.default_value
|
|
277
|
-
):
|
|
278
|
-
containers[container_id].properties[container_property_id] = dataclasses.replace(
|
|
279
|
-
existing_property, default_value=None
|
|
280
|
-
)
|
|
281
|
-
|
|
282
|
-
# scenario: property holds multiple values -> set is_list to True
|
|
283
|
-
if (
|
|
284
|
-
not isinstance(existing_property.type, DirectRelation)
|
|
285
|
-
and not isinstance(api_container_property.type, DirectRelation)
|
|
286
|
-
and isinstance(existing_property.type, ListablePropertyType)
|
|
287
|
-
and isinstance(api_container_property.type, ListablePropertyType)
|
|
288
|
-
and existing_property.type.is_list != api_container_property.type.is_list
|
|
289
|
-
):
|
|
290
|
-
type_ = containers[container_id].properties[container_property_id].type
|
|
291
|
-
if isinstance(type_, ListablePropertyType):
|
|
292
|
-
type_.is_list = True
|
|
293
|
-
|
|
294
|
-
if errors:
|
|
295
|
-
raise ExceptionGroup("Properties value types have been redefined! This is prohibited! Aborting!", errors)
|
|
296
|
-
|
|
297
|
-
return containers
|
|
298
|
-
|
|
299
|
-
@classmethod
|
|
300
|
-
def as_api_container_property(cls, property_: Property) -> ContainerProperty:
|
|
301
|
-
is_one_to_many = property_.max_count != 1
|
|
302
|
-
|
|
303
|
-
# Literal, i.e. Node attribute
|
|
304
|
-
if property_.property_type is EntityTypes.data_property:
|
|
305
|
-
property_type = cast(ValueTypeMapping, property_.expected_value_type.mapping).dms
|
|
306
|
-
return ContainerProperty(
|
|
307
|
-
type=property_type(is_list=is_one_to_many),
|
|
308
|
-
nullable=property_.min_count == 0,
|
|
309
|
-
default_value=property_.default,
|
|
310
|
-
name=property_.property_name,
|
|
311
|
-
description=property_.description,
|
|
312
|
-
)
|
|
313
|
-
|
|
314
|
-
# URIRef, i.e. edge 1-1 , aka direct relation between Nodes
|
|
315
|
-
elif property_.property_type is EntityTypes.object_property and not is_one_to_many:
|
|
316
|
-
return ContainerProperty(
|
|
317
|
-
type=DirectRelation(),
|
|
318
|
-
nullable=True,
|
|
319
|
-
name=property_.property_name,
|
|
320
|
-
description=property_.description,
|
|
321
|
-
)
|
|
322
|
-
|
|
323
|
-
# return type=None if property cannot be created
|
|
324
|
-
else:
|
|
325
|
-
return ContainerProperty(type=EmptyPropertyType())
|
|
326
|
-
|
|
327
|
-
@classmethod
|
|
328
|
-
def as_api_container(cls, container: ContainerEntity) -> ContainerApply:
|
|
329
|
-
return ContainerApply(
|
|
330
|
-
space=container.space,
|
|
331
|
-
external_id=container.external_id,
|
|
332
|
-
description=container.description,
|
|
333
|
-
name=container.name,
|
|
334
|
-
# properties are added later
|
|
335
|
-
properties={},
|
|
336
|
-
)
|
|
337
|
-
|
|
338
|
-
@classmethod
|
|
339
|
-
def views_from_rules(cls, rules: Rules) -> dict[str, ViewApply]:
|
|
340
|
-
"""Generates a dictionary of ViewApply instances from a Rules instance.
|
|
341
|
-
|
|
342
|
-
Args:
|
|
343
|
-
rule: Instance of Rules.
|
|
344
|
-
space: Name of the space to place the views.
|
|
345
|
-
|
|
346
|
-
Returns:
|
|
347
|
-
Dictionary of ViewApply instances.
|
|
348
|
-
"""
|
|
349
|
-
views: dict[str, ViewApply] = {
|
|
350
|
-
f"{rules.metadata.space}:{class_.class_id}": cls.as_api_view(
|
|
351
|
-
class_, rules.metadata.space, rules.metadata.version
|
|
352
|
-
)
|
|
353
|
-
for class_ in rules.classes.values()
|
|
354
|
-
}
|
|
355
|
-
errors: list = []
|
|
356
|
-
|
|
357
|
-
# Create views from property-class definitions
|
|
358
|
-
for row, property_ in rules.properties.items():
|
|
359
|
-
view_property = cls.as_api_view_property(property_, rules.metadata.space)
|
|
360
|
-
id_ = f"{rules.metadata.space}:{property_.class_id}"
|
|
361
|
-
|
|
362
|
-
# scenario: view exist but property does not so it is added
|
|
363
|
-
if view_property and (property_.property_id not in cast(dict, views[id_].properties)):
|
|
364
|
-
cast(dict, views[id_].properties)[property_.property_id] = view_property
|
|
365
|
-
|
|
366
|
-
# scenario: view exist, property exists but it is differently defined -> raise error
|
|
367
|
-
# type: ignore
|
|
368
|
-
elif (
|
|
369
|
-
view_property
|
|
370
|
-
and property_.property_id in cast(dict, views[id_].properties)
|
|
371
|
-
and view_property is not cast(dict, views[id_].properties)[property_.property_id]
|
|
372
|
-
):
|
|
373
|
-
errors.append(
|
|
374
|
-
exceptions.ViewPropertyRedefinition(
|
|
375
|
-
view_id=id_,
|
|
376
|
-
property_id=cast(str, property_.property_id),
|
|
377
|
-
loc=f"[Properties/Property/{row}]",
|
|
378
|
-
)
|
|
379
|
-
)
|
|
380
|
-
|
|
381
|
-
if errors:
|
|
382
|
-
raise ExceptionGroup("View properties have been redefined! This is prohibited! Aborting!", errors)
|
|
383
|
-
|
|
384
|
-
return views
|
|
385
|
-
|
|
386
|
-
@classmethod
|
|
387
|
-
def as_api_view(cls, class_: Class, space: str, version: str) -> ViewApply:
|
|
388
|
-
return ViewApply(
|
|
389
|
-
space=space,
|
|
390
|
-
external_id=class_.class_id,
|
|
391
|
-
version=version,
|
|
392
|
-
name=class_.class_name,
|
|
393
|
-
description=class_.description,
|
|
394
|
-
properties={},
|
|
395
|
-
implements=(
|
|
396
|
-
[parent_class.view_id for parent_class in cast(list[ParentClass], class_.parent_class)]
|
|
397
|
-
if class_.parent_class
|
|
398
|
-
else None
|
|
399
|
-
),
|
|
400
|
-
)
|
|
401
|
-
|
|
402
|
-
@classmethod
|
|
403
|
-
def as_api_view_property(
|
|
404
|
-
cls, property_: Property, space: str
|
|
405
|
-
) -> MappedPropertyApply | ConnectionDefinitionApply | None:
|
|
406
|
-
property_.container = cast(ContainerEntity, property_.container)
|
|
407
|
-
property_.container_property = cast(str, property_.container_property)
|
|
408
|
-
if property_.property_type is EntityTypes.data_property:
|
|
409
|
-
return MappedPropertyApply(
|
|
410
|
-
container=ContainerId(space=property_.container.space, external_id=property_.container.external_id),
|
|
411
|
-
container_property_identifier=property_.container_property,
|
|
412
|
-
name=property_.property_name,
|
|
413
|
-
description=property_.description,
|
|
414
|
-
)
|
|
415
|
-
|
|
416
|
-
# edge 1-1 == directRelation
|
|
417
|
-
elif property_.property_type is EntityTypes.object_property and property_.max_count == 1:
|
|
418
|
-
return MappedPropertyApply(
|
|
419
|
-
container=ContainerId(space=property_.container.space, external_id=property_.container.external_id),
|
|
420
|
-
container_property_identifier=property_.container_property,
|
|
421
|
-
name=property_.property_name,
|
|
422
|
-
description=property_.description,
|
|
423
|
-
source=ViewId(
|
|
424
|
-
space=property_.expected_value_type.space,
|
|
425
|
-
external_id=property_.expected_value_type.external_id,
|
|
426
|
-
version=property_.expected_value_type.version,
|
|
427
|
-
),
|
|
428
|
-
)
|
|
429
|
-
|
|
430
|
-
# edge 1-many == EDGE, this does not have container! type is here source ViewId ?!
|
|
431
|
-
elif property_.property_type is EntityTypes.object_property and property_.max_count != 1:
|
|
432
|
-
if property_.container and property_.expected_value_type.space != property_.container.space:
|
|
433
|
-
type_ = DirectRelationReference(
|
|
434
|
-
space=property_.container.space,
|
|
435
|
-
external_id=f"{property_.container.suffix}.{property_.container_property}",
|
|
436
|
-
)
|
|
437
|
-
else:
|
|
438
|
-
type_ = DirectRelationReference(
|
|
439
|
-
space=space, external_id=f"{property_.class_id}.{property_.property_id}"
|
|
440
|
-
)
|
|
441
|
-
|
|
442
|
-
return SingleHopConnectionDefinitionApply(
|
|
443
|
-
type=type_,
|
|
444
|
-
# Here we create ViewID to the container that the edge is pointing to.
|
|
445
|
-
source=ViewId(
|
|
446
|
-
space=property_.expected_value_type.space,
|
|
447
|
-
external_id=property_.expected_value_type.external_id,
|
|
448
|
-
version=property_.expected_value_type.version,
|
|
449
|
-
),
|
|
450
|
-
direction="outwards",
|
|
451
|
-
name=property_.property_name,
|
|
452
|
-
description=property_.description,
|
|
453
|
-
)
|
|
454
|
-
else:
|
|
455
|
-
return None
|
|
456
|
-
|
|
457
|
-
def find_existing_spaces(self, client: CogniteClient) -> set[str]:
|
|
458
|
-
"""Checks if the spaces exist in CDF.
|
|
459
|
-
|
|
460
|
-
Args:
|
|
461
|
-
client: Cognite client.
|
|
462
|
-
|
|
463
|
-
Returns:
|
|
464
|
-
External ids of spaces which are part of DMS Schema components that already exist in CDF.
|
|
465
|
-
"""
|
|
466
|
-
|
|
467
|
-
return set(client.data_modeling.spaces.retrieve(list(self.spaces)).as_ids())
|
|
468
|
-
|
|
469
|
-
def find_existing_containers(self, client: CogniteClient) -> set[str]:
|
|
470
|
-
"""Checks if the containers exist in CDF.
|
|
471
|
-
|
|
472
|
-
Args:
|
|
473
|
-
client: Cognite client.
|
|
474
|
-
|
|
475
|
-
Returns:
|
|
476
|
-
External ids of containers which are part of DMS Schema components that already exist in CDF.
|
|
477
|
-
"""
|
|
478
|
-
|
|
479
|
-
return {
|
|
480
|
-
f"{id_.space}:{id_.external_id}"
|
|
481
|
-
for id_ in client.data_modeling.containers.retrieve(
|
|
482
|
-
[container.as_id() for container in self.containers.values()]
|
|
483
|
-
).as_ids()
|
|
484
|
-
}
|
|
485
|
-
|
|
486
|
-
def find_existing_views(self, client: CogniteClient) -> set[str]:
|
|
487
|
-
"""Checks if the views exist in CDF.
|
|
488
|
-
|
|
489
|
-
Args:
|
|
490
|
-
client: Cognite client.
|
|
491
|
-
|
|
492
|
-
Returns:
|
|
493
|
-
External ids of views which are part of DMS Schema components that already exist in CDF.
|
|
494
|
-
"""
|
|
495
|
-
|
|
496
|
-
return {
|
|
497
|
-
f"{id_.space}:{id_.external_id}"
|
|
498
|
-
for id_ in client.data_modeling.views.retrieve([view.as_id() for view in self.views.values()]).as_ids()
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
def find_existing_data_model(self, client: CogniteClient) -> DataModelId | None:
|
|
502
|
-
"""Checks if the data model exists in CDF.
|
|
503
|
-
|
|
504
|
-
Args:
|
|
505
|
-
client: Cognite client.
|
|
506
|
-
|
|
507
|
-
Returns:
|
|
508
|
-
True if the data model exists, False otherwise.
|
|
509
|
-
"""
|
|
510
|
-
if model := client.data_modeling.data_models.retrieve((self.space, self.external_id, self.version)):
|
|
511
|
-
cdf_data_model = model.latest_version()
|
|
512
|
-
return cdf_data_model.as_id()
|
|
513
|
-
return None
|
|
514
|
-
|
|
515
|
-
# mypy unsatisfied with overload , tried all combination and gave up
|
|
516
|
-
@no_type_check
|
|
517
|
-
def to_cdf(
|
|
518
|
-
self,
|
|
519
|
-
client: CogniteClient,
|
|
520
|
-
components_to_create: set | None = None,
|
|
521
|
-
existing_component_handling: Literal["fail", "skip", "update"] = "fail",
|
|
522
|
-
multi_space_components_create: bool = False,
|
|
523
|
-
return_report: bool = False,
|
|
524
|
-
) -> None | tuple[dict[str, list], dict[str, list]]:
|
|
525
|
-
"""Write the the data model to CDF.
|
|
526
|
-
|
|
527
|
-
Args:
|
|
528
|
-
client: Connected Cognite client.
|
|
529
|
-
components_to_create: Which components to create. Takes set
|
|
530
|
-
existing_component_handling: How to handle existing components. Takes Literal["fail", "skip", "update"]
|
|
531
|
-
multi_space_comp_create: Whether to create only components belonging to the data model space,
|
|
532
|
-
or also additionally components outside of the data model space. Default is False.
|
|
533
|
-
|
|
534
|
-
!!! note "Multi Space DMS Schema Components"
|
|
535
|
-
If multi_space_components_create is set to True, the components will be created
|
|
536
|
-
in all spaces used or defined in `Rules`. If set to False, the
|
|
537
|
-
components will only be created in the space defined in `Rules` metadata.
|
|
538
|
-
|
|
539
|
-
!!! note "Component Creation Policy"
|
|
540
|
-
Here is more information about the different component creation policies
|
|
541
|
-
configured through components_to_create argument:
|
|
542
|
-
|
|
543
|
-
- `all`: all components of the data model will be created, meaning space, containers, views and data model
|
|
544
|
-
- `data model`: only the data model will be created
|
|
545
|
-
- `view`: only views will be created
|
|
546
|
-
- `container`: only the containers will be created
|
|
547
|
-
|
|
548
|
-
for creation of containers beyond the data model space, set `multi_space_components_create` argument
|
|
549
|
-
to True.
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
!!! note "Existing Component Handling Policy"
|
|
553
|
-
Here is more information about the different existing component handling policies:
|
|
554
|
-
|
|
555
|
-
- `fail`: if any component of the data model (DMS schema) already exists
|
|
556
|
-
- `skip`: skip DMS components that exist
|
|
557
|
-
- `update`: create DMS components that do not exist and update those that do exist
|
|
558
|
-
|
|
559
|
-
`update` policy is currently not implemented !
|
|
560
|
-
|
|
561
|
-
"""
|
|
562
|
-
|
|
563
|
-
logs, errors = {}, {}
|
|
564
|
-
|
|
565
|
-
components_to_create = components_to_create or {"all"}
|
|
566
|
-
|
|
567
|
-
existing_spaces = self.find_existing_spaces(client)
|
|
568
|
-
existing_containers = self.find_existing_containers(client)
|
|
569
|
-
existing_views = self.find_existing_views(client)
|
|
570
|
-
existing_data_model = self.find_existing_data_model(client)
|
|
571
|
-
|
|
572
|
-
if (
|
|
573
|
-
existing_spaces or existing_containers or existing_data_model or existing_views
|
|
574
|
-
) and existing_component_handling == "fail":
|
|
575
|
-
raise exceptions.DataModelOrItsComponentsAlreadyExist(
|
|
576
|
-
existing_spaces,
|
|
577
|
-
existing_data_model,
|
|
578
|
-
existing_containers,
|
|
579
|
-
existing_views,
|
|
580
|
-
)
|
|
581
|
-
|
|
582
|
-
if "space" in components_to_create or "all" in components_to_create:
|
|
583
|
-
logs["space"], errors["space"] = self.create_space(
|
|
584
|
-
client,
|
|
585
|
-
existing_spaces,
|
|
586
|
-
existing_component_handling == "update",
|
|
587
|
-
multi_space_components_create,
|
|
588
|
-
)
|
|
589
|
-
if "container" in components_to_create or "all" in components_to_create:
|
|
590
|
-
logs["container"], errors["container"] = self.create_containers(
|
|
591
|
-
client,
|
|
592
|
-
existing_containers,
|
|
593
|
-
existing_component_handling == "update",
|
|
594
|
-
multi_space_components_create,
|
|
595
|
-
)
|
|
596
|
-
|
|
597
|
-
if "view" in components_to_create or "all" in components_to_create:
|
|
598
|
-
logs["view"], errors["view"] = self.create_views(
|
|
599
|
-
client, existing_views, existing_component_handling == "update"
|
|
600
|
-
)
|
|
601
|
-
if "data model" in components_to_create or "all" in components_to_create:
|
|
602
|
-
logs["data model"], errors["data model"] = self.create_data_model(
|
|
603
|
-
client, existing_data_model, existing_component_handling == "update"
|
|
604
|
-
)
|
|
605
|
-
|
|
606
|
-
if return_report:
|
|
607
|
-
return logs, errors
|
|
608
|
-
|
|
609
|
-
def create_space(
|
|
610
|
-
self,
|
|
611
|
-
client: CogniteClient,
|
|
612
|
-
existing_spaces: set,
|
|
613
|
-
update: bool = False,
|
|
614
|
-
multi_space_components_create: bool = False,
|
|
615
|
-
) -> tuple[list, list]:
|
|
616
|
-
logs, errors = [], []
|
|
617
|
-
|
|
618
|
-
spaces_to_create = (
|
|
619
|
-
self.spaces - existing_spaces
|
|
620
|
-
if multi_space_components_create
|
|
621
|
-
else ((self.spaces - existing_spaces) & {self.space})
|
|
622
|
-
)
|
|
623
|
-
|
|
624
|
-
spaces_to_update = existing_spaces if multi_space_components_create else (existing_spaces & {self.space})
|
|
625
|
-
|
|
626
|
-
if spaces_to_create:
|
|
627
|
-
try:
|
|
628
|
-
_ = client.data_modeling.spaces.apply([SpaceApply(space=space) for space in spaces_to_create])
|
|
629
|
-
logs.append(f"Created space {spaces_to_create}")
|
|
630
|
-
except CogniteAPIError as e:
|
|
631
|
-
errors.append(f"Failed to create space {spaces_to_create}! Reason: {e.message}")
|
|
632
|
-
|
|
633
|
-
if update and spaces_to_update:
|
|
634
|
-
try:
|
|
635
|
-
_ = client.data_modeling.spaces.apply([SpaceApply(space=space) for space in spaces_to_update])
|
|
636
|
-
logs.append(f"Updated space {spaces_to_update}")
|
|
637
|
-
except CogniteAPIError as e:
|
|
638
|
-
errors.append(f"Failed to update spaces {spaces_to_update}! Reason: {e.message}")
|
|
639
|
-
|
|
640
|
-
return logs, errors
|
|
641
|
-
|
|
642
|
-
def create_containers(
|
|
643
|
-
self,
|
|
644
|
-
client: CogniteClient,
|
|
645
|
-
existing_containers: set,
|
|
646
|
-
update: bool = False,
|
|
647
|
-
multi_space_components_create: bool = False,
|
|
648
|
-
) -> tuple[list, list]:
|
|
649
|
-
logs, errors = [], []
|
|
650
|
-
|
|
651
|
-
containers_to_create = (
|
|
652
|
-
set(self.containers.keys()) - existing_containers
|
|
653
|
-
if multi_space_components_create
|
|
654
|
-
else {k for k in (set(self.containers.keys()) - existing_containers) if k.split(":")[0] == self.space}
|
|
655
|
-
)
|
|
656
|
-
|
|
657
|
-
containers_to_update = (
|
|
658
|
-
existing_containers
|
|
659
|
-
if multi_space_components_create
|
|
660
|
-
else {k for k in existing_containers if k.split(":")[0] == self.space}
|
|
661
|
-
)
|
|
662
|
-
|
|
663
|
-
if containers_to_create:
|
|
664
|
-
try:
|
|
665
|
-
_ = client.data_modeling.containers.apply([self.containers[id_] for id_ in containers_to_create])
|
|
666
|
-
logs.append(f"Created container {containers_to_create}")
|
|
667
|
-
except CogniteAPIError as e:
|
|
668
|
-
errors.append(f"Failed to create containers {containers_to_create}! Reason: {e.message}")
|
|
669
|
-
|
|
670
|
-
if update and containers_to_update:
|
|
671
|
-
try:
|
|
672
|
-
_ = client.data_modeling.containers.apply([self.containers[id_] for id_ in containers_to_update])
|
|
673
|
-
logs.append(f"Updated containers {containers_to_update}")
|
|
674
|
-
except CogniteAPIError as e:
|
|
675
|
-
errors.append(f"Failed to update containers {containers_to_update}! Reason: {e.message}")
|
|
676
|
-
|
|
677
|
-
return logs, errors
|
|
678
|
-
|
|
679
|
-
def create_views(
|
|
680
|
-
self,
|
|
681
|
-
client: CogniteClient,
|
|
682
|
-
existing_views: set,
|
|
683
|
-
update: bool = False,
|
|
684
|
-
) -> tuple[list, list]:
|
|
685
|
-
logs, errors = [], []
|
|
686
|
-
|
|
687
|
-
non_existing_views = set(self.views.keys()) - existing_views
|
|
688
|
-
if non_existing_views:
|
|
689
|
-
try:
|
|
690
|
-
_ = client.data_modeling.views.apply([self.views[id_] for id_ in non_existing_views])
|
|
691
|
-
logs.append(f"Created views {non_existing_views}")
|
|
692
|
-
except CogniteAPIError as e:
|
|
693
|
-
errors.append(f"Failed to update views {non_existing_views}! Reason: {e.message}")
|
|
694
|
-
|
|
695
|
-
if update and existing_views:
|
|
696
|
-
try:
|
|
697
|
-
_ = client.data_modeling.views.apply([self.views[id_] for id_ in existing_views])
|
|
698
|
-
logs.append(f"Updated views {existing_views}")
|
|
699
|
-
except CogniteAPIError as e:
|
|
700
|
-
errors.append(f"Failed to update views {existing_views}! Reason: {e.message}")
|
|
701
|
-
|
|
702
|
-
return logs, errors
|
|
703
|
-
|
|
704
|
-
def create_data_model(
|
|
705
|
-
self,
|
|
706
|
-
client: CogniteClient,
|
|
707
|
-
existing_data_model: DataModelId | None = None,
|
|
708
|
-
update: bool = False,
|
|
709
|
-
) -> tuple[list, list]:
|
|
710
|
-
logs, errors = [], []
|
|
711
|
-
|
|
712
|
-
if not existing_data_model:
|
|
713
|
-
try:
|
|
714
|
-
_ = client.data_modeling.data_models.apply(
|
|
715
|
-
DataModelApply(
|
|
716
|
-
name=self.name,
|
|
717
|
-
description=self.description,
|
|
718
|
-
space=self.space,
|
|
719
|
-
external_id=self.external_id,
|
|
720
|
-
version=self.version,
|
|
721
|
-
views=[view.as_id() for view in self.views.values()],
|
|
722
|
-
)
|
|
723
|
-
)
|
|
724
|
-
logs.append(f"Created data model {{{self.space}:{self.external_id}/{self.version}}}")
|
|
725
|
-
except CogniteAPIError as e:
|
|
726
|
-
errors.append(
|
|
727
|
-
f"Failed to create data model {{{self.space}:{self.external_id}/{self.version}}}!"
|
|
728
|
-
f" Reason: {e.message}"
|
|
729
|
-
)
|
|
730
|
-
elif update:
|
|
731
|
-
try:
|
|
732
|
-
_ = client.data_modeling.data_models.apply(
|
|
733
|
-
DataModelApply(
|
|
734
|
-
name=self.name,
|
|
735
|
-
description=self.description,
|
|
736
|
-
space=self.space,
|
|
737
|
-
external_id=self.external_id,
|
|
738
|
-
version=self.version,
|
|
739
|
-
views=[view.as_id() for view in self.views.values()],
|
|
740
|
-
)
|
|
741
|
-
)
|
|
742
|
-
logs.append("Updated data model " f"{{{self.space}:{self.external_id}/{self.version}}}")
|
|
743
|
-
except CogniteAPIError as e:
|
|
744
|
-
errors.append(
|
|
745
|
-
"Failed to update data model"
|
|
746
|
-
f" {{{self.space}:{self.external_id}/{self.version}}}! Reason: {e.message}"
|
|
747
|
-
)
|
|
748
|
-
|
|
749
|
-
else:
|
|
750
|
-
logs.append(f"Skipped update of data model {{{self.space}:{self.external_id}/{self.version}}}!")
|
|
751
|
-
|
|
752
|
-
return logs, errors
|
|
753
|
-
|
|
754
|
-
@no_type_check
|
|
755
|
-
def remove(
|
|
756
|
-
self,
|
|
757
|
-
client: CogniteClient,
|
|
758
|
-
components_to_remove: set | None = None,
|
|
759
|
-
multi_space_components_removal: bool = False,
|
|
760
|
-
return_report: bool = False,
|
|
761
|
-
) -> None | tuple[dict[str, list], dict[str, list]]:
|
|
762
|
-
"""Remove DMS schema components from CDF.
|
|
763
|
-
|
|
764
|
-
Args:
|
|
765
|
-
client: Connected Cognite client.
|
|
766
|
-
components_to_remove: Which components to remove. Takes set
|
|
767
|
-
multi_space_components_removal: Whether to remove components in multiple spaces,
|
|
768
|
-
or only in the space of the data model. Default is False.
|
|
769
|
-
|
|
770
|
-
!!! note "Component Creation Policy"
|
|
771
|
-
Here is more information about the different component creation policies:
|
|
772
|
-
- `all`: all components of the data model will be created, meaning space, containers, views and data model
|
|
773
|
-
- `data model`: only the data model will be created
|
|
774
|
-
- `view`: only the views will be created
|
|
775
|
-
- `container`: only the containers will be created
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
!!! note "Multi Space DMS Schema Components"
|
|
779
|
-
If multi_space_components_removal is set to True, the components will be removed
|
|
780
|
-
in all spaces used or defined in `Rules`. If set to False, the
|
|
781
|
-
components will only be removed in the space defined in `Rules` metadata.
|
|
782
|
-
"""
|
|
783
|
-
|
|
784
|
-
logs, errors = {}, {}
|
|
785
|
-
|
|
786
|
-
components_to_remove = components_to_remove or {"all"}
|
|
787
|
-
|
|
788
|
-
if "data model" in components_to_remove or "all" in components_to_remove:
|
|
789
|
-
logs["data model"], errors["data model"] = self.remove_data_model(client)
|
|
790
|
-
if "view" in components_to_remove or "all" in components_to_remove:
|
|
791
|
-
logs["view"], errors["view"] = self.remove_views(client)
|
|
792
|
-
if "container" in components_to_remove or "all" in components_to_remove:
|
|
793
|
-
logs["container"], errors["container"] = self.remove_containers(client, multi_space_components_removal)
|
|
794
|
-
if "space" in components_to_remove or "all" in components_to_remove:
|
|
795
|
-
logs["space"], errors["space"] = self.remove_spaces(client, multi_space_components_removal)
|
|
796
|
-
|
|
797
|
-
if return_report:
|
|
798
|
-
return logs, errors
|
|
799
|
-
|
|
800
|
-
def remove_data_model(self, client: CogniteClient) -> tuple[list, list]:
|
|
801
|
-
logs, errors = [], []
|
|
802
|
-
|
|
803
|
-
if client.data_modeling.data_models.retrieve((self.space, self.external_id, self.version)):
|
|
804
|
-
try:
|
|
805
|
-
_ = client.data_modeling.data_models.delete((self.space, self.external_id, self.version))
|
|
806
|
-
logs.append(f"Removed data model {{{self.space}:{self.external_id}/{self.version}}}")
|
|
807
|
-
except CogniteAPIError as e:
|
|
808
|
-
errors.append(
|
|
809
|
-
f"Failed to remove data model "
|
|
810
|
-
f"{{{self.space}:{self.external_id}/{self.version}}}! Reason: {e.message}"
|
|
811
|
-
)
|
|
812
|
-
else:
|
|
813
|
-
logs.append("No Data Model to remove")
|
|
814
|
-
|
|
815
|
-
return logs, errors
|
|
816
|
-
|
|
817
|
-
def remove_views(self, client: CogniteClient) -> tuple[list, list]:
|
|
818
|
-
logs, errors = [], []
|
|
819
|
-
|
|
820
|
-
if existing_views := self.find_existing_views(client):
|
|
821
|
-
try:
|
|
822
|
-
_ = client.data_modeling.views.delete([self.views[id_].as_id() for id_ in existing_views])
|
|
823
|
-
logs.append(f"Removed views {existing_views}!")
|
|
824
|
-
except CogniteAPIError as e:
|
|
825
|
-
errors.append(f"Failed to remove views {existing_views}! Reason: {e.message}")
|
|
826
|
-
|
|
827
|
-
else:
|
|
828
|
-
logs.append("No Views to remove!")
|
|
829
|
-
|
|
830
|
-
return logs, errors
|
|
831
|
-
|
|
832
|
-
def remove_containers(
|
|
833
|
-
self, client: CogniteClient, multi_space_components_removal: bool = False
|
|
834
|
-
) -> tuple[list, list]:
|
|
835
|
-
logs, errors = [], []
|
|
836
|
-
|
|
837
|
-
existing_containers = self.find_existing_containers(client)
|
|
838
|
-
existing_container_in_rules_space = {k for k in existing_containers if k.split(":")[0] == self.space}
|
|
839
|
-
|
|
840
|
-
if existing_containers and multi_space_components_removal:
|
|
841
|
-
try:
|
|
842
|
-
_ = client.data_modeling.containers.delete(
|
|
843
|
-
[self.containers[id_].as_id() for id_ in existing_containers]
|
|
844
|
-
)
|
|
845
|
-
logs.append(f"Removed containers {existing_containers}!")
|
|
846
|
-
except CogniteAPIError as e:
|
|
847
|
-
errors.append(f"Failed to remove containers {existing_containers}! Reason: {e.message}")
|
|
848
|
-
|
|
849
|
-
elif existing_container_in_rules_space and not multi_space_components_removal:
|
|
850
|
-
try:
|
|
851
|
-
_ = client.data_modeling.containers.delete(
|
|
852
|
-
[self.containers[id_].as_id() for id_ in existing_container_in_rules_space]
|
|
853
|
-
)
|
|
854
|
-
logs.append(f"Removed containers {existing_container_in_rules_space}!")
|
|
855
|
-
except CogniteAPIError as e:
|
|
856
|
-
errors.append(f"Failed to remove containers {existing_container_in_rules_space}! Reason: {e.message}")
|
|
857
|
-
|
|
858
|
-
else:
|
|
859
|
-
logs.append("No Containers to remove")
|
|
860
|
-
|
|
861
|
-
return logs, errors
|
|
862
|
-
|
|
863
|
-
def remove_spaces(self, client: CogniteClient, multi_space_components_removal: bool = False) -> tuple[list, list]:
|
|
864
|
-
logs, errors = [], []
|
|
865
|
-
existing_spaces = self.find_existing_spaces(client)
|
|
866
|
-
|
|
867
|
-
if existing_spaces and multi_space_components_removal:
|
|
868
|
-
for space in existing_spaces:
|
|
869
|
-
try:
|
|
870
|
-
_ = client.data_modeling.spaces.delete(space)
|
|
871
|
-
logs.append(f"Removed spaces {space}!")
|
|
872
|
-
except CogniteAPIError as e:
|
|
873
|
-
errors.append(f"Failed to remove {space}! Reason: {e.message}")
|
|
874
|
-
|
|
875
|
-
elif self.space in existing_spaces and not multi_space_components_removal:
|
|
876
|
-
try:
|
|
877
|
-
_ = client.data_modeling.spaces.delete(self.space)
|
|
878
|
-
logs.append(f"Removed space {self.space}!")
|
|
879
|
-
except CogniteAPIError as e:
|
|
880
|
-
errors.append(f"Failed to remove space {self.space}! Reason: {e.message}")
|
|
881
|
-
|
|
882
|
-
else:
|
|
883
|
-
logs.append("No Spaces to remove")
|
|
884
|
-
|
|
885
|
-
return logs, errors
|