cognite-neat 0.87.6__py3-none-any.whl → 0.88.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (125) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/data_classes/rest.py +0 -19
  3. cognite/neat/app/api/explorer.py +6 -4
  4. cognite/neat/app/api/routers/crud.py +11 -21
  5. cognite/neat/app/api/routers/workflows.py +24 -94
  6. cognite/neat/graph/stores/_base.py +5 -0
  7. cognite/neat/rules/importers/_inference2rules.py +31 -35
  8. cognite/neat/workflows/steps/data_contracts.py +17 -43
  9. cognite/neat/workflows/steps/lib/current/graph_extractor.py +28 -24
  10. cognite/neat/workflows/steps/lib/current/graph_loader.py +4 -21
  11. cognite/neat/workflows/steps/lib/current/graph_store.py +18 -134
  12. cognite/neat/workflows/steps_registry.py +5 -7
  13. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/METADATA +1 -1
  14. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/RECORD +17 -125
  15. cognite/neat/app/api/routers/core.py +0 -91
  16. cognite/neat/app/api/routers/data_exploration.py +0 -336
  17. cognite/neat/app/api/routers/rules.py +0 -203
  18. cognite/neat/legacy/__init__.py +0 -0
  19. cognite/neat/legacy/graph/__init__.py +0 -3
  20. cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -20182
  21. cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44.xml +0 -20163
  22. cognite/neat/legacy/graph/examples/__init__.py +0 -10
  23. cognite/neat/legacy/graph/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
  24. cognite/neat/legacy/graph/exceptions.py +0 -90
  25. cognite/neat/legacy/graph/extractors/__init__.py +0 -6
  26. cognite/neat/legacy/graph/extractors/_base.py +0 -14
  27. cognite/neat/legacy/graph/extractors/_dexpi.py +0 -44
  28. cognite/neat/legacy/graph/extractors/_graph_capturing_sheet.py +0 -403
  29. cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +0 -361
  30. cognite/neat/legacy/graph/loaders/__init__.py +0 -23
  31. cognite/neat/legacy/graph/loaders/_asset_loader.py +0 -511
  32. cognite/neat/legacy/graph/loaders/_base.py +0 -67
  33. cognite/neat/legacy/graph/loaders/_exceptions.py +0 -85
  34. cognite/neat/legacy/graph/loaders/core/__init__.py +0 -0
  35. cognite/neat/legacy/graph/loaders/core/labels.py +0 -58
  36. cognite/neat/legacy/graph/loaders/core/models.py +0 -136
  37. cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +0 -1046
  38. cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +0 -559
  39. cognite/neat/legacy/graph/loaders/rdf_to_dms.py +0 -309
  40. cognite/neat/legacy/graph/loaders/validator.py +0 -87
  41. cognite/neat/legacy/graph/models.py +0 -6
  42. cognite/neat/legacy/graph/stores/__init__.py +0 -13
  43. cognite/neat/legacy/graph/stores/_base.py +0 -400
  44. cognite/neat/legacy/graph/stores/_graphdb_store.py +0 -52
  45. cognite/neat/legacy/graph/stores/_memory_store.py +0 -43
  46. cognite/neat/legacy/graph/stores/_oxigraph_store.py +0 -151
  47. cognite/neat/legacy/graph/stores/_oxrdflib.py +0 -247
  48. cognite/neat/legacy/graph/stores/_rdf_to_graph.py +0 -42
  49. cognite/neat/legacy/graph/transformations/__init__.py +0 -0
  50. cognite/neat/legacy/graph/transformations/entity_matcher.py +0 -101
  51. cognite/neat/legacy/graph/transformations/query_generator/__init__.py +0 -3
  52. cognite/neat/legacy/graph/transformations/query_generator/sparql.py +0 -575
  53. cognite/neat/legacy/graph/transformations/transformer.py +0 -322
  54. cognite/neat/legacy/rules/__init__.py +0 -0
  55. cognite/neat/legacy/rules/analysis.py +0 -231
  56. cognite/neat/legacy/rules/examples/Rules-Nordic44-to-graphql.xlsx +0 -0
  57. cognite/neat/legacy/rules/examples/Rules-Nordic44.xlsx +0 -0
  58. cognite/neat/legacy/rules/examples/__init__.py +0 -18
  59. cognite/neat/legacy/rules/examples/power-grid-containers.yaml +0 -124
  60. cognite/neat/legacy/rules/examples/power-grid-example.xlsx +0 -0
  61. cognite/neat/legacy/rules/examples/power-grid-model.yaml +0 -224
  62. cognite/neat/legacy/rules/examples/rules-template.xlsx +0 -0
  63. cognite/neat/legacy/rules/examples/sheet2cdf-transformation-rules.xlsx +0 -0
  64. cognite/neat/legacy/rules/examples/skos-rules.xlsx +0 -0
  65. cognite/neat/legacy/rules/examples/source-to-solution-mapping-rules.xlsx +0 -0
  66. cognite/neat/legacy/rules/examples/wind-energy.owl +0 -1511
  67. cognite/neat/legacy/rules/exceptions.py +0 -2972
  68. cognite/neat/legacy/rules/exporters/__init__.py +0 -20
  69. cognite/neat/legacy/rules/exporters/_base.py +0 -45
  70. cognite/neat/legacy/rules/exporters/_core/__init__.py +0 -5
  71. cognite/neat/legacy/rules/exporters/_core/rules2labels.py +0 -24
  72. cognite/neat/legacy/rules/exporters/_rules2dms.py +0 -885
  73. cognite/neat/legacy/rules/exporters/_rules2excel.py +0 -213
  74. cognite/neat/legacy/rules/exporters/_rules2graphql.py +0 -183
  75. cognite/neat/legacy/rules/exporters/_rules2ontology.py +0 -524
  76. cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +0 -748
  77. cognite/neat/legacy/rules/exporters/_rules2rules.py +0 -105
  78. cognite/neat/legacy/rules/exporters/_rules2triples.py +0 -38
  79. cognite/neat/legacy/rules/exporters/_validation.py +0 -146
  80. cognite/neat/legacy/rules/importers/__init__.py +0 -22
  81. cognite/neat/legacy/rules/importers/_base.py +0 -66
  82. cognite/neat/legacy/rules/importers/_dict2rules.py +0 -158
  83. cognite/neat/legacy/rules/importers/_dms2rules.py +0 -194
  84. cognite/neat/legacy/rules/importers/_graph2rules.py +0 -308
  85. cognite/neat/legacy/rules/importers/_json2rules.py +0 -39
  86. cognite/neat/legacy/rules/importers/_owl2rules/__init__.py +0 -3
  87. cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +0 -239
  88. cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +0 -260
  89. cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +0 -217
  90. cognite/neat/legacy/rules/importers/_owl2rules/_owl2rules.py +0 -290
  91. cognite/neat/legacy/rules/importers/_spreadsheet2rules.py +0 -45
  92. cognite/neat/legacy/rules/importers/_xsd2rules.py +0 -20
  93. cognite/neat/legacy/rules/importers/_yaml2rules.py +0 -39
  94. cognite/neat/legacy/rules/models/__init__.py +0 -5
  95. cognite/neat/legacy/rules/models/_base.py +0 -151
  96. cognite/neat/legacy/rules/models/raw_rules.py +0 -316
  97. cognite/neat/legacy/rules/models/rdfpath.py +0 -237
  98. cognite/neat/legacy/rules/models/rules.py +0 -1289
  99. cognite/neat/legacy/rules/models/tables.py +0 -9
  100. cognite/neat/legacy/rules/models/value_types.py +0 -118
  101. cognite/neat/legacy/workflows/examples/Export_DMS/workflow.yaml +0 -89
  102. cognite/neat/legacy/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
  103. cognite/neat/legacy/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
  104. cognite/neat/legacy/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
  105. cognite/neat/legacy/workflows/examples/Import_DMS/workflow.yaml +0 -65
  106. cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
  107. cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml +0 -67
  108. cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
  109. cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml +0 -95
  110. cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml +0 -111
  111. cognite/neat/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
  112. cognite/neat/workflows/migration/__init__.py +0 -0
  113. cognite/neat/workflows/migration/steps.py +0 -91
  114. cognite/neat/workflows/migration/wf_manifests.py +0 -33
  115. cognite/neat/workflows/steps/lib/legacy/__init__.py +0 -7
  116. cognite/neat/workflows/steps/lib/legacy/graph_contextualization.py +0 -82
  117. cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +0 -746
  118. cognite/neat/workflows/steps/lib/legacy/graph_loader.py +0 -606
  119. cognite/neat/workflows/steps/lib/legacy/graph_store.py +0 -307
  120. cognite/neat/workflows/steps/lib/legacy/graph_transformer.py +0 -58
  121. cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +0 -511
  122. cognite/neat/workflows/steps/lib/legacy/rules_importer.py +0 -612
  123. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/LICENSE +0 -0
  124. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/WHEEL +0 -0
  125. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/entry_points.txt +0 -0
@@ -1,885 +0,0 @@
1
- """Exports rules to CDF Data Model Storage (DMS) through cognite-sdk."""
2
-
3
- import dataclasses
4
- import sys
5
- import warnings
6
- from dataclasses import dataclass
7
- from pathlib import Path
8
- from typing import ClassVar, Literal, cast, no_type_check
9
-
10
- import yaml
11
- from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
12
-
13
- from cognite.neat.legacy.rules.models.value_types import ValueTypeMapping
14
-
15
- if sys.version_info >= (3, 11):
16
- from typing import Self
17
- else:
18
- from typing_extensions import Self
19
-
20
- from cognite.client import CogniteClient
21
- from cognite.client import data_modeling as dm
22
- from cognite.client.data_classes.data_modeling import (
23
- ContainerApply,
24
- ContainerApplyList,
25
- ContainerId,
26
- ContainerProperty,
27
- DataModelApply,
28
- DataModelId,
29
- DirectRelation,
30
- DirectRelationReference,
31
- MappedPropertyApply,
32
- PropertyType,
33
- SpaceApply,
34
- ViewApply,
35
- ViewId,
36
- )
37
- from cognite.client.data_classes.data_modeling.views import (
38
- ConnectionDefinitionApply,
39
- SingleHopConnectionDefinitionApply,
40
- )
41
- from cognite.client.exceptions import CogniteAPIError
42
- from pydantic import BaseModel, ConfigDict, field_validator
43
-
44
- from cognite.neat.legacy.rules import exceptions
45
- from cognite.neat.legacy.rules.exporters._base import BaseExporter
46
- from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant
47
- from cognite.neat.legacy.rules.models._base import ContainerEntity, EntityTypes, ParentClass
48
- from cognite.neat.legacy.rules.models.rules import Class, Property, Rules
49
- from cognite.neat.utils.auxiliary import generate_exception_report
50
-
51
- if sys.version_info < (3, 11):
52
- from exceptiongroup import ExceptionGroup
53
-
54
-
55
- @dataclass
56
- class EmptyPropertyType(PropertyType):
57
- _type = "No property"
58
-
59
-
60
- @dataclass
61
- class DMSSchema:
62
- data_model: DataModelApply
63
- containers: ContainerApplyList
64
-
65
-
66
- class DMSExporter(BaseExporter[DMSSchema]):
67
- """Class for exporting transformation rules object to CDF Data Model Storage (DMS).
68
-
69
- Args:
70
- rules: Transformation rules object.
71
- data_model_id: The id of the data model to be created.
72
- container_policy: How to create/reuse existing containers.
73
- existing_model: In the case of updating an existing model, this is the existing model.
74
- report: Report. This is used when the exporter object is created from RawRules
75
-
76
- !!! note "Container policy"
77
- Here is more information about the different container policies:
78
- - `create`: assumes no containers exist in CDF, will attempt to create them
79
- - `reuse`: assumes containers exists in CDF, will attempt to only re-use them
80
- - `extend`: will re-use existing, extend and/or create only missing containers
81
- - `optimize`: create containers of size's prescribed by NEAT's optimization algorithm
82
- """
83
-
84
- def __init__(
85
- self,
86
- rules: Rules,
87
- data_model_id: dm.DataModelId | None = None,
88
- container_policy: Literal["create", "reuse", "extend", "optimize"] = "create",
89
- existing_model: dm.DataModel[dm.View] | None = None,
90
- report: str | None = None,
91
- ):
92
- super().__init__(rules, report)
93
- self.data_model_id = data_model_id
94
- self.container_policy = container_policy
95
- if container_policy == "extend" and existing_model is None:
96
- raise ValueError("Container policy is extend-existing, but no existing model is provided")
97
- if container_policy != "create":
98
- raise NotImplementedError("Only one-to-one-view container policy is currently supported")
99
- self.existing_model = existing_model
100
-
101
- def _export_to_file(self, filepath: Path) -> None:
102
- if filepath.suffix not in {".yaml", ".yml"}:
103
- warnings.warn("File extension is not .yaml, adding it to the file name", stacklevel=2)
104
- filepath = filepath.with_suffix(".yaml")
105
- schema = self.export()
106
- filepath.write_text(
107
- yaml.safe_dump(
108
- {
109
- "data_models": [schema.data_model.dump(camel_case=True)],
110
- "containers": schema.containers.dump(camel_case=True),
111
- }
112
- )
113
- )
114
-
115
- def export(self) -> DMSSchema:
116
- model = DMSSchemaComponents.from_rules(self.rules, self.data_model_id)
117
- return DMSSchema(
118
- data_model=DataModelApply(
119
- space=model.space,
120
- external_id=model.external_id,
121
- version=model.version,
122
- description=model.description,
123
- name=model.name,
124
- views=list(model.views.values()),
125
- ),
126
- containers=ContainerApplyList(model.containers.values()),
127
- )
128
-
129
-
130
- class DMSSchemaComponents(BaseModel):
131
- """
132
- DMS Schema Components pydantic class used to create space(s), containers, views and data model in CDF.
133
-
134
- This can be used to create a data model in CDF from rules.
135
-
136
- Args:
137
- space: Name of the space to place the resulting data model.
138
- external_id: External id of the data model.
139
- version: Version of the data model.
140
- description: Description of the data model.
141
- name: Name of the data model.
142
- containers: Containers connected to the data model.
143
- views: Views connected to the data model.
144
-
145
- """
146
-
147
- space: str
148
- external_id: str
149
- version: str
150
- description: str | None = None
151
- name: str | None = None
152
- containers: dict[str, ContainerApply]
153
- views: dict[str, ViewApply]
154
-
155
- model_config: ClassVar[ConfigDict] = ConfigDict(
156
- populate_by_name=True, str_strip_whitespace=True, arbitrary_types_allowed=True, strict=False, extra="allow"
157
- )
158
-
159
- @field_validator("views", mode="after")
160
- def remove_empty_views(cls, value):
161
- """This validator removes views that do not have any properties or implements."""
162
- for view_id, view in list(value.items()):
163
- if not view.properties and not view.implements:
164
- del value[view_id]
165
-
166
- return value
167
-
168
- @property
169
- def spaces(self):
170
- return set(
171
- [container.space for container in self.containers.values()]
172
- + [self.space]
173
- + [view.space for view in self.views.values()]
174
- )
175
-
176
- @classmethod
177
- def from_rules(
178
- cls, rules: Rules, data_model_id: dm.DataModelId | None = None, set_expected_source: bool = True
179
- ) -> Self:
180
- """Generates a DataModel class instance from a Rules instance.
181
-
182
- Args:
183
- rules: instance of Rules.
184
- data_model_id: The id of the data model to be created.
185
- set_expected_source: Whether to set the expected source on views with direct relation properties.
186
-
187
- Returns:
188
- Instance of DataModel.
189
- """
190
- if data_model_id and data_model_id.space:
191
- # update space in rules to match space
192
- rules.update_space(data_model_id.space)
193
-
194
- if data_model_id and data_model_id.version:
195
- # update version in rules to match version
196
- rules.update_version(data_model_id.version)
197
-
198
- if data_model_id and data_model_id.external_id:
199
- # update data model name to match external_id
200
- rules.metadata.suffix = data_model_id.external_id
201
-
202
- names_compliant, name_warnings = are_entity_names_dms_compliant(rules, return_report=True)
203
- if not names_compliant:
204
- raise exceptions.EntitiesContainNonDMSCompliantCharacters(report=generate_exception_report(name_warnings))
205
-
206
- if rules.metadata.external_id is None:
207
- raise exceptions.DataModelIdMissing(prefix=rules.metadata.space)
208
-
209
- return cls(
210
- space=rules.metadata.space,
211
- external_id=rules.metadata.external_id,
212
- version=rules.metadata.version,
213
- description=rules.metadata.description,
214
- name=rules.metadata.name,
215
- containers=cls.containers_from_rules(rules),
216
- views=cls.views_from_rules(rules),
217
- )
218
-
219
- @classmethod
220
- def containers_from_rules(cls, rules: Rules) -> dict[str, ContainerApply]:
221
- """Create a dictionary of ContainerApply instances from a Rules instance.
222
-
223
- Args:
224
- rules: instance of Rules.`
225
-
226
- Returns:
227
- Dictionary of ContainerApply instances.
228
- """
229
-
230
- containers: dict[str, ContainerApply] = {}
231
- errors: list = []
232
-
233
- for row, property_ in rules.properties.items():
234
- if property_.container:
235
- container_property_id: str = cast(str, property_.container_property)
236
- container_id = property_.container.id
237
-
238
- api_container = cls.as_api_container(property_.container)
239
- api_container_property = cls.as_api_container_property(property_)
240
-
241
- # scenario: adding new container to the data model for the first time
242
- if not isinstance(api_container_property.type, EmptyPropertyType) and container_id not in containers:
243
- containers[container_id] = api_container
244
- containers[container_id].properties[container_property_id] = api_container_property
245
-
246
- # scenario: adding new property to an existing container
247
- elif (
248
- not isinstance(api_container_property.type, EmptyPropertyType)
249
- and container_property_id not in containers[container_id].properties
250
- ):
251
- containers[container_id].properties[container_property_id] = api_container_property
252
-
253
- # scenario: property is redefined checking for potential sub-scenarios
254
- elif (
255
- not isinstance(api_container_property.type, EmptyPropertyType)
256
- and container_property_id in containers[container_id].properties
257
- ):
258
- existing_property = containers[container_id].properties[container_property_id]
259
-
260
- # scenario: property is redefined with a different value type -> raise error
261
- if not isinstance(existing_property.type, type(api_container_property.type)):
262
- errors.append(
263
- exceptions.ContainerPropertyValueTypeRedefinition(
264
- container_id=container_id,
265
- property_id=container_property_id,
266
- current_value_type=str(existing_property.type),
267
- redefined_value_type=str(api_container_property.type),
268
- loc=f"[Properties/Type/{row}]",
269
- )
270
- )
271
-
272
- # scenario: default value is redefined -> set default value to None
273
- if (
274
- not isinstance(existing_property.type, DirectRelation)
275
- and not isinstance(api_container_property.type, DirectRelation)
276
- and existing_property.default_value != api_container_property.default_value
277
- ):
278
- containers[container_id].properties[container_property_id] = dataclasses.replace(
279
- existing_property, default_value=None
280
- )
281
-
282
- # scenario: property holds multiple values -> set is_list to True
283
- if (
284
- not isinstance(existing_property.type, DirectRelation)
285
- and not isinstance(api_container_property.type, DirectRelation)
286
- and isinstance(existing_property.type, ListablePropertyType)
287
- and isinstance(api_container_property.type, ListablePropertyType)
288
- and existing_property.type.is_list != api_container_property.type.is_list
289
- ):
290
- type_ = containers[container_id].properties[container_property_id].type
291
- if isinstance(type_, ListablePropertyType):
292
- type_.is_list = True
293
-
294
- if errors:
295
- raise ExceptionGroup("Properties value types have been redefined! This is prohibited! Aborting!", errors)
296
-
297
- return containers
298
-
299
- @classmethod
300
- def as_api_container_property(cls, property_: Property) -> ContainerProperty:
301
- is_one_to_many = property_.max_count != 1
302
-
303
- # Literal, i.e. Node attribute
304
- if property_.property_type is EntityTypes.data_property:
305
- property_type = cast(ValueTypeMapping, property_.expected_value_type.mapping).dms
306
- return ContainerProperty(
307
- type=property_type(is_list=is_one_to_many),
308
- nullable=property_.min_count == 0,
309
- default_value=property_.default,
310
- name=property_.property_name,
311
- description=property_.description,
312
- )
313
-
314
- # URIRef, i.e. edge 1-1 , aka direct relation between Nodes
315
- elif property_.property_type is EntityTypes.object_property and not is_one_to_many:
316
- return ContainerProperty(
317
- type=DirectRelation(),
318
- nullable=True,
319
- name=property_.property_name,
320
- description=property_.description,
321
- )
322
-
323
- # return type=None if property cannot be created
324
- else:
325
- return ContainerProperty(type=EmptyPropertyType())
326
-
327
- @classmethod
328
- def as_api_container(cls, container: ContainerEntity) -> ContainerApply:
329
- return ContainerApply(
330
- space=container.space,
331
- external_id=container.external_id,
332
- description=container.description,
333
- name=container.name,
334
- # properties are added later
335
- properties={},
336
- )
337
-
338
- @classmethod
339
- def views_from_rules(cls, rules: Rules) -> dict[str, ViewApply]:
340
- """Generates a dictionary of ViewApply instances from a Rules instance.
341
-
342
- Args:
343
- rule: Instance of Rules.
344
- space: Name of the space to place the views.
345
-
346
- Returns:
347
- Dictionary of ViewApply instances.
348
- """
349
- views: dict[str, ViewApply] = {
350
- f"{rules.metadata.space}:{class_.class_id}": cls.as_api_view(
351
- class_, rules.metadata.space, rules.metadata.version
352
- )
353
- for class_ in rules.classes.values()
354
- }
355
- errors: list = []
356
-
357
- # Create views from property-class definitions
358
- for row, property_ in rules.properties.items():
359
- view_property = cls.as_api_view_property(property_, rules.metadata.space)
360
- id_ = f"{rules.metadata.space}:{property_.class_id}"
361
-
362
- # scenario: view exist but property does not so it is added
363
- if view_property and (property_.property_id not in cast(dict, views[id_].properties)):
364
- cast(dict, views[id_].properties)[property_.property_id] = view_property
365
-
366
- # scenario: view exist, property exists but it is differently defined -> raise error
367
- # type: ignore
368
- elif (
369
- view_property
370
- and property_.property_id in cast(dict, views[id_].properties)
371
- and view_property is not cast(dict, views[id_].properties)[property_.property_id]
372
- ):
373
- errors.append(
374
- exceptions.ViewPropertyRedefinition(
375
- view_id=id_,
376
- property_id=cast(str, property_.property_id),
377
- loc=f"[Properties/Property/{row}]",
378
- )
379
- )
380
-
381
- if errors:
382
- raise ExceptionGroup("View properties have been redefined! This is prohibited! Aborting!", errors)
383
-
384
- return views
385
-
386
- @classmethod
387
- def as_api_view(cls, class_: Class, space: str, version: str) -> ViewApply:
388
- return ViewApply(
389
- space=space,
390
- external_id=class_.class_id,
391
- version=version,
392
- name=class_.class_name,
393
- description=class_.description,
394
- properties={},
395
- implements=(
396
- [parent_class.view_id for parent_class in cast(list[ParentClass], class_.parent_class)]
397
- if class_.parent_class
398
- else None
399
- ),
400
- )
401
-
402
- @classmethod
403
- def as_api_view_property(
404
- cls, property_: Property, space: str
405
- ) -> MappedPropertyApply | ConnectionDefinitionApply | None:
406
- property_.container = cast(ContainerEntity, property_.container)
407
- property_.container_property = cast(str, property_.container_property)
408
- if property_.property_type is EntityTypes.data_property:
409
- return MappedPropertyApply(
410
- container=ContainerId(space=property_.container.space, external_id=property_.container.external_id),
411
- container_property_identifier=property_.container_property,
412
- name=property_.property_name,
413
- description=property_.description,
414
- )
415
-
416
- # edge 1-1 == directRelation
417
- elif property_.property_type is EntityTypes.object_property and property_.max_count == 1:
418
- return MappedPropertyApply(
419
- container=ContainerId(space=property_.container.space, external_id=property_.container.external_id),
420
- container_property_identifier=property_.container_property,
421
- name=property_.property_name,
422
- description=property_.description,
423
- source=ViewId(
424
- space=property_.expected_value_type.space,
425
- external_id=property_.expected_value_type.external_id,
426
- version=property_.expected_value_type.version,
427
- ),
428
- )
429
-
430
- # edge 1-many == EDGE, this does not have container! type is here source ViewId ?!
431
- elif property_.property_type is EntityTypes.object_property and property_.max_count != 1:
432
- if property_.container and property_.expected_value_type.space != property_.container.space:
433
- type_ = DirectRelationReference(
434
- space=property_.container.space,
435
- external_id=f"{property_.container.suffix}.{property_.container_property}",
436
- )
437
- else:
438
- type_ = DirectRelationReference(
439
- space=space, external_id=f"{property_.class_id}.{property_.property_id}"
440
- )
441
-
442
- return SingleHopConnectionDefinitionApply(
443
- type=type_,
444
- # Here we create ViewID to the container that the edge is pointing to.
445
- source=ViewId(
446
- space=property_.expected_value_type.space,
447
- external_id=property_.expected_value_type.external_id,
448
- version=property_.expected_value_type.version,
449
- ),
450
- direction="outwards",
451
- name=property_.property_name,
452
- description=property_.description,
453
- )
454
- else:
455
- return None
456
-
457
- def find_existing_spaces(self, client: CogniteClient) -> set[str]:
458
- """Checks if the spaces exist in CDF.
459
-
460
- Args:
461
- client: Cognite client.
462
-
463
- Returns:
464
- External ids of spaces which are part of DMS Schema components that already exist in CDF.
465
- """
466
-
467
- return set(client.data_modeling.spaces.retrieve(list(self.spaces)).as_ids())
468
-
469
- def find_existing_containers(self, client: CogniteClient) -> set[str]:
470
- """Checks if the containers exist in CDF.
471
-
472
- Args:
473
- client: Cognite client.
474
-
475
- Returns:
476
- External ids of containers which are part of DMS Schema components that already exist in CDF.
477
- """
478
-
479
- return {
480
- f"{id_.space}:{id_.external_id}"
481
- for id_ in client.data_modeling.containers.retrieve(
482
- [container.as_id() for container in self.containers.values()]
483
- ).as_ids()
484
- }
485
-
486
- def find_existing_views(self, client: CogniteClient) -> set[str]:
487
- """Checks if the views exist in CDF.
488
-
489
- Args:
490
- client: Cognite client.
491
-
492
- Returns:
493
- External ids of views which are part of DMS Schema components that already exist in CDF.
494
- """
495
-
496
- return {
497
- f"{id_.space}:{id_.external_id}"
498
- for id_ in client.data_modeling.views.retrieve([view.as_id() for view in self.views.values()]).as_ids()
499
- }
500
-
501
- def find_existing_data_model(self, client: CogniteClient) -> DataModelId | None:
502
- """Checks if the data model exists in CDF.
503
-
504
- Args:
505
- client: Cognite client.
506
-
507
- Returns:
508
- True if the data model exists, False otherwise.
509
- """
510
- if model := client.data_modeling.data_models.retrieve((self.space, self.external_id, self.version)):
511
- cdf_data_model = model.latest_version()
512
- return cdf_data_model.as_id()
513
- return None
514
-
515
- # mypy unsatisfied with overload , tried all combination and gave up
516
- @no_type_check
517
- def to_cdf(
518
- self,
519
- client: CogniteClient,
520
- components_to_create: set | None = None,
521
- existing_component_handling: Literal["fail", "skip", "update"] = "fail",
522
- multi_space_components_create: bool = False,
523
- return_report: bool = False,
524
- ) -> None | tuple[dict[str, list], dict[str, list]]:
525
- """Write the the data model to CDF.
526
-
527
- Args:
528
- client: Connected Cognite client.
529
- components_to_create: Which components to create. Takes set
530
- existing_component_handling: How to handle existing components. Takes Literal["fail", "skip", "update"]
531
- multi_space_comp_create: Whether to create only components belonging to the data model space,
532
- or also additionally components outside of the data model space. Default is False.
533
-
534
- !!! note "Multi Space DMS Schema Components"
535
- If multi_space_components_create is set to True, the components will be created
536
- in all spaces used or defined in `Rules`. If set to False, the
537
- components will only be created in the space defined in `Rules` metadata.
538
-
539
- !!! note "Component Creation Policy"
540
- Here is more information about the different component creation policies
541
- configured through components_to_create argument:
542
-
543
- - `all`: all components of the data model will be created, meaning space, containers, views and data model
544
- - `data model`: only the data model will be created
545
- - `view`: only views will be created
546
- - `container`: only the containers will be created
547
-
548
- for creation of containers beyond the data model space, set `multi_space_components_create` argument
549
- to True.
550
-
551
-
552
- !!! note "Existing Component Handling Policy"
553
- Here is more information about the different existing component handling policies:
554
-
555
- - `fail`: if any component of the data model (DMS schema) already exists
556
- - `skip`: skip DMS components that exist
557
- - `update`: create DMS components that do not exist and update those that do exist
558
-
559
- `update` policy is currently not implemented !
560
-
561
- """
562
-
563
- logs, errors = {}, {}
564
-
565
- components_to_create = components_to_create or {"all"}
566
-
567
- existing_spaces = self.find_existing_spaces(client)
568
- existing_containers = self.find_existing_containers(client)
569
- existing_views = self.find_existing_views(client)
570
- existing_data_model = self.find_existing_data_model(client)
571
-
572
- if (
573
- existing_spaces or existing_containers or existing_data_model or existing_views
574
- ) and existing_component_handling == "fail":
575
- raise exceptions.DataModelOrItsComponentsAlreadyExist(
576
- existing_spaces,
577
- existing_data_model,
578
- existing_containers,
579
- existing_views,
580
- )
581
-
582
- if "space" in components_to_create or "all" in components_to_create:
583
- logs["space"], errors["space"] = self.create_space(
584
- client,
585
- existing_spaces,
586
- existing_component_handling == "update",
587
- multi_space_components_create,
588
- )
589
- if "container" in components_to_create or "all" in components_to_create:
590
- logs["container"], errors["container"] = self.create_containers(
591
- client,
592
- existing_containers,
593
- existing_component_handling == "update",
594
- multi_space_components_create,
595
- )
596
-
597
- if "view" in components_to_create or "all" in components_to_create:
598
- logs["view"], errors["view"] = self.create_views(
599
- client, existing_views, existing_component_handling == "update"
600
- )
601
- if "data model" in components_to_create or "all" in components_to_create:
602
- logs["data model"], errors["data model"] = self.create_data_model(
603
- client, existing_data_model, existing_component_handling == "update"
604
- )
605
-
606
- if return_report:
607
- return logs, errors
608
-
609
- def create_space(
610
- self,
611
- client: CogniteClient,
612
- existing_spaces: set,
613
- update: bool = False,
614
- multi_space_components_create: bool = False,
615
- ) -> tuple[list, list]:
616
- logs, errors = [], []
617
-
618
- spaces_to_create = (
619
- self.spaces - existing_spaces
620
- if multi_space_components_create
621
- else ((self.spaces - existing_spaces) & {self.space})
622
- )
623
-
624
- spaces_to_update = existing_spaces if multi_space_components_create else (existing_spaces & {self.space})
625
-
626
- if spaces_to_create:
627
- try:
628
- _ = client.data_modeling.spaces.apply([SpaceApply(space=space) for space in spaces_to_create])
629
- logs.append(f"Created space {spaces_to_create}")
630
- except CogniteAPIError as e:
631
- errors.append(f"Failed to create space {spaces_to_create}! Reason: {e.message}")
632
-
633
- if update and spaces_to_update:
634
- try:
635
- _ = client.data_modeling.spaces.apply([SpaceApply(space=space) for space in spaces_to_update])
636
- logs.append(f"Updated space {spaces_to_update}")
637
- except CogniteAPIError as e:
638
- errors.append(f"Failed to update spaces {spaces_to_update}! Reason: {e.message}")
639
-
640
- return logs, errors
641
-
642
- def create_containers(
643
- self,
644
- client: CogniteClient,
645
- existing_containers: set,
646
- update: bool = False,
647
- multi_space_components_create: bool = False,
648
- ) -> tuple[list, list]:
649
- logs, errors = [], []
650
-
651
- containers_to_create = (
652
- set(self.containers.keys()) - existing_containers
653
- if multi_space_components_create
654
- else {k for k in (set(self.containers.keys()) - existing_containers) if k.split(":")[0] == self.space}
655
- )
656
-
657
- containers_to_update = (
658
- existing_containers
659
- if multi_space_components_create
660
- else {k for k in existing_containers if k.split(":")[0] == self.space}
661
- )
662
-
663
- if containers_to_create:
664
- try:
665
- _ = client.data_modeling.containers.apply([self.containers[id_] for id_ in containers_to_create])
666
- logs.append(f"Created container {containers_to_create}")
667
- except CogniteAPIError as e:
668
- errors.append(f"Failed to create containers {containers_to_create}! Reason: {e.message}")
669
-
670
- if update and containers_to_update:
671
- try:
672
- _ = client.data_modeling.containers.apply([self.containers[id_] for id_ in containers_to_update])
673
- logs.append(f"Updated containers {containers_to_update}")
674
- except CogniteAPIError as e:
675
- errors.append(f"Failed to update containers {containers_to_update}! Reason: {e.message}")
676
-
677
- return logs, errors
678
-
679
- def create_views(
680
- self,
681
- client: CogniteClient,
682
- existing_views: set,
683
- update: bool = False,
684
- ) -> tuple[list, list]:
685
- logs, errors = [], []
686
-
687
- non_existing_views = set(self.views.keys()) - existing_views
688
- if non_existing_views:
689
- try:
690
- _ = client.data_modeling.views.apply([self.views[id_] for id_ in non_existing_views])
691
- logs.append(f"Created views {non_existing_views}")
692
- except CogniteAPIError as e:
693
- errors.append(f"Failed to update views {non_existing_views}! Reason: {e.message}")
694
-
695
- if update and existing_views:
696
- try:
697
- _ = client.data_modeling.views.apply([self.views[id_] for id_ in existing_views])
698
- logs.append(f"Updated views {existing_views}")
699
- except CogniteAPIError as e:
700
- errors.append(f"Failed to update views {existing_views}! Reason: {e.message}")
701
-
702
- return logs, errors
703
-
704
- def create_data_model(
705
- self,
706
- client: CogniteClient,
707
- existing_data_model: DataModelId | None = None,
708
- update: bool = False,
709
- ) -> tuple[list, list]:
710
- logs, errors = [], []
711
-
712
- if not existing_data_model:
713
- try:
714
- _ = client.data_modeling.data_models.apply(
715
- DataModelApply(
716
- name=self.name,
717
- description=self.description,
718
- space=self.space,
719
- external_id=self.external_id,
720
- version=self.version,
721
- views=[view.as_id() for view in self.views.values()],
722
- )
723
- )
724
- logs.append(f"Created data model {{{self.space}:{self.external_id}/{self.version}}}")
725
- except CogniteAPIError as e:
726
- errors.append(
727
- f"Failed to create data model {{{self.space}:{self.external_id}/{self.version}}}!"
728
- f" Reason: {e.message}"
729
- )
730
- elif update:
731
- try:
732
- _ = client.data_modeling.data_models.apply(
733
- DataModelApply(
734
- name=self.name,
735
- description=self.description,
736
- space=self.space,
737
- external_id=self.external_id,
738
- version=self.version,
739
- views=[view.as_id() for view in self.views.values()],
740
- )
741
- )
742
- logs.append("Updated data model " f"{{{self.space}:{self.external_id}/{self.version}}}")
743
- except CogniteAPIError as e:
744
- errors.append(
745
- "Failed to update data model"
746
- f" {{{self.space}:{self.external_id}/{self.version}}}! Reason: {e.message}"
747
- )
748
-
749
- else:
750
- logs.append(f"Skipped update of data model {{{self.space}:{self.external_id}/{self.version}}}!")
751
-
752
- return logs, errors
753
-
754
- @no_type_check
755
- def remove(
756
- self,
757
- client: CogniteClient,
758
- components_to_remove: set | None = None,
759
- multi_space_components_removal: bool = False,
760
- return_report: bool = False,
761
- ) -> None | tuple[dict[str, list], dict[str, list]]:
762
- """Remove DMS schema components from CDF.
763
-
764
- Args:
765
- client: Connected Cognite client.
766
- components_to_remove: Which components to remove. Takes set
767
- multi_space_components_removal: Whether to remove components in multiple spaces,
768
- or only in the space of the data model. Default is False.
769
-
770
- !!! note "Component Creation Policy"
771
- Here is more information about the different component creation policies:
772
- - `all`: all components of the data model will be created, meaning space, containers, views and data model
773
- - `data model`: only the data model will be created
774
- - `view`: only the views will be created
775
- - `container`: only the containers will be created
776
-
777
-
778
- !!! note "Multi Space DMS Schema Components"
779
- If multi_space_components_removal is set to True, the components will be removed
780
- in all spaces used or defined in `Rules`. If set to False, the
781
- components will only be removed in the space defined in `Rules` metadata.
782
- """
783
-
784
- logs, errors = {}, {}
785
-
786
- components_to_remove = components_to_remove or {"all"}
787
-
788
- if "data model" in components_to_remove or "all" in components_to_remove:
789
- logs["data model"], errors["data model"] = self.remove_data_model(client)
790
- if "view" in components_to_remove or "all" in components_to_remove:
791
- logs["view"], errors["view"] = self.remove_views(client)
792
- if "container" in components_to_remove or "all" in components_to_remove:
793
- logs["container"], errors["container"] = self.remove_containers(client, multi_space_components_removal)
794
- if "space" in components_to_remove or "all" in components_to_remove:
795
- logs["space"], errors["space"] = self.remove_spaces(client, multi_space_components_removal)
796
-
797
- if return_report:
798
- return logs, errors
799
-
800
- def remove_data_model(self, client: CogniteClient) -> tuple[list, list]:
801
- logs, errors = [], []
802
-
803
- if client.data_modeling.data_models.retrieve((self.space, self.external_id, self.version)):
804
- try:
805
- _ = client.data_modeling.data_models.delete((self.space, self.external_id, self.version))
806
- logs.append(f"Removed data model {{{self.space}:{self.external_id}/{self.version}}}")
807
- except CogniteAPIError as e:
808
- errors.append(
809
- f"Failed to remove data model "
810
- f"{{{self.space}:{self.external_id}/{self.version}}}! Reason: {e.message}"
811
- )
812
- else:
813
- logs.append("No Data Model to remove")
814
-
815
- return logs, errors
816
-
817
- def remove_views(self, client: CogniteClient) -> tuple[list, list]:
818
- logs, errors = [], []
819
-
820
- if existing_views := self.find_existing_views(client):
821
- try:
822
- _ = client.data_modeling.views.delete([self.views[id_].as_id() for id_ in existing_views])
823
- logs.append(f"Removed views {existing_views}!")
824
- except CogniteAPIError as e:
825
- errors.append(f"Failed to remove views {existing_views}! Reason: {e.message}")
826
-
827
- else:
828
- logs.append("No Views to remove!")
829
-
830
- return logs, errors
831
-
832
- def remove_containers(
833
- self, client: CogniteClient, multi_space_components_removal: bool = False
834
- ) -> tuple[list, list]:
835
- logs, errors = [], []
836
-
837
- existing_containers = self.find_existing_containers(client)
838
- existing_container_in_rules_space = {k for k in existing_containers if k.split(":")[0] == self.space}
839
-
840
- if existing_containers and multi_space_components_removal:
841
- try:
842
- _ = client.data_modeling.containers.delete(
843
- [self.containers[id_].as_id() for id_ in existing_containers]
844
- )
845
- logs.append(f"Removed containers {existing_containers}!")
846
- except CogniteAPIError as e:
847
- errors.append(f"Failed to remove containers {existing_containers}! Reason: {e.message}")
848
-
849
- elif existing_container_in_rules_space and not multi_space_components_removal:
850
- try:
851
- _ = client.data_modeling.containers.delete(
852
- [self.containers[id_].as_id() for id_ in existing_container_in_rules_space]
853
- )
854
- logs.append(f"Removed containers {existing_container_in_rules_space}!")
855
- except CogniteAPIError as e:
856
- errors.append(f"Failed to remove containers {existing_container_in_rules_space}! Reason: {e.message}")
857
-
858
- else:
859
- logs.append("No Containers to remove")
860
-
861
- return logs, errors
862
-
863
- def remove_spaces(self, client: CogniteClient, multi_space_components_removal: bool = False) -> tuple[list, list]:
864
- logs, errors = [], []
865
- existing_spaces = self.find_existing_spaces(client)
866
-
867
- if existing_spaces and multi_space_components_removal:
868
- for space in existing_spaces:
869
- try:
870
- _ = client.data_modeling.spaces.delete(space)
871
- logs.append(f"Removed spaces {space}!")
872
- except CogniteAPIError as e:
873
- errors.append(f"Failed to remove {space}! Reason: {e.message}")
874
-
875
- elif self.space in existing_spaces and not multi_space_components_removal:
876
- try:
877
- _ = client.data_modeling.spaces.delete(self.space)
878
- logs.append(f"Removed space {self.space}!")
879
- except CogniteAPIError as e:
880
- errors.append(f"Failed to remove space {self.space}! Reason: {e.message}")
881
-
882
- else:
883
- logs.append("No Spaces to remove")
884
-
885
- return logs, errors