cognite-neat 0.75.7__py3-none-any.whl → 0.75.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/configuration.py +4 -9
- cognite/neat/app/api/routers/configuration.py +2 -1
- cognite/neat/app/api/routers/crud.py +5 -5
- cognite/neat/app/api/routers/data_exploration.py +3 -1
- cognite/neat/app/api/routers/rules.py +3 -3
- cognite/neat/app/api/routers/workflows.py +3 -3
- cognite/neat/app/ui/neat-app/build/asset-manifest.json +3 -3
- cognite/neat/app/ui/neat-app/build/index.html +1 -1
- cognite/neat/app/ui/neat-app/build/static/js/{main.4345d42f.js → main.ec7f72e2.js} +3 -3
- cognite/neat/app/ui/neat-app/build/static/js/{main.4345d42f.js.map → main.ec7f72e2.js.map} +1 -1
- cognite/neat/config.py +147 -12
- cognite/neat/constants.py +1 -0
- cognite/neat/graph/exceptions.py +1 -2
- cognite/neat/graph/extractors/_mock_graph_generator.py +6 -5
- cognite/neat/legacy/graph/exceptions.py +1 -2
- cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +1 -2
- cognite/neat/legacy/graph/loaders/_asset_loader.py +8 -13
- cognite/neat/legacy/graph/loaders/_base.py +2 -4
- cognite/neat/legacy/graph/loaders/_exceptions.py +1 -3
- cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +4 -8
- cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +2 -4
- cognite/neat/legacy/graph/loaders/rdf_to_dms.py +2 -4
- cognite/neat/legacy/graph/loaders/validator.py +1 -1
- cognite/neat/legacy/graph/transformations/transformer.py +1 -2
- cognite/neat/legacy/rules/exporters/_rules2dms.py +1 -2
- cognite/neat/legacy/rules/exporters/_validation.py +4 -8
- cognite/neat/legacy/rules/importers/_base.py +0 -4
- cognite/neat/legacy/rules/importers/_dms2rules.py +0 -2
- cognite/neat/legacy/rules/models/rdfpath.py +1 -2
- cognite/neat/legacy/workflows/examples/Export_DMS/workflow.yaml +89 -0
- cognite/neat/legacy/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +152 -0
- cognite/neat/legacy/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +139 -0
- cognite/neat/legacy/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +270 -0
- cognite/neat/legacy/workflows/examples/Import_DMS/workflow.yaml +65 -0
- cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml +116 -0
- cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml +67 -0
- cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml +64 -0
- cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml +95 -0
- cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml +111 -0
- cognite/neat/rules/analysis/_base.py +1 -1
- cognite/neat/rules/analysis/_information_rules.py +4 -4
- cognite/neat/rules/exporters/_rules2excel.py +2 -2
- cognite/neat/rules/exporters/_rules2ontology.py +20 -17
- cognite/neat/rules/exporters/_validation.py +8 -10
- cognite/neat/rules/importers/_base.py +2 -4
- cognite/neat/rules/importers/_dms2rules.py +16 -19
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +21 -19
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +2 -4
- cognite/neat/rules/importers/_dtdl2rules/spec.py +3 -5
- cognite/neat/rules/importers/_owl2rules/_owl2rules.py +4 -6
- cognite/neat/rules/importers/_spreadsheet2rules.py +10 -9
- cognite/neat/rules/importers/_yaml2rules.py +10 -6
- cognite/neat/rules/issues/dms.py +3 -5
- cognite/neat/rules/issues/formatters.py +3 -1
- cognite/neat/rules/models/data_types.py +54 -31
- cognite/neat/rules/models/entities.py +184 -42
- cognite/neat/rules/models/rdfpath.py +112 -13
- cognite/neat/rules/models/rules/_base.py +2 -2
- cognite/neat/rules/models/rules/_dms_architect_rules.py +119 -189
- cognite/neat/rules/models/rules/_dms_rules_write.py +344 -0
- cognite/neat/rules/models/rules/_dms_schema.py +3 -3
- cognite/neat/rules/models/rules/_domain_rules.py +6 -3
- cognite/neat/rules/models/rules/_information_rules.py +68 -61
- cognite/neat/rules/models/rules/_types/__init__.py +0 -47
- cognite/neat/rules/models/rules/_types/_base.py +1 -309
- cognite/neat/rules/models/rules/_types/_field.py +0 -225
- cognite/neat/utils/cdf_loaders/_data_modeling.py +3 -1
- cognite/neat/utils/cdf_loaders/_ingestion.py +2 -4
- cognite/neat/utils/spreadsheet.py +2 -4
- cognite/neat/utils/utils.py +2 -4
- cognite/neat/workflows/base.py +5 -5
- cognite/neat/workflows/manager.py +32 -22
- cognite/neat/workflows/model.py +3 -3
- cognite/neat/workflows/steps/lib/__init__.py +0 -7
- cognite/neat/workflows/steps/lib/current/__init__.py +6 -0
- cognite/neat/workflows/steps/lib/{rules_exporter.py → current/rules_exporter.py} +8 -8
- cognite/neat/workflows/steps/lib/{rules_importer.py → current/rules_importer.py} +7 -7
- cognite/neat/workflows/steps/lib/io/__init__.py +1 -0
- cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_contextualization.py +2 -2
- cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_extractor.py +9 -9
- cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_loader.py +9 -9
- cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_store.py +4 -4
- cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_transformer.py +2 -2
- cognite/neat/workflows/steps/lib/{v1 → legacy}/rules_exporter.py +15 -17
- cognite/neat/workflows/steps/lib/{v1 → legacy}/rules_importer.py +7 -7
- cognite/neat/workflows/steps/step_model.py +5 -9
- cognite/neat/workflows/steps_registry.py +20 -11
- {cognite_neat-0.75.7.dist-info → cognite_neat-0.75.9.dist-info}/METADATA +1 -1
- {cognite_neat-0.75.7.dist-info → cognite_neat-0.75.9.dist-info}/RECORD +100 -90
- cognite/neat/app/api/data_classes/configuration.py +0 -121
- cognite/neat/rules/models/_entity.py +0 -142
- cognite/neat/rules/models/rules/_types/_value.py +0 -159
- /cognite/neat/app/ui/neat-app/build/static/js/{main.4345d42f.js.LICENSE.txt → main.ec7f72e2.js.LICENSE.txt} +0 -0
- /cognite/neat/workflows/steps/lib/{graph_extractor.py → current/graph_extractor.py} +0 -0
- /cognite/neat/workflows/steps/lib/{graph_loader.py → current/graph_loader.py} +0 -0
- /cognite/neat/workflows/steps/lib/{graph_store.py → current/graph_store.py} +0 -0
- /cognite/neat/workflows/steps/lib/{rules_validator.py → current/rules_validator.py} +0 -0
- /cognite/neat/workflows/steps/lib/{io_steps.py → io/io_steps.py} +0 -0
- /cognite/neat/workflows/steps/lib/{v1 → legacy}/__init__.py +0 -0
- {cognite_neat-0.75.7.dist-info → cognite_neat-0.75.9.dist-info}/LICENSE +0 -0
- {cognite_neat-0.75.7.dist-info → cognite_neat-0.75.9.dist-info}/WHEEL +0 -0
- {cognite_neat-0.75.7.dist-info → cognite_neat-0.75.9.dist-info}/entry_points.txt +0 -0
|
@@ -11,36 +11,35 @@ from cognite.client import data_modeling as dm
|
|
|
11
11
|
from cognite.client.data_classes.data_modeling import PropertyType as CognitePropertyType
|
|
12
12
|
from cognite.client.data_classes.data_modeling.containers import BTreeIndex
|
|
13
13
|
from cognite.client.data_classes.data_modeling.views import SingleReverseDirectRelationApply, ViewPropertyApply
|
|
14
|
-
from pydantic import Field, field_validator, model_serializer, model_validator
|
|
14
|
+
from pydantic import Field, field_serializer, field_validator, model_serializer, model_validator
|
|
15
15
|
from pydantic_core.core_schema import SerializationInfo, ValidationInfo
|
|
16
16
|
from rdflib import Namespace
|
|
17
17
|
|
|
18
18
|
import cognite.neat.rules.issues.spreadsheet
|
|
19
19
|
from cognite.neat.rules import issues
|
|
20
|
+
from cognite.neat.rules.models.data_types import DataType
|
|
21
|
+
from cognite.neat.rules.models.entities import (
|
|
22
|
+
ClassEntity,
|
|
23
|
+
ContainerEntity,
|
|
24
|
+
ContainerEntityList,
|
|
25
|
+
DMSUnknownEntity,
|
|
26
|
+
ParentClassEntity,
|
|
27
|
+
ReferenceEntity,
|
|
28
|
+
UnknownEntity,
|
|
29
|
+
URLEntity,
|
|
30
|
+
ViewEntity,
|
|
31
|
+
ViewEntityList,
|
|
32
|
+
ViewPropertyEntity,
|
|
33
|
+
)
|
|
20
34
|
from cognite.neat.rules.models.rules._domain_rules import DomainRules
|
|
21
35
|
|
|
22
36
|
from ._base import BaseMetadata, BaseRules, ExtensionCategory, RoleTypes, SchemaCompleteness, SheetEntity, SheetList
|
|
23
37
|
from ._dms_schema import DMSSchema, PipelineSchema
|
|
24
38
|
from ._types import (
|
|
25
|
-
CdfValueType,
|
|
26
|
-
ClassEntity,
|
|
27
|
-
ContainerEntity,
|
|
28
|
-
ContainerListType,
|
|
29
|
-
ContainerType,
|
|
30
|
-
DMSValueType,
|
|
31
39
|
ExternalIdType,
|
|
32
|
-
ParentClassEntity,
|
|
33
40
|
PropertyType,
|
|
34
|
-
ReferenceEntity,
|
|
35
|
-
ReferenceType,
|
|
36
41
|
StrListType,
|
|
37
|
-
Undefined,
|
|
38
42
|
VersionType,
|
|
39
|
-
ViewEntity,
|
|
40
|
-
ViewListType,
|
|
41
|
-
ViewPropEntity,
|
|
42
|
-
ViewType,
|
|
43
|
-
XSDValueType,
|
|
44
43
|
)
|
|
45
44
|
|
|
46
45
|
if TYPE_CHECKING:
|
|
@@ -51,6 +50,7 @@ if sys.version_info >= (3, 11):
|
|
|
51
50
|
else:
|
|
52
51
|
from typing_extensions import Self
|
|
53
52
|
|
|
53
|
+
_DEFAULT_VERSION = "1"
|
|
54
54
|
|
|
55
55
|
subclasses = list(CognitePropertyType.__subclasses__())
|
|
56
56
|
_PropertyType_by_name: dict[str, type[CognitePropertyType]] = {}
|
|
@@ -164,14 +164,14 @@ class DMSMetadata(BaseMetadata):
|
|
|
164
164
|
class DMSProperty(SheetEntity):
|
|
165
165
|
property_: PropertyType = Field(alias="Property")
|
|
166
166
|
relation: Literal["direct", "reversedirect", "multiedge"] | None = Field(None, alias="Relation")
|
|
167
|
-
value_type:
|
|
167
|
+
value_type: DataType | ViewPropertyEntity | ViewEntity | DMSUnknownEntity = Field(alias="Value Type")
|
|
168
168
|
nullable: bool | None = Field(default=None, alias="Nullable")
|
|
169
169
|
is_list: bool | None = Field(default=None, alias="IsList")
|
|
170
170
|
default: str | int | dict | None = Field(None, alias="Default")
|
|
171
|
-
reference:
|
|
172
|
-
container:
|
|
171
|
+
reference: URLEntity | ReferenceEntity | None = Field(default=None, alias="Reference", union_mode="left_to_right")
|
|
172
|
+
container: ContainerEntity | None = Field(None, alias="Container")
|
|
173
173
|
container_property: str | None = Field(None, alias="ContainerProperty")
|
|
174
|
-
view:
|
|
174
|
+
view: ViewEntity = Field(alias="View")
|
|
175
175
|
view_property: str = Field(alias="ViewProperty")
|
|
176
176
|
index: StrListType | None = Field(None, alias="Index")
|
|
177
177
|
constraint: StrListType | None = Field(None, alias="Constraint")
|
|
@@ -189,12 +189,13 @@ class DMSProperty(SheetEntity):
|
|
|
189
189
|
return value
|
|
190
190
|
|
|
191
191
|
@field_validator("value_type", mode="after")
|
|
192
|
-
def relations_value_type(cls, value:
|
|
192
|
+
def relations_value_type(cls, value: DataType | ClassEntity, info: ValidationInfo) -> DataType | ClassEntity:
|
|
193
193
|
if (relation := info.data["relation"]) is None:
|
|
194
194
|
return value
|
|
195
|
-
if not isinstance(value,
|
|
195
|
+
if not isinstance(value, ViewEntity | ViewPropertyEntity | DMSUnknownEntity):
|
|
196
196
|
raise ValueError(f"Relations must have a value type that points to another view, got {value}")
|
|
197
197
|
if relation == "reversedirect" and value.property_ is None:
|
|
198
|
+
# Todo fix this error message. It have the wrong syntax for how to have a property
|
|
198
199
|
raise ValueError(
|
|
199
200
|
"Reverse direct relation must set what it is the reverse property of. "
|
|
200
201
|
f"Which property in {value.versioned_id} is this the reverse of? Expecting"
|
|
@@ -202,17 +203,25 @@ class DMSProperty(SheetEntity):
|
|
|
202
203
|
)
|
|
203
204
|
return value
|
|
204
205
|
|
|
206
|
+
@field_serializer("value_type", when_used="always")
|
|
207
|
+
@staticmethod
|
|
208
|
+
def as_dms_type(value_type: DataType | ViewPropertyEntity | ViewEntity) -> str:
|
|
209
|
+
if isinstance(value_type, DataType):
|
|
210
|
+
return value_type.dms._type
|
|
211
|
+
else:
|
|
212
|
+
return str(value_type)
|
|
213
|
+
|
|
205
214
|
|
|
206
215
|
class DMSContainer(SheetEntity):
|
|
207
|
-
container:
|
|
208
|
-
reference:
|
|
209
|
-
constraint:
|
|
216
|
+
container: ContainerEntity = Field(alias="Container")
|
|
217
|
+
reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
|
|
218
|
+
constraint: ContainerEntityList | None = Field(None, alias="Constraint")
|
|
210
219
|
|
|
211
|
-
def as_container(self
|
|
212
|
-
container_id = self.container.as_id(
|
|
220
|
+
def as_container(self) -> dm.ContainerApply:
|
|
221
|
+
container_id = self.container.as_id()
|
|
213
222
|
constraints: dict[str, dm.Constraint] = {}
|
|
214
223
|
for constraint in self.constraint or []:
|
|
215
|
-
requires = dm.RequiresConstraint(constraint.as_id(
|
|
224
|
+
requires = dm.RequiresConstraint(constraint.as_id())
|
|
216
225
|
constraints[f"{constraint.space}_{constraint.external_id}"] = requires
|
|
217
226
|
|
|
218
227
|
return dm.ContainerApply(
|
|
@@ -233,7 +242,7 @@ class DMSContainer(SheetEntity):
|
|
|
233
242
|
# UniquenessConstraint it handled in the properties
|
|
234
243
|
return cls(
|
|
235
244
|
class_=ClassEntity(prefix=container.space, suffix=container.external_id),
|
|
236
|
-
container=
|
|
245
|
+
container=ContainerEntity(space=container.space, externalId=container.external_id),
|
|
237
246
|
name=container.name or None,
|
|
238
247
|
description=container.description,
|
|
239
248
|
constraint=constraints or None,
|
|
@@ -241,22 +250,21 @@ class DMSContainer(SheetEntity):
|
|
|
241
250
|
|
|
242
251
|
|
|
243
252
|
class DMSView(SheetEntity):
|
|
244
|
-
view:
|
|
245
|
-
implements:
|
|
246
|
-
reference:
|
|
253
|
+
view: ViewEntity = Field(alias="View")
|
|
254
|
+
implements: ViewEntityList | None = Field(None, alias="Implements")
|
|
255
|
+
reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
|
|
247
256
|
filter_: Literal["hasData", "nodeType"] | None = Field(None, alias="Filter")
|
|
248
257
|
in_model: bool = Field(True, alias="InModel")
|
|
249
258
|
|
|
250
|
-
def as_view(self
|
|
251
|
-
view_id = self.view.as_id(
|
|
259
|
+
def as_view(self) -> dm.ViewApply:
|
|
260
|
+
view_id = self.view.as_id()
|
|
252
261
|
return dm.ViewApply(
|
|
253
262
|
space=view_id.space,
|
|
254
263
|
external_id=view_id.external_id,
|
|
255
|
-
version=view_id.version or
|
|
264
|
+
version=view_id.version or _DEFAULT_VERSION,
|
|
256
265
|
name=self.name or None,
|
|
257
266
|
description=self.description,
|
|
258
|
-
implements=[parent.as_id(
|
|
259
|
-
or None,
|
|
267
|
+
implements=[parent.as_id() for parent in self.implements or []] or None,
|
|
260
268
|
properties={},
|
|
261
269
|
)
|
|
262
270
|
|
|
@@ -264,11 +272,13 @@ class DMSView(SheetEntity):
|
|
|
264
272
|
def from_view(cls, view: dm.ViewApply, data_model_view_ids: set[dm.ViewId]) -> "DMSView":
|
|
265
273
|
return cls(
|
|
266
274
|
class_=ClassEntity(prefix=view.space, suffix=view.external_id),
|
|
267
|
-
view=
|
|
275
|
+
view=ViewEntity(space=view.space, externalId=view.external_id, version=view.version),
|
|
268
276
|
description=view.description,
|
|
269
277
|
name=view.name,
|
|
270
278
|
implements=[
|
|
271
|
-
|
|
279
|
+
ViewEntity(
|
|
280
|
+
space=parent.space, externalId=parent.external_id, version=parent.version or _DEFAULT_VERSION
|
|
281
|
+
)
|
|
272
282
|
for parent in view.implements
|
|
273
283
|
]
|
|
274
284
|
or None,
|
|
@@ -283,79 +293,6 @@ class DMSRules(BaseRules):
|
|
|
283
293
|
containers: SheetList[DMSContainer] | None = Field(None, alias="Containers")
|
|
284
294
|
reference: "DMSRules | None" = Field(None, alias="Reference")
|
|
285
295
|
|
|
286
|
-
@model_validator(mode="after")
|
|
287
|
-
def set_default_space_and_version(self) -> "DMSRules":
|
|
288
|
-
default_space = self.metadata.space
|
|
289
|
-
default_view_version = self.metadata.default_view_version
|
|
290
|
-
for entity in self.properties:
|
|
291
|
-
if entity.class_.prefix is Undefined or entity.class_.version is None:
|
|
292
|
-
entity.class_ = ClassEntity(
|
|
293
|
-
prefix=default_space if entity.class_.prefix is Undefined else entity.class_.prefix,
|
|
294
|
-
suffix=entity.class_.suffix,
|
|
295
|
-
version=default_view_version if entity.class_.version is None else entity.class_.version,
|
|
296
|
-
)
|
|
297
|
-
if entity.container and entity.container.space is Undefined:
|
|
298
|
-
entity.container = ContainerEntity(prefix=default_space, suffix=entity.container.external_id)
|
|
299
|
-
|
|
300
|
-
if entity.view and (entity.view.space is Undefined or entity.view.version is None):
|
|
301
|
-
entity.view = ViewEntity(
|
|
302
|
-
prefix=default_space if entity.view.space is Undefined else entity.view.space,
|
|
303
|
-
suffix=entity.view.external_id,
|
|
304
|
-
version=default_view_version if entity.view.version is None else entity.view.version,
|
|
305
|
-
)
|
|
306
|
-
if isinstance(entity.value_type, ViewPropEntity) and (
|
|
307
|
-
entity.value_type.space is Undefined or entity.value_type.version is None
|
|
308
|
-
):
|
|
309
|
-
entity.value_type = ViewPropEntity(
|
|
310
|
-
prefix=default_space if entity.value_type.space is Undefined else entity.value_type.space,
|
|
311
|
-
suffix=entity.value_type.suffix,
|
|
312
|
-
version=default_view_version if entity.value_type.version is None else entity.value_type.version,
|
|
313
|
-
property_=entity.value_type.property_,
|
|
314
|
-
)
|
|
315
|
-
|
|
316
|
-
for container in self.containers or []:
|
|
317
|
-
if container.class_.prefix is Undefined:
|
|
318
|
-
container.class_ = ClassEntity(prefix=default_space, suffix=container.class_.suffix)
|
|
319
|
-
if container.container.space is Undefined:
|
|
320
|
-
container.container = ContainerEntity(prefix=default_space, suffix=container.container.external_id)
|
|
321
|
-
container.constraint = [
|
|
322
|
-
(
|
|
323
|
-
ContainerEntity(prefix=default_space, suffix=constraint.external_id)
|
|
324
|
-
if constraint.space is Undefined
|
|
325
|
-
else constraint
|
|
326
|
-
)
|
|
327
|
-
for constraint in container.constraint or []
|
|
328
|
-
] or None
|
|
329
|
-
|
|
330
|
-
for view in self.views or []:
|
|
331
|
-
if view.class_.prefix is Undefined or view.class_.version is None:
|
|
332
|
-
view.class_ = ClassEntity(
|
|
333
|
-
prefix=default_space if view.class_.prefix is Undefined else view.class_.prefix,
|
|
334
|
-
suffix=view.class_.suffix,
|
|
335
|
-
version=default_view_version if view.class_.version is None else view.class_.version,
|
|
336
|
-
)
|
|
337
|
-
|
|
338
|
-
if view.view.space is Undefined or view.view.version is None:
|
|
339
|
-
view.view = ViewEntity(
|
|
340
|
-
prefix=default_space if view.view.space is Undefined else view.view.space,
|
|
341
|
-
suffix=view.view.external_id,
|
|
342
|
-
version=default_view_version if view.view.version is None else view.view.version,
|
|
343
|
-
)
|
|
344
|
-
view.implements = [
|
|
345
|
-
(
|
|
346
|
-
ViewEntity(
|
|
347
|
-
prefix=default_space if parent.space is Undefined else parent.space,
|
|
348
|
-
suffix=parent.external_id,
|
|
349
|
-
version=default_view_version if parent.version is None else parent.version,
|
|
350
|
-
)
|
|
351
|
-
if parent.space is Undefined or parent.version is None
|
|
352
|
-
else parent
|
|
353
|
-
)
|
|
354
|
-
for parent in view.implements or []
|
|
355
|
-
] or None
|
|
356
|
-
|
|
357
|
-
return self
|
|
358
|
-
|
|
359
296
|
@model_validator(mode="after")
|
|
360
297
|
def consistent_container_properties(self) -> "DMSRules":
|
|
361
298
|
container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, DMSProperty]]] = defaultdict(list)
|
|
@@ -367,13 +304,16 @@ class DMSRules(BaseRules):
|
|
|
367
304
|
for (container, prop_name), properties in container_properties_by_id.items():
|
|
368
305
|
if len(properties) == 1:
|
|
369
306
|
continue
|
|
370
|
-
container_id = container.as_id(
|
|
307
|
+
container_id = container.as_id()
|
|
371
308
|
row_numbers = {prop_no for prop_no, _ in properties}
|
|
372
309
|
value_types = {prop.value_type for _, prop in properties if prop.value_type}
|
|
373
310
|
if len(value_types) > 1:
|
|
374
311
|
errors.append(
|
|
375
312
|
cognite.neat.rules.issues.spreadsheet.MultiValueTypeError(
|
|
376
|
-
container_id,
|
|
313
|
+
container_id,
|
|
314
|
+
prop_name,
|
|
315
|
+
row_numbers,
|
|
316
|
+
{v.dms._type if isinstance(v, DataType) else str(v) for v in value_types},
|
|
377
317
|
)
|
|
378
318
|
)
|
|
379
319
|
list_definitions = {prop.is_list for _, prop in properties if prop.is_list is not None}
|
|
@@ -437,14 +377,11 @@ class DMSRules(BaseRules):
|
|
|
437
377
|
@model_validator(mode="after")
|
|
438
378
|
def referenced_views_and_containers_are_existing(self) -> "DMSRules":
|
|
439
379
|
# There two checks are done in the same method to raise all the errors at once.
|
|
440
|
-
defined_views = {view.view.as_id(
|
|
380
|
+
defined_views = {view.view.as_id() for view in self.views}
|
|
441
381
|
|
|
442
382
|
errors: list[issues.NeatValidationError] = []
|
|
443
383
|
for prop_no, prop in enumerate(self.properties):
|
|
444
|
-
if (
|
|
445
|
-
prop.view
|
|
446
|
-
and (view_id := prop.view.as_id(False, self.metadata.space, self.metadata.version)) not in defined_views
|
|
447
|
-
):
|
|
384
|
+
if prop.view and (view_id := prop.view.as_id()) not in defined_views:
|
|
448
385
|
errors.append(
|
|
449
386
|
cognite.neat.rules.issues.spreadsheet.NonExistingViewError(
|
|
450
387
|
column="View",
|
|
@@ -457,12 +394,9 @@ class DMSRules(BaseRules):
|
|
|
457
394
|
)
|
|
458
395
|
)
|
|
459
396
|
if self.metadata.schema_ is SchemaCompleteness.complete:
|
|
460
|
-
defined_containers = {container.container.as_id(
|
|
397
|
+
defined_containers = {container.container.as_id() for container in self.containers or []}
|
|
461
398
|
for prop_no, prop in enumerate(self.properties):
|
|
462
|
-
if (
|
|
463
|
-
prop.container
|
|
464
|
-
and (container_id := prop.container.as_id(self.metadata.space)) not in defined_containers
|
|
465
|
-
):
|
|
399
|
+
if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
|
|
466
400
|
errors.append(
|
|
467
401
|
cognite.neat.rules.issues.spreadsheet.NonExistingContainerError(
|
|
468
402
|
column="Container",
|
|
@@ -476,13 +410,13 @@ class DMSRules(BaseRules):
|
|
|
476
410
|
)
|
|
477
411
|
for _container_no, container in enumerate(self.containers or []):
|
|
478
412
|
for constraint_no, constraint in enumerate(container.constraint or []):
|
|
479
|
-
if constraint.as_id(
|
|
413
|
+
if constraint.as_id() not in defined_containers:
|
|
480
414
|
errors.append(
|
|
481
415
|
cognite.neat.rules.issues.spreadsheet.NonExistingContainerError(
|
|
482
416
|
column="Constraint",
|
|
483
417
|
row=constraint_no,
|
|
484
418
|
type="value_error.missing",
|
|
485
|
-
container_id=constraint.as_id(
|
|
419
|
+
container_id=constraint.as_id(),
|
|
486
420
|
msg="",
|
|
487
421
|
input=None,
|
|
488
422
|
url=None,
|
|
@@ -584,17 +518,15 @@ class DMSRules(BaseRules):
|
|
|
584
518
|
# This is an extension of the reference rules, we need to merge the two
|
|
585
519
|
rules = self.model_copy(deep=True)
|
|
586
520
|
rules.properties.extend(self.reference.properties.data)
|
|
587
|
-
existing_views = {view.view.as_id(
|
|
588
|
-
rules.views.extend([view for view in self.reference.views if view.view.as_id(
|
|
521
|
+
existing_views = {view.view.as_id() for view in rules.views}
|
|
522
|
+
rules.views.extend([view for view in self.reference.views if view.view.as_id() not in existing_views])
|
|
589
523
|
if rules.containers and self.reference.containers:
|
|
590
|
-
existing_containers = {
|
|
591
|
-
container.container.as_id(self.metadata.space) for container in rules.containers.data
|
|
592
|
-
}
|
|
524
|
+
existing_containers = {container.container.as_id() for container in rules.containers.data}
|
|
593
525
|
rules.containers.extend(
|
|
594
526
|
[
|
|
595
527
|
container
|
|
596
528
|
for container in self.reference.containers
|
|
597
|
-
if container.container.as_id(
|
|
529
|
+
if container.container.as_id() not in existing_containers
|
|
598
530
|
]
|
|
599
531
|
)
|
|
600
532
|
elif not rules.containers and self.reference.containers:
|
|
@@ -683,7 +615,7 @@ class DMSRules(BaseRules):
|
|
|
683
615
|
new_rules = self.model_copy(deep=True)
|
|
684
616
|
for prop in new_rules.properties:
|
|
685
617
|
prop.reference = ReferenceEntity(
|
|
686
|
-
prefix=prop.view.prefix, suffix=prop.view.suffix, version=prop.view.version,
|
|
618
|
+
prefix=prop.view.prefix, suffix=prop.view.suffix, version=prop.view.version, property=prop.property_
|
|
687
619
|
)
|
|
688
620
|
view: DMSView
|
|
689
621
|
for view in new_rules.views:
|
|
@@ -713,25 +645,17 @@ class _DMSExporter:
|
|
|
713
645
|
self.instance_space = instance_space
|
|
714
646
|
|
|
715
647
|
def to_schema(self, rules: DMSRules) -> DMSSchema:
|
|
716
|
-
|
|
717
|
-
default_space = rules.metadata.space
|
|
718
|
-
|
|
719
|
-
container_properties_by_id, view_properties_by_id = self._gather_properties(
|
|
720
|
-
rules, default_space, default_version
|
|
721
|
-
)
|
|
648
|
+
container_properties_by_id, view_properties_by_id = self._gather_properties(rules)
|
|
722
649
|
|
|
723
|
-
containers = self._create_containers(rules.containers, container_properties_by_id
|
|
650
|
+
containers = self._create_containers(rules.containers, container_properties_by_id)
|
|
724
651
|
|
|
725
|
-
views, node_types = self._create_views_with_node_types(
|
|
726
|
-
rules.views, view_properties_by_id, default_space, default_version
|
|
727
|
-
)
|
|
652
|
+
views, node_types = self._create_views_with_node_types(rules.views, view_properties_by_id)
|
|
728
653
|
|
|
729
|
-
views_not_in_model = {
|
|
730
|
-
view.view.as_id(False, default_space, default_version) for view in rules.views if not view.in_model
|
|
731
|
-
}
|
|
654
|
+
views_not_in_model = {view.view.as_id() for view in rules.views if not view.in_model}
|
|
732
655
|
data_model = rules.metadata.as_data_model()
|
|
733
656
|
data_model.views = sorted(
|
|
734
|
-
[view_id for view_id in views.as_ids() if view_id not in views_not_in_model],
|
|
657
|
+
[view_id for view_id in views.as_ids() if view_id not in views_not_in_model],
|
|
658
|
+
key=lambda v: v.as_tuple(), # type: ignore[union-attr]
|
|
735
659
|
)
|
|
736
660
|
|
|
737
661
|
spaces = self._create_spaces(rules.metadata, containers, views, data_model)
|
|
@@ -770,13 +694,9 @@ class _DMSExporter:
|
|
|
770
694
|
self,
|
|
771
695
|
dms_views: SheetList[DMSView],
|
|
772
696
|
view_properties_by_id: dict[dm.ViewId, list[DMSProperty]],
|
|
773
|
-
default_space: str,
|
|
774
|
-
default_version: str,
|
|
775
697
|
) -> tuple[dm.ViewApplyList, dm.NodeApplyList]:
|
|
776
|
-
views = dm.ViewApplyList([dms_view.as_view(
|
|
777
|
-
dms_view_by_id = {
|
|
778
|
-
dms_view.view.as_id(False, default_space, default_version): dms_view for dms_view in dms_views
|
|
779
|
-
}
|
|
698
|
+
views = dm.ViewApplyList([dms_view.as_view() for dms_view in dms_views])
|
|
699
|
+
dms_view_by_id = {dms_view.view.as_id(): dms_view for dms_view in dms_views}
|
|
780
700
|
|
|
781
701
|
for view in views:
|
|
782
702
|
view_id = view.as_id()
|
|
@@ -789,7 +709,9 @@ class _DMSExporter:
|
|
|
789
709
|
# This is not yet supported in the CDF API, a warning has already been issued, here we convert it to
|
|
790
710
|
# a multi-edge connection.
|
|
791
711
|
if isinstance(prop.value_type, ViewEntity):
|
|
792
|
-
|
|
712
|
+
source_view_id = prop.value_type.as_id()
|
|
713
|
+
elif isinstance(prop.value_type, ViewPropertyEntity):
|
|
714
|
+
source_view_id = prop.value_type.as_view_id()
|
|
793
715
|
else:
|
|
794
716
|
raise ValueError(
|
|
795
717
|
"Direct relation must have a view as value type. "
|
|
@@ -797,10 +719,10 @@ class _DMSExporter:
|
|
|
797
719
|
)
|
|
798
720
|
view_property = dm.MultiEdgeConnectionApply(
|
|
799
721
|
type=dm.DirectRelationReference(
|
|
800
|
-
space=
|
|
722
|
+
space=source_view_id.space,
|
|
801
723
|
external_id=f"{prop.view.external_id}.{prop.view_property}",
|
|
802
724
|
),
|
|
803
|
-
source=
|
|
725
|
+
source=source_view_id,
|
|
804
726
|
direction="outwards",
|
|
805
727
|
name=prop.name,
|
|
806
728
|
description=prop.description,
|
|
@@ -809,14 +731,16 @@ class _DMSExporter:
|
|
|
809
731
|
container_prop_identifier = prop.container_property
|
|
810
732
|
extra_args: dict[str, Any] = {}
|
|
811
733
|
if prop.relation == "direct" and isinstance(prop.value_type, ViewEntity):
|
|
812
|
-
extra_args["source"] = prop.value_type.as_id(
|
|
734
|
+
extra_args["source"] = prop.value_type.as_id()
|
|
735
|
+
elif isinstance(prop.value_type, DMSUnknownEntity):
|
|
736
|
+
extra_args["source"] = None
|
|
813
737
|
elif prop.relation == "direct" and not isinstance(prop.value_type, ViewEntity):
|
|
814
738
|
raise ValueError(
|
|
815
739
|
"Direct relation must have a view as value type. "
|
|
816
740
|
"This should have been validated in the rules"
|
|
817
741
|
)
|
|
818
742
|
view_property = dm.MappedPropertyApply(
|
|
819
|
-
container=prop.container.as_id(
|
|
743
|
+
container=prop.container.as_id(),
|
|
820
744
|
container_property_identifier=container_prop_identifier,
|
|
821
745
|
name=prop.name,
|
|
822
746
|
description=prop.description,
|
|
@@ -824,34 +748,35 @@ class _DMSExporter:
|
|
|
824
748
|
)
|
|
825
749
|
elif prop.view and prop.view_property and prop.relation == "multiedge":
|
|
826
750
|
if isinstance(prop.value_type, ViewEntity):
|
|
827
|
-
|
|
751
|
+
source_view_id = prop.value_type.as_id()
|
|
828
752
|
else:
|
|
829
753
|
raise ValueError(
|
|
830
754
|
"Multiedge relation must have a view as value type. "
|
|
831
755
|
"This should have been validated in the rules"
|
|
832
756
|
)
|
|
833
757
|
if isinstance(prop.reference, ReferenceEntity):
|
|
834
|
-
ref_view_prop = prop.reference.
|
|
758
|
+
ref_view_prop = prop.reference.as_view_property_id()
|
|
835
759
|
edge_type = dm.DirectRelationReference(
|
|
836
760
|
space=ref_view_prop.source.space,
|
|
837
761
|
external_id=f"{ref_view_prop.source.external_id}.{ref_view_prop.property}",
|
|
838
762
|
)
|
|
839
763
|
else:
|
|
840
764
|
edge_type = dm.DirectRelationReference(
|
|
841
|
-
space=
|
|
765
|
+
space=source_view_id.space,
|
|
842
766
|
external_id=f"{prop.view.external_id}.{prop.view_property}",
|
|
843
767
|
)
|
|
844
768
|
|
|
845
769
|
view_property = dm.MultiEdgeConnectionApply(
|
|
846
770
|
type=edge_type,
|
|
847
|
-
source=
|
|
771
|
+
source=source_view_id,
|
|
848
772
|
direction="outwards",
|
|
849
773
|
name=prop.name,
|
|
850
774
|
description=prop.description,
|
|
851
775
|
)
|
|
852
776
|
elif prop.view and prop.view_property and prop.relation == "reversedirect":
|
|
853
|
-
if isinstance(prop.value_type,
|
|
854
|
-
|
|
777
|
+
if isinstance(prop.value_type, ViewPropertyEntity):
|
|
778
|
+
source_prop_id = prop.value_type.as_id()
|
|
779
|
+
source_view_id = cast(dm.ViewId, source_prop_id.source)
|
|
855
780
|
else:
|
|
856
781
|
raise ValueError(
|
|
857
782
|
"Reverse direct relation must have a view as value type. "
|
|
@@ -865,34 +790,39 @@ class _DMSExporter:
|
|
|
865
790
|
f"{prop.value_type.versioned_id}:<property>"
|
|
866
791
|
)
|
|
867
792
|
reverse_prop = next(
|
|
868
|
-
(
|
|
793
|
+
(
|
|
794
|
+
prop
|
|
795
|
+
for prop in view_properties_by_id.get(source_view_id, [])
|
|
796
|
+
if prop.property_ == source_prop
|
|
797
|
+
),
|
|
798
|
+
None,
|
|
869
799
|
)
|
|
870
800
|
if reverse_prop and reverse_prop.relation == "direct" and reverse_prop.is_list:
|
|
871
801
|
warnings.warn(
|
|
872
802
|
issues.dms.ReverseOfDirectRelationListWarning(view_id, prop.property_), stacklevel=2
|
|
873
803
|
)
|
|
874
804
|
if isinstance(reverse_prop.reference, ReferenceEntity):
|
|
875
|
-
ref_view_prop = reverse_prop.reference.
|
|
805
|
+
ref_view_prop = reverse_prop.reference.as_view_property_id()
|
|
876
806
|
edge_type = dm.DirectRelationReference(
|
|
877
807
|
space=ref_view_prop.source.space,
|
|
878
808
|
external_id=f"{ref_view_prop.source.external_id}.{ref_view_prop.property}",
|
|
879
809
|
)
|
|
880
810
|
else:
|
|
881
811
|
edge_type = dm.DirectRelationReference(
|
|
882
|
-
space=source.space,
|
|
812
|
+
space=source_prop_id.source.space,
|
|
883
813
|
external_id=f"{reverse_prop.view.external_id}.{reverse_prop.view_property}",
|
|
884
814
|
)
|
|
885
815
|
view_property = dm.MultiEdgeConnectionApply(
|
|
886
816
|
type=edge_type,
|
|
887
|
-
source=source,
|
|
817
|
+
source=source_prop_id.source, # type: ignore[arg-type]
|
|
888
818
|
direction="inwards",
|
|
889
819
|
name=prop.name,
|
|
890
820
|
description=prop.description,
|
|
891
821
|
)
|
|
892
822
|
else:
|
|
893
823
|
args: dict[str, Any] = dict(
|
|
894
|
-
source=
|
|
895
|
-
through=dm.PropertyId(source, source_prop),
|
|
824
|
+
source=source_view_id,
|
|
825
|
+
through=dm.PropertyId(source_prop_id.source, source_prop),
|
|
896
826
|
description=prop.description,
|
|
897
827
|
name=prop.name,
|
|
898
828
|
)
|
|
@@ -925,7 +855,7 @@ class _DMSExporter:
|
|
|
925
855
|
if dms_view and isinstance(dms_view.reference, ReferenceEntity):
|
|
926
856
|
# If the view is a reference, we implement the reference view,
|
|
927
857
|
# and need the filter to match the reference
|
|
928
|
-
ref_view = dms_view.reference.
|
|
858
|
+
ref_view = dms_view.reference.as_view_id()
|
|
929
859
|
node_type = dm.filters.Equals(
|
|
930
860
|
["node", "type"], {"space": ref_view.space, "externalId": ref_view.external_id}
|
|
931
861
|
)
|
|
@@ -959,11 +889,8 @@ class _DMSExporter:
|
|
|
959
889
|
self,
|
|
960
890
|
dms_container: SheetList[DMSContainer] | None,
|
|
961
891
|
container_properties_by_id: dict[dm.ContainerId, list[DMSProperty]],
|
|
962
|
-
default_space: str,
|
|
963
892
|
) -> dm.ContainerApplyList:
|
|
964
|
-
containers = dm.ContainerApplyList(
|
|
965
|
-
[dms_container.as_container(default_space) for dms_container in dms_container or []]
|
|
966
|
-
)
|
|
893
|
+
containers = dm.ContainerApplyList([dms_container.as_container() for dms_container in dms_container or []])
|
|
967
894
|
container_to_drop = set()
|
|
968
895
|
for container in containers:
|
|
969
896
|
container_id = container.as_id()
|
|
@@ -974,7 +901,7 @@ class _DMSExporter:
|
|
|
974
901
|
for prop in container_properties:
|
|
975
902
|
if prop.container_property is None:
|
|
976
903
|
continue
|
|
977
|
-
if isinstance(prop.value_type,
|
|
904
|
+
if isinstance(prop.value_type, DataType):
|
|
978
905
|
type_cls = prop.value_type.dms
|
|
979
906
|
else:
|
|
980
907
|
type_cls = dm.DirectRelation
|
|
@@ -1032,26 +959,26 @@ class _DMSExporter:
|
|
|
1032
959
|
)
|
|
1033
960
|
|
|
1034
961
|
def _gather_properties(
|
|
1035
|
-
self, rules: DMSRules
|
|
962
|
+
self, rules: DMSRules
|
|
1036
963
|
) -> tuple[dict[dm.ContainerId, list[DMSProperty]], dict[dm.ViewId, list[DMSProperty]]]:
|
|
1037
964
|
container_properties_by_id: dict[dm.ContainerId, list[DMSProperty]] = defaultdict(list)
|
|
1038
965
|
view_properties_by_id: dict[dm.ViewId, list[DMSProperty]] = defaultdict(list)
|
|
1039
966
|
for prop in rules.properties:
|
|
1040
|
-
view_id = prop.view.as_id(
|
|
967
|
+
view_id = prop.view.as_id()
|
|
1041
968
|
view_properties_by_id[view_id].append(prop)
|
|
1042
969
|
|
|
1043
970
|
if prop.container and prop.container_property:
|
|
1044
971
|
if prop.relation == "direct" and prop.is_list:
|
|
1045
972
|
warnings.warn(
|
|
1046
973
|
issues.dms.DirectRelationListWarning(
|
|
1047
|
-
container_id=prop.container.as_id(
|
|
1048
|
-
view_id=prop.view.as_id(
|
|
974
|
+
container_id=prop.container.as_id(),
|
|
975
|
+
view_id=prop.view.as_id(),
|
|
1049
976
|
property=prop.container_property,
|
|
1050
977
|
),
|
|
1051
978
|
stacklevel=2,
|
|
1052
979
|
)
|
|
1053
980
|
continue
|
|
1054
|
-
container_id = prop.container.as_id(
|
|
981
|
+
container_id = prop.container.as_id()
|
|
1055
982
|
container_properties_by_id[container_id].append(prop)
|
|
1056
983
|
|
|
1057
984
|
return container_properties_by_id, view_properties_by_id
|
|
@@ -1090,7 +1017,7 @@ class _DMSRulesConverter:
|
|
|
1090
1017
|
classes = [
|
|
1091
1018
|
InformationClass(
|
|
1092
1019
|
# we do not want a version in class as we use URI for the class
|
|
1093
|
-
class_=view.class_.
|
|
1020
|
+
class_=ClassEntity(prefix=view.class_.prefix, suffix=view.class_.suffix),
|
|
1094
1021
|
description=view.description,
|
|
1095
1022
|
parent=[
|
|
1096
1023
|
# we do not want a version in class as we use URI for the class
|
|
@@ -1105,23 +1032,26 @@ class _DMSRulesConverter:
|
|
|
1105
1032
|
]
|
|
1106
1033
|
|
|
1107
1034
|
properties: list[InformationProperty] = []
|
|
1108
|
-
value_type:
|
|
1035
|
+
value_type: DataType | ClassEntity | str
|
|
1109
1036
|
for property_ in self.dms.properties:
|
|
1110
|
-
if isinstance(property_.value_type,
|
|
1111
|
-
value_type =
|
|
1112
|
-
elif isinstance(property_.value_type, ViewEntity):
|
|
1037
|
+
if isinstance(property_.value_type, DataType):
|
|
1038
|
+
value_type = property_.value_type
|
|
1039
|
+
elif isinstance(property_.value_type, ViewEntity | ViewPropertyEntity):
|
|
1113
1040
|
value_type = ClassEntity(
|
|
1114
1041
|
prefix=property_.value_type.prefix,
|
|
1115
1042
|
suffix=property_.value_type.suffix,
|
|
1116
1043
|
)
|
|
1044
|
+
elif isinstance(property_.value_type, DMSUnknownEntity):
|
|
1045
|
+
value_type = UnknownEntity()
|
|
1117
1046
|
else:
|
|
1118
1047
|
raise ValueError(f"Unsupported value type: {property_.value_type.type_}")
|
|
1119
1048
|
|
|
1120
1049
|
properties.append(
|
|
1121
1050
|
InformationProperty(
|
|
1122
|
-
|
|
1051
|
+
# Removing version
|
|
1052
|
+
class_=ClassEntity(suffix=property_.class_.suffix, prefix=property_.class_.prefix),
|
|
1123
1053
|
property_=property_.view_property,
|
|
1124
|
-
value_type=
|
|
1054
|
+
value_type=value_type,
|
|
1125
1055
|
description=property_.description,
|
|
1126
1056
|
min_count=0 if property_.nullable or property_.nullable is None else 1,
|
|
1127
1057
|
max_count=float("inf") if property_.is_list or property_.nullable is None else 1,
|
|
@@ -1161,5 +1091,5 @@ class _DMSRulesConverter:
|
|
|
1161
1091
|
return ReferenceEntity(
|
|
1162
1092
|
prefix=container.prefix,
|
|
1163
1093
|
suffix=container.suffix,
|
|
1164
|
-
|
|
1094
|
+
property=property_.container_property,
|
|
1165
1095
|
)
|