cognite-neat 0.97.3__py3-none-any.whl → 0.98.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_graph/loaders/__init__.py +1 -2
- cognite/neat/_issues/warnings/_models.py +9 -0
- cognite/neat/_rules/_shared.py +3 -8
- cognite/neat/_rules/analysis/__init__.py +1 -2
- cognite/neat/_rules/analysis/_base.py +2 -23
- cognite/neat/_rules/analysis/_dms.py +4 -10
- cognite/neat/_rules/analysis/_information.py +2 -10
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_rules2excel.py +15 -72
- cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/_rules/importers/_base.py +3 -4
- cognite/neat/_rules/importers/_dms2rules.py +17 -40
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
- cognite/neat/_rules/importers/_rdf/_base.py +17 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +30 -18
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
- cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
- cognite/neat/_rules/models/__init__.py +2 -16
- cognite/neat/_rules/models/_base_rules.py +98 -52
- cognite/neat/_rules/models/dms/_exporter.py +7 -160
- cognite/neat/_rules/models/dms/_rules.py +18 -126
- cognite/neat/_rules/models/dms/_rules_input.py +20 -48
- cognite/neat/_rules/models/dms/_schema.py +11 -0
- cognite/neat/_rules/models/dms/_validation.py +9 -107
- cognite/neat/_rules/models/information/_rules.py +19 -114
- cognite/neat/_rules/models/information/_rules_input.py +32 -41
- cognite/neat/_rules/models/information/_validation.py +34 -102
- cognite/neat/_rules/transformers/__init__.py +1 -4
- cognite/neat/_rules/transformers/_converters.py +18 -195
- cognite/neat/_rules/transformers/_mapping.py +1 -5
- cognite/neat/_rules/transformers/_verification.py +0 -14
- cognite/neat/_session/_base.py +34 -13
- cognite/neat/_session/_collector.py +126 -0
- cognite/neat/_session/_inspect.py +5 -5
- cognite/neat/_session/_prepare.py +4 -4
- cognite/neat/_session/_read.py +62 -9
- cognite/neat/_session/_set.py +2 -2
- cognite/neat/_session/_show.py +11 -11
- cognite/neat/_session/_to.py +24 -11
- cognite/neat/_session/exceptions.py +20 -3
- cognite/neat/_store/_provenance.py +2 -2
- cognite/neat/_utils/auxiliary.py +19 -0
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/data_contracts.py +2 -10
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +6 -46
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +2 -7
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.98.0.dist-info}/METADATA +2 -1
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.98.0.dist-info}/RECORD +58 -64
- cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
- cognite/neat/_rules/analysis/_asset.py +0 -173
- cognite/neat/_rules/models/asset/__init__.py +0 -13
- cognite/neat/_rules/models/asset/_rules.py +0 -109
- cognite/neat/_rules/models/asset/_rules_input.py +0 -101
- cognite/neat/_rules/models/asset/_validation.py +0 -45
- cognite/neat/_rules/models/domain.py +0 -136
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.98.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.98.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.98.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import math
|
|
2
1
|
import warnings
|
|
3
2
|
from collections.abc import Hashable
|
|
4
|
-
from datetime import datetime
|
|
5
3
|
from typing import Any, ClassVar, Literal
|
|
6
4
|
|
|
7
5
|
import pandas as pd
|
|
@@ -10,31 +8,24 @@ from pydantic import Field, field_serializer, field_validator, model_validator
|
|
|
10
8
|
from pydantic_core.core_schema import SerializationInfo, ValidationInfo
|
|
11
9
|
from rdflib import URIRef
|
|
12
10
|
|
|
13
|
-
from cognite.neat._constants import COGNITE_SPACES
|
|
11
|
+
from cognite.neat._constants import COGNITE_SPACES
|
|
14
12
|
from cognite.neat._issues import MultiValueError
|
|
15
13
|
from cognite.neat._issues.warnings import (
|
|
16
14
|
PrincipleMatchingSpaceAndVersionWarning,
|
|
17
|
-
PrincipleSolutionBuildsOnEnterpriseWarning,
|
|
18
15
|
)
|
|
19
16
|
from cognite.neat._rules.models._base_rules import (
|
|
20
17
|
BaseMetadata,
|
|
21
18
|
BaseRules,
|
|
22
|
-
|
|
23
|
-
ExtensionCategory,
|
|
19
|
+
DataModelAspect,
|
|
24
20
|
RoleTypes,
|
|
25
|
-
SchemaCompleteness,
|
|
26
21
|
SheetList,
|
|
27
22
|
SheetRow,
|
|
28
23
|
)
|
|
29
24
|
from cognite.neat._rules.models._types import (
|
|
30
25
|
ClassEntityType,
|
|
31
26
|
ContainerEntityType,
|
|
32
|
-
DataModelExternalIdType,
|
|
33
27
|
DmsPropertyType,
|
|
34
|
-
InformationPropertyType,
|
|
35
|
-
SpaceType,
|
|
36
28
|
StrListType,
|
|
37
|
-
VersionType,
|
|
38
29
|
ViewEntityType,
|
|
39
30
|
)
|
|
40
31
|
from cognite.neat._rules.models.data_types import DataType
|
|
@@ -48,9 +39,7 @@ from cognite.neat._rules.models.entities import (
|
|
|
48
39
|
HasDataFilter,
|
|
49
40
|
NodeTypeFilter,
|
|
50
41
|
RawFilter,
|
|
51
|
-
ReferenceEntity,
|
|
52
42
|
ReverseConnectionEntity,
|
|
53
|
-
URLEntity,
|
|
54
43
|
ViewEntity,
|
|
55
44
|
ViewEntityList,
|
|
56
45
|
)
|
|
@@ -62,54 +51,8 @@ _DEFAULT_VERSION = "1"
|
|
|
62
51
|
|
|
63
52
|
class DMSMetadata(BaseMetadata):
|
|
64
53
|
role: ClassVar[RoleTypes] = RoleTypes.dms
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
extension: ExtensionCategory = ExtensionCategory.addition
|
|
68
|
-
space: SpaceType
|
|
69
|
-
name: str | None = Field(
|
|
70
|
-
None,
|
|
71
|
-
description="Human readable name of the data model",
|
|
72
|
-
min_length=1,
|
|
73
|
-
max_length=255,
|
|
74
|
-
)
|
|
75
|
-
description: str | None = Field(None, min_length=1, max_length=1024)
|
|
76
|
-
external_id: DataModelExternalIdType = Field(alias="externalId")
|
|
77
|
-
version: VersionType
|
|
78
|
-
creator: StrListType
|
|
79
|
-
created: datetime = Field(
|
|
80
|
-
description=("Date of the data model creation"),
|
|
81
|
-
)
|
|
82
|
-
updated: datetime = Field(
|
|
83
|
-
description=("Date of the data model update"),
|
|
84
|
-
)
|
|
85
|
-
|
|
86
|
-
@field_validator("*", mode="before")
|
|
87
|
-
def strip_string(cls, value: Any) -> Any:
|
|
88
|
-
if isinstance(value, str):
|
|
89
|
-
return value.strip()
|
|
90
|
-
return value
|
|
91
|
-
|
|
92
|
-
@field_serializer("schema_", "extension", "data_model_type", when_used="always")
|
|
93
|
-
def as_string(self, value: SchemaCompleteness | ExtensionCategory | DataModelType) -> str:
|
|
94
|
-
return str(value)
|
|
95
|
-
|
|
96
|
-
@field_validator("schema_", mode="plain")
|
|
97
|
-
def as_enum_schema(cls, value: str) -> SchemaCompleteness:
|
|
98
|
-
return SchemaCompleteness(value.strip())
|
|
99
|
-
|
|
100
|
-
@field_validator("extension", mode="plain")
|
|
101
|
-
def as_enum_extension(cls, value: str) -> ExtensionCategory:
|
|
102
|
-
return ExtensionCategory(value.strip())
|
|
103
|
-
|
|
104
|
-
@field_validator("data_model_type", mode="plain")
|
|
105
|
-
def as_enum_model_type(cls, value: str) -> DataModelType:
|
|
106
|
-
return DataModelType(value.strip())
|
|
107
|
-
|
|
108
|
-
@field_validator("description", mode="before")
|
|
109
|
-
def nan_as_none(cls, value):
|
|
110
|
-
if isinstance(value, float) and math.isnan(value):
|
|
111
|
-
return None
|
|
112
|
-
return value
|
|
54
|
+
aspect: ClassVar[DataModelAspect] = DataModelAspect.physical
|
|
55
|
+
logical: str | None = None
|
|
113
56
|
|
|
114
57
|
def as_space(self) -> dm.SpaceApply:
|
|
115
58
|
return dm.SpaceApply(
|
|
@@ -159,13 +102,15 @@ class DMSProperty(SheetRow):
|
|
|
159
102
|
immutable: bool | None = Field(default=None, alias="Immutable")
|
|
160
103
|
is_list: bool | None = Field(default=None, alias="Is List")
|
|
161
104
|
default: str | int | dict | None = Field(None, alias="Default")
|
|
162
|
-
reference: URLEntity | ReferenceEntity | None = Field(default=None, alias="Reference", union_mode="left_to_right")
|
|
163
105
|
container: ContainerEntityType | None = Field(None, alias="Container")
|
|
164
106
|
container_property: DmsPropertyType | None = Field(None, alias="Container Property")
|
|
165
107
|
index: StrListType | None = Field(None, alias="Index")
|
|
166
108
|
constraint: StrListType | None = Field(None, alias="Constraint")
|
|
167
|
-
|
|
168
|
-
|
|
109
|
+
logical: URIRef | None = Field(
|
|
110
|
+
None,
|
|
111
|
+
alias="Logical",
|
|
112
|
+
description="Used to make connection between physical and logical data model aspect",
|
|
113
|
+
)
|
|
169
114
|
|
|
170
115
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
171
116
|
return self.view, self.view_property
|
|
@@ -208,19 +153,6 @@ class DMSProperty(SheetRow):
|
|
|
208
153
|
)
|
|
209
154
|
return value
|
|
210
155
|
|
|
211
|
-
@field_serializer("reference", when_used="always")
|
|
212
|
-
def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
|
|
213
|
-
if isinstance(info.context, dict) and info.context.get("as_reference") is True:
|
|
214
|
-
return str(
|
|
215
|
-
ReferenceEntity(
|
|
216
|
-
prefix=self.view.prefix,
|
|
217
|
-
suffix=self.view.suffix,
|
|
218
|
-
version=self.view.version,
|
|
219
|
-
property=self.view_property,
|
|
220
|
-
)
|
|
221
|
-
)
|
|
222
|
-
return str(value) if value is not None else None
|
|
223
|
-
|
|
224
156
|
@field_serializer("value_type", when_used="always")
|
|
225
157
|
def as_dms_type(self, value_type: DataType | EdgeEntity | ViewEntity, info: SerializationInfo) -> str:
|
|
226
158
|
if isinstance(value_type, DataType):
|
|
@@ -229,7 +161,7 @@ class DMSProperty(SheetRow):
|
|
|
229
161
|
return value_type.dump(space=metadata.space, version=metadata.version)
|
|
230
162
|
return str(value_type)
|
|
231
163
|
|
|
232
|
-
@field_serializer("view", "container",
|
|
164
|
+
@field_serializer("view", "container", when_used="unless-none")
|
|
233
165
|
def remove_default_space(self, value: str, info: SerializationInfo) -> str:
|
|
234
166
|
if (metadata := _metadata(info.context)) and isinstance(value, Entity):
|
|
235
167
|
if info.field_name == "container" and info.context.get("as_reference") is True:
|
|
@@ -253,10 +185,8 @@ class DMSContainer(SheetRow):
|
|
|
253
185
|
container: ContainerEntityType = Field(alias="Container")
|
|
254
186
|
name: str | None = Field(alias="Name", default=None)
|
|
255
187
|
description: str | None = Field(alias="Description", default=None)
|
|
256
|
-
reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
|
|
257
188
|
constraint: ContainerEntityList | None = Field(None, alias="Constraint")
|
|
258
189
|
used_for: Literal["node", "edge", "all"] | None = Field("all", alias="Used For")
|
|
259
|
-
class_: ClassEntityType = Field(alias="Class (linage)")
|
|
260
190
|
|
|
261
191
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
262
192
|
return (self.container,)
|
|
@@ -278,13 +208,7 @@ class DMSContainer(SheetRow):
|
|
|
278
208
|
used_for=self.used_for,
|
|
279
209
|
)
|
|
280
210
|
|
|
281
|
-
@field_serializer("
|
|
282
|
-
def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
|
|
283
|
-
if isinstance(info.context, dict) and info.context.get("as_reference") is True:
|
|
284
|
-
return self.container.dump()
|
|
285
|
-
return str(value) if value is not None else None
|
|
286
|
-
|
|
287
|
-
@field_serializer("container", "class_", when_used="unless-none")
|
|
211
|
+
@field_serializer("container", when_used="unless-none")
|
|
288
212
|
def remove_default_space(self, value: Any, info: SerializationInfo) -> str:
|
|
289
213
|
if metadata := _metadata(info.context):
|
|
290
214
|
if isinstance(value, DMSEntity):
|
|
@@ -310,21 +234,18 @@ class DMSView(SheetRow):
|
|
|
310
234
|
name: str | None = Field(alias="Name", default=None)
|
|
311
235
|
description: str | None = Field(alias="Description", default=None)
|
|
312
236
|
implements: ViewEntityList | None = Field(None, alias="Implements")
|
|
313
|
-
reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
|
|
314
237
|
filter_: HasDataFilter | NodeTypeFilter | RawFilter | None = Field(None, alias="Filter")
|
|
315
238
|
in_model: bool = Field(True, alias="In Model")
|
|
316
|
-
|
|
239
|
+
logical: URIRef | None = Field(
|
|
240
|
+
None,
|
|
241
|
+
alias="Logical",
|
|
242
|
+
description="Used to make connection between physical and logical data model aspect",
|
|
243
|
+
)
|
|
317
244
|
|
|
318
245
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
319
246
|
return (self.view,)
|
|
320
247
|
|
|
321
|
-
@field_serializer("
|
|
322
|
-
def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
|
|
323
|
-
if isinstance(info.context, dict) and info.context.get("as_reference") is True:
|
|
324
|
-
return self.view.dump()
|
|
325
|
-
return str(value) if value is not None else None
|
|
326
|
-
|
|
327
|
-
@field_serializer("view", "class_", when_used="unless-none")
|
|
248
|
+
@field_serializer("view", when_used="unless-none")
|
|
328
249
|
def remove_default_space(self, value: Any, info: SerializationInfo) -> str:
|
|
329
250
|
if (metadata := _metadata(info.context)) and isinstance(value, Entity):
|
|
330
251
|
return value.dump(prefix=metadata.space, version=metadata.version)
|
|
@@ -344,11 +265,6 @@ class DMSView(SheetRow):
|
|
|
344
265
|
def as_view(self) -> dm.ViewApply:
|
|
345
266
|
view_id = self.view.as_id()
|
|
346
267
|
implements = [parent.as_id() for parent in self.implements or []] or None
|
|
347
|
-
if implements is None and isinstance(self.reference, ReferenceEntity):
|
|
348
|
-
# Fallback to the reference if no implements are provided
|
|
349
|
-
parent = self.reference.as_view_id()
|
|
350
|
-
if (parent.space, parent.external_id) != (view_id.space, view_id.external_id):
|
|
351
|
-
implements = [parent]
|
|
352
268
|
|
|
353
269
|
return dm.ViewApply(
|
|
354
270
|
space=view_id.space,
|
|
@@ -408,24 +324,6 @@ class DMSRules(BaseRules):
|
|
|
408
324
|
containers: SheetList[DMSContainer] | None = Field(None, alias="Containers")
|
|
409
325
|
enum: SheetList[DMSEnum] | None = Field(None, alias="Enum")
|
|
410
326
|
nodes: SheetList[DMSNode] | None = Field(None, alias="Nodes")
|
|
411
|
-
last: "DMSRules | None" = Field(None, alias="Last", description="The previous version of the data model")
|
|
412
|
-
reference: "DMSRules | None" = Field(None, alias="Reference")
|
|
413
|
-
|
|
414
|
-
@field_validator("reference")
|
|
415
|
-
def check_reference_of_reference(cls, value: "DMSRules | None", info: ValidationInfo) -> "DMSRules | None":
|
|
416
|
-
if value is None:
|
|
417
|
-
return None
|
|
418
|
-
if value.reference is not None:
|
|
419
|
-
raise ValueError("Reference rules cannot have a reference")
|
|
420
|
-
if value.metadata.data_model_type == DataModelType.solution and (metadata := info.data.get("metadata")):
|
|
421
|
-
warnings.warn(
|
|
422
|
-
PrincipleSolutionBuildsOnEnterpriseWarning(
|
|
423
|
-
f"The solution model {metadata.as_data_model_id()} is referencing another "
|
|
424
|
-
f"solution model {value.metadata.as_data_model_id()}",
|
|
425
|
-
),
|
|
426
|
-
stacklevel=2,
|
|
427
|
-
)
|
|
428
|
-
return value
|
|
429
327
|
|
|
430
328
|
@field_validator("views")
|
|
431
329
|
def matching_version_and_space(cls, value: SheetList[DMSView], info: ValidationInfo) -> SheetList[DMSView]:
|
|
@@ -476,7 +374,7 @@ class DMSRules(BaseRules):
|
|
|
476
374
|
|
|
477
375
|
def _repr_html_(self) -> str:
|
|
478
376
|
summary = {
|
|
479
|
-
"
|
|
377
|
+
"aspect": self.metadata.aspect,
|
|
480
378
|
"intended for": "DMS Architect",
|
|
481
379
|
"name": self.metadata.name,
|
|
482
380
|
"space": self.metadata.space,
|
|
@@ -488,9 +386,3 @@ class DMSRules(BaseRules):
|
|
|
488
386
|
}
|
|
489
387
|
|
|
490
388
|
return pd.DataFrame([summary]).T.rename(columns={0: ""})._repr_html_() # type: ignore
|
|
491
|
-
|
|
492
|
-
@property
|
|
493
|
-
def id_(self) -> URIRef:
|
|
494
|
-
return DEFAULT_NAMESPACE[
|
|
495
|
-
f"data-model/verified/dms/{self.metadata.space}/{self.metadata.external_id}/{self.metadata.version}"
|
|
496
|
-
]
|
|
@@ -5,13 +5,12 @@ from typing import Any, Literal
|
|
|
5
5
|
|
|
6
6
|
import pandas as pd
|
|
7
7
|
from cognite.client import data_modeling as dm
|
|
8
|
-
from rdflib import URIRef
|
|
8
|
+
from rdflib import Namespace, URIRef
|
|
9
9
|
|
|
10
10
|
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
11
11
|
from cognite.neat._rules.models._base_input import InputComponent, InputRules
|
|
12
12
|
from cognite.neat._rules.models.data_types import DataType
|
|
13
13
|
from cognite.neat._rules.models.entities import (
|
|
14
|
-
ClassEntity,
|
|
15
14
|
ContainerEntity,
|
|
16
15
|
DMSNodeEntity,
|
|
17
16
|
DMSUnknownEntity,
|
|
@@ -27,17 +26,15 @@ from ._rules import _DEFAULT_VERSION, DMSContainer, DMSEnum, DMSMetadata, DMSNod
|
|
|
27
26
|
|
|
28
27
|
@dataclass
|
|
29
28
|
class DMSInputMetadata(InputComponent[DMSMetadata]):
|
|
30
|
-
schema_: Literal["complete", "partial", "extended"]
|
|
31
29
|
space: str
|
|
32
30
|
external_id: str
|
|
33
31
|
creator: str
|
|
34
32
|
version: str
|
|
35
|
-
extension: Literal["addition", "reshape", "rebuild"] = "addition"
|
|
36
|
-
data_model_type: Literal["solution", "enterprise"] = "solution"
|
|
37
33
|
name: str | None = None
|
|
38
34
|
description: str | None = None
|
|
39
35
|
created: datetime | str | None = None
|
|
40
36
|
updated: datetime | str | None = None
|
|
37
|
+
logical: str | None = None
|
|
41
38
|
|
|
42
39
|
@classmethod
|
|
43
40
|
def _get_verified_cls(cls) -> type[DMSMetadata]:
|
|
@@ -52,11 +49,9 @@ class DMSInputMetadata(InputComponent[DMSMetadata]):
|
|
|
52
49
|
return output
|
|
53
50
|
|
|
54
51
|
@classmethod
|
|
55
|
-
def from_data_model(cls, data_model: dm.DataModelApply
|
|
52
|
+
def from_data_model(cls, data_model: dm.DataModelApply) -> "DMSInputMetadata":
|
|
56
53
|
description, creator = cls._get_description_and_creator(data_model.description)
|
|
57
54
|
return cls(
|
|
58
|
-
schema_="complete",
|
|
59
|
-
data_model_type="solution" if has_reference else "enterprise",
|
|
60
55
|
space=data_model.space,
|
|
61
56
|
name=data_model.name or None,
|
|
62
57
|
description=description,
|
|
@@ -80,14 +75,27 @@ class DMSInputMetadata(InputComponent[DMSMetadata]):
|
|
|
80
75
|
description = None
|
|
81
76
|
return description, creator
|
|
82
77
|
|
|
78
|
+
@property
|
|
79
|
+
def identifier(self) -> URIRef:
|
|
80
|
+
"""Globally unique identifier for the data model.
|
|
81
|
+
|
|
82
|
+
!!! note
|
|
83
|
+
Unlike namespace, the identifier does not end with "/" or "#".
|
|
84
|
+
|
|
85
|
+
"""
|
|
86
|
+
return DEFAULT_NAMESPACE[f"data-model/unverified/physical/{self.space}/{self.external_id}/{self.version}"]
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def namespace(self) -> Namespace:
|
|
90
|
+
"""Namespace for the data model used for the entities in the data model."""
|
|
91
|
+
return Namespace(f"{self.identifier}/")
|
|
92
|
+
|
|
83
93
|
|
|
84
94
|
@dataclass
|
|
85
95
|
class DMSInputProperty(InputComponent[DMSProperty]):
|
|
86
96
|
view: str
|
|
87
97
|
view_property: str | None
|
|
88
98
|
value_type: str | DataType | ViewEntity | DMSUnknownEntity
|
|
89
|
-
property_: str | None = None
|
|
90
|
-
class_: str | None = None
|
|
91
99
|
name: str | None = None
|
|
92
100
|
description: str | None = None
|
|
93
101
|
connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | str | None = None
|
|
@@ -95,11 +103,11 @@ class DMSInputProperty(InputComponent[DMSProperty]):
|
|
|
95
103
|
immutable: bool | None = None
|
|
96
104
|
is_list: bool | None = None
|
|
97
105
|
default: str | int | dict | None = None
|
|
98
|
-
reference: str | None = None
|
|
99
106
|
container: str | None = None
|
|
100
107
|
container_property: str | None = None
|
|
101
108
|
index: str | list[str] | None = None
|
|
102
109
|
constraint: str | list[str] | None = None
|
|
110
|
+
logical: str | None = None
|
|
103
111
|
|
|
104
112
|
@classmethod
|
|
105
113
|
def _get_verified_cls(cls) -> type[DMSProperty]:
|
|
@@ -110,12 +118,6 @@ class DMSInputProperty(InputComponent[DMSProperty]):
|
|
|
110
118
|
output["View"] = ViewEntity.load(self.view, space=default_space, version=default_version)
|
|
111
119
|
output["Value Type"] = load_dms_value_type(self.value_type, default_space, default_version)
|
|
112
120
|
output["Connection"] = load_connection(self.connection, default_space, default_version)
|
|
113
|
-
output["Property (linage)"] = self.property_ or self.view_property
|
|
114
|
-
output["Class (linage)"] = (
|
|
115
|
-
ClassEntity.load(self.class_ or self.view, prefix=default_space, version=default_version)
|
|
116
|
-
if self.class_ or self.view
|
|
117
|
-
else None
|
|
118
|
-
)
|
|
119
121
|
output["Container"] = (
|
|
120
122
|
ContainerEntity.load(self.container, space=default_space, version=default_version)
|
|
121
123
|
if self.container
|
|
@@ -127,10 +129,8 @@ class DMSInputProperty(InputComponent[DMSProperty]):
|
|
|
127
129
|
@dataclass
|
|
128
130
|
class DMSInputContainer(InputComponent[DMSContainer]):
|
|
129
131
|
container: str
|
|
130
|
-
class_: str | None = None
|
|
131
132
|
name: str | None = None
|
|
132
133
|
description: str | None = None
|
|
133
|
-
reference: str | None = None
|
|
134
134
|
constraint: str | None = None
|
|
135
135
|
used_for: Literal["node", "edge", "all"] | None = None
|
|
136
136
|
|
|
@@ -142,9 +142,6 @@ class DMSInputContainer(InputComponent[DMSContainer]):
|
|
|
142
142
|
output = super().dump()
|
|
143
143
|
container = ContainerEntity.load(self.container, space=default_space)
|
|
144
144
|
output["Container"] = container
|
|
145
|
-
output["Class (linage)"] = (
|
|
146
|
-
ClassEntity.load(self.class_, prefix=default_space) if self.class_ else container.as_class()
|
|
147
|
-
)
|
|
148
145
|
output["Constraint"] = (
|
|
149
146
|
[ContainerEntity.load(constraint.strip(), space=default_space) for constraint in self.constraint.split(",")]
|
|
150
147
|
if self.constraint
|
|
@@ -161,7 +158,6 @@ class DMSInputContainer(InputComponent[DMSContainer]):
|
|
|
161
158
|
# UniquenessConstraint it handled in the properties
|
|
162
159
|
container_entity = ContainerEntity.from_id(container.as_id())
|
|
163
160
|
return cls(
|
|
164
|
-
class_=str(container_entity.as_class()),
|
|
165
161
|
container=str(container_entity),
|
|
166
162
|
name=container.name or None,
|
|
167
163
|
description=container.description,
|
|
@@ -173,13 +169,12 @@ class DMSInputContainer(InputComponent[DMSContainer]):
|
|
|
173
169
|
@dataclass
|
|
174
170
|
class DMSInputView(InputComponent[DMSView]):
|
|
175
171
|
view: str
|
|
176
|
-
class_: str | None = None
|
|
177
172
|
name: str | None = None
|
|
178
173
|
description: str | None = None
|
|
179
174
|
implements: str | None = None
|
|
180
|
-
reference: str | None = None
|
|
181
175
|
filter_: Literal["hasData", "nodeType", "rawFilter"] | None = None
|
|
182
176
|
in_model: bool = True
|
|
177
|
+
logical: str | None = None
|
|
183
178
|
|
|
184
179
|
@classmethod
|
|
185
180
|
def _get_verified_cls(cls) -> type[DMSView]:
|
|
@@ -189,11 +184,6 @@ class DMSInputView(InputComponent[DMSView]):
|
|
|
189
184
|
output = super().dump()
|
|
190
185
|
view = ViewEntity.load(self.view, space=default_space, version=default_version)
|
|
191
186
|
output["View"] = view
|
|
192
|
-
output["Class (linage)"] = (
|
|
193
|
-
ClassEntity.load(self.class_, prefix=default_space, version=default_version)
|
|
194
|
-
if self.class_
|
|
195
|
-
else view.as_class()
|
|
196
|
-
)
|
|
197
187
|
output["Implements"] = (
|
|
198
188
|
[
|
|
199
189
|
ViewEntity.load(implement, space=default_space, version=default_version)
|
|
@@ -207,10 +197,8 @@ class DMSInputView(InputComponent[DMSView]):
|
|
|
207
197
|
@classmethod
|
|
208
198
|
def from_view(cls, view: dm.ViewApply, in_model: bool) -> "DMSInputView":
|
|
209
199
|
view_entity = ViewEntity.from_id(view.as_id())
|
|
210
|
-
class_entity = view_entity.as_class(skip_version=True)
|
|
211
200
|
|
|
212
201
|
return cls(
|
|
213
|
-
class_=str(class_entity),
|
|
214
202
|
view=str(view_entity),
|
|
215
203
|
description=view.description,
|
|
216
204
|
name=view.name,
|
|
@@ -261,8 +249,6 @@ class DMSInputRules(InputRules[DMSRules]):
|
|
|
261
249
|
containers: list[DMSInputContainer] | None = None
|
|
262
250
|
enum: list[DMSInputEnum] | None = None
|
|
263
251
|
nodes: list[DMSInputNode] | None = None
|
|
264
|
-
last: "DMSInputRules | None" = None
|
|
265
|
-
reference: "DMSInputRules | None" = None
|
|
266
252
|
|
|
267
253
|
@classmethod
|
|
268
254
|
def _get_verified_cls(cls) -> type[DMSRules]:
|
|
@@ -271,18 +257,6 @@ class DMSInputRules(InputRules[DMSRules]):
|
|
|
271
257
|
def dump(self) -> dict[str, Any]:
|
|
272
258
|
default_space = self.metadata.space
|
|
273
259
|
default_version = str(self.metadata.version)
|
|
274
|
-
reference: dict[str, Any] | None = None
|
|
275
|
-
if isinstance(self.reference, DMSInputRules):
|
|
276
|
-
reference = self.reference.dump()
|
|
277
|
-
elif isinstance(self.reference, DMSRules):
|
|
278
|
-
# We need to load through the DMSRulesInput to set the correct default space and version
|
|
279
|
-
reference = DMSInputRules.load(self.reference.model_dump()).dump()
|
|
280
|
-
last: dict[str, Any] | None = None
|
|
281
|
-
if isinstance(self.last, DMSInputRules):
|
|
282
|
-
last = self.last.dump()
|
|
283
|
-
elif isinstance(self.last, DMSRules):
|
|
284
|
-
# We need to load through the DMSRulesInput to set the correct default space and version
|
|
285
|
-
last = DMSInputRules.load(self.last.model_dump()).dump()
|
|
286
260
|
|
|
287
261
|
return {
|
|
288
262
|
"Metadata": self.metadata.dump(),
|
|
@@ -291,8 +265,6 @@ class DMSInputRules(InputRules[DMSRules]):
|
|
|
291
265
|
"Containers": [container.dump(default_space) for container in self.containers or []] or None,
|
|
292
266
|
"Enum": [enum.dump() for enum in self.enum or []] or None,
|
|
293
267
|
"Nodes": [node_type.dump(default_space) for node_type in self.nodes or []] or None,
|
|
294
|
-
"Last": last,
|
|
295
|
-
"Reference": reference,
|
|
296
268
|
}
|
|
297
269
|
|
|
298
270
|
def _repr_html_(self) -> str:
|
|
@@ -539,6 +539,10 @@ class DMSSchema:
|
|
|
539
539
|
raise ValueError(f"Cannot sort item of type {type(item)}")
|
|
540
540
|
|
|
541
541
|
def validate(self) -> list[NeatError]:
|
|
542
|
+
# TODO: This type of validation should be done in NeatSession where all the
|
|
543
|
+
# schema components which are not part of Rules are imported and the model as
|
|
544
|
+
# the whole is validated.
|
|
545
|
+
|
|
542
546
|
errors: set[NeatError] = set()
|
|
543
547
|
defined_spaces = self.spaces.copy()
|
|
544
548
|
defined_containers = self.containers.copy()
|
|
@@ -708,6 +712,13 @@ class DMSSchema:
|
|
|
708
712
|
referenced_spaces |= {s.space for s in self.spaces.values()}
|
|
709
713
|
return referenced_spaces
|
|
710
714
|
|
|
715
|
+
def referenced_container(self) -> set[dm.ContainerId]:
|
|
716
|
+
referenced_containers = {
|
|
717
|
+
container for view in self.views.values() for container in view.referenced_containers()
|
|
718
|
+
}
|
|
719
|
+
referenced_containers |= set(self.containers.keys())
|
|
720
|
+
return referenced_containers
|
|
721
|
+
|
|
711
722
|
def as_read_model(self) -> dm.DataModel[dm.View]:
|
|
712
723
|
if self.data_model is None:
|
|
713
724
|
raise ValueError("Data model is not defined")
|
|
@@ -7,7 +7,6 @@ from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_
|
|
|
7
7
|
from cognite.neat._issues import IssueList, NeatError, NeatIssue, NeatIssueList
|
|
8
8
|
from cognite.neat._issues.errors import (
|
|
9
9
|
PropertyDefinitionDuplicatedError,
|
|
10
|
-
ResourceChangedError,
|
|
11
10
|
ResourceNotDefinedError,
|
|
12
11
|
)
|
|
13
12
|
from cognite.neat._issues.errors._properties import ReversedConnectionNotFeasibleError
|
|
@@ -20,7 +19,6 @@ from cognite.neat._issues.warnings.user_modeling import (
|
|
|
20
19
|
NotNeatSupportedFilterWarning,
|
|
21
20
|
ViewPropertyLimitWarning,
|
|
22
21
|
)
|
|
23
|
-
from cognite.neat._rules.models._base_rules import DataModelType, ExtensionCategory, SchemaCompleteness
|
|
24
22
|
from cognite.neat._rules.models.data_types import DataType
|
|
25
23
|
from cognite.neat._rules.models.entities import ContainerEntity, RawFilter
|
|
26
24
|
from cognite.neat._rules.models.entities._single_value import (
|
|
@@ -55,12 +53,7 @@ class DMSPostValidation:
|
|
|
55
53
|
self._validate_reverse_connections()
|
|
56
54
|
|
|
57
55
|
self._referenced_views_and_containers_are_existing_and_proper_size()
|
|
58
|
-
if self.metadata.schema_ is SchemaCompleteness.extended:
|
|
59
|
-
self._validate_extension()
|
|
60
|
-
if self.metadata.schema_ is SchemaCompleteness.partial:
|
|
61
|
-
return self.issue_list
|
|
62
56
|
dms_schema = self.rules.as_schema()
|
|
63
|
-
self.issue_list.extend(dms_schema.validate())
|
|
64
57
|
self._validate_performance(dms_schema)
|
|
65
58
|
return self.issue_list
|
|
66
59
|
|
|
@@ -160,9 +153,10 @@ class DMSPostValidation:
|
|
|
160
153
|
self.issue_list.extend(errors)
|
|
161
154
|
|
|
162
155
|
def _referenced_views_and_containers_are_existing_and_proper_size(self) -> None:
|
|
156
|
+
# TODO: Split this method and keep only validation that should be independent of
|
|
157
|
+
# whether view and/or container exist in the pydantic model instance
|
|
158
|
+
# other validation should be done through NeatSession.verify()
|
|
163
159
|
defined_views = {view.view.as_id() for view in self.views}
|
|
164
|
-
if self.metadata.schema_ is SchemaCompleteness.extended and self.rules.last:
|
|
165
|
-
defined_views |= {view.view.as_id() for view in self.rules.last.views}
|
|
166
160
|
|
|
167
161
|
property_count_by_view: dict[dm.ViewId, int] = defaultdict(int)
|
|
168
162
|
errors: list[NeatIssue] = []
|
|
@@ -184,103 +178,9 @@ class DMSPostValidation:
|
|
|
184
178
|
for view_id, count in property_count_by_view.items():
|
|
185
179
|
if count > DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
|
|
186
180
|
errors.append(ViewPropertyLimitWarning(view_id, count))
|
|
187
|
-
if self.metadata.schema_ is SchemaCompleteness.complete:
|
|
188
|
-
defined_containers = {container.container.as_id() for container in self.containers or []}
|
|
189
|
-
if self.metadata.data_model_type == DataModelType.solution and self.rules.reference:
|
|
190
|
-
defined_containers |= {
|
|
191
|
-
container.container.as_id() for container in self.rules.reference.containers or []
|
|
192
|
-
}
|
|
193
181
|
|
|
194
|
-
for prop_no, prop in enumerate(self.properties):
|
|
195
|
-
if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
|
|
196
|
-
errors.append(
|
|
197
|
-
ResourceNotDefinedError(
|
|
198
|
-
identifier=container_id,
|
|
199
|
-
resource_type="container",
|
|
200
|
-
location="Containers Sheet",
|
|
201
|
-
column_name="Container",
|
|
202
|
-
row_number=prop_no,
|
|
203
|
-
sheet_name="Properties",
|
|
204
|
-
)
|
|
205
|
-
)
|
|
206
|
-
for _container_no, container in enumerate(self.containers or []):
|
|
207
|
-
for constraint_no, constraint in enumerate(container.constraint or []):
|
|
208
|
-
if constraint.as_id() not in defined_containers:
|
|
209
|
-
errors.append(
|
|
210
|
-
ResourceNotDefinedError(
|
|
211
|
-
identifier=constraint.as_id(),
|
|
212
|
-
resource_type="container",
|
|
213
|
-
location="Containers Sheet",
|
|
214
|
-
column_name="Constraint",
|
|
215
|
-
row_number=constraint_no,
|
|
216
|
-
sheet_name="Properties",
|
|
217
|
-
)
|
|
218
|
-
)
|
|
219
182
|
self.issue_list.extend(errors)
|
|
220
183
|
|
|
221
|
-
def _validate_extension(self) -> None:
|
|
222
|
-
if self.metadata.schema_ is not SchemaCompleteness.extended:
|
|
223
|
-
return None
|
|
224
|
-
if not self.rules.last:
|
|
225
|
-
raise ValueError("The schema is set to 'extended', but no last rules are provided to validate against")
|
|
226
|
-
if self.metadata.extension is ExtensionCategory.rebuild:
|
|
227
|
-
# Everything is allowed
|
|
228
|
-
return None
|
|
229
|
-
user_schema = self.rules.as_schema()
|
|
230
|
-
new_containers = user_schema.containers.copy()
|
|
231
|
-
|
|
232
|
-
last_schema = self.rules.last.as_schema()
|
|
233
|
-
existing_containers = last_schema.containers.copy()
|
|
234
|
-
|
|
235
|
-
for container_id, container in new_containers.items():
|
|
236
|
-
existing_container = existing_containers.get(container_id)
|
|
237
|
-
if not existing_container or existing_container == container:
|
|
238
|
-
# No problem
|
|
239
|
-
continue
|
|
240
|
-
new_dumped = container.dump()
|
|
241
|
-
existing_dumped = existing_container.dump()
|
|
242
|
-
changed_attributes, changed_properties = self._changed_attributes_and_properties(
|
|
243
|
-
new_dumped, existing_dumped
|
|
244
|
-
)
|
|
245
|
-
self.issue_list.append(
|
|
246
|
-
ResourceChangedError(
|
|
247
|
-
container_id,
|
|
248
|
-
"container",
|
|
249
|
-
changed_properties=frozenset(changed_properties),
|
|
250
|
-
changed_attributes=frozenset(changed_attributes),
|
|
251
|
-
)
|
|
252
|
-
)
|
|
253
|
-
|
|
254
|
-
if self.metadata.extension is ExtensionCategory.reshape:
|
|
255
|
-
# Reshape allows changes to views
|
|
256
|
-
return None
|
|
257
|
-
|
|
258
|
-
new_views = user_schema.views.copy()
|
|
259
|
-
existing_views = last_schema.views.copy()
|
|
260
|
-
for view_id, view in new_views.items():
|
|
261
|
-
existing_view = existing_views.get(view_id)
|
|
262
|
-
if not existing_view or existing_view == view:
|
|
263
|
-
# No problem
|
|
264
|
-
continue
|
|
265
|
-
changed_attributes, changed_properties = self._changed_attributes_and_properties(
|
|
266
|
-
view.dump(), existing_view.dump()
|
|
267
|
-
)
|
|
268
|
-
existing_properties = existing_view.properties or {}
|
|
269
|
-
changed_properties = [prop for prop in changed_properties if prop in existing_properties]
|
|
270
|
-
changed_attributes = [attr for attr in changed_attributes if attr not in self.changeable_view_attributes]
|
|
271
|
-
|
|
272
|
-
if not changed_attributes and not changed_properties:
|
|
273
|
-
# Only added new properties, no problem
|
|
274
|
-
continue
|
|
275
|
-
self.issue_list.append(
|
|
276
|
-
ResourceChangedError(
|
|
277
|
-
view_id,
|
|
278
|
-
"view",
|
|
279
|
-
changed_properties=frozenset(changed_properties),
|
|
280
|
-
changed_attributes=frozenset(changed_attributes),
|
|
281
|
-
)
|
|
282
|
-
)
|
|
283
|
-
|
|
284
184
|
def _validate_performance(self, dms_schema: DMSSchema) -> None:
|
|
285
185
|
for view_id, view in dms_schema.views.items():
|
|
286
186
|
mapped_containers = dms_schema._get_mapped_container_from_view(view_id)
|
|
@@ -320,7 +220,7 @@ class DMSPostValidation:
|
|
|
320
220
|
UndefinedViewWarning(
|
|
321
221
|
str(prop_.view),
|
|
322
222
|
str(prop_.value_type),
|
|
323
|
-
prop_.
|
|
223
|
+
prop_.view_property,
|
|
324
224
|
)
|
|
325
225
|
)
|
|
326
226
|
|
|
@@ -329,7 +229,7 @@ class DMSPostValidation:
|
|
|
329
229
|
if self.metadata.as_data_model_id() in COGNITE_MODELS:
|
|
330
230
|
return None
|
|
331
231
|
|
|
332
|
-
properties_by_ids = {f"{prop_.view!s}.{prop_.
|
|
232
|
+
properties_by_ids = {f"{prop_.view!s}.{prop_.view_property}": prop_ for prop_ in self.properties}
|
|
333
233
|
reversed_by_ids = {
|
|
334
234
|
id_: prop_
|
|
335
235
|
for id_, prop_ in properties_by_ids.items()
|
|
@@ -339,11 +239,12 @@ class DMSPostValidation:
|
|
|
339
239
|
for id_, prop_ in reversed_by_ids.items():
|
|
340
240
|
source_id = f"{prop_.value_type!s}." f"{cast(ReverseConnectionEntity, prop_.connection).property_}"
|
|
341
241
|
if source_id not in properties_by_ids:
|
|
242
|
+
print(f"source_id: {source_id}, first issue")
|
|
342
243
|
self.issue_list.append(
|
|
343
244
|
ReversedConnectionNotFeasibleError(
|
|
344
245
|
id_,
|
|
345
246
|
"reversed connection",
|
|
346
|
-
prop_.
|
|
247
|
+
prop_.view_property,
|
|
347
248
|
str(prop_.view),
|
|
348
249
|
str(prop_.value_type),
|
|
349
250
|
cast(ReverseConnectionEntity, prop_.connection).property_,
|
|
@@ -351,11 +252,12 @@ class DMSPostValidation:
|
|
|
351
252
|
)
|
|
352
253
|
|
|
353
254
|
elif source_id in properties_by_ids and properties_by_ids[source_id].value_type != prop_.view:
|
|
255
|
+
print(f"source_id: {source_id}, second issue")
|
|
354
256
|
self.issue_list.append(
|
|
355
257
|
ReversedConnectionNotFeasibleError(
|
|
356
258
|
id_,
|
|
357
259
|
"view property",
|
|
358
|
-
prop_.
|
|
260
|
+
prop_.view_property,
|
|
359
261
|
str(prop_.view),
|
|
360
262
|
str(prop_.value_type),
|
|
361
263
|
cast(ReverseConnectionEntity, prop_.connection).property_,
|