cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/__init__.py +4 -0
- cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
- cognite/neat/_client/_api/schema.py +50 -0
- cognite/neat/_client/_api_client.py +17 -0
- cognite/neat/_client/data_classes/__init__.py +0 -0
- cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
- cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
- cognite/neat/_graph/_shared.py +14 -15
- cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
- cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
- cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
- cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
- cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
- cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
- cognite/neat/_graph/extractors/_rdf_file.py +6 -7
- cognite/neat/_graph/loaders/__init__.py +1 -2
- cognite/neat/_graph/queries/_base.py +17 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
- cognite/neat/_graph/transformers/_prune_graph.py +1 -1
- cognite/neat/_graph/transformers/_rdfpath.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -0
- cognite/neat/_issues/warnings/_external.py +8 -0
- cognite/neat/_issues/warnings/_models.py +9 -0
- cognite/neat/_issues/warnings/_properties.py +16 -0
- cognite/neat/_rules/_constants.py +7 -6
- cognite/neat/_rules/_shared.py +3 -8
- cognite/neat/_rules/analysis/__init__.py +1 -2
- cognite/neat/_rules/analysis/_base.py +10 -27
- cognite/neat/_rules/analysis/_dms.py +4 -10
- cognite/neat/_rules/analysis/_information.py +2 -10
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_base.py +3 -4
- cognite/neat/_rules/exporters/_rules2dms.py +29 -40
- cognite/neat/_rules/exporters/_rules2excel.py +15 -72
- cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/_rules/importers/_base.py +3 -4
- cognite/neat/_rules/importers/_dms2rules.py +21 -45
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
- cognite/neat/_rules/importers/_rdf/_base.py +17 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
- cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
- cognite/neat/_rules/models/__init__.py +3 -17
- cognite/neat/_rules/models/_base_rules.py +118 -62
- cognite/neat/_rules/models/dms/__init__.py +2 -2
- cognite/neat/_rules/models/dms/_exporter.py +20 -178
- cognite/neat/_rules/models/dms/_rules.py +65 -128
- cognite/neat/_rules/models/dms/_rules_input.py +72 -56
- cognite/neat/_rules/models/dms/_validation.py +16 -109
- cognite/neat/_rules/models/entities/_single_value.py +32 -4
- cognite/neat/_rules/models/information/_rules.py +19 -122
- cognite/neat/_rules/models/information/_rules_input.py +32 -41
- cognite/neat/_rules/models/information/_validation.py +34 -102
- cognite/neat/_rules/models/mapping/__init__.py +2 -3
- cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
- cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
- cognite/neat/_rules/transformers/__init__.py +3 -6
- cognite/neat/_rules/transformers/_converters.py +128 -206
- cognite/neat/_rules/transformers/_mapping.py +105 -34
- cognite/neat/_rules/transformers/_verification.py +5 -16
- cognite/neat/_session/_base.py +83 -21
- cognite/neat/_session/_collector.py +126 -0
- cognite/neat/_session/_drop.py +35 -0
- cognite/neat/_session/_inspect.py +22 -10
- cognite/neat/_session/_mapping.py +39 -0
- cognite/neat/_session/_prepare.py +222 -27
- cognite/neat/_session/_read.py +109 -19
- cognite/neat/_session/_set.py +2 -2
- cognite/neat/_session/_show.py +11 -11
- cognite/neat/_session/_to.py +27 -14
- cognite/neat/_session/exceptions.py +20 -3
- cognite/neat/_store/_base.py +27 -24
- cognite/neat/_store/_provenance.py +2 -2
- cognite/neat/_utils/auxiliary.py +19 -0
- cognite/neat/_utils/rdf_.py +28 -1
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/data_contracts.py +2 -10
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
- cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
- cognite/neat/_rules/analysis/_asset.py +0 -173
- cognite/neat/_rules/models/asset/__init__.py +0 -13
- cognite/neat/_rules/models/asset/_rules.py +0 -109
- cognite/neat/_rules/models/asset/_rules_input.py +0 -101
- cognite/neat/_rules/models/asset/_validation.py +0 -45
- cognite/neat/_rules/models/domain.py +0 -136
- cognite/neat/_rules/models/mapping/_base.py +0 -131
- cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
- cognite/neat/_utils/cdf/loaders/_base.py +0 -54
- cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
- cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
- /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import math
|
|
2
1
|
import warnings
|
|
3
2
|
from collections.abc import Hashable
|
|
4
|
-
from datetime import datetime
|
|
5
3
|
from typing import Any, ClassVar, Literal
|
|
6
4
|
|
|
7
5
|
import pandas as pd
|
|
@@ -10,31 +8,29 @@ from pydantic import Field, field_serializer, field_validator, model_validator
|
|
|
10
8
|
from pydantic_core.core_schema import SerializationInfo, ValidationInfo
|
|
11
9
|
from rdflib import URIRef
|
|
12
10
|
|
|
13
|
-
from cognite.neat.
|
|
11
|
+
from cognite.neat._client.data_classes.schema import DMSSchema
|
|
12
|
+
from cognite.neat._constants import COGNITE_SPACES
|
|
14
13
|
from cognite.neat._issues import MultiValueError
|
|
14
|
+
from cognite.neat._issues.errors import NeatValueError
|
|
15
15
|
from cognite.neat._issues.warnings import (
|
|
16
16
|
PrincipleMatchingSpaceAndVersionWarning,
|
|
17
|
-
PrincipleSolutionBuildsOnEnterpriseWarning,
|
|
18
17
|
)
|
|
19
18
|
from cognite.neat._rules.models._base_rules import (
|
|
20
19
|
BaseMetadata,
|
|
21
20
|
BaseRules,
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
ContainerProperty,
|
|
22
|
+
DataModelAspect,
|
|
24
23
|
RoleTypes,
|
|
25
|
-
SchemaCompleteness,
|
|
26
24
|
SheetList,
|
|
27
25
|
SheetRow,
|
|
26
|
+
ViewProperty,
|
|
27
|
+
ViewRef,
|
|
28
28
|
)
|
|
29
29
|
from cognite.neat._rules.models._types import (
|
|
30
30
|
ClassEntityType,
|
|
31
31
|
ContainerEntityType,
|
|
32
|
-
DataModelExternalIdType,
|
|
33
32
|
DmsPropertyType,
|
|
34
|
-
InformationPropertyType,
|
|
35
|
-
SpaceType,
|
|
36
33
|
StrListType,
|
|
37
|
-
VersionType,
|
|
38
34
|
ViewEntityType,
|
|
39
35
|
)
|
|
40
36
|
from cognite.neat._rules.models.data_types import DataType
|
|
@@ -48,68 +44,18 @@ from cognite.neat._rules.models.entities import (
|
|
|
48
44
|
HasDataFilter,
|
|
49
45
|
NodeTypeFilter,
|
|
50
46
|
RawFilter,
|
|
51
|
-
ReferenceEntity,
|
|
52
47
|
ReverseConnectionEntity,
|
|
53
|
-
URLEntity,
|
|
54
48
|
ViewEntity,
|
|
55
49
|
ViewEntityList,
|
|
56
50
|
)
|
|
57
51
|
|
|
58
|
-
from ._schema import DMSSchema
|
|
59
|
-
|
|
60
52
|
_DEFAULT_VERSION = "1"
|
|
61
53
|
|
|
62
54
|
|
|
63
55
|
class DMSMetadata(BaseMetadata):
|
|
64
56
|
role: ClassVar[RoleTypes] = RoleTypes.dms
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
extension: ExtensionCategory = ExtensionCategory.addition
|
|
68
|
-
space: SpaceType
|
|
69
|
-
name: str | None = Field(
|
|
70
|
-
None,
|
|
71
|
-
description="Human readable name of the data model",
|
|
72
|
-
min_length=1,
|
|
73
|
-
max_length=255,
|
|
74
|
-
)
|
|
75
|
-
description: str | None = Field(None, min_length=1, max_length=1024)
|
|
76
|
-
external_id: DataModelExternalIdType = Field(alias="externalId")
|
|
77
|
-
version: VersionType
|
|
78
|
-
creator: StrListType
|
|
79
|
-
created: datetime = Field(
|
|
80
|
-
description=("Date of the data model creation"),
|
|
81
|
-
)
|
|
82
|
-
updated: datetime = Field(
|
|
83
|
-
description=("Date of the data model update"),
|
|
84
|
-
)
|
|
85
|
-
|
|
86
|
-
@field_validator("*", mode="before")
|
|
87
|
-
def strip_string(cls, value: Any) -> Any:
|
|
88
|
-
if isinstance(value, str):
|
|
89
|
-
return value.strip()
|
|
90
|
-
return value
|
|
91
|
-
|
|
92
|
-
@field_serializer("schema_", "extension", "data_model_type", when_used="always")
|
|
93
|
-
def as_string(self, value: SchemaCompleteness | ExtensionCategory | DataModelType) -> str:
|
|
94
|
-
return str(value)
|
|
95
|
-
|
|
96
|
-
@field_validator("schema_", mode="plain")
|
|
97
|
-
def as_enum_schema(cls, value: str) -> SchemaCompleteness:
|
|
98
|
-
return SchemaCompleteness(value.strip())
|
|
99
|
-
|
|
100
|
-
@field_validator("extension", mode="plain")
|
|
101
|
-
def as_enum_extension(cls, value: str) -> ExtensionCategory:
|
|
102
|
-
return ExtensionCategory(value.strip())
|
|
103
|
-
|
|
104
|
-
@field_validator("data_model_type", mode="plain")
|
|
105
|
-
def as_enum_model_type(cls, value: str) -> DataModelType:
|
|
106
|
-
return DataModelType(value.strip())
|
|
107
|
-
|
|
108
|
-
@field_validator("description", mode="before")
|
|
109
|
-
def nan_as_none(cls, value):
|
|
110
|
-
if isinstance(value, float) and math.isnan(value):
|
|
111
|
-
return None
|
|
112
|
-
return value
|
|
57
|
+
aspect: ClassVar[DataModelAspect] = DataModelAspect.physical
|
|
58
|
+
logical: str | None = None
|
|
113
59
|
|
|
114
60
|
def as_space(self) -> dm.SpaceApply:
|
|
115
61
|
return dm.SpaceApply(
|
|
@@ -159,13 +105,15 @@ class DMSProperty(SheetRow):
|
|
|
159
105
|
immutable: bool | None = Field(default=None, alias="Immutable")
|
|
160
106
|
is_list: bool | None = Field(default=None, alias="Is List")
|
|
161
107
|
default: str | int | dict | None = Field(None, alias="Default")
|
|
162
|
-
reference: URLEntity | ReferenceEntity | None = Field(default=None, alias="Reference", union_mode="left_to_right")
|
|
163
108
|
container: ContainerEntityType | None = Field(None, alias="Container")
|
|
164
109
|
container_property: DmsPropertyType | None = Field(None, alias="Container Property")
|
|
165
110
|
index: StrListType | None = Field(None, alias="Index")
|
|
166
111
|
constraint: StrListType | None = Field(None, alias="Constraint")
|
|
167
|
-
|
|
168
|
-
|
|
112
|
+
logical: URIRef | None = Field(
|
|
113
|
+
None,
|
|
114
|
+
alias="Logical",
|
|
115
|
+
description="Used to make connection between physical and logical data model aspect",
|
|
116
|
+
)
|
|
169
117
|
|
|
170
118
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
171
119
|
return self.view, self.view_property
|
|
@@ -208,19 +156,6 @@ class DMSProperty(SheetRow):
|
|
|
208
156
|
)
|
|
209
157
|
return value
|
|
210
158
|
|
|
211
|
-
@field_serializer("reference", when_used="always")
|
|
212
|
-
def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
|
|
213
|
-
if isinstance(info.context, dict) and info.context.get("as_reference") is True:
|
|
214
|
-
return str(
|
|
215
|
-
ReferenceEntity(
|
|
216
|
-
prefix=self.view.prefix,
|
|
217
|
-
suffix=self.view.suffix,
|
|
218
|
-
version=self.view.version,
|
|
219
|
-
property=self.view_property,
|
|
220
|
-
)
|
|
221
|
-
)
|
|
222
|
-
return str(value) if value is not None else None
|
|
223
|
-
|
|
224
159
|
@field_serializer("value_type", when_used="always")
|
|
225
160
|
def as_dms_type(self, value_type: DataType | EdgeEntity | ViewEntity, info: SerializationInfo) -> str:
|
|
226
161
|
if isinstance(value_type, DataType):
|
|
@@ -229,7 +164,7 @@ class DMSProperty(SheetRow):
|
|
|
229
164
|
return value_type.dump(space=metadata.space, version=metadata.version)
|
|
230
165
|
return str(value_type)
|
|
231
166
|
|
|
232
|
-
@field_serializer("view", "container",
|
|
167
|
+
@field_serializer("view", "container", when_used="unless-none")
|
|
233
168
|
def remove_default_space(self, value: str, info: SerializationInfo) -> str:
|
|
234
169
|
if (metadata := _metadata(info.context)) and isinstance(value, Entity):
|
|
235
170
|
if info.field_name == "container" and info.context.get("as_reference") is True:
|
|
@@ -248,15 +183,21 @@ class DMSProperty(SheetRow):
|
|
|
248
183
|
return value.dump(space=metadata.space, version=metadata.version, type=default_type)
|
|
249
184
|
return str(value)
|
|
250
185
|
|
|
186
|
+
def as_container_reference(self) -> ContainerProperty:
|
|
187
|
+
if self.container is None or self.container_property is None:
|
|
188
|
+
raise NeatValueError("Accessing container reference without container and container property set")
|
|
189
|
+
return ContainerProperty(container=self.container, property_=self.container_property)
|
|
190
|
+
|
|
191
|
+
def as_view_reference(self) -> ViewProperty:
|
|
192
|
+
return ViewProperty(view=self.view, property_=self.view_property)
|
|
193
|
+
|
|
251
194
|
|
|
252
195
|
class DMSContainer(SheetRow):
|
|
253
196
|
container: ContainerEntityType = Field(alias="Container")
|
|
254
197
|
name: str | None = Field(alias="Name", default=None)
|
|
255
198
|
description: str | None = Field(alias="Description", default=None)
|
|
256
|
-
reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
|
|
257
199
|
constraint: ContainerEntityList | None = Field(None, alias="Constraint")
|
|
258
200
|
used_for: Literal["node", "edge", "all"] | None = Field("all", alias="Used For")
|
|
259
|
-
class_: ClassEntityType = Field(alias="Class (linage)")
|
|
260
201
|
|
|
261
202
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
262
203
|
return (self.container,)
|
|
@@ -278,13 +219,7 @@ class DMSContainer(SheetRow):
|
|
|
278
219
|
used_for=self.used_for,
|
|
279
220
|
)
|
|
280
221
|
|
|
281
|
-
@field_serializer("
|
|
282
|
-
def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
|
|
283
|
-
if isinstance(info.context, dict) and info.context.get("as_reference") is True:
|
|
284
|
-
return self.container.dump()
|
|
285
|
-
return str(value) if value is not None else None
|
|
286
|
-
|
|
287
|
-
@field_serializer("container", "class_", when_used="unless-none")
|
|
222
|
+
@field_serializer("container", when_used="unless-none")
|
|
288
223
|
def remove_default_space(self, value: Any, info: SerializationInfo) -> str:
|
|
289
224
|
if metadata := _metadata(info.context):
|
|
290
225
|
if isinstance(value, DMSEntity):
|
|
@@ -310,21 +245,18 @@ class DMSView(SheetRow):
|
|
|
310
245
|
name: str | None = Field(alias="Name", default=None)
|
|
311
246
|
description: str | None = Field(alias="Description", default=None)
|
|
312
247
|
implements: ViewEntityList | None = Field(None, alias="Implements")
|
|
313
|
-
reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
|
|
314
248
|
filter_: HasDataFilter | NodeTypeFilter | RawFilter | None = Field(None, alias="Filter")
|
|
315
249
|
in_model: bool = Field(True, alias="In Model")
|
|
316
|
-
|
|
250
|
+
logical: URIRef | None = Field(
|
|
251
|
+
None,
|
|
252
|
+
alias="Logical",
|
|
253
|
+
description="Used to make connection between physical and logical data model aspect",
|
|
254
|
+
)
|
|
317
255
|
|
|
318
256
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
319
257
|
return (self.view,)
|
|
320
258
|
|
|
321
|
-
@field_serializer("
|
|
322
|
-
def set_reference(self, value: Any, info: SerializationInfo) -> str | None:
|
|
323
|
-
if isinstance(info.context, dict) and info.context.get("as_reference") is True:
|
|
324
|
-
return self.view.dump()
|
|
325
|
-
return str(value) if value is not None else None
|
|
326
|
-
|
|
327
|
-
@field_serializer("view", "class_", when_used="unless-none")
|
|
259
|
+
@field_serializer("view", when_used="unless-none")
|
|
328
260
|
def remove_default_space(self, value: Any, info: SerializationInfo) -> str:
|
|
329
261
|
if (metadata := _metadata(info.context)) and isinstance(value, Entity):
|
|
330
262
|
return value.dump(prefix=metadata.space, version=metadata.version)
|
|
@@ -344,11 +276,6 @@ class DMSView(SheetRow):
|
|
|
344
276
|
def as_view(self) -> dm.ViewApply:
|
|
345
277
|
view_id = self.view.as_id()
|
|
346
278
|
implements = [parent.as_id() for parent in self.implements or []] or None
|
|
347
|
-
if implements is None and isinstance(self.reference, ReferenceEntity):
|
|
348
|
-
# Fallback to the reference if no implements are provided
|
|
349
|
-
parent = self.reference.as_view_id()
|
|
350
|
-
if (parent.space, parent.external_id) != (view_id.space, view_id.external_id):
|
|
351
|
-
implements = [parent]
|
|
352
279
|
|
|
353
280
|
return dm.ViewApply(
|
|
354
281
|
space=view_id.space,
|
|
@@ -356,10 +283,14 @@ class DMSView(SheetRow):
|
|
|
356
283
|
version=view_id.version or _DEFAULT_VERSION,
|
|
357
284
|
name=self.name or None,
|
|
358
285
|
description=self.description,
|
|
286
|
+
filter=None if self.filter_ is None else self.filter_.as_dms_filter(),
|
|
359
287
|
implements=implements,
|
|
360
288
|
properties={},
|
|
361
289
|
)
|
|
362
290
|
|
|
291
|
+
def as_view_reference(self) -> ViewRef:
|
|
292
|
+
return ViewRef(view=self.view)
|
|
293
|
+
|
|
363
294
|
|
|
364
295
|
class DMSNode(SheetRow):
|
|
365
296
|
node: DMSNodeEntity = Field(alias="Node")
|
|
@@ -408,24 +339,9 @@ class DMSRules(BaseRules):
|
|
|
408
339
|
containers: SheetList[DMSContainer] | None = Field(None, alias="Containers")
|
|
409
340
|
enum: SheetList[DMSEnum] | None = Field(None, alias="Enum")
|
|
410
341
|
nodes: SheetList[DMSNode] | None = Field(None, alias="Nodes")
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
@field_validator("reference")
|
|
415
|
-
def check_reference_of_reference(cls, value: "DMSRules | None", info: ValidationInfo) -> "DMSRules | None":
|
|
416
|
-
if value is None:
|
|
417
|
-
return None
|
|
418
|
-
if value.reference is not None:
|
|
419
|
-
raise ValueError("Reference rules cannot have a reference")
|
|
420
|
-
if value.metadata.data_model_type == DataModelType.solution and (metadata := info.data.get("metadata")):
|
|
421
|
-
warnings.warn(
|
|
422
|
-
PrincipleSolutionBuildsOnEnterpriseWarning(
|
|
423
|
-
f"The solution model {metadata.as_data_model_id()} is referencing another "
|
|
424
|
-
f"solution model {value.metadata.as_data_model_id()}",
|
|
425
|
-
),
|
|
426
|
-
stacklevel=2,
|
|
427
|
-
)
|
|
428
|
-
return value
|
|
342
|
+
# This is a hack to allow the post_validation to be turned off when needed
|
|
343
|
+
# Will likely be moved completely out of the rules in the future
|
|
344
|
+
post_validate: bool = Field(default=True, exclude=True, repr=False)
|
|
429
345
|
|
|
430
346
|
@field_validator("views")
|
|
431
347
|
def matching_version_and_space(cls, value: SheetList[DMSView], info: ValidationInfo) -> SheetList[DMSView]:
|
|
@@ -462,6 +378,8 @@ class DMSRules(BaseRules):
|
|
|
462
378
|
def post_validation(self) -> "DMSRules":
|
|
463
379
|
from ._validation import DMSPostValidation
|
|
464
380
|
|
|
381
|
+
if not self.post_validate:
|
|
382
|
+
return self
|
|
465
383
|
issue_list = DMSPostValidation(self).validate()
|
|
466
384
|
if issue_list.warnings:
|
|
467
385
|
issue_list.trigger_warnings()
|
|
@@ -472,11 +390,11 @@ class DMSRules(BaseRules):
|
|
|
472
390
|
def as_schema(self, include_pipeline: bool = False, instance_space: str | None = None) -> DMSSchema:
|
|
473
391
|
from ._exporter import _DMSExporter
|
|
474
392
|
|
|
475
|
-
return _DMSExporter(self,
|
|
393
|
+
return _DMSExporter(self, instance_space).to_schema()
|
|
476
394
|
|
|
477
395
|
def _repr_html_(self) -> str:
|
|
478
396
|
summary = {
|
|
479
|
-
"
|
|
397
|
+
"aspect": self.metadata.aspect,
|
|
480
398
|
"intended for": "DMS Architect",
|
|
481
399
|
"name": self.metadata.name,
|
|
482
400
|
"space": self.metadata.space,
|
|
@@ -489,8 +407,27 @@ class DMSRules(BaseRules):
|
|
|
489
407
|
|
|
490
408
|
return pd.DataFrame([summary]).T.rename(columns={0: ""})._repr_html_() # type: ignore
|
|
491
409
|
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
]
|
|
410
|
+
def imported_views_and_containers_ids(
|
|
411
|
+
self, include_model_views_with_no_properties: bool = True
|
|
412
|
+
) -> tuple[set[dm.ViewId], set[dm.ContainerId]]:
|
|
413
|
+
existing_views = {view.view for view in self.views}
|
|
414
|
+
imported_views: set[dm.ViewId] = set()
|
|
415
|
+
for view in self.views:
|
|
416
|
+
for parent in view.implements or []:
|
|
417
|
+
if parent not in existing_views:
|
|
418
|
+
imported_views.add(parent.as_id())
|
|
419
|
+
existing_containers = {container.container for container in self.containers or []}
|
|
420
|
+
imported_containers: set[dm.ContainerId] = set()
|
|
421
|
+
view_with_properties: set[ViewEntity] = set()
|
|
422
|
+
for prop in self.properties:
|
|
423
|
+
if prop.container and prop.container not in existing_containers:
|
|
424
|
+
imported_containers.add(prop.container.as_id())
|
|
425
|
+
if prop.view not in existing_views:
|
|
426
|
+
imported_views.add(prop.view.as_id())
|
|
427
|
+
view_with_properties.add(prop.view)
|
|
428
|
+
|
|
429
|
+
if include_model_views_with_no_properties:
|
|
430
|
+
extra_views = existing_views - view_with_properties
|
|
431
|
+
imported_views.update({view.as_id() for view in extra_views})
|
|
432
|
+
|
|
433
|
+
return imported_views, imported_containers
|
|
@@ -5,13 +5,13 @@ from typing import Any, Literal
|
|
|
5
5
|
|
|
6
6
|
import pandas as pd
|
|
7
7
|
from cognite.client import data_modeling as dm
|
|
8
|
-
from
|
|
8
|
+
from cognite.client.data_classes.data_modeling import ContainerId, ViewId
|
|
9
|
+
from rdflib import Namespace, URIRef
|
|
9
10
|
|
|
10
11
|
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
11
12
|
from cognite.neat._rules.models._base_input import InputComponent, InputRules
|
|
12
13
|
from cognite.neat._rules.models.data_types import DataType
|
|
13
14
|
from cognite.neat._rules.models.entities import (
|
|
14
|
-
ClassEntity,
|
|
15
15
|
ContainerEntity,
|
|
16
16
|
DMSNodeEntity,
|
|
17
17
|
DMSUnknownEntity,
|
|
@@ -27,17 +27,15 @@ from ._rules import _DEFAULT_VERSION, DMSContainer, DMSEnum, DMSMetadata, DMSNod
|
|
|
27
27
|
|
|
28
28
|
@dataclass
|
|
29
29
|
class DMSInputMetadata(InputComponent[DMSMetadata]):
|
|
30
|
-
schema_: Literal["complete", "partial", "extended"]
|
|
31
30
|
space: str
|
|
32
31
|
external_id: str
|
|
33
32
|
creator: str
|
|
34
33
|
version: str
|
|
35
|
-
extension: Literal["addition", "reshape", "rebuild"] = "addition"
|
|
36
|
-
data_model_type: Literal["solution", "enterprise"] = "solution"
|
|
37
34
|
name: str | None = None
|
|
38
35
|
description: str | None = None
|
|
39
36
|
created: datetime | str | None = None
|
|
40
37
|
updated: datetime | str | None = None
|
|
38
|
+
logical: str | None = None
|
|
41
39
|
|
|
42
40
|
@classmethod
|
|
43
41
|
def _get_verified_cls(cls) -> type[DMSMetadata]:
|
|
@@ -52,11 +50,9 @@ class DMSInputMetadata(InputComponent[DMSMetadata]):
|
|
|
52
50
|
return output
|
|
53
51
|
|
|
54
52
|
@classmethod
|
|
55
|
-
def from_data_model(cls, data_model: dm.DataModelApply
|
|
53
|
+
def from_data_model(cls, data_model: dm.DataModelApply) -> "DMSInputMetadata":
|
|
56
54
|
description, creator = cls._get_description_and_creator(data_model.description)
|
|
57
55
|
return cls(
|
|
58
|
-
schema_="complete",
|
|
59
|
-
data_model_type="solution" if has_reference else "enterprise",
|
|
60
56
|
space=data_model.space,
|
|
61
57
|
name=data_model.name or None,
|
|
62
58
|
description=description,
|
|
@@ -80,14 +76,27 @@ class DMSInputMetadata(InputComponent[DMSMetadata]):
|
|
|
80
76
|
description = None
|
|
81
77
|
return description, creator
|
|
82
78
|
|
|
79
|
+
@property
|
|
80
|
+
def identifier(self) -> URIRef:
|
|
81
|
+
"""Globally unique identifier for the data model.
|
|
82
|
+
|
|
83
|
+
!!! note
|
|
84
|
+
Unlike namespace, the identifier does not end with "/" or "#".
|
|
85
|
+
|
|
86
|
+
"""
|
|
87
|
+
return DEFAULT_NAMESPACE[f"data-model/unverified/physical/{self.space}/{self.external_id}/{self.version}"]
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def namespace(self) -> Namespace:
|
|
91
|
+
"""Namespace for the data model used for the entities in the data model."""
|
|
92
|
+
return Namespace(f"{self.identifier}/")
|
|
93
|
+
|
|
83
94
|
|
|
84
95
|
@dataclass
|
|
85
96
|
class DMSInputProperty(InputComponent[DMSProperty]):
|
|
86
97
|
view: str
|
|
87
98
|
view_property: str | None
|
|
88
99
|
value_type: str | DataType | ViewEntity | DMSUnknownEntity
|
|
89
|
-
property_: str | None = None
|
|
90
|
-
class_: str | None = None
|
|
91
100
|
name: str | None = None
|
|
92
101
|
description: str | None = None
|
|
93
102
|
connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | str | None = None
|
|
@@ -95,11 +104,11 @@ class DMSInputProperty(InputComponent[DMSProperty]):
|
|
|
95
104
|
immutable: bool | None = None
|
|
96
105
|
is_list: bool | None = None
|
|
97
106
|
default: str | int | dict | None = None
|
|
98
|
-
reference: str | None = None
|
|
99
107
|
container: str | None = None
|
|
100
108
|
container_property: str | None = None
|
|
101
109
|
index: str | list[str] | None = None
|
|
102
110
|
constraint: str | list[str] | None = None
|
|
111
|
+
logical: str | None = None
|
|
103
112
|
|
|
104
113
|
@classmethod
|
|
105
114
|
def _get_verified_cls(cls) -> type[DMSProperty]:
|
|
@@ -110,12 +119,6 @@ class DMSInputProperty(InputComponent[DMSProperty]):
|
|
|
110
119
|
output["View"] = ViewEntity.load(self.view, space=default_space, version=default_version)
|
|
111
120
|
output["Value Type"] = load_dms_value_type(self.value_type, default_space, default_version)
|
|
112
121
|
output["Connection"] = load_connection(self.connection, default_space, default_version)
|
|
113
|
-
output["Property (linage)"] = self.property_ or self.view_property
|
|
114
|
-
output["Class (linage)"] = (
|
|
115
|
-
ClassEntity.load(self.class_ or self.view, prefix=default_space, version=default_version)
|
|
116
|
-
if self.class_ or self.view
|
|
117
|
-
else None
|
|
118
|
-
)
|
|
119
122
|
output["Container"] = (
|
|
120
123
|
ContainerEntity.load(self.container, space=default_space, version=default_version)
|
|
121
124
|
if self.container
|
|
@@ -123,14 +126,18 @@ class DMSInputProperty(InputComponent[DMSProperty]):
|
|
|
123
126
|
)
|
|
124
127
|
return output
|
|
125
128
|
|
|
129
|
+
def referenced_view(self, default_space: str, default_version: str) -> ViewEntity:
|
|
130
|
+
return ViewEntity.load(self.view, strict=True, space=default_space, version=default_version)
|
|
131
|
+
|
|
132
|
+
def referenced_container(self, default_space: str) -> ContainerEntity | None:
|
|
133
|
+
return ContainerEntity.load(self.container, strict=True, space=default_space) if self.container else None
|
|
134
|
+
|
|
126
135
|
|
|
127
136
|
@dataclass
|
|
128
137
|
class DMSInputContainer(InputComponent[DMSContainer]):
|
|
129
138
|
container: str
|
|
130
|
-
class_: str | None = None
|
|
131
139
|
name: str | None = None
|
|
132
140
|
description: str | None = None
|
|
133
|
-
reference: str | None = None
|
|
134
141
|
constraint: str | None = None
|
|
135
142
|
used_for: Literal["node", "edge", "all"] | None = None
|
|
136
143
|
|
|
@@ -140,11 +147,7 @@ class DMSInputContainer(InputComponent[DMSContainer]):
|
|
|
140
147
|
|
|
141
148
|
def dump(self, default_space: str) -> dict[str, Any]: # type: ignore[override]
|
|
142
149
|
output = super().dump()
|
|
143
|
-
|
|
144
|
-
output["Container"] = container
|
|
145
|
-
output["Class (linage)"] = (
|
|
146
|
-
ClassEntity.load(self.class_, prefix=default_space) if self.class_ else container.as_class()
|
|
147
|
-
)
|
|
150
|
+
output["Container"] = self.as_entity_id(default_space)
|
|
148
151
|
output["Constraint"] = (
|
|
149
152
|
[ContainerEntity.load(constraint.strip(), space=default_space) for constraint in self.constraint.split(",")]
|
|
150
153
|
if self.constraint
|
|
@@ -152,6 +155,9 @@ class DMSInputContainer(InputComponent[DMSContainer]):
|
|
|
152
155
|
)
|
|
153
156
|
return output
|
|
154
157
|
|
|
158
|
+
def as_entity_id(self, default_space: str) -> ContainerEntity:
|
|
159
|
+
return ContainerEntity.load(self.container, strict=True, space=default_space)
|
|
160
|
+
|
|
155
161
|
@classmethod
|
|
156
162
|
def from_container(cls, container: dm.ContainerApply) -> "DMSInputContainer":
|
|
157
163
|
constraints: list[str] = []
|
|
@@ -161,7 +167,6 @@ class DMSInputContainer(InputComponent[DMSContainer]):
|
|
|
161
167
|
# UniquenessConstraint it handled in the properties
|
|
162
168
|
container_entity = ContainerEntity.from_id(container.as_id())
|
|
163
169
|
return cls(
|
|
164
|
-
class_=str(container_entity.as_class()),
|
|
165
170
|
container=str(container_entity),
|
|
166
171
|
name=container.name or None,
|
|
167
172
|
description=container.description,
|
|
@@ -173,13 +178,12 @@ class DMSInputContainer(InputComponent[DMSContainer]):
|
|
|
173
178
|
@dataclass
|
|
174
179
|
class DMSInputView(InputComponent[DMSView]):
|
|
175
180
|
view: str
|
|
176
|
-
class_: str | None = None
|
|
177
181
|
name: str | None = None
|
|
178
182
|
description: str | None = None
|
|
179
183
|
implements: str | None = None
|
|
180
|
-
|
|
181
|
-
filter_: Literal["hasData", "nodeType", "rawFilter"] | None = None
|
|
184
|
+
filter_: Literal["hasData", "nodeType", "rawFilter"] | str | None = None
|
|
182
185
|
in_model: bool = True
|
|
186
|
+
logical: str | None = None
|
|
183
187
|
|
|
184
188
|
@classmethod
|
|
185
189
|
def _get_verified_cls(cls) -> type[DMSView]:
|
|
@@ -187,30 +191,31 @@ class DMSInputView(InputComponent[DMSView]):
|
|
|
187
191
|
|
|
188
192
|
def dump(self, default_space: str, default_version: str) -> dict[str, Any]: # type: ignore[override]
|
|
189
193
|
output = super().dump()
|
|
190
|
-
|
|
191
|
-
output["
|
|
192
|
-
output
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
194
|
+
output["View"] = self.as_entity_id(default_space, default_version)
|
|
195
|
+
output["Implements"] = self._load_implements(default_space, default_version)
|
|
196
|
+
return output
|
|
197
|
+
|
|
198
|
+
def as_entity_id(self, default_space: str, default_version: str) -> ViewEntity:
|
|
199
|
+
return ViewEntity.load(self.view, strict=True, space=default_space, version=default_version)
|
|
200
|
+
|
|
201
|
+
def _load_implements(self, default_space: str, default_version: str) -> list[ViewEntity] | None:
|
|
202
|
+
return (
|
|
198
203
|
[
|
|
199
|
-
ViewEntity.load(implement, space=default_space, version=default_version)
|
|
204
|
+
ViewEntity.load(implement, strict=True, space=default_space, version=default_version)
|
|
200
205
|
for implement in self.implements.split(",")
|
|
201
206
|
]
|
|
202
207
|
if self.implements
|
|
203
208
|
else None
|
|
204
209
|
)
|
|
205
|
-
|
|
210
|
+
|
|
211
|
+
def referenced_views(self, default_space: str, default_version: str) -> list[ViewEntity]:
|
|
212
|
+
return self._load_implements(default_space, default_version) or []
|
|
206
213
|
|
|
207
214
|
@classmethod
|
|
208
215
|
def from_view(cls, view: dm.ViewApply, in_model: bool) -> "DMSInputView":
|
|
209
216
|
view_entity = ViewEntity.from_id(view.as_id())
|
|
210
|
-
class_entity = view_entity.as_class(skip_version=True)
|
|
211
217
|
|
|
212
218
|
return cls(
|
|
213
|
-
class_=str(class_entity),
|
|
214
219
|
view=str(view_entity),
|
|
215
220
|
description=view.description,
|
|
216
221
|
name=view.name,
|
|
@@ -261,8 +266,6 @@ class DMSInputRules(InputRules[DMSRules]):
|
|
|
261
266
|
containers: list[DMSInputContainer] | None = None
|
|
262
267
|
enum: list[DMSInputEnum] | None = None
|
|
263
268
|
nodes: list[DMSInputNode] | None = None
|
|
264
|
-
last: "DMSInputRules | None" = None
|
|
265
|
-
reference: "DMSInputRules | None" = None
|
|
266
269
|
|
|
267
270
|
@classmethod
|
|
268
271
|
def _get_verified_cls(cls) -> type[DMSRules]:
|
|
@@ -271,18 +274,6 @@ class DMSInputRules(InputRules[DMSRules]):
|
|
|
271
274
|
def dump(self) -> dict[str, Any]:
|
|
272
275
|
default_space = self.metadata.space
|
|
273
276
|
default_version = str(self.metadata.version)
|
|
274
|
-
reference: dict[str, Any] | None = None
|
|
275
|
-
if isinstance(self.reference, DMSInputRules):
|
|
276
|
-
reference = self.reference.dump()
|
|
277
|
-
elif isinstance(self.reference, DMSRules):
|
|
278
|
-
# We need to load through the DMSRulesInput to set the correct default space and version
|
|
279
|
-
reference = DMSInputRules.load(self.reference.model_dump()).dump()
|
|
280
|
-
last: dict[str, Any] | None = None
|
|
281
|
-
if isinstance(self.last, DMSInputRules):
|
|
282
|
-
last = self.last.dump()
|
|
283
|
-
elif isinstance(self.last, DMSRules):
|
|
284
|
-
# We need to load through the DMSRulesInput to set the correct default space and version
|
|
285
|
-
last = DMSInputRules.load(self.last.model_dump()).dump()
|
|
286
277
|
|
|
287
278
|
return {
|
|
288
279
|
"Metadata": self.metadata.dump(),
|
|
@@ -291,8 +282,6 @@ class DMSInputRules(InputRules[DMSRules]):
|
|
|
291
282
|
"Containers": [container.dump(default_space) for container in self.containers or []] or None,
|
|
292
283
|
"Enum": [enum.dump() for enum in self.enum or []] or None,
|
|
293
284
|
"Nodes": [node_type.dump(default_space) for node_type in self.nodes or []] or None,
|
|
294
|
-
"Last": last,
|
|
295
|
-
"Reference": reference,
|
|
296
285
|
}
|
|
297
286
|
|
|
298
287
|
def _repr_html_(self) -> str:
|
|
@@ -315,3 +304,30 @@ class DMSInputRules(InputRules[DMSRules]):
|
|
|
315
304
|
return DEFAULT_NAMESPACE[
|
|
316
305
|
f"data-model/unverified/dms/{self.metadata.space}/{self.metadata.external_id}/{self.metadata.version}"
|
|
317
306
|
]
|
|
307
|
+
|
|
308
|
+
def referenced_views_and_containers(self) -> tuple[set[ViewEntity], set[ContainerEntity]]:
|
|
309
|
+
default_space = self.metadata.space
|
|
310
|
+
default_version = self.metadata.version
|
|
311
|
+
|
|
312
|
+
containers: set[ContainerEntity] = set()
|
|
313
|
+
views = {parent for view in self.views for parent in view.referenced_views(default_space, default_version)}
|
|
314
|
+
for prop in self.properties:
|
|
315
|
+
views.add(prop.referenced_view(default_space, default_version))
|
|
316
|
+
if ref_container := prop.referenced_container(default_space):
|
|
317
|
+
containers.add(ref_container)
|
|
318
|
+
|
|
319
|
+
return views, containers
|
|
320
|
+
|
|
321
|
+
def as_view_entities(self) -> list[ViewEntity]:
|
|
322
|
+
return [view.as_entity_id(self.metadata.space, self.metadata.version) for view in self.views]
|
|
323
|
+
|
|
324
|
+
def as_container_entities(self) -> list[ContainerEntity]:
|
|
325
|
+
return [container.as_entity_id(self.metadata.space) for container in self.containers or []]
|
|
326
|
+
|
|
327
|
+
def imported_views_and_containers(self) -> tuple[set[ViewEntity], set[ContainerEntity]]:
|
|
328
|
+
views, containers = self.referenced_views_and_containers()
|
|
329
|
+
return views - set(self.as_view_entities()), containers - set(self.as_container_entities())
|
|
330
|
+
|
|
331
|
+
def imported_views_and_containers_ids(self) -> tuple[set[ViewId], set[ContainerId]]:
|
|
332
|
+
views, containers = self.imported_views_and_containers()
|
|
333
|
+
return {view.as_id() for view in views}, {container.as_id() for container in containers}
|