cognite-neat 0.76.1__py3-none-any.whl → 0.76.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/routers/core.py +1 -1
- cognite/neat/app/api/routers/rules.py +1 -1
- cognite/neat/graph/extractors/_mock_graph_generator.py +2 -2
- cognite/neat/rules/_shared.py +1 -1
- cognite/neat/rules/analysis/_information_rules.py +3 -3
- cognite/neat/rules/exporters/_base.py +1 -1
- cognite/neat/rules/exporters/_rules2dms.py +8 -49
- cognite/neat/rules/exporters/_rules2excel.py +71 -40
- cognite/neat/rules/exporters/_rules2ontology.py +2 -2
- cognite/neat/rules/exporters/_rules2yaml.py +1 -1
- cognite/neat/rules/exporters/_validation.py +2 -2
- cognite/neat/rules/importers/_base.py +1 -1
- cognite/neat/rules/importers/_dms2rules.py +93 -108
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +1 -1
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +2 -3
- cognite/neat/rules/importers/_owl2rules/_owl2classes.py +1 -1
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +2 -2
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +1 -1
- cognite/neat/rules/importers/_owl2rules/_owl2rules.py +1 -1
- cognite/neat/rules/importers/_spreadsheet2rules.py +87 -62
- cognite/neat/rules/importers/_yaml2rules.py +3 -3
- cognite/neat/rules/issues/base.py +5 -0
- cognite/neat/rules/issues/dms.py +65 -0
- cognite/neat/rules/models/__init__.py +27 -0
- cognite/neat/rules/models/dms/__init__.py +18 -0
- cognite/neat/rules/models/dms/_converter.py +140 -0
- cognite/neat/rules/models/dms/_exporter.py +405 -0
- cognite/neat/rules/models/dms/_rules.py +379 -0
- cognite/neat/rules/models/{rules/_dms_rules_write.py → dms/_rules_input.py} +42 -33
- cognite/neat/rules/models/{rules/_dms_schema.py → dms/_schema.py} +36 -4
- cognite/neat/rules/models/dms/_serializer.py +126 -0
- cognite/neat/rules/models/dms/_validation.py +288 -0
- cognite/neat/rules/models/{rules/_domain_rules.py → domain.py} +1 -0
- cognite/neat/rules/models/information/__init__.py +3 -0
- cognite/neat/rules/models/information/_converter.py +195 -0
- cognite/neat/rules/models/{rules/_information_rules.py → information/_rules.py} +35 -202
- cognite/neat/workflows/steps/data_contracts.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +10 -3
- cognite/neat/workflows/steps/lib/current/rules_importer.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_validator.py +1 -2
- {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/METADATA +1 -1
- {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/RECORD +51 -44
- cognite/neat/rules/models/rules/__init__.py +0 -14
- cognite/neat/rules/models/rules/_dms_architect_rules.py +0 -1255
- /cognite/neat/rules/models/{rules/_base.py → _base.py} +0 -0
- /cognite/neat/rules/models/{rdfpath.py → _rdfpath.py} +0 -0
- /cognite/neat/rules/models/{rules/_types → _types}/__init__.py +0 -0
- /cognite/neat/rules/models/{rules/_types → _types}/_base.py +0 -0
- /cognite/neat/rules/models/{rules/_types → _types}/_field.py +0 -0
- {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/LICENSE +0 -0
- {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/WHEEL +0 -0
- {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/entry_points.txt +0 -0
|
@@ -1,1255 +0,0 @@
|
|
|
1
|
-
import abc
|
|
2
|
-
import math
|
|
3
|
-
import re
|
|
4
|
-
import sys
|
|
5
|
-
import warnings
|
|
6
|
-
from collections import defaultdict
|
|
7
|
-
from collections.abc import Callable
|
|
8
|
-
from datetime import datetime
|
|
9
|
-
from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
|
|
10
|
-
|
|
11
|
-
from cognite.client import data_modeling as dm
|
|
12
|
-
from cognite.client.data_classes.data_modeling import PropertyType as CognitePropertyType
|
|
13
|
-
from cognite.client.data_classes.data_modeling.containers import BTreeIndex
|
|
14
|
-
from cognite.client.data_classes.data_modeling.views import (
|
|
15
|
-
SingleEdgeConnectionApply,
|
|
16
|
-
SingleReverseDirectRelationApply,
|
|
17
|
-
ViewPropertyApply,
|
|
18
|
-
)
|
|
19
|
-
from pydantic import Field, field_serializer, field_validator, model_serializer, model_validator
|
|
20
|
-
from pydantic_core.core_schema import SerializationInfo, ValidationInfo
|
|
21
|
-
from rdflib import Namespace
|
|
22
|
-
|
|
23
|
-
import cognite.neat.rules.issues.spreadsheet
|
|
24
|
-
from cognite.neat.rules import issues
|
|
25
|
-
from cognite.neat.rules.models.data_types import DataType
|
|
26
|
-
from cognite.neat.rules.models.entities import (
|
|
27
|
-
ClassEntity,
|
|
28
|
-
ContainerEntity,
|
|
29
|
-
ContainerEntityList,
|
|
30
|
-
DMSNodeEntity,
|
|
31
|
-
DMSUnknownEntity,
|
|
32
|
-
ParentClassEntity,
|
|
33
|
-
ReferenceEntity,
|
|
34
|
-
UnknownEntity,
|
|
35
|
-
URLEntity,
|
|
36
|
-
ViewEntity,
|
|
37
|
-
ViewEntityList,
|
|
38
|
-
ViewPropertyEntity,
|
|
39
|
-
)
|
|
40
|
-
from cognite.neat.rules.models.rules._domain_rules import DomainRules
|
|
41
|
-
from cognite.neat.rules.models.wrapped_entities import DMSFilter, HasDataFilter, NodeTypeFilter
|
|
42
|
-
|
|
43
|
-
from ._base import (
|
|
44
|
-
BaseMetadata,
|
|
45
|
-
BaseRules,
|
|
46
|
-
DataModelType,
|
|
47
|
-
ExtensionCategory,
|
|
48
|
-
RoleTypes,
|
|
49
|
-
SchemaCompleteness,
|
|
50
|
-
SheetEntity,
|
|
51
|
-
SheetList,
|
|
52
|
-
)
|
|
53
|
-
from ._dms_schema import DMSSchema, PipelineSchema
|
|
54
|
-
from ._types import (
|
|
55
|
-
ExternalIdType,
|
|
56
|
-
PropertyType,
|
|
57
|
-
StrListType,
|
|
58
|
-
VersionType,
|
|
59
|
-
)
|
|
60
|
-
|
|
61
|
-
if TYPE_CHECKING:
|
|
62
|
-
from ._information_rules import InformationRules
|
|
63
|
-
|
|
64
|
-
if sys.version_info >= (3, 11):
|
|
65
|
-
from typing import Self
|
|
66
|
-
else:
|
|
67
|
-
from typing_extensions import Self
|
|
68
|
-
|
|
69
|
-
_DEFAULT_VERSION = "1"
|
|
70
|
-
|
|
71
|
-
subclasses = list(CognitePropertyType.__subclasses__())
|
|
72
|
-
_PropertyType_by_name: dict[str, type[CognitePropertyType]] = {}
|
|
73
|
-
for subclass in subclasses:
|
|
74
|
-
subclasses.extend(subclass.__subclasses__())
|
|
75
|
-
if abc.ABC in subclass.__bases__:
|
|
76
|
-
continue
|
|
77
|
-
try:
|
|
78
|
-
_PropertyType_by_name[subclass._type.casefold()] = subclass
|
|
79
|
-
except AttributeError:
|
|
80
|
-
...
|
|
81
|
-
del subclasses # cleanup namespace
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
class DMSMetadata(BaseMetadata):
|
|
85
|
-
role: ClassVar[RoleTypes] = RoleTypes.dms_architect
|
|
86
|
-
data_model_type: DataModelType = Field(DataModelType.solution, alias="dataModelType")
|
|
87
|
-
schema_: SchemaCompleteness = Field(alias="schema")
|
|
88
|
-
extension: ExtensionCategory = ExtensionCategory.addition
|
|
89
|
-
space: ExternalIdType
|
|
90
|
-
name: str | None = Field(
|
|
91
|
-
None,
|
|
92
|
-
description="Human readable name of the data model",
|
|
93
|
-
min_length=1,
|
|
94
|
-
max_length=255,
|
|
95
|
-
)
|
|
96
|
-
description: str | None = Field(None, min_length=1, max_length=1024)
|
|
97
|
-
external_id: ExternalIdType = Field(alias="externalId")
|
|
98
|
-
version: VersionType
|
|
99
|
-
creator: StrListType
|
|
100
|
-
created: datetime = Field(
|
|
101
|
-
description=("Date of the data model creation"),
|
|
102
|
-
)
|
|
103
|
-
updated: datetime = Field(
|
|
104
|
-
description=("Date of the data model update"),
|
|
105
|
-
)
|
|
106
|
-
|
|
107
|
-
@field_validator("*", mode="before")
|
|
108
|
-
def strip_string(cls, value: Any) -> Any:
|
|
109
|
-
if isinstance(value, str):
|
|
110
|
-
return value.strip()
|
|
111
|
-
return value
|
|
112
|
-
|
|
113
|
-
@field_serializer("schema_", "extension", "data_model_type", when_used="always")
|
|
114
|
-
@staticmethod
|
|
115
|
-
def as_string(value: SchemaCompleteness | ExtensionCategory | DataModelType) -> str:
|
|
116
|
-
return str(value)
|
|
117
|
-
|
|
118
|
-
@field_validator("schema_", mode="plain")
|
|
119
|
-
def as_enum_schema(cls, value: str) -> SchemaCompleteness:
|
|
120
|
-
return SchemaCompleteness(value)
|
|
121
|
-
|
|
122
|
-
@field_validator("extension", mode="plain")
|
|
123
|
-
def as_enum_extension(cls, value: str) -> ExtensionCategory:
|
|
124
|
-
return ExtensionCategory(value)
|
|
125
|
-
|
|
126
|
-
@field_validator("data_model_type", mode="plain")
|
|
127
|
-
def as_enum_model_type(cls, value: str) -> DataModelType:
|
|
128
|
-
return DataModelType(value)
|
|
129
|
-
|
|
130
|
-
@field_validator("description", mode="before")
|
|
131
|
-
def nan_as_none(cls, value):
|
|
132
|
-
if isinstance(value, float) and math.isnan(value):
|
|
133
|
-
return None
|
|
134
|
-
return value
|
|
135
|
-
|
|
136
|
-
def as_space(self) -> dm.SpaceApply:
|
|
137
|
-
return dm.SpaceApply(
|
|
138
|
-
space=self.space,
|
|
139
|
-
)
|
|
140
|
-
|
|
141
|
-
def as_data_model_id(self) -> dm.DataModelId:
|
|
142
|
-
return dm.DataModelId(space=self.space, external_id=self.external_id, version=self.version)
|
|
143
|
-
|
|
144
|
-
def as_data_model(self) -> dm.DataModelApply:
|
|
145
|
-
suffix = f"Creator: {', '.join(self.creator)}"
|
|
146
|
-
if self.description:
|
|
147
|
-
description = f"{self.description} Creator: {', '.join(self.creator)}"
|
|
148
|
-
else:
|
|
149
|
-
description = suffix
|
|
150
|
-
|
|
151
|
-
return dm.DataModelApply(
|
|
152
|
-
space=self.space,
|
|
153
|
-
external_id=self.external_id,
|
|
154
|
-
name=self.name or None,
|
|
155
|
-
version=self.version or "missing",
|
|
156
|
-
description=description,
|
|
157
|
-
views=[],
|
|
158
|
-
)
|
|
159
|
-
|
|
160
|
-
@classmethod
|
|
161
|
-
def _get_description_and_creator(cls, description_raw: str | None) -> tuple[str | None, list[str]]:
|
|
162
|
-
if description_raw and (description_match := re.search(r"Creator: (.+)", description_raw)):
|
|
163
|
-
creator = description_match.group(1).split(", ")
|
|
164
|
-
description = description_raw.replace(description_match.string, "").strip() or None
|
|
165
|
-
elif description_raw:
|
|
166
|
-
creator = ["MISSING"]
|
|
167
|
-
description = description_raw
|
|
168
|
-
else:
|
|
169
|
-
creator = ["MISSING"]
|
|
170
|
-
description = None
|
|
171
|
-
return description, creator
|
|
172
|
-
|
|
173
|
-
@classmethod
|
|
174
|
-
def from_data_model(cls, data_model: dm.DataModelApply) -> "DMSMetadata":
|
|
175
|
-
description, creator = cls._get_description_and_creator(data_model.description)
|
|
176
|
-
return cls(
|
|
177
|
-
schema_=SchemaCompleteness.complete,
|
|
178
|
-
space=data_model.space,
|
|
179
|
-
name=data_model.name or None,
|
|
180
|
-
description=description,
|
|
181
|
-
external_id=data_model.external_id,
|
|
182
|
-
version=data_model.version,
|
|
183
|
-
creator=creator,
|
|
184
|
-
created=datetime.now(),
|
|
185
|
-
updated=datetime.now(),
|
|
186
|
-
)
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
class DMSProperty(SheetEntity):
|
|
190
|
-
view: ViewEntity = Field(alias="View")
|
|
191
|
-
view_property: str = Field(alias="View Property")
|
|
192
|
-
name: str | None = Field(alias="Name", default=None)
|
|
193
|
-
description: str | None = Field(alias="Description", default=None)
|
|
194
|
-
connection: Literal["direct", "edge", "reverse"] | None = Field(None, alias="Connection")
|
|
195
|
-
value_type: DataType | ViewPropertyEntity | ViewEntity | DMSUnknownEntity = Field(alias="Value Type")
|
|
196
|
-
nullable: bool | None = Field(default=None, alias="Nullable")
|
|
197
|
-
is_list: bool | None = Field(default=None, alias="Is List")
|
|
198
|
-
default: str | int | dict | None = Field(None, alias="Default")
|
|
199
|
-
reference: URLEntity | ReferenceEntity | None = Field(default=None, alias="Reference", union_mode="left_to_right")
|
|
200
|
-
container: ContainerEntity | None = Field(None, alias="Container")
|
|
201
|
-
container_property: str | None = Field(None, alias="Container Property")
|
|
202
|
-
index: StrListType | None = Field(None, alias="Index")
|
|
203
|
-
constraint: StrListType | None = Field(None, alias="Constraint")
|
|
204
|
-
class_: ClassEntity = Field(alias="Class (linage)")
|
|
205
|
-
property_: PropertyType = Field(alias="Property (linage)")
|
|
206
|
-
|
|
207
|
-
@field_validator("nullable")
|
|
208
|
-
def direct_relation_must_be_nullable(cls, value: Any, info: ValidationInfo) -> None:
|
|
209
|
-
if info.data.get("connection") == "direct" and value is False:
|
|
210
|
-
raise ValueError("Direct relation must be nullable")
|
|
211
|
-
return value
|
|
212
|
-
|
|
213
|
-
@field_validator("value_type", mode="after")
|
|
214
|
-
def connections_value_type(
|
|
215
|
-
cls, value: ViewPropertyEntity | ViewEntity | DMSUnknownEntity, info: ValidationInfo
|
|
216
|
-
) -> DataType | ViewPropertyEntity | ViewEntity | DMSUnknownEntity:
|
|
217
|
-
if (connection := info.data.get("connection")) is None:
|
|
218
|
-
return value
|
|
219
|
-
if connection == "direct" and not isinstance(value, ViewEntity | DMSUnknownEntity):
|
|
220
|
-
raise ValueError(f"Direct relation must have a value type that points to a view, got {value}")
|
|
221
|
-
elif connection == "edge" and not isinstance(value, ViewEntity):
|
|
222
|
-
raise ValueError(f"Edge connection must have a value type that points to a view, got {value}")
|
|
223
|
-
elif connection == "reverse" and not isinstance(value, ViewPropertyEntity | ViewEntity):
|
|
224
|
-
raise ValueError(
|
|
225
|
-
f"Reverse connection must have a value type that points to a view or view property, got {value}"
|
|
226
|
-
)
|
|
227
|
-
return value
|
|
228
|
-
|
|
229
|
-
@field_serializer("value_type", when_used="always")
|
|
230
|
-
@staticmethod
|
|
231
|
-
def as_dms_type(value_type: DataType | ViewPropertyEntity | ViewEntity) -> str:
|
|
232
|
-
if isinstance(value_type, DataType):
|
|
233
|
-
return value_type.dms._type
|
|
234
|
-
else:
|
|
235
|
-
return str(value_type)
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
class DMSContainer(SheetEntity):
|
|
239
|
-
container: ContainerEntity = Field(alias="Container")
|
|
240
|
-
name: str | None = Field(alias="Name", default=None)
|
|
241
|
-
description: str | None = Field(alias="Description", default=None)
|
|
242
|
-
reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
|
|
243
|
-
constraint: ContainerEntityList | None = Field(None, alias="Constraint")
|
|
244
|
-
class_: ClassEntity = Field(alias="Class (linage)")
|
|
245
|
-
|
|
246
|
-
def as_container(self) -> dm.ContainerApply:
|
|
247
|
-
container_id = self.container.as_id()
|
|
248
|
-
constraints: dict[str, dm.Constraint] = {}
|
|
249
|
-
for constraint in self.constraint or []:
|
|
250
|
-
requires = dm.RequiresConstraint(constraint.as_id())
|
|
251
|
-
constraints[f"{constraint.space}_{constraint.external_id}"] = requires
|
|
252
|
-
|
|
253
|
-
return dm.ContainerApply(
|
|
254
|
-
space=container_id.space,
|
|
255
|
-
external_id=container_id.external_id,
|
|
256
|
-
name=self.name or None,
|
|
257
|
-
description=self.description,
|
|
258
|
-
constraints=constraints or None,
|
|
259
|
-
properties={},
|
|
260
|
-
)
|
|
261
|
-
|
|
262
|
-
@classmethod
|
|
263
|
-
def from_container(cls, container: dm.ContainerApply) -> "DMSContainer":
|
|
264
|
-
constraints: list[ContainerEntity] = []
|
|
265
|
-
for _, constraint_obj in (container.constraints or {}).items():
|
|
266
|
-
if isinstance(constraint_obj, dm.RequiresConstraint):
|
|
267
|
-
constraints.append(ContainerEntity.from_id(constraint_obj.require))
|
|
268
|
-
# UniquenessConstraint it handled in the properties
|
|
269
|
-
container_entity = ContainerEntity.from_id(container.as_id())
|
|
270
|
-
return cls(
|
|
271
|
-
class_=container_entity.as_class(),
|
|
272
|
-
container=container_entity,
|
|
273
|
-
name=container.name or None,
|
|
274
|
-
description=container.description,
|
|
275
|
-
constraint=constraints or None,
|
|
276
|
-
)
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
class DMSView(SheetEntity):
|
|
280
|
-
view: ViewEntity = Field(alias="View")
|
|
281
|
-
name: str | None = Field(alias="Name", default=None)
|
|
282
|
-
description: str | None = Field(alias="Description", default=None)
|
|
283
|
-
implements: ViewEntityList | None = Field(None, alias="Implements")
|
|
284
|
-
reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
|
|
285
|
-
filter_: HasDataFilter | NodeTypeFilter | None = Field(None, alias="Filter")
|
|
286
|
-
in_model: bool = Field(True, alias="In Model")
|
|
287
|
-
class_: ClassEntity = Field(alias="Class (linage)")
|
|
288
|
-
|
|
289
|
-
def as_view(self) -> dm.ViewApply:
|
|
290
|
-
view_id = self.view.as_id()
|
|
291
|
-
return dm.ViewApply(
|
|
292
|
-
space=view_id.space,
|
|
293
|
-
external_id=view_id.external_id,
|
|
294
|
-
version=view_id.version or _DEFAULT_VERSION,
|
|
295
|
-
name=self.name or None,
|
|
296
|
-
description=self.description,
|
|
297
|
-
implements=[parent.as_id() for parent in self.implements or []] or None,
|
|
298
|
-
properties={},
|
|
299
|
-
)
|
|
300
|
-
|
|
301
|
-
@classmethod
|
|
302
|
-
def from_view(cls, view: dm.ViewApply, in_model: bool) -> "DMSView":
|
|
303
|
-
view_entity = ViewEntity.from_id(view.as_id())
|
|
304
|
-
class_entity = view_entity.as_class(skip_version=True)
|
|
305
|
-
|
|
306
|
-
return cls(
|
|
307
|
-
class_=class_entity,
|
|
308
|
-
view=view_entity,
|
|
309
|
-
description=view.description,
|
|
310
|
-
name=view.name,
|
|
311
|
-
implements=[ViewEntity.from_id(parent, _DEFAULT_VERSION) for parent in view.implements] or None,
|
|
312
|
-
in_model=in_model,
|
|
313
|
-
)
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
class DMSRules(BaseRules):
|
|
317
|
-
metadata: DMSMetadata = Field(alias="Metadata")
|
|
318
|
-
properties: SheetList[DMSProperty] = Field(alias="Properties")
|
|
319
|
-
views: SheetList[DMSView] = Field(alias="Views")
|
|
320
|
-
containers: SheetList[DMSContainer] | None = Field(None, alias="Containers")
|
|
321
|
-
reference: "DMSRules | None" = Field(None, alias="Reference")
|
|
322
|
-
|
|
323
|
-
@field_validator("reference")
|
|
324
|
-
def check_reference_of_reference(cls, value: "DMSRules | None", info: ValidationInfo) -> "DMSRules | None":
|
|
325
|
-
if value is None:
|
|
326
|
-
return None
|
|
327
|
-
if value.reference is not None:
|
|
328
|
-
raise ValueError("Reference rules cannot have a reference")
|
|
329
|
-
if value.metadata.data_model_type == DataModelType.solution and (metadata := info.data.get("metadata")):
|
|
330
|
-
warnings.warn(
|
|
331
|
-
issues.dms.SolutionOnTopOfSolutionModelWarning(
|
|
332
|
-
metadata.as_data_model_id(), value.metadata.as_data_model_id()
|
|
333
|
-
),
|
|
334
|
-
stacklevel=2,
|
|
335
|
-
)
|
|
336
|
-
return value
|
|
337
|
-
|
|
338
|
-
@field_validator("views")
|
|
339
|
-
def matching_version_and_space(cls, value: SheetList[DMSView], info: ValidationInfo) -> SheetList[DMSView]:
|
|
340
|
-
if not (metadata := info.data.get("metadata")):
|
|
341
|
-
return value
|
|
342
|
-
model_version = metadata.version
|
|
343
|
-
if different_version := [view.view.as_id() for view in value if view.view.version != model_version]:
|
|
344
|
-
warnings.warn(issues.dms.ViewModelVersionNotMatchingWarning(different_version, model_version), stacklevel=2)
|
|
345
|
-
if different_space := [view.view.as_id() for view in value if view.view.space != metadata.space]:
|
|
346
|
-
warnings.warn(issues.dms.ViewModelSpaceNotMatchingWarning(different_space, metadata.space), stacklevel=2)
|
|
347
|
-
return value
|
|
348
|
-
|
|
349
|
-
@field_validator("views")
|
|
350
|
-
def matching_version(cls, value: SheetList[DMSView], info: ValidationInfo) -> SheetList[DMSView]:
|
|
351
|
-
if not (metadata := info.data.get("metadata")):
|
|
352
|
-
return value
|
|
353
|
-
model_version = metadata.version
|
|
354
|
-
if different_version := [view.view.as_id() for view in value if view.view.version != model_version]:
|
|
355
|
-
warnings.warn(issues.dms.ViewModelVersionNotMatchingWarning(different_version, model_version), stacklevel=2)
|
|
356
|
-
return value
|
|
357
|
-
|
|
358
|
-
@model_validator(mode="after")
|
|
359
|
-
def consistent_container_properties(self) -> "DMSRules":
|
|
360
|
-
container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, DMSProperty]]] = defaultdict(list)
|
|
361
|
-
for prop_no, prop in enumerate(self.properties):
|
|
362
|
-
if prop.container and prop.container_property:
|
|
363
|
-
container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
|
|
364
|
-
|
|
365
|
-
errors: list[cognite.neat.rules.issues.spreadsheet.InconsistentContainerDefinitionError] = []
|
|
366
|
-
for (container, prop_name), properties in container_properties_by_id.items():
|
|
367
|
-
if len(properties) == 1:
|
|
368
|
-
continue
|
|
369
|
-
container_id = container.as_id()
|
|
370
|
-
row_numbers = {prop_no for prop_no, _ in properties}
|
|
371
|
-
value_types = {prop.value_type for _, prop in properties if prop.value_type}
|
|
372
|
-
if len(value_types) > 1:
|
|
373
|
-
errors.append(
|
|
374
|
-
cognite.neat.rules.issues.spreadsheet.MultiValueTypeError(
|
|
375
|
-
container_id,
|
|
376
|
-
prop_name,
|
|
377
|
-
row_numbers,
|
|
378
|
-
{v.dms._type if isinstance(v, DataType) else str(v) for v in value_types},
|
|
379
|
-
)
|
|
380
|
-
)
|
|
381
|
-
list_definitions = {prop.is_list for _, prop in properties if prop.is_list is not None}
|
|
382
|
-
if len(list_definitions) > 1:
|
|
383
|
-
errors.append(
|
|
384
|
-
cognite.neat.rules.issues.spreadsheet.MultiValueIsListError(
|
|
385
|
-
container_id, prop_name, row_numbers, list_definitions
|
|
386
|
-
)
|
|
387
|
-
)
|
|
388
|
-
nullable_definitions = {prop.nullable for _, prop in properties if prop.nullable is not None}
|
|
389
|
-
if len(nullable_definitions) > 1:
|
|
390
|
-
errors.append(
|
|
391
|
-
cognite.neat.rules.issues.spreadsheet.MultiNullableError(
|
|
392
|
-
container_id, prop_name, row_numbers, nullable_definitions
|
|
393
|
-
)
|
|
394
|
-
)
|
|
395
|
-
default_definitions = {prop.default for _, prop in properties if prop.default is not None}
|
|
396
|
-
if len(default_definitions) > 1:
|
|
397
|
-
errors.append(
|
|
398
|
-
cognite.neat.rules.issues.spreadsheet.MultiDefaultError(
|
|
399
|
-
container_id, prop_name, row_numbers, list(default_definitions)
|
|
400
|
-
)
|
|
401
|
-
)
|
|
402
|
-
index_definitions = {",".join(prop.index) for _, prop in properties if prop.index is not None}
|
|
403
|
-
if len(index_definitions) > 1:
|
|
404
|
-
errors.append(
|
|
405
|
-
cognite.neat.rules.issues.spreadsheet.MultiIndexError(
|
|
406
|
-
container_id, prop_name, row_numbers, index_definitions
|
|
407
|
-
)
|
|
408
|
-
)
|
|
409
|
-
constraint_definitions = {
|
|
410
|
-
",".join(prop.constraint) for _, prop in properties if prop.constraint is not None
|
|
411
|
-
}
|
|
412
|
-
if len(constraint_definitions) > 1:
|
|
413
|
-
errors.append(
|
|
414
|
-
cognite.neat.rules.issues.spreadsheet.MultiUniqueConstraintError(
|
|
415
|
-
container_id, prop_name, row_numbers, constraint_definitions
|
|
416
|
-
)
|
|
417
|
-
)
|
|
418
|
-
|
|
419
|
-
# This sets the container definition for all the properties where it is not defined.
|
|
420
|
-
# This allows the user to define the container only once.
|
|
421
|
-
value_type = next(iter(value_types))
|
|
422
|
-
list_definition = next(iter(list_definitions)) if list_definitions else None
|
|
423
|
-
nullable_definition = next(iter(nullable_definitions)) if nullable_definitions else None
|
|
424
|
-
default_definition = next(iter(default_definitions)) if default_definitions else None
|
|
425
|
-
index_definition = next(iter(index_definitions)).split(",") if index_definitions else None
|
|
426
|
-
constraint_definition = next(iter(constraint_definitions)).split(",") if constraint_definitions else None
|
|
427
|
-
for _, prop in properties:
|
|
428
|
-
prop.value_type = value_type
|
|
429
|
-
prop.is_list = prop.is_list or list_definition
|
|
430
|
-
prop.nullable = prop.nullable or nullable_definition
|
|
431
|
-
prop.default = prop.default or default_definition
|
|
432
|
-
prop.index = prop.index or index_definition
|
|
433
|
-
prop.constraint = prop.constraint or constraint_definition
|
|
434
|
-
|
|
435
|
-
if errors:
|
|
436
|
-
raise issues.MultiValueError(errors)
|
|
437
|
-
return self
|
|
438
|
-
|
|
439
|
-
@model_validator(mode="after")
|
|
440
|
-
def referenced_views_and_containers_are_existing(self) -> "DMSRules":
|
|
441
|
-
# There two checks are done in the same method to raise all the errors at once.
|
|
442
|
-
defined_views = {view.view.as_id() for view in self.views}
|
|
443
|
-
|
|
444
|
-
errors: list[issues.NeatValidationError] = []
|
|
445
|
-
for prop_no, prop in enumerate(self.properties):
|
|
446
|
-
if prop.view and (view_id := prop.view.as_id()) not in defined_views:
|
|
447
|
-
errors.append(
|
|
448
|
-
cognite.neat.rules.issues.spreadsheet.NonExistingViewError(
|
|
449
|
-
column="View",
|
|
450
|
-
row=prop_no,
|
|
451
|
-
type="value_error.missing",
|
|
452
|
-
view_id=view_id,
|
|
453
|
-
msg="",
|
|
454
|
-
input=None,
|
|
455
|
-
url=None,
|
|
456
|
-
)
|
|
457
|
-
)
|
|
458
|
-
if self.metadata.schema_ is SchemaCompleteness.complete:
|
|
459
|
-
defined_containers = {container.container.as_id() for container in self.containers or []}
|
|
460
|
-
for prop_no, prop in enumerate(self.properties):
|
|
461
|
-
if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
|
|
462
|
-
errors.append(
|
|
463
|
-
cognite.neat.rules.issues.spreadsheet.NonExistingContainerError(
|
|
464
|
-
column="Container",
|
|
465
|
-
row=prop_no,
|
|
466
|
-
type="value_error.missing",
|
|
467
|
-
container_id=container_id,
|
|
468
|
-
msg="",
|
|
469
|
-
input=None,
|
|
470
|
-
url=None,
|
|
471
|
-
)
|
|
472
|
-
)
|
|
473
|
-
for _container_no, container in enumerate(self.containers or []):
|
|
474
|
-
for constraint_no, constraint in enumerate(container.constraint or []):
|
|
475
|
-
if constraint.as_id() not in defined_containers:
|
|
476
|
-
errors.append(
|
|
477
|
-
cognite.neat.rules.issues.spreadsheet.NonExistingContainerError(
|
|
478
|
-
column="Constraint",
|
|
479
|
-
row=constraint_no,
|
|
480
|
-
type="value_error.missing",
|
|
481
|
-
container_id=constraint.as_id(),
|
|
482
|
-
msg="",
|
|
483
|
-
input=None,
|
|
484
|
-
url=None,
|
|
485
|
-
)
|
|
486
|
-
)
|
|
487
|
-
if errors:
|
|
488
|
-
raise issues.MultiValueError(errors)
|
|
489
|
-
return self
|
|
490
|
-
|
|
491
|
-
@model_validator(mode="after")
|
|
492
|
-
def validate_extension(self) -> "DMSRules":
|
|
493
|
-
if self.metadata.schema_ is not SchemaCompleteness.extended:
|
|
494
|
-
return self
|
|
495
|
-
if not self.reference:
|
|
496
|
-
raise ValueError("The schema is set to 'extended', but no reference rules are provided to validate against")
|
|
497
|
-
is_solution = self.metadata.space != self.reference.metadata.space
|
|
498
|
-
if is_solution:
|
|
499
|
-
return self
|
|
500
|
-
if self.metadata.extension is ExtensionCategory.rebuild:
|
|
501
|
-
# Everything is allowed
|
|
502
|
-
return self
|
|
503
|
-
# Is an extension of an existing model.
|
|
504
|
-
user_schema = self.as_schema(include_ref=False)
|
|
505
|
-
ref_schema = self.reference.as_schema()
|
|
506
|
-
new_containers = {container.as_id(): container for container in user_schema.containers}
|
|
507
|
-
existing_containers = {container.as_id(): container for container in ref_schema.containers}
|
|
508
|
-
|
|
509
|
-
errors: list[issues.NeatValidationError] = []
|
|
510
|
-
for container_id, container in new_containers.items():
|
|
511
|
-
existing_container = existing_containers.get(container_id)
|
|
512
|
-
if not existing_container or existing_container == container:
|
|
513
|
-
# No problem
|
|
514
|
-
continue
|
|
515
|
-
new_dumped = container.dump()
|
|
516
|
-
existing_dumped = existing_container.dump()
|
|
517
|
-
changed_attributes, changed_properties = self._changed_attributes_and_properties(
|
|
518
|
-
new_dumped, existing_dumped
|
|
519
|
-
)
|
|
520
|
-
errors.append(
|
|
521
|
-
issues.dms.ChangingContainerError(
|
|
522
|
-
container_id=container_id,
|
|
523
|
-
changed_properties=changed_properties or None,
|
|
524
|
-
changed_attributes=changed_attributes or None,
|
|
525
|
-
)
|
|
526
|
-
)
|
|
527
|
-
|
|
528
|
-
if self.metadata.extension is ExtensionCategory.reshape and errors:
|
|
529
|
-
raise issues.MultiValueError(errors)
|
|
530
|
-
elif self.metadata.extension is ExtensionCategory.reshape:
|
|
531
|
-
# Reshape allows changes to views
|
|
532
|
-
return self
|
|
533
|
-
|
|
534
|
-
new_views = {view.as_id(): view for view in user_schema.views}
|
|
535
|
-
existing_views = {view.as_id(): view for view in ref_schema.views}
|
|
536
|
-
for view_id, view in new_views.items():
|
|
537
|
-
existing_view = existing_views.get(view_id)
|
|
538
|
-
if not existing_view or existing_view == view:
|
|
539
|
-
# No problem
|
|
540
|
-
continue
|
|
541
|
-
changed_attributes, changed_properties = self._changed_attributes_and_properties(
|
|
542
|
-
view.dump(), existing_view.dump()
|
|
543
|
-
)
|
|
544
|
-
errors.append(
|
|
545
|
-
issues.dms.ChangingViewError(
|
|
546
|
-
view_id=view_id,
|
|
547
|
-
changed_properties=changed_properties or None,
|
|
548
|
-
changed_attributes=changed_attributes or None,
|
|
549
|
-
)
|
|
550
|
-
)
|
|
551
|
-
|
|
552
|
-
if errors:
|
|
553
|
-
raise issues.MultiValueError(errors)
|
|
554
|
-
return self
|
|
555
|
-
|
|
556
|
-
@staticmethod
|
|
557
|
-
def _changed_attributes_and_properties(
|
|
558
|
-
new_dumped: dict[str, Any], existing_dumped: dict[str, Any]
|
|
559
|
-
) -> tuple[list[str], list[str]]:
|
|
560
|
-
"""Helper method to find the changed attributes and properties between two containers or views."""
|
|
561
|
-
new_attributes = {key: value for key, value in new_dumped.items() if key != "properties"}
|
|
562
|
-
existing_attributes = {key: value for key, value in existing_dumped.items() if key != "properties"}
|
|
563
|
-
changed_attributes = [key for key in new_attributes if new_attributes[key] != existing_attributes.get(key)]
|
|
564
|
-
new_properties = new_dumped.get("properties", {})
|
|
565
|
-
existing_properties = existing_dumped.get("properties", {})
|
|
566
|
-
changed_properties = [prop for prop in new_properties if new_properties[prop] != existing_properties.get(prop)]
|
|
567
|
-
return changed_attributes, changed_properties
|
|
568
|
-
|
|
569
|
-
@model_validator(mode="after")
|
|
570
|
-
def validate_schema(self) -> "DMSRules":
|
|
571
|
-
if self.metadata.schema_ is SchemaCompleteness.partial:
|
|
572
|
-
return self
|
|
573
|
-
elif self.metadata.schema_ is SchemaCompleteness.complete:
|
|
574
|
-
rules: DMSRules = self
|
|
575
|
-
elif self.metadata.schema_ is SchemaCompleteness.extended:
|
|
576
|
-
if not self.reference:
|
|
577
|
-
raise ValueError(
|
|
578
|
-
"The schema is set to 'extended', but no reference rules are provided to validate against"
|
|
579
|
-
)
|
|
580
|
-
# This is an extension of the reference rules, we need to merge the two
|
|
581
|
-
rules = self.model_copy(deep=True)
|
|
582
|
-
rules.properties.extend(self.reference.properties.data)
|
|
583
|
-
existing_views = {view.view.as_id() for view in rules.views}
|
|
584
|
-
rules.views.extend([view for view in self.reference.views if view.view.as_id() not in existing_views])
|
|
585
|
-
if rules.containers and self.reference.containers:
|
|
586
|
-
existing_containers = {container.container.as_id() for container in rules.containers.data}
|
|
587
|
-
rules.containers.extend(
|
|
588
|
-
[
|
|
589
|
-
container
|
|
590
|
-
for container in self.reference.containers
|
|
591
|
-
if container.container.as_id() not in existing_containers
|
|
592
|
-
]
|
|
593
|
-
)
|
|
594
|
-
elif not rules.containers and self.reference.containers:
|
|
595
|
-
rules.containers = self.reference.containers
|
|
596
|
-
else:
|
|
597
|
-
raise ValueError("Unknown schema completeness")
|
|
598
|
-
|
|
599
|
-
schema = rules.as_schema()
|
|
600
|
-
errors = schema.validate()
|
|
601
|
-
if errors:
|
|
602
|
-
raise issues.MultiValueError(errors)
|
|
603
|
-
return self
|
|
604
|
-
|
|
605
|
-
@model_serializer(mode="wrap", when_used="always")
|
|
606
|
-
def dms_rules_serialization(
|
|
607
|
-
self,
|
|
608
|
-
handler: Callable,
|
|
609
|
-
info: SerializationInfo,
|
|
610
|
-
) -> dict[str, Any]:
|
|
611
|
-
dumped = cast(dict[str, Any], handler(self, info))
|
|
612
|
-
space, version = self.metadata.space, self.metadata.version
|
|
613
|
-
return _DMSRulesSerializer(info, space, version).clean(dumped)
|
|
614
|
-
|
|
615
|
-
def as_schema(
|
|
616
|
-
self, include_ref: bool = False, include_pipeline: bool = False, instance_space: str | None = None
|
|
617
|
-
) -> DMSSchema:
|
|
618
|
-
return _DMSExporter(self, include_ref, include_pipeline, instance_space).to_schema()
|
|
619
|
-
|
|
620
|
-
def as_information_architect_rules(self) -> "InformationRules":
|
|
621
|
-
return _DMSRulesConverter(self).as_information_architect_rules()
|
|
622
|
-
|
|
623
|
-
def as_domain_expert_rules(self) -> DomainRules:
|
|
624
|
-
return _DMSRulesConverter(self).as_domain_rules()
|
|
625
|
-
|
|
626
|
-
def reference_self(self) -> Self:
|
|
627
|
-
new_rules = self.model_copy(deep=True)
|
|
628
|
-
for prop in new_rules.properties:
|
|
629
|
-
prop.reference = ReferenceEntity(
|
|
630
|
-
prefix=prop.view.prefix, suffix=prop.view.suffix, version=prop.view.version, property=prop.property_
|
|
631
|
-
)
|
|
632
|
-
view: DMSView
|
|
633
|
-
for view in new_rules.views:
|
|
634
|
-
view.reference = ReferenceEntity(
|
|
635
|
-
prefix=view.view.prefix, suffix=view.view.suffix, version=view.view.version
|
|
636
|
-
)
|
|
637
|
-
container: DMSContainer
|
|
638
|
-
for container in new_rules.containers or []:
|
|
639
|
-
container.reference = ReferenceEntity(prefix=container.container.prefix, suffix=container.container.suffix)
|
|
640
|
-
return new_rules
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
class _DMSExporter:
|
|
644
|
-
"""The DMS Exporter is responsible for exporting the DMSRules to a DMSSchema.
|
|
645
|
-
|
|
646
|
-
This kept in this location such that it can be used by the DMSRules to validate the schema.
|
|
647
|
-
(This module cannot have a dependency on the exporter module, as it would create a circular dependency.)
|
|
648
|
-
|
|
649
|
-
Args
|
|
650
|
-
include_pipeline (bool): If True, the pipeline will be included with the schema. Pipeline means the
|
|
651
|
-
raw tables and transformations necessary to populate the data model.
|
|
652
|
-
instance_space (str): The space to use for the instance. Defaults to None,`Rules.metadata.space` will be used
|
|
653
|
-
"""
|
|
654
|
-
|
|
655
|
-
def __init__(
|
|
656
|
-
self,
|
|
657
|
-
rules: DMSRules,
|
|
658
|
-
include_ref: bool = True,
|
|
659
|
-
include_pipeline: bool = False,
|
|
660
|
-
instance_space: str | None = None,
|
|
661
|
-
):
|
|
662
|
-
self.include_ref = include_ref
|
|
663
|
-
self.include_pipeline = include_pipeline
|
|
664
|
-
self.instance_space = instance_space
|
|
665
|
-
self.rules = rules
|
|
666
|
-
self._ref_schema = rules.reference.as_schema() if rules.reference else None
|
|
667
|
-
if self._ref_schema:
|
|
668
|
-
# We skip version as that will always be missing in the reference
|
|
669
|
-
self._ref_views_by_id = {dm.ViewId(view.space, view.external_id): view for view in self._ref_schema.views}
|
|
670
|
-
else:
|
|
671
|
-
self._ref_views_by_id = {}
|
|
672
|
-
|
|
673
|
-
def to_schema(self) -> DMSSchema:
|
|
674
|
-
rules = self.rules
|
|
675
|
-
container_properties_by_id, view_properties_by_id = self._gather_properties()
|
|
676
|
-
containers = self._create_containers(container_properties_by_id)
|
|
677
|
-
|
|
678
|
-
views, node_types = self._create_views_with_node_types(view_properties_by_id)
|
|
679
|
-
|
|
680
|
-
views_not_in_model = {view.view.as_id() for view in rules.views if not view.in_model}
|
|
681
|
-
data_model = rules.metadata.as_data_model()
|
|
682
|
-
data_model.views = sorted(
|
|
683
|
-
[view_id for view_id in views.as_ids() if view_id not in views_not_in_model],
|
|
684
|
-
key=lambda v: v.as_tuple(), # type: ignore[union-attr]
|
|
685
|
-
)
|
|
686
|
-
|
|
687
|
-
spaces = self._create_spaces(rules.metadata, containers, views, data_model)
|
|
688
|
-
|
|
689
|
-
output = DMSSchema(
|
|
690
|
-
spaces=spaces,
|
|
691
|
-
data_models=dm.DataModelApplyList([data_model]),
|
|
692
|
-
views=views,
|
|
693
|
-
containers=containers,
|
|
694
|
-
node_types=node_types,
|
|
695
|
-
)
|
|
696
|
-
if self.include_pipeline:
|
|
697
|
-
return PipelineSchema.from_dms(output, self.instance_space)
|
|
698
|
-
|
|
699
|
-
if self._ref_schema and self.include_ref:
|
|
700
|
-
output.frozen_ids.update(self._ref_schema.node_types.as_ids())
|
|
701
|
-
output.frozen_ids.update(self._ref_schema.views.as_ids())
|
|
702
|
-
output.frozen_ids.update(self._ref_schema.containers.as_ids())
|
|
703
|
-
output.node_types.extend(self._ref_schema.node_types)
|
|
704
|
-
output.views.extend(self._ref_schema.views)
|
|
705
|
-
output.containers.extend(self._ref_schema.containers)
|
|
706
|
-
output.data_models.extend(self._ref_schema.data_models)
|
|
707
|
-
|
|
708
|
-
return output
|
|
709
|
-
|
|
710
|
-
def _create_spaces(
|
|
711
|
-
self,
|
|
712
|
-
metadata: DMSMetadata,
|
|
713
|
-
containers: dm.ContainerApplyList,
|
|
714
|
-
views: dm.ViewApplyList,
|
|
715
|
-
data_model: dm.DataModelApply,
|
|
716
|
-
) -> dm.SpaceApplyList:
|
|
717
|
-
used_spaces = {container.space for container in containers} | {view.space for view in views}
|
|
718
|
-
if len(used_spaces) == 1:
|
|
719
|
-
# We skip the default space and only use this space for the data model
|
|
720
|
-
data_model.space = used_spaces.pop()
|
|
721
|
-
spaces = dm.SpaceApplyList([dm.SpaceApply(space=data_model.space)])
|
|
722
|
-
else:
|
|
723
|
-
used_spaces.add(metadata.space)
|
|
724
|
-
spaces = dm.SpaceApplyList([dm.SpaceApply(space=space) for space in used_spaces])
|
|
725
|
-
if self.instance_space and self.instance_space not in {space.space for space in spaces}:
|
|
726
|
-
spaces.append(dm.SpaceApply(space=self.instance_space, name=self.instance_space))
|
|
727
|
-
return spaces
|
|
728
|
-
|
|
729
|
-
def _create_views_with_node_types(
|
|
730
|
-
self,
|
|
731
|
-
view_properties_by_id: dict[dm.ViewId, list[DMSProperty]],
|
|
732
|
-
) -> tuple[dm.ViewApplyList, dm.NodeApplyList]:
|
|
733
|
-
views = dm.ViewApplyList([dms_view.as_view() for dms_view in self.rules.views])
|
|
734
|
-
dms_view_by_id = {dms_view.view.as_id(): dms_view for dms_view in self.rules.views}
|
|
735
|
-
|
|
736
|
-
for view in views:
|
|
737
|
-
view_id = view.as_id()
|
|
738
|
-
view.properties = {}
|
|
739
|
-
if not (view_properties := view_properties_by_id.get(view_id)):
|
|
740
|
-
continue
|
|
741
|
-
for prop in view_properties:
|
|
742
|
-
view_property = self._create_view_property(prop, view_properties_by_id)
|
|
743
|
-
if view_property is not None:
|
|
744
|
-
view.properties[prop.view_property] = view_property
|
|
745
|
-
|
|
746
|
-
data_model_type = self.rules.metadata.data_model_type
|
|
747
|
-
unique_node_types: set[dm.NodeId] = set()
|
|
748
|
-
parent_views = {parent for view in views for parent in view.implements or []}
|
|
749
|
-
for view in views:
|
|
750
|
-
dms_view = dms_view_by_id.get(view.as_id())
|
|
751
|
-
dms_properties = view_properties_by_id.get(view.as_id(), [])
|
|
752
|
-
view_filter = self._create_view_filter(view, dms_view, data_model_type, dms_properties)
|
|
753
|
-
|
|
754
|
-
view.filter = view_filter.as_dms_filter()
|
|
755
|
-
|
|
756
|
-
if isinstance(view_filter, NodeTypeFilter):
|
|
757
|
-
unique_node_types.update(view_filter.nodes)
|
|
758
|
-
if view.as_id() in parent_views:
|
|
759
|
-
warnings.warn(issues.dms.NodeTypeFilterOnParentViewWarning(view.as_id()), stacklevel=2)
|
|
760
|
-
elif isinstance(view_filter, HasDataFilter) and data_model_type == DataModelType.solution:
|
|
761
|
-
if dms_view and isinstance(dms_view.reference, ReferenceEntity):
|
|
762
|
-
references = {dms_view.reference.as_view_id()}
|
|
763
|
-
elif any(True for prop in dms_properties if isinstance(prop.reference, ReferenceEntity)):
|
|
764
|
-
references = {
|
|
765
|
-
prop.reference.as_view_id()
|
|
766
|
-
for prop in dms_properties
|
|
767
|
-
if isinstance(prop.reference, ReferenceEntity)
|
|
768
|
-
}
|
|
769
|
-
else:
|
|
770
|
-
continue
|
|
771
|
-
warnings.warn(
|
|
772
|
-
issues.dms.HasDataFilterOnViewWithReferencesWarning(view.as_id(), list(references)), stacklevel=2
|
|
773
|
-
)
|
|
774
|
-
|
|
775
|
-
return views, dm.NodeApplyList(
|
|
776
|
-
[dm.NodeApply(space=node.space, external_id=node.external_id) for node in unique_node_types]
|
|
777
|
-
)
|
|
778
|
-
|
|
779
|
-
@classmethod
|
|
780
|
-
def _create_edge_type_from_prop(cls, prop: DMSProperty) -> dm.DirectRelationReference:
|
|
781
|
-
if isinstance(prop.reference, ReferenceEntity):
|
|
782
|
-
ref_view_prop = prop.reference.as_view_property_id()
|
|
783
|
-
return cls._create_edge_type_from_view_id(cast(dm.ViewId, ref_view_prop.source), ref_view_prop.property)
|
|
784
|
-
else:
|
|
785
|
-
return cls._create_edge_type_from_view_id(prop.view.as_id(), prop.view_property)
|
|
786
|
-
|
|
787
|
-
@staticmethod
|
|
788
|
-
def _create_edge_type_from_view_id(view_id: dm.ViewId, property_: str) -> dm.DirectRelationReference:
|
|
789
|
-
return dm.DirectRelationReference(
|
|
790
|
-
space=view_id.space,
|
|
791
|
-
# This is the same convention as used when converting GraphQL to DMS
|
|
792
|
-
external_id=f"{view_id.external_id}.{property_}",
|
|
793
|
-
)
|
|
794
|
-
|
|
795
|
-
def _create_containers(
|
|
796
|
-
self,
|
|
797
|
-
container_properties_by_id: dict[dm.ContainerId, list[DMSProperty]],
|
|
798
|
-
) -> dm.ContainerApplyList:
|
|
799
|
-
containers = dm.ContainerApplyList(
|
|
800
|
-
[dms_container.as_container() for dms_container in self.rules.containers or []]
|
|
801
|
-
)
|
|
802
|
-
container_to_drop = set()
|
|
803
|
-
for container in containers:
|
|
804
|
-
container_id = container.as_id()
|
|
805
|
-
if not (container_properties := container_properties_by_id.get(container_id)):
|
|
806
|
-
warnings.warn(issues.dms.EmptyContainerWarning(container_id=container_id), stacklevel=2)
|
|
807
|
-
container_to_drop.add(container_id)
|
|
808
|
-
continue
|
|
809
|
-
for prop in container_properties:
|
|
810
|
-
if prop.container_property is None:
|
|
811
|
-
continue
|
|
812
|
-
if isinstance(prop.value_type, DataType):
|
|
813
|
-
type_cls = prop.value_type.dms
|
|
814
|
-
else:
|
|
815
|
-
type_cls = dm.DirectRelation
|
|
816
|
-
|
|
817
|
-
type_ = type_cls(is_list=prop.is_list or False)
|
|
818
|
-
container.properties[prop.container_property] = dm.ContainerProperty(
|
|
819
|
-
type=type_,
|
|
820
|
-
nullable=prop.nullable if prop.nullable is not None else True,
|
|
821
|
-
default_value=prop.default,
|
|
822
|
-
name=prop.name,
|
|
823
|
-
description=prop.description,
|
|
824
|
-
)
|
|
825
|
-
|
|
826
|
-
uniqueness_properties: dict[str, set[str]] = defaultdict(set)
|
|
827
|
-
for prop in container_properties:
|
|
828
|
-
if prop.container_property is not None:
|
|
829
|
-
for constraint in prop.constraint or []:
|
|
830
|
-
uniqueness_properties[constraint].add(prop.container_property)
|
|
831
|
-
for constraint_name, properties in uniqueness_properties.items():
|
|
832
|
-
container.constraints = container.constraints or {}
|
|
833
|
-
container.constraints[constraint_name] = dm.UniquenessConstraint(properties=list(properties))
|
|
834
|
-
|
|
835
|
-
index_properties: dict[str, set[str]] = defaultdict(set)
|
|
836
|
-
for prop in container_properties:
|
|
837
|
-
if prop.container_property is not None:
|
|
838
|
-
for index in prop.index or []:
|
|
839
|
-
index_properties[index].add(prop.container_property)
|
|
840
|
-
for index_name, properties in index_properties.items():
|
|
841
|
-
container.indexes = container.indexes or {}
|
|
842
|
-
container.indexes[index_name] = BTreeIndex(properties=list(properties))
|
|
843
|
-
|
|
844
|
-
# We might drop containers we convert direct relations of list into multi-edge connections
|
|
845
|
-
# which do not have a container.
|
|
846
|
-
for container in containers:
|
|
847
|
-
if container.constraints:
|
|
848
|
-
container.constraints = {
|
|
849
|
-
name: const
|
|
850
|
-
for name, const in container.constraints.items()
|
|
851
|
-
if not (isinstance(const, dm.RequiresConstraint) and const.require in container_to_drop)
|
|
852
|
-
}
|
|
853
|
-
return dm.ContainerApplyList(
|
|
854
|
-
[container for container in containers if container.as_id() not in container_to_drop]
|
|
855
|
-
)
|
|
856
|
-
|
|
857
|
-
def _gather_properties(self) -> tuple[dict[dm.ContainerId, list[DMSProperty]], dict[dm.ViewId, list[DMSProperty]]]:
|
|
858
|
-
container_properties_by_id: dict[dm.ContainerId, list[DMSProperty]] = defaultdict(list)
|
|
859
|
-
view_properties_by_id: dict[dm.ViewId, list[DMSProperty]] = defaultdict(list)
|
|
860
|
-
for prop in self.rules.properties:
|
|
861
|
-
view_id = prop.view.as_id()
|
|
862
|
-
view_properties_by_id[view_id].append(prop)
|
|
863
|
-
|
|
864
|
-
if prop.container and prop.container_property:
|
|
865
|
-
container_id = prop.container.as_id()
|
|
866
|
-
container_properties_by_id[container_id].append(prop)
|
|
867
|
-
|
|
868
|
-
return container_properties_by_id, view_properties_by_id
|
|
869
|
-
|
|
870
|
-
def _create_view_filter(
|
|
871
|
-
self,
|
|
872
|
-
view: dm.ViewApply,
|
|
873
|
-
dms_view: DMSView | None,
|
|
874
|
-
data_model_type: DataModelType,
|
|
875
|
-
dms_properties: list[DMSProperty],
|
|
876
|
-
) -> DMSFilter:
|
|
877
|
-
selected_filter_name = (dms_view and dms_view.filter_ and dms_view.filter_.name) or ""
|
|
878
|
-
if dms_view and dms_view.filter_ and not dms_view.filter_.is_empty:
|
|
879
|
-
# Has Explicit Filter, use it
|
|
880
|
-
return dms_view.filter_
|
|
881
|
-
|
|
882
|
-
if data_model_type == DataModelType.solution and selected_filter_name in [NodeTypeFilter.name, ""]:
|
|
883
|
-
if (
|
|
884
|
-
dms_view
|
|
885
|
-
and isinstance(dms_view.reference, ReferenceEntity)
|
|
886
|
-
and not dms_properties
|
|
887
|
-
and (ref_view := self._ref_views_by_id.get(dms_view.reference.as_view_id()))
|
|
888
|
-
and ref_view.filter
|
|
889
|
-
):
|
|
890
|
-
# No new properties, only reference, reuse the reference filter
|
|
891
|
-
return DMSFilter.from_dms_filter(ref_view.filter)
|
|
892
|
-
else:
|
|
893
|
-
referenced_node_ids = {
|
|
894
|
-
prop.reference.as_node_entity()
|
|
895
|
-
for prop in dms_properties
|
|
896
|
-
if isinstance(prop.reference, ReferenceEntity)
|
|
897
|
-
}
|
|
898
|
-
if dms_view and isinstance(dms_view.reference, ReferenceEntity):
|
|
899
|
-
referenced_node_ids.add(dms_view.reference.as_node_entity())
|
|
900
|
-
if referenced_node_ids:
|
|
901
|
-
return NodeTypeFilter(inner=list(referenced_node_ids))
|
|
902
|
-
|
|
903
|
-
# Enterprise Model or (Solution + HasData)
|
|
904
|
-
ref_containers = view.referenced_containers()
|
|
905
|
-
if not ref_containers or selected_filter_name == HasDataFilter.name:
|
|
906
|
-
# Child filter without container properties
|
|
907
|
-
if selected_filter_name == HasDataFilter.name:
|
|
908
|
-
warnings.warn(issues.dms.HasDataFilterOnNoPropertiesViewWarning(view.as_id()), stacklevel=2)
|
|
909
|
-
return NodeTypeFilter(inner=[DMSNodeEntity(space=view.space, externalId=view.external_id)])
|
|
910
|
-
else:
|
|
911
|
-
# HasData or not provided (this is the default)
|
|
912
|
-
return HasDataFilter(inner=[ContainerEntity.from_id(id_) for id_ in ref_containers])
|
|
913
|
-
|
|
914
|
-
def _create_view_property(
|
|
915
|
-
self, prop: DMSProperty, view_properties_by_id: dict[dm.ViewId, list[DMSProperty]]
|
|
916
|
-
) -> ViewPropertyApply | None:
|
|
917
|
-
if prop.container and prop.container_property:
|
|
918
|
-
container_prop_identifier = prop.container_property
|
|
919
|
-
extra_args: dict[str, Any] = {}
|
|
920
|
-
if prop.connection == "direct":
|
|
921
|
-
if isinstance(prop.value_type, ViewEntity):
|
|
922
|
-
extra_args["source"] = prop.value_type.as_id()
|
|
923
|
-
elif isinstance(prop.value_type, DMSUnknownEntity):
|
|
924
|
-
extra_args["source"] = None
|
|
925
|
-
else:
|
|
926
|
-
# Should have been validated.
|
|
927
|
-
raise ValueError(
|
|
928
|
-
"If this error occurs it is a bug in NEAT, please report"
|
|
929
|
-
f"Debug Info, Invalid valueType direct: {prop.model_dump_json()}"
|
|
930
|
-
)
|
|
931
|
-
elif prop.connection is not None:
|
|
932
|
-
# Should have been validated.
|
|
933
|
-
raise ValueError(
|
|
934
|
-
"If this error occurs it is a bug in NEAT, please report"
|
|
935
|
-
f"Debug Info, Invalid connection: {prop.model_dump_json()}"
|
|
936
|
-
)
|
|
937
|
-
return dm.MappedPropertyApply(
|
|
938
|
-
container=prop.container.as_id(),
|
|
939
|
-
container_property_identifier=container_prop_identifier,
|
|
940
|
-
name=prop.name,
|
|
941
|
-
description=prop.description,
|
|
942
|
-
**extra_args,
|
|
943
|
-
)
|
|
944
|
-
elif prop.connection == "edge":
|
|
945
|
-
if isinstance(prop.value_type, ViewEntity):
|
|
946
|
-
source_view_id = prop.value_type.as_id()
|
|
947
|
-
else:
|
|
948
|
-
# Should have been validated.
|
|
949
|
-
raise ValueError(
|
|
950
|
-
"If this error occurs it is a bug in NEAT, please report"
|
|
951
|
-
f"Debug Info, Invalid valueType edge: {prop.model_dump_json()}"
|
|
952
|
-
)
|
|
953
|
-
edge_cls: type[dm.EdgeConnectionApply] = dm.MultiEdgeConnectionApply
|
|
954
|
-
# If is_list is not set, we default to a MultiEdgeConnection
|
|
955
|
-
if prop.is_list is False:
|
|
956
|
-
edge_cls = SingleEdgeConnectionApply
|
|
957
|
-
|
|
958
|
-
return edge_cls(
|
|
959
|
-
type=self._create_edge_type_from_prop(prop),
|
|
960
|
-
source=source_view_id,
|
|
961
|
-
direction="outwards",
|
|
962
|
-
name=prop.name,
|
|
963
|
-
description=prop.description,
|
|
964
|
-
)
|
|
965
|
-
elif prop.connection == "reverse":
|
|
966
|
-
reverse_prop_id: str | None = None
|
|
967
|
-
if isinstance(prop.value_type, ViewPropertyEntity):
|
|
968
|
-
source_view_id = prop.value_type.as_view_id()
|
|
969
|
-
reverse_prop_id = prop.value_type.property_
|
|
970
|
-
elif isinstance(prop.value_type, ViewEntity):
|
|
971
|
-
source_view_id = prop.value_type.as_id()
|
|
972
|
-
else:
|
|
973
|
-
# Should have been validated.
|
|
974
|
-
raise ValueError(
|
|
975
|
-
"If this error occurs it is a bug in NEAT, please report"
|
|
976
|
-
f"Debug Info, Invalid valueType reverse connection: {prop.model_dump_json()}"
|
|
977
|
-
)
|
|
978
|
-
reverse_prop: DMSProperty | None = None
|
|
979
|
-
if reverse_prop_id is not None:
|
|
980
|
-
reverse_prop = next(
|
|
981
|
-
(
|
|
982
|
-
prop
|
|
983
|
-
for prop in view_properties_by_id.get(source_view_id, [])
|
|
984
|
-
if prop.property_ == reverse_prop_id
|
|
985
|
-
),
|
|
986
|
-
None,
|
|
987
|
-
)
|
|
988
|
-
|
|
989
|
-
if reverse_prop is None:
|
|
990
|
-
warnings.warn(
|
|
991
|
-
issues.dms.ReverseRelationMissingOtherSideWarning(source_view_id, prop.view_property),
|
|
992
|
-
stacklevel=2,
|
|
993
|
-
)
|
|
994
|
-
|
|
995
|
-
if reverse_prop is None or reverse_prop.connection == "edge":
|
|
996
|
-
inwards_edge_cls = (
|
|
997
|
-
dm.MultiEdgeConnectionApply if prop.is_list in [True, None] else SingleEdgeConnectionApply
|
|
998
|
-
)
|
|
999
|
-
return inwards_edge_cls(
|
|
1000
|
-
type=self._create_edge_type_from_prop(reverse_prop or prop),
|
|
1001
|
-
source=source_view_id,
|
|
1002
|
-
name=prop.name,
|
|
1003
|
-
description=prop.description,
|
|
1004
|
-
direction="inwards",
|
|
1005
|
-
)
|
|
1006
|
-
elif reverse_prop_id and reverse_prop and reverse_prop.connection == "direct":
|
|
1007
|
-
reverse_direct_cls = (
|
|
1008
|
-
dm.MultiReverseDirectRelationApply if prop.is_list is True else SingleReverseDirectRelationApply
|
|
1009
|
-
)
|
|
1010
|
-
return reverse_direct_cls(
|
|
1011
|
-
source=source_view_id,
|
|
1012
|
-
through=dm.PropertyId(source=source_view_id, property=reverse_prop_id),
|
|
1013
|
-
name=prop.name,
|
|
1014
|
-
description=prop.description,
|
|
1015
|
-
)
|
|
1016
|
-
else:
|
|
1017
|
-
return None
|
|
1018
|
-
|
|
1019
|
-
elif prop.view and prop.view_property and prop.connection:
|
|
1020
|
-
warnings.warn(
|
|
1021
|
-
issues.dms.UnsupportedConnectionWarning(prop.view.as_id(), prop.view_property, prop.connection or ""),
|
|
1022
|
-
stacklevel=2,
|
|
1023
|
-
)
|
|
1024
|
-
return None
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
class _DMSRulesConverter:
|
|
1028
|
-
def __init__(self, dms: DMSRules):
|
|
1029
|
-
self.dms = dms
|
|
1030
|
-
|
|
1031
|
-
def as_domain_rules(self) -> "DomainRules":
|
|
1032
|
-
raise NotImplementedError("DomainRules not implemented yet")
|
|
1033
|
-
|
|
1034
|
-
def as_information_architect_rules(
|
|
1035
|
-
self,
|
|
1036
|
-
created: datetime | None = None,
|
|
1037
|
-
updated: datetime | None = None,
|
|
1038
|
-
name: str | None = None,
|
|
1039
|
-
namespace: Namespace | None = None,
|
|
1040
|
-
) -> "InformationRules":
|
|
1041
|
-
from ._information_rules import InformationClass, InformationMetadata, InformationProperty, InformationRules
|
|
1042
|
-
|
|
1043
|
-
dms = self.dms.metadata
|
|
1044
|
-
prefix = dms.space
|
|
1045
|
-
|
|
1046
|
-
metadata = InformationMetadata(
|
|
1047
|
-
schema_=dms.schema_,
|
|
1048
|
-
prefix=prefix,
|
|
1049
|
-
namespace=namespace or Namespace(f"https://purl.orgl/neat/{prefix}/"),
|
|
1050
|
-
version=dms.version,
|
|
1051
|
-
name=name or dms.name or "Missing name",
|
|
1052
|
-
creator=dms.creator,
|
|
1053
|
-
created=dms.created or created or datetime.now(),
|
|
1054
|
-
updated=dms.updated or updated or datetime.now(),
|
|
1055
|
-
)
|
|
1056
|
-
|
|
1057
|
-
classes = [
|
|
1058
|
-
InformationClass(
|
|
1059
|
-
# we do not want a version in class as we use URI for the class
|
|
1060
|
-
class_=ClassEntity(prefix=view.class_.prefix, suffix=view.class_.suffix),
|
|
1061
|
-
description=view.description,
|
|
1062
|
-
parent=[
|
|
1063
|
-
# we do not want a version in class as we use URI for the class
|
|
1064
|
-
ParentClassEntity(prefix=implemented_view.prefix, suffix=implemented_view.suffix)
|
|
1065
|
-
# We only want parents in the same namespace, parent in a different namespace is a reference
|
|
1066
|
-
for implemented_view in view.implements or []
|
|
1067
|
-
if implemented_view.prefix == view.class_.prefix
|
|
1068
|
-
],
|
|
1069
|
-
reference=self._get_class_reference(view),
|
|
1070
|
-
)
|
|
1071
|
-
for view in self.dms.views
|
|
1072
|
-
]
|
|
1073
|
-
|
|
1074
|
-
properties: list[InformationProperty] = []
|
|
1075
|
-
value_type: DataType | ClassEntity | str
|
|
1076
|
-
for property_ in self.dms.properties:
|
|
1077
|
-
if isinstance(property_.value_type, DataType):
|
|
1078
|
-
value_type = property_.value_type
|
|
1079
|
-
elif isinstance(property_.value_type, ViewEntity | ViewPropertyEntity):
|
|
1080
|
-
value_type = ClassEntity(
|
|
1081
|
-
prefix=property_.value_type.prefix,
|
|
1082
|
-
suffix=property_.value_type.suffix,
|
|
1083
|
-
)
|
|
1084
|
-
elif isinstance(property_.value_type, DMSUnknownEntity):
|
|
1085
|
-
value_type = UnknownEntity()
|
|
1086
|
-
else:
|
|
1087
|
-
raise ValueError(f"Unsupported value type: {property_.value_type.type_}")
|
|
1088
|
-
|
|
1089
|
-
properties.append(
|
|
1090
|
-
InformationProperty(
|
|
1091
|
-
# Removing version
|
|
1092
|
-
class_=ClassEntity(suffix=property_.class_.suffix, prefix=property_.class_.prefix),
|
|
1093
|
-
property_=property_.view_property,
|
|
1094
|
-
value_type=value_type,
|
|
1095
|
-
description=property_.description,
|
|
1096
|
-
min_count=0 if property_.nullable or property_.nullable is None else 1,
|
|
1097
|
-
max_count=float("inf") if property_.is_list or property_.nullable is None else 1,
|
|
1098
|
-
reference=self._get_property_reference(property_),
|
|
1099
|
-
)
|
|
1100
|
-
)
|
|
1101
|
-
|
|
1102
|
-
return InformationRules(
|
|
1103
|
-
metadata=metadata,
|
|
1104
|
-
properties=SheetList[InformationProperty](data=properties),
|
|
1105
|
-
classes=SheetList[InformationClass](data=classes),
|
|
1106
|
-
reference=self.dms.reference and self.dms.reference.as_information_architect_rules(), # type: ignore[arg-type]
|
|
1107
|
-
)
|
|
1108
|
-
|
|
1109
|
-
@classmethod
|
|
1110
|
-
def _get_class_reference(cls, view: DMSView) -> ReferenceEntity | None:
|
|
1111
|
-
parents_other_namespace = [parent for parent in view.implements or [] if parent.prefix != view.class_.prefix]
|
|
1112
|
-
if len(parents_other_namespace) == 0:
|
|
1113
|
-
return None
|
|
1114
|
-
if len(parents_other_namespace) > 1:
|
|
1115
|
-
warnings.warn(
|
|
1116
|
-
issues.dms.MultipleReferenceWarning(
|
|
1117
|
-
view_id=view.view.as_id(), implements=[v.as_id() for v in parents_other_namespace]
|
|
1118
|
-
),
|
|
1119
|
-
stacklevel=2,
|
|
1120
|
-
)
|
|
1121
|
-
other_parent = parents_other_namespace[0]
|
|
1122
|
-
|
|
1123
|
-
return ReferenceEntity(prefix=other_parent.prefix, suffix=other_parent.suffix)
|
|
1124
|
-
|
|
1125
|
-
@classmethod
|
|
1126
|
-
def _get_property_reference(cls, property_: DMSProperty) -> ReferenceEntity | None:
|
|
1127
|
-
has_container_other_namespace = property_.container and property_.container.prefix != property_.class_.prefix
|
|
1128
|
-
if not has_container_other_namespace:
|
|
1129
|
-
return None
|
|
1130
|
-
container = cast(ContainerEntity, property_.container)
|
|
1131
|
-
return ReferenceEntity(
|
|
1132
|
-
prefix=container.prefix,
|
|
1133
|
-
suffix=container.suffix,
|
|
1134
|
-
property=property_.container_property,
|
|
1135
|
-
)
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
class _DMSRulesSerializer:
|
|
1139
|
-
# These are the fields that need to be cleaned from the default space and version
|
|
1140
|
-
PROPERTIES_FIELDS: ClassVar[list[str]] = ["class_", "view", "value_type", "container"]
|
|
1141
|
-
VIEWS_FIELDS: ClassVar[list[str]] = ["class_", "view", "implements"]
|
|
1142
|
-
CONTAINERS_FIELDS: ClassVar[list[str]] = ["class_", "container"]
|
|
1143
|
-
|
|
1144
|
-
def __init__(self, info: SerializationInfo, default_space: str, default_version: str) -> None:
|
|
1145
|
-
self.default_space = f"{default_space}:"
|
|
1146
|
-
self.default_version = f"version={default_version}"
|
|
1147
|
-
self.default_version_wrapped = f"({self.default_version})"
|
|
1148
|
-
|
|
1149
|
-
self.properties_fields = self.PROPERTIES_FIELDS
|
|
1150
|
-
self.views_fields = self.VIEWS_FIELDS
|
|
1151
|
-
self.containers_fields = self.CONTAINERS_FIELDS
|
|
1152
|
-
self.prop_name = "properties"
|
|
1153
|
-
self.view_name = "views"
|
|
1154
|
-
self.container_name = "containers"
|
|
1155
|
-
self.metadata_name = "metadata"
|
|
1156
|
-
self.prop_view = "view"
|
|
1157
|
-
self.prop_view_property = "view_property"
|
|
1158
|
-
self.prop_value_type = "value_type"
|
|
1159
|
-
self.view_view = "view"
|
|
1160
|
-
self.view_implements = "implements"
|
|
1161
|
-
self.container_container = "container"
|
|
1162
|
-
self.container_constraint = "constraint"
|
|
1163
|
-
|
|
1164
|
-
if info.by_alias:
|
|
1165
|
-
self.properties_fields = [
|
|
1166
|
-
DMSProperty.model_fields[field].alias or field for field in self.properties_fields
|
|
1167
|
-
]
|
|
1168
|
-
self.views_fields = [DMSView.model_fields[field].alias or field for field in self.views_fields]
|
|
1169
|
-
self.container_fields = [
|
|
1170
|
-
DMSContainer.model_fields[field].alias or field for field in self.containers_fields
|
|
1171
|
-
]
|
|
1172
|
-
self.prop_view = DMSProperty.model_fields[self.prop_view].alias or self.prop_view
|
|
1173
|
-
self.prop_view_property = DMSProperty.model_fields[self.prop_view_property].alias or self.prop_view_property
|
|
1174
|
-
self.prop_value_type = DMSProperty.model_fields[self.prop_value_type].alias or self.prop_value_type
|
|
1175
|
-
self.view_view = DMSView.model_fields[self.view_view].alias or self.view_view
|
|
1176
|
-
self.view_implements = DMSView.model_fields[self.view_implements].alias or self.view_implements
|
|
1177
|
-
self.container_container = (
|
|
1178
|
-
DMSContainer.model_fields[self.container_container].alias or self.container_container
|
|
1179
|
-
)
|
|
1180
|
-
self.container_constraint = (
|
|
1181
|
-
DMSContainer.model_fields[self.container_constraint].alias or self.container_constraint
|
|
1182
|
-
)
|
|
1183
|
-
self.prop_name = DMSRules.model_fields[self.prop_name].alias or self.prop_name
|
|
1184
|
-
self.view_name = DMSRules.model_fields[self.view_name].alias or self.view_name
|
|
1185
|
-
self.container_name = DMSRules.model_fields[self.container_name].alias or self.container_name
|
|
1186
|
-
self.metadata_name = DMSRules.model_fields[self.metadata_name].alias or self.metadata_name
|
|
1187
|
-
|
|
1188
|
-
if isinstance(info.exclude, dict):
|
|
1189
|
-
# Just for happy mypy
|
|
1190
|
-
exclude = cast(dict, info.exclude)
|
|
1191
|
-
self.exclude_properties = exclude.get("properties", {}).get("__all__", set())
|
|
1192
|
-
self.exclude_views = exclude.get("views", {}).get("__all__", set()) or set()
|
|
1193
|
-
self.exclude_containers = exclude.get("containers", {}).get("__all__", set()) or set()
|
|
1194
|
-
self.metadata_exclude = exclude.get("metadata", set()) or set()
|
|
1195
|
-
self.exclude_top = {k for k, v in exclude.items() if not v}
|
|
1196
|
-
else:
|
|
1197
|
-
self.exclude_top = set(info.exclude or {})
|
|
1198
|
-
self.exclude_properties = set()
|
|
1199
|
-
self.exclude_views = set()
|
|
1200
|
-
self.exclude_containers = set()
|
|
1201
|
-
self.metadata_exclude = set()
|
|
1202
|
-
|
|
1203
|
-
def clean(self, dumped: dict[str, Any]) -> dict[str, Any]:
|
|
1204
|
-
# Sorting to get a deterministic order
|
|
1205
|
-
dumped[self.prop_name] = sorted(
|
|
1206
|
-
dumped[self.prop_name]["data"], key=lambda p: (p[self.prop_view], p[self.prop_view_property])
|
|
1207
|
-
)
|
|
1208
|
-
dumped[self.view_name] = sorted(dumped[self.view_name]["data"], key=lambda v: v[self.view_view])
|
|
1209
|
-
if self.container_name in dumped:
|
|
1210
|
-
dumped[self.container_name] = sorted(
|
|
1211
|
-
dumped[self.container_name]["data"], key=lambda c: c[self.container_container]
|
|
1212
|
-
)
|
|
1213
|
-
|
|
1214
|
-
for prop in dumped[self.prop_name]:
|
|
1215
|
-
for field_name in self.properties_fields:
|
|
1216
|
-
if value := prop.get(field_name):
|
|
1217
|
-
prop[field_name] = value.removeprefix(self.default_space).removesuffix(self.default_version_wrapped)
|
|
1218
|
-
# Value type can have a property as well
|
|
1219
|
-
prop[self.prop_value_type] = prop[self.prop_value_type].replace(self.default_version, "")
|
|
1220
|
-
if self.exclude_properties:
|
|
1221
|
-
for field in self.exclude_properties:
|
|
1222
|
-
prop.pop(field, None)
|
|
1223
|
-
|
|
1224
|
-
for view in dumped[self.view_name]:
|
|
1225
|
-
for field_name in self.views_fields:
|
|
1226
|
-
if value := view.get(field_name):
|
|
1227
|
-
view[field_name] = value.removeprefix(self.default_space).removesuffix(self.default_version_wrapped)
|
|
1228
|
-
if value := view.get(self.view_implements):
|
|
1229
|
-
view[self.view_implements] = ",".join(
|
|
1230
|
-
parent.strip().removeprefix(self.default_space).removesuffix(self.default_version_wrapped)
|
|
1231
|
-
for parent in value.split(",")
|
|
1232
|
-
)
|
|
1233
|
-
if self.exclude_views:
|
|
1234
|
-
for field in self.exclude_views:
|
|
1235
|
-
view.pop(field, None)
|
|
1236
|
-
|
|
1237
|
-
for container in dumped[self.container_name]:
|
|
1238
|
-
for field_name in self.containers_fields:
|
|
1239
|
-
if value := container.get(field_name):
|
|
1240
|
-
container[field_name] = value.removeprefix(self.default_space)
|
|
1241
|
-
|
|
1242
|
-
if value := container.get(self.container_constraint):
|
|
1243
|
-
container[self.container_constraint] = ",".join(
|
|
1244
|
-
constraint.strip().removeprefix(self.default_space) for constraint in value.split(",")
|
|
1245
|
-
)
|
|
1246
|
-
if self.exclude_containers:
|
|
1247
|
-
for field in self.exclude_containers:
|
|
1248
|
-
container.pop(field, None)
|
|
1249
|
-
|
|
1250
|
-
if self.metadata_exclude:
|
|
1251
|
-
for field in self.metadata_exclude:
|
|
1252
|
-
dumped[self.metadata_name].pop(field, None)
|
|
1253
|
-
for field in self.exclude_top:
|
|
1254
|
-
dumped.pop(field, None)
|
|
1255
|
-
return dumped
|