cognite-neat 0.76.0__py3-none-any.whl → 0.76.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/routers/core.py +1 -1
  3. cognite/neat/app/api/routers/rules.py +1 -1
  4. cognite/neat/graph/extractors/_mock_graph_generator.py +2 -2
  5. cognite/neat/rules/_shared.py +1 -1
  6. cognite/neat/rules/analysis/_information_rules.py +3 -3
  7. cognite/neat/rules/exporters/_base.py +1 -1
  8. cognite/neat/rules/exporters/_rules2dms.py +8 -49
  9. cognite/neat/rules/exporters/_rules2excel.py +9 -3
  10. cognite/neat/rules/exporters/_rules2ontology.py +2 -2
  11. cognite/neat/rules/exporters/_rules2yaml.py +1 -1
  12. cognite/neat/rules/exporters/_validation.py +2 -2
  13. cognite/neat/rules/importers/_base.py +1 -1
  14. cognite/neat/rules/importers/_dms2rules.py +93 -108
  15. cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +1 -1
  16. cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +2 -3
  17. cognite/neat/rules/importers/_owl2rules/_owl2classes.py +1 -1
  18. cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +2 -2
  19. cognite/neat/rules/importers/_owl2rules/_owl2properties.py +1 -1
  20. cognite/neat/rules/importers/_owl2rules/_owl2rules.py +1 -1
  21. cognite/neat/rules/importers/_spreadsheet2rules.py +10 -4
  22. cognite/neat/rules/importers/_yaml2rules.py +3 -3
  23. cognite/neat/rules/issues/base.py +5 -0
  24. cognite/neat/rules/models/__init__.py +27 -0
  25. cognite/neat/rules/models/dms/__init__.py +18 -0
  26. cognite/neat/rules/models/dms/_converter.py +140 -0
  27. cognite/neat/rules/models/dms/_exporter.py +405 -0
  28. cognite/neat/rules/models/dms/_rules.py +379 -0
  29. cognite/neat/rules/models/{rules/_dms_rules_write.py → dms/_rules_input.py} +65 -57
  30. cognite/neat/rules/models/{rules/_dms_schema.py → dms/_schema.py} +10 -4
  31. cognite/neat/rules/models/dms/_serializer.py +126 -0
  32. cognite/neat/rules/models/dms/_validation.py +255 -0
  33. cognite/neat/rules/models/information/__init__.py +3 -0
  34. cognite/neat/rules/models/information/_converter.py +193 -0
  35. cognite/neat/rules/models/{rules/_information_rules.py → information/_rules.py} +35 -202
  36. cognite/neat/workflows/steps/data_contracts.py +1 -1
  37. cognite/neat/workflows/steps/lib/current/rules_exporter.py +9 -3
  38. cognite/neat/workflows/steps/lib/current/rules_importer.py +1 -1
  39. cognite/neat/workflows/steps/lib/current/rules_validator.py +1 -2
  40. {cognite_neat-0.76.0.dist-info → cognite_neat-0.76.2.dist-info}/METADATA +1 -1
  41. {cognite_neat-0.76.0.dist-info → cognite_neat-0.76.2.dist-info}/RECORD +50 -43
  42. cognite/neat/rules/models/rules/__init__.py +0 -14
  43. cognite/neat/rules/models/rules/_dms_architect_rules.py +0 -1255
  44. /cognite/neat/rules/models/{rules/_base.py → _base.py} +0 -0
  45. /cognite/neat/rules/models/{rdfpath.py → _rdfpath.py} +0 -0
  46. /cognite/neat/rules/models/{rules/_types → _types}/__init__.py +0 -0
  47. /cognite/neat/rules/models/{rules/_types → _types}/_base.py +0 -0
  48. /cognite/neat/rules/models/{rules/_types → _types}/_field.py +0 -0
  49. /cognite/neat/rules/models/{rules/_domain_rules.py → domain.py} +0 -0
  50. {cognite_neat-0.76.0.dist-info → cognite_neat-0.76.2.dist-info}/LICENSE +0 -0
  51. {cognite_neat-0.76.0.dist-info → cognite_neat-0.76.2.dist-info}/WHEEL +0 -0
  52. {cognite_neat-0.76.0.dist-info → cognite_neat-0.76.2.dist-info}/entry_points.txt +0 -0
@@ -1,1255 +0,0 @@
1
- import abc
2
- import math
3
- import re
4
- import sys
5
- import warnings
6
- from collections import defaultdict
7
- from collections.abc import Callable
8
- from datetime import datetime
9
- from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
10
-
11
- from cognite.client import data_modeling as dm
12
- from cognite.client.data_classes.data_modeling import PropertyType as CognitePropertyType
13
- from cognite.client.data_classes.data_modeling.containers import BTreeIndex
14
- from cognite.client.data_classes.data_modeling.views import (
15
- SingleEdgeConnectionApply,
16
- SingleReverseDirectRelationApply,
17
- ViewPropertyApply,
18
- )
19
- from pydantic import Field, field_serializer, field_validator, model_serializer, model_validator
20
- from pydantic_core.core_schema import SerializationInfo, ValidationInfo
21
- from rdflib import Namespace
22
-
23
- import cognite.neat.rules.issues.spreadsheet
24
- from cognite.neat.rules import issues
25
- from cognite.neat.rules.models.data_types import DataType
26
- from cognite.neat.rules.models.entities import (
27
- ClassEntity,
28
- ContainerEntity,
29
- ContainerEntityList,
30
- DMSNodeEntity,
31
- DMSUnknownEntity,
32
- ParentClassEntity,
33
- ReferenceEntity,
34
- UnknownEntity,
35
- URLEntity,
36
- ViewEntity,
37
- ViewEntityList,
38
- ViewPropertyEntity,
39
- )
40
- from cognite.neat.rules.models.rules._domain_rules import DomainRules
41
- from cognite.neat.rules.models.wrapped_entities import DMSFilter, HasDataFilter, NodeTypeFilter
42
-
43
- from ._base import (
44
- BaseMetadata,
45
- BaseRules,
46
- DataModelType,
47
- ExtensionCategory,
48
- RoleTypes,
49
- SchemaCompleteness,
50
- SheetEntity,
51
- SheetList,
52
- )
53
- from ._dms_schema import DMSSchema, PipelineSchema
54
- from ._types import (
55
- ExternalIdType,
56
- PropertyType,
57
- StrListType,
58
- VersionType,
59
- )
60
-
61
- if TYPE_CHECKING:
62
- from ._information_rules import InformationRules
63
-
64
- if sys.version_info >= (3, 11):
65
- from typing import Self
66
- else:
67
- from typing_extensions import Self
68
-
69
- _DEFAULT_VERSION = "1"
70
-
71
- subclasses = list(CognitePropertyType.__subclasses__())
72
- _PropertyType_by_name: dict[str, type[CognitePropertyType]] = {}
73
- for subclass in subclasses:
74
- subclasses.extend(subclass.__subclasses__())
75
- if abc.ABC in subclass.__bases__:
76
- continue
77
- try:
78
- _PropertyType_by_name[subclass._type.casefold()] = subclass
79
- except AttributeError:
80
- ...
81
- del subclasses # cleanup namespace
82
-
83
-
84
- class DMSMetadata(BaseMetadata):
85
- role: ClassVar[RoleTypes] = RoleTypes.dms_architect
86
- data_model_type: DataModelType = Field(DataModelType.solution, alias="dataModelType")
87
- schema_: SchemaCompleteness = Field(alias="schema")
88
- extension: ExtensionCategory = ExtensionCategory.addition
89
- space: ExternalIdType
90
- name: str | None = Field(
91
- None,
92
- description="Human readable name of the data model",
93
- min_length=1,
94
- max_length=255,
95
- )
96
- description: str | None = Field(None, min_length=1, max_length=1024)
97
- external_id: ExternalIdType = Field(alias="externalId")
98
- version: VersionType
99
- creator: StrListType
100
- created: datetime = Field(
101
- description=("Date of the data model creation"),
102
- )
103
- updated: datetime = Field(
104
- description=("Date of the data model update"),
105
- )
106
-
107
- @field_validator("*", mode="before")
108
- def strip_string(cls, value: Any) -> Any:
109
- if isinstance(value, str):
110
- return value.strip()
111
- return value
112
-
113
- @field_serializer("schema_", "extension", "data_model_type", when_used="always")
114
- @staticmethod
115
- def as_string(value: SchemaCompleteness | ExtensionCategory | DataModelType) -> str:
116
- return str(value)
117
-
118
- @field_validator("schema_", mode="plain")
119
- def as_enum_schema(cls, value: str) -> SchemaCompleteness:
120
- return SchemaCompleteness(value)
121
-
122
- @field_validator("extension", mode="plain")
123
- def as_enum_extension(cls, value: str) -> ExtensionCategory:
124
- return ExtensionCategory(value)
125
-
126
- @field_validator("data_model_type", mode="plain")
127
- def as_enum_model_type(cls, value: str) -> DataModelType:
128
- return DataModelType(value)
129
-
130
- @field_validator("description", mode="before")
131
- def nan_as_none(cls, value):
132
- if isinstance(value, float) and math.isnan(value):
133
- return None
134
- return value
135
-
136
- def as_space(self) -> dm.SpaceApply:
137
- return dm.SpaceApply(
138
- space=self.space,
139
- )
140
-
141
- def as_data_model_id(self) -> dm.DataModelId:
142
- return dm.DataModelId(space=self.space, external_id=self.external_id, version=self.version)
143
-
144
- def as_data_model(self) -> dm.DataModelApply:
145
- suffix = f"Creator: {', '.join(self.creator)}"
146
- if self.description:
147
- description = f"{self.description} Creator: {', '.join(self.creator)}"
148
- else:
149
- description = suffix
150
-
151
- return dm.DataModelApply(
152
- space=self.space,
153
- external_id=self.external_id,
154
- name=self.name or None,
155
- version=self.version or "missing",
156
- description=description,
157
- views=[],
158
- )
159
-
160
- @classmethod
161
- def _get_description_and_creator(cls, description_raw: str | None) -> tuple[str | None, list[str]]:
162
- if description_raw and (description_match := re.search(r"Creator: (.+)", description_raw)):
163
- creator = description_match.group(1).split(", ")
164
- description = description_raw.replace(description_match.string, "").strip() or None
165
- elif description_raw:
166
- creator = ["MISSING"]
167
- description = description_raw
168
- else:
169
- creator = ["MISSING"]
170
- description = None
171
- return description, creator
172
-
173
- @classmethod
174
- def from_data_model(cls, data_model: dm.DataModelApply) -> "DMSMetadata":
175
- description, creator = cls._get_description_and_creator(data_model.description)
176
- return cls(
177
- schema_=SchemaCompleteness.complete,
178
- space=data_model.space,
179
- name=data_model.name or None,
180
- description=description,
181
- external_id=data_model.external_id,
182
- version=data_model.version,
183
- creator=creator,
184
- created=datetime.now(),
185
- updated=datetime.now(),
186
- )
187
-
188
-
189
- class DMSProperty(SheetEntity):
190
- view: ViewEntity = Field(alias="View")
191
- view_property: str = Field(alias="View Property")
192
- name: str | None = Field(alias="Name", default=None)
193
- description: str | None = Field(alias="Description", default=None)
194
- connection: Literal["direct", "edge", "reverse"] | None = Field(None, alias="Connection")
195
- value_type: DataType | ViewPropertyEntity | ViewEntity | DMSUnknownEntity = Field(alias="Value Type")
196
- nullable: bool | None = Field(default=None, alias="Nullable")
197
- is_list: bool | None = Field(default=None, alias="Is List")
198
- default: str | int | dict | None = Field(None, alias="Default")
199
- reference: URLEntity | ReferenceEntity | None = Field(default=None, alias="Reference", union_mode="left_to_right")
200
- container: ContainerEntity | None = Field(None, alias="Container")
201
- container_property: str | None = Field(None, alias="Container Property")
202
- index: StrListType | None = Field(None, alias="Index")
203
- constraint: StrListType | None = Field(None, alias="Constraint")
204
- class_: ClassEntity = Field(alias="Class (linage)")
205
- property_: PropertyType = Field(alias="Property (linage)")
206
-
207
- @field_validator("nullable")
208
- def direct_relation_must_be_nullable(cls, value: Any, info: ValidationInfo) -> None:
209
- if info.data.get("connection") == "direct" and value is False:
210
- raise ValueError("Direct relation must be nullable")
211
- return value
212
-
213
- @field_validator("value_type", mode="after")
214
- def connections_value_type(
215
- cls, value: ViewPropertyEntity | ViewEntity | DMSUnknownEntity, info: ValidationInfo
216
- ) -> DataType | ViewPropertyEntity | ViewEntity | DMSUnknownEntity:
217
- if (connection := info.data.get("connection")) is None:
218
- return value
219
- if connection == "direct" and not isinstance(value, ViewEntity | DMSUnknownEntity):
220
- raise ValueError(f"Direct relation must have a value type that points to a view, got {value}")
221
- elif connection == "edge" and not isinstance(value, ViewEntity):
222
- raise ValueError(f"Edge connection must have a value type that points to a view, got {value}")
223
- elif connection == "reverse" and not isinstance(value, ViewPropertyEntity | ViewEntity):
224
- raise ValueError(
225
- f"Reverse connection must have a value type that points to a view or view property, got {value}"
226
- )
227
- return value
228
-
229
- @field_serializer("value_type", when_used="always")
230
- @staticmethod
231
- def as_dms_type(value_type: DataType | ViewPropertyEntity | ViewEntity) -> str:
232
- if isinstance(value_type, DataType):
233
- return value_type.dms._type
234
- else:
235
- return str(value_type)
236
-
237
-
238
- class DMSContainer(SheetEntity):
239
- container: ContainerEntity = Field(alias="Container")
240
- name: str | None = Field(alias="Name", default=None)
241
- description: str | None = Field(alias="Description", default=None)
242
- reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
243
- constraint: ContainerEntityList | None = Field(None, alias="Constraint")
244
- class_: ClassEntity = Field(alias="Class (linage)")
245
-
246
- def as_container(self) -> dm.ContainerApply:
247
- container_id = self.container.as_id()
248
- constraints: dict[str, dm.Constraint] = {}
249
- for constraint in self.constraint or []:
250
- requires = dm.RequiresConstraint(constraint.as_id())
251
- constraints[f"{constraint.space}_{constraint.external_id}"] = requires
252
-
253
- return dm.ContainerApply(
254
- space=container_id.space,
255
- external_id=container_id.external_id,
256
- name=self.name or None,
257
- description=self.description,
258
- constraints=constraints or None,
259
- properties={},
260
- )
261
-
262
- @classmethod
263
- def from_container(cls, container: dm.ContainerApply) -> "DMSContainer":
264
- constraints: list[ContainerEntity] = []
265
- for _, constraint_obj in (container.constraints or {}).items():
266
- if isinstance(constraint_obj, dm.RequiresConstraint):
267
- constraints.append(ContainerEntity.from_id(constraint_obj.require))
268
- # UniquenessConstraint it handled in the properties
269
- container_entity = ContainerEntity.from_id(container.as_id())
270
- return cls(
271
- class_=container_entity.as_class(),
272
- container=container_entity,
273
- name=container.name or None,
274
- description=container.description,
275
- constraint=constraints or None,
276
- )
277
-
278
-
279
- class DMSView(SheetEntity):
280
- view: ViewEntity = Field(alias="View")
281
- name: str | None = Field(alias="Name", default=None)
282
- description: str | None = Field(alias="Description", default=None)
283
- implements: ViewEntityList | None = Field(None, alias="Implements")
284
- reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
285
- filter_: HasDataFilter | NodeTypeFilter | None = Field(None, alias="Filter")
286
- in_model: bool = Field(True, alias="In Model")
287
- class_: ClassEntity = Field(alias="Class (linage)")
288
-
289
- def as_view(self) -> dm.ViewApply:
290
- view_id = self.view.as_id()
291
- return dm.ViewApply(
292
- space=view_id.space,
293
- external_id=view_id.external_id,
294
- version=view_id.version or _DEFAULT_VERSION,
295
- name=self.name or None,
296
- description=self.description,
297
- implements=[parent.as_id() for parent in self.implements or []] or None,
298
- properties={},
299
- )
300
-
301
- @classmethod
302
- def from_view(cls, view: dm.ViewApply, in_model: bool) -> "DMSView":
303
- view_entity = ViewEntity.from_id(view.as_id())
304
- class_entity = view_entity.as_class(skip_version=True)
305
-
306
- return cls(
307
- class_=class_entity,
308
- view=view_entity,
309
- description=view.description,
310
- name=view.name,
311
- implements=[ViewEntity.from_id(parent, _DEFAULT_VERSION) for parent in view.implements] or None,
312
- in_model=in_model,
313
- )
314
-
315
-
316
- class DMSRules(BaseRules):
317
- metadata: DMSMetadata = Field(alias="Metadata")
318
- properties: SheetList[DMSProperty] = Field(alias="Properties")
319
- views: SheetList[DMSView] = Field(alias="Views")
320
- containers: SheetList[DMSContainer] | None = Field(None, alias="Containers")
321
- reference: "DMSRules | None" = Field(None, alias="Reference")
322
-
323
- @field_validator("reference")
324
- def check_reference_of_reference(cls, value: "DMSRules | None", info: ValidationInfo) -> "DMSRules | None":
325
- if value is None:
326
- return None
327
- if value.reference is not None:
328
- raise ValueError("Reference rules cannot have a reference")
329
- if value.metadata.data_model_type == DataModelType.solution and (metadata := info.data.get("metadata")):
330
- warnings.warn(
331
- issues.dms.SolutionOnTopOfSolutionModelWarning(
332
- metadata.as_data_model_id(), value.metadata.as_data_model_id()
333
- ),
334
- stacklevel=2,
335
- )
336
- return value
337
-
338
- @field_validator("views")
339
- def matching_version_and_space(cls, value: SheetList[DMSView], info: ValidationInfo) -> SheetList[DMSView]:
340
- if not (metadata := info.data.get("metadata")):
341
- return value
342
- model_version = metadata.version
343
- if different_version := [view.view.as_id() for view in value if view.view.version != model_version]:
344
- warnings.warn(issues.dms.ViewModelVersionNotMatchingWarning(different_version, model_version), stacklevel=2)
345
- if different_space := [view.view.as_id() for view in value if view.view.space != metadata.space]:
346
- warnings.warn(issues.dms.ViewModelSpaceNotMatchingWarning(different_space, metadata.space), stacklevel=2)
347
- return value
348
-
349
- @field_validator("views")
350
- def matching_version(cls, value: SheetList[DMSView], info: ValidationInfo) -> SheetList[DMSView]:
351
- if not (metadata := info.data.get("metadata")):
352
- return value
353
- model_version = metadata.version
354
- if different_version := [view.view.as_id() for view in value if view.view.version != model_version]:
355
- warnings.warn(issues.dms.ViewModelVersionNotMatchingWarning(different_version, model_version), stacklevel=2)
356
- return value
357
-
358
- @model_validator(mode="after")
359
- def consistent_container_properties(self) -> "DMSRules":
360
- container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, DMSProperty]]] = defaultdict(list)
361
- for prop_no, prop in enumerate(self.properties):
362
- if prop.container and prop.container_property:
363
- container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
364
-
365
- errors: list[cognite.neat.rules.issues.spreadsheet.InconsistentContainerDefinitionError] = []
366
- for (container, prop_name), properties in container_properties_by_id.items():
367
- if len(properties) == 1:
368
- continue
369
- container_id = container.as_id()
370
- row_numbers = {prop_no for prop_no, _ in properties}
371
- value_types = {prop.value_type for _, prop in properties if prop.value_type}
372
- if len(value_types) > 1:
373
- errors.append(
374
- cognite.neat.rules.issues.spreadsheet.MultiValueTypeError(
375
- container_id,
376
- prop_name,
377
- row_numbers,
378
- {v.dms._type if isinstance(v, DataType) else str(v) for v in value_types},
379
- )
380
- )
381
- list_definitions = {prop.is_list for _, prop in properties if prop.is_list is not None}
382
- if len(list_definitions) > 1:
383
- errors.append(
384
- cognite.neat.rules.issues.spreadsheet.MultiValueIsListError(
385
- container_id, prop_name, row_numbers, list_definitions
386
- )
387
- )
388
- nullable_definitions = {prop.nullable for _, prop in properties if prop.nullable is not None}
389
- if len(nullable_definitions) > 1:
390
- errors.append(
391
- cognite.neat.rules.issues.spreadsheet.MultiNullableError(
392
- container_id, prop_name, row_numbers, nullable_definitions
393
- )
394
- )
395
- default_definitions = {prop.default for _, prop in properties if prop.default is not None}
396
- if len(default_definitions) > 1:
397
- errors.append(
398
- cognite.neat.rules.issues.spreadsheet.MultiDefaultError(
399
- container_id, prop_name, row_numbers, list(default_definitions)
400
- )
401
- )
402
- index_definitions = {",".join(prop.index) for _, prop in properties if prop.index is not None}
403
- if len(index_definitions) > 1:
404
- errors.append(
405
- cognite.neat.rules.issues.spreadsheet.MultiIndexError(
406
- container_id, prop_name, row_numbers, index_definitions
407
- )
408
- )
409
- constraint_definitions = {
410
- ",".join(prop.constraint) for _, prop in properties if prop.constraint is not None
411
- }
412
- if len(constraint_definitions) > 1:
413
- errors.append(
414
- cognite.neat.rules.issues.spreadsheet.MultiUniqueConstraintError(
415
- container_id, prop_name, row_numbers, constraint_definitions
416
- )
417
- )
418
-
419
- # This sets the container definition for all the properties where it is not defined.
420
- # This allows the user to define the container only once.
421
- value_type = next(iter(value_types))
422
- list_definition = next(iter(list_definitions)) if list_definitions else None
423
- nullable_definition = next(iter(nullable_definitions)) if nullable_definitions else None
424
- default_definition = next(iter(default_definitions)) if default_definitions else None
425
- index_definition = next(iter(index_definitions)).split(",") if index_definitions else None
426
- constraint_definition = next(iter(constraint_definitions)).split(",") if constraint_definitions else None
427
- for _, prop in properties:
428
- prop.value_type = value_type
429
- prop.is_list = prop.is_list or list_definition
430
- prop.nullable = prop.nullable or nullable_definition
431
- prop.default = prop.default or default_definition
432
- prop.index = prop.index or index_definition
433
- prop.constraint = prop.constraint or constraint_definition
434
-
435
- if errors:
436
- raise issues.MultiValueError(errors)
437
- return self
438
-
439
- @model_validator(mode="after")
440
- def referenced_views_and_containers_are_existing(self) -> "DMSRules":
441
- # There two checks are done in the same method to raise all the errors at once.
442
- defined_views = {view.view.as_id() for view in self.views}
443
-
444
- errors: list[issues.NeatValidationError] = []
445
- for prop_no, prop in enumerate(self.properties):
446
- if prop.view and (view_id := prop.view.as_id()) not in defined_views:
447
- errors.append(
448
- cognite.neat.rules.issues.spreadsheet.NonExistingViewError(
449
- column="View",
450
- row=prop_no,
451
- type="value_error.missing",
452
- view_id=view_id,
453
- msg="",
454
- input=None,
455
- url=None,
456
- )
457
- )
458
- if self.metadata.schema_ is SchemaCompleteness.complete:
459
- defined_containers = {container.container.as_id() for container in self.containers or []}
460
- for prop_no, prop in enumerate(self.properties):
461
- if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
462
- errors.append(
463
- cognite.neat.rules.issues.spreadsheet.NonExistingContainerError(
464
- column="Container",
465
- row=prop_no,
466
- type="value_error.missing",
467
- container_id=container_id,
468
- msg="",
469
- input=None,
470
- url=None,
471
- )
472
- )
473
- for _container_no, container in enumerate(self.containers or []):
474
- for constraint_no, constraint in enumerate(container.constraint or []):
475
- if constraint.as_id() not in defined_containers:
476
- errors.append(
477
- cognite.neat.rules.issues.spreadsheet.NonExistingContainerError(
478
- column="Constraint",
479
- row=constraint_no,
480
- type="value_error.missing",
481
- container_id=constraint.as_id(),
482
- msg="",
483
- input=None,
484
- url=None,
485
- )
486
- )
487
- if errors:
488
- raise issues.MultiValueError(errors)
489
- return self
490
-
491
- @model_validator(mode="after")
492
- def validate_extension(self) -> "DMSRules":
493
- if self.metadata.schema_ is not SchemaCompleteness.extended:
494
- return self
495
- if not self.reference:
496
- raise ValueError("The schema is set to 'extended', but no reference rules are provided to validate against")
497
- is_solution = self.metadata.space != self.reference.metadata.space
498
- if is_solution:
499
- return self
500
- if self.metadata.extension is ExtensionCategory.rebuild:
501
- # Everything is allowed
502
- return self
503
- # Is an extension of an existing model.
504
- user_schema = self.as_schema(include_ref=False)
505
- ref_schema = self.reference.as_schema()
506
- new_containers = {container.as_id(): container for container in user_schema.containers}
507
- existing_containers = {container.as_id(): container for container in ref_schema.containers}
508
-
509
- errors: list[issues.NeatValidationError] = []
510
- for container_id, container in new_containers.items():
511
- existing_container = existing_containers.get(container_id)
512
- if not existing_container or existing_container == container:
513
- # No problem
514
- continue
515
- new_dumped = container.dump()
516
- existing_dumped = existing_container.dump()
517
- changed_attributes, changed_properties = self._changed_attributes_and_properties(
518
- new_dumped, existing_dumped
519
- )
520
- errors.append(
521
- issues.dms.ChangingContainerError(
522
- container_id=container_id,
523
- changed_properties=changed_properties or None,
524
- changed_attributes=changed_attributes or None,
525
- )
526
- )
527
-
528
- if self.metadata.extension is ExtensionCategory.reshape and errors:
529
- raise issues.MultiValueError(errors)
530
- elif self.metadata.extension is ExtensionCategory.reshape:
531
- # Reshape allows changes to views
532
- return self
533
-
534
- new_views = {view.as_id(): view for view in user_schema.views}
535
- existing_views = {view.as_id(): view for view in ref_schema.views}
536
- for view_id, view in new_views.items():
537
- existing_view = existing_views.get(view_id)
538
- if not existing_view or existing_view == view:
539
- # No problem
540
- continue
541
- changed_attributes, changed_properties = self._changed_attributes_and_properties(
542
- view.dump(), existing_view.dump()
543
- )
544
- errors.append(
545
- issues.dms.ChangingViewError(
546
- view_id=view_id,
547
- changed_properties=changed_properties or None,
548
- changed_attributes=changed_attributes or None,
549
- )
550
- )
551
-
552
- if errors:
553
- raise issues.MultiValueError(errors)
554
- return self
555
-
556
- @staticmethod
557
- def _changed_attributes_and_properties(
558
- new_dumped: dict[str, Any], existing_dumped: dict[str, Any]
559
- ) -> tuple[list[str], list[str]]:
560
- """Helper method to find the changed attributes and properties between two containers or views."""
561
- new_attributes = {key: value for key, value in new_dumped.items() if key != "properties"}
562
- existing_attributes = {key: value for key, value in existing_dumped.items() if key != "properties"}
563
- changed_attributes = [key for key in new_attributes if new_attributes[key] != existing_attributes.get(key)]
564
- new_properties = new_dumped.get("properties", {})
565
- existing_properties = existing_dumped.get("properties", {})
566
- changed_properties = [prop for prop in new_properties if new_properties[prop] != existing_properties.get(prop)]
567
- return changed_attributes, changed_properties
568
-
569
- @model_validator(mode="after")
570
- def validate_schema(self) -> "DMSRules":
571
- if self.metadata.schema_ is SchemaCompleteness.partial:
572
- return self
573
- elif self.metadata.schema_ is SchemaCompleteness.complete:
574
- rules: DMSRules = self
575
- elif self.metadata.schema_ is SchemaCompleteness.extended:
576
- if not self.reference:
577
- raise ValueError(
578
- "The schema is set to 'extended', but no reference rules are provided to validate against"
579
- )
580
- # This is an extension of the reference rules, we need to merge the two
581
- rules = self.model_copy(deep=True)
582
- rules.properties.extend(self.reference.properties.data)
583
- existing_views = {view.view.as_id() for view in rules.views}
584
- rules.views.extend([view for view in self.reference.views if view.view.as_id() not in existing_views])
585
- if rules.containers and self.reference.containers:
586
- existing_containers = {container.container.as_id() for container in rules.containers.data}
587
- rules.containers.extend(
588
- [
589
- container
590
- for container in self.reference.containers
591
- if container.container.as_id() not in existing_containers
592
- ]
593
- )
594
- elif not rules.containers and self.reference.containers:
595
- rules.containers = self.reference.containers
596
- else:
597
- raise ValueError("Unknown schema completeness")
598
-
599
- schema = rules.as_schema()
600
- errors = schema.validate()
601
- if errors:
602
- raise issues.MultiValueError(errors)
603
- return self
604
-
605
- @model_serializer(mode="wrap", when_used="always")
606
- def dms_rules_serialization(
607
- self,
608
- handler: Callable,
609
- info: SerializationInfo,
610
- ) -> dict[str, Any]:
611
- dumped = cast(dict[str, Any], handler(self, info))
612
- space, version = self.metadata.space, self.metadata.version
613
- return _DMSRulesSerializer(info, space, version).clean(dumped)
614
-
615
- def as_schema(
616
- self, include_ref: bool = False, include_pipeline: bool = False, instance_space: str | None = None
617
- ) -> DMSSchema:
618
- return _DMSExporter(self, include_ref, include_pipeline, instance_space).to_schema()
619
-
620
- def as_information_architect_rules(self) -> "InformationRules":
621
- return _DMSRulesConverter(self).as_information_architect_rules()
622
-
623
- def as_domain_expert_rules(self) -> DomainRules:
624
- return _DMSRulesConverter(self).as_domain_rules()
625
-
626
- def reference_self(self) -> Self:
627
- new_rules = self.model_copy(deep=True)
628
- for prop in new_rules.properties:
629
- prop.reference = ReferenceEntity(
630
- prefix=prop.view.prefix, suffix=prop.view.suffix, version=prop.view.version, property=prop.property_
631
- )
632
- view: DMSView
633
- for view in new_rules.views:
634
- view.reference = ReferenceEntity(
635
- prefix=view.view.prefix, suffix=view.view.suffix, version=view.view.version
636
- )
637
- container: DMSContainer
638
- for container in new_rules.containers or []:
639
- container.reference = ReferenceEntity(prefix=container.container.prefix, suffix=container.container.suffix)
640
- return new_rules
641
-
642
-
643
- class _DMSExporter:
644
- """The DMS Exporter is responsible for exporting the DMSRules to a DMSSchema.
645
-
646
- This kept in this location such that it can be used by the DMSRules to validate the schema.
647
- (This module cannot have a dependency on the exporter module, as it would create a circular dependency.)
648
-
649
- Args
650
- include_pipeline (bool): If True, the pipeline will be included with the schema. Pipeline means the
651
- raw tables and transformations necessary to populate the data model.
652
- instance_space (str): The space to use for the instance. Defaults to None,`Rules.metadata.space` will be used
653
- """
654
-
655
- def __init__(
656
- self,
657
- rules: DMSRules,
658
- include_ref: bool = True,
659
- include_pipeline: bool = False,
660
- instance_space: str | None = None,
661
- ):
662
- self.include_ref = include_ref
663
- self.include_pipeline = include_pipeline
664
- self.instance_space = instance_space
665
- self.rules = rules
666
- self._ref_schema = rules.reference.as_schema() if rules.reference else None
667
- if self._ref_schema:
668
- # We skip version as that will always be missing in the reference
669
- self._ref_views_by_id = {dm.ViewId(view.space, view.external_id): view for view in self._ref_schema.views}
670
- else:
671
- self._ref_views_by_id = {}
672
-
673
- def to_schema(self) -> DMSSchema:
674
- rules = self.rules
675
- container_properties_by_id, view_properties_by_id = self._gather_properties()
676
- containers = self._create_containers(container_properties_by_id)
677
-
678
- views, node_types = self._create_views_with_node_types(view_properties_by_id)
679
-
680
- views_not_in_model = {view.view.as_id() for view in rules.views if not view.in_model}
681
- data_model = rules.metadata.as_data_model()
682
- data_model.views = sorted(
683
- [view_id for view_id in views.as_ids() if view_id not in views_not_in_model],
684
- key=lambda v: v.as_tuple(), # type: ignore[union-attr]
685
- )
686
-
687
- spaces = self._create_spaces(rules.metadata, containers, views, data_model)
688
-
689
- output = DMSSchema(
690
- spaces=spaces,
691
- data_models=dm.DataModelApplyList([data_model]),
692
- views=views,
693
- containers=containers,
694
- node_types=node_types,
695
- )
696
- if self.include_pipeline:
697
- return PipelineSchema.from_dms(output, self.instance_space)
698
-
699
- if self._ref_schema and self.include_ref:
700
- output.frozen_ids.update(self._ref_schema.node_types.as_ids())
701
- output.frozen_ids.update(self._ref_schema.views.as_ids())
702
- output.frozen_ids.update(self._ref_schema.containers.as_ids())
703
- output.node_types.extend(self._ref_schema.node_types)
704
- output.views.extend(self._ref_schema.views)
705
- output.containers.extend(self._ref_schema.containers)
706
- output.data_models.extend(self._ref_schema.data_models)
707
-
708
- return output
709
-
710
- def _create_spaces(
711
- self,
712
- metadata: DMSMetadata,
713
- containers: dm.ContainerApplyList,
714
- views: dm.ViewApplyList,
715
- data_model: dm.DataModelApply,
716
- ) -> dm.SpaceApplyList:
717
- used_spaces = {container.space for container in containers} | {view.space for view in views}
718
- if len(used_spaces) == 1:
719
- # We skip the default space and only use this space for the data model
720
- data_model.space = used_spaces.pop()
721
- spaces = dm.SpaceApplyList([dm.SpaceApply(space=data_model.space)])
722
- else:
723
- used_spaces.add(metadata.space)
724
- spaces = dm.SpaceApplyList([dm.SpaceApply(space=space) for space in used_spaces])
725
- if self.instance_space and self.instance_space not in {space.space for space in spaces}:
726
- spaces.append(dm.SpaceApply(space=self.instance_space, name=self.instance_space))
727
- return spaces
728
-
729
- def _create_views_with_node_types(
730
- self,
731
- view_properties_by_id: dict[dm.ViewId, list[DMSProperty]],
732
- ) -> tuple[dm.ViewApplyList, dm.NodeApplyList]:
733
- views = dm.ViewApplyList([dms_view.as_view() for dms_view in self.rules.views])
734
- dms_view_by_id = {dms_view.view.as_id(): dms_view for dms_view in self.rules.views}
735
-
736
- for view in views:
737
- view_id = view.as_id()
738
- view.properties = {}
739
- if not (view_properties := view_properties_by_id.get(view_id)):
740
- continue
741
- for prop in view_properties:
742
- view_property = self._create_view_property(prop, view_properties_by_id)
743
- if view_property is not None:
744
- view.properties[prop.view_property] = view_property
745
-
746
- data_model_type = self.rules.metadata.data_model_type
747
- unique_node_types: set[dm.NodeId] = set()
748
- parent_views = {parent for view in views for parent in view.implements or []}
749
- for view in views:
750
- dms_view = dms_view_by_id.get(view.as_id())
751
- dms_properties = view_properties_by_id.get(view.as_id(), [])
752
- view_filter = self._create_view_filter(view, dms_view, data_model_type, dms_properties)
753
-
754
- view.filter = view_filter.as_dms_filter()
755
-
756
- if isinstance(view_filter, NodeTypeFilter):
757
- unique_node_types.update(view_filter.nodes)
758
- if view.as_id() in parent_views:
759
- warnings.warn(issues.dms.NodeTypeFilterOnParentViewWarning(view.as_id()), stacklevel=2)
760
- elif isinstance(view_filter, HasDataFilter) and data_model_type == DataModelType.solution:
761
- if dms_view and isinstance(dms_view.reference, ReferenceEntity):
762
- references = {dms_view.reference.as_view_id()}
763
- elif any(True for prop in dms_properties if isinstance(prop.reference, ReferenceEntity)):
764
- references = {
765
- prop.reference.as_view_id()
766
- for prop in dms_properties
767
- if isinstance(prop.reference, ReferenceEntity)
768
- }
769
- else:
770
- continue
771
- warnings.warn(
772
- issues.dms.HasDataFilterOnViewWithReferencesWarning(view.as_id(), list(references)), stacklevel=2
773
- )
774
-
775
- return views, dm.NodeApplyList(
776
- [dm.NodeApply(space=node.space, external_id=node.external_id) for node in unique_node_types]
777
- )
778
-
779
- @classmethod
780
- def _create_edge_type_from_prop(cls, prop: DMSProperty) -> dm.DirectRelationReference:
781
- if isinstance(prop.reference, ReferenceEntity):
782
- ref_view_prop = prop.reference.as_view_property_id()
783
- return cls._create_edge_type_from_view_id(cast(dm.ViewId, ref_view_prop.source), ref_view_prop.property)
784
- else:
785
- return cls._create_edge_type_from_view_id(prop.view.as_id(), prop.view_property)
786
-
787
- @staticmethod
788
- def _create_edge_type_from_view_id(view_id: dm.ViewId, property_: str) -> dm.DirectRelationReference:
789
- return dm.DirectRelationReference(
790
- space=view_id.space,
791
- # This is the same convention as used when converting GraphQL to DMS
792
- external_id=f"{view_id.external_id}.{property_}",
793
- )
794
-
795
- def _create_containers(
796
- self,
797
- container_properties_by_id: dict[dm.ContainerId, list[DMSProperty]],
798
- ) -> dm.ContainerApplyList:
799
- containers = dm.ContainerApplyList(
800
- [dms_container.as_container() for dms_container in self.rules.containers or []]
801
- )
802
- container_to_drop = set()
803
- for container in containers:
804
- container_id = container.as_id()
805
- if not (container_properties := container_properties_by_id.get(container_id)):
806
- warnings.warn(issues.dms.EmptyContainerWarning(container_id=container_id), stacklevel=2)
807
- container_to_drop.add(container_id)
808
- continue
809
- for prop in container_properties:
810
- if prop.container_property is None:
811
- continue
812
- if isinstance(prop.value_type, DataType):
813
- type_cls = prop.value_type.dms
814
- else:
815
- type_cls = dm.DirectRelation
816
-
817
- type_ = type_cls(is_list=prop.is_list or False)
818
- container.properties[prop.container_property] = dm.ContainerProperty(
819
- type=type_,
820
- nullable=prop.nullable if prop.nullable is not None else True,
821
- default_value=prop.default,
822
- name=prop.name,
823
- description=prop.description,
824
- )
825
-
826
- uniqueness_properties: dict[str, set[str]] = defaultdict(set)
827
- for prop in container_properties:
828
- if prop.container_property is not None:
829
- for constraint in prop.constraint or []:
830
- uniqueness_properties[constraint].add(prop.container_property)
831
- for constraint_name, properties in uniqueness_properties.items():
832
- container.constraints = container.constraints or {}
833
- container.constraints[constraint_name] = dm.UniquenessConstraint(properties=list(properties))
834
-
835
- index_properties: dict[str, set[str]] = defaultdict(set)
836
- for prop in container_properties:
837
- if prop.container_property is not None:
838
- for index in prop.index or []:
839
- index_properties[index].add(prop.container_property)
840
- for index_name, properties in index_properties.items():
841
- container.indexes = container.indexes or {}
842
- container.indexes[index_name] = BTreeIndex(properties=list(properties))
843
-
844
- # We might drop containers we convert direct relations of list into multi-edge connections
845
- # which do not have a container.
846
- for container in containers:
847
- if container.constraints:
848
- container.constraints = {
849
- name: const
850
- for name, const in container.constraints.items()
851
- if not (isinstance(const, dm.RequiresConstraint) and const.require in container_to_drop)
852
- }
853
- return dm.ContainerApplyList(
854
- [container for container in containers if container.as_id() not in container_to_drop]
855
- )
856
-
857
- def _gather_properties(self) -> tuple[dict[dm.ContainerId, list[DMSProperty]], dict[dm.ViewId, list[DMSProperty]]]:
858
- container_properties_by_id: dict[dm.ContainerId, list[DMSProperty]] = defaultdict(list)
859
- view_properties_by_id: dict[dm.ViewId, list[DMSProperty]] = defaultdict(list)
860
- for prop in self.rules.properties:
861
- view_id = prop.view.as_id()
862
- view_properties_by_id[view_id].append(prop)
863
-
864
- if prop.container and prop.container_property:
865
- container_id = prop.container.as_id()
866
- container_properties_by_id[container_id].append(prop)
867
-
868
- return container_properties_by_id, view_properties_by_id
869
-
870
- def _create_view_filter(
871
- self,
872
- view: dm.ViewApply,
873
- dms_view: DMSView | None,
874
- data_model_type: DataModelType,
875
- dms_properties: list[DMSProperty],
876
- ) -> DMSFilter:
877
- selected_filter_name = (dms_view and dms_view.filter_ and dms_view.filter_.name) or ""
878
- if dms_view and dms_view.filter_ and not dms_view.filter_.is_empty:
879
- # Has Explicit Filter, use it
880
- return dms_view.filter_
881
-
882
- if data_model_type == DataModelType.solution and selected_filter_name in [NodeTypeFilter.name, ""]:
883
- if (
884
- dms_view
885
- and isinstance(dms_view.reference, ReferenceEntity)
886
- and not dms_properties
887
- and (ref_view := self._ref_views_by_id.get(dms_view.reference.as_view_id()))
888
- and ref_view.filter
889
- ):
890
- # No new properties, only reference, reuse the reference filter
891
- return DMSFilter.from_dms_filter(ref_view.filter)
892
- else:
893
- referenced_node_ids = {
894
- prop.reference.as_node_entity()
895
- for prop in dms_properties
896
- if isinstance(prop.reference, ReferenceEntity)
897
- }
898
- if dms_view and isinstance(dms_view.reference, ReferenceEntity):
899
- referenced_node_ids.add(dms_view.reference.as_node_entity())
900
- if referenced_node_ids:
901
- return NodeTypeFilter(inner=list(referenced_node_ids))
902
-
903
- # Enterprise Model or (Solution + HasData)
904
- ref_containers = view.referenced_containers()
905
- if not ref_containers or selected_filter_name == HasDataFilter.name:
906
- # Child filter without container properties
907
- if selected_filter_name == HasDataFilter.name:
908
- warnings.warn(issues.dms.HasDataFilterOnNoPropertiesViewWarning(view.as_id()), stacklevel=2)
909
- return NodeTypeFilter(inner=[DMSNodeEntity(space=view.space, externalId=view.external_id)])
910
- else:
911
- # HasData or not provided (this is the default)
912
- return HasDataFilter(inner=[ContainerEntity.from_id(id_) for id_ in ref_containers])
913
-
914
- def _create_view_property(
915
- self, prop: DMSProperty, view_properties_by_id: dict[dm.ViewId, list[DMSProperty]]
916
- ) -> ViewPropertyApply | None:
917
- if prop.container and prop.container_property:
918
- container_prop_identifier = prop.container_property
919
- extra_args: dict[str, Any] = {}
920
- if prop.connection == "direct":
921
- if isinstance(prop.value_type, ViewEntity):
922
- extra_args["source"] = prop.value_type.as_id()
923
- elif isinstance(prop.value_type, DMSUnknownEntity):
924
- extra_args["source"] = None
925
- else:
926
- # Should have been validated.
927
- raise ValueError(
928
- "If this error occurs it is a bug in NEAT, please report"
929
- f"Debug Info, Invalid valueType direct: {prop.model_dump_json()}"
930
- )
931
- elif prop.connection is not None:
932
- # Should have been validated.
933
- raise ValueError(
934
- "If this error occurs it is a bug in NEAT, please report"
935
- f"Debug Info, Invalid connection: {prop.model_dump_json()}"
936
- )
937
- return dm.MappedPropertyApply(
938
- container=prop.container.as_id(),
939
- container_property_identifier=container_prop_identifier,
940
- name=prop.name,
941
- description=prop.description,
942
- **extra_args,
943
- )
944
- elif prop.connection == "edge":
945
- if isinstance(prop.value_type, ViewEntity):
946
- source_view_id = prop.value_type.as_id()
947
- else:
948
- # Should have been validated.
949
- raise ValueError(
950
- "If this error occurs it is a bug in NEAT, please report"
951
- f"Debug Info, Invalid valueType edge: {prop.model_dump_json()}"
952
- )
953
- edge_cls: type[dm.EdgeConnectionApply] = dm.MultiEdgeConnectionApply
954
- # If is_list is not set, we default to a MultiEdgeConnection
955
- if prop.is_list is False:
956
- edge_cls = SingleEdgeConnectionApply
957
-
958
- return edge_cls(
959
- type=self._create_edge_type_from_prop(prop),
960
- source=source_view_id,
961
- direction="outwards",
962
- name=prop.name,
963
- description=prop.description,
964
- )
965
- elif prop.connection == "reverse":
966
- reverse_prop_id: str | None = None
967
- if isinstance(prop.value_type, ViewPropertyEntity):
968
- source_view_id = prop.value_type.as_view_id()
969
- reverse_prop_id = prop.value_type.property_
970
- elif isinstance(prop.value_type, ViewEntity):
971
- source_view_id = prop.value_type.as_id()
972
- else:
973
- # Should have been validated.
974
- raise ValueError(
975
- "If this error occurs it is a bug in NEAT, please report"
976
- f"Debug Info, Invalid valueType reverse connection: {prop.model_dump_json()}"
977
- )
978
- reverse_prop: DMSProperty | None = None
979
- if reverse_prop_id is not None:
980
- reverse_prop = next(
981
- (
982
- prop
983
- for prop in view_properties_by_id.get(source_view_id, [])
984
- if prop.property_ == reverse_prop_id
985
- ),
986
- None,
987
- )
988
-
989
- if reverse_prop is None:
990
- warnings.warn(
991
- issues.dms.ReverseRelationMissingOtherSideWarning(source_view_id, prop.view_property),
992
- stacklevel=2,
993
- )
994
-
995
- if reverse_prop is None or reverse_prop.connection == "edge":
996
- inwards_edge_cls = (
997
- dm.MultiEdgeConnectionApply if prop.is_list in [True, None] else SingleEdgeConnectionApply
998
- )
999
- return inwards_edge_cls(
1000
- type=self._create_edge_type_from_prop(reverse_prop or prop),
1001
- source=source_view_id,
1002
- name=prop.name,
1003
- description=prop.description,
1004
- direction="inwards",
1005
- )
1006
- elif reverse_prop_id and reverse_prop and reverse_prop.connection == "direct":
1007
- reverse_direct_cls = (
1008
- dm.MultiReverseDirectRelationApply if prop.is_list is True else SingleReverseDirectRelationApply
1009
- )
1010
- return reverse_direct_cls(
1011
- source=source_view_id,
1012
- through=dm.PropertyId(source=source_view_id, property=reverse_prop_id),
1013
- name=prop.name,
1014
- description=prop.description,
1015
- )
1016
- else:
1017
- return None
1018
-
1019
- elif prop.view and prop.view_property and prop.connection:
1020
- warnings.warn(
1021
- issues.dms.UnsupportedConnectionWarning(prop.view.as_id(), prop.view_property, prop.connection or ""),
1022
- stacklevel=2,
1023
- )
1024
- return None
1025
-
1026
-
1027
- class _DMSRulesConverter:
1028
- def __init__(self, dms: DMSRules):
1029
- self.dms = dms
1030
-
1031
- def as_domain_rules(self) -> "DomainRules":
1032
- raise NotImplementedError("DomainRules not implemented yet")
1033
-
1034
- def as_information_architect_rules(
1035
- self,
1036
- created: datetime | None = None,
1037
- updated: datetime | None = None,
1038
- name: str | None = None,
1039
- namespace: Namespace | None = None,
1040
- ) -> "InformationRules":
1041
- from ._information_rules import InformationClass, InformationMetadata, InformationProperty, InformationRules
1042
-
1043
- dms = self.dms.metadata
1044
- prefix = dms.space
1045
-
1046
- metadata = InformationMetadata(
1047
- schema_=dms.schema_,
1048
- prefix=prefix,
1049
- namespace=namespace or Namespace(f"https://purl.orgl/neat/{prefix}/"),
1050
- version=dms.version,
1051
- name=name or dms.name or "Missing name",
1052
- creator=dms.creator,
1053
- created=dms.created or created or datetime.now(),
1054
- updated=dms.updated or updated or datetime.now(),
1055
- )
1056
-
1057
- classes = [
1058
- InformationClass(
1059
- # we do not want a version in class as we use URI for the class
1060
- class_=ClassEntity(prefix=view.class_.prefix, suffix=view.class_.suffix),
1061
- description=view.description,
1062
- parent=[
1063
- # we do not want a version in class as we use URI for the class
1064
- ParentClassEntity(prefix=implemented_view.prefix, suffix=implemented_view.suffix)
1065
- # We only want parents in the same namespace, parent in a different namespace is a reference
1066
- for implemented_view in view.implements or []
1067
- if implemented_view.prefix == view.class_.prefix
1068
- ],
1069
- reference=self._get_class_reference(view),
1070
- )
1071
- for view in self.dms.views
1072
- ]
1073
-
1074
- properties: list[InformationProperty] = []
1075
- value_type: DataType | ClassEntity | str
1076
- for property_ in self.dms.properties:
1077
- if isinstance(property_.value_type, DataType):
1078
- value_type = property_.value_type
1079
- elif isinstance(property_.value_type, ViewEntity | ViewPropertyEntity):
1080
- value_type = ClassEntity(
1081
- prefix=property_.value_type.prefix,
1082
- suffix=property_.value_type.suffix,
1083
- )
1084
- elif isinstance(property_.value_type, DMSUnknownEntity):
1085
- value_type = UnknownEntity()
1086
- else:
1087
- raise ValueError(f"Unsupported value type: {property_.value_type.type_}")
1088
-
1089
- properties.append(
1090
- InformationProperty(
1091
- # Removing version
1092
- class_=ClassEntity(suffix=property_.class_.suffix, prefix=property_.class_.prefix),
1093
- property_=property_.view_property,
1094
- value_type=value_type,
1095
- description=property_.description,
1096
- min_count=0 if property_.nullable or property_.nullable is None else 1,
1097
- max_count=float("inf") if property_.is_list or property_.nullable is None else 1,
1098
- reference=self._get_property_reference(property_),
1099
- )
1100
- )
1101
-
1102
- return InformationRules(
1103
- metadata=metadata,
1104
- properties=SheetList[InformationProperty](data=properties),
1105
- classes=SheetList[InformationClass](data=classes),
1106
- reference=self.dms.reference and self.dms.reference.as_information_architect_rules(), # type: ignore[arg-type]
1107
- )
1108
-
1109
- @classmethod
1110
- def _get_class_reference(cls, view: DMSView) -> ReferenceEntity | None:
1111
- parents_other_namespace = [parent for parent in view.implements or [] if parent.prefix != view.class_.prefix]
1112
- if len(parents_other_namespace) == 0:
1113
- return None
1114
- if len(parents_other_namespace) > 1:
1115
- warnings.warn(
1116
- issues.dms.MultipleReferenceWarning(
1117
- view_id=view.view.as_id(), implements=[v.as_id() for v in parents_other_namespace]
1118
- ),
1119
- stacklevel=2,
1120
- )
1121
- other_parent = parents_other_namespace[0]
1122
-
1123
- return ReferenceEntity(prefix=other_parent.prefix, suffix=other_parent.suffix)
1124
-
1125
- @classmethod
1126
- def _get_property_reference(cls, property_: DMSProperty) -> ReferenceEntity | None:
1127
- has_container_other_namespace = property_.container and property_.container.prefix != property_.class_.prefix
1128
- if not has_container_other_namespace:
1129
- return None
1130
- container = cast(ContainerEntity, property_.container)
1131
- return ReferenceEntity(
1132
- prefix=container.prefix,
1133
- suffix=container.suffix,
1134
- property=property_.container_property,
1135
- )
1136
-
1137
-
1138
- class _DMSRulesSerializer:
1139
- # These are the fields that need to be cleaned from the default space and version
1140
- PROPERTIES_FIELDS: ClassVar[list[str]] = ["class_", "view", "value_type", "container"]
1141
- VIEWS_FIELDS: ClassVar[list[str]] = ["class_", "view", "implements"]
1142
- CONTAINERS_FIELDS: ClassVar[list[str]] = ["class_", "container"]
1143
-
1144
- def __init__(self, info: SerializationInfo, default_space: str, default_version: str) -> None:
1145
- self.default_space = f"{default_space}:"
1146
- self.default_version = f"version={default_version}"
1147
- self.default_version_wrapped = f"({self.default_version})"
1148
-
1149
- self.properties_fields = self.PROPERTIES_FIELDS
1150
- self.views_fields = self.VIEWS_FIELDS
1151
- self.containers_fields = self.CONTAINERS_FIELDS
1152
- self.prop_name = "properties"
1153
- self.view_name = "views"
1154
- self.container_name = "containers"
1155
- self.metadata_name = "metadata"
1156
- self.prop_view = "view"
1157
- self.prop_view_property = "view_property"
1158
- self.prop_value_type = "value_type"
1159
- self.view_view = "view"
1160
- self.view_implements = "implements"
1161
- self.container_container = "container"
1162
- self.container_constraint = "constraint"
1163
-
1164
- if info.by_alias:
1165
- self.properties_fields = [
1166
- DMSProperty.model_fields[field].alias or field for field in self.properties_fields
1167
- ]
1168
- self.views_fields = [DMSView.model_fields[field].alias or field for field in self.views_fields]
1169
- self.container_fields = [
1170
- DMSContainer.model_fields[field].alias or field for field in self.containers_fields
1171
- ]
1172
- self.prop_view = DMSProperty.model_fields[self.prop_view].alias or self.prop_view
1173
- self.prop_view_property = DMSProperty.model_fields[self.prop_view_property].alias or self.prop_view_property
1174
- self.prop_value_type = DMSProperty.model_fields[self.prop_value_type].alias or self.prop_value_type
1175
- self.view_view = DMSView.model_fields[self.view_view].alias or self.view_view
1176
- self.view_implements = DMSView.model_fields[self.view_implements].alias or self.view_implements
1177
- self.container_container = (
1178
- DMSContainer.model_fields[self.container_container].alias or self.container_container
1179
- )
1180
- self.container_constraint = (
1181
- DMSContainer.model_fields[self.container_constraint].alias or self.container_constraint
1182
- )
1183
- self.prop_name = DMSRules.model_fields[self.prop_name].alias or self.prop_name
1184
- self.view_name = DMSRules.model_fields[self.view_name].alias or self.view_name
1185
- self.container_name = DMSRules.model_fields[self.container_name].alias or self.container_name
1186
- self.metadata_name = DMSRules.model_fields[self.metadata_name].alias or self.metadata_name
1187
-
1188
- if isinstance(info.exclude, dict):
1189
- # Just for happy mypy
1190
- exclude = cast(dict, info.exclude)
1191
- self.exclude_properties = exclude.get("properties", {}).get("__all__", set())
1192
- self.exclude_views = exclude.get("views", {}).get("__all__", set()) or set()
1193
- self.exclude_containers = exclude.get("containers", {}).get("__all__", set()) or set()
1194
- self.metadata_exclude = exclude.get("metadata", set()) or set()
1195
- self.exclude_top = {k for k, v in exclude.items() if not v}
1196
- else:
1197
- self.exclude_top = set(info.exclude or {})
1198
- self.exclude_properties = set()
1199
- self.exclude_views = set()
1200
- self.exclude_containers = set()
1201
- self.metadata_exclude = set()
1202
-
1203
- def clean(self, dumped: dict[str, Any]) -> dict[str, Any]:
1204
- # Sorting to get a deterministic order
1205
- dumped[self.prop_name] = sorted(
1206
- dumped[self.prop_name]["data"], key=lambda p: (p[self.prop_view], p[self.prop_view_property])
1207
- )
1208
- dumped[self.view_name] = sorted(dumped[self.view_name]["data"], key=lambda v: v[self.view_view])
1209
- if self.container_name in dumped:
1210
- dumped[self.container_name] = sorted(
1211
- dumped[self.container_name]["data"], key=lambda c: c[self.container_container]
1212
- )
1213
-
1214
- for prop in dumped[self.prop_name]:
1215
- for field_name in self.properties_fields:
1216
- if value := prop.get(field_name):
1217
- prop[field_name] = value.removeprefix(self.default_space).removesuffix(self.default_version_wrapped)
1218
- # Value type can have a property as well
1219
- prop[self.prop_value_type] = prop[self.prop_value_type].replace(self.default_version, "")
1220
- if self.exclude_properties:
1221
- for field in self.exclude_properties:
1222
- prop.pop(field, None)
1223
-
1224
- for view in dumped[self.view_name]:
1225
- for field_name in self.views_fields:
1226
- if value := view.get(field_name):
1227
- view[field_name] = value.removeprefix(self.default_space).removesuffix(self.default_version_wrapped)
1228
- if value := view.get(self.view_implements):
1229
- view[self.view_implements] = ",".join(
1230
- parent.strip().removeprefix(self.default_space).removesuffix(self.default_version_wrapped)
1231
- for parent in value.split(",")
1232
- )
1233
- if self.exclude_views:
1234
- for field in self.exclude_views:
1235
- view.pop(field, None)
1236
-
1237
- for container in dumped[self.container_name]:
1238
- for field_name in self.containers_fields:
1239
- if value := container.get(field_name):
1240
- container[field_name] = value.removeprefix(self.default_space)
1241
-
1242
- if value := container.get(self.container_constraint):
1243
- container[self.container_constraint] = ",".join(
1244
- constraint.strip().removeprefix(self.default_space) for constraint in value.split(",")
1245
- )
1246
- if self.exclude_containers:
1247
- for field in self.exclude_containers:
1248
- container.pop(field, None)
1249
-
1250
- if self.metadata_exclude:
1251
- for field in self.metadata_exclude:
1252
- dumped[self.metadata_name].pop(field, None)
1253
- for field in self.exclude_top:
1254
- dumped.pop(field, None)
1255
- return dumped