cognite-neat 0.77.3__py3-none-any.whl → 0.77.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/rules/analysis/_information_rules.py +2 -12
- cognite/neat/rules/exporters/_rules2excel.py +4 -4
- cognite/neat/rules/importers/_dms2rules.py +15 -6
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +1 -0
- cognite/neat/rules/importers/_spreadsheet2rules.py +21 -8
- cognite/neat/rules/issues/spreadsheet.py +60 -5
- cognite/neat/rules/models/_base.py +6 -0
- cognite/neat/rules/models/dms/_converter.py +2 -0
- cognite/neat/rules/models/dms/_exporter.py +2 -1
- cognite/neat/rules/models/dms/_rules.py +3 -2
- cognite/neat/rules/models/dms/_rules_input.py +4 -11
- cognite/neat/rules/models/dms/_schema.py +5 -4
- cognite/neat/rules/models/dms/_serializer.py +1 -1
- cognite/neat/rules/models/dms/_validation.py +27 -60
- cognite/neat/rules/models/information/__init__.py +8 -1
- cognite/neat/rules/models/information/_rules.py +41 -82
- cognite/neat/rules/models/information/_rules_input.py +266 -0
- cognite/neat/rules/models/information/_serializer.py +85 -0
- cognite/neat/rules/models/information/_validation.py +164 -0
- cognite/neat/utils/cdf.py +35 -0
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +30 -7
- cognite/neat/workflows/steps/lib/current/rules_importer.py +21 -2
- {cognite_neat-0.77.3.dist-info → cognite_neat-0.77.4.dist-info}/METADATA +1 -1
- {cognite_neat-0.77.3.dist-info → cognite_neat-0.77.4.dist-info}/RECORD +28 -25
- {cognite_neat-0.77.3.dist-info → cognite_neat-0.77.4.dist-info}/LICENSE +0 -0
- {cognite_neat-0.77.3.dist-info → cognite_neat-0.77.4.dist-info}/WHEEL +0 -0
- {cognite_neat-0.77.3.dist-info → cognite_neat-0.77.4.dist-info}/entry_points.txt +0 -0
cognite/neat/_version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.77.
|
|
1
|
+
__version__ = "0.77.4"
|
|
@@ -13,23 +13,13 @@ from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, ParentC
|
|
|
13
13
|
from cognite.neat.rules.models.information import InformationClass, InformationProperty, InformationRules
|
|
14
14
|
from cognite.neat.utils.utils import get_inheritance_path
|
|
15
15
|
|
|
16
|
-
from ._base import BaseAnalysis
|
|
16
|
+
from ._base import BaseAnalysis
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
class InformationArchitectRulesAnalysis(BaseAnalysis):
|
|
20
20
|
def __init__(self, rules: InformationRules):
|
|
21
21
|
self.rules = rules
|
|
22
22
|
|
|
23
|
-
@property
|
|
24
|
-
def data_modeling_scenario(self) -> DataModelingScenario:
|
|
25
|
-
if not self.rules.reference:
|
|
26
|
-
return DataModelingScenario.from_scratch
|
|
27
|
-
|
|
28
|
-
if self.rules.metadata.namespace == self.rules.reference.metadata.namespace:
|
|
29
|
-
return DataModelingScenario.extend_reference
|
|
30
|
-
else:
|
|
31
|
-
return DataModelingScenario.build_solution
|
|
32
|
-
|
|
33
23
|
@property
|
|
34
24
|
def referred_classes(self) -> set[ClassEntity]:
|
|
35
25
|
return self.directly_referred_classes.union(self.inherited_referred_classes)
|
|
@@ -362,7 +352,7 @@ class InformationArchitectRulesAnalysis(BaseAnalysis):
|
|
|
362
352
|
|
|
363
353
|
rules = cast(InformationRules, self.rules.reference if use_reference else self.rules)
|
|
364
354
|
|
|
365
|
-
if
|
|
355
|
+
if rules.metadata.schema_ is not SchemaCompleteness.complete:
|
|
366
356
|
raise ValueError("Rules are not complete cannot perform reduction!")
|
|
367
357
|
class_as_dict = self.as_class_dict()
|
|
368
358
|
class_parents_pairs = self.class_parent_pairs()
|
|
@@ -132,11 +132,11 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
132
132
|
self._write_sheets(workbook, dumped_user_rules, rules)
|
|
133
133
|
if dumped_last_rules:
|
|
134
134
|
self._write_sheets(workbook, dumped_last_rules, rules, sheet_prefix="Last")
|
|
135
|
+
self._write_metadata_sheet(workbook, dumped_last_rules["Metadata"], sheet_prefix="Last")
|
|
135
136
|
|
|
136
137
|
if dumped_reference_rules:
|
|
137
|
-
|
|
138
|
-
self.
|
|
139
|
-
self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix=prefix)
|
|
138
|
+
self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix="Ref")
|
|
139
|
+
self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix="Ref")
|
|
140
140
|
|
|
141
141
|
if self._styling_level > 0:
|
|
142
142
|
self._adjust_column_widths(workbook)
|
|
@@ -291,7 +291,7 @@ class _MetadataCreator:
|
|
|
291
291
|
prefix = self.new_model_id[0]
|
|
292
292
|
return InformationMetadata(
|
|
293
293
|
data_model_type=DataModelType.solution,
|
|
294
|
-
schema_=SchemaCompleteness.
|
|
294
|
+
schema_=SchemaCompleteness.complete,
|
|
295
295
|
extension=ExtensionCategory.addition,
|
|
296
296
|
prefix=prefix,
|
|
297
297
|
namespace=f"http://purl.org/neat/{prefix}/", # type: ignore[arg-type]
|
|
@@ -109,7 +109,7 @@ class DMSImporter(BaseImporter):
|
|
|
109
109
|
if result.result == "failure" or issue_list.has_errors:
|
|
110
110
|
return cls(DMSSchema(), issue_list)
|
|
111
111
|
|
|
112
|
-
metadata = cls._create_metadata_from_model(user_model)
|
|
112
|
+
metadata = cls._create_metadata_from_model(user_model, has_reference=ref_model is not None)
|
|
113
113
|
ref_metadata = cls._create_metadata_from_model(ref_model) if ref_model else None
|
|
114
114
|
|
|
115
115
|
return cls(schema, issue_list, metadata, ref_metadata)
|
|
@@ -131,6 +131,7 @@ class DMSImporter(BaseImporter):
|
|
|
131
131
|
def _create_metadata_from_model(
|
|
132
132
|
cls,
|
|
133
133
|
model: dm.DataModel[dm.View] | dm.DataModelApply,
|
|
134
|
+
has_reference: bool = False,
|
|
134
135
|
) -> DMSMetadata:
|
|
135
136
|
description, creator = DMSMetadata._get_description_and_creator(model.description)
|
|
136
137
|
|
|
@@ -143,6 +144,7 @@ class DMSImporter(BaseImporter):
|
|
|
143
144
|
updated = now
|
|
144
145
|
return DMSMetadata(
|
|
145
146
|
schema_=SchemaCompleteness.complete,
|
|
147
|
+
data_model_type=DataModelType.solution if has_reference else DataModelType.enterprise,
|
|
146
148
|
extension=ExtensionCategory.addition,
|
|
147
149
|
space=model.space,
|
|
148
150
|
external_id=model.external_id,
|
|
@@ -202,16 +204,21 @@ class DMSImporter(BaseImporter):
|
|
|
202
204
|
**self._create_rule_components(
|
|
203
205
|
ref_model,
|
|
204
206
|
ref_schema,
|
|
205
|
-
self.ref_metadata
|
|
207
|
+
self.ref_metadata
|
|
208
|
+
or self._create_default_metadata(list(ref_schema.views.values()), is_ref=True),
|
|
206
209
|
DataModelType.enterprise,
|
|
207
210
|
)
|
|
208
211
|
)
|
|
209
|
-
schema_completeness = SchemaCompleteness.extended
|
|
210
212
|
data_model_type = DataModelType.solution
|
|
211
213
|
|
|
212
214
|
user_rules = DMSRules(
|
|
213
215
|
**self._create_rule_components(
|
|
214
|
-
model,
|
|
216
|
+
model,
|
|
217
|
+
self.root_schema,
|
|
218
|
+
self.metadata,
|
|
219
|
+
data_model_type,
|
|
220
|
+
schema_completeness,
|
|
221
|
+
has_reference=reference is not None,
|
|
215
222
|
),
|
|
216
223
|
reference=reference,
|
|
217
224
|
)
|
|
@@ -228,6 +235,7 @@ class DMSImporter(BaseImporter):
|
|
|
228
235
|
metadata: DMSMetadata | None = None,
|
|
229
236
|
data_model_type: DataModelType | None = None,
|
|
230
237
|
schema_completeness: SchemaCompleteness | None = None,
|
|
238
|
+
has_reference: bool = False,
|
|
231
239
|
) -> dict[str, Any]:
|
|
232
240
|
properties = SheetList[DMSProperty]()
|
|
233
241
|
for view_id, view in schema.views.items():
|
|
@@ -242,7 +250,7 @@ class DMSImporter(BaseImporter):
|
|
|
242
250
|
view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
|
|
243
251
|
}
|
|
244
252
|
|
|
245
|
-
metadata = metadata or DMSMetadata.from_data_model(data_model)
|
|
253
|
+
metadata = metadata or DMSMetadata.from_data_model(data_model, has_reference)
|
|
246
254
|
if data_model_type is not None:
|
|
247
255
|
metadata.data_model_type = data_model_type
|
|
248
256
|
if schema_completeness is not None:
|
|
@@ -262,12 +270,13 @@ class DMSImporter(BaseImporter):
|
|
|
262
270
|
)
|
|
263
271
|
|
|
264
272
|
@classmethod
|
|
265
|
-
def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply]) -> DMSMetadata:
|
|
273
|
+
def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply], is_ref: bool = False) -> DMSMetadata:
|
|
266
274
|
now = datetime.now().replace(microsecond=0)
|
|
267
275
|
space = Counter(view.space for view in views).most_common(1)[0][0]
|
|
268
276
|
return DMSMetadata(
|
|
269
277
|
schema_=SchemaCompleteness.complete,
|
|
270
278
|
extension=ExtensionCategory.addition,
|
|
279
|
+
data_model_type=DataModelType.enterprise if is_ref else DataModelType.solution,
|
|
271
280
|
space=space,
|
|
272
281
|
external_id="Unknown",
|
|
273
282
|
version="0.1.0",
|
|
@@ -22,6 +22,7 @@ from cognite.neat.rules.models import (
|
|
|
22
22
|
SchemaCompleteness,
|
|
23
23
|
)
|
|
24
24
|
from cognite.neat.rules.models.dms import DMSRulesInput
|
|
25
|
+
from cognite.neat.rules.models.information import InformationRulesInput
|
|
25
26
|
from cognite.neat.utils.auxiliary import local_import
|
|
26
27
|
from cognite.neat.utils.spreadsheet import SpreadsheetRead, read_individual_sheet
|
|
27
28
|
|
|
@@ -110,17 +111,27 @@ class SpreadsheetReader:
|
|
|
110
111
|
self.required = required
|
|
111
112
|
self.metadata = metadata
|
|
112
113
|
self._sheet_prefix = sheet_prefix
|
|
114
|
+
self._seen_files: set[Path] = set()
|
|
115
|
+
self._seen_sheets: set[str] = set()
|
|
113
116
|
|
|
114
117
|
@property
|
|
115
118
|
def metadata_sheet_name(self) -> str:
|
|
116
119
|
return f"{self._sheet_prefix}Metadata"
|
|
117
120
|
|
|
121
|
+
@property
|
|
122
|
+
def seen_sheets(self) -> set[str]:
|
|
123
|
+
if not self._seen_files:
|
|
124
|
+
raise ValueError("No files have been read yet.")
|
|
125
|
+
return self._seen_sheets
|
|
126
|
+
|
|
118
127
|
def sheet_names(self, role: RoleTypes) -> set[str]:
|
|
119
128
|
names = MANDATORY_SHEETS_BY_ROLE[role]
|
|
120
129
|
return {f"{self._sheet_prefix}{sheet_name}" for sheet_name in names if sheet_name != "Metadata"}
|
|
121
130
|
|
|
122
131
|
def read(self, filepath: Path) -> None | ReadResult:
|
|
123
132
|
with pd.ExcelFile(filepath) as excel_file:
|
|
133
|
+
self._seen_files.add(filepath)
|
|
134
|
+
self._seen_sheets.update(map(str, excel_file.sheet_names))
|
|
124
135
|
metadata: MetadataRaw | None
|
|
125
136
|
if self.metadata is not None:
|
|
126
137
|
metadata = self.metadata
|
|
@@ -212,20 +223,20 @@ class ExcelImporter(BaseImporter):
|
|
|
212
223
|
issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
|
|
213
224
|
return self._return_or_raise(issue_list, errors)
|
|
214
225
|
|
|
215
|
-
|
|
226
|
+
user_reader = SpreadsheetReader(issue_list)
|
|
227
|
+
user_read = user_reader.read(self.filepath)
|
|
216
228
|
if user_read is None or issue_list.has_errors:
|
|
217
229
|
return self._return_or_raise(issue_list, errors)
|
|
218
230
|
|
|
219
231
|
last_read: ReadResult | None = None
|
|
232
|
+
if any(sheet_name.startswith("Last") for sheet_name in user_reader.seen_sheets):
|
|
233
|
+
last_read = SpreadsheetReader(issue_list, required=False, sheet_prefix="Last").read(self.filepath)
|
|
220
234
|
reference_read: ReadResult | None = None
|
|
221
|
-
if
|
|
222
|
-
# Last does not have its own metadata sheet. It is the same as the user's metadata sheet.
|
|
223
|
-
last_read = SpreadsheetReader(
|
|
224
|
-
issue_list, required=False, metadata=user_read.metadata, sheet_prefix="Last"
|
|
225
|
-
).read(self.filepath)
|
|
235
|
+
if any(sheet_name.startswith("Ref") for sheet_name in user_reader.seen_sheets):
|
|
226
236
|
reference_read = SpreadsheetReader(issue_list, sheet_prefix="Ref").read(self.filepath)
|
|
227
|
-
|
|
228
|
-
|
|
237
|
+
|
|
238
|
+
if issue_list.has_errors:
|
|
239
|
+
return self._return_or_raise(issue_list, errors)
|
|
229
240
|
|
|
230
241
|
if reference_read and user_read.role != reference_read.role:
|
|
231
242
|
issue_list.append(issues.spreadsheet_file.RoleMismatchError(self.filepath))
|
|
@@ -253,6 +264,8 @@ class ExcelImporter(BaseImporter):
|
|
|
253
264
|
rules: Rules
|
|
254
265
|
if rules_cls is DMSRules:
|
|
255
266
|
rules = DMSRulesInput.load(sheets).as_rules()
|
|
267
|
+
elif rules_cls is InformationRules:
|
|
268
|
+
rules = InformationRulesInput.load(sheets).as_rules()
|
|
256
269
|
else:
|
|
257
270
|
rules = rules_cls.model_validate(sheets) # type: ignore[attr-defined]
|
|
258
271
|
|
|
@@ -10,7 +10,7 @@ from pydantic_core import ErrorDetails
|
|
|
10
10
|
|
|
11
11
|
from cognite.neat.utils.spreadsheet import SpreadsheetRead
|
|
12
12
|
|
|
13
|
-
from .base import DefaultPydanticError, MultiValueError, NeatValidationError
|
|
13
|
+
from .base import DefaultPydanticError, MultiValueError, NeatValidationError
|
|
14
14
|
|
|
15
15
|
if sys.version_info >= (3, 11):
|
|
16
16
|
from typing import Self
|
|
@@ -27,7 +27,7 @@ __all__ = [
|
|
|
27
27
|
"InvalidRowUnknownSheetError",
|
|
28
28
|
"NonExistingContainerError",
|
|
29
29
|
"NonExistingViewError",
|
|
30
|
-
"
|
|
30
|
+
"ClassNoPropertiesNoParentError",
|
|
31
31
|
"InconsistentContainerDefinitionError",
|
|
32
32
|
"MultiValueTypeError",
|
|
33
33
|
"MultiValueIsListError",
|
|
@@ -239,7 +239,26 @@ class NonExistingViewError(InvalidPropertyError):
|
|
|
239
239
|
|
|
240
240
|
|
|
241
241
|
@dataclass(frozen=True)
|
|
242
|
-
class
|
|
242
|
+
class PropertiesDefinedForUndefinedClassesError(NeatValidationError):
|
|
243
|
+
description = "Properties are defined for undefined classes."
|
|
244
|
+
fix = "Make sure to define class in the Classes sheet."
|
|
245
|
+
|
|
246
|
+
classes: list[str]
|
|
247
|
+
|
|
248
|
+
def dump(self) -> dict[str, list[str]]:
|
|
249
|
+
output = super().dump()
|
|
250
|
+
output["classes"] = self.classes
|
|
251
|
+
return output
|
|
252
|
+
|
|
253
|
+
def message(self) -> str:
|
|
254
|
+
return (
|
|
255
|
+
f"Classes {', '.join(self.classes)} have properties assigned to them, but"
|
|
256
|
+
" they are not defined in the Classes sheet."
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
@dataclass(frozen=True)
|
|
261
|
+
class ClassNoPropertiesNoParentError(NeatValidationError):
|
|
243
262
|
description = "Class has no properties and no parents."
|
|
244
263
|
fix = "Check if the class should have properties or parents."
|
|
245
264
|
|
|
@@ -252,8 +271,44 @@ class ClassNoPropertiesNoParentsWarning(ValidationWarning):
|
|
|
252
271
|
|
|
253
272
|
def message(self) -> str:
|
|
254
273
|
if len(self.classes) > 1:
|
|
255
|
-
return f"Classes {', '.join(self.classes)} have no
|
|
256
|
-
return f"Class {self.classes[0]}
|
|
274
|
+
return f"Classes {', '.join(self.classes)} have no direct or inherited properties. This may be a mistake."
|
|
275
|
+
return f"Class {self.classes[0]} have no direct or inherited properties. This may be a mistake."
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
@dataclass(frozen=True)
|
|
279
|
+
class ParentClassesNotDefinedError(NeatValidationError):
|
|
280
|
+
description = "Parent classes are not defined."
|
|
281
|
+
fix = "Check if the parent classes are defined in Classes sheet."
|
|
282
|
+
|
|
283
|
+
classes: list[str]
|
|
284
|
+
|
|
285
|
+
def dump(self) -> dict[str, list[str]]:
|
|
286
|
+
output = super().dump()
|
|
287
|
+
output["classes"] = self.classes
|
|
288
|
+
return output
|
|
289
|
+
|
|
290
|
+
def message(self) -> str:
|
|
291
|
+
if len(self.classes) > 1:
|
|
292
|
+
return f"Parent classes {', '.join(self.classes)} are not defined. This may be a mistake."
|
|
293
|
+
return f"Parent classes {', '.join(self.classes[0])} are not defined. This may be a mistake."
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
@dataclass(frozen=True)
|
|
297
|
+
class ValueTypeNotDefinedError(NeatValidationError):
|
|
298
|
+
description = "Value types referred by properties are not defined in Rules."
|
|
299
|
+
fix = "Make sure that all value types are defined in Rules."
|
|
300
|
+
|
|
301
|
+
value_types: list[str]
|
|
302
|
+
|
|
303
|
+
def dump(self) -> dict[str, list[str]]:
|
|
304
|
+
output = super().dump()
|
|
305
|
+
output["classes"] = self.value_types
|
|
306
|
+
return output
|
|
307
|
+
|
|
308
|
+
def message(self) -> str:
|
|
309
|
+
if len(self.value_types) > 1:
|
|
310
|
+
return f"Value types {', '.join(self.value_types)} are not defined. This may be a mistake."
|
|
311
|
+
return f"Value types {', '.join(self.value_types[0])} are not defined. This may be a mistake."
|
|
257
312
|
|
|
258
313
|
|
|
259
314
|
@dataclass(frozen=True)
|
|
@@ -38,6 +38,12 @@ else:
|
|
|
38
38
|
METADATA_VALUE_MAX_LENGTH = 5120
|
|
39
39
|
|
|
40
40
|
|
|
41
|
+
def _add_alias(data: dict[str, Any], base_model: type[BaseModel]) -> None:
|
|
42
|
+
for field_name, field_ in base_model.model_fields.items():
|
|
43
|
+
if field_name not in data and field_.alias in data:
|
|
44
|
+
data[field_name] = data[field_.alias]
|
|
45
|
+
|
|
46
|
+
|
|
41
47
|
def replace_nan_floats_with_default(values: dict, model_fields: dict[str, FieldInfo]) -> dict:
|
|
42
48
|
output = {}
|
|
43
49
|
for field_name, value in values.items():
|
|
@@ -105,6 +105,8 @@ class _DMSRulesConverter:
|
|
|
105
105
|
prefix = metadata.space
|
|
106
106
|
return InformationMetadata(
|
|
107
107
|
schema_=metadata.schema_,
|
|
108
|
+
data_model_type=metadata.data_model_type,
|
|
109
|
+
extension=metadata.extension,
|
|
108
110
|
prefix=prefix,
|
|
109
111
|
namespace=Namespace(f"https://purl.orgl/neat/{prefix}/"),
|
|
110
112
|
version=metadata.version,
|
|
@@ -58,7 +58,7 @@ _DEFAULT_VERSION = "1"
|
|
|
58
58
|
|
|
59
59
|
class DMSMetadata(BaseMetadata):
|
|
60
60
|
role: ClassVar[RoleTypes] = RoleTypes.dms_architect
|
|
61
|
-
data_model_type: DataModelType = Field(DataModelType.
|
|
61
|
+
data_model_type: DataModelType = Field(DataModelType.enterprise, alias="dataModelType")
|
|
62
62
|
schema_: SchemaCompleteness = Field(alias="schema")
|
|
63
63
|
extension: ExtensionCategory = ExtensionCategory.addition
|
|
64
64
|
space: ExternalIdType
|
|
@@ -146,10 +146,11 @@ class DMSMetadata(BaseMetadata):
|
|
|
146
146
|
return description, creator
|
|
147
147
|
|
|
148
148
|
@classmethod
|
|
149
|
-
def from_data_model(cls, data_model: dm.DataModelApply) -> "DMSMetadata":
|
|
149
|
+
def from_data_model(cls, data_model: dm.DataModelApply, has_reference: bool) -> "DMSMetadata":
|
|
150
150
|
description, creator = cls._get_description_and_creator(data_model.description)
|
|
151
151
|
return cls(
|
|
152
152
|
schema_=SchemaCompleteness.complete,
|
|
153
|
+
data_model_type=DataModelType.solution if has_reference else DataModelType.enterprise,
|
|
153
154
|
space=data_model.space,
|
|
154
155
|
name=data_model.name or None,
|
|
155
156
|
description=description,
|
|
@@ -3,9 +3,7 @@ from dataclasses import dataclass
|
|
|
3
3
|
from datetime import datetime
|
|
4
4
|
from typing import Any, Literal, cast, overload
|
|
5
5
|
|
|
6
|
-
from
|
|
7
|
-
|
|
8
|
-
from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
|
|
6
|
+
from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness, _add_alias
|
|
9
7
|
from cognite.neat.rules.models.data_types import DataType
|
|
10
8
|
from cognite.neat.rules.models.entities import (
|
|
11
9
|
ClassEntity,
|
|
@@ -44,8 +42,9 @@ class DMSMetadataInput:
|
|
|
44
42
|
external_id=data.get("external_id"), # type: ignore[arg-type]
|
|
45
43
|
creator=data.get("creator"), # type: ignore[arg-type]
|
|
46
44
|
version=data.get("version"), # type: ignore[arg-type]
|
|
47
|
-
|
|
48
|
-
|
|
45
|
+
# safeguard from empty cell, i.e. if key provided by value None
|
|
46
|
+
extension=data.get("extension", "addition") or "addition",
|
|
47
|
+
data_model_type=data.get("data_model_type", "solution") or "solution",
|
|
49
48
|
name=data.get("name"),
|
|
50
49
|
description=data.get("description"),
|
|
51
50
|
created=data.get("created"),
|
|
@@ -355,9 +354,3 @@ class DMSRulesInput:
|
|
|
355
354
|
Last=last,
|
|
356
355
|
Reference=reference,
|
|
357
356
|
)
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
def _add_alias(data: dict[str, Any], base_model: type[BaseModel]) -> None:
|
|
361
|
-
for field_name, field_ in base_model.model_fields.items():
|
|
362
|
-
if field_name not in data and field_.alias in data:
|
|
363
|
-
data[field_name] = data[field_.alias]
|
|
@@ -516,10 +516,11 @@ class DMSSchema:
|
|
|
516
516
|
defined_spaces = self.spaces.copy()
|
|
517
517
|
defined_containers = self.containers.copy()
|
|
518
518
|
defined_views = self.views.copy()
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
519
|
+
for other_schema in [self.reference, self.last]:
|
|
520
|
+
if other_schema:
|
|
521
|
+
defined_spaces |= other_schema.spaces
|
|
522
|
+
defined_containers |= other_schema.containers
|
|
523
|
+
defined_views |= other_schema.views
|
|
523
524
|
|
|
524
525
|
for container in self.containers.values():
|
|
525
526
|
if container.space not in defined_spaces:
|
|
@@ -37,7 +37,7 @@ class _DMSRulesSerializer:
|
|
|
37
37
|
DMSProperty.model_fields[field].alias or field for field in self.properties_fields
|
|
38
38
|
]
|
|
39
39
|
self.views_fields = [DMSView.model_fields[field].alias or field for field in self.views_fields]
|
|
40
|
-
self.
|
|
40
|
+
self.containers_fields = [
|
|
41
41
|
DMSContainer.model_fields[field].alias or field for field in self.containers_fields
|
|
42
42
|
]
|
|
43
43
|
self.prop_view = DMSProperty.model_fields[self.prop_view].alias or self.prop_view
|
|
@@ -5,12 +5,13 @@ from cognite.client import data_modeling as dm
|
|
|
5
5
|
|
|
6
6
|
from cognite.neat.rules import issues
|
|
7
7
|
from cognite.neat.rules.issues import IssueList
|
|
8
|
-
from cognite.neat.rules.models._base import ExtensionCategory, SchemaCompleteness
|
|
8
|
+
from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
|
|
9
9
|
from cognite.neat.rules.models.data_types import DataType
|
|
10
10
|
from cognite.neat.rules.models.entities import ContainerEntity
|
|
11
11
|
from cognite.neat.rules.models.wrapped_entities import RawFilter
|
|
12
12
|
|
|
13
13
|
from ._rules import DMSProperty, DMSRules
|
|
14
|
+
from ._schema import DMSSchema
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
class DMSPostValidation:
|
|
@@ -26,12 +27,17 @@ class DMSPostValidation:
|
|
|
26
27
|
self.issue_list = IssueList()
|
|
27
28
|
|
|
28
29
|
def validate(self) -> IssueList:
|
|
29
|
-
self.
|
|
30
|
+
self._validate_raw_filter()
|
|
30
31
|
self._consistent_container_properties()
|
|
32
|
+
|
|
31
33
|
self._referenced_views_and_containers_are_existing()
|
|
32
|
-
self.
|
|
33
|
-
|
|
34
|
-
self.
|
|
34
|
+
if self.metadata.schema_ is SchemaCompleteness.extended:
|
|
35
|
+
self._validate_extension()
|
|
36
|
+
if self.metadata.schema_ is SchemaCompleteness.partial:
|
|
37
|
+
return self.issue_list
|
|
38
|
+
dms_schema = self.rules.as_schema()
|
|
39
|
+
self.issue_list.extend(dms_schema.validate())
|
|
40
|
+
self._validate_performance(dms_schema)
|
|
35
41
|
return self.issue_list
|
|
36
42
|
|
|
37
43
|
def _consistent_container_properties(self) -> None:
|
|
@@ -106,8 +112,9 @@ class DMSPostValidation:
|
|
|
106
112
|
self.issue_list.extend(errors)
|
|
107
113
|
|
|
108
114
|
def _referenced_views_and_containers_are_existing(self) -> None:
|
|
109
|
-
# There two checks are done in the same method to raise all the errors at once.
|
|
110
115
|
defined_views = {view.view.as_id() for view in self.views}
|
|
116
|
+
if self.metadata.schema_ is SchemaCompleteness.extended and self.rules.last:
|
|
117
|
+
defined_views |= {view.view.as_id() for view in self.rules.last.views}
|
|
111
118
|
|
|
112
119
|
errors: list[issues.NeatValidationError] = []
|
|
113
120
|
for prop_no, prop in enumerate(self.properties):
|
|
@@ -125,6 +132,11 @@ class DMSPostValidation:
|
|
|
125
132
|
)
|
|
126
133
|
if self.metadata.schema_ is SchemaCompleteness.complete:
|
|
127
134
|
defined_containers = {container.container.as_id() for container in self.containers or []}
|
|
135
|
+
if self.metadata.data_model_type == DataModelType.solution and self.rules.reference:
|
|
136
|
+
defined_containers |= {
|
|
137
|
+
container.container.as_id() for container in self.rules.reference.containers or []
|
|
138
|
+
}
|
|
139
|
+
|
|
128
140
|
for prop_no, prop in enumerate(self.properties):
|
|
129
141
|
if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
|
|
130
142
|
errors.append(
|
|
@@ -157,19 +169,16 @@ class DMSPostValidation:
|
|
|
157
169
|
def _validate_extension(self) -> None:
|
|
158
170
|
if self.metadata.schema_ is not SchemaCompleteness.extended:
|
|
159
171
|
return None
|
|
160
|
-
if not self.rules.
|
|
161
|
-
raise ValueError("The schema is set to 'extended', but no
|
|
162
|
-
is_solution = self.metadata.space != self.rules.reference.metadata.space
|
|
163
|
-
if is_solution:
|
|
164
|
-
return None
|
|
172
|
+
if not self.rules.last:
|
|
173
|
+
raise ValueError("The schema is set to 'extended', but no last rules are provided to validate against")
|
|
165
174
|
if self.metadata.extension is ExtensionCategory.rebuild:
|
|
166
175
|
# Everything is allowed
|
|
167
176
|
return None
|
|
168
|
-
# Is an extension of an existing model.
|
|
169
177
|
user_schema = self.rules.as_schema()
|
|
170
|
-
ref_schema = self.rules.reference.as_schema()
|
|
171
178
|
new_containers = user_schema.containers.copy()
|
|
172
|
-
|
|
179
|
+
|
|
180
|
+
last_schema = self.rules.last.as_schema()
|
|
181
|
+
existing_containers = last_schema.containers.copy()
|
|
173
182
|
|
|
174
183
|
for container_id, container in new_containers.items():
|
|
175
184
|
existing_container = existing_containers.get(container_id)
|
|
@@ -189,14 +198,12 @@ class DMSPostValidation:
|
|
|
189
198
|
)
|
|
190
199
|
)
|
|
191
200
|
|
|
192
|
-
if self.metadata.extension is ExtensionCategory.reshape
|
|
193
|
-
return None
|
|
194
|
-
elif self.metadata.extension is ExtensionCategory.reshape:
|
|
201
|
+
if self.metadata.extension is ExtensionCategory.reshape:
|
|
195
202
|
# Reshape allows changes to views
|
|
196
203
|
return None
|
|
197
204
|
|
|
198
205
|
new_views = user_schema.views.copy()
|
|
199
|
-
existing_views =
|
|
206
|
+
existing_views = last_schema.views.copy()
|
|
200
207
|
for view_id, view in new_views.items():
|
|
201
208
|
existing_view = existing_views.get(view_id)
|
|
202
209
|
if not existing_view or existing_view == view:
|
|
@@ -213,14 +220,7 @@ class DMSPostValidation:
|
|
|
213
220
|
)
|
|
214
221
|
)
|
|
215
222
|
|
|
216
|
-
def _validate_performance(self) -> None:
|
|
217
|
-
# we can only validate performance on complete schemas due to the need
|
|
218
|
-
# to access all the container mappings
|
|
219
|
-
if self.metadata.schema_ is not SchemaCompleteness.complete:
|
|
220
|
-
return None
|
|
221
|
-
|
|
222
|
-
dms_schema = self.rules.as_schema()
|
|
223
|
-
|
|
223
|
+
def _validate_performance(self, dms_schema: DMSSchema) -> None:
|
|
224
224
|
for view_id, view in dms_schema.views.items():
|
|
225
225
|
mapped_containers = dms_schema._get_mapped_container_from_view(view_id)
|
|
226
226
|
|
|
@@ -243,7 +243,7 @@ class DMSPostValidation:
|
|
|
243
243
|
)
|
|
244
244
|
)
|
|
245
245
|
|
|
246
|
-
def
|
|
246
|
+
def _validate_raw_filter(self) -> None:
|
|
247
247
|
for view in self.views:
|
|
248
248
|
if view.filter_ and isinstance(view.filter_, RawFilter):
|
|
249
249
|
self.issue_list.append(
|
|
@@ -264,36 +264,3 @@ class DMSPostValidation:
|
|
|
264
264
|
existing_properties = existing_dumped.get("properties", {})
|
|
265
265
|
changed_properties = [prop for prop in new_properties if new_properties[prop] != existing_properties.get(prop)]
|
|
266
266
|
return changed_attributes, changed_properties
|
|
267
|
-
|
|
268
|
-
def _validate_schema(self) -> None:
|
|
269
|
-
if self.metadata.schema_ is SchemaCompleteness.partial:
|
|
270
|
-
return None
|
|
271
|
-
elif self.metadata.schema_ is SchemaCompleteness.complete:
|
|
272
|
-
rules: DMSRules = self.rules
|
|
273
|
-
elif self.metadata.schema_ is SchemaCompleteness.extended:
|
|
274
|
-
if not self.rules.reference:
|
|
275
|
-
raise ValueError(
|
|
276
|
-
"The schema is set to 'extended', but no reference rules are provided to validate against"
|
|
277
|
-
)
|
|
278
|
-
# This is an extension of the reference rules, we need to merge the two
|
|
279
|
-
rules = self.rules.model_copy(deep=True)
|
|
280
|
-
rules.properties.extend(self.rules.reference.properties.data)
|
|
281
|
-
existing_views = {view.view.as_id() for view in rules.views}
|
|
282
|
-
rules.views.extend([view for view in self.rules.reference.views if view.view.as_id() not in existing_views])
|
|
283
|
-
if rules.containers and self.rules.reference.containers:
|
|
284
|
-
existing_containers = {container.container.as_id() for container in rules.containers.data}
|
|
285
|
-
rules.containers.extend(
|
|
286
|
-
[
|
|
287
|
-
container
|
|
288
|
-
for container in self.rules.reference.containers
|
|
289
|
-
if container.container.as_id() not in existing_containers
|
|
290
|
-
]
|
|
291
|
-
)
|
|
292
|
-
elif not rules.containers and self.rules.reference.containers:
|
|
293
|
-
rules.containers = self.rules.reference.containers
|
|
294
|
-
else:
|
|
295
|
-
raise ValueError("Unknown schema completeness")
|
|
296
|
-
|
|
297
|
-
schema = rules.as_schema()
|
|
298
|
-
errors = schema.validate()
|
|
299
|
-
self.issue_list.extend(errors)
|
|
@@ -1,3 +1,10 @@
|
|
|
1
1
|
from ._rules import InformationClass, InformationMetadata, InformationProperty, InformationRules
|
|
2
|
+
from ._rules_input import InformationRulesInput
|
|
2
3
|
|
|
3
|
-
__all__ = [
|
|
4
|
+
__all__ = [
|
|
5
|
+
"InformationRules",
|
|
6
|
+
"InformationMetadata",
|
|
7
|
+
"InformationClass",
|
|
8
|
+
"InformationProperty",
|
|
9
|
+
"InformationRulesInput",
|
|
10
|
+
]
|