cognite-neat 0.77.3__py3-none-any.whl → 0.77.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (28) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/rules/analysis/_information_rules.py +2 -12
  3. cognite/neat/rules/exporters/_rules2excel.py +4 -4
  4. cognite/neat/rules/importers/_dms2rules.py +15 -6
  5. cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +1 -0
  6. cognite/neat/rules/importers/_spreadsheet2rules.py +21 -8
  7. cognite/neat/rules/issues/spreadsheet.py +60 -5
  8. cognite/neat/rules/models/_base.py +6 -0
  9. cognite/neat/rules/models/dms/_converter.py +2 -0
  10. cognite/neat/rules/models/dms/_exporter.py +2 -1
  11. cognite/neat/rules/models/dms/_rules.py +3 -2
  12. cognite/neat/rules/models/dms/_rules_input.py +4 -11
  13. cognite/neat/rules/models/dms/_schema.py +5 -4
  14. cognite/neat/rules/models/dms/_serializer.py +1 -1
  15. cognite/neat/rules/models/dms/_validation.py +27 -60
  16. cognite/neat/rules/models/information/__init__.py +8 -1
  17. cognite/neat/rules/models/information/_rules.py +41 -82
  18. cognite/neat/rules/models/information/_rules_input.py +266 -0
  19. cognite/neat/rules/models/information/_serializer.py +85 -0
  20. cognite/neat/rules/models/information/_validation.py +164 -0
  21. cognite/neat/utils/cdf.py +35 -0
  22. cognite/neat/workflows/steps/lib/current/rules_exporter.py +30 -7
  23. cognite/neat/workflows/steps/lib/current/rules_importer.py +21 -2
  24. {cognite_neat-0.77.3.dist-info → cognite_neat-0.77.4.dist-info}/METADATA +1 -1
  25. {cognite_neat-0.77.3.dist-info → cognite_neat-0.77.4.dist-info}/RECORD +28 -25
  26. {cognite_neat-0.77.3.dist-info → cognite_neat-0.77.4.dist-info}/LICENSE +0 -0
  27. {cognite_neat-0.77.3.dist-info → cognite_neat-0.77.4.dist-info}/WHEEL +0 -0
  28. {cognite_neat-0.77.3.dist-info → cognite_neat-0.77.4.dist-info}/entry_points.txt +0 -0
cognite/neat/_version.py CHANGED
@@ -1 +1 @@
1
- __version__ = "0.77.3"
1
+ __version__ = "0.77.4"
@@ -13,23 +13,13 @@ from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, ParentC
13
13
  from cognite.neat.rules.models.information import InformationClass, InformationProperty, InformationRules
14
14
  from cognite.neat.utils.utils import get_inheritance_path
15
15
 
16
- from ._base import BaseAnalysis, DataModelingScenario
16
+ from ._base import BaseAnalysis
17
17
 
18
18
 
19
19
  class InformationArchitectRulesAnalysis(BaseAnalysis):
20
20
  def __init__(self, rules: InformationRules):
21
21
  self.rules = rules
22
22
 
23
- @property
24
- def data_modeling_scenario(self) -> DataModelingScenario:
25
- if not self.rules.reference:
26
- return DataModelingScenario.from_scratch
27
-
28
- if self.rules.metadata.namespace == self.rules.reference.metadata.namespace:
29
- return DataModelingScenario.extend_reference
30
- else:
31
- return DataModelingScenario.build_solution
32
-
33
23
  @property
34
24
  def referred_classes(self) -> set[ClassEntity]:
35
25
  return self.directly_referred_classes.union(self.inherited_referred_classes)
@@ -362,7 +352,7 @@ class InformationArchitectRulesAnalysis(BaseAnalysis):
362
352
 
363
353
  rules = cast(InformationRules, self.rules.reference if use_reference else self.rules)
364
354
 
365
- if not rules.metadata.schema_ is not SchemaCompleteness.complete:
355
+ if rules.metadata.schema_ is not SchemaCompleteness.complete:
366
356
  raise ValueError("Rules are not complete cannot perform reduction!")
367
357
  class_as_dict = self.as_class_dict()
368
358
  class_parents_pairs = self.class_parent_pairs()
@@ -132,11 +132,11 @@ class ExcelExporter(BaseExporter[Workbook]):
132
132
  self._write_sheets(workbook, dumped_user_rules, rules)
133
133
  if dumped_last_rules:
134
134
  self._write_sheets(workbook, dumped_last_rules, rules, sheet_prefix="Last")
135
+ self._write_metadata_sheet(workbook, dumped_last_rules["Metadata"], sheet_prefix="Last")
135
136
 
136
137
  if dumped_reference_rules:
137
- prefix = "Ref"
138
- self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix=prefix)
139
- self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix=prefix)
138
+ self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix="Ref")
139
+ self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix="Ref")
140
140
 
141
141
  if self._styling_level > 0:
142
142
  self._adjust_column_widths(workbook)
@@ -291,7 +291,7 @@ class _MetadataCreator:
291
291
  prefix = self.new_model_id[0]
292
292
  return InformationMetadata(
293
293
  data_model_type=DataModelType.solution,
294
- schema_=SchemaCompleteness.extended,
294
+ schema_=SchemaCompleteness.complete,
295
295
  extension=ExtensionCategory.addition,
296
296
  prefix=prefix,
297
297
  namespace=f"http://purl.org/neat/{prefix}/", # type: ignore[arg-type]
@@ -109,7 +109,7 @@ class DMSImporter(BaseImporter):
109
109
  if result.result == "failure" or issue_list.has_errors:
110
110
  return cls(DMSSchema(), issue_list)
111
111
 
112
- metadata = cls._create_metadata_from_model(user_model)
112
+ metadata = cls._create_metadata_from_model(user_model, has_reference=ref_model is not None)
113
113
  ref_metadata = cls._create_metadata_from_model(ref_model) if ref_model else None
114
114
 
115
115
  return cls(schema, issue_list, metadata, ref_metadata)
@@ -131,6 +131,7 @@ class DMSImporter(BaseImporter):
131
131
  def _create_metadata_from_model(
132
132
  cls,
133
133
  model: dm.DataModel[dm.View] | dm.DataModelApply,
134
+ has_reference: bool = False,
134
135
  ) -> DMSMetadata:
135
136
  description, creator = DMSMetadata._get_description_and_creator(model.description)
136
137
 
@@ -143,6 +144,7 @@ class DMSImporter(BaseImporter):
143
144
  updated = now
144
145
  return DMSMetadata(
145
146
  schema_=SchemaCompleteness.complete,
147
+ data_model_type=DataModelType.solution if has_reference else DataModelType.enterprise,
146
148
  extension=ExtensionCategory.addition,
147
149
  space=model.space,
148
150
  external_id=model.external_id,
@@ -202,16 +204,21 @@ class DMSImporter(BaseImporter):
202
204
  **self._create_rule_components(
203
205
  ref_model,
204
206
  ref_schema,
205
- self.ref_metadata or self._create_default_metadata(list(ref_schema.views.values())),
207
+ self.ref_metadata
208
+ or self._create_default_metadata(list(ref_schema.views.values()), is_ref=True),
206
209
  DataModelType.enterprise,
207
210
  )
208
211
  )
209
- schema_completeness = SchemaCompleteness.extended
210
212
  data_model_type = DataModelType.solution
211
213
 
212
214
  user_rules = DMSRules(
213
215
  **self._create_rule_components(
214
- model, self.root_schema, self.metadata, data_model_type, schema_completeness
216
+ model,
217
+ self.root_schema,
218
+ self.metadata,
219
+ data_model_type,
220
+ schema_completeness,
221
+ has_reference=reference is not None,
215
222
  ),
216
223
  reference=reference,
217
224
  )
@@ -228,6 +235,7 @@ class DMSImporter(BaseImporter):
228
235
  metadata: DMSMetadata | None = None,
229
236
  data_model_type: DataModelType | None = None,
230
237
  schema_completeness: SchemaCompleteness | None = None,
238
+ has_reference: bool = False,
231
239
  ) -> dict[str, Any]:
232
240
  properties = SheetList[DMSProperty]()
233
241
  for view_id, view in schema.views.items():
@@ -242,7 +250,7 @@ class DMSImporter(BaseImporter):
242
250
  view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
243
251
  }
244
252
 
245
- metadata = metadata or DMSMetadata.from_data_model(data_model)
253
+ metadata = metadata or DMSMetadata.from_data_model(data_model, has_reference)
246
254
  if data_model_type is not None:
247
255
  metadata.data_model_type = data_model_type
248
256
  if schema_completeness is not None:
@@ -262,12 +270,13 @@ class DMSImporter(BaseImporter):
262
270
  )
263
271
 
264
272
  @classmethod
265
- def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply]) -> DMSMetadata:
273
+ def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply], is_ref: bool = False) -> DMSMetadata:
266
274
  now = datetime.now().replace(microsecond=0)
267
275
  space = Counter(view.space for view in views).most_common(1)[0][0]
268
276
  return DMSMetadata(
269
277
  schema_=SchemaCompleteness.complete,
270
278
  extension=ExtensionCategory.addition,
279
+ data_model_type=DataModelType.enterprise if is_ref else DataModelType.solution,
271
280
  space=space,
272
281
  external_id="Unknown",
273
282
  version="0.1.0",
@@ -134,6 +134,7 @@ class DTDLImporter(BaseImporter):
134
134
 
135
135
  metadata = self._default_metadata()
136
136
  metadata["schema"] = self._schema_completeness.value
137
+
137
138
  if self.title:
138
139
  metadata["title"] = to_pascal(self.title)
139
140
  try:
@@ -22,6 +22,7 @@ from cognite.neat.rules.models import (
22
22
  SchemaCompleteness,
23
23
  )
24
24
  from cognite.neat.rules.models.dms import DMSRulesInput
25
+ from cognite.neat.rules.models.information import InformationRulesInput
25
26
  from cognite.neat.utils.auxiliary import local_import
26
27
  from cognite.neat.utils.spreadsheet import SpreadsheetRead, read_individual_sheet
27
28
 
@@ -110,17 +111,27 @@ class SpreadsheetReader:
110
111
  self.required = required
111
112
  self.metadata = metadata
112
113
  self._sheet_prefix = sheet_prefix
114
+ self._seen_files: set[Path] = set()
115
+ self._seen_sheets: set[str] = set()
113
116
 
114
117
  @property
115
118
  def metadata_sheet_name(self) -> str:
116
119
  return f"{self._sheet_prefix}Metadata"
117
120
 
121
+ @property
122
+ def seen_sheets(self) -> set[str]:
123
+ if not self._seen_files:
124
+ raise ValueError("No files have been read yet.")
125
+ return self._seen_sheets
126
+
118
127
  def sheet_names(self, role: RoleTypes) -> set[str]:
119
128
  names = MANDATORY_SHEETS_BY_ROLE[role]
120
129
  return {f"{self._sheet_prefix}{sheet_name}" for sheet_name in names if sheet_name != "Metadata"}
121
130
 
122
131
  def read(self, filepath: Path) -> None | ReadResult:
123
132
  with pd.ExcelFile(filepath) as excel_file:
133
+ self._seen_files.add(filepath)
134
+ self._seen_sheets.update(map(str, excel_file.sheet_names))
124
135
  metadata: MetadataRaw | None
125
136
  if self.metadata is not None:
126
137
  metadata = self.metadata
@@ -212,20 +223,20 @@ class ExcelImporter(BaseImporter):
212
223
  issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
213
224
  return self._return_or_raise(issue_list, errors)
214
225
 
215
- user_read = SpreadsheetReader(issue_list).read(self.filepath)
226
+ user_reader = SpreadsheetReader(issue_list)
227
+ user_read = user_reader.read(self.filepath)
216
228
  if user_read is None or issue_list.has_errors:
217
229
  return self._return_or_raise(issue_list, errors)
218
230
 
219
231
  last_read: ReadResult | None = None
232
+ if any(sheet_name.startswith("Last") for sheet_name in user_reader.seen_sheets):
233
+ last_read = SpreadsheetReader(issue_list, required=False, sheet_prefix="Last").read(self.filepath)
220
234
  reference_read: ReadResult | None = None
221
- if user_read.schema == SchemaCompleteness.extended:
222
- # Last does not have its own metadata sheet. It is the same as the user's metadata sheet.
223
- last_read = SpreadsheetReader(
224
- issue_list, required=False, metadata=user_read.metadata, sheet_prefix="Last"
225
- ).read(self.filepath)
235
+ if any(sheet_name.startswith("Ref") for sheet_name in user_reader.seen_sheets):
226
236
  reference_read = SpreadsheetReader(issue_list, sheet_prefix="Ref").read(self.filepath)
227
- if issue_list.has_errors:
228
- return self._return_or_raise(issue_list, errors)
237
+
238
+ if issue_list.has_errors:
239
+ return self._return_or_raise(issue_list, errors)
229
240
 
230
241
  if reference_read and user_read.role != reference_read.role:
231
242
  issue_list.append(issues.spreadsheet_file.RoleMismatchError(self.filepath))
@@ -253,6 +264,8 @@ class ExcelImporter(BaseImporter):
253
264
  rules: Rules
254
265
  if rules_cls is DMSRules:
255
266
  rules = DMSRulesInput.load(sheets).as_rules()
267
+ elif rules_cls is InformationRules:
268
+ rules = InformationRulesInput.load(sheets).as_rules()
256
269
  else:
257
270
  rules = rules_cls.model_validate(sheets) # type: ignore[attr-defined]
258
271
 
@@ -10,7 +10,7 @@ from pydantic_core import ErrorDetails
10
10
 
11
11
  from cognite.neat.utils.spreadsheet import SpreadsheetRead
12
12
 
13
- from .base import DefaultPydanticError, MultiValueError, NeatValidationError, ValidationWarning
13
+ from .base import DefaultPydanticError, MultiValueError, NeatValidationError
14
14
 
15
15
  if sys.version_info >= (3, 11):
16
16
  from typing import Self
@@ -27,7 +27,7 @@ __all__ = [
27
27
  "InvalidRowUnknownSheetError",
28
28
  "NonExistingContainerError",
29
29
  "NonExistingViewError",
30
- "ClassNoPropertiesNoParentsWarning",
30
+ "ClassNoPropertiesNoParentError",
31
31
  "InconsistentContainerDefinitionError",
32
32
  "MultiValueTypeError",
33
33
  "MultiValueIsListError",
@@ -239,7 +239,26 @@ class NonExistingViewError(InvalidPropertyError):
239
239
 
240
240
 
241
241
  @dataclass(frozen=True)
242
- class ClassNoPropertiesNoParentsWarning(ValidationWarning):
242
+ class PropertiesDefinedForUndefinedClassesError(NeatValidationError):
243
+ description = "Properties are defined for undefined classes."
244
+ fix = "Make sure to define class in the Classes sheet."
245
+
246
+ classes: list[str]
247
+
248
+ def dump(self) -> dict[str, list[str]]:
249
+ output = super().dump()
250
+ output["classes"] = self.classes
251
+ return output
252
+
253
+ def message(self) -> str:
254
+ return (
255
+ f"Classes {', '.join(self.classes)} have properties assigned to them, but"
256
+ " they are not defined in the Classes sheet."
257
+ )
258
+
259
+
260
+ @dataclass(frozen=True)
261
+ class ClassNoPropertiesNoParentError(NeatValidationError):
243
262
  description = "Class has no properties and no parents."
244
263
  fix = "Check if the class should have properties or parents."
245
264
 
@@ -252,8 +271,44 @@ class ClassNoPropertiesNoParentsWarning(ValidationWarning):
252
271
 
253
272
  def message(self) -> str:
254
273
  if len(self.classes) > 1:
255
- return f"Classes {', '.join(self.classes)} have no properties and no parents. This may be a mistake."
256
- return f"Class {self.classes[0]} has no properties and no parents. This may be a mistake."
274
+ return f"Classes {', '.join(self.classes)} have no direct or inherited properties. This may be a mistake."
275
+ return f"Class {self.classes[0]} have no direct or inherited properties. This may be a mistake."
276
+
277
+
278
+ @dataclass(frozen=True)
279
+ class ParentClassesNotDefinedError(NeatValidationError):
280
+ description = "Parent classes are not defined."
281
+ fix = "Check if the parent classes are defined in Classes sheet."
282
+
283
+ classes: list[str]
284
+
285
+ def dump(self) -> dict[str, list[str]]:
286
+ output = super().dump()
287
+ output["classes"] = self.classes
288
+ return output
289
+
290
+ def message(self) -> str:
291
+ if len(self.classes) > 1:
292
+ return f"Parent classes {', '.join(self.classes)} are not defined. This may be a mistake."
293
+ return f"Parent classes {', '.join(self.classes[0])} are not defined. This may be a mistake."
294
+
295
+
296
+ @dataclass(frozen=True)
297
+ class ValueTypeNotDefinedError(NeatValidationError):
298
+ description = "Value types referred by properties are not defined in Rules."
299
+ fix = "Make sure that all value types are defined in Rules."
300
+
301
+ value_types: list[str]
302
+
303
+ def dump(self) -> dict[str, list[str]]:
304
+ output = super().dump()
305
+ output["classes"] = self.value_types
306
+ return output
307
+
308
+ def message(self) -> str:
309
+ if len(self.value_types) > 1:
310
+ return f"Value types {', '.join(self.value_types)} are not defined. This may be a mistake."
311
+ return f"Value types {', '.join(self.value_types[0])} are not defined. This may be a mistake."
257
312
 
258
313
 
259
314
  @dataclass(frozen=True)
@@ -38,6 +38,12 @@ else:
38
38
  METADATA_VALUE_MAX_LENGTH = 5120
39
39
 
40
40
 
41
+ def _add_alias(data: dict[str, Any], base_model: type[BaseModel]) -> None:
42
+ for field_name, field_ in base_model.model_fields.items():
43
+ if field_name not in data and field_.alias in data:
44
+ data[field_name] = data[field_.alias]
45
+
46
+
41
47
  def replace_nan_floats_with_default(values: dict, model_fields: dict[str, FieldInfo]) -> dict:
42
48
  output = {}
43
49
  for field_name, value in values.items():
@@ -105,6 +105,8 @@ class _DMSRulesConverter:
105
105
  prefix = metadata.space
106
106
  return InformationMetadata(
107
107
  schema_=metadata.schema_,
108
+ data_model_type=metadata.data_model_type,
109
+ extension=metadata.extension,
108
110
  prefix=prefix,
109
111
  namespace=Namespace(f"https://purl.orgl/neat/{prefix}/"),
110
112
  version=metadata.version,
@@ -86,7 +86,8 @@ class _DMSExporter:
86
86
 
87
87
  if self._ref_schema:
88
88
  output.reference = self._ref_schema
89
-
89
+ if self.rules.last:
90
+ output.last = self.rules.last.as_schema()
90
91
  return output
91
92
 
92
93
  def _create_spaces(
@@ -58,7 +58,7 @@ _DEFAULT_VERSION = "1"
58
58
 
59
59
  class DMSMetadata(BaseMetadata):
60
60
  role: ClassVar[RoleTypes] = RoleTypes.dms_architect
61
- data_model_type: DataModelType = Field(DataModelType.solution, alias="dataModelType")
61
+ data_model_type: DataModelType = Field(DataModelType.enterprise, alias="dataModelType")
62
62
  schema_: SchemaCompleteness = Field(alias="schema")
63
63
  extension: ExtensionCategory = ExtensionCategory.addition
64
64
  space: ExternalIdType
@@ -146,10 +146,11 @@ class DMSMetadata(BaseMetadata):
146
146
  return description, creator
147
147
 
148
148
  @classmethod
149
- def from_data_model(cls, data_model: dm.DataModelApply) -> "DMSMetadata":
149
+ def from_data_model(cls, data_model: dm.DataModelApply, has_reference: bool) -> "DMSMetadata":
150
150
  description, creator = cls._get_description_and_creator(data_model.description)
151
151
  return cls(
152
152
  schema_=SchemaCompleteness.complete,
153
+ data_model_type=DataModelType.solution if has_reference else DataModelType.enterprise,
153
154
  space=data_model.space,
154
155
  name=data_model.name or None,
155
156
  description=description,
@@ -3,9 +3,7 @@ from dataclasses import dataclass
3
3
  from datetime import datetime
4
4
  from typing import Any, Literal, cast, overload
5
5
 
6
- from pydantic import BaseModel
7
-
8
- from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
6
+ from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness, _add_alias
9
7
  from cognite.neat.rules.models.data_types import DataType
10
8
  from cognite.neat.rules.models.entities import (
11
9
  ClassEntity,
@@ -44,8 +42,9 @@ class DMSMetadataInput:
44
42
  external_id=data.get("external_id"), # type: ignore[arg-type]
45
43
  creator=data.get("creator"), # type: ignore[arg-type]
46
44
  version=data.get("version"), # type: ignore[arg-type]
47
- extension=data.get("extension", "addition"),
48
- data_model_type=data.get("data_model_type", "solution"),
45
+ # safeguard from empty cell, i.e. if key provided by value None
46
+ extension=data.get("extension", "addition") or "addition",
47
+ data_model_type=data.get("data_model_type", "solution") or "solution",
49
48
  name=data.get("name"),
50
49
  description=data.get("description"),
51
50
  created=data.get("created"),
@@ -355,9 +354,3 @@ class DMSRulesInput:
355
354
  Last=last,
356
355
  Reference=reference,
357
356
  )
358
-
359
-
360
- def _add_alias(data: dict[str, Any], base_model: type[BaseModel]) -> None:
361
- for field_name, field_ in base_model.model_fields.items():
362
- if field_name not in data and field_.alias in data:
363
- data[field_name] = data[field_.alias]
@@ -516,10 +516,11 @@ class DMSSchema:
516
516
  defined_spaces = self.spaces.copy()
517
517
  defined_containers = self.containers.copy()
518
518
  defined_views = self.views.copy()
519
- if self.reference:
520
- defined_spaces |= self.reference.spaces
521
- defined_containers |= self.reference.containers
522
- defined_views |= self.reference.views
519
+ for other_schema in [self.reference, self.last]:
520
+ if other_schema:
521
+ defined_spaces |= other_schema.spaces
522
+ defined_containers |= other_schema.containers
523
+ defined_views |= other_schema.views
523
524
 
524
525
  for container in self.containers.values():
525
526
  if container.space not in defined_spaces:
@@ -37,7 +37,7 @@ class _DMSRulesSerializer:
37
37
  DMSProperty.model_fields[field].alias or field for field in self.properties_fields
38
38
  ]
39
39
  self.views_fields = [DMSView.model_fields[field].alias or field for field in self.views_fields]
40
- self.container_fields = [
40
+ self.containers_fields = [
41
41
  DMSContainer.model_fields[field].alias or field for field in self.containers_fields
42
42
  ]
43
43
  self.prop_view = DMSProperty.model_fields[self.prop_view].alias or self.prop_view
@@ -5,12 +5,13 @@ from cognite.client import data_modeling as dm
5
5
 
6
6
  from cognite.neat.rules import issues
7
7
  from cognite.neat.rules.issues import IssueList
8
- from cognite.neat.rules.models._base import ExtensionCategory, SchemaCompleteness
8
+ from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
9
9
  from cognite.neat.rules.models.data_types import DataType
10
10
  from cognite.neat.rules.models.entities import ContainerEntity
11
11
  from cognite.neat.rules.models.wrapped_entities import RawFilter
12
12
 
13
13
  from ._rules import DMSProperty, DMSRules
14
+ from ._schema import DMSSchema
14
15
 
15
16
 
16
17
  class DMSPostValidation:
@@ -26,12 +27,17 @@ class DMSPostValidation:
26
27
  self.issue_list = IssueList()
27
28
 
28
29
  def validate(self) -> IssueList:
29
- self._validate_best_practices()
30
+ self._validate_raw_filter()
30
31
  self._consistent_container_properties()
32
+
31
33
  self._referenced_views_and_containers_are_existing()
32
- self._validate_extension()
33
- self._validate_schema()
34
- self._validate_performance()
34
+ if self.metadata.schema_ is SchemaCompleteness.extended:
35
+ self._validate_extension()
36
+ if self.metadata.schema_ is SchemaCompleteness.partial:
37
+ return self.issue_list
38
+ dms_schema = self.rules.as_schema()
39
+ self.issue_list.extend(dms_schema.validate())
40
+ self._validate_performance(dms_schema)
35
41
  return self.issue_list
36
42
 
37
43
  def _consistent_container_properties(self) -> None:
@@ -106,8 +112,9 @@ class DMSPostValidation:
106
112
  self.issue_list.extend(errors)
107
113
 
108
114
  def _referenced_views_and_containers_are_existing(self) -> None:
109
- # There two checks are done in the same method to raise all the errors at once.
110
115
  defined_views = {view.view.as_id() for view in self.views}
116
+ if self.metadata.schema_ is SchemaCompleteness.extended and self.rules.last:
117
+ defined_views |= {view.view.as_id() for view in self.rules.last.views}
111
118
 
112
119
  errors: list[issues.NeatValidationError] = []
113
120
  for prop_no, prop in enumerate(self.properties):
@@ -125,6 +132,11 @@ class DMSPostValidation:
125
132
  )
126
133
  if self.metadata.schema_ is SchemaCompleteness.complete:
127
134
  defined_containers = {container.container.as_id() for container in self.containers or []}
135
+ if self.metadata.data_model_type == DataModelType.solution and self.rules.reference:
136
+ defined_containers |= {
137
+ container.container.as_id() for container in self.rules.reference.containers or []
138
+ }
139
+
128
140
  for prop_no, prop in enumerate(self.properties):
129
141
  if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
130
142
  errors.append(
@@ -157,19 +169,16 @@ class DMSPostValidation:
157
169
  def _validate_extension(self) -> None:
158
170
  if self.metadata.schema_ is not SchemaCompleteness.extended:
159
171
  return None
160
- if not self.rules.reference:
161
- raise ValueError("The schema is set to 'extended', but no reference rules are provided to validate against")
162
- is_solution = self.metadata.space != self.rules.reference.metadata.space
163
- if is_solution:
164
- return None
172
+ if not self.rules.last:
173
+ raise ValueError("The schema is set to 'extended', but no last rules are provided to validate against")
165
174
  if self.metadata.extension is ExtensionCategory.rebuild:
166
175
  # Everything is allowed
167
176
  return None
168
- # Is an extension of an existing model.
169
177
  user_schema = self.rules.as_schema()
170
- ref_schema = self.rules.reference.as_schema()
171
178
  new_containers = user_schema.containers.copy()
172
- existing_containers = ref_schema.containers.copy()
179
+
180
+ last_schema = self.rules.last.as_schema()
181
+ existing_containers = last_schema.containers.copy()
173
182
 
174
183
  for container_id, container in new_containers.items():
175
184
  existing_container = existing_containers.get(container_id)
@@ -189,14 +198,12 @@ class DMSPostValidation:
189
198
  )
190
199
  )
191
200
 
192
- if self.metadata.extension is ExtensionCategory.reshape and self.issue_list:
193
- return None
194
- elif self.metadata.extension is ExtensionCategory.reshape:
201
+ if self.metadata.extension is ExtensionCategory.reshape:
195
202
  # Reshape allows changes to views
196
203
  return None
197
204
 
198
205
  new_views = user_schema.views.copy()
199
- existing_views = ref_schema.views.copy()
206
+ existing_views = last_schema.views.copy()
200
207
  for view_id, view in new_views.items():
201
208
  existing_view = existing_views.get(view_id)
202
209
  if not existing_view or existing_view == view:
@@ -213,14 +220,7 @@ class DMSPostValidation:
213
220
  )
214
221
  )
215
222
 
216
- def _validate_performance(self) -> None:
217
- # we can only validate performance on complete schemas due to the need
218
- # to access all the container mappings
219
- if self.metadata.schema_ is not SchemaCompleteness.complete:
220
- return None
221
-
222
- dms_schema = self.rules.as_schema()
223
-
223
+ def _validate_performance(self, dms_schema: DMSSchema) -> None:
224
224
  for view_id, view in dms_schema.views.items():
225
225
  mapped_containers = dms_schema._get_mapped_container_from_view(view_id)
226
226
 
@@ -243,7 +243,7 @@ class DMSPostValidation:
243
243
  )
244
244
  )
245
245
 
246
- def _validate_best_practices(self) -> None:
246
+ def _validate_raw_filter(self) -> None:
247
247
  for view in self.views:
248
248
  if view.filter_ and isinstance(view.filter_, RawFilter):
249
249
  self.issue_list.append(
@@ -264,36 +264,3 @@ class DMSPostValidation:
264
264
  existing_properties = existing_dumped.get("properties", {})
265
265
  changed_properties = [prop for prop in new_properties if new_properties[prop] != existing_properties.get(prop)]
266
266
  return changed_attributes, changed_properties
267
-
268
- def _validate_schema(self) -> None:
269
- if self.metadata.schema_ is SchemaCompleteness.partial:
270
- return None
271
- elif self.metadata.schema_ is SchemaCompleteness.complete:
272
- rules: DMSRules = self.rules
273
- elif self.metadata.schema_ is SchemaCompleteness.extended:
274
- if not self.rules.reference:
275
- raise ValueError(
276
- "The schema is set to 'extended', but no reference rules are provided to validate against"
277
- )
278
- # This is an extension of the reference rules, we need to merge the two
279
- rules = self.rules.model_copy(deep=True)
280
- rules.properties.extend(self.rules.reference.properties.data)
281
- existing_views = {view.view.as_id() for view in rules.views}
282
- rules.views.extend([view for view in self.rules.reference.views if view.view.as_id() not in existing_views])
283
- if rules.containers and self.rules.reference.containers:
284
- existing_containers = {container.container.as_id() for container in rules.containers.data}
285
- rules.containers.extend(
286
- [
287
- container
288
- for container in self.rules.reference.containers
289
- if container.container.as_id() not in existing_containers
290
- ]
291
- )
292
- elif not rules.containers and self.rules.reference.containers:
293
- rules.containers = self.rules.reference.containers
294
- else:
295
- raise ValueError("Unknown schema completeness")
296
-
297
- schema = rules.as_schema()
298
- errors = schema.validate()
299
- self.issue_list.extend(errors)
@@ -1,3 +1,10 @@
1
1
  from ._rules import InformationClass, InformationMetadata, InformationProperty, InformationRules
2
+ from ._rules_input import InformationRulesInput
2
3
 
3
- __all__ = ["InformationRules", "InformationMetadata", "InformationClass", "InformationProperty"]
4
+ __all__ = [
5
+ "InformationRules",
6
+ "InformationMetadata",
7
+ "InformationClass",
8
+ "InformationProperty",
9
+ "InformationRulesInput",
10
+ ]