cognite-neat 0.77.2__py3-none-any.whl → 0.77.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (31) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/rules/analysis/_information_rules.py +2 -12
  3. cognite/neat/rules/exporters/_rules2excel.py +78 -89
  4. cognite/neat/rules/importers/_dms2rules.py +24 -11
  5. cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +1 -0
  6. cognite/neat/rules/importers/_spreadsheet2rules.py +21 -8
  7. cognite/neat/rules/issues/dms.py +19 -0
  8. cognite/neat/rules/issues/importing.py +16 -0
  9. cognite/neat/rules/issues/spreadsheet.py +60 -5
  10. cognite/neat/rules/models/_base.py +6 -0
  11. cognite/neat/rules/models/dms/_converter.py +25 -19
  12. cognite/neat/rules/models/dms/_exporter.py +2 -1
  13. cognite/neat/rules/models/dms/_rules.py +3 -2
  14. cognite/neat/rules/models/dms/_rules_input.py +4 -11
  15. cognite/neat/rules/models/dms/_schema.py +41 -5
  16. cognite/neat/rules/models/dms/_serializer.py +1 -1
  17. cognite/neat/rules/models/dms/_validation.py +27 -60
  18. cognite/neat/rules/models/information/__init__.py +8 -1
  19. cognite/neat/rules/models/information/_converter.py +27 -19
  20. cognite/neat/rules/models/information/_rules.py +41 -82
  21. cognite/neat/rules/models/information/_rules_input.py +266 -0
  22. cognite/neat/rules/models/information/_serializer.py +85 -0
  23. cognite/neat/rules/models/information/_validation.py +164 -0
  24. cognite/neat/utils/cdf.py +35 -0
  25. cognite/neat/workflows/steps/lib/current/rules_exporter.py +30 -7
  26. cognite/neat/workflows/steps/lib/current/rules_importer.py +21 -2
  27. {cognite_neat-0.77.2.dist-info → cognite_neat-0.77.4.dist-info}/METADATA +1 -1
  28. {cognite_neat-0.77.2.dist-info → cognite_neat-0.77.4.dist-info}/RECORD +31 -28
  29. {cognite_neat-0.77.2.dist-info → cognite_neat-0.77.4.dist-info}/LICENSE +0 -0
  30. {cognite_neat-0.77.2.dist-info → cognite_neat-0.77.4.dist-info}/WHEEL +0 -0
  31. {cognite_neat-0.77.2.dist-info → cognite_neat-0.77.4.dist-info}/entry_points.txt +0 -0
cognite/neat/_version.py CHANGED
@@ -1 +1 @@
1
- __version__ = "0.77.2"
1
+ __version__ = "0.77.4"
@@ -13,23 +13,13 @@ from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, ParentC
13
13
  from cognite.neat.rules.models.information import InformationClass, InformationProperty, InformationRules
14
14
  from cognite.neat.utils.utils import get_inheritance_path
15
15
 
16
- from ._base import BaseAnalysis, DataModelingScenario
16
+ from ._base import BaseAnalysis
17
17
 
18
18
 
19
19
  class InformationArchitectRulesAnalysis(BaseAnalysis):
20
20
  def __init__(self, rules: InformationRules):
21
21
  self.rules = rules
22
22
 
23
- @property
24
- def data_modeling_scenario(self) -> DataModelingScenario:
25
- if not self.rules.reference:
26
- return DataModelingScenario.from_scratch
27
-
28
- if self.rules.metadata.namespace == self.rules.reference.metadata.namespace:
29
- return DataModelingScenario.extend_reference
30
- else:
31
- return DataModelingScenario.build_solution
32
-
33
23
  @property
34
24
  def referred_classes(self) -> set[ClassEntity]:
35
25
  return self.directly_referred_classes.union(self.inherited_referred_classes)
@@ -362,7 +352,7 @@ class InformationArchitectRulesAnalysis(BaseAnalysis):
362
352
 
363
353
  rules = cast(InformationRules, self.rules.reference if use_reference else self.rules)
364
354
 
365
- if not rules.metadata.schema_ is not SchemaCompleteness.complete:
355
+ if rules.metadata.schema_ is not SchemaCompleteness.complete:
366
356
  raise ValueError("Rules are not complete cannot perform reduction!")
367
357
  class_as_dict = self.as_class_dict()
368
358
  class_parents_pairs = self.class_parent_pairs()
@@ -1,7 +1,8 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import itertools
4
- from datetime import datetime
4
+ import json
5
+ from datetime import datetime, timezone
5
6
  from pathlib import Path
6
7
  from types import GenericAlias
7
8
  from typing import Any, ClassVar, Literal, cast, get_args
@@ -14,14 +15,14 @@ from openpyxl.worksheet.worksheet import Worksheet
14
15
  from cognite.neat.rules._shared import Rules
15
16
  from cognite.neat.rules.models import (
16
17
  DataModelType,
17
- DMSRules,
18
- DomainRules,
19
18
  ExtensionCategory,
20
- InformationRules,
21
19
  RoleTypes,
22
20
  SchemaCompleteness,
23
21
  SheetEntity,
24
22
  )
23
+ from cognite.neat.rules.models.dms import DMSMetadata
24
+ from cognite.neat.rules.models.domain import DomainMetadata
25
+ from cognite.neat.rules.models.information import InformationMetadata
25
26
 
26
27
  from ._base import BaseExporter
27
28
 
@@ -34,9 +35,6 @@ class ExcelExporter(BaseExporter[Workbook]):
34
35
  on the different styles.
35
36
  output_role: The role to use for the exported spreadsheet. If provided, the rules will be converted to
36
37
  this role formate before being written to excel. If not provided, the role from the rules will be used.
37
- new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
38
- rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
39
- new metadata sheet in the Excel file.
40
38
  dump_as: This determines how the rules are written to the Excel file. An Excel file has up to three sets of
41
39
  sheets: user, last, and reference. The user sheets are used for inputting rules from a user. The last sheets
42
40
  are used for the last version of the same model as the user, while the reference sheets are used for
@@ -49,6 +47,10 @@ class ExcelExporter(BaseExporter[Workbook]):
49
47
  change a model that has already been published to CDF and that model is in production.
50
48
  * "reference": The rules are written to the reference sheets. This is typically used when you want to build
51
49
  a new solution on top of an enterprise model.
50
+ new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
51
+ rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
52
+ new metadata sheet in the Excel file. The model id is expected to be a tuple of (prefix, title)
53
+ (space, external_id) for InformationRules and DMSRules respectively.
52
54
 
53
55
  The following styles are available:
54
56
 
@@ -74,8 +76,8 @@ class ExcelExporter(BaseExporter[Workbook]):
74
76
  self,
75
77
  styling: Style = "default",
76
78
  output_role: RoleTypes | None = None,
77
- new_model_id: tuple[str, str, str] | None = None,
78
79
  dump_as: DumpOptions = "user",
80
+ new_model_id: tuple[str, str] | None = None,
79
81
  ):
80
82
  if styling not in self.style_options:
81
83
  raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
@@ -106,9 +108,11 @@ class ExcelExporter(BaseExporter[Workbook]):
106
108
  dumped_last_rules: dict[str, Any] | None = None
107
109
  dumped_reference_rules: dict[str, Any] | None = None
108
110
  if self.dump_as != "user":
109
- # Writes empty reference sheets
111
+ action = {"last": "update", "reference": "create"}[self.dump_as]
112
+ metadata_creator = _MetadataCreator(action, self.new_model_id) # type: ignore[arg-type]
113
+
110
114
  dumped_user_rules = {
111
- "Metadata": self._create_metadata_sheet_user_rules(rules),
115
+ "Metadata": metadata_creator.create(rules.metadata),
112
116
  }
113
117
 
114
118
  if self.dump_as == "last":
@@ -128,11 +132,11 @@ class ExcelExporter(BaseExporter[Workbook]):
128
132
  self._write_sheets(workbook, dumped_user_rules, rules)
129
133
  if dumped_last_rules:
130
134
  self._write_sheets(workbook, dumped_last_rules, rules, sheet_prefix="Last")
135
+ self._write_metadata_sheet(workbook, dumped_last_rules["Metadata"], sheet_prefix="Last")
131
136
 
132
137
  if dumped_reference_rules:
133
- prefix = "Ref"
134
- self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix=prefix)
135
- self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix=prefix)
138
+ self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix="Ref")
139
+ self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix="Ref")
136
140
 
137
141
  if self._styling_level > 0:
138
142
  self._adjust_column_widths(workbook)
@@ -234,82 +238,67 @@ class ExcelExporter(BaseExporter[Workbook]):
234
238
  sheet.column_dimensions[selected_column.column_letter].width = max(current, max_length + 0.5)
235
239
  return None
236
240
 
237
- def _create_metadata_sheet_user_rules(self, rules: Rules) -> dict[str, Any]:
238
- metadata: dict[str, Any] = {
239
- field_alias: None for field_alias in rules.metadata.model_dump(by_alias=True).keys()
240
- }
241
- if "creator" in metadata:
242
- metadata["creator"] = "YOUR NAME"
243
-
244
- if isinstance(rules, DomainRules):
245
- return metadata
246
- elif isinstance(rules, DMSRules):
247
- existing_model_id = (rules.metadata.space, rules.metadata.external_id, rules.metadata.version)
248
- elif isinstance(rules, InformationRules):
249
- existing_model_id = (rules.metadata.prefix, rules.metadata.name, rules.metadata.version)
250
- else:
251
- raise ValueError(f"Unsupported rules type: {type(rules)}")
252
- existing_metadata = rules.metadata.model_dump(by_alias=True)
253
- if isinstance(existing_metadata["created"], datetime):
254
- metadata["created"] = existing_metadata["created"].replace(tzinfo=None)
255
- if isinstance(existing_metadata["updated"], datetime):
256
- metadata["updated"] = existing_metadata["updated"].replace(tzinfo=None)
257
- # Excel does not support timezone in datetime strings
258
- now_iso = datetime.now().replace(tzinfo=None).isoformat()
259
- is_info = isinstance(rules, InformationRules)
260
- is_dms = isinstance(rules, DMSRules)
261
- is_extension = self.new_model_id is not None or rules.reference is not None
262
- is_solution = rules.metadata.data_model_type == DataModelType.solution
263
-
264
- if is_solution and self.new_model_id:
265
- metadata["prefix" if is_info else "space"] = self.new_model_id[0] # type: ignore[index]
266
- metadata["title" if is_info else "externalId"] = self.new_model_id[1] # type: ignore[index]
267
- metadata["version"] = self.new_model_id[2] # type: ignore[index]
268
- elif is_solution and self.dump_as == "reference" and rules.reference:
269
- metadata["prefix" if is_info else "space"] = "YOUR_PREFIX"
270
- metadata["title" if is_info else "externalId"] = "YOUR_TITLE"
271
- metadata["version"] = "1"
272
- else:
273
- metadata["prefix" if is_info else "space"] = existing_model_id[0]
274
- metadata["title" if is_info else "externalId"] = existing_model_id[1]
275
- metadata["version"] = existing_model_id[2]
276
-
277
- if is_solution and is_info and self.new_model_id:
278
- metadata["namespace"] = f"http://purl.org/{self.new_model_id[0]}/" # type: ignore[index]
279
- elif is_info:
280
- metadata["namespace"] = existing_metadata["namespace"]
281
-
282
- if is_solution and is_dms and self.new_model_id:
283
- metadata["name"] = self.new_model_id[1] # type: ignore[index]
284
-
285
- if is_solution:
286
- metadata["created"] = now_iso
287
- else:
288
- metadata["created"] = existing_metadata["created"]
289
-
290
- if is_solution or is_extension:
291
- metadata["updated"] = now_iso
292
- else:
293
- metadata["updated"] = existing_metadata["updated"]
294
241
 
295
- if is_solution:
296
- metadata["creator"] = "YOUR NAME"
297
- else:
298
- metadata["creator"] = existing_metadata["creator"]
299
-
300
- if not is_solution:
301
- metadata["description"] = existing_metadata["description"]
242
+ class _MetadataCreator:
243
+ creator_name = "<YOUR NAME>"
302
244
 
303
- if is_extension:
304
- metadata["schema"] = SchemaCompleteness.extended.value
305
- else:
306
- metadata["schema"] = SchemaCompleteness.complete.value
307
-
308
- if is_solution:
309
- metadata["dataModelType"] = DataModelType.solution.value
245
+ def __init__(
246
+ self,
247
+ action: Literal["create", "update"],
248
+ new_model_id: tuple[str, str] | None = None,
249
+ ):
250
+ self.action = action
251
+ self.new_model_id = new_model_id or ("YOUR_PREFIX", "YOUR_TITLE")
252
+
253
+ def create(self, metadata: DomainMetadata | InformationMetadata | DMSMetadata) -> dict[str, Any]:
254
+ now = datetime.now(timezone.utc).replace(microsecond=0, tzinfo=None)
255
+ if self.action == "update":
256
+ output = json.loads(metadata.model_dump_json(by_alias=True))
257
+ # This is the same for Information and DMS
258
+ output["updated"] = now.isoformat()
259
+ output["schema"] = SchemaCompleteness.extended.value
260
+ output["extension"] = ExtensionCategory.addition.value
261
+ if value := output.get("creator"):
262
+ output["creator"] = f"{value}, {self.creator_name}"
263
+ else:
264
+ output["creator"] = self.creator_name
265
+ return output
266
+
267
+ # Action "create"
268
+ if isinstance(metadata, DomainMetadata):
269
+ output = {field_alias: None for field_alias in metadata.model_dump(by_alias=True).keys()}
270
+ output["role"] = metadata.role.value
271
+ output["creator"] = self.creator_name
272
+ return output
273
+
274
+ new_metadata = self._create_new_info(now)
275
+ if isinstance(metadata, DMSMetadata):
276
+ from cognite.neat.rules.models.information._converter import _InformationRulesConverter
277
+
278
+ output_metadata: DMSMetadata | InformationMetadata = _InformationRulesConverter._convert_metadata_to_dms(
279
+ new_metadata
280
+ )
281
+ elif isinstance(metadata, InformationMetadata):
282
+ output_metadata = new_metadata
310
283
  else:
311
- metadata["dataModelType"] = DataModelType.enterprise.value
312
-
313
- metadata["extension"] = ExtensionCategory.addition.value
314
- metadata["role"] = (self.output_role and self.output_role.value) or rules.metadata.role.value
315
- return metadata
284
+ raise ValueError(f"Bug in Neat: Unknown metadata type: {type(metadata)}")
285
+
286
+ created = json.loads(output_metadata.model_dump_json(by_alias=True))
287
+ created.pop("extension", None)
288
+ return created
289
+
290
+ def _create_new_info(self, now: datetime) -> InformationMetadata:
291
+ prefix = self.new_model_id[0]
292
+ return InformationMetadata(
293
+ data_model_type=DataModelType.solution,
294
+ schema_=SchemaCompleteness.complete,
295
+ extension=ExtensionCategory.addition,
296
+ prefix=prefix,
297
+ namespace=f"http://purl.org/neat/{prefix}/", # type: ignore[arg-type]
298
+ description=None,
299
+ version="1",
300
+ created=now,
301
+ updated=now,
302
+ creator=[self.creator_name],
303
+ name=self.new_model_id[1],
304
+ )
@@ -61,8 +61,10 @@ class DMSImporter(BaseImporter):
61
61
  self.ref_metadata = ref_metadata
62
62
  self.issue_list = IssueList(read_issues)
63
63
  self._all_containers_by_id = schema.containers.copy()
64
+ self._all_view_ids = set(self.root_schema.views.keys())
64
65
  if self.root_schema.reference:
65
66
  self._all_containers_by_id.update(self.root_schema.reference.containers)
67
+ self._all_view_ids.update(self.root_schema.reference.views.keys())
66
68
 
67
69
  @classmethod
68
70
  def from_data_model_id(
@@ -100,15 +102,17 @@ class DMSImporter(BaseImporter):
100
102
  else:
101
103
  ref_model = None
102
104
 
103
- try:
105
+ issue_list = IssueList()
106
+ with _handle_issues(issue_list) as result:
104
107
  schema = DMSSchema.from_data_model(client, user_model, ref_model)
105
- except Exception as e:
106
- return cls(DMSSchema(), [issues.importing.APIError(str(e))])
107
108
 
108
- metadata = cls._create_metadata_from_model(user_model)
109
+ if result.result == "failure" or issue_list.has_errors:
110
+ return cls(DMSSchema(), issue_list)
111
+
112
+ metadata = cls._create_metadata_from_model(user_model, has_reference=ref_model is not None)
109
113
  ref_metadata = cls._create_metadata_from_model(ref_model) if ref_model else None
110
114
 
111
- return cls(schema, [], metadata, ref_metadata)
115
+ return cls(schema, issue_list, metadata, ref_metadata)
112
116
 
113
117
  @classmethod
114
118
  def _find_model_in_list(
@@ -127,6 +131,7 @@ class DMSImporter(BaseImporter):
127
131
  def _create_metadata_from_model(
128
132
  cls,
129
133
  model: dm.DataModel[dm.View] | dm.DataModelApply,
134
+ has_reference: bool = False,
130
135
  ) -> DMSMetadata:
131
136
  description, creator = DMSMetadata._get_description_and_creator(model.description)
132
137
 
@@ -139,6 +144,7 @@ class DMSImporter(BaseImporter):
139
144
  updated = now
140
145
  return DMSMetadata(
141
146
  schema_=SchemaCompleteness.complete,
147
+ data_model_type=DataModelType.solution if has_reference else DataModelType.enterprise,
142
148
  extension=ExtensionCategory.addition,
143
149
  space=model.space,
144
150
  external_id=model.external_id,
@@ -198,16 +204,21 @@ class DMSImporter(BaseImporter):
198
204
  **self._create_rule_components(
199
205
  ref_model,
200
206
  ref_schema,
201
- self.ref_metadata or self._create_default_metadata(list(ref_schema.views.values())),
207
+ self.ref_metadata
208
+ or self._create_default_metadata(list(ref_schema.views.values()), is_ref=True),
202
209
  DataModelType.enterprise,
203
210
  )
204
211
  )
205
- schema_completeness = SchemaCompleteness.extended
206
212
  data_model_type = DataModelType.solution
207
213
 
208
214
  user_rules = DMSRules(
209
215
  **self._create_rule_components(
210
- model, self.root_schema, self.metadata, data_model_type, schema_completeness
216
+ model,
217
+ self.root_schema,
218
+ self.metadata,
219
+ data_model_type,
220
+ schema_completeness,
221
+ has_reference=reference is not None,
211
222
  ),
212
223
  reference=reference,
213
224
  )
@@ -224,6 +235,7 @@ class DMSImporter(BaseImporter):
224
235
  metadata: DMSMetadata | None = None,
225
236
  data_model_type: DataModelType | None = None,
226
237
  schema_completeness: SchemaCompleteness | None = None,
238
+ has_reference: bool = False,
227
239
  ) -> dict[str, Any]:
228
240
  properties = SheetList[DMSProperty]()
229
241
  for view_id, view in schema.views.items():
@@ -238,7 +250,7 @@ class DMSImporter(BaseImporter):
238
250
  view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
239
251
  }
240
252
 
241
- metadata = metadata or DMSMetadata.from_data_model(data_model)
253
+ metadata = metadata or DMSMetadata.from_data_model(data_model, has_reference)
242
254
  if data_model_type is not None:
243
255
  metadata.data_model_type = data_model_type
244
256
  if schema_completeness is not None:
@@ -258,12 +270,13 @@ class DMSImporter(BaseImporter):
258
270
  )
259
271
 
260
272
  @classmethod
261
- def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply]) -> DMSMetadata:
273
+ def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply], is_ref: bool = False) -> DMSMetadata:
262
274
  now = datetime.now().replace(microsecond=0)
263
275
  space = Counter(view.space for view in views).most_common(1)[0][0]
264
276
  return DMSMetadata(
265
277
  schema_=SchemaCompleteness.complete,
266
278
  extension=ExtensionCategory.addition,
279
+ data_model_type=DataModelType.enterprise if is_ref else DataModelType.solution,
267
280
  space=space,
268
281
  external_id="Unknown",
269
282
  version="0.1.0",
@@ -361,7 +374,7 @@ class DMSImporter(BaseImporter):
361
374
  elif isinstance(prop, dm.MappedPropertyApply):
362
375
  container_prop = self._container_prop_unsafe(cast(dm.MappedPropertyApply, prop))
363
376
  if isinstance(container_prop.type, dm.DirectRelation):
364
- if prop.source is None:
377
+ if prop.source is None or prop.source not in self._all_view_ids:
365
378
  # The warning is issued when the DMS Rules are created.
366
379
  return DMSUnknownEntity()
367
380
  else:
@@ -134,6 +134,7 @@ class DTDLImporter(BaseImporter):
134
134
 
135
135
  metadata = self._default_metadata()
136
136
  metadata["schema"] = self._schema_completeness.value
137
+
137
138
  if self.title:
138
139
  metadata["title"] = to_pascal(self.title)
139
140
  try:
@@ -22,6 +22,7 @@ from cognite.neat.rules.models import (
22
22
  SchemaCompleteness,
23
23
  )
24
24
  from cognite.neat.rules.models.dms import DMSRulesInput
25
+ from cognite.neat.rules.models.information import InformationRulesInput
25
26
  from cognite.neat.utils.auxiliary import local_import
26
27
  from cognite.neat.utils.spreadsheet import SpreadsheetRead, read_individual_sheet
27
28
 
@@ -110,17 +111,27 @@ class SpreadsheetReader:
110
111
  self.required = required
111
112
  self.metadata = metadata
112
113
  self._sheet_prefix = sheet_prefix
114
+ self._seen_files: set[Path] = set()
115
+ self._seen_sheets: set[str] = set()
113
116
 
114
117
  @property
115
118
  def metadata_sheet_name(self) -> str:
116
119
  return f"{self._sheet_prefix}Metadata"
117
120
 
121
+ @property
122
+ def seen_sheets(self) -> set[str]:
123
+ if not self._seen_files:
124
+ raise ValueError("No files have been read yet.")
125
+ return self._seen_sheets
126
+
118
127
  def sheet_names(self, role: RoleTypes) -> set[str]:
119
128
  names = MANDATORY_SHEETS_BY_ROLE[role]
120
129
  return {f"{self._sheet_prefix}{sheet_name}" for sheet_name in names if sheet_name != "Metadata"}
121
130
 
122
131
  def read(self, filepath: Path) -> None | ReadResult:
123
132
  with pd.ExcelFile(filepath) as excel_file:
133
+ self._seen_files.add(filepath)
134
+ self._seen_sheets.update(map(str, excel_file.sheet_names))
124
135
  metadata: MetadataRaw | None
125
136
  if self.metadata is not None:
126
137
  metadata = self.metadata
@@ -212,20 +223,20 @@ class ExcelImporter(BaseImporter):
212
223
  issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
213
224
  return self._return_or_raise(issue_list, errors)
214
225
 
215
- user_read = SpreadsheetReader(issue_list).read(self.filepath)
226
+ user_reader = SpreadsheetReader(issue_list)
227
+ user_read = user_reader.read(self.filepath)
216
228
  if user_read is None or issue_list.has_errors:
217
229
  return self._return_or_raise(issue_list, errors)
218
230
 
219
231
  last_read: ReadResult | None = None
232
+ if any(sheet_name.startswith("Last") for sheet_name in user_reader.seen_sheets):
233
+ last_read = SpreadsheetReader(issue_list, required=False, sheet_prefix="Last").read(self.filepath)
220
234
  reference_read: ReadResult | None = None
221
- if user_read.schema == SchemaCompleteness.extended:
222
- # Last does not have its own metadata sheet. It is the same as the user's metadata sheet.
223
- last_read = SpreadsheetReader(
224
- issue_list, required=False, metadata=user_read.metadata, sheet_prefix="Last"
225
- ).read(self.filepath)
235
+ if any(sheet_name.startswith("Ref") for sheet_name in user_reader.seen_sheets):
226
236
  reference_read = SpreadsheetReader(issue_list, sheet_prefix="Ref").read(self.filepath)
227
- if issue_list.has_errors:
228
- return self._return_or_raise(issue_list, errors)
237
+
238
+ if issue_list.has_errors:
239
+ return self._return_or_raise(issue_list, errors)
229
240
 
230
241
  if reference_read and user_read.role != reference_read.role:
231
242
  issue_list.append(issues.spreadsheet_file.RoleMismatchError(self.filepath))
@@ -253,6 +264,8 @@ class ExcelImporter(BaseImporter):
253
264
  rules: Rules
254
265
  if rules_cls is DMSRules:
255
266
  rules = DMSRulesInput.load(sheets).as_rules()
267
+ elif rules_cls is InformationRules:
268
+ rules = InformationRulesInput.load(sheets).as_rules()
256
269
  else:
257
270
  rules = rules_cls.model_validate(sheets) # type: ignore[attr-defined]
258
271
 
@@ -30,6 +30,7 @@ __all__ = [
30
30
  "HasDataFilterAppliedToTooManyContainersWarning",
31
31
  "ReverseRelationMissingOtherSideWarning",
32
32
  "NodeTypeFilterOnParentViewWarning",
33
+ "MissingViewInModelWarning",
33
34
  "ChangingContainerError",
34
35
  "ChangingViewError",
35
36
  ]
@@ -293,6 +294,24 @@ class ViewMapsToTooManyContainersWarning(DMSSchemaWarning):
293
294
  return output
294
295
 
295
296
 
297
+ @dataclass(frozen=True)
298
+ class MissingViewInModelWarning(DMSSchemaWarning):
299
+ description = "The data model contains view pointing to views not present in the data model"
300
+ fix = "Add the view(s) to the data model"
301
+ error_name: ClassVar[str] = "MissingViewInModel"
302
+ data_model_id: dm.DataModelId
303
+ view_ids: set[dm.ViewId]
304
+
305
+ def message(self) -> str:
306
+ return f"The view(s) {self.view_ids} are missing in the data model {self.data_model_id}"
307
+
308
+ def dump(self) -> dict[str, Any]:
309
+ output = super().dump()
310
+ output["data_model_id"] = self.data_model_id.dump()
311
+ output["view_id"] = [view_id.dump() for view_id in self.view_ids]
312
+ return output
313
+
314
+
296
315
  @dataclass(frozen=True)
297
316
  class ContainerPropertyUsedMultipleTimesError(DMSSchemaError):
298
317
  description = "The container property is used multiple times by the same view property"
@@ -1,5 +1,6 @@
1
1
  from abc import ABC
2
2
  from dataclasses import dataclass
3
+ from typing import Any
3
4
 
4
5
  from .base import NeatValidationError, ValidationWarning
5
6
 
@@ -23,6 +24,7 @@ __all__ = [
23
24
  "MissingIdentifierError",
24
25
  "UnsupportedPropertyTypeError",
25
26
  "APIError",
27
+ "FailedImportWarning",
26
28
  ]
27
29
 
28
30
 
@@ -259,6 +261,20 @@ class APIWarning(ModelImportWarning):
259
261
  return {"error_message": self.error_message}
260
262
 
261
263
 
264
+ @dataclass(frozen=True)
265
+ class FailedImportWarning(ModelImportWarning):
266
+ description = "Failed to import part of the model."
267
+ fix = "No fix is available."
268
+
269
+ identifier: set[str]
270
+
271
+ def message(self) -> str:
272
+ return f"Failed to import: {self.identifier}. This will be skipped."
273
+
274
+ def dump(self) -> dict[str, Any]:
275
+ return {"identifier": list(self.identifier)}
276
+
277
+
262
278
  @dataclass(frozen=True)
263
279
  class ModelImportError(NeatValidationError, ABC):
264
280
  description = "An error was raised during importing."
@@ -10,7 +10,7 @@ from pydantic_core import ErrorDetails
10
10
 
11
11
  from cognite.neat.utils.spreadsheet import SpreadsheetRead
12
12
 
13
- from .base import DefaultPydanticError, MultiValueError, NeatValidationError, ValidationWarning
13
+ from .base import DefaultPydanticError, MultiValueError, NeatValidationError
14
14
 
15
15
  if sys.version_info >= (3, 11):
16
16
  from typing import Self
@@ -27,7 +27,7 @@ __all__ = [
27
27
  "InvalidRowUnknownSheetError",
28
28
  "NonExistingContainerError",
29
29
  "NonExistingViewError",
30
- "ClassNoPropertiesNoParentsWarning",
30
+ "ClassNoPropertiesNoParentError",
31
31
  "InconsistentContainerDefinitionError",
32
32
  "MultiValueTypeError",
33
33
  "MultiValueIsListError",
@@ -239,7 +239,26 @@ class NonExistingViewError(InvalidPropertyError):
239
239
 
240
240
 
241
241
  @dataclass(frozen=True)
242
- class ClassNoPropertiesNoParentsWarning(ValidationWarning):
242
+ class PropertiesDefinedForUndefinedClassesError(NeatValidationError):
243
+ description = "Properties are defined for undefined classes."
244
+ fix = "Make sure to define class in the Classes sheet."
245
+
246
+ classes: list[str]
247
+
248
+ def dump(self) -> dict[str, list[str]]:
249
+ output = super().dump()
250
+ output["classes"] = self.classes
251
+ return output
252
+
253
+ def message(self) -> str:
254
+ return (
255
+ f"Classes {', '.join(self.classes)} have properties assigned to them, but"
256
+ " they are not defined in the Classes sheet."
257
+ )
258
+
259
+
260
+ @dataclass(frozen=True)
261
+ class ClassNoPropertiesNoParentError(NeatValidationError):
243
262
  description = "Class has no properties and no parents."
244
263
  fix = "Check if the class should have properties or parents."
245
264
 
@@ -252,8 +271,44 @@ class ClassNoPropertiesNoParentsWarning(ValidationWarning):
252
271
 
253
272
  def message(self) -> str:
254
273
  if len(self.classes) > 1:
255
- return f"Classes {', '.join(self.classes)} have no properties and no parents. This may be a mistake."
256
- return f"Class {self.classes[0]} has no properties and no parents. This may be a mistake."
274
+ return f"Classes {', '.join(self.classes)} have no direct or inherited properties. This may be a mistake."
275
+ return f"Class {self.classes[0]} have no direct or inherited properties. This may be a mistake."
276
+
277
+
278
+ @dataclass(frozen=True)
279
+ class ParentClassesNotDefinedError(NeatValidationError):
280
+ description = "Parent classes are not defined."
281
+ fix = "Check if the parent classes are defined in Classes sheet."
282
+
283
+ classes: list[str]
284
+
285
+ def dump(self) -> dict[str, list[str]]:
286
+ output = super().dump()
287
+ output["classes"] = self.classes
288
+ return output
289
+
290
+ def message(self) -> str:
291
+ if len(self.classes) > 1:
292
+ return f"Parent classes {', '.join(self.classes)} are not defined. This may be a mistake."
293
+ return f"Parent classes {', '.join(self.classes[0])} are not defined. This may be a mistake."
294
+
295
+
296
+ @dataclass(frozen=True)
297
+ class ValueTypeNotDefinedError(NeatValidationError):
298
+ description = "Value types referred by properties are not defined in Rules."
299
+ fix = "Make sure that all value types are defined in Rules."
300
+
301
+ value_types: list[str]
302
+
303
+ def dump(self) -> dict[str, list[str]]:
304
+ output = super().dump()
305
+ output["classes"] = self.value_types
306
+ return output
307
+
308
+ def message(self) -> str:
309
+ if len(self.value_types) > 1:
310
+ return f"Value types {', '.join(self.value_types)} are not defined. This may be a mistake."
311
+ return f"Value types {', '.join(self.value_types[0])} are not defined. This may be a mistake."
257
312
 
258
313
 
259
314
  @dataclass(frozen=True)
@@ -38,6 +38,12 @@ else:
38
38
  METADATA_VALUE_MAX_LENGTH = 5120
39
39
 
40
40
 
41
+ def _add_alias(data: dict[str, Any], base_model: type[BaseModel]) -> None:
42
+ for field_name, field_ in base_model.model_fields.items():
43
+ if field_name not in data and field_.alias in data:
44
+ data[field_name] = data[field_.alias]
45
+
46
+
41
47
  def replace_nan_floats_with_default(values: dict, model_fields: dict[str, FieldInfo]) -> dict:
42
48
  output = {}
43
49
  for field_name, value in values.items():