cognite-neat 0.76.1__py3-none-any.whl → 0.76.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (53) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/routers/core.py +1 -1
  3. cognite/neat/app/api/routers/rules.py +1 -1
  4. cognite/neat/graph/extractors/_mock_graph_generator.py +2 -2
  5. cognite/neat/rules/_shared.py +1 -1
  6. cognite/neat/rules/analysis/_information_rules.py +3 -3
  7. cognite/neat/rules/exporters/_base.py +1 -1
  8. cognite/neat/rules/exporters/_rules2dms.py +8 -49
  9. cognite/neat/rules/exporters/_rules2excel.py +71 -40
  10. cognite/neat/rules/exporters/_rules2ontology.py +2 -2
  11. cognite/neat/rules/exporters/_rules2yaml.py +1 -1
  12. cognite/neat/rules/exporters/_validation.py +2 -2
  13. cognite/neat/rules/importers/_base.py +1 -1
  14. cognite/neat/rules/importers/_dms2rules.py +93 -108
  15. cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +1 -1
  16. cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +2 -3
  17. cognite/neat/rules/importers/_owl2rules/_owl2classes.py +1 -1
  18. cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +2 -2
  19. cognite/neat/rules/importers/_owl2rules/_owl2properties.py +1 -1
  20. cognite/neat/rules/importers/_owl2rules/_owl2rules.py +1 -1
  21. cognite/neat/rules/importers/_spreadsheet2rules.py +87 -62
  22. cognite/neat/rules/importers/_yaml2rules.py +3 -3
  23. cognite/neat/rules/issues/base.py +5 -0
  24. cognite/neat/rules/issues/dms.py +65 -0
  25. cognite/neat/rules/models/__init__.py +27 -0
  26. cognite/neat/rules/models/dms/__init__.py +18 -0
  27. cognite/neat/rules/models/dms/_converter.py +140 -0
  28. cognite/neat/rules/models/dms/_exporter.py +405 -0
  29. cognite/neat/rules/models/dms/_rules.py +379 -0
  30. cognite/neat/rules/models/{rules/_dms_rules_write.py → dms/_rules_input.py} +42 -33
  31. cognite/neat/rules/models/{rules/_dms_schema.py → dms/_schema.py} +36 -4
  32. cognite/neat/rules/models/dms/_serializer.py +126 -0
  33. cognite/neat/rules/models/dms/_validation.py +288 -0
  34. cognite/neat/rules/models/{rules/_domain_rules.py → domain.py} +1 -0
  35. cognite/neat/rules/models/information/__init__.py +3 -0
  36. cognite/neat/rules/models/information/_converter.py +195 -0
  37. cognite/neat/rules/models/{rules/_information_rules.py → information/_rules.py} +35 -202
  38. cognite/neat/workflows/steps/data_contracts.py +1 -1
  39. cognite/neat/workflows/steps/lib/current/rules_exporter.py +10 -3
  40. cognite/neat/workflows/steps/lib/current/rules_importer.py +1 -1
  41. cognite/neat/workflows/steps/lib/current/rules_validator.py +1 -2
  42. {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/METADATA +1 -1
  43. {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/RECORD +51 -44
  44. cognite/neat/rules/models/rules/__init__.py +0 -14
  45. cognite/neat/rules/models/rules/_dms_architect_rules.py +0 -1255
  46. /cognite/neat/rules/models/{rules/_base.py → _base.py} +0 -0
  47. /cognite/neat/rules/models/{rdfpath.py → _rdfpath.py} +0 -0
  48. /cognite/neat/rules/models/{rules/_types → _types}/__init__.py +0 -0
  49. /cognite/neat/rules/models/{rules/_types → _types}/_base.py +0 -0
  50. /cognite/neat/rules/models/{rules/_types → _types}/_field.py +0 -0
  51. {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/LICENSE +0 -0
  52. {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/WHEEL +0 -0
  53. {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/entry_points.txt +0 -0
@@ -2,7 +2,7 @@ from collections import Counter
2
2
  from collections.abc import Sequence
3
3
  from datetime import datetime
4
4
  from pathlib import Path
5
- from typing import Literal, cast, overload
5
+ from typing import Any, Literal, cast, overload
6
6
 
7
7
  from cognite.client import CogniteClient
8
8
  from cognite.client import data_modeling as dm
@@ -20,7 +20,22 @@ from cognite.client.utils import ms_to_datetime
20
20
  from cognite.neat.rules import issues
21
21
  from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
22
22
  from cognite.neat.rules.issues import IssueList, ValidationIssue
23
+ from cognite.neat.rules.models import (
24
+ DataModelType,
25
+ DMSRules,
26
+ DMSSchema,
27
+ ExtensionCategory,
28
+ RoleTypes,
29
+ SchemaCompleteness,
30
+ SheetList,
31
+ )
23
32
  from cognite.neat.rules.models.data_types import DataType
33
+ from cognite.neat.rules.models.dms import (
34
+ DMSContainer,
35
+ DMSMetadata,
36
+ DMSProperty,
37
+ DMSView,
38
+ )
24
39
  from cognite.neat.rules.models.entities import (
25
40
  ClassEntity,
26
41
  ContainerEntity,
@@ -29,15 +44,6 @@ from cognite.neat.rules.models.entities import (
29
44
  ViewEntity,
30
45
  ViewPropertyEntity,
31
46
  )
32
- from cognite.neat.rules.models.rules import DMSRules, DMSSchema, RoleTypes
33
- from cognite.neat.rules.models.rules._base import DataModelType, ExtensionCategory, SchemaCompleteness
34
- from cognite.neat.rules.models.rules._dms_architect_rules import (
35
- DMSContainer,
36
- DMSMetadata,
37
- DMSProperty,
38
- DMSView,
39
- SheetList,
40
- )
41
47
 
42
48
 
43
49
  class DMSImporter(BaseImporter):
@@ -47,10 +53,17 @@ class DMSImporter(BaseImporter):
47
53
  read_issues: Sequence[ValidationIssue] | None = None,
48
54
  metadata: DMSMetadata | None = None,
49
55
  ):
50
- self.schema = schema
56
+ # Calling this root schema to distinguish it from
57
+ # * User Schema
58
+ # * Reference Schema
59
+ self.root_schema = schema
51
60
  self.metadata = metadata
52
61
  self.issue_list = IssueList(read_issues)
53
- self._container_by_id = {container.as_id(): container for container in schema.containers}
62
+ self._all_containers_by_id = {container.as_id(): container for container in schema.containers}
63
+ if self.root_schema.reference:
64
+ self._all_containers_by_id.update(
65
+ {container.as_id(): container for container in self.root_schema.reference.containers}
66
+ )
54
67
 
55
68
  @classmethod
56
69
  def from_data_model_id(cls, client: CogniteClient, data_model_id: DataModelIdentifier) -> "DMSImporter":
@@ -134,130 +147,102 @@ class DMSImporter(BaseImporter):
134
147
  # In case there were errors during the import, the to_rules method will return None
135
148
  return self._return_or_raise(self.issue_list, errors)
136
149
 
137
- if len(self.schema.data_models) == 0:
150
+ if len(self.root_schema.data_models) == 0:
138
151
  self.issue_list.append(issues.importing.NoDataModelError("No data model found."))
139
152
  return self._return_or_raise(self.issue_list, errors)
140
153
 
141
- if len(self.schema.data_models) > 2:
154
+ with _handle_issues(
155
+ self.issue_list,
156
+ ) as future:
157
+ schema_completeness = SchemaCompleteness.complete
158
+ data_model_type = DataModelType.enterprise
159
+ reference: DMSRules | None = None
160
+ if ref_schema := self.root_schema.reference:
161
+ # Reference should always be an enterprise model.
162
+ reference = DMSRules(
163
+ **self._create_rule_components(
164
+ ref_schema, self._create_default_metadata(ref_schema.views), DataModelType.enterprise
165
+ )
166
+ )
167
+ schema_completeness = SchemaCompleteness.extended
168
+ data_model_type = DataModelType.solution
169
+
170
+ user_rules = DMSRules(
171
+ **self._create_rule_components(self.root_schema, self.metadata, data_model_type, schema_completeness),
172
+ reference=reference,
173
+ )
174
+
175
+ if future.result == "failure" or self.issue_list.has_errors:
176
+ return self._return_or_raise(self.issue_list, errors)
177
+
178
+ return self._to_output(user_rules, self.issue_list, errors, role)
179
+
180
+ def _create_rule_components(
181
+ self,
182
+ schema: DMSSchema,
183
+ metadata: DMSMetadata | None = None,
184
+ data_model_type: DataModelType | None = None,
185
+ schema_completeness: SchemaCompleteness | None = None,
186
+ ) -> dict[str, Any]:
187
+ if len(schema.data_models) > 2:
142
188
  # Creating a DataModelEntity to convert the data model id to a string.
143
189
  self.issue_list.append(
144
190
  issues.importing.MultipleDataModelsWarning(
145
- [str(DataModelEntity.from_id(model.as_id())) for model in self.schema.data_models]
191
+ [str(DataModelEntity.from_id(model.as_id())) for model in schema.data_models]
146
192
  )
147
193
  )
148
194
 
149
- data_model = self.schema.data_models[0]
195
+ data_model = schema.data_models[0]
150
196
 
151
197
  properties = SheetList[DMSProperty]()
152
- ref_properties = SheetList[DMSProperty]()
153
- for view in self.schema.views:
198
+ for view in schema.views:
154
199
  view_id = view.as_id()
155
200
  view_entity = ViewEntity.from_id(view_id)
156
201
  class_entity = view_entity.as_class()
157
202
  for prop_id, prop in (view.properties or {}).items():
158
203
  dms_property = self._create_dms_property(prop_id, prop, view_entity, class_entity)
159
204
  if dms_property is not None:
160
- if view_id in self.schema.frozen_ids:
161
- ref_properties.append(dms_property)
162
- else:
163
- properties.append(dms_property)
205
+ properties.append(dms_property)
164
206
 
165
207
  data_model_view_ids: set[dm.ViewId] = {
166
208
  view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
167
209
  }
168
210
 
169
- metadata = self.metadata or DMSMetadata.from_data_model(data_model)
170
- metadata.data_model_type = self._infer_data_model_type(metadata.space)
171
- if ref_properties:
172
- metadata.schema_ = SchemaCompleteness.extended
173
-
174
- with _handle_issues(
175
- self.issue_list,
176
- ) as future:
177
- user_rules = DMSRules(
178
- metadata=metadata,
179
- properties=properties,
180
- containers=SheetList[DMSContainer](
181
- data=[
182
- DMSContainer.from_container(container)
183
- for container in self.schema.containers
184
- if container.as_id() not in self.schema.frozen_ids
185
- ]
186
- ),
187
- views=SheetList[DMSView](
188
- data=[
189
- DMSView.from_view(view, in_model=view.as_id() in data_model_view_ids)
190
- for view in self.schema.views
191
- if view.as_id() not in self.schema.frozen_ids
192
- ]
193
- ),
194
- reference=self._create_reference_rules(ref_properties),
195
- )
196
-
197
- if future.result == "failure" or self.issue_list.has_errors:
198
- return self._return_or_raise(self.issue_list, errors)
199
-
200
- return self._to_output(user_rules, self.issue_list, errors, role)
201
-
202
- def _create_reference_rules(self, properties: SheetList[DMSProperty]) -> DMSRules | None:
203
- if not properties:
204
- return None
205
-
206
- if len(self.schema.data_models) == 2:
207
- data_model = self.schema.data_models[1]
208
- data_model_view_ids: set[dm.ViewId] = {
209
- view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
210
- }
211
- metadata = self._create_metadata_from_model(data_model)
212
- else:
213
- data_model_view_ids = set()
214
- now = datetime.now().replace(microsecond=0)
215
- space = Counter(prop.view.space for prop in properties).most_common(1)[0][0]
216
- metadata = DMSMetadata(
217
- schema_=SchemaCompleteness.complete,
218
- extension=ExtensionCategory.addition,
219
- space=space,
220
- external_id="Unknown",
221
- version="0.1.0",
222
- creator=["Unknown"],
223
- created=now,
224
- updated=now,
225
- )
226
-
227
- metadata.data_model_type = DataModelType.enterprise
228
- return DMSRules(
211
+ metadata = metadata or DMSMetadata.from_data_model(data_model)
212
+ if data_model_type is not None:
213
+ metadata.data_model_type = data_model_type
214
+ if schema_completeness is not None:
215
+ metadata.schema_ = schema_completeness
216
+ return dict(
229
217
  metadata=metadata,
230
218
  properties=properties,
231
- views=SheetList[DMSView](
232
- data=[
233
- DMSView.from_view(view, in_model=not data_model_view_ids or (view.as_id() in data_model_view_ids))
234
- for view in self.schema.views
235
- if view.as_id() in self.schema.frozen_ids
236
- ]
237
- ),
238
219
  containers=SheetList[DMSContainer](
239
- data=[
240
- DMSContainer.from_container(container)
241
- for container in self.schema.containers
242
- if container.as_id() in self.schema.frozen_ids
243
- ]
220
+ data=[DMSContainer.from_container(container) for container in schema.containers]
221
+ ),
222
+ views=SheetList[DMSView](
223
+ data=[DMSView.from_view(view, in_model=view.as_id() in data_model_view_ids) for view in schema.views]
244
224
  ),
245
- reference=None,
246
225
  )
247
226
 
248
- def _infer_data_model_type(self, space: str) -> DataModelType:
249
- if self.schema.referenced_spaces() - {space}:
250
- # If the data model has containers, views, node types in another space
251
- # we assume it is a solution model.
252
- return DataModelType.solution
253
- else:
254
- # All containers, views, node types are in the same space as the data model
255
- return DataModelType.enterprise
227
+ @classmethod
228
+ def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply]) -> DMSMetadata:
229
+ now = datetime.now().replace(microsecond=0)
230
+ space = Counter(view.space for view in views).most_common(1)[0][0]
231
+ return DMSMetadata(
232
+ schema_=SchemaCompleteness.complete,
233
+ extension=ExtensionCategory.addition,
234
+ space=space,
235
+ external_id="Unknown",
236
+ version="0.1.0",
237
+ creator=["Unknown"],
238
+ created=now,
239
+ updated=now,
240
+ )
256
241
 
257
242
  def _create_dms_property(
258
243
  self, prop_id: str, prop: ViewPropertyApply, view_entity: ViewEntity, class_entity: ClassEntity
259
244
  ) -> DMSProperty | None:
260
- if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._container_by_id:
245
+ if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._all_containers_by_id:
261
246
  self.issue_list.append(
262
247
  issues.importing.MissingContainerWarning(
263
248
  view_id=str(view_entity),
@@ -268,7 +253,7 @@ class DMSImporter(BaseImporter):
268
253
  return None
269
254
  if (
270
255
  isinstance(prop, dm.MappedPropertyApply)
271
- and prop.container_property_identifier not in self._container_by_id[prop.container].properties
256
+ and prop.container_property_identifier not in self._all_containers_by_id[prop.container].properties
272
257
  ):
273
258
  self.issue_list.append(
274
259
  issues.importing.MissingContainerPropertyWarning(
@@ -315,7 +300,7 @@ class DMSImporter(BaseImporter):
315
300
 
316
301
  def _container_prop_unsafe(self, prop: dm.MappedPropertyApply) -> dm.ContainerProperty:
317
302
  """This method assumes you have already checked that the container with property exists."""
318
- return self._container_by_id[prop.container].properties[prop.container_property_identifier]
303
+ return self._all_containers_by_id[prop.container].properties[prop.container_property_identifier]
319
304
 
320
305
  def _get_relation_type(self, prop: ViewPropertyApply) -> Literal["edge", "reverse", "direct"] | None:
321
306
  if isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "outwards":
@@ -380,7 +365,7 @@ class DMSImporter(BaseImporter):
380
365
  def _get_index(self, prop: ViewPropertyApply, prop_id) -> list[str] | None:
381
366
  if not isinstance(prop, dm.MappedPropertyApply):
382
367
  return None
383
- container = self._container_by_id[prop.container]
368
+ container = self._all_containers_by_id[prop.container]
384
369
  index: list[str] = []
385
370
  for index_name, index_obj in (container.indexes or {}).items():
386
371
  if isinstance(index_obj, BTreeIndex | InvertedIndex) and prop_id in index_obj.properties:
@@ -390,7 +375,7 @@ class DMSImporter(BaseImporter):
390
375
  def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[str] | None:
391
376
  if not isinstance(prop, dm.MappedPropertyApply):
392
377
  return None
393
- container = self._container_by_id[prop.container]
378
+ container = self._all_containers_by_id[prop.container]
394
379
  unique_constraints: list[str] = []
395
380
  for constraint_name, constraint_obj in (container.constraints or {}).items():
396
381
  if isinstance(constraint_obj, dm.RequiresConstraint):
@@ -23,7 +23,7 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
23
23
  from cognite.neat.rules.issues import IssueList, ValidationIssue
24
24
  from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
25
25
  from cognite.neat.rules.models.entities import ClassEntity, ParentClassEntity
26
- from cognite.neat.rules.models.rules._information_rules import InformationClass, InformationProperty
26
+ from cognite.neat.rules.models.information import InformationClass, InformationProperty
27
27
 
28
28
 
29
29
  class _DTDLConverter:
@@ -12,9 +12,8 @@ from cognite.neat.rules.importers._base import BaseImporter, _handle_issues
12
12
  from cognite.neat.rules.importers._dtdl2rules.dtdl_converter import _DTDLConverter
13
13
  from cognite.neat.rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
14
14
  from cognite.neat.rules.issues import IssueList, ValidationIssue
15
- from cognite.neat.rules.models.rules import InformationRules, RoleTypes
16
- from cognite.neat.rules.models.rules._base import SchemaCompleteness, SheetList
17
- from cognite.neat.rules.models.rules._information_rules import InformationClass, InformationProperty
15
+ from cognite.neat.rules.models import InformationRules, RoleTypes, SchemaCompleteness, SheetList
16
+ from cognite.neat.rules.models.information import InformationClass, InformationProperty
18
17
  from cognite.neat.utils.text import to_pascal
19
18
 
20
19
 
@@ -4,7 +4,7 @@ import numpy as np
4
4
  import pandas as pd
5
5
  from rdflib import OWL, Graph
6
6
 
7
- from cognite.neat.rules.models.rules._base import MatchType
7
+ from cognite.neat.rules.models._base import MatchType
8
8
  from cognite.neat.utils.utils import remove_namespace
9
9
 
10
10
 
@@ -4,8 +4,8 @@ import re
4
4
  from rdflib import Graph, Namespace
5
5
 
6
6
  from cognite.neat.constants import DEFAULT_NAMESPACE
7
- from cognite.neat.rules.models.rules._base import RoleTypes, SchemaCompleteness
8
- from cognite.neat.rules.models.rules._types._base import (
7
+ from cognite.neat.rules.models import RoleTypes, SchemaCompleteness
8
+ from cognite.neat.rules.models._types._base import (
9
9
  PREFIX_COMPLIANCE_REGEX,
10
10
  VERSION_COMPLIANCE_REGEX,
11
11
  )
@@ -4,7 +4,7 @@ import numpy as np
4
4
  import pandas as pd
5
5
  from rdflib import Graph
6
6
 
7
- from cognite.neat.rules.models.rules._base import MatchType
7
+ from cognite.neat.rules.models._base import MatchType
8
8
  from cognite.neat.utils.utils import remove_namespace
9
9
 
10
10
  from ._owl2classes import _data_type_property_class, _object_property_class, _thing_class
@@ -10,8 +10,8 @@ from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SKOS, Graph
10
10
 
11
11
  from cognite.neat.rules.importers._base import BaseImporter, Rules
12
12
  from cognite.neat.rules.issues import IssueList
13
+ from cognite.neat.rules.models import InformationRules, RoleTypes
13
14
  from cognite.neat.rules.models.data_types import _XSD_TYPES
14
- from cognite.neat.rules.models.rules import InformationRules, RoleTypes
15
15
 
16
16
  from ._owl2classes import parse_owl_classes
17
17
  from ._owl2metadata import parse_owl_metadata
@@ -13,9 +13,15 @@ from pandas import ExcelFile
13
13
 
14
14
  from cognite.neat.rules import issues
15
15
  from cognite.neat.rules.issues import IssueList
16
- from cognite.neat.rules.models.rules import RULES_PER_ROLE, DMSRules, DomainRules, InformationRules
17
- from cognite.neat.rules.models.rules._base import RoleTypes, SchemaCompleteness
18
- from cognite.neat.rules.models.rules._dms_rules_write import DMSRulesWrite
16
+ from cognite.neat.rules.models import (
17
+ RULES_PER_ROLE,
18
+ DMSRules,
19
+ DomainRules,
20
+ InformationRules,
21
+ RoleTypes,
22
+ SchemaCompleteness,
23
+ )
24
+ from cognite.neat.rules.models.dms import DMSRulesInput
19
25
  from cognite.neat.utils.auxiliary import local_import
20
26
  from cognite.neat.utils.spreadsheet import SpreadsheetRead, read_individual_sheet
21
27
 
@@ -81,71 +87,95 @@ class MetadataRaw(UserDict):
81
87
  class ReadResult:
82
88
  sheets: dict[str, dict | list]
83
89
  read_info_by_sheet: dict[str, SpreadsheetRead]
84
- role: RoleTypes
85
- schema: SchemaCompleteness | None
90
+ metadata: MetadataRaw
91
+
92
+ @property
93
+ def role(self) -> RoleTypes:
94
+ return self.metadata.role
95
+
96
+ @property
97
+ def schema(self) -> SchemaCompleteness | None:
98
+ return self.metadata.schema
86
99
 
87
100
 
88
101
  class SpreadsheetReader:
89
- def __init__(self, issue_list: IssueList, is_reference: bool = False):
102
+ def __init__(
103
+ self,
104
+ issue_list: IssueList,
105
+ required: bool = True,
106
+ metadata: MetadataRaw | None = None,
107
+ sheet_prefix: Literal["", "Last", "Ref"] = "",
108
+ ):
90
109
  self.issue_list = issue_list
91
- self._is_reference = is_reference
110
+ self.required = required
111
+ self.metadata = metadata
112
+ self._sheet_prefix = sheet_prefix
92
113
 
93
114
  @property
94
115
  def metadata_sheet_name(self) -> str:
95
- metadata_name = "Metadata"
96
- return self.to_reference_sheet(metadata_name) if self._is_reference else metadata_name
116
+ return f"{self._sheet_prefix}Metadata"
97
117
 
98
118
  def sheet_names(self, role: RoleTypes) -> set[str]:
99
119
  names = MANDATORY_SHEETS_BY_ROLE[role]
100
- return {self.to_reference_sheet(sheet_name) for sheet_name in names} if self._is_reference else names
101
-
102
- @classmethod
103
- def to_reference_sheet(cls, sheet_name: str) -> str:
104
- return f"Ref{sheet_name}"
120
+ return {f"{self._sheet_prefix}{sheet_name}" for sheet_name in names if sheet_name != "Metadata"}
105
121
 
106
122
  def read(self, filepath: Path) -> None | ReadResult:
107
123
  with pd.ExcelFile(filepath) as excel_file:
108
- if self.metadata_sheet_name not in excel_file.sheet_names:
124
+ metadata: MetadataRaw | None
125
+ if self.metadata is not None:
126
+ metadata = self.metadata
127
+ else:
128
+ metadata = self._read_metadata(excel_file, filepath)
129
+ if metadata is None:
130
+ # The reading of metadata failed, so we can't continue
131
+ return None
132
+
133
+ sheets, read_info_by_sheet = self._read_sheets(excel_file, metadata.role)
134
+ if sheets is None or self.issue_list.has_errors:
135
+ return None
136
+ sheets["Metadata"] = dict(metadata)
137
+
138
+ return ReadResult(sheets, read_info_by_sheet, metadata)
139
+
140
+ def _read_metadata(self, excel_file: ExcelFile, filepath: Path) -> MetadataRaw | None:
141
+ if self.metadata_sheet_name not in excel_file.sheet_names:
142
+ if self.required:
109
143
  self.issue_list.append(
110
144
  issues.spreadsheet_file.MetadataSheetMissingOrFailedError(
111
145
  filepath, sheet_name=self.metadata_sheet_name
112
146
  )
113
147
  )
114
- return None
115
-
116
- metadata = MetadataRaw.from_excel(excel_file, self.metadata_sheet_name)
148
+ return None
117
149
 
118
- if not metadata.is_valid(self.issue_list, filepath):
119
- return None
150
+ metadata = MetadataRaw.from_excel(excel_file, self.metadata_sheet_name)
120
151
 
121
- sheets, read_info_by_sheet = self._read_sheets(metadata, excel_file)
122
- if sheets is None or self.issue_list.has_errors:
123
- return None
124
-
125
- return ReadResult(sheets, read_info_by_sheet, metadata.role, metadata.schema)
152
+ if not metadata.is_valid(self.issue_list, filepath):
153
+ return None
154
+ return metadata
126
155
 
127
156
  def _read_sheets(
128
- self, metadata: MetadataRaw, excel_file: ExcelFile
157
+ self, excel_file: ExcelFile, read_role: RoleTypes
129
158
  ) -> tuple[dict[str, dict | list] | None, dict[str, SpreadsheetRead]]:
130
159
  read_info_by_sheet: dict[str, SpreadsheetRead] = defaultdict(SpreadsheetRead)
131
160
 
132
- sheets: dict[str, dict | list] = {"Metadata": dict(metadata)}
161
+ sheets: dict[str, dict | list] = {}
133
162
 
134
- expected_sheet_names = self.sheet_names(metadata.role)
163
+ expected_sheet_names = self.sheet_names(read_role)
135
164
 
136
165
  if missing_sheets := expected_sheet_names.difference(set(excel_file.sheet_names)):
137
- self.issue_list.append(
138
- issues.spreadsheet_file.SheetMissingError(cast(Path, excel_file.io), list(missing_sheets))
139
- )
166
+ if self.required:
167
+ self.issue_list.append(
168
+ issues.spreadsheet_file.SheetMissingError(cast(Path, excel_file.io), list(missing_sheets))
169
+ )
140
170
  return None, read_info_by_sheet
141
171
 
142
172
  for source_sheet_name, target_sheet_name, headers_input in SOURCE_SHEET__TARGET_FIELD__HEADERS:
143
- source_sheet_name = self.to_reference_sheet(source_sheet_name) if self._is_reference else source_sheet_name
173
+ source_sheet_name = f"{self._sheet_prefix}{source_sheet_name}"
144
174
 
145
175
  if source_sheet_name not in excel_file.sheet_names:
146
176
  continue
147
177
  if isinstance(headers_input, dict):
148
- headers = headers_input[metadata.role]
178
+ headers = headers_input[read_role]
149
179
  else:
150
180
  headers = headers_input
151
181
 
@@ -182,42 +212,37 @@ class ExcelImporter(BaseImporter):
182
212
  issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
183
213
  return self._return_or_raise(issue_list, errors)
184
214
 
185
- user_result = SpreadsheetReader(issue_list, is_reference=False).read(self.filepath)
186
- if user_result is None or issue_list.has_errors:
215
+ user_read = SpreadsheetReader(issue_list).read(self.filepath)
216
+ if user_read is None or issue_list.has_errors:
187
217
  return self._return_or_raise(issue_list, errors)
188
218
 
189
- reference_result: ReadResult | None = None
190
- if (
191
- user_result
192
- and user_result.role != RoleTypes.domain_expert
193
- and user_result.schema == SchemaCompleteness.extended
194
- ):
195
- reference_result = SpreadsheetReader(issue_list, is_reference=True).read(self.filepath)
219
+ last_read: ReadResult | None = None
220
+ reference_read: ReadResult | None = None
221
+ if user_read.schema == SchemaCompleteness.extended:
222
+ # Last does not have its own metadata sheet. It is the same as the user's metadata sheet.
223
+ last_read = SpreadsheetReader(
224
+ issue_list, required=False, metadata=user_read.metadata, sheet_prefix="Last"
225
+ ).read(self.filepath)
226
+ reference_read = SpreadsheetReader(issue_list, sheet_prefix="Ref").read(self.filepath)
196
227
  if issue_list.has_errors:
197
228
  return self._return_or_raise(issue_list, errors)
198
229
 
199
- if user_result and reference_result and user_result.role != reference_result.role:
230
+ if reference_read and user_read.role != reference_read.role:
200
231
  issue_list.append(issues.spreadsheet_file.RoleMismatchError(self.filepath))
201
232
  return self._return_or_raise(issue_list, errors)
202
233
 
203
- if user_result and reference_result:
204
- user_result.sheets["reference"] = reference_result.sheets
205
- sheets = user_result.sheets
206
- original_role = user_result.role
207
- read_info_by_sheet = user_result.read_info_by_sheet
208
- read_info_by_sheet.update(reference_result.read_info_by_sheet)
209
- elif user_result:
210
- sheets = user_result.sheets
211
- original_role = user_result.role
212
- read_info_by_sheet = user_result.read_info_by_sheet
213
- elif reference_result:
214
- sheets = reference_result.sheets
215
- original_role = reference_result.role
216
- read_info_by_sheet = reference_result.read_info_by_sheet
217
- else:
218
- raise ValueError(
219
- "No rules were generated. This should have been caught earlier. " f"Bug in {type(self).__name__}."
220
- )
234
+ sheets = user_read.sheets
235
+ original_role = user_read.role
236
+ read_info_by_sheet = user_read.read_info_by_sheet
237
+ if last_read:
238
+ sheets["last"] = last_read.sheets
239
+ read_info_by_sheet.update(last_read.read_info_by_sheet)
240
+ if reference_read:
241
+ # The last rules will also be validated against the reference rules
242
+ sheets["last"]["reference"] = reference_read.sheets # type: ignore[call-overload]
243
+ if reference_read:
244
+ sheets["reference"] = reference_read.sheets
245
+ read_info_by_sheet.update(reference_read.read_info_by_sheet)
221
246
 
222
247
  rules_cls = RULES_PER_ROLE[original_role]
223
248
  with _handle_issues(
@@ -227,7 +252,7 @@ class ExcelImporter(BaseImporter):
227
252
  ) as future:
228
253
  rules: Rules
229
254
  if rules_cls is DMSRules:
230
- rules = DMSRulesWrite.load(sheets).as_read()
255
+ rules = DMSRulesInput.load(sheets).as_rules()
231
256
  else:
232
257
  rules = rules_cls.model_validate(sheets) # type: ignore[attr-defined]
233
258
 
@@ -5,8 +5,8 @@ import yaml
5
5
 
6
6
  from cognite.neat.rules import issues
7
7
  from cognite.neat.rules.issues import IssueList, NeatValidationError, ValidationIssue
8
- from cognite.neat.rules.models.rules import RULES_PER_ROLE, DMSRules, RoleTypes
9
- from cognite.neat.rules.models.rules._dms_rules_write import DMSRulesWrite
8
+ from cognite.neat.rules.models import RULES_PER_ROLE, DMSRules, RoleTypes
9
+ from cognite.neat.rules.models.dms import DMSRulesInput
10
10
 
11
11
  from ._base import BaseImporter, Rules, _handle_issues
12
12
 
@@ -98,7 +98,7 @@ class YAMLImporter(BaseImporter):
98
98
  with _handle_issues(issue_list) as future:
99
99
  rules: Rules
100
100
  if rules_model is DMSRules:
101
- rules = DMSRulesWrite.load(self.raw_data).as_read()
101
+ rules = DMSRulesInput.load(self.raw_data).as_rules()
102
102
  else:
103
103
  rules = rules_model.model_validate(self.raw_data)
104
104
 
@@ -1,4 +1,5 @@
1
1
  import sys
2
+ import warnings
2
3
  from abc import ABC, abstractmethod
3
4
  from collections import UserList
4
5
  from collections.abc import Sequence
@@ -182,6 +183,10 @@ class IssueList(UserList[ValidationIssue]):
182
183
  [ValueError(issue.message()) for issue in self if isinstance(issue, NeatValidationError)],
183
184
  )
184
185
 
186
+ def trigger_warnings(self) -> None:
187
+ for warning in [issue for issue in self if isinstance(issue, ValidationWarning)]:
188
+ warnings.warn(warning, stacklevel=2)
189
+
185
190
  def to_pandas(self) -> pd.DataFrame:
186
191
  return pd.DataFrame([issue.dump() for issue in self])
187
192