cognite-neat 0.76.2__py3-none-any.whl → 0.77.0__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.76.2"
+__version__ = "0.77.0"

cognite/neat/rules/exporters/_rules2dms.py

@@ -5,7 +5,7 @@ from typing import Literal, TypeAlias, cast
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes._base import CogniteResource, CogniteResourceList
-from cognite.client.data_classes.data_modeling import DataModelApply, DataModelId
+from cognite.client.data_classes.data_modeling import DataModelApply, DataModelApplyList, DataModelId
 from cognite.client.exceptions import CogniteAPIError
 
 from cognite.neat.rules import issues
@@ -114,9 +114,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
             dms_rules = rules.as_dms_architect_rules()
         else:
             raise ValueError(f"{type(rules).__name__} cannot be exported to DMS")
-        return dms_rules.as_schema(
-            include_ref=True, include_pipeline=self.export_pipeline, instance_space=self.instance_space
-        )
+        return dms_rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
 
     def delete_from_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
         schema, to_export = self._prepare_schema_and_exporters(rules, client)
@@ -272,7 +270,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
         if self.export_components.intersection({"all", "views"}):
             to_export.append((schema.views, ViewLoader(client, self.existing_handling)))
         if self.export_components.intersection({"all", "data_models"}):
-            to_export.append((schema.data_models, DataModelLoader(client)))
+            to_export.append((DataModelApplyList([schema.data_model]), DataModelLoader(client)))
         if isinstance(schema, PipelineSchema):
             to_export.append((schema.databases, RawDatabaseLoader(client)))
             to_export.append((schema.raw_tables, RawTableLoader(client)))

cognite/neat/rules/exporters/_rules2excel.py

@@ -13,8 +13,10 @@ from openpyxl.worksheet.worksheet import Worksheet
 
 from cognite.neat.rules._shared import Rules
 from cognite.neat.rules.models import (
+    DataModelType,
     DMSRules,
     DomainRules,
+    ExtensionCategory,
     InformationRules,
     RoleTypes,
     SchemaCompleteness,
@@ -35,11 +37,18 @@ class ExcelExporter(BaseExporter[Workbook]):
         new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
             rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
             new metadata sheet in the Excel file.
-        is_reference: If True, the rules are considered to be a reference model. The exported Excel file will
-            then contain empty sheets for the main rules and this data model will be dumped to the reference sheets.
-            This is useful when you are building a solution model based on an Enterprise model, then the
-            Enterprise model will serve as the reference model. It is also useful when you are extending an existing
-            model, then the existing model will serve as the reference model. Defaults to False.
+        dump_as: This determines how the rules are written to the Excel file. An Excel file has up to three sets of
+           sheets: user, last, and reference. The user sheets are used for inputting rules from a user. The last sheets
+           are used for the last version of the same model as the user, while the reference sheets are used for
+           the model the user is building on. The options are:
+             * "user": The rules are written to the user sheets. This is used when you want to modify the rules
+                directly and potentially change the model. This is useful when you have imported the data model
+                from outside CDF and you want to modify it before you write it to CDF.
+             * "last": The rules are written to the last sheets. This is used when you want to extend the rules,
+               but have validation that you are not breaking the existing model. This is used when you want to
+               change a model that has already been published to CDF and that model is in production.
+             * "reference": The rules are written to the reference sheets. This is typically used when you want to build
+               a new solution on top of an enterprise model.
 
     The following styles are available:
 
@@ -51,7 +60,7 @@ class ExcelExporter(BaseExporter[Workbook]):
     """
 
     Style = Literal["none", "minimal", "default", "maximal"]
-
+    DumpOptions = Literal["user", "last", "reference"]
     _main_header_by_sheet_name: ClassVar[dict[str, str]] = {
         "Properties": "Definition of Properties per Class",
         "Classes": "Definition of Classes",
@@ -59,21 +68,24 @@ class ExcelExporter(BaseExporter[Workbook]):
         "Containers": "Definition of Containers",
     }
     style_options = get_args(Style)
+    dump_options = get_args(DumpOptions)
 
     def __init__(
         self,
         styling: Style = "default",
         output_role: RoleTypes | None = None,
         new_model_id: tuple[str, str, str] | None = None,
-        is_reference: bool = False,
+        dump_as: DumpOptions = "user",
     ):
         if styling not in self.style_options:
             raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
+        if dump_as not in self.dump_options:
+            raise ValueError(f"Invalid dump_as: {dump_as}. Valid options are {self.dump_options}")
         self.styling = styling
         self._styling_level = self.style_options.index(styling)
         self.output_role = output_role
         self.new_model_id = new_model_id
-        self.is_reference = is_reference
+        self.dump_as = dump_as
 
     def export_to_file(self, rules: Rules, filepath: Path) -> None:
         """Exports transformation rules to excel file."""
@@ -90,41 +102,48 @@ class ExcelExporter(BaseExporter[Workbook]):
         # Remove default sheet named "Sheet"
         workbook.remove(workbook["Sheet"])
 
-        dumped_rules: dict[str, Any]
+        dumped_user_rules: dict[str, Any]
+        dumped_last_rules: dict[str, Any] | None = None
         dumped_reference_rules: dict[str, Any] | None = None
-        if self.is_reference:
+        if self.dump_as != "user":
             # Writes empty reference sheets
-            dumped_rules = {
+            dumped_user_rules = {
                 "Metadata": self._create_metadata_sheet_user_rules(rules),
             }
-            dumped_rules["Metadata"]["role"] = (
-                self.output_role and self.output_role.value
-            ) or rules.metadata.role.value
-            dumped_reference_rules = rules.reference_self().model_dump(by_alias=True)
+
+            if self.dump_as == "last":
+                dumped_last_rules = rules.model_dump(by_alias=True)
+                if rules.reference:
+                    dumped_reference_rules = rules.reference.model_dump(by_alias=True)
+            elif self.dump_as == "reference":
+                dumped_reference_rules = rules.reference_self().model_dump(by_alias=True)
         else:
-            dumped_rules = rules.model_dump(by_alias=True)
+            dumped_user_rules = rules.model_dump(by_alias=True)
+            if rules.last:
+                dumped_last_rules = rules.last.model_dump(by_alias=True)
             if rules.reference:
                 dumped_reference_rules = rules.reference.model_dump(by_alias=True)
 
-        self._write_metadata_sheet(workbook, dumped_rules["Metadata"])
-        self._write_sheets(workbook, dumped_rules, rules)
+        self._write_metadata_sheet(workbook, dumped_user_rules["Metadata"])
+        self._write_sheets(workbook, dumped_user_rules, rules)
+        if dumped_last_rules:
+            self._write_sheets(workbook, dumped_last_rules, rules, sheet_prefix="Last")
+
         if dumped_reference_rules:
-            self._write_sheets(workbook, dumped_reference_rules, rules, is_reference=True)
-            self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], is_reference=True)
+            prefix = "Ref"
+            self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix=prefix)
+            self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix=prefix)
 
         if self._styling_level > 0:
             self._adjust_column_widths(workbook)
 
         return workbook
 
-    def _write_sheets(self, workbook: Workbook, dumped_rules: dict[str, Any], rules: Rules, is_reference: bool = False):
+    def _write_sheets(self, workbook: Workbook, dumped_rules: dict[str, Any], rules: Rules, sheet_prefix: str = ""):
         for sheet_name, headers in rules.headers_by_sheet(by_alias=True).items():
             if sheet_name in ("Metadata", "prefixes", "Reference", "Last"):
                 continue
-            if is_reference:
-                sheet = workbook.create_sheet(f"Ref{sheet_name}")
-            else:
-                sheet = workbook.create_sheet(sheet_name)
+            sheet = workbook.create_sheet(f"{sheet_prefix}{sheet_name}")
 
             main_header = self._main_header_by_sheet_name[sheet_name]
             sheet.append([main_header] + [""] * (len(headers) - 1))
@@ -170,17 +189,14 @@ class ExcelExporter(BaseExporter[Workbook]):
                 for cell in sheet["2"]:
                     cell.font = Font(bold=True, size=14)
 
-    def _write_metadata_sheet(self, workbook: Workbook, metadata: dict[str, Any], is_reference: bool = False) -> None:
+    def _write_metadata_sheet(self, workbook: Workbook, metadata: dict[str, Any], sheet_prefix: str = "") -> None:
         # Excel does not support timezone in datetime strings
         if isinstance(metadata.get("created"), datetime):
             metadata["created"] = metadata["created"].replace(tzinfo=None)
         if isinstance(metadata.get("updated"), datetime):
             metadata["updated"] = metadata["updated"].replace(tzinfo=None)
 
-        if is_reference:
-            metadata_sheet = workbook.create_sheet("RefMetadata")
-        else:
-            metadata_sheet = workbook.create_sheet("Metadata")
+        metadata_sheet = workbook.create_sheet(f"{sheet_prefix}Metadata")
         for key, value in metadata.items():
             metadata_sheet.append([key, value])
 
@@ -241,25 +257,29 @@ class ExcelExporter(BaseExporter[Workbook]):
         # Excel does not support timezone in datetime strings
         now_iso = datetime.now().replace(tzinfo=None).isoformat()
         is_info = isinstance(rules, InformationRules)
-        is_dms = not is_info
-        is_extension = self.new_model_id is not None
-        is_solution = is_extension and self.new_model_id != existing_model_id
+        is_dms = isinstance(rules, DMSRules)
+        is_extension = self.new_model_id is not None or rules.reference is not None
+        is_solution = rules.metadata.data_model_type == DataModelType.solution
 
-        if is_solution:
+        if is_solution and self.new_model_id:
             metadata["prefix" if is_info else "space"] = self.new_model_id[0]  # type: ignore[index]
             metadata["title" if is_info else "externalId"] = self.new_model_id[1]  # type: ignore[index]
             metadata["version"] = self.new_model_id[2]  # type: ignore[index]
+        elif is_solution and self.dump_as == "reference" and rules.reference:
+            metadata["prefix" if is_info else "space"] = "YOUR_PREFIX"
+            metadata["title" if is_info else "externalId"] = "YOUR_TITLE"
+            metadata["version"] = "1"
         else:
             metadata["prefix" if is_info else "space"] = existing_model_id[0]
             metadata["title" if is_info else "externalId"] = existing_model_id[1]
             metadata["version"] = existing_model_id[2]
 
-        if is_solution and is_info:
+        if is_solution and is_info and self.new_model_id:
             metadata["namespace"] = f"http://purl.org/{self.new_model_id[0]}/"  # type: ignore[index]
         elif is_info:
             metadata["namespace"] = existing_metadata["namespace"]
 
-        if is_solution and is_dms:
+        if is_solution and is_dms and self.new_model_id:
             metadata["name"] = self.new_model_id[1]  # type: ignore[index]
 
         if is_solution:
@@ -285,6 +305,11 @@ class ExcelExporter(BaseExporter[Workbook]):
         else:
             metadata["schema"] = SchemaCompleteness.complete.value
 
-        metadata["extension"] = "addition"
+        if is_solution:
+            metadata["dataModelType"] = DataModelType.solution.value
+        else:
+            metadata["dataModelType"] = DataModelType.enterprise.value
 
+        metadata["extension"] = ExtensionCategory.addition.value
+        metadata["role"] = (self.output_role and self.output_role.value) or rules.metadata.role.value
         return metadata

cognite/neat/rules/importers/_dms2rules.py

@@ -6,7 +6,7 @@ from typing import Any, Literal, cast, overload
 
 from cognite.client import CogniteClient
 from cognite.client import data_modeling as dm
-from cognite.client.data_classes.data_modeling import DataModelIdentifier
+from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
 from cognite.client.data_classes.data_modeling.containers import BTreeIndex, InvertedIndex
 from cognite.client.data_classes.data_modeling.views import (
     MultiEdgeConnectionApply,
@@ -39,7 +39,6 @@ from cognite.neat.rules.models.dms import (
 from cognite.neat.rules.models.entities import (
     ClassEntity,
     ContainerEntity,
-    DataModelEntity,
     DMSUnknownEntity,
     ViewEntity,
     ViewPropertyEntity,
@@ -52,12 +51,14 @@ class DMSImporter(BaseImporter):
         schema: DMSSchema,
         read_issues: Sequence[ValidationIssue] | None = None,
         metadata: DMSMetadata | None = None,
+        ref_metadata: DMSMetadata | None = None,
     ):
         # Calling this root schema to distinguish it from
         # * User Schema
         # * Reference Schema
         self.root_schema = schema
         self.metadata = metadata
+        self.ref_metadata = ref_metadata
         self.issue_list = IssueList(read_issues)
         self._all_containers_by_id = {container.as_id(): container for container in schema.containers}
         if self.root_schema.reference:
@@ -66,42 +67,78 @@ class DMSImporter(BaseImporter):
             )
 
     @classmethod
-    def from_data_model_id(cls, client: CogniteClient, data_model_id: DataModelIdentifier) -> "DMSImporter":
+    def from_data_model_id(
+        cls,
+        client: CogniteClient,
+        data_model_id: DataModelIdentifier,
+        reference_model_id: DataModelIdentifier | None = None,
+    ) -> "DMSImporter":
         """Create a DMSImporter ready to convert the given data model to rules.
 
         Args:
             client: Instantiated CogniteClient to retrieve data model.
+            reference_model_id: The reference data model to retrieve. This is the data model that
+                the given data model is built on top of, typically, an enterprise data model.
             data_model_id: Data Model to retrieve.
 
         Returns:
             DMSImporter: DMSImporter instance
         """
-        data_models = client.data_modeling.data_models.retrieve(data_model_id, inline_views=True)
-        if len(data_models) == 0:
+        data_model_ids = [data_model_id, reference_model_id] if reference_model_id else [data_model_id]
+        data_models = client.data_modeling.data_models.retrieve(data_model_ids, inline_views=True)
+
+        user_models = cls._find_model_in_list(data_models, data_model_id)
+        if len(user_models) == 0:
             return cls(DMSSchema(), [issues.importing.NoDataModelError(f"Data model {data_model_id} not found")])
-        data_model = data_models.latest_version()
+        user_model = user_models.latest_version()
+
+        if reference_model_id:
+            ref_models = cls._find_model_in_list(data_models, reference_model_id)
+            if len(ref_models) == 0:
+                return cls(
+                    DMSSchema(), [issues.importing.NoDataModelError(f"Data model {reference_model_id} not found")]
+                )
+            ref_model: dm.DataModel[dm.View] | None = ref_models.latest_version()
+        else:
+            ref_model = None
 
         try:
-            schema = DMSSchema.from_data_model(client, data_model)
+            schema = DMSSchema.from_data_model(client, user_model, ref_model)
         except Exception as e:
             return cls(DMSSchema(), [issues.importing.APIError(str(e))])
 
-        created = ms_to_datetime(data_model.created_time)
-        updated = ms_to_datetime(data_model.last_updated_time)
+        metadata = cls._create_metadata_from_model(user_model)
+        ref_metadata = cls._create_metadata_from_model(ref_model) if ref_model else None
 
-        metadata = cls._create_metadata_from_model(data_model, created, updated)
+        return cls(schema, [], metadata, ref_metadata)
 
-        return cls(schema, [], metadata)
+    @classmethod
+    def _find_model_in_list(
+        cls, data_models: dm.DataModelList[dm.View], model_id: DataModelIdentifier
+    ) -> dm.DataModelList[dm.View]:
+        identifier = DataModelId.load(model_id)
+        return dm.DataModelList[dm.View](
+            [
+                model
+                for model in data_models
+                if (model.space, model.external_id) == (identifier.space, identifier.external_id)
+            ]
+        )
 
     @classmethod
     def _create_metadata_from_model(
         cls,
         model: dm.DataModel[dm.View] | dm.DataModelApply,
-        created: datetime | None = None,
-        updated: datetime | None = None,
     ) -> DMSMetadata:
         description, creator = DMSMetadata._get_description_and_creator(model.description)
-        now = datetime.now().replace(microsecond=0)
+
+        if isinstance(model, dm.DataModel):
+            created = ms_to_datetime(model.created_time)
+            updated = ms_to_datetime(model.last_updated_time)
+        else:
+            now = datetime.now().replace(microsecond=0)
+            created = now
+            updated = now
         return DMSMetadata(
             schema_=SchemaCompleteness.complete,
             extension=ExtensionCategory.addition,
@@ -109,8 +146,8 @@ class DMSImporter(BaseImporter):
             external_id=model.external_id,
             name=model.name or model.external_id,
             version=model.version or "0.1.0",
-            updated=updated or now,
-            created=created or now,
+            updated=updated,
+            created=created,
             creator=creator,
             description=description,
         )
@@ -147,28 +184,33 @@ class DMSImporter(BaseImporter):
             # In case there were errors during the import, the to_rules method will return None
             return self._return_or_raise(self.issue_list, errors)
 
-        if len(self.root_schema.data_models) == 0:
+        if not self.root_schema.data_model:
             self.issue_list.append(issues.importing.NoDataModelError("No data model found."))
             return self._return_or_raise(self.issue_list, errors)
-
+        model = self.root_schema.data_model
         with _handle_issues(
             self.issue_list,
         ) as future:
             schema_completeness = SchemaCompleteness.complete
             data_model_type = DataModelType.enterprise
             reference: DMSRules | None = None
-            if ref_schema := self.root_schema.reference:
+            if (ref_schema := self.root_schema.reference) and (ref_model := ref_schema.data_model):
                 # Reference should always be an enterprise model.
                 reference = DMSRules(
                     **self._create_rule_components(
-                        ref_schema, self._create_default_metadata(ref_schema.views), DataModelType.enterprise
+                        ref_model,
+                        ref_schema,
+                        self.ref_metadata or self._create_default_metadata(ref_schema.views),
+                        DataModelType.enterprise,
                     )
                 )
                 schema_completeness = SchemaCompleteness.extended
                 data_model_type = DataModelType.solution
 
             user_rules = DMSRules(
-                **self._create_rule_components(self.root_schema, self.metadata, data_model_type, schema_completeness),
+                **self._create_rule_components(
+                    model, self.root_schema, self.metadata, data_model_type, schema_completeness
+                ),
                 reference=reference,
             )
 
@@ -179,21 +221,12 @@ class DMSImporter(BaseImporter):
 
     def _create_rule_components(
         self,
+        data_model: dm.DataModelApply,
         schema: DMSSchema,
         metadata: DMSMetadata | None = None,
         data_model_type: DataModelType | None = None,
         schema_completeness: SchemaCompleteness | None = None,
     ) -> dict[str, Any]:
-        if len(schema.data_models) > 2:
-            # Creating a DataModelEntity to convert the data model id to a string.
-            self.issue_list.append(
-                issues.importing.MultipleDataModelsWarning(
-                    [str(DataModelEntity.from_id(model.as_id())) for model in schema.data_models]
-                )
-            )
-
-        data_model = schema.data_models[0]
-
         properties = SheetList[DMSProperty]()
         for view in schema.views:
             view_id = view.as_id()

cognite/neat/rules/importers/_spreadsheet2rules.py

@@ -87,71 +87,95 @@ class MetadataRaw(UserDict):
 class ReadResult:
     sheets: dict[str, dict | list]
     read_info_by_sheet: dict[str, SpreadsheetRead]
-    role: RoleTypes
-    schema: SchemaCompleteness | None
+    metadata: MetadataRaw
+
+    @property
+    def role(self) -> RoleTypes:
+        return self.metadata.role
+
+    @property
+    def schema(self) -> SchemaCompleteness | None:
+        return self.metadata.schema
 
 
 class SpreadsheetReader:
-    def __init__(self, issue_list: IssueList, is_reference: bool = False):
+    def __init__(
+        self,
+        issue_list: IssueList,
+        required: bool = True,
+        metadata: MetadataRaw | None = None,
+        sheet_prefix: Literal["", "Last", "Ref"] = "",
+    ):
         self.issue_list = issue_list
-        self._is_reference = is_reference
+        self.required = required
+        self.metadata = metadata
+        self._sheet_prefix = sheet_prefix
 
     @property
     def metadata_sheet_name(self) -> str:
-        metadata_name = "Metadata"
-        return self.to_reference_sheet(metadata_name) if self._is_reference else metadata_name
+        return f"{self._sheet_prefix}Metadata"
 
     def sheet_names(self, role: RoleTypes) -> set[str]:
         names = MANDATORY_SHEETS_BY_ROLE[role]
-        return {self.to_reference_sheet(sheet_name) for sheet_name in names} if self._is_reference else names
-
-    @classmethod
-    def to_reference_sheet(cls, sheet_name: str) -> str:
-        return f"Ref{sheet_name}"
+        return {f"{self._sheet_prefix}{sheet_name}" for sheet_name in names if sheet_name != "Metadata"}
 
     def read(self, filepath: Path) -> None | ReadResult:
         with pd.ExcelFile(filepath) as excel_file:
-            if self.metadata_sheet_name not in excel_file.sheet_names:
+            metadata: MetadataRaw | None
+            if self.metadata is not None:
+                metadata = self.metadata
+            else:
+                metadata = self._read_metadata(excel_file, filepath)
+                if metadata is None:
+                    # The reading of metadata failed, so we can't continue
+                    return None
+
+            sheets, read_info_by_sheet = self._read_sheets(excel_file, metadata.role)
+            if sheets is None or self.issue_list.has_errors:
+                return None
+            sheets["Metadata"] = dict(metadata)
+
+            return ReadResult(sheets, read_info_by_sheet, metadata)
+
+    def _read_metadata(self, excel_file: ExcelFile, filepath: Path) -> MetadataRaw | None:
+        if self.metadata_sheet_name not in excel_file.sheet_names:
+            if self.required:
                 self.issue_list.append(
                     issues.spreadsheet_file.MetadataSheetMissingOrFailedError(
                         filepath, sheet_name=self.metadata_sheet_name
                     )
                 )
-                return None
-
-            metadata = MetadataRaw.from_excel(excel_file, self.metadata_sheet_name)
+            return None
 
-            if not metadata.is_valid(self.issue_list, filepath):
-                return None
+        metadata = MetadataRaw.from_excel(excel_file, self.metadata_sheet_name)
 
-            sheets, read_info_by_sheet = self._read_sheets(metadata, excel_file)
-            if sheets is None or self.issue_list.has_errors:
-                return None
-
-            return ReadResult(sheets, read_info_by_sheet, metadata.role, metadata.schema)
+        if not metadata.is_valid(self.issue_list, filepath):
+            return None
+        return metadata
 
     def _read_sheets(
-        self, metadata: MetadataRaw, excel_file: ExcelFile
+        self, excel_file: ExcelFile, read_role: RoleTypes
     ) -> tuple[dict[str, dict | list] | None, dict[str, SpreadsheetRead]]:
         read_info_by_sheet: dict[str, SpreadsheetRead] = defaultdict(SpreadsheetRead)
 
-        sheets: dict[str, dict | list] = {"Metadata": dict(metadata)}
+        sheets: dict[str, dict | list] = {}
 
-        expected_sheet_names = self.sheet_names(metadata.role)
+        expected_sheet_names = self.sheet_names(read_role)
 
         if missing_sheets := expected_sheet_names.difference(set(excel_file.sheet_names)):
-            self.issue_list.append(
-                issues.spreadsheet_file.SheetMissingError(cast(Path, excel_file.io), list(missing_sheets))
-            )
+            if self.required:
+                self.issue_list.append(
+                    issues.spreadsheet_file.SheetMissingError(cast(Path, excel_file.io), list(missing_sheets))
+                )
             return None, read_info_by_sheet
 
         for source_sheet_name, target_sheet_name, headers_input in SOURCE_SHEET__TARGET_FIELD__HEADERS:
-            source_sheet_name = self.to_reference_sheet(source_sheet_name) if self._is_reference else source_sheet_name
+            source_sheet_name = f"{self._sheet_prefix}{source_sheet_name}"
 
             if source_sheet_name not in excel_file.sheet_names:
                 continue
             if isinstance(headers_input, dict):
-                headers = headers_input[metadata.role]
+                headers = headers_input[read_role]
             else:
                 headers = headers_input
 
@@ -188,42 +212,37 @@ class ExcelImporter(BaseImporter):
             issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
             return self._return_or_raise(issue_list, errors)
 
-        user_result = SpreadsheetReader(issue_list, is_reference=False).read(self.filepath)
-        if user_result is None or issue_list.has_errors:
+        user_read = SpreadsheetReader(issue_list).read(self.filepath)
+        if user_read is None or issue_list.has_errors:
             return self._return_or_raise(issue_list, errors)
 
-        reference_result: ReadResult | None = None
-        if (
-            user_result
-            and user_result.role != RoleTypes.domain_expert
-            and user_result.schema == SchemaCompleteness.extended
-        ):
-            reference_result = SpreadsheetReader(issue_list, is_reference=True).read(self.filepath)
+        last_read: ReadResult | None = None
+        reference_read: ReadResult | None = None
+        if user_read.schema == SchemaCompleteness.extended:
+            # Last does not have its own metadata sheet. It is the same as the user's metadata sheet.
+            last_read = SpreadsheetReader(
+                issue_list, required=False, metadata=user_read.metadata, sheet_prefix="Last"
+            ).read(self.filepath)
+            reference_read = SpreadsheetReader(issue_list, sheet_prefix="Ref").read(self.filepath)
             if issue_list.has_errors:
                 return self._return_or_raise(issue_list, errors)
 
-        if user_result and reference_result and user_result.role != reference_result.role:
+        if reference_read and user_read.role != reference_read.role:
             issue_list.append(issues.spreadsheet_file.RoleMismatchError(self.filepath))
             return self._return_or_raise(issue_list, errors)
 
-        if user_result and reference_result:
-            user_result.sheets["reference"] = reference_result.sheets
-            sheets = user_result.sheets
-            original_role = user_result.role
-            read_info_by_sheet = user_result.read_info_by_sheet
-            read_info_by_sheet.update(reference_result.read_info_by_sheet)
-        elif user_result:
-            sheets = user_result.sheets
-            original_role = user_result.role
-            read_info_by_sheet = user_result.read_info_by_sheet
-        elif reference_result:
-            sheets = reference_result.sheets
-            original_role = reference_result.role
-            read_info_by_sheet = reference_result.read_info_by_sheet
-        else:
-            raise ValueError(
-                "No rules were generated. This should have been caught earlier. " f"Bug in {type(self).__name__}."
-            )
+        sheets = user_read.sheets
+        original_role = user_read.role
+        read_info_by_sheet = user_read.read_info_by_sheet
+        if last_read:
+            sheets["last"] = last_read.sheets
+            read_info_by_sheet.update(last_read.read_info_by_sheet)
+            if reference_read:
+                # The last rules will also be validated against the reference rules
+                sheets["last"]["reference"] = reference_read.sheets  # type: ignore[call-overload]
+        if reference_read:
+            sheets["reference"] = reference_read.sheets
+            read_info_by_sheet.update(reference_read.read_info_by_sheet)
 
         rules_cls = RULES_PER_ROLE[original_role]
         with _handle_issues(