cognite-neat 0.76.2__py3-none-any.whl → 0.76.3__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.76.2"
+__version__ = "0.76.3"

cognite/neat/rules/exporters/_rules2excel.py

@@ -13,8 +13,10 @@ from openpyxl.worksheet.worksheet import Worksheet
 
 from cognite.neat.rules._shared import Rules
 from cognite.neat.rules.models import (
+    DataModelType,
     DMSRules,
     DomainRules,
+    ExtensionCategory,
     InformationRules,
     RoleTypes,
     SchemaCompleteness,
@@ -35,11 +37,18 @@ class ExcelExporter(BaseExporter[Workbook]):
         new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
             rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
             new metadata sheet in the Excel file.
-        is_reference: If True, the rules are considered to be a reference model. The exported Excel file will
-            then contain empty sheets for the main rules and this data model will be dumped to the reference sheets.
-            This is useful when you are building a solution model based on an Enterprise model, then the
-            Enterprise model will serve as the reference model. It is also useful when you are extending an existing
-            model, then the existing model will serve as the reference model. Defaults to False.
+        dump_as: This determines how the rules are written to the Excel file. An Excel file has up to three sets of
+           sheets: user, last, and reference. The user sheets are used for inputting rules from a user. The last sheets
+           are used for the last version of the same model as the user, while the reference sheets are used for
+           the model the user is building on. The options are:
+             * "user": The rules are written to the user sheets. This is used when you want to modify the rules
+                directly and potentially change the model. This is useful when you have imported the data model
+                from outside CDF and you want to modify it before you write it to CDF.
+             * "last": The rules are written to the last sheets. This is used when you want to extend the rules,
+               but have validation that you are not breaking the existing model. This is used when you want to
+               change a model that has already been published to CDF and that model is in production.
+             * "reference": The rules are written to the reference sheets. This is typically used when you want to build
+               a new solution on top of an enterprise model.
 
     The following styles are available:
 
@@ -51,7 +60,7 @@ class ExcelExporter(BaseExporter[Workbook]):
     """
 
     Style = Literal["none", "minimal", "default", "maximal"]
-
+    DumpOptions = Literal["user", "last", "reference"]
     _main_header_by_sheet_name: ClassVar[dict[str, str]] = {
         "Properties": "Definition of Properties per Class",
         "Classes": "Definition of Classes",
@@ -59,21 +68,24 @@ class ExcelExporter(BaseExporter[Workbook]):
         "Containers": "Definition of Containers",
     }
     style_options = get_args(Style)
+    dump_options = get_args(DumpOptions)
 
     def __init__(
         self,
         styling: Style = "default",
         output_role: RoleTypes | None = None,
         new_model_id: tuple[str, str, str] | None = None,
-        is_reference: bool = False,
+        dump_as: DumpOptions = "user",
     ):
         if styling not in self.style_options:
             raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
+        if dump_as not in self.dump_options:
+            raise ValueError(f"Invalid dump_as: {dump_as}. Valid options are {self.dump_options}")
         self.styling = styling
         self._styling_level = self.style_options.index(styling)
         self.output_role = output_role
         self.new_model_id = new_model_id
-        self.is_reference = is_reference
+        self.dump_as = dump_as
 
     def export_to_file(self, rules: Rules, filepath: Path) -> None:
         """Exports transformation rules to excel file."""
@@ -90,41 +102,48 @@ class ExcelExporter(BaseExporter[Workbook]):
         # Remove default sheet named "Sheet"
         workbook.remove(workbook["Sheet"])
 
-        dumped_rules: dict[str, Any]
+        dumped_user_rules: dict[str, Any]
+        dumped_last_rules: dict[str, Any] | None = None
         dumped_reference_rules: dict[str, Any] | None = None
-        if self.is_reference:
+        if self.dump_as != "user":
             # Writes empty reference sheets
-            dumped_rules = {
+            dumped_user_rules = {
                 "Metadata": self._create_metadata_sheet_user_rules(rules),
             }
-            dumped_rules["Metadata"]["role"] = (
-                self.output_role and self.output_role.value
-            ) or rules.metadata.role.value
-            dumped_reference_rules = rules.reference_self().model_dump(by_alias=True)
+
+            if self.dump_as == "last":
+                dumped_last_rules = rules.model_dump(by_alias=True)
+                if rules.reference:
+                    dumped_reference_rules = rules.reference.model_dump(by_alias=True)
+            elif self.dump_as == "reference":
+                dumped_reference_rules = rules.reference_self().model_dump(by_alias=True)
         else:
-            dumped_rules = rules.model_dump(by_alias=True)
+            dumped_user_rules = rules.model_dump(by_alias=True)
+            if rules.last:
+                dumped_last_rules = rules.last.model_dump(by_alias=True)
             if rules.reference:
                 dumped_reference_rules = rules.reference.model_dump(by_alias=True)
 
-        self._write_metadata_sheet(workbook, dumped_rules["Metadata"])
-        self._write_sheets(workbook, dumped_rules, rules)
+        self._write_metadata_sheet(workbook, dumped_user_rules["Metadata"])
+        self._write_sheets(workbook, dumped_user_rules, rules)
+        if dumped_last_rules:
+            self._write_sheets(workbook, dumped_last_rules, rules, sheet_prefix="Last")
+
         if dumped_reference_rules:
-            self._write_sheets(workbook, dumped_reference_rules, rules, is_reference=True)
-            self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], is_reference=True)
+            prefix = "Ref"
+            self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix=prefix)
+            self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix=prefix)
 
         if self._styling_level > 0:
             self._adjust_column_widths(workbook)
 
         return workbook
 
-    def _write_sheets(self, workbook: Workbook, dumped_rules: dict[str, Any], rules: Rules, is_reference: bool = False):
+    def _write_sheets(self, workbook: Workbook, dumped_rules: dict[str, Any], rules: Rules, sheet_prefix: str = ""):
         for sheet_name, headers in rules.headers_by_sheet(by_alias=True).items():
             if sheet_name in ("Metadata", "prefixes", "Reference", "Last"):
                 continue
-            if is_reference:
-                sheet = workbook.create_sheet(f"Ref{sheet_name}")
-            else:
-                sheet = workbook.create_sheet(sheet_name)
+            sheet = workbook.create_sheet(f"{sheet_prefix}{sheet_name}")
 
             main_header = self._main_header_by_sheet_name[sheet_name]
             sheet.append([main_header] + [""] * (len(headers) - 1))
@@ -170,17 +189,14 @@ class ExcelExporter(BaseExporter[Workbook]):
                 for cell in sheet["2"]:
                     cell.font = Font(bold=True, size=14)
 
-    def _write_metadata_sheet(self, workbook: Workbook, metadata: dict[str, Any], is_reference: bool = False) -> None:
+    def _write_metadata_sheet(self, workbook: Workbook, metadata: dict[str, Any], sheet_prefix: str = "") -> None:
         # Excel does not support timezone in datetime strings
         if isinstance(metadata.get("created"), datetime):
             metadata["created"] = metadata["created"].replace(tzinfo=None)
         if isinstance(metadata.get("updated"), datetime):
             metadata["updated"] = metadata["updated"].replace(tzinfo=None)
 
-        if is_reference:
-            metadata_sheet = workbook.create_sheet("RefMetadata")
-        else:
-            metadata_sheet = workbook.create_sheet("Metadata")
+        metadata_sheet = workbook.create_sheet(f"{sheet_prefix}Metadata")
         for key, value in metadata.items():
             metadata_sheet.append([key, value])
 
@@ -241,25 +257,29 @@ class ExcelExporter(BaseExporter[Workbook]):
         # Excel does not support timezone in datetime strings
         now_iso = datetime.now().replace(tzinfo=None).isoformat()
         is_info = isinstance(rules, InformationRules)
-        is_dms = not is_info
-        is_extension = self.new_model_id is not None
-        is_solution = is_extension and self.new_model_id != existing_model_id
+        is_dms = isinstance(rules, DMSRules)
+        is_extension = self.new_model_id is not None or rules.reference is not None
+        is_solution = rules.metadata.data_model_type == DataModelType.solution
 
-        if is_solution:
+        if is_solution and self.new_model_id:
             metadata["prefix" if is_info else "space"] = self.new_model_id[0]  # type: ignore[index]
             metadata["title" if is_info else "externalId"] = self.new_model_id[1]  # type: ignore[index]
             metadata["version"] = self.new_model_id[2]  # type: ignore[index]
+        elif is_solution and self.dump_as == "reference" and rules.reference:
+            metadata["prefix" if is_info else "space"] = "YOUR_PREFIX"
+            metadata["title" if is_info else "externalId"] = "YOUR_TITLE"
+            metadata["version"] = "1"
         else:
             metadata["prefix" if is_info else "space"] = existing_model_id[0]
             metadata["title" if is_info else "externalId"] = existing_model_id[1]
             metadata["version"] = existing_model_id[2]
 
-        if is_solution and is_info:
+        if is_solution and is_info and self.new_model_id:
             metadata["namespace"] = f"http://purl.org/{self.new_model_id[0]}/"  # type: ignore[index]
         elif is_info:
             metadata["namespace"] = existing_metadata["namespace"]
 
-        if is_solution and is_dms:
+        if is_solution and is_dms and self.new_model_id:
             metadata["name"] = self.new_model_id[1]  # type: ignore[index]
 
         if is_solution:
@@ -285,6 +305,11 @@ class ExcelExporter(BaseExporter[Workbook]):
         else:
             metadata["schema"] = SchemaCompleteness.complete.value
 
-        metadata["extension"] = "addition"
+        if is_solution:
+            metadata["dataModelType"] = DataModelType.solution.value
+        else:
+            metadata["dataModelType"] = DataModelType.enterprise.value
 
+        metadata["extension"] = ExtensionCategory.addition.value
+        metadata["role"] = (self.output_role and self.output_role.value) or rules.metadata.role.value
         return metadata

cognite/neat/rules/importers/_spreadsheet2rules.py

@@ -87,71 +87,95 @@ class MetadataRaw(UserDict):
 class ReadResult:
     sheets: dict[str, dict | list]
     read_info_by_sheet: dict[str, SpreadsheetRead]
-    role: RoleTypes
-    schema: SchemaCompleteness | None
+    metadata: MetadataRaw
+
+    @property
+    def role(self) -> RoleTypes:
+        return self.metadata.role
+
+    @property
+    def schema(self) -> SchemaCompleteness | None:
+        return self.metadata.schema
 
 
 class SpreadsheetReader:
-    def __init__(self, issue_list: IssueList, is_reference: bool = False):
+    def __init__(
+        self,
+        issue_list: IssueList,
+        required: bool = True,
+        metadata: MetadataRaw | None = None,
+        sheet_prefix: Literal["", "Last", "Ref"] = "",
+    ):
         self.issue_list = issue_list
-        self._is_reference = is_reference
+        self.required = required
+        self.metadata = metadata
+        self._sheet_prefix = sheet_prefix
 
     @property
     def metadata_sheet_name(self) -> str:
-        metadata_name = "Metadata"
-        return self.to_reference_sheet(metadata_name) if self._is_reference else metadata_name
+        return f"{self._sheet_prefix}Metadata"
 
     def sheet_names(self, role: RoleTypes) -> set[str]:
         names = MANDATORY_SHEETS_BY_ROLE[role]
-        return {self.to_reference_sheet(sheet_name) for sheet_name in names} if self._is_reference else names
-
-    @classmethod
-    def to_reference_sheet(cls, sheet_name: str) -> str:
-        return f"Ref{sheet_name}"
+        return {f"{self._sheet_prefix}{sheet_name}" for sheet_name in names if sheet_name != "Metadata"}
 
     def read(self, filepath: Path) -> None | ReadResult:
         with pd.ExcelFile(filepath) as excel_file:
-            if self.metadata_sheet_name not in excel_file.sheet_names:
+            metadata: MetadataRaw | None
+            if self.metadata is not None:
+                metadata = self.metadata
+            else:
+                metadata = self._read_metadata(excel_file, filepath)
+                if metadata is None:
+                    # The reading of metadata failed, so we can't continue
+                    return None
+
+            sheets, read_info_by_sheet = self._read_sheets(excel_file, metadata.role)
+            if sheets is None or self.issue_list.has_errors:
+                return None
+            sheets["Metadata"] = dict(metadata)
+
+            return ReadResult(sheets, read_info_by_sheet, metadata)
+
+    def _read_metadata(self, excel_file: ExcelFile, filepath: Path) -> MetadataRaw | None:
+        if self.metadata_sheet_name not in excel_file.sheet_names:
+            if self.required:
                 self.issue_list.append(
                     issues.spreadsheet_file.MetadataSheetMissingOrFailedError(
                         filepath, sheet_name=self.metadata_sheet_name
                     )
                 )
-                return None
-
-            metadata = MetadataRaw.from_excel(excel_file, self.metadata_sheet_name)
+            return None
 
-            if not metadata.is_valid(self.issue_list, filepath):
-                return None
+        metadata = MetadataRaw.from_excel(excel_file, self.metadata_sheet_name)
 
-            sheets, read_info_by_sheet = self._read_sheets(metadata, excel_file)
-            if sheets is None or self.issue_list.has_errors:
-                return None
-
-            return ReadResult(sheets, read_info_by_sheet, metadata.role, metadata.schema)
+        if not metadata.is_valid(self.issue_list, filepath):
+            return None
+        return metadata
 
     def _read_sheets(
-        self, metadata: MetadataRaw, excel_file: ExcelFile
+        self, excel_file: ExcelFile, read_role: RoleTypes
     ) -> tuple[dict[str, dict | list] | None, dict[str, SpreadsheetRead]]:
         read_info_by_sheet: dict[str, SpreadsheetRead] = defaultdict(SpreadsheetRead)
 
-        sheets: dict[str, dict | list] = {"Metadata": dict(metadata)}
+        sheets: dict[str, dict | list] = {}
 
-        expected_sheet_names = self.sheet_names(metadata.role)
+        expected_sheet_names = self.sheet_names(read_role)
 
         if missing_sheets := expected_sheet_names.difference(set(excel_file.sheet_names)):
-            self.issue_list.append(
-                issues.spreadsheet_file.SheetMissingError(cast(Path, excel_file.io), list(missing_sheets))
-            )
+            if self.required:
+                self.issue_list.append(
+                    issues.spreadsheet_file.SheetMissingError(cast(Path, excel_file.io), list(missing_sheets))
+                )
             return None, read_info_by_sheet
 
         for source_sheet_name, target_sheet_name, headers_input in SOURCE_SHEET__TARGET_FIELD__HEADERS:
-            source_sheet_name = self.to_reference_sheet(source_sheet_name) if self._is_reference else source_sheet_name
+            source_sheet_name = f"{self._sheet_prefix}{source_sheet_name}"
 
             if source_sheet_name not in excel_file.sheet_names:
                 continue
             if isinstance(headers_input, dict):
-                headers = headers_input[metadata.role]
+                headers = headers_input[read_role]
             else:
                 headers = headers_input
 
@@ -188,42 +212,37 @@ class ExcelImporter(BaseImporter):
             issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
             return self._return_or_raise(issue_list, errors)
 
-        user_result = SpreadsheetReader(issue_list, is_reference=False).read(self.filepath)
-        if user_result is None or issue_list.has_errors:
+        user_read = SpreadsheetReader(issue_list).read(self.filepath)
+        if user_read is None or issue_list.has_errors:
             return self._return_or_raise(issue_list, errors)
 
-        reference_result: ReadResult | None = None
-        if (
-            user_result
-            and user_result.role != RoleTypes.domain_expert
-            and user_result.schema == SchemaCompleteness.extended
-        ):
-            reference_result = SpreadsheetReader(issue_list, is_reference=True).read(self.filepath)
+        last_read: ReadResult | None = None
+        reference_read: ReadResult | None = None
+        if user_read.schema == SchemaCompleteness.extended:
+            # Last does not have its own metadata sheet. It is the same as the user's metadata sheet.
+            last_read = SpreadsheetReader(
+                issue_list, required=False, metadata=user_read.metadata, sheet_prefix="Last"
+            ).read(self.filepath)
+            reference_read = SpreadsheetReader(issue_list, sheet_prefix="Ref").read(self.filepath)
             if issue_list.has_errors:
                 return self._return_or_raise(issue_list, errors)
 
-        if user_result and reference_result and user_result.role != reference_result.role:
+        if reference_read and user_read.role != reference_read.role:
             issue_list.append(issues.spreadsheet_file.RoleMismatchError(self.filepath))
             return self._return_or_raise(issue_list, errors)
 
-        if user_result and reference_result:
-            user_result.sheets["reference"] = reference_result.sheets
-            sheets = user_result.sheets
-            original_role = user_result.role
-            read_info_by_sheet = user_result.read_info_by_sheet
-            read_info_by_sheet.update(reference_result.read_info_by_sheet)
-        elif user_result:
-            sheets = user_result.sheets
-            original_role = user_result.role
-            read_info_by_sheet = user_result.read_info_by_sheet
-        elif reference_result:
-            sheets = reference_result.sheets
-            original_role = reference_result.role
-            read_info_by_sheet = reference_result.read_info_by_sheet
-        else:
-            raise ValueError(
-                "No rules were generated. This should have been caught earlier. " f"Bug in {type(self).__name__}."
-            )
+        sheets = user_read.sheets
+        original_role = user_read.role
+        read_info_by_sheet = user_read.read_info_by_sheet
+        if last_read:
+            sheets["last"] = last_read.sheets
+            read_info_by_sheet.update(last_read.read_info_by_sheet)
+            if reference_read:
+                # The last rules will also be validated against the reference rules
+                sheets["last"]["reference"] = reference_read.sheets  # type: ignore[call-overload]
+        if reference_read:
+            sheets["reference"] = reference_read.sheets
+            read_info_by_sheet.update(reference_read.read_info_by_sheet)
 
         rules_cls = RULES_PER_ROLE[original_role]
         with _handle_issues(

cognite/neat/rules/issues/dms.py

@@ -9,6 +9,7 @@ from .base import NeatValidationError, ValidationWarning
 __all__ = [
     "DMSSchemaError",
     "DMSSchemaWarning",
+    "IncompleteSchemaError",
     "MissingSpaceError",
     "MissingContainerError",
     "MissingContainerPropertyError",
@@ -19,12 +20,14 @@ __all__ = [
     "DirectRelationMissingSourceWarning",
     "ViewModelVersionNotMatchingWarning",
     "ViewModelSpaceNotMatchingWarning",
+    "ViewMapsToTooManyContainersWarning",
     "DuplicatedViewInDataModelError",
     "ContainerPropertyUsedMultipleTimesError",
     "EmptyContainerWarning",
     "UnsupportedConnectionWarning",
     "MultipleReferenceWarning",
     "HasDataFilterOnNoPropertiesViewWarning",
+    "HasDataFilterAppliedToTooManyContainersWarning",
     "ReverseRelationMissingOtherSideWarning",
     "NodeTypeFilterOnParentViewWarning",
     "ChangingContainerError",
@@ -40,6 +43,24 @@ class DMSSchemaError(NeatValidationError, ABC): ...
 class DMSSchemaWarning(ValidationWarning, ABC): ...
 
 
+@dataclass(frozen=True)
+class IncompleteSchemaError(DMSSchemaError):
+    description = "This error is raised when the schema is claimed to be complete but missing some components"
+    fix = "Either provide the missing components or change the schema to partial"
+    missing_component: dm.ContainerId | dm.ViewId
+
+    def message(self) -> str:
+        return (
+            "The data model schema is set to be complete, however, "
+            f"the referred component {self.missing_component} is not preset."
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["missing_component"] = self.missing_component
+        return output
+
+
 @dataclass(frozen=True)
 class MissingSpaceError(DMSSchemaError):
     description = "The spaced referred to by the Container/View/Node/Edge/DataModel does not exist"
@@ -250,6 +271,28 @@ class ViewModelSpaceNotMatchingWarning(DMSSchemaWarning):
         return output
 
 
+@dataclass(frozen=True)
+class ViewMapsToTooManyContainersWarning(DMSSchemaWarning):
+    description = "The view maps to more than 10 containers which impacts read/write performance of data model"
+    fix = "Try to have as few containers as possible to which the view maps to"
+    error_name: ClassVar[str] = "ViewMapsToTooManyContainers"
+    view_id: dm.ViewId
+    container_ids: set[dm.ContainerId]
+
+    def message(self) -> str:
+        return (
+            f"The view {self.view_id} maps to total of {len(self.container_ids)},."
+            "Mapping to more than 10 containers is not recommended and can lead to poor performances."
+            "Re-iterate the data model design to reduce the number of containers to which the view maps to."
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = self.view_id.dump()
+        output["container_ids"] = [container_id.dump() for container_id in self.container_ids]
+        return output
+
+
 @dataclass(frozen=True)
 class ContainerPropertyUsedMultipleTimesError(DMSSchemaError):
     description = "The container property is used multiple times by the same view property"
@@ -442,6 +485,28 @@ class HasDataFilterOnNoPropertiesViewWarning(DMSSchemaWarning):
         return output
 
 
+@dataclass(frozen=True)
+class HasDataFilterAppliedToTooManyContainersWarning(DMSSchemaWarning):
+    description = "The view filter hasData applied to more than 10 containers this will cause DMS API Error"
+    fix = "Do not map to more than 10 containers, alternatively override the filter by using rawFilter"
+    error_name: ClassVar[str] = "HasDataFilterAppliedToTooManyContainers"
+    view_id: dm.ViewId
+    container_ids: set[dm.ContainerId]
+
+    def message(self) -> str:
+        return (
+            f"The view {self.view_id} HasData filter applied to total of {len(self.container_ids)},."
+            "Applying HasData filter to more than 10 containers is not recommended and can lead to DMS API error."
+            "Re-iterate the data model design to reduce the number of containers to which the view maps to."
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = self.view_id.dump()
+        output["container_ids"] = [container_id.dump() for container_id in self.container_ids]
+        return output
+
+
 @dataclass(frozen=True)
 class NodeTypeFilterOnParentViewWarning(DMSSchemaWarning):
     description = (

cognite/neat/rules/models/dms/_rules_input.py

@@ -303,6 +303,7 @@ class DMSRulesInput:
     properties: Sequence[DMSPropertyInput]
     views: Sequence[DMSViewInput]
     containers: Sequence[DMSContainerInput] | None = None
+    last: "DMSRulesInput | DMSRules | None" = None
     reference: "DMSRulesInput | DMSRules | None" = None
 
     @classmethod
@@ -323,6 +324,7 @@ class DMSRulesInput:
             properties=DMSPropertyInput.load(data.get("properties")),  # type: ignore[arg-type]
             views=DMSViewInput.load(data.get("views")),  # type: ignore[arg-type]
             containers=DMSContainerInput.load(data.get("containers")) or [],
+            last=DMSRulesInput.load(data.get("last")),
             reference=DMSRulesInput.load(data.get("reference")),
         )
 
@@ -338,12 +340,19 @@ class DMSRulesInput:
         elif isinstance(self.reference, DMSRules):
             # We need to load through the DMSRulesInput to set the correct default space and version
             reference = DMSRulesInput.load(self.reference.model_dump()).dump()
+        last: dict[str, Any] | None = None
+        if isinstance(self.last, DMSRulesInput):
+            last = self.last.dump()
+        elif isinstance(self.last, DMSRules):
+            # We need to load through the DMSRulesInput to set the correct default space and version
+            last = DMSRulesInput.load(self.last.model_dump()).dump()
 
         return dict(
             Metadata=self.metadata.dump(),
             Properties=[prop.dump(default_space, default_version) for prop in self.properties],
             Views=[view.dump(default_space, default_version) for view in self.views],
             Containers=[container.dump(default_space) for container in self.containers or []] or None,
+            Last=last,
             Reference=reference,
         )
 

cognite/neat/rules/models/dms/_schema.py

@@ -20,6 +20,7 @@ from cognite.neat.rules.issues.dms import (
     DirectRelationMissingSourceWarning,
     DMSSchemaError,
     DuplicatedViewInDataModelError,
+    IncompleteSchemaError,
     MissingContainerError,
     MissingContainerPropertyError,
     MissingEdgeViewError,
@@ -32,6 +33,7 @@ from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
 from cognite.neat.utils.cdf_loaders import ViewLoader
 from cognite.neat.utils.cdf_loaders.data_classes import RawTableWrite, RawTableWriteList
 from cognite.neat.utils.text import to_camel
+from cognite.neat.utils.utils import get_inheritance_path
 
 if sys.version_info >= (3, 11):
     from typing import Self
@@ -60,6 +62,30 @@ class DMSSchema:
         "node": "node_types",
     }
 
+    def _get_mapped_container_from_view(self, view_id: dm.ViewId) -> set[dm.ContainerId]:
+        # index all views, including ones from reference
+        indexed_views = {
+            **{view.as_id(): view for view in self.views},
+            **({view.as_id(): view for view in self.reference.views} if self.reference else {}),
+        }
+
+        if view_id not in indexed_views:
+            raise ValueError(f"View {view_id} not found")
+
+        indexed_implemented_views = {id_: view.implements for id_, view in indexed_views.items()}
+        view_inheritance = get_inheritance_path(view_id, indexed_implemented_views)
+
+        directly_referenced_containers = indexed_views[view_id].referenced_containers()
+        inherited_referenced_containers = set()
+
+        for view_id in view_inheritance:
+            if implemented_view := indexed_views.get(view_id):
+                inherited_referenced_containers |= implemented_view.referenced_containers()
+            else:
+                raise IncompleteSchemaError(missing_component=view_id).as_exception()
+
+        return directly_referenced_containers | inherited_referenced_containers
+
     @classmethod
     def from_model_id(cls, client: CogniteClient, data_model_id: dm.DataModelIdentifier) -> "DMSSchema":
         data_models = client.data_modeling.data_models.retrieve(data_model_id, inline_views=True)

cognite/neat/rules/models/dms/_serializer.py

@@ -77,10 +77,10 @@ class _DMSRulesSerializer:
             dumped[self.prop_name]["data"], key=lambda p: (p[self.prop_view], p[self.prop_view_property])
         )
         dumped[self.view_name] = sorted(dumped[self.view_name]["data"], key=lambda v: v[self.view_view])
-        if self.container_name in dumped:
-            dumped[self.container_name] = sorted(
-                dumped[self.container_name]["data"], key=lambda c: c[self.container_container]
-            )
+        if container_data := dumped.get(self.container_name):
+            dumped[self.container_name] = sorted(container_data["data"], key=lambda c: c[self.container_container])
+        else:
+            dumped.pop(self.container_name, None)
 
         for prop in dumped[self.prop_name]:
             for field_name in self.properties_fields:
@@ -105,7 +105,7 @@ class _DMSRulesSerializer:
                 for field in self.exclude_views:
                     view.pop(field, None)
 
-        for container in dumped[self.container_name]:
+        for container in dumped.get(self.container_name, []):
             for field_name in self.containers_fields:
                 if value := container.get(field_name):
                     container[field_name] = value.removeprefix(self.default_space)

cognite/neat/rules/models/dms/_validation.py

@@ -1,6 +1,8 @@
 from collections import defaultdict
 from typing import Any
 
+from cognite.client import data_modeling as dm
+
 from cognite.neat.rules import issues
 from cognite.neat.rules.issues import IssueList
 from cognite.neat.rules.models._base import ExtensionCategory, SchemaCompleteness
@@ -27,6 +29,7 @@ class DMSPostValidation:
         self._referenced_views_and_containers_are_existing()
         self._validate_extension()
         self._validate_schema()
+        self._validate_performance()
         return self.issue_list
 
     def _consistent_container_properties(self) -> None:
@@ -208,6 +211,36 @@ class DMSPostValidation:
                 )
             )
 
+    def _validate_performance(self) -> None:
+        # we can only validate performance on complete schemas due to the need
+        # to access all the container mappings
+        if self.metadata.schema_ is not SchemaCompleteness.complete:
+            return None
+
+        dms_schema = self.rules.as_schema()
+
+        for view in dms_schema.views:
+            mapped_containers = dms_schema._get_mapped_container_from_view(view.as_id())
+
+            if mapped_containers and len(mapped_containers) > 10:
+                self.issue_list.append(
+                    issues.dms.ViewMapsToTooManyContainersWarning(
+                        view_id=view.as_id(),
+                        container_ids=mapped_containers,
+                    )
+                )
+                if (
+                    view.filter
+                    and isinstance(view.filter, dm.filters.HasData)
+                    and len(view.filter.dump()["hasData"]) > 10
+                ):
+                    self.issue_list.append(
+                        issues.dms.HasDataFilterAppliedToTooManyContainersWarning(
+                            view_id=view.as_id(),
+                            container_ids=mapped_containers,
+                        )
+                    )
+
     @staticmethod
     def _changed_attributes_and_properties(
         new_dumped: dict[str, Any], existing_dumped: dict[str, Any]

cognite/neat/rules/models/domain.py

@@ -55,6 +55,7 @@ class DomainRules(RuleModel):
     metadata: DomainMetadata = Field(alias="Metadata")
     properties: SheetList[DomainProperty] = Field(alias="Properties")
     classes: SheetList[DomainClass] | None = Field(None, alias="Classes")
+    last: "DomainRules | None" = Field(None, alias="Last")
     reference: "DomainRules | None" = Field(None, alias="Reference")
 
     @model_serializer(mode="plain", when_used="always")

cognite/neat/rules/models/information/_converter.py

@@ -4,6 +4,7 @@ from datetime import datetime
 from typing import Literal
 
 from cognite.neat.rules.models._base import (
+    DataModelType,
     SheetList,
 )
 from cognite.neat.rules.models.data_types import DataType
@@ -45,6 +46,7 @@ class _InformationRulesConverter:
         metadata = DMSMetadata(
             schema_=info_metadata.schema_,
             space=space,
+            data_model_type=DataModelType.solution if self.information.reference else DataModelType.enterprise,
             version=info_metadata.version,
             external_id=info_metadata.name.replace(" ", "_").lower(),
             creator=info_metadata.creator,

cognite/neat/workflows/steps/lib/current/rules_exporter.py

@@ -292,7 +292,8 @@ class RulesToExcel(Step):
         if role != "input" and role is not None:
             output_role = RoleTypes[role]
 
-        excel_exporter = exporters.ExcelExporter(styling=styling, output_role=output_role, is_reference=is_reference)
+        dump_as = "reference" if is_reference else "user"
+        excel_exporter = exporters.ExcelExporter(styling=styling, output_role=output_role, dump_as=dump_as)  # type: ignore[arg-type]
 
         rule_instance: Rules
         if rules.domain:

{cognite_neat-0.76.2.dist-info → cognite_neat-0.76.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.76.2
+Version: 0.76.3
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

{cognite_neat-0.76.2.dist-info → cognite_neat-0.76.3.dist-info}/RECORD

@@ -1,5 +1,5 @@
 cognite/neat/__init__.py,sha256=v-rRiDOgZ3sQSMQKq0vgUQZvpeOkoHFXissAx6Ktg84,61
-cognite/neat/_version.py,sha256=HtluIH-Z1E-exBrYUiTjSYnxRCglf6JdDjRnCX5A30k,23
+cognite/neat/_version.py,sha256=OtWfvEnAHBJj0F1t4KUE2UmEyH4tIaTqHJzrcWyW6tI,23
 cognite/neat/app/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/app/api/asgi/metrics.py,sha256=nxFy7L5cChTI0a-zkCiJ59Aq8yLuIJp5c9Dg0wRXtV0,152
 cognite/neat/app/api/configuration.py,sha256=2U5M6M252swvQPQyooA1EBzFUZNtcTmuSaywfJDgckM,4232
@@ -166,7 +166,7 @@ cognite/neat/rules/exporters/__init__.py,sha256=Gn3CjkVKHJF9Po1ZPH4wAJ-sRW9up7b2
 cognite/neat/rules/exporters/_base.py,sha256=m63iw8xjlZbZAxGL8mn7pjGf1pW3rVv8C20_RSiu4t0,1511
 cognite/neat/rules/exporters/_models.py,sha256=vRd0P_YsrZ1eaAGGHfdTeFunaqHdaa0ZtnWiVZBR1nc,1976
 cognite/neat/rules/exporters/_rules2dms.py,sha256=BNznUtTdJ__M10I7QQf3_zdIQTET8SGFvHv5a-5louM,13529
-cognite/neat/rules/exporters/_rules2excel.py,sha256=zjGmUxQmsnHCEmSM5KvGQCFapdaDCCMJQRaLkVsI8mM,13034
+cognite/neat/rules/exporters/_rules2excel.py,sha256=K3D_AC6UZ-cG9ZFkqFvuDiMTdBC9ZUW9_IkkY9KsYW0,14934
 cognite/neat/rules/exporters/_rules2ontology.py,sha256=NWS3cn2927LQqW_PdQ-92OLIlmIKGNk7xh5yOMAyj94,20120
 cognite/neat/rules/exporters/_rules2yaml.py,sha256=sOSdnTJ5mXuyAJECdNnNsX6oLvgETptkpgPUQbK0n2w,3026
 cognite/neat/rules/exporters/_validation.py,sha256=OlKIyf4nhSDehJwFHDQ8Zdf6HpNfW7dSe2s67eywHu4,4078
@@ -183,11 +183,11 @@ cognite/neat/rules/importers/_owl2rules/_owl2classes.py,sha256=LInFeBq-NbBIuMEAw
 cognite/neat/rules/importers/_owl2rules/_owl2metadata.py,sha256=NdPN0dBB0NYkAcfC0yrYdIrGfdPbl5gfeGnSV3EtUPM,7786
 cognite/neat/rules/importers/_owl2rules/_owl2properties.py,sha256=BLptGmH-Aa5gZu0hDIxSZTrn9GmB2FicWgRYoETLSnQ,7437
 cognite/neat/rules/importers/_owl2rules/_owl2rules.py,sha256=H2Vv56hXGFnq_b0obWGWr5ErDFcoWpT8G2uy89100cU,6925
-cognite/neat/rules/importers/_spreadsheet2rules.py,sha256=eCDxFImXn_tQagjET_fxC9wriQmrWqRhghqilPfDXPI,11353
+cognite/neat/rules/importers/_spreadsheet2rules.py,sha256=dKTue97yZlPQagegfdyOWthDQ6X07hfz18KLXZlUPtA,11882
 cognite/neat/rules/importers/_yaml2rules.py,sha256=F0uksSz1A3po5OlRM2152_w5j8D9oYTLB9NFTkSMlWI,4275
 cognite/neat/rules/issues/__init__.py,sha256=Ms6jgCxCezc5IgTOwCFtXQPtoVFfOvdcXj84_rs917I,563
 cognite/neat/rules/issues/base.py,sha256=i2aTC-wq3UVW2bj_7wKeuhYxCpMD06Bd9-m00bWcTBs,6438
-cognite/neat/rules/issues/dms.py,sha256=0xBTYa3b3CPFjsdfKv59l3yX8DLTYB1Jo0LWDhfG6oQ,19178
+cognite/neat/rules/issues/dms.py,sha256=7UyVCqLfky6-KR9-pGNDhJhhMpAxuvOXj2FG-OR5kSA,22035
 cognite/neat/rules/issues/fileread.py,sha256=ao199mtvhPSW0IA8ZQZ0RzuLIIipYtL0jp6fLqxb4_c,5748
 cognite/neat/rules/issues/formatters.py,sha256=_ag2bJ9hncOj8pAGJvTTEPs9kTtxbD7vkqvS9Zcnizc,3385
 cognite/neat/rules/issues/importing.py,sha256=p90847g_TbUAXMyxalcEaWXaPygJoSE5E85dQO4syoo,12288
@@ -204,14 +204,14 @@ cognite/neat/rules/models/dms/__init__.py,sha256=Wzyqzz2ZIjpUbDg04CMuuIAw-f2A02D
 cognite/neat/rules/models/dms/_converter.py,sha256=x3u3jLnkknozoXXoAXXOWFHCsppqUwSvWv9wMOJ2F1Y,5706
 cognite/neat/rules/models/dms/_exporter.py,sha256=9EVToiib2QSxEXOzP7-xSlitpUlWSZTHar1HLIrtqxI,18908
 cognite/neat/rules/models/dms/_rules.py,sha256=CesPmLT4rC7rB_Q895nIV34lwfg3RC9NoU1wsz5l-hA,15599
-cognite/neat/rules/models/dms/_rules_input.py,sha256=WcdL_ztq1fAd8dv0Ob6X865pMMc_T1TVd4k2-kwlE2c,13147
-cognite/neat/rules/models/dms/_schema.py,sha256=-sMP5_2_DjY_QwdNpDWvSErEorTtZqUa7dATz0S9mZs,36236
-cognite/neat/rules/models/dms/_serializer.py,sha256=MYPpkbuor75PoY6kIk6O4elFqnKU8_0ON39nMtkG3dU,6619
-cognite/neat/rules/models/dms/_validation.py,sha256=k1awA0nMpv3G1dyWQd3q1GX1RBjJgEzN_G-uVlomaUY,12960
-cognite/neat/rules/models/domain.py,sha256=2S74P9YPPtb6myx8wg3-el9jrEBMH9AOBg9dAfwzlh4,2934
+cognite/neat/rules/models/dms/_rules_input.py,sha256=qfInh3JYf7XGEghxPOtIj7GY0f5_aVvnYeUBmfGV9mk,13620
+cognite/neat/rules/models/dms/_schema.py,sha256=cUAkIp7TCYVCshLZwQJcJMj_BduGSI3Y4_LwVxHUUUM,37463
+cognite/neat/rules/models/dms/_serializer.py,sha256=Zulj__rnaVNtrbGJPkn4dYMfMXWYyRmtNPR2Yb5zYW0,6668
+cognite/neat/rules/models/dms/_validation.py,sha256=QaNBqKi9YGqkYw_cwO_FcH3vRI2dXAORC6fjqV3D9HM,14290
+cognite/neat/rules/models/domain.py,sha256=13OhG-XavE5ipU2ICaYaUhz60volkuVfbJrsp0PhaUU,2993
 cognite/neat/rules/models/entities.py,sha256=iBG84Jr1qQ7PvkMJUJzJ1oWApeONb1IACixdJSztUhk,16395
 cognite/neat/rules/models/information/__init__.py,sha256=KvbYxVk38qReGbGTrU_Y3P3Gz6Bfghk5lHSKs8DlTOI,195
-cognite/neat/rules/models/information/_converter.py,sha256=cNWMBTsJ412M9MW2ytcafDKuWYZc_xcjwcsUag0Fs54,7833
+cognite/neat/rules/models/information/_converter.py,sha256=jzaIk7Q2CeU3TIGULEINwUNNyhWu-VdOW646EjH_FrI,7964
 cognite/neat/rules/models/information/_rules.py,sha256=YE7X8MsPQv-AVtl4vYtQW99moT45sYk2dI2DDS1YRO0,15546
 cognite/neat/rules/models/wrapped_entities.py,sha256=c5GkzOrYrE6SSRzIS2r8OAjhwxXpOoAO1WGc8kwiPPo,6154
 cognite/neat/utils/__init__.py,sha256=l5Nyqhqo25bcQXCOb_lk01cr-UXsG8cczz_y_I0u6bg,68
@@ -253,7 +253,7 @@ cognite/neat/workflows/steps/lib/current/__init__.py,sha256=c22IznGdCSNCpXCi_yon
 cognite/neat/workflows/steps/lib/current/graph_extractor.py,sha256=vW9UpJScx5dFVCSairpOdWRdBdLpkCt2kNh6litbF0o,5161
 cognite/neat/workflows/steps/lib/current/graph_loader.py,sha256=HfGg1HRZhbV58TFu89FTjKeUxGsbCYLeFJIQFDN_pQM,2341
 cognite/neat/workflows/steps/lib/current/graph_store.py,sha256=r7VTxdaz8jJQU7FJbnRDMxvEYbSAZFNMABhPyfNwiFk,6295
-cognite/neat/workflows/steps/lib/current/rules_exporter.py,sha256=Xe0b-ngLvfkc_uxvoG-BOzIuYdUup-QfSoJR31cIOwE,22824
+cognite/neat/workflows/steps/lib/current/rules_exporter.py,sha256=wUQAZXWBCqWXe0241QSREtnNTii_tSmOkeiSPwNQRjk,22898
 cognite/neat/workflows/steps/lib/current/rules_importer.py,sha256=yDq06cvxLvEpSnTXTjwhxDie_MzHa3wO1A4cbKnrH6c,10338
 cognite/neat/workflows/steps/lib/current/rules_validator.py,sha256=fDRQiRHN9Cuph38-WruK0T1UG5H448S_GsbzdOpi0h4,4729
 cognite/neat/workflows/steps/lib/io/__init__.py,sha256=k7IPbIq3ey19oRc5sA_15F99-O6dxzqbm1LihGRRo5A,32
@@ -271,8 +271,8 @@ cognite/neat/workflows/steps_registry.py,sha256=fkTX14ZA7_gkUYfWIlx7A1XbCidvqR23
 cognite/neat/workflows/tasks.py,sha256=dqlJwKAb0jlkl7abbY8RRz3m7MT4SK8-7cntMWkOYjw,788
 cognite/neat/workflows/triggers.py,sha256=_BLNplzoz0iic367u1mhHMHiUrCwP-SLK6_CZzfODX0,7071
 cognite/neat/workflows/utils.py,sha256=gKdy3RLG7ctRhbCRwaDIWpL9Mi98zm56-d4jfHDqP1E,453
-cognite_neat-0.76.2.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.76.2.dist-info/METADATA,sha256=0DLiYko7Fua8EdVbh9D7gvftXhrk9TyltRtIs_HLvkY,9316
-cognite_neat-0.76.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cognite_neat-0.76.2.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
-cognite_neat-0.76.2.dist-info/RECORD,,
+cognite_neat-0.76.3.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.76.3.dist-info/METADATA,sha256=N-fbQdrmuZ820x4fcdayOruzEbkDoCD1TU3scIJhWxo,9316
+cognite_neat-0.76.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_neat-0.76.3.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
+cognite_neat-0.76.3.dist-info/RECORD,,