cognite-neat 0.76.1__py3-none-any.whl → 0.76.3__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.76.1"
+__version__ = "0.76.3"

cognite/neat/app/api/routers/core.py

@@ -9,7 +9,7 @@ from fastapi import APIRouter, UploadFile
 
 from cognite.neat.app.api.configuration import NEAT_APP
 from cognite.neat.rules import exporters, importers
-from cognite.neat.rules.models.rules import DMSRules, RoleTypes
+from cognite.neat.rules.models import DMSRules, RoleTypes
 
 router = APIRouter()
 

cognite/neat/app/api/routers/rules.py

@@ -12,7 +12,7 @@ from cognite.neat.legacy.rules import importers as legacy_importers
 from cognite.neat.legacy.rules.models._base import EntityTypes
 from cognite.neat.legacy.rules.models.rules import Class, Classes, Metadata, Properties, Property, Rules
 from cognite.neat.rules import importers
-from cognite.neat.rules.models.rules import RoleTypes
+from cognite.neat.rules.models import RoleTypes
 from cognite.neat.workflows.steps.data_contracts import RulesData
 from cognite.neat.workflows.steps.lib.current.rules_exporter import RulesToExcel
 from cognite.neat.workflows.steps.lib.current.rules_importer import ExcelToRules

cognite/neat/graph/extractors/_mock_graph_generator.py

@@ -14,10 +14,10 @@ from rdflib import RDF, Literal, Namespace, URIRef
 
 from cognite.neat.graph.models import Triple
 from cognite.neat.rules.analysis import InformationArchitectRulesAnalysis
+from cognite.neat.rules.models import DMSRules, InformationRules
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
-from cognite.neat.rules.models.rules import DMSRules, InformationRules
-from cognite.neat.rules.models.rules._information_rules import InformationProperty
+from cognite.neat.rules.models.information import InformationProperty
 from cognite.neat.utils.utils import remove_namespace
 
 from ._base import BaseExtractor

cognite/neat/rules/_shared.py

@@ -1,5 +1,5 @@
 from typing import TypeAlias
 
-from cognite.neat.rules.models.rules import DMSRules, DomainRules, InformationRules
+from cognite.neat.rules.models import DMSRules, DomainRules, InformationRules
 
 Rules: TypeAlias = DomainRules | InformationRules | DMSRules

cognite/neat/rules/analysis/_information_rules.py

@@ -7,10 +7,10 @@ from typing import Any, cast
 import pandas as pd
 from pydantic import ValidationError
 
+from cognite.neat.rules.models import SchemaCompleteness
+from cognite.neat.rules.models._rdfpath import TransformationRuleType
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, ParentClassEntity, ReferenceEntity
-from cognite.neat.rules.models.rdfpath import TransformationRuleType
-from cognite.neat.rules.models.rules._base import SchemaCompleteness
-from cognite.neat.rules.models.rules._information_rules import InformationClass, InformationProperty, InformationRules
+from cognite.neat.rules.models.information import InformationClass, InformationProperty, InformationRules
 from cognite.neat.utils.utils import get_inheritance_path
 
 from ._base import BaseAnalysis, DataModelingScenario

cognite/neat/rules/exporters/_base.py

@@ -6,7 +6,7 @@ from typing import Generic, TypeVar
 from cognite.client import CogniteClient
 
 from cognite.neat.rules._shared import Rules
-from cognite.neat.rules.models.rules import DMSRules, InformationRules, RoleTypes
+from cognite.neat.rules.models import DMSRules, InformationRules, RoleTypes
 
 from ._models import UploadResult
 

cognite/neat/rules/exporters/_rules2dms.py

@@ -11,10 +11,8 @@ from cognite.client.exceptions import CogniteAPIError
 from cognite.neat.rules import issues
 from cognite.neat.rules._shared import Rules
 from cognite.neat.rules.issues import IssueList
-from cognite.neat.rules.models.rules import InformationRules
-from cognite.neat.rules.models.rules._base import ExtensionCategory, SheetList
-from cognite.neat.rules.models.rules._dms_architect_rules import DMSContainer, DMSRules
-from cognite.neat.rules.models.rules._dms_schema import DMSSchema, PipelineSchema
+from cognite.neat.rules.models import InformationRules
+from cognite.neat.rules.models.dms import DMSRules, DMSSchema, PipelineSchema
 from cognite.neat.utils.cdf_loaders import (
     ContainerLoader,
     DataModelingLoader,
@@ -116,47 +114,9 @@ class DMSExporter(CDFExporter[DMSSchema]):
             dms_rules = rules.as_dms_architect_rules()
         else:
             raise ValueError(f"{type(rules).__name__} cannot be exported to DMS")
-
-        is_solution_model = (
-            dms_rules.reference and dms_rules.metadata.external_id != dms_rules.reference.metadata.external_id
+        return dms_rules.as_schema(
+            include_ref=True, include_pipeline=self.export_pipeline, instance_space=self.instance_space
         )
-        is_new_model = dms_rules.reference is None
-        if is_new_model or is_solution_model:
-            return dms_rules.as_schema(False, self.export_pipeline, self.instance_space)
-
-        # This is an extension of an existing model.
-        reference_rules = cast(DMSRules, dms_rules.reference).model_copy(deep=True)
-        reference_schema = reference_rules.as_schema(include_ref=False, include_pipeline=self.export_pipeline)
-
-        # Todo Move this to an appropriate location
-        # Merging Reference with User Rules
-        combined_rules = dms_rules.model_copy(deep=True)
-        existing_containers = {container.class_ for container in combined_rules.containers or []}
-        if combined_rules.containers is None:
-            combined_rules.containers = SheetList[DMSContainer](data=[])
-        for container in reference_rules.containers or []:
-            if container.class_ not in existing_containers:
-                container.reference = None
-                combined_rules.containers.append(container)
-        existing_views = {view.class_ for view in combined_rules.views}
-        for view in reference_rules.views:
-            if view.class_ not in existing_views:
-                view.reference = None
-                combined_rules.views.append(view)
-        existing_properties = {(property_.class_, property_.property_) for property_ in combined_rules.properties}
-        for property_ in reference_rules.properties:
-            if (property_.class_, property_.property_) not in existing_properties:
-                property_.reference = None
-                combined_rules.properties.append(property_)
-
-        schema = combined_rules.as_schema(True, self.export_pipeline, self.instance_space)
-
-        if dms_rules.metadata.extension in (ExtensionCategory.addition, ExtensionCategory.reshape):
-            # We do not freeze views as they might be changed, even for addition,
-            # in case, for example, new properties are added. The validation will catch this.
-            schema.frozen_ids.update(set(reference_schema.containers.as_ids()))
-            schema.frozen_ids.update(set(reference_schema.node_types.as_ids()))
-        return schema
 
     def delete_from_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
         schema, to_export = self._prepare_schema_and_exporters(rules, client)
@@ -165,8 +125,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
         # as they are sorted in the order of creation and we need to delete them in reverse order
         for all_items, loader in reversed(to_export):
             all_item_ids = loader.get_ids(all_items)
-            skipped = sum(1 for item_id in all_item_ids if item_id in schema.frozen_ids)
-            item_ids = [item_id for item_id in all_item_ids if item_id not in schema.frozen_ids]
+            item_ids = [item_id for item_id in all_item_ids]
             cdf_items = loader.retrieve(item_ids)
             cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
             items = [item for item in all_items if loader.get_id(item) in item_ids]
@@ -200,7 +159,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
             yield UploadResult(
                 name=loader.resource_name,
                 deleted=deleted,
-                skipped=skipped,
+                skipped=0,
                 failed_deleted=failed_deleted,
                 error_messages=error_messages,
             )
@@ -215,8 +174,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
         for all_items, loader in to_export:
             issue_list = IssueList()
             all_item_ids = loader.get_ids(all_items)
-            skipped = sum(1 for item_id in all_item_ids if item_id in schema.frozen_ids)
-            item_ids = [item_id for item_id in all_item_ids if item_id not in schema.frozen_ids]
+            item_ids = [item_id for item_id in all_item_ids]
             cdf_items = loader.retrieve(item_ids)
             cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
             items = [item for item in all_items if loader.get_id(item) in item_ids]
@@ -243,6 +201,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
 
             created = len(to_create)
             failed_created = 0
+            skipped = 0
 
             if self.existing_handling in ["update", "force"]:
                 changed = len(to_update)

cognite/neat/rules/exporters/_rules2excel.py

@@ -12,8 +12,16 @@ from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
 from openpyxl.worksheet.worksheet import Worksheet
 
 from cognite.neat.rules._shared import Rules
-from cognite.neat.rules.models.rules import DMSRules, DomainRules, InformationRules
-from cognite.neat.rules.models.rules._base import RoleTypes, SchemaCompleteness, SheetEntity
+from cognite.neat.rules.models import (
+    DataModelType,
+    DMSRules,
+    DomainRules,
+    ExtensionCategory,
+    InformationRules,
+    RoleTypes,
+    SchemaCompleteness,
+    SheetEntity,
+)
 
 from ._base import BaseExporter
 
@@ -29,11 +37,18 @@ class ExcelExporter(BaseExporter[Workbook]):
         new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
             rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
             new metadata sheet in the Excel file.
-        is_reference: If True, the rules are considered to be a reference model. The exported Excel file will
-            then contain empty sheets for the main rules and this data model will be dumped to the reference sheets.
-            This is useful when you are building a solution model based on an Enterprise model, then the
-            Enterprise model will serve as the reference model. It is also useful when you are extending an existing
-            model, then the existing model will serve as the reference model. Defaults to False.
+        dump_as: This determines how the rules are written to the Excel file. An Excel file has up to three sets of
+           sheets: user, last, and reference. The user sheets are used for inputting rules from a user. The last sheets
+           are used for the last version of the same model as the user, while the reference sheets are used for
+           the model the user is building on. The options are:
+             * "user": The rules are written to the user sheets. This is used when you want to modify the rules
+                directly and potentially change the model. This is useful when you have imported the data model
+                from outside CDF and you want to modify it before you write it to CDF.
+             * "last": The rules are written to the last sheets. This is used when you want to extend the rules,
+               but have validation that you are not breaking the existing model. This is used when you want to
+               change a model that has already been published to CDF and that model is in production.
+             * "reference": The rules are written to the reference sheets. This is typically used when you want to build
+               a new solution on top of an enterprise model.
 
     The following styles are available:
 
@@ -45,7 +60,7 @@ class ExcelExporter(BaseExporter[Workbook]):
     """
 
     Style = Literal["none", "minimal", "default", "maximal"]
-
+    DumpOptions = Literal["user", "last", "reference"]
     _main_header_by_sheet_name: ClassVar[dict[str, str]] = {
         "Properties": "Definition of Properties per Class",
         "Classes": "Definition of Classes",
@@ -53,21 +68,24 @@ class ExcelExporter(BaseExporter[Workbook]):
         "Containers": "Definition of Containers",
     }
     style_options = get_args(Style)
+    dump_options = get_args(DumpOptions)
 
     def __init__(
         self,
         styling: Style = "default",
         output_role: RoleTypes | None = None,
         new_model_id: tuple[str, str, str] | None = None,
-        is_reference: bool = False,
+        dump_as: DumpOptions = "user",
     ):
         if styling not in self.style_options:
             raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
+        if dump_as not in self.dump_options:
+            raise ValueError(f"Invalid dump_as: {dump_as}. Valid options are {self.dump_options}")
         self.styling = styling
         self._styling_level = self.style_options.index(styling)
         self.output_role = output_role
         self.new_model_id = new_model_id
-        self.is_reference = is_reference
+        self.dump_as = dump_as
 
     def export_to_file(self, rules: Rules, filepath: Path) -> None:
         """Exports transformation rules to excel file."""
@@ -84,41 +102,48 @@ class ExcelExporter(BaseExporter[Workbook]):
         # Remove default sheet named "Sheet"
         workbook.remove(workbook["Sheet"])
 
-        dumped_rules: dict[str, Any]
+        dumped_user_rules: dict[str, Any]
+        dumped_last_rules: dict[str, Any] | None = None
         dumped_reference_rules: dict[str, Any] | None = None
-        if self.is_reference:
+        if self.dump_as != "user":
             # Writes empty reference sheets
-            dumped_rules = {
+            dumped_user_rules = {
                 "Metadata": self._create_metadata_sheet_user_rules(rules),
             }
-            dumped_rules["Metadata"]["role"] = (
-                self.output_role and self.output_role.value
-            ) or rules.metadata.role.value
-            dumped_reference_rules = rules.reference_self().model_dump(by_alias=True)
+
+            if self.dump_as == "last":
+                dumped_last_rules = rules.model_dump(by_alias=True)
+                if rules.reference:
+                    dumped_reference_rules = rules.reference.model_dump(by_alias=True)
+            elif self.dump_as == "reference":
+                dumped_reference_rules = rules.reference_self().model_dump(by_alias=True)
         else:
-            dumped_rules = rules.model_dump(by_alias=True)
+            dumped_user_rules = rules.model_dump(by_alias=True)
+            if rules.last:
+                dumped_last_rules = rules.last.model_dump(by_alias=True)
             if rules.reference:
                 dumped_reference_rules = rules.reference.model_dump(by_alias=True)
 
-        self._write_metadata_sheet(workbook, dumped_rules["Metadata"])
-        self._write_sheets(workbook, dumped_rules, rules)
+        self._write_metadata_sheet(workbook, dumped_user_rules["Metadata"])
+        self._write_sheets(workbook, dumped_user_rules, rules)
+        if dumped_last_rules:
+            self._write_sheets(workbook, dumped_last_rules, rules, sheet_prefix="Last")
+
         if dumped_reference_rules:
-            self._write_sheets(workbook, dumped_reference_rules, rules, is_reference=True)
-            self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], is_reference=True)
+            prefix = "Ref"
+            self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix=prefix)
+            self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix=prefix)
 
         if self._styling_level > 0:
             self._adjust_column_widths(workbook)
 
         return workbook
 
-    def _write_sheets(self, workbook: Workbook, dumped_rules: dict[str, Any], rules: Rules, is_reference: bool = False):
+    def _write_sheets(self, workbook: Workbook, dumped_rules: dict[str, Any], rules: Rules, sheet_prefix: str = ""):
         for sheet_name, headers in rules.headers_by_sheet(by_alias=True).items():
-            if sheet_name in ("Metadata", "prefixes", "Reference", "is_reference"):
+            if sheet_name in ("Metadata", "prefixes", "Reference", "Last"):
                 continue
-            if is_reference:
-                sheet = workbook.create_sheet(f"Ref{sheet_name}")
-            else:
-                sheet = workbook.create_sheet(sheet_name)
+            sheet = workbook.create_sheet(f"{sheet_prefix}{sheet_name}")
 
             main_header = self._main_header_by_sheet_name[sheet_name]
             sheet.append([main_header] + [""] * (len(headers) - 1))
@@ -164,17 +189,14 @@ class ExcelExporter(BaseExporter[Workbook]):
                 for cell in sheet["2"]:
                     cell.font = Font(bold=True, size=14)
 
-    def _write_metadata_sheet(self, workbook: Workbook, metadata: dict[str, Any], is_reference: bool = False) -> None:
+    def _write_metadata_sheet(self, workbook: Workbook, metadata: dict[str, Any], sheet_prefix: str = "") -> None:
         # Excel does not support timezone in datetime strings
         if isinstance(metadata.get("created"), datetime):
             metadata["created"] = metadata["created"].replace(tzinfo=None)
         if isinstance(metadata.get("updated"), datetime):
             metadata["updated"] = metadata["updated"].replace(tzinfo=None)
 
-        if is_reference:
-            metadata_sheet = workbook.create_sheet("RefMetadata")
-        else:
-            metadata_sheet = workbook.create_sheet("Metadata")
+        metadata_sheet = workbook.create_sheet(f"{sheet_prefix}Metadata")
         for key, value in metadata.items():
             metadata_sheet.append([key, value])
 
@@ -235,25 +257,29 @@ class ExcelExporter(BaseExporter[Workbook]):
         # Excel does not support timezone in datetime strings
         now_iso = datetime.now().replace(tzinfo=None).isoformat()
         is_info = isinstance(rules, InformationRules)
-        is_dms = not is_info
-        is_extension = self.new_model_id is not None
-        is_solution = is_extension and self.new_model_id != existing_model_id
+        is_dms = isinstance(rules, DMSRules)
+        is_extension = self.new_model_id is not None or rules.reference is not None
+        is_solution = rules.metadata.data_model_type == DataModelType.solution
 
-        if is_solution:
+        if is_solution and self.new_model_id:
             metadata["prefix" if is_info else "space"] = self.new_model_id[0]  # type: ignore[index]
             metadata["title" if is_info else "externalId"] = self.new_model_id[1]  # type: ignore[index]
             metadata["version"] = self.new_model_id[2]  # type: ignore[index]
+        elif is_solution and self.dump_as == "reference" and rules.reference:
+            metadata["prefix" if is_info else "space"] = "YOUR_PREFIX"
+            metadata["title" if is_info else "externalId"] = "YOUR_TITLE"
+            metadata["version"] = "1"
         else:
             metadata["prefix" if is_info else "space"] = existing_model_id[0]
             metadata["title" if is_info else "externalId"] = existing_model_id[1]
             metadata["version"] = existing_model_id[2]
 
-        if is_solution and is_info:
+        if is_solution and is_info and self.new_model_id:
             metadata["namespace"] = f"http://purl.org/{self.new_model_id[0]}/"  # type: ignore[index]
         elif is_info:
             metadata["namespace"] = existing_metadata["namespace"]
 
-        if is_solution and is_dms:
+        if is_solution and is_dms and self.new_model_id:
             metadata["name"] = self.new_model_id[1]  # type: ignore[index]
 
         if is_solution:
@@ -279,6 +305,11 @@ class ExcelExporter(BaseExporter[Workbook]):
         else:
             metadata["schema"] = SchemaCompleteness.complete.value
 
-        metadata["extension"] = "addition"
+        if is_solution:
+            metadata["dataModelType"] = DataModelType.solution.value
+        else:
+            metadata["dataModelType"] = DataModelType.enterprise.value
 
+        metadata["extension"] = ExtensionCategory.addition.value
+        metadata["role"] = (self.output_role and self.output_role.value) or rules.metadata.role.value
         return metadata

cognite/neat/rules/exporters/_rules2ontology.py

@@ -11,10 +11,10 @@ from rdflib.collection import Collection as GraphCollection
 from cognite.neat.constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
 from cognite.neat.rules import exceptions
 from cognite.neat.rules.analysis import InformationArchitectRulesAnalysis
+from cognite.neat.rules.models import DMSRules
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
-from cognite.neat.rules.models.rules import DMSRules
-from cognite.neat.rules.models.rules._information_rules import (
+from cognite.neat.rules.models.information import (
     InformationClass,
     InformationMetadata,
     InformationProperty,

cognite/neat/rules/exporters/_rules2yaml.py

@@ -6,7 +6,7 @@ from typing import Literal, get_args
 import yaml
 
 from cognite.neat.rules._shared import Rules
-from cognite.neat.rules.models.rules._base import RoleTypes
+from cognite.neat.rules.models import RoleTypes
 
 from ._base import BaseExporter
 

cognite/neat/rules/exporters/_validation.py

@@ -4,8 +4,8 @@ from typing import Literal, overload
 
 from cognite.neat.exceptions import wrangle_warnings
 from cognite.neat.rules import exceptions
-from cognite.neat.rules.models.rules import InformationRules
-from cognite.neat.rules.models.rules._types._base import DMS_PROPERTY_ID_COMPLIANCE_REGEX, VIEW_ID_COMPLIANCE_REGEX
+from cognite.neat.rules.models import InformationRules
+from cognite.neat.rules.models._types._base import DMS_PROPERTY_ID_COMPLIANCE_REGEX, VIEW_ID_COMPLIANCE_REGEX
 
 
 @overload

cognite/neat/rules/importers/_base.py

@@ -11,7 +11,7 @@ from rdflib import Namespace
 
 from cognite.neat.rules._shared import Rules
 from cognite.neat.rules.issues.base import IssueList, NeatValidationError, ValidationWarning
-from cognite.neat.rules.models.rules import DMSRules, InformationRules, RoleTypes
+from cognite.neat.rules.models import DMSRules, InformationRules, RoleTypes
 
 
 class BaseImporter(ABC):