cognite-neat 0.77.3__tar.gz → 0.77.5__tar.gz

{cognite_neat-0.77.3 → cognite_neat-0.77.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.77.3
+Version: 0.77.5
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.77.5/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.77.5"

{cognite_neat-0.77.3 → cognite_neat-0.77.5}/cognite/neat/rules/analysis/_information_rules.py

@@ -13,23 +13,13 @@ from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, ParentC
 from cognite.neat.rules.models.information import InformationClass, InformationProperty, InformationRules
 from cognite.neat.utils.utils import get_inheritance_path
 
-from ._base import BaseAnalysis, DataModelingScenario
+from ._base import BaseAnalysis
 
 
 class InformationArchitectRulesAnalysis(BaseAnalysis):
     def __init__(self, rules: InformationRules):
         self.rules = rules
 
-    @property
-    def data_modeling_scenario(self) -> DataModelingScenario:
-        if not self.rules.reference:
-            return DataModelingScenario.from_scratch
-
-        if self.rules.metadata.namespace == self.rules.reference.metadata.namespace:
-            return DataModelingScenario.extend_reference
-        else:
-            return DataModelingScenario.build_solution
-
     @property
     def referred_classes(self) -> set[ClassEntity]:
         return self.directly_referred_classes.union(self.inherited_referred_classes)
@@ -362,7 +352,7 @@ class InformationArchitectRulesAnalysis(BaseAnalysis):
 
         rules = cast(InformationRules, self.rules.reference if use_reference else self.rules)
 
-        if not rules.metadata.schema_ is not SchemaCompleteness.complete:
+        if rules.metadata.schema_ is not SchemaCompleteness.complete:
             raise ValueError("Rules are not complete cannot perform reduction!")
         class_as_dict = self.as_class_dict()
         class_parents_pairs = self.class_parent_pairs()

{cognite_neat-0.77.3 → cognite_neat-0.77.5}/cognite/neat/rules/exporters/_rules2excel.py

@@ -116,27 +116,27 @@ class ExcelExporter(BaseExporter[Workbook]):
             }
 
             if self.dump_as == "last":
-                dumped_last_rules = rules.model_dump(by_alias=True)
+                dumped_last_rules = rules.dump(by_alias=True)
                 if rules.reference:
-                    dumped_reference_rules = rules.reference.model_dump(by_alias=True)
+                    dumped_reference_rules = rules.reference.dump(by_alias=True, as_reference=True)
             elif self.dump_as == "reference":
-                dumped_reference_rules = rules.reference_self().model_dump(by_alias=True)
+                dumped_reference_rules = rules.dump(by_alias=True, as_reference=True)
         else:
-            dumped_user_rules = rules.model_dump(by_alias=True)
+            dumped_user_rules = rules.dump(by_alias=True)
             if rules.last:
-                dumped_last_rules = rules.last.model_dump(by_alias=True)
+                dumped_last_rules = rules.last.dump(by_alias=True)
             if rules.reference:
-                dumped_reference_rules = rules.reference.model_dump(by_alias=True)
+                dumped_reference_rules = rules.reference.dump(by_alias=True, as_reference=True)
 
         self._write_metadata_sheet(workbook, dumped_user_rules["Metadata"])
         self._write_sheets(workbook, dumped_user_rules, rules)
         if dumped_last_rules:
             self._write_sheets(workbook, dumped_last_rules, rules, sheet_prefix="Last")
+            self._write_metadata_sheet(workbook, dumped_last_rules["Metadata"], sheet_prefix="Last")
 
         if dumped_reference_rules:
-            prefix = "Ref"
-            self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix=prefix)
-            self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix=prefix)
+            self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix="Ref")
+            self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix="Ref")
 
         if self._styling_level > 0:
             self._adjust_column_widths(workbook)
@@ -291,7 +291,7 @@ class _MetadataCreator:
         prefix = self.new_model_id[0]
         return InformationMetadata(
             data_model_type=DataModelType.solution,
-            schema_=SchemaCompleteness.extended,
+            schema_=SchemaCompleteness.complete,
             extension=ExtensionCategory.addition,
             prefix=prefix,
             namespace=f"http://purl.org/neat/{prefix}/",  # type: ignore[arg-type]

{cognite_neat-0.77.3 → cognite_neat-0.77.5}/cognite/neat/rules/exporters/_rules2yaml.py

@@ -69,10 +69,10 @@ class YAMLExporter(BaseExporter[str]):
         rules = self._convert_to_output_role(rules, self.output_role)
         # model_dump_json ensures that the output is in JSON format,
         # if we don't do this, we will get Enums and other types that are not serializable to YAML
-        json_output = rules.model_dump_json()
+        json_output = rules.dump(mode="json")
         if self.output == "json":
-            return json_output
+            return json.dumps(json_output)
         elif self.output == "yaml":
-            return yaml.safe_dump(json.loads(json_output))
+            return yaml.safe_dump(json_output)
         else:
             raise ValueError(f"Invalid output: {self.output}. Valid options are {self.format_option}")

{cognite_neat-0.77.3 → cognite_neat-0.77.5}/cognite/neat/rules/importers/_dms2rules.py

@@ -109,7 +109,7 @@ class DMSImporter(BaseImporter):
         if result.result == "failure" or issue_list.has_errors:
             return cls(DMSSchema(), issue_list)
 
-        metadata = cls._create_metadata_from_model(user_model)
+        metadata = cls._create_metadata_from_model(user_model, has_reference=ref_model is not None)
         ref_metadata = cls._create_metadata_from_model(ref_model) if ref_model else None
 
         return cls(schema, issue_list, metadata, ref_metadata)
@@ -131,6 +131,7 @@ class DMSImporter(BaseImporter):
     def _create_metadata_from_model(
         cls,
         model: dm.DataModel[dm.View] | dm.DataModelApply,
+        has_reference: bool = False,
     ) -> DMSMetadata:
         description, creator = DMSMetadata._get_description_and_creator(model.description)
 
@@ -143,6 +144,7 @@ class DMSImporter(BaseImporter):
             updated = now
         return DMSMetadata(
             schema_=SchemaCompleteness.complete,
+            data_model_type=DataModelType.solution if has_reference else DataModelType.enterprise,
             extension=ExtensionCategory.addition,
             space=model.space,
             external_id=model.external_id,
@@ -202,16 +204,21 @@ class DMSImporter(BaseImporter):
                     **self._create_rule_components(
                         ref_model,
                         ref_schema,
-                        self.ref_metadata or self._create_default_metadata(list(ref_schema.views.values())),
+                        self.ref_metadata
+                        or self._create_default_metadata(list(ref_schema.views.values()), is_ref=True),
                         DataModelType.enterprise,
                     )
                 )
-                schema_completeness = SchemaCompleteness.extended
                 data_model_type = DataModelType.solution
 
             user_rules = DMSRules(
                 **self._create_rule_components(
-                    model, self.root_schema, self.metadata, data_model_type, schema_completeness
+                    model,
+                    self.root_schema,
+                    self.metadata,
+                    data_model_type,
+                    schema_completeness,
+                    has_reference=reference is not None,
                 ),
                 reference=reference,
             )
@@ -228,6 +235,7 @@ class DMSImporter(BaseImporter):
         metadata: DMSMetadata | None = None,
         data_model_type: DataModelType | None = None,
         schema_completeness: SchemaCompleteness | None = None,
+        has_reference: bool = False,
     ) -> dict[str, Any]:
         properties = SheetList[DMSProperty]()
         for view_id, view in schema.views.items():
@@ -242,7 +250,7 @@ class DMSImporter(BaseImporter):
             view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
         }
 
-        metadata = metadata or DMSMetadata.from_data_model(data_model)
+        metadata = metadata or DMSMetadata.from_data_model(data_model, has_reference)
         if data_model_type is not None:
             metadata.data_model_type = data_model_type
         if schema_completeness is not None:
@@ -262,12 +270,13 @@ class DMSImporter(BaseImporter):
         )
 
     @classmethod
-    def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply]) -> DMSMetadata:
+    def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply], is_ref: bool = False) -> DMSMetadata:
         now = datetime.now().replace(microsecond=0)
         space = Counter(view.space for view in views).most_common(1)[0][0]
         return DMSMetadata(
             schema_=SchemaCompleteness.complete,
             extension=ExtensionCategory.addition,
+            data_model_type=DataModelType.enterprise if is_ref else DataModelType.solution,
             space=space,
             external_id="Unknown",
             version="0.1.0",

{cognite_neat-0.77.3 → cognite_neat-0.77.5}/cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py

@@ -134,6 +134,7 @@ class DTDLImporter(BaseImporter):
 
         metadata = self._default_metadata()
         metadata["schema"] = self._schema_completeness.value
+
         if self.title:
             metadata["title"] = to_pascal(self.title)
         try:

{cognite_neat-0.77.3 → cognite_neat-0.77.5}/cognite/neat/rules/importers/_spreadsheet2rules.py

@@ -22,6 +22,7 @@ from cognite.neat.rules.models import (
     SchemaCompleteness,
 )
 from cognite.neat.rules.models.dms import DMSRulesInput
+from cognite.neat.rules.models.information import InformationRulesInput
 from cognite.neat.utils.auxiliary import local_import
 from cognite.neat.utils.spreadsheet import SpreadsheetRead, read_individual_sheet
 
@@ -110,17 +111,27 @@ class SpreadsheetReader:
         self.required = required
         self.metadata = metadata
         self._sheet_prefix = sheet_prefix
+        self._seen_files: set[Path] = set()
+        self._seen_sheets: set[str] = set()
 
     @property
     def metadata_sheet_name(self) -> str:
         return f"{self._sheet_prefix}Metadata"
 
+    @property
+    def seen_sheets(self) -> set[str]:
+        if not self._seen_files:
+            raise ValueError("No files have been read yet.")
+        return self._seen_sheets
+
     def sheet_names(self, role: RoleTypes) -> set[str]:
         names = MANDATORY_SHEETS_BY_ROLE[role]
         return {f"{self._sheet_prefix}{sheet_name}" for sheet_name in names if sheet_name != "Metadata"}
 
     def read(self, filepath: Path) -> None | ReadResult:
         with pd.ExcelFile(filepath) as excel_file:
+            self._seen_files.add(filepath)
+            self._seen_sheets.update(map(str, excel_file.sheet_names))
             metadata: MetadataRaw | None
             if self.metadata is not None:
                 metadata = self.metadata
@@ -212,20 +223,20 @@ class ExcelImporter(BaseImporter):
             issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
             return self._return_or_raise(issue_list, errors)
 
-        user_read = SpreadsheetReader(issue_list).read(self.filepath)
+        user_reader = SpreadsheetReader(issue_list)
+        user_read = user_reader.read(self.filepath)
         if user_read is None or issue_list.has_errors:
             return self._return_or_raise(issue_list, errors)
 
         last_read: ReadResult | None = None
+        if any(sheet_name.startswith("Last") for sheet_name in user_reader.seen_sheets):
+            last_read = SpreadsheetReader(issue_list, required=False, sheet_prefix="Last").read(self.filepath)
         reference_read: ReadResult | None = None
-        if user_read.schema == SchemaCompleteness.extended:
-            # Last does not have its own metadata sheet. It is the same as the user's metadata sheet.
-            last_read = SpreadsheetReader(
-                issue_list, required=False, metadata=user_read.metadata, sheet_prefix="Last"
-            ).read(self.filepath)
+        if any(sheet_name.startswith("Ref") for sheet_name in user_reader.seen_sheets):
             reference_read = SpreadsheetReader(issue_list, sheet_prefix="Ref").read(self.filepath)
-            if issue_list.has_errors:
-                return self._return_or_raise(issue_list, errors)
+
+        if issue_list.has_errors:
+            return self._return_or_raise(issue_list, errors)
 
         if reference_read and user_read.role != reference_read.role:
             issue_list.append(issues.spreadsheet_file.RoleMismatchError(self.filepath))
@@ -253,6 +264,8 @@ class ExcelImporter(BaseImporter):
             rules: Rules
             if rules_cls is DMSRules:
                 rules = DMSRulesInput.load(sheets).as_rules()
+            elif rules_cls is InformationRules:
+                rules = InformationRulesInput.load(sheets).as_rules()
             else:
                 rules = rules_cls.model_validate(sheets)  # type: ignore[attr-defined]
 

{cognite_neat-0.77.3 → cognite_neat-0.77.5}/cognite/neat/rules/issues/spreadsheet.py

@@ -10,7 +10,7 @@ from pydantic_core import ErrorDetails
 
 from cognite.neat.utils.spreadsheet import SpreadsheetRead
 
-from .base import DefaultPydanticError, MultiValueError, NeatValidationError, ValidationWarning
+from .base import DefaultPydanticError, MultiValueError, NeatValidationError
 
 if sys.version_info >= (3, 11):
     from typing import Self
@@ -27,7 +27,7 @@ __all__ = [
     "InvalidRowUnknownSheetError",
     "NonExistingContainerError",
     "NonExistingViewError",
-    "ClassNoPropertiesNoParentsWarning",
+    "ClassNoPropertiesNoParentError",
     "InconsistentContainerDefinitionError",
     "MultiValueTypeError",
     "MultiValueIsListError",
@@ -239,7 +239,26 @@ class NonExistingViewError(InvalidPropertyError):
 
 
 @dataclass(frozen=True)
-class ClassNoPropertiesNoParentsWarning(ValidationWarning):
+class PropertiesDefinedForUndefinedClassesError(NeatValidationError):
+    description = "Properties are defined for undefined classes."
+    fix = "Make sure to define class in the Classes sheet."
+
+    classes: list[str]
+
+    def dump(self) -> dict[str, list[str]]:
+        output = super().dump()
+        output["classes"] = self.classes
+        return output
+
+    def message(self) -> str:
+        return (
+            f"Classes {', '.join(self.classes)} have properties assigned to them, but"
+            " they are not defined in the Classes sheet."
+        )
+
+
+@dataclass(frozen=True)
+class ClassNoPropertiesNoParentError(NeatValidationError):
     description = "Class has no properties and no parents."
     fix = "Check if the class should have properties or parents."
 
@@ -252,8 +271,44 @@ class ClassNoPropertiesNoParentsWarning(ValidationWarning):
 
     def message(self) -> str:
         if len(self.classes) > 1:
-            return f"Classes {', '.join(self.classes)} have no properties and no parents. This may be a mistake."
-        return f"Class {self.classes[0]} has no properties and no parents. This may be a mistake."
+            return f"Classes {', '.join(self.classes)} have no direct or inherited properties. This may be a mistake."
+        return f"Class {self.classes[0]} have no direct or inherited properties. This may be a mistake."
+
+
+@dataclass(frozen=True)
+class ParentClassesNotDefinedError(NeatValidationError):
+    description = "Parent classes are not defined."
+    fix = "Check if the parent classes are defined in Classes sheet."
+
+    classes: list[str]
+
+    def dump(self) -> dict[str, list[str]]:
+        output = super().dump()
+        output["classes"] = self.classes
+        return output
+
+    def message(self) -> str:
+        if len(self.classes) > 1:
+            return f"Parent classes {', '.join(self.classes)} are not defined. This may be a mistake."
+        return f"Parent classes {', '.join(self.classes[0])} are not defined. This may be a mistake."
+
+
+@dataclass(frozen=True)
+class ValueTypeNotDefinedError(NeatValidationError):
+    description = "Value types referred by properties are not defined in Rules."
+    fix = "Make sure that all value types are defined in Rules."
+
+    value_types: list[str]
+
+    def dump(self) -> dict[str, list[str]]:
+        output = super().dump()
+        output["classes"] = self.value_types
+        return output
+
+    def message(self) -> str:
+        if len(self.value_types) > 1:
+            return f"Value types {', '.join(self.value_types)} are not defined. This may be a mistake."
+        return f"Value types {', '.join(self.value_types[0])} are not defined. This may be a mistake."
 
 
 @dataclass(frozen=True)

{cognite_neat-0.77.3 → cognite_neat-0.77.5}/cognite/neat/rules/models/_base.py

@@ -7,10 +7,9 @@ from __future__ import annotations
 import math
 import sys
 import types
-from abc import abstractmethod
 from collections.abc import Callable, Iterator
 from functools import wraps
-from typing import Annotated, Any, ClassVar, Generic, TypeAlias, TypeVar
+from typing import Annotated, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar
 
 import pandas as pd
 from pydantic import (
@@ -26,18 +25,23 @@ from pydantic import (
     model_validator,
 )
 from pydantic.fields import FieldInfo
+from pydantic.main import IncEx
 
 if sys.version_info >= (3, 11):
     from enum import StrEnum
-    from typing import Self
 else:
     from backports.strenum import StrEnum
-    from typing_extensions import Self
 
 
 METADATA_VALUE_MAX_LENGTH = 5120
 
 
+def _add_alias(data: dict[str, Any], base_model: type[BaseModel]) -> None:
+    for field_name, field_ in base_model.model_fields.items():
+        if field_name not in data and field_.alias in data:
+            data[field_name] = data[field_.alias]
+
+
 def replace_nan_floats_with_default(values: dict, model_fields: dict[str, FieldInfo]) -> dict:
     output = {}
     for field_name, value in values.items():
@@ -253,26 +257,33 @@ class BaseRules(RuleModel):
 
     Args:
         metadata: Data model metadata
-        classes: Classes defined in the data model
-        properties: Class properties defined in the data model with accompanying transformation rules
-                    to transform data from source to target representation
-        prefixes: Prefixes used in the data model. Defaults to PREFIXES
-        instances: Instances defined in the data model. Defaults to None
         validators_to_skip: List of validators to skip. Defaults to []
     """
 
     metadata: BaseMetadata
 
-    @abstractmethod
-    def reference_self(self) -> Self:
-        """
-        Returns a copy of the rules with reference fields set to itself
-
-        For example, if the rules have a property with a reference field, then
-        the reference field will be set to the property itself. This is used when
-        exporting a reference model.
+    def dump(
+        self,
+        mode: Literal["python", "json"] = "python",
+        by_alias: bool = False,
+        exclude: IncEx = None,
+        exclude_none: bool = False,
+        exclude_unset: bool = False,
+        exclude_defaults: bool = False,
+        as_reference: bool = False,
+    ) -> dict[str, Any]:
+        """Dump the model to a dictionary.
+
+        This is used in the Exporters to dump rules in the required format.
         """
-        raise NotImplementedError
+        return self.model_dump(
+            mode=mode,
+            by_alias=by_alias,
+            exclude=exclude,
+            exclude_none=exclude_none,
+            exclude_unset=exclude_unset,
+            exclude_defaults=exclude_defaults,
+        )
 
 
 # An sheet entity is either a class or a property.

{cognite_neat-0.77.3 → cognite_neat-0.77.5}/cognite/neat/rules/models/dms/_converter.py

@@ -105,6 +105,8 @@ class _DMSRulesConverter:
         prefix = metadata.space
         return InformationMetadata(
             schema_=metadata.schema_,
+            data_model_type=metadata.data_model_type,
+            extension=metadata.extension,
             prefix=prefix,
             namespace=Namespace(f"https://purl.orgl/neat/{prefix}/"),
             version=metadata.version,