cognite-neat 0.121.1__py3-none-any.whl → 0.122.0__py3-none-any.whl

cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py}

@@ -1,21 +1,24 @@
-"""This module performs importing of graph to TransformationRules pydantic class.
-In more details, it traverses the graph and abstracts class and properties, basically
-generating a list of rules based on which nodes that form the graph are made.
-"""
+"""This module performs importing of data model from spreadsheets."""
 
+import tempfile
 from collections import UserDict, defaultdict
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Literal, cast
 
 import pandas as pd
+from openpyxl import load_workbook
+from openpyxl.worksheet.worksheet import Worksheet
 from pandas import ExcelFile
 from rdflib import Namespace, URIRef
 
-from cognite.neat.core._data_model._shared import ReadRules, T_InputRules
+from cognite.neat.core._data_model._shared import (
+    ImportedDataModel,
+    T_UnverifiedDataModel,
+)
 from cognite.neat.core._data_model.models import (
-    INPUT_RULES_BY_ROLE,
-    VERIFIED_RULES_BY_ROLE,
+    UNVERIFIED_DATA_MODEL_BY_ROLE,
+    VERIFIED_DATA_MODEL_BY_ROLE,
     RoleTypes,
     SchemaCompleteness,
 )
@@ -36,11 +39,11 @@ SOURCE_SHEET__TARGET_FIELD__HEADERS = [
         "Properties",
         "Properties",
         {
-            RoleTypes.information: ["Class", "Property"],
+            RoleTypes.information: ["Concept", "Property"],
             RoleTypes.dms: ["View", "View Property"],
         },
     ),
-    ("Classes", "Classes", ["Class"]),
+    ("Concepts", "Concepts", ["Concept"]),
     ("Containers", "Containers", ["Container"]),
     ("Views", "Views", ["View"]),
     ("Enum", "Enum", ["Collection"]),
@@ -52,8 +55,8 @@ MANDATORY_SHEETS_BY_ROLE: dict[RoleTypes, set[str]] = {
     role_type: {
         str(sheet_name)
         for sheet_name in (
-            VERIFIED_RULES_BY_ROLE.get(role_type).mandatory_fields(use_alias=True)  # type: ignore
-            if VERIFIED_RULES_BY_ROLE.get(role_type)
+            VERIFIED_DATA_MODEL_BY_ROLE.get(role_type).mandatory_fields(use_alias=True)  # type: ignore
+            if VERIFIED_DATA_MODEL_BY_ROLE.get(role_type)
             else []
         )
         if sheet_name is not None
@@ -242,8 +245,8 @@ class SpreadsheetReader:
         return sheets, read_info_by_sheet
 
 
-class ExcelImporter(BaseImporter[T_InputRules]):
-    """Import rules from an Excel file.
+class ExcelImporter(BaseImporter[T_UnverifiedDataModel]):
+    """Import data_model from an Excel file.
 
     Args:
         filepath (Path): The path to the Excel file.
@@ -252,14 +255,15 @@ class ExcelImporter(BaseImporter[T_InputRules]):
     def __init__(self, filepath: Path):
         self.filepath = filepath
 
-    def to_rules(self) -> ReadRules[T_InputRules]:
+    def to_data_model(self) -> ImportedDataModel[T_UnverifiedDataModel]:
         issue_list = IssueList(title=f"'{self.filepath.name}'")
         if not self.filepath.exists():
             raise FileNotFoundNeatError(self.filepath)
 
+        self.filepath = self._make_forward_compatible_spreadsheet(self.filepath)
+
         with pd.ExcelFile(self.filepath) as excel_file:
             user_reader = SpreadsheetReader(issue_list)
-
             user_read = user_reader.read(excel_file, self.filepath)
 
         issue_list.trigger_warnings()
@@ -267,15 +271,23 @@ class ExcelImporter(BaseImporter[T_InputRules]):
             raise MultiValueError(issue_list.errors)
 
         if user_read is None:
-            return ReadRules(None, {})
+            return ImportedDataModel(None, {})
 
         sheets = user_read.sheets
         original_role = user_read.role
         read_info_by_sheet = user_read.read_info_by_sheet
 
-        rules_cls = INPUT_RULES_BY_ROLE[original_role]
-        rules = cast(T_InputRules, rules_cls.load(sheets))
-        return ReadRules(rules, read_info_by_sheet)
+        data_model_cls = UNVERIFIED_DATA_MODEL_BY_ROLE[original_role]
+        data_model = cast(T_UnverifiedDataModel, data_model_cls.load(sheets))
+
+        # Delete the temporary file if it was created
+        if "temp_neat_file" in self.filepath.name:
+            try:
+                self.filepath.unlink()
+            except Exception as e:
+                issue_list.append(FileReadError(self.filepath, f"Failed to delete temporary file: {e}"))
+
+        return ImportedDataModel(data_model, read_info_by_sheet)
 
     @property
     def description(self) -> str:
@@ -284,3 +296,48 @@ class ExcelImporter(BaseImporter[T_InputRules]):
     @property
     def source_uri(self) -> URIRef:
         return URIRef(f"file://{self.filepath.name}")
+
+    def _make_forward_compatible_spreadsheet(self, filepath: Path) -> Path:
+        """Makes the spreadsheet forward compatible by renaming legacy class with concept
+
+        Args:
+            filepath (Path): The path to the Excel file.
+
+        """
+
+        workbook = load_workbook(filepath)
+
+        if "Classes" in workbook.sheetnames:
+            print(
+                (
+                    "You are using a legacy spreadsheet format, "
+                    "which we will support until v1.0 of neat."
+                    " Please update your spreadsheet to the new format."
+                ),
+            )
+            _replace_class_with_concept_cell(workbook["Classes"])
+            sheet = workbook["Classes"]
+            sheet.title = "Concepts"
+
+            if "Properties" in workbook.sheetnames:
+                _replace_class_with_concept_cell(workbook["Properties"])
+
+            with tempfile.NamedTemporaryFile(prefix="temp_neat_file", suffix=".xlsx", delete=False) as temp_file:
+                workbook.save(temp_file.name)
+                return Path(temp_file.name)
+
+        else:
+            return filepath
+
+
+def _replace_class_with_concept_cell(sheet: Worksheet) -> None:
+    """
+    Replaces the word "Class" with "Concept" in the first row of the given sheet.
+
+    Args:
+        sheet (Worksheet): The sheet in which to replace the word "Class".
+    """
+    for row in sheet.iter_rows():
+        for cell in row:
+            if cell.value == "Class":
+                cell.value = "Concept"

cognite/neat/core/_data_model/models/__init__.py

@@ -7,30 +7,32 @@ from cognite.neat.core._data_model.models.conceptual._verified import (
 )
 
 from ._base_verified import DataModelType, ExtensionCategory, RoleTypes, SchemaCompleteness, SheetList, SheetRow
-from .dms._rules import DMSRules
-from .dms._rules_input import DMSInputRules
+from .physical._unverified import UnverifiedPhysicalDataModel
+from .physical._verified import PhysicalDataModel
 
-INPUT_RULES_BY_ROLE: dict[RoleTypes, type[UnverifiedConceptualDataModel] | type[DMSInputRules]] = {
+UNVERIFIED_DATA_MODEL_BY_ROLE: dict[
+    RoleTypes, type[UnverifiedConceptualDataModel] | type[UnverifiedPhysicalDataModel]
+] = {
     RoleTypes.information: UnverifiedConceptualDataModel,
-    RoleTypes.dms: DMSInputRules,
+    RoleTypes.dms: UnverifiedPhysicalDataModel,
 }
-VERIFIED_RULES_BY_ROLE: dict[RoleTypes, type[ConceptualDataModel] | type[DMSRules]] = {
+VERIFIED_DATA_MODEL_BY_ROLE: dict[RoleTypes, type[ConceptualDataModel] | type[PhysicalDataModel]] = {
     RoleTypes.information: ConceptualDataModel,
-    RoleTypes.dms: DMSRules,
+    RoleTypes.dms: PhysicalDataModel,
 }
 
 
 __all__ = [
-    "INPUT_RULES_BY_ROLE",
+    "UNVERIFIED_DATA_MODEL_BY_ROLE",
     "ConceptualDataModel",
-    "DMSInputRules",
-    "DMSRules",
     "DMSSchema",
     "DataModelType",
     "ExtensionCategory",
+    "PhysicalDataModel",
     "RoleTypes",
     "SchemaCompleteness",
     "SheetList",
     "SheetRow",
     "UnverifiedConceptualDataModel",
+    "UnverifiedPhysicalDataModel",
 ]

cognite/neat/core/_data_model/models/_base_unverified.py

@@ -27,29 +27,29 @@ if sys.version_info >= (3, 11):
 else:
     from typing_extensions import Self
 
-T_BaseRules = TypeVar("T_BaseRules", bound=BaseVerifiedDataModel)
-T_RuleModel = TypeVar("T_RuleModel", bound=SchemaModel)
+T_BaseDataModel = TypeVar("T_BaseDataModel", bound=BaseVerifiedDataModel)
+T_DataModel = TypeVar("T_DataModel", bound=SchemaModel)
 
 
 @dataclass
-class UnverifiedDataModel(Generic[T_BaseRules], ABC):
-    """Input rules are raw data that is not yet validated."""
+class UnverifiedDataModel(Generic[T_BaseDataModel], ABC):
+    """Input data model are raw data that is not yet validated."""
 
     @classmethod
     @abstractmethod
-    def _get_verified_cls(cls) -> type[T_BaseRules]:
+    def _get_verified_cls(cls) -> type[T_BaseDataModel]:
         raise NotImplementedError("This method should be implemented in the subclass.")
 
     @classmethod
     @overload
-    def load(cls: "type[T_InputRules]", data: dict[str, Any]) -> "T_InputRules": ...
+    def load(cls: "type[T_UnverifiedDataModel]", data: dict[str, Any]) -> "T_UnverifiedDataModel": ...
 
     @classmethod
     @overload
-    def load(cls: "type[T_InputRules]", data: None) -> None: ...
+    def load(cls: "type[T_UnverifiedDataModel]", data: None) -> None: ...
 
     @classmethod
-    def load(cls: "type[T_InputRules]", data: dict | None) -> "T_InputRules | None":
+    def load(cls: "type[T_UnverifiedDataModel]", data: dict | None) -> "T_UnverifiedDataModel | None":
         if data is None:
             return None
         return cls._load(data)
@@ -110,7 +110,7 @@ class UnverifiedDataModel(Generic[T_BaseRules], ABC):
     def _dataclass_fields(self) -> list[Field]:
         return list(fields(self))
 
-    def as_verified_rules(self) -> T_BaseRules:
+    def as_verified_data_model(self) -> T_BaseDataModel:
         cls_ = self._get_verified_cls()
         return cls_.model_validate(self.dump())
 
@@ -127,14 +127,14 @@ class UnverifiedDataModel(Generic[T_BaseRules], ABC):
         return output
 
 
-T_InputRules = TypeVar("T_InputRules", bound=UnverifiedDataModel)
+T_UnverifiedDataModel = TypeVar("T_UnverifiedDataModel", bound=UnverifiedDataModel)
 
 
 @dataclass
-class UnverifiedComponent(ABC, Generic[T_RuleModel]):
+class UnverifiedComponent(ABC, Generic[T_DataModel]):
     @classmethod
     @abstractmethod
-    def _get_verified_cls(cls) -> type[T_RuleModel]:
+    def _get_verified_cls(cls) -> type[T_DataModel]:
         raise NotImplementedError("This method should be implemented in the subclass.")
 
     @classmethod

cognite/neat/core/_data_model/models/_base_verified.py

@@ -1,4 +1,4 @@
-"""This module contains the definition of `TransformationRules` pydantic model and all
+"""This module contains the definition of `DataModel` pydantic model and all
 its sub-models and validators.
 """
 
@@ -30,7 +30,7 @@ from cognite.neat.core._constants import DEFAULT_NAMESPACE
 from cognite.neat.core._data_model.models._types import (
     ContainerEntityType,
     DataModelExternalIdType,
-    DmsPropertyType,
+    PhysicalPropertyType,
     SpaceType,
     StrListType,
     URIRefType,
@@ -160,7 +160,7 @@ class BaseVerifiedMetadata(SchemaModel):
 
     creator: StrListType = Field(
         description=(
-            "List of contributors (comma seperated) to the data model creation, "
+            "List of contributors (comma separated) to the data model creation, "
             "typically information architects are considered as contributors."
         ),
     )
@@ -175,7 +175,7 @@ class BaseVerifiedMetadata(SchemaModel):
 
     source_id: URIRefType | None = Field(
         None,
-        description="Id of source that produced this rules",
+        description="Id of source that produced this data model",
         alias="sourceId",
     )
 
@@ -254,12 +254,7 @@ class BaseVerifiedMetadata(SchemaModel):
 
 class BaseVerifiedDataModel(SchemaModel, ABC):
     """
-    Rules is a core concept in `neat`. This represents fusion of data model
-    definitions and (optionally) the transformation rules used to transform the data/graph
-    from the source representation to the target representation defined by the data model.
-    The rules are defined in an Excel sheet and then parsed into a `Rules` object. The
-    `Rules` object is then used to generate data model and the `RDF` graph made of data
-    model instances.
+    Data Model is a core concept in `neat`.
 
     Args:
         metadata: Data model metadata
@@ -317,11 +312,11 @@ class BaseVerifiedDataModel(SchemaModel, ABC):
     ) -> dict[str, Any]:
         """Dump the model to a dictionary.
 
-        This is used in the Exporters to dump rules in the required format.
+        This is used in the Exporters to the dump data model in the required format.
 
         Args:
             entities_exclude_defaults: Whether to exclude default prefix (and version) for entities.
-                For example, given a class that is dumped as 'my_prefix:MyClass', if the prefix for the rules
+                For example, given a class that is dumped as 'my_prefix:MyClass', if the prefix for the data model
                 set in metadata.prefix = 'my_prefix', then this class will be dumped as 'MyClass' when this flag is set.
                 Defaults to True.
             sort: Whether to sort the entities in the output.
@@ -438,7 +433,7 @@ ExtensionCategoryType = Annotated[
 # Immutable such that this can be used as a key in a dictionary
 class ContainerProperty(BaseModel, frozen=True):
     container: ContainerEntityType
-    property_: DmsPropertyType
+    property_: PhysicalPropertyType
 
 
 class ContainerDestinationProperty(ContainerProperty, frozen=True):
@@ -451,4 +446,4 @@ class ViewRef(BaseModel, frozen=True):
 
 
 class ViewProperty(ViewRef, frozen=True):
-    property_: DmsPropertyType
+    property_: PhysicalPropertyType

cognite/neat/core/_data_model/models/_types.py

@@ -23,14 +23,14 @@ from cognite.neat.core._data_model._constants import (
 )
 from cognite.neat.core._data_model.models.entities._multi_value import MultiValueTypeInfo
 from cognite.neat.core._data_model.models.entities._single_value import (
-    ClassEntity,
+    ConceptEntity,
     ContainerEntity,
     ViewEntity,
 )
 from cognite.neat.core._issues.errors import RegexViolationError
 from cognite.neat.core._issues.warnings import RegexViolationWarning
 
-Entities: TypeAlias = ClassEntity | ViewEntity | ContainerEntity
+Entities: TypeAlias = ConceptEntity | ViewEntity | ContainerEntity
 T_Entities = TypeVar("T_Entities", bound=Entities)
 
 
@@ -139,9 +139,9 @@ ConceptualPropertyType = Annotated[
     str,
     AfterValidator(_external_id_validation_factory(EntityTypes.conceptual_property, "Property column in properties")),
 ]
-DmsPropertyType = Annotated[
+PhysicalPropertyType = Annotated[
     str,
-    AfterValidator(_external_id_validation_factory(EntityTypes.dms_property, "Property column in properties")),
+    AfterValidator(_external_id_validation_factory(EntityTypes.physical_property, "Property column in properties")),
 ]
 
 
@@ -152,7 +152,7 @@ def _entity_validation(value: Entities, location: str) -> Entities:
     return value
 
 
-ClassEntityType = Annotated[ClassEntity, AfterValidator(lambda v: _entity_validation(v, "the Class column"))]
+ConceptEntityType = Annotated[ConceptEntity, AfterValidator(lambda v: _entity_validation(v, "the Class column"))]
 ViewEntityType = Annotated[ViewEntity, AfterValidator(lambda v: _entity_validation(v, "the View column"))]
 ContainerEntityType = Annotated[
     ContainerEntity, AfterValidator(lambda v: _entity_validation(v, "the Container column"))
@@ -161,7 +161,7 @@ ContainerEntityType = Annotated[
 
 def _multi_value_type_validation(value: MultiValueTypeInfo, location: str) -> MultiValueTypeInfo:
     for type_ in value.types:
-        if isinstance(type_, ClassEntity):
+        if isinstance(type_, ConceptEntity):
             _entity_validation(type_, location)
     return value
 

cognite/neat/core/_data_model/models/conceptual/__init__.py

@@ -1,24 +1,24 @@
 from ._unverified import (
-    UnverifiedConceptualClass,
+    UnverifiedConcept,
     UnverifiedConceptualDataModel,
     UnverifiedConceptualMetadata,
     UnverifiedConceptualProperty,
 )
-from ._validation import InformationValidation
+from ._validation import ConceptualValidation
 from ._verified import (
-    ConceptualClass,
+    Concept,
     ConceptualDataModel,
     ConceptualMetadata,
     ConceptualProperty,
 )
 
 __all__ = [
-    "ConceptualClass",
+    "Concept",
     "ConceptualDataModel",
     "ConceptualMetadata",
     "ConceptualProperty",
-    "InformationValidation",
-    "UnverifiedConceptualClass",
+    "ConceptualValidation",
+    "UnverifiedConcept",
     "UnverifiedConceptualDataModel",
     "UnverifiedConceptualMetadata",
     "UnverifiedConceptualProperty",

cognite/neat/core/_data_model/models/conceptual/_unverified.py

@@ -13,7 +13,7 @@ from cognite.neat.core._data_model.models._base_unverified import (
 )
 from cognite.neat.core._data_model.models.data_types import DataType
 from cognite.neat.core._data_model.models.entities import (
-    ClassEntity,
+    ConceptEntity,
     MultiValueTypeInfo,
     UnknownEntity,
     load_value_type,
@@ -21,7 +21,7 @@ from cognite.neat.core._data_model.models.entities import (
 from cognite.neat.core._utils.rdf_ import uri_display_name
 
 from ._verified import (
-    ConceptualClass,
+    Concept,
     ConceptualDataModel,
     ConceptualMetadata,
     ConceptualProperty,
@@ -68,7 +68,7 @@ class UnverifiedConceptualMetadata(UnverifiedComponent[ConceptualMetadata]):
             Unlike namespace, the identifier does not end with "/" or "#".
 
         """
-        return DEFAULT_NAMESPACE[f"data-model/unverified/logical/{self.space}/{self.external_id}/{self.version}"]
+        return DEFAULT_NAMESPACE[f"data-model/unverified/conceptual/{self.space}/{self.external_id}/{self.version}"]
 
     @property
     def namespace(self) -> Namespace:
@@ -78,9 +78,9 @@ class UnverifiedConceptualMetadata(UnverifiedComponent[ConceptualMetadata]):
 
 @dataclass
 class UnverifiedConceptualProperty(UnverifiedComponent[ConceptualProperty]):
-    class_: ClassEntity | str
+    concept: ConceptEntity | str
     property_: str
-    value_type: DataType | ClassEntity | MultiValueTypeInfo | UnknownEntity | str
+    value_type: DataType | ConceptEntity | MultiValueTypeInfo | UnknownEntity | str
     name: str | None = None
     description: str | None = None
     min_count: int | None = None
@@ -100,7 +100,7 @@ class UnverifiedConceptualProperty(UnverifiedComponent[ConceptualProperty]):
 
     def dump(self, default_prefix: str, **kwargs) -> dict[str, Any]:  # type: ignore
         output = super().dump()
-        output["Class"] = ClassEntity.load(self.class_, prefix=default_prefix)
+        output["Concept"] = ConceptEntity.load(self.concept, prefix=default_prefix)
         output["Value Type"] = load_value_type(self.value_type, default_prefix)
         return output
 
@@ -112,33 +112,33 @@ class UnverifiedConceptualProperty(UnverifiedComponent[ConceptualProperty]):
 
 
 @dataclass
-class UnverifiedConceptualClass(UnverifiedComponent[ConceptualClass]):
-    class_: ClassEntity | str
+class UnverifiedConcept(UnverifiedComponent[Concept]):
+    concept: ConceptEntity | str
     name: str | None = None
     description: str | None = None
-    implements: str | list[ClassEntity] | None = None
+    implements: str | list[ConceptEntity] | None = None
     instance_source: str | None = None
     neatId: str | URIRef | None = None
     # linking
     physical: str | URIRef | None = None
 
     @classmethod
-    def _get_verified_cls(cls) -> type[ConceptualClass]:
-        return ConceptualClass
+    def _get_verified_cls(cls) -> type[Concept]:
+        return Concept
 
     @property
-    def class_str(self) -> str:
-        return str(self.class_)
+    def concept_str(self) -> str:
+        return str(self.concept)
 
     def dump(self, default_prefix: str, **kwargs) -> dict[str, Any]:  # type: ignore
         output = super().dump()
-        parent: list[ClassEntity] | None = None
+        parent: list[ConceptEntity] | None = None
         if isinstance(self.implements, str):
             self.implements = self.implements.strip()
-            parent = [ClassEntity.load(parent, prefix=default_prefix) for parent in self.implements.split(",")]
+            parent = [ConceptEntity.load(parent, prefix=default_prefix) for parent in self.implements.split(",")]
         elif isinstance(self.implements, list):
-            parent = [ClassEntity.load(parent_, prefix=default_prefix) for parent_ in self.implements]
-        output["Class"] = ClassEntity.load(self.class_, prefix=default_prefix)
+            parent = [ConceptEntity.load(parent_, prefix=default_prefix) for parent_ in self.implements]
+        output["Concept"] = ConceptEntity.load(self.concept, prefix=default_prefix)
         output["Implements"] = parent
         return output
 
@@ -147,7 +147,7 @@ class UnverifiedConceptualClass(UnverifiedComponent[ConceptualClass]):
 class UnverifiedConceptualDataModel(UnverifiedDataModel[ConceptualDataModel]):
     metadata: UnverifiedConceptualMetadata
     properties: list[UnverifiedConceptualProperty] = field(default_factory=list)
-    classes: list[UnverifiedConceptualClass] = field(default_factory=list)
+    concepts: list[UnverifiedConcept] = field(default_factory=list)
     prefixes: dict[str, Namespace] | None = None
 
     @classmethod
@@ -160,7 +160,7 @@ class UnverifiedConceptualDataModel(UnverifiedDataModel[ConceptualDataModel]):
         return dict(
             Metadata=self.metadata.dump(),
             Properties=[prop.dump(default_prefix) for prop in self.properties],
-            Classes=[class_.dump(default_prefix) for class_ in self.classes],
+            Concepts=[concept.dump(default_prefix) for concept in self.concepts],
             Prefixes=self.prefixes,
         )
 
@@ -180,7 +180,7 @@ class UnverifiedConceptualDataModel(UnverifiedDataModel[ConceptualDataModel]):
             "external_id": self.metadata.external_id,
             "space": self.metadata.space,
             "version": self.metadata.version,
-            "classes": len(self.classes),
+            "concepts": len(self.concepts),
             "properties": len(self.properties),
         }