cognite-neat 0.121.1__py3-none-any.whl → 0.122.0__py3-none-any.whl

cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py}

@@ -10,15 +10,16 @@ from rdflib.collection import Collection as GraphCollection
 
 from cognite.neat.core._constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
 from cognite.neat.core._data_model._constants import EntityTypes
-from cognite.neat.core._data_model.analysis import RulesAnalysis
+from cognite.neat.core._data_model.analysis import DataModelAnalysis
 from cognite.neat.core._data_model.models.conceptual import (
-    ConceptualClass,
+    Concept,
     ConceptualDataModel,
     ConceptualMetadata,
     ConceptualProperty,
 )
+from cognite.neat.core._data_model.models.conceptual._validation import duplicated_properties
 from cognite.neat.core._data_model.models.data_types import DataType
-from cognite.neat.core._data_model.models.entities import ClassEntity
+from cognite.neat.core._data_model.models.entities import ConceptEntity
 from cognite.neat.core._issues import MultiValueError
 from cognite.neat.core._issues.errors import (
     PropertyDefinitionDuplicatedError,
@@ -27,7 +28,6 @@ from cognite.neat.core._issues.warnings import PropertyDefinitionDuplicatedWarni
 from cognite.neat.core._utils.rdf_ import remove_namespace_from_uri
 
 from ._base import BaseExporter
-from ._validation import duplicated_properties
 
 if sys.version_info >= (3, 11):
     from typing import Self
@@ -36,26 +36,26 @@ else:
 
 
 class GraphExporter(BaseExporter[ConceptualDataModel, Graph], ABC):
-    def export_to_file(self, rules: ConceptualDataModel, filepath: Path) -> None:
-        self.export(rules).serialize(destination=filepath, encoding=self._encoding, newline=self._new_line)
+    def export_to_file(self, data_model: ConceptualDataModel, filepath: Path) -> None:
+        self.export(data_model).serialize(destination=filepath, encoding=self._encoding, newline=self._new_line)
 
 
 class OWLExporter(GraphExporter):
-    """Exports verified information rules to an OWL ontology."""
+    """Exports verified conceptual data model to an OWL ontology."""
 
-    def export(self, rules: ConceptualDataModel) -> Graph:
-        return Ontology.from_rules(rules).as_owl()
+    def export(self, data_model: ConceptualDataModel) -> Graph:
+        return Ontology.from_data_model(data_model).as_owl()
 
     @property
     def description(self) -> str:
-        return "Export verified information model to OWL."
+        return "Export verified conceptual data model to OWL."
 
 
 class SHACLExporter(GraphExporter):
-    """Exports rules to a SHACL graph."""
+    """Exports data_model to a SHACL graph."""
 
-    def export(self, rules: ConceptualDataModel) -> Graph:
-        return Ontology.from_rules(rules).as_shacl()
+    def export(self, data_model: ConceptualDataModel) -> Graph:
+        return Ontology.from_data_model(data_model).as_shacl()
 
     @property
     def description(self) -> str:
@@ -65,8 +65,8 @@ class SHACLExporter(GraphExporter):
 class SemanticDataModelExporter(GraphExporter):
     """Exports verified information model to a semantic data model."""
 
-    def export(self, rules: ConceptualDataModel) -> Graph:
-        return Ontology.from_rules(rules).as_semantic_data_model()
+    def export(self, data_model: ConceptualDataModel) -> Graph:
+        return Ontology.from_data_model(data_model).as_semantic_data_model()
 
     @property
     def description(self) -> str:
@@ -79,7 +79,7 @@ class OntologyModel(BaseModel):
 
 class Ontology(OntologyModel):
     """
-    Represents an ontology. Thi class is used to generate an OWL ontology from a set of transformation rules.
+    Represents an ontology. This class is used to generate an OWL ontology from conceptual data model.
 
     Args:
         properties: A list of OWL properties.
@@ -96,23 +96,23 @@ class Ontology(OntologyModel):
     prefixes: dict[str, Namespace]
 
     @classmethod
-    def from_rules(cls, rules: ConceptualDataModel) -> Self:
+    def from_data_model(cls, data_model: ConceptualDataModel) -> Self:
         """
-        Generates an ontology from a set of transformation rules.
+        Generates an ontology from a set of transformation data_model.
 
         Args:
-            rules: The rules to generate the ontology from.
+            data_model: The data_model to generate the ontology from.
 
         Returns:
             An instance of Ontology.
         """
-        if duplicates := duplicated_properties(rules.properties):
+        if duplicates := duplicated_properties(data_model.properties):
             errors = []
-            for (class_, property_), definitions in duplicates.items():
+            for (concept, property_), definitions in duplicates.items():
                 errors.append(
                     PropertyDefinitionDuplicatedError(
-                        class_,
-                        "class",
+                        concept,
+                        "concept",
                         property_,
                         frozenset({str(definition[1].value_type) for definition in definitions}),
                         tuple(definition[0] for definition in definitions),
@@ -121,35 +121,35 @@ class Ontology(OntologyModel):
                 )
             raise MultiValueError(errors)
 
-        analysis = RulesAnalysis(rules)
-        class_dict = analysis.class_by_suffix()
+        analysis = DataModelAnalysis(data_model)
+        concept_by_suffix = analysis.concept_by_suffix()
         return cls(
             properties=[
-                OWLProperty.from_list_of_properties(definition, rules.metadata.namespace)
+                OWLProperty.from_list_of_properties(definition, data_model.metadata.namespace)
                 for definition in analysis.property_by_id().values()
             ],
             classes=[
-                OWLClass.from_class(definition, rules.metadata.namespace, rules.prefixes)
-                for definition in rules.classes
+                OWLClass.from_concept(definition, data_model.metadata.namespace, data_model.prefixes)
+                for definition in data_model.concepts
             ],
             shapes=[
-                SHACLNodeShape.from_rules(
-                    class_dict[str(class_.suffix)],
+                SHACLNodeShape.from_data_model(
+                    concept_by_suffix[str(concept.suffix)],
                     list(properties.values()),
-                    rules.metadata.namespace,
+                    data_model.metadata.namespace,
                 )
-                for class_, properties in analysis.properties_by_id_by_class().items()
+                for concept, properties in analysis.properties_by_id_by_concept().items()
             ]
             + [
-                SHACLNodeShape.from_rules(
-                    class_,
+                SHACLNodeShape.from_data_model(
+                    concept,
                     [],
-                    rules.metadata.namespace,
+                    data_model.metadata.namespace,
                 )
-                for class_ in class_dict.values()
+                for concept in concept_by_suffix.values()
             ],
-            metadata=OWLMetadata(**rules.metadata.model_dump()),
-            prefixes=rules.prefixes,
+            metadata=OWLMetadata(**data_model.metadata.model_dump()),
+            prefixes=data_model.prefixes,
         )
 
     def as_shacl(self) -> Graph:
@@ -259,7 +259,7 @@ class OWLClass(OntologyModel):
     namespace: Namespace
 
     @classmethod
-    def from_class(cls, definition: ConceptualClass, namespace: Namespace, prefixes: dict) -> Self:
+    def from_concept(cls, definition: Concept, namespace: Namespace, prefixes: dict) -> Self:
         if definition.implements and isinstance(definition.implements, list):
             sub_class_of = []
             for parent_class in definition.implements:
@@ -271,7 +271,7 @@ class OWLClass(OntologyModel):
             sub_class_of = None
 
         return cls(
-            id_=namespace[str(definition.class_.suffix)],
+            id_=namespace[str(definition.concept.suffix)],
             label=definition.name if definition.name else None,
             comment=definition.description,
             sub_class_of=sub_class_of,
@@ -338,7 +338,7 @@ class OWLProperty(OntologyModel):
         property_ids = {definition.property_ for definition in definitions}
         if len(property_ids) != 1:
             raise PropertyDefinitionDuplicatedError(
-                definitions[0].class_,
+                definitions[0].concept,
                 "class",
                 definitions[0].property_,
                 frozenset(property_ids),
@@ -358,11 +358,11 @@ class OWLProperty(OntologyModel):
 
             if isinstance(definition.value_type, DataType):
                 owl_property.range_.add(XSD[definition.value_type.xsd])
-            elif isinstance(definition.value_type, ClassEntity):
+            elif isinstance(definition.value_type, ConceptEntity):
                 owl_property.range_.add(namespace[str(definition.value_type.suffix)])
             else:
                 raise ValueError(f"Value type {definition.value_type.type_} is not supported")
-            owl_property.domain.add(namespace[str(definition.class_.suffix)])
+            owl_property.domain.add(namespace[str(definition.concept.suffix)])
             if definition.name:
                 owl_property.label.add(definition.name)
             if definition.description:
@@ -572,19 +572,19 @@ class SHACLNodeShape(OntologyModel):
         )
 
     @classmethod
-    def from_rules(
+    def from_data_model(
         cls,
-        class_definition: ConceptualClass,
+        concept_definition: Concept,
         property_definitions: list[ConceptualProperty],
         namespace: Namespace,
     ) -> "SHACLNodeShape":
-        if class_definition.implements:
-            parent = [namespace[str(parent.suffix) + "Shape"] for parent in class_definition.implements]
+        if concept_definition.implements:
+            parent = [namespace[str(parent.suffix) + "Shape"] for parent in concept_definition.implements]
         else:
             parent = None
         return cls(
-            id_=namespace[f"{class_definition.class_.suffix!s}Shape"],
-            target_class=namespace[str(class_definition.class_.suffix)],
+            id_=namespace[f"{concept_definition.concept.suffix!s}Shape"],
+            target_class=namespace[str(concept_definition.concept.suffix)],
             parent=parent,
             property_shapes=[SHACLPropertyShape.from_property(prop, namespace) for prop in property_definitions],
             namespace=namespace,
@@ -633,7 +633,7 @@ class SHACLPropertyShape(OntologyModel):
     @classmethod
     def from_property(cls, definition: ConceptualProperty, namespace: Namespace) -> "SHACLPropertyShape":
         # TODO requires PR to fix MultiValueType and UnknownValueType
-        if isinstance(definition.value_type, ClassEntity):
+        if isinstance(definition.value_type, ConceptEntity):
             expected_value_type = namespace[f"{definition.value_type.suffix}Shape"]
         elif isinstance(definition.value_type, DataType):
             expected_value_type = XSD[definition.value_type.xsd]

cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py}

@@ -5,27 +5,28 @@ from typing import Literal, get_args
 
 import yaml
 
-from cognite.neat.core._data_model._shared import VerifiedRules
+from cognite.neat.core._data_model._shared import VerifiedDataModel
 
 from ._base import BaseExporter
 
 
-class YAMLExporter(BaseExporter[VerifiedRules, str]):
-    """Export rules (Information, DMS or Domain) to YAML.
+class YAMLExporter(BaseExporter[VerifiedDataModel, str]):
+    """Export data_model (Information, DMS or Domain) to YAML.
 
     Args:
         files: The number of files to output. Defaults to "single".
-        output: The format to output the rules. Defaults to "yaml".
+        output: The format to output the data_model. Defaults to "yaml".
 
     The following formats are available:
 
-    - "single": A single YAML file will contain the entire rules.
+    - "single": A single YAML file will contain the entire data_model.
 
     .. note::
 
-        YAML files are typically used for storing rules when checked into version control systems, e.g., git-history.
-        The advantage of using YAML files over Excel is that tools like git can show the differences between different
-        versions of the rules.
+        YAML files are typically used for storing data_model when checked into version
+        control systems, e.g., git-history.The advantage of using YAML files over
+        Excel is that tools like git can show the differences between different
+        versions of the data_model.
 
     """
 
@@ -47,32 +48,34 @@ class YAMLExporter(BaseExporter[VerifiedRules, str]):
     def description(self) -> str:
         return "Export verified model to YAML."
 
-    def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
-        """Exports transformation rules to YAML/JSON file(s)."""
+    def export_to_file(self, data_model: VerifiedDataModel, filepath: Path) -> None:
+        """Exports transformation data_model to YAML/JSON file(s)."""
         if self.files == "single":
             if filepath.suffix != f".{self.output}":
                 warnings.warn(f"File extension is not .{self.output}, adding it to the file name", stacklevel=2)
                 filepath = filepath.with_suffix(f".{self.output}")
-            filepath.write_text(self.export(rules), encoding=self._encoding, newline=self._new_line)
+            filepath.write_text(self.export(data_model), encoding=self._encoding, newline=self._new_line)
         else:
             raise NotImplementedError(f"Exporting to {self.files} files is not supported")
 
-    def export(self, rules: VerifiedRules) -> str:
-        """Export rules to YAML (or JSON) format.
+    def export(self, data_model: VerifiedDataModel) -> str:
+        """Export data_model to YAML (or JSON) format.
 
-        This will export the rules to YAML format if the output is set to "yaml" and JSON format if the output is set.
-        All None and Unset values are excluded from the output to keep the output clean, i.e., only the values the user
+        This will export the data_model to YAML format if the output is
+        set to "yaml" and JSON format if the output is set.
+        All None and Unset values are excluded from the output
+        to keep the output clean, i.e., only the values the user
         has set.
 
         Args:
-            rules: The rules to be exported.
+            data_model: The data_model to be exported.
 
         Returns:
-            str: The rules in YAML (or JSON) format.
+            str: The data_model in YAML (or JSON) format.
         """
         # model_dump_json ensures that the output is in JSON format,
         # if we don't do this, we will get Enums and other types that are not serializable to YAML
-        json_output = rules.dump(mode="json", sort=True, exclude_none=True, exclude_unset=True)
+        json_output = data_model.dump(mode="json", sort=True, exclude_none=True, exclude_unset=True)
         if self.output == "json":
             return json.dumps(json_output)
         elif self.output == "yaml":

cognite/neat/core/_data_model/importers/__init__.py

@@ -1,29 +1,29 @@
 from ._base import BaseImporter
-from ._dms2rules import DMSImporter
-from ._dtdl2rules import DTDLImporter
+from ._dict2data_model import DictImporter
+from ._dms2data_model import DMSImporter
+from ._dtdl2data_model import DTDLImporter
 from ._rdf import IMFImporter, InferenceImporter, OWLImporter, SubclassInferenceImporter
-from ._spreadsheet2rules import ExcelImporter
-from ._yaml2rules import YAMLImporter
+from ._spreadsheet2data_model import ExcelImporter
 
 __all__ = [
     "BaseImporter",
     "DMSImporter",
     "DTDLImporter",
+    "DictImporter",
     "ExcelImporter",
     "IMFImporter",
     "InferenceImporter",
     "OWLImporter",
     "SubclassInferenceImporter",
-    "YAMLImporter",
 ]
 
-RulesImporters = (
+DataModelImporters = (
     OWLImporter
     | IMFImporter
     | DMSImporter
     | ExcelImporter
     | DTDLImporter
-    | YAMLImporter
+    | DictImporter
     | InferenceImporter
     | SubclassInferenceImporter
 )
@@ -47,5 +47,5 @@ def _repr_html_() -> str:
 
     return (
         "<strong>Importer</strong> An importer reads data/schema/data model from a source"
-        f" and converts it into Neat's representation of a data model called <em>Rules</em>.<br />{table}"
+        f" and converts it into Neat's representation of a data model.<br />{table}"
     )

cognite/neat/core/_data_model/importers/_base.py

@@ -6,21 +6,24 @@ from typing import TYPE_CHECKING, Any, Generic
 from rdflib import URIRef
 
 from cognite.neat.core._constants import DEFAULT_NAMESPACE
-from cognite.neat.core._data_model._shared import ReadRules, T_InputRules
+from cognite.neat.core._data_model._shared import (
+    ImportedDataModel,
+    T_UnverifiedDataModel,
+)
 from cognite.neat.core._utils.auxiliary import class_html_doc
 
 if TYPE_CHECKING:
     from cognite.neat.core._store._provenance import Agent as ProvenanceAgent
 
 
-class BaseImporter(ABC, Generic[T_InputRules]):
+class BaseImporter(ABC, Generic[T_UnverifiedDataModel]):
     """
-    BaseImporter class which all importers inherit from.
+    BaseImporter class which all data model importers inherit from.
     """
 
     @abstractmethod
-    def to_rules(self) -> ReadRules[T_InputRules]:
-        """Creates `Rules` object from the data for target role."""
+    def to_data_model(self) -> ImportedDataModel[T_UnverifiedDataModel]:
+        """Creates `DataModel` object from the data for target role."""
         raise NotImplementedError()
 
     def _default_metadata(self) -> dict[str, Any]:
@@ -31,7 +34,6 @@ class BaseImporter(ABC, Generic[T_InputRules]):
             creator = getpass.getuser()
 
         return {
-            "schema": "partial",
             "space": "neat",
             "external_id": "NeatImportedDataModel",
             "version": "0.1.0",

cognite/neat/core/_data_model/importers/_base_file_reader.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+from typing import Protocol, TypeVar
+
+from cognite.neat.core._issues import NeatIssue
+from cognite.neat.core._issues.errors import (
+    FileNotAFileError,
+    FileNotFoundNeatError,
+    FileTypeUnexpectedError,
+)
+
+T = TypeVar("T")
+DataT = TypeVar("DataT", covariant=True)
+
+
+class FileReader(Protocol[DataT]):
+    """Protocol for file readers that parse files into structured data."""
+
+    def read_file(self, filepath: Path) -> tuple[DataT, list[NeatIssue]]:
+        """Read a file and return the data with any issues encountered.
+
+        Args:
+            filepath: Path to the file to read
+
+        Returns:
+            Tuple of (parsed_data, issues_list)
+        """
+        ...
+
+
+class BaseFileReader:
+    """Base implementation with common file validation logic."""
+
+    @staticmethod
+    def validate_file(filepath: Path, allowed_extensions: frozenset[str]) -> list[NeatIssue]:
+        """Validate that a file exists and has the correct extension.
+
+        Args:
+            filepath: Path to the file to validate
+            allowed_extensions: Set of allowed file extensions
+
+        Returns:
+            List of issues found during validation, empty if valid
+        """
+        # Check if file exists
+        if not filepath.exists():
+            return [FileNotFoundNeatError(filepath)]
+
+        # Check if it's a file
+        if not filepath.is_file():
+            return [FileNotAFileError(filepath)]
+
+        # Check file extension
+        if filepath.suffix not in allowed_extensions:
+            return [FileTypeUnexpectedError(filepath, allowed_extensions)]
+
+        return []

cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py}

@@ -3,32 +3,62 @@ from typing import Any, cast
 
 import yaml
 
-from cognite.neat.core._data_model._shared import ReadRules, T_InputRules
-from cognite.neat.core._data_model.models import INPUT_RULES_BY_ROLE, RoleTypes
+from cognite.neat.core._data_model._shared import (
+    ImportedDataModel,
+    T_UnverifiedDataModel,
+)
+from cognite.neat.core._data_model.models import (
+    UNVERIFIED_DATA_MODEL_BY_ROLE,
+    RoleTypes,
+)
 from cognite.neat.core._issues import IssueList, MultiValueError, NeatIssue
 from cognite.neat.core._issues.errors import (
     FileMissingRequiredFieldError,
-    FileNotAFileError,
-    FileNotFoundNeatError,
     FileReadError,
-    FileTypeUnexpectedError,
 )
 from cognite.neat.core._issues.warnings import NeatValueWarning
 
 from ._base import BaseImporter
+from ._base_file_reader import BaseFileReader
+
+
+class YAMLReader:
+    """Handles reading and parsing YAML files with error handling."""
+
+    @staticmethod
+    def read_file(
+        filepath: Path, allowed_extensions: frozenset[str] = frozenset([".yaml", ".yml"])
+    ) -> tuple[dict[str, Any], list[NeatIssue]]:
+        """Read a YAML file and return the data with any issues encountered."""
+        issues = BaseFileReader.validate_file(filepath, allowed_extensions)
+        if issues:
+            return {}, issues
+        # Try to load the YAML
+        try:
+            data = yaml.safe_load(filepath.read_text())
+            if not isinstance(data, dict):
+                issues.append(FileReadError(filepath, "YAML content is not a dictionary"))
+                return {}, issues
+            return data, issues
+        except yaml.YAMLError as exc:
+            return {}, [FileReadError(filepath, f"Invalid YAML: {exc!s}")]
+        except Exception as exc:
+            return {}, [FileReadError(filepath, f"Error reading file: {exc!s}")]
 
 
-class YAMLImporter(BaseImporter[T_InputRules]):
-    """Imports the rules from a YAML file.
+class DictImporter(BaseImporter[T_UnverifiedDataModel]):
+    """Imports the data model from a YAML file.
 
     Args:
         raw_data: The raw data to be imported.
 
     .. note::
 
-        YAML files are typically used for storing rules when checked into version control systems, e.g., git-history.
-        The advantage of using YAML files over Excel is that tools like git can show the differences between different
-        versions of the rules.
+        YAML files are typically used for storing data model when checked into version
+        control systems, e.g., git-history.
+        The advantage of using YAML files over Excel is that tools like git can
+        show the differences between different
+        versions of the data model.
 
     """
 
@@ -49,21 +79,16 @@ class YAMLImporter(BaseImporter[T_InputRules]):
         return f"YAML file {self._source_name} read as unverified data model"
 
     @classmethod
-    def from_file(cls, filepath: Path, source_name: str = "Unknown") -> "YAMLImporter":
-        if not filepath.exists():
-            return cls({}, [FileNotFoundNeatError(filepath)])
-        elif not filepath.is_file():
-            return cls({}, [FileNotAFileError(filepath)])
-        elif filepath.suffix not in [".yaml", ".yml"]:
-            return cls({}, [FileTypeUnexpectedError(filepath, frozenset([".yaml", ".yml"]))])
-        try:
-            data = yaml.safe_load(filepath.read_text())
-        except yaml.YAMLError as exc:
-            return cls({}, [FileReadError(filepath, f"Invalid YAML: {exc!s}")])
+    def from_yaml_file(cls, filepath: Path, source_name: str = "Unknown") -> "DictImporter":
+        """Create a DictImporter from a YAML file."""
+        data, issues = YAMLReader.read_file(filepath)
+
+        if issues:
+            return cls({}, issues)
 
         return cls(data, filepaths=[filepath], source_name=source_name)
 
-    def to_rules(self) -> ReadRules[T_InputRules]:
+    def to_data_model(self) -> ImportedDataModel[T_UnverifiedDataModel]:
         if self._read_issues.has_errors or not self.raw_data:
             self._read_issues.trigger_warnings()
             raise MultiValueError(self._read_issues.errors)
@@ -95,12 +120,12 @@ class YAMLImporter(BaseImporter[T_InputRules]):
 
         role_input = RoleTypes(metadata["role"])
         role_enum = RoleTypes(role_input)
-        rules_cls = INPUT_RULES_BY_ROLE[role_enum]
+        data_model_cls = UNVERIFIED_DATA_MODEL_BY_ROLE[role_enum]
 
-        rules = cast(T_InputRules, rules_cls.load(self.raw_data))
+        data_model = cast(T_UnverifiedDataModel, data_model_cls.load(self.raw_data))
 
         issue_list.trigger_warnings()
         if self._read_issues.has_errors:
             raise MultiValueError(self._read_issues.errors)
 
-        return ReadRules[T_InputRules](rules, {})
+        return ImportedDataModel[T_UnverifiedDataModel](data_model, {})