cognite-neat 0.88.3__tar.gz → 0.89.0__tar.gz

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.88.3
+Version: 0.89.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.89.0/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.89.0"

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/constants.py

@@ -30,3 +30,6 @@ def get_default_prefixes() -> dict[str, Namespace]:
 DEFAULT_URI = ""
 
 DEFAULT_DOCS_URL = "https://cognite-neat.readthedocs-hosted.com/en/latest/"
+
+DMS_CONTAINER_PROPERTY_SIZE_LIMIT = 100
+DMS_VIEW_CONTAINER_SIZE_LIMIT = 10

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/graph/extractors/_mock_graph_generator.py

@@ -17,6 +17,7 @@ from cognite.neat.rules.models import DMSRules, InformationRules
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
 from cognite.neat.rules.models.information import InformationProperty
+from cognite.neat.rules.transformers import DMSToInformation
 from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 from ._base import BaseExtractor
@@ -42,7 +43,7 @@ class MockGraphGenerator(BaseExtractor):
         allow_isolated_classes: bool = True,
     ):
         if isinstance(rules, DMSRules):
-            self.rules = rules.as_information_rules()
+            self.rules = DMSToInformation().transform(rules).rules
         elif isinstance(rules, InformationRules):
             self.rules = rules
         else:

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/issues/_base.py

@@ -50,6 +50,7 @@ ResourceType: TypeAlias = (
         "data model",
         "edge",
         "node",
+        "enum collection",
         "unknown",
     ]
     # String to handle all unknown types in different importers.
@@ -339,7 +340,7 @@ class DefaultWarning(NeatWarning):
 T_NeatIssue = TypeVar("T_NeatIssue", bound=NeatIssue)
 
 
-class NeatIssueList(UserList[T_NeatIssue], ABC):
+class NeatIssueList(UserList[T_NeatIssue], Sequence[T_NeatIssue], ABC):
     """This is a generic list of NeatIssues."""
 
     def __init__(self, issues: Sequence[T_NeatIssue] | None = None, title: str | None = None):

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/issues/errors/__init__.py

@@ -9,7 +9,7 @@ from ._external import (
     FileTypeUnexpectedError,
     NeatYamlError,
 )
-from ._general import NeatImportError, NeatValueError, RegexViolationError
+from ._general import NeatImportError, NeatTypeError, NeatValueError, RegexViolationError
 from ._properties import (
     PropertyDefinitionDuplicatedError,
     PropertyDefinitionError,
@@ -67,6 +67,7 @@ __all__ = [
     "DefaultPydanticError",
     "PropertyMappingDuplicatedError",
     "RowError",
+    "NeatTypeError",
 ]
 
 _NEAT_ERRORS_BY_NAME = {error.__name__: error for error in _get_subclasses(NeatError, include_base=True)}

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/issues/errors/_general.py

@@ -10,6 +10,13 @@ class NeatValueError(NeatError, ValueError):
     raw_message: str
 
 
+@dataclass(frozen=True)
+class NeatTypeError(NeatError, TypeError):
+    """{raw_message}"""
+
+    raw_message: str
+
+
 @dataclass(frozen=True)
 class RegexViolationError(NeatError, ValueError):
     """Value, {value} failed regex, {regex}, validation. Make sure that the name follows the regex pattern."""

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/issues/warnings/_models.py

@@ -4,8 +4,8 @@ from typing import ClassVar
 
 from cognite.client.data_classes.data_modeling import ViewId
 
+from cognite.neat.constants import DMS_VIEW_CONTAINER_SIZE_LIMIT
 from cognite.neat.issues import NeatWarning
-from cognite.neat.rules.models._constants import DMS_VIEW_CONTAINER_SIZE_LIMIT
 
 _BASE_URL = "https://cognite-neat.readthedocs-hosted.com/en/latest/data-modeling-principles.html"
 

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/issues/warnings/user_modeling.py

@@ -7,7 +7,7 @@ from dataclasses import dataclass
 
 from cognite.client.data_classes.data_modeling import ContainerId, ViewId
 
-from cognite.neat.rules.models._constants import DMS_CONTAINER_PROPERTY_SIZE_LIMIT
+from cognite.neat.constants import DMS_CONTAINER_PROPERTY_SIZE_LIMIT
 
 from ._models import UserModelingWarning
 

cognite_neat-0.89.0/cognite/neat/rules/_shared.py

@@ -0,0 +1,61 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Generic, TypeAlias, TypeVar
+
+from cognite.neat.issues import IssueList
+from cognite.neat.rules.models import (
+    AssetRules,
+    DMSRules,
+    DomainRules,
+    InformationRules,
+)
+from cognite.neat.rules.models.asset._rules_input import AssetInputRules
+from cognite.neat.rules.models.dms._rules_input import DMSInputRules
+from cognite.neat.rules.models.information._rules_input import InformationInputRules
+
+VerifiedRules: TypeAlias = DomainRules | InformationRules | DMSRules | AssetRules
+InputRules: TypeAlias = AssetInputRules | DMSInputRules | InformationInputRules
+Rules: TypeAlias = (
+    AssetInputRules | DMSInputRules | InformationInputRules | DomainRules | InformationRules | DMSRules | AssetRules
+)
+T_Rules = TypeVar("T_Rules", bound=Rules)
+T_VerifiedRules = TypeVar("T_VerifiedRules", bound=VerifiedRules)
+T_InputRules = TypeVar("T_InputRules", bound=InputRules)
+
+
+@dataclass
+class OutRules(Generic[T_Rules], ABC):
+    """This is a base class for all rule states."""
+
+    @abstractmethod
+    def get_rules(self) -> T_Rules | None:
+        """Get the rules from the state."""
+        raise NotImplementedError()
+
+
+@dataclass
+class JustRules(OutRules[T_Rules]):
+    """This represents a rule that exists"""
+
+    rules: T_Rules
+
+    def get_rules(self) -> T_Rules:
+        return self.rules
+
+
+@dataclass
+class MaybeRules(OutRules[T_Rules]):
+    """This represents a rule that may or may not exist"""
+
+    rules: T_Rules | None
+    issues: IssueList
+
+    def get_rules(self) -> T_Rules | None:
+        return self.rules
+
+
+@dataclass
+class ReadRules(MaybeRules[T_Rules]):
+    """This represents a rule that does not exist"""
+
+    read_context: dict[str, Any]

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/rules/analysis/_base.py

@@ -9,7 +9,7 @@ from typing import Generic, TypeVar
 import pandas as pd
 from pydantic import BaseModel
 
-from cognite.neat.rules.models._base import BaseRules
+from cognite.neat.rules.models._base_rules import BaseRules
 from cognite.neat.rules.models._rdfpath import RDFPath
 from cognite.neat.rules.models.entities import (
     ClassEntity,

cognite_neat-0.89.0/cognite/neat/rules/exporters/_base.py

@@ -0,0 +1,40 @@
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Generic, TypeVar
+
+from cognite.client import CogniteClient
+
+from cognite.neat.rules._shared import T_VerifiedRules
+from cognite.neat.utils.auxiliary import class_html_doc
+from cognite.neat.utils.upload import UploadResult, UploadResultList
+
+T_Export = TypeVar("T_Export")
+
+
+class BaseExporter(ABC, Generic[T_VerifiedRules, T_Export]):
+    _new_line = "\n"
+    _encoding = "utf-8"
+
+    @abstractmethod
+    def export_to_file(self, rules: T_VerifiedRules, filepath: Path) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def export(self, rules: T_VerifiedRules) -> T_Export:
+        raise NotImplementedError
+
+    @classmethod
+    def _repr_html_(cls) -> str:
+        return class_html_doc(cls, include_factory_methods=False)
+
+
+class CDFExporter(BaseExporter[T_VerifiedRules, T_Export]):
+    @abstractmethod
+    def export_to_cdf_iterable(
+        self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False
+    ) -> Iterable[UploadResult]:
+        raise NotImplementedError
+
+    def export_to_cdf(self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False) -> UploadResultList:
+        return UploadResultList(self.export_to_cdf_iterable(rules, client, dry_run))

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/rules/exporters/_rules2dms.py

@@ -20,8 +20,6 @@ from cognite.neat.issues.warnings import (
     PrincipleOneModelOneSpaceWarning,
     ResourceRetrievalWarning,
 )
-from cognite.neat.rules._shared import VerifiedRules
-from cognite.neat.rules.models import InformationRules
 from cognite.neat.rules.models.dms import DMSRules, DMSSchema, PipelineSchema
 from cognite.neat.utils.cdf.loaders import (
     ContainerLoader,
@@ -41,7 +39,7 @@ from ._base import CDFExporter
 Component: TypeAlias = Literal["all", "spaces", "data_models", "views", "containers", "node_types"]
 
 
-class DMSExporter(CDFExporter[DMSSchema]):
+class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
     """Export rules to Cognite Data Fusion's Data Model Storage (DMS) service.
 
     Args:
@@ -82,7 +80,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
         self.suppress_warnings = suppress_warnings
         self._schema: DMSSchema | None = None
 
-    def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
+    def export_to_file(self, rules: DMSRules, filepath: Path) -> None:
         """Export the rules to a file(s).
 
         If the file is a directory, the components will be exported to separate files, otherwise they will be
@@ -97,12 +95,12 @@ class DMSExporter(CDFExporter[DMSSchema]):
         else:
             self._export_to_zip_file(filepath, rules)
 
-    def _export_to_directory(self, directory: Path, rules: VerifiedRules) -> None:
+    def _export_to_directory(self, directory: Path, rules: DMSRules) -> None:
         schema = self.export(rules)
         exclude = self._create_exclude_set()
         schema.to_directory(directory, exclude=exclude, new_line=self._new_line, encoding=self._encoding)
 
-    def _export_to_zip_file(self, filepath: Path, rules: VerifiedRules) -> None:
+    def _export_to_zip_file(self, filepath: Path, rules: DMSRules) -> None:
         if filepath.suffix not in {".zip"}:
             warnings.warn("File extension is not .zip, adding it to the file name", stacklevel=2)
             filepath = filepath.with_suffix(".zip")
@@ -117,18 +115,10 @@ class DMSExporter(CDFExporter[DMSSchema]):
             exclude = {"spaces", "data_models", "views", "containers", "node_types"} - self.export_components
         return exclude
 
-    def export(self, rules: VerifiedRules) -> DMSSchema:
-        if isinstance(rules, DMSRules):
-            dms_rules = rules
-        elif isinstance(rules, InformationRules):
-            dms_rules = rules.as_dms_rules()
-        else:
-            raise ValueError(f"{type(rules).__name__} cannot be exported to DMS")
-        return dms_rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
+    def export(self, rules: DMSRules) -> DMSSchema:
+        return rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
 
-    def delete_from_cdf(
-        self, rules: VerifiedRules, client: CogniteClient, dry_run: bool = False
-    ) -> Iterable[UploadResult]:
+    def delete_from_cdf(self, rules: DMSRules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
         to_export = self._prepare_exporters(rules, client)
 
         # we need to reverse order in which we are picking up the items to delete
@@ -172,7 +162,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
             )
 
     def export_to_cdf_iterable(
-        self, rules: VerifiedRules, client: CogniteClient, dry_run: bool = False
+        self, rules: DMSRules, client: CogniteClient, dry_run: bool = False
     ) -> Iterable[UploadResult]:
         to_export = self._prepare_exporters(rules, client)
 

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/rules/exporters/_rules2excel.py

@@ -16,7 +16,6 @@ from cognite.neat.rules._shared import VerifiedRules
 from cognite.neat.rules.models import (
     DataModelType,
     ExtensionCategory,
-    RoleTypes,
     SchemaCompleteness,
     SheetEntity,
 )
@@ -27,7 +26,7 @@ from cognite.neat.rules.models.information import InformationMetadata
 from ._base import BaseExporter
 
 
-class ExcelExporter(BaseExporter[Workbook]):
+class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
     """Export rules to Excel.
 
     Args:
@@ -68,16 +67,14 @@ class ExcelExporter(BaseExporter[Workbook]):
         "Classes": "Definition of Classes",
         "Views": "Definition of Views",
         "Containers": "Definition of Containers",
+        "Nodes": "Definition of Nodes",
+        "Enum": "Definition of Enum Collections",
     }
     style_options = get_args(Style)
     dump_options = get_args(DumpOptions)
 
     def __init__(
-        self,
-        styling: Style = "default",
-        output_role: RoleTypes | None = None,
-        dump_as: DumpOptions = "user",
-        new_model_id: tuple[str, str] | None = None,
+        self, styling: Style = "default", dump_as: DumpOptions = "user", new_model_id: tuple[str, str] | None = None
     ):
         if styling not in self.style_options:
             raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
@@ -85,7 +82,6 @@ class ExcelExporter(BaseExporter[Workbook]):
             raise ValueError(f"Invalid dump_as: {dump_as}. Valid options are {self.dump_options}")
         self.styling = styling
         self._styling_level = self.style_options.index(styling)
-        self.output_role = output_role
         self.new_model_id = new_model_id
         self.dump_as = dump_as
 
@@ -99,7 +95,6 @@ class ExcelExporter(BaseExporter[Workbook]):
         return None
 
     def export(self, rules: VerifiedRules) -> Workbook:
-        rules = self._convert_to_output_role(rules, self.output_role)
         workbook = Workbook()
         # Remove default sheet named "Sheet"
         workbook.remove(workbook["Sheet"])
@@ -279,9 +274,7 @@ class _MetadataCreator:
 
         new_metadata = self._create_new_info(now)
         if isinstance(metadata, DMSMetadata):
-            from cognite.neat.rules.models.information._converter import (
-                _InformationRulesConverter,
-            )
+            from cognite.neat.rules.transformers._converters import _InformationRulesConverter
 
             output_metadata: DMSMetadata | InformationMetadata = _InformationRulesConverter._convert_metadata_to_dms(
                 new_metadata

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/rules/exporters/_rules2ontology.py

@@ -15,7 +15,6 @@ from cognite.neat.issues.errors import (
 )
 from cognite.neat.issues.warnings import PropertyDefinitionDuplicatedWarning
 from cognite.neat.rules.analysis import InformationAnalysis
-from cognite.neat.rules.models import DMSRules
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
 from cognite.neat.rules.models.information import (
@@ -34,32 +33,30 @@ if sys.version_info >= (3, 11):
 else:
     from typing_extensions import Self
 
-from cognite.neat.rules._shared import VerifiedRules
 
-
-class GraphExporter(BaseExporter[Graph], ABC):
-    def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
+class GraphExporter(BaseExporter[InformationRules, Graph], ABC):
+    def export_to_file(self, rules: InformationRules, filepath: Path) -> None:
         self.export(rules).serialize(destination=filepath, encoding=self._encoding, newline=self._new_line)
 
 
 class OWLExporter(GraphExporter):
-    """Exports rules to an OWL ontology."""
+    """Exports verified information rules to an OWL ontology."""
 
-    def export(self, rules: VerifiedRules) -> Graph:
+    def export(self, rules: InformationRules) -> Graph:
         return Ontology.from_rules(rules).as_owl()
 
 
 class SHACLExporter(GraphExporter):
     """Exports rules to a SHACL graph."""
 
-    def export(self, rules: VerifiedRules) -> Graph:
+    def export(self, rules: InformationRules) -> Graph:
         return Ontology.from_rules(rules).as_shacl()
 
 
 class SemanticDataModelExporter(GraphExporter):
-    """Exports rules to a semantic data model."""
+    """Exports verified information rules to a semantic data model."""
 
-    def export(self, rules: VerifiedRules) -> Graph:
+    def export(self, rules: InformationRules) -> Graph:
         return Ontology.from_rules(rules).as_semantic_data_model()
 
 
@@ -86,23 +83,16 @@ class Ontology(OntologyModel):
     prefixes: dict[str, Namespace]
 
     @classmethod
-    def from_rules(cls, input_rules: VerifiedRules) -> Self:
+    def from_rules(cls, rules: InformationRules) -> Self:
         """
         Generates an ontology from a set of transformation rules.
 
         Args:
-            input_rules: The rules to generate the ontology from.
+            rules: The rules to generate the ontology from.
 
         Returns:
             An instance of Ontology.
         """
-        if isinstance(input_rules, InformationRules):
-            rules = input_rules
-        elif isinstance(input_rules, DMSRules):
-            rules = input_rules.as_information_rules()
-        else:
-            raise ValueError(f"{type(input_rules).__name__} cannot be exported to Ontology")
-
         if duplicates := duplicated_properties(rules.properties):
             errors = []
             for (class_, property_), definitions in duplicates.items():

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/rules/exporters/_rules2yaml.py

@@ -6,13 +6,12 @@ from typing import Literal, get_args
 import yaml
 
 from cognite.neat.rules._shared import VerifiedRules
-from cognite.neat.rules.models import RoleTypes
 
 from ._base import BaseExporter
 
 
-class YAMLExporter(BaseExporter[str]):
-    """Export rules to YAML.
+class YAMLExporter(BaseExporter[VerifiedRules, str]):
+    """Export rules (Information, DMS or Domain) to YAML.
 
     Args:
         files: The number of files to output. Defaults to "single".
@@ -38,14 +37,13 @@ class YAMLExporter(BaseExporter[str]):
     file_option = get_args(Files)
     format_option = get_args(Format)
 
-    def __init__(self, files: Files = "single", output: Format = "yaml", output_role: RoleTypes | None = None):
+    def __init__(self, files: Files = "single", output: Format = "yaml"):
         if files not in self.file_option:
             raise ValueError(f"Invalid files: {files}. Valid options are {self.file_option}")
         if output not in self.format_option:
             raise ValueError(f"Invalid output: {output}. Valid options are {self.format_option}")
         self.files = files
         self.output = output
-        self.output_role = output_role
 
     def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
         """Exports transformation rules to YAML/JSON file(s)."""
@@ -66,7 +64,6 @@ class YAMLExporter(BaseExporter[str]):
         Returns:
             str: The rules in YAML (or JSON) format.
         """
-        rules = self._convert_to_output_role(rules, self.output_role)
         # model_dump_json ensures that the output is in JSON format,
         # if we don't do this, we will get Enums and other types that are not serializable to YAML
         json_output = rules.dump(mode="json")

{cognite_neat-0.88.3 → cognite_neat-0.89.0}/cognite/neat/rules/importers/_base.py

@@ -4,72 +4,27 @@ from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from contextlib import contextmanager
 from datetime import datetime
-from typing import Any, Literal, overload
+from typing import Any, Generic, Literal
 
 from pydantic import ValidationError
 from rdflib import Namespace
 
 from cognite.neat.issues import IssueList, NeatError, NeatWarning
-from cognite.neat.rules._shared import VerifiedRules
-from cognite.neat.rules.models import AssetRules, DMSRules, InformationRules, RoleTypes
+from cognite.neat.rules._shared import ReadRules, T_InputRules
 from cognite.neat.utils.auxiliary import class_html_doc
 
 
-class BaseImporter(ABC):
+class BaseImporter(ABC, Generic[T_InputRules]):
     """
     BaseImporter class which all importers inherit from.
     """
 
-    @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
-
-    @overload
-    def to_rules(
-        self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
-    ) -> tuple[VerifiedRules | None, IssueList]: ...
-
     @abstractmethod
-    def to_rules(
-        self,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-    ) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
-        """
-        Creates `Rules` object from the data for target role.
-        """
-        ...
-
-    @classmethod
-    def _to_output(
-        cls,
-        rules: VerifiedRules,
-        issues: IssueList,
-        errors: Literal["raise", "continue"] = "continue",
-        role: RoleTypes | None = None,
-    ) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
-        """Converts the rules to the output format."""
-
-        if rules.metadata.role is role or role is None:
-            output = rules
-        elif isinstance(rules, DMSRules) or isinstance(rules, AssetRules) and role is RoleTypes.information:
-            output = rules.as_information_rules()
-        elif isinstance(rules, InformationRules) or isinstance(rules, AssetRules) and role is RoleTypes.dms:
-            output = rules.as_dms_rules()
-        else:
-            raise NotImplementedError(f"Role {role} is not supported for {type(rules).__name__} rules")
-
-        if errors == "raise":
-            return output
-        else:
-            return output, issues
-
-    @classmethod
-    def _return_or_raise(cls, issue_list: IssueList, errors: Literal["raise", "continue"]) -> tuple[None, IssueList]:
-        if errors == "raise":
-            raise issue_list.as_errors()
-        return None, issue_list
+    def to_rules(self) -> ReadRules[T_InputRules]:
+        """Creates `Rules` object from the data for target role."""
+        raise NotImplementedError()
 
-    def _default_metadata(self):
+    def _default_metadata(self) -> dict[str, Any]:
         return {
             "prefix": "neat",
             "schema": "partial",