cognite-neat 0.92.3__py3-none-any.whl → 0.93.0__py3-none-any.whl

cognite/neat/{rules → _rules}/models/entities/_single_value.py

@@ -22,7 +22,7 @@ from pydantic import (
     model_validator,
 )
 
-from cognite.neat.utils.text import replace_non_alphanumeric_with_underscore
+from cognite.neat._utils.text import replace_non_alphanumeric_with_underscore
 
 if sys.version_info >= (3, 11):
     from enum import StrEnum
@@ -31,12 +31,15 @@ else:
     from backports.strenum import StrEnum
     from typing_extensions import Self
 
-from cognite.neat.utils.regex_patterns import SPLIT_ON_COMMA_PATTERN, SPLIT_ON_EQUAL_PATTERN
+from cognite.neat._rules._constants import (
+    ENTITY_PATTERN,
+    SPLIT_ON_COMMA_PATTERN,
+    SPLIT_ON_EQUAL_PATTERN,
+    EntityTypes,
+)
 
 from ._constants import (
     _PARSE,
-    ENTITY_PATTERN,
-    EntityTypes,
     Undefined,
     Unknown,
     _UndefinedType,

cognite/neat/{rules → _rules}/models/information/_rules.py

@@ -8,38 +8,42 @@ from pydantic import Field, field_serializer, field_validator, model_validator
 from pydantic_core.core_schema import SerializationInfo
 from rdflib import Namespace
 
-from cognite.neat.constants import get_default_prefixes
-from cognite.neat.issues.errors import PropertyDefinitionError
-from cognite.neat.rules.models._base_rules import (
+from cognite.neat._constants import get_default_prefixes
+from cognite.neat._issues.errors import NeatValueError, PropertyDefinitionError
+from cognite.neat._rules._constants import EntityTypes
+from cognite.neat._rules.models._base_rules import (
     BaseMetadata,
     BaseRules,
+    ClassRef,
     DataModelType,
     ExtensionCategory,
     ExtensionCategoryType,
     MatchType,
+    PropertyRef,
     RoleTypes,
     SchemaCompleteness,
     SheetList,
     SheetRow,
 )
-from cognite.neat.rules.models._rdfpath import (
+from cognite.neat._rules.models._rdfpath import (
     RDFPath,
     TransformationRuleType,
     parse_rule,
 )
-from cognite.neat.rules.models._types import (
+from cognite.neat._rules.models._types import (
+    ClassEntityType,
+    InformationPropertyType,
+    MultiValueTypeType,
     NamespaceType,
     PrefixType,
-    PropertyType,
     StrListType,
     VersionType,
 )
-from cognite.neat.rules.models.data_types import DataType
-from cognite.neat.rules.models.entities import (
+from cognite.neat._rules.models.data_types import DataType
+from cognite.neat._rules.models.entities import (
     ClassEntity,
     ClassEntityList,
     Entity,
-    EntityTypes,
     MultiValueTypeInfo,
     ReferenceEntity,
     Undefined,
@@ -48,7 +52,7 @@ from cognite.neat.rules.models.entities import (
 )
 
 if TYPE_CHECKING:
-    from cognite.neat.rules.models import DMSRules
+    from cognite.neat._rules.models import DMSRules
 
 
 if sys.version_info >= (3, 11):
@@ -135,7 +139,7 @@ class InformationClass(SheetRow):
         match_type: The match type of the resource being described and the source entity.
     """
 
-    class_: ClassEntity = Field(alias="Class")
+    class_: ClassEntityType = Field(alias="Class")
     name: str | None = Field(alias="Name", default=None)
     description: str | None = Field(alias="Description", default=None)
     parent: ClassEntityList | None = Field(alias="Parent Class", default=None)
@@ -169,6 +173,9 @@ class InformationClass(SheetRow):
             )
         return ",".join(str(value) for value in value)
 
+    def as_reference(self) -> ClassRef:
+        return ClassRef(Class=self.class_)
+
 
 class InformationProperty(SheetRow):
     """
@@ -189,11 +196,11 @@ class InformationProperty(SheetRow):
               knowledge graph. Defaults to None (no transformation)
     """
 
-    class_: ClassEntity = Field(alias="Class")
-    property_: PropertyType = Field(alias="Property")
+    class_: ClassEntityType = Field(alias="Class")
+    property_: InformationPropertyType = Field(alias="Property")
     name: str | None = Field(alias="Name", default=None)
     description: str | None = Field(alias="Description", default=None)
-    value_type: DataType | ClassEntity | MultiValueTypeInfo | UnknownEntity = Field(
+    value_type: DataType | ClassEntityType | MultiValueTypeType | UnknownEntity = Field(
         alias="Value Type", union_mode="left_to_right"
     )
     min_count: int | None = Field(alias="Min Count", default=None)
@@ -201,7 +208,7 @@ class InformationProperty(SheetRow):
     default: Any | None = Field(alias="Default", default=None)
     reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
     match_type: MatchType | None = Field(alias="Match Type", default=None)
-    transformation: str | RDFPath | None = Field(alias="Transformation", default=None)
+    transformation: RDFPath | None = Field(alias="Transformation", default=None)
     comment: str | None = Field(alias="Comment", default=None)
     inherited: bool = Field(
         default=False,
@@ -219,12 +226,14 @@ class InformationProperty(SheetRow):
             return float("inf")
         return value
 
-    @model_validator(mode="after")
-    def generate_valid_transformation(self):
-        # TODO: Currently only supporting RDFpath
-        if self.transformation:
-            self.transformation = parse_rule(self.transformation, TransformationRuleType.rdfpath)
-        return self
+    @field_validator("transformation", mode="before")
+    def generate_rdfpath(cls, value: str | RDFPath | None) -> RDFPath | None:
+        if value is None or isinstance(value, RDFPath):
+            return value
+        elif isinstance(value, str):
+            return parse_rule(value, TransformationRuleType.rdfpath)
+        else:
+            raise NeatValueError(f"Invalid RDF Path: {value!s}")
 
     @model_validator(mode="after")
     def set_default_as_list(self):
@@ -312,6 +321,9 @@ class InformationProperty(SheetRow):
             isinstance(self.max_count, int | float) and self.max_count > 1
         )
 
+    def as_reference(self) -> PropertyRef:
+        return PropertyRef(Class=self.class_, Property=self.property_)
+
 
 class InformationRules(BaseRules):
     metadata: InformationMetadata = Field(alias="Metadata")
@@ -365,6 +377,6 @@ class InformationRules(BaseRules):
         return self
 
     def as_dms_rules(self) -> "DMSRules":
-        from cognite.neat.rules.transformers._converters import _InformationRulesConverter
+        from cognite.neat._rules.transformers._converters import _InformationRulesConverter
 
         return _InformationRulesConverter(self).as_dms_rules()

cognite/neat/{rules → _rules}/models/information/_rules_input.py

@@ -4,9 +4,9 @@ from typing import Any, Literal
 
 from rdflib import Namespace
 
-from cognite.neat.rules.models._base_input import InputComponent, InputRules
-from cognite.neat.rules.models.data_types import DataType
-from cognite.neat.rules.models.entities import (
+from cognite.neat._rules.models._base_input import InputComponent, InputRules
+from cognite.neat._rules.models.data_types import DataType
+from cognite.neat._rules.models.entities import (
     ClassEntity,
     MultiValueTypeInfo,
     UnknownEntity,

cognite/neat/{rules → _rules}/models/information/_validation.py

@@ -2,11 +2,12 @@ import itertools
 from collections import Counter
 from typing import cast
 
-from cognite.neat.issues import IssueList
-from cognite.neat.issues.errors import NeatValueError, ResourceNotDefinedError
-from cognite.neat.rules.models._base_rules import DataModelType, SchemaCompleteness
-from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, UnknownEntity
-from cognite.neat.utils.rdf_ import get_inheritance_path
+from cognite.neat._issues import IssueList
+from cognite.neat._issues.errors import NeatValueError, ResourceNotDefinedError
+from cognite.neat._rules._constants import EntityTypes
+from cognite.neat._rules.models._base_rules import DataModelType, SchemaCompleteness
+from cognite.neat._rules.models.entities import ClassEntity, UnknownEntity
+from cognite.neat._utils.rdf_ import get_inheritance_path
 
 from ._rules import InformationRules
 

cognite/neat/_rules/models/mapping/__init__.py

@@ -0,0 +1,4 @@
+from ._base import Mapping, MappingList, RuleMapping
+from ._classic2core import create_classic_to_core_mapping
+
+__all__ = ["RuleMapping", "Mapping", "MappingList", "create_classic_to_core_mapping"]

cognite/neat/_rules/models/mapping/_base.py

@@ -0,0 +1,131 @@
+from collections import Counter, defaultdict
+from collections.abc import Iterator, MutableSequence, Sequence
+from pathlib import Path
+from typing import Any, Generic, SupportsIndex, TypeVar, cast, get_args, overload
+
+import pandas as pd
+from pydantic import BaseModel, GetCoreSchemaHandler, field_validator
+from pydantic_core import core_schema
+from pydantic_core.core_schema import ValidationInfo
+
+from cognite.neat._issues.errors import NeatValueError
+from cognite.neat._rules.models._base_rules import ClassRef, PropertyRef
+from cognite.neat._rules.models.entities import ClassEntity, Undefined
+
+T_Mapping = TypeVar("T_Mapping", bound=ClassRef | PropertyRef)
+
+
+class Mapping(BaseModel, Generic[T_Mapping]):
+    source: T_Mapping
+    destination: T_Mapping
+
+
+class MappingList(list, MutableSequence[Mapping[T_Mapping]]):
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
+        if args := get_args(source):
+            item_type = args[0]
+        else:
+            # Someone use SheetList without specifying the type
+            raise TypeError("SheetList must be used with a type argument, e.g., SheetList[InformationProperty]")
+
+        instance_schema = core_schema.is_instance_schema(cls)
+        sequence_row_schema = handler.generate_schema(Sequence[item_type])  # type: ignore[valid-type]
+
+        non_instance_schema = core_schema.no_info_after_validator_function(MappingList, sequence_row_schema)
+        return core_schema.union_schema([instance_schema, non_instance_schema])
+
+    def as_destination_by_source(self) -> dict[T_Mapping, T_Mapping]:
+        return {mapping.source: mapping.destination for mapping in self}
+
+    def to_pandas(self, drop_na_columns: bool = True, include: list[str] | None = None) -> pd.DataFrame:
+        """Converts ResourceDict to pandas DataFrame."""
+        df = pd.DataFrame([entity.model_dump() for entity in self])
+        if drop_na_columns:
+            df = df.dropna(axis=1, how="all")
+        if include is not None:
+            df = df[include]
+        return df
+
+    def _repr_html_(self) -> str:
+        """Returns HTML representation of ResourceDict."""
+        return self.to_pandas(drop_na_columns=True)._repr_html_()  # type: ignore[operator]
+
+    # Implemented to get correct type hints
+    def __iter__(self) -> Iterator[Mapping[T_Mapping]]:
+        return super().__iter__()
+
+    @overload
+    def __getitem__(self, index: SupportsIndex) -> Mapping[T_Mapping]: ...
+
+    @overload
+    def __getitem__(self, index: slice) -> "MappingList[T_Mapping]": ...
+
+    def __getitem__(self, index: SupportsIndex | slice, /) -> "Mapping[T_Mapping] | MappingList[T_Mapping]":
+        if isinstance(index, slice):
+            return MappingList[T_Mapping](super().__getitem__(index))
+        return super().__getitem__(index)
+
+
+class RuleMapping(BaseModel):
+    properties: MappingList[PropertyRef]
+    classes: MappingList[ClassRef]
+
+    @field_validator("properties", "classes", mode="before")
+    def as_mapping_list(cls, value: Sequence[Any], info: ValidationInfo) -> Any:
+        if isinstance(value, Sequence) and not isinstance(value, MappingList):
+            annotation = cast(type, cls.model_fields[info.field_name].annotation)  # type: ignore[index]
+            ref_cls = get_args(annotation)[0]
+            return annotation([Mapping[ref_cls].model_validate(item) for item in value])  # type: ignore[valid-type, index]
+        return value
+
+    @classmethod
+    def load_spreadsheet(
+        cls, path: str | Path, source_prefix: str | None = None, destination_prefix: str | None = None
+    ) -> "RuleMapping":
+        """Loads mapping from Excel spreadsheet.
+
+        This method expects four columns in the spreadsheet. The first two columns are the source class and
+        property, and the last two columns are the destination class and property. The method will create
+        a mapping for each row in the spreadsheet.
+
+        The class mapping will be inferred from the property mappings. If a source class has multiple
+        destination classes, the most common destination class will be used.
+
+        Args:
+            path: Path to Excel spreadsheet.
+            source_prefix: Default prefix for source classes.
+            destination_prefix: Default prefix for destination classes.
+
+        Returns:
+            Mapping object.
+
+        """
+        df = pd.read_excel(path).dropna(axis=1, how="all")
+        properties = MappingList[PropertyRef]()
+        destination_classes_by_source: dict[ClassEntity, Counter[ClassEntity]] = defaultdict(Counter)
+        for _, row in df.iterrows():
+            if len(row) < 4:
+                raise NeatValueError(f"Row {row} is not valid. Expected 4 columns, got {len(row)}")
+
+            if any(pd.isna(row.iloc[:4])):
+                continue
+            source_class, source_property, destination_class, destination_property = row.iloc[:4]
+            source_entity = ClassEntity.load(source_class, prefix=source_prefix or Undefined)
+            destination_entity = ClassEntity.load(destination_class, prefix=destination_prefix or Undefined)
+            properties.append(
+                Mapping(
+                    source=PropertyRef(Class=source_entity, Property=source_property),
+                    destination=PropertyRef(Class=destination_entity, Property=destination_property),
+                )
+            )
+            destination_classes_by_source[source_entity][destination_entity] += 1
+
+        classes = MappingList[ClassRef]()
+        for source_entity, destination_classes in destination_classes_by_source.items():
+            destination_entity = destination_classes.most_common(1)[0][0]
+            classes.append(
+                Mapping(source=ClassRef(Class=source_entity), destination=ClassRef(Class=destination_entity))
+            )
+
+        return cls(properties=properties, classes=classes)

cognite/neat/_rules/models/mapping/_classic2core.py

@@ -0,0 +1,150 @@
+from ._base import RuleMapping
+
+
+def create_classic_to_core_mapping() -> RuleMapping:
+    return RuleMapping.model_validate(_CLASSIC_TO_CORE)
+
+
+_CLASSIC_TO_CORE = {
+    "classes": [
+        {"destination": {"Class": "cdf_cdm:CogniteAsset"}, "source": {"Class": "classic:Asset"}},
+        {"destination": {"Class": "cdf_cdm:DataProduct"}, "source": {"Class": "classic:DataSet"}},
+        {"destination": {"Class": "cdf_cdm:CogniteActivity"}, "source": {"Class": "classic:Event"}},
+        {"destination": {"Class": "cdf_cdm:CogniteFile"}, "source": {"Class": "classic:File"}},
+        {"destination": {"Class": "cdf_cdm:CogniteTimeSeries"}, "source": {"Class": "classic:TimeSeries"}},
+    ],
+    "properties": [
+        {
+            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "externalId"},
+            "source": {"Class": "classic:Asset", "Property": "externalId"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "name"},
+            "source": {"Class": "classic:Asset", "Property": "name"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "parent"},
+            "source": {"Class": "classic:Asset", "Property": "parentId"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "description"},
+            "source": {"Class": "classic:Asset", "Property": "description"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteSourceSystem", "Property": "name"},
+            "source": {"Class": "classic:Asset", "Property": "source"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "tag"},
+            "source": {"Class": "classic:Asset", "Property": "labels"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "externalId"},
+            "source": {"Class": "classic:DataSet", "Property": "externalId"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "name"},
+            "source": {"Class": "classic:DataSet", "Property": "name"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "description"},
+            "source": {"Class": "classic:DataSet", "Property": "description"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "metadata"},
+            "source": {"Class": "classic:DataSet", "Property": "metadata"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "writeProtected"},
+            "source": {"Class": "classic:DataSet", "Property": "writeProtected"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteActivity", "Property": "externalId"},
+            "source": {"Class": "classic:Event", "Property": "externalId"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteSchedulable", "Property": "startTime"},
+            "source": {"Class": "classic:Event", "Property": "startTime"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteSchedulable", "Property": "endTime"},
+            "source": {"Class": "classic:Event", "Property": "endTime"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteActivity", "Property": "description"},
+            "source": {"Class": "classic:Event", "Property": "description"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteActivity", "Property": "assets"},
+            "source": {"Class": "classic:Event", "Property": "assetIds"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteSourceSystem", "Property": "name"},
+            "source": {"Class": "classic:Event", "Property": "source"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "externalId"},
+            "source": {"Class": "classic:File", "Property": "externalId"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "name"},
+            "source": {"Class": "classic:File", "Property": "name"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "directory"},
+            "source": {"Class": "classic:File", "Property": "directory"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteSourceSystem", "Property": "name"},
+            "source": {"Class": "classic:File", "Property": "source"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "mimeType"},
+            "source": {"Class": "classic:File", "Property": "mimeType"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "assets"},
+            "source": {"Class": "classic:File", "Property": "assetIds"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "sourceCreatedTime"},
+            "source": {"Class": "classic:File", "Property": "sourceCreatedTime"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "sourceUpdatedTime"},
+            "source": {"Class": "classic:File", "Property": "sourceModifiedTime"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "externalId"},
+            "source": {"Class": "classic:TimeSeries", "Property": "externalId"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "name"},
+            "source": {"Class": "classic:TimeSeries", "Property": "name"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "type"},
+            "source": {"Class": "classic:TimeSeries", "Property": "isString"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "sourceUnit"},
+            "source": {"Class": "classic:TimeSeries", "Property": "unit"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "unit"},
+            "source": {"Class": "classic:TimeSeries", "Property": "unitExternalId"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "assets"},
+            "source": {"Class": "classic:TimeSeries", "Property": "assetId"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "isStep"},
+            "source": {"Class": "classic:TimeSeries", "Property": "isStep"},
+        },
+        {
+            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "description"},
+            "source": {"Class": "classic:TimeSeries", "Property": "description"},
+        },
+    ],
+}

cognite/neat/{rules → _rules}/transformers/__init__.py

@@ -1,6 +1,15 @@
 from ._base import RulesPipeline, RulesTransformer
-from ._converters import AssetToInformation, ConvertToRules, DMSToInformation, InformationToAsset, InformationToDMS
-from ._map_onto import MapOneToOne
+from ._converters import (
+    AssetToInformation,
+    ConvertToRules,
+    DMSToInformation,
+    InformationToAsset,
+    InformationToDMS,
+    ReduceCogniteModel,
+    ToCompliantEntities,
+    ToExtension,
+)
+from ._mapping import MapOneToOne, RuleMapper
 from ._pipelines import ImporterPipeline
 from ._verification import VerifyAnyRules, VerifyAssetRules, VerifyDMSRules, VerifyInformationRules
 
@@ -18,4 +27,8 @@ __all__ = [
     "VerifyInformationRules",
     "VerifyAnyRules",
     "MapOneToOne",
+    "ToCompliantEntities",
+    "RuleMapper",
+    "ToExtension",
+    "ReduceCogniteModel",
 ]

cognite/neat/{rules → _rules}/transformers/_base.py

@@ -2,9 +2,9 @@ from abc import ABC, abstractmethod
 from collections.abc import MutableSequence
 from typing import Generic, TypeVar
 
-from cognite.neat.issues import IssueList, NeatError
-from cognite.neat.issues.errors import NeatTypeError, NeatValueError
-from cognite.neat.rules._shared import (
+from cognite.neat._issues import IssueList, NeatError
+from cognite.neat._issues.errors import NeatTypeError, NeatValueError
+from cognite.neat._rules._shared import (
     InputRules,
     JustRules,
     MaybeRules,