cognite-neat 0.98.0__py3-none-any.whl → 0.99.0__py3-none-any.whl

cognite/neat/_rules/models/entities/_single_value.py

@@ -3,7 +3,7 @@ import sys
 from abc import ABC, abstractmethod
 from functools import total_ordering
 from types import UnionType
-from typing import Any, ClassVar, Generic, Literal, TypeVar, Union, cast, get_args, get_origin
+from typing import Any, ClassVar, Generic, Literal, TypeVar, Union, cast, get_args, get_origin, overload
 
 from cognite.client.data_classes.data_modeling import DirectRelationReference
 from cognite.client.data_classes.data_modeling.data_types import UnitReference
@@ -22,6 +22,7 @@ from pydantic import (
     model_validator,
 )
 
+from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._utils.text import replace_non_alphanumeric_with_underscore
 
 if sys.version_info >= (3, 11):
@@ -56,14 +57,29 @@ class Entity(BaseModel, extra="ignore"):
     suffix: str
 
     @classmethod
-    def load(cls: "type[T_Entity]", data: Any, **defaults) -> "T_Entity | UnknownEntity":
+    @overload
+    def load(cls: "type[T_Entity]", data: Any, strict: Literal[True], **defaults) -> "T_Entity": ...
+
+    @classmethod
+    @overload
+    def load(
+        cls: "type[T_Entity]", data: Any, strict: Literal[False] = False, **defaults
+    ) -> "T_Entity | UnknownEntity": ...
+
+    @classmethod
+    def load(cls: "type[T_Entity]", data: Any, strict: bool = False, **defaults) -> "T_Entity | UnknownEntity":
         if isinstance(data, cls):
             return data
         elif isinstance(data, str) and data == str(Unknown):
+            if strict:
+                raise NeatValueError(f"Failed to load entity {data!s}")
             return UnknownEntity(prefix=Undefined, suffix=Unknown)
         if defaults and isinstance(defaults, dict):
             # This is a trick to pass in default values
-            return cls.model_validate({_PARSE: data, "defaults": defaults})
+            try:
+                return cls.model_validate({_PARSE: data, "defaults": defaults})
+            except ValueError:
+                raise
         else:
             return cls.model_validate(data)
 
@@ -291,9 +307,21 @@ class DMSEntity(Entity, Generic[T_ID], ABC):
             defaults["prefix"] = defaults.pop("space")
         return super().dump(**defaults)
 
+    @classmethod  # type: ignore[override]
+    @overload
+    def load(cls: "type[T_DMSEntity]", data: Any, strict: Literal[True], **defaults) -> "T_DMSEntity": ...
+
+    @classmethod
+    @overload
+    def load(
+        cls: "type[T_DMSEntity]", data: Any, strict: Literal[False] = False, **defaults
+    ) -> "T_DMSEntity | DMSUnknownEntity": ...
+
     @classmethod
-    def load(cls: "type[T_DMSEntity]", data: Any, **defaults) -> "T_DMSEntity | DMSUnknownEntity":  # type: ignore[override]
+    def load(cls: "type[T_DMSEntity]", data: Any, strict: bool = False, **defaults) -> "T_DMSEntity | DMSUnknownEntity":  # type: ignore[override]
         if isinstance(data, str) and data == str(Unknown):
+            if strict:
+                raise NeatValueError(f"Failed to load entity {data!s}")
             return DMSUnknownEntity.from_id(None)
         return cast(T_DMSEntity, super().load(data, **defaults))
 

cognite/neat/_rules/models/information/_rules.py

@@ -14,9 +14,7 @@ from cognite.neat._rules._constants import EntityTypes
 from cognite.neat._rules.models._base_rules import (
     BaseMetadata,
     BaseRules,
-    ClassRef,
     DataModelAspect,
-    PropertyRef,
     RoleTypes,
     SheetList,
     SheetRow,
@@ -103,9 +101,6 @@ class InformationClass(SheetRow):
             )
         return ",".join(str(value) for value in value)
 
-    def as_reference(self) -> ClassRef:
-        return ClassRef(Class=self.class_)
-
 
 class InformationProperty(SheetRow):
     """
@@ -233,9 +228,6 @@ class InformationProperty(SheetRow):
             isinstance(self.max_count, int | float) and self.max_count > 1
         )
 
-    def as_reference(self) -> PropertyRef:
-        return PropertyRef(Class=self.class_, Property=self.property_)
-
 
 class InformationRules(BaseRules):
     metadata: InformationMetadata = Field(alias="Metadata")

cognite/neat/_rules/models/mapping/__init__.py

@@ -1,4 +1,3 @@
-from ._base import Mapping, MappingList, RuleMapping
-from ._classic2core import create_classic_to_core_mapping
+from ._classic2core import load_classic_to_core_mapping
 
-__all__ = ["RuleMapping", "Mapping", "MappingList", "create_classic_to_core_mapping"]
+__all__ = ["load_classic_to_core_mapping"]

cognite/neat/_rules/models/mapping/_classic2core.py

@@ -1,150 +1,40 @@
-from ._base import RuleMapping
+from functools import lru_cache
+from pathlib import Path
 
+import yaml
 
-def create_classic_to_core_mapping() -> RuleMapping:
-    return RuleMapping.model_validate(_CLASSIC_TO_CORE)
+from cognite.neat._issues.errors import NeatValueError
+from cognite.neat._rules._shared import ReadRules
+from cognite.neat._rules.models.dms import DMSInputRules, DMSRules
 
+_CLASSIC_TO_CORE_MAPPING = Path(__file__).resolve().parent / "_classic2core.yaml"
 
-_CLASSIC_TO_CORE = {
-    "classes": [
-        {"destination": {"Class": "cdf_cdm:CogniteAsset"}, "source": {"Class": "classic:Asset"}},
-        {"destination": {"Class": "cdf_cdm:DataProduct"}, "source": {"Class": "classic:DataSet"}},
-        {"destination": {"Class": "cdf_cdm:CogniteActivity"}, "source": {"Class": "classic:Event"}},
-        {"destination": {"Class": "cdf_cdm:CogniteFile"}, "source": {"Class": "classic:File"}},
-        {"destination": {"Class": "cdf_cdm:CogniteTimeSeries"}, "source": {"Class": "classic:TimeSeries"}},
-    ],
-    "properties": [
-        {
-            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "externalId"},
-            "source": {"Class": "classic:Asset", "Property": "externalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "name"},
-            "source": {"Class": "classic:Asset", "Property": "name"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "parent"},
-            "source": {"Class": "classic:Asset", "Property": "parentId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "description"},
-            "source": {"Class": "classic:Asset", "Property": "description"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteSourceSystem", "Property": "name"},
-            "source": {"Class": "classic:Asset", "Property": "source"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "tag"},
-            "source": {"Class": "classic:Asset", "Property": "labels"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "externalId"},
-            "source": {"Class": "classic:DataSet", "Property": "externalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "name"},
-            "source": {"Class": "classic:DataSet", "Property": "name"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "description"},
-            "source": {"Class": "classic:DataSet", "Property": "description"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "metadata"},
-            "source": {"Class": "classic:DataSet", "Property": "metadata"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "writeProtected"},
-            "source": {"Class": "classic:DataSet", "Property": "writeProtected"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteActivity", "Property": "externalId"},
-            "source": {"Class": "classic:Event", "Property": "externalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteSchedulable", "Property": "startTime"},
-            "source": {"Class": "classic:Event", "Property": "startTime"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteSchedulable", "Property": "endTime"},
-            "source": {"Class": "classic:Event", "Property": "endTime"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteActivity", "Property": "description"},
-            "source": {"Class": "classic:Event", "Property": "description"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteActivity", "Property": "assets"},
-            "source": {"Class": "classic:Event", "Property": "assetIds"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteSourceSystem", "Property": "name"},
-            "source": {"Class": "classic:Event", "Property": "source"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "externalId"},
-            "source": {"Class": "classic:File", "Property": "externalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "name"},
-            "source": {"Class": "classic:File", "Property": "name"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "directory"},
-            "source": {"Class": "classic:File", "Property": "directory"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteSourceSystem", "Property": "name"},
-            "source": {"Class": "classic:File", "Property": "source"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "mimeType"},
-            "source": {"Class": "classic:File", "Property": "mimeType"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "assets"},
-            "source": {"Class": "classic:File", "Property": "assetIds"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "sourceCreatedTime"},
-            "source": {"Class": "classic:File", "Property": "sourceCreatedTime"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "sourceUpdatedTime"},
-            "source": {"Class": "classic:File", "Property": "sourceModifiedTime"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "externalId"},
-            "source": {"Class": "classic:TimeSeries", "Property": "externalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "name"},
-            "source": {"Class": "classic:TimeSeries", "Property": "name"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "type"},
-            "source": {"Class": "classic:TimeSeries", "Property": "isString"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "sourceUnit"},
-            "source": {"Class": "classic:TimeSeries", "Property": "unit"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "unit"},
-            "source": {"Class": "classic:TimeSeries", "Property": "unitExternalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "assets"},
-            "source": {"Class": "classic:TimeSeries", "Property": "assetId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "isStep"},
-            "source": {"Class": "classic:TimeSeries", "Property": "isStep"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "description"},
-            "source": {"Class": "classic:TimeSeries", "Property": "description"},
-        },
-    ],
-}
+
+@lru_cache(maxsize=1)
+def _read_source_file() -> str:
+    return _CLASSIC_TO_CORE_MAPPING.read_text()
+
+
+def load_classic_to_core_mapping(org_name: str, source_space: str, source_version: str) -> DMSRules:
+    if not org_name:
+        raise NeatValueError("Organization name must be provided.")
+
+    from cognite.neat._rules.importers import YAMLImporter
+    from cognite.neat._rules.transformers import VerifyDMSRules
+
+    raw_str = _read_source_file().replace("Classic", org_name)
+
+    loaded = yaml.safe_load(raw_str)
+    loaded["metadata"]["space"] = source_space
+    loaded["metadata"]["version"] = source_version
+
+    read: ReadRules[DMSInputRules] = YAMLImporter(loaded).to_rules()
+    if not isinstance(read.rules, DMSInputRules):
+        raise NeatValueError(f"Expected DMS rules, but got {type(read.rules).__name__}")
+
+    verified = VerifyDMSRules(errors="raise", post_validate=False).transform(read)
+
+    if verified.rules is None:
+        raise NeatValueError("Failed to verify the rules.")
+
+    return verified.rules

cognite/neat/_rules/models/mapping/_classic2core.yaml

@@ -0,0 +1,339 @@
+enum:
+- collection: timeseriesType
+  description: Time series with double floating point data points.
+  name: numeric
+  value: numeric
+- collection: timeseriesType
+  description: Time series with string data points.
+  name: string
+  value: string
+metadata:
+  created: '2024-08-29T13:49:56.696000'
+  creator: Anders Albert,Nikola Vasiljevic,Thorkild Stray
+  description: Mapping from Classic to Core
+  external_id: ClassicCoreMapping
+  name: data_product data model
+  role: DMS Architect
+  space: mapping
+  updated: '2024-08-29T13:49:56.696000'
+  version: v1.0
+properties:
+- container: cdf_cdm:CogniteDescribable
+  container_property: description
+  description: Description of the instance
+  immutable: false
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicAsset
+  view_property: description
+- container: cdf_cdm:CogniteDescribable
+  container_property: tags
+  description: Text based labels for generic use, limited to 1000
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicAsset
+  view_property: labels
+- container: cdf_cdm:CogniteDescribable
+  container_property: name
+  description: Name of the instance
+  immutable: false
+  index: name
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicAsset
+  view_property: name
+- connection: direct
+  container: cdf_cdm:CogniteAsset
+  container_property: assetHierarchy_parent
+  description: The parent of the asset.
+  immutable: false
+  is_list: false
+  name: Parent
+  nullable: true
+  value_type: ClassicAsset
+  view: ClassicAsset
+  view_property: parentId
+- connection: direct
+  container: cdf_cdm:CogniteSourceable
+  container_property: source
+  description: Direct relation to a source system
+  immutable: false
+  index: source
+  is_list: false
+  nullable: true
+  value_type: cdf_cdm:SourceSystem(version=v1)
+  view: ClassicAsset
+  view_property: source
+- container: cdf_cdm:CogniteDescribable
+  container_property: aliases
+  description: Alternative names for the node
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicEvent
+  view_property: aliases
+- connection: direct
+  container: cdf_cdm:CogniteActivity
+  container_property: assets
+  description: A list of assets the activity is related to.
+  immutable: false
+  is_list: true
+  name: Assets
+  nullable: true
+  value_type: ClassicAsset
+  view: ClassicEvent
+  view_property: assetIds
+- container: cdf_cdm:CogniteDescribable
+  container_property: description
+  description: Description of the instance
+  immutable: false
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicEvent
+  view_property: description
+- container: cdf_cdm:CogniteSchedulable
+  container_property: endTime
+  description: The actual end time of an activity (or similar that extends this)
+  immutable: false
+  index: endTime
+  is_list: false
+  nullable: true
+  value_type: timestamp
+  view: ClassicEvent
+  view_property: endTime
+- container: cdf_cdm:CogniteDescribable
+  container_property: tags
+  description: Text based labels for generic use, limited to 1000
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicEvent
+  view_property: labels
+- connection: direct
+  container: cdf_cdm:CogniteSourceable
+  container_property: source
+  description: Direct relation to a source system
+  immutable: false
+  index: source
+  is_list: false
+  nullable: true
+  value_type: cdf_cdm:SourceSystem(version=v1)
+  view: ClassicEvent
+  view_property: source
+- container: cdf_cdm:CogniteSchedulable
+  container_property: startTime
+  description: The actual start time of an activity (or similar that extends this)
+  immutable: false
+  index: startTime
+  is_list: false
+  nullable: true
+  value_type: timestamp
+  view: ClassicEvent
+  view_property: startTime
+- connection: direct
+  container: cdf_cdm:CogniteFile
+  container_property: assets
+  description: A list of assets this file is related to.
+  immutable: false
+  is_list: true
+  name: Assets
+  nullable: true
+  value_type: ClassicAsset
+  view: ClassicFile
+  view_property: assetIds
+- container: cdf_cdm:CogniteFile
+  container_property: directory
+  description: Contains the path elements from the source (if the source system has
+    a file system hierarchy or similar.)
+  immutable: false
+  is_list: false
+  name: Directory
+  nullable: true
+  value_type: text
+  view: ClassicFile
+  view_property: directory
+- container: cdf_cdm:CogniteDescribable
+  container_property: tags
+  description: Text based labels for generic use, limited to 1000
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicFile
+  view_property: labels
+- container: cdf_cdm:CogniteFile
+  container_property: mimeType
+  description: The MIME type of the file.
+  immutable: false
+  is_list: false
+  name: MIME type
+  nullable: true
+  value_type: text
+  view: ClassicFile
+  view_property: mimeType
+- container: cdf_cdm:CogniteDescribable
+  container_property: name
+  description: Name of the instance
+  immutable: false
+  index: name
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicFile
+  view_property: name
+- connection: direct
+  container: cdf_cdm:CogniteSourceable
+  container_property: source
+  description: Direct relation to a source system
+  immutable: false
+  index: source
+  is_list: false
+  nullable: true
+  value_type: cdf_cdm:SourceSystem(version=v1)
+  view: ClassicFile
+  view_property: source
+- container: cdf_cdm:CogniteSourceable
+  container_property: sourceCreatedTime
+  description: When the instance was created in source system (if available)
+  immutable: false
+  index: sourceCreatedTime
+  is_list: false
+  nullable: true
+  value_type: timestamp
+  view: ClassicFile
+  view_property: sourceCreatedTime
+- container: cdf_cdm:CogniteSourceable
+  container_property: sourceUpdatedTime
+  description: When the instance was last updated in the source system (if available)
+  immutable: false
+  index: sourceUpdatedTime
+  is_list: false
+  nullable: true
+  value_type: timestamp
+  view: ClassicFile
+  view_property: sourceModifiedTime
+- connection: direct
+  container: cdf_cdm:CogniteTimeSeries
+  container_property: assets
+  description: A list of assets the time series is related to.
+  immutable: false
+  is_list: true
+  name: Assets
+  nullable: true
+  value_type: ClassicAsset
+  view: ClassicTimeSeries
+  view_property: assetId
+- container: cdf_cdm:CogniteDescribable
+  container_property: description
+  description: Description of the instance
+  immutable: false
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicTimeSeries
+  view_property: description
+- container: cdf_cdm:CogniteTimeSeries
+  container_property: isStep
+  default: 0
+  description: Specifies whether the time series is a step time series or not.
+  immutable: false
+  is_list: false
+  name: Is step
+  nullable: false
+  value_type: boolean
+  view: ClassicTimeSeries
+  view_property: isStep
+- container: cdf_cdm:CogniteTimeSeries
+  container_property: type
+  description: Specifies the data type of the data points.
+  immutable: true
+  is_list: false
+  name: Type
+  nullable: false
+  value_type: enum(collection=timeseriesType)
+  view: ClassicTimeSeries
+  view_property: isString
+- container: cdf_cdm:CogniteDescribable
+  container_property: aliases
+  description: Alternative names for the node
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicTimeSeries
+  view_property: legacyName
+- container: cdf_cdm:CogniteDescribable
+  container_property: name
+  description: Name of the instance
+  immutable: false
+  index: name
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicTimeSeries
+  view_property: name
+- container: cdf_cdm:CogniteDescribable
+  container_property: tags
+  description: Text based labels for generic use, limited to 1000
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicTimeSeries
+  view_property: securityCategories
+- container: cdf_cdm:CogniteTimeSeries
+  container_property: sourceUnit
+  description: The unit specified in the source system.
+  immutable: false
+  is_list: false
+  name: Source unit
+  nullable: true
+  value_type: text
+  view: ClassicTimeSeries
+  view_property: unit
+- connection: direct
+  container: cdf_cdm:CogniteTimeSeries
+  container_property: unit
+  description: The unit of the time series.
+  immutable: false
+  is_list: false
+  name: Unit
+  nullable: true
+  value_type: Unit
+  view: ClassicTimeSeries
+  view_property: unitExternalId
+views:
+- description: The CogniteSourceSystem core concept is used to standardize the way
+    source system is stored.
+  implements: cdf_cdm:CogniteDescribable(version=v1)
+  in_model: true
+  view: cdf_cdm:CogniteSourceSystem(version=v1)
+- description: Assets represent systems that support industrial functions or processes.
+    Assets are often called 'functional location'.
+  implements: cdf_cdm:CogniteAsset(version=v1)
+  in_model: true
+  name: Asset
+  view: ClassicAsset
+- description: Represents activities. Activities typically happen over a period and
+    have a start and end time.
+  implements: cdf_cdm:CogniteActivity(version=v1)
+  in_model: true
+  name: Activity
+  view: ClassicEvent
+- description: Represents files.
+  implements: cdf_cdm:CogniteFile(version=v1)
+  in_model: true
+  name: File
+  view: ClassicFile
+- description: Represents a series of data points in time order.
+  implements: cdf_cdm:CogniteTimeSeries(version=v1)
+  in_model: true
+  name: Time series
+  view: ClassicTimeSeries

cognite/neat/_rules/transformers/__init__.py

@@ -3,6 +3,7 @@ from ._converters import (
     ConvertToRules,
     DMSToInformation,
     InformationToDMS,
+    PrefixEntities,
     ReduceCogniteModel,
     SetIDDMSModel,
     ToCompliantEntities,
@@ -17,9 +18,8 @@ __all__ = [
     "RulesTransformer",
     "RulesPipeline",
     "InformationToDMS",
-    "InformationToAsset",
     "ConvertToRules",
-    "AssetToInformation",
+    "PrefixEntities",
     "DMSToInformation",
     "VerifyDMSRules",
     "VerifyInformationRules",