cognite-neat 0.98.0__py3-none-any.whl → 0.99.1__py3-none-any.whl

cognite/neat/_rules/models/mapping/_classic2core.yaml

@@ -0,0 +1,343 @@
+enum:
+- collection: timeseriesType
+  description: Time series with double floating point data points.
+  name: numeric
+  value: numeric
+- collection: timeseriesType
+  description: Time series with string data points.
+  name: string
+  value: string
+metadata:
+  created: '2024-08-29T13:49:56.696000'
+  creator: Anders Albert,Nikola Vasiljevic,Thorkild Stray
+  description: Mapping from Classic to Core
+  external_id: ClassicCoreMapping
+  name: data_product data model
+  role: DMS Architect
+  space: mapping
+  updated: '2024-08-29T13:49:56.696000'
+  version: v1.0
+properties:
+- container: cdf_cdm:CogniteDescribable
+  container_property: description
+  description: Description of the instance
+  immutable: false
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicAsset
+  view_property: description
+- container: cdf_cdm:CogniteDescribable
+  container_property: tags
+  description: Text based labels for generic use, limited to 1000
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicAsset
+  view_property: labels
+- container: cdf_cdm:CogniteDescribable
+  container_property: name
+  description: Name of the instance
+  immutable: false
+  index: name
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicAsset
+  view_property: name
+- connection: direct
+  container: cdf_cdm:CogniteAsset
+  container_property: assetHierarchy_parent
+  description: The parent of the asset.
+  immutable: false
+  is_list: false
+  name: Parent
+  nullable: true
+  value_type: ClassicAsset
+  view: ClassicAsset
+  view_property: parentId
+- connection: direct
+  container: cdf_cdm:CogniteSourceable
+  container_property: source
+  description: Direct relation to a source system
+  immutable: false
+  index: source
+  is_list: false
+  nullable: true
+  value_type: cdf_cdm:CogniteSourceSystem(version=v1)
+  view: ClassicAsset
+  view_property: source
+- container: cdf_cdm:CogniteDescribable
+  container_property: aliases
+  description: Alternative names for the node
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicEvent
+  view_property: aliases
+- connection: direct
+  container: cdf_cdm:CogniteActivity
+  container_property: assets
+  description: A list of assets the activity is related to.
+  immutable: false
+  is_list: true
+  name: Assets
+  nullable: true
+  value_type: ClassicAsset
+  view: ClassicEvent
+  view_property: assetIds
+- container: cdf_cdm:CogniteDescribable
+  container_property: description
+  description: Description of the instance
+  immutable: false
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicEvent
+  view_property: description
+- container: cdf_cdm:CogniteSchedulable
+  container_property: endTime
+  description: The actual end time of an activity (or similar that extends this)
+  immutable: false
+  index: endTime
+  is_list: false
+  nullable: true
+  value_type: timestamp
+  view: ClassicEvent
+  view_property: endTime
+- container: cdf_cdm:CogniteDescribable
+  container_property: tags
+  description: Text based labels for generic use, limited to 1000
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicEvent
+  view_property: labels
+- connection: direct
+  container: cdf_cdm:CogniteSourceable
+  container_property: source
+  description: Direct relation to a source system
+  immutable: false
+  index: source
+  is_list: false
+  nullable: true
+  value_type: cdf_cdm:CogniteSourceSystem(version=v1)
+  view: ClassicEvent
+  view_property: source
+- container: cdf_cdm:CogniteSchedulable
+  container_property: startTime
+  description: The actual start time of an activity (or similar that extends this)
+  immutable: false
+  index: startTime
+  is_list: false
+  nullable: true
+  value_type: timestamp
+  view: ClassicEvent
+  view_property: startTime
+- connection: direct
+  container: cdf_cdm:CogniteFile
+  container_property: assets
+  description: A list of assets this file is related to.
+  immutable: false
+  is_list: true
+  name: Assets
+  nullable: true
+  value_type: ClassicAsset
+  view: ClassicFile
+  view_property: assetIds
+- container: cdf_cdm:CogniteFile
+  container_property: directory
+  description: Contains the path elements from the source (if the source system has
+    a file system hierarchy or similar.)
+  immutable: false
+  is_list: false
+  name: Directory
+  nullable: true
+  value_type: text
+  view: ClassicFile
+  view_property: directory
+- container: cdf_cdm:CogniteDescribable
+  container_property: tags
+  description: Text based labels for generic use, limited to 1000
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicFile
+  view_property: labels
+- container: cdf_cdm:CogniteFile
+  container_property: mimeType
+  description: The MIME type of the file.
+  immutable: false
+  is_list: false
+  name: MIME type
+  nullable: true
+  value_type: text
+  view: ClassicFile
+  view_property: mimeType
+- container: cdf_cdm:CogniteDescribable
+  container_property: name
+  description: Name of the instance
+  immutable: false
+  index: name
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicFile
+  view_property: name
+- connection: direct
+  container: cdf_cdm:CogniteSourceable
+  container_property: source
+  description: Direct relation to a source system
+  immutable: false
+  index: source
+  is_list: false
+  nullable: true
+  value_type: cdf_cdm:CogniteSourceSystem(version=v1)
+  view: ClassicFile
+  view_property: source
+- container: cdf_cdm:CogniteSourceable
+  container_property: sourceCreatedTime
+  description: When the instance was created in source system (if available)
+  immutable: false
+  index: sourceCreatedTime
+  is_list: false
+  nullable: true
+  value_type: timestamp
+  view: ClassicFile
+  view_property: sourceCreatedTime
+- container: cdf_cdm:CogniteSourceable
+  container_property: sourceUpdatedTime
+  description: When the instance was last updated in the source system (if available)
+  immutable: false
+  index: sourceUpdatedTime
+  is_list: false
+  nullable: true
+  value_type: timestamp
+  view: ClassicFile
+  view_property: sourceModifiedTime
+- connection: direct
+  container: cdf_cdm:CogniteTimeSeries
+  container_property: assets
+  description: A list of assets the time series is related to.
+  immutable: false
+  is_list: true
+  name: Assets
+  nullable: true
+  value_type: ClassicAsset
+  view: ClassicTimeSeries
+  view_property: assetId
+- container: cdf_cdm:CogniteDescribable
+  container_property: description
+  description: Description of the instance
+  immutable: false
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicTimeSeries
+  view_property: description
+- container: cdf_cdm:CogniteTimeSeries
+  container_property: isStep
+  default: 0
+  description: Specifies whether the time series is a step time series or not.
+  immutable: false
+  is_list: false
+  name: Is step
+  nullable: false
+  value_type: boolean
+  view: ClassicTimeSeries
+  view_property: isStep
+- container: cdf_cdm:CogniteTimeSeries
+  container_property: type
+  description: Specifies the data type of the data points.
+  immutable: true
+  is_list: false
+  name: Type
+  nullable: false
+  value_type: enum(collection=timeseriesType)
+  view: ClassicTimeSeries
+  view_property: isString
+- container: cdf_cdm:CogniteDescribable
+  container_property: aliases
+  description: Alternative names for the node
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicTimeSeries
+  view_property: legacyName
+- container: cdf_cdm:CogniteDescribable
+  container_property: name
+  description: Name of the instance
+  immutable: false
+  index: name
+  is_list: false
+  nullable: true
+  value_type: text
+  view: ClassicTimeSeries
+  view_property: name
+- container: cdf_cdm:CogniteDescribable
+  container_property: tags
+  description: Text based labels for generic use, limited to 1000
+  immutable: false
+  is_list: true
+  nullable: true
+  value_type: text
+  view: ClassicTimeSeries
+  view_property: securityCategories
+- container: cdf_cdm:CogniteTimeSeries
+  container_property: sourceUnit
+  description: The unit specified in the source system.
+  immutable: false
+  is_list: false
+  name: Source unit
+  nullable: true
+  value_type: text
+  view: ClassicTimeSeries
+  view_property: unit
+- connection: direct
+  container: cdf_cdm:CogniteTimeSeries
+  container_property: unit
+  description: The unit of the time series.
+  immutable: false
+  is_list: false
+  name: Unit
+  nullable: true
+  value_type: cdf_cdm:CogniteUnit(version=v1)
+  view: ClassicTimeSeries
+  view_property: unitExternalId
+views:
+- description: The CogniteSourceSystem core concept is used to standardize the way
+    source system is stored.
+  implements: cdf_cdm:CogniteDescribable(version=v1)
+  in_model: true
+  view: cdf_cdm:CogniteSourceSystem(version=v1)
+- description: Represents a single unit of measurement
+  implements: CogniteDescribable
+  in_model: true
+  view: cdf_cdm:CogniteUnit(version=v1)
+- description: Assets represent systems that support industrial functions or processes.
+    Assets are often called 'functional location'.
+  implements: cdf_cdm:CogniteAsset(version=v1)
+  in_model: true
+  name: Asset
+  view: ClassicAsset
+- description: Represents activities. Activities typically happen over a period and
+    have a start and end time.
+  implements: cdf_cdm:CogniteActivity(version=v1)
+  in_model: true
+  name: Activity
+  view: ClassicEvent
+- description: Represents files.
+  implements: cdf_cdm:CogniteFile(version=v1)
+  in_model: true
+  name: File
+  view: ClassicFile
+- description: Represents a series of data points in time order.
+  implements: cdf_cdm:CogniteTimeSeries(version=v1)
+  in_model: true
+  name: Time series
+  view: ClassicTimeSeries

cognite/neat/_rules/transformers/__init__.py

@@ -3,6 +3,7 @@ from ._converters import (
     ConvertToRules,
     DMSToInformation,
     InformationToDMS,
+    PrefixEntities,
     ReduceCogniteModel,
     SetIDDMSModel,
     ToCompliantEntities,
@@ -17,9 +18,8 @@ __all__ = [
     "RulesTransformer",
     "RulesPipeline",
     "InformationToDMS",
-    "InformationToAsset",
     "ConvertToRules",
-    "AssetToInformation",
+    "PrefixEntities",
     "DMSToInformation",
     "VerifyDMSRules",
     "VerifyInformationRules",

cognite/neat/_rules/transformers/_converters.py

@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
 from collections import Counter, defaultdict
 from collections.abc import Collection, Mapping
 from datetime import date, datetime
-from typing import Literal, TypeVar, cast
+from typing import Literal, TypeVar, cast, overload
 
 from cognite.client.data_classes import data_modeling as dms
 from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
@@ -42,8 +42,10 @@ from cognite.neat._rules.models.entities import (
     ContainerEntity,
     DMSUnknownEntity,
     EdgeEntity,
+    Entity,
     MultiValueTypeInfo,
     ReverseConnectionEntity,
+    T_Entity,
     UnknownEntity,
     ViewEntity,
 )
@@ -176,6 +178,74 @@ class ToCompliantEntities(RulesTransformer[InformationInputRules, InformationInp
         return fixed_definitions
 
 
+class PrefixEntities(RulesTransformer[InputRules, InputRules]):  # type: ignore[misc]
+    """Prefixes all entities with a given prefix."""
+
+    def __init__(self, prefix: str) -> None:
+        self._prefix = prefix
+
+    def transform(self, rules: InputRules | OutRules[InputRules]) -> ReadRules[InputRules]:
+        return ReadRules(self._transform(self._to_rules(rules)), IssueList(), {})
+
+    def _transform(self, rules: InputRules) -> InputRules:
+        rules.metadata.version += f"_prefixed_{self._prefix}"
+
+        if isinstance(rules, InformationInputRules):
+            # Todo Make Not mutate input class
+            prefixed_by_class: dict[str, str] = {}
+            for cls in rules.classes:
+                prefixed = str(self._with_prefix(cls.class_))
+                prefixed_by_class[str(cls.class_)] = prefixed
+                cls.class_ = prefixed
+            for prop in rules.properties:
+                prop.class_ = self._with_prefix(prop.class_)
+                if str(prop.value_type) in prefixed_by_class:
+                    prop.value_type = prefixed_by_class[str(prop.value_type)]
+            return rules
+        elif isinstance(rules, DMSInputRules):
+            # Todo not mutate input class new_dms = copy.deepcopy(rules)
+            prefixed_by_view: dict[str, str] = {}
+            for view in rules.views:
+                prefixed = str(self._with_prefix(view.view))
+                prefixed_by_view[str(view.view)] = prefixed
+                view.view = prefixed
+            for dms_prop in rules.properties:
+                dms_prop.view = self._with_prefix(dms_prop.view)
+                if str(dms_prop.value_type) in prefixed_by_view:
+                    dms_prop.value_type = prefixed_by_view[str(dms_prop.value_type)]
+            if rules.containers:
+                for container in rules.containers:
+                    container.container = self._with_prefix(container.container)
+            return rules
+        raise NeatValueError(f"Unsupported rules type: {type(rules)}")
+
+    @overload
+    def _with_prefix(self, raw: str) -> str: ...
+
+    @overload
+    def _with_prefix(self, raw: T_Entity) -> T_Entity: ...
+
+    def _with_prefix(self, raw: str | T_Entity) -> str | T_Entity:
+        is_entity_format = not isinstance(raw, str)
+        entity = Entity.load(raw)
+        output: ClassEntity | ViewEntity | ContainerEntity
+        if isinstance(entity, ClassEntity):
+            output = ClassEntity(prefix=entity.prefix, suffix=f"{self._prefix}{entity.suffix}", version=entity.version)
+        elif isinstance(entity, ViewEntity):
+            output = ViewEntity(
+                space=entity.space, externalId=f"{self._prefix}{entity.external_id}", version=entity.version
+            )
+        elif isinstance(entity, ContainerEntity):
+            output = ContainerEntity(space=entity.space, externalId=f"{self._prefix}{entity.external_id}")
+        elif isinstance(entity, UnknownEntity | Entity):
+            return f"{self._prefix}{raw}"
+        else:
+            raise NeatValueError(f"Unsupported entity type: {type(entity)}")
+        if is_entity_format:
+            return cast(T_Entity, output)
+        return str(output)
+
+
 class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
     """Converts InformationRules to DMSRules."""
 
@@ -233,10 +303,11 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
         self,
         new_model_id: DataModelIdentifier,
         org_name: str = "My",
-        type_: Literal["enterprise", "solution"] = "enterprise",
+        type_: Literal["enterprise", "solution", "data_product"] = "enterprise",
         mode: Literal["read", "write"] = "read",
         dummy_property: str = "GUID",
         move_connections: bool = False,
+        include: Literal["same-space", "all"] = "same-space",
     ):
         self.new_model_id = DataModelId.load(new_model_id)
         if not self.new_model_id.version:
@@ -247,6 +318,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
         self.type_ = type_
         self.dummy_property = dummy_property
         self.move_connections = move_connections
+        self.include = include
 
     def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
         # Copy to ensure immutability
@@ -274,6 +346,16 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
                 )
 
             return self._to_enterprise(reference_model)
+        elif self.type_ == "data_product":
+            expanded = self._expand_properties(reference_model.model_copy(deep=True))
+            if self.include == "same-space":
+                expanded.properties = SheetList[DMSProperty](
+                    [prop for prop in expanded.properties if prop.view.space == expanded.metadata.space]
+                )
+                expanded.views = SheetList[DMSView](
+                    [view for view in expanded.views if view.view.space == expanded.metadata.space]
+                )
+            return self._to_solution(expanded, remove_views_in_other_space=False)
 
         else:
             raise NeatValueError(f"Unsupported data model type: {self.type_}")
@@ -281,7 +363,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
     def _has_views_in_multiple_space(self, rules: DMSRules) -> bool:
         return any(view.view.space != rules.metadata.space for view in rules.views)
 
-    def _to_solution(self, reference_rules: DMSRules) -> JustRules[DMSRules]:
+    def _to_solution(self, reference_rules: DMSRules, remove_views_in_other_space: bool = True) -> JustRules[DMSRules]:
         """For creation of solution data model / rules specifically for mapping over existing containers."""
 
         dump = reference_rules.dump()
@@ -292,16 +374,11 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
         dump["metadata"]["external_id"] = self.new_model_id.external_id
         dump["metadata"]["version"] = self.new_model_id.version
 
-        # dropping reference and last from the dump as they can cause validation
-        # issues especially if reference is enterprise model build on top of CDM
-        dump.pop("reference", None)
-        dump.pop("last", None)
-
         # Set implement to NONE for all views
         for view in dump["views"]:
             view["implements"] = None
 
-        if self._has_views_in_multiple_space(reference_rules):
+        if remove_views_in_other_space and self._has_views_in_multiple_space(reference_rules):
             views_to_remove = []
             for view in dump["views"]:
                 if ":" in view["view"]:
@@ -312,7 +389,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
         solution_model = DMSRules.model_validate(DMSInputRules.load(dump).dump())
 
         # Dropping containers coming from reference model
-        solution_model.containers = SheetList[DMSContainer]()
+        solution_model.containers = None
 
         # We want to map properties to existing containers allowing extension
         for prop in solution_model.properties:
@@ -338,7 +415,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
 
             # Here we add ONLY dummy properties of the solution model and
             # corresponding solution model space containers to hold them
-            solution_model.containers.extend(new_containers)
+            solution_model.containers = new_containers
             solution_model.properties.extend(new_properties)
 
         return JustRules(solution_model)
@@ -377,6 +454,28 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
 
         return JustRules(enterprise_model)
 
+    @staticmethod
+    def _expand_properties(rules: DMSRules) -> DMSRules:
+        probe = DMSAnalysis(rules)
+        ancestor_properties_by_view = probe.classes_with_properties(
+            consider_inheritance=True, allow_different_namespace=True
+        )
+        property_ids_by_view = {
+            view: {prop.view_property for prop in properties}
+            for view, properties in probe.classes_with_properties(consider_inheritance=False).items()
+        }
+        for view, property_ids in property_ids_by_view.items():
+            ancestor_properties = ancestor_properties_by_view.get(view, [])
+            for prop in ancestor_properties:
+                if isinstance(prop.connection, ReverseConnectionEntity):
+                    # If you try to add a reverse direct relation of a parent, it will fail as the ValueType of the
+                    # original property will point to the parent view, and not the child.
+                    continue
+                if prop.view_property not in property_ids:
+                    rules.properties.append(prop)
+                    property_ids.add(prop.view_property)
+        return rules
+
     def _remove_cognite_affix(self, entity: _T_Entity) -> _T_Entity:
         """This method removes `Cognite` affix from the entity."""
         new_suffix = entity.suffix.replace("Cognite", self.org_name or "")