cognite-neat 0.114.0__tar.gz → 0.115.0__tar.gz

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cognite-neat
-Version: 0.114.0
+Version: 0.115.0
 Summary: Knowledge graph transformation
 License: Apache-2.0
 Author: Nikola Vasiljevic

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/cognite/neat/_constants.py

@@ -117,9 +117,12 @@ DEFAULT_URI = ""
 
 DEFAULT_DOCS_URL = "https://cognite-neat.readthedocs-hosted.com/en/latest/"
 
+# These are the API limits for the DMS API, https://docs.cognite.com/cdf/dm/dm_reference/dm_limits_and_restrictions
 DMS_CONTAINER_PROPERTY_SIZE_LIMIT = 100
 DMS_VIEW_CONTAINER_SIZE_LIMIT = 10
-DMS_DIRECT_RELATION_LIST_LIMIT = 100
+DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT = 100
+DMS_PRIMITIVE_LIST_DEFAULT_LIMIT = 1000
+DMS_CONTAINER_LIST_MAX_LIMIT = 2000
 
 _ASSET_ROOT_PROPERTY = {
     "connection": "direct",

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/cognite/neat/_graph/loaders/_rdf2dms.py

@@ -21,7 +21,7 @@ from rdflib import RDF, URIRef
 
 from cognite.neat._client import NeatClient
 from cognite.neat._client._api_client import SchemaAPI
-from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_LIMIT, is_readonly_property
+from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT, is_readonly_property
 from cognite.neat._issues import IssueList, NeatError, NeatIssue, catch_issues
 from cognite.neat._issues.errors import (
     AuthorizationError,
@@ -417,20 +417,20 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                     ids = (self._create_instance_id(v, "node", stop_on_exception=True) for v in value)
                     result = [id_.dump(camel_case=True, include_instance_type=False) for id_ in ids]
                     # Todo: Account for max_list_limit
-                    if len(result) <= DMS_DIRECT_RELATION_LIST_LIMIT:
+                    if len(result) <= DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT:
                         return result
                     warnings.warn(
                         PropertyDirectRelationLimitWarning(
                             identifier="unknown",
                             resource_type="view property",
                             property_name=cast(str, cls.model_fields[info.field_name].alias or info.field_name),
-                            limit=DMS_DIRECT_RELATION_LIST_LIMIT,
+                            limit=DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT,
                         ),
                         stacklevel=2,
                     )
                     # To get deterministic results, we sort by space and externalId
                     result.sort(key=lambda x: (x["space"], x["externalId"]))
-                    return result[:DMS_DIRECT_RELATION_LIST_LIMIT]
+                    return result[:DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT]
                 elif value:
                     return self._create_instance_id(value[0], "node", stop_on_exception=True).dump(
                         camel_case=True, include_instance_type=False

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/cognite/neat/_issues/warnings/__init__.py

@@ -13,7 +13,13 @@ from ._external import (
     FileReadWarning,
     FileTypeUnexpectedWarning,
 )
-from ._general import MissingCogniteClientWarning, NeatValueWarning, NotSupportedWarning, RegexViolationWarning
+from ._general import (
+    DeprecatedWarning,
+    MissingCogniteClientWarning,
+    NeatValueWarning,
+    NotSupportedWarning,
+    RegexViolationWarning,
+)
 from ._models import (
     BreakingModelingPrincipleWarning,
     CDFNotSupportedWarning,
@@ -50,6 +56,7 @@ __all__ = [
     "CDFAuthWarning",
     "CDFMaxIterationsWarning",
     "CDFNotSupportedWarning",
+    "DeprecatedWarning",
     "FileItemNotSupportedWarning",
     "FileMissingRequiredFieldWarning",
     "FileReadWarning",

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/cognite/neat/_issues/warnings/_general.py

@@ -34,3 +34,13 @@ class MissingCogniteClientWarning(NeatWarning):
     """Missing Cognite Client required for {functionality}"""
 
     functionality: str
+
+
+@dataclass(unsafe_hash=True)
+class DeprecatedWarning(NeatWarning):
+    """{feature} is deprecated"""
+
+    extra = "{replacement}"
+
+    feature: str
+    replacement: str | None = None

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/cognite/neat/_issues/warnings/_properties.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from typing import Generic
 
-from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_LIMIT
+from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT
 from cognite.neat._issues._base import ResourceType
 
 from ._resources import ResourceNeatWarning, T_Identifier, T_ReferenceIdentifier
@@ -79,7 +79,7 @@ class PropertyDirectRelationLimitWarning(PropertyWarning[T_Identifier]):
 
     resource_type = "view"
 
-    limit: int = DMS_DIRECT_RELATION_LIST_LIMIT
+    limit: int = DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT
 
 
 @dataclass(unsafe_hash=True)

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/cognite/neat/_rules/exporters/_rules2excel.py

@@ -170,16 +170,12 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
         dv_value_types = generate_data_validation(self._helper_sheet_name, "C", no_header_rows=0, no_rows=no_rows)
 
         dv_immutable = generate_data_validation(self._helper_sheet_name, "D", no_header_rows=0, no_rows=3)
-        dv_nullable = generate_data_validation(self._helper_sheet_name, "D", no_header_rows=0, no_rows=3)
-        dv_is_list = generate_data_validation(self._helper_sheet_name, "D", no_header_rows=0, no_rows=3)
         dv_in_model = generate_data_validation(self._helper_sheet_name, "D", no_header_rows=0, no_rows=3)
         dv_used_for = generate_data_validation(self._helper_sheet_name, "E", no_header_rows=0, no_rows=3)
 
         workbook["Properties"].add_data_validation(dv_views)
         workbook["Properties"].add_data_validation(dv_containers)
         workbook["Properties"].add_data_validation(dv_value_types)
-        workbook["Properties"].add_data_validation(dv_nullable)
-        workbook["Properties"].add_data_validation(dv_is_list)
         workbook["Properties"].add_data_validation(dv_immutable)
         workbook["Views"].add_data_validation(dv_in_model)
         workbook["Containers"].add_data_validation(dv_used_for)
@@ -194,12 +190,6 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
         if column := find_column_with_value(workbook["Properties"], "Value Type"):
             dv_value_types.add(f"{column}{3}:{column}{no_rows * 100}")
 
-        if column := find_column_with_value(workbook["Properties"], "Nullable"):
-            dv_nullable.add(f"{column}{3}:{column}{no_rows * 100}")
-
-        if column := find_column_with_value(workbook["Properties"], "Is List"):
-            dv_is_list.add(f"{column}{3}:{column}{no_rows * 100}")
-
         if column := find_column_with_value(workbook["Properties"], "Immutable"):
             dv_immutable.add(f"{column}{3}:{column}{no_rows * 100}")
 

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/cognite/neat/_rules/importers/_dms2rules.py

@@ -7,8 +7,12 @@ from typing import Literal, cast
 from cognite.client import data_modeling as dm
 from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
 from cognite.client.data_classes.data_modeling.containers import BTreeIndex, InvertedIndex
+from cognite.client.data_classes.data_modeling.data_types import (
+    DirectRelation,
+    ListablePropertyType,
+    PropertyTypeWithUnit,
+)
 from cognite.client.data_classes.data_modeling.data_types import Enum as DMSEnum
-from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType, PropertyTypeWithUnit
 from cognite.client.data_classes.data_modeling.views import (
     MultiEdgeConnectionApply,
     MultiReverseDirectRelationApply,
@@ -19,6 +23,7 @@ from cognite.client.data_classes.data_modeling.views import (
 from cognite.client.utils import ms_to_datetime
 
 from cognite.neat._client import NeatClient
+from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT, DMS_PRIMITIVE_LIST_DEFAULT_LIMIT
 from cognite.neat._issues import IssueList, MultiValueError, NeatIssue, catch_issues
 from cognite.neat._issues.errors import (
     FileTypeUnexpectedError,
@@ -336,8 +341,8 @@ class DMSImporter(BaseImporter[DMSInputRules]):
             name=prop.name,
             connection=self._get_connection_type(prop),
             value_type=str(value_type),
-            is_list=self._get_is_list(prop),
-            nullable=self._get_nullable(prop),
+            min_count=self._get_min_count(prop),
+            max_count=self._get_max_count(prop),
             immutable=self._get_immutable(prop),
             default=self._get_default(prop),
             container=(
@@ -419,9 +424,9 @@ class DMSImporter(BaseImporter[DMSInputRules]):
             )
             return None
 
-    def _get_nullable(self, prop: ViewPropertyApply) -> bool | None:
+    def _get_min_count(self, prop: ViewPropertyApply) -> int | None:
         if isinstance(prop, dm.MappedPropertyApply):
-            return self._container_prop_unsafe(prop).nullable
+            return int(not self._container_prop_unsafe(prop).nullable)
         else:
             return None
 
@@ -431,15 +436,26 @@ class DMSImporter(BaseImporter[DMSInputRules]):
         else:
             return None
 
-    def _get_is_list(self, prop: ViewPropertyApply) -> bool | None:
+    def _get_max_count(self, prop: ViewPropertyApply) -> int | float | None:
         if isinstance(prop, dm.MappedPropertyApply):
             prop_type = self._container_prop_unsafe(prop).type
-            return isinstance(prop_type, ListablePropertyType) and prop_type.is_list
+            if isinstance(prop_type, ListablePropertyType):
+                if prop_type.is_list is False:
+                    return 1
+                elif isinstance(prop_type.max_list_size, int):
+                    return prop_type.max_list_size
+                elif isinstance(prop_type, DirectRelation):
+                    return DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT
+                else:
+                    return DMS_PRIMITIVE_LIST_DEFAULT_LIMIT
+            else:
+                return 1
         elif isinstance(prop, MultiEdgeConnectionApply | MultiReverseDirectRelationApply):
-            return True
+            return float("inf")
         elif isinstance(prop, SingleEdgeConnectionApply | SingleReverseDirectRelationApply):
-            return False
+            return 1
         else:
+            # Unknown type.
             return None
 
     def _get_default(self, prop: ViewPropertyApply) -> str | None:

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/cognite/neat/_rules/models/dms/_exporter.py

@@ -20,7 +20,11 @@ from cognite.neat._client.data_classes.data_modeling import (
     ViewApplyDict,
 )
 from cognite.neat._client.data_classes.schema import DMSSchema
-from cognite.neat._constants import COGNITE_SPACES
+from cognite.neat._constants import (
+    COGNITE_SPACES,
+    DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT,
+    DMS_PRIMITIVE_LIST_DEFAULT_LIMIT,
+)
 from cognite.neat._issues.errors import NeatTypeError, NeatValueError, ResourceNotFoundError
 from cognite.neat._issues.warnings import NotSupportedWarning, PropertyNotFoundWarning
 from cognite.neat._issues.warnings.user_modeling import (
@@ -310,7 +314,16 @@ class _DMSExporter:
 
                 args: dict[str, Any] = {}
                 if issubclass(type_cls, ListablePropertyType):
-                    args["is_list"] = prop.is_list or False
+                    is_list = args["is_list"] = prop.is_list or False
+                    if is_list:
+                        if type_cls is dm.DirectRelation and prop.max_count == DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT:
+                            # Use default of API.
+                            args["max_list_size"] = None
+                        elif type_cls is not dm.DirectRelation and prop.max_count == DMS_PRIMITIVE_LIST_DEFAULT_LIMIT:
+                            # Use default of API.
+                            args["max_list_size"] = None
+                        else:
+                            args["max_list_size"] = prop.max_count
                 if isinstance(prop.value_type, Double | Float) and isinstance(prop.value_type.unit, UnitEntity):
                     args["unit"] = prop.value_type.unit.as_reference()
                 if isinstance(prop.value_type, Enum):

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/cognite/neat/_rules/models/dms/_rules.py

@@ -1,3 +1,4 @@
+import math
 import warnings
 from collections.abc import Hashable
 from typing import TYPE_CHECKING, Any, ClassVar, Literal
@@ -8,6 +9,7 @@ from pydantic import Field, field_serializer, field_validator, model_validator
 from pydantic_core.core_schema import SerializationInfo, ValidationInfo
 
 from cognite.neat._client.data_classes.schema import DMSSchema
+from cognite.neat._constants import DMS_CONTAINER_LIST_MAX_LIMIT
 from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._issues.warnings._general import NeatValueWarning
 from cognite.neat._rules.models._base_rules import (
@@ -102,22 +104,25 @@ class DMSProperty(SheetRow):
         alias="Value Type",
         description="Value type that the property can hold. It takes either subset of CDF primitive types or a View id",
     )
-    nullable: bool | None = Field(
+    min_count: int | None = Field(
+        alias="Min Count",
         default=None,
-        alias="Nullable",
-        description="Used to indicate whether the property is required or not. Only applies to primitive type.",
+        description="Minimum number of values that the property can hold. "
+        "If no value is provided, the default value is  `0`, "
+        "which means that the property is optional.",
+    )
+    max_count: int | float | None = Field(
+        alias="Max Count",
+        default=None,
+        description="Maximum number of values that the property can hold. "
+        "If no value is provided, the default value is  `inf`, "
+        "which means that the property can hold any number of values (listable).",
     )
     immutable: bool | None = Field(
         default=None,
         alias="Immutable",
         description="sed to indicate whether the property is can only be set once. Only applies to primitive type.",
     )
-    is_list: bool | None = Field(
-        default=None,
-        alias="Is List",
-        description="Used to indicate whether the property holds single or multiple values (list). "
-        "Only applies to primitive types.",
-    )
     default: bool | str | int | float | dict | None = Field(
         None, alias="Default", description="Specifies default value for the property."
     )
@@ -147,13 +152,43 @@ class DMSProperty(SheetRow):
         description="Used to make connection between physical and logical data model aspect",
     )
 
+    @property
+    def nullable(self) -> bool | None:
+        """Used to indicate whether the property is required or not. Only applies to primitive type."""
+        return self.min_count in {0, None}
+
+    @property
+    def is_list(self) -> bool | None:
+        """Used to indicate whether the property holds single or multiple values (list). "
+        "Only applies to primitive types."""
+        if self.max_count is None:
+            return None
+        return self.max_count is float("inf") or (isinstance(self.max_count, int | float) and self.max_count > 1)
+
     def _identifier(self) -> tuple[Hashable, ...]:
         return self.view, self.view_property
 
-    @field_validator("nullable")
+    @field_validator("min_count")
     def direct_relation_must_be_nullable(cls, value: Any, info: ValidationInfo) -> None:
-        if info.data.get("connection") == "direct" and value is False:
-            raise ValueError("Direct relation must be nullable")
+        if info.data.get("connection") == "direct" and value not in {0, None}:
+            raise ValueError("Direct relation must have min count set to 0")
+        return value
+
+    @field_validator("max_count", mode="before")
+    def as_integer(cls, value: Any) -> Any:
+        if isinstance(value, float) and not math.isinf(value):
+            return int(value)
+        return value
+
+    @field_validator("max_count")
+    def max_list_size(cls, value: Any, info: ValidationInfo) -> Any:
+        if isinstance(info.data.get("connection"), EdgeEntity | ReverseConnectionEntity):
+            if value is not None and value != float("inf") and not (isinstance(value, int) and value == 1):
+                raise ValueError("Edge and reverse connections must have max count set to inf or 1")
+            return value
+        # We do not have a connection, so we can check the max list size.
+        if isinstance(value, int) and value > DMS_CONTAINER_LIST_MAX_LIMIT:
+            raise ValueError(f"Max list size cannot be greater than {DMS_CONTAINER_LIST_MAX_LIMIT}")
         return value
 
     @field_validator("value_type", mode="after")

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/cognite/neat/_rules/models/dms/_rules_input.py

@@ -1,4 +1,6 @@
 import re
+import sys
+import warnings
 from dataclasses import dataclass
 from datetime import datetime
 from typing import Any, Literal
@@ -8,7 +10,8 @@ from cognite.client import data_modeling as dm
 from cognite.client.data_classes.data_modeling import ContainerId, ViewId
 from rdflib import Namespace, URIRef
 
-from cognite.neat._constants import DEFAULT_NAMESPACE
+from cognite.neat._constants import DEFAULT_NAMESPACE, DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT
+from cognite.neat._issues.warnings import DeprecatedWarning
 from cognite.neat._rules.models._base_input import InputComponent, InputRules
 from cognite.neat._rules.models.data_types import DataType
 from cognite.neat._rules.models.entities import (
@@ -26,6 +29,11 @@ from cognite.neat._utils.rdf_ import uri_display_name
 
 from ._rules import _DEFAULT_VERSION, DMSContainer, DMSEnum, DMSMetadata, DMSNode, DMSProperty, DMSRules, DMSView
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
 
 @dataclass
 class DMSInputMetadata(InputComponent[DMSMetadata]):
@@ -106,9 +114,9 @@ class DMSInputProperty(InputComponent[DMSProperty]):
     name: str | None = None
     description: str | None = None
     connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | str | None = None
-    nullable: bool | None = None
+    min_count: int | None = None
+    max_count: int | float | None = None
     immutable: bool | None = None
-    is_list: bool | None = None
     default: str | int | dict | None = None
     container: str | None = None
     container_property: str | None = None
@@ -117,6 +125,19 @@ class DMSInputProperty(InputComponent[DMSProperty]):
     neatId: str | URIRef | None = None
     logical: str | URIRef | None = None
 
+    @property
+    def nullable(self) -> bool | None:
+        """Used to indicate whether the property is required or not. Only applies to primitive type."""
+        return self.min_count in {0, None}
+
+    @property
+    def is_list(self) -> bool | None:
+        """Used to indicate whether the property holds single or multiple values (list). "
+        "Only applies to primitive types."""
+        return self.max_count in {float("inf"), None} or (
+            isinstance(self.max_count, int | float) and self.max_count > 1
+        )
+
     @classmethod
     def _get_verified_cls(cls) -> type[DMSProperty]:
         return DMSProperty
@@ -139,6 +160,44 @@ class DMSInputProperty(InputComponent[DMSProperty]):
     def referenced_container(self, default_space: str) -> ContainerEntity | None:
         return ContainerEntity.load(self.container, strict=True, space=default_space) if self.container else None
 
+    @classmethod
+    def _load(cls, data: dict[str, Any]) -> Self:
+        # For backwards compatability, we need to convert nullable and Is List to min and max count
+        for min_count_key, nullable_key in [("Min Count", "Nullable"), ("min_count", "nullable")]:
+            if nullable_key in data and min_count_key not in data:
+                if isinstance(data[nullable_key], bool | float):
+                    data[min_count_key] = 0 if data[nullable_key] else 1
+                else:
+                    data[min_count_key] = None
+                warnings.warn(
+                    DeprecatedWarning(f"{nullable_key} column", replacement=f"{min_count_key} column"), stacklevel=2
+                )
+                data.pop(nullable_key)
+                break
+        for max_count_key, is_list_key, connection_key in [
+            ("Max Count", "Is List", "Connection"),
+            ("max_count", "is_list", "connection"),
+        ]:
+            if is_list_key in data and max_count_key not in data:
+                if isinstance(data[is_list_key], bool | float):
+                    if not data[is_list_key]:
+                        data[max_count_key] = 1
+                    elif "direct" in (data.get(connection_key, "") or "") and "edge" not in (
+                        data.get(connection_key, "") or ""
+                    ):
+                        data[max_count_key] = DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT
+                    else:
+                        # Reverse or edge connection
+                        data[max_count_key] = float("inf")
+                else:
+                    data[max_count_key] = 1
+                warnings.warn(
+                    DeprecatedWarning(f"{is_list_key} column", replacement=f"{max_count_key} column"), stacklevel=2
+                )
+                data.pop(is_list_key)
+                break
+        return super()._load(data)
+
 
 @dataclass
 class DMSInputContainer(InputComponent[DMSContainer]):

{cognite_neat-0.114.0 → cognite_neat-0.115.0}/cognite/neat/_rules/models/information/_rules.py

@@ -237,18 +237,6 @@ class InformationProperty(SheetRow):
         else:
             return EntityTypes.undefined
 
-    @property
-    def is_mandatory(self) -> bool:
-        """Returns True if property is mandatory."""
-        return self.min_count not in {0, None}
-
-    @property
-    def is_list(self) -> bool:
-        """Returns True if property contains a list of values."""
-        return self.max_count in {float("inf"), None} or (
-            isinstance(self.max_count, int | float) and self.max_count > 1
-        )
-
 
 class InformationRules(BaseRules):
     metadata: InformationMetadata = Field(alias="Metadata", description="Metadata for the logical data model")