cognite-neat 0.117.9__tar.gz → 0.117.10__tar.gz

{cognite_neat-0.117.9 → cognite_neat-0.117.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cognite-neat
-Version: 0.117.9
+Version: 0.117.10
 Summary: Knowledge graph transformation
 License: Apache-2.0
 Author: Nikola Vasiljevic

{cognite_neat-0.117.9 → cognite_neat-0.117.10}/cognite/neat/_client/data_classes/schema.py

@@ -307,6 +307,45 @@ class DMSSchema:
                 resources.append(loaded_instance)
         return resources
 
+    @classmethod
+    def from_read_model(cls, model: dm.DataModel[dm.View]) -> Self:
+        """Load schema from a read model.
+
+        CAVEAT: This method infers the containers and spaces from the views. This means that
+        for example indexes and constraints will not be captured in the containers.
+
+        Args:
+            model (dm.DataModel): The read model to load the schema from.
+        """
+        write_model = model.as_write()
+        write_model.views = [view.as_id() for view in model.views or []]
+        views = ViewApplyDict([view.as_write() for view in model.views])
+        containers = ContainerApplyDict()
+        for view in model.views:
+            for prop in view.properties.values():
+                if not isinstance(prop, dm.MappedProperty):
+                    continue
+                if prop.container not in containers:
+                    containers[prop.container] = dm.ContainerApply(
+                        space=prop.container.space,
+                        external_id=prop.container.external_id,
+                        properties={},
+                        used_for=view.used_for,
+                    )
+                containers[prop.container].properties[prop.container_property_identifier] = dm.ContainerProperty(
+                    type=prop.type,
+                    nullable=prop.nullable,
+                    auto_increment=prop.auto_increment,
+                    immutable=prop.immutable,
+                    default_value=prop.default_value,
+                    name=prop.name,
+                    description=prop.description,
+                )
+
+        schema = cls(data_model=write_model, views=views, containers=containers)
+        schema.spaces = SpaceApplyDict(dm.SpaceApply(space) for space in schema.referenced_spaces())
+        return schema
+
     def dump(self, camel_case: bool = True, sort: bool = True) -> dict[str, Any]:
         """Dump the schema to a dictionary that can be serialized to JSON.
 

{cognite_neat-0.117.9 → cognite_neat-0.117.10}/cognite/neat/_graph/loaders/_rdf2dms.py

@@ -205,6 +205,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                 for missing_view in missing:
                     issues.append(ResourceNotFoundError(missing_view, "view", more="The view is not found in CDF."))
                 return [], issues
+            self._lookup_max_limits_size(self._client, views)
         else:
             views = dm.ViewList([])
             with catch_issues() as issues:
@@ -231,6 +232,29 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             view_iterations.append(view_iteration)
         return view_iterations, issues
 
+    @staticmethod
+    def _lookup_max_limits_size(client: NeatClient, views: dm.ViewList) -> None:
+        """For listable container properties (mapped properties), the read version of the view does not
+        contain the max_list_size. This method will lookup the max_list_size from the containers definitions."""
+        containers = client.data_modeling.containers.retrieve(list(views.referenced_containers()))
+        properties_by_container_and_prop_id = {
+            (container.as_id(), prop_id): prop
+            for container in containers
+            for prop_id, prop in container.properties.items()
+        }
+
+        for view in views:
+            for prop in view.properties.values():
+                if not isinstance(prop, dm.MappedProperty):
+                    continue
+                if not isinstance(prop.type, ListablePropertyType):
+                    continue
+                prop_definition = properties_by_container_and_prop_id.get(
+                    (prop.container, prop.container_property_identifier)
+                )
+                if prop_definition and isinstance(prop_definition.type, ListablePropertyType):
+                    prop.type.max_list_size = prop_definition.type.max_list_size
+
     def _select_views_with_instances(self, view_query_by_id: ViewQueryDict) -> dict[dm.ViewId, _ViewIterator]:
         """Selects the views with data."""
         view_iterations: dict[dm.ViewId, _ViewIterator] = {}
@@ -414,23 +438,27 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             def parse_direct_relation(cls: Any, value: list, info: ValidationInfo) -> dict | list[dict]:
                 # We validate above that we only get one value for single direct relations.
                 if list.__name__ in _get_field_value_types(cls, info):
-                    ids = (self._create_instance_id(v, "node", stop_on_exception=True) for v in value)
-                    result = [id_.dump(camel_case=True, include_instance_type=False) for id_ in ids]
-                    # Todo: Account for max_list_limit
-                    if len(result) <= DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT:
-                        return result
-                    warnings.warn(
-                        PropertyDirectRelationLimitWarning(
-                            identifier="unknown",
-                            resource_type="view property",
-                            property_name=cast(str, cls.model_fields[info.field_name].alias or info.field_name),
-                            limit=DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT,
-                        ),
-                        stacklevel=2,
+                    # To get deterministic results
+                    value.sort()
+                    limit = (
+                        # We know that info.field_name will always be set due to *direct_relation_by_property.keys()
+                        direct_relation_by_property[cast(str, info.field_name)].max_list_size
+                        or DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT
                     )
-                    # To get deterministic results, we sort by space and externalId
-                    result.sort(key=lambda x: (x["space"], x["externalId"]))
-                    return result[:DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT]
+                    if len(value) > limit:
+                        warnings.warn(
+                            PropertyDirectRelationLimitWarning(
+                                identifier="unknown",
+                                resource_type="view property",
+                                property_name=cast(str, cls.model_fields[info.field_name].alias or info.field_name),
+                                limit=limit,
+                            ),
+                            stacklevel=2,
+                        )
+                        value = value[:limit]
+
+                    ids = (self._create_instance_id(v, "node", stop_on_exception=True) for v in value)
+                    return [id_.dump(camel_case=True, include_instance_type=False) for id_ in ids]
                 elif value:
                     return self._create_instance_id(value[0], "node", stop_on_exception=True).dump(
                         camel_case=True, include_instance_type=False

cognite_neat-0.117.10/cognite/neat/_version.py

@@ -0,0 +1,2 @@
+__version__ = "0.117.10"
+__engine__ = "^2.0.4"

{cognite_neat-0.117.9 → cognite_neat-0.117.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.117.9"
+version = "0.117.10"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.117.9/cognite/neat/_version.py

@@ -1,2 +0,0 @@
-__version__ = "0.117.9"
-__engine__ = "^2.0.4"