cognite-neat 0.75.9__py3-none-any.whl → 0.76.1__py3-none-any.whl

cognite/neat/rules/models/rules/_dms_rules_write.py

@@ -15,7 +15,7 @@ from cognite.neat.rules.models.entities import (
     ViewPropertyEntity,
 )
 
-from ._base import ExtensionCategory, SchemaCompleteness
+from ._base import DataModelType, ExtensionCategory, SchemaCompleteness
 from ._dms_architect_rules import DMSContainer, DMSMetadata, DMSProperty, DMSRules, DMSView
 
 
@@ -27,11 +27,11 @@ class DMSMetadataWrite:
     creator: str
     version: str
     extension: Literal["addition", "reshape", "rebuild"] = "addition"
+    data_model_type: Literal["solution", "enterprise"] = "solution"
     name: str | None = None
     description: str | None = None
     created: datetime | str | None = None
     updated: datetime | str | None = None
-    default_view_version: str | None = None
 
     @classmethod
     def load(cls, data: dict[str, Any] | None) -> "DMSMetadataWrite | None":
@@ -45,11 +45,11 @@ class DMSMetadataWrite:
             creator=data.get("creator"),  # type: ignore[arg-type]
             version=data.get("version"),  # type: ignore[arg-type]
             extension=data.get("extension", "addition"),
+            data_model_type=data.get("data_model_type", "solution"),
             name=data.get("name"),
             description=data.get("description"),
             created=data.get("created"),
             updated=data.get("updated"),
-            default_view_version=data.get("default_view_version"),
         )
 
     def dump(self) -> dict[str, Any]:
@@ -58,13 +58,13 @@ class DMSMetadataWrite:
             extension=ExtensionCategory(self.extension),
             space=self.space,
             externalId=self.external_id,
+            dataModelType=DataModelType(self.data_model_type),
             creator=self.creator,
             version=self.version,
             name=self.name,
             description=self.description,
             created=self.created or datetime.now(),
             updated=self.updated or datetime.now(),
-            default_view_version=self.default_view_version or self.version,
         )
 
 
@@ -77,7 +77,7 @@ class DMSPropertyWrite:
     class_: str | None = None
     name: str | None = None
     description: str | None = None
-    relation: Literal["direct", "reversedirect", "multiedge"] | None = None
+    connection: Literal["direct", "edge", "reverse"] | None = None
     nullable: bool | None = None
     is_list: bool | None = None
     default: str | int | dict | None = None
@@ -118,7 +118,7 @@ class DMSPropertyWrite:
             class_=data.get("class_"),
             name=data.get("name"),
             description=data.get("description"),
-            relation=data.get("relation"),
+            connection=data.get("connection"),
             nullable=data.get("nullable"),
             is_list=data.get("is_list"),
             default=data.get("default"),
@@ -143,23 +143,25 @@ class DMSPropertyWrite:
 
         return {
             "View": ViewEntity.load(self.view, space=default_space, version=default_version),
-            "ViewProperty": self.view_property,
+            "View Property": self.view_property,
             "Value Type": value_type,
-            "Property": self.property_ or self.view_property,
-            "Class": ClassEntity.load(self.class_, prefix=default_space, version=default_version)
-            if self.class_
-            else None,
+            "Property (linage)": self.property_ or self.view_property,
+            "Class (linage)": (
+                ClassEntity.load(self.class_, prefix=default_space, version=default_version) if self.class_ else None
+            ),
             "Name": self.name,
             "Description": self.description,
-            "Relation": self.relation,
+            "Connection": self.connection,
             "Nullable": self.nullable,
-            "IsList": self.is_list,
+            "Is List": self.is_list,
             "Default": self.default,
             "Reference": self.reference,
-            "Container": ContainerEntity.load(self.container, space=default_space, version=default_version)
-            if self.container
-            else None,
-            "ContainerProperty": self.container_property,
+            "Container": (
+                ContainerEntity.load(self.container, space=default_space, version=default_version)
+                if self.container
+                else None
+            ),
+            "Container Property": self.container_property,
             "Index": self.index,
             "Constraint": self.constraint,
         }
@@ -208,19 +210,23 @@ class DMSContainerWrite:
 
     def dump(self, default_space: str) -> dict[str, Any]:
         container = ContainerEntity.load(self.container, space=default_space)
-        return dict(
-            Container=container,
-            Class=ClassEntity.load(self.class_, prefix=default_space) if self.class_ else container.as_class(),
-            Name=self.name,
-            Description=self.description,
-            Reference=self.reference,
-            Constraint=[
-                ContainerEntity.load(constraint.strip(), space=default_space)
-                for constraint in self.constraint.split(",")
-            ]
-            if self.constraint
-            else None,
-        )
+        return {
+            "Container": container,
+            "Class (linage)": (
+                ClassEntity.load(self.class_, prefix=default_space) if self.class_ else container.as_class()
+            ),
+            "Name": self.name,
+            "Description": self.description,
+            "Reference": self.reference,
+            "Constraint": (
+                [
+                    ContainerEntity.load(constraint.strip(), space=default_space)
+                    for constraint in self.constraint.split(",")
+                ]
+                if self.constraint
+                else None
+            ),
+        }
 
 
 @dataclass
@@ -268,23 +274,27 @@ class DMSViewWrite:
 
     def dump(self, default_space: str, default_version: str) -> dict[str, Any]:
         view = ViewEntity.load(self.view, space=default_space, version=default_version)
-        return dict(
-            View=view,
-            Class=ClassEntity.load(self.class_, prefix=default_space, version=default_version)
-            if self.class_
-            else view.as_class(),
-            Name=self.name,
-            Description=self.description,
-            Implements=[
-                ViewEntity.load(implement, space=default_space, version=default_version)
-                for implement in self.implements.split(",")
-            ]
-            if self.implements
-            else None,
-            Reference=self.reference,
-            Filter=self.filter_,
-            InModel=self.in_model,
-        )
+        return {
+            "View": view,
+            "Class (linage)": (
+                ClassEntity.load(self.class_, prefix=default_space, version=default_version)
+                if self.class_
+                else view.as_class()
+            ),
+            "Name": self.name,
+            "Description": self.description,
+            "Implements": (
+                [
+                    ViewEntity.load(implement, space=default_space, version=default_version)
+                    for implement in self.implements.split(",")
+                ]
+                if self.implements
+                else None
+            ),
+            "Reference": self.reference,
+            "Filter": self.filter_,
+            "In Model": self.in_model,
+        }
 
 
 @dataclass

cognite/neat/rules/models/rules/_dms_schema.py

@@ -3,7 +3,7 @@ import sys
 import warnings
 import zipfile
 from collections import Counter, defaultdict
-from dataclasses import dataclass, field, fields
+from dataclasses import Field, dataclass, field, fields
 from pathlib import Path
 from typing import Any, ClassVar, cast
 
@@ -14,6 +14,7 @@ from cognite.client.data_classes import DatabaseWrite, DatabaseWriteList, Transf
 from cognite.client.data_classes.data_modeling import ViewApply
 from cognite.client.data_classes.transformations.common import Edges, EdgeType, Nodes, ViewInfo
 
+from cognite.neat.rules import issues
 from cognite.neat.rules.issues.dms import (
     ContainerPropertyUsedMultipleTimesError,
     DirectRelationMissingSourceWarning,
@@ -106,23 +107,33 @@ class DMSSchema:
         The directory is expected to follow the Cognite-Toolkit convention
         where each file is named as `resource_type.resource_name.yaml`.
         """
-        data = cls._read_directory(Path(directory))
-        return cls.load(data)
+        data, context = cls._read_directory(Path(directory))
+        return cls.load(data, context)
 
     @classmethod
-    def _read_directory(cls, directory: Path) -> dict[str, list[Any]]:
+    def _read_directory(cls, directory: Path) -> tuple[dict[str, list[Any]], dict[str, list[Path]]]:
         data: dict[str, Any] = {}
+        context: dict[str, list[Path]] = {}
         for yaml_file in directory.rglob("*.yaml"):
             if "." in yaml_file.stem:
                 resource_type = yaml_file.stem.rsplit(".", 1)[-1]
                 if attr_name := cls._FIELD_NAME_BY_RESOURCE_TYPE.get(resource_type):
                     data.setdefault(attr_name, [])
-                    loaded = yaml.safe_load(yaml_file.read_text())
+                    context.setdefault(attr_name, [])
+                    try:
+                        # Using CSafeLoader over safe_load for ~10x speedup
+                        loaded = yaml.safe_load(yaml_file.read_text())
+                    except Exception as e:
+                        warnings.warn(issues.fileread.InvalidFileFormatWarning(yaml_file, str(e)), stacklevel=2)
+                        continue
+
                     if isinstance(loaded, list):
                         data[attr_name].extend(loaded)
+                        context[attr_name].extend([yaml_file] * len(loaded))
                     else:
                         data[attr_name].append(loaded)
-        return data
+                        context[attr_name].append(yaml_file)
+        return data, context
 
     def to_directory(
         self,
@@ -182,12 +193,13 @@ class DMSSchema:
         The ZIP file is expected to follow the Cognite-Toolkit convention
         where each file is named as `resource_type.resource_name.yaml`.
         """
-        data = cls._read_zip(Path(zip_file))
-        return cls.load(data)
+        data, context = cls._read_zip(Path(zip_file))
+        return cls.load(data, context)
 
     @classmethod
-    def _read_zip(cls, zip_file: Path) -> dict[str, list[Any]]:
+    def _read_zip(cls, zip_file: Path) -> tuple[dict[str, list[Any]], dict[str, list[Path]]]:
         data: dict[str, list[Any]] = {}
+        context: dict[str, list[Path]] = {}
         with zipfile.ZipFile(zip_file, "r") as zip_ref:
             for file_info in zip_ref.infolist():
                 if file_info.filename.endswith(".yaml"):
@@ -199,12 +211,19 @@ class DMSSchema:
                     resource_type = filename.stem.rsplit(".", 1)[-1]
                     if attr_name := cls._FIELD_NAME_BY_RESOURCE_TYPE.get(resource_type):
                         data.setdefault(attr_name, [])
-                        loaded = yaml.safe_load(zip_ref.read(file_info).decode())
+                        context.setdefault(attr_name, [])
+                        try:
+                            loaded = yaml.safe_load(zip_ref.read(file_info).decode())
+                        except Exception as e:
+                            warnings.warn(issues.fileread.InvalidFileFormatWarning(filename, str(e)), stacklevel=2)
+                            continue
                         if isinstance(loaded, list):
                             data[attr_name].extend(loaded)
+                            context[attr_name].extend([filename] * len(loaded))
                         else:
                             data[attr_name].append(loaded)
-        return data
+                            context[attr_name].append(filename)
+        return data, context
 
     def to_zip(self, zip_file: str | Path, exclude: set[str] | None = None) -> None:
         """Save the schema to a ZIP file as YAML files. This is compatible with the Cognite-Toolkit convention.
@@ -234,18 +253,63 @@ class DMSSchema:
                     zip_ref.writestr(f"data_models/nodes/{node.external_id}.node.yaml", node.dump_yaml())
 
     @classmethod
-    def load(cls, data: str | dict[str, Any]) -> Self:
+    def load(cls, data: str | dict[str, list[Any]], context: dict[str, list[Path]] | None = None) -> Self:
+        """Loads a schema from a dictionary or a YAML or JSON formatted string.
+
+        Args:
+            data: The data to load the schema from. This can be a dictionary, a YAML or JSON formatted string.
+            context: This provides linage for where the data was loaded from. This is used in Warnings
+                if a single item fails to load.
+
+        Returns:
+            DMSSchema: The loaded schema.
+        """
+        context = context or {}
         if isinstance(data, str):
             # YAML is a superset of JSON, so we can use the same parser
-            data_dict = yaml.safe_load(data)
+            try:
+                data_dict = yaml.safe_load(data)
+            except Exception as e:
+                raise issues.fileread.FailedStringLoadError(".yaml", str(e)).as_exception() from None
+            if not isinstance(data_dict, dict) and all(isinstance(v, list) for v in data_dict.values()):
+                raise issues.fileread.FailedStringLoadError(
+                    "dict[str, list[Any]]", f"Invalid data structure: {type(data)}"
+                ).as_exception() from None
         else:
             data_dict = data
         loaded: dict[str, Any] = {}
         for attr in fields(cls):
             if items := data_dict.get(attr.name) or data_dict.get(to_camel(attr.name)):
-                loaded[attr.name] = attr.type.load(items)
+                try:
+                    loaded[attr.name] = attr.type.load(items)
+                except Exception as e:
+                    loaded[attr.name] = cls._load_individual_resources(items, attr, str(e), context.get(attr.name, []))
         return cls(**loaded)
 
+    @classmethod
+    def _load_individual_resources(cls, items: list, attr: Field, trigger_error: str, resource_context) -> list[Any]:
+        resources = attr.type([])
+        if not hasattr(attr.type, "_RESOURCE"):
+            warnings.warn(
+                issues.fileread.FailedLoadWarning(Path("UNKNOWN"), attr.type.__name__, trigger_error), stacklevel=2
+            )
+            return resources
+        # Fallback to load individual resources.
+        single_cls = attr.type._RESOURCE
+        for no, item in enumerate(items):
+            try:
+                loaded_instance = single_cls.load(item)
+            except Exception as e:
+                try:
+                    filepath = resource_context[no]
+                except IndexError:
+                    filepath = Path("UNKNOWN")
+                # We use repr(e) instead of str(e) to include the exception type in the warning message
+                warnings.warn(issues.fileread.FailedLoadWarning(filepath, single_cls.__name__, repr(e)), stacklevel=2)
+            else:
+                resources.append(loaded_instance)
+        return resources
+
     def dump(self, camel_case: bool = True, sort: bool = True) -> dict[str, Any]:
         """Dump the schema to a dictionary that can be serialized to JSON.
 
@@ -410,6 +474,18 @@ class DMSSchema:
             )
         return None
 
+    def referenced_spaces(self) -> set[str]:
+        referenced_spaces = {container.space for container in self.containers}
+        referenced_spaces |= {view.space for view in self.views}
+        referenced_spaces |= {container.space for view in self.views for container in view.referenced_containers()}
+        referenced_spaces |= {parent.space for view in self.views for parent in view.implements or []}
+        referenced_spaces |= {node.space for node in self.node_types}
+        referenced_spaces |= {model.space for model in self.data_models}
+        referenced_spaces |= {view.space for model in self.data_models for view in model.views or []}
+        referenced_spaces |= {s.space for s in self.spaces}
+
+        return referenced_spaces
+
 
 @dataclass
 class PipelineSchema(DMSSchema):
@@ -429,18 +505,25 @@ class PipelineSchema(DMSSchema):
             self.databases.extend([DatabaseWrite(name=database) for database in missing])
 
     @classmethod
-    def _read_directory(cls, directory: Path) -> dict[str, list[Any]]:
-        data = super()._read_directory(directory)
+    def _read_directory(cls, directory: Path) -> tuple[dict[str, list[Any]], dict[str, list[Path]]]:
+        data, context = super()._read_directory(directory)
         for yaml_file in directory.rglob("*.yaml"):
             if yaml_file.parent.name in ("transformations", "raw"):
                 attr_name = cls._FIELD_NAME_BY_RESOURCE_TYPE.get(yaml_file.parent.name, yaml_file.parent.name)
                 data.setdefault(attr_name, [])
-                loaded = yaml.safe_load(yaml_file.read_text())
+                context.setdefault(attr_name, [])
+                try:
+                    loaded = yaml.safe_load(yaml_file.read_text())
+                except Exception as e:
+                    warnings.warn(issues.fileread.InvalidFileFormatWarning(yaml_file, str(e)), stacklevel=2)
+                    continue
                 if isinstance(loaded, list):
                     data[attr_name].extend(loaded)
+                    context[attr_name].extend([yaml_file] * len(loaded))
                 else:
                     data[attr_name].append(loaded)
-        return data
+                    context[attr_name].append(yaml_file)
+        return data, context
 
     def to_directory(
         self,
@@ -483,8 +566,8 @@ class PipelineSchema(DMSSchema):
                     zip_ref.writestr(f"raw/{raw_table.name}.yaml", raw_table.dump_yaml())
 
     @classmethod
-    def _read_zip(cls, zip_file: Path) -> dict[str, list[Any]]:
-        data = super()._read_zip(zip_file)
+    def _read_zip(cls, zip_file: Path) -> tuple[dict[str, list[Any]], dict[str, list[Path]]]:
+        data, context = super()._read_zip(zip_file)
         with zipfile.ZipFile(zip_file, "r") as zip_ref:
             for file_info in zip_ref.infolist():
                 if file_info.filename.endswith(".yaml"):
@@ -494,12 +577,19 @@ class PipelineSchema(DMSSchema):
                     if (parent := filepath.parent.name) in ("transformations", "raw"):
                         attr_name = cls._FIELD_NAME_BY_RESOURCE_TYPE.get(parent, parent)
                         data.setdefault(attr_name, [])
-                        loaded = yaml.safe_load(zip_ref.read(file_info).decode())
+                        context.setdefault(attr_name, [])
+                        try:
+                            loaded = yaml.safe_load(zip_ref.read(file_info).decode())
+                        except Exception as e:
+                            warnings.warn(issues.fileread.InvalidFileFormatWarning(filepath, str(e)), stacklevel=2)
+                            continue
                         if isinstance(loaded, list):
                             data[attr_name].extend(loaded)
+                            context[attr_name].extend([filepath] * len(loaded))
                         else:
                             data[attr_name].append(loaded)
-        return data
+                            context[attr_name].append(filepath)
+        return data, context
 
     @classmethod
     def from_dms(cls, schema: DMSSchema, instance_space: str | None = None) -> "PipelineSchema":

cognite/neat/rules/models/rules/_domain_rules.py

@@ -23,7 +23,10 @@ class DomainMetadata(BaseMetadata):
 
 
 class DomainProperty(SheetEntity):
+    class_: ClassEntity = Field(alias="Class")
     property_: PropertyType = Field(alias="Property")
+    name: str | None = Field(alias="Name", default=None)
+    description: str | None = Field(alias="Description", default=None)
     value_type: DataType | ClassEntity = Field(alias="Value Type")
     min_count: int | None = Field(alias="Min Count", default=None)
     max_count: int | float | None = Field(alias="Max Count", default=None)
@@ -42,6 +45,8 @@ class DomainProperty(SheetEntity):
 
 
 class DomainClass(SheetEntity):
+    class_: ClassEntity = Field(alias="Class")
+    name: str | None = Field(alias="Name", default=None)
     description: str | None = Field(None, alias="Description")
     parent: ParentEntityList | None = Field(alias="Parent Class")
 

cognite/neat/rules/models/rules/_information_rules.py

@@ -45,6 +45,7 @@ from cognite.neat.rules.models.rdfpath import (
 
 from ._base import (
     BaseMetadata,
+    DataModelType,
     ExtensionCategory,
     ExtensionCategoryType,
     MatchType,
@@ -75,6 +76,7 @@ else:
 
 class InformationMetadata(BaseMetadata):
     role: ClassVar[RoleTypes] = RoleTypes.information_architect
+    data_model_type: DataModelType = Field(DataModelType.solution, alias="dataModelType")
     schema_: SchemaCompleteness = Field(alias="schema")
     extension: ExtensionCategoryType | None = ExtensionCategory.addition
     prefix: PrefixType
@@ -123,6 +125,9 @@ class InformationClass(SheetEntity):
         match_type: The match type of the resource being described and the source entity.
     """
 
+    class_: ClassEntity = Field(alias="Class")
+    name: str | None = Field(alias="Name", default=None)
+    description: str | None = Field(alias="Description", default=None)
     parent: ParentEntityList | None = Field(alias="Parent Class", default=None)
     reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
     match_type: MatchType | None = Field(alias="Match Type", default=None)
@@ -150,7 +155,10 @@ class InformationProperty(SheetEntity):
               knowledge graph. Defaults to None (no transformation)
     """
 
+    class_: ClassEntity = Field(alias="Class")
     property_: PropertyType = Field(alias="Property")
+    name: str | None = Field(alias="Name", default=None)
+    description: str | None = Field(alias="Description", default=None)
     value_type: DataType | ClassEntity | UnknownEntity = Field(alias="Value Type", union_mode="left_to_right")
     min_count: int | None = Field(alias="Min Count", default=None)
     max_count: int | float | None = Field(alias="Max Count", default=None)
@@ -430,7 +438,7 @@ class _InformationRulesConverter:
         for class_ in self.information.classes:
             properties: list[DMSProperty] = properties_by_class.get(class_.class_.versioned_id, [])
             if not properties or all(
-                isinstance(prop.value_type, ViewPropertyEntity) and prop.relation != "direct" for prop in properties
+                isinstance(prop.value_type, ViewPropertyEntity) and prop.connection != "direct" for prop in properties
             ):
                 classes_without_properties.add(class_.class_.versioned_id)
 
@@ -480,16 +488,15 @@ class _InformationRulesConverter:
         else:
             raise ValueError(f"Unsupported value type: {prop.value_type.type_}")
 
-        relation: Literal["direct", "multiedge"] | None = None
+        relation: Literal["direct", "edge", "reverse"] | None = None
         if isinstance(value_type, ViewEntity | ViewPropertyEntity):
-            relation = "multiedge" if prop.is_list else "direct"
+            relation = "edge" if prop.is_list else "direct"
 
         container: ContainerEntity | None = None
         container_property: str | None = None
         is_list: bool | None = prop.is_list
         nullable: bool | None = not prop.is_mandatory
-        if relation == "multiedge":
-            is_list = None
+        if relation == "edge":
             nullable = None
         elif relation == "direct":
             nullable = True
@@ -504,7 +511,7 @@ class _InformationRulesConverter:
             value_type=value_type,
             nullable=nullable,
             is_list=is_list,
-            relation=relation,
+            connection=relation,
             default=prop.default,
             reference=prop.reference,
             container=container,