cognite-neat 0.123.32__py3-none-any.whl → 0.127.30__py3-none-any.whl

cognite/neat/_config.py

@@ -0,0 +1,185 @@
+import sys
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from cognite.neat._issues import ConsistencyError, ModelSyntaxError
+from cognite.neat._utils.useful_types import ModusOperandi
+
+if sys.version_info >= (3, 11):
+    import tomllib as tomli  # Python 3.11+
+else:
+    import tomli  # type: ignore
+
+
+class ValidationConfig(BaseModel, populate_by_name=True):
+    """Validation configuration."""
+
+    exclude: list[str] = Field(default_factory=list)
+
+    def can_run_validator(self, code: str, issue_type: type) -> bool:
+        """
+        Check if a specific validator should run.
+
+        Args:
+            code: Validation code (e.g., "NEAT-DMS-CONTAINER-001")
+            issue_type: Issue type (e.g., ModelSyntaxError, ConsistencyError, Recommendation)
+
+        Returns:
+            True if validator should run, False otherwise
+        """
+
+        is_excluded = self._is_excluded(code, self.exclude)
+
+        if issue_type in [ModelSyntaxError, ConsistencyError] and is_excluded:
+            print(f"Validator {code} was excluded however it is a critical validator and will still run.")
+            return True
+        else:
+            return not is_excluded
+
+    @classmethod
+    def _is_excluded(cls, code: str, patterns: list[str]) -> bool:
+        """Check if code matches any pattern (supports wildcards)."""
+        for pattern in patterns:
+            if "*" in pattern:
+                # Split both pattern and code by hyphens
+                pattern_parts = pattern.split("-")
+                code_parts = code.split("-")
+
+                # Pattern must have same or fewer parts than code
+                if len(pattern_parts) > len(code_parts):
+                    continue
+
+                # Check if all pattern parts match (allowing wildcards)
+                match = True
+                for p_part, c_part in zip(pattern_parts, code_parts, strict=False):
+                    if p_part != "*" and p_part != c_part:
+                        match = False
+                        break
+
+                if match:
+                    return True
+            elif code == pattern:
+                return True
+
+        return False
+
+    def __str__(self) -> str:
+        """Human-readable configuration summary."""
+        if not self.exclude:
+            return "All validators enabled"
+        return f"Excluded Rules: {', '.join(self.exclude)}"
+
+
+class ModelingConfig(BaseModel, populate_by_name=True):
+    """Modeling configuration."""
+
+    mode: ModusOperandi = "additive"
+
+
+class NeatConfig(BaseModel, populate_by_name=True):
+    """Configuration for a custom profile."""
+
+    profile: str
+    validation: ValidationConfig
+    modeling: ModelingConfig
+
+    def __str__(self) -> str:
+        """Human-readable configuration summary."""
+        lines = [
+            f"Profile: {self.profile}",
+            f"Modeling Mode: {self.modeling.mode}",
+            f"Validation: {self.validation}",
+        ]
+        return "\n".join(lines)
+
+
+def internal_profiles() -> dict[str, NeatConfig]:
+    """Get internal NeatConfig profile by name."""
+    return {
+        "legacy-additive": NeatConfig(
+            profile="legacy-additive",
+            modeling=ModelingConfig(mode="additive"),
+            validation=ValidationConfig(
+                exclude=[
+                    "NEAT-DMS-AI-READINESS-*",
+                    "NEAT-DMS-CONNECTIONS-002",
+                    "NEAT-DMS-CONNECTIONS-REVERSE-007",
+                    "NEAT-DMS-CONNECTIONS-REVERSE-008",
+                    "NEAT-DMS-CONSISTENCY-001",
+                ]
+            ),
+        ),
+        "legacy-rebuild": NeatConfig(
+            profile="legacy-rebuild",
+            modeling=ModelingConfig(mode="rebuild"),
+            validation=ValidationConfig(
+                exclude=[
+                    "NEAT-DMS-AI-READINESS-*",
+                    "NEAT-DMS-CONNECTIONS-002",
+                    "NEAT-DMS-CONNECTIONS-REVERSE-007",
+                    "NEAT-DMS-CONNECTIONS-REVERSE-008",
+                    "NEAT-DMS-CONSISTENCY-001",
+                ]
+            ),
+        ),
+        "deep-additive": NeatConfig(
+            profile="deep-additive",
+            modeling=ModelingConfig(mode="additive"),
+            validation=ValidationConfig(exclude=[]),
+        ),
+        "deep-rebuild": NeatConfig(
+            profile="deep-rebuild",
+            modeling=ModelingConfig(mode="rebuild"),
+            validation=ValidationConfig(exclude=[]),
+        ),
+    }
+
+
+def get_neat_config(config_file_name: str, profile: str) -> NeatConfig:
+    """Get NeatConfig from file or internal profiles.
+
+    Args:
+        config_file_name: Path to configuration file.
+        profile: Profile name to use.
+    Returns:
+        NeatConfig instance.
+    """
+
+    if not config_file_name.endswith(".toml"):
+        raise ValueError("config_file_name must end with '.toml'")
+
+    file_path = Path.cwd() / config_file_name
+
+    if file_path.exists():
+        with file_path.open("rb") as f:
+            toml = tomli.load(f)
+
+        if "tool" in toml and "neat" in toml["tool"]:
+            data = toml["tool"]["neat"]
+        elif "neat" in toml:
+            data = toml["neat"]
+        else:
+            raise ValueError("No [tool.neat] or [neat] section found in the configuration file.")
+
+        toml_profile = data.get("profile")
+        toml_profiles = data.get("profiles")
+        hardcoded_profiles = internal_profiles()
+
+        if toml_profile and toml_profile in hardcoded_profiles:
+            raise ValueError(f"Internal profile '{toml_profile}' cannot be used in external configuration file.")
+
+        if toml_profiles and any(p in hardcoded_profiles for p in toml_profiles.keys()):
+            raise ValueError(
+                "Internal profiles cannot be redefined in external configuration file: "
+                f"{set(hardcoded_profiles.keys()).intersection(toml_profiles.keys())}"
+            )
+
+        if toml_profile and profile == toml_profile:
+            return NeatConfig(**data)
+        elif (built_in_profiles := data.get("profiles")) and profile in built_in_profiles:
+            return NeatConfig(profile=profile, **data["profiles"][profile])
+        else:
+            raise ValueError(f"Profile '{profile}' not found in configuration file.")
+    else:
+        raise FileNotFoundError(f"Configuration file '{file_path}' not found.")

cognite/neat/_data_model/_analysis.py

@@ -0,0 +1,196 @@
+from graphlib import TopologicalSorter
+
+from cognite.neat._data_model.models.dms._constraints import RequiresConstraintDefinition
+from cognite.neat._data_model.models.dms._container import ContainerPropertyDefinition, ContainerRequest
+from cognite.neat._data_model.models.dms._data_types import DirectNodeRelation
+from cognite.neat._data_model.models.dms._references import (
+    ContainerDirectReference,
+    ContainerReference,
+    ViewDirectReference,
+    ViewReference,
+)
+from cognite.neat._data_model.models.dms._schema import RequestSchema
+from cognite.neat._data_model.models.dms._view_property import (
+    EdgeProperty,
+    ReverseDirectRelationProperty,
+    ViewCorePropertyRequest,
+    ViewRequestProperty,
+)
+from cognite.neat._data_model.models.dms._views import ViewRequest
+
+
+class DataModelAnalysis:
+    def __init__(
+        self,
+        physical: RequestSchema | None = None,
+    ) -> None:
+        self._physical = physical
+
+    @property
+    def physical(self) -> RequestSchema:
+        if self._physical is None:
+            raise ValueError("Physical Data Model is required for this analysis")
+        return self._physical
+
+    def referenced_views(self, include_connection_end_node_types: bool = False) -> set[ViewReference]:
+        """Get all referenced views in the physical data model."""
+        referenced_views = set()
+
+        for view in self.physical.views:
+            referenced_views.add(view.as_reference())
+            if view.implements:
+                for implement in view.implements:
+                    referenced_views.add(implement)
+
+        if include_connection_end_node_types:
+            referenced_views |= {view for view in self.connection_end_node_types.values() if view is not None}
+
+        return referenced_views
+
+    @property
+    def referenced_containers(self) -> set[ContainerReference]:
+        """Get all referenced containers in the physical data model."""
+        referenced_containers = set()
+
+        for view in self.physical.views:
+            for property_ in view.properties.values():
+                if isinstance(property_, ViewCorePropertyRequest):
+                    referenced_containers.add(property_.container)
+
+        for container in self.physical.containers:
+            referenced_containers.add(container.as_reference())
+            if container.constraints:
+                for constraint in container.constraints.values():
+                    if isinstance(constraint, RequiresConstraintDefinition):
+                        referenced_containers.add(constraint.require)
+
+        return referenced_containers
+
+    def view_by_reference(self, include_inherited_properties: bool = True) -> dict[ViewReference, ViewRequest]:
+        """Get a mapping of view references to their corresponding ViewRequest objects."""
+        view_ancestors = self.ancestors_by_view(self.physical.views)
+
+        view_by_reference: dict[ViewReference, ViewRequest] = {
+            view.as_reference(): view.model_copy(deep=True) for view in self.physical.views
+        }
+
+        if include_inherited_properties:
+            for ref, view in view_by_reference.items():
+                for ancestor in view_ancestors.get(ref, set()):
+                    if ancestor_view := view_by_reference.get(ancestor):
+                        if ancestor_view.properties:
+                            view.properties.update(ancestor_view.properties)
+
+        return view_by_reference
+
+    @property
+    def properties_by_view(self) -> dict[ViewReference, dict[str, ViewRequestProperty]]:
+        """Get a mapping of view references to their corresponding properties."""
+        view_by_reference = self.view_by_reference(include_inherited_properties=False)
+        return {
+            view_ref: {prop_name: prop for prop_name, prop in view.properties.items()}
+            for view_ref, view in view_by_reference.items()
+        }
+
+    @staticmethod
+    def ancestors_by_view(views: list[ViewRequest]) -> dict[ViewReference, set[ViewReference]]:
+        """Get a mapping of each view to its ancestors in the physical data model."""
+        implements_by_view = DataModelAnalysis.implements_by_view(views)
+
+        # Topological sort to ensure that concepts include all ancestors
+        for view in list(TopologicalSorter(implements_by_view).static_order()):
+            if view not in implements_by_view:
+                continue
+            implements_by_view[view] |= {
+                grand_parent
+                for parent in implements_by_view[view]
+                for grand_parent in implements_by_view.get(parent, set())
+            }
+
+        return implements_by_view
+
+    @staticmethod
+    def implements_by_view(views: list[ViewRequest]) -> dict[ViewReference, set[ViewReference]]:
+        """Get a mapping of each view to the views it implements."""
+        implements_mapping: dict[ViewReference, set[ViewReference]] = {}
+
+        for view in views:
+            view_ref = view.as_reference()
+            if view_ref not in implements_mapping:
+                implements_mapping[view_ref] = set()
+            if view.implements:
+                for implement in view.implements:
+                    implements_mapping[view_ref].add(implement)
+
+        return implements_mapping
+
+    @property
+    def container_by_reference(self) -> dict[ContainerReference, ContainerRequest]:
+        """Get a mapping of container references to their corresponding ContainerRequest objects."""
+        return {container.as_reference(): container.model_copy(deep=True) for container in self.physical.containers}
+
+    @property
+    def container_properties(self) -> dict[tuple[ContainerReference, str], ContainerPropertyDefinition]:
+        """Get a mapping from (container reference, property name) to the property definition."""
+
+        return {
+            (container.as_reference(), prop_name): property_
+            for container in self.physical.containers
+            for prop_name, property_ in container.properties.items()
+        }
+
+    @property
+    def connection_end_node_types(self) -> dict[tuple[ViewReference, str], ViewReference | None]:
+        """Get a mapping of view references to their corresponding ViewRequest objects."""
+        view_by_reference = self.view_by_reference(include_inherited_properties=False)
+        connection_end_node_types: dict[tuple[ViewReference, str], ViewReference | None] = {}
+        container_properties = self.container_properties
+
+        for view_ref, view in view_by_reference.items():
+            if not view.properties:
+                continue
+            for prop_ref, property_ in view.properties.items():
+                # direct relation
+                if isinstance(property_, ViewCorePropertyRequest):
+                    # explicit set of end node type via 'source' which is View reference
+                    if property_.source:
+                        connection_end_node_types[(view_ref, prop_ref)] = property_.source
+
+                    # implicit end node type via container property, without actual knowledge of end node type
+                    elif (
+                        container_property := container_properties.get(
+                            (property_.container, property_.container_property_identifier)
+                        )
+                    ) and isinstance(container_property.type, DirectNodeRelation):
+                        connection_end_node_types[(view_ref, prop_ref)] = None
+
+                # reverse direct relation
+                if isinstance(property_, ReverseDirectRelationProperty) and property_.source:
+                    connection_end_node_types[(view_ref, prop_ref)] = property_.source
+
+                # edge property
+                if isinstance(property_, EdgeProperty) and property_.source:
+                    connection_end_node_types[(view_ref, prop_ref)] = property_.source
+
+        return connection_end_node_types
+
+    @property
+    def reverse_to_direct_mapping(
+        self,
+    ) -> dict[tuple[ViewReference, str], tuple[ViewReference, ContainerDirectReference | ViewDirectReference]]:
+        """Get a mapping of reverse direct relations to their corresponding source view and 'through' property."""
+        view_by_reference = self.view_by_reference(include_inherited_properties=False)
+        bidirectional_connections = {}
+
+        for view_ref, view in view_by_reference.items():
+            if not view.properties:
+                continue
+            for prop_ref, property_ in view.properties.items():
+                # reverse direct relation
+                if isinstance(property_, ReverseDirectRelationProperty):
+                    bidirectional_connections[(view_ref, prop_ref)] = (
+                        property_.source,
+                        property_.through,
+                    )
+
+        return bidirectional_connections

cognite/neat/_data_model/_constants.py

@@ -0,0 +1,67 @@
+from ._identifiers import NameSpace
+
+XML_SCHEMA_NAMESPACE = NameSpace("http://www.w3.org/2001/XMLSchema#")
+
+CDF_CDM_SPACE = "cdf_cdm"
+CDF_CDM_VERSION = "v1"
+
+COGNITE_CONCEPTS_MAIN = (
+    "CogniteAsset",
+    "CogniteEquipment",
+    "CogniteActivity",
+    "CogniteTimeSeries",
+    "CogniteFile",
+)
+
+COGNITE_CONCEPTS_INTERFACES = (
+    "CogniteDescribable",
+    "CogniteSourceable",
+    "CogniteSchedulable",
+    "CogniteVisualizable",
+)
+
+COGNITE_CONCEPTS_CONFIGURATIONS = (
+    "CogniteSourceSystem",
+    "CogniteUnit",
+    "CogniteAssetClass",
+    "CogniteAssetType",
+    "CogniteEquipmentType",
+    "CogniteFileCategory",
+)
+COGNITE_CONCEPTS_ANNOTATIONS = (
+    "CogniteAnnotation",
+    "CogniteDiagramAnnotation",
+)
+COGNITE_CONCEPTS_3D = (
+    "CogniteCubeMap",
+    "CogniteCADRevision",
+    "CognitePointCloudVolume",
+    "Cognite360ImageAnnotation",
+    "Cognite3DObject",
+    "Cognite3DRevision",
+    "Cognite360Image",
+    "Cognite360ImageCollection",
+    "Cognite360ImageStation",
+    "CognitePointCloudModel",
+    "Cognite3DTransformation",
+    "Cognite360ImageModel",
+    "Cognite3DModel",
+    "CogniteCADModel",
+    "CognitePointCloudRevision",
+    "CogniteCADNode",
+)
+
+COGNITE_CONCEPTS: tuple[str, ...] = (
+    *COGNITE_CONCEPTS_MAIN,
+    *COGNITE_CONCEPTS_INTERFACES,
+    *COGNITE_CONCEPTS_CONFIGURATIONS,
+    *COGNITE_CONCEPTS_ANNOTATIONS,
+    *COGNITE_CONCEPTS_3D,
+)
+
+COGNITE_SPACES = (CDF_CDM_SPACE,)
+
+# Defaults from https://docs.cognite.com/cdf/dm/dm_reference/dm_limits_and_restrictions#list-size-limits
+
+DEFAULT_MAX_LIST_SIZE = 1000
+DEFAULT_MAX_LIST_SIZE_DIRECT_RELATIONS = 100

cognite/neat/_data_model/_identifiers.py

@@ -0,0 +1,61 @@
+from pydantic import HttpUrl, RootModel, ValidationError
+
+from cognite.neat.v0.core._utils.auxiliary import local_import
+
+
+class URI(RootModel[str]):
+    def __init__(self, value: str):
+        try:
+            # Use Pydantic's AnyUrl to validate the URI
+            _ = HttpUrl(value)
+        except ValidationError as e:
+            raise ValueError(f"Invalid URI: {value}") from e
+        super().__init__(value)
+
+    def __str__(self) -> str:
+        return self.root
+
+    def __repr__(self) -> str:
+        return f"URI({self.root!r})"
+
+    def as_rdflib_uriref(self):  # type: ignore[no-untyped-def]
+        # rdflib is an optional dependency, so import here
+        local_import("rdflib", "rdflib")
+        from rdflib import URIRef
+
+        return URIRef(self.root)
+
+
+class NameSpace(RootModel[str]):
+    def __init__(self, value: str):
+        try:
+            # Use Pydantic's AnyUrl to validate the URI
+            _ = HttpUrl(value)
+        except ValidationError as e:
+            raise ValueError(f"Invalid Namespace: {value}") from e
+        super().__init__(value)
+
+    def __str__(self) -> str:
+        return self.root
+
+    def __repr__(self) -> str:
+        return f"NameSpace({self.root!r})"
+
+    def term(self, name: str) -> URI:
+        # need to handle slices explicitly because of __getitem__ override
+        return URI(self.root + (name if isinstance(name, str) else ""))
+
+    def __getitem__(self, key: str) -> URI:  # type: ignore[override]
+        return self.term(key)
+
+    def __getattr__(self, name: str) -> URI:
+        if name.startswith("__"):  # ignore any special Python names!
+            raise AttributeError
+        return self.term(name)
+
+    def as_rdflib_namespace(self):  # type: ignore[no-untyped-def]
+        # rdflib is an optional dependency, so import here
+        local_import("rdflib", "rdflib")
+        from rdflib import Namespace
+
+        return Namespace(self.root)

cognite/neat/_data_model/_shared.py

@@ -0,0 +1,41 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from cognite.neat._data_model.deployer.data_classes import DeploymentResult
+from cognite.neat._issues import IssueList
+
+
+class OnSuccess(ABC):
+    """Abstract base class for post-activity success handlers."""
+
+    @abstractmethod
+    def run(self, data_model: Any) -> None:
+        """Execute the success handler on the data model."""
+        pass
+
+
+class OnSuccessIssuesChecker(OnSuccess, ABC):
+    """Abstract base class for post-activity success handlers that check for issues of the data model."""
+
+    def __init__(self) -> None:
+        self._issues = IssueList()
+        self._has_run = False
+
+    @property
+    def issues(self) -> IssueList:
+        if not self._has_run:
+            raise RuntimeError(f"{type(self).__name__} has not been run yet.")
+        return IssueList(self._issues)
+
+
+class OnSuccessResultProducer(OnSuccess, ABC):
+    """Abstract base class for post-activity success handlers that produce desired outcomes using the data model."""
+
+    def __init__(self) -> None:
+        self._results: DeploymentResult | None = None
+
+    @property
+    def result(self) -> DeploymentResult:
+        if self._results is None:
+            raise RuntimeError("SchemaDeployer has not been run yet.")
+        return self._results

cognite/neat/_data_model/deployer/_differ.py

@@ -0,0 +1,140 @@
+from abc import ABC, abstractmethod
+from typing import Generic
+
+from cognite.neat._utils.useful_types import T_Item
+
+from .data_classes import (
+    AddedField,
+    ChangedField,
+    FieldChange,
+    FieldChanges,
+    RemovedField,
+    SeverityType,
+)
+
+
+class Differ(Generic[T_Item], ABC):
+    def __init__(self, parent_path: str | None = None) -> None:
+        self.parent_path = parent_path
+
+    def _get_path(self, field: str) -> str:
+        if self.parent_path:
+            return f"{self.parent_path}.{field}"
+        return field
+
+    def _diff_name_description(self, current: T_Item, new: T_Item, identifier: str | None = None) -> list[FieldChange]:
+        changes: list[FieldChange] = []
+        if hasattr(current, "name") and hasattr(new, "name"):
+            if current.name != new.name:
+                field_path = self._get_path(f"{identifier}.name" if identifier else "name")
+                changes.append(
+                    ChangedField(
+                        item_severity=SeverityType.SAFE,
+                        field_path=field_path,
+                        current_value=current.name,
+                        new_value=new.name,
+                    )
+                )
+        if hasattr(current, "description") and hasattr(new, "description"):
+            if current.description != new.description:
+                field_path = self._get_path(f"{identifier}.description" if identifier else "description")
+                changes.append(
+                    ChangedField(
+                        item_severity=SeverityType.SAFE,
+                        field_path=field_path,
+                        current_value=current.description,
+                        new_value=new.description,
+                    )
+                )
+        return changes
+
+
+class ItemDiffer(Differ[T_Item], ABC):
+    """A generic class for comparing two items of the same type and reporting the differences."""
+
+    @abstractmethod
+    def diff(self, current: T_Item, new: T_Item) -> list[FieldChange]:
+        """Compare two items and return a list of changes.
+
+        Args:
+            current: The resource as it is in CDF.
+            new: The resource as it is desired to be.
+
+        Returns:
+            A list of changes between the two resources.
+        """
+        raise NotImplementedError()
+
+
+class ObjectDiffer(Differ[T_Item], ABC):
+    @abstractmethod
+    def diff(self, current: T_Item, new: T_Item, identifier: str) -> list[FieldChange]:
+        """Compare two dict-like objects and return a list of changes.
+
+        Args:
+            current: The resource as it is in CDF.
+            new: The resource as it is desired to be.
+            identifier: The field used to identify individual items within the objects.
+
+        Returns:
+            A list of changes between the two resources.
+        """
+        raise NotImplementedError()
+
+
+def field_differences(
+    parent_path: str,
+    current: dict[str, T_Item] | None,
+    new: dict[str, T_Item] | None,
+    add_severity: SeverityType,
+    remove_severity: SeverityType,
+    differ: ObjectDiffer[T_Item],
+) -> list[FieldChange]:
+    """Diff two containers of items.
+
+    A container is for example the properties, constraints, or indexes of a container,
+    properties of a space, views of a data model, etc.
+
+    Args:
+        parent_path: The JSON path to the container being compared.
+        current: The items as they are in CDF.
+        new: The items as they are desired to be.
+        add_severity: The severity to assign to added items.
+        remove_severity: The severity to assign to removed items.
+        differ: The differ to use for comparing individual items.
+
+    """
+    changes: list[FieldChange] = []
+    current_map = current or {}
+    new_map = new or {}
+    current_keys = set(current_map.keys())
+    new_keys = set(new_map.keys())
+
+    for key in sorted(new_keys - current_keys):
+        item_path = f"{parent_path}.{key}"
+        changes.append(
+            AddedField(
+                item_severity=add_severity,
+                field_path=item_path,
+                new_value=new_map[key],
+            )
+        )
+
+    for key in sorted(current_keys - new_keys):
+        changes.append(
+            RemovedField(
+                item_severity=remove_severity,
+                field_path=f"{parent_path}.{key}",
+                current_value=current_map[key],
+            )
+        )
+
+    for key in sorted(current_keys & new_keys):
+        item_path = f"{parent_path}.{key}"
+        cdf_item = current_map[key]
+        desired_item = new_map[key]
+        diffs = differ.diff(cdf_item, desired_item, identifier=key)
+        if diffs:
+            changes.append(FieldChanges(field_path=item_path, changes=diffs))
+
+    return changes