cognite-neat 0.123.24__py3-none-any.whl → 1.0.22__py3-none-any.whl

cognite/neat/_client/spaces_api.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+from cognite.neat._data_model.models.dms import DataModelBody, SpaceRequest, SpaceResponse
+from cognite.neat._data_model.models.dms._references import SpaceReference
+from cognite.neat._utils.http_client import ItemIDBody, ItemsRequest, ParametersRequest
+from cognite.neat._utils.useful_types import PrimitiveType
+
+from .api import NeatAPI
+from .data_classes import PagedResponse
+
+
+class SpacesAPI(NeatAPI):
+    ENDPOINT = "/models/spaces"
+
+    def apply(self, spaces: list[SpaceRequest]) -> list[SpaceResponse]:
+        """Apply (create or update) spaces in CDF.
+
+        Args:
+            spaces: List of SpaceRequest objects to apply.
+        Returns:
+            List of SpaceResponse objects.
+        """
+        if not spaces:
+            return []
+        if len(spaces) > 100:
+            raise ValueError("Cannot apply more than 100 spaces at once.")
+        result = self._http_client.request_with_retries(
+            ItemsRequest(
+                endpoint_url=self._config.create_api_url(self.ENDPOINT),
+                method="POST",
+                body=DataModelBody(items=spaces),
+            )
+        )
+        result.raise_for_status()
+        result = PagedResponse[SpaceResponse].model_validate_json(result.success_response.body)
+        return result.items
+
+    def retrieve(self, spaces: list[SpaceReference]) -> list[SpaceResponse]:
+        """Retrieve spaces by their identifiers.
+
+        Args:
+            spaces: List of space identifiers to retrieve.
+
+        Returns:
+            List of SpaceResponse objects.
+        """
+        if not spaces:
+            return []
+        if len(spaces) > 1000:
+            raise ValueError("Cannot retrieve more than 1000 spaces at once.")
+
+        result = self._http_client.request_with_retries(
+            ItemsRequest(
+                endpoint_url=self._config.create_api_url(f"{self.ENDPOINT}/byids"),
+                method="POST",
+                body=ItemIDBody(items=spaces),
+            )
+        )
+        result.raise_for_status()
+        result = PagedResponse[SpaceResponse].model_validate_json(result.success_response.body)
+        return result.items
+
+    def delete(self, spaces: list[SpaceReference]) -> list[SpaceReference]:
+        """Delete spaces by their identifiers.
+
+        Args:
+            spaces: List of space identifiers to delete.
+        Returns:
+            List of SpaceReference objects representing the deleted spaces.
+        """
+        if not spaces:
+            return []
+        if len(spaces) > 100:
+            raise ValueError("Cannot delete more than 100 spaces at once.")
+        result = self._http_client.request_with_retries(
+            ItemsRequest(
+                endpoint_url=self._config.create_api_url(f"{self.ENDPOINT}/delete"),
+                method="POST",
+                body=ItemIDBody(items=spaces),
+            )
+        )
+        result.raise_for_status()
+        result = PagedResponse[SpaceReference].model_validate_json(result.success_response.body)
+        return result.items
+
+    def list(
+        self,
+        include_global: bool = False,
+        limit: int = 10,
+    ) -> list[SpaceResponse]:
+        """List spaces in CDF Project.
+
+        Args:
+            include_global: If True, include global spaces.
+            limit: Maximum number of spaces to return. Max is 1000.
+
+        Returns:
+            List of SpaceResponse objects.
+        """
+        if limit > 1000:
+            raise ValueError("Pagination is not (yet) supported for listing spaces. The maximum limit is 1000.")
+        parameters: dict[str, PrimitiveType] = {
+            "includeGlobal": include_global,
+            "limit": limit,
+        }
+        result = self._http_client.request_with_retries(
+            ParametersRequest(
+                endpoint_url=self._config.create_api_url(self.ENDPOINT),
+                method="GET",
+                parameters=parameters,
+            )
+        )
+        result.raise_for_status()
+        result = PagedResponse[SpaceResponse].model_validate_json(result.success_response.body)
+        return result.items

cognite/neat/_client/statistics_api.py

@@ -0,0 +1,24 @@
+from cognite.neat._client.api import NeatAPI
+from cognite.neat._client.data_classes import StatisticsResponse
+from cognite.neat._utils.http_client import ParametersRequest
+
+
+class StatisticsAPI(NeatAPI):
+    def project(self) -> StatisticsResponse:
+        """Retrieve project-wide usage data and limits.
+
+        Returns:
+            StatisticsResponse object.
+        """
+
+        result = self._http_client.request_with_retries(
+            ParametersRequest(
+                endpoint_url=self._config.create_api_url("/models/statistics"),
+                method="GET",
+                parameters=None,
+            )
+        )
+
+        result.raise_for_status()
+        result = StatisticsResponse.model_validate_json(result.success_response.body)
+        return result

cognite/neat/_client/views_api.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from cognite.neat._data_model.models.dms import DataModelBody, ViewReference, ViewRequest, ViewResponse
+from cognite.neat._utils.collection import chunker_sequence
+from cognite.neat._utils.http_client import ItemIDBody, ItemsRequest, ParametersRequest
+from cognite.neat._utils.useful_types import PrimitiveType
+
+from .api import NeatAPI
+from .data_classes import PagedResponse
+
+
+class ViewsAPI(NeatAPI):
+    ENDPOINT = "/models/views"
+    LIST_REQUEST_LIMIT = 1000
+
+    def apply(self, items: Sequence[ViewRequest]) -> list[ViewResponse]:
+        """Create or update views in CDF Project.
+        Args:
+            items: List of ViewRequest objects to create or update.
+        Returns:
+            List of ViewResponse objects.
+        """
+        if not items:
+            return []
+        if len(items) > 100:
+            raise ValueError("Cannot apply more than 100 views at once.")
+        result = self._http_client.request_with_retries(
+            ItemsRequest(
+                endpoint_url=self._config.create_api_url(self.ENDPOINT),
+                method="POST",
+                body=DataModelBody(items=items),
+            )
+        )
+        result.raise_for_status()
+        result = PagedResponse[ViewResponse].model_validate_json(result.success_response.body)
+        return result.items
+
+    def retrieve(
+        self,
+        items: list[ViewReference],
+        include_inherited_properties: bool = True,
+    ) -> list[ViewResponse]:
+        """Retrieve views by their identifiers.
+
+        Args:
+            items: List of (space, external_id, version) tuples identifying the views to retrieve.
+            include_inherited_properties: If True, include properties inherited from parent views.
+
+        Returns:
+            List of ViewResponse objects.
+        """
+        results: list[ViewResponse] = []
+        for chunk in chunker_sequence(items, 100):
+            batch = self._http_client.request_with_retries(
+                ItemsRequest(
+                    endpoint_url=self._config.create_api_url(f"{self.ENDPOINT}/byids"),
+                    method="POST",
+                    body=ItemIDBody(items=chunk),
+                    parameters={"includeInheritedProperties": include_inherited_properties},
+                )
+            )
+            batch.raise_for_status()
+            result = PagedResponse[ViewResponse].model_validate_json(batch.success_response.body)
+            results.extend(result.items)
+        return results
+
+    def delete(self, items: list[ViewReference]) -> list[ViewReference]:
+        """Delete views by their identifiers.
+
+        Args:
+            items: List of (space, external_id, version) tuples identifying the views to delete.
+        """
+        if not items:
+            return []
+        if len(items) > 100:
+            raise ValueError("Cannot delete more than 100 views at once.")
+
+        result = self._http_client.request_with_retries(
+            ItemsRequest(
+                endpoint_url=self._config.create_api_url(f"{self.ENDPOINT}/delete"),
+                method="POST",
+                body=ItemIDBody(items=items),
+            )
+        )
+        result.raise_for_status()
+        result = PagedResponse[ViewReference].model_validate_json(result.success_response.body)
+        return result.items
+
+    def list(
+        self,
+        space: str | None = None,
+        all_versions: bool = False,
+        include_inherited_properties: bool = True,
+        include_global: bool = False,
+        limit: int | None = 10,
+    ) -> list[ViewResponse]:
+        """List views in CDF Project.
+
+        Args:
+            space: If specified, only views in this space are returned.
+            all_versions: If True, return all versions. If False, only return the latest version.
+            include_inherited_properties: If True, include properties inherited from parent views.
+            include_global: If True, include global views.
+            limit: Maximum number of views to return. If None, return all views.
+
+        Returns:
+            List of ViewResponse objects.
+        """
+        if limit is not None and limit < 0:
+            raise ValueError("Limit must be non-negative.")
+        elif limit is not None and limit == 0:
+            return []
+        parameters: dict[str, PrimitiveType] = {
+            "allVersions": all_versions,
+            "includeInheritedProperties": include_inherited_properties,
+            "includeGlobal": include_global,
+        }
+        if space is not None:
+            parameters["space"] = space
+        cursor: str | None = None
+        view_responses: list[ViewResponse] = []
+        while True:
+            if cursor is not None:
+                parameters["cursor"] = cursor
+            if limit is None:
+                parameters["limit"] = self.LIST_REQUEST_LIMIT
+            else:
+                parameters["limit"] = min(self.LIST_REQUEST_LIMIT, limit - len(view_responses))
+            result = self._http_client.request_with_retries(
+                ParametersRequest(
+                    endpoint_url=self._config.create_api_url(self.ENDPOINT),
+                    method="GET",
+                    parameters=parameters,
+                )
+            )
+            result.raise_for_status()
+            result = PagedResponse[ViewResponse].model_validate_json(result.success_response.body)
+            view_responses.extend(result.items)
+            cursor = result.next_cursor
+            if cursor is None or (limit is not None and len(view_responses) >= limit):
+                break
+        return view_responses

cognite/neat/_config.py

@@ -0,0 +1,266 @@
+import sys
+from pathlib import Path
+from typing import Any, Literal, TypeAlias
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from cognite.neat._exceptions import UserInputError
+from cognite.neat._issues import ConsistencyError, ModelSyntaxError
+from cognite.neat._utils.text import humanize_collection
+from cognite.neat._utils.useful_types import ModusOperandi
+
+if sys.version_info >= (3, 11):
+    import tomllib as tomli  # Python 3.11+
+else:
+    import tomli  # type: ignore
+
+# Public profiles
+PredefinedProfile: TypeAlias = Literal["legacy-additive", "legacy-rebuild", "deep-additive", "deep-rebuild"]
+
+# Private profiles only
+_PrivateProfiles: TypeAlias = Literal["no-validation-additive", "no-validation-rebuild"]
+
+# All profiles (union of public and private)
+_AllProfiles: TypeAlias = PredefinedProfile | _PrivateProfiles
+
+
+class ConfigModel(BaseModel):
+    model_config = ConfigDict(populate_by_name=True, validate_assignment=True)
+
+
+class ValidationConfig(ConfigModel):
+    """Validation configuration."""
+
+    exclude: list[str] = Field(default_factory=list)
+    override: bool = Field(False, description="If enabled, all validators are skipped.")
+
+    def can_run_validator(self, code: str, issue_type: type) -> bool:
+        """
+        Check if a specific validator should run.
+
+        Args:
+            code: Validation code (e.g., "NEAT-DMS-CONTAINER-001")
+            issue_type: Issue type (e.g., ModelSyntaxError, ConsistencyError, Recommendation)
+
+        Returns:
+            True if validator should run, False otherwise
+        """
+
+        is_excluded = self._is_excluded(code, self.exclude)
+
+        if issue_type in [ModelSyntaxError, ConsistencyError] and is_excluded and not self.override:
+            print(f"Validator {code} was excluded however it is a critical validator and will still run.")
+            return True
+        else:
+            return not is_excluded
+
+    @classmethod
+    def _is_excluded(cls, code: str, patterns: list[str]) -> bool:
+        """Check if code matches any pattern (supports wildcards)."""
+        for pattern in patterns:
+            if "*" in pattern:
+                # Split both pattern and code by hyphens
+                pattern_parts = pattern.split("-")
+                code_parts = code.split("-")
+
+                # Pattern must have same or fewer parts than code
+                if len(pattern_parts) > len(code_parts):
+                    continue
+
+                # Check if all pattern parts match (allowing wildcards)
+                match = True
+                for p_part, c_part in zip(pattern_parts, code_parts, strict=False):
+                    if p_part != "*" and p_part != c_part:
+                        match = False
+                        break
+
+                if match:
+                    return True
+            elif code == pattern:
+                return True
+
+        return False
+
+    def __str__(self) -> str:
+        """Human-readable configuration summary."""
+        if not self.exclude:
+            return "All validators enabled"
+        return f"Excluded Rules: {', '.join(self.exclude)}"
+
+
+class ModelingConfig(ConfigModel):
+    """Modeling configuration."""
+
+    mode: ModusOperandi = "additive"
+
+
+class AlphaFlagConfig(ConfigModel):
+    """Alpha feature flags configuration."""
+
+    fix_validation_issues: bool = Field(
+        default=False,
+        description="If enabled, Neat will attempt to automatically fix certain validation issues in the data model.",
+    )
+    enable_experimental_validators: bool = Field(
+        default=False,
+        description="If enabled, Neat will run experimental validators that are still in alpha stage.",
+    )
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        """Set attribute value or raise AttributeError."""
+        if key in self.model_fields:
+            super().__setattr__(key, value)
+        else:
+            available_flags = humanize_collection(type(self).model_fields.keys())
+            raise UserInputError(f"Alpha flag '{key}' not found. Available flags: {available_flags}.")
+
+    def _repr_html_(self) -> str:
+        """HTML representation for Jupyter notebooks."""
+        lines = ["<b>Alpha Feature Flags:</b>"]
+        for field_name, field in type(self).model_fields.items():
+            value = getattr(self, field_name)
+            display = "Enabled" if value else "Disabled"
+            lines.append(f"<li><b>{field_name}</b>: {display} - {field.description}</li>")
+        return "<ul>" + "\n".join(lines) + "</ul>"
+
+
+class NeatConfig(ConfigModel):
+    """NeatSession configuration."""
+
+    profile: str
+    validation: ValidationConfig
+    modeling: ModelingConfig
+    alpha: AlphaFlagConfig = Field(default_factory=AlphaFlagConfig)
+
+    def __str__(self) -> str:
+        """Human-readable configuration summary."""
+        lines = [
+            f"Profile: {self.profile}",
+            f"Modeling Mode: {self.modeling.mode}",
+            f"Validation: {self.validation}",
+        ]
+        return "\n".join(lines)
+
+    @classmethod
+    def create_predefined(cls, profile: PredefinedProfile = "legacy-additive") -> "NeatConfig":
+        """Create NeatConfig from internal profiles.
+
+        Args:
+            profile: Profile name to use. Defaults to "legacy-additive".
+
+        !!! note "Predefined Profiles"
+            The following predefined profiles are available:
+
+            - `legacy-additive`: Additive modeling with legacy validation rules.
+            - `legacy-rebuild`: Rebuild modeling with legacy validation rules.
+            - `deep-additive`: Additive modeling with deep validation rules.
+            - `deep-rebuild`: Rebuild modeling with deep validation rules.
+        """
+        available_profiles = internal_profiles()
+        if profile not in available_profiles:
+            raise UserInputError(
+                f"Profile {profile!r} not found. Available predefined profiles: "
+                f"{humanize_collection(available_profiles.keys())}."
+            )
+        return available_profiles[profile]
+
+
+def internal_profiles() -> dict[_AllProfiles, NeatConfig]:
+    """Get internal NeatConfig profile by name."""
+    return {
+        "legacy-additive": NeatConfig(
+            profile="legacy-additive",
+            modeling=ModelingConfig(mode="additive"),
+            validation=ValidationConfig(
+                exclude=[
+                    "NEAT-DMS-AI-READINESS-*",
+                    "NEAT-DMS-CONNECTIONS-002",
+                    "NEAT-DMS-CONNECTIONS-REVERSE-007",
+                    "NEAT-DMS-CONNECTIONS-REVERSE-008",
+                    "NEAT-DMS-CONSISTENCY-001",
+                ]
+            ),
+        ),
+        "legacy-rebuild": NeatConfig(
+            profile="legacy-rebuild",
+            modeling=ModelingConfig(mode="rebuild"),
+            validation=ValidationConfig(
+                exclude=[
+                    "NEAT-DMS-AI-READINESS-*",
+                    "NEAT-DMS-CONNECTIONS-002",
+                    "NEAT-DMS-CONNECTIONS-REVERSE-007",
+                    "NEAT-DMS-CONNECTIONS-REVERSE-008",
+                    "NEAT-DMS-CONSISTENCY-001",
+                ]
+            ),
+        ),
+        "deep-additive": NeatConfig(
+            profile="deep-additive",
+            modeling=ModelingConfig(mode="additive"),
+            validation=ValidationConfig(exclude=[]),
+        ),
+        "deep-rebuild": NeatConfig(
+            profile="deep-rebuild",
+            modeling=ModelingConfig(mode="rebuild"),
+            validation=ValidationConfig(exclude=[]),
+        ),
+        "no-validation-rebuild": NeatConfig(
+            profile="no-validation-rebuild",
+            modeling=ModelingConfig(mode="rebuild"),
+            validation=ValidationConfig(exclude=["*"], override=True),
+        ),
+        "no-validation-additive": NeatConfig(
+            profile="no-validation-additive",
+            modeling=ModelingConfig(mode="additive"),
+            validation=ValidationConfig(exclude=["*"], override=True),
+        ),
+    }
+
+
+def get_neat_config_from_file(config_file_name: str, profile: str) -> NeatConfig:
+    """Get NeatConfig from file or internal profiles.
+
+    Args:
+        config_file_name: Path to configuration file.
+        profile: Profile name to use.
+    Returns:
+        NeatConfig instance.
+    """
+
+    if not config_file_name.endswith(".toml"):
+        raise ValueError("config_file_name must end with '.toml'")
+
+    file_path = Path.cwd() / config_file_name
+
+    if file_path.exists():
+        with file_path.open("rb") as f:
+            toml = tomli.load(f)
+
+        if "tool" in toml and "neat" in toml["tool"]:
+            data = toml["tool"]["neat"]
+        elif "neat" in toml:
+            data = toml["neat"]
+        else:
+            raise ValueError("No [tool.neat] or [neat] section found in the configuration file.")
+
+        toml_profile = data.get("profile")
+        toml_profiles = data.get("profiles")
+        hardcoded_profiles = internal_profiles()
+
+        if toml_profile and toml_profile in hardcoded_profiles:
+            raise ValueError(f"Internal profile '{toml_profile}' cannot be used in external configuration file.")
+
+        if toml_profiles and any(p in hardcoded_profiles for p in toml_profiles.keys()):
+            raise ValueError(
+                "Internal profiles cannot be redefined in external configuration file: "
+                f"{set(hardcoded_profiles.keys()).intersection(toml_profiles.keys())}"
+            )
+
+        if toml_profile and profile == toml_profile:
+            return NeatConfig(**data)
+        elif (built_in_profiles := data.get("profiles")) and profile in built_in_profiles:
+            return NeatConfig(profile=profile, **data["profiles"][profile])
+        else:
+            raise ValueError(f"Profile '{profile}' not found in configuration file.")
+    else:
+        raise FileNotFoundError(f"Configuration file '{file_path}' not found.")