cognite-neat 0.123.26__py3-none-any.whl → 1.0.22__py3-none-any.whl

cognite/neat/_session/_issues.py

@@ -0,0 +1,81 @@
+import json
+import uuid
+from collections import defaultdict
+from typing import Any
+
+from cognite.neat._issues import ConsistencyError, IssueList, ModelSyntaxError, Recommendation
+from cognite.neat._session._html._render import render
+from cognite.neat._store import NeatStore
+
+
+class Issues:
+    """Class to handle issues in the NeatSession."""
+
+    def __init__(self, store: NeatStore) -> None:
+        self._store = store
+
+    @property
+    def _issues(self) -> IssueList:
+        """Get all issues from the last change in the store."""
+        issues = IssueList()
+        if change := self._store.provenance.last_change:
+            issues += change.errors or IssueList()
+            issues += change.issues or IssueList()
+        return issues
+
+    @property
+    def _stats(self) -> dict[str, Any]:
+        """Compute statistics about issues."""
+        by_type: defaultdict[str, int] = defaultdict(int)
+        by_code: defaultdict[str, int] = defaultdict(int)
+
+        stats: dict[str, Any] = {
+            "total": len(self._issues),
+            "by_type": by_type,
+            "by_code": by_code,
+            "severity_order": [ModelSyntaxError.__name__, ConsistencyError.__name__, Recommendation.__name__],
+        }
+
+        for issue in self._issues:
+            stats["by_type"][issue.issue_type()] += 1
+
+            if issue.code:
+                stats["by_code"][f"{issue.issue_type()}:{issue.code}"] += 1
+
+        return stats
+
+    @property
+    def _serialized_issues(self) -> list[dict[str, Any]]:
+        """Convert issues to JSON-serializable format."""
+        serialized = []
+        for idx, issue in enumerate(self._issues):
+            serialized.append(
+                {
+                    "id": idx,
+                    "type": issue.issue_type(),
+                    "code": issue.code or "",
+                    "message": issue.message,
+                    "fix": issue.fix or "",
+                }
+            )
+        return serialized
+
+    def _repr_html_(self) -> str:
+        """Generate interactive HTML representation."""
+        if not self._issues:
+            return "<b>No issues found.</b>"
+        stats = self._stats
+
+        # Generate unique ID for this render to avoid conflicts in Jupyter
+        unique_id = uuid.uuid4().hex[:8]
+
+        template_vars = {
+            "JSON": json.dumps(self._serialized_issues),
+            "total": stats["total"],
+            "syntax_errors": stats["by_type"].get("ModelSyntaxError", 0),
+            "consistency_errors": stats["by_type"].get("ConsistencyError", 0),
+            "recommendations": stats["by_type"].get("Recommendation", 0),
+            "unique_id": unique_id,
+        }
+
+        return render("issues", template_vars)

cognite/neat/_session/_physical.py

@@ -0,0 +1,294 @@
+from typing import Any, Literal
+
+from cognite.neat._client import NeatClient
+from cognite.neat._config import NeatConfig
+from cognite.neat._data_model.deployer.deployer import DeploymentOptions, SchemaDeployer
+from cognite.neat._data_model.exporters import (
+    DMSAPIExporter,
+    DMSAPIJSONExporter,
+    DMSAPIYAMLExporter,
+    DMSExcelExporter,
+    DMSExporter,
+    DMSTableJSONExporter,
+    DMSTableYamlExporter,
+)
+from cognite.neat._data_model.exporters._table_exporter.workbook import WorkbookOptions
+from cognite.neat._data_model.importers import DMSAPIImporter, DMSImporter, DMSTableImporter
+from cognite.neat._data_model.models.dms import DataModelReference
+from cognite.neat._data_model.validation.dms import DmsDataModelValidation
+from cognite.neat._exceptions import UserInputError
+from cognite.neat._state_machine import PhysicalState
+from cognite.neat._store._store import NeatStore
+from cognite.neat._utils._reader import NeatReader
+
+from ._wrappers import session_wrapper
+
+
+class PhysicalDataModel:
+    """Read from a data source into NeatSession graph store."""
+
+    def __init__(self, store: NeatStore, client: NeatClient, config: NeatConfig) -> None:
+        self._store = store
+        self._client = client
+        self._config = config
+        self.read = ReadPhysicalDataModel(self._store, self._client, self._config)
+        self.write = WritePhysicalDataModel(self._store, self._client, self._config)
+
+    def _repr_html_(self) -> str:
+        if not isinstance(self._store.state, PhysicalState):
+            return "No physical data model. Get started by reading physical data model <em>.physica_data_mode.read</em>"
+
+        dm = self._store.physical_data_model[-1]
+
+        html = ["<div>"]
+        html.append(
+            f"<h3>Data Model: {dm.data_model.space}:{dm.data_model.external_id}(version={dm.data_model.version})</h3>"
+        )
+        html.append("<table style='border-collapse: collapse;'>")
+        html.append("<tr><th style='text-align: left; padding: 4px; border: 1px solid #ddd;'>Component</th>")
+        html.append("<th style='text-align: left; padding: 4px; border: 1px solid #ddd;'>Count</th></tr>")
+
+        html.append("<tr><td style='padding: 4px; border: 1px solid #ddd;'>Views</td>")
+        html.append(f"<td style='padding: 4px; border: 1px solid #ddd;'>{len(dm.views)}</td></tr>")
+
+        html.append("<tr><td style='padding: 4px; border: 1px solid #ddd;'>Containers</td>")
+        html.append(f"<td style='padding: 4px; border: 1px solid #ddd;'>{len(dm.containers)}</td></tr>")
+
+        html.append("<tr><td style='padding: 4px; border: 1px solid #ddd;'>Spaces</td>")
+        html.append(f"<td style='padding: 4px; border: 1px solid #ddd;'>{len(dm.spaces)}</td></tr>")
+
+        html.append("<tr><td style='padding: 4px; border: 1px solid #ddd;'>Node Types</td>")
+        html.append(f"<td style='padding: 4px; border: 1px solid #ddd;'>{len(dm.node_types)}</td></tr>")
+
+        html.append("</table>")
+        html.append("</div>")
+
+        return "".join(html)
+
+
+@session_wrapper
+class ReadPhysicalDataModel:
+    """Read physical data model from various sources into NeatSession store.
+
+    Available methods:
+
+    - `neat.physical_data_model.read.yaml`
+    - `neat.physical_data_model.read.json`
+    - `neat.physical_data_model.read.excel`
+    - `neat.physical_data_model.read.cdf`
+    """
+
+    def __init__(self, store: NeatStore, client: NeatClient, config: NeatConfig) -> None:
+        self._store = store
+        self._client = client
+        self._config = config
+
+    def yaml(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None:
+        """Read physical data model from YAML file(s)
+
+        Args:
+            io (Any): The file or directory path or buffer to read from.
+            format (Literal["neat", "toolkit"]): The format of the input file(s).
+                - "neat": Neat's DMS table format.
+                - "toolkit": Cognite DMS API format which is the format used by Cognite Toolkit.
+        """
+
+        path = NeatReader.create(io).materialize_path()
+
+        reader: DMSImporter
+        if format == "neat":
+            reader = DMSTableImporter.from_yaml(path)
+        elif format == "toolkit":
+            reader = DMSAPIImporter.from_yaml(path)
+        else:
+            raise UserInputError(f"Unsupported format: {format}. Supported formats are 'neat' and 'toolkit'.")
+
+        on_success = DmsDataModelValidation(
+            modus_operandi=self._config.modeling.mode,
+            cdf_snapshot=self._store.cdf_snapshot,
+            limits=self._store.cdf_limits,
+            can_run_validator=self._config.validation.can_run_validator,
+            enable_alpha_validators=self._config.alpha.enable_experimental_validators,
+        )
+        return self._store.read_physical(reader, on_success)
+
+    def json(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None:
+        """Read physical data model from JSON file(s)
+
+        Args:
+            io (Any): The file or directory path or buffer to read from.
+            format (Literal["neat", "toolkit"]): The format of the input file(s).
+                - "neat": Neat's DMS table format.
+                - "toolkit": Cognite DMS API format which is the format used by Cognite Toolkit.
+        """
+
+        path = NeatReader.create(io).materialize_path()
+
+        reader: DMSImporter
+        if format == "neat":
+            reader = DMSTableImporter.from_json(path)
+        elif format == "toolkit":
+            reader = DMSAPIImporter.from_json(path)
+        else:
+            raise UserInputError(f"Unsupported format: {format}. Supported formats are 'neat' and 'toolkit'.")
+
+        on_success = DmsDataModelValidation(
+            modus_operandi=self._config.modeling.mode,
+            cdf_snapshot=self._store.cdf_snapshot,
+            limits=self._store.cdf_limits,
+            can_run_validator=self._config.validation.can_run_validator,
+            enable_alpha_validators=self._config.alpha.enable_experimental_validators,
+        )
+        return self._store.read_physical(reader, on_success)
+
+    def excel(self, io: Any) -> None:
+        """Read physical data model from Excel file
+
+        Args:
+            io (Any): The file path or buffer to read from.
+
+        """
+
+        path = NeatReader.create(io).materialize_path()
+        reader = DMSTableImporter.from_excel(path)
+
+        on_success = DmsDataModelValidation(
+            modus_operandi=self._config.modeling.mode,
+            cdf_snapshot=self._store.cdf_snapshot,
+            limits=self._store.cdf_limits,
+            can_run_validator=self._config.validation.can_run_validator,
+            enable_alpha_validators=self._config.alpha.enable_experimental_validators,
+        )
+
+        return self._store.read_physical(reader, on_success)
+
+    def cdf(self, space: str, external_id: str, version: str) -> None:
+        """Read physical data model from CDF
+
+        Args:
+            space (str): The schema space of the data model.
+            external_id (str): The external id of the data model.
+            version (str): The version of the data model.
+
+        """
+        reader = DMSAPIImporter.from_cdf(
+            DataModelReference(space=space, external_id=external_id, version=version), self._client
+        )
+
+        on_success = DmsDataModelValidation(
+            modus_operandi=self._config.modeling.mode,
+            cdf_snapshot=self._store.cdf_snapshot,
+            limits=self._store.cdf_limits,
+            can_run_validator=self._config.validation.can_run_validator,
+            enable_alpha_validators=self._config.alpha.enable_experimental_validators,
+        )
+
+        return self._store.read_physical(reader, on_success)
+
+
+@session_wrapper
+class WritePhysicalDataModel:
+    """Write physical data model to various sources from NeatSession store.
+
+    Available methods:
+
+    - `neat.physical_data_model.write.yaml`
+    - `neat.physical_data_model.write.json`
+    - `neat.physical_data_model.write.excel`
+    - `neat.physical_data_model.write.cdf`
+    """
+
+    def __init__(self, store: NeatStore, client: NeatClient, config: NeatConfig) -> None:
+        self._store = store
+        self._client = client
+        self._config = config
+
+    def yaml(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None:
+        """Write physical data model to YAML file
+
+        Args:
+            io (Any): The file path or buffer to write to.
+            format (Literal["neat", "toolkit"]): The format of the output file
+                - "neat": Neat's DMS table format.
+                - "toolkit": Cognite DMS API format which is the format used by Cognite Toolkit.
+        """
+
+        file_path = NeatReader.create(io).materialize_path()
+        writer: DMSExporter
+        if format == "neat":
+            writer = DMSTableYamlExporter()
+        elif format == "toolkit":
+            writer = DMSAPIYAMLExporter()
+        else:
+            raise UserInputError(f"Unsupported format: {format}. Supported formats are 'neat' and 'toolkit'.")
+
+        return self._store.write_physical(writer, file_path=file_path)
+
+    def json(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None:
+        """Write physical data model to JSON file
+
+        Args:
+            io (Any): The file path or buffer to write to.
+            format (Literal["neat", "toolkit"]): The format of the output file
+                - "neat": Neat's DMS table format.
+                - "toolkit": Cognite DMS API format which is the format used by Cognite Toolkit.
+        """
+
+        file_path = NeatReader.create(io).materialize_path()
+        writer: DMSExporter
+        if format == "neat":
+            writer = DMSTableJSONExporter()
+        elif format == "toolkit":
+            writer = DMSAPIJSONExporter()
+        else:
+            raise UserInputError(f"Unsupported format: {format}. Supported formats are 'neat' and 'toolkit'.")
+
+        return self._store.write_physical(writer, file_path=file_path)
+
+    def excel(self, io: Any, skip_other_spaces: bool = True) -> None:
+        """Write physical data model to Excel file
+
+        Args:
+            io (Any): The file path or buffer to write to.
+            skip_other_spaces (bool): If true, only properties in the same space as the data model will be written.
+
+        """
+
+        file_path = NeatReader.create(io).materialize_path()
+        options = WorkbookOptions(skip_properties_in_other_spaces=skip_other_spaces)
+        writer = DMSExcelExporter(options=options)
+
+        return self._store.write_physical(writer, file_path=file_path)
+
+    def cdf(self, dry_run: bool = True, rollback: bool = False, drop_data: bool = False) -> None:
+        """Write physical data model with views, containers, and spaces that are in the same space as the data model
+        to CDF.
+
+        Args:
+            dry_run (bool): If true, the changes will not be applied to CDF. Instead, Neat will
+                report what changes would have been made.
+            rollback (bool): If true, all changes will be rolled back if any error occurs.
+            drop_data (bool): Only applicable if the session mode is 'rebuild'. If
+
+        !!! note "Impact of governance profile"
+            This method depends on the session governance profile for data modeling set when creating the NeatSession:
+
+            - In `additive` mode, only new or updates to data models/views/containers will be applied.
+              You cannot remove views from data models, properties from views or containers, or
+              indexes or constraints from containers.
+
+            - In `rebuild` mode, the data model in CDF will be made to exactly match the data model in Neat.
+              If there are any breaking changes, Neat will delete and recreate the relevant
+              data model/view/container. However, if drop_data is set to False, Neat will treat
+              containers as `additive` and will not delete any containers or remove properties,
+              indexes, or constraints. To fully rebuild the data model, including containers, set drop_data to True.
+        """
+        writer = DMSAPIExporter()
+        options = DeploymentOptions(
+            dry_run=dry_run,
+            auto_rollback=rollback,
+            drop_data=drop_data,
+            modus_operandi=self._config.modeling.mode,
+        )
+        on_success = SchemaDeployer(self._client, options)
+        return self._store.write_physical(writer, on_success)

cognite/neat/_session/_result/__init__.py

@@ -0,0 +1,3 @@
+from ._result import Result
+
+__all__ = ["Result"]

cognite/neat/_session/_result/_deployment/_physical/_changes.py

@@ -0,0 +1,196 @@
+import itertools
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from cognite.neat._data_model.deployer.data_classes import (
+    AddedField,
+    AppliedChanges,
+    ChangedField,
+    ChangeResult,
+    DeploymentResult,
+    FieldChange,
+    FieldChanges,
+    RemovedField,
+    ResourceChange,
+)
+
+
+class SerializedFieldChange(BaseModel):
+    """Serialized field change for JSON output."""
+
+    field_path: str
+    severity: str
+    description: str
+
+    @classmethod
+    def from_field_change(cls, field_change: FieldChange) -> list["SerializedFieldChange"]:
+        """Serialize a field change, handling nested FieldChanges recursively.
+
+        Args:
+            field_change: The field change to serialize.
+
+        Returns:
+            List of serialized field changes (may be multiple if nested).
+        """
+        serialized_changes: list[SerializedFieldChange] = []
+
+        if isinstance(field_change, FieldChanges):
+            # Recursively handle nested changes
+            for change in field_change.changes:
+                serialized_changes.extend(cls.from_field_change(change))
+        else:
+            # Base case: single field change
+            serialized_changes.append(cls._from_single_field_change(field_change))
+
+        return serialized_changes
+
+    @classmethod
+    def _from_single_field_change(cls, field_change: FieldChange) -> "SerializedFieldChange":
+        """Serialize a single non-nested field change.
+
+        Args:
+            field_change: The single field change to serialize.
+
+        Returns:
+            Serialized field change.
+        """
+        return cls(
+            field_path=field_change.field_path,
+            severity=field_change.severity.name,
+            description=field_change.description
+            if isinstance(field_change, AddedField | RemovedField | ChangedField)
+            else "Field changed",
+        )
+
+
+class SerializedResourceChange(BaseModel):
+    """Serialized resource change for JSON output."""
+
+    id: int
+    endpoint: str
+    change_type: str
+    severity: str
+    resource_id: str
+    message: str | None = None
+    changes: list[SerializedFieldChange] = Field(default_factory=list)
+
+    @classmethod
+    def from_resource_change(
+        cls, resource: ResourceChange, endpoint: str, change_id: int
+    ) -> "SerializedResourceChange":
+        """Serialize a single resource change.
+
+        Args:
+            resource: The resource change to serialize.
+            endpoint: The endpoint type.
+            change_id: Unique ID for this change.
+
+        Returns:
+            Serialized resource change.
+        """
+        changes: list[SerializedFieldChange] = []
+
+        for change in resource.changes:
+            changes.extend(SerializedFieldChange.from_field_change(change))
+
+        return cls(
+            id=change_id,
+            endpoint=endpoint,
+            change_type=resource.change_type,
+            severity=resource.severity.name,
+            resource_id=str(resource.resource_id),
+            changes=changes,
+            message=resource.message,
+        )
+
+    @classmethod
+    def from_change_result(cls, change_id: int, response: ChangeResult) -> "SerializedResourceChange":
+        """Serialize from a change result (actual deployment).
+
+        Args:
+            change_id: Unique ID for this change.
+            response: The change result from deployment.
+
+        Returns:
+            Serialized resource change with deployment status.
+        """
+        serialized_resource_change = cls.from_resource_change(
+            resource=response.change,
+            endpoint=response.endpoint,
+            change_id=change_id,
+        )
+
+        serialized_resource_change.message = response.message
+        if not response.is_success:
+            serialized_resource_change.change_type = "failed"
+
+        return serialized_resource_change
+
+
+class SerializedChanges(BaseModel):
+    """Container for all serialized changes."""
+
+    changes: list[SerializedResourceChange] = Field(default_factory=list)
+
+    @classmethod
+    def from_deployment_result(cls, result: DeploymentResult) -> "SerializedChanges":
+        """Create SerializedChanges from a DeploymentResult.
+
+        Args:
+            result: The deployment result to serialize changes from.
+
+        Returns:
+            SerializedChanges instance with all changes.
+        """
+        serialized = cls()
+
+        if not result.responses:
+            serialized._add_from_dry_run(result)
+        else:
+            serialized._add_from_applied_changes(result.responses)
+
+        return serialized
+
+    def _add_from_dry_run(self, result: DeploymentResult) -> None:
+        """Add changes from dry run deployment.
+
+        Args:
+            result: The deployment result in dry run mode.
+        """
+        # Iterate over each endpoint plan
+        for endpoint_plan in result.plan:
+            # Then per resource in the endpoint
+            for resource in endpoint_plan.resources:
+                # Then serialize individual resource change
+                serialized_resource_change = SerializedResourceChange.from_resource_change(
+                    resource=resource,
+                    endpoint=endpoint_plan.endpoint,
+                    change_id=len(self.changes),
+                )
+                self.changes.append(serialized_resource_change)
+
+    def _add_from_applied_changes(self, applied_changes: AppliedChanges) -> None:
+        """Add changes from actual deployment.
+        Args:
+            result: The deployment result from actual deployment.
+        """
+        for response in itertools.chain(
+            applied_changes.created,
+            applied_changes.merged_updated,
+            applied_changes.deletions,
+            applied_changes.unchanged,
+            applied_changes.skipped,
+        ):
+            self.changes.append(SerializedResourceChange.from_change_result(len(self.changes), response))
+
+    def model_dump_json_flat(self, **kwargs: Any) -> str:
+        """Dump changes as JSON array without the wrapper key.
+        Returns:
+            JSON string of the changes array.
+        """
+        if not self.changes:
+            return "[]"
+
+        iterator = (change.model_dump_json(**kwargs) for change in self.changes)
+        return f"[{','.join(iterator)}]"