cognite-neat 0.123.2__py3-none-any.whl → 0.127.30__py3-none-any.whl

cognite/neat/_session/_physical.py

@@ -0,0 +1,261 @@
+from typing import Any, Literal
+
+from cognite.neat._client import NeatClient
+from cognite.neat._config import NeatConfig
+from cognite.neat._data_model.deployer.deployer import DeploymentOptions, SchemaDeployer
+from cognite.neat._data_model.exporters import (
+    DMSAPIExporter,
+    DMSAPIJSONExporter,
+    DMSAPIYAMLExporter,
+    DMSExcelExporter,
+    DMSExporter,
+    DMSTableJSONExporter,
+    DMSTableYamlExporter,
+)
+from cognite.neat._data_model.exporters._table_exporter.workbook import WorkbookOptions
+from cognite.neat._data_model.importers import DMSAPIImporter, DMSImporter, DMSTableImporter
+from cognite.neat._data_model.models.dms import DataModelReference
+from cognite.neat._data_model.validation.dms import DmsDataModelValidation
+from cognite.neat._exceptions import UserInputError
+from cognite.neat._state_machine import PhysicalState
+from cognite.neat._store._store import NeatStore
+from cognite.neat._utils._reader import NeatReader
+
+from ._wrappers import session_wrapper
+
+
+class PhysicalDataModel:
+    """Read from a data source into NeatSession graph store."""
+
+    def __init__(self, store: NeatStore, client: NeatClient, config: NeatConfig) -> None:
+        self._store = store
+        self._client = client
+        self._config = config
+        self.read = ReadPhysicalDataModel(self._store, self._client, self._config)
+        self.write = WritePhysicalDataModel(self._store, self._client, self._config)
+
+    def _repr_html_(self) -> str:
+        if not isinstance(self._store.state, PhysicalState):
+            return "No physical data model. Get started by reading physical data model <em>.physica_data_mode.read</em>"
+
+        dm = self._store.physical_data_model[-1]
+
+        html = ["<div>"]
+        html.append(
+            f"<h3>Data Model: {dm.data_model.space}:{dm.data_model.external_id}(version={dm.data_model.version})</h3>"
+        )
+        html.append("<table style='border-collapse: collapse;'>")
+        html.append("<tr><th style='text-align: left; padding: 4px; border: 1px solid #ddd;'>Component</th>")
+        html.append("<th style='text-align: left; padding: 4px; border: 1px solid #ddd;'>Count</th></tr>")
+
+        html.append("<tr><td style='padding: 4px; border: 1px solid #ddd;'>Views</td>")
+        html.append(f"<td style='padding: 4px; border: 1px solid #ddd;'>{len(dm.views)}</td></tr>")
+
+        html.append("<tr><td style='padding: 4px; border: 1px solid #ddd;'>Containers</td>")
+        html.append(f"<td style='padding: 4px; border: 1px solid #ddd;'>{len(dm.containers)}</td></tr>")
+
+        html.append("<tr><td style='padding: 4px; border: 1px solid #ddd;'>Spaces</td>")
+        html.append(f"<td style='padding: 4px; border: 1px solid #ddd;'>{len(dm.spaces)}</td></tr>")
+
+        html.append("<tr><td style='padding: 4px; border: 1px solid #ddd;'>Node Types</td>")
+        html.append(f"<td style='padding: 4px; border: 1px solid #ddd;'>{len(dm.node_types)}</td></tr>")
+
+        html.append("</table>")
+        html.append("</div>")
+
+        return "".join(html)
+
+
+@session_wrapper
+class ReadPhysicalDataModel:
+    """Read physical data model from various sources into NeatSession graph store."""
+
+    def __init__(self, store: NeatStore, client: NeatClient, config: NeatConfig) -> None:
+        self._store = store
+        self._client = client
+        self._config = config
+
+    def yaml(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None:
+        """Read physical data model from YAML file(s)
+
+        Args:
+            io (Any): The file or directory path or buffer to read from.
+            format (Literal["neat", "toolkit"]): The format of the input file(s).
+                - "neat": Neat's DMS table format.
+                - "toolkit": Cognite DMS API format which is the format used by Cognite Toolkit.
+        """
+
+        path = NeatReader.create(io).materialize_path()
+
+        reader: DMSImporter
+        if format == "neat":
+            reader = DMSTableImporter.from_yaml(path)
+        elif format == "toolkit":
+            reader = DMSAPIImporter.from_yaml(path)
+        else:
+            raise UserInputError(f"Unsupported format: {format}. Supported formats are 'neat' and 'toolkit'.")
+        on_success = DmsDataModelValidation(
+            self._client,
+            modus_operandi=self._config.modeling.mode,
+            can_run_validator=self._config.validation.can_run_validator,
+        )
+
+        return self._store.read_physical(reader, on_success)
+
+    def json(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None:
+        """Read physical data model from JSON file(s)
+
+        Args:
+            io (Any): The file or directory path or buffer to read from.
+            format (Literal["neat", "toolkit"]): The format of the input file(s).
+                - "neat": Neat's DMS table format.
+                - "toolkit": Cognite DMS API format which is the format used by Cognite Toolkit.
+        """
+
+        path = NeatReader.create(io).materialize_path()
+
+        reader: DMSImporter
+        if format == "neat":
+            reader = DMSTableImporter.from_json(path)
+        elif format == "toolkit":
+            reader = DMSAPIImporter.from_json(path)
+        else:
+            raise UserInputError(f"Unsupported format: {format}. Supported formats are 'neat' and 'toolkit'.")
+        on_success = DmsDataModelValidation(
+            self._client,
+            modus_operandi=self._config.modeling.mode,
+            can_run_validator=self._config.validation.can_run_validator,
+        )
+
+        return self._store.read_physical(reader, on_success)
+
+    def excel(self, io: Any) -> None:
+        """Read physical data model from Excel file"""
+
+        path = NeatReader.create(io).materialize_path()
+        reader = DMSTableImporter.from_excel(path)
+        on_success = DmsDataModelValidation(
+            self._client,
+            modus_operandi=self._config.modeling.mode,
+            can_run_validator=self._config.validation.can_run_validator,
+        )
+
+        return self._store.read_physical(reader, on_success)
+
+    def cdf(self, space: str, external_id: str, version: str) -> None:
+        """Read physical data model from CDF
+
+        Args:
+            space (str): The schema space of the data model.
+            external_id (str): The external id of the data model.
+            version (str): The version of the data model.
+
+        """
+        reader = DMSAPIImporter.from_cdf(
+            DataModelReference(space=space, external_id=external_id, version=version), self._client
+        )
+        on_success = DmsDataModelValidation(
+            self._client,
+            modus_operandi=self._config.modeling.mode,
+            can_run_validator=self._config.validation.can_run_validator,
+        )
+
+        return self._store.read_physical(reader, on_success)
+
+
+@session_wrapper
+class WritePhysicalDataModel:
+    """Write physical data model to various sources from NeatSession graph store."""
+
+    def __init__(self, store: NeatStore, client: NeatClient, config: NeatConfig) -> None:
+        self._store = store
+        self._client = client
+        self._config = config
+
+    def yaml(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None:
+        """Write physical data model to YAML file
+
+        Args:
+            io (Any): The file path or buffer to write to.
+            format (Literal["neat", "toolkit"]): The format of the output file
+                - "neat": Neat's DMS table format.
+                - "toolkit": Cognite DMS API format which is the format used by Cognite Toolkit.
+        """
+
+        file_path = NeatReader.create(io).materialize_path()
+        writer: DMSExporter
+        if format == "neat":
+            writer = DMSTableYamlExporter()
+        elif format == "toolkit":
+            writer = DMSAPIYAMLExporter()
+        else:
+            raise UserInputError(f"Unsupported format: {format}. Supported formats are 'neat' and 'toolkit'.")
+
+        return self._store.write_physical(writer, file_path=file_path)
+
+    def json(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None:
+        """Write physical data model to JSON file
+
+        Args:
+            io (Any): The file path or buffer to write to.
+            format (Literal["neat", "toolkit"]): The format of the output file
+                - "neat": Neat's DMS table format.
+                - "toolkit": Cognite DMS API format which is the format used by Cognite Toolkit.
+        """
+
+        file_path = NeatReader.create(io).materialize_path()
+        writer: DMSExporter
+        if format == "neat":
+            writer = DMSTableJSONExporter()
+        elif format == "toolkit":
+            writer = DMSAPIJSONExporter()
+        else:
+            raise UserInputError(f"Unsupported format: {format}. Supported formats are 'neat' and 'toolkit'.")
+
+        return self._store.write_physical(writer, file_path=file_path)
+
+    def excel(self, io: Any, skip_other_spaces: bool = True) -> None:
+        """Write physical data model to Excel file
+
+        Args:
+            io (Any): The file path or buffer to write to.
+            skip_other_spaces (bool): If true, only properties in the same space as the data model will be written.
+
+        """
+
+        file_path = NeatReader.create(io).materialize_path()
+        options = WorkbookOptions(skip_properties_in_other_spaces=skip_other_spaces)
+        writer = DMSExcelExporter(options=options)
+
+        return self._store.write_physical(writer, file_path=file_path)
+
+    def cdf(self, dry_run: bool = True, rollback: bool = True, drop_data: bool = False) -> None:
+        """Write physical data model with views, containers, and spaces that are in the same space as the data model
+        to CDF.
+
+        This method depends on the session governance profile for data modeling set when creating the NeatSession.
+            - In 'additive' mode, only new or updates to data models/views/containers will be applied.
+                You cannot remove views from data models, properties from views or containers, or
+                indexes or constraints from containers.
+            - In 'rebuild' mode, the data model in CDF will be made to exactly match the data model in Neat.
+                If there are any breaking changes, Neat will delete and recreate the relevant
+                data model/view/container. However, if drop_data is set to False, Neat will treat
+                containers as 'additive' and will not delete any containers or remove properties,
+                indexes, or constraints. To fully rebuild the data model, including containers, set drop_data to True.
+
+        Args:
+            dry_run (bool): If true, the changes will not be applied to CDF. Instead, Neat will
+                report what changes would have been made.
+            rollback (bool): If true, all changes will be rolled back if any error occurs.
+            drop_data (bool): Only applicable if the session mode is 'rebuild'. If
+
+        """
+        writer = DMSAPIExporter()
+        options = DeploymentOptions(
+            dry_run=dry_run,
+            auto_rollback=rollback,
+            drop_data=drop_data,
+            modus_operandi=self._config.modeling.mode,
+        )
+        on_success = SchemaDeployer(self._client, options)
+        return self._store.write_physical(writer, on_success)

cognite/neat/_session/_result.py

@@ -0,0 +1,236 @@
+import json
+import uuid
+from typing import Any, get_args
+
+from pydantic import BaseModel, Field
+
+from cognite.neat._data_model.deployer.data_classes import (
+    AddedField,
+    ChangedField,
+    DataModelEndpoint,
+    DeploymentResult,
+    FieldChange,
+    FieldChanges,
+    RemovedField,
+    ResourceChange,
+)
+from cognite.neat._session._html._render import render
+from cognite.neat._store import NeatStore
+
+
+class EndpointStatistics(BaseModel):
+    """Statistics for a single endpoint."""
+
+    total: int = 0
+    create: int = 0
+    update: int = 0
+    delete: int = 0
+    skip: int = 0
+    unchanged: int = 0
+
+
+class ChangeTypeStatistics(BaseModel):
+    """Statistics grouped by change type."""
+
+    create: int = 0
+    update: int = 0
+    delete: int = 0
+    skip: int = 0
+    unchanged: int = 0
+
+
+class SeverityStatistics(BaseModel):
+    """Statistics grouped by severity."""
+
+    SAFE: int = 0
+    WARNING: int = 0
+    BREAKING: int = 0
+
+
+class DeploymentStatistics(BaseModel):
+    """Overall deployment statistics."""
+
+    status: str
+    is_dry_run: bool
+    total_changes: int = 0
+    by_endpoint: dict[str, EndpointStatistics] = Field(default_factory=dict)
+    by_change_type: ChangeTypeStatistics = Field(default_factory=ChangeTypeStatistics)
+    by_severity: SeverityStatistics = Field(default_factory=SeverityStatistics)
+    has_recovery: bool = False
+
+
+class SerializedFieldChange(BaseModel):
+    """Serialized field change for JSON output."""
+
+    field_path: str
+    severity: str
+    description: str
+
+
+class SerializedResourceChange(BaseModel):
+    """Serialized resource change for JSON output."""
+
+    id: int
+    endpoint: str
+    change_type: str
+    severity: str
+    resource_id: str
+    changes: list[SerializedFieldChange] = Field(default_factory=list)
+
+
+class Result:
+    """Class to handle deployment results in the NeatSession."""
+
+    def __init__(self, store: NeatStore) -> None:
+        self._store = store
+
+    @property
+    def _result(self) -> DeploymentResult | None:
+        """Get deployment result from the last change in the store."""
+        if change := self._store.provenance.last_change:
+            if change.result:
+                return change.result
+        return None
+
+    def _initialize_stats(self, result: DeploymentResult) -> DeploymentStatistics:
+        """Initialize the statistics structure."""
+        # Get all possible endpoints from the DataModelEndpoint type alias
+        endpoints = get_args(DataModelEndpoint)
+
+        return DeploymentStatistics(
+            status=result.status,
+            is_dry_run=result.is_dry_run,
+            total_changes=0,
+            by_endpoint={endpoint: EndpointStatistics() for endpoint in endpoints},
+            by_change_type=ChangeTypeStatistics(),
+            by_severity=SeverityStatistics(),
+            has_recovery=result.recovery is not None,
+        )
+
+    def _update_statistics(self, stats: DeploymentStatistics, endpoint: str, change_type: str, severity: str) -> None:
+        """Update all statistics counters for a single resource change."""
+        stats.total_changes += 1
+
+        # Update change type statistics
+        setattr(stats.by_change_type, change_type, getattr(stats.by_change_type, change_type) + 1)
+
+        # Update endpoint statistics
+        endpoint_stats = stats.by_endpoint[endpoint]
+        endpoint_stats.total += 1
+        setattr(endpoint_stats, change_type, getattr(endpoint_stats, change_type) + 1)
+
+        # Update severity statistics
+        setattr(stats.by_severity, severity, getattr(stats.by_severity, severity) + 1)
+
+    @property
+    def _stats(self) -> DeploymentStatistics | None:
+        """Compute statistics about deployment result."""
+        if not self._result:
+            return None
+
+        stats = self._initialize_stats(self._result)
+
+        for plan in self._result.plan:
+            for resource in plan.resources:
+                self._update_statistics(
+                    stats=stats,
+                    endpoint=plan.endpoint,
+                    change_type=resource.change_type,
+                    severity=resource.severity.name,
+                )
+
+        return stats
+
+    def _serialize_single_field_change(self, field_change: FieldChange) -> SerializedFieldChange:
+        """Serialize a single non-nested field change."""
+        return SerializedFieldChange(
+            field_path=field_change.field_path,
+            severity=field_change.severity.name,
+            description=field_change.description
+            if isinstance(field_change, AddedField | RemovedField | ChangedField)
+            else "Field changed",
+        )
+
+    def _serialize_field_change(self, field_change: FieldChange) -> list[SerializedFieldChange]:
+        """Serialize a field change, handling nested FieldChanges recursively."""
+        serialized_changes: list[SerializedFieldChange] = []
+
+        if isinstance(field_change, FieldChanges):
+            # Recursively handle nested changes
+            for change in field_change.changes:
+                serialized_changes.extend(self._serialize_field_change(change))
+        else:
+            # Base case: single field change
+            serialized_changes.append(self._serialize_single_field_change(field_change))
+
+        return serialized_changes
+
+    def _serialize_resource_change(
+        self, resource: ResourceChange, endpoint: str, change_id: int
+    ) -> SerializedResourceChange:
+        """Serialize a single resource change."""
+        changes: list[SerializedFieldChange] = []
+
+        for change in resource.changes:
+            changes.extend(self._serialize_field_change(change))
+
+        return SerializedResourceChange(
+            id=change_id,
+            endpoint=endpoint,
+            change_type=resource.change_type,
+            severity=resource.severity.name,
+            resource_id=str(resource.resource_id),
+            changes=changes,
+        )
+
+    @property
+    def _serialized_changes(self) -> list[SerializedResourceChange]:
+        """Convert deployment changes to JSON-serializable format."""
+        if not self._result:
+            return []
+
+        all_changes: list[SerializedResourceChange] = []
+
+        for plan in self._result.plan:
+            for resource in plan.resources:
+                change_data = self._serialize_resource_change(
+                    resource=resource,
+                    endpoint=plan.endpoint,
+                    change_id=len(all_changes),
+                )
+                all_changes.append(change_data)
+
+        return all_changes
+
+    def _build_template_vars(self, stats: DeploymentStatistics) -> dict[str, Any]:
+        """Build template variables from statistics."""
+        # Generate unique ID for this render to avoid conflicts in Jupyter
+        unique_id = uuid.uuid4().hex[:8]
+
+        # Convert Pydantic models to dicts for JSON serialization
+        serialized_changes = [change.model_dump() for change in self._serialized_changes]
+
+        return {
+            "CHANGES_JSON": json.dumps(serialized_changes),
+            "STATS_JSON": stats.model_dump_json(),
+            "status": stats.status,
+            "total_changes": stats.total_changes,
+            "created": stats.by_change_type.create,
+            "updated": stats.by_change_type.update,
+            "deleted": stats.by_change_type.delete,
+            "skipped": stats.by_change_type.skip,
+            "unchanged": stats.by_change_type.unchanged,
+            "unique_id": unique_id,
+        }
+
+    def _repr_html_(self) -> str:
+        """Generate interactive HTML representation."""
+        if not self._result:
+            return "<p>No deployment result available</p>"
+
+        stats = self._stats
+        if not stats:
+            return "<p>Unable to compute deployment statistics</p>"
+
+        template_vars = self._build_template_vars(stats)
+        return render("deployment", template_vars)

cognite/neat/_session/_session.py

@@ -0,0 +1,88 @@
+import json
+from typing import Literal
+
+from cognite.client import ClientConfig, CogniteClient
+
+from cognite.neat import _version
+from cognite.neat._client import NeatClient
+from cognite.neat._config import internal_profiles
+from cognite.neat._state_machine import EmptyState, PhysicalState
+from cognite.neat._store import NeatStore
+from cognite.neat._utils.http_client import ParametersRequest, SuccessResponse
+
+from ._issues import Issues
+from ._opt import Opt
+from ._physical import PhysicalDataModel
+from ._result import Result
+
+
+class NeatSession:
+    """A session is an interface for neat operations."""
+
+    def __init__(
+        self,
+        client: CogniteClient | ClientConfig,
+        config: Literal["legacy-additive", "legacy-rebuild", "deep-additive", "deep-rebuild"] | None = None,
+    ) -> None:
+        """Initialize a Neat session.
+
+        Args:
+            client (CogniteClient | ClientConfig): The Cognite client or client configuration to use for the session.
+            config (Literal["legacy-additive", "legacy-rebuild", "deep-additive", "deep-rebuild"] | None):
+                The configuration profile to use for the session.
+                If None, the default profile "legacy-additive" is used. Meaning that Neat will perform additive modeling
+                and apply only validations that were part of the legacy Neat version."""
+
+        # Load configuration
+        if config and config not in internal_profiles():
+            raise ValueError(f"Profile '{config}' not found among internal profiles.")
+
+        self._config = internal_profiles()[config or "legacy-additive"]
+
+        # Use configuration for physical data model
+        self._store = NeatStore()
+        self._client = NeatClient(client)
+        self.physical_data_model = PhysicalDataModel(self._store, self._client, self._config)
+        self.issues = Issues(self._store)
+        self.result = Result(self._store)
+        self.opt = Opt(self._store)
+
+        if self.opt._collector.can_collect:
+            self.opt._collector.collect("initSession", {"mode": self._config.modeling.mode})
+
+        self._welcome_message()
+
+    def _welcome_message(self) -> None:
+        cdf_project = self._client.config.project
+        message = f"Neat session started for CDF project: '{cdf_project}'"
+        responses = self._client.http_client.request(
+            ParametersRequest(endpoint_url=self._client.config.create_api_url(""), method="GET")
+        )
+        if len(responses) == 1 and isinstance(response := responses[0], SuccessResponse):
+            organization = ""
+            try:
+                organization = json.loads(response.body)["organization"]
+            except (KeyError, ValueError):
+                ...
+            if organization:
+                message += f" (Organization: '{organization}')"
+
+        print(message)
+        print(self._config)
+
+    @property
+    def version(self) -> str:
+        """Get the current version of neat."""
+        return _version.__version__
+
+    def _repr_html_(self) -> str:
+        if isinstance(self._store.state, EmptyState):
+            return (
+                "<strong>Empty session</strong>. Get started by reading for example physical data model"
+                " <em>.physical_data_model.read</em>"
+            )
+
+        if isinstance(self._store.state, PhysicalState):
+            return self.physical_data_model._repr_html_()
+
+        raise RuntimeError("Unknown session state, contact support.")