cognite-neat 0.126.0__py3-none-any.whl → 0.126.1__py3-none-any.whl

cognite/neat/_data_model/deployer/data_classes.py

@@ -0,0 +1,176 @@
+from abc import ABC, abstractmethod
+from datetime import datetime
+from enum import Enum
+from typing import Generic, Literal, TypeAlias
+
+from pydantic import BaseModel, Field
+from pydantic.alias_generators import to_camel
+
+from cognite.neat._data_model.models.dms import (
+    BaseModelObject,
+    ContainerReference,
+    ContainerRequest,
+    DataModelReference,
+    DataModelRequest,
+    NodeReference,
+    SpaceRequest,
+    T_Resource,
+    ViewReference,
+    ViewRequest,
+)
+from cognite.neat._utils.http_client._data_classes import HTTPMessage
+from cognite.neat._utils.useful_types import T_Reference
+
+JsonPath: TypeAlias = str  # e.g., 'properties.temperature', 'constraints.uniqueKey'
+DataModelEndpoint: TypeAlias = Literal["spaces", "containers", "views", "datamodels", "instances"]
+
+
+class SeverityType(Enum):
+    SAFE = 1
+    WARNING = 2
+    BREAKING = 3
+
+    @classmethod
+    def max_severity(cls, severities: list["SeverityType"], default: "SeverityType") -> "SeverityType":
+        value = max([severity.value for severity in severities], default=default.value)
+        return cls(value)
+
+
+class BaseDeployObject(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_name=True):
+    """Base class for all deployer data model objects."""
+
+    ...
+
+
+class FieldChange(BaseDeployObject, ABC):
+    """Represents a change to a specific property or field."""
+
+    field_path: JsonPath
+
+    @property
+    @abstractmethod
+    def severity(self) -> SeverityType:
+        """The severity of the change."""
+        raise NotImplementedError()
+
+
+class PrimitiveField(FieldChange, ABC):
+    """Base class for changes to primitive properties."""
+
+    item_severity: SeverityType
+
+    @property
+    def severity(self) -> SeverityType:
+        return self.item_severity
+
+
+class AddedField(PrimitiveField):
+    new_value: BaseModelObject | str | int | float | bool | None
+
+    @property
+    def description(self) -> str:
+        return f"added with value {self.new_value!r}"
+
+
+class RemovedField(PrimitiveField):
+    current_value: BaseModelObject | str | int | float | bool | None
+
+    @property
+    def description(self) -> str:
+        return f"removed (was {self.current_value!r})"
+
+
+class ChangedField(PrimitiveField):
+    new_value: BaseModelObject | str | int | float | bool | None
+    current_value: BaseModelObject | str | int | float | bool | None
+
+    @property
+    def description(self) -> str:
+        if self.new_value is None:
+            return f"removed (was {self.current_value!r})"
+        elif self.current_value is None:
+            return f"added with value {self.new_value!r}"
+        return f"changed from {self.current_value!r} to {self.new_value!r}"
+
+
+class FieldChanges(FieldChange):
+    """Represents a nested property, i.e., a property that contains other properties."""
+
+    changes: list[FieldChange]
+
+    @property
+    def severity(self) -> SeverityType:
+        return SeverityType.max_severity([item.severity for item in self.changes], default=SeverityType.SAFE)
+
+
+class ResourceChange(BaseDeployObject, Generic[T_Reference, T_Resource]):
+    resource_id: T_Reference
+    new_value: T_Resource
+    old_value: T_Resource | None = None
+    changes: list[FieldChange] = Field(default_factory=list)
+
+    @property
+    def change_type(self) -> Literal["create", "update", "delete", "unchanged"]:
+        if self.old_value is None:
+            return "create"
+        elif self.changes:
+            return "update"
+        else:
+            return "unchanged"
+
+    @property
+    def severity(self) -> SeverityType:
+        return SeverityType.max_severity([change.severity for change in self.changes], default=SeverityType.SAFE)
+
+
+class ResourceDeploymentPlan(BaseDeployObject, Generic[T_Reference, T_Resource]):
+    endpoint: DataModelEndpoint
+    resources: list[ResourceChange[T_Reference, T_Resource]]
+
+    @property
+    def to_upsert(self) -> list[ResourceChange[T_Reference, T_Resource]]:
+        return [change for change in self.resources if change.change_type in ("create", "update")]
+
+    @property
+    def to_delete(self) -> list[ResourceChange[T_Reference, T_Resource]]:
+        return [change for change in self.resources if change.change_type == "delete"]
+
+    @property
+    def unchanged(self) -> list[ResourceChange[T_Reference, T_Resource]]:
+        return [change for change in self.resources if change.change_type == "unchanged"]
+
+
+class SchemaSnapshot(BaseDeployObject):
+    timestamp: datetime
+    data_model: dict[DataModelReference, DataModelRequest]
+    views: dict[ViewReference, ViewRequest]
+    containers: dict[ContainerReference, ContainerRequest]
+    spaces: dict[str, SpaceRequest]
+    node_types: dict[NodeReference, NodeReference]
+
+
+class ChangeResult(BaseDeployObject, Generic[T_Reference, T_Resource]):
+    change: ResourceChange[T_Reference, T_Resource]
+    message: HTTPMessage
+
+
+class AppliedChanges(BaseDeployObject, Generic[T_Reference, T_Resource]):
+    created: list[ChangeResult[T_Reference, T_Resource]] = Field(default_factory=list)
+    updated: list[ChangeResult[T_Reference, T_Resource]] = Field(default_factory=list)
+    deletions: list[ChangeResult[T_Reference, T_Resource]] = Field(default_factory=list)
+
+
+class DeploymentResult(BaseDeployObject):
+    status: Literal["success", "failure", "partial", "pending"]
+    plan: list[ResourceDeploymentPlan]
+    snapshot: SchemaSnapshot
+    responses: AppliedChanges | None = None
+    recovery: AppliedChanges | None = None
+
+    @property
+    def is_dry_run(self) -> bool:
+        return self.status == "pending"
+
+    @property
+    def is_success(self) -> bool:
+        return self.status == "success"

cognite/neat/_data_model/exporters/__init__.py

@@ -0,0 +1,4 @@
+from ._base import DMSExporter
+from ._table_exporter.exporter import DMSExcelExporter, DMSTableExporter, DMSYamlExporter
+
+__all__ = ["DMSExcelExporter", "DMSExporter", "DMSTableExporter", "DMSYamlExporter"]

cognite/neat/_data_model/exporters/_base.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from pathlib import Path
 from typing import Generic, TypeVar
 
 from cognite.neat._data_model.models.dms import RequestSchema
@@ -13,5 +14,9 @@ class DMSExporter(ABC, Generic[T_Export]):
     ENCODING = "utf-8"
 
     @abstractmethod
-    def export(self, data_model: RequestSchema) -> T_Export:
+    def _export(self, data_model: RequestSchema) -> T_Export:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def export(self, data_model: RequestSchema, file_path: Path) -> None:
         raise NotImplementedError()

cognite/neat/_data_model/exporters/_table_exporter/exporter.py

@@ -0,0 +1,106 @@
+from pathlib import Path
+from typing import cast
+
+import yaml
+from pyparsing import ABC
+
+from cognite.neat._data_model.exporters._base import DMSExporter
+from cognite.neat._data_model.importers._table_importer.data_classes import DMSProperty, TableDMS
+from cognite.neat._data_model.models.dms import RequestSchema
+from cognite.neat._utils.useful_types import DataModelTableType
+
+from .workbook import WorkbookCreator, WorkbookOptions
+from .writer import DMSTableWriter
+
+
+class DMSTableExporter(DMSExporter[DataModelTableType], ABC):
+    """Exports DMS to a table structure.
+
+    The tables can are expected to be a dictionary where the keys are the table names and the values
+    are lists of dictionaries representing the rows in the table.
+    """
+
+    class Sheets:
+        properties = cast(str, TableDMS.model_fields["properties"].validation_alias)
+
+    def __init__(self, exclude_none: bool = False) -> None:
+        self._exclude_none = exclude_none
+
+    def _export(self, data_model: RequestSchema) -> DataModelTableType:
+        model = data_model.data_model
+        tables = DMSTableWriter(model.space, model.version).write_tables(data_model)
+        exclude: set[str] = set()
+        if self._exclude_none:
+            if not tables.enum:
+                exclude.add("enum")
+            if not tables.nodes:
+                exclude.add("nodes")
+            if not tables.containers:
+                exclude.add("containers")
+
+        output = tables.model_dump(mode="json", by_alias=True, exclude_none=self._exclude_none, exclude=exclude)
+        # When we have exclude_none we only want to exclude none of optional properties, not required.
+        # Thus, we do the implementation below
+        required_properties = [
+            field_.serialization_alias for field_ in DMSProperty.model_fields.values() if field_.is_required()
+        ]
+        for row in output[self.Sheets.properties]:
+            for prop in required_properties:
+                if prop not in row:
+                    row[prop] = None
+        return output
+
+
+class DMSYamlExporter(DMSTableExporter):
+    """Exports DMS to YAML."""
+
+    def __init__(self) -> None:
+        super().__init__(exclude_none=True)
+
+    def export(self, data_model: RequestSchema, file_path: Path) -> None:
+        """Exports the data model as a flat YAML file, which is identical to the spreadsheet representation
+
+        Args:
+            data_model (RequestSchema): The data model to export.
+            file_path (Path): The path to the YAML file to create.
+        """
+        table_format = self._export(data_model)
+        file_path.write_text(
+            yaml.safe_dump(table_format, sort_keys=False), encoding=self.ENCODING, newline=self.NEW_LINE
+        )
+
+
+class DMSExcelExporter(DMSTableExporter):
+    """Exports DMS to Excel file."""
+
+    def __init__(self, options: WorkbookOptions | None = None) -> None:
+        super().__init__(exclude_none=False)
+        self._options = options or WorkbookOptions()
+
+    def export(self, data_model: RequestSchema, file_path: Path) -> None:
+        """Exports the data model as a Excel file.
+
+        Args:
+            data_model (RequestSchema): The data model to export.
+            file_path (Path): The path to the Excel file to create.
+            options (WorkbookOptions | None): Options for creating the workbook.
+        """
+        table_format = self._export(data_model)
+        workbook = WorkbookCreator(self._options).create_workbook(table_format)
+        try:
+            workbook.save(file_path)
+        finally:
+            workbook.close()
+
+
+class DMSCsvExporter(DMSTableExporter):
+    """Exports DMS to CSV files in a directory."""
+
+    def export(self, data_model: RequestSchema, directory_path: Path) -> None:
+        """Exports the data model as a set of CSV files, one for each table.
+
+        Args:
+            data_model (RequestSchema): The data model to export.
+            directory_path (Path): The path to the directory to create the CSV files in.
+        """
+        raise NotImplementedError()