cognite-neat 1.0.5__py3-none-any.whl → 1.0.7__py3-none-any.whl

cognite/neat/_data_model/deployer/data_classes.py

@@ -2,6 +2,7 @@ import itertools
 import sys
 from abc import ABC, abstractmethod
 from collections import UserList, defaultdict
+from collections.abc import Hashable, Sequence
 from datetime import datetime
 from enum import Enum
 from typing import Any, Generic, Literal, TypeAlias, cast
@@ -379,15 +380,88 @@ class ResourceDeploymentPlanList(UserList[ResourceDeploymentPlan]):
         return type(self)(forced_plans)
 
 
-class ChangeResult(BaseDeployObject, Generic[T_ResourceId, T_DataModelResource]):
+class ChangeResult(BaseDeployObject, Generic[T_ResourceId, T_DataModelResource], ABC):
     endpoint: DataModelEndpoint
     change: ResourceChange[T_ResourceId, T_DataModelResource]
-    message: SuccessResponseItems[T_ResourceId] | FailedResponseItems[T_ResourceId] | FailedRequestItems[T_ResourceId]
 
+    @property
+    @abstractmethod
+    def message(self) -> str:
+        """Human-readable message about the change result."""
+        ...
 
-class ChangedFieldResult(BaseDeployObject, Generic[T_Reference]):
+    @property
+    @abstractmethod
+    def is_success(self) -> bool:
+        """Whether the change was successful."""
+        ...
+
+
+class HTTPChangeResult(ChangeResult[T_ResourceId, T_DataModelResource]):
+    http_message: (
+        SuccessResponseItems[T_ResourceId] | FailedResponseItems[T_ResourceId] | FailedRequestItems[T_ResourceId]
+    )
+
+    @property
+    def message(self) -> str:
+        if isinstance(self.http_message, SuccessResponse):
+            return "Success"
+        elif isinstance(self.http_message, FailedResponseItems):
+            error = self.http_message.error
+            return f"Failed: {error.code} | {error.message}"
+        elif isinstance(self.http_message, FailedRequestItems):
+            return f"Request Failed: {self.http_message.message}"
+        else:
+            return "Unknown result"
+
+    @property
+    def is_success(self) -> bool:
+        return isinstance(self.http_message, SuccessResponse)
+
+
+class MultiHTTPChangeResult(ChangeResult[T_ResourceId, T_DataModelResource]):
+    http_messages: list[
+        SuccessResponseItems[T_ResourceId] | FailedResponseItems[T_ResourceId] | FailedRequestItems[T_ResourceId]
+    ]
+
+    @property
+    def message(self) -> str:
+        error_messages: list[str] = []
+        for msg in self.http_messages:
+            if isinstance(msg, SuccessResponse):
+                continue
+            elif isinstance(msg, FailedResponseItems):
+                error = msg.error
+                error_messages.append(f"Failed: {error.code} | {error.message}")
+            elif isinstance(msg, FailedRequestItems):
+                error_messages.append(f"Request Failed: {msg.message}")
+        if not error_messages:
+            return "Success"
+        return "; ".join(error_messages)
+
+    @property
+    def is_success(self) -> bool:
+        return all(isinstance(msg, SuccessResponse) for msg in self.http_messages)
+
+
+class NoOpChangeResult(ChangeResult[T_ResourceId, T_DataModelResource]):
+    """A change result representing a no-op, e.g., when a change was skipped or unchanged."""
+
+    reason: str
+
+    @property
+    def message(self) -> str:
+        return self.reason
+
+    @property
+    def is_success(self) -> bool:
+        return True
+
+
+class ChangedFieldResult(BaseDeployObject, Generic[T_ResourceId, T_Reference]):
+    resource_id: T_ResourceId
     field_change: FieldChange
-    message: SuccessResponseItems[T_Reference] | FailedResponseItems[T_Reference] | FailedRequestItems[T_Reference]
+    http_message: SuccessResponseItems[T_Reference] | FailedResponseItems[T_Reference] | FailedRequestItems[T_Reference]
 
 
 class AppliedChanges(BaseDeployObject):
@@ -399,26 +473,55 @@ class AppliedChanges(BaseDeployObject):
     This is needed as these changes are done with a separate API call per change.
     """
 
-    created: list[ChangeResult] = Field(default_factory=list)
-    updated: list[ChangeResult] = Field(default_factory=list)
-    deletions: list[ChangeResult] = Field(default_factory=list)
-    unchanged: list[ResourceChange] = Field(default_factory=list)
-    skipped: list[ResourceChange] = Field(default_factory=list)
+    created: list[HTTPChangeResult] = Field(default_factory=list)
+    updated: list[HTTPChangeResult] = Field(default_factory=list)
+    deletions: list[HTTPChangeResult] = Field(default_factory=list)
+    unchanged: list[NoOpChangeResult] = Field(default_factory=list)
+    skipped: list[NoOpChangeResult] = Field(default_factory=list)
     changed_fields: list[ChangedFieldResult] = Field(default_factory=list)
 
     @property
     def is_success(self) -> bool:
         return all(
             # MyPy fails to understand that ChangeFieldResult.message has the same structure as ChangeResult.message
-            isinstance(change.message, SuccessResponse)  # type: ignore[attr-defined]
+            isinstance(change.http_message, SuccessResponse)  # type: ignore[attr-defined]
             for change in itertools.chain(self.created, self.updated, self.deletions, self.changed_fields)
         )
 
+    @property
+    def merged_updated(self) -> Sequence[ChangeResult]:
+        """Merges the changed field into the updated changes."""
+        if not self.changed_fields:
+            return self.updated
+        changed_fields_by_id: dict[Hashable, list[ChangedFieldResult]] = defaultdict(list)
+        for changed_field in self.changed_fields:
+            changed_fields_by_id[changed_field.resource_id].append(changed_field)
+        merged_changes: list[ChangeResult] = []
+        for update in self.updated:
+            if update.change.resource_id not in changed_fields_by_id:
+                merged_changes.append(update)
+                continue
+
+            field_changes = changed_fields_by_id[update.change.resource_id]
+            merged_change = update.change.model_copy(
+                update={"changes": update.change.changes + [fc.field_change for fc in field_changes]}
+            )
+
+            # MyPy wants an annotation were we want this to be generic.
+            merged_result = MultiHTTPChangeResult(  # type: ignore[var-annotated]
+                endpoint=update.endpoint,
+                change=merged_change,
+                http_messages=[update.http_message] + [fc.http_message for fc in field_changes],
+            )
+            merged_changes.append(merged_result)
+
+        return merged_changes
+
     def as_recovery_plan(self) -> list[ResourceDeploymentPlan]:
         """Generate a recovery plan based on the applied changes."""
         recovery_plan: dict[DataModelEndpoint, ResourceDeploymentPlan] = {}
         for change_result in itertools.chain(self.created, self.updated, self.deletions):
-            if not isinstance(change_result.message, SuccessResponse):
+            if not isinstance(change_result.http_message, SuccessResponse):
                 continue  # Skip failed changes.
             change = change_result.change
             if change.change_type == "create":
@@ -504,7 +607,7 @@ class DeploymentResult(BaseDeployObject):
 
     @property
     def is_dry_run(self) -> bool:
-        return self.status == "pending"
+        return self.responses is None
 
     @property
     def is_success(self) -> bool:
@@ -520,10 +623,10 @@ class DeploymentResult(BaseDeployObject):
         if self.responses:
             counts: dict[str, int] = defaultdict(int)
             for change in itertools.chain(self.responses.created, self.responses.updated, self.responses.deletions):
-                suffix = type(change.message).__name__.removesuffix("[TypeVar]").removesuffix("[~T_ResourceId]")
+                suffix = type(change.http_message).__name__.removesuffix("[TypeVar]").removesuffix("[~T_ResourceId]")
                 # For example: containers.created.successResponseItems
                 prefix = f"{change.endpoint}.{change.change.change_type}.{suffix}"
-                counts[prefix] += len(change.message.ids)
+                counts[prefix] += len(change.http_message.ids)
 
             output.update(counts)
         return output

cognite/neat/_data_model/deployer/deployer.py

@@ -35,12 +35,13 @@ from .data_classes import (
     AddedField,
     AppliedChanges,
     ChangedFieldResult,
-    ChangeResult,
     ContainerDeploymentPlan,
     DataModelEndpoint,
     DeploymentResult,
     FieldChange,
     FieldChanges,
+    HTTPChangeResult,
+    NoOpChangeResult,
     RemovedField,
     ResourceChange,
     ResourceDeploymentPlan,
@@ -289,24 +290,58 @@ class SchemaDeployer(OnSuccessResultProducer):
             AppliedChanges: The result of applying the changes.
         """
         applied_changes = AppliedChanges()
+        # If any HTTP request fails, the skip_message will be set and subsequent operations will be skipped
+        failure_message: str | None = None
         for resource in reversed(plan):
-            deletions = self._delete_items(resource)
-            applied_changes.deletions.extend(deletions)
+            if failure_message is None:
+                deletions = self._delete_items(resource)
+                applied_changes.deletions.extend(deletions)
+                if any(not deletion.is_success for deletion in deletions):
+                    failure_message = f"Skipping due to {resource.endpoint} deletions failing."
+            else:
+                applied_changes.skipped.extend(
+                    [
+                        NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=failure_message)
+                        for change in resource.to_delete
+                    ]
+                )
 
         for resource in plan:
-            if isinstance(resource, ContainerDeploymentPlan):
-                applied_changes.changed_fields.extend(self._remove_container_constraints(resource))
-                applied_changes.changed_fields.extend(self._remove_container_indexes(resource))
-
-            creations, updated = self._upsert_items(resource)
-            applied_changes.created.extend(creations)
-            applied_changes.updated.extend(updated)
+            if failure_message is None:
+                if isinstance(resource, ContainerDeploymentPlan):
+                    # Note that we continue to deploy even if removing constraints/indexes fail,
+                    # as the creation/update of views and data models will still succeed.
+                    applied_changes.changed_fields.extend(self._remove_container_constraints(resource))
+                    applied_changes.changed_fields.extend(self._remove_container_indexes(resource))
+
+                creations, updated = self._upsert_items(resource)
+                applied_changes.created.extend(creations)
+                applied_changes.updated.extend(updated)
+                if any(not change.is_success for change in creations + updated):
+                    failure_message = f"Skipping due to {resource.endpoint} upsert failing."
+            else:
+                applied_changes.skipped.extend(
+                    [
+                        NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=failure_message)
+                        for change in resource.to_upsert
+                    ]
+                )
 
-            applied_changes.unchanged.extend(resource.unchanged)
-            applied_changes.skipped.extend(resource.skip)
+            applied_changes.unchanged.extend(
+                [
+                    NoOpChangeResult(endpoint=resource.endpoint, change=change, reason="No changes detected.")
+                    for change in resource.unchanged
+                ]
+            )
+            applied_changes.skipped.extend(
+                [
+                    NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=change.message or "Unknown")
+                    for change in resource.skip
+                ]
+            )
         return applied_changes
 
-    def _delete_items(self, resource: ResourceDeploymentPlan) -> list[ChangeResult]:
+    def _delete_items(self, resource: ResourceDeploymentPlan) -> list[HTTPChangeResult]:
         to_delete_by_id = {change.resource_id: change for change in resource.to_delete}
         if not to_delete_by_id:
             return []
@@ -319,7 +354,7 @@ class SchemaDeployer(OnSuccessResultProducer):
         )
         return self._process_resource_responses(responses, to_delete_by_id, resource.endpoint)
 
-    def _upsert_items(self, resource: ResourceDeploymentPlan) -> tuple[list[ChangeResult], list[ChangeResult]]:
+    def _upsert_items(self, resource: ResourceDeploymentPlan) -> tuple[list[HTTPChangeResult], list[HTTPChangeResult]]:
         to_upsert = [
             resource_change.new_value for resource_change in resource.to_upsert if resource_change.new_value is not None
         ]
@@ -375,14 +410,14 @@ class SchemaDeployer(OnSuccessResultProducer):
     @classmethod
     def _process_resource_responses(
         cls, responses: APIResponse, change_by_id: dict[T_ResourceId, ResourceChange], endpoint: DataModelEndpoint
-    ) -> list[ChangeResult]:
-        results: list[ChangeResult] = []
+    ) -> list[HTTPChangeResult]:
+        results: list[HTTPChangeResult] = []
         for response in responses:
             if isinstance(response, SuccessResponseItems | FailedResponseItems | FailedRequestItems):
                 for id in response.ids:
                     if id not in change_by_id:
                         continue
-                    results.append(ChangeResult(change=change_by_id[id], message=response, endpoint=endpoint))
+                    results.append(HTTPChangeResult(change=change_by_id[id], http_message=response, endpoint=endpoint))
             else:
                 # This should never happen as we do a ItemsRequest should always return ItemMessage responses
                 raise ValueError("Bug in Neat. Got an unexpected response type.")
@@ -398,7 +433,9 @@ class SchemaDeployer(OnSuccessResultProducer):
                 for id in response.ids:
                     if id not in change_by_id:
                         continue
-                    results.append(ChangedFieldResult(field_change=change_by_id[id], message=response))
+                    results.append(
+                        ChangedFieldResult(resource_id=id, field_change=change_by_id[id], http_message=response)
+                    )
             else:
                 # This should never happen as we do a ItemsRequest should always return ItemMessage responses
                 raise RuntimeError("Bug in Neat. Got an unexpected response type.")

cognite/neat/_session/_html/static/deployment.css

@@ -132,6 +132,16 @@
     color: #9ca3af;
 }
 
+.change-failed {
+    background: #fee2e2;
+    color: #991b1b;
+}
+
+.dark-mode .change-failed {
+    background: #7f1d1d;
+    color: #fca5a5;
+}
+
 .severity-badge {
     padding: 4px 10px;
     border-radius: 4px;
@@ -286,6 +296,16 @@
     color: white;
 }
 
+.status-recovered {
+    background: #10b981;
+    color: white;
+}
+
+.status-recovery_failed {
+    background: #dc2626;
+    color: white;
+}
+
 .dark-mode .status-success {
     background: #059669;
 }
@@ -300,4 +320,157 @@
 
 .dark-mode .status-pending {
     background: #2563eb;
+}
+
+.dark-mode .status-recovered {
+    background: #059669;
+}
+
+.dark-mode .status-recovery_failed {
+    background: #b91c1c;
+}
+
+
+/* Error message handling */
+
+.error-message-box {
+    background: #fee2e2;
+    border: 2px solid #dc2626;
+    border-left: 6px solid #dc2626;
+    border-radius: 8px;
+    padding: 16px;
+    margin-top: 12px;
+    animation: slideIn 0.3s ease-out;
+}
+
+.dark-mode .error-message-box {
+    background: #7f1d1d;
+    border-color: #ef4444;
+}
+
+.error-message-header {
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    margin-bottom: 8px;
+}
+
+.error-icon {
+    font-size: 24px;
+}
+
+.error-title {
+    font-weight: 700;
+    font-size: 14px;
+    color: #991b1b;
+}
+
+.dark-mode .error-title {
+    color: #fca5a5;
+}
+
+.error-message-content {
+    color: #7f1d1d;
+    font-size: 13px;
+    line-height: 1.6;
+    padding-left: 34px;
+    font-family: 'Monaco', 'Courier New', monospace;
+}
+
+.dark-mode .error-message-content {
+    color: #fca5a5;
+}
+
+.change-item.failed-change {
+    border-left: 4px solid #dc2626;
+    background: rgba(254, 226, 226, 0.1);
+}
+
+.dark-mode .change-item.failed-change {
+    border-left-color: #ef4444;
+    background: rgba(127, 29, 29, 0.2);
+}
+
+.info-message-box {
+    border-radius: 6px;
+    padding: 12px 16px;
+    margin-top: 12px;
+    font-size: 13px;
+    line-height: 1.6;
+}
+
+.info-message-content {
+    color: inherit;
+}
+
+.info-message-create {
+    background: #d1fae5;
+    border-left: 4px solid #065f46;
+    color: #065f46;
+}
+
+.info-message-update {
+    background: #dbeafe;
+    border-left: 4px solid #1e40af;
+    color: #1e40af;
+}
+
+.info-message-delete {
+    background: #fee2e2;
+    border-left: 4px solid #991b1b;
+    color: #991b1b;
+}
+
+.info-message-skip {
+    background: #fef3c7;
+    border-left: 4px solid #92400e;
+    color: #92400e;
+}
+
+.info-message-unchanged {
+    background: #f3f4f6;
+    border-left: 4px solid #4b5563;
+    color: #4b5563;
+}
+
+.dark-mode .info-message-create {
+    background: #064e3b;
+    border-left-color: #6ee7b7;
+    color: #6ee7b7;
+}
+
+.dark-mode .info-message-update {
+    background: #1e3a8a;
+    border-left-color: #93c5fd;
+    color: #93c5fd;
+}
+
+.dark-mode .info-message-delete {
+    background: #7f1d1d;
+    border-left-color: #fca5a5;
+    color: #fca5a5;
+}
+
+.dark-mode .info-message-skip {
+    background: #78350f;
+    border-left-color: #fde68a;
+    color: #fde68a;
+}
+
+.dark-mode .info-message-unchanged {
+    background: #374151;
+    border-left-color: #9ca3af;
+    color: #9ca3af;
+}
+
+
+@keyframes slideIn {
+    from {
+        opacity: 0;
+        transform: translateY(-10px);
+    }
+    to {
+        opacity: 1;
+        transform: translateY(0);
+    }
 }

cognite/neat/_session/_html/static/deployment.js

@@ -17,7 +17,9 @@ const STATUS_CONFIG = {
     success: { icon: '✅', text: 'Success' },
     failure: { icon: '❌', text: 'Failure' },
     partial: { icon: '⚠️', text: 'Partial Success' },
-    pending: { icon: '⏳', text: 'Pending (Dry Run)' }
+    pending: { icon: '⏳', text: 'Pending (Dry Run)' },
+    recovered: { icon: '🔄', text: 'Recovered' },
+    recovery_failed: { icon: '💔', text: 'Recovery Failed' }
 };
 
 // Initialize status badge
@@ -26,7 +28,7 @@ function initializeStatus() {
     const statusIcon = document.getElementById('statusIcon-' + uniqueId);
     const statusText = document.getElementById('statusText-' + uniqueId);
 
-    const statusConfig = STATUS_CONFIG[stats.status] || STATUS_CONFIG.pending;
+    const statusConfig = STATUS_CONFIG[deploymentStatus] || STATUS_CONFIG.pending;
     statusIcon.textContent = statusConfig.icon;
     statusText.textContent = statusConfig.text;
 }
@@ -43,6 +45,31 @@ function updateTheme() {
     }
 }
 
+function renderChangeMessage(change) {
+    if (!change.message) return '';
+
+    // Using textContent to set the message prevents XSS vulnerabilities
+    // by automatically escaping HTML characters.
+    const messageHolder = document.createElement('div');
+    messageHolder.textContent = change.message;
+    const escapedMessage = messageHolder.innerHTML;
+
+    if (change.change_type === 'failed') {
+        return `
+            <div class="error-message-box">
+                <div class="error-message-header">
+                    <span class="error-icon">❌</span>
+                    <span class="error-title">Deployment Failed</span>
+                </div>
+                <div class="error-message-content">${escapedMessage}</div>
+            </div>`;
+    }
+    return `
+        <div class="info-message-box info-message-${change.change_type}">
+            <div class="info-message-content">${escapedMessage}</div>
+        </div>`;
+}
+
 updateTheme();
 initializeStatus();
 
@@ -76,13 +103,14 @@ function renderChanges() {
     }
 
     listContainer.innerHTML = filtered.map(change => `
-        <div class="change-item">
+        <div class="change-item ${change.change_type === 'failed' ? 'failed-change' : ''}">
             <div class="change-header">
                 <span class="endpoint-badge endpoint-${change.endpoint}">${change.endpoint}</span>
                 <span class="change-type-badge change-${change.change_type}">${change.change_type}</span>
                 <span class="severity-badge severity-${change.severity}">${change.severity}</span>
             </div>
             <div class="resource-id">${change.resource_id}</div>
+
             ${change.changes.length > 0 ? `
                 <div class="field-changes">
                     ${change.changes.map(fc => `
@@ -93,6 +121,9 @@ function renderChanges() {
                     `).join('')}
                 </div>
             ` : ''}
+
+            ${renderChangeMessage(change)}
+
         </div>
     `).join('');
 }
@@ -130,8 +161,8 @@ document.getElementById('searchInput-' + uniqueId).addEventListener('input', fun
 window['exportDeployment_' + uniqueId] = function() {
     const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
     const report = {
-        status: stats.status,
-        is_dry_run: stats.is_dry_run,
+        status: deploymentStatus,
+        is_dry_run: is_dry_run === 'True',
         timestamp: timestamp,
         statistics: stats,
         changes: changes,

cognite/neat/_session/_html/templates/deployment.html

@@ -20,6 +20,9 @@
             <div class="stat-item active" data-filter="all">
                 <span class="stat-number">{{total_changes}}</span> Total Changes
             </div>
+            <div class="stat-item" data-filter="failed">
+                <span class="stat-number">{{failed}}</span> Failed
+            </div>
             <div class="stat-item" data-filter="create">
                 <span class="stat-number">{{created}}</span> Created
             </div>
@@ -29,12 +32,12 @@
             <div class="stat-item" data-filter="delete">
                 <span class="stat-number">{{deleted}}</span> Deleted
             </div>
-            <div class="stat-item" data-filter="skip">
-                <span class="stat-number">{{skipped}}</span> Skipped
-            </div>
             <div class="stat-item" data-filter="unchanged">
                 <span class="stat-number">{{unchanged}}</span> Unchanged
             </div>
+            <div class="stat-item" data-filter="skip">
+                <span class="stat-number">{{skipped}}</span> Skipped
+            </div>
             <button class="export-btn" onclick="exportDeployment_{{unique_id}}()">Export Report</button>
         </div>
     </div>
@@ -70,6 +73,8 @@
     const changes = {{CHANGES_JSON}};
     const stats = {{STATS_JSON}};
     const uniqueId = '{{unique_id}}';
+    const is_dry_run = '{{is_dry_run}}';
+    const deploymentStatus = '{{status}}';
     {{SCRIPTS}}
 })();
 </script>

cognite/neat/_session/_result/__init__.py

@@ -0,0 +1,3 @@
+from ._result import Result
+
+__all__ = ["Result"]