cognite-neat 1.0.4__py3-none-any.whl → 1.0.6__py3-none-any.whl

cognite/neat/_config.py

@@ -14,8 +14,15 @@ if sys.version_info >= (3, 11):
 else:
     import tomli  # type: ignore
 
+# Public profiles
 PredefinedProfile: TypeAlias = Literal["legacy-additive", "legacy-rebuild", "deep-additive", "deep-rebuild"]
 
+# Private profiles only
+_PrivateProfiles: TypeAlias = Literal["no-validation-additive", "no-validation-rebuild"]
+
+# All profiles (union of public and private)
+_AllProfiles: TypeAlias = PredefinedProfile | _PrivateProfiles
+
 
 class ConfigModel(BaseModel):
     model_config = ConfigDict(populate_by_name=True, validate_assignment=True)
@@ -25,6 +32,7 @@ class ValidationConfig(ConfigModel):
     """Validation configuration."""
 
     exclude: list[str] = Field(default_factory=list)
+    override: bool = Field(False, description="If enabled, all validators are skipped.")
 
     def can_run_validator(self, code: str, issue_type: type) -> bool:
         """
@@ -40,7 +48,7 @@ class ValidationConfig(ConfigModel):
 
         is_excluded = self._is_excluded(code, self.exclude)
 
-        if issue_type in [ModelSyntaxError, ConsistencyError] and is_excluded:
+        if issue_type in [ModelSyntaxError, ConsistencyError] and is_excluded and not self.override:
             print(f"Validator {code} was excluded however it is a critical validator and will still run.")
             return True
         else:
@@ -141,7 +149,7 @@ class NeatConfig(ConfigModel):
         return available_profiles[profile]
 
 
-def internal_profiles() -> dict[PredefinedProfile, NeatConfig]:
+def internal_profiles() -> dict[_AllProfiles, NeatConfig]:
     """Get internal NeatConfig profile by name."""
     return {
         "legacy-additive": NeatConfig(
@@ -180,6 +188,16 @@ def internal_profiles() -> dict[PredefinedProfile, NeatConfig]:
             modeling=ModelingConfig(mode="rebuild"),
             validation=ValidationConfig(exclude=[]),
         ),
+        "no-validation-rebuild": NeatConfig(
+            profile="no-validation-rebuild",
+            modeling=ModelingConfig(mode="rebuild"),
+            validation=ValidationConfig(exclude=["*"], override=True),
+        ),
+        "no-validation-additive": NeatConfig(
+            profile="no-validation-additive",
+            modeling=ModelingConfig(mode="additive"),
+            validation=ValidationConfig(exclude=["*"], override=True),
+        ),
     }
 
 

cognite/neat/_data_model/deployer/data_classes.py

@@ -2,6 +2,7 @@ import itertools
 import sys
 from abc import ABC, abstractmethod
 from collections import UserList, defaultdict
+from collections.abc import Hashable, Sequence
 from datetime import datetime
 from enum import Enum
 from typing import Any, Generic, Literal, TypeAlias, cast
@@ -379,15 +380,88 @@ class ResourceDeploymentPlanList(UserList[ResourceDeploymentPlan]):
         return type(self)(forced_plans)
 
 
-class ChangeResult(BaseDeployObject, Generic[T_ResourceId, T_DataModelResource]):
+class ChangeResult(BaseDeployObject, Generic[T_ResourceId, T_DataModelResource], ABC):
     endpoint: DataModelEndpoint
     change: ResourceChange[T_ResourceId, T_DataModelResource]
-    message: SuccessResponseItems[T_ResourceId] | FailedResponseItems[T_ResourceId] | FailedRequestItems[T_ResourceId]
 
+    @property
+    @abstractmethod
+    def message(self) -> str:
+        """Human-readable message about the change result."""
+        ...
 
-class ChangedFieldResult(BaseDeployObject, Generic[T_Reference]):
+    @property
+    @abstractmethod
+    def is_success(self) -> bool:
+        """Whether the change was successful."""
+        ...
+
+
+class HTTPChangeResult(ChangeResult[T_ResourceId, T_DataModelResource]):
+    http_message: (
+        SuccessResponseItems[T_ResourceId] | FailedResponseItems[T_ResourceId] | FailedRequestItems[T_ResourceId]
+    )
+
+    @property
+    def message(self) -> str:
+        if isinstance(self.http_message, SuccessResponse):
+            return "Success"
+        elif isinstance(self.http_message, FailedResponseItems):
+            error = self.http_message.error
+            return f"Failed: {error.code} | {error.message}"
+        elif isinstance(self.http_message, FailedRequestItems):
+            return f"Request Failed: {self.http_message.message}"
+        else:
+            return "Unknown result"
+
+    @property
+    def is_success(self) -> bool:
+        return isinstance(self.http_message, SuccessResponse)
+
+
+class MultiHTTPChangeResult(ChangeResult[T_ResourceId, T_DataModelResource]):
+    http_messages: list[
+        SuccessResponseItems[T_ResourceId] | FailedResponseItems[T_ResourceId] | FailedRequestItems[T_ResourceId]
+    ]
+
+    @property
+    def message(self) -> str:
+        error_messages: list[str] = []
+        for msg in self.http_messages:
+            if isinstance(msg, SuccessResponse):
+                continue
+            elif isinstance(msg, FailedResponseItems):
+                error = msg.error
+                error_messages.append(f"Failed: {error.code} | {error.message}")
+            elif isinstance(msg, FailedRequestItems):
+                error_messages.append(f"Request Failed: {msg.message}")
+        if not error_messages:
+            return "Success"
+        return "; ".join(error_messages)
+
+    @property
+    def is_success(self) -> bool:
+        return all(isinstance(msg, SuccessResponse) for msg in self.http_messages)
+
+
+class NoOpChangeResult(ChangeResult[T_ResourceId, T_DataModelResource]):
+    """A change result representing a no-op, e.g., when a change was skipped or unchanged."""
+
+    reason: str
+
+    @property
+    def message(self) -> str:
+        return self.reason
+
+    @property
+    def is_success(self) -> bool:
+        return True
+
+
+class ChangedFieldResult(BaseDeployObject, Generic[T_ResourceId, T_Reference]):
+    resource_id: T_ResourceId
     field_change: FieldChange
-    message: SuccessResponseItems[T_Reference] | FailedResponseItems[T_Reference] | FailedRequestItems[T_Reference]
+    http_message: SuccessResponseItems[T_Reference] | FailedResponseItems[T_Reference] | FailedRequestItems[T_Reference]
 
 
 class AppliedChanges(BaseDeployObject):
@@ -399,26 +473,55 @@ class AppliedChanges(BaseDeployObject):
     This is needed as these changes are done with a separate API call per change.
     """
 
-    created: list[ChangeResult] = Field(default_factory=list)
-    updated: list[ChangeResult] = Field(default_factory=list)
-    deletions: list[ChangeResult] = Field(default_factory=list)
-    unchanged: list[ResourceChange] = Field(default_factory=list)
-    skipped: list[ResourceChange] = Field(default_factory=list)
+    created: list[HTTPChangeResult] = Field(default_factory=list)
+    updated: list[HTTPChangeResult] = Field(default_factory=list)
+    deletions: list[HTTPChangeResult] = Field(default_factory=list)
+    unchanged: list[NoOpChangeResult] = Field(default_factory=list)
+    skipped: list[NoOpChangeResult] = Field(default_factory=list)
     changed_fields: list[ChangedFieldResult] = Field(default_factory=list)
 
     @property
     def is_success(self) -> bool:
         return all(
             # MyPy fails to understand that ChangeFieldResult.message has the same structure as ChangeResult.message
-            isinstance(change.message, SuccessResponse)  # type: ignore[attr-defined]
+            isinstance(change.http_message, SuccessResponse)  # type: ignore[attr-defined]
             for change in itertools.chain(self.created, self.updated, self.deletions, self.changed_fields)
         )
 
+    @property
+    def merged_updated(self) -> Sequence[ChangeResult]:
+        """Merges the changed field into the updated changes."""
+        if not self.changed_fields:
+            return self.updated
+        changed_fields_by_id: dict[Hashable, list[ChangedFieldResult]] = defaultdict(list)
+        for changed_field in self.changed_fields:
+            changed_fields_by_id[changed_field.resource_id].append(changed_field)
+        merged_changes: list[ChangeResult] = []
+        for update in self.updated:
+            if update.change.resource_id not in changed_fields_by_id:
+                merged_changes.append(update)
+                continue
+
+            field_changes = changed_fields_by_id[update.change.resource_id]
+            merged_change = update.change.model_copy(
+                update={"changes": update.change.changes + [fc.field_change for fc in field_changes]}
+            )
+
+            # MyPy wants an annotation were we want this to be generic.
+            merged_result = MultiHTTPChangeResult(  # type: ignore[var-annotated]
+                endpoint=update.endpoint,
+                change=merged_change,
+                http_messages=[update.http_message] + [fc.http_message for fc in field_changes],
+            )
+            merged_changes.append(merged_result)
+
+        return merged_changes
+
     def as_recovery_plan(self) -> list[ResourceDeploymentPlan]:
         """Generate a recovery plan based on the applied changes."""
         recovery_plan: dict[DataModelEndpoint, ResourceDeploymentPlan] = {}
         for change_result in itertools.chain(self.created, self.updated, self.deletions):
-            if not isinstance(change_result.message, SuccessResponse):
+            if not isinstance(change_result.http_message, SuccessResponse):
                 continue  # Skip failed changes.
             change = change_result.change
             if change.change_type == "create":
@@ -520,10 +623,10 @@ class DeploymentResult(BaseDeployObject):
         if self.responses:
             counts: dict[str, int] = defaultdict(int)
             for change in itertools.chain(self.responses.created, self.responses.updated, self.responses.deletions):
-                suffix = type(change.message).__name__.removesuffix("[TypeVar]").removesuffix("[~T_ResourceId]")
+                suffix = type(change.http_message).__name__.removesuffix("[TypeVar]").removesuffix("[~T_ResourceId]")
                 # For example: containers.created.successResponseItems
                 prefix = f"{change.endpoint}.{change.change.change_type}.{suffix}"
-                counts[prefix] += len(change.message.ids)
+                counts[prefix] += len(change.http_message.ids)
 
             output.update(counts)
         return output

cognite/neat/_data_model/deployer/deployer.py

@@ -35,12 +35,13 @@ from .data_classes import (
     AddedField,
     AppliedChanges,
     ChangedFieldResult,
-    ChangeResult,
     ContainerDeploymentPlan,
     DataModelEndpoint,
     DeploymentResult,
     FieldChange,
     FieldChanges,
+    HTTPChangeResult,
+    NoOpChangeResult,
     RemovedField,
     ResourceChange,
     ResourceDeploymentPlan,
@@ -289,24 +290,58 @@ class SchemaDeployer(OnSuccessResultProducer):
             AppliedChanges: The result of applying the changes.
         """
         applied_changes = AppliedChanges()
+        # If any HTTP request fails, the skip_message will be set and subsequent operations will be skipped
+        failure_message: str | None = None
         for resource in reversed(plan):
-            deletions = self._delete_items(resource)
-            applied_changes.deletions.extend(deletions)
+            if failure_message is None:
+                deletions = self._delete_items(resource)
+                applied_changes.deletions.extend(deletions)
+                if any(not deletion.is_success for deletion in deletions):
+                    failure_message = f"Skipping due to {resource.endpoint} deletions failing."
+            else:
+                applied_changes.skipped.extend(
+                    [
+                        NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=failure_message)
+                        for change in resource.to_delete
+                    ]
+                )
 
         for resource in plan:
-            if isinstance(resource, ContainerDeploymentPlan):
-                applied_changes.changed_fields.extend(self._remove_container_constraints(resource))
-                applied_changes.changed_fields.extend(self._remove_container_indexes(resource))
-
-            creations, updated = self._upsert_items(resource)
-            applied_changes.created.extend(creations)
-            applied_changes.updated.extend(updated)
+            if failure_message is None:
+                if isinstance(resource, ContainerDeploymentPlan):
+                    # Note that we continue to deploy even if removing constraints/indexes fail,
+                    # as the creation/update of views and data models will still succeed.
+                    applied_changes.changed_fields.extend(self._remove_container_constraints(resource))
+                    applied_changes.changed_fields.extend(self._remove_container_indexes(resource))
+
+                creations, updated = self._upsert_items(resource)
+                applied_changes.created.extend(creations)
+                applied_changes.updated.extend(updated)
+                if any(not change.is_success for change in creations + updated):
+                    failure_message = f"Skipping due to {resource.endpoint} upsert failing."
+            else:
+                applied_changes.skipped.extend(
+                    [
+                        NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=failure_message)
+                        for change in resource.to_upsert
+                    ]
+                )
 
-            applied_changes.unchanged.extend(resource.unchanged)
-            applied_changes.skipped.extend(resource.skip)
+            applied_changes.unchanged.extend(
+                [
+                    NoOpChangeResult(endpoint=resource.endpoint, change=change, reason="No changes detected.")
+                    for change in resource.unchanged
+                ]
+            )
+            applied_changes.skipped.extend(
+                [
+                    NoOpChangeResult(endpoint=resource.endpoint, change=change, reason=change.message or "Unknown")
+                    for change in resource.skip
+                ]
+            )
         return applied_changes
 
-    def _delete_items(self, resource: ResourceDeploymentPlan) -> list[ChangeResult]:
+    def _delete_items(self, resource: ResourceDeploymentPlan) -> list[HTTPChangeResult]:
         to_delete_by_id = {change.resource_id: change for change in resource.to_delete}
         if not to_delete_by_id:
             return []
@@ -319,7 +354,7 @@ class SchemaDeployer(OnSuccessResultProducer):
         )
         return self._process_resource_responses(responses, to_delete_by_id, resource.endpoint)
 
-    def _upsert_items(self, resource: ResourceDeploymentPlan) -> tuple[list[ChangeResult], list[ChangeResult]]:
+    def _upsert_items(self, resource: ResourceDeploymentPlan) -> tuple[list[HTTPChangeResult], list[HTTPChangeResult]]:
         to_upsert = [
             resource_change.new_value for resource_change in resource.to_upsert if resource_change.new_value is not None
         ]
@@ -375,14 +410,14 @@ class SchemaDeployer(OnSuccessResultProducer):
     @classmethod
     def _process_resource_responses(
         cls, responses: APIResponse, change_by_id: dict[T_ResourceId, ResourceChange], endpoint: DataModelEndpoint
-    ) -> list[ChangeResult]:
-        results: list[ChangeResult] = []
+    ) -> list[HTTPChangeResult]:
+        results: list[HTTPChangeResult] = []
         for response in responses:
             if isinstance(response, SuccessResponseItems | FailedResponseItems | FailedRequestItems):
                 for id in response.ids:
                     if id not in change_by_id:
                         continue
-                    results.append(ChangeResult(change=change_by_id[id], message=response, endpoint=endpoint))
+                    results.append(HTTPChangeResult(change=change_by_id[id], http_message=response, endpoint=endpoint))
             else:
                 # This should never happen as we do a ItemsRequest should always return ItemMessage responses
                 raise ValueError("Bug in Neat. Got an unexpected response type.")
@@ -398,7 +433,9 @@ class SchemaDeployer(OnSuccessResultProducer):
                 for id in response.ids:
                     if id not in change_by_id:
                         continue
-                    results.append(ChangedFieldResult(field_change=change_by_id[id], message=response))
+                    results.append(
+                        ChangedFieldResult(resource_id=id, field_change=change_by_id[id], http_message=response)
+                    )
             else:
                 # This should never happen as we do a ItemsRequest should always return ItemMessage responses
                 raise RuntimeError("Bug in Neat. Got an unexpected response type.")

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "1.0.4"
+__version__ = "1.0.6"
 __engine__ = "^2.0.4"

{cognite_neat-1.0.4.dist-info → cognite_neat-1.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-neat
-Version: 1.0.4
+Version: 1.0.6
 Summary: Knowledge graph transformation
 Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/

{cognite_neat-1.0.4.dist-info → cognite_neat-1.0.6.dist-info}/RECORD

@@ -1,8 +1,8 @@
 cognite/neat/__init__.py,sha256=u5EhnGuNS2GKydU6lVFCNrBpHBBKUnCDAE63Cqk__eo,244
-cognite/neat/_config.py,sha256=NXObeA-860LV40KlY4orsqjMGACa0jKRz2UE5L9kH6U,8401
+cognite/neat/_config.py,sha256=MzQiZer0Wyk6VEtfDtl_tTF9KYAjYu2Q8jE7RugMJuM,9201
 cognite/neat/_exceptions.py,sha256=ox-5hXpee4UJlPE7HpuEHV2C96aLbLKo-BhPDoOAzhA,1650
 cognite/neat/_issues.py,sha256=wH1mnkrpBsHUkQMGUHFLUIQWQlfJ_qMfdF7q0d9wNhY,1871
-cognite/neat/_version.py,sha256=PG0hTrgdtokkHUdUcZQhG_UiE_vFIC9YHCENYSPb91w,44
+cognite/neat/_version.py,sha256=mbpsAx7QFBCcnMi7Q3BurnEcb8MiqmNPURjepyY33lk,44
 cognite/neat/legacy.py,sha256=eI2ecxOV8ilGHyLZlN54ve_abtoK34oXognkFv3yvF0,219
 cognite/neat/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/_client/__init__.py,sha256=75Bh7eGhaN4sOt3ZcRzHl7pXaheu1z27kmTHeaI05vo,114
@@ -26,8 +26,8 @@ cognite/neat/_data_model/deployer/_differ_container.py,sha256=mcy7PhUOfnvAxnZWNo
 cognite/neat/_data_model/deployer/_differ_data_model.py,sha256=iA7Xp-7NRvzZJXLLpJaLebkKKpv_VCBKPX6f-RU9wBk,1864
 cognite/neat/_data_model/deployer/_differ_space.py,sha256=J_AaqiseLpwQsOkKc7gmho4U2oSWAGVeEdQNepZiWw0,343
 cognite/neat/_data_model/deployer/_differ_view.py,sha256=g1xHwsoxFUaTOTtQa19nntKF3rxFzc2FxpKKFAUN_NE,11412
-cognite/neat/_data_model/deployer/data_classes.py,sha256=NTvLUYb_HbpzByKe_ZPvb-arOOs_YVfTjk3LzGpEHs0,23082
-cognite/neat/_data_model/deployer/deployer.py,sha256=9ZFf9QAZEnyqmX3b4zSkdf5t3A1wnU6cYGDNut_KKuQ,18390
+cognite/neat/_data_model/deployer/data_classes.py,sha256=Rz3-6zX6_BJNdoiPmJCJYmKXPi5AwWPoAEvqQ-8qTI4,26781
+cognite/neat/_data_model/deployer/deployer.py,sha256=8W6Zy8MVLaUWRqNlNrP-A1HUJlOMdzpDsyPqClxfyrU,20288
 cognite/neat/_data_model/exporters/__init__.py,sha256=AskjmB_0Vqib4kN84bWt8-M8nO42QypFf-l-E8oA5W8,482
 cognite/neat/_data_model/exporters/_api_exporter.py,sha256=G9cqezy_SH8VdhW4o862qBHh_QcbzfP6O1Yyjvdpeog,1579
 cognite/neat/_data_model/exporters/_base.py,sha256=rG_qAU5i5Hh5hUMep2UmDFFZID4x3LEenL6Z5C6N8GQ,646
@@ -312,7 +312,7 @@ cognite/neat/_v0/session/_to.py,sha256=AnsRSDDdfFyYwSgi0Z-904X7WdLtPfLlR0x1xsu_j
 cognite/neat/_v0/session/_wizard.py,sha256=baPJgXAAF3d1bn4nbIzon1gWfJOeS5T43UXRDJEnD3c,1490
 cognite/neat/_v0/session/exceptions.py,sha256=jv52D-SjxGfgqaHR8vnpzo0SOJETIuwbyffSWAxSDJw,3495
 cognite/neat/_v0/session/_state/README.md,sha256=o6N7EL98lgyWffw8IoEUf2KG5uSKveD5__TW45YzVjA,902
-cognite_neat-1.0.4.dist-info/METADATA,sha256=5n7OsUlUVMaXmo5m9O4VQsSsWleV2xcjomJdMjyMBTE,6030
-cognite_neat-1.0.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-cognite_neat-1.0.4.dist-info/licenses/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-1.0.4.dist-info/RECORD,,
+cognite_neat-1.0.6.dist-info/METADATA,sha256=UTHB6fm94ONhx2QoDJWgmvdXE0oIyZmHvrAzycCNBYU,6030
+cognite_neat-1.0.6.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+cognite_neat-1.0.6.dist-info/licenses/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-1.0.6.dist-info/RECORD,,