validio-sdk 7.4.2__tar.gz → 7.5.0__tar.gz

{validio_sdk-7.4.2 → validio_sdk-7.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validio-sdk
-Version: 7.4.2
+Version: 7.5.0
 Summary: SDK to interact with the Validio platform
 Home-page: https://validio.io/
 License: Apache-2.0

{validio_sdk-7.4.2 → validio_sdk-7.5.0}/pyproject.toml

@@ -3,7 +3,7 @@ name = "validio-sdk"
 # This version does not represent the released version or any tag. For each
 # release we automatically bump this before building and publishing so this
 # should be kept at 0.0.1dev1
-version = "7.4.2"
+version = "7.5.0"
 description = "SDK to interact with the Validio platform"
 authors = ["Validio <support@validio.io>"]
 license = "Apache-2.0"

{validio_sdk-7.4.2 → validio_sdk-7.5.0}/validio_sdk/_api/api.py

@@ -975,6 +975,19 @@ async def get_credentials(
           optBaseUrl: baseUrl
         }
       }
+      ... on AwsBedrockCredential {
+        config {
+          model
+          region
+          optBaseUrl: baseUrl
+          auth {
+            __typename
+            ... on AwsBedrockCredentialAccessKey {
+              accessKeyId
+            }
+          }
+        }
+      }
       ... on AwsCredential {
         config {
           accessKey
@@ -1312,6 +1325,29 @@ async def get_channels(
           }
         }
       }
+      ... on ServiceNowChannel {
+        config {
+          applicationLinkUrl
+          instanceUrl
+          statusMapping {
+            triage
+            investigating
+            resolved
+            serviceNowActive
+            resolvedCloseCode
+            notAnAnomalyCloseCode
+          }
+          serviceNowAuth: auth {
+            __typename
+            ... on ServiceNowBasicAuth {
+              username
+            }
+            ... on ServiceNowOAuth {
+              clientId
+            }
+          }
+        }
+      }
     }
     """
 
@@ -1360,6 +1396,7 @@ async def get_channels(
             [
                 ("msTeamsInteractiveMessageEnabled", "interactiveMessageEnabled"),
                 ("emailInteractiveMessageEnabled", "interactiveMessageEnabled"),
+                ("serviceNowAuth", "auth"),
             ],
         )
 
@@ -1952,6 +1989,31 @@ async def get_validators(
         }}
         {reference_source_config}
       }}
+      ... on ReferentialIntegrityValidator {{
+        config {{
+          keyFields {{
+            sourceField
+            referenceField
+          }}
+          valueFields {{
+            sourceField
+            referenceField
+          }}
+          referenceSource {{
+            resourceName
+          }}
+          referenceWindow {{
+            resourceName
+          }}
+          referenceFilter {{
+            resourceName
+          }}
+          initializeWithBackfill
+          threshold {{
+            ...ThresholdDetails
+          }}
+        }}
+      }}
       ... on SqlValidator {{
         config {{
           query

{validio_sdk-7.4.2 → validio_sdk-7.5.0}/validio_sdk/code/plan.py

@@ -24,7 +24,6 @@ from validio_sdk.resource._resource import (
 )
 from validio_sdk.resource._server_resources import load_resources
 from validio_sdk.resource._util import SourceSchemaReinference
-from validio_sdk.resource.channels import WebhookChannel
 from validio_sdk.resource.tags import Tag
 
 
@@ -228,7 +227,7 @@ def _create_resource_diff_object(
     show_secrets: bool,
     rewrites: dict[str, Any] | None = None,
     secret_fields_changed: dict[str, Any] | None = None,
-    is_manifest: bool = False,
+    manifest: Resource | Diffable | dict | None = None,
 ) -> dict[str, object]:
     if rewrites is None:
         rewrites = {}
@@ -236,13 +235,24 @@ def _create_resource_diff_object(
     if secret_fields_changed is None:
         secret_fields_changed = {}
 
+    is_manifest = manifest is None
+
     data = dict(r) if not isinstance(r, dict) else r
+    manifest_data = (
+        dict(manifest)
+        if not isinstance(manifest, dict) and manifest is not None
+        else manifest
+    )
 
     diff_object = {}
     for k, v in data.items():
         if k.startswith("_"):
             continue
 
+        nested_manifest_data = (
+            manifest_data.get(k) if isinstance(manifest_data, dict) else None
+        )
+
         if k in rewrites:
             diff_object[k] = rewrites[k]
         elif isinstance(v, Resource | Diffable | dict):
@@ -251,7 +261,7 @@ def _create_resource_diff_object(
                 v,
                 show_secrets,
                 secret_fields_changed=ch if not isinstance(ch, bool) else {},
-                is_manifest=is_manifest,
+                manifest=nested_manifest_data,
             )
         elif hasattr(v, "__dict__") and not isinstance(v, Enum):
             ch = secret_fields_changed.get(k)
@@ -259,7 +269,7 @@ def _create_resource_diff_object(
                 v.__dict__,
                 show_secrets,
                 secret_fields_changed=ch if not isinstance(ch, bool) else {},
-                is_manifest=is_manifest,
+                manifest=nested_manifest_data,
             )
         elif isinstance(v, list):
             if len(v) == 0 or not isinstance(v[0], Diffable):
@@ -272,7 +282,7 @@ def _create_resource_diff_object(
                     _create_resource_diff_object(
                         item.__dict__,
                         show_secrets,
-                        is_manifest=is_manifest,
+                        manifest=None,
                     )
                 )
 
@@ -280,41 +290,28 @@ def _create_resource_diff_object(
         else:
             diff_object[k] = v
 
-    # Mask any sensitive info if requested.
-    if not show_secrets and hasattr(r, "_secret_fields"):
-        secret_fields = r._secret_fields()
-
-        if secret_fields:
-            for field in secret_fields:
-                # We want to avoid overriding webhook_url if it is None already
-                # which may give a misleading impression that it is set to a value.
-                #
-                # For any channel that is not the WebhookChannel, we will always
-                # flag it as changed if any secrets have changed, which is consistent
-                # with existing behavior.
-                #
-                # This workaround until webhook URL is deprecated fully from some
-                # channels (VR-4047).
-                flag_webhook_url_as_changed = False
-                if not isinstance(r, WebhookChannel) and field == "webhook_url":
-                    if hasattr(r, "webhook_url") and not r.webhook_url:
-                        continue
-                    if any(
-                        secret_fields_changed[secret_field]
-                        for secret_field in secret_fields_changed
-                    ):
-                        flag_webhook_url_as_changed = True
-
-                if (
-                    # Special case handling for webhook URL
-                    flag_webhook_url_as_changed
-                    or
-                    # If the field does not exist in the secrets changed dict then
-                    # don't flag it as changed
-                    secret_fields_changed.get(field)
-                ) and not is_manifest:
-                    diff_object[field] = "REDACTED-PREVIOUS"
-                else:
-                    diff_object[field] = "REDACTED"
+    # Nothing to mask, no secrets
+    if not hasattr(r, "_secret_fields"):
+        return diff_object
+
+    for secret_field in r._secret_fields():
+        # The field can either be in the map set to False, or absent, to be
+        # treated as unchanged.
+        field_changed = (
+            secret_field in secret_fields_changed
+            and secret_fields_changed[secret_field]
+        )
+
+        if is_manifest:
+            if not show_secrets:
+                diff_object[secret_field] = "REDACTED"
+        elif field_changed:
+            diff_object[secret_field] = (
+                "UNKNOWN" if show_secrets else "REDACTED-PREVIOUS"
+            )
+        elif not show_secrets:
+            diff_object[secret_field] = "REDACTED"
+        elif manifest_data and secret_field in manifest_data:
+            diff_object[secret_field] = manifest_data[secret_field]
 
     return diff_object

{validio_sdk-7.4.2 → validio_sdk-7.5.0}/validio_sdk/resource/_diff.py

@@ -594,52 +594,87 @@ def diff(
 
     # No changes yet. Next, descend into the nested fields. If we find any
     # changes, then mark _this_ resource as changed.
-    server_nested_objects = server_object._nested_objects()
-    for field, manifest in manifest_object._nested_objects().items():
-        server = server_nested_objects[field]
-
-        # No possible change if both are unset.
-        if manifest is None and server is None:
-            continue
-
-        # If exactly one of them is None, then we definitely have a change.
-        if manifest is None or server is None:
-            return ResourceUpdate(manifest_resource, server_resource)
+    # For immutable nested objects, changes require the resource to be recreated.
+    nested_object_sources = [
+        (
+            manifest_object._mutable_nested_objects(),
+            server_object._mutable_nested_objects(),
+            False,
+        ),
+        (
+            manifest_object._immutable_nested_objects(),
+            server_object._immutable_nested_objects(),
+            True,
+        ),
+    ]
+    for manifest_nested, server_nested, is_immutable in nested_object_sources:
+        for field, manifest in manifest_nested.items():
+            server = server_nested[field]
+            replacement_field = field if is_immutable else None
 
-        if isinstance(server, list) != isinstance(manifest, list):
-            return ResourceUpdate(manifest_resource, server_resource)
+            # No possible change if both are unset.
+            if manifest is None and server is None:
+                continue
 
-        if isinstance(server, list) and isinstance(manifest, list):
-            if len(server) != len(manifest):
-                return ResourceUpdate(manifest_resource, server_resource)
+            # If exactly one of them is None, then we definitely have a change.
+            if manifest is None or server is None:
+                return ResourceUpdate(
+                    manifest_resource,
+                    server_resource,
+                    replacement_field=replacement_field,
+                )
 
-            for i in range(len(server)):
-                # Descend into both objects to diff.
-                collected_diff = diff(
-                    manifest[i],
-                    server[i],
-                    curr_depth + 1,
+            if isinstance(server, list) != isinstance(manifest, list):
+                return ResourceUpdate(
                     manifest_resource,
                     server_resource,
+                    replacement_field=replacement_field,
                 )
 
-                if collected_diff:
-                    # The returned diff is that of the full resource (not
-                    # just the nested object that had a change). So we can
-                    # exit with it.
-                    return collected_diff
+            if isinstance(server, list) and isinstance(manifest, list):
+                if len(server) != len(manifest):
+                    return ResourceUpdate(
+                        manifest_resource,
+                        server_resource,
+                        replacement_field=replacement_field,
+                    )
+
+                for i in range(len(server)):
+                    # Descend into both objects to diff.
+                    collected_diff = diff(
+                        manifest[i],
+                        server[i],
+                        curr_depth + 1,
+                        manifest_resource,
+                        server_resource,
+                    )
+
+                    if collected_diff:
+                        # For immutable, flag for replacement. For mutable,
+                        # return the collected diff directly.
+                        if is_immutable:
+                            return ResourceUpdate(
+                                manifest_resource,
+                                server_resource,
+                                replacement_field=replacement_field,
+                            )
+                        return collected_diff
 
-            continue
+                continue
 
-        # Descend into both objects to diff.
-        collected_diff = diff(
-            manifest, server, curr_depth + 1, manifest_resource, server_resource
-        )
-        if collected_diff:
-            # The returned diff is that of the full resource (not
-            # just the nested object that had a change). So we can
-            # exit with it.
-            return collected_diff
+            collected_diff = diff(
+                manifest, server, curr_depth + 1, manifest_resource, server_resource
+            )
+            if collected_diff:
+                # For immutable, flag for replacement. For mutable,
+                # return the collected diff directly.
+                if is_immutable:
+                    return ResourceUpdate(
+                        manifest_resource,
+                        server_resource,
+                        replacement_field=replacement_field,
+                    )
+                return collected_diff
 
     # No update to make
     return None

{validio_sdk-7.4.2 → validio_sdk-7.5.0}/validio_sdk/resource/_diffable.py

@@ -41,7 +41,7 @@ class Diffable(ABC):
     within a resource be sure to account for how that field should be diffed.
     Every field reachable from a resource, should be returned by exactly one
     of the listed APIs here, with the exception of `_replace_on_type_change_fields`
-    which can overlap with `_nested_objects`.
+    which can overlap with `_mutable_nested_objects`.
     """
 
     @abstractmethod
@@ -57,16 +57,27 @@ class Diffable(ABC):
         return set({})
 
     @abstractmethod
-    def _nested_objects(
+    def _mutable_nested_objects(
         self,
     ) -> dict[str, "Diffable | list[Diffable] | None"]:
-        """Returns any nested objects contained within this object.
+        """Returns any mutable nested objects contained within this object.
 
-        Nested objects will be diff-ed recursively.
+        Mutable nested objects will be diff-ed recursively.
         ...
         :returns dict[field, Optional[object]].
         """
 
+    def _immutable_nested_objects(
+        self,
+    ) -> dict[str, "Diffable | list[Diffable] | None"]:
+        """Returns any immutable nested objects contained within this object.
+
+        Immutable nested objects cannot be updated after creation.
+        ...
+        :returns dict[field, Optional[object]].
+        """
+        return {}
+
     def __iter__(self) -> Iterator[Any]:
         yield from self.__dict__.items()
 
@@ -83,7 +94,7 @@ class Diffable(ABC):
         if self._secret_fields():
             return True
 
-        for nested in self._nested_objects().values():
+        for nested in self._mutable_nested_objects().values():
             if isinstance(nested, list):
                 for n in nested:
                     if n._has_secret_fields(curr_depth=curr_depth + 1):
@@ -98,7 +109,8 @@ class Diffable(ABC):
         return {
             *self._immutable_fields(),
             *self._mutable_fields(),
-            *self._nested_objects().keys(),
+            *self._mutable_nested_objects().keys(),
+            *self._immutable_nested_objects().keys(),
             *self._ignored_fields(),
         }
 
@@ -239,7 +251,7 @@ class ApiSecretChangeNestedResource(Diffable):
         data[NODE_TYPE_FIELD_NAME] = self.__class__.__name__
         return data
 
-    def _nested_objects(
+    def _mutable_nested_objects(
         self,
     ) -> dict[str, "Diffable | list[Diffable] | None"]:
         return {}

{validio_sdk-7.4.2 → validio_sdk-7.5.0}/validio_sdk/resource/_resource.py

@@ -323,7 +323,9 @@ class Resource(Diffable):
 
         return {"_children": children}
 
-    def _nested_objects(self) -> dict[str, Optional["Diffable | list[Diffable]"]]:
+    def _mutable_nested_objects(
+        self,
+    ) -> dict[str, Optional["Diffable | list[Diffable]"]]:
         return {}
 
     def _nested_mutable_parents(self) -> dict[str, str | None]:
@@ -568,7 +570,7 @@ class Resource(Diffable):
 
     def _nested_secret_objects(self) -> dict[str, ApiSecretChangeNestedResource]:
         objects = {}
-        for f in self._nested_objects():
+        for f in self._mutable_nested_objects():
             obj = getattr(self, f)
             if isinstance(obj, ApiSecretChangeNestedResource):
                 objects[f] = obj

{validio_sdk-7.4.2 → validio_sdk-7.5.0}/validio_sdk/resource/_serde.py

@@ -206,7 +206,8 @@ def _api_input(
 
     skip_fields = {
         *set(overrides.keys()),
-        *set(resource._nested_objects().keys()),
+        *set(resource._mutable_nested_objects().keys()),
+        *set(resource._immutable_nested_objects().keys()),
         *skip_fields,
         "name",
         "resourceName",

{validio_sdk-7.4.2 → validio_sdk-7.5.0}/validio_sdk/resource/_server_resources.py

@@ -29,11 +29,17 @@ from validio_sdk.resource._diff_util import (
     must_find_window,
 )
 from validio_sdk.resource._resource import Resource, ResourceGraph
-from validio_sdk.resource._util import _rename_dict_key, _sanitized_error_str
+from validio_sdk.resource._util import (
+    _rename_dict_key,
+    _sanitized_error_str,
+    _to_snake_recursive,
+)
 from validio_sdk.resource.channels import (
     Channel,
     EmailChannel,
     MsTeamsChannel,
+    ServiceNowChannel,
+    ServiceNowStatusMapping,
     SlackChannel,
     WebhookChannel,
 )
@@ -43,6 +49,10 @@ from validio_sdk.resource.credentials import (
     AnthropicCredentialAuth,
     AtlanCredential,
     AwsAthenaCredential,
+    AwsBedrockCredential,
+    AwsBedrockCredentialAccessKey,
+    AwsBedrockCredentialApiKey,
+    AwsBedrockCredentialAuth,
     AwsCredential,
     AwsRedshiftCredential,
     AzureSynapseEntraIdCredential,
@@ -90,7 +100,13 @@ from validio_sdk.resource.segmentations import Segmentation
 from validio_sdk.resource.sources import AzureSynapseSource, Source, SqlSource
 from validio_sdk.resource.tags import Tag
 from validio_sdk.resource.thresholds import Threshold
-from validio_sdk.resource.validators import Reference, SqlValidator, Validator
+from validio_sdk.resource.validators import (
+    FieldMapping,
+    Reference,
+    ReferentialIntegrityValidator,
+    SqlValidator,
+    Validator,
+)
 
 # Some credentials depend on other credentials, i.e. wrapping credentials. This
 # list contains all of those and can be used when sorting to ensure they always
@@ -227,6 +243,34 @@ async def load_credentials(
                     display_name=display_name,
                     __internal__=g,
                 )
+            case "AwsBedrockCredential":
+                match c["config"]["auth"]["__typename"]:
+                    case "AwsBedrockCredentialAccessKey":
+                        bedrock_auth: AwsBedrockCredentialAuth = (
+                            AwsBedrockCredentialAccessKey(
+                                access_key_id=c["config"]["auth"]["accessKeyId"],
+                                secret_access_key="UNSET",
+                            )
+                        )
+                    case "AwsBedrockCredentialApiKey":
+                        bedrock_auth = AwsBedrockCredentialApiKey(
+                            api_key="UNSET",
+                        )
+                    case _:
+                        raise ValidioBugError(
+                            f"Unknown AWS Bedrock auth type on {name}: "
+                            f"'{c['config']['auth']['__typename']}'"
+                        )
+
+                credential = AwsBedrockCredential(
+                    name=name,
+                    model=c["config"]["model"],
+                    region=c["config"]["region"],
+                    base_url=c["config"]["baseUrl"],
+                    auth=bedrock_auth,
+                    display_name=display_name,
+                    __internal__=g,
+                )
             case "AtlanCredential":
                 credential = AtlanCredential(
                     name=name,
@@ -646,6 +690,28 @@ async def load_channels(
                         "__internal__": g,
                     }
                 )
+            case "ServiceNowChannel":
+                auth = cfg["auth"]
+                auth_cls = eval(
+                    f"validio_sdk.resource.channels.{auth.pop('__typename')}"
+                )
+
+                channel = ServiceNowChannel(
+                    **{
+                        **_to_snake_recursive(cfg),
+                        "name": name,
+                        "application_link_url": application_link_url,
+                        "display_name": display_name,
+                        "webhook_secret": "UNSET",
+                        "auth": auth_cls(
+                            **{
+                                f: auth.get(to_camel(f), "UNSET")
+                                for f in inspect.signature(auth_cls).parameters
+                            }
+                        ),
+                        "__internal__": g,
+                    }
+                )
             case _:
                 raise ValidioError(f"unsupported channel '{name}' of type '{type(ch)}'")
 
@@ -838,6 +904,26 @@ def convert_reference(ctx: DiffContext, r: dict[str, Any]) -> Reference:
     )
 
 
+def extract_reference_resources(
+    ctx: DiffContext, config: dict[str, Any]
+) -> dict[str, Any]:
+    out = {
+        "reference_source": must_find_source(
+            ctx, config["referenceSource"]["resourceName"]
+        ),
+        "reference_window": must_find_window(
+            ctx, config["referenceWindow"]["resourceName"]
+        ),
+    }
+
+    if config.get("referenceFilter"):
+        out["reference_filter"] = must_find_filter(
+            ctx, config["referenceFilter"]["resourceName"]
+        )
+
+    return out
+
+
 async def load_validators(
     namespace: str,
     ctx: DiffContext,
@@ -880,13 +966,19 @@ async def load_validators(
             else {}
         )
 
+        maybe_reference_resources = (
+            extract_reference_resources(ctx, config)
+            if "referenceSource" in config
+            else {}
+        )
+
         # Volume validator still have a deprecated field that we use. It's
         # called sourceField in the API still but `optional_source_field` on
         # the resource class so we rename it here.
         if v["__typename"] == "VolumeValidator":
             _rename_dict_key(config, "sourceField", "optionalSourceField")
 
-        config = {to_snake(k): v for k, v in config.items() if k != "threshold"}
+        config = _to_snake_recursive(config)
 
         cls = eval(f"validio_sdk.resource.validators.{v['__typename']}")
 
@@ -895,6 +987,7 @@ async def load_validators(
                 **config,
                 **maybe_reference,
                 **maybe_filter,
+                **maybe_reference_resources,
                 "threshold": threshold,
                 "name": name,
                 "window": window,
@@ -1028,6 +1121,14 @@ async def apply_deletes(
         list(deletes.notification_rules.values()), session, progress_bar
     )
 
+    # Delete any validators with cross source references before sources since
+    # they might prevent source deletes.
+    await _delete_resources(
+        [v for v in deletes.validators.values() if v.has_cross_source_references()],
+        session,
+        progress_bar,
+    )
+
     # For pipeline resources, start with sources (This cascades deletes,
     # so we don't have to individually delete child resources).
     await _delete_resources(list(deletes.sources.values()), session, progress_bar)

{validio_sdk-7.4.2 → validio_sdk-7.5.0}/validio_sdk/resource/_util.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from typing import Any
 
+from camel_converter import to_snake
 from gql.transport.exceptions import TransportServerError
 
 
@@ -46,3 +47,20 @@ def _rename_dict_key(d: dict[str, Any], from_key: str, to_key: str) -> None:
 
     d[to_key] = d[from_key]
     del d[from_key]
+
+
+def _to_snake_recursive(obj: Any, max_depth: int = 10, _depth: int = 0) -> Any:
+    """Recursively convert dictionary keys to snake_case."""
+    if _depth > max_depth:
+        return obj
+
+    if isinstance(obj, dict):
+        return {
+            to_snake(k): _to_snake_recursive(v, max_depth, _depth + 1)
+            for k, v in obj.items()
+        }
+
+    if isinstance(obj, list):
+        return [_to_snake_recursive(item, max_depth, _depth + 1) for item in obj]
+
+    return obj