validio-sdk 5.2.0__tar.gz → 5.3.1__tar.gz

{validio_sdk-5.2.0 → validio_sdk-5.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validio-sdk
-Version: 5.2.0
+Version: 5.3.1
 Summary: SDK to interact with the Validio platform
 Home-page: https://validio.io/
 License: Apache-2.0

{validio_sdk-5.2.0 → validio_sdk-5.3.1}/pyproject.toml

@@ -3,7 +3,7 @@ name = "validio-sdk"
 # This version does not represent the released version or any tag. For each
 # release we automatically bump this before building and publishing so this
 # should be kept at 0.0.1dev1
-version = "5.2.0"
+version = "5.3.1"
 description = "SDK to interact with the Validio platform"
 authors = ["Validio <support@validio.io>"]
 license = "Apache-2.0"

{validio_sdk-5.2.0 → validio_sdk-5.3.1}/validio_sdk/_api/api.py

@@ -453,6 +453,22 @@ async def get_sources(
           schedule
         }
       }
+      ... on MsSqlServerSource {
+        config {
+          database
+          schema
+          table
+          schedule
+        }
+      }
+      ... on OracleSource {
+        config {
+          database
+          schema
+          table
+          schedule
+        }
+      }
       ... on PostgreSqlSource {
         config {
           database
@@ -992,6 +1008,37 @@ async def get_credentials(
         }
         enableCatalog
       }
+      ... on MsSqlServerCredential {
+        config {
+          host
+          port
+          database
+          auth {
+            __typename
+            ... on MsSqlServerCredentialEntraId {
+              clientId
+            }
+            ... on MsSqlServerCredentialUserPassword {
+              user
+            }
+          }
+        }
+        enableCatalog
+      }
+      ... on OracleCredential {
+        config {
+          host
+          port
+          databaseRequired: database
+          auth {
+            __typename
+            ... on OracleCredentialUserPassword {
+              user
+            }
+          }
+        }
+        enableCatalog
+      }
       ... on PostgreSqlCredential {
         config {
           host
@@ -1117,7 +1164,13 @@ async def get_credentials(
         result = [result]
 
     for i, credential in enumerate(result):
-        _replace_aliases(credential.get("config", {}), [("powerBiAuth", "auth")])
+        _replace_aliases(
+            credential.get("config", {}),
+            [
+                ("powerBiAuth", "auth"),
+                ("databaseRequired", "database"),
+            ],
+        )
         result[i] = credential
 
     return result[0] if return_first else result
@@ -1141,7 +1194,6 @@ async def get_channels(
       }
       ... on SlackChannel {
         config {
-          slackWebhookUrl: webhookUrl
           applicationLinkUrl
           slackChannelId
           interactiveMessageEnabled
@@ -1154,7 +1206,6 @@ async def get_channels(
       }
       ... on MsTeamsChannel {
         config {
-          msTeamsWebhookUrl: webhookUrl
           applicationLinkUrl
           msTeamsChannelId
           msTeamsInteractiveMessageEnabled: interactiveMessageEnabled
@@ -1223,8 +1274,6 @@ async def get_channels(
         _replace_aliases(
             ftr["config"],
             [
-                ("slackWebhookUrl", "webhookUrl"),
-                ("msTeamsWebhookUrl", "webhookUrl"),
                 ("msTeamsInteractiveMessageEnabled", "interactiveMessageEnabled"),
                 ("emailInteractiveMessageEnabled", "interactiveMessageEnabled"),
             ],
@@ -2523,6 +2572,7 @@ async def apply_source_action(
             result = {f"source{action.value}": {"errors": e.errors}}
         else:
             result = {f"source{action.value}": {"errors": [{"message": str(e)}]}}
+
     return result[f"source{action.value}"]
 
 

{validio_sdk-5.2.0 → validio_sdk-5.3.1}/validio_sdk/code/_import.py

@@ -277,7 +277,7 @@ async def _import(
             ("tags", resource_tags(r), None),
         ]
 
-        if v._filter_resource_mode() and v.filter_name:
+        if v.filter_name:
             filter_ = import_ctx.get_variable(Filter, v.filter_name)
             inits.append(("filter", filter_, None))
 

{validio_sdk-5.2.0 → validio_sdk-5.3.1}/validio_sdk/code/plan.py

@@ -24,7 +24,7 @@ from validio_sdk.resource._resource import (
 )
 from validio_sdk.resource._server_resources import load_resources
 from validio_sdk.resource._util import SourceSchemaReinference
-from validio_sdk.resource.channels import Channel, WebhookChannel
+from validio_sdk.resource.channels import WebhookChannel
 from validio_sdk.resource.tags import Tag
 
 
@@ -286,10 +286,6 @@ def _create_resource_diff_object(
     if not show_secrets and hasattr(r, "_secret_fields"):
         secret_fields = r._secret_fields()
 
-        # Workaround for Channel having a special secret field
-        if isinstance(r, Channel):
-            secret_fields.update(r._deprecated_secret_fields())
-
         if secret_fields:
             for field in secret_fields:
                 # We want to avoid overriding webhook_url if it is None already

{validio_sdk-5.2.0 → validio_sdk-5.3.1}/validio_sdk/resource/_diff.py

@@ -1,7 +1,5 @@
 import asyncio
-import copy
 import dataclasses
-import re
 import typing
 from dataclasses import dataclass
 from typing import (
@@ -9,7 +7,6 @@ from typing import (
     Any,
     Optional,
     TypeVar,
-    cast,
 )
 
 from camel_converter import to_snake
@@ -18,7 +15,6 @@ from gql.transport.exceptions import TransportQueryError
 from validio_sdk._api.api import secrets_changed_by_field, split_to_n_chunks
 from validio_sdk.client import Session
 from validio_sdk.exception import ValidioBugError, ValidioError
-from validio_sdk.resource._diff_util import must_find_source
 from validio_sdk.resource._diffable import (
     ApiSecretChangeNestedResource,
     Diffable,
@@ -28,10 +24,8 @@ from validio_sdk.resource._errors import (
     max_resource_depth_exceeded,
     updated_resource_type_mismatch_exception,
 )
-from validio_sdk.resource._field_selector import SelectorWithFieldName
 from validio_sdk.resource._resource import CREATE_ONLY_RESOURCES, DiffContext, Resource
 from validio_sdk.resource._util import SourceSchemaReinference, _sanitize_error
-from validio_sdk.resource.channels import Channel
 from validio_sdk.resource.replacement import (
     CascadeReplacementReason,
     ImmutableFieldReplacementReason,
@@ -41,7 +35,6 @@ from validio_sdk.resource.replacement import (
 from validio_sdk.resource.segmentations import Segmentation
 from validio_sdk.resource.sources import Source
 from validio_sdk.resource.tags import Tag
-from validio_sdk.resource.validators import Reference, Validator
 from validio_sdk.resource.windows import TumblingWindow
 
 if TYPE_CHECKING:
@@ -179,51 +172,29 @@ async def diff_resource_graph(
     server_ctx: DiffContext,
     import_mode: bool = False,
 ) -> GraphDiff:
-    partial_diff = _diff_resource_graph(
+    graph_diff = _diff_resource_graph(
         namespace=namespace,
         manifest_ctx=manifest_ctx,
         server_ctx=server_ctx,
-        for_validators=False,
         import_mode=import_mode,
     )
 
-    # Enrich resource_graph. We do this before expanding field selectors
-    # because the latter depends on the schema for all sources being present.
     await enrich_resource_graph(
         manifest_ctx=manifest_ctx,
         server_ctx=server_ctx,
-        graph=partial_diff,
+        graph=graph_diff,
         session=session,
         schema_reinference=schema_reinference,
         show_secrets=show_secrets,
     )
 
-    # Expand field selector for validators.
-    expand_validator_field_selectors(manifest_ctx)
-
-    # Now after expanding field selectors, we can diff validators.
-    validators_diff = _diff_resource_graph(
-        namespace=namespace,
-        manifest_ctx=manifest_ctx,
-        server_ctx=server_ctx,
-        for_validators=True,
-        import_mode=import_mode,
-    )
-
-    final_diff = partial_diff
-    final_diff.to_create.validators = validators_diff.to_create.validators
-    final_diff.to_update.validators = validators_diff.to_update.validators
-    final_diff.to_delete.validators = validators_diff.to_delete.validators
-    for k, v in validators_diff.replacement_ctx.validators.items():
-        final_diff.replacement_ctx.validators[k] = v
-
     _compute_replacements(
         manifest_ctx=manifest_ctx,
         server_ctx=server_ctx,
-        graph_diff=final_diff,
+        graph_diff=graph_diff,
     )
 
-    return final_diff
+    return graph_diff
 
 
 def _compute_replacements(
@@ -281,8 +252,7 @@ def _visit_resource_to_replace(
     to_delete: dict[str, Resource] = getattr(graph_diff.to_delete, resource_type)
 
     # Remove the resource from the update list.
-    if resource_name in to_update:
-        del to_update[resource_name]
+    to_update.pop(resource_name, None)
 
     # Flag the resource to be created.
     manifest_resources: dict[str, Resource] = getattr(manifest_ctx, resource_type)
@@ -406,85 +376,10 @@ def _visit_resource_to_cascade_update(
     )
 
 
-def expand_validator_field_selectors(manifest_ctx: DiffContext) -> None:
-    for name in list(manifest_ctx.validators.keys()):
-        template = manifest_ctx.validators[name]
-
-        source_field_selector: SelectorWithFieldName | None = getattr(
-            template, "_field_selector", None
-        )
-        reference_field_selector: dict[str, str] | None = getattr(
-            template, "_reference_field_selector", None
-        )
-        filter_field_selector: SelectorWithFieldName | None = (
-            getattr(template.filter, "_field_selector", None)
-            if template.filter
-            else None
-        )
-
-        if source_field_selector:
-            field_selector = source_field_selector
-        elif filter_field_selector:
-            field_selector = filter_field_selector
-        else:
-            continue
-
-        source = must_find_source(manifest_ctx, template.source_name)
-        if source.jtd_schema is None:
-            # If we have no schema then it means we're doing a diff where the
-            # source's credential has not yet been created - which means we can't
-            # do schema inference. As a result, we can't yet expand the selector.
-            continue
-
-        if source_field_selector:
-            delattr(template, "_field_selector")
-        if reference_field_selector:
-            delattr(template, "_reference_field_selector")
-        if filter_field_selector:
-            delattr(template.filter, "_field_selector")
-
-        # Remove the placeholder validator.
-        del manifest_ctx.validators[name]
-
-        fields = field_selector.field_selector._get_matching_fields(
-            cast(Any, source.jtd_schema)
-        )
-
-        # We don't want to deep copy the entire resource graph
-        resource_graph = template._resource_graph
-        template._resource_graph = None  # type: ignore
-        for field in fields:
-            validator = copy.deepcopy(template)
-            validator._resource_graph = resource_graph
-
-            # We must obey to our own naming convention so we drop everything
-            # that's not valid.
-            resource_name = validator.name % {"field": field}
-            resource_name = resource_name.replace(" ", "_").lower()
-            resource_name = re.sub(r"[^a-z0-9_.-]", "_", resource_name)
-            validator.name = resource_name
-
-            if source_field_selector:
-                setattr(validator, field_selector.field_name, field)
-
-            if reference_field_selector:
-                setattr(validator, reference_field_selector["field_name"], field)
-
-            if filter_field_selector:
-                setattr(
-                    validator.filter,
-                    field_selector.field_name,
-                    field,
-                )
-
-            manifest_ctx.validators[resource_name] = validator
-
-
 def _diff_resource_graph(
     namespace: str,
     manifest_ctx: DiffContext,
     server_ctx: DiffContext,
-    for_validators: bool | None = None,
     import_mode: bool = False,
 ) -> GraphDiff:
     # Backwards compatibility: Here we note the filters that
@@ -506,14 +401,6 @@ def _diff_resource_graph(
         for field in fields:
             if field in CREATE_ONLY_RESOURCES:
                 continue
-
-            include_diff = (
-                True
-                if for_validators is None
-                else (field == "validators") == for_validators
-            )
-            if not include_diff:
-                continue
             diff_by_resource[field] = diff_fn(
                 namespace, getattr(manifest_ctx, field), getattr(server_ctx, field)
             )
@@ -676,23 +563,6 @@ def diff(
     if isinstance(manifest_object, Resource) and manifest_object.ignore_changes:
         return None
 
-    # NOTE: Backwards compatibility while we transition to Filter resources.
-    # If we're about to diff a validator where the manifest validator still uses
-    # the JSONFilterExpression mode, then force the server object to work
-    # in the JSONFilterExpression mode so that we can compare the same fields.
-    # Same for validator reference.
-    if (
-        isinstance(manifest_object, (Reference, Validator))
-        and
-        # NOTE: the server resource is always the Validator. Even if
-        # the server object is potentially the nested 'Reference'.
-        # This is mostly for type-checking.
-        isinstance(server_resource, Validator)
-        and isinstance(manifest_resource, Validator)
-    ) and not manifest_object._filter_resource_mode():
-        manifest_resource._force_filter_expression_mode()
-        server_resource._force_filter_expression_mode()
-
     if isinstance(server_resource, Segmentation) and isinstance(
         manifest_resource, Segmentation
     ):
@@ -752,20 +622,6 @@ def diff(
         server = getattr(server_object, field)
 
         if manifest != server:
-            # This is a temporary fix to support marking deprecated secret fields as
-            # changed on a `Channel`. If the field is both in `_mutable_fields`
-            # and also in `_deprecated_secret_fields` we will flag the field in
-            # `secret_fields_changed` if there has been an update. This ensures
-            # that the value is redacted by default when displaying the config.
-            secret_fields_changed = None
-            if (
-                isinstance(manifest_object, Channel)
-                and field in manifest_object._deprecated_secret_fields()
-            ):
-                secret_fields_changed = {
-                    field: True,
-                }
-
             # Whenever we find there's a change to make, grab all fields on both
             # resource versions so that we can present them as a diff.
             # Regardless of at what depth we find a change, the diff we present
@@ -774,7 +630,6 @@ def diff(
             return ResourceUpdate(
                 manifest_resource,
                 server_resource,
-                secret_fields_changed=secret_fields_changed,
             )
 
     # Some fields require the resource to be re-created if the type changes
@@ -974,7 +829,7 @@ async def check_for_source_schema_changes(
                 manifest_source, server_source
             )
 
-        # Ensure the source has a schema in case we need it in a field selector.
+        # Ensure the source has a schema in case we need it later on.
         if manifest_source.jtd_schema is None:
             manifest_source.jtd_schema = server_source.jtd_schema
 

{validio_sdk-5.2.0 → validio_sdk-5.3.1}/validio_sdk/resource/_server_resources.py

@@ -10,16 +10,13 @@ from gql.transport.exceptions import TransportQueryError
 import validio_sdk
 from validio_sdk._api import api
 from validio_sdk.client import Session
-from validio_sdk.client.client import Client
 from validio_sdk.code._progress import ProgressBar
-from validio_sdk.config import Config, ValidioConfig
 from validio_sdk.exception import ValidioBugError, ValidioError
 from validio_sdk.resource._diff import (
     DiffContext,
     GraphDiff,
     ResourceUpdate,
     ResourceUpdates,
-    expand_validator_field_selectors,
     infer_schemas,
 )
 from validio_sdk.resource._diff_util import (
@@ -59,6 +56,13 @@ from validio_sdk.resource.credentials import (
     LookerCredential,
     MsPowerBiCredential,
     MsPowerBiCredentialAuthEntraId,
+    MsSqlServerCredential,
+    MsSqlServerCredentialAuth,
+    MsSqlServerCredentialEntraId,
+    MsSqlServerCredentialUserPassword,
+    OracleCredential,
+    OracleCredentialAuth,
+    OracleCredentialUserPassword,
     PostgreSqlCredential,
     SigmaCredential,
     SnowflakeCredential,
@@ -314,12 +318,12 @@ async def load_credentials(
                 auth_type = c["config"]["auth"]["__typename"]
 
                 if auth_type == "SnowflakeCredentialUserPassword":
-                    auth: SnowflakeCredentialAuth = SnowflakeCredentialUserPassword(
+                    sf_auth: SnowflakeCredentialAuth = SnowflakeCredentialUserPassword(
                         user=c["config"]["auth"]["user"],
                         password="UNSET",
                     )
                 elif auth_type == "SnowflakeCredentialKeyPair":
-                    auth = SnowflakeCredentialKeyPair(
+                    sf_auth = SnowflakeCredentialKeyPair(
                         user=c["config"]["auth"]["user"],
                         private_key="UNSET",
                         private_key_passphrase="UNSET",
@@ -330,13 +334,64 @@ async def load_credentials(
                 credential = SnowflakeCredential(
                     name=name,
                     account=c["config"]["account"],
-                    auth=auth,
+                    auth=sf_auth,
                     warehouse=c["config"]["warehouse"],
                     role=c["config"]["role"],
                     display_name=display_name,
                     enable_catalog=c["enableCatalog"],
                     __internal__=g,
                 )
+            case "MsSqlServerCredential":
+                auth_type = c["config"]["auth"]["__typename"]
+
+                if auth_type == "MsSqlServerCredentialUserPassword":
+                    ms_auth: MsSqlServerCredentialAuth = (
+                        MsSqlServerCredentialUserPassword(
+                            user=c["config"]["auth"]["user"],
+                            password="UNSET",
+                        )
+                    )
+                elif auth_type == "MsSqlServerCredentialEntraId":
+                    ms_auth = MsSqlServerCredentialEntraId(
+                        client_id=c["config"]["auth"]["clientId"],
+                        client_secret="UNSET",
+                    )
+                else:
+                    raise ValidioBugError(
+                        f"Unknown MsSqlServer auth type: '{auth_type}'"
+                    )
+
+                credential = MsSqlServerCredential(
+                    name=name,
+                    host=c["config"]["host"],
+                    port=c["config"]["port"],
+                    database=c["config"]["database"],
+                    auth=ms_auth,
+                    display_name=display_name,
+                    enable_catalog=c["enableCatalog"],
+                    __internal__=g,
+                )
+            case "OracleCredential":
+                auth_type = c["config"]["auth"]["__typename"]
+
+                if auth_type == "OracleCredentialUserPassword":
+                    oracle_auth: OracleCredentialAuth = OracleCredentialUserPassword(
+                        user=c["config"]["auth"]["user"],
+                        password="UNSET",
+                    )
+                else:
+                    raise ValidioBugError(f"Unknown Oracle auth type: '{auth_type}'")
+
+                credential = OracleCredential(
+                    name=name,
+                    host=c["config"]["host"],
+                    port=c["config"]["port"],
+                    database=c["config"]["database"],
+                    auth=oracle_auth,
+                    display_name=display_name,
+                    enable_catalog=c["enableCatalog"],
+                    __internal__=g,
+                )
             case "KafkaSslCredential":
                 credential = KafkaSslCredential(
                     name=name,
@@ -443,7 +498,6 @@ async def load_channels(
 
         cfg = ch["config"]
         application_link_url = cfg["applicationLinkUrl"]
-        unset_or_none = None if cfg.get("webhookUrl") else "UNSET"
         interactive_message_enabled = cfg.get("interactiveMessageEnabled", None)
         display_name = ch["name"]
         # The 'secret' parts of a channel are left unset since they are not
@@ -454,10 +508,9 @@ async def load_channels(
                     name=name,
                     application_link_url=application_link_url,
                     ms_teams_channel_id=cfg["msTeamsChannelId"],
-                    client_id=unset_or_none,
-                    client_secret=unset_or_none,
+                    client_id="UNSET",
+                    client_secret="UNSET",
                     interactive_message_enabled=interactive_message_enabled,
-                    webhook_url=cfg["webhookUrl"],
                     display_name=display_name,
                     __internal__=g,
                 )
@@ -466,11 +519,10 @@ async def load_channels(
                     name=name,
                     application_link_url=application_link_url,
                     slack_channel_id=cfg["slackChannelId"],
-                    token=unset_or_none,
-                    signing_secret=unset_or_none,
-                    app_token=unset_or_none,
+                    token="UNSET",
+                    signing_secret="UNSET",
+                    app_token="UNSET",
                     interactive_message_enabled=interactive_message_enabled,
-                    webhook_url=cfg["webhookUrl"],
                     display_name=display_name,
                     __internal__=g,
                 )
@@ -839,10 +891,6 @@ async def apply_updates_on_server(
         dry_run_sql=dry_run_sql,
     )
 
-    # Now we should have all source schemas available. We can expand
-    # field selectors.
-    expand_validator_field_selectors(ctx)
-
     # Then apply updates.
     await apply_updates(
         namespace=namespace,