validio-sdk 0.28.0__tar.gz → 5.2.0__tar.gz

{validio_sdk-0.28.0 → validio_sdk-5.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validio-sdk
-Version: 0.28.0
+Version: 5.2.0
 Summary: SDK to interact with the Validio platform
 Home-page: https://validio.io/
 License: Apache-2.0

{validio_sdk-0.28.0 → validio_sdk-5.2.0}/pyproject.toml

@@ -3,7 +3,7 @@ name = "validio-sdk"
 # This version does not represent the released version or any tag. For each
 # release we automatically bump this before building and publishing so this
 # should be kept at 0.0.1dev1
-version = "0.28.0"
+version = "5.2.0"
 description = "SDK to interact with the Validio platform"
 authors = ["Validio <support@validio.io>"]
 license = "Apache-2.0"

{validio_sdk-0.28.0 → validio_sdk-5.2.0}/validio_sdk/_api/api.py

@@ -367,20 +367,6 @@ async def get_sources(
         key
         value
       }
-      ... on GcpStorageSource {
-        config {
-          project
-          bucket
-          folder
-          csv {
-            nullMarker
-            delimiter
-          }
-          schedule
-          filePattern
-          fileFormat
-        }
-      }
       ... on GcpBigQuerySource {
         config {
           project
@@ -432,19 +418,6 @@ async def get_sources(
           schedule
         }
       }
-      ... on AwsS3Source {
-        config {
-          bucket
-          prefix
-          csv {
-            nullMarker
-            delimiter
-          }
-          schedule
-          filePattern
-          fileFormat
-        }
-      }
       ... on AzureSynapseSource {
         config {
           database
@@ -502,6 +475,12 @@ async def get_sources(
           schedule
         }
       }
+      ... on SqlSource {
+        config {
+          sqlQuery
+          schedule
+        }
+      }
       ... on KafkaSource {
         config {
           topic
@@ -605,6 +584,42 @@ async def secrets_changed_by_field(
     return await execute(session, query, variable_values=variable_values)
 
 
+async def sql_source_preview(
+    session: Session,
+    credential_id: str,
+    sql_query: str,
+    dry_run: bool,
+    source_name: str,
+) -> Any:
+    query = """
+    query Op($input: SqlSourcePreviewInput!) {
+      sqlSourcePreview(input: $input) {
+        jtdSchema
+        queryError
+      }
+    }
+    """
+    input_variable_values = {
+        "input": {
+            "credentialId": credential_id,
+            "sqlQuery": sql_query,
+            "dryRun": dry_run,
+        }
+    }
+
+    result = await execute(
+        session=session, query=query, variable_values=input_variable_values
+    )
+
+    query_error = result.get("queryError")
+    if query_error is not None:
+        raise ValidioError(
+            f"""Sql query for source '{source_name}' is not valid: {query_error}"""
+        )
+
+    return result
+
+
 async def infer_schema(
     session: Session,
     class_name: str,
@@ -1412,12 +1427,6 @@ async def get_windows(
       source {
         resourceName
       }
-      ... on FileWindow {
-        dataTimeField
-        config {
-          segmentRetentionPeriodDays
-        }
-      }
       ... on GlobalWindow {
         config {
           segmentRetentionPeriodDays
@@ -1677,6 +1686,7 @@ async def get_validators(
         sensitivity
         decisionBoundsType
         adaptionRate
+        algorithm
       }}
       ... on DifferenceThreshold {{
         differenceOperator: operator

{validio_sdk-0.28.0 → validio_sdk-5.2.0}/validio_sdk/resource/_diff.py

@@ -907,11 +907,20 @@ async def check_for_source_schema_changes(
 
         manifest_source = r.manifest
         reinfer = schema_reinference.should_reinfer_schema_for_source(name)
+
         if manifest_source.jtd_schema is None:
             if reinfer:
                 sources_to_infer.append(manifest_source)
             else:
                 manifest_source.jtd_schema = r.server.jtd_schema
+        elif not reinfer:
+            # Unless we specify to update schemas we let the server schema be
+            # source of truth. This is to not end up mutating or changing
+            # anything schema related when we only wanted to update something
+            # unrelated, e.g. a description.
+            # Only if the source is specified to update its schema do we use the
+            # manifest as source of truth.
+            manifest_source.jtd_schema = r.server.jtd_schema
 
     # For unchanged sources, check if there is a schema diff now that all schemas
     # have been resolved. If we find a diff, flag the source as updated.
@@ -938,11 +947,22 @@ async def check_for_source_schema_changes(
         sources=sources_to_infer,
         session=session,
     )
+
     for name, server_source in server_ctx.sources.items():
         if name not in unchanged_sources:
             continue
 
         manifest_source = unchanged_sources[name]
+
+        # Unless we are actively looking to update the schema, force the server
+        # manifest to be whatever is specified in the manifest,  if in manual
+        # schema mode.
+        if (
+            manifest_source.jtd_schema is not None
+            and not schema_reinference.should_reinfer_schema_for_source(name)
+        ):
+            manifest_source.jtd_schema = server_source.jtd_schema
+
         if (
             manifest_source.jtd_schema is not None
             and manifest_source.jtd_schema != server_source.jtd_schema

{validio_sdk-0.28.0 → validio_sdk-5.2.0}/validio_sdk/resource/_server_resources.py

@@ -71,7 +71,7 @@ from validio_sdk.resource.credentials import (
 from validio_sdk.resource.filters import Filter, SqlFilter
 from validio_sdk.resource.notification_rules import Conditions
 from validio_sdk.resource.segmentations import Segmentation
-from validio_sdk.resource.sources import AzureSynapseSource
+from validio_sdk.resource.sources import AzureSynapseSource, Source, SqlSource
 from validio_sdk.resource.tags import Tag
 from validio_sdk.resource.thresholds import Threshold
 from validio_sdk.resource.validators import Reference, SqlValidator, Validator
@@ -468,6 +468,7 @@ async def load_channels(
                     slack_channel_id=cfg["slackChannelId"],
                     token=unset_or_none,
                     signing_secret=unset_or_none,
+                    app_token=unset_or_none,
                     interactive_message_enabled=interactive_message_enabled,
                     webhook_url=cfg["webhookUrl"],
                     display_name=display_name,
@@ -835,6 +836,7 @@ async def apply_updates_on_server(
         show_secrets=show_secrets,
         session=session,
         progress_bar=progress_bar,
+        dry_run_sql=dry_run_sql,
     )
 
     # Now we should have all source schemas available. We can expand
@@ -1071,6 +1073,16 @@ async def _maybe_validate_queries(
             [v for v in resources_list if isinstance(v, SqlFilter)],
             session,
         )
+    if (
+        len(resources_list) > 0
+        and isinstance(resources_list[0], Source)
+        and dry_run_sql
+    ):
+        await validate_source_sql_queries(
+            manifest_ctx,
+            [v for v in resources_list if isinstance(v, SqlSource)],
+            session,
+        )
 
 
 async def apply_updates(
@@ -1229,6 +1241,38 @@ async def validate_filter_sql_query(
     )
 
 
+async def validate_source_sql_queries(
+    manifest_ctx: DiffContext,
+    sources: list[SqlSource],
+    session: Session,
+) -> None:
+    for s in sources:
+        await validate_source_sql_query(manifest_ctx, s, session)
+
+
+async def validate_source_sql_query(
+    manifest_ctx: DiffContext,
+    source: SqlSource,
+    session: Session,
+) -> None:
+    credential = manifest_ctx.credentials.get(source.credential_name)
+    if credential is None:
+        return
+
+    if isinstance(credential, AzureSynapseSqlCredential):
+        # Skipping Azure Synapse credentials since dry run is not supported for query
+        # validation
+        return
+
+    await api.sql_source_preview(
+        session=session,
+        credential_id=credential._must_id(),
+        sql_query=source.sql_query,
+        dry_run=True,
+        source_name=source.name,
+    )
+
+
 async def test_credential(
     namespace: str,
     manifest_ctx: DiffContext,

{validio_sdk-0.28.0 → validio_sdk-5.2.0}/validio_sdk/resource/channels.py

@@ -191,6 +191,7 @@ class SlackChannel(Channel):
         slack_channel_id: str | None = None,
         token: str | None = None,
         signing_secret: str | None = None,
+        app_token: str | None = None,
         interactive_message_enabled: bool | None = None,
         webhook_url: str | None = None,
         display_name: str | None = None,
@@ -204,7 +205,8 @@ class SlackChannel(Channel):
             instance, used to send notifications.
         :param slack_channel_id: Slack channel ID to send to.
         :param token: Slack API token.
-        :param signing_secret: Slack API signing secret.
+        :param signing_secret: Slack API signing secret. Deprecated.
+        :param app_token: App level token used to retrieve user events from Slack.
         :param interactive_message_enabled: If interactive notification messages should
          be used.
         :param webhook_url: Webhook URL provided by Slack to the
@@ -224,14 +226,22 @@ class SlackChannel(Channel):
         self.slack_channel_id = slack_channel_id
         self.token = token
         self.signing_secret = signing_secret
+        self.app_token = app_token
         self.interactive_message_enabled = interactive_message_enabled
 
+        if self.signing_secret:
+            self.add_field_deprecation("signing_secret", "app_token")
+
         if webhook_url and (
-            slack_channel_id or token or signing_secret or interactive_message_enabled
+            slack_channel_id
+            or token
+            or signing_secret
+            or interactive_message_enabled
+            or app_token
         ):
             raise ManifestConfigurationError(
                 f"invalid configuration for Slack channel {self.name}: either"
-                " webhook_url or slack_channel_id, token, signing_secret and"
+                " webhook_url or slack_channel_id, token, app_token and"
                 " interactive_message_enabled can be used."
             )
 
@@ -241,7 +251,15 @@ class SlackChannel(Channel):
     def _secret_fields(self) -> set[str]:
         # If the webhook URL is set, we return an empty set so that the caller can
         # avoid calling the secrets changed endpoint as it's not supported
-        return set({}) if self.webhook_url else {"signing_secret", "token"}
+        return (
+            set({})
+            if self.webhook_url
+            else {
+                "app_token",
+                "signing_secret",
+                "token",
+            }
+        )
 
     def _immutable_fields(self) -> set[str]:
         return set({})

{validio_sdk-0.28.0 → validio_sdk-5.2.0}/validio_sdk/resource/credentials.py

@@ -1641,3 +1641,15 @@ class SigmaCredential(Credential):
 
     def _secret_fields(self) -> set[str]:
         return {"client_secret"}
+
+
+WarehouseCredential = Union[
+    AzureSynapseCredential,
+    AwsAthenaCredential,
+    AwsRedshiftCredential,
+    ClickHouseCredential,
+    DatabricksCredential,
+    GcpCredential,
+    PostgreSqlCredential,
+    SnowflakeCredential,
+]