validio-sdk 0.25.2__tar.gz → 0.26.0__tar.gz

{validio_sdk-0.25.2 → validio_sdk-0.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validio-sdk
-Version: 0.25.2
+Version: 0.26.0
 Summary: SDK to interact with the Validio platform
 Home-page: https://validio.io/
 License: Apache-2.0

{validio_sdk-0.25.2 → validio_sdk-0.26.0}/pyproject.toml

@@ -3,7 +3,7 @@ name = "validio-sdk"
 # This version does not represent the released version or any tag. For each
 # release we automatically bump this before building and publishing so this
 # should be kept at 0.0.1dev1
-version = "0.25.2"
+version = "0.26.0"
 description = "SDK to interact with the Validio platform"
 authors = ["Validio <support@validio.io>"]
 license = "Apache-2.0"
@@ -31,7 +31,7 @@ licensecheck = "^2023.1.1"
 mypy = "^1.2.0"
 pytest = "^7.3.1"
 pytest-asyncio = "^0.21.0"
-ruff = "^0.1.11"
+ruff = "^0.9.6"
 sphinx = "^6.2.1"           # We use <7 because most themes only work with that.
 sphinx-rtd-theme = "^1.2.1"
 ariadne-codegen = "0.14.0"
@@ -126,6 +126,7 @@ exclude = [
     "docs/*",
 ]
 
+[tool.ruff.lint]
 select = [
     "ARG", # flake8-unused-argument
     "C4",  # flake8-comprehension
@@ -152,7 +153,7 @@ ignore = [
     "PLR0913", # Too many arguments to function call (named arguments take care of this)
 ]
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "__init__.py" = [
     # We don't need docstrings in __init__
     "D104",
@@ -165,13 +166,13 @@ ignore = [
 "*/graphql_client/*.py" = ["D"]
 "*/tests/*.py" = ["D"]
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "google"
 
 [tool.mypy]
 ignore_missing_imports = true
 disallow_untyped_calls = true
-disallow_untyped_defs = true
+disallow_untyped_defs = false # TODO(Set to true when generated client is deleted)
 disallow_incomplete_defs = true
 check_untyped_defs = true
 disallow_untyped_decorators = true

{validio_sdk-0.25.2 → validio_sdk-0.26.0}/validio_sdk/__init__.py

@@ -25,24 +25,21 @@ from validio_sdk.resource.filters import (
 from validio_sdk.util import load_jtd_schema
 
 __all__ = [
-    # Filters
     "BooleanFilter",
     "BooleanFilterOperator",
+    "ConfigInvalidError",
+    "ConfigNotFoundError",
     "EnumFilter",
     "EnumFilterOperator",
     "NullFilter",
     "NullFilterOperator",
+    "SqlFilter",
     "StringFilter",
     "StringFilterOperator",
     "ThresholdFilter",
     "ThresholdFilterOperator",
-    "SqlFilter",
-    # Exceptions
-    "ValidioError",
-    "ConfigNotFoundError",
-    "ConfigInvalidError",
     "UnauthorizedError",
     "ValidioConnectionError",
-    # Methods
+    "ValidioError",
     "load_jtd_schema",
 ]

{validio_sdk-0.25.2 → validio_sdk-0.26.0}/validio_sdk/_api/api.py

@@ -241,6 +241,38 @@ class APIClient:
         async with self.client as session:
             return await delete_namespace(session, namespace_id)
 
+    async def delete_credentials(self, credential_ids: list[str]) -> Any:
+        async with self.client as session:
+            return await delete_credentials(session, credential_ids)
+
+    async def delete_channels(self, channel_ids: list[str]) -> Any:
+        async with self.client as session:
+            return await delete_channels(session, channel_ids)
+
+    async def delete_sources(self, source_ids: list[str]) -> Any:
+        async with self.client as session:
+            return await delete_sources(session, source_ids)
+
+    async def delete_windows(self, window_ids: list[str]) -> Any:
+        async with self.client as session:
+            return await delete_windows(session, window_ids)
+
+    async def delete_segmentations(self, segmentation_ids: list[str]) -> Any:
+        async with self.client as session:
+            return await delete_segmentations(session, segmentation_ids)
+
+    async def delete_filters(self, filter_ids: list[str]) -> Any:
+        async with self.client as session:
+            return await delete_filters(session, filter_ids)
+
+    async def delete_notification_rules(self, notification_rule_ids: list[str]) -> Any:
+        async with self.client as session:
+            return await delete_notification_rules(session, notification_rule_ids)
+
+    async def delete_validators(self, validator_ids: list[str]) -> Any:
+        async with self.client as session:
+            return await delete_validators(session, validator_ids)
+
     async def get_recommendations(self, source_id: str) -> Any:
         async with self.client as session:
             return await get_recommendations(session, source_id)
@@ -368,17 +400,6 @@ async def get_sources(
           }
         }
       }
-      ... on GcpPubSubLiteSource {
-        config {
-          location
-          project
-          subscriptionId
-          messageFormat {
-            format
-            schema
-          }
-        }
-      }
       ... on AwsAthenaSource {
         config {
           catalog
@@ -554,7 +575,7 @@ async def get_sources(
 async def secrets_changed_by_field(
     session: Session,
     class_name: str,
-    return_fields: set[str],
+    query_response_fields: str,
     variable_values: dict[str, Any],
 ) -> Any:
     """Check if secrets changed. This is only compatible with APIs that return
@@ -562,8 +583,7 @@ async def secrets_changed_by_field(
 
     Create a query and input types for the passed class name.
     :param class_name: Name of resource class, e.g. RedshiftCredential
-    :param return_fields: Names of the secret fields which will be used to retrieve
-    the results of change per field
+    :param query_response_fields: Query response that contains secret fields changed.
     :param variable_values: Dictionary with secret fields
     :returns: GraphQL response containing which fields have changed
     """
@@ -571,13 +591,11 @@ async def secrets_changed_by_field(
     method_name = f"{lc_first}SecretChanged"
     input_type = f"{class_name}SecretChangedInput!"
 
-    return_fields_query: str = "\n".join(return_fields)
-
     query = f"""
     query Op($input: {input_type}) {{
       {method_name}(input: $input) {{
-        {return_fields_query}
         errors {{ code message }}
+        {query_response_fields}
       }}
     }}
     """
@@ -631,7 +649,7 @@ async def dbt_artifact_multipart_upload_create(
 
     if response["errors"]:
         raise ValidioError(
-            f'Failed to create dbt multipart upload: {response["errors"]}',
+            f"Failed to create dbt multipart upload: {response['errors']}",
         )
 
     return response["id"]
@@ -654,7 +672,7 @@ async def dbt_artifact_multipart_upload_append_part(
 
     if response["errors"]:
         raise ValidioError(
-            f'Failed to append dbt multipart upload: {response["errors"]}',
+            f"Failed to append dbt multipart upload: {response['errors']}",
         )
 
 
@@ -670,7 +688,7 @@ async def dbt_artifact_multipart_upload_complete(
 
     if response["errors"]:
         raise ValidioError(
-            f'Failed to finalize dbt multipart upload: {response["errors"]}',
+            f"Failed to finalize dbt multipart upload: {response['errors']}",
         )
 
 
@@ -710,7 +728,6 @@ async def validator_segment_metrics(
       isIncident
       value
       severity
-      deviation
       ... on ValidatorMetricWithFixedThreshold {
         operator
         bound
@@ -1038,6 +1055,8 @@ async def get_credentials(
         variable_values = _namespace_filter(namespace_id)
 
     result = await execute(session, query, variable_values=variable_values)
+    if not result:
+        return result
 
     if isinstance(result, list):
         return_first = False
@@ -1071,7 +1090,6 @@ async def get_channels(
       ... on SlackChannel {
         config {
           slackWebhookUrl: webhookUrl
-          timezone
           applicationLinkUrl
           slackChannelId
           interactiveMessageEnabled
@@ -1081,13 +1099,11 @@ async def get_channels(
         config {
           webhookUrl
           applicationLinkUrl
-          authHeader
         }
       }
       ... on MsTeamsChannel {
         config {
           msTeamsWebhookUrl: webhookUrl
-          timezone
           applicationLinkUrl
           msTeamsChannelId
           msTeamsInteractiveMessageEnabled: interactiveMessageEnabled
@@ -1126,6 +1142,8 @@ async def get_channels(
         variable_values = _namespace_filter(namespace_id)
 
     result = await execute(session, query, variable_values=variable_values)
+    if not result:
+        return result
 
     if isinstance(result, list):
         return_first = False
@@ -1273,7 +1291,6 @@ async def get_segments(
     segment_details = """
     fragment SegmentDetails on Segment {
         id
-        muted
         fields {
         field
         value
@@ -1342,12 +1359,21 @@ async def get_windows(
       }
       ... on FileWindow {
         dataTimeField
+        config {
+          segmentRetentionPeriodDays
+        }
+      }
+      ... on GlobalWindow {
+        config {
+          segmentRetentionPeriodDays
+        }
       }
       ... on FixedBatchWindow {
         config {
           batchSize
           segmentedBatching
           batchTimeoutSecs
+          segmentRetentionPeriodDays
         }
         dataTimeField
       }
@@ -1356,6 +1382,7 @@ async def get_windows(
           windowSize
           timeUnit
           windowTimeoutDisabled
+          segmentRetentionPeriodDays
         }
         dataTimeField
       }
@@ -1565,12 +1592,6 @@ async def get_validators(
 
     reference_source_config = """
     referenceSourceConfig {
-      source {
-        ...ReferenceSourceConfigDetails
-      }
-      window {
-        ...WindowDetails
-      }
       history
       offset
       sourceFilter {
@@ -1598,28 +1619,6 @@ async def get_validators(
       }}
     }}
 
-    fragment ReferenceSourceConfigDetails on Source {{
-      __typename
-      id
-      name
-      resourceName
-      namespaceId
-      namespace {{
-        id
-      }}
-    }}
-
-    fragment WindowDetails on Window {{
-      __typename
-      id
-      name
-      resourceName
-      namespaceId
-      namespace {{
-        id
-      }}
-    }}
-
     fragment ValidatorDetails on Validator {{
       __typename
       id
@@ -1695,6 +1694,7 @@ async def get_validators(
           sourceFields
           volumeMetric: metric
           initializeWithBackfill
+          metadataEnabled
           threshold {{
             ...ThresholdDetails
           }}
@@ -1730,6 +1730,7 @@ async def get_validators(
       ... on FreshnessValidator {{
         config {{
           initializeWithBackfill
+          metadataEnabled
           optionalSourceField: sourceField
           threshold {{
             ...ThresholdDetails
@@ -2125,6 +2126,80 @@ async def delete_namespace(session: Session, namespace_id: str) -> Any:
     )
 
 
+async def delete_credentials(session: Session, window_ids: list[str]) -> Any:
+    return await execute_mutation(
+        session,
+        "credentialsDelete",
+        argument_types={"ids": "[CredentialId!]!"},
+        variable_values={"ids": window_ids},
+    )
+
+
+async def delete_channels(session: Session, channel_ids: list[str]) -> Any:
+    return await execute_mutation(
+        session,
+        "channelsDelete",
+        argument_types={"ids": "[ChannelId!]!"},
+        variable_values={"ids": channel_ids},
+    )
+
+
+async def delete_sources(session: Session, source_ids: list[str]) -> Any:
+    return await execute_mutation(
+        session,
+        "sourcesDelete",
+        argument_types={"ids": "[SourceId!]!"},
+        variable_values={"ids": source_ids},
+    )
+
+
+async def delete_windows(session: Session, window_ids: list[str]) -> Any:
+    return await execute_mutation(
+        session,
+        "windowsDelete",
+        argument_types={"ids": "[WindowId!]!"},
+        variable_values={"ids": window_ids},
+    )
+
+
+async def delete_segmentations(session: Session, segmentation_ids: list[str]) -> Any:
+    return await execute_mutation(
+        session,
+        "segmentationsDelete",
+        argument_types={"ids": "[SegmentationId!]!"},
+        variable_values={"ids": segmentation_ids},
+    )
+
+
+async def delete_filters(session: Session, filter_ids: list[str]) -> Any:
+    return await execute_mutation(
+        session,
+        "filtersDelete",
+        argument_types={"ids": "[FilterId!]!"},
+        variable_values={"ids": filter_ids},
+    )
+
+
+async def delete_notification_rules(
+    session: Session, notification_rule_ids: list[str]
+) -> Any:
+    return await execute_mutation(
+        session,
+        "notificationRulesDelete",
+        argument_types={"ids": "[NotificationRuleId!]!"},
+        variable_values={"ids": notification_rule_ids},
+    )
+
+
+async def delete_validators(session: Session, validator_ids: list[str]) -> Any:
+    return await execute_mutation(
+        session,
+        "validatorsDelete",
+        argument_types={"ids": "[ValidatorId!]!"},
+        variable_values={"ids": validator_ids},
+    )
+
+
 async def get_recommendations(session: Session, source_id: str) -> Any:
     query = """
     query GetSourceRecommendedValidators($id: SourceId!) {
@@ -2437,6 +2512,73 @@ async def apply_source_action(
     return result[f"source{action.value}"]
 
 
+async def validate_sql_validator_query(
+    session: Session,
+    query: str,
+    validator_display_name: str,
+    source_id: str,
+    segmentation_id: str,
+    window_id: str,
+) -> None:
+    gql_query = """
+        query S($input: SqlValidatorQueryVerificationInput!) {
+            sqlValidatorQueryVerification(input: $input) {
+                queryError
+            }
+        }
+    """
+    result = await execute(
+        session,
+        gql_query,
+        variable_values={
+            "input": {
+                "query": query,
+                "sourceConfig": {
+                    "segmentationId": segmentation_id,
+                    "windowId": window_id,
+                    "sourceId": source_id,
+                },
+                "dryRun": True,
+            }
+        },
+    )
+    if "queryError" in result and result["queryError"] is not None:
+        raise ValidioError(
+            f"""Sql query for validator '{validator_display_name}' is not valid:
+    {result["queryError"]}"""
+        )
+
+
+async def validate_sql_filter_query(
+    session: Session,
+    query: str,
+    filter_display_name: str,
+    source_id: str,
+) -> None:
+    gql_query = """
+        query S($input: SqlFilterVerificationInput!) {
+            sqlFilterVerification(input: $input) {
+                queryError
+            }
+        }
+    """
+    result = await execute(
+        session,
+        gql_query,
+        variable_values={
+            "input": {
+                "query": query,
+                "sourceId": source_id,
+            }
+        },
+    )
+    if "queryError" in result and result["queryError"] is not None:
+        raise ValidioError(
+            f"""Sql query for filter '{filter_display_name}' is not valid:
+    {result["queryError"]}"""
+        )
+
+
 def split_to_chunks(data: list[Any]) -> list[list[Any]]:
     return split_to_n_chunks(data, NUM_CONCURRENT_CONNECTIONS)
 

{validio_sdk-0.25.2 → validio_sdk-0.26.0}/validio_sdk/client/__init__.py

@@ -1,3 +1,3 @@
 from .client import Client, Session
 
-__all__ = ["Session", "Client"]
+__all__ = ["Client", "Session"]

{validio_sdk-0.25.2 → validio_sdk-0.26.0}/validio_sdk/code/apply.py

@@ -14,6 +14,7 @@ async def apply(
     diff: GraphDiff,
     show_secrets: bool,
     show_progress: bool = True,
+    dry_run_sql: bool = False,
 ) -> None:
     """Applies the provided diff operations on the server."""
     progress_bar = ProgressBar(
@@ -24,5 +25,11 @@ async def apply(
     with progress_bar:
         async with client.client as session:
             await apply_updates_on_server(
-                namespace, ctx, diff, session, show_secrets, progress_bar
+                namespace,
+                ctx,
+                diff,
+                session,
+                show_secrets,
+                progress_bar,
+                dry_run_sql,
             )

{validio_sdk-0.25.2 → validio_sdk-0.26.0}/validio_sdk/code/plan.py

@@ -24,7 +24,7 @@ from validio_sdk.resource._resource import (
 )
 from validio_sdk.resource._server_resources import load_resources
 from validio_sdk.resource._util import SourceSchemaReinference
-from validio_sdk.resource.channels import Channel
+from validio_sdk.resource.channels import Channel, WebhookChannel
 from validio_sdk.resource.tags import Tag
 
 
@@ -229,12 +229,15 @@ def _create_resource_diff_object(
     r: Resource | Diffable | dict,
     show_secrets: bool,
     rewrites: dict[str, Any] | None = None,
-    secrets_changed: bool = False,
+    secret_fields_changed: dict[str, Any] | None = None,
     is_manifest: bool = False,
 ) -> dict[str, object]:
     if rewrites is None:
         rewrites = {}
 
+    if secret_fields_changed is None:
+        secret_fields_changed = {}
+
     data = dict(r) if not isinstance(r, dict) else r
 
     diff_object = {}
@@ -245,17 +248,19 @@ def _create_resource_diff_object(
         if k in rewrites:
             diff_object[k] = rewrites[k]
         elif isinstance(v, Resource | Diffable | dict):
+            ch = secret_fields_changed.get(k)
             diff_object[k] = _create_resource_diff_object(
                 v,
                 show_secrets,
-                secrets_changed=secrets_changed,
+                secret_fields_changed=ch if not isinstance(ch, bool) else {},
                 is_manifest=is_manifest,
             )
         elif hasattr(v, "__dict__") and not isinstance(v, Enum):
+            ch = secret_fields_changed.get(k)
             diff_object[k] = _create_resource_diff_object(
                 v.__dict__,
                 show_secrets,
-                secrets_changed=secrets_changed,
+                secret_fields_changed=ch if not isinstance(ch, bool) else {},
                 is_manifest=is_manifest,
             )
         elif isinstance(v, list):
@@ -269,7 +274,6 @@ def _create_resource_diff_object(
                     _create_resource_diff_object(
                         item.__dict__,
                         show_secrets,
-                        secrets_changed=secrets_changed,
                         is_manifest=is_manifest,
                     )
                 )
@@ -288,7 +292,33 @@ def _create_resource_diff_object(
 
         if secret_fields:
             for field in secret_fields:
-                if secrets_changed and not is_manifest:
+                # We want to avoid overriding webhook_url if it is None already
+                # which may give a misleading impression that it is set to a value.
+                #
+                # For any channel that is not the WebhookChannel, we will always
+                # flag it as changed if any secrets have changed, which is consistent
+                # with existing behavior.
+                #
+                # This workaround until webhook URL is deprecated fully from some
+                # channels (VR-4047).
+                flag_webhook_url_as_changed = False
+                if not isinstance(r, WebhookChannel) and field == "webhook_url":
+                    if hasattr(r, "webhook_url") and not r.webhook_url:
+                        continue
+                    if any(
+                        secret_fields_changed[secret_field]
+                        for secret_field in secret_fields_changed
+                    ):
+                        flag_webhook_url_as_changed = True
+
+                if (
+                    # Special case handling for webhook URL
+                    flag_webhook_url_as_changed
+                    or
+                    # If the field does not exist in the secrets changed dict then
+                    # don't flag it as changed
+                    secret_fields_changed.get(field)
+                ) and not is_manifest:
                     diff_object[field] = "REDACTED-PREVIOUS"
                 else:
                     diff_object[field] = "REDACTED"

{validio_sdk-0.25.2 → validio_sdk-0.26.0}/validio_sdk/config.py

@@ -77,10 +77,10 @@ class ValidioConfig:
         Will ensure we don't print sensitive information.
         """
         return (
-            f'{"Default namespace":<20} | {self.default_namespace}\n'
-            f'{"Endpoint":<20} | {self.endpoint}\n'
-            f'{"Access key":<20} | {self.access_key}\n'
-            f'{"Access secret":<20} | {self._none_or_hidden(self.access_secret)}\n'
+            f"{'Default namespace':<20} | {self.default_namespace}\n"
+            f"{'Endpoint':<20} | {self.endpoint}\n"
+            f"{'Access key':<20} | {self.access_key}\n"
+            f"{'Access secret':<20} | {self._none_or_hidden(self.access_secret)}\n"
         )
 
 

{validio_sdk-0.25.2 → validio_sdk-0.26.0}/validio_sdk/dbt.py

@@ -10,6 +10,7 @@ from validio_sdk._api.api import (
     dbt_artifact_multipart_upload_complete,
     dbt_artifact_multipart_upload_create,
 )
+from validio_sdk.client.client import Client
 
 _DBT_MULTIPART_UPLOAD_CHUNK_SIZE_BYTES = 2 * 1000 * 1000  # 2MB
 
@@ -164,7 +165,7 @@ def _extract_sources(manifest: dict, relevant_sources: set) -> dict:
 
 
 async def upload_artifacts(
-    client: APIClient,
+    client: APIClient | Client,
     credential_id: str,
     job_name: str,
     manifest: dict,
@@ -174,13 +175,15 @@ async def upload_artifacts(
     Helper function to upload dbt artifacts.
     The artifacts are additionally trimmed before uploading.
 
-    :param client: ValidioAPIClient to use for the artifacts upload.
+    :param client: Client to use for the artifacts upload.
     :param credential_id: DBTCore credential id to upload artifacts on behalf of.
     :param job_name:  Job associated with the artifacts.
     :param manifest: manifest.json payload
     :param run_results: run_results.json payload
     """
-    async with client.client as session:
+    client = client if isinstance(client, Client) else client.client
+
+    async with client as session:
         upload_id = await dbt_artifact_multipart_upload_create(
             session,
             credential_id=credential_id,

{validio_sdk-0.25.2 → validio_sdk-0.26.0}/validio_sdk/graphql_client/client.py

@@ -27967,7 +27967,6 @@ class Client(AsyncBaseClient):
               isIncident
               value
               severity
-              deviation
               ... on ValidatorMetricWithFixedThreshold {
                 operator
                 bound