validio-sdk 7.4.0__tar.gz → 7.4.2__tar.gz

{validio_sdk-7.4.0 → validio_sdk-7.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validio-sdk
-Version: 7.4.0
+Version: 7.4.2
 Summary: SDK to interact with the Validio platform
 Home-page: https://validio.io/
 License: Apache-2.0

{validio_sdk-7.4.0 → validio_sdk-7.4.2}/pyproject.toml

@@ -3,7 +3,7 @@ name = "validio-sdk"
 # This version does not represent the released version or any tag. For each
 # release we automatically bump this before building and publishing so this
 # should be kept at 0.0.1dev1
-version = "7.4.0"
+version = "7.4.2"
 description = "SDK to interact with the Validio platform"
 authors = ["Validio <support@validio.io>"]
 license = "Apache-2.0"

{validio_sdk-7.4.0 → validio_sdk-7.4.2}/validio_sdk/_api/api.py

@@ -1023,6 +1023,12 @@ async def get_credentials(
           host
           port
           httpPath
+          databricksAuth: auth {
+            __typename
+            ... on DatabricksCredentialOAuth {
+              clientId
+            }
+          }
         }
         enableCatalog
       }
@@ -1242,6 +1248,7 @@ async def get_credentials(
             credential.get("config", {}),
             [
                 ("powerBiAuth", "auth"),
+                ("databricksAuth", "auth"),
                 ("databaseRequired", "database"),
                 ("optUser", "user"),
                 ("optBaseUrl", "baseUrl"),

{validio_sdk-7.4.0 → validio_sdk-7.4.2}/validio_sdk/code/_import.py

@@ -101,6 +101,7 @@ class ImportContext:
 async def _import(
     ctx: DiffContext,
     tags_ctx: DiffContext,
+    include_schema: bool = False,
 ) -> str:
     import_ctx = ImportContext()
 
@@ -189,16 +190,21 @@ async def _import(
     def import_sources(cls: type, name: str, r: Resource) -> None:
         credential = import_ctx.get_variable(Credential, cast(Any, r).credential_name)
 
+        inits: list[tuple[str, Any, str | None]] = [
+            ("credential", credential, None),
+            ("tags", resource_tags(r), None),
+        ]
+
+        if include_schema and isinstance(r, Source):
+            inits.append(("jtd_schema", r.jtd_schema, None))
+
         import_ctx.add_resource_decl(
             cls,
             name,
             r._import_str(
                 indent_level=0,
                 import_ctx=import_ctx,
-                inits=[
-                    ("credential", credential, None),
-                    ("tags", resource_tags(r), None),
-                ],
+                inits=inits,
                 skip={"tag_names"},
             ),
         )

{validio_sdk-7.4.0 → validio_sdk-7.4.2}/validio_sdk/resource/_diffable.py

@@ -23,6 +23,12 @@ could enter a cycle.
 """
 MAX_RESOURCE_DEPTH = 15
 
+"""
+When we format dictionaries this is the limit of how deep we can go to build a
+formatted output.
+"""
+MAX_FORMAT_DICT_DEPTH = 20
+
 
 class Diffable(ABC):
     """
@@ -162,8 +168,13 @@ class Diffable(ABC):
         import_args = []
 
         for field, arg, comment in sorted_params:
+            if isinstance(arg, dict):
+                value = _format_dict_value(arg, indent_level + 1, 0)
+            else:
+                value = arg
+
             comment_str = "" if not comment else f" # {comment}"
-            import_args.append(f"{field}={arg},{comment_str}")
+            import_args.append(f"{field}={value},{comment_str}")
 
         params_str = line_indent.join(import_args)
         closing_indent = " " * self._num_ident_spaces(indent_level)
@@ -175,6 +186,39 @@ class Diffable(ABC):
         return 4 * indent_level
 
 
+def _format_dict_value(value: Any, indent_level: int, recursion_depth: int) -> str:
+    """
+    Format dictionary for import.
+
+    This is only a best effort to try to wrap dict values instead of having a
+    long line for e.g. jtd schemas on import.
+    """
+    if recursion_depth >= MAX_FORMAT_DICT_DEPTH:
+        return value
+
+    indent = " " * (indent_level * 4)
+    if isinstance(value, dict):
+        if not value:
+            return "{}"
+
+        inner = indent + " " * 4
+
+        items = []
+        for k, v in value.items():
+            formatted = _format_dict_value(v, indent_level + 1, recursion_depth + 1)
+            items.append(f"{inner}'{k}': {formatted}")
+
+        return "{\n" + ",\n".join(items) + ",\n" + indent + "}"
+
+    if isinstance(value, bool):
+        return "True" if value else "False"
+
+    if value is None:
+        return "None"
+
+    return repr(value)
+
+
 class ApiSecretChangeNestedResource(Diffable):
     """An interface for nested structure inside a list of secret fields."""
 
@@ -195,6 +239,11 @@ class ApiSecretChangeNestedResource(Diffable):
         data[NODE_TYPE_FIELD_NAME] = self.__class__.__name__
         return data
 
+    def _nested_objects(
+        self,
+    ) -> dict[str, "Diffable | list[Diffable] | None"]:
+        return {}
+
     @staticmethod
     def _decode(
         type_: type,
@@ -218,6 +267,9 @@ class ApiSecretChangeNestedResource(Diffable):
             }
         }
 
+    def _api_secret_change_input(self) -> dict[str, Any]:
+        return self._api_input()
+
     def _api_secret_change_auth_query(self) -> str:
         """Returns the GraphQL query with fragments for the API secret change."""
         fields = "\n".join([to_camel(f) for f in self._secret_fields()])

{validio_sdk-7.4.0 → validio_sdk-7.4.2}/validio_sdk/resource/_resource.py

@@ -546,7 +546,7 @@ class Resource(Diffable):
 
     def _api_secret_change_input(self) -> dict[str, Any]:
         nested_objects = {
-            to_camel(f): obj._api_input()
+            to_camel(f): obj._api_secret_change_input()
             for f, obj in self._nested_secret_objects().items()
         }
 

{validio_sdk-7.4.0 → validio_sdk-7.4.2}/validio_sdk/resource/_server_resources.py

@@ -50,6 +50,9 @@ from validio_sdk.resource.credentials import (
     ClickHouseCredential,
     Credential,
     DatabricksCredential,
+    DatabricksCredentialAuth,
+    DatabricksCredentialOAuth,
+    DatabricksCredentialPersonalAccessToken,
     DbtCloudCredential,
     DbtCoreCredential,
     DemoCredential,
@@ -326,12 +329,31 @@ async def load_credentials(
                     __internal__=g,
                 )
             case "DatabricksCredential":
+                access_token = None
+                auth: DatabricksCredentialAuth | None = None
+
+                auth_type_config = c.get("config", {}).get("auth", {}) or {}
+                auth_type = auth_type_config.get("__typename")
+
+                if auth_type == "DatabricksCredentialPersonalAccessToken":
+                    auth = DatabricksCredentialPersonalAccessToken(token="UNSET")
+                elif auth_type == "DatabricksCredentialOAuth":
+                    auth = DatabricksCredentialOAuth(
+                        client_id=c["config"]["auth"]["clientId"],
+                        client_secret="UNSET",
+                    )
+                elif auth_type is None:
+                    access_token = "UNSET"
+                else:
+                    raise ValidioBugError(f"unknown auth type '{auth_type}'")
+
                 credential = DatabricksCredential(
                     name=name,
                     host=c["config"]["host"],
                     port=c["config"]["port"],
-                    access_token="UNSET",
                     http_path=c["config"]["httpPath"],
+                    access_token=access_token,
+                    auth=auth,
                     display_name=display_name,
                     enable_catalog=c["enableCatalog"],
                     __internal__=g,

{validio_sdk-7.4.0 → validio_sdk-7.4.2}/validio_sdk/resource/credentials.py

@@ -8,6 +8,8 @@ from camel_converter import to_camel
 
 from validio_sdk._api.api import APIClient, test_credential
 from validio_sdk.client.client import Session
+from validio_sdk.config import ValidioConfig
+from validio_sdk.exception import ValidioError
 from validio_sdk.resource._resource import (
     ApiSecretChangeNestedResource,
     Resource,
@@ -749,11 +751,6 @@ class SnowflakeCredentialKeyPair(ApiSecretChangeNestedResource):
     def _secret_fields(self) -> set[str]:
         return {"private_key", "private_key_passphrase"}
 
-    def _nested_objects(
-        self,
-    ) -> dict[str, "Diffable | list[Diffable] | None"]:
-        return {}
-
 
 class SnowflakeCredentialUserPassword(ApiSecretChangeNestedResource):
     """Snowflake password-based authentication."""
@@ -785,11 +782,6 @@ class SnowflakeCredentialUserPassword(ApiSecretChangeNestedResource):
     def _secret_fields(self) -> set[str]:
         return {"password"}
 
-    def _nested_objects(
-        self,
-    ) -> dict[str, "Diffable | list[Diffable] | None"]:
-        return {}
-
 
 SnowflakeCredentialAuth = Union[
     SnowflakeCredentialKeyPair, SnowflakeCredentialUserPassword
@@ -1185,6 +1177,81 @@ class LookerCredential(Credential):
         return {"client_secret"}
 
 
+class DatabricksCredentialPersonalAccessToken(ApiSecretChangeNestedResource):
+    """Databricks personal access token authentication."""
+
+    def __init__(
+        self,
+        *,
+        token: str,
+    ):
+        """
+        Constructor.
+
+        :param token: Databricks personal access token.
+        """
+        self.token = token
+
+    def _api_variant_name(self) -> str:
+        return "personalAccessToken"
+
+    def _immutable_fields(self) -> set[str]:
+        return set({})
+
+    def _mutable_fields(self) -> set[str]:
+        return set({})
+
+    def _secret_fields(self) -> set[str]:
+        return {"token"}
+
+
+class DatabricksCredentialOAuth(ApiSecretChangeNestedResource):
+    """Databricks OAuth token authentication."""
+
+    def __init__(
+        self,
+        *,
+        client_id: str,
+        client_secret: str,
+    ):
+        """
+        Constructor.
+
+        :param client_id: Databricks OAuth client ID.
+        :param client_secret: Databricks OAuth client secret.
+        """
+        self.client_id = client_id
+        self.client_secret = client_secret
+
+    def _api_variant_name(self) -> str:
+        return "oauth"
+
+    def _immutable_fields(self) -> set[str]:
+        return set({})
+
+    def _mutable_fields(self) -> set[str]:
+        return {"client_id"}
+
+    def _secret_fields(self) -> set[str]:
+        return {"client_secret"}
+
+    def _api_secret_change_input(self) -> dict[str, Any]:
+        # We override this for Databricks since Databricks is the only one that
+        # doesn't reuse the input type for creating credentials when checking if
+        # secrets are changed. This mean we can use the API input as is, we need
+        # to filter out non secrets.
+        return {
+            self._api_variant_name(): {
+                to_camel(f): getattr(self, f) for f in self._secret_fields()
+            }
+        }
+
+
+DatabricksCredentialAuth = (
+    DatabricksCredentialPersonalAccessToken | DatabricksCredentialOAuth
+)
+
+
 class DatabricksCredential(Credential):
     """A credential resource that can be used to connect to a Databricks table."""
 
@@ -1194,8 +1261,9 @@ class DatabricksCredential(Credential):
         name: str,
         host: str,
         port: int,
-        access_token: str,
         http_path: str,
+        access_token: str | None = None,
+        auth: DatabricksCredentialAuth | None = None,
         enable_catalog: bool = False,
         display_name: str | None = None,
         ignore_changes: bool = False,
@@ -1207,8 +1275,9 @@ class DatabricksCredential(Credential):
         :param name: Unique resource name assigned to the credential
         :param host: A host of Databricks deployment
         :param port: A port of Databricks deployment
-        :param access_token: An access token of system principal or a user
         :param http_path: Connection path of the compute resource to use.
+        :param access_token: Deprecated.
+        :param auth: Authentication method and config.
         :param enable_catalog: If set to true, this credential will
             be used to fetch catalog information.
         :param display_name: Human-readable name for the credential. This name is
@@ -1221,11 +1290,22 @@ class DatabricksCredential(Credential):
             ignore_changes=ignore_changes,
             __internal__=__internal__,
         )
+
+        if access_token and auth:
+            raise ValidioError("only one of 'access_token' and 'auth' can be specified")
+
+        if not access_token and not auth:
+            raise ValidioError("auth configuration missing")
+
+        if access_token:
+            self.add_field_deprecation("access_token", "auth")
+
         self.host = host
         self.port = port
         self.access_token = access_token
         self.http_path = http_path
         self.enable_catalog = enable_catalog
+        self.auth = auth
 
     def _immutable_fields(self) -> set[str]:
         return set({})
@@ -1237,8 +1317,28 @@ class DatabricksCredential(Credential):
         }
 
     def _secret_fields(self) -> set[str]:
+        # We must always include access token even when it's not used to ensure
+        # we can go from the legacy way of using this to the new auth config.
+        # When we pass auth we must also pass `access_token` as `null` to clear
+        # it.
         return {"access_token"}
 
+    def _nested_objects(
+        self,
+    ) -> dict[str, "Diffable | list[Diffable] | None"]:
+        return {"auth": self.auth}
+
+    def _api_update_input(self, namespace: str, ctx: "DiffContext") -> Any:
+        # Nested objects are not included in the input if they are `None` but we
+        # want to pass `auth` in all requests to ensure we can go from the new
+        # config (auth being set) to the old (access_token being set).
+        #
+        # Once we've dropped the legacy support we can also drop this override.
+        if self.auth is None:
+            return _api_update_input_params(self, overrides={"auth": None})
+
+        return super()._api_update_input(namespace, ctx)
+
 
 class AzureSynapseCredential(Credential):
     """A base class of Azure Credential resource."""

{validio_sdk-7.4.0 → validio_sdk-7.4.2}/validio_sdk/resource/tests/test_import.py

@@ -165,7 +165,7 @@ notificationrule_0 = NotificationRule(
     assert doc.strip() == expected.strip()
 
 
-def test_import_sql_source() -> None:
+def test_import_sql_source_with_include_schema() -> None:
     credential = PostgreSqlCredential(
         name="postgres",
         host="127.0.0.1",
@@ -188,7 +188,14 @@ def test_import_sql_source() -> None:
             "nullable": False,
             "optionalProperties": {},
             "properties": {
-                "value": {"type": "int32"},
+                "value": {
+                    "type": "int32",
+                    "metadata": {
+                        "json_path": "value",
+                        "catalog_type": "NUMBER",
+                        "sampled_field": False,
+                    },
+                },
             },
         },
     )
@@ -224,6 +231,20 @@ source_0 = SqlSource(
     credential=credential_0,
     description=None,
     display_name='sql-source-postgres',
+    jtd_schema={
+        'nullable': False,
+        'optionalProperties': {},
+        'properties': {
+            'value': {
+                'type': 'int32',
+                'metadata': {
+                    'json_path': 'value',
+                    'catalog_type': 'NUMBER',
+                    'sampled_field': False,
+                },
+            },
+        },
+    },
     owner=None,
     priority=None,
     schedule='* * * * *',
@@ -240,7 +261,7 @@ source_0 = SqlSource(
         sources={source.name: source},
     )
 
-    doc = asyncio.run(_import(ctx, DiffContext()))
+    doc = asyncio.run(_import(ctx, DiffContext(), include_schema=True))
 
     assert doc.strip() == expected.strip()