dasl-client 1.0.7__tar.gz → 1.0.11__tar.gz

{dasl_client-1.0.7 → dasl_client-1.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dasl_client
-Version: 1.0.7
+Version: 1.0.11
 Summary: The DASL client library used for interacting with the DASL workspace
 Home-page: https://github.com/antimatter/asl
 Author: Antimatter Team
@@ -8,9 +8,10 @@ Author-email: Antimatter Team <support@antimatter.io>
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: dasl_api==0.1.12
+Requires-Dist: dasl_api==0.1.14
 Requires-Dist: databricks-sdk>=0.41.0
 Requires-Dist: pydantic>=2
+Requires-Dist: typing_extensions==4.10.0
 
 # DASL Client Library
 

{dasl_client-1.0.7 → dasl_client-1.0.11}/dasl_client/auth/auth.py

@@ -1,4 +1,5 @@
 import abc
+import base64
 import time
 from datetime import datetime
 
@@ -176,15 +177,15 @@ class DatabricksSecretAuth(Authorization):
             principalName=self._principal,
         )
         handler = api.WorkspaceV1Api(api_client=self._client)
-        resp = handler.workspace_v1_begin_authentication(
-            workspace=self._workspace, workspace_v1_begin_authenticate_request=req
+        resp = handler.workspace_v1_request_secret(
+            workspace=self._workspace, workspace_v1_request_secret_request=req
         )
         secret_name = resp.secret_name
         secret_value = ""
         for tries in range(3):
             try:
-                secret_value = WorkspaceClient().secrets.get_secret(
-                    secret_name, "token"
+                secret_value = (
+                    WorkspaceClient().secrets.get_secret(secret_name, "token").value
                 )
                 break
             except ResourceDoesNotExist:
@@ -194,7 +195,7 @@ class DatabricksSecretAuth(Authorization):
             raise RuntimeError(f"failed to complete secret auth")
 
         req = WorkspaceV1AuthenticateRequest(
-            databricks_secret=secret_value,
+            databricks_secret=base64.b64decode(secret_value).decode("utf-8"),
         )
         handler = api.WorkspaceV1Api(api_client=self._client)
 

{dasl_client-1.0.7 → dasl_client-1.0.11}/dasl_client/client.py

@@ -3,8 +3,8 @@ from typing import Any, Callable, Iterator, List, Optional, TypeVar
 
 from dasl_api import (
     CoreV1Api,
-    CoreV1DataSourceAutoloaderSpec,
     DbuiV1Api,
+    ContentV1Api,
     WorkspaceV1Api,
     WorkspaceV1CreateWorkspaceRequest,
     api,
@@ -13,6 +13,7 @@ from pydantic import Field
 
 from dasl_client.auth.auth import (
     Authorization,
+    DatabricksSecretAuth,
     DatabricksTokenAuth,
     ServiceAccountKeyAuth,
 )
@@ -29,6 +30,8 @@ from .types import (
     WorkspaceConfig,
     TransformRequest,
     TransformResponse,
+    DataSourcePresetsList,
+    DataSourcePreset,
 )
 
 
@@ -51,7 +54,7 @@ class Client:
 
         :param auth: Authorization instance for authorizing requests to
             the dasl control plane.
-        :returns Client
+        :returns: Client
         """
         self.auth = auth
 
@@ -83,7 +86,7 @@ class Client:
         :param dasl_host: The URL of the DASL server. This value should
             not generally be specified unless you are testing against
             an alternative environment.
-        :returns Client
+        :returns: Client for the newly created workspace.
         """
         with error_handler():
             if workspace_url is None:
@@ -131,7 +134,7 @@ class Client:
         :param dasl_host: The URL of the DASL server. This value should
             not generally be specified unless you are testing against
             an alternative environment.
-        :returns Client
+        :returns: Client for the existing workspace.
         """
         with error_handler():
             if workspace_url is None:
@@ -195,7 +198,7 @@ class Client:
         :param dasl_host: The URL of the DASL server. This value should
             not generally be specified unless you are testing against
             an alternative environment.
-        :returns Client
+        :returns: Client for the newly created or existing workspace.
         """
         try:
             return Client.new_workspace(
@@ -229,6 +232,9 @@ class Client:
     def _dbui_client(self) -> DbuiV1Api:
         return DbuiV1Api(self.auth.client())
 
+    def _content_client(self) -> ContentV1Api:
+        return ContentV1Api(self.auth.client())
+
     def _workspace(self) -> str:
         return self.auth.workspace()
 
@@ -274,7 +280,7 @@ class Client:
         you will need to repopulate the service_principal_secret correctly
         before passing the result back to put_admin_config.
 
-        :returns AdminConfig
+        :returns: AdminConfig containing the current settings.
         """
         with error_handler():
             return AdminConfig.from_api_obj(
@@ -306,6 +312,8 @@ class Client:
         Retrieve the WorkspaceConfig from the DASL server. The returned
         value can be updated directly and passed to put_config in order
         to make changes.
+
+        :returns: WorkspaceConfig containing the current configuration.
         """
         with error_handler():
             return WorkspaceConfig.from_api_obj(
@@ -348,7 +356,7 @@ class Client:
         in order to make changes.
 
         :param name: The unique name of the DataSource within this workspace
-        :returns DataSource
+        :returns: DataSource
         """
         with error_handler():
             return DataSource.from_api_obj(
@@ -459,7 +467,7 @@ class Client:
         in order to make changes.
 
         :param name: The unique name of the Rule within this workspace
-        :returns Rule
+        :returns: Rule
         """
         with error_handler():
             return Rule.from_api_obj(
@@ -601,7 +609,7 @@ class Client:
         :param value: The observable value
         :param cursor: A cursor to be used when paginating results
         :param limit: A limit of the number of results to return
-        :returns EventsList
+        :returns: EventsList
         """
         with error_handler():
             return Dbui.ObservableEvents.EventsList.from_api_obj(
@@ -614,3 +622,31 @@ class Client:
                     limit=limit,
                 )
             )
+
+    def list_presets(self) -> DataSourcePresetsList:
+        """
+        List the Presets in this workspace. This will include any user defined
+        presets if a custom presets path has been configured in the workspace.
+
+        :returns: DataSourcePresetsList
+        """
+        with error_handler():
+            return DataSourcePresetsList.from_api_obj(
+                self._content_client().content_v1_get_preset_data_sources(
+                    self._workspace(),
+                )
+            )
+
+    def get_preset(self, name: str) -> DataSourcePreset:
+        """
+        Get the preset with the argument name from the DASL server. If the preset name
+        begins with 'internal_' it will instead be collected from the user catalog,
+        provided a preset path is set in the workspace config.
+
+        :param name: The unique name of the DataSource preset within this workspace.
+        :returns: DataSourcePreset
+        """
+        with error_handler():
+            return DataSourcePreset.from_api_obj(
+                self._content_client().content_v1_get_preset_datasource(self._workspace(), name)
+            )

dasl_client-1.0.11/dasl_client/preset_development/__init__.py

@@ -0,0 +1,4 @@
+# dasl_client/preset_development/__init__.py
+from .preview_parameters import *
+from .errors import *
+from .preview_engine import *

dasl_client-1.0.11/dasl_client/preset_development/errors.py

@@ -0,0 +1,159 @@
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.types import *
+from pyspark.sql.dataframe import DataFrame
+from typing import Dict, Any, List, Mapping, Tuple
+from IPython import get_ipython
+
+
+class PresetError(Exception):
+    pass
+
+
+class InvalidGoldTableSchemaError(PresetError):
+    def __init__(self, schema: str, additional_message: str = ""):
+        self.schema = schema
+        message = (
+            f"Malformed gold schema provided {schema}. {additional_message}".strip()
+        )
+        super().__init__(message)
+
+
+class NoSilverStageProvdedError(PresetError):
+    def __init__(self, additional_msg: str = ""):
+        message = f"No silver stage provided{additional_msg}."
+        super().__init__(message)
+
+
+class NoSilverTransformStageProvdedError(PresetError):
+    def __init__(
+        self,
+        message: str = "No silver transform stage provided, but gold stage is present.",
+    ):
+        super().__init__(message)
+
+
+class PreTransformNotFound(PresetError):
+    def __init__(
+        self,
+        message: str = "Requested silver pretransform name not found in preset's silver pretransforms.",
+    ):
+        super().__init__(message)
+
+
+class NoSilverPreTransformStageProvdedError(PresetError):
+    def __init__(
+        self,
+        message: str = "No silver transform stage provided, but prestransform name provided.",
+    ):
+        super().__init__(message)
+
+
+class MissingTableFieldError(PresetError):
+    def __init__(self, layer: str, table_name: str, field_name: str):
+        self.layer = layer
+        self.table_name = table_name
+        self.field_name = field_name
+        message = f"{layer} stage {table_name} is missing {field_name} field."
+        super().__init__(message)
+
+
+class DuplicateFieldNameError(PresetError):
+    def __init__(self, stage: str, stage_name: str, field_name: str):
+        self.stage = stage
+        self.stage_name = stage_name
+        self.field_name = field_name
+        message = f"Duplicate field specification name found in {stage} stage {stage_name} named {field_name}."
+        super().__init__(message)
+
+
+class MalformedFieldError(PresetError):
+    def __init__(self, stage: str, stage_name: str, field_name: str):
+        self.stage = stage
+        self.stage_name = stage_name
+        self.field_name = field_name
+        message = f"Please provide 1 operation only in {stage} stage {stage_name}'s field specification named {field_name}."
+        super().__init__(message)
+
+
+class MissingFieldNameError(PresetError):
+    def __init__(self, stage: str, stage_name: str):
+        self.stage = stage
+        self.stage_name = stage_name
+        message = (
+            f"Field specification in {stage} stage {stage_name} missing name field."
+        )
+        super().__init__(message)
+
+
+class MissingSilverKeysError(PresetError):
+    def __init__(self, missing_keys: str):
+        self.missing_keys = missing_keys
+        message = f"Gold table/s have no corresponding input from silver table/s: {missing_keys}"
+        super().__init__(message)
+
+
+class MissingAutoloaderConfigError(PresetError):
+    def __init__(
+        self,
+        message: str = "Autoloader mode selected, but no autoloader configuration found in preset.autoloader.",
+    ):
+        super().__init__(message)
+
+
+class AutoloaderMissingFieldError(PresetError):
+    def __init__(self, field_name: str):
+        self.field_name = field_name
+        message = f"Autoloader mode selected, but missing field {field_name} in preset."
+        super().__init__(message)
+
+
+class UnknownGoldTableError(PresetError):
+    def __init__(self, table_name: str, schema: str):
+        self.table_name = table_name
+        self.schema = schema
+        message = (
+            f"The referenced Gold table name {table_name} does not exist in {schema}."
+        )
+        super().__init__(message)
+
+
+class GoldTableCompatibilityError(PresetError):
+    def __init__(self, message: str):
+        super().__init__(message)
+
+
+class ReferencedColumnMissingError(PresetError):
+    def __init__(self, operation: str, column_name: str):
+        self.operation = operation
+        self.column_name = column_name
+        message = f"The referenced column {column_name} was not found in the dataframe during {operation} operation."
+        super().__init__(message)
+
+
+class MissingJoinFieldError(PresetError):
+    def __init__(self, field_name: str):
+        self.field_name = field_name
+        message = f"Join operation is missing required field {field_name}."
+        super().__init__(message)
+
+
+class MissingUtilityConfigurationFieldError(PresetError):
+    def __init__(self, operation: str, field_name: str):
+        self.operation = operation
+        self.field_name = field_name
+        message = f"The required configuration field {field_name} was not suppled in the {operation} operation."
+        super().__init__(message)
+
+
+class AssertionFailedError(PresetError):
+    def __init__(self, expr: str, assertion_message: str, df: DataFrame):
+        # Get the Databricks built-in functions out the namespace.
+        ipython = get_ipython()
+        display = ipython.user_ns["display"]
+
+        self.expr = expr
+        self.assertion_message = assertion_message
+        self.df = df
+        message = f"The above rows failed the assertion expression {expr} with reason: {assertion_message}\n"
+        display(df)
+        super().__init__(message)