dasl-client 1.0.28__tar.gz → 1.0.29__tar.gz

{dasl_client-1.0.28 → dasl_client-1.0.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dasl_client
-Version: 1.0.28
+Version: 1.0.29
 Summary: The DASL client library used for interacting with the DASL workspace
 Author-email: Antimatter Team <support@antimatter.io>
 Requires-Python: >=3.8

{dasl_client-1.0.28 → dasl_client-1.0.29}/dasl_client/client.py

@@ -1,24 +1,22 @@
 from copy import deepcopy
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 from time import sleep
-from typing import Any, Callable, Iterator, List, Optional, Tuple, TypeVar
-from pydantic import Field
-from pyspark.sql import DataFrame
+from typing import Any, Callable, Iterator, List, Optional, TypeVar
 
 from dasl_api import (
+    ContentV1Api,
     CoreV1Api,
+    CoreV1QueryExtendRequestDateRange,
     DbuiV1Api,
     DbuiV1QueryExtendRequest,
-    CoreV1QueryExtendRequestDateRange,
     DbuiV1QueryGenerateRequest,
     DbuiV1QueryGenerateRequestTimeRange,
     DbuiV1QueryGenerateStatus,
-    DbuiV1QueryLookupRequest,
-    DbuiV1QueryLookupResult,
     DbuiV1QueryHistogramRequest,
     DbuiV1QueryHistogramResult,
+    DbuiV1QueryLookupRequest,
     DbuiV1QueryLookupRequestPagination,
-    ContentV1Api,
+    DbuiV1QueryLookupResult,
     WorkspaceV1Api,
     WorkspaceV1CreateWorkspaceRequest,
     api,
@@ -26,30 +24,28 @@ from dasl_api import (
 from dasl_client.auth.auth import (
     Authorization,
     DatabricksSecretAuth,
-    DatabricksTokenAuth,
     ServiceAccountKeyAuth,
 )
-from dasl_client.metadata import WorkspaceMetadata
 from dasl_client.conn.conn import get_base_conn
 from dasl_client.errors.errors import ConflictError, error_handler
-from .helpers import Helpers
+from dasl_client.metadata import WorkspaceMetadata
+
 from .exec_rule import ExecRule
+from .helpers import Helpers
 from .regions import Regions
-
 from .types import (
     AdminConfig,
     DataSource,
+    DataSourcePreset,
+    DataSourcePresetsList,
     Dbui,
     Metadata,
     Rule,
-    WorkspaceConfig,
     TransformRequest,
     TransformResponse,
-    DataSourcePresetsList,
-    DataSourcePreset,
+    WorkspaceConfig,
 )
 
-
 T = TypeVar("T")
 
 

{dasl_client-1.0.28 → dasl_client-1.0.29}/dasl_client/errors/errors.py

@@ -3,7 +3,6 @@ from collections.abc import Callable
 from contextlib import contextmanager
 
 from dasl_api import ApiException
-from urllib3.exceptions import MaxRetryError, RequestError
 
 
 class ConflictError(Exception):
@@ -136,3 +135,7 @@ def error_handler():
             raise e
     except Exception as e:
         raise e
+
+
+class WorkspaceLookupError(Exception):
+    """Internal exception wrapper for workspace lookup errors"""

{dasl_client-1.0.28 → dasl_client-1.0.29}/dasl_client/metadata.py

@@ -5,6 +5,8 @@ from dasl_api import ApiClient, Configuration, WorkspaceV1Api
 from dasl_api.models import WorkspaceV1WorkspaceMetadata
 from dasl_api.exceptions import ApiException
 
+from .errors.errors import WorkspaceLookupError
+
 
 class WorkspaceMetadata:
     """Workspace metadata lookup functionality for auto-detecting API endpoints."""
@@ -21,13 +23,36 @@ class WorkspaceMetadata:
         :param dasl_host: Optional DASL host to use for the lookup. If None, uses default region.
         :returns: WorkspaceV1WorkspaceMetadata if successful, None if workspace not found
         """
+        hosts = []
         if dasl_host is None:
             # Use default region for metadata lookup
             from .regions import Regions
-            from .helpers import Helpers
 
-            dasl_host = Regions.lookup(Helpers.default_region)
+            for region in Regions.list():
+                hosts.append(Regions.lookup(region))
+        else:
+            hosts.append(dasl_host)
+
+        last_exception = None
+        for host in hosts:
+            try:
+                metadata = WorkspaceMetadata._get_workspace_metadata(
+                    workspace_url, host
+                )
+                if metadata:
+                    return metadata
+            except WorkspaceLookupError as e:
+                last_exception = e
+                continue
+
+        if last_exception:
+            raise last_exception
+        return None
 
+    @staticmethod
+    def _get_workspace_metadata(
+        workspace_url: str, dasl_host: str
+    ) -> Optional[WorkspaceV1WorkspaceMetadata]:
         try:
             # Create an unauthenticated client for the public metadata endpoint
             configuration = Configuration(host=dasl_host)
@@ -55,10 +80,10 @@ class WorkspaceMetadata:
                 raise ValueError(f"Invalid workspace URL: {workspace_url}")
             else:
                 # Other API errors
-                raise Exception(f"Failed to get workspace metadata: {e}")
+                raise WorkspaceLookupError(f"Failed to get workspace metadata: {e}")
         except Exception as e:
             # Network errors, encoding errors, etc.
-            raise Exception(f"Failed to lookup workspace metadata: {e}")
+            raise WorkspaceLookupError(f"Failed to get workspace metadata: {e}")
 
     @staticmethod
     def get_endpoint_for_workspace(workspace_url: str) -> Optional[str]:

{dasl_client-1.0.28 → dasl_client-1.0.29}/dasl_client/preset_development/preview_engine.py

@@ -2,13 +2,17 @@ from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.types import *
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.functions import lit, col as col_, sum as sum_, when
+
 from dasl_client.preset_development.preview_parameters import *
 from dasl_client.preset_development.stage import *
 from dasl_client.preset_development.errors import *
+
 import yaml
-from IPython import get_ipython
+import os
 from itertools import count
 
+from IPython import get_ipython
+
 
 @udf(StringType())
 def constant_udf(*args):
@@ -362,7 +366,7 @@ class PreviewEngine:
             display = ipython.user_ns["display"]
         else:
             displayHTML = lambda x: print(x)
-            display = lambda x: x.show()
+            display = lambda x, **kwargs: x.show()
 
         def d(txt, lvl) -> None:
             displayHTML(
@@ -376,25 +380,50 @@ class PreviewEngine:
 
         (pre_silver, silver, gold, pre_bronze) = stage_dataframes
         d("Autoloader Input", 1)
-        display(input_df)
+        display(
+            input_df,
+            checkpointLocation=os.path.join(
+                self._ds_params.get_checkpoint_temp_location(), "input"
+            ),
+        )
         d("Bronze Pre-Transform", 1)
         for name, df in pre_bronze.items():
             d(f"{name}", 2)
-            display(df)
+            display(
+                df,
+                checkpointLocation=os.path.join(
+                    self._ds_params.get_checkpoint_temp_location(), f"pre_bronze-{name}"
+                ),
+            )
         d("Silver Pre-Transform", 1)
         if pre_silver:
-            display(pre_silver)
+            display(
+                pre_silver,
+                checkpointLocation=os.path.join(
+                    self._ds_params.get_checkpoint_temp_location(), "pre_silver"
+                ),
+            )
         else:
             d("Skipped", 2)
         d("Silver Transform", 1)
         for name, df in silver.items():
             d(f"{name}", 2)
-            display(df)
+            display(
+                df,
+                checkpointLocation=os.path.join(
+                    self._ds_params.get_checkpoint_temp_location(), f"silver-{name}"
+                ),
+            )
         d("Gold", 1)
         for full_name, df in gold.items():
             d(f"{full_name}", 2)
             d("Stage output", 3)
-            display(df)
+            display(
+                df,
+                checkpointLocation=os.path.join(
+                    self._ds_params.get_checkpoint_temp_location(), f"gold-{full_name}"
+                ),
+            )
 
             # NOTE: Name is stored as Gold_name/Silver_input. So we need to get just the Gold table
             # name that we are comparing the dataframe metadata to.
@@ -440,7 +469,13 @@ class PreviewEngine:
                 # alls good. display the output.
                 d("Resultant gold table preview", 3)
                 unioned_df = delta_df.unionByName(df, allowMissingColumns=True)
-                display(unioned_df)
+                display(
+                    unioned_df,
+                    checkpointLocation=os.path.join(
+                        self._ds_params.get_checkpoint_temp_location(),
+                        f"gold-unioned-{full_name}",
+                    ),
+                )
 
     def is_backtick_escaped(self, name: str) -> bool:
         """

{dasl_client-1.0.28 → dasl_client-1.0.29}/dasl_client/preset_development/preview_parameters.py

@@ -1,13 +1,19 @@
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
 
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.types import *
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.functions import col, lit, udf
 from dasl_client.preset_development.errors import *
+
 import uuid
+import os
+
 from IPython import get_ipython
 
+if TYPE_CHECKING:
+    from dasl_client import Client
+
 
 class PreviewParameters:
     """
@@ -128,13 +134,24 @@ class PreviewParameters:
     ```
 
     **Note:**
-    When using autoloader mode, this implementation requires a location to store a temporary schema for
-    the loaded records. By default, this is set to `"dbfs:/tmp/schemas"`. You can change this using
-    `set_autoloader_temp_schema_location`. Regardless of whether you use the default or a custom path,
-    you must have write permissions for that location.
+    When using autoloader mode, this implementation requires locations to store temporary schemas and
+    checkpoints. By default, these paths are automatically determined from your workspace's
+    `daslStoragePath` configuration:
+    - Schema location: `{daslStoragePath}/preset_preview/schemas`
+    - Checkpoint location: `{daslStoragePath}/preset_preview/checkpoints`
+
+    The workspace configuration is retrieved automatically via `Client.for_workspace()`. If you need
+    to use custom paths or don't have access to the DASL API, you can set them explicitly:
+    ```python
+    ds_params = (PreviewParameters(spark)
+        .set_autoloader_temp_schema_location('/Volumes/catalog/schema/volume/schemas')
+        .set_checkpoint_temp_location_base('/Volumes/catalog/schema/volume/checkpoints'))
+    ```
+
+    Regardless of the paths used, you must have write permissions for those locations.
     """
 
-    def __init__(self, spark: SparkSession) -> None:
+    def __init__(self, spark: SparkSession, client: Optional["Client"] = None) -> None:
         """
         Initializes the PreviewParameters instance with sparse default settings.
 
@@ -142,10 +159,19 @@ class PreviewParameters:
         of records at a time. By default, the record limit is set to 10, but this can be overridden
         if needed.
 
+        Args:
+            spark: SparkSession for DataFrame operations.
+            client: Optional DASL client for retrieving workspace configuration.
+                If not provided and storage paths are not set explicitly,
+                a client will be created automatically via Client.for_workspace().
+
         Instance Attributes:
             mode (str): Indicates the source type ("input" or "autoloader").
             record_limit (int): Maximum number of records to load. Defaults to 10.
             autoloader_temp_schema_location (str): Temporary location to store the autoloader schema.
+                Defaults to {daslStoragePath}/preset_preview/schemas.
+            checkpoint_temp_location_base (str): Temporary location to store checkpoints for stream and display.
+                Defaults to {daslStoragePath}/preset_preview/checkpoints.
             time_column (str): Column name used for time-based filtering.
             start_time (str): Start time for filtering.
             end_time (str): End time for filtering.
@@ -161,10 +187,12 @@ class PreviewParameters:
             df (DataFrame): Internal Spark DataFrame loaded using the specified parameters.
         """
         self._spark = spark
+        self._client = client  # Store client for lazy path resolution
         self._mode = None  # [input, table, autoloader, silverbronze]
         self._record_limit = 10
-        self._autoloader_temp_schema_location = "dbfs:/tmp/schemas"
+        self._autoloader_temp_schema_location = None  # Will be resolved lazily
         self._gold_test_schemas = []
+        self._checkpoint_temp_location_base = None  # Will be resolved lazily
 
         self._time_column = None
         self._start_time = None
@@ -192,6 +220,69 @@ class PreviewParameters:
 
         self._df = None
 
+    def _ensure_storage_paths_configured(self) -> None:
+        """
+        Ensure storage paths are configured, either from explicit user settings
+        or from WorkspaceConfig. Only creates Client if paths are not explicitly set.
+
+        Raises:
+            RuntimeError: If daslStoragePath cannot be determined and paths not set
+        """
+        # If both paths already set explicitly, nothing to do
+        if (
+            self._autoloader_temp_schema_location is not None
+            and self._checkpoint_temp_location_base is not None
+        ):
+            return
+
+        # Need to get daslStoragePath from WorkspaceConfig
+        if self._client is None:
+            # Try to auto-create client
+            try:
+                from dasl_client import Client
+
+                self._client = Client.for_workspace()
+            except Exception as e:
+                raise RuntimeError(
+                    "Could not create DASL client to retrieve workspace configuration. "
+                    "Either provide a client explicitly: PreviewParameters(spark, client=client), "
+                    "or set storage paths manually:\n"
+                    "  .set_autoloader_temp_schema_location('/path/to/schemas')\n"
+                    "  .set_checkpoint_temp_location_base('/path/to/checkpoints')\n"
+                    f"Client creation error: {e}"
+                )
+
+        # Get config and extract daslStoragePath
+        try:
+            config = self._client.get_config()
+            dasl_storage_path = config.dasl_storage_path
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to retrieve workspace configuration: {e}\n"
+                "Set storage paths manually if WorkspaceConfig is not available:\n"
+                "  .set_autoloader_temp_schema_location('/path/to/schemas')\n"
+                "  .set_checkpoint_temp_location_base('/path/to/checkpoints')"
+            )
+
+        if not dasl_storage_path:
+            raise RuntimeError(
+                "WorkspaceConfig.dasl_storage_path is not set. "
+                "Configure this in your workspace settings or set paths explicitly:\n"
+                "  .set_autoloader_temp_schema_location('/path/to/schemas')\n"
+                "  .set_checkpoint_temp_location_base('/path/to/checkpoints')"
+            )
+
+        # Build default paths from daslStoragePath
+        if self._autoloader_temp_schema_location is None:
+            self._autoloader_temp_schema_location = os.path.join(
+                dasl_storage_path, "preset_preview", "schemas"
+            )
+
+        if self._checkpoint_temp_location_base is None:
+            self._checkpoint_temp_location_base = os.path.join(
+                dasl_storage_path, "preset_preview", "checkpoints"
+            )
+
     def __create_from_autoloader(self) -> DataFrame:
         stream_df = (
             self._spark.readStream.format("cloudFiles")
@@ -220,7 +311,10 @@ class PreviewParameters:
                 .option("cloudFiles.inferColumnTypes", "true")
                 .option(
                     "cloudFiles.schemaLocation",
-                    f"{self._autoloader_temp_schema_location}/{self._schema_uuid_str}",
+                    os.path.join(
+                        self.get_autoloader_temp_schema_location(),
+                        self._schema_uuid_str,
+                    ),
                 )
             )
 
@@ -239,6 +333,10 @@ class PreviewParameters:
             stream_df.writeStream.format("memory")
             .queryName("batch_data")
             .trigger(availableNow=True)
+            .option(
+                "checkpointLocation",
+                os.path.join(self.get_checkpoint_temp_location(), "memory"),
+            )
             .start()
         )
 
@@ -294,6 +392,13 @@ class PreviewParameters:
         """
         Cleans up the temporary schema created for streaming mode, if it was created.
         """
+        # Only clean up if paths were actually configured
+        # This handles the case where __exit__ is called after an exception in __enter__
+        if (
+            self._autoloader_temp_schema_location is None
+            or self._checkpoint_temp_location_base is None
+        ):
+            return
 
         # Get the Databricks built-in functions out the namespace.
         ipython = get_ipython()
@@ -301,19 +406,32 @@ class PreviewParameters:
             dbutils = ipython.user_ns["dbutils"]
 
             dbutils.fs.rm(
-                f"{self._autoloader_temp_schema_location}/{self._schema_uuid_str}",
+                os.path.join(
+                    self._autoloader_temp_schema_location, self._schema_uuid_str
+                ),
+                recurse=True,
+            )
+            dbutils.fs.rm(
+                os.path.join(
+                    self._checkpoint_temp_location_base, self._schema_uuid_str
+                ),
                 recurse=True,
             )
             for gold_test_schema in self._gold_test_schemas:
                 dbutils.fs.rm(
-                    f"{self._autoloader_temp_schema_location}/{gold_test_schema}",
+                    os.path.join(
+                        self._autoloader_temp_schema_location, gold_test_schema
+                    ),
                     recurse=True,
                 )
         else:
             leaked_lines = [
-                f"FYI, we are leaking temp data {self._autoloader_temp_schema_location}/{self._schema_uuid_str}",
+                f"FYI, we are leaking temp data {os.path.join(self._autoloader_temp_schema_location, self._schema_uuid_str)}",
+                os.path.join(
+                    self._checkpoint_temp_location_base, self._schema_uuid_str
+                ),
                 *[
-                    f"{self._autoloader_temp_schema_location}/{x}"
+                    os.path.join(self._autoloader_temp_schema_location, x)
                     for x in self._gold_test_schemas
                 ],
             ]
@@ -396,11 +514,56 @@ class PreviewParameters:
         """
         Get the location for the autoloader's streaming mode schema to be created.
 
+        If not explicitly set, defaults to {daslStoragePath}/preset_preview/schemas.
+
         Returns:
              str: The location for the autoloader's streaming mode schema to be created.
+
+        Raises:
+            RuntimeError: If path cannot be determined from WorkspaceConfig
         """
+        self._ensure_storage_paths_configured()
         return self._autoloader_temp_schema_location
 
+    def set_checkpoint_temp_location_base(self, path: str):
+        """
+        Set the base location for the checkpoint to be created. This is
+        deleted at the end of a run.
+
+        Returns:
+            PreviewParameters: The current instance with updated configuration.
+        """
+        self._checkpoint_temp_location_base = path
+        return self
+
+    def get_checkpoint_temp_location_base(self) -> str:
+        """
+        Get the location for the checkpoint to be created.
+
+        If not explicitly set, defaults to {daslStoragePath}/preset_preview/checkpoints.
+
+        Returns:
+             str: The location for the checkpoint to be created.
+
+        Raises:
+            RuntimeError: If path cannot be determined from WorkspaceConfig
+        """
+        self._ensure_storage_paths_configured()
+        return self._checkpoint_temp_location_base
+
+    def get_checkpoint_temp_location(self) -> str:
+        """
+        Get the location where checkpoints to be created.
+
+        Returns:
+             str: The location where checkpoints to be created.
+
+        Raises:
+            RuntimeError: If path cannot be determined from WorkspaceConfig
+        """
+        self._ensure_storage_paths_configured()
+        return os.path.join(self._checkpoint_temp_location_base, self._schema_uuid_str)
+
     def set_data_schema(self, schema: StructType):
         """
         Set the input schema for "input" mode. For example:

dasl_client-1.0.29/dasl_client/regions.json

@@ -0,0 +1,6 @@
+{
+	"aws-us-east-1": "https://api.sl.us-east-1.cloud.databricks.com",
+	"aws-us-west-2": "https://api.sl.us-west-2.cloud.databricks.com",
+	"aws-ap-southeast-2": "https://api.sl.ap-southeast-2.cloud.databricks.com",
+	"aws-eu-central-1": "https://api.sl.eu-central-1.cloud.databricks.com"
+}

{dasl_client-1.0.28 → dasl_client-1.0.29}/dasl_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dasl_client
-Version: 1.0.28
+Version: 1.0.29
 Summary: The DASL client library used for interacting with the DASL workspace
 Author-email: Antimatter Team <support@antimatter.io>
 Requires-Python: >=3.8

{dasl_client-1.0.28 → dasl_client-1.0.29}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dasl_client"
-version = "1.0.28"
+version = "1.0.29"
 description = "The DASL client library used for interacting with the DASL workspace"
 readme = "README.md"
 authors = [

dasl_client-1.0.28/dasl_client/regions.json

@@ -1,4 +0,0 @@
-{
-	"aws-us-east-1": "https://api.sl.us-east-1.cloud.databricks.com",
-	"aws-us-west-2": "https://api.sl.us-west-2.cloud.databricks.com"
-}