dasl-client 1.0.27__py3-none-any.whl → 1.0.29__py3-none-any.whl

dasl_client/client.py

@@ -1,24 +1,22 @@
 from copy import deepcopy
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 from time import sleep
-from typing import Any, Callable, Iterator, List, Optional, Tuple, TypeVar
-from pydantic import Field
-from pyspark.sql import DataFrame
+from typing import Any, Callable, Iterator, List, Optional, TypeVar
 
 from dasl_api import (
+    ContentV1Api,
     CoreV1Api,
+    CoreV1QueryExtendRequestDateRange,
     DbuiV1Api,
     DbuiV1QueryExtendRequest,
-    CoreV1QueryExtendRequestDateRange,
     DbuiV1QueryGenerateRequest,
     DbuiV1QueryGenerateRequestTimeRange,
     DbuiV1QueryGenerateStatus,
-    DbuiV1QueryLookupRequest,
-    DbuiV1QueryLookupResult,
     DbuiV1QueryHistogramRequest,
     DbuiV1QueryHistogramResult,
+    DbuiV1QueryLookupRequest,
     DbuiV1QueryLookupRequestPagination,
-    ContentV1Api,
+    DbuiV1QueryLookupResult,
     WorkspaceV1Api,
     WorkspaceV1CreateWorkspaceRequest,
     api,
@@ -26,29 +24,28 @@ from dasl_api import (
 from dasl_client.auth.auth import (
     Authorization,
     DatabricksSecretAuth,
-    DatabricksTokenAuth,
     ServiceAccountKeyAuth,
 )
 from dasl_client.conn.conn import get_base_conn
 from dasl_client.errors.errors import ConflictError, error_handler
-from .helpers import Helpers
+from dasl_client.metadata import WorkspaceMetadata
+
 from .exec_rule import ExecRule
+from .helpers import Helpers
 from .regions import Regions
-
 from .types import (
     AdminConfig,
     DataSource,
+    DataSourcePreset,
+    DataSourcePresetsList,
     Dbui,
     Metadata,
     Rule,
-    WorkspaceConfig,
     TransformRequest,
     TransformResponse,
-    DataSourcePresetsList,
-    DataSourcePreset,
+    WorkspaceConfig,
 )
 
-
 T = TypeVar("T")
 
 
@@ -79,7 +76,7 @@ class Client:
         service_principal_id: str,
         service_principal_secret: str,
         workspace_url: Optional[str] = None,
-        region: str = Helpers.default_region,
+        region: Optional[str] = None,
         dasl_host: Optional[str] = None,
     ) -> "Client":
         """
@@ -98,18 +95,24 @@ class Client:
             being registered. If you omit this value, it will be inferred
             if you are running within a Databricks notebook. Otherwise, an
             exception will be raised.
-        :param region: The name of the DASL region.
+        :param region: The name of the DASL region. If not specified,
+            the client will auto-detect the region from the workspace
+            URL. For a DASL region, this includes the cloud host, e.g.
+            aws-us-east-1.
         :param dasl_host: The URL of the DASL server. This value should
             not generally be specified. When specified, this value
-            overrides region.
+            overrides both region and auto-detection.
         :returns: Client for the newly created workspace.
         """
-        if dasl_host is None:
-            dasl_host = Regions.lookup(region)
-
         with error_handler():
             if workspace_url is None:
                 workspace_url = Helpers.current_workspace_url()
+
+            # Determine the DASL host to use
+            dasl_host = Client._dasl_host_from_workspace_metadata(
+                workspace_url, dasl_host, region
+            )
+
             admin_config = AdminConfig(
                 workspace_url=workspace_url,
                 app_client_id=app_client_id,
@@ -137,7 +140,7 @@ class Client:
     def for_workspace(
         workspace_url: Optional[str] = None,
         service_account_token: Optional[str] = None,
-        region: str = Helpers.default_region,
+        region: Optional[str] = None,
         dasl_host: Optional[str] = None,
     ) -> "Client":
         """
@@ -151,19 +154,24 @@ class Client:
         :param service_account_token: Antimatter service account token.
             If provided, the client will use this token for auth instead
             of (automatic) secret-based auth.
-        :param region: The name of the DASL region.
+        :param region: The name of the DASL region. If not specified,
+            the client will auto-detect the region from the workspace
+            URL. For a DASL region, this includes the cloud host, e.g.
+            aws-us-east-1.
         :param dasl_host: The URL of the DASL server. This value should
             not generally be specified. When specified, this value
-            overrides region.
+            overrides both region and auto-detection.
         :returns: Client for the existing workspace.
         """
-        if dasl_host is None:
-            dasl_host = Regions.lookup(region)
-
         with error_handler():
             if workspace_url is None:
                 workspace_url = Helpers.current_workspace_url()
 
+            # Determine the DASL host to use
+            dasl_host = Client._dasl_host_from_workspace_metadata(
+                workspace_url, dasl_host, region
+            )
+
             if service_account_token is None:
                 return Client(
                     DatabricksSecretAuth(
@@ -188,7 +196,7 @@ class Client:
         service_principal_secret: str,
         workspace_url: Optional[str] = None,
         service_account_token: Optional[str] = None,
-        region: str = Helpers.default_region,
+        region: Optional[str] = None,
         dasl_host: Optional[str] = None,
     ) -> "Client":
         """
@@ -220,10 +228,13 @@ class Client:
             If provided, the client will use this token for auth instead
             of (automatic) secret-based auth. Ignored if the workspace
             doesn't exist.
-        :param region: The name of the DASL region.
+        :param region: The name of the DASL region. If not specified,
+            the client will auto-detect the region from the workspace
+            URL. For a DASL region, this includes the cloud host, e.g.
+            aws-us-east-1.
         :param dasl_host: The URL of the DASL server. This value should
             not generally be specified. When specified, this value
-            overrides region.
+            overrides both region and auto-detection.
         :returns: Client for the newly created or existing workspace.
         """
         try:
@@ -990,3 +1001,52 @@ class Client:
         """
         with error_handler():
             return self._dbui_client().dbui_v1_query_cancel(self._workspace(), id)
+
+    @staticmethod
+    def _dasl_host_from_workspace_metadata(
+        workspace_url: str, dasl_host: Optional[str], region: Optional[str]
+    ) -> str:
+        """
+        If the dasl_host is already set, it will be returned as-is. If a region
+        is set, this will return the host that region maps to. If neither are
+        set, this gets the DASL host from the workspace URL with a workspace
+        metadata lookup.
+
+        :param workspace_url: The full base URL of the Databricks workspace
+            being registered. If you omit this value, it will be inferred
+            if you are running within a Databricks notebook. Otherwise, an
+            exception will be raised.
+        :param dasl_host: The URL of the DASL server. This value should
+            not generally be specified. When specified, this value
+            overrides both region and auto-detection.
+        :param region: The name of the DASL region. If not specified,
+            the client will auto-detect the region from the workspace
+            URL. For a DASL region, this includes the cloud host, e.g.
+            aws-us-east-1.
+        :return: The DASL host to use.
+        """
+        if dasl_host is None:
+            if region is not None:
+                # Use explicit region
+                dasl_host = Regions.lookup(region)
+            else:
+                # Attempt auto-detection from workspace URL
+                try:
+                    metadata = WorkspaceMetadata.get_workspace_metadata(workspace_url)
+                    if metadata is not None:
+                        dasl_host = metadata.api_url
+                    else:
+                        raise Exception(
+                            f"Could not determine API endpoint for workspace '{workspace_url}'. "
+                            f"The workspace may not be in a supported region. "
+                            f"Please specify 'region' or 'dasl_host' explicitly."
+                        )
+                except Exception as e:
+                    if "Could not determine API endpoint" in str(e):
+                        raise
+                    else:
+                        raise Exception(
+                            f"Failed to auto-detect API endpoint for workspace '{workspace_url}': {e}. "
+                            f"Please specify 'region' or 'dasl_host' explicitly."
+                        )
+        return dasl_host

dasl_client/errors/errors.py

@@ -3,7 +3,6 @@ from collections.abc import Callable
 from contextlib import contextmanager
 
 from dasl_api import ApiException
-from urllib3.exceptions import MaxRetryError, RequestError
 
 
 class ConflictError(Exception):
@@ -136,3 +135,7 @@ def error_handler():
             raise e
     except Exception as e:
         raise e
+
+
+class WorkspaceLookupError(Exception):
+    """Internal exception wrapper for workspace lookup errors"""

dasl_client/metadata.py

@@ -0,0 +1,99 @@
+import base64
+from typing import Optional
+
+from dasl_api import ApiClient, Configuration, WorkspaceV1Api
+from dasl_api.models import WorkspaceV1WorkspaceMetadata
+from dasl_api.exceptions import ApiException
+
+from .errors.errors import WorkspaceLookupError
+
+
+class WorkspaceMetadata:
+    """Workspace metadata lookup functionality for auto-detecting API endpoints."""
+
+    @staticmethod
+    def get_workspace_metadata(
+        workspace_url: str, dasl_host: Optional[str] = None
+    ) -> Optional[WorkspaceV1WorkspaceMetadata]:
+        """
+        Query the workspace metadata endpoint to auto-detect the correct region
+        and API endpoint for a given Databricks workspace.
+
+        :param workspace_url: The Databricks workspace URL to lookup
+        :param dasl_host: Optional DASL host to use for the lookup. If None, uses default region.
+        :returns: WorkspaceV1WorkspaceMetadata if successful, None if workspace not found
+        """
+        hosts = []
+        if dasl_host is None:
+            # Use default region for metadata lookup
+            from .regions import Regions
+
+            for region in Regions.list():
+                hosts.append(Regions.lookup(region))
+        else:
+            hosts.append(dasl_host)
+
+        last_exception = None
+        for host in hosts:
+            try:
+                metadata = WorkspaceMetadata._get_workspace_metadata(
+                    workspace_url, host
+                )
+                if metadata:
+                    return metadata
+            except WorkspaceLookupError as e:
+                last_exception = e
+                continue
+
+        if last_exception:
+            raise last_exception
+        return None
+
+    @staticmethod
+    def _get_workspace_metadata(
+        workspace_url: str, dasl_host: str
+    ) -> Optional[WorkspaceV1WorkspaceMetadata]:
+        try:
+            # Create an unauthenticated client for the public metadata endpoint
+            configuration = Configuration(host=dasl_host)
+            api_client = ApiClient(configuration)
+            workspace_api = WorkspaceV1Api(api_client)
+
+            # Base64 encode the workspace URL
+            encoded_workspace = base64.urlsafe_b64encode(
+                workspace_url.encode()
+            ).decode()
+
+            # Call the metadata endpoint
+            metadata = workspace_api.workspace_v1_get_workspace_metadata(
+                databricks_workspace=encoded_workspace
+            )
+
+            return metadata
+
+        except ApiException as e:
+            if e.status == 404:
+                # Workspace not found or not in supported region
+                return None
+            elif e.status == 400:
+                # Invalid workspace URL
+                raise ValueError(f"Invalid workspace URL: {workspace_url}")
+            else:
+                # Other API errors
+                raise WorkspaceLookupError(f"Failed to get workspace metadata: {e}")
+        except Exception as e:
+            # Network errors, encoding errors, etc.
+            raise WorkspaceLookupError(f"Failed to get workspace metadata: {e}")
+
+    @staticmethod
+    def get_endpoint_for_workspace(workspace_url: str) -> Optional[str]:
+        """
+        Get the API endpoint URL for a workspace.
+
+        :param workspace_url: The Databricks workspace URL
+        :returns: API endpoint URL if successful, None if workspace not found
+        """
+        metadata = WorkspaceMetadata.get_workspace_metadata(workspace_url)
+        if metadata is not None:
+            return metadata.api_url
+        return None

dasl_client/preset_development/preview_engine.py

@@ -2,13 +2,17 @@ from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.types import *
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.functions import lit, col as col_, sum as sum_, when
+
 from dasl_client.preset_development.preview_parameters import *
 from dasl_client.preset_development.stage import *
 from dasl_client.preset_development.errors import *
+
 import yaml
-from IPython import get_ipython
+import os
 from itertools import count
 
+from IPython import get_ipython
+
 
 @udf(StringType())
 def constant_udf(*args):
@@ -362,7 +366,7 @@ class PreviewEngine:
             display = ipython.user_ns["display"]
         else:
             displayHTML = lambda x: print(x)
-            display = lambda x: x.show()
+            display = lambda x, **kwargs: x.show()
 
         def d(txt, lvl) -> None:
             displayHTML(
@@ -376,25 +380,50 @@ class PreviewEngine:
 
         (pre_silver, silver, gold, pre_bronze) = stage_dataframes
         d("Autoloader Input", 1)
-        display(input_df)
+        display(
+            input_df,
+            checkpointLocation=os.path.join(
+                self._ds_params.get_checkpoint_temp_location(), "input"
+            ),
+        )
         d("Bronze Pre-Transform", 1)
         for name, df in pre_bronze.items():
             d(f"{name}", 2)
-            display(df)
+            display(
+                df,
+                checkpointLocation=os.path.join(
+                    self._ds_params.get_checkpoint_temp_location(), f"pre_bronze-{name}"
+                ),
+            )
         d("Silver Pre-Transform", 1)
         if pre_silver:
-            display(pre_silver)
+            display(
+                pre_silver,
+                checkpointLocation=os.path.join(
+                    self._ds_params.get_checkpoint_temp_location(), "pre_silver"
+                ),
+            )
         else:
             d("Skipped", 2)
         d("Silver Transform", 1)
         for name, df in silver.items():
             d(f"{name}", 2)
-            display(df)
+            display(
+                df,
+                checkpointLocation=os.path.join(
+                    self._ds_params.get_checkpoint_temp_location(), f"silver-{name}"
+                ),
+            )
         d("Gold", 1)
         for full_name, df in gold.items():
             d(f"{full_name}", 2)
             d("Stage output", 3)
-            display(df)
+            display(
+                df,
+                checkpointLocation=os.path.join(
+                    self._ds_params.get_checkpoint_temp_location(), f"gold-{full_name}"
+                ),
+            )
 
             # NOTE: Name is stored as Gold_name/Silver_input. So we need to get just the Gold table
             # name that we are comparing the dataframe metadata to.
@@ -440,7 +469,13 @@ class PreviewEngine:
                 # alls good. display the output.
                 d("Resultant gold table preview", 3)
                 unioned_df = delta_df.unionByName(df, allowMissingColumns=True)
-                display(unioned_df)
+                display(
+                    unioned_df,
+                    checkpointLocation=os.path.join(
+                        self._ds_params.get_checkpoint_temp_location(),
+                        f"gold-unioned-{full_name}",
+                    ),
+                )
 
     def is_backtick_escaped(self, name: str) -> bool:
         """

dasl_client/preset_development/preview_parameters.py

@@ -1,13 +1,19 @@
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
 
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.types import *
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.functions import col, lit, udf
 from dasl_client.preset_development.errors import *
+
 import uuid
+import os
+
 from IPython import get_ipython
 
+if TYPE_CHECKING:
+    from dasl_client import Client
+
 
 class PreviewParameters:
     """
@@ -128,13 +134,24 @@ class PreviewParameters:
     ```
 
     **Note:**
-    When using autoloader mode, this implementation requires a location to store a temporary schema for
-    the loaded records. By default, this is set to `"dbfs:/tmp/schemas"`. You can change this using
-    `set_autoloader_temp_schema_location`. Regardless of whether you use the default or a custom path,
-    you must have write permissions for that location.
+    When using autoloader mode, this implementation requires locations to store temporary schemas and
+    checkpoints. By default, these paths are automatically determined from your workspace's
+    `daslStoragePath` configuration:
+    - Schema location: `{daslStoragePath}/preset_preview/schemas`
+    - Checkpoint location: `{daslStoragePath}/preset_preview/checkpoints`
+
+    The workspace configuration is retrieved automatically via `Client.for_workspace()`. If you need
+    to use custom paths or don't have access to the DASL API, you can set them explicitly:
+    ```python
+    ds_params = (PreviewParameters(spark)
+        .set_autoloader_temp_schema_location('/Volumes/catalog/schema/volume/schemas')
+        .set_checkpoint_temp_location_base('/Volumes/catalog/schema/volume/checkpoints'))
+    ```
+
+    Regardless of the paths used, you must have write permissions for those locations.
     """
 
-    def __init__(self, spark: SparkSession) -> None:
+    def __init__(self, spark: SparkSession, client: Optional["Client"] = None) -> None:
         """
         Initializes the PreviewParameters instance with sparse default settings.
 
@@ -142,10 +159,19 @@ class PreviewParameters:
         of records at a time. By default, the record limit is set to 10, but this can be overridden
         if needed.
 
+        Args:
+            spark: SparkSession for DataFrame operations.
+            client: Optional DASL client for retrieving workspace configuration.
+                If not provided and storage paths are not set explicitly,
+                a client will be created automatically via Client.for_workspace().
+
         Instance Attributes:
             mode (str): Indicates the source type ("input" or "autoloader").
             record_limit (int): Maximum number of records to load. Defaults to 10.
             autoloader_temp_schema_location (str): Temporary location to store the autoloader schema.
+                Defaults to {daslStoragePath}/preset_preview/schemas.
+            checkpoint_temp_location_base (str): Temporary location to store checkpoints for stream and display.
+                Defaults to {daslStoragePath}/preset_preview/checkpoints.
             time_column (str): Column name used for time-based filtering.
             start_time (str): Start time for filtering.
             end_time (str): End time for filtering.
@@ -161,10 +187,12 @@ class PreviewParameters:
             df (DataFrame): Internal Spark DataFrame loaded using the specified parameters.
         """
         self._spark = spark
+        self._client = client  # Store client for lazy path resolution
         self._mode = None  # [input, table, autoloader, silverbronze]
         self._record_limit = 10
-        self._autoloader_temp_schema_location = "dbfs:/tmp/schemas"
+        self._autoloader_temp_schema_location = None  # Will be resolved lazily
         self._gold_test_schemas = []
+        self._checkpoint_temp_location_base = None  # Will be resolved lazily
 
         self._time_column = None
         self._start_time = None
@@ -192,6 +220,69 @@ class PreviewParameters:
 
         self._df = None
 
+    def _ensure_storage_paths_configured(self) -> None:
+        """
+        Ensure storage paths are configured, either from explicit user settings
+        or from WorkspaceConfig. Only creates Client if paths are not explicitly set.
+
+        Raises:
+            RuntimeError: If daslStoragePath cannot be determined and paths not set
+        """
+        # If both paths already set explicitly, nothing to do
+        if (
+            self._autoloader_temp_schema_location is not None
+            and self._checkpoint_temp_location_base is not None
+        ):
+            return
+
+        # Need to get daslStoragePath from WorkspaceConfig
+        if self._client is None:
+            # Try to auto-create client
+            try:
+                from dasl_client import Client
+
+                self._client = Client.for_workspace()
+            except Exception as e:
+                raise RuntimeError(
+                    "Could not create DASL client to retrieve workspace configuration. "
+                    "Either provide a client explicitly: PreviewParameters(spark, client=client), "
+                    "or set storage paths manually:\n"
+                    "  .set_autoloader_temp_schema_location('/path/to/schemas')\n"
+                    "  .set_checkpoint_temp_location_base('/path/to/checkpoints')\n"
+                    f"Client creation error: {e}"
+                )
+
+        # Get config and extract daslStoragePath
+        try:
+            config = self._client.get_config()
+            dasl_storage_path = config.dasl_storage_path
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to retrieve workspace configuration: {e}\n"
+                "Set storage paths manually if WorkspaceConfig is not available:\n"
+                "  .set_autoloader_temp_schema_location('/path/to/schemas')\n"
+                "  .set_checkpoint_temp_location_base('/path/to/checkpoints')"
+            )
+
+        if not dasl_storage_path:
+            raise RuntimeError(
+                "WorkspaceConfig.dasl_storage_path is not set. "
+                "Configure this in your workspace settings or set paths explicitly:\n"
+                "  .set_autoloader_temp_schema_location('/path/to/schemas')\n"
+                "  .set_checkpoint_temp_location_base('/path/to/checkpoints')"
+            )
+
+        # Build default paths from daslStoragePath
+        if self._autoloader_temp_schema_location is None:
+            self._autoloader_temp_schema_location = os.path.join(
+                dasl_storage_path, "preset_preview", "schemas"
+            )
+
+        if self._checkpoint_temp_location_base is None:
+            self._checkpoint_temp_location_base = os.path.join(
+                dasl_storage_path, "preset_preview", "checkpoints"
+            )
+
     def __create_from_autoloader(self) -> DataFrame:
         stream_df = (
             self._spark.readStream.format("cloudFiles")
@@ -220,7 +311,10 @@ class PreviewParameters:
                 .option("cloudFiles.inferColumnTypes", "true")
                 .option(
                     "cloudFiles.schemaLocation",
-                    f"{self._autoloader_temp_schema_location}/{self._schema_uuid_str}",
+                    os.path.join(
+                        self.get_autoloader_temp_schema_location(),
+                        self._schema_uuid_str,
+                    ),
                 )
             )
 
@@ -239,6 +333,10 @@ class PreviewParameters:
             stream_df.writeStream.format("memory")
             .queryName("batch_data")
             .trigger(availableNow=True)
+            .option(
+                "checkpointLocation",
+                os.path.join(self.get_checkpoint_temp_location(), "memory"),
+            )
             .start()
         )
 
@@ -294,6 +392,13 @@ class PreviewParameters:
         """
         Cleans up the temporary schema created for streaming mode, if it was created.
         """
+        # Only clean up if paths were actually configured
+        # This handles the case where __exit__ is called after an exception in __enter__
+        if (
+            self._autoloader_temp_schema_location is None
+            or self._checkpoint_temp_location_base is None
+        ):
+            return
 
         # Get the Databricks built-in functions out the namespace.
         ipython = get_ipython()
@@ -301,19 +406,32 @@ class PreviewParameters:
             dbutils = ipython.user_ns["dbutils"]
 
             dbutils.fs.rm(
-                f"{self._autoloader_temp_schema_location}/{self._schema_uuid_str}",
+                os.path.join(
+                    self._autoloader_temp_schema_location, self._schema_uuid_str
+                ),
+                recurse=True,
+            )
+            dbutils.fs.rm(
+                os.path.join(
+                    self._checkpoint_temp_location_base, self._schema_uuid_str
+                ),
                 recurse=True,
             )
             for gold_test_schema in self._gold_test_schemas:
                 dbutils.fs.rm(
-                    f"{self._autoloader_temp_schema_location}/{gold_test_schema}",
+                    os.path.join(
+                        self._autoloader_temp_schema_location, gold_test_schema
+                    ),
                     recurse=True,
                 )
         else:
             leaked_lines = [
-                f"FYI, we are leaking temp data {self._autoloader_temp_schema_location}/{self._schema_uuid_str}",
+                f"FYI, we are leaking temp data {os.path.join(self._autoloader_temp_schema_location, self._schema_uuid_str)}",
+                os.path.join(
+                    self._checkpoint_temp_location_base, self._schema_uuid_str
+                ),
                 *[
-                    f"{self._autoloader_temp_schema_location}/{x}"
+                    os.path.join(self._autoloader_temp_schema_location, x)
                     for x in self._gold_test_schemas
                 ],
             ]
@@ -396,11 +514,56 @@ class PreviewParameters:
         """
         Get the location for the autoloader's streaming mode schema to be created.
 
+        If not explicitly set, defaults to {daslStoragePath}/preset_preview/schemas.
+
         Returns:
              str: The location for the autoloader's streaming mode schema to be created.
+
+        Raises:
+            RuntimeError: If path cannot be determined from WorkspaceConfig
         """
+        self._ensure_storage_paths_configured()
         return self._autoloader_temp_schema_location
 
+    def set_checkpoint_temp_location_base(self, path: str):
+        """
+        Set the base location for the checkpoint to be created. This is
+        deleted at the end of a run.
+
+        Returns:
+            PreviewParameters: The current instance with updated configuration.
+        """
+        self._checkpoint_temp_location_base = path
+        return self
+
+    def get_checkpoint_temp_location_base(self) -> str:
+        """
+        Get the location for the checkpoint to be created.
+
+        If not explicitly set, defaults to {daslStoragePath}/preset_preview/checkpoints.
+
+        Returns:
+             str: The location for the checkpoint to be created.
+
+        Raises:
+            RuntimeError: If path cannot be determined from WorkspaceConfig
+        """
+        self._ensure_storage_paths_configured()
+        return self._checkpoint_temp_location_base
+
+    def get_checkpoint_temp_location(self) -> str:
+        """
+        Get the location where checkpoints to be created.
+
+        Returns:
+             str: The location where checkpoints to be created.
+
+        Raises:
+            RuntimeError: If path cannot be determined from WorkspaceConfig
+        """
+        self._ensure_storage_paths_configured()
+        return os.path.join(self._checkpoint_temp_location_base, self._schema_uuid_str)
+
     def set_data_schema(self, schema: StructType):
         """
         Set the input schema for "input" mode. For example:

dasl_client/regions.json

@@ -1,4 +1,6 @@
 {
 	"aws-us-east-1": "https://api.sl.us-east-1.cloud.databricks.com",
-	"aws-us-west-2": "https://api.sl.us-west-2.cloud.databricks.com"
+	"aws-us-west-2": "https://api.sl.us-west-2.cloud.databricks.com",
+	"aws-ap-southeast-2": "https://api.sl.ap-southeast-2.cloud.databricks.com",
+	"aws-eu-central-1": "https://api.sl.eu-central-1.cloud.databricks.com"
 }

{dasl_client-1.0.27.dist-info → dasl_client-1.0.29.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: dasl_client
-Version: 1.0.27
+Version: 1.0.29
 Summary: The DASL client library used for interacting with the DASL workspace
 Author-email: Antimatter Team <support@antimatter.io>
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: dasl_api==0.1.26
+Requires-Dist: dasl_api==0.1.27
 Requires-Dist: databricks-sdk>=0.41.0
 Requires-Dist: pydantic>=2
 Requires-Dist: typing_extensions>=4.10.0

{dasl_client-1.0.27.dist-info → dasl_client-1.0.29.dist-info}/RECORD

@@ -1,8 +1,9 @@
 dasl_client/__init__.py,sha256=MuaH74tnEHwfymHtlK6GqeMHRRWaSDfS6hSzbMrd7iQ,150
-dasl_client/client.py,sha256=L6Vlzj96CRZ-SeSW9le005IsA52YPcgb6KZBCD29LrI,38983
+dasl_client/client.py,sha256=34P9E2SMQZEMbVo3tyqWR7XfHHnwTPdHpPI88MnTy4Y,42138
 dasl_client/exec_rule.py,sha256=kn-Yo-9L0fjxbulyAghiIKO1SYcqv2XHZn45F8FvUzE,3599
 dasl_client/helpers.py,sha256=kdOoNiyoVzfDHAZ5DGg5YTU4Fj9A5a8gz2RljrY8hbY,1095
-dasl_client/regions.json,sha256=Rvs_gslMuaod_n6LYLLrTAugtr7VNTljnFUiKf_0qNA,137
+dasl_client/metadata.py,sha256=D0DkKfY9fTCKL483Gh9wtCg2h1DKlXuXN2ziwvcjIOo,3553
+dasl_client/regions.json,sha256=1ADGIvA3gS4P_CODULFOaCFEPFCk4gg-LNn_z9V0t_k,287
 dasl_client/regions.py,sha256=1TIlyJ4iMLsfgeb0ShQsfsju2_NBSXnMAfjdSNMQgi8,442
 dasl_client/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dasl_client/auth/auth.py,sha256=yTeijYYpfJVJ_wYyq0U6kAntg4xz5MzIR37_CpVR57k,7277
@@ -10,11 +11,11 @@ dasl_client/conn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 dasl_client/conn/client_identifier.py,sha256=kPrX0wPa6y7ifhKSb6dQriDSlIgPOUhBq7OoC73E7NU,657
 dasl_client/conn/conn.py,sha256=tMUjxsK7MOSOcudbf2iNunbz5gbvWu3AmNXQjRW-vP8,1467
 dasl_client/errors/__init__.py,sha256=lpH2HGF5kCRTk6MxpPEyY9ulTvsLBFKb4NnLuFFLZZA,40
-dasl_client/errors/errors.py,sha256=u-B8dR8zlxdNVeEdHi6UozX178jwJJ5ZJOGl9YjONRc,4008
+dasl_client/errors/errors.py,sha256=mryJNKrsfni0WUHsAgQ7lLCKQtQHGC5sLr1qf8janVM,4055
 dasl_client/preset_development/__init__.py,sha256=9yC4gmQfombvYLThzo0pSfT5JMolfNVWFVQIuIg_XUA,131
 dasl_client/preset_development/errors.py,sha256=KhcMEQ0o58WC9meGSgmGJuE6s9ztkWyn9md0LQ5srWE,7708
-dasl_client/preset_development/preview_engine.py,sha256=yubIjQitXJaQZHA1Mq-q8qMhskYerFGsD7-GQdqoxo8,22086
-dasl_client/preset_development/preview_parameters.py,sha256=aZrpCkB2JrqZCp7Lqb2Tz5DMJOFc0qDMdmDxAI5tv98,20183
+dasl_client/preset_development/preview_engine.py,sha256=HFknqZhGx0Ltuo1Wqmr0EfoiSn126d9q4-WEYX2Ns1U,23243
+dasl_client/preset_development/preview_parameters.py,sha256=JpoVeByweNI_jYTRRWiLkjfaTx_qDSn0ciJlCSn7e60,26871
 dasl_client/preset_development/stage.py,sha256=JLkV6Dmomw-lAWIuJv82z2XvorEOEWZz0GHAr_CMJ9A,27253
 dasl_client/types/__init__.py,sha256=GsXC3eWuv21VTLPLPH9pzM95JByaKnKrPjJkh2rlZfQ,170
 dasl_client/types/admin_config.py,sha256=Kmx3Kuai9_LWMeO2NpWasRUgLihYSEXGtuYVfG0FkjU,2200
@@ -25,8 +26,8 @@ dasl_client/types/helpers.py,sha256=gLGTvrssAKrdkQT9h80twEosld2egwhvj-zAudxWFPs,
 dasl_client/types/rule.py,sha256=5zvRsqzsej5kYTFjIpP1lptb3Rtit06cxvA4sl89OMU,28734
 dasl_client/types/types.py,sha256=DeUOfdYGOhUGEy7yKOfo0OYTXYRrs57yYgNLUbu7Tlc,8806
 dasl_client/types/workspace_config.py,sha256=XkSmeuOA3OPpWQkx7apPcPNuD4Ai2QO3YKU-aFoOW8k,31030
-dasl_client-1.0.27.dist-info/licenses/LICENSE,sha256=M35UepUPyKmFkvENlkweeaMElheQqNoM5Emh8ADO-rs,4
-dasl_client-1.0.27.dist-info/METADATA,sha256=pTORHSM3o9x_3u1J47vEwpzbosz_0Ojv--r45_fMtjI,4046
-dasl_client-1.0.27.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dasl_client-1.0.27.dist-info/top_level.txt,sha256=kIv8ox_2oJPjGB8_yuey5vvuPCyfY8kywG138f9oSOY,12
-dasl_client-1.0.27.dist-info/RECORD,,
+dasl_client-1.0.29.dist-info/licenses/LICENSE,sha256=M35UepUPyKmFkvENlkweeaMElheQqNoM5Emh8ADO-rs,4
+dasl_client-1.0.29.dist-info/METADATA,sha256=-cEDVkDtXcuz5JI9gUby5v0u7nePZOZoa7L4Z2azZoA,4046
+dasl_client-1.0.29.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dasl_client-1.0.29.dist-info/top_level.txt,sha256=kIv8ox_2oJPjGB8_yuey5vvuPCyfY8kywG138f9oSOY,12
+dasl_client-1.0.29.dist-info/RECORD,,