dasl-client 1.0.24__tar.gz → 1.0.26__tar.gz

{dasl_client-1.0.24 → dasl_client-1.0.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dasl_client
-Version: 1.0.24
+Version: 1.0.26
 Summary: The DASL client library used for interacting with the DASL workspace
 Home-page: https://github.com/antimatter/asl
 Author: Antimatter Team
@@ -8,10 +8,11 @@ Author-email: Antimatter Team <support@antimatter.io>
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: dasl_api==0.1.24
+Requires-Dist: dasl_api==0.1.25
 Requires-Dist: databricks-sdk>=0.41.0
 Requires-Dist: pydantic>=2
 Requires-Dist: typing_extensions>=4.10.0
+Requires-Dist: pyyaml==6.0.2
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{dasl_client-1.0.24 → dasl_client-1.0.26}/dasl_client/client.py

@@ -8,6 +8,8 @@ from pyspark.sql import DataFrame
 from dasl_api import (
     CoreV1Api,
     DbuiV1Api,
+    DbuiV1QueryExtendRequest,
+    DbuiV1QueryExtendRequestTimeRange,
     DbuiV1QueryGenerateRequest,
     DbuiV1QueryGenerateRequestTimeRange,
     DbuiV1QueryGenerateStatus,
@@ -597,7 +599,7 @@ class Client:
     def exec_rule(
         self,
         spark,
-        rule_in: Rule,
+        rule_in: Rule | str,
     ) -> ExecRule:
         """
         Locally execute a Rule. Must be run from within a Databricks
@@ -607,19 +609,25 @@ class Client:
         :param spark: Spark context from Databricks notebook. Will be
             injected into the execution environment for use by the
             Rule notebook.
-        :param rule_in: The specification of the Rule to execute.
+        :param rule_in:
+            The specification of the Rule to execute. If specified as
+            a string, it should be in YAML format.
         :returns ExecRule: A class containing various information and
             functionality relating to the execution. See the docs for
             ExecRule for additional details, but note that you must
             call its cleanup function or tables created just for this
             request will leak.
         """
+        rule = rule_in
+        if isinstance(rule_in, str):
+            rule = Rule.from_yaml_str(rule_in)
+
         Helpers.ensure_databricks()
 
         with error_handler():
             result = self._core_client().core_v1_render_rule(
                 self._workspace(),
-                rule_in.to_api_obj(),
+                rule.to_api_obj(),
             )
 
             try:
@@ -794,6 +802,60 @@ class Client:
                 .id
             )
 
+    def extend_query(
+        self,
+        id: str,
+        warehouse: Optional[str] = None,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+    ) -> str:
+        """
+        Extend an existing query to cover a larger time range . If the query
+        is ordered by time and contains no aggregations, this will add the
+        additional data to the existing underlying query, returning the
+        existing ID. If the existing table cannot be extended, a new table
+        will be created to cover the updated time range.
+
+        :param id: The ID of the query to extend.
+        :param warehouse: The SQL warehouse use to execute the SQL. If
+            omitted, the default SQL warehouse specified in the workspace
+            config will be used.
+        :param start_date: An optional starting date to extend the existing
+            query by. If not provided, the current start date of the query
+            will be used.
+        :param end_date: An optional end date to extend the existing
+            query by. If not provided, the current end date of the query
+            will be used.
+        :returns str: The ID of the query generation operation. This value
+            can be used with get_query_status to track the progress of
+            the generation process, and eventually to perform lookups
+            on the completed query. If the current query could be extended,
+            this id will be the same as the one provided. If a new query had
+            to be generated, the new ID is returned.
+        """
+        time_range = None
+        if start_date is not None or end_date is not None:
+            time_range = DbuiV1QueryExtendRequestTimeRange(
+                startDate=start_date,
+                endDate=end_date,
+            )
+
+        req = DbuiV1QueryExtendRequest(
+            warehouse=warehouse,
+            timeRange=time_range,
+        )
+
+        with error_handler():
+            return (
+                self._dbui_client()
+                .dbui_v1_query_extend(
+                    self._workspace(),
+                    id,
+                    req,
+                )
+                .id
+            )
+
     def get_query_status(
         self,
         id: str,

{dasl_client-1.0.24 → dasl_client-1.0.26}/dasl_client/conn/conn.py

@@ -19,7 +19,9 @@ def get_base_conn(enable_retries: bool = True, host: Optional[str] = None) -> Ap
     :return: An API conn without any auth
     """
     if host is None:
-        host = os.getenv("DASL_API_URL", "https://api.prod.sl.antimatter.io")
+        host = os.getenv(
+            "DASL_API_URL", "https://api.sl.us-east-1.cloud.databricks.com"
+        )
     config = Configuration(host=host)
     if enable_retries:
         # configure retries with backup for all HTTP verbs; we do not limit this to only

{dasl_client-1.0.24 → dasl_client-1.0.26}/dasl_client/preset_development/errors.py

@@ -9,6 +9,26 @@ class PresetError(Exception):
     pass
 
 
+class StageExecutionException(PresetError):
+    def __init__(
+        self,
+        medallion_layer="unknown",
+        exception_map: Dict[str, List[str]] = {},
+        verbose: bool = False,
+    ):
+        self.exception_map = exception_map
+        message = (
+            f"Field specification errors encountered in {medallion_layer} stage.\n\n"
+        )
+        for table, exceptions in exception_map.items():
+            message += f"Table: {table}\n"
+            count = 1
+            for exception in exceptions:
+                message += f"Exception {count}:\n{exception.split('JVM')[0] if not verbose else exception}\n\n"
+                count += 1
+        super().__init__(message)
+
+
 class InvalidGoldTableSchemaError(PresetError):
     def __init__(self, schema: str, additional_message: str = ""):
         self.schema = schema

{dasl_client-1.0.24 → dasl_client-1.0.26}/dasl_client/preset_development/preview_engine.py

@@ -49,6 +49,7 @@ class PreviewEngine:
         """
         self._spark = spark
         self._ds_params = ds_params
+        self.__stage_exception = {}
         self._preset = yaml.safe_load(preset_yaml_str)
         self._pretransform_name = ds_params._pretransform_name
 
@@ -129,7 +130,7 @@ class PreviewEngine:
         if missing_keys:
             raise MissingSilverKeysError(missing_keys)
 
-    def _compile_stages(self) -> None:
+    def _compile_stages(self, force_evaluation: bool = False) -> None:
         """
         Creates Stage objects, setting silver pretransform to None if not provided.
         """
@@ -160,15 +161,21 @@ class PreviewEngine:
                     break
 
         self._silver = [
-            Stage(self._spark, "silver transform", table)
+            Stage(
+                self._spark,
+                "silver transform",
+                table,
+                force_evaluation=force_evaluation,
+            )
             for table in self._preset.get("silver", {}).get("transform", [])
         ]
         self._gold = [
-            Stage(self._spark, "gold", table) for table in self._preset.get("gold", [])
+            Stage(self._spark, "gold", table, force_evaluation=force_evaluation)
+            for table in self._preset.get("gold", [])
         ]
 
     def _run(
-        self, df: DataFrame
+        self, df: DataFrame, verbose: bool = False
     ) -> Tuple[DataFrame, Dict[str, DataFrame], Dict[str, DataFrame]]:
         """
         Runs all stages, in medallion stage order. This allows prior stage outputs to feed
@@ -232,6 +239,14 @@ class PreviewEngine:
         for table in self._silver:
             silver_output_map[table._name] = table.run(df)
 
+        # Check for silver stage exceptions.
+        # NOTE: These exception lists only get populated if force_evaluation is enabled.
+        for table in self._silver:
+            if exceptions := table.get_exceptions():
+                self.__stage_exception[table._name] = exceptions
+        if self.__stage_exception:
+            raise StageExecutionException("silver", self.__stage_exception, verbose)
+
         gold_output_map = {}
         for table in self._gold:
             # We store as gold_name/silver_input to prevent clobbering on duplicate gold table use.
@@ -239,12 +254,92 @@ class PreviewEngine:
                 silver_output_map[table._input]
             )
 
+        # Check for gold stage exceptions.
+        # NOTE: These exception lists only get populated if force_evaluation is enabled.
+        for table in self._gold:
+            if exceptions := table.get_exceptions():
+                self.__stage_exception[table._name] = exceptions
+        if self.__stage_exception:
+            raise StageExecutionException("gold", self.__stage_exception, verbose)
+
         return (
             (df, silver_output_map, gold_output_map, pre_bronze_output)
             if self._pre_silver
             else (None, silver_output_map, gold_output_map, pre_bronze_output)
         )
 
+    def __get_sql_type(self, data_type) -> str:
+        """
+        Helper to convert Spark data type objects to SQL type strings.
+        """
+        if isinstance(data_type, StringType):
+            return "STRING"
+        elif isinstance(data_type, IntegerType):
+            return "INT"
+        elif isinstance(data_type, LongType):
+            return "BIGINT"
+        elif isinstance(data_type, FloatType):
+            return "FLOAT"
+        elif isinstance(data_type, DoubleType):
+            return "DOUBLE"
+        elif isinstance(data_type, BooleanType):
+            return "BOOLEAN"
+        elif isinstance(data_type, TimestampType):
+            return "TIMESTAMP"
+        elif isinstance(data_type, DateType):
+            return "DATE"
+        elif isinstance(data_type, ArrayType):
+            return f"ARRAY<{self.__get_sql_type(data_type.elementType)}>"
+        elif isinstance(data_type, MapType):
+            return f"MAP<{self.__get_sql_type(data_type.keyType)}, {self.__get_sql_type(data_type.valueType)}>"
+        elif isinstance(data_type, StructType):
+            fields = ", ".join(
+                [
+                    f"{field.name}: {self.__get_sql_type(field.dataType)}"
+                    for field in data_type.fields
+                ]
+            )
+            return f"STRUCT<{fields}>"
+        elif isinstance(data_type, VariantType):
+            return f"VARIANT"
+        else:
+            return f"UNKNOWN ({data_type})"
+
+    def __format_gold_column_merge_exception(
+        self,
+        columns: Dict[str, List[Exception]],
+        gold_df: DataFrame,
+        verbose: bool = False,
+    ):
+        """
+        Formatter for various exceptions that occur during the merge of gold tables.
+        """
+        missing_column_flag = False
+        for column, info in columns.items():
+            # RANT: it is annoying, but basically every exception comes back from the
+            # query analyzer as pyspark.errors.exceptions.connect.AnalysisException,
+            # so we are forced into this awkward string search.
+            str_e = str(info["exception"])
+            str_e = str_e.split("JVM")[0] if not verbose else str_e
+            if "LEGACY_ERROR_TEMP_DELTA_0007" in str_e:
+                print(
+                    f"-> Column \"{column}\" of type \"{self.__get_sql_type(info['type'])}\" does not exist in gold table \"{info['table']}\"."
+                )
+                missing_column_flag = True
+            elif "DELTA_FAILED_TO_MERGE_FIELDS" in str_e:
+                print(
+                    f"-> Column \"{column}\" of type \"{self.__get_sql_type(info['type'])}\" is not compatiable with gold table \"{info['table']}\"'s \"{column}\" of type \"{self.__get_sql_type(gold_df.schema[column].dataType)}\""
+                )
+            else:
+                print(
+                    f"-> Column \"{column}\" raised the following unformatted exception when appending to gold table \"{info['table']}\":\n{str_e}"
+                )
+
+        if missing_column_flag:
+            print(
+                f"\nA write to 1 or more non-existent columns occured - available columns are: {', '.join(gold_df.columns)}"
+            )
+
     def _render_output(
         self,
         input_df: DataFrame,
@@ -253,6 +348,7 @@ class PreviewEngine:
         ],
         gold_table_catalog: str,
         gold_table_schema: str,
+        verbose: bool = False,
     ) -> None:
         """
         Displays formatted HTML output from executed Stages' DataFrames.
@@ -278,31 +374,6 @@ class PreviewEngine:
                 """
             )
 
-        def check_struct_compatibility(
-            target_field: StructField, df_field: StructField, prefix=""
-        ):
-            if not (
-                isinstance(target_field.dataType, StructType)
-                and isinstance(df_field.dataType, StructType)
-            ):
-                return
-
-            target_fields = {
-                field.name: field for field in target_field.dataType.fields
-            }
-            for field in df_field.dataType.fields:
-                if field.name not in target_fields:
-                    raise GoldTableCompatibilityError(
-                        f"Extra field found in gold stage output STRUCT column {prefix}{target_field.name}: {field.name}"
-                    )
-                else:
-                    if isinstance(field.dataType, StructType):
-                        check_struct_compatibility(
-                            target_fields[field.name],
-                            field,
-                            prefix=prefix + target_field.name + ".",
-                        )
-
         (pre_silver, silver, gold, pre_bronze) = stage_dataframes
         d("Autoloader Input", 1)
         display(input_df)
@@ -343,17 +414,33 @@ class PreviewEngine:
             self._ds_params.add_gold_schema_table(full_name)
 
             # Perform the type checks by trying to insert data into the table
-            try:
-                df.write.mode("append").save(
-                    f"{self._ds_params.get_autoloader_temp_schema_location()}/{full_name}"
-                )
-            except Exception as e:
-                raise GoldTableCompatibilityError(
-                    f"Preset gold table '{full_name}' did not match the gold schema for {fqn_gold_table_name}: {repr(e)}"
-                )
 
-            d("Resultant gold table preview", 3)
-            display(df)
+            df_columns = df.columns
+            df_single_columns = {}
+            df_append_exceptions = {}
+            for column in df_columns:
+                df_single_columns[column] = df.select(column)
+            for column, df_single_column in df_single_columns.items():
+                try:
+                    df_single_column.write.mode("append").save(
+                        f"{self._ds_params.get_autoloader_temp_schema_location()}/{full_name}"
+                    )
+                except Exception as e:
+                    df_append_exceptions[column] = {
+                        "type": df_single_column.schema[column].dataType,
+                        "exception": e,
+                        "table": name,
+                    }
+
+            self.__format_gold_column_merge_exception(
+                df_append_exceptions, delta_df, verbose
+            )
+
+            if not df_append_exceptions:
+                # alls good. display the output.
+                d("Resultant gold table preview", 3)
+                unioned_df = delta_df.unionByName(df, allowMissingColumns=True)
+                display(unioned_df)
 
     def is_backtick_escaped(self, name: str) -> bool:
         """
@@ -374,7 +461,13 @@ class PreviewEngine:
             return name
         return f"`{name}`"
 
-    def evaluate(self, gold_table_schema: str, display: bool = True) -> None:
+    def evaluate(
+        self,
+        gold_table_schema: str,
+        display: bool = True,
+        force_evaluation: bool = False,
+        verbose: bool = False,
+    ) -> None:
         """
         Evaluates the loaded preset YAML using the input datasource configuration to load
         records. Finally, checks that the output from the Gold stages is compatible with
@@ -429,16 +522,17 @@ class PreviewEngine:
                         schema_hints_file
                     )
 
-        self._compile_stages()
+        self._compile_stages(force_evaluation=force_evaluation)
 
         with self._ds_params as df:
-            self._result_df_map = self._run(df)
+            self._result_df_map = self._run(df, verbose)
             if display:
                 self._render_output(
                     df,
                     self._result_df_map,
                     self.force_apply_backticks(catalog_name),
                     self.force_apply_backticks(schema_name),
+                    verbose,
                 )
 
     def results(

{dasl_client-1.0.24 → dasl_client-1.0.26}/dasl_client/preset_development/stage.py

@@ -98,7 +98,13 @@ class Stage:
                         "Malformed column name referenced",
                     )
 
-    def __init__(self, spark: SparkSession, stage: str, table: Dict[str, any]):
+    def __init__(
+        self,
+        spark: SparkSession,
+        stage: str,
+        table: Dict[str, any],
+        force_evaluation: bool = False,
+    ):
         """
         Initializes a Stage object that encapsulates all operations required for a single
         table within a stage.
@@ -115,11 +121,13 @@ class Stage:
         """
         self._spark = spark
         self._stage = stage
+        self.__force_evaluation = force_evaluation
         self._name = table.get("name", "")
         self._filter = table.get("filter", "")
         self._postFilter = table.get("postFilter", "")
         self._utils = table.get("utils", {})
         self._input = table.get("input", None)
+        self.__exceptions = []
 
         # The dasl_id does not exist before bronze or when dealing with temp fields.
         fields = (
@@ -134,6 +142,12 @@ class Stage:
 
         self.__validate_field_spec(self._fields, self._stage)
 
+    def get_exceptions(self) -> List[str]:
+        """
+        Get the list of exceptions encountered during field spec evaluation.
+        """
+        return self.__exceptions
+
     def _referenced_columns(self) -> List[str]:
         """
         Get a list of columns referenced in the table's field specifications.
@@ -569,7 +583,14 @@ class Stage:
             if field.get("join", None):
                 joins_cols += [field["name"]]
 
-        return df.selectExpr(select_fields + joins_cols)
+        expressions = select_fields + joins_cols
+        if self.__force_evaluation:
+            for expression in expressions:
+                try:
+                    df.selectExpr(expression).collect()  # Collect to make unlazy.
+                except Exception as e:
+                    self.__exceptions += [f"expression: {expression}\n{str(e)}"]
+        return df.selectExpr(expressions)
 
     def run_joins(self, df: DataFrame) -> DataFrame:
         """

dasl_client-1.0.26/dasl_client/regions.json

@@ -0,0 +1,3 @@
+{
+	"us-east-1": "https://api.sl.us-east-1.cloud.databricks.com"
+}

{dasl_client-1.0.24 → dasl_client-1.0.26}/dasl_client/regions.py

@@ -9,7 +9,7 @@ class Regions:
     @staticmethod
     def lookup(name: str) -> str:
         try:
-            return _data(name)
+            return _data[name]
         except KeyError as e:
             raise ValueError(f"unknown region {name}") from e
 

{dasl_client-1.0.24 → dasl_client-1.0.26}/dasl_client/types/datasource.py

@@ -1201,6 +1201,7 @@ class DataSource(BaseModel):
             The name of the originator of the data.
         source_type (Optional[str]):
             The type of data being imported.
+        epoch (Optional[int]):
         schedule (Optional[Schedule]):
             The schedule for data ingestion.
         custom (Optional[DataSource.CustomNotebook]):
@@ -1353,6 +1354,7 @@ class DataSource(BaseModel):
     metadata: Optional[Metadata] = None
     source: Optional[str] = None
     source_type: Optional[str] = None
+    epoch: Optional[int] = None
     schedule: Optional[Schedule] = None
     custom: Optional["DataSource.CustomNotebook"] = None
     primary_key: Optional["DataSource.PrimaryKey"] = None
@@ -1371,6 +1373,7 @@ class DataSource(BaseModel):
             metadata=Metadata.from_api_obj(obj.metadata),
             source=obj.spec.source,
             source_type=obj.spec.source_type,
+            epoch=obj.spec.epoch,
             schedule=Schedule.from_api_obj(obj.spec.schedule),
             custom=DataSource.CustomNotebook.from_api_obj(obj.spec.custom),
             primary_key=DataSource.PrimaryKey.from_api_obj(obj.spec.primary_key),
@@ -1393,6 +1396,7 @@ class DataSource(BaseModel):
             spec=CoreV1DataSourceSpec(
                 source=self.source,
                 source_type=self.source_type,
+                epoch=self.epoch,
                 schedule=Helpers.maybe(to_api_obj, self.schedule),
                 custom=Helpers.maybe(to_api_obj, self.custom),
                 primary_key=Helpers.maybe(to_api_obj, self.primary_key),

{dasl_client-1.0.24 → dasl_client-1.0.26}/dasl_client/types/rule.py

@@ -1,6 +1,7 @@
 from pydantic import BaseModel
-from typing import Dict, List, Optional, Union
+from typing import Dict, Iterable, List, Optional, Union
 from datetime import datetime, timezone
+import yaml
 
 from dasl_api import (
     CoreV1Rule,
@@ -37,6 +38,8 @@ class Rule(BaseModel):
             The rule configuration metadata.
         schedule (Schedule):
             The rule schedule.
+        compute_mode (Optional[str]):
+            The compute mode to use for this rule's job.
         input (Rule.Input):
             The rule input configuration.
         observables (Optional[List[Rule.Observable]]):
@@ -613,10 +616,12 @@ class Rule(BaseModel):
         Attributes:
             summary (Optional[str]):
             context (Optional[Dict[str, str]]):
+            default_context (Optional[bool]):
         """
 
         summary: Optional[str] = None
         context: Optional[Dict[str, str]] = None
+        default_context: Optional[bool] = False
 
         @staticmethod
         def from_api_obj(obj: Optional[CoreV1RuleSpecOutput]) -> "Rule.Output":
@@ -625,12 +630,14 @@ class Rule(BaseModel):
             return Rule.Output(
                 summary=obj.summary,
                 context=obj.context,
+                default_context=obj.default_context,
             )
 
         def to_api_obj(self) -> CoreV1RuleSpecOutput:
             return CoreV1RuleSpecOutput(
                 summary=self.summary,
                 context=self.context,
+                default_context=self.default_context,
             )
 
     class Collate(BaseModel):
@@ -673,6 +680,7 @@ class Rule(BaseModel):
     metadata: Optional[Metadata] = None
     rule_metadata: Optional["Rule.RuleMetadata"] = None
     schedule: Schedule
+    compute_mode: Optional[str] = None
     input: "Rule.Input"
     observables: Optional[List["Rule.Observable"]] = None
     output: "Rule.Output"
@@ -690,6 +698,7 @@ class Rule(BaseModel):
             metadata=Metadata.from_api_obj(obj.metadata),
             rule_metadata=Rule.RuleMetadata.from_api_obj(obj.spec.metadata),
             schedule=Schedule.from_api_obj(obj.spec.schedule),
+            compute_mode=obj.spec.compute_mode,
             input=Rule.Input.from_api_obj(obj.spec.input),
             observables=observables,
             output=Rule.Output.from_api_obj(obj.spec.output),
@@ -697,6 +706,24 @@ class Rule(BaseModel):
             status=ResourceStatus.from_api_obj(obj.status),
         )
 
+    @staticmethod
+    def from_yaml_str(s: str) -> "Rule":
+        docs = yaml.safe_load_all(s)
+        docs = list(docs)
+
+        if not docs:
+            raise ValueError("YAML is empty")
+        if len(docs) > 1:
+            raise ValueError(f"Expected a single YAML document, got {len(docs)}")
+
+        data = docs[0]
+        if not isinstance(data, dict):
+            raise ValueError(
+                f"Expected a mapping at top-level, got {type(data).__name__}"
+            )
+
+        return Rule.model_validate(data)
+
     def to_api_obj(self) -> CoreV1Rule:
         observables = None
         if self.observables is not None:
@@ -709,6 +736,7 @@ class Rule(BaseModel):
             spec=CoreV1RuleSpec(
                 metadata=Helpers.maybe(to_api_obj, self.rule_metadata),
                 schedule=self.schedule.to_api_obj(),
+                compute_mode=self.compute_mode,
                 input=self.input.to_api_obj(),
                 observables=observables,
                 output=self.output.to_api_obj(),

{dasl_client-1.0.24 → dasl_client-1.0.26}/dasl_client/types/workspace_config.py

@@ -19,6 +19,7 @@ from dasl_api import (
     WorkspaceV1WorkspaceConfigSpecSystemTablesConfig,
     WorkspaceV1DefaultConfig,
     WorkspaceV1DefaultConfigComputeGroupOverridesValue,
+    WorkspaceV1ExportConfigSlackConfigToken,
 )
 
 from .helpers import Helpers
@@ -104,18 +105,53 @@ class ExportConfig(BaseModel):
                 destination=self.destination.to_api_obj(),
             )
 
+    class SlackToken(BaseModel):
+        """
+        Configuration settings for access a slack token.
+
+        Attributes:
+            value (Optional[str]):
+            scope (Optional[str]):
+            key (Optional[str]):
+        """
+
+        value: Optional[str] = None
+        scope: Optional[str] = None
+        key: Optional[str] = None
+
+        @staticmethod
+        def from_api_obj(
+            obj: Optional[WorkspaceV1ExportConfigSlackConfigToken],
+        ) -> "ExportConfig.WebhookDestination":
+            if obj is None:
+                return None
+            return ExportConfig.SlackToken(
+                value=obj.value,
+                scope=obj.scope,
+                key=obj.key,
+            )
+
+        def to_api_obj(self) -> WorkspaceV1ExportConfigSlackConfigToken:
+            return WorkspaceV1ExportConfigSlackConfigToken(
+                value=self.value,
+                scope=self.scope,
+                key=self.key,
+            )
+
     class SlackConfig(BaseModel):
         """
         Configuration settings for exporting to Slack.
 
         Attributes:
-            token (Optional[ExportConfig.WebhookDestination]):
+            token   (Optional[ExportConfig.SlackToken]):
             channel (Optional[str]):
+            url     (Optional[str]):
             message (Optional[str]):
         """
 
-        token: Optional["ExportConfig.WebhookDestination"] = None
+        token: Optional["ExportConfig.SlackToken"] = None
         channel: Optional[str] = None
+        url: Optional[str] = None
         message: Optional[str] = None
 
         @staticmethod
@@ -125,8 +161,9 @@ class ExportConfig(BaseModel):
             if obj is None:
                 return None
             return ExportConfig.SlackConfig(
-                token=ExportConfig.WebhookDestination.from_api_obj(obj.token),
+                token=ExportConfig.SlackToken.from_api_obj(obj.token),
                 channel=obj.channel,
+                url=obj.url,
                 message=obj.message,
             )
 
@@ -137,6 +174,7 @@ class ExportConfig(BaseModel):
             return WorkspaceV1ExportConfigSlackConfig(
                 token=token,
                 channel=self.channel,
+                url=self.url,
                 message=self.message,
             )
 
@@ -301,9 +339,12 @@ class RulesConfig(BaseModel):
         checkpoint_location (Optional[str]):
             The location to store checkpoints for streaming writes. If
             not provided, the daslStoragePath will be used.
+        default_compute_mode (Optional[str]):
+            The default compute mode to use for rule jobs.
     """
 
     checkpoint_location: Optional[str] = None
+    default_compute_mode: Optional[str] = None
 
     @staticmethod
     def from_api_obj(
@@ -314,11 +355,13 @@ class RulesConfig(BaseModel):
 
         return RulesConfig(
             checkpoint_location=obj.checkpoint_location,
+            default_compute_mode=obj.default_compute_mode,
         )
 
     def to_api_obj(self) -> WorkspaceV1WorkspaceConfigSpecRules:
         return WorkspaceV1WorkspaceConfigSpecRules(
             checkpoint_location=self.checkpoint_location,
+            default_compute_mode=self.default_compute_mode,
         )
 
 
@@ -332,18 +375,18 @@ class DefaultConfig(BaseModel):
     for alternatives.
 
     Attributes:
-        datasources (Optional[DefaultConfig.Config]):
+        datasources (Optional[DefaultConfig.ResourceConfig]):
             Configuration that applies to DataSources. May be omitted.
-        transforms (Optional[DefaultConfig.Config]):
+        transforms (Optional[DefaultConfig.ResourceConfig]):
             Configuration that applies to Transforms. May be omitted.
-        rules (Optional[DefaultConfig.Config]):
+        rules (Optional[DefaultConfig.ResourceConfig]):
             Configuration that applies to Rules. May be omitted.
-        var_global (Optional[DefaultConfig.Config]):
+        var_global (Optional[DefaultConfig.ResourceConfig]):
             Configuration that applies globally to resources without a
             resource-specific configuration specified. Must be specified.
     """
 
-    class Config(BaseModel):
+    class ResourceConfig(BaseModel):
         """
         Default configuration for a specific resource type.
 
@@ -364,7 +407,7 @@ class DefaultConfig(BaseModel):
             checkpoint_location (Optional[str]):
                 The location to store checkpoints for streaming writes. If
                 not provided, the daslStoragePath will be used.
-            compute_group_overrides (Optional[Dict[str, DefaultConfig.Config.ComputeGroupOverrides]]):
+            compute_group_overrides (Optional[Dict[str, DefaultConfig.ResourceConfig.ComputeGroupOverrides]]):
                 Overrides for the maximum number of resources that can be
                 placed into a single job, keyed by the compute group name.
         """
@@ -384,9 +427,9 @@ class DefaultConfig(BaseModel):
 
             @staticmethod
             def from_api_obj(
-                obj: Optional["DefaultConfig.Config.ComputeGroupOverrides"],
-            ) -> "DefaultConfig.Config.ComputeGroupOverrides":
-                return DefaultConfig.Config.ComputeGroupOverrides(
+                obj: Optional["DefaultConfig.ResourceConfig.ComputeGroupOverrides"],
+            ) -> "DefaultConfig.ResourceConfig.ComputeGroupOverrides":
+                return DefaultConfig.ResourceConfig.ComputeGroupOverrides(
                     max_resources_per_job=obj.max_resources_per_job,
                 )
 
@@ -403,23 +446,25 @@ class DefaultConfig(BaseModel):
         default_max_resources_per_job: Optional[int] = None
         checkpoint_location: Optional[str] = None
         compute_group_overrides: Optional[
-            Dict[str, "DefaultConfig.Config.ComputeGroupOverrides"]
+            Dict[str, "DefaultConfig.ResourceConfig.ComputeGroupOverrides"]
         ] = None
 
         @staticmethod
         def from_api_obj(
             obj: Optional[WorkspaceV1DefaultConfig],
-        ) -> "DefaultConfig.Config":
+        ) -> "DefaultConfig.ResourceConfig":
             if obj is None:
                 return None
 
             compute_group_overrides = None
             if obj.compute_group_overrides is not None:
                 compute_group_overrides = {
-                    key: DefaultConfig.Config.ComputeGroupOverrides.from_api_obj(value)
+                    key: DefaultConfig.ResourceConfig.ComputeGroupOverrides.from_api_obj(
+                        value
+                    )
                     for key, value in obj.compute_group_overrides.items()
                 }
-            return DefaultConfig.Config(
+            return DefaultConfig.ResourceConfig(
                 notebook_location=obj.notebook_location,
                 bronze_schema=obj.bronze_schema,
                 silver_schema=obj.silver_schema,
@@ -448,10 +493,10 @@ class DefaultConfig(BaseModel):
                 compute_group_overrides=compute_group_overrides,
             )
 
-    datasources: Optional["DefaultConfig.Config"] = None
-    transforms: Optional["DefaultConfig.Config"] = None
-    rules: Optional["DefaultConfig.Config"] = None
-    var_global: Optional["DefaultConfig.Config"] = None
+    datasources: Optional["DefaultConfig.ResourceConfig"] = None
+    transforms: Optional["DefaultConfig.ResourceConfig"] = None
+    rules: Optional["DefaultConfig.ResourceConfig"] = None
+    var_global: Optional["DefaultConfig.ResourceConfig"] = None
 
     @staticmethod
     def from_api_obj(
@@ -461,10 +506,10 @@ class DefaultConfig(BaseModel):
             return None
 
         return DefaultConfig(
-            datasources=DefaultConfig.Config.from_api_obj(obj.datasources),
-            transforms=DefaultConfig.Config.from_api_obj(obj.transforms),
-            rules=DefaultConfig.Config.from_api_obj(obj.rules),
-            var_global=DefaultConfig.Config.from_api_obj(obj.var_global),
+            datasources=DefaultConfig.ResourceConfig.from_api_obj(obj.datasources),
+            transforms=DefaultConfig.ResourceConfig.from_api_obj(obj.transforms),
+            rules=DefaultConfig.ResourceConfig.from_api_obj(obj.rules),
+            var_global=DefaultConfig.ResourceConfig.from_api_obj(obj.var_global),
         )
 
     def to_api_obj(self) -> WorkspaceV1WorkspaceConfigSpecDefaultConfig:

{dasl_client-1.0.24 → dasl_client-1.0.26}/dasl_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dasl_client
-Version: 1.0.24
+Version: 1.0.26
 Summary: The DASL client library used for interacting with the DASL workspace
 Home-page: https://github.com/antimatter/asl
 Author: Antimatter Team
@@ -8,10 +8,11 @@ Author-email: Antimatter Team <support@antimatter.io>
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: dasl_api==0.1.24
+Requires-Dist: dasl_api==0.1.25
 Requires-Dist: databricks-sdk>=0.41.0
 Requires-Dist: pydantic>=2
 Requires-Dist: typing_extensions>=4.10.0
+Requires-Dist: pyyaml==6.0.2
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{dasl_client-1.0.24 → dasl_client-1.0.26}/dasl_client.egg-info/requires.txt

@@ -1,4 +1,5 @@
-dasl_api==0.1.24
+dasl_api==0.1.25
 databricks-sdk>=0.41.0
 pydantic>=2
 typing_extensions>=4.10.0
+pyyaml==6.0.2

{dasl_client-1.0.24 → dasl_client-1.0.26}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dasl_client"
-version = "1.0.24"
+version = "1.0.26"
 description = "The DASL client library used for interacting with the DASL workspace"
 readme = "README.md"
 authors = [
@@ -13,10 +13,11 @@ authors = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "dasl_api==0.1.24",
+    "dasl_api==0.1.25",
     "databricks-sdk>=0.41.0",
     "pydantic>=2",
     "typing_extensions>=4.10.0",
+    "pyyaml==6.0.2",
 ]
 
 [tool.setuptools]

dasl_client-1.0.24/dasl_client/regions.json

@@ -1,3 +0,0 @@
-{
-	"us-east-1": "https://api.prod.sl.antimatter.io"
-}