dasl-client 1.0.21__tar.gz → 1.0.23__tar.gz

{dasl_client-1.0.21 → dasl_client-1.0.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dasl_client
-Version: 1.0.21
+Version: 1.0.23
 Summary: The DASL client library used for interacting with the DASL workspace
 Home-page: https://github.com/antimatter/asl
 Author: Antimatter Team

{dasl_client-1.0.21 → dasl_client-1.0.23}/dasl_client/preset_development/preview_engine.py

@@ -240,60 +240,40 @@ class PreviewEngine:
             d(f"{name}", 2)
             display(df)
         d("Gold", 1)
-        for name, df in gold.items():
-            d(f"{name}", 2)
+        for full_name, df in gold.items():
+            d(f"{full_name}", 2)
             d("Stage output", 3)
             display(df)
 
             # NOTE: Name is stored as Gold_name/Silver_input. So we need to get just the Gold table
             # name that we are comparing the dataframe metadata to.
-            name = name.split("/")[0]
+            name = full_name.split("/")[0]
             fqn_gold_table_name = f"{self.force_apply_backticks(gold_table_catalog)}.{self.force_apply_backticks(gold_table_schema)}.{self.force_apply_backticks(name)}"
 
             if not self._spark.catalog.tableExists(f"{fqn_gold_table_name}"):
                 raise UnknownGoldTableError(name, gold_table_schema)
 
-            # Performs the type check.
+            # Create a temporary table to perform the type check
             delta_df = self._spark.table(f"{fqn_gold_table_name}").limit(0)
-            unioned_df = delta_df.unionByName(df, allowMissingColumns=True)
+            delta_df.write.mode("overwrite").save(
+                f"{self._ds_params.get_autoloader_temp_schema_location()}/{full_name}"
+            )
 
-            # Now we check no new columns.
-            if not set(df.columns).issubset(delta_df.columns):
-                raise GoldTableCompatibilityError(
-                    f"Extra columns provided: {', '.join([col for col in df.columns if col not in delta_df.columns])}"
-                )
+            # Update the params to indicate we've added a testing temp gold table
+            self._ds_params.add_gold_schema_table(full_name)
 
-            # Now we check no new fields in STRUCT columns.
-            for field in delta_df.schema.fields:
-                if isinstance(field.dataType, StructType) and field.name in df.columns:
-                    # Retrieve the corresponding field from the DataFrame's schema.
-                    df_field = next(f for f in df.schema.fields if f.name == field.name)
-                    check_struct_compatibility(field, df_field)
-
-            # Check nullable columns exist, and data what we are inserting is set.
-            non_nullable_cols = [
-                field.name for field in delta_df.schema.fields if not field.nullable
-            ]
-            null_checks = [
-                sum_(when(col_(col).isNull(), 1).otherwise(0)).alias(col)
-                for col in non_nullable_cols
-            ]
-            null_counts = df.select(null_checks).collect()[0].asDict()
-            cols_with_nulls = []
+            # Perform the type checks by trying to insert data into the table
             try:
-                cols_with_nulls = [
-                    col_name for col_name, count in null_counts.items() if count > 0
-                ]
-            except TypeError:
-                # There were no records returned and so null_counts == None.
-                pass
-            if cols_with_nulls:
+                df.write.mode("append").save(
+                    f"{self._ds_params.get_autoloader_temp_schema_location()}/{full_name}"
+                )
+            except Exception as e:
                 raise GoldTableCompatibilityError(
-                    f"Record with null data found for non-nullable columns: {', '.join([col for col in cols_with_nulls])}"
+                    f"Preset gold table '{full_name}' did not match the gold schema for {fqn_gold_table_name}: {repr(e)}"
                 )
 
             d("Resultant gold table preview", 3)
-            display(unioned_df)
+            display(df)
 
     def is_backtick_escaped(self, name: str) -> bool:
         """

{dasl_client-1.0.21 → dasl_client-1.0.23}/dasl_client/preset_development/preview_parameters.py

@@ -97,6 +97,7 @@ class PreviewParameters:
         self._mode = None  # [input, autoloader]
         self._record_limit = 10
         self._autoloader_temp_schema_location = "dbfs:/tmp/schemas"
+        self._gold_test_schemas = []
 
         self._time_column = None
         self._start_time = None
@@ -206,10 +207,21 @@ class PreviewParameters:
                 f"{self._autoloader_temp_schema_location}/{self._schema_uuid_str}",
                 recurse=True,
             )
+            for gold_test_schema in self._gold_test_schemas:
+                dbutils.fs.rm(
+                    f"{self._autoloader_temp_schema_location}/{gold_test_schema}",
+                    recurse=True,
+                )
         else:
-            print(
-                f"FYI, we are leaking temp data {self._autoloader_temp_schema_location}/{self._schema_uuid_str}"
-            )
+            leaked_lines = [
+                f"FYI, we are leaking temp data {self._autoloader_temp_schema_location}/{self._schema_uuid_str}",
+                *[
+                    f"{self._autoloader_temp_schema_location}/{x}"
+                    for x in self._gold_test_schemas
+                ],
+            ]
+            print(", ".join(leaked_lines))
+        self._gold_test_schemas = []
 
     def from_input(self):
         """
@@ -253,6 +265,15 @@ class PreviewParameters:
         self._autoloader_temp_schema_location = path
         return self
 
+    def get_autoloader_temp_schema_location(self) -> str:
+        """
+        Get the location for the autoloader's streaming mode schema to be created.
+
+        Returns:
+             str: The location for the autoloader's streaming mode schema to be created.
+        """
+        return self._autoloader_temp_schema_location
+
     def set_data_schema(self, schema: StructType):
         """
         Set the input schema for "input" mode. For example:
@@ -409,3 +430,10 @@ class PreviewParameters:
         """
         self._table = table_name
         return self
+
+    def add_gold_schema_table(self, gold_schema_table_name: str):
+        """
+        Add a gold schema temporary table name that will need to be cleaned
+        up at the end of the run.
+        """
+        self._gold_test_schemas.append(gold_schema_table_name)

{dasl_client-1.0.21 → dasl_client-1.0.23}/dasl_client/types/datasource.py

@@ -1311,6 +1311,7 @@ class DataSource(BaseModel):
     schedule: Optional[Schedule] = None
     custom: Optional["DataSource.CustomNotebook"] = None
     use_preset: Optional[str] = None
+    use_preset_version: Optional[int] = None
     autoloader: Optional["DataSource.Autoloader"] = None
     bronze: Optional[BronzeSpec] = None
     silver: Optional[SilverSpec] = None
@@ -1326,6 +1327,7 @@ class DataSource(BaseModel):
             schedule=Schedule.from_api_obj(obj.spec.schedule),
             custom=DataSource.CustomNotebook.from_api_obj(obj.spec.custom),
             use_preset=obj.spec.use_preset,
+            use_preset_version=obj.spec.use_preset_version,
             autoloader=DataSource.Autoloader.from_api_obj(obj.spec.autoloader),
             bronze=BronzeSpec.from_api_obj(obj.spec.bronze),
             silver=SilverSpec.from_api_obj(obj.spec.silver),
@@ -1345,6 +1347,7 @@ class DataSource(BaseModel):
                 schedule=Helpers.maybe(to_api_obj, self.schedule),
                 custom=Helpers.maybe(to_api_obj, self.custom),
                 use_preset=self.use_preset,
+                use_preset_version=self.use_preset_version,
                 autoloader=Helpers.maybe(to_api_obj, self.autoloader),
                 bronze=Helpers.maybe(to_api_obj, self.bronze),
                 silver=Helpers.maybe(to_api_obj, self.silver),

{dasl_client-1.0.21 → dasl_client-1.0.23}/dasl_client/types/workspace_config.py

@@ -8,12 +8,14 @@ from dasl_api import (
     WorkspaceV1ExportConfigWebhookConfigDestination,
     WorkspaceV1WorkspaceConfig,
     WorkspaceV1WorkspaceConfigSpec,
+    WorkspaceV1WorkspaceConfigSpecDatasources,
     WorkspaceV1WorkspaceConfigSpecDefaultConfig,
     WorkspaceV1WorkspaceConfigSpecDetectionRuleMetadata,
     WorkspaceV1WorkspaceConfigSpecManagedRetentionInner,
     WorkspaceV1WorkspaceConfigSpecManagedRetentionInnerOverridesInner,
     WorkspaceV1WorkspaceConfigSpecObservables,
     WorkspaceV1WorkspaceConfigSpecObservablesKindsInner,
+    WorkspaceV1WorkspaceConfigSpecRules,
     WorkspaceV1WorkspaceConfigSpecSystemTablesConfig,
     WorkspaceV1DefaultConfig,
     WorkspaceV1DefaultConfigComputeGroupOverridesValue,
@@ -238,12 +240,91 @@ class WorkspaceConfigObservables(BaseModel):
         )
 
 
+class DatasourcesConfig(BaseModel):
+    """
+    Configuration settings used by Datasources.
+
+    Attributes:
+        bronze_schema (Optional[str]):
+            Name of the bronze schema in the catalog.
+        silver_schema (Optional[str]):
+            Name of the silver schema in the catalog.
+        gold_schema (Optional[str]):
+            Name of the gold schema in the catalog.
+        catalog_name (Optional[str]):
+            The catalog name to use as the resource's default.
+        checkpoint_location (Optional[str]):
+            The base checkpoint location to use in Rule notebooks.
+    """
+
+    catalog_name: Optional[str] = None
+    bronze_schema: Optional[str] = None
+    silver_schema: Optional[str] = None
+    gold_schema: Optional[str] = None
+    checkpoint_location: Optional[str] = None
+
+    @staticmethod
+    def from_api_obj(
+        obj: Optional[WorkspaceV1WorkspaceConfigSpecDatasources],
+    ) -> Optional["DatasourcesConfig"]:
+        if obj is None:
+            return None
+
+        return DatasourcesConfig(
+            catalog_name=obj.catalog_name,
+            bronze_schema=obj.bronze_schema,
+            silver_schema=obj.silver_schema,
+            gold_schema=obj.gold_schema,
+            checkpoint_location=obj.checkpoint_location,
+        )
+
+    def to_api_obj(self) -> WorkspaceV1WorkspaceConfigSpecDatasources:
+        return WorkspaceV1WorkspaceConfigSpecDatasources(
+            catalog_name=self.catalog_name,
+            bronze_schema=self.bronze_schema,
+            silver_schema=self.silver_schema,
+            gold_schema=self.gold_schema,
+            checkpoint_location=self.checkpoint_location,
+        )
+
+
+class RulesConfig(BaseModel):
+    """
+    Configuration settings used by Rules.
+
+    Attributes:
+        checkpoint_location (Optional[str]):
+            The location to store checkpoints for streaming writes. If
+            not provided, the daslStoragePath will be used.
+    """
+
+    checkpoint_location: Optional[str] = None
+
+    @staticmethod
+    def from_api_obj(
+        obj: Optional[WorkspaceV1WorkspaceConfigSpecRules],
+    ) -> "RulesConfig":
+        if obj is None:
+            return None
+
+        return RulesConfig(
+            checkpoint_location=obj.checkpoint_location,
+        )
+
+    def to_api_obj(self) -> WorkspaceV1WorkspaceConfigSpecRules:
+        return WorkspaceV1WorkspaceConfigSpecRules(
+            checkpoint_location=self.checkpoint_location,
+        )
+
+
 class DefaultConfig(BaseModel):
     """
-    Configuration of the schemas, notebook storage locations, checkpoint
-    storage locations, and so forth, for each concrete resource type and
-    a global fallback that applies to resources which do not have a
-    specified DefaultConfig.
+    (DEPRECATED) Configuration of the schemas, notebook storage locations,
+    checkpoint storage locations, and so forth, for each concrete resource
+    type and a global fallback that applies to resources which do not have a
+    specified DefaultConfig. While it does still work, this field is
+    deprecated and should not be used; see DatasourcesConfig and RulesConfig
+    for alternatives.
 
     Attributes:
         datasources (Optional[DefaultConfig.Config]):
@@ -369,8 +450,11 @@ class DefaultConfig(BaseModel):
 
     @staticmethod
     def from_api_obj(
-        obj: WorkspaceV1WorkspaceConfigSpecDefaultConfig,
-    ) -> "DefaultConfig":
+        obj: Optional[WorkspaceV1WorkspaceConfigSpecDefaultConfig],
+    ) -> Optional["DefaultConfig"]:
+        if obj is None:
+            return None
+
         return DefaultConfig(
             datasources=DefaultConfig.Config.from_api_obj(obj.datasources),
             transforms=DefaultConfig.Config.from_api_obj(obj.transforms),
@@ -579,8 +663,15 @@ class WorkspaceConfig(BaseModel):
         dasl_custom_presets_path (Optional[str]):
             An optional path to a directory containing user defined presets.
         default_config (Optional[DefaultConfig]):
-            Configuration settings regarding storage of bronze, silver, and
-            gold tables and related assets for each resource type.
+            (DEPRECATED) Configuration settings regarding storage of bronze,
+            silver, and gold tables and related assets for each resource type.
+        default_custom_notebook_location (Optional[str]):
+            The storage location for custom user-provided notebooks. Also
+            used as the prefix for relative paths to custom notebooks.
+        datasources (Optional[DatasourcesConfig]):
+            Configuration items that apply specifically to datasources.
+        rules (Optional[RulesConfig]):
+            Configuration items that apply specifically to rules.
         managed_retention (Optional[List[ManagedRetention]]):
             Configuration of regular cleanup (i.e. pruning) jobs for various
             catalogs, schemas, and tables.
@@ -598,6 +689,9 @@ class WorkspaceConfig(BaseModel):
     dasl_storage_path: Optional[str] = None
     dasl_custom_presets_path: Optional[str] = None
     default_config: Optional[DefaultConfig] = None
+    default_custom_notebook_location: Optional[str] = None
+    datasources: Optional[DatasourcesConfig] = None
+    rules: Optional[RulesConfig] = None
     managed_retention: Optional[List[ManagedRetention]] = None
     status: Optional[ResourceStatus] = None
 
@@ -628,6 +722,9 @@ class WorkspaceConfig(BaseModel):
             dasl_storage_path=spec.dasl_storage_path,
             dasl_custom_presets_path=spec.dasl_custom_presets_path,
             default_config=DefaultConfig.from_api_obj(spec.default_config),
+            default_custom_notebook_location=spec.default_custom_notebook_location,
+            datasources=DatasourcesConfig.from_api_obj(spec.datasources),
+            rules=RulesConfig.from_api_obj(spec.rules),
             managed_retention=managed_retention,
             status=ResourceStatus.from_api_obj(obj.status),
         )
@@ -658,6 +755,9 @@ class WorkspaceConfig(BaseModel):
                 dasl_storage_path=self.dasl_storage_path,
                 dasl_custom_presets_path=self.dasl_custom_presets_path,
                 default_config=Helpers.maybe(to_api_obj, self.default_config),
+                default_custom_notebook_location=self.default_custom_notebook_location,
+                datasources=Helpers.maybe(to_api_obj, self.datasources),
+                rules=Helpers.maybe(to_api_obj, self.rules),
                 managed_retention=managed_retention,
             ),
             status=Helpers.maybe(to_api_obj, self.status),

{dasl_client-1.0.21 → dasl_client-1.0.23}/dasl_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dasl_client
-Version: 1.0.21
+Version: 1.0.23
 Summary: The DASL client library used for interacting with the DASL workspace
 Home-page: https://github.com/antimatter/asl
 Author: Antimatter Team

{dasl_client-1.0.21 → dasl_client-1.0.23}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dasl_client"
-version = "1.0.21"
+version = "1.0.23"
 description = "The DASL client library used for interacting with the DASL workspace"
 readme = "README.md"
 authors = [

{dasl_client-1.0.21 → dasl_client-1.0.23}/test/test_api_changes.py

@@ -123,3 +123,15 @@ def test_api_model_for_changes(tpe):
     with open(f"test/expected_api_models/{checked_dasl_types[tpe]}", "r") as f:
         expected_val = f.read()
     assert dasl_model_to_string(tpe) == expected_val
+
+
+@pytest.mark.update
+@pytest.mark.parametrize(
+    "tpe",
+    checked_dasl_types.keys(),
+    ids=[f"updating {tpe.__name__} model" for tpe in checked_dasl_types.keys()],
+)
+def test_apply_api_model_changes(tpe):
+    model_ser = dasl_model_to_string(tpe)
+    with open(f"test/expected_api_models/{checked_dasl_types[tpe]}", "w") as f:
+        f.write(model_ser)

{dasl_client-1.0.21 → dasl_client-1.0.23}/test/test_api_surface.py

@@ -47,13 +47,15 @@ def test_workspace_config(api_client):
             catalog_name="automated_test_cases",
             var_schema="default",
         ),
-        default_config=DefaultConfig(
-            var_global=DefaultConfig.Config(
-                bronze_schema="bronze",
-                silver_schema="silver",
-                gold_schema="gold",
-                catalog_name="automated_test_cases",
-            ),
+        default_custom_notebook_location="/Users/test/notebooks",
+        datasources=DatasourcesConfig(
+            bronze_schema="bronze",
+            silver_schema="silver",
+            gold_schema="gold",
+            catalog_name="automated_test_cases",
+        ),
+        rules=RulesConfig(
+            checkpoint_location="/Users/test/checkpoints",
         ),
     )
 
@@ -69,7 +71,7 @@ def test_workspace_config(api_client):
 
     assert api_client.get_config() == base_workspace_config
 
-    base_workspace_config.default_config.var_global.bronze_schema = "bronze_new"
+    base_workspace_config.datasources.bronze_schema = "bronze_new"
     api_client.put_config(base_workspace_config)
     got = api_client.get_config()
     base_workspace_config.metadata.modified_timestamp = got.metadata.modified_timestamp
@@ -172,7 +174,7 @@ def test_minimal_rule(api_client):
             stream=Rule.Input.Stream(
                 tables=[
                     Rule.Input.Stream.Table(
-                        name="test",
+                        name="automated_test_cases.pylib.test",
                     ),
                 ],
             ),
@@ -223,7 +225,7 @@ def test_minimal_rule(api_client):
     base_rule_2.metadata.resource_status = got_2.metadata.resource_status
     assert api_client.get_rule("test_1") == base_rule_2
 
-    base_rule_2.input.stream.tables[0].name = "test_1"
+    base_rule_2.input.stream.tables[0].name = "databricks_dev.antimatter_meta.test_ip_summaries"
     api_client.replace_rule("test_1", base_rule_2)
 
     got_2 = api_client.get_rule("test_1")
@@ -253,7 +255,7 @@ def test_list_pagination(api_client):
             stream=Rule.Input.Stream(
                 tables=[
                     Rule.Input.Stream.Table(
-                        name="test",
+                        name="automated_test_cases.pylib.test",
                     ),
                 ],
             ),

{dasl_client-1.0.21 → dasl_client-1.0.23}/test/test_databricks_secret_auth.py

@@ -10,6 +10,7 @@ from .constants import *
 
 pylib_volume_path = os.environ["PYLIB_VOLUME_PATH"]
 pylib_wheel_path = os.environ["PYLIB_WHEEL_PATH"]
+api_wheel_path = os.environ["API_WHEEL_PATH"]
 
 
 def test_secret_auth(api_client):
@@ -18,6 +19,8 @@ def test_secret_auth(api_client):
 
     # need to do an API operation using databricks secret auth.
     notebook_data = f"""
+    %pip uninstall -y dasl-client dasl-api
+    %pip install {api_wheel_path}
     %pip install {pylib_wheel_path}
     dbutils.library.restartPython()
     # COMMAND ----------

{dasl_client-1.0.21 → dasl_client-1.0.23}/test/test_marshaling.py

@@ -72,6 +72,17 @@ def test_workspace_config_marshal_unmarshal():
             relationships=["rel1", "rel2"],
         ),
         dasl_storage_path="/random/storage/path",
+        default_custom_notebook_location="/tmp/notebooks",
+        datasources=DatasourcesConfig(
+            catalog_name="test_catalog",
+            bronze_schema="bronze",
+            silver_schema="silver",
+            gold_schema="gold",
+            checkpoint_location="/tmp/checkpoints",
+        ),
+        rules=RulesConfig(
+            checkpoint_location="/tmp/checkpoints",
+        ),
         default_config=DefaultConfig(
             datasources=DefaultConfig.Config(
                 notebook_location="notebook_ds",