dasl-client 1.0.9__tar.gz → 1.0.12__tar.gz

{dasl_client-1.0.9 → dasl_client-1.0.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dasl_client
-Version: 1.0.9
+Version: 1.0.12
 Summary: The DASL client library used for interacting with the DASL workspace
 Home-page: https://github.com/antimatter/asl
 Author: Antimatter Team
@@ -8,7 +8,7 @@ Author-email: Antimatter Team <support@antimatter.io>
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: dasl_api==0.1.12
+Requires-Dist: dasl_api==0.1.15
 Requires-Dist: databricks-sdk>=0.41.0
 Requires-Dist: pydantic>=2
 Requires-Dist: typing_extensions==4.10.0

{dasl_client-1.0.9 → dasl_client-1.0.12}/dasl_client/auth/auth.py

@@ -1,4 +1,5 @@
 import abc
+import base64
 import time
 from datetime import datetime
 
@@ -176,15 +177,15 @@ class DatabricksSecretAuth(Authorization):
             principalName=self._principal,
         )
         handler = api.WorkspaceV1Api(api_client=self._client)
-        resp = handler.workspace_v1_begin_authentication(
-            workspace=self._workspace, workspace_v1_begin_authenticate_request=req
+        resp = handler.workspace_v1_request_secret(
+            workspace=self._workspace, workspace_v1_request_secret_request=req
         )
         secret_name = resp.secret_name
         secret_value = ""
         for tries in range(3):
             try:
-                secret_value = WorkspaceClient().secrets.get_secret(
-                    secret_name, "token"
+                secret_value = (
+                    WorkspaceClient().secrets.get_secret(secret_name, "token").value
                 )
                 break
             except ResourceDoesNotExist:
@@ -194,7 +195,7 @@ class DatabricksSecretAuth(Authorization):
             raise RuntimeError(f"failed to complete secret auth")
 
         req = WorkspaceV1AuthenticateRequest(
-            databricks_secret=secret_value,
+            databricks_secret=base64.b64decode(secret_value).decode("utf-8"),
         )
         handler = api.WorkspaceV1Api(api_client=self._client)
 

{dasl_client-1.0.9 → dasl_client-1.0.12}/dasl_client/client.py

@@ -3,8 +3,8 @@ from typing import Any, Callable, Iterator, List, Optional, TypeVar
 
 from dasl_api import (
     CoreV1Api,
-    CoreV1DataSourceAutoloaderSpec,
     DbuiV1Api,
+    ContentV1Api,
     WorkspaceV1Api,
     WorkspaceV1CreateWorkspaceRequest,
     api,
@@ -13,6 +13,7 @@ from pydantic import Field
 
 from dasl_client.auth.auth import (
     Authorization,
+    DatabricksSecretAuth,
     DatabricksTokenAuth,
     ServiceAccountKeyAuth,
 )
@@ -29,6 +30,8 @@ from .types import (
     WorkspaceConfig,
     TransformRequest,
     TransformResponse,
+    DataSourcePresetsList,
+    DataSourcePreset,
 )
 
 
@@ -229,6 +232,9 @@ class Client:
     def _dbui_client(self) -> DbuiV1Api:
         return DbuiV1Api(self.auth.client())
 
+    def _content_client(self) -> ContentV1Api:
+        return ContentV1Api(self.auth.client())
+
     def _workspace(self) -> str:
         return self.auth.workspace()
 
@@ -616,3 +622,33 @@ class Client:
                     limit=limit,
                 )
             )
+
+    def list_presets(self) -> DataSourcePresetsList:
+        """
+        List the Presets in this workspace. This will include any user defined
+        presets if a custom presets path has been configured in the workspace.
+
+        :returns: DataSourcePresetsList
+        """
+        with error_handler():
+            return DataSourcePresetsList.from_api_obj(
+                self._content_client().content_v1_get_preset_data_sources(
+                    self._workspace(),
+                )
+            )
+
+    def get_preset(self, name: str) -> DataSourcePreset:
+        """
+        Get the preset with the argument name from the DASL server. If the preset name
+        begins with 'internal_' it will instead be collected from the user catalog,
+        provided a preset path is set in the workspace config.
+
+        :param name: The unique name of the DataSource preset within this workspace.
+        :returns: DataSourcePreset
+        """
+        with error_handler():
+            return DataSourcePreset.from_api_obj(
+                self._content_client().content_v1_get_preset_datasource(
+                    self._workspace(), name
+                )
+            )

{dasl_client-1.0.9 → dasl_client-1.0.12}/dasl_client/types/__init__.py

@@ -4,3 +4,4 @@ from .workspace_config import *
 from .datasource import *
 from .rule import *
 from .dbui import *
+from .content import *

dasl_client-1.0.12/dasl_client/types/content.py

@@ -0,0 +1,243 @@
+from typing import Optional, List, Dict
+from pydantic import BaseModel
+
+from dasl_api import (
+    ContentV1PresetDataSourceList,
+    ContentV1PresetDataSourceListItemsInner,
+    ContentV1DatasourcePresetAutoloader,
+    ContentV1DatasourcePresetSilver,
+    ContentV1DatasourcePresetSilverPreTransformInner,
+    ContentV1DatasourcePresetSilverTransformInner,
+    ContentV1DatasourcePresetGoldInner,
+    ContentV1DatasourcePreset,
+    ContentV1DatasourcePresetAutoloaderCloudFiles,
+)
+
+from .datasource import DataSource, FieldSpec, FieldUtils
+from .helpers import Helpers
+
+
+class SilverPreset(BaseModel):
+    class PreTransform(BaseModel):
+        name: Optional[str] = None
+        filter: Optional[str] = None
+        post_filter: Optional[str] = None
+        fields: Optional[List[Optional[FieldSpec]]] = None
+        utils: Optional[FieldUtils] = None
+
+        @staticmethod
+        def from_api_obj(
+            obj: Optional[ContentV1DatasourcePresetSilverPreTransformInner],
+        ) -> Optional["SilverPreset.PreTransform"]:
+            if obj is None:
+                return None
+            if obj.fields is not None:
+                fields = [FieldSpec.from_api_obj(item) for item in obj.fields]
+            else:
+                fields = None
+
+            return SilverPreset.PreTransform(
+                name=obj.name,
+                filter=obj.filter,
+                post_filter=obj.post_filter,
+                fields=fields,
+                utils=FieldUtils.from_api_obj(obj.utils),
+            )
+
+    class Transform(BaseModel):
+        name: Optional[str] = None
+        filter: Optional[str] = None
+        post_filter: Optional[str] = None
+        fields: Optional[List[Optional[FieldSpec]]] = None
+        utils: Optional[FieldUtils] = None
+
+        @staticmethod
+        def from_api_obj(
+            obj: Optional[ContentV1DatasourcePresetSilverTransformInner],
+        ) -> Optional["SilverPreset.Transform"]:
+            if obj is None:
+                return None
+            if obj.fields is not None:
+                fields = [FieldSpec.from_api_obj(item) for item in obj.fields]
+            else:
+                fields = None
+
+            return SilverPreset.Transform(
+                name=obj.name,
+                filter=obj.filter,
+                post_filter=obj.post_filter,
+                fields=fields,
+                utils=FieldUtils.from_api_obj(obj.utils),
+            )
+
+    pre_transform: Optional[List[PreTransform]] = None
+    transform: Optional[List[Transform]] = None
+
+    @staticmethod
+    def from_api_obj(
+        obj: Optional[ContentV1DatasourcePresetSilver],
+    ) -> Optional["SilverPreset"]:
+        if obj is None:
+            return None
+
+        if obj.pre_transform is not None:
+            pre_transform = [
+                SilverPreset.PreTransform.from_api_obj(item)
+                for item in obj.pre_transform
+            ]
+        else:
+            pre_transform = None
+
+        if obj.transform is not None:
+            transform = [
+                SilverPreset.Transform.from_api_obj(item) for item in obj.transform
+            ]
+        else:
+            transform = None
+
+        return SilverPreset(
+            pre_transform=pre_transform,
+            transform=transform,
+        )
+
+
+class GoldPreset(BaseModel):
+    name: str
+    input: str
+    filter: Optional[str] = None
+    post_filter: Optional[str] = None
+    fields: Optional[List[Optional[FieldSpec]]] = None
+    utils: Optional[FieldUtils] = None
+
+    @staticmethod
+    def from_api_obj(
+        obj: Optional[ContentV1DatasourcePresetGoldInner],
+    ) -> Optional["GoldPreset"]:
+        if obj is None:
+            return None
+        if obj.fields is not None:
+            fields = [FieldSpec.from_api_obj(item) for item in obj.fields]
+        else:
+            fields = None
+
+        return GoldPreset(
+            name=obj.name,
+            input=obj.input,
+            filter=obj.filter,
+            post_filter=obj.post_filter,
+            fields=fields,
+            utils=FieldUtils.from_api_obj(obj.utils),
+        )
+
+
+class PresetCloudFiles(BaseModel):
+    schema_hints_file: Optional[str] = None
+    schema_hints: Optional[str] = None
+
+    @staticmethod
+    def from_api_obj(
+        obj: Optional[ContentV1DatasourcePresetAutoloaderCloudFiles],
+    ) -> Optional["PresetCloudFiles"]:
+        if obj is None:
+            return None
+
+        return PresetCloudFiles(
+            schema_hints_file=obj.schema_hints_file,
+            schema_hints=obj.schema_hints,
+        )
+
+
+class PresetAutoloader(BaseModel):
+    format: str
+    schema_file: Optional[str] = None
+    var_schema: Optional[str] = None
+    cloud_files: Optional[PresetCloudFiles] = None
+
+    @staticmethod
+    def from_api_obj(
+        obj: Optional[ContentV1DatasourcePresetAutoloader],
+    ) -> Optional["PresetAutoloader"]:
+        if obj is None:
+            return None
+
+        return PresetAutoloader(
+            format=obj.format,
+            schema_file=obj.schema_file,
+            var_schema=obj.var_schema,
+            cloud_files=PresetCloudFiles.from_api_obj(obj.cloud_files),
+        )
+
+
+class DataSourcePreset(BaseModel):
+
+    name: Optional[str] = None
+    author: Optional[str] = None
+    description: Optional[str] = None
+    autoloader: Optional[PresetAutoloader] = None
+    silver: Optional[SilverPreset] = None
+    gold: Optional[List[GoldPreset]] = None
+
+    @staticmethod
+    def from_api_obj(
+        obj: Optional[ContentV1DatasourcePreset],
+    ) -> Optional["DataSourcePreset"]:
+        if obj is None:
+            return None
+
+        return DataSourcePreset(
+            name=obj.name,
+            author=obj.author,
+            description=obj.description,
+            autoloader=PresetAutoloader.from_api_obj(obj.autoloader),
+            silver=SilverPreset.from_api_obj(obj.silver),
+            gold=[GoldPreset.from_api_obj(item) for item in obj.gold],
+        )
+
+
+class DataSourcePresetSummary(BaseModel):
+    name: Optional[str] = None
+    source: Optional[str] = None
+    source_type: Optional[str] = None
+    description: Optional[str] = None
+    title: Optional[str] = None
+    author: Optional[str] = None
+    icon_url: Optional[str] = None
+
+    @staticmethod
+    def from_api_obj(
+        obj: Optional[ContentV1PresetDataSourceListItemsInner],
+    ) -> Optional["DataSourcePresetSummary"]:
+        if obj is None:
+            return None
+
+        return DataSourcePresetSummary(
+            name=obj.name,
+            source=obj.source,
+            source_type=obj.source_type,
+            description=obj.description,
+            title=obj.title,
+            author=obj.author,
+            icon_url=obj.icon_url,
+        )
+
+
+class DataSourcePresetsList(BaseModel):
+    kind: Optional[str] = None
+    cursor: Optional[str] = None
+    items: List["DataSourcePresetSummary"] = []
+
+    @staticmethod
+    def from_api_obj(
+        obj: Optional[ContentV1PresetDataSourceList],
+    ) -> Optional["DataSourcePresetsList"]:
+        if obj is None:
+            return None
+        cursor = None
+        if obj.metadata is not None:
+            cursor = obj.metadata.cursor
+
+        return DataSourcePresetsList(
+            kind=obj.kind,
+            cursor=cursor,
+            items=[DataSourcePresetSummary.from_api_obj(item) for item in obj.items],
+        )

{dasl_client-1.0.9 → dasl_client-1.0.12}/dasl_client/types/rule.py

@@ -17,7 +17,7 @@ from dasl_api import (
     CoreV1RuleObservable,
     CoreV1RuleObservableRisk,
     CoreV1RuleSpecOutput,
-    CoreV1RuleSpecCollate,
+    CoreV1RuleSpecCollate, CoreV1RuleSpecInputBatchCustom,
 )
 
 from .helpers import Helpers
@@ -251,9 +251,9 @@ class Rule(BaseModel):
                 Input if the rule should operate on batched input data.
         """
 
-        class Custom(BaseModel):
+        class CustomStream(BaseModel):
             """
-            Specification of a custom notebook for generating input to
+            Specification of a stream custom notebook for generating input to
             the Rule.
 
             Attributes:
@@ -267,10 +267,10 @@ class Rule(BaseModel):
             @staticmethod
             def from_api_obj(
                 obj: Optional[CoreV1RuleSpecInputStreamCustom],
-            ) -> "Rule.Input.Custom":
+            ) -> "Rule.Input.CustomStream":
                 if obj is None:
                     return None
-                return Rule.Input.Custom(
+                return Rule.Input.CustomStream(
                     notebook=obj.notebook,
                     options=obj.options,
                 )
@@ -281,6 +281,36 @@ class Rule(BaseModel):
                     options=self.options,
                 )
 
+        class CustomBatch(BaseModel):
+            """
+            Specification of a batch custom notebook for generating input to
+            the Rule.
+
+            Attributes:
+                notebook (Optional[str]):
+                options (Optional[Dict[str, str]]):
+            """
+
+            notebook: Optional[str] = None
+            options: Optional[Dict[str, str]] = None
+
+            @staticmethod
+            def from_api_obj(
+                obj: Optional[CoreV1RuleSpecInputBatchCustom],
+            ) -> "Rule.Input.CustomBatch":
+                if obj is None:
+                    return None
+                return Rule.Input.CustomBatch(
+                    notebook=obj.notebook,
+                    options=obj.options,
+                )
+
+            def to_api_obj(self) -> CoreV1RuleSpecInputBatchCustom:
+                return CoreV1RuleSpecInputBatchCustom(
+                    notebook=self.notebook,
+                    options=self.options,
+                )
+
         class Stream(BaseModel):
             """
             Specification for streaming input from a table or tables,
@@ -290,7 +320,7 @@ class Rule(BaseModel):
                 tables (Optional[List[Rule.Input.Stream.Table]]):
                 filter (Optional[str]):
                 sql (Optional[str]):
-                custom (Optional[Rule.Input.Custom]):
+                custom (Optional[Rule.Input.CustomStream]):
             """
 
             class Table(BaseModel):
@@ -395,7 +425,7 @@ class Rule(BaseModel):
             tables: Optional[List["Rule.Input.Stream.Table"]] = None
             filter: Optional[str] = None
             sql: Optional[str] = None
-            custom: Optional["Rule.Input.Custom"] = None
+            custom: Optional["Rule.Input.CustomStream"] = None
 
             @staticmethod
             def from_api_obj(
@@ -413,7 +443,7 @@ class Rule(BaseModel):
                     tables=tables,
                     filter=obj.filter,
                     sql=obj.sql,
-                    custom=Rule.Input.Custom.from_api_obj(obj.custom),
+                    custom=Rule.Input.CustomStream.from_api_obj(obj.custom),
                 )
 
             def to_api_obj(self) -> CoreV1RuleSpecInputStream:
@@ -434,11 +464,11 @@ class Rule(BaseModel):
 
             Attributes:
                 sql (Optional[str]):
-                custom (Optional[Rule.Input.Custom]):
+                custom (Optional[Rule.Input.CustomBatch]):
             """
 
             sql: Optional[str] = None
-            custom: Optional["Rule.Input.Custom"] = None
+            custom: Optional["Rule.Input.CustomBatch"] = None
 
             @staticmethod
             def from_api_obj(
@@ -448,7 +478,7 @@ class Rule(BaseModel):
                     return None
                 return Rule.Input.Batch(
                     sql=obj.sql,
-                    custom=Rule.Input.Custom.from_api_obj(obj.custom),
+                    custom=Rule.Input.CustomBatch.from_api_obj(obj.custom),
                 )
 
             def to_api_obj(self) -> CoreV1RuleSpecInputBatch:

{dasl_client-1.0.9 → dasl_client-1.0.12}/dasl_client/types/workspace_config.py

@@ -576,6 +576,8 @@ class WorkspaceConfig(BaseModel):
         dasl_storage_path (Optional[str]):
             The path to a directory where DASL can store internal files and
             state.
+        dasl_custom_presets_path (Optional[str]):
+            An optional path to a directory containing user defined presets.
         default_config (Optional[DefaultConfig]):
             Configuration settings regarding storage of bronze, silver, and
             gold tables and related assets for each resource type.
@@ -594,6 +596,7 @@ class WorkspaceConfig(BaseModel):
     operational_alert_export: Optional[ExportConfig] = None
     observables: Optional[WorkspaceConfigObservables] = None
     dasl_storage_path: Optional[str] = None
+    dasl_custom_presets_path: Optional[str] = None
     default_config: Optional[DefaultConfig] = None
     managed_retention: Optional[List[ManagedRetention]] = None
     status: Optional[ResourceStatus] = None
@@ -623,6 +626,7 @@ class WorkspaceConfig(BaseModel):
             ),
             observables=WorkspaceConfigObservables.from_api_obj(spec.observables),
             dasl_storage_path=spec.dasl_storage_path,
+            dasl_custom_presets_path=spec.dasl_custom_presets_path,
             default_config=DefaultConfig.from_api_obj(spec.default_config),
             managed_retention=managed_retention,
             status=ResourceStatus.from_api_obj(obj.status),

{dasl_client-1.0.9 → dasl_client-1.0.12}/dasl_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dasl_client
-Version: 1.0.9
+Version: 1.0.12
 Summary: The DASL client library used for interacting with the DASL workspace
 Home-page: https://github.com/antimatter/asl
 Author: Antimatter Team
@@ -8,7 +8,7 @@ Author-email: Antimatter Team <support@antimatter.io>
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: dasl_api==0.1.12
+Requires-Dist: dasl_api==0.1.15
 Requires-Dist: databricks-sdk>=0.41.0
 Requires-Dist: pydantic>=2
 Requires-Dist: typing_extensions==4.10.0

{dasl_client-1.0.9 → dasl_client-1.0.12}/dasl_client.egg-info/SOURCES.txt

@@ -24,6 +24,7 @@ dasl_client/preset_development/preview_parameters.py
 dasl_client/preset_development/stage.py
 dasl_client/types/__init__.py
 dasl_client/types/admin_config.py
+dasl_client/types/content.py
 dasl_client/types/datasource.py
 dasl_client/types/dbui.py
 dasl_client/types/helpers.py

{dasl_client-1.0.9 → dasl_client-1.0.12}/dasl_client.egg-info/requires.txt

@@ -1,4 +1,4 @@
-dasl_api==0.1.12
+dasl_api==0.1.15
 databricks-sdk>=0.41.0
 pydantic>=2
 typing_extensions==4.10.0

{dasl_client-1.0.9 → dasl_client-1.0.12}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dasl_client"
-version = "1.0.9"
+version = "1.0.12"
 description = "The DASL client library used for interacting with the DASL workspace"
 readme = "README.md"
 authors = [
@@ -13,7 +13,7 @@ authors = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "dasl_api==0.1.12",
+    "dasl_api==0.1.15",
     "databricks-sdk>=0.41.0",
     "pydantic>=2",
     "typing_extensions==4.10.0"

{dasl_client-1.0.9 → dasl_client-1.0.12}/test/test_api_changes.py

@@ -18,6 +18,7 @@ checked_dasl_types = {
     CoreV1DataSource: "data_source.json",
     CoreV1Rule: "rule.json",
     WorkspaceV1WorkspaceConfig: "workspace_config.json",
+    ContentV1DatasourcePreset: "datasource_preset.json",
     # Data
     DbuiV1ObservableEventsList: "observable_events_list.json",
 }

{dasl_client-1.0.9 → dasl_client-1.0.12}/test/test_marshaling.py

@@ -613,14 +613,14 @@ def test_rule_marshal_unmarshal():
                 ],
                 filter="status = 'active'",
                 sql="SELECT * FROM streaming_source",
-                custom=Rule.Input.Custom(
+                custom=Rule.Input.CustomStream(
                     notebook="stream_custom.ipynb",
                     options={"filter": "recent", "limit": "1000"},
                 ),
             ),
             batch=Rule.Input.Batch(
                 sql="SELECT * FROM historical_source",
-                custom=Rule.Input.Custom(
+                custom=Rule.Input.CustomBatch(
                     notebook="batch_custom.ipynb",
                     options={"start_date": "2022-01-01", "end_date": "2022-12-31"},
                 ),