dasl-client 1.0.13__py3-none-any.whl → 1.0.14__py3-none-any.whl

dasl_client/client.py

@@ -1,5 +1,7 @@
 from copy import deepcopy
-from typing import Any, Callable, Iterator, List, Optional, TypeVar
+from typing import Any, Callable, Iterator, List, Optional, Tuple, TypeVar
+from pydantic import Field
+from pyspark.sql import DataFrame
 
 from dasl_api import (
     CoreV1Api,
@@ -9,8 +11,6 @@ from dasl_api import (
     WorkspaceV1CreateWorkspaceRequest,
     api,
 )
-from pydantic import Field
-
 from dasl_client.auth.auth import (
     Authorization,
     DatabricksSecretAuth,
@@ -569,6 +569,41 @@ class Client:
             )
         return Rule.from_api_obj(result)
 
+    def exec_rule(
+        self, rule_in: Rule, df: DataFrame
+    ) -> Tuple[DataFrame, Optional[DataFrame]]:
+        """
+        Locally execute a Rule. Must be run from within a Databricks
+        notebook or else an exception will be raised. This is intended
+        to facilitate Rule development.
+
+        :param rule_in: The specification of the Rule to execute.
+        :param df: The DataFrame to use as the input to the Rule.
+        :returns Tuple[DataFrame, Optional[DataFrame]]: The first
+            element of the tuple contains the notables produced by
+            the rule, and the second element contains the observables
+            or None if no observables were produced.
+        """
+        Helpers.ensure_databricks()
+        with error_handler():
+            result = self._core_client().core_v1_render_rule(
+                self._workspace(),
+                rule_in.to_api_obj(),
+            )
+
+            try:
+                import notebook_utils
+            except ImportError as e:
+                raise ImportError(
+                    "Package 'notebook_utils' not found. "
+                    "Install it within this this notebook using "
+                    f"%pip install {result.notebook_utils_path}"
+                )
+
+            namespace = {}
+            exec(result.content, namespace)
+            return namespace["generate"](df)
+
     def adhoc_transform(
         self,
         warehouse: str,

dasl_client/helpers.py

@@ -6,15 +6,18 @@ class Helpers:
     default_dasl_host = "https://api.prod.sl.antimatter.io"
 
     @staticmethod
-    def databricks_context():
-        # This import raises an exception if outside a notebook context, so only
-        # import if this method is called
+    def ensure_databricks():
         if "DATABRICKS_RUNTIME_VERSION" not in os.environ:
             raise Exception(
                 "attempted to access databricks context outside "
                 + "of databricks notebook"
             )
 
+    @staticmethod
+    def databricks_context():
+        # This import raises an exception if outside a notebook context, so only
+        # import if this method is called
+        Helpers.ensure_databricks()
         from databricks.sdk.runtime import dbutils
 
         return dbutils.notebook.entry_point.getDbutils().notebook().getContext()

dasl_client/types/content.py

@@ -173,6 +173,8 @@ class DataSourcePreset(BaseModel):
     name: Optional[str] = None
     author: Optional[str] = None
     description: Optional[str] = None
+    title: Optional[str] = None
+    icon_url: Optional[str] = None
     autoloader: Optional[PresetAutoloader] = None
     silver: Optional[SilverPreset] = None
     gold: Optional[List[GoldPreset]] = None
@@ -188,6 +190,8 @@ class DataSourcePreset(BaseModel):
             name=obj.name,
             author=obj.author,
             description=obj.description,
+            title=obj.title,
+            icon_url=obj.icon_url,
             autoloader=PresetAutoloader.from_api_obj(obj.autoloader),
             silver=SilverPreset.from_api_obj(obj.silver),
             gold=[GoldPreset.from_api_obj(item) for item in obj.gold],

dasl_client/types/dbui.py

@@ -11,7 +11,8 @@ from dasl_api import (
     DbuiV1TransformRequestTransformsInner,
     DbuiV1TransformRequestTransformsInnerPresetOverrides,
     DbuiV1TransformResponse,
-    DbuiV1TransformResponseStagesInner,
+    DbuiV1TransformResponseStagesInner, ContentV1DatasourcePresetAutoloaderCloudFiles,
+    DbuiV1TransformRequestAutoloaderInput,
 )
 
 from .datasource import DataSource, FieldSpec, FieldUtils
@@ -131,7 +132,7 @@ class TransformRequest(BaseModel):
     Attributes:
         input (TransformRequest.Input):
             The input block containing the columns metadata and data.
-        autoloader_input (DataSource.Autoloader):
+        autoloader_input (Autoloader):
             The autoloader input configuration.
         use_preset (str):
             Indicates which preset to use for the transforms.
@@ -172,6 +173,84 @@ class TransformRequest(BaseModel):
                 data=self.data,
             )
 
+    class Autoloader(BaseModel):
+        """
+        Autoloader configuration for the DataSource.
+
+        Attributes:
+            format (Optional[str]):
+                The format of the data (e.g., json, parquet, csv, etc.).
+            location (str):
+                External location for the volume in Unity Catalog.
+            schema_file (Optional[str]):
+                An optional file containing the schema of the data source.
+            cloud_files (Optional[Autoloader.CloudFiles]):
+                CloudFiles configuration.
+        """
+
+        class CloudFiles(BaseModel):
+            """
+            CloudFiles configuration for the Autoloader.
+
+            Attributes:
+                schema_hints_file (Optional[str]):
+                schema_hints (Optional[str]):
+            """
+
+            schema_hints_file: Optional[str] = None
+            schema_hints: Optional[str] = None
+
+            @staticmethod
+            def from_api_obj(
+                    obj: Optional[ContentV1DatasourcePresetAutoloaderCloudFiles],
+            ) -> "TransformRequest.Autoloader.CloudFiles":
+                if obj is None:
+                    return None
+                return TransformRequest.Autoloader.CloudFiles(
+                    schema_hints_file=obj.schema_hints_file,
+                    schema_hints=obj.schema_hints,
+                )
+
+            def to_api_obj(self) -> ContentV1DatasourcePresetAutoloaderCloudFiles:
+                return ContentV1DatasourcePresetAutoloaderCloudFiles(
+                    schema_hints_file=self.schema_hints_file,
+                    schema_hints=self.schema_hints,
+                )
+
+        format: Optional[str] = None
+        location: str
+        schema_file: Optional[str] = None
+        schema: Optional[str] = None
+        cloud_files: Optional["TransformRequest.Autoloader.CloudFiles"] = None
+        row_count: Optional[int] = None
+        row_offset: Optional[int] = None
+
+        @staticmethod
+        def from_api_obj(
+                obj: Optional[DbuiV1TransformRequestAutoloaderInput],
+        ) -> "Optional[TransformRequest.Autoloader]":
+            if obj is None:
+                return None
+            return TransformRequest.Autoloader(
+                format=obj.format,
+                location=obj.location,
+                schema_file=obj.schema_file,
+                cloud_files=TransformRequest.Autoloader.CloudFiles.from_api_obj(obj.cloud_files),
+                row_count=obj.row_count,
+                row_offset=obj.row_offset,
+            )
+        def to_api_obj(self) -> DbuiV1TransformRequestAutoloaderInput:
+            return DbuiV1TransformRequestAutoloaderInput(
+                format=self.format,
+                location=self.location,
+                schemaFile=self.schema_file,
+                schema=self.schema_file,
+                cloudFiles=Helpers.maybe(lambda o: o.to_api_obj(), self.cloud_files),
+                rowCount=self.row_count,
+                rowOffset=self.row_offset
+            )
+
+
     class Transform(BaseModel):
         """
         A transform configuration to apply to the data.
@@ -273,7 +352,7 @@ class TransformRequest(BaseModel):
             )
 
     input: Optional["TransformRequest.Input"] = None
-    autoloader_input: Optional[DataSource.Autoloader] = None
+    autoloader_input: Optional["TransformRequest.Autoloader"] = None
     use_preset: Optional[str] = None
     transforms: List["TransformRequest.Transform"]
 
@@ -281,7 +360,7 @@ class TransformRequest(BaseModel):
     def from_api_obj(obj: DbuiV1TransformRequest) -> "TransformRequest":
         return TransformRequest(
             input=TransformRequest.Input.from_api_obj(obj.input),
-            autoloader_input=DataSource.Autoloader.from_api_obj(obj.autoloader_input),
+            autoloader_input=TransformRequest.Autoloader.from_api_obj(obj.autoloader_input),
             use_preset=obj.use_preset,
             transforms=[
                 TransformRequest.Transform.from_api_obj(item) for item in obj.transforms

{dasl_client-1.0.13.dist-info → dasl_client-1.0.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dasl_client
-Version: 1.0.13
+Version: 1.0.14
 Summary: The DASL client library used for interacting with the DASL workspace
 Home-page: https://github.com/antimatter/asl
 Author: Antimatter Team
@@ -8,9 +8,10 @@ Author-email: Antimatter Team <support@antimatter.io>
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: dasl-api ==0.1.15
+Requires-Dist: dasl-api ==0.1.17
 Requires-Dist: databricks-sdk >=0.41.0
 Requires-Dist: pydantic >=2
+Requires-Dist: pyspark >=3.5.5
 Requires-Dist: typing-extensions ==4.10.0
 
 # DASL Client Library

{dasl_client-1.0.13.dist-info → dasl_client-1.0.14.dist-info}/RECORD

@@ -1,6 +1,6 @@
 dasl_client/__init__.py,sha256=E6gOgO8qg96Y38JKA-4LyNBvc2ytQPEfhdniYsCWBxA,127
-dasl_client/client.py,sha256=jiILjQOB4OuXXj7J0ZBYEN-pnHm8VQjj1Jqf54eJvS8,25419
-dasl_client/helpers.py,sha256=hi_SrFhEqBuLWOteuQlv__Atzq2VMCgY7A8xSt3ztuA,1035
+dasl_client/client.py,sha256=Eb_A7UKjRaTqmD0Q5Keq-JMA4MGGtKFv1Oft9vXE4Yo,26831
+dasl_client/helpers.py,sha256=L7ycxrqyG28glRRGZgsrVBdCJzXYCW7DB0hAvupGMuA,1118
 dasl_client/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dasl_client/auth/auth.py,sha256=yTeijYYpfJVJ_wYyq0U6kAntg4xz5MzIR37_CpVR57k,7277
 dasl_client/conn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -15,15 +15,15 @@ dasl_client/preset_development/preview_parameters.py,sha256=KRk3lTImyvJTeEcepwOS
 dasl_client/preset_development/stage.py,sha256=NiDuFiKvoXgqVPPjzB3TZQN1dQeVff6he6oaSVGwkCs,20154
 dasl_client/types/__init__.py,sha256=GsXC3eWuv21VTLPLPH9pzM95JByaKnKrPjJkh2rlZfQ,170
 dasl_client/types/admin_config.py,sha256=Kmx3Kuai9_LWMeO2NpWasRUgLihYSEXGtuYVfG0FkjU,2200
-dasl_client/types/content.py,sha256=b_4-6rcA6uP0aPN-nBd1QonwQ5Bc4lnrHlL6MKYnl8U,7311
+dasl_client/types/content.py,sha256=uZAO-Vm_orvqsH2CkiwBUHgn6fWGXR90hOGKc256lcA,7442
 dasl_client/types/datasource.py,sha256=-ABmBh5yZwHeY-PKQMnNCNa9FSzod5n1O817m8ZCL6o,52519
-dasl_client/types/dbui.py,sha256=RZV_YxCc5KIHLcDLO5Gb1t3KnS8JKN4PbhnYGsVJiws,13200
+dasl_client/types/dbui.py,sha256=CLGEaBudddi7AmXoykHw3Mxv8wU2cWm20DZuM-bAjQA,16302
 dasl_client/types/helpers.py,sha256=gLGTvrssAKrdkQT9h80twEosld2egwhvj-zAudxWFPs,109
 dasl_client/types/rule.py,sha256=BqhWhT8Eh95UXNytd0PxVcjqYuWQcdN1tfKjUB4Tk74,25781
 dasl_client/types/types.py,sha256=DeUOfdYGOhUGEy7yKOfo0OYTXYRrs57yYgNLUbu7Tlc,8806
 dasl_client/types/workspace_config.py,sha256=RThg_THS_4leITWdzBPTWdR2ytq5Uk36m6nIOUMzFCM,24878
-dasl_client-1.0.13.dist-info/LICENSE,sha256=M35UepUPyKmFkvENlkweeaMElheQqNoM5Emh8ADO-rs,4
-dasl_client-1.0.13.dist-info/METADATA,sha256=fXnfud7SFjMSDN_6MTVTwjpDmhV3Rio_ECGDZ36RZIA,741
-dasl_client-1.0.13.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-dasl_client-1.0.13.dist-info/top_level.txt,sha256=kIv8ox_2oJPjGB8_yuey5vvuPCyfY8kywG138f9oSOY,12
-dasl_client-1.0.13.dist-info/RECORD,,
+dasl_client-1.0.14.dist-info/LICENSE,sha256=M35UepUPyKmFkvENlkweeaMElheQqNoM5Emh8ADO-rs,4
+dasl_client-1.0.14.dist-info/METADATA,sha256=G12U4jgWU43fwrPGNh392IArKnjuAO33s8Wvb9nPTvQ,772
+dasl_client-1.0.14.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+dasl_client-1.0.14.dist-info/top_level.txt,sha256=kIv8ox_2oJPjGB8_yuey5vvuPCyfY8kywG138f9oSOY,12
+dasl_client-1.0.14.dist-info/RECORD,,