dasl-client 1.0.7__py3-none-any.whl → 1.0.11__py3-none-any.whl

dasl_client/types/datasource.py

@@ -44,24 +44,28 @@ class FieldSpec(BaseModel):
 
     Attributes:
         name (Optional[str]):
-        comment (Optional[str]): The comment to apply to the field.
-        var_assert (Optional[List[FieldSpec.Assert]]): A list of SQL
-            expressions that must evaluate to true for every processed row.
-            If the assertion is false, an operational alert is raised using
-            'message' for each row.
-        var_from (Optional[str]): This field obtains its value from the source
-            column of this name. Use this to bring in a column from some
-            upstream table.
-        alias (Optional[str]): This field obtains its value from the
-            destination (transformed) column of this name. Use this to alias
-            a column from within the same table (ie. silver table). You cannot
-            alias a column from some upstream table.
-        expr (Optional[str]): This field obtains its value from the given SQL
-            expression.
-        literal (Optional[str]): This field obtains its value from the given
-            literal string. For other data types, use expr.
-        join (Optional[FieldSpec.Join]): This field obtains its value from
-            joining to another table.
+            The name of the field.
+        comment (Optional[str]):
+            The comment to apply to the field.
+        var_assert (Optional[List[FieldSpec.Assert]]):
+            A list of SQL expressions that must evaluate to true for every
+            processed row. If the assertion is false, an operational alert
+            is raised using 'message' for each row.
+        var_from (Optional[str]):
+            This field obtains its value from the source column of this name.
+            Use this to bring in a column from some upstream table.
+        alias (Optional[str]):
+            This field obtains its value from the destination (transformed)
+            column of this name. Use this to alias a column from within the
+            same table (ie. silver table). You cannot alias a column from
+            some upstream table.
+        expr (Optional[str]):
+            This field obtains its value from the given SQL expression.
+        literal (Optional[str]):
+            This field obtains its value from the given literal string. For
+            other data types, use expr.
+        join (Optional[FieldSpec.Join]):
+            This field obtains its value from joining to another table.
     """
 
     class Assert(BaseModel):
@@ -69,10 +73,12 @@ class FieldSpec(BaseModel):
         An assertion within a FieldSpec.
 
         Attributes:
-            expr (Optional[str]): The SQL expression that must evaluate to
-                true for every processed row.
-            message (Optional[str]): The message to include in the operational
-                alert if the assertion fails.
+            expr (Optional[str]):
+                The SQL expression that must evaluate to true for every
+                processed row.
+            message (Optional[str]):
+                The message to include in the operational alert if the
+                assertion fails.
         """
 
         expr: Optional[str] = None
@@ -100,13 +106,17 @@ class FieldSpec(BaseModel):
         A join expression within a FieldSpec.
 
         Attributes:
-            with_table (Optional[str]): The table to join to.
-            with_csv (Optional[FieldSpec.Join.WithCSV]): The CSV configuration
-                used for the join.
-            lhs (Optional[str]): The column in the source dataframe to join on.
-            rhs (Optional[str]): The column in the joined table to join on.
-            select (Optional[str]): A SQL expression to create the new field
-                from the joined dataset.
+            with_table (Optional[str]):
+                The table to join to.
+            with_csv (Optional[FieldSpec.Join.WithCSV]):
+                The CSV configuration used for the join.
+            lhs (Optional[str]):
+                The column in the source dataframe to join on.
+            rhs (Optional[str]):
+                The column in the joined table to join on.
+            select (Optional[str]):
+                A SQL expression to create the new field from the joined
+                dataset.
         """
 
         class WithCSV(BaseModel):
@@ -114,7 +124,8 @@ class FieldSpec(BaseModel):
             A CSV file used for joins within a FieldSpec.
 
             Attributes:
-                path (Optional[str]): The path to the CSV file.
+                path (Optional[str]):
+                    The path to the CSV file.
             """
 
             path: Optional[str] = None
@@ -213,8 +224,8 @@ class FieldUtils(BaseModel):
         unreferenced_columns (Optional[FieldUtils.UnreferencedColumns]):
             Defines whether columns not referenced in the FieldSpecs should
             be preserved or omitted.
-        json_extract (Optional[List[FieldUtils.JsonExtract]]): A list of
-            configurations for extracting JSON fields from a column.
+        json_extract (Optional[List[FieldUtils.JsonExtract]]):
+            A list of configurations for extracting JSON fields from a column.
     """
 
     class UnreferencedColumns(BaseModel):
@@ -222,14 +233,16 @@ class FieldUtils(BaseModel):
         Configuration related to unreferenced columns.
 
         Attributes:
-            preserve (Optional[bool]): Indicates whether columns not
-                referenced in the FieldSpecs should be preserved.
-            embed_column (Optional[str]): Specifies a name for a new column to
-                contain all unreferenced fields.
-            omit_columns (Optional[List[str]]): Lists columns to exclude from
-                the output.
-            duplicate_prefix (Optional[str]): Adds a prefix to resolve
-                ambiguous duplicate field names.
+            preserve (Optional[bool]):
+                Indicates whether columns not referenced in the FieldSpecs
+                should be preserved.
+            embed_column (Optional[str]):
+                Specifies a name for a new column to contain all unreferenced
+                fields.
+            omit_columns (Optional[List[str]]):
+                Lists columns to exclude from the output.
+            duplicate_prefix (Optional[str]):
+                Adds a prefix to resolve ambiguous duplicate field names.
         """
 
         preserve: Optional[bool] = None
@@ -263,14 +276,15 @@ class FieldUtils(BaseModel):
         Configuration for extracting JSON fields from table columns.
 
         Attributes:
-            source (Optional[str]): The column name containing JSON string(s)
-                to extract from.
-            omit_fields (Optional[List[str]]): Specifies high-level fields to
-                exclude from extraction.
-            duplicate_prefix (Optional[str]): Adds a prefix to resolve
-                duplicate field names during extraction.
-            embed_column (Optional[str]): Specifies a column name to store the
-                extracted JSON object.
+            source (Optional[str]):
+                The column name containing JSON string(s) to extract from.
+            omit_fields (Optional[List[str]]):
+                Specifies high-level fields to exclude from extraction.
+            duplicate_prefix (Optional[str]):
+                Adds a prefix to resolve duplicate field names during
+                extraction.
+            embed_column (Optional[str]):
+                Specifies a column name to store the extracted JSON object.
         """
 
         source: Optional[str] = None
@@ -333,12 +347,13 @@ class BronzeSpec(BaseModel):
     Configuration for bronze table within a DataSource.
 
     Attributes:
-        clustering (Optional[BronzeSpec.Clustering]): Describes optional
-            liquid clustering configuration for the bronze table.
-        bronze_table (Optional[str]): The name of the bronze table to create
-            and hold the imported data.
-        skip_bronze_loading (Optional[bool]): Indicates whether to skip the
-            bronze loading step.
+        clustering (Optional[BronzeSpec.Clustering]):
+            Describes optional liquid clustering configuration for the
+            bronze table.
+        bronze_table (Optional[str]):
+            The name of the bronze table to create and hold the imported data.
+        skip_bronze_loading (Optional[bool]):
+            Indicates whether to skip the bronze loading step.
     """
 
     class Clustering(BaseModel):
@@ -346,10 +361,11 @@ class BronzeSpec(BaseModel):
         Configuration of liquid clustering for a bronze table.
 
         Attributes:
-            column_names (Optional[List[str]]): List of column names to
-                include in liquid clustering.
-            time_column (Optional[str]): Name of the column that holds 'time'
-                information for clustering.
+            column_names (Optional[List[str]]):
+                List of column names to include in liquid clustering.
+            time_column (Optional[str]):
+                Name of the column that holds 'time' information for
+                clustering.
         """
 
         column_names: Optional[List[str]] = None
@@ -399,12 +415,12 @@ class SilverSpec(BaseModel):
     Configuration for silver table in a DataSource.
 
     Attributes:
-        bronze_tables (Optional[List[SilverSpec.BronzeTable]]): A list of
-            bronze tables to be joined for silver transformation.
-        pre_transform (Optional[SilverSpec.PreTransform]): Pretransformation
-            configuration.
-        transform (Optional[SilverSpec.Transform]): Transformation
-            configuration for silver processing.
+        bronze_tables (Optional[List[SilverSpec.BronzeTable]]):
+            A list of bronze tables to be joined for silver transformation.
+        pre_transform (Optional[SilverSpec.PreTransform]):
+            Pretransformation configuration.
+        transform (Optional[SilverSpec.Transform]):
+            Transformation configuration for silver processing.
     """
 
     class BronzeTable(BaseModel):
@@ -412,12 +428,18 @@ class SilverSpec(BaseModel):
         Reference to a bronze table for a silver table.
 
         Attributes:
-            name (Optional[str]): Name of the bronze table.
+            name (Optional[str]):
+                Name of the bronze table.
             streaming (Optional[bool]):
+                True if the input should be streamed from the bronze table.
             watermark (Optional[SilverSpec.BronzeTable.Watermark]):
-            alias (Optional[str]): Alias name for the table.
-            join_type (Optional[str]): How to join to the preceding table.
-            join_expr (Optional[str]): The join condition expression.
+                Bronze table watermark.
+            alias (Optional[str]):
+                Alias name for the table.
+            join_type (Optional[str]):
+                How to join to the preceding table.
+            join_expr (Optional[str]):
+                The join condition expression.
         """
 
         class Watermark(BaseModel):
@@ -425,12 +447,12 @@ class SilverSpec(BaseModel):
             Watermark for a bronze source table within a silver table.
 
             Attributes:
-                event_time_column (Optional[str]): Which column is the event
-                    time for the delay threshold.
-                delay_threshold (Optional[str]): A time duration string for
-                    the watermark delay.
-                drop_duplicates (Optional[List[str]]): Columns to pass to
-                    pyspark dropDuplicates.
+                event_time_column (Optional[str]):
+                    Which column is the event time for the delay threshold.
+                delay_threshold (Optional[str]):
+                    A time duration string for the watermark delay.
+                drop_duplicates (Optional[List[str]]):
+                    Columns to pass to pyspark dropDuplicates.
             """
 
             event_time_column: Optional[str] = None
@@ -496,17 +518,20 @@ class SilverSpec(BaseModel):
 
         Attributes:
             use_preset (Optional[str]):
+                Preset to use.
             skip_pre_transform (Optional[bool]):
-            custom (Optional[SilverSpec.PreTransform.Custom]): Custom
-                pretransform function and options.
-            filter (Optional[str]): A SQL filter to apply at the beginning of
-                the preTransform phase.
-            post_filter (Optional[str]): A SQL filter to apply at the end of
-                the preTransform phase.
+                If True, skip pre-transform entirely.
+            custom (Optional[SilverSpec.PreTransform.Custom]):
+                Custom pretransform function and options.
+            filter (Optional[str]):
+                A SQL filter to apply at the beginning of the preTransform
+                phase.
+            post_filter (Optional[str]):
+                A SQL filter to apply at the end of the preTransform phase.
             preset_overrides (Optional[SilverSpec.PreTransform.PresetOverrides]):
                 Overrides for preset filters.
-            add_fields (Optional[List[FieldSpec]]): User defined fields to
-                add to the transformation.
+            add_fields (Optional[List[FieldSpec]]):
+                User defined fields to add to the transformation.
         """
 
         class Custom(BaseModel):
@@ -543,8 +568,8 @@ class SilverSpec(BaseModel):
             Overrides for the preset.
 
             Attributes:
-                omit_fields (Optional[List[str]]): A list of fields to omit
-                    from the chosen preset.
+                omit_fields (Optional[List[str]]):
+                    A list of fields to omit from the chosen preset.
             """
 
             omit_fields: Optional[List[str]] = None
@@ -616,6 +641,7 @@ class SilverSpec(BaseModel):
 
         Attributes:
             skip_silver_transform (Optional[bool]):
+                If True, skip transform entirely.
             preset_overrides (Optional[SilverSpec.Transform.PresetOverrides]):
                 Preset overrides for the silver transformation.
         """
@@ -625,14 +651,12 @@ class SilverSpec(BaseModel):
             Overrides for preset transform settings.
 
             Attributes:
-                modify_tables (Optional[List[SilverSpec.Transform.
-                    PresetOverrides.ModifyTables]]): Modifications for
-                    existing tables.
-                omit_tables (Optional[List[str]]): A list of tables to omit
-                    from the preset.
-                add_tables (Optional[List[SilverSpec.Transform.PresetOverrides.
-                    AddTables]]): User defined tables to include in the
-                    transformation.
+                modify_tables (Optional[List[SilverSpec.Transform.PresetOverrides.ModifyTables]]):
+                    Modifications forexisting tables.
+                omit_tables (Optional[List[str]]):
+                    A list of tables to omit from the preset.
+                add_tables (Optional[List[SilverSpec.Transform.PresetOverrides.AddTables]]):
+                    User defined tables to include in the transformation.
             """
 
             class Custom(BaseModel):
@@ -909,12 +933,12 @@ class GoldSpec(BaseModel):
     Configuration for gold table in a DataSource.
 
     Attributes:
-        omit_tables (Optional[List[str]]): A list of tables to omit from the
-            preset.
-        modify_tables (Optional[List[GoldSpec.ModifyTables]]): Modifications
-            for existing gold table definitions.
-        add_tables (Optional[List[GoldSpec.AddTables]]): User defined tables
-            to add to the gold configuration.
+        omit_tables (Optional[List[str]]):
+            A list of tables to omit from the preset.
+        modify_tables (Optional[List[GoldSpec.ModifyTables]]):
+            Modifications for existing gold table definitions.
+        add_tables (Optional[List[GoldSpec.AddTables]]):
+            User defined tables to add to the gold configuration.
     """
 
     class ModifyTables(BaseModel):
@@ -923,15 +947,19 @@ class GoldSpec(BaseModel):
 
         Attributes:
             name (Optional[str]):
-            source_table (Optional[str]): Used to match against the preset's
-                gold stanzas input fields.
-            custom (Optional[GoldSpec.ModifyTables.Custom]): Custom function
-                for modifying tables.
-            omit_fields (Optional[List[str]]): A list of fields to omit.
-            add_fields (Optional[List[FieldSpec]]): Fields to add.
-            filter (Optional[str]): A SQL filter to apply before processing.
-            post_filter (Optional[str]): A SQL filter to apply after
-                processing.
+                Table name.
+            source_table (Optional[str]):
+                Used to match against the preset's gold stanzas input fields.
+            custom (Optional[GoldSpec.ModifyTables.Custom]):
+                Custom function for modifying tables.
+            omit_fields (Optional[List[str]]):
+                A list of fields to omit.
+            add_fields (Optional[List[FieldSpec]]):
+                Fields to add.
+            filter (Optional[str]):
+                A SQL filter to apply before processing.
+            post_filter (Optional[str]):
+                A SQL filter to apply after processing.
         """
 
         class Custom(BaseModel):
@@ -1016,16 +1044,18 @@ class GoldSpec(BaseModel):
         Tables to add during gold table transformation.
 
         Attributes:
-            custom (Optional[GoldSpec.AddTables.Custom]): Custom function for
-                adding tables.
-            name (Optional[str]): The name of the table to add.
-            source_table (Optional[str]): The source table/dataframe for the
-                gold table.
-            filter (Optional[str]): A SQL filter to apply.
-            post_filter (Optional[str]): A SQL filter to apply after
-                processing.
-            fields (Optional[List[FieldSpec]]): Field specifications for the
-                new table.
+            custom (Optional[GoldSpec.AddTables.Custom]):
+                Custom function for adding tables.
+            name (Optional[str]):
+                The name of the table to add.
+            source_table (Optional[str]):
+                The source table/dataframe for the gold table.
+            filter (Optional[str]):
+                A SQL filter to apply.
+            post_filter (Optional[str]):
+                A SQL filter to apply after processing.
+            fields (Optional[List[FieldSpec]]):
+                Field specifications for the new table.
         """
 
         class Custom(BaseModel):
@@ -1150,20 +1180,28 @@ class DataSource(BaseModel):
     A DataSource resource.
 
     Attributes:
-        metadata (Optional[Metadata]): Standard object metadata.
-        source (Optional[str]): The name of the originator of the data.
-        source_type (Optional[str]): The type of data being imported.
-        schedule (Optional[Schedule]): The schedule for data ingestion.
-        custom (Optional[DataSource.CustomNotebook]): A custom notebook for
-            the datasource.
-        use_preset (Optional[str]): The name of the preset to use for this
-            data source.
-        autoloader (Optional[DataSource.Autoloader]): Autoloader configuration.
-        bronze (Optional[BronzeSpec]): Bronze table configuration.
-        silver (Optional[SilverSpec]): Silver transformation configuration.
-        gold (Optional[GoldSpec]): Gold transformation configuration.
-        status (Optional[ResourceStatus]): The current status of the
-            datasource.
+        metadata (Optional[Metadata]):
+            Standard object metadata.
+        source (Optional[str]):
+            The name of the originator of the data.
+        source_type (Optional[str]):
+            The type of data being imported.
+        schedule (Optional[Schedule]):
+            The schedule for data ingestion.
+        custom (Optional[DataSource.CustomNotebook]):
+            A custom notebook for the datasource.
+        use_preset (Optional[str]):
+            The name of the preset to use for this data source.
+        autoloader (Optional[DataSource.Autoloader]):
+            Autoloader configuration.
+        bronze (Optional[BronzeSpec]):
+            Bronze table configuration.
+        silver (Optional[SilverSpec]):
+            Silver transformation configuration.
+        gold (Optional[GoldSpec]):
+            Gold transformation configuration.
+        status (Optional[ResourceStatus]):
+            The current status of the datasource.
     """
 
     class CustomNotebook(BaseModel):
@@ -1171,8 +1209,8 @@ class DataSource(BaseModel):
         A custom notebook for generating data.
 
         Attributes:
-            notebook (Optional[str]): Path to the notebook in the Databricks
-                workspace.
+            notebook (Optional[str]):
+                Path to the notebook in the Databricks workspace.
         """
 
         notebook: Optional[str] = None
@@ -1197,11 +1235,12 @@ class DataSource(BaseModel):
         Autoloader configuration for the DataSource.
 
         Attributes:
-            format (Optional[str]): The format of the data (e.g., json,
-                parquet, csv, etc.).
-            location (str): External location for the volume in Unity Catalog.
-            schema_file (Optional[str]): An optional file containing the
-                schema of the data source.
+            format (Optional[str]):
+                The format of the data (e.g., json, parquet, csv, etc.).
+            location (str):
+                External location for the volume in Unity Catalog.
+            schema_file (Optional[str]):
+                An optional file containing the schema of the data source.
             cloud_files (Optional[DataSource.Autoloader.CloudFiles]):
                 CloudFiles configuration.
         """

dasl_client/types/dbui.py

@@ -24,12 +24,16 @@ class Dbui(BaseModel):
         Table column details.
 
         Attributes:
-            name (Optional[str]): The column name.
-            type_name (Optional[str]): The name of the column type.
-            type_detail (Optional[str]): Additional information about the
-                column's type.
-            position (Optional[int]): The column's index in the table.
-            nullable (Optional[bool]): Indicates if this column is nullable.
+            name (Optional[str]):
+                The column name.
+            type_name (Optional[str]):
+                The name of the column type.
+            type_detail (Optional[str]):
+                Additional information about the column's type.
+            position (Optional[int]):
+                The column's index in the table.
+            nullable (Optional[bool]):
+                Indicates if this column is nullable.
         """
 
         name: Optional[str] = None
@@ -125,12 +129,14 @@ class TransformRequest(BaseModel):
     silver).
 
     Attributes:
-        input (TransformRequest.Input): The input block containing the
-            columns metadata and data.
+        input (TransformRequest.Input):
+            The input block containing the columns metadata and data.
         autoloader_input (DataSource.Autoloader):
-        use_preset (str): Indicates which preset to use for the transforms.
-        transforms (List[TransformRequest.Transform]): A list of transform
-            configurations.
+            The autoloader input configuration.
+        use_preset (str):
+            Indicates which preset to use for the transforms.
+        transforms (List[TransformRequest.Transform]):
+            A list of transform configurations.
     """
 
     class Input(BaseModel):
@@ -138,10 +144,10 @@ class TransformRequest(BaseModel):
         Input data for the transform request.
 
         Attributes:
-            columns (List[Dbui.TableColumnDetails]): A list of metadata about
-                the columns.
-            data (List[Dict[str, str]]): The data represented as a list of
-                dictionaries.
+            columns (List[Dbui.TableColumnDetails]):
+                A list of metadata about the columns.
+            data (List[Dict[str, str]]):
+                The data represented as a list of dictionaries.
         """
 
         columns: List[Dbui.TableColumnDetails]
@@ -171,19 +177,24 @@ class TransformRequest(BaseModel):
         A transform configuration to apply to the data.
 
         Attributes:
-            transform_type (str): The type of transform (one of
-                SilverPreTransform, SilverTransform, Gold).
-            use_pre_transform_preset (str): Indicates which pre-transform
-                preset to use if requested.
-            use_preset_table (str): Indicates which table to use within the
-                preset's transform type for Silver and Gold.
+            transform_type (str):
+                The type of transform (one of SilverPreTransform,
+                SilverTransform, Gold).
+            use_pre_transform_preset (str):
+                Indicates which pre-transform preset to use if requested.
+            use_preset_table (str):
+                Indicates which table to use within the preset's transform
+                type for Silver and Gold.
             filter (str):
+                Filter expression.
             post_filter (str):
+                Filter expression applied after the transform.
             preset_overrides (TransformRequest.Transform.PresetOverrides):
                 Overrides for the preset configuration.
-            add_fields (List[FieldSpec]): Additional field specifications
-                to add.
-            utils (FieldUtils): Utility configurations for handling fields.
+            add_fields (List[FieldSpec]):
+                Additional field specifications to add.
+            utils (FieldUtils):
+                Utility configurations for handling fields.
         """
 
         class PresetOverrides(BaseModel):
@@ -191,8 +202,8 @@ class TransformRequest(BaseModel):
             Preset overrides for a transform configuration.
 
             Attributes:
-                omit_fields (List[str]): A list of fields to omit from
-                    the preset.
+                omit_fields (List[str]):
+                    A list of fields to omit from the preset.
             """
 
             omit_fields: Optional[List[str]] = None
@@ -293,8 +304,8 @@ class TransformResponse(BaseModel):
     applied on the data.
 
     Attributes:
-        stages (List[TransformResponse.Stages]): A list of stages
-            representing each intermediate transform step.
+        stages (List[TransformResponse.Stages]):
+            A list of stages representing each intermediate transform step.
     """
 
     class Stages(BaseModel):
@@ -302,12 +313,13 @@ class TransformResponse(BaseModel):
         A stage in the transform response.
 
         Attributes:
-            transform_type (str): The type of transform applied in this stage
-                (one of SilverPreTransform, SilverTransform, Gold, Input).
-            columns (List[Dbui.TableColumnDetails]): A list of metadata about
-                the columns returned in this stage.
-            data (List[Dict[str, str]]): The data represented as a list
-                of dictionaries.
+            transform_type (str):
+                The type of transform applied in this stage (one of
+                SilverPreTransform, SilverTransform, Gold, Input).
+            columns (List[Dbui.TableColumnDetails]):
+                A list of metadata about the columns returned in this stage.
+            data (List[Dict[str, str]]):
+                The data represented as a list of dictionaries.
         """
 
         transform_type: str