dasl-client 1.0.6__py3-none-any.whl → 1.0.9__py3-none-any.whl

dasl_client/types/datasource.py

@@ -44,24 +44,28 @@ class FieldSpec(BaseModel):
 
     Attributes:
         name (Optional[str]):
-        comment (Optional[str]): The comment to apply to the field.
-        var_assert (Optional[List[FieldSpec.Assert]]): A list of SQL
-            expressions that must evaluate to true for every processed row.
-            If the assertion is false, an operational alert is raised using
-            'message' for each row.
-        var_from (Optional[str]): This field obtains its value from the source
-            column of this name. Use this to bring in a column from some
-            upstream table.
-        alias (Optional[str]): This field obtains its value from the
-            destination (transformed) column of this name. Use this to alias
-            a column from within the same table (ie. silver table). You cannot
-            alias a column from some upstream table.
-        expr (Optional[str]): This field obtains its value from the given SQL
-            expression.
-        literal (Optional[str]): This field obtains its value from the given
-            literal string. For other data types, use expr.
-        join (Optional[FieldSpec.Join]): This field obtains its value from
-            joining to another table.
+            The name of the field.
+        comment (Optional[str]):
+            The comment to apply to the field.
+        var_assert (Optional[List[FieldSpec.Assert]]):
+            A list of SQL expressions that must evaluate to true for every
+            processed row. If the assertion is false, an operational alert
+            is raised using 'message' for each row.
+        var_from (Optional[str]):
+            This field obtains its value from the source column of this name.
+            Use this to bring in a column from some upstream table.
+        alias (Optional[str]):
+            This field obtains its value from the destination (transformed)
+            column of this name. Use this to alias a column from within the
+            same table (ie. silver table). You cannot alias a column from
+            some upstream table.
+        expr (Optional[str]):
+            This field obtains its value from the given SQL expression.
+        literal (Optional[str]):
+            This field obtains its value from the given literal string. For
+            other data types, use expr.
+        join (Optional[FieldSpec.Join]):
+            This field obtains its value from joining to another table.
     """
 
     class Assert(BaseModel):
@@ -69,10 +73,12 @@ class FieldSpec(BaseModel):
         An assertion within a FieldSpec.
 
         Attributes:
-            expr (Optional[str]): The SQL expression that must evaluate to
-                true for every processed row.
-            message (Optional[str]): The message to include in the operational
-                alert if the assertion fails.
+            expr (Optional[str]):
+                The SQL expression that must evaluate to true for every
+                processed row.
+            message (Optional[str]):
+                The message to include in the operational alert if the
+                assertion fails.
         """
 
         expr: Optional[str] = None
@@ -100,13 +106,17 @@ class FieldSpec(BaseModel):
         A join expression within a FieldSpec.
 
         Attributes:
-            with_table (Optional[str]): The table to join to.
-            with_csv (Optional[FieldSpec.Join.WithCSV]): The CSV configuration
-                used for the join.
-            lhs (Optional[str]): The column in the source dataframe to join on.
-            rhs (Optional[str]): The column in the joined table to join on.
-            select (Optional[str]): A SQL expression to create the new field
-                from the joined dataset.
+            with_table (Optional[str]):
+                The table to join to.
+            with_csv (Optional[FieldSpec.Join.WithCSV]):
+                The CSV configuration used for the join.
+            lhs (Optional[str]):
+                The column in the source dataframe to join on.
+            rhs (Optional[str]):
+                The column in the joined table to join on.
+            select (Optional[str]):
+                A SQL expression to create the new field from the joined
+                dataset.
         """
 
         class WithCSV(BaseModel):
@@ -114,7 +124,8 @@ class FieldSpec(BaseModel):
             A CSV file used for joins within a FieldSpec.
 
             Attributes:
-                path (Optional[str]): The path to the CSV file.
+                path (Optional[str]):
+                    The path to the CSV file.
             """
 
             path: Optional[str] = None
@@ -213,8 +224,8 @@ class FieldUtils(BaseModel):
         unreferenced_columns (Optional[FieldUtils.UnreferencedColumns]):
             Defines whether columns not referenced in the FieldSpecs should
             be preserved or omitted.
-        json_extract (Optional[List[FieldUtils.JsonExtract]]): A list of
-            configurations for extracting JSON fields from a column.
+        json_extract (Optional[List[FieldUtils.JsonExtract]]):
+            A list of configurations for extracting JSON fields from a column.
     """
 
     class UnreferencedColumns(BaseModel):
@@ -222,14 +233,16 @@ class FieldUtils(BaseModel):
         Configuration related to unreferenced columns.
 
         Attributes:
-            preserve (Optional[bool]): Indicates whether columns not
-                referenced in the FieldSpecs should be preserved.
-            embed_column (Optional[str]): Specifies a name for a new column to
-                contain all unreferenced fields.
-            omit_columns (Optional[List[str]]): Lists columns to exclude from
-                the output.
-            duplicate_prefix (Optional[str]): Adds a prefix to resolve
-                ambiguous duplicate field names.
+            preserve (Optional[bool]):
+                Indicates whether columns not referenced in the FieldSpecs
+                should be preserved.
+            embed_column (Optional[str]):
+                Specifies a name for a new column to contain all unreferenced
+                fields.
+            omit_columns (Optional[List[str]]):
+                Lists columns to exclude from the output.
+            duplicate_prefix (Optional[str]):
+                Adds a prefix to resolve ambiguous duplicate field names.
         """
 
         preserve: Optional[bool] = None
@@ -263,14 +276,15 @@ class FieldUtils(BaseModel):
         Configuration for extracting JSON fields from table columns.
 
         Attributes:
-            source (Optional[str]): The column name containing JSON string(s)
-                to extract from.
-            omit_fields (Optional[List[str]]): Specifies high-level fields to
-                exclude from extraction.
-            duplicate_prefix (Optional[str]): Adds a prefix to resolve
-                duplicate field names during extraction.
-            embed_column (Optional[str]): Specifies a column name to store the
-                extracted JSON object.
+            source (Optional[str]):
+                The column name containing JSON string(s) to extract from.
+            omit_fields (Optional[List[str]]):
+                Specifies high-level fields to exclude from extraction.
+            duplicate_prefix (Optional[str]):
+                Adds a prefix to resolve duplicate field names during
+                extraction.
+            embed_column (Optional[str]):
+                Specifies a column name to store the extracted JSON object.
         """
 
         source: Optional[str] = None
@@ -333,12 +347,13 @@ class BronzeSpec(BaseModel):
     Configuration for bronze table within a DataSource.
 
     Attributes:
-        clustering (Optional[BronzeSpec.Clustering]): Describes optional
-            liquid clustering configuration for the bronze table.
-        bronze_table (Optional[str]): The name of the bronze table to create
-            and hold the imported data.
-        skip_bronze_loading (Optional[bool]): Indicates whether to skip the
-            bronze loading step.
+        clustering (Optional[BronzeSpec.Clustering]):
+            Describes optional liquid clustering configuration for the
+            bronze table.
+        bronze_table (Optional[str]):
+            The name of the bronze table to create and hold the imported data.
+        skip_bronze_loading (Optional[bool]):
+            Indicates whether to skip the bronze loading step.
     """
 
     class Clustering(BaseModel):
@@ -346,10 +361,11 @@ class BronzeSpec(BaseModel):
         Configuration of liquid clustering for a bronze table.
 
         Attributes:
-            column_names (Optional[List[str]]): List of column names to
-                include in liquid clustering.
-            time_column (Optional[str]): Name of the column that holds 'time'
-                information for clustering.
+            column_names (Optional[List[str]]):
+                List of column names to include in liquid clustering.
+            time_column (Optional[str]):
+                Name of the column that holds 'time' information for
+                clustering.
         """
 
         column_names: Optional[List[str]] = None
@@ -399,12 +415,12 @@ class SilverSpec(BaseModel):
     Configuration for silver table in a DataSource.
 
     Attributes:
-        bronze_tables (Optional[List[SilverSpec.BronzeTable]]): A list of
-            bronze tables to be joined for silver transformation.
-        pre_transform (Optional[SilverSpec.PreTransform]): Pretransformation
-            configuration.
-        transform (Optional[SilverSpec.Transform]): Transformation
-            configuration for silver processing.
+        bronze_tables (Optional[List[SilverSpec.BronzeTable]]):
+            A list of bronze tables to be joined for silver transformation.
+        pre_transform (Optional[SilverSpec.PreTransform]):
+            Pretransformation configuration.
+        transform (Optional[SilverSpec.Transform]):
+            Transformation configuration for silver processing.
     """
 
     class BronzeTable(BaseModel):
@@ -412,12 +428,18 @@ class SilverSpec(BaseModel):
         Reference to a bronze table for a silver table.
 
         Attributes:
-            name (Optional[str]): Name of the bronze table.
+            name (Optional[str]):
+                Name of the bronze table.
             streaming (Optional[bool]):
+                True if the input should be streamed from the bronze table.
             watermark (Optional[SilverSpec.BronzeTable.Watermark]):
-            alias (Optional[str]): Alias name for the table.
-            join_type (Optional[str]): How to join to the preceding table.
-            join_expr (Optional[str]): The join condition expression.
+                Bronze table watermark.
+            alias (Optional[str]):
+                Alias name for the table.
+            join_type (Optional[str]):
+                How to join to the preceding table.
+            join_expr (Optional[str]):
+                The join condition expression.
         """
 
         class Watermark(BaseModel):
@@ -425,12 +447,12 @@ class SilverSpec(BaseModel):
             Watermark for a bronze source table within a silver table.
 
             Attributes:
-                event_time_column (Optional[str]): Which column is the event
-                    time for the delay threshold.
-                delay_threshold (Optional[str]): A time duration string for
-                    the watermark delay.
-                drop_duplicates (Optional[List[str]]): Columns to pass to
-                    pyspark dropDuplicates.
+                event_time_column (Optional[str]):
+                    Which column is the event time for the delay threshold.
+                delay_threshold (Optional[str]):
+                    A time duration string for the watermark delay.
+                drop_duplicates (Optional[List[str]]):
+                    Columns to pass to pyspark dropDuplicates.
             """
 
             event_time_column: Optional[str] = None
@@ -496,18 +518,20 @@ class SilverSpec(BaseModel):
 
         Attributes:
             use_preset (Optional[str]):
+                Preset to use.
             skip_pre_transform (Optional[bool]):
-            custom (Optional[SilverSpec.PreTransform.Custom]): Custom
-                pretransform function and options.
-            filter (Optional[str]): A SQL filter to apply at the beginning of
-                the preTransform phase.
-            post_filter (Optional[str]): A SQL filter to apply at the end of
-                the preTransform phase.
+                If True, skip pre-transform entirely.
+            custom (Optional[SilverSpec.PreTransform.Custom]):
+                Custom pretransform function and options.
+            filter (Optional[str]):
+                A SQL filter to apply at the beginning of the preTransform
+                phase.
+            post_filter (Optional[str]):
+                A SQL filter to apply at the end of the preTransform phase.
             preset_overrides (Optional[SilverSpec.PreTransform.PresetOverrides]):
                 Overrides for preset filters.
-            add_fields (Optional[List[FieldSpec]]): User defined fields to
-                add to the transformation.
-            utils (Optional[FieldUtils]): Utilities for handling fields.
+            add_fields (Optional[List[FieldSpec]]):
+                User defined fields to add to the transformation.
         """
 
         class Custom(BaseModel):
@@ -544,8 +568,8 @@ class SilverSpec(BaseModel):
             Overrides for the preset.
 
             Attributes:
-                omit_fields (Optional[List[str]]): A list of fields to omit
-                    from the chosen preset.
+                omit_fields (Optional[List[str]]):
+                    A list of fields to omit from the chosen preset.
             """
 
             omit_fields: Optional[List[str]] = None
@@ -574,7 +598,6 @@ class SilverSpec(BaseModel):
         post_filter: Optional[str] = None
         preset_overrides: Optional["SilverSpec.PreTransform.PresetOverrides"] = None
         add_fields: Optional[List[FieldSpec]] = None
-        utils: Optional[FieldUtils] = None
 
         @staticmethod
         def from_api_obj(
@@ -595,7 +618,6 @@ class SilverSpec(BaseModel):
                     obj.preset_overrides
                 ),
                 add_fields=add_fields,
-                utils=FieldUtils.from_api_obj(obj.utils),
             )
 
         def to_api_obj(self) -> CoreV1DataSourceSpecSilverPreTransform:
@@ -611,7 +633,6 @@ class SilverSpec(BaseModel):
                 post_filter=self.post_filter,
                 preset_overrides=Helpers.maybe(to_api_obj, self.preset_overrides),
                 add_fields=add_fields,
-                utils=Helpers.maybe(to_api_obj, self.utils),
             )
 
     class Transform(BaseModel):
@@ -620,6 +641,7 @@ class SilverSpec(BaseModel):
 
         Attributes:
             skip_silver_transform (Optional[bool]):
+                If True, skip transform entirely.
             preset_overrides (Optional[SilverSpec.Transform.PresetOverrides]):
                 Preset overrides for the silver transformation.
         """
@@ -629,14 +651,12 @@ class SilverSpec(BaseModel):
             Overrides for preset transform settings.
 
             Attributes:
-                modify_tables (Optional[List[SilverSpec.Transform.
-                    PresetOverrides.ModifyTables]]): Modifications for
-                    existing tables.
-                omit_tables (Optional[List[str]]): A list of tables to omit
-                    from the preset.
-                add_tables (Optional[List[SilverSpec.Transform.PresetOverrides.
-                    AddTables]]): User defined tables to include in the
-                    transformation.
+                modify_tables (Optional[List[SilverSpec.Transform.PresetOverrides.ModifyTables]]):
+                    Modifications forexisting tables.
+                omit_tables (Optional[List[str]]):
+                    A list of tables to omit from the preset.
+                add_tables (Optional[List[SilverSpec.Transform.PresetOverrides.AddTables]]):
+                    User defined tables to include in the transformation.
             """
 
             class Custom(BaseModel):
@@ -913,12 +933,12 @@ class GoldSpec(BaseModel):
     Configuration for gold table in a DataSource.
 
     Attributes:
-        omit_tables (Optional[List[str]]): A list of tables to omit from the
-            preset.
-        modify_tables (Optional[List[GoldSpec.ModifyTables]]): Modifications
-            for existing gold table definitions.
-        add_tables (Optional[List[GoldSpec.AddTables]]): User defined tables
-            to add to the gold configuration.
+        omit_tables (Optional[List[str]]):
+            A list of tables to omit from the preset.
+        modify_tables (Optional[List[GoldSpec.ModifyTables]]):
+            Modifications for existing gold table definitions.
+        add_tables (Optional[List[GoldSpec.AddTables]]):
+            User defined tables to add to the gold configuration.
     """
 
     class ModifyTables(BaseModel):
@@ -927,16 +947,19 @@ class GoldSpec(BaseModel):
 
         Attributes:
             name (Optional[str]):
-            source_table (Optional[str]): Used to match against the preset's
-                gold stanzas input fields.
-            custom (Optional[GoldSpec.ModifyTables.Custom]): Custom function
-                for modifying tables.
-            omit_fields (Optional[List[str]]): A list of fields to omit.
-            add_fields (Optional[List[FieldSpec]]): Fields to add.
-            filter (Optional[str]): A SQL filter to apply before processing.
-            post_filter (Optional[str]): A SQL filter to apply after
-                processing.
-            utils (Optional[FieldUtils]): Utilities for field handling.
+                Table name.
+            source_table (Optional[str]):
+                Used to match against the preset's gold stanzas input fields.
+            custom (Optional[GoldSpec.ModifyTables.Custom]):
+                Custom function for modifying tables.
+            omit_fields (Optional[List[str]]):
+                A list of fields to omit.
+            add_fields (Optional[List[FieldSpec]]):
+                Fields to add.
+            filter (Optional[str]):
+                A SQL filter to apply before processing.
+            post_filter (Optional[str]):
+                A SQL filter to apply after processing.
         """
 
         class Custom(BaseModel):
@@ -979,7 +1002,6 @@ class GoldSpec(BaseModel):
         add_fields: Optional[List[FieldSpec]] = None
         filter: Optional[str] = None
         post_filter: Optional[str] = None
-        utils: Optional[FieldUtils] = None
 
         @staticmethod
         def from_api_obj(
@@ -998,7 +1020,6 @@ class GoldSpec(BaseModel):
                 add_fields=add_fields,
                 filter=obj.filter,
                 post_filter=obj.post_filter,
-                utils=FieldUtils.from_api_obj(obj.utils),
             )
 
         def to_api_obj(
@@ -1016,7 +1037,6 @@ class GoldSpec(BaseModel):
                 add_fields=add_fields,
                 filter=self.filter,
                 post_filter=self.post_filter,
-                utils=Helpers.maybe(to_api_obj, self.utils),
             )
 
     class AddTables(BaseModel):
@@ -1024,19 +1044,18 @@ class GoldSpec(BaseModel):
         Tables to add during gold table transformation.
 
         Attributes:
-            custom (Optional[GoldSpec.AddTables.Custom]): Custom function for
-                adding tables.
-            name (Optional[str]): The name of the table to add.
-            source_table (Optional[str]): The source table/dataframe for the
-                gold table.
-            filter (Optional[str]): A SQL filter to apply.
-            post_filter (Optional[str]): A SQL filter to apply after
-                processing.
-            override_liquid_columns (Optional[List[str]]): Columns to override
-                in the preset.
-            fields (Optional[List[FieldSpec]]): Field specifications for the
-                new table.
-            utils (Optional[FieldUtils]): Utilities for field handling.
+            custom (Optional[GoldSpec.AddTables.Custom]):
+                Custom function for adding tables.
+            name (Optional[str]):
+                The name of the table to add.
+            source_table (Optional[str]):
+                The source table/dataframe for the gold table.
+            filter (Optional[str]):
+                A SQL filter to apply.
+            post_filter (Optional[str]):
+                A SQL filter to apply after processing.
+            fields (Optional[List[FieldSpec]]):
+                Field specifications for the new table.
         """
 
         class Custom(BaseModel):
@@ -1077,9 +1096,7 @@ class GoldSpec(BaseModel):
         custom: Optional["GoldSpec.AddTables.Custom"] = None
         filter: Optional[str] = None
         post_filter: Optional[str] = None
-        override_liquid_columns: Optional[List[str]] = None
         fields: Optional[List[FieldSpec]] = None
-        utils: Optional[FieldUtils] = None
 
         @staticmethod
         def from_api_obj(
@@ -1096,9 +1113,7 @@ class GoldSpec(BaseModel):
                 source_table=obj.source_table,
                 filter=obj.filter,
                 post_filter=obj.post_filter,
-                override_liquid_columns=obj.override_liquid_columns,
                 fields=fields,
-                utils=FieldUtils.from_api_obj(obj.utils),
             )
 
         def to_api_obj(self) -> CoreV1DataSourceSpecGoldPresetOverridesAddTablesInner:
@@ -1112,9 +1127,7 @@ class GoldSpec(BaseModel):
                 source_table=self.source_table,
                 filter=self.filter,
                 post_filter=self.post_filter,
-                override_liquid_columns=self.override_liquid_columns,
                 fields=fields,
-                utils=Helpers.maybe(to_api_obj, self.utils),
             )
 
     omit_tables: Optional[List[str]] = None
@@ -1167,20 +1180,28 @@ class DataSource(BaseModel):
     A DataSource resource.
 
     Attributes:
-        metadata (Optional[Metadata]): Standard object metadata.
-        source (Optional[str]): The name of the originator of the data.
-        source_type (Optional[str]): The type of data being imported.
-        schedule (Optional[Schedule]): The schedule for data ingestion.
-        custom (Optional[DataSource.CustomNotebook]): A custom notebook for
-            the datasource.
-        use_preset (Optional[str]): The name of the preset to use for this
-            data source.
-        autoloader (Optional[DataSource.Autoloader]): Autoloader configuration.
-        bronze (Optional[BronzeSpec]): Bronze table configuration.
-        silver (Optional[SilverSpec]): Silver transformation configuration.
-        gold (Optional[GoldSpec]): Gold transformation configuration.
-        status (Optional[ResourceStatus]): The current status of the
-            datasource.
+        metadata (Optional[Metadata]):
+            Standard object metadata.
+        source (Optional[str]):
+            The name of the originator of the data.
+        source_type (Optional[str]):
+            The type of data being imported.
+        schedule (Optional[Schedule]):
+            The schedule for data ingestion.
+        custom (Optional[DataSource.CustomNotebook]):
+            A custom notebook for the datasource.
+        use_preset (Optional[str]):
+            The name of the preset to use for this data source.
+        autoloader (Optional[DataSource.Autoloader]):
+            Autoloader configuration.
+        bronze (Optional[BronzeSpec]):
+            Bronze table configuration.
+        silver (Optional[SilverSpec]):
+            Silver transformation configuration.
+        gold (Optional[GoldSpec]):
+            Gold transformation configuration.
+        status (Optional[ResourceStatus]):
+            The current status of the datasource.
     """
 
     class CustomNotebook(BaseModel):
@@ -1188,8 +1209,8 @@ class DataSource(BaseModel):
         A custom notebook for generating data.
 
         Attributes:
-            notebook (Optional[str]): Path to the notebook in the Databricks
-                workspace.
+            notebook (Optional[str]):
+                Path to the notebook in the Databricks workspace.
         """
 
         notebook: Optional[str] = None
@@ -1214,11 +1235,12 @@ class DataSource(BaseModel):
         Autoloader configuration for the DataSource.
 
         Attributes:
-            format (Optional[str]): The format of the data (e.g., json,
-                parquet, csv, etc.).
-            location (str): External location for the volume in Unity Catalog.
-            schema_file (Optional[str]): An optional file containing the
-                schema of the data source.
+            format (Optional[str]):
+                The format of the data (e.g., json, parquet, csv, etc.).
+            location (str):
+                External location for the volume in Unity Catalog.
+            schema_file (Optional[str]):
+                An optional file containing the schema of the data source.
             cloud_files (Optional[DataSource.Autoloader.CloudFiles]):
                 CloudFiles configuration.
         """