dasl-client 1.0.18__tar.gz → 1.0.20__tar.gz

{dasl_client-1.0.18 → dasl_client-1.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dasl_client
-Version: 1.0.18
+Version: 1.0.20
 Summary: The DASL client library used for interacting with the DASL workspace
 Home-page: https://github.com/antimatter/asl
 Author: Antimatter Team

{dasl_client-1.0.18 → dasl_client-1.0.20}/dasl_client/preset_development/preview_engine.py

@@ -175,6 +175,7 @@ class PreviewEngine:
         self,
         input_df: DataFrame,
         stage_dataframes: Tuple[DataFrame, Dict[str, DataFrame], Dict[str, DataFrame]],
+        gold_table_catalog: str,
         gold_table_schema: str,
     ) -> None:
         """
@@ -247,12 +248,13 @@ class PreviewEngine:
             # NOTE: Name is stored as Gold_name/Silver_input. So we need to get just the Gold table
             # name that we are comparing the dataframe metadata to.
             name = name.split("/")[0]
+            fqn_gold_table_name = f"{self.force_apply_backticks(gold_table_catalog)}.{self.force_apply_backticks(gold_table_schema)}.{self.force_apply_backticks(name)}"
 
-            if not self._spark.catalog.tableExists(f"{gold_table_schema}.{name}"):
+            if not self._spark.catalog.tableExists(f"{fqn_gold_table_name}"):
                 raise UnknownGoldTableError(name, gold_table_schema)
 
             # Performs the type check.
-            delta_df = self._spark.table(f"`{gold_table_schema}`.`{name}`").limit(0)
+            delta_df = self._spark.table(f"{fqn_gold_table_name}").limit(0)
             unioned_df = delta_df.unionByName(df, allowMissingColumns=True)
 
             # Now we check no new columns.
@@ -293,6 +295,25 @@ class PreviewEngine:
             d("Resultant gold table preview", 3)
             display(unioned_df)
 
+    def is_backtick_escaped(self, name: str) -> bool:
+        """
+        check if a given (column) name is backtick escaped or not
+        :param name: column name
+        :return: bool
+        """
+        return name.startswith("`") and name.endswith("`")
+
+    def force_apply_backticks(self, name: str) -> str:
+        """
+        forces application of backticks to the given (column) name as a single unit
+        if it already has backticks this is a noop
+        :param name: column name
+        :return: str
+        """
+        if self.is_backtick_escaped(name):
+            return name
+        return f"`{name}`"
+
     def evaluate(self, gold_table_schema: str, display: bool = True) -> None:
         """
         Evaluates the loaded preset YAML using the input datasource configuration to load
@@ -302,8 +323,8 @@ class PreviewEngine:
         s = gold_table_schema.split(".")
         if len(s) != 2:
             raise InvalidGoldTableSchemaError(gold_table_schema)
-        catalog_name = s[0]
-        schema_name = s[1]
+        catalog_name = s[0].lstrip("`").rstrip("`")
+        schema_name = s[1].lstrip("`").rstrip("`")
         if any(
             row.catalog == catalog_name
             for row in self._spark.sql("SHOW CATALOGS").collect()
@@ -352,7 +373,12 @@ class PreviewEngine:
             self._bronze = df
             self._result_df_map = self._run(df)
             if display:
-                self._render_output(df, self._result_df_map, gold_table_schema)
+                self._render_output(
+                    df,
+                    self._result_df_map,
+                    self.force_apply_backticks(catalog_name),
+                    self.force_apply_backticks(schema_name),
+                )
 
     def results(
         self,

{dasl_client-1.0.18 → dasl_client-1.0.20}/dasl_client/preset_development/stage.py

@@ -224,10 +224,15 @@ class Stage:
 
         referenced_columns = self._referenced_columns()
         omitted_columns = self._omitted_columns()
+        # omitted and referenced cols MAY be wrapped in backticks
+        # so we need to check for their existence here on top of just plain col name
         preserved_columns = [
             col
             for col in df.columns
-            if col not in referenced_columns and col not in omitted_columns
+            if col not in referenced_columns
+            and col not in omitted_columns
+            and self.force_apply_backticks(col) not in referenced_columns
+            and self.force_apply_backticks(col) not in omitted_columns
         ]
         duplicate_prefix = self._duplicate_prefix()
         column_names = self._column_names()

{dasl_client-1.0.18 → dasl_client-1.0.20}/dasl_client/types/content.py

@@ -107,7 +107,6 @@ class GoldPreset(BaseModel):
     filter: Optional[str] = None
     post_filter: Optional[str] = None
     fields: Optional[List[Optional[FieldSpec]]] = None
-    utils: Optional[FieldUtils] = None
 
     @staticmethod
     def from_api_obj(
@@ -126,7 +125,6 @@ class GoldPreset(BaseModel):
             filter=obj.filter,
             post_filter=obj.post_filter,
             fields=fields,
-            utils=FieldUtils.from_api_obj(obj.utils),
         )
 
 

{dasl_client-1.0.18 → dasl_client-1.0.20}/dasl_client/types/dbui.py

@@ -263,8 +263,6 @@ class TransformRequest(BaseModel):
             transform_type (str):
                 The type of transform (one of SilverPreTransform,
                 SilverTransform, Gold).
-            use_pre_transform_preset (str):
-                Indicates which pre-transform preset to use if requested.
             use_preset_table (str):
                 Indicates which table to use within the preset's transform
                 type for Silver and Gold.
@@ -309,7 +307,6 @@ class TransformRequest(BaseModel):
                 )
 
         transform_type: str
-        use_pre_transform_preset: Optional[str] = None
         use_preset_table: Optional[str] = None
         filter: Optional[str] = None
         post_filter: Optional[str] = None
@@ -328,7 +325,6 @@ class TransformRequest(BaseModel):
                 add_fields = [FieldSpec.from_api_obj(item) for item in obj.add_fields]
             return TransformRequest.Transform(
                 transform_type=obj.transform_type,
-                use_pre_transform_preset=obj.use_pre_transform_preset,
                 use_preset_table=obj.use_preset_table,
                 filter=obj.filter,
                 post_filter=obj.post_filter,
@@ -346,7 +342,6 @@ class TransformRequest(BaseModel):
             to_api_obj = lambda o: o.to_api_obj()
             return DbuiV1TransformRequestTransformsInner(
                 transform_type=self.transform_type,
-                use_pre_transform_preset=self.use_pre_transform_preset,
                 use_preset_table=self.use_preset_table,
                 filter=self.filter,
                 post_filter=self.post_filter,

{dasl_client-1.0.18 → dasl_client-1.0.20}/dasl_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dasl_client
-Version: 1.0.18
+Version: 1.0.20
 Summary: The DASL client library used for interacting with the DASL workspace
 Home-page: https://github.com/antimatter/asl
 Author: Antimatter Team

{dasl_client-1.0.18 → dasl_client-1.0.20}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dasl_client"
-version = "1.0.18"
+version = "1.0.20"
 description = "The DASL client library used for interacting with the DASL workspace"
 readme = "README.md"
 authors = [

{dasl_client-1.0.18 → dasl_client-1.0.20}/test/test_api_surface.py

@@ -81,6 +81,7 @@ def test_workspace_config(api_client):
 def test_minimal_data_source(api_client):
     base_data_source = DataSource(
         source="test",
+        source_type="test",
         schedule=Schedule(
             at_least_every="2h",
             enabled=True,
@@ -89,8 +90,10 @@ def test_minimal_data_source(api_client):
             bronze_table="test_bronze_table",
             skip_bronze_loading=False,
         ),
-        silver=SilverSpec(),
-        gold=GoldSpec(),
+        autoloader=DataSource.Autoloader(
+            location="s3://aws-security-data-lake-us-east-1-k8vskbicklrtekgxvyufaavf36jjql/aws/S3_DATA/2.0/region=us-east-1/",
+            format="json",
+        ),
     )
 
     base_ds_1 = api_client.create_datasource("test_1", base_data_source)

{dasl_client-1.0.18 → dasl_client-1.0.20}/test/test_marshaling.py

@@ -712,7 +712,6 @@ def test_transform_request_marshal_unmarshal():
         transforms=[
             TransformRequest.Transform(
                 transform_type="Gold",
-                use_pre_transform_preset="pre_preset",
                 use_preset_table="table_name",
                 filter="col > 0",
                 post_filter="col < 100",
@@ -784,7 +783,6 @@ def test_transform_request_marshal_unmarshal():
             ),
             TransformRequest.Transform(
                 transform_type="SilverTransform",
-                use_pre_transform_preset="pre_preset_b",
                 use_preset_table="table_b",
                 filter="col >= 10",
                 post_filter="col <= 50",