google-pso-data-validator 7.6.0__tar.gz → 7.6.1__tar.gz

{google_pso_data_validator-7.6.0/google_pso_data_validator.egg-info → google_pso_data_validator-7.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 7.6.0
+Version: 7.6.1
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com

{google_pso_data_validator-7.6.0 → google_pso_data_validator-7.6.1}/data_validation/combiner.py

@@ -406,18 +406,18 @@ def _pivot_result(
                             consts.AGGREGATION_TYPE
                         ),
                         ibis.literal(validation.get_table_name(result_type)).name(
-                            "table_name"
+                            consts.COMBINER_TABLE_NAME
                         ),
                         # Cast to string to ensure types match, even when column
                         # name is NULL (such as for count aggregations).
                         ibis.literal(validation.get_column_name(result_type))
                         .cast("string")
-                        .name("column_name"),
+                        .name(consts.COMBINER_COLUMN_NAME),
                         primary_keys,
                         ibis.literal(validation.num_random_rows).name(
                             consts.NUM_RANDOM_ROWS
                         ),
-                        result[field].cast("string").name("agg_value"),
+                        result[field].cast("string").name(consts.COMBINER_AGG_VALUE),
                     )
                     + join_on_fields
                 )
@@ -469,11 +469,11 @@ def _join_pivots(
         + [
             source[consts.VALIDATION_TYPE],
             source[consts.AGGREGATION_TYPE],
-            source[consts.CONFIG_TABLE_NAME],
-            source["column_name"],
+            source[consts.COMBINER_TABLE_NAME],
+            source[consts.COMBINER_COLUMN_NAME],
             source[consts.CONFIG_PRIMARY_KEYS],
             source[consts.NUM_RANDOM_ROWS],
-            source["agg_value"],
+            source[consts.COMBINER_AGG_VALUE],
             differences[consts.VALIDATION_DIFFERENCE],
             differences[consts.VALIDATION_PCT_DIFFERENCE],
             differences[consts.VALIDATION_PCT_THRESHOLD],
@@ -488,12 +488,12 @@ def _join_pivots(
         source_difference[consts.AGGREGATION_TYPE]
         .fillna(target[consts.AGGREGATION_TYPE])
         .name(consts.AGGREGATION_TYPE),
-        source_difference["table_name"].name(consts.SOURCE_TABLE_NAME),
-        source_difference["column_name"].name(consts.SOURCE_COLUMN_NAME),
-        source_difference["agg_value"].name(consts.SOURCE_AGG_VALUE),
-        target["table_name"].name(consts.TARGET_TABLE_NAME),
-        target["column_name"].name(consts.TARGET_COLUMN_NAME),
-        target["agg_value"].name(consts.TARGET_AGG_VALUE),
+        source_difference[consts.COMBINER_TABLE_NAME].name(consts.SOURCE_TABLE_NAME),
+        source_difference[consts.COMBINER_COLUMN_NAME].name(consts.SOURCE_COLUMN_NAME),
+        source_difference[consts.COMBINER_AGG_VALUE].name(consts.SOURCE_AGG_VALUE),
+        target[consts.COMBINER_TABLE_NAME].name(consts.TARGET_TABLE_NAME),
+        target[consts.COMBINER_COLUMN_NAME].name(consts.TARGET_COLUMN_NAME),
+        target[consts.COMBINER_AGG_VALUE].name(consts.TARGET_AGG_VALUE),
         group_by_columns,
         source_difference[consts.CONFIG_PRIMARY_KEYS],
         source_difference[consts.NUM_RANDOM_ROWS],

{google_pso_data_validator-7.6.0 → google_pso_data_validator-7.6.1}/data_validation/config_manager.py

@@ -611,7 +611,7 @@ class ConfigManager(object):
                     self.build_config_calculated_fields(
                         [casefold_source_columns[field.casefold()]],
                         [casefold_target_columns[field.casefold()]],
-                        "rstrip",
+                        consts.CALC_FIELD_RSTRIP,
                         alias,
                         0,
                     )
@@ -812,16 +812,16 @@ class ConfigManager(object):
                 consts.CONFIG_FIELD_ALIAS
             ]
             depth = 1
-            calc_func = "length"
+            calc_func = consts.CALC_FIELD_LENGTH
         elif column_type in ["string", "!string"]:
-            calc_func = "length"
+            calc_func = consts.CALC_FIELD_LENGTH
 
         elif self._is_uuid(column_type, target_column_type):
             calc_func = consts.CONFIG_CAST
             cast_type = consts.CONFIG_CAST_UUID_STRING
 
         elif column_type in ["binary", "!binary"]:
-            calc_func = "byte_length"
+            calc_func = consts.CALC_FIELD_BYTE_LENGTH
 
         elif column_type in ["timestamp", "!timestamp", "date", "!date"]:
             if (
@@ -841,7 +841,7 @@ class ConfigManager(object):
                 ]
                 depth = 1
 
-            calc_func = "epoch_seconds"
+            calc_func = consts.CALC_FIELD_EPOCH_SECONDS
             if agg_type == consts.CONFIG_TYPE_SUM:
                 # It is possible to exceed int64 when summing epoch_seconds therefore cast to string.
                 # See issue 1391 for details.
@@ -1138,7 +1138,7 @@ class ConfigManager(object):
         target_table: "ibis.expr.types.Table",
     ) -> dict:
         """Mutates col_config to contain any overrides. Also returns col_config for convenience."""
-        if col_config["calc_type"] != "cast":
+        if col_config["calc_type"] != consts.CALC_FIELD_CAST:
             return col_config
 
         source_table_schema = {k: v for k, v in source_table.schema().items()}
@@ -1186,13 +1186,19 @@ class ConfigManager(object):
 
     def _get_order_of_operations(self, calc_type: str) -> List[str]:
         """Return order of operations for row validation."""
-        order_of_operations = ["cast", "ifnull", "rstrip"]
+        order_of_operations = [
+            consts.CALC_FIELD_CAST,
+            consts.CALC_FIELD_IFNULL,
+            consts.CALC_FIELD_RSTRIP,
+        ]
         if self.case_insensitive_match():
-            order_of_operations.append("upper")
-        if calc_type == "hash":
-            order_of_operations.extend(["concat", "hash"])
-        elif calc_type == "concat":
-            order_of_operations.append("concat")
+            order_of_operations.append(consts.CALC_FIELD_UPPER)
+        if calc_type == consts.CALC_FIELD_HASH:
+            order_of_operations.extend(
+                [consts.CALC_FIELD_CONCAT, consts.CALC_FIELD_HASH]
+            )
+        elif calc_type == consts.CALC_FIELD_CONCAT:
+            order_of_operations.append(consts.CALC_FIELD_CONCAT)
 
         return order_of_operations
 
@@ -1241,7 +1247,7 @@ class ConfigManager(object):
                 previous_level = [x for x in casefold_source_columns.keys()]
             else:
                 previous_level = [k for k, v in column_aliases.items() if v == i - 1]
-            if calc in ["concat", "hash"]:
+            if calc in [consts.CALC_FIELD_CONCAT, consts.CALC_FIELD_HASH]:
                 col = {}
                 col["source_reference"] = previous_level
                 col["target_reference"] = previous_level

{google_pso_data_validator-7.6.0 → google_pso_data_validator-7.6.1}/data_validation/consts.py

@@ -181,6 +181,11 @@ FAILED_SOURCE_NOT_IN_TARGET = "failed_rows_present_in_source_not_in_target"
 FAILED_TARGET_NOT_IN_SOURCE = "failed_rows_present_in_target_not_in_source"
 FAILED_PRESENT_IN_BOTH_TABLES = "failed_rows_present_in_both_source_and_target"
 
+# Combiner only constants
+COMBINER_TABLE_NAME = "dvt_table_name"
+COMBINER_COLUMN_NAME = "dvt_column_name"
+COMBINER_AGG_VALUE = "dvt_agg_value"
+
 # SQL Template Formatting
 # TODO: should this be managed in query_builder if that is the only place its used?
 COUNT_STAR = "{count_star}"
@@ -239,3 +244,15 @@ MAX_CONCAT_COLUMNS_DEFAULTS = {
     # Minimizing risk of: [Error 3556] Too many columns defined for this table.
     "teradata": 500,
 }
+
+# CalculatedField expression constants.
+CALC_FIELD_CAST = "cast"
+CALC_FIELD_CONCAT = "concat"
+CALC_FIELD_BYTE_LENGTH = "byte_length"
+CALC_FIELD_EPOCH_SECONDS = "epoch_seconds"
+CALC_FIELD_HASH = "hash"
+CALC_FIELD_IFNULL = "ifnull"
+CALC_FIELD_LENGTH = "length"
+CALC_FIELD_PADDED_CHAR_LENGTH = "padded_char_length"
+CALC_FIELD_RSTRIP = "rstrip"
+CALC_FIELD_UPPER = "upper"

{google_pso_data_validator-7.6.0 → google_pso_data_validator-7.6.1}/data_validation/query_builder/query_builder.py

@@ -358,6 +358,14 @@ class CalculatedField(object):
             fields,
         )
 
+    @staticmethod
+    def padded_char_length(config, fields):
+        return CalculatedField(
+            ibis.expr.types.StringValue.padded_char_length,
+            config,
+            fields,
+        )
+
     @staticmethod
     def byte_length(config, fields):
         return CalculatedField(
@@ -403,7 +411,7 @@ class CalculatedField(object):
     @staticmethod
     def custom(config, fields):
         """Returns a CalculatedField instance built for any custom ibis expression
-        i.e. 'ibis.expr.api.StringValue.replace'. For a list of supported functions,
+        e.g. 'ibis.expr.api.StringValue.replace'. For a list of supported functions,
         see https://github.com/ibis-project/ibis/blob/1.4.0/ibis/expr/api.py
         Args:
             expr (Str): A custom ibis expression to be used as a calc field

{google_pso_data_validator-7.6.0 → google_pso_data_validator-7.6.1}/data_validation/validation_builder.py

@@ -11,8 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import logging
 from copy import deepcopy
+import logging
+from typing import TYPE_CHECKING
 
 from data_validation import consts, metadata
 from data_validation.clients import get_max_in_list_size
@@ -25,6 +26,9 @@ from data_validation.query_builder.query_builder import (
     QueryBuilder,
 )
 
+if TYPE_CHECKING:
+    import ibis
+
 
 def list_to_sublists(id_list: list, max_size: int) -> list:
     """Return a list of items as a list of lists based on a max list length of max_size."""
@@ -87,6 +91,23 @@ class ValidationBuilder(object):
         else:
             return FilterField.isin(column_name, in_list)
 
+    def _is_padded_char(
+        self,
+        client: "ibis.backends.base.BaseBackend",
+        raw_column_metadata: dict,
+        column_name: str,
+    ) -> bool:
+        # Clients only need to implement _is_char_type_padded method if they support padded strings.
+        return (
+            (
+                client.is_char_type_padded(raw_column_metadata[column_name])
+                if raw_column_metadata.get(column_name)
+                else False
+            )
+            if hasattr(client, "is_char_type_padded") and raw_column_metadata
+            else False
+        )
+
     def clone(self):
         cloned_builder = ValidationBuilder(self.config_manager)
 
@@ -347,32 +368,63 @@ class ValidationBuilder(object):
             threshold=self.config_manager.threshold,
         )
 
-    def add_calc(self, calc_field):
+    def _get_calc_type(
+        self,
+        calc_field: dict,
+        column_name: str,
+        client: "ibis.backends.base.BaseBackend",
+        raw_data_types: dict,
+    ) -> str:
+        if calc_field[
+            consts.CONFIG_TYPE
+        ] == consts.CALC_FIELD_LENGTH and self._is_padded_char(
+            client,
+            raw_data_types,
+            column_name,
+        ):
+            calc_type = consts.CALC_FIELD_PADDED_CHAR_LENGTH
+        else:
+            calc_type = calc_field[consts.CONFIG_TYPE]
+        # Check if valid calc field and return correct object.
+        if not hasattr(CalculatedField, calc_type):
+            raise Exception("Unknown Calculation Type: {}".format(calc_type))
+        return calc_type
+
+    def add_calc(self, calc_field: dict):
         """Add CalculatedField to Queries
 
         Args:
             calc_field (Dict): An object with source, target, and cast info
         """
-        # prepare source and target payloads
+        # Prepare source and target payloads
         source_config = deepcopy(calc_field)
         source_fields = calc_field[consts.CONFIG_CALCULATED_SOURCE_COLUMNS]
         target_config = deepcopy(calc_field)
         target_fields = calc_field[consts.CONFIG_CALCULATED_TARGET_COLUMNS]
-        # grab calc field metadata
+        # Grab calc field metadata
         alias = calc_field[consts.CONFIG_FIELD_ALIAS]
-        calc_type = calc_field[consts.CONFIG_TYPE]
-        # check if valid calc field and return correct object
-        if not hasattr(CalculatedField, calc_type):
-            raise Exception("Unknown Calculation Type: {}".format(calc_type))
-        source_field = getattr(CalculatedField, calc_type)(
+        source_calc_type = self._get_calc_type(
+            calc_field,
+            source_fields[0],
+            self.source_client,
+            self.config_manager.get_source_raw_data_types(),
+        )
+        target_calc_type = self._get_calc_type(
+            calc_field,
+            target_fields[0],
+            self.target_client,
+            self.config_manager.get_target_raw_data_types(),
+        )
+
+        source_field = getattr(CalculatedField, source_calc_type)(
             config=source_config, fields=source_fields
         )
-        target_field = getattr(CalculatedField, calc_type)(
+        target_field = getattr(CalculatedField, target_calc_type)(
             config=target_config, fields=target_fields
         )
         self.source_builder.add_calculated_field(source_field)
         self.target_builder.add_calculated_field(target_field)
-        # register calc field under alias
+        # Register calc field under alias
         self.calculated_aliases[alias] = calc_field
 
     def get_source_query(self):

{google_pso_data_validator-7.6.0 → google_pso_data_validator-7.6.1/google_pso_data_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 7.6.0
+Version: 7.6.1
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com

{google_pso_data_validator-7.6.0 → google_pso_data_validator-7.6.1}/setup.py

@@ -18,7 +18,7 @@ import setuptools
 
 name = "google-pso-data-validator"
 description = "A package to enable easy data validation"
-version = "7.6.0"
+version = "7.6.1"
 release_status = "Development Status :: 4 - Beta"
 
 with open("README.md", "r") as fh:

{google_pso_data_validator-7.6.0 → google_pso_data_validator-7.6.1}/third_party/ibis/ibis_addon/operations.py

@@ -58,7 +58,7 @@ from ibis.backends.mysql.compiler import MySQLExprTranslator
 from ibis.backends.pandas.dispatch import execute_node
 from ibis.backends.pandas.execution.temporal import execute_epoch_seconds
 from ibis.backends.postgres.compiler import PostgreSQLExprTranslator
-from ibis.expr.types import BinaryValue, NumericValue, TemporalValue
+from ibis.expr.types import BinaryValue, NumericValue, StringValue, TemporalValue
 
 # Do not remove these lines, they trigger patching of Ibis code.
 import third_party.ibis.ibis_biquery.api  # noqa
@@ -106,6 +106,12 @@ class BinaryLength(ops.Value):
     output_shape = rlz.shape_like("arg")
 
 
+class PaddedCharLength(ops.Value):
+    arg = rlz.one_of([rlz.value(dt.String)])
+    output_dtype = dt.int32
+    output_shape = rlz.shape_like("arg")
+
+
 class ToChar(ops.Value):
     arg = rlz.one_of(
         [
@@ -128,6 +134,10 @@ def compile_binary_length(binary_value):
     return BinaryLength(binary_value).to_expr()
 
 
+def compile_padded_char_length(char_value):
+    return PaddedCharLength(char_value).to_expr()
+
+
 def compile_to_char(numeric_value, fmt):
     return ToChar(numeric_value, fmt=fmt).to_expr()
 
@@ -598,6 +608,8 @@ execute_epoch_seconds = execute_epoch_seconds_new
 
 BinaryValue.byte_length = compile_binary_length
 
+StringValue.padded_char_length = compile_padded_char_length
+
 NumericValue.to_char = compile_to_char
 TemporalValue.to_char = compile_to_char
 
@@ -614,6 +626,8 @@ AlchemyExprTranslator._registry[RawSQL] = format_raw_sql
 AlchemyExprTranslator._registry[ops.HashBytes] = format_hashbytes_alchemy
 ExprTranslator._registry[RawSQL] = format_raw_sql
 ExprTranslator._registry[ops.HashBytes] = format_hashbytes_base
+# Base length of padded string is the same as for a standard string.
+ExprTranslator._registry[PaddedCharLength] = ExprTranslator._registry[ops.StringLength]
 
 ImpalaExprTranslator._registry[ops.Cast] = sa_cast_hive
 ImpalaExprTranslator._registry[RawSQL] = format_raw_sql
@@ -639,6 +653,10 @@ PostgreSQLExprTranslator._registry[BinaryLength] = sa_format_binary_length
 PostgreSQLExprTranslator._registry[
     ops.ExtractEpochSeconds
 ] = postgres_registry.sa_epoch_seconds
+PostgreSQLExprTranslator._registry[
+    PaddedCharLength
+] = postgres_registry.sa_format_postgres_padded_char_length
+
 
 MsSqlExprTranslator._registry[ops.HashBytes] = mssql_registry.sa_format_hashbytes
 MsSqlExprTranslator._registry[RawSQL] = sa_format_raw_sql

{google_pso_data_validator-7.6.0 → google_pso_data_validator-7.6.1}/third_party/ibis/ibis_postgres/client.py

@@ -207,8 +207,15 @@ def _raw_column_metadata(
         )
 
 
+def _is_char_type_padded(self, char_type: Tuple) -> bool:
+    """Define this method if the backend supports character/string types that are padded and returns
+    padded values, which DVT may want to trim"""
+    return char_type[0] == "character"
+
+
 PostgresBackend._metadata = _metadata
 PostgresBackend.list_databases = list_schemas
 PostgresBackend.do_connect = do_connect
 PostgresBackend.list_primary_key_columns = _list_primary_key_columns
 PostgresBackend.raw_column_metadata = _raw_column_metadata
+PostgresBackend.is_char_type_padded = _is_char_type_padded

{google_pso_data_validator-7.6.0 → google_pso_data_validator-7.6.1}/third_party/ibis/ibis_postgres/registry.py

@@ -32,3 +32,12 @@ def sa_epoch_seconds(translator, op):
         sa.extract("epoch", sa.func.date_trunc(sa.sql.literal_column("'second'"), arg)),
         sa.BIGINT,
     )
+
+
+def sa_format_postgres_padded_char_length(translator, op):
+    """Inject concat of empty string into length(bpchar) to protect trailing spaces.
+
+    Without this workaround the bpchar value is implicitly cast to varchar and loses trailing spaces.
+    """
+    arg = translator.translate(op.arg)
+    return sa.func.char_length(sa.func.concat(arg, sa.text("''")))