google-pso-data-validator 8.1.0__tar.gz → 8.2.0__tar.gz

{google_pso_data_validator-8.1.0/google_pso_data_validator.egg-info → google_pso_data_validator-8.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 8.1.0
+Version: 8.2.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -449,6 +449,8 @@ data-validation
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
                         File containing the target sql command. Supports GCS and local paths.
+  [--grouped-columns or -gc GROUPED_COLUMNS]
+                        Comma separated list of columns for Group By i.e col_a,col_b
   [--count COLUMNS]     Comma separated list of columns for count or * for all columns
   [--sum COLUMNS]       Comma separated list of columns for sum or * for all numeric
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/README.md

@@ -390,6 +390,8 @@ data-validation
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
                         File containing the target sql command. Supports GCS and local paths.
+  [--grouped-columns or -gc GROUPED_COLUMNS]
+                        Comma separated list of columns for Group By i.e col_a,col_b
   [--count COLUMNS]     Comma separated list of columns for count or * for all columns
   [--sum COLUMNS]       Comma separated list of columns for sum or * for all numeric
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/data_validation/__init__.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from pkg_resources import get_distribution
+from importlib.metadata import version
 
 
-__version__ = get_distribution("google-pso-data-validator").version
+__version__ = version("google-pso-data-validator")
 __all__ = ["__version__"]

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/data_validation/__main__.py

@@ -294,10 +294,7 @@ def build_config_from_args(args: "Namespace", config_manager: ConfigManager):
         and args.custom_query_type == consts.COLUMN_VALIDATION.lower()
     ):
         config_manager.append_aggregates(get_aggregate_config(args, config_manager))
-        if (
-            config_manager.validation_type == consts.COLUMN_VALIDATION
-            and args.grouped_columns  # grouped_columns not supported in custom queries - at least now.
-        ):
+        if args.grouped_columns:
             grouped_columns = cli_tools.get_arg_list(args.grouped_columns)
             config_manager.append_query_groups(
                 config_manager.build_column_configs(grouped_columns)

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/data_validation/cli_tools.py

@@ -948,6 +948,11 @@ def _configure_custom_query_column_parser(custom_query_column_parser):
                 equivalent) that are comparable across all other supported SQL engines. This option may produce
                 unreliable results.""",
     )
+    optional_arguments.add_argument(
+        "--grouped-columns",
+        "-gc",
+        help="Comma separated list of columns to group results by, e.g. 'col_a,col_b'. Applies to column validations, including custom-query column.",
+    )
     optional_arguments.add_argument(
         "--exclude-columns",
         "-ec",

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/data_validation/clients.py

@@ -69,20 +69,20 @@ def _raise_missing_client_error(msg):
 # Teradata requires teradatasql and licensing
 try:
     from third_party.ibis.ibis_teradata.api import teradata_connect
-except Exception:
+except ImportError:
     msg = "pip install teradatasql (requires Teradata licensing)"
     teradata_connect = _raise_missing_client_error(msg)
 
 # Oracle requires python-oracldb driver
 try:
     from third_party.ibis.ibis_oracle.api import oracle_connect
-except Exception:
+except ImportError:
     oracle_connect = _raise_missing_client_error("pip install oracledb")
 
 # Snowflake requires snowflake-connector-python and snowflake-sqlalchemy
 try:
     from third_party.ibis.ibis_snowflake.api import snowflake_connect
-except Exception:
+except ImportError:
     snowflake_connect = _raise_missing_client_error(
         "pip install snowflake-connector-python && pip install snowflake-sqlalchemy"
     )
@@ -90,8 +90,7 @@ except Exception:
 # DB2 requires ibm_db_sa
 try:
     from third_party.ibis.ibis_db2.api import db2_connect
-except Exception as e:
-    logging.error(f"Exception {str(e)} while importing db2_connect")
+except ImportError:
     db2_connect = _raise_missing_client_error("pip install ibm_db_sa")
 
 

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/data_validation/config_manager.py

@@ -664,6 +664,30 @@ class ConfigManager(object):
                 isinstance(source_type, dt.UUID) or isinstance(target_type, dt.UUID)
             )
 
+    def _is_sql_server_text(
+        self, source_column_name: str, target_column_name: str
+    ) -> bool:
+        """Returns True when either source or target column is of SQL Server text data type.
+
+        This is because SQL Server text does not support the len() function to get the length in characters.
+        Instead we must fall back to using ByteLength."""
+
+        raw_source_types = self.get_source_raw_data_types()
+        raw_target_types = self.get_target_raw_data_types()
+        text_types = ("text", "ntext")
+        return bool(
+            (
+                self.source_client.name == "mssql"
+                and raw_source_types
+                and raw_source_types.get(source_column_name, [None])[0] in text_types
+            )
+            or (
+                self.target_client.name == "mssql"
+                and raw_target_types
+                and raw_target_types.get(target_column_name, [None])[0] in text_types
+            )
+        )
+
     def build_config_comparison_fields(self, fields, depth=None):
         """Return list of field config objects."""
         field_configs = []
@@ -809,7 +833,10 @@ class ConfigManager(object):
             depth = 1
             calc_func = consts.CALC_FIELD_LENGTH
         elif column_type in ["string", "!string"]:
-            calc_func = consts.CALC_FIELD_LENGTH
+            if self._is_sql_server_text(source_column, target_column):
+                calc_func = consts.CALC_FIELD_BYTE_LENGTH
+            else:
+                calc_func = consts.CALC_FIELD_LENGTH
 
         elif self._is_uuid(column_type, target_column_type):
             calc_func = consts.CONFIG_CAST

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0/google_pso_data_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 8.1.0
+Version: 8.2.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -449,6 +449,8 @@ data-validation
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
                         File containing the target sql command. Supports GCS and local paths.
+  [--grouped-columns or -gc GROUPED_COLUMNS]
+                        Comma separated list of columns for Group By i.e col_a,col_b
   [--count COLUMNS]     Comma separated list of columns for count or * for all columns
   [--sum COLUMNS]       Comma separated list of columns for sum or * for all numeric
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/google_pso_data_validator.egg-info/SOURCES.txt

@@ -45,6 +45,7 @@ third_party/ibis/ibis_addon/api.py
 third_party/ibis/ibis_addon/operations.py
 third_party/ibis/ibis_bigquery/__init__.py
 third_party/ibis/ibis_bigquery/api.py
+third_party/ibis/ibis_bigquery/registry.py
 third_party/ibis/ibis_cloud_spanner/__init__.py
 third_party/ibis/ibis_cloud_spanner/api.py
 third_party/ibis/ibis_cloud_spanner/client.py

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/setup.py

@@ -18,7 +18,7 @@ import setuptools
 
 name = "google-pso-data-validator"
 description = "A package to enable easy data validation"
-version = "8.1.0"
+version = "8.2.0"
 release_status = "Development Status :: 4 - Beta"
 
 with open("README.md", "r") as fh:

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/third_party/ibis/ibis_addon/operations.py

@@ -48,10 +48,7 @@ from ibis.backends.bigquery.client import (
     _LEGACY_TO_STANDARD as _BQ_LEGACY_TO_STANDARD,
 )
 from ibis.backends.bigquery.compiler import BigQueryExprTranslator
-from ibis.backends.bigquery.registry import (
-    STRFTIME_FORMAT_FUNCTIONS as BQ_STRFTIME_FORMAT_FUNCTIONS,
-    bigquery_cast,
-)
+from ibis.backends.bigquery.registry import bigquery_cast
 from ibis.backends.impala.compiler import ImpalaExprTranslator
 from ibis.backends.mssql.compiler import MsSqlExprTranslator
 from ibis.backends.mysql.compiler import MySQLExprTranslator
@@ -62,6 +59,7 @@ from ibis.expr.types import BinaryValue, NumericValue, StringValue, TemporalValu
 
 # Do not remove these lines, they trigger patching of Ibis code.
 import third_party.ibis.ibis_bigquery.api  # noqa
+from third_party.ibis.ibis_bigquery import registry as bigquery_registry
 import third_party.ibis.ibis_mysql.compiler  # noqa
 from third_party.ibis.ibis_mssql import registry as mssql_registry
 from third_party.ibis.ibis_postgres import registry as postgres_registry
@@ -101,7 +99,7 @@ NAT_INT64_MIN_IN_SECONDS = np.iinfo(np.int64).min // 1_000_000_000
 
 
 class BinaryLength(ops.Value):
-    arg = rlz.one_of([rlz.value(dt.Binary)])
+    arg = rlz.one_of([rlz.value(dt.Binary), rlz.value(dt.String)])
     output_dtype = dt.int32
     output_shape = rlz.shape_like("arg")
 
@@ -154,24 +152,6 @@ def bigquery_cast_to_binary_generate(compiled_arg, from_, to):
     return f"FROM_HEX({compiled_arg})"
 
 
-def format_hash_bigquery(translator, op):
-    arg = translator.translate(op.arg)
-    if op.how == "farm_fingerprint":
-        return f"FARM_FINGERPRINT({arg})"
-    else:
-        raise ValueError(f"unexpected value for 'how': {op.how}")
-
-
-def format_hashbytes_bigquery(translator, op):
-    arg = translator.translate(op.arg)
-    if op.how == "sha256":
-        return f"TO_HEX(SHA256({arg}))"
-    elif op.how == "farm_fingerprint":
-        return f"FARM_FINGERPRINT({arg})"
-    else:
-        raise ValueError(f"unexpected value for 'how': {op.how}")
-
-
 def format_hashbytes_teradata(translator, op):
     arg = translator.translate(op.arg)
     if op.how == "sha256":
@@ -184,30 +164,6 @@ def format_hashbytes_teradata(translator, op):
         raise ValueError(f"unexpected value for 'how': {op.how}")
 
 
-def strftime_bigquery(translator, op):
-    """Timestamp formatting."""
-    arg = op.arg
-    format_str = op.format_str
-    arg_type = arg.output_dtype
-    strftime_format_func_name = BQ_STRFTIME_FORMAT_FUNCTIONS[type(arg_type)]
-    fmt_string = translator.translate(format_str)
-    # Deal with issue 1181 due a GoogleSQL bug with dates before 1000 CE affects both date and timestamp types
-    if format_str.value.startswith("%Y"):
-        fmt_string = fmt_string.replace("%Y", "%E4Y", 1)
-    arg_formatted = translator.translate(arg)
-    if isinstance(arg_type, dt.Timestamp):
-        return "FORMAT_{}({}, {}({}), {!r})".format(
-            strftime_format_func_name,
-            fmt_string,
-            strftime_format_func_name,
-            arg_formatted,
-            arg_type.timezone if arg_type.timezone is not None else "UTC",
-        )
-    return "FORMAT_{}({}, {})".format(
-        strftime_format_func_name, fmt_string, arg_formatted
-    )
-
-
 def strftime_mysql(translator, op):
     arg = op.arg
     format_string = op.format_str
@@ -528,10 +484,10 @@ TemporalValue.to_char = compile_to_char
 # so we can piggy back Ibis code rather than writing metadata queries for all engines.
 BaseAlchemyBackend.dvt_list_tables = _dvt_list_tables
 
-BigQueryExprTranslator._registry[ops.HashBytes] = format_hashbytes_bigquery
+BigQueryExprTranslator._registry[ops.HashBytes] = bigquery_registry.format_hashbytes
 BigQueryExprTranslator._registry[RawSQL] = format_raw_sql
-BigQueryExprTranslator._registry[ops.Strftime] = strftime_bigquery
-BigQueryExprTranslator._registry[BinaryLength] = sa_format_binary_length
+BigQueryExprTranslator._registry[ops.Strftime] = bigquery_registry.strftime
+BigQueryExprTranslator._registry[BinaryLength] = bigquery_registry.format_binary_length
 
 AlchemyExprTranslator._registry[RawSQL] = format_raw_sql
 AlchemyExprTranslator._registry[ops.HashBytes] = format_hashbytes_alchemy
@@ -580,6 +536,7 @@ MsSqlExprTranslator._registry[RawSQL] = sa_format_raw_sql
 MsSqlExprTranslator._registry[ops.IfNull] = sa_fixed_arity(sa.func.isnull, 2)
 MsSqlExprTranslator._registry[ops.StringJoin] = mssql_registry.sa_string_join
 MsSqlExprTranslator._registry[ops.RandomScalar] = mssql_registry.sa_format_new_id
+MsSqlExprTranslator._registry[ops.StringLength] = mssql_registry.sa_format_string_length
 MsSqlExprTranslator._registry[ops.Strftime] = mssql_registry.strftime
 MsSqlExprTranslator._registry[ops.Cast] = mssql_registry.sa_cast_mssql
 MsSqlExprTranslator._registry[BinaryLength] = mssql_registry.sa_format_binary_length
@@ -614,7 +571,7 @@ if Db2ExprTranslator:
     ]
 
 SpannerExprTranslator._registry[RawSQL] = format_raw_sql
-SpannerExprTranslator._registry[ops.HashBytes] = format_hashbytes_bigquery
+SpannerExprTranslator._registry[ops.HashBytes] = bigquery_registry.format_hashbytes
 SpannerExprTranslator._registry[BinaryLength] = sa_format_binary_length
 
 if TeradataExprTranslator:

google_pso_data_validator-8.2.0/third_party/ibis/ibis_bigquery/registry.py

@@ -0,0 +1,57 @@
+# Copyright 2025 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ibis.backends.bigquery.registry import (
+    STRFTIME_FORMAT_FUNCTIONS as BQ_STRFTIME_FORMAT_FUNCTIONS,
+)
+import ibis.expr.datatypes as dt
+
+
+def format_hashbytes(translator, op):
+    arg = translator.translate(op.arg)
+    if op.how == "sha256":
+        return f"TO_HEX(SHA256({arg}))"
+    elif op.how == "farm_fingerprint":
+        return f"FARM_FINGERPRINT({arg})"
+    else:
+        raise ValueError(f"unexpected value for 'how': {op.how}")
+
+
+def strftime(translator, op):
+    """Timestamp formatting."""
+    arg = op.arg
+    format_str = op.format_str
+    arg_type = arg.output_dtype
+    strftime_format_func_name = BQ_STRFTIME_FORMAT_FUNCTIONS[type(arg_type)]
+    fmt_string = translator.translate(format_str)
+    # Deal with issue 1181 due a GoogleSQL bug with dates before 1000 CE affects both date and timestamp types
+    if format_str.value.startswith("%Y"):
+        fmt_string = fmt_string.replace("%Y", "%E4Y", 1)
+    arg_formatted = translator.translate(arg)
+    if isinstance(arg_type, dt.Timestamp):
+        return "FORMAT_{}({}, {}({}), {!r})".format(
+            strftime_format_func_name,
+            fmt_string,
+            strftime_format_func_name,
+            arg_formatted,
+            arg_type.timezone if arg_type.timezone is not None else "UTC",
+        )
+    return "FORMAT_{}({}, {})".format(
+        strftime_format_func_name, fmt_string, arg_formatted
+    )
+
+
+def format_binary_length(translator, op):
+    arg = translator.translate(op.arg)
+    return f"LENGTH({arg})"

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/third_party/ibis/ibis_db2/__init__.py

@@ -104,7 +104,7 @@ class Backend(BaseAlchemyBackend):
 
     def raw_column_metadata_not_implemented(
         self, database: str = None, table: str = None, query: str = None
-    ) -> list[Tuple]:
+    ) -> list:
         """Define this method to allow DVT to test if backend specific transformations may be needed for comparison.
         Partner method to _metadata that retains raw data type information instead of converting
         to Ibis types.  This works in the same way as _metadata by running a query over the DVT
@@ -117,7 +117,7 @@ class Backend(BaseAlchemyBackend):
             list: A list of tuples containing the standard 7 DB API fields:
                   https://peps.python.org/pep-0249/#description
         """
-        return ()
+        return []
 
     def is_char_type_padded(self, char_type: Tuple) -> bool:
         """Define this method if the backend supports character/string types that are padded and returns

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/third_party/ibis/ibis_mssql/registry.py

@@ -84,9 +84,25 @@ def sa_epoch_seconds(translator, op):
     )
 
 
+def sa_format_string_length(translator, op):
+    """Calculate SQL Server string length in characters, not bytes.
+
+    This uses len() rather than datalength() to give an output compatible with BigQuery.
+    We need to protect trailing spaces due to an odd quirk in SQL Server len()
+    behaviour that ignores them, so this is done by first replacing spaces with
+    underscores before passing the string to len().
+
+    The len expression is cast to BIGINT to prevent int overflow for large tables."""
+    arg = translator.translate(op.arg)
+    return sa.func.cast(sa.func.len(sa.func.replace(arg, " ", "_")), sa.BIGINT)
+
+
 def sa_format_binary_length(translator, op):
+    """Calculate SQL Server bytes/string length in bytes, not characters.
+
+    The len expression is cast to BIGINT to prevent int overflow for large tables."""
     arg = translator.translate(op.arg)
-    return sa.func.datalength(arg)
+    return sa.func.cast(sa.func.datalength(arg), sa.BIGINT)
 
 
 def sa_format_hashbytes(translator, op):
@@ -130,6 +146,12 @@ def sa_cast_mssql(t, op):
         formatted_value = sa.func.format(sa_arg, format_string)
         # Replace trailing '.0' with ''
         return sa.func.replace(formatted_value, ".0", "")
+    elif arg_dtype.is_boolean() and typ.is_string():
+        return sa.case(
+            (sa_arg == 0, sa.literal_column("'false'")),
+            (sa_arg == 1, sa.literal_column("'true'")),
+            else_=sa.null(),
+        )
 
     # Follow the original Ibis code path.
     return sa_fixed_cast(t, op)

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/third_party/ibis/ibis_oracle/datatypes.py

@@ -84,6 +84,7 @@ _type_mapping = {
     oracledb.DB_TYPE_BINARY_DOUBLE: dt.Float64,
     oracledb.DB_TYPE_INTERVAL_DS: dt.Interval,
     oracledb.DB_TYPE_INTERVAL_YM: dt.Interval,
+    oracledb.DB_TYPE_BOOLEAN: dt.Boolean,
 }
 
 # SQL Alchemy doesn't support LONG RAW which drops us into Ibis 5.1.0 method:
@@ -100,6 +101,9 @@ if "TIMESTAMP WITH LOCAL TIME ZONE" not in OracleDialect_oracledb.ischema_names:
         "TIMESTAMP WITH LOCAL TIME ZONE"
     ] = oracle.TIMESTAMP(timezone=True)
 
+if "BOOLEAN" not in OracleDialect_oracledb.ischema_names:
+    OracleDialect_oracledb.ischema_names["BOOLEAN"] = sat.BOOLEAN
+
 
 @dt.dtype.register(OracleDialect_oracledb, sa.dialects.oracle.CLOB)
 def sa_oracle_CLOB(_, satype, nullable=True):

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/third_party/ibis/ibis_oracle/registry.py

@@ -160,6 +160,8 @@ def _cast(t, op):
             (sa_arg.in_(["1", "Y"]), sa.literal(1)),
             else_=sa.literal_column("TO_NUMBER(NULL)"),
         )
+    elif arg_dtype.is_boolean() and typ.is_string():
+        return sa.func.lower(sa.func.to_char(sa_arg))
     elif (arg_dtype.is_timestamp() or arg_dtype.is_date()) and typ.is_date():
         # If we are casting to Date then simulate what all other engines
         # understand to be a Date, which is without the time element.

{google_pso_data_validator-8.1.0 → google_pso_data_validator-8.2.0}/third_party/ibis/ibis_teradata/registry.py

@@ -352,6 +352,8 @@ def _literal(t, op):
     dtype = op.output_dtype
     if dtype.is_string():
         return _string_literal_format(t, op)
+    elif dtype.is_time():  # The base backend does not have a renderer for TIME Literals
+        return f"TIME '{op.value.isoformat()}'"
     else:
         return ibis_literal(t, op)