google-pso-data-validator 8.5.0__tar.gz → 8.5.2__tar.gz

{google_pso_data_validator-8.5.0/google_pso_data_validator.egg-info → google_pso_data_validator-8.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 8.5.0
+Version: 8.5.2
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com

{google_pso_data_validator-8.5.0 → google_pso_data_validator-8.5.2}/data_validation/combiner.py

@@ -113,11 +113,11 @@ def generate_report(
     # Get the first validation metadata object to fill source and/or target empty table names.
     first = run_metadata.validations[next(iter(run_metadata.validations))]
     if first.validation_type != consts.CUSTOM_QUERY:
-        result_df.source_table_name.fillna(
-            first.get_table_name(consts.RESULT_TYPE_SOURCE), inplace=True
+        result_df["source_table_name"] = result_df["source_table_name"].fillna(
+            first.get_table_name(consts.RESULT_TYPE_SOURCE)
         )
-        result_df.target_table_name.fillna(
-            first.get_table_name(consts.RESULT_TYPE_TARGET), inplace=True
+        result_df["target_table_name"] = result_df["target_table_name"].fillna(
+            first.get_table_name(consts.RESULT_TYPE_TARGET)
         )
 
     _get_summary(run_metadata, result_df, source_df, target_df)
@@ -194,7 +194,9 @@ def _generate_report_slice(
         logging.debug(documented.compile())
 
     result_df = client.execute(documented)
-    result_df.validation_status.fillna(consts.VALIDATION_STATUS_FAIL, inplace=True)
+    result_df["validation_status"] = result_df["validation_status"].fillna(
+        consts.VALIDATION_STATUS_FAIL
+    )
     return result_df
 
 

{google_pso_data_validator-8.5.0 → google_pso_data_validator-8.5.2}/data_validation/config_manager.py

@@ -667,6 +667,27 @@ class ConfigManager(object):
                 isinstance(source_type, dt.UUID) or isinstance(target_type, dt.UUID)
             )
 
+    def _is_db2_xml(self, source_column_name: str, target_column_name: str) -> bool:
+        """Returns True when either source or target column is Oracle LOB data type."""
+        return self._is_raw_data_type(
+            "db2", source_column_name, target_column_name, ["XML"]
+        )
+
+    def _is_oracle_lob(self, source_column_name: str, target_column_name: str) -> bool:
+        """Returns True when either source or target column is Oracle LOB data type.
+
+        Unexpectedly the raw types for for LOB types are:
+            BLOB: LONG_RAW
+            CLOB: LONG
+            NCLOB: LONG_NVARCHAR
+        """
+        return self._is_raw_data_type(
+            "oracle",
+            source_column_name,
+            target_column_name,
+            ["LONG", "LONG_NVARCHAR", "LONG_RAW"],
+        )
+
     def _is_sql_server_text(
         self, source_column_name: str, target_column_name: str
     ) -> bool:
@@ -693,19 +714,36 @@ class ConfigManager(object):
         self, source_column_name: str, target_column_name: str, type_list: List[str]
     ) -> bool:
         """Returns True when either source or target column is of a SQL Server type listed in type_list."""
+        return self._is_raw_data_type(
+            "mssql",
+            source_column_name,
+            target_column_name,
+            type_list,
+        )
 
+    def _is_raw_data_type(
+        self,
+        client_name: str,
+        source_column_name: str,
+        target_column_name: str,
+        type_list: List[str],
+    ) -> bool:
+        """Returns True when either source or target column is of a client & type."""
         raw_source_types = self.get_source_raw_data_types()
         raw_target_types = self.get_target_raw_data_types()
+        # Raw data type map uses casefold column name as the key.
         return bool(
             (
-                self.source_client.name == "mssql"
+                self.source_client.name == client_name
                 and raw_source_types
-                and raw_source_types.get(source_column_name, [None])[0] in type_list
+                and raw_source_types.get(source_column_name.casefold(), [None])[0]
+                in type_list
             )
             or (
-                self.target_client.name == "mssql"
+                self.target_client.name == client_name
                 and raw_target_types
-                and raw_target_types.get(target_column_name, [None])[0] in type_list
+                and raw_target_types.get(target_column_name.casefold(), [None])[0]
+                in type_list
             )
         )
 
@@ -853,6 +891,7 @@ class ConfigManager(object):
             ]
             depth = 1
             calc_func = consts.CALC_FIELD_LENGTH
+
         elif column_type in ["string", "!string"]:
             if self._is_sql_server_text(source_column, target_column):
                 calc_func = consts.CALC_FIELD_BYTE_LENGTH
@@ -992,6 +1031,8 @@ class ConfigManager(object):
         """Return list of aggregate objects of given agg_type."""
 
         def require_pre_agg_calc_field(
+            source_column: str,
+            target_column: str,
             column_type: str,
             target_column_type: str,
             agg_type: str,
@@ -1001,8 +1042,15 @@ class ConfigManager(object):
                 _ in ["string", "!string", "json", "!json"]
                 for _ in [column_type, target_column_type]
             ):
-                # These data types are aggregated using their lengths.
-                return True
+                # These data types are aggregated using their lengths, except for count().
+                if agg_type == "count":
+                    # Oracle LOBs & SQL Server TEXT need a length before the count().
+                    # TODO As does Sybase TEXT, see issue-1675.
+                    return self._is_oracle_lob(
+                        source_column, target_column
+                    ) or self._is_sql_server_text(source_column, target_column)
+                else:
+                    return True
             elif self._is_uuid(column_type, target_column_type):
                 return True
             elif column_type in ["binary", "!binary"]:
@@ -1010,10 +1058,7 @@ class ConfigManager(object):
                     # Oracle BLOB is invalid for use with SQL COUNT function.
                     # The expression below returns True if client is Oracle which
                     # has the effect of triggering use of byte_length transformation.
-                    return bool(
-                        self.source_client.name == "oracle"
-                        or self.target_client.name == "oracle"
-                    )
+                    return self._is_oracle_lob(source_column, target_column)
                 else:
                     # Convert to length for any min/max/sum on binary columns.
                     return True
@@ -1101,9 +1146,19 @@ class ConfigManager(object):
                     f"Skipping {agg_type} on {column} due to SQL Server image data type"
                 )
                 continue
+            elif agg_type != "count" and self._is_db2_xml(column, column):
+                logging.info(
+                    f"Skipping {agg_type} on {column} due to Db2 XML data type"
+                )
+                continue
 
             if require_pre_agg_calc_field(
-                column_type, target_column_type, agg_type, cast_to_bigint
+                casefold_source_columns[column],
+                casefold_target_columns[column],
+                column_type,
+                target_column_type,
+                agg_type,
+                cast_to_bigint,
             ):
                 aggregate_config = self.append_pre_agg_calc_field(
                     casefold_source_columns[column],

{google_pso_data_validator-8.5.0 → google_pso_data_validator-8.5.2/google_pso_data_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 8.5.0
+Version: 8.5.2
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com

{google_pso_data_validator-8.5.0 → google_pso_data_validator-8.5.2}/setup.py

@@ -18,7 +18,7 @@ import setuptools
 
 name = "google-pso-data-validator"
 description = "A package to enable easy data validation"
-version = "8.5.0"
+version = "8.5.2"
 release_status = "Development Status :: 4 - Beta"
 
 with open("README.md", "r") as fh:

{google_pso_data_validator-8.5.0 → google_pso_data_validator-8.5.2}/third_party/ibis/ibis_addon/api.py

@@ -173,6 +173,14 @@ def db2_type_string_length(
         # Timestamp when converted to string will be at least 19 characters (YYYY-MM-DD HH:MI:SS).
         return 19
     elif ibis_column.is_integer():
+        return ibis_integer_string_length(ibis_column)
+    else:
+        return None
+
+
+def ibis_integer_string_length(ibis_column: dt.DataType) -> Optional[int]:
+    """Return the maximum string length of an integer column, including minus sign."""
+    if ibis_column.is_integer():
         if isinstance(ibis_column, dt.Int64):
             return 20
         elif isinstance(ibis_column, dt.Int32):
@@ -181,7 +189,8 @@ def db2_type_string_length(
             return 6
         elif isinstance(ibis_column, dt.Int8):
             return 4
-        return 20
+        # Safe catch all length.
+        return 38
     else:
         return None
 

{google_pso_data_validator-8.5.0 → google_pso_data_validator-8.5.2}/third_party/ibis/ibis_addon/operations.py

@@ -439,7 +439,7 @@ def string_to_epoch(ts: str) -> int:
 
 @execute_node.register(ops.ExtractEpochSeconds, (datetime.datetime, pd.Series))
 def execute_epoch_seconds_new(op, data, **kwargs):
-    convert = getattr(data, "view", data.astype)
+    convert = data.astype
     try:
         series = convert(np.int64)
         # We need int64 below because NaT overflows int32.

{google_pso_data_validator-8.5.0 → google_pso_data_validator-8.5.2}/third_party/ibis/ibis_db2/__init__.py

@@ -11,11 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import sqlalchemy as sa
+
 import re
+from typing import Iterable, Optional, Tuple
+
+import sqlalchemy as sa
 
 import ibis.expr.datatypes as dt
-from typing import Iterable, Tuple
 from ibis.backends.base.sql.alchemy import BaseAlchemyBackend
 from third_party.ibis.ibis_db2.compiler import Db2Compiler
 from third_party.ibis.ibis_db2.datatypes import _get_type
@@ -28,11 +30,11 @@ class Backend(BaseAlchemyBackend):
     def do_connect(
         self,
         host: str = "localhost",
-        user: str = None,
-        password: str = None,
+        user: Optional[str] = None,
+        password: Optional[str] = None,
         port: int = 50000,
-        database: str = None,
-        url: str = None,
+        database: Optional[str] = None,
+        url: Optional[str] = None,
         driver: str = "ibm_db_sa",
     ) -> None:
         if url is None:
@@ -102,24 +104,69 @@ class Backend(BaseAlchemyBackend):
             )
             return [_[0] for _ in result.cursor.fetchall()]
 
-    def raw_column_metadata_not_implemented(
+    def raw_column_metadata(
         self, database: str = None, table: str = None, query: str = None
     ) -> list:
         """Define this method to allow DVT to test if backend specific transformations may be needed for comparison.
-        Partner method to _metadata that retains raw data type information instead of converting
-        to Ibis types.  This works in the same way as _metadata by running a query over the DVT
-        source, either schema.table or a custom query, and fetching the metadata using sp_describe_first_result_set.
-
-        THIS METHOD IS NOT IMPLEMENTED YET.
+        Partner method to _metadata that retains raw data type information instead of converting to Ibis types.
+        This works in the same way as _metadata by running a query over the DVT source, either schema.table or a
+        custom query, and fetching the first row. From the cursor we can detect data types of the row's columns.
 
+        NOTE: This only works for table look-ups. For custom queries the raw data types are not available to us
+              due to the IBM Db2 driver hiding the real data types.
 
         Returns:
             list: A list of tuples containing the standard 7 DB API fields:
                   https://peps.python.org/pep-0249/#description
         """
-        return []
+        assert (database and table) or query, "We should never receive all args=None"
+        if database and table:
+            # For table-based validation, query the system catalog to get the true data type.
+            get_column_metadata_sql = """
+                SELECT COLNAME, TYPENAME, LENGTH, SCALE, NULLS
+                FROM SYSCAT.COLUMNS
+                WHERE TABSCHEMA = ? AND TABNAME = ?
+                ORDER BY COLNO
+            """
+            with self.begin() as con:
+                result = con.exec_driver_sql(
+                    get_column_metadata_sql,
+                    parameters=(database.upper(), table.upper()),
+                )
+                # Yield a 7-tuple mimicking cursor.description, with the true typename.
+                for (
+                    colname,
+                    typename,
+                    col_length,
+                    col_scale,
+                    nullable,
+                ) in result.cursor.fetchall():
+                    yield (
+                        colname,
+                        typename,
+                        col_length,
+                        col_length,
+                        col_length,
+                        col_scale,
+                        nullable,
+                    )
+        elif query:
+            # For custom queries, the system catalog cannot be used. Fall back to
+            # cursor.description, which may not distinguish padded char types.
+            source = f"({query})"
+            with self.begin() as con:
+                result = con.exec_driver_sql(f"SELECT * FROM {source} t0 LIMIT 0")
+                cursor = result.cursor
+                yield from (column for column in cursor.description)
 
     def is_char_type_padded(self, char_type: Tuple) -> bool:
         """Define this method if the backend supports character/string types that are padded and returns
         padded values, which DVT may want to trim"""
-        return char_type[0] == "CHARACTER"
+        type_code = char_type[0]
+        if isinstance(type_code, str):
+            return type_code.upper() == "CHARACTER"
+        else:
+            # From cursor.description for custom queries, this is a DBAPITypeObject.
+            # It's not possible to distinguish padded char types in this case,
+            # so we default to False to be safe and avoid trimming incorrectly.
+            return False

{google_pso_data_validator-8.5.0 → google_pso_data_validator-8.5.2}/third_party/ibis/ibis_db2/datatypes.py

@@ -15,11 +15,11 @@
 import ibm_db_dbi
 
 import ibis.expr.datatypes as dt
-from ibis.backends.base.sql.alchemy.datatypes import ibis_type_to_sqla
 from ibm_db_sa.ibm_db import DB2Dialect_ibm_db
 import sqlalchemy as sa
 import sqlalchemy.types as sat
 
+from ibis.backends.base.sql.alchemy import to_sqla_type
 
 # Types from https://github.com/ibmdb/python-ibmdb/blob/master/IBM_DB/ibm_db/ibm_db_dbi.py
 _type_mapping = {
@@ -37,7 +37,12 @@ _type_mapping = {
     ibm_db_dbi.ROWID: dt.String,
 }
 
-ibis_type_to_sqla[dt.String] = sa.sql.sqltypes.String(length=3000)
+
+@to_sqla_type.register(DB2Dialect_ibm_db, dt.String)
+def _string(_, itype):
+    """Include a limit for casts to String due to line size limits supported by Db2 concat()."""
+    return sa.sql.sqltypes.String(length=3000)
+
 
 DB2Dialect_ibm_db.ischema_names["DECFLOAT"] = sat.DOUBLE
 

{google_pso_data_validator-8.5.0 → google_pso_data_validator-8.5.2}/third_party/ibis/ibis_db2/registry.py

@@ -33,6 +33,9 @@ from ibis.backends.base.sql.alchemy import (
 )
 from ibis.backends.base.sql.alchemy.registry import variance_reduction
 
+from third_party.ibis.ibis_addon.api import ibis_integer_string_length
+
+
 operation_registry = sqlalchemy_operation_registry.copy()
 operation_registry.update(sqlalchemy_window_functions_registry)
 
@@ -121,9 +124,41 @@ def _cast(t, op):
 
     sa_arg = t.translate(arg)
 
-    # specialize going from an integer type to a timestamp
-    if arg_dtype.is_integer() and typ.is_timestamp():
-        return t.integer_to_timestamp(sa_arg, tz=typ.timezone)
+    if arg_dtype.is_integer():
+        if typ.is_timestamp():
+            # specialize going from an integer type to a timestamp
+            return t.integer_to_timestamp(sa_arg, tz=typ.timezone)
+        elif typ.is_string():
+            string_length = ibis_integer_string_length(arg_dtype)
+            if string_length:
+                # Restring target string length to appropriate length for integer size.
+                return sa.cast(sa_arg, sa.String(string_length))
+    elif arg_dtype.is_decimal() and typ.is_string():
+        if arg_dtype.scale is not None and arg_dtype.scale > 0:
+            # Db2 always pads fractional part of the number out to length of scale.
+            # We need to remove those insignificant digits.
+            precision = arg_dtype.precision or 31
+            fmt = (
+                ("9" * (precision - arg_dtype.scale - 1))
+                + "0."
+                + ("9" * arg_dtype.scale)
+            )
+            return sa.func.ltrim(
+                sa.func.regexp_replace(
+                    sa.func.to_char(sa_arg, fmt),
+                    sa.literal_column("'\\.?0+$'"),
+                    sa.literal_column("''"),
+                )
+            )
+        # Max expected precision 38 plus 2 for minus sign and decimal place.
+        return sa.cast(sa_arg, sa.String(40))
+    elif arg_dtype.is_floating() and typ.is_string():
+        # Max expected precision 38 plus 2 for minus sign and decimal place.
+        return sa.cast(sa_arg, sa.String(40))
+
+    if arg_dtype.is_time() and typ.is_string():
+        # Force colons as time separator with CHAR(column,JIS) expression.
+        return sa.func.char(sa_arg, sa.literal_column("JIS"))
 
     if arg_dtype.is_binary() and typ.is_string():
         # Binary to string cast is a "to hex" conversion for DVT.
@@ -297,10 +332,6 @@ def _reduction(func_name):
     return reduction_compiler
 
 
-def _count_start(sa_func):
-    return sa_func
-
-
 def _reduction_count(sa_func):
     def formatter(t, op):
         *args, where = op.args

{google_pso_data_validator-8.5.0 → google_pso_data_validator-8.5.2}/third_party/ibis/ibis_redshift/__init__.py

@@ -82,14 +82,16 @@ class Backend(BaseAlchemyBackend):
         return self._filter_with_like(databases, like)
 
     def _metadata(self, query: str) -> Iterable[Tuple[str, dt.DataType]]:
-        raw_name = util.guid()
+        raw_name = util.guid().lower()
         name = self._quote(raw_name)
-        type_info_sql = """\
-    SELECT
-    "column", "type"
-    FROM PG_TABLE_DEF
-    WHERE tablename = :raw_name
-    """
+        type_info_sql = """
+        SELECT 
+            "column_name", 
+            "data_type"
+        FROM SVV_ALL_COLUMNS
+        WHERE table_name = :raw_name
+        ORDER BY ordinal_position
+        """
         if self.inspector.has_table(query):
             query = f"TABLE {query}"
         with self.begin() as con:

{google_pso_data_validator-8.5.0 → google_pso_data_validator-8.5.2}/third_party/ibis/ibis_sybase/registry.py

@@ -104,7 +104,7 @@ def sa_format_hashbytes(translator, op):
     Sybase ASE does not support a SHA256 algorithm:
     https://userapps.support.sap.com/sap/support/knowledge/en/3108409
 
-    It might be possible to add a customer UDF for this, see issue-xxxx for details.
+    It might be possible to add a custom UDF for this, see issue-1538 for details.
     """
     arg = translator.translate(op.arg)
     cast_arg = sa.func.convert(sa.sql.literal_column("VARCHAR(16384)"), arg)