google-pso-data-validator 7.2.0__tar.gz → 7.3.0__tar.gz

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: google-pso-data-validator
-Version: 7.2.0
+Version: 7.3.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/data_validation/__main__.py

@@ -570,7 +570,6 @@ def run_validations(args, config_managers):
     Args:
         config_managers (list[ConfigManager]): List of config manager instances.
     """
-    # TODO(issue/31): Add parallel execution logic
     for config_manager in config_managers:
         run_validation(config_manager, dry_run=args.dry_run, verbose=args.verbose)
 

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/data_validation/combiner.py

@@ -25,17 +25,20 @@ import logging
 import ibis
 import ibis.expr.datatypes as dt
 
+from typing import TYPE_CHECKING
 from data_validation import consts
 
-DEFAULT_SOURCE = "source"
-DEFAULT_TARGET = "target"
+if TYPE_CHECKING:
+    from pandas import DataFrame
+    import ibis.expr.types.relations.table as IbisTable
+    from data_validation.metadata import RunMetadata, ValidationMetadata
 
 
 def generate_report(
     client,
-    run_metadata,
-    source,
-    target,
+    run_metadata: "RunMetadata",
+    source: "IbisTable",
+    target: "IbisTable",
     join_on_fields=(),
     is_value_comparison=False,
     verbose=False,
@@ -44,8 +47,7 @@ def generate_report(
 
     Args:
         client (ibis.client.Client): Ibis client used to combine results.
-        run_metadata (data_validation.metadata.RunMetadata):
-            Metadata about the run and validations.
+        run_metadata: Metadata about the run and validations.
         source (ibis.QUERY): Ibis query / table object.
         target (ibis.QUERY): Ibis query / table object.
         join_on_fields (Sequence[str]):
@@ -69,7 +71,7 @@ def generate_report(
     if source_names != target_names:
         raise ValueError(
             "Expected source and target to have same schema, got "
-            f"source: {source_names} target: {target_names}"
+            f"{consts.RESULT_TYPE_SOURCE}: {source_names}; {consts.RESULT_TYPE_TARGET}: {target_names}"
         )
     differences_pivot = _calculate_differences(
         source, target, join_on_fields, run_metadata.validations, is_value_comparison
@@ -87,13 +89,17 @@ def generate_report(
     target_df = client.execute(target_pivot)
 
     con = ibis.pandas.connect(
-        {"source": source_df, "differences": differences_df, "target": target_df}
+        {
+            consts.RESULT_TYPE_SOURCE: source_df,
+            "differences": differences_df,
+            consts.RESULT_TYPE_TARGET: target_df,
+        }
     )
     joined = _join_pivots(
         con.tables.source, con.tables.target, con.tables.differences, join_on_fields
     )
 
-    documented = _add_metadata(joined, run_metadata)
+    documented, run_metadata = _add_metadata(joined, run_metadata)
 
     if verbose:
         logging.debug("-- ** Combiner Query ** --")
@@ -112,15 +118,17 @@ def generate_report(
             first.get_table_name(consts.RESULT_TYPE_TARGET), inplace=True
         )
 
+    _get_summary(run_metadata, result_df, source_df, target_df)
+
     return result_df
 
 
 def _calculate_difference(
-    field_differences,
+    field_differences: "IbisTable",
     datatype: dt.DataType,
     target_type: dt.DataType,
-    validation,
-    is_value_comparison,
+    validation: "ValidationMetadata",
+    is_value_comparison: bool,
 ):
     pct_threshold = ibis.literal(validation.threshold)
     if datatype.is_timestamp() or datatype.is_date():
@@ -210,12 +218,16 @@ def _calculate_difference(
         difference.name("difference"),
         pct_difference.name("pct_difference"),
         pct_threshold.name("pct_threshold"),
-        validation_status.name("validation_status"),
+        validation_status.name(consts.VALIDATION_STATUS),
     )
 
 
 def _calculate_differences(
-    source, target, join_on_fields, validations, is_value_comparison
+    source: "IbisTable",
+    target: "IbisTable",
+    join_on_fields: tuple,
+    validations: "dict[ValidationMetadata]",
+    is_value_comparison: bool,
 ):
     """Calculate differences between source and target fields.
 
@@ -267,7 +279,12 @@ def _calculate_differences(
     return differences_pivot
 
 
-def _pivot_result(result, join_on_fields, validations, result_type):
+def _pivot_result(
+    result: "IbisTable",
+    join_on_fields: tuple,
+    validations: "dict[ValidationMetadata]",
+    result_type: str,
+):
     all_fields = frozenset(result.schema().names)
     validation_fields = (
         all_fields - frozenset(join_on_fields)
@@ -285,19 +302,21 @@ def _pivot_result(result, join_on_fields, validations, result_type):
                     ibis.literal("{")
                     + ibis.literal(", ").join(validation.primary_keys)
                     + ibis.literal("}")
-                ).name("primary_keys")
+                ).name(consts.CONFIG_PRIMARY_KEYS)
             else:
-                primary_keys = ibis.literal(None).cast("string").name("primary_keys")
+                primary_keys = (
+                    ibis.literal(None).cast("string").name(consts.CONFIG_PRIMARY_KEYS)
+                )
 
             pivots.append(
                 result.projection(
                     (
                         ibis.literal(field).name("validation_name"),
                         ibis.literal(validation.validation_type).name(
-                            "validation_type"
+                            consts.VALIDATION_TYPE
                         ),
                         ibis.literal(validation.aggregation_type).name(
-                            "aggregation_type"
+                            consts.AGGREGATION_TYPE
                         ),
                         ibis.literal(validation.get_table_name(result_type)).name(
                             "table_name"
@@ -333,7 +352,12 @@ def _as_json(expr):
     )
 
 
-def _join_pivots(source, target, differences, join_on_fields):
+def _join_pivots(
+    source: "IbisTable",
+    target: "IbisTable",
+    differences: "IbisTable",
+    join_on_fields: tuple,
+):
     if join_on_fields:
         join_values = []
         for field in join_on_fields:
@@ -346,62 +370,125 @@ def _join_pivots(source, target, differences, join_on_fields):
 
         group_by_columns = (
             ibis.literal("{") + ibis.literal(", ").join(join_values) + ibis.literal("}")
-        ).name("group_by_columns")
+        ).name(consts.GROUP_BY_COLUMNS)
     else:
-        group_by_columns = ibis.literal(None).cast("string").name("group_by_columns")
+        group_by_columns = (
+            ibis.literal(None).cast("string").name(consts.GROUP_BY_COLUMNS)
+        )
 
     join_keys = ("validation_name",) + join_on_fields
     source_difference = source.join(differences, join_keys, how="outer")[
         [source[field] for field in join_keys]
         + [
-            source["validation_type"],
-            source["aggregation_type"],
-            source["table_name"],
+            source[consts.VALIDATION_TYPE],
+            source[consts.AGGREGATION_TYPE],
+            source[consts.CONFIG_TABLE_NAME],
             source["column_name"],
-            source["primary_keys"],
+            source[consts.CONFIG_PRIMARY_KEYS],
             source["num_random_rows"],
             source["agg_value"],
             differences["difference"],
             differences["pct_difference"],
             differences["pct_threshold"],
-            differences["validation_status"],
+            differences[consts.VALIDATION_STATUS],
         ]
     ]
     joined = source_difference.join(target, join_keys, how="outer")[
         source_difference["validation_name"],
-        source_difference["validation_type"]
-        .fillna(target["validation_type"])
-        .name("validation_type"),
-        source_difference["aggregation_type"]
-        .fillna(target["aggregation_type"])
-        .name("aggregation_type"),
-        source_difference["table_name"].name("source_table_name"),
-        source_difference["column_name"].name("source_column_name"),
-        source_difference["agg_value"].name("source_agg_value"),
-        target["table_name"].name("target_table_name"),
-        target["column_name"].name("target_column_name"),
-        target["agg_value"].name("target_agg_value"),
+        source_difference[consts.VALIDATION_TYPE]
+        .fillna(target[consts.VALIDATION_TYPE])
+        .name(consts.VALIDATION_TYPE),
+        source_difference[consts.AGGREGATION_TYPE]
+        .fillna(target[consts.AGGREGATION_TYPE])
+        .name(consts.AGGREGATION_TYPE),
+        source_difference["table_name"].name(consts.SOURCE_TABLE_NAME),
+        source_difference["column_name"].name(consts.SOURCE_COLUMN_NAME),
+        source_difference["agg_value"].name(consts.SOURCE_AGG_VALUE),
+        target["table_name"].name(consts.TARGET_TABLE_NAME),
+        target["column_name"].name(consts.TARGET_COLUMN_NAME),
+        target["agg_value"].name(consts.TARGET_AGG_VALUE),
         group_by_columns,
-        source_difference["primary_keys"],
+        source_difference[consts.CONFIG_PRIMARY_KEYS],
         source_difference["num_random_rows"],
         source_difference["difference"],
         source_difference["pct_difference"],
         source_difference["pct_threshold"],
-        source_difference["validation_status"],
+        source_difference[consts.VALIDATION_STATUS],
     ]
     return joined
 
 
-def _add_metadata(joined, run_metadata):
+def _add_metadata(joined: "IbisTable", run_metadata: "RunMetadata"):
     # TODO: Add source and target queries to metadata
     run_metadata.end_time = datetime.datetime.now(datetime.timezone.utc)
 
     joined = joined[
         joined,
-        ibis.literal(run_metadata.run_id).name("run_id"),
-        ibis.literal(run_metadata.labels).name("labels"),
-        ibis.literal(run_metadata.start_time).name("start_time"),
-        ibis.literal(run_metadata.end_time).name("end_time"),
+        ibis.literal(run_metadata.run_id).name(consts.CONFIG_RUN_ID),
+        ibis.literal(run_metadata.labels).name(consts.CONFIG_LABELS),
+        ibis.literal(run_metadata.start_time).name(consts.CONFIG_START_TIME),
+        ibis.literal(run_metadata.end_time).name(consts.CONFIG_END_TIME),
     ]
 
-    return joined
+    return (joined, run_metadata)
+
+
+def _get_summary(
+    run_metadata: "RunMetadata",
+    result_df: "DataFrame",
+    source_df: "DataFrame",
+    target_df: "DataFrame",
+):
+    """Logs a summary report/stats of row validation results."""
+    try:
+        if result_df.loc[0, consts.VALIDATION_TYPE] == consts.ROW_VALIDATION:
+            # Vectorized calculations for all counts
+            success_condition = (
+                result_df[consts.VALIDATION_STATUS] == consts.VALIDATION_STATUS_SUCCESS
+            )
+            fail_condition = ~success_condition  # Invert success for fail condition
+
+            source_not_in_target = (
+                result_df[consts.SOURCE_AGG_VALUE].notnull()
+                & result_df[consts.TARGET_AGG_VALUE].isnull()
+            )
+            target_not_in_source = (
+                result_df[consts.SOURCE_AGG_VALUE].isnull()
+                & result_df[consts.TARGET_AGG_VALUE].notnull()
+            )
+            present_in_both_tables = (
+                result_df[consts.SOURCE_AGG_VALUE].notnull()
+                & result_df[consts.TARGET_AGG_VALUE].notnull()
+            )
+
+            logging.info(
+                {
+                    consts.CONFIG_RUN_ID: run_metadata.run_id,
+                    consts.CONFIG_START_TIME: run_metadata.start_time.strftime(
+                        "%Y-%m-%d %H:%M:%S %Z"
+                    ),
+                    consts.CONFIG_END_TIME: run_metadata.end_time.strftime(
+                        "%Y-%m-%d %H:%M:%S %Z"
+                    ),
+                    consts.TOTAL_SOURCE_ROWS: source_df.shape[0],
+                    consts.TOTAL_TARGET_ROWS: target_df.shape[0],
+                    consts.TOTAL_ROWS_VALIDATED: result_df.shape[0],
+                    # Using .sum() on boolean Series for much faster counting
+                    consts.TOTAL_ROWS_SUCCESS: success_condition.sum(),
+                    consts.TOTAL_ROWS_FAIL: fail_condition.sum(),
+                    consts.FAILED_SOURCE_NOT_IN_TARGET: (
+                        fail_condition & source_not_in_target
+                    ).sum(),
+                    consts.FAILED_TARGET_NOT_IN_SOURCE: (
+                        fail_condition & target_not_in_source
+                    ).sum(),
+                    consts.FAILED_PRESENT_IN_BOTH_TABLES: (
+                        fail_condition & present_in_both_tables
+                    ).sum(),
+                }
+            )
+    except Exception as e:
+        logging.warning(
+            f"Error while generating summary report of row validation results: {e}",
+            exc_info=True,
+        )

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/data_validation/consts.py

@@ -56,6 +56,8 @@ CONFIG_CASE_INSENSITIVE_MATCH = "case_insensitive_match"
 CONFIG_ROW_CONCAT = "concat"
 CONFIG_ROW_HASH = "hash"
 CONFIG_RUN_ID = "run_id"
+CONFIG_START_TIME = "start_time"
+CONFIG_END_TIME = "end_time"
 CONFIG_SOURCE_COLUMN = "source_column"
 CONFIG_TARGET_COLUMN = "target_column"
 CONFIG_THRESHOLD = "threshold"
@@ -142,12 +144,21 @@ TARGET_AGG_VALUE = "target_agg_value"
 VALIDATION_STATUS = "validation_status"
 VALIDATION_STATUS_SUCCESS = "success"
 VALIDATION_STATUS_FAIL = "fail"
-# TODO: update if we start to support other statuses
 VALIDATION_STATUSES = [
     VALIDATION_STATUS_SUCCESS,
     VALIDATION_STATUS_FAIL,
 ]
 
+# Summary stats of Row Validation results
+TOTAL_SOURCE_ROWS = "total_source_rows"
+TOTAL_TARGET_ROWS = "total_target_rows"
+TOTAL_ROWS_VALIDATED = "total_rows_validated"
+TOTAL_ROWS_SUCCESS = "total_rows_success_validation_status"
+TOTAL_ROWS_FAIL = "total_rows_fail_validation_status"
+FAILED_SOURCE_NOT_IN_TARGET = "failed_rows_present_in_source_not_in_target"
+FAILED_TARGET_NOT_IN_SOURCE = "failed_rows_present_in_target_not_in_source"
+FAILED_PRESENT_IN_BOTH_TABLES = "failed_rows_present_in_both_source_and_target"
+
 # SQL Template Formatting
 # TODO: should this be managed in query_builder if that is the only place its used?
 COUNT_STAR = "{count_star}"
@@ -160,25 +171,25 @@ FORMAT_TYPES = ["csv", "json", "table", "text"]
 
 # Text Result Handler column filter list
 COLUMN_FILTER_LIST = [
-    "aggregation_type",
-    "end_time",
-    "labels",
+    AGGREGATION_TYPE,
+    CONFIG_END_TIME,
+    CONFIG_LABELS,
     "pct_threshold",
-    "start_time",
-    "target_table_name",
-    "target_column_name",
+    CONFIG_START_TIME,
+    TARGET_TABLE_NAME,
+    TARGET_COLUMN_NAME,
     "difference",
-    "primary_keys",
-    "group_by_columns",
+    CONFIG_PRIMARY_KEYS,
+    GROUP_BY_COLUMNS,
     "num_random_rows",
 ]
 SCHEMA_VALIDATION_COLUMN_FILTER_LIST = [
-    "start_time",
-    "end_time",
-    "aggregation_type",
+    CONFIG_START_TIME,
+    CONFIG_END_TIME,
+    AGGREGATION_TYPE,
     "difference",
-    "primary_keys",
-    "group_by_columns",
+    CONFIG_PRIMARY_KEYS,
+    GROUP_BY_COLUMNS,
     "num_random_rows",
     "pct_threshold",
 ]

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/data_validation/data_validation.py

@@ -353,7 +353,10 @@ class DataValidation(object):
                 target_df = futures[1].result()
 
             pandas_client = ibis.pandas.connect(
-                {combiner.DEFAULT_SOURCE: source_df, combiner.DEFAULT_TARGET: target_df}
+                {
+                    consts.RESULT_TYPE_SOURCE: source_df,
+                    consts.RESULT_TYPE_TARGET: target_df,
+                }
             )
 
             try:
@@ -362,8 +365,8 @@ class DataValidation(object):
                     combiner.generate_report,
                     pandas_client,
                     self.run_metadata,
-                    pandas_client.table(combiner.DEFAULT_SOURCE),
-                    pandas_client.table(combiner.DEFAULT_TARGET),
+                    pandas_client.table(consts.RESULT_TYPE_SOURCE),
+                    pandas_client.table(consts.RESULT_TYPE_TARGET),
                     join_on_fields=join_on_fields,
                     is_value_comparison=is_value_comparison,
                     verbose=self.verbose,

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/data_validation/result_handlers/bigquery.py

@@ -113,7 +113,7 @@ class BigQueryResultHandler(object):
         else:
             logging.info(f'{BQRH_WRITE_MESSAGE}, run id: {result_df.iloc[0]["run_id"]}')
 
-        # Handler also logs results after saving to BigQuery.
+        # Handler can also output results after saving to BigQuery.
         logger = logging.getLogger()
         if logger.isEnabledFor(logging.DEBUG):
             # Checking log level to avoid evaluating a large Dataframe that will never be logged.

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/data_validation/secret_manager.py

@@ -53,6 +53,10 @@ class GCPSecretManager:
             # Return the decoded payload.
             payload = response.payload.data.decode("UTF-8")
             return payload
-        except (exceptions.NotFound, exceptions.InvalidArgument):
-            # The secret does not exist and therefore we can assume it is just a token to be returned.
+        except (
+            exceptions.NotFound,
+            exceptions.InvalidArgument,
+            exceptions.PermissionDenied,
+        ):
+            # The secret does not exist or is inaccessible and therefore we can assume it is just a token to be returned.
             return secret_id

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/data_validation/util.py

@@ -12,10 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import logging
 import re
 import time
 
+from data_validation import exceptions
+
 
 def timed_call(log_txt, fn, *args, **kwargs):
     t0 = time.time()
@@ -44,3 +47,18 @@ def split_not_in_quotes(
         return [t for t in re.split(pattern, to_split) if t]
     else:
         return re.split(pattern, to_split)
+
+
+def dvt_config_string_to_dict(config_string: str) -> dict:
+    """Convert JSON in a string to a dict."""
+    if not config_string:
+        return None
+    if isinstance(config_string, dict):
+        return config_string
+    try:
+        param_dict = json.loads(config_string.replace("'", '"'))
+        return param_dict
+    except json.JSONDecodeError as exc:
+        raise exceptions.ValidationException(
+            f"Invalid JSON format in connection parameter dictionary string: {config_string}"
+        ) from exc

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/google_pso_data_validator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: google-pso-data-validator
-Version: 7.2.0
+Version: 7.3.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/setup.py

@@ -18,7 +18,7 @@ import setuptools
 
 name = "google-pso-data-validator"
 description = "A package to enable easy data validation"
-version = "7.2.0"
+version = "7.3.0"
 release_status = "Development Status :: 4 - Beta"
 
 with open("README.md", "r") as fh:

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/third_party/ibis/ibis_oracle/__init__.py

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import sqlalchemy as sa
-from sqlalchemy.dialects.oracle.base import OracleIdentifierPreparer
+from sqlalchemy.dialects.oracle.base import (
+    OracleIdentifierPreparer,
+    RESERVED_WORDS as ORACLE_RESERVED_WORDS,
+)
 from sqlalchemy.dialects.oracle.cx_oracle import OracleDialect_cx_oracle
 import re
 
@@ -23,6 +26,13 @@ from third_party.ibis.ibis_oracle.compiler import OracleCompiler
 from third_party.ibis.ibis_oracle.datatypes import _get_type
 
 
+EXTRA_RESERVED_WORDS = set(
+    [
+        "COLUMN",
+    ]
+)
+
+
 def _ora_denormalize_name(self, name):
     """Oracle specific version of sqlalchemy/engine/default.py.denormalize_name()
 
@@ -47,6 +57,11 @@ OracleDialect_cx_oracle.denormalize_name = _ora_denormalize_name
 
 
 class DVTOracleIdentifierPreparer(OracleIdentifierPreparer):
+
+    reserved_words = {
+        x.lower() for x in ORACLE_RESERVED_WORDS.union(EXTRA_RESERVED_WORDS)
+    }
+
     def quote_identifier(self, value):
         """Quote an identifier.
 

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/third_party/ibis/ibis_snowflake/api.py

@@ -22,6 +22,7 @@ import snowflake.sqlalchemy
 
 logging.getLogger("snowflake.connector").setLevel(logging.WARNING)
 
+from data_validation.util import dvt_config_string_to_dict
 import third_party.ibis.ibis_snowflake.datatypes
 
 
@@ -30,8 +31,10 @@ def snowflake_connect(
     password: str,
     account: str,
     database: str,
-    connect_args: Mapping[str, Any] = None,
+    connect_args: str = None,
 ):
+    if connect_args:
+        connect_args = dvt_config_string_to_dict(connect_args)
     return ibis.snowflake.connect(
         user=user,
         password=password,

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/third_party/ibis/ibis_teradata/__init__.py

@@ -14,13 +14,13 @@
 import pandas
 import warnings
 
-import json
 import teradatasql
 import ibis.expr.datatypes as dt
 import ibis.expr.types as ir
 from typing import Mapping, Any
 import ibis.expr.schema as sch
 from ibis.backends.base.sql import BaseSQLBackend
+
 from third_party.ibis.ibis_teradata.compiler import TeradataCompiler
 from third_party.ibis.ibis_teradata.datatypes import (
     TeradataTypeTranslator,
@@ -41,7 +41,7 @@ class Backend(BaseSQLBackend):
         port: int = 1025,
         logmech: str = "TD2",
         use_no_lock_tables: str = "False",
-        json_params: str = None,
+        json_params: Mapping[str, Any] = None,
     ) -> None:
         self.teradata_config = {
             "host": host,
@@ -50,15 +50,8 @@ class Backend(BaseSQLBackend):
             "dbs_port": port,
             "logmech": logmech,
         }
-
         if json_params:
-            try:
-                param_dict = json.loads(json_params.replace("'", '"'))
-                self.teradata_config.update(param_dict)
-            except json.JSONDecodeError:
-                print(
-                    f"Invalid JSON format in the parameter dictionary string: {json_params}"
-                )
+            self.teradata_config.update(json_params)
 
         self.client = teradatasql.connect(**self.teradata_config)
         self.con = self.client.cursor()

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.3.0}/third_party/ibis/ibis_teradata/api.py

@@ -14,6 +14,8 @@
 from third_party.ibis.ibis_teradata import Backend as TeradataBackend
 import teradatasql  # NOQA fail early if the package is missing
 
+from data_validation.util import dvt_config_string_to_dict
+
 
 def teradata_connect(
     host: str = "localhost",
@@ -25,6 +27,9 @@ def teradata_connect(
     json_params: str = None,
 ):
     backend = TeradataBackend()
+    if json_params:
+        json_params = dvt_config_string_to_dict(json_params)
+
     backend.do_connect(
         host=host,
         user_name=user_name,