google-pso-data-validator 7.2.0__tar.gz → 7.4.0__tar.gz

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: google-pso-data-validator
-Version: 7.2.0
+Version: 7.4.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -651,6 +651,8 @@ data-validation query
           The connection name to be queried
   --query or -q QUERY
           The raw query to run against the supplied connection
+  [--format or -f {minimal,python}]
+          Format for query output (default: python)
 ```
 
 ### Building Matched Table Lists

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.4.0}/README.md

@@ -593,6 +593,8 @@ data-validation query
           The connection name to be queried
   --query or -q QUERY
           The raw query to run against the supplied connection
+  [--format or -f {minimal,python}]
+          Format for query output (default: python)
 ```
 
 ### Building Matched Table Lists

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.4.0}/data_validation/__main__.py

@@ -16,9 +16,9 @@ import json
 import logging
 import os
 import sys
+from typing import TYPE_CHECKING
 
 from yaml import Dumper, dump
-from argparse import Namespace
 from typing import List
 from data_validation import (
     cli_tools,
@@ -34,6 +34,9 @@ from data_validation.data_validation import DataValidation
 from data_validation.find_tables import find_tables_using_string_matching
 from data_validation.partition_builder import PartitionBuilder
 
+if TYPE_CHECKING:
+    from argparse import Namespace
+
 # by default yaml dumps lists as pointers. This disables that feature
 Dumper.ignore_aliases = lambda *args: True
 
@@ -246,7 +249,7 @@ def _get_comparison_config(
     return config_manager.build_config_comparison_fields(comparison_fields)
 
 
-def build_config_from_args(args: Namespace, config_manager: ConfigManager):
+def build_config_from_args(args: "Namespace", config_manager: ConfigManager):
     """This function is used to append build configs to the config manager for all validation commands and generate-table-partitions.
     Instead of having two separate commands, e.g. validate row and validate custom-query row, generate-table-partitions
     uses implicit choice of table or custom-query. A user can specify either tables or source/target query/file,
@@ -337,7 +340,7 @@ def build_config_from_args(args: Namespace, config_manager: ConfigManager):
 
 
 def build_config_managers_from_args(
-    args: Namespace, validate_cmd: str = None
+    args: "Namespace", validate_cmd: str = None
 ) -> List[ConfigManager]:
     """Return a list of config managers ready to execute."""
 
@@ -570,7 +573,6 @@ def run_validations(args, config_managers):
     Args:
         config_managers (list[ConfigManager]): List of config manager instances.
     """
-    # TODO(issue/31): Add parallel execution logic
     for config_manager in config_managers:
         run_validation(config_manager, dry_run=args.dry_run, verbose=args.verbose)
 
@@ -597,7 +599,7 @@ def store_json_config_file(args, config_managers):
     cli_tools.store_validation(config_file_path, json_config)
 
 
-def partition_and_store_config_files(args: Namespace) -> None:
+def partition_and_store_config_files(args: "Namespace") -> None:
     """Build multiple YAML Config files using user specified partition logic
 
     Args:
@@ -692,7 +694,7 @@ def main():
         print(find_tables_using_string_matching(args))
     elif args.command == "query":
         raw_query.print_raw_query_output(
-            raw_query.run_raw_query_against_connection(args)
+            raw_query.run_raw_query_against_connection(args), format=args.output_format
         )
     elif args.command == "validate":
         validate(args)

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.4.0}/data_validation/cli_tools.py

@@ -52,8 +52,7 @@ import sys
 import uuid
 import os
 import math
-from argparse import Namespace
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, TYPE_CHECKING
 from yaml import Dumper, Loader, dump, load
 
 from data_validation import (
@@ -66,6 +65,9 @@ from data_validation import (
 )
 from data_validation.validation_builder import list_to_sublists
 
+if TYPE_CHECKING:
+    from argparse import Namespace
+
 
 CONNECTION_SOURCE_FIELDS = {
     "BigQuery": [
@@ -191,7 +193,7 @@ VALIDATE_SCHEMA_HELP_TEXT = "Run a schema validation"
 VALIDATE_CUSTOM_QUERY_HELP_TEXT = "Run a custom query validation"
 
 
-def _check_custom_query_args(parser: argparse.ArgumentParser, parsed_args: Namespace):
+def _check_custom_query_args(parser: argparse.ArgumentParser, parsed_args: "Namespace"):
     # This is where we make additional checks if the arguments provided are what we expect
     # For example, only one of -tbls and custom query options can be provided
     if hasattr(parsed_args, "tables_list") and hasattr(
@@ -223,7 +225,7 @@ def _check_custom_query_args(parser: argparse.ArgumentParser, parsed_args: Names
         return  # old format - only one of them is present
 
 
-def get_parsed_args() -> Namespace:
+def get_parsed_args() -> "Namespace":
     """Return ArgParser with configured CLI arguments."""
     parser = configure_arg_parser()
     args = ["--help"] if len(sys.argv) == 1 else None
@@ -393,6 +395,14 @@ def _configure_raw_query(subparsers):
     )
     query_parser.add_argument("--conn", "-c", help="Connection name to query")
     query_parser.add_argument("--query", "-q", help="Raw query to execute")
+    query_parser.add_argument(
+        "--format",
+        "-f",
+        dest="output_format",
+        choices=consts.RAW_QUERY_FORMAT_TYPES,
+        default=consts.FORMAT_TYPE_PYTHON,
+        help=f"Format for query output (default: {consts.FORMAT_TYPE_PYTHON})",
+    )
 
 
 def _configure_validation_config_parser(subparsers):
@@ -1027,7 +1037,7 @@ def _add_common_arguments(
     optional_arguments.add_argument(
         "--format",
         "-fmt",
-        default="table",
+        default=consts.FORMAT_TYPE_TABLE,
         help="Set the format for printing command output, Supported formats are (text, csv, json, table). Defaults "
         "to table",
     )
@@ -1443,7 +1453,7 @@ def _concat_column_count_configs(
     return return_list
 
 
-def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
+def get_pre_build_configs(args: "Namespace", validate_cmd: str) -> List[Dict]:
     """Return a dict of configurations to build ConfigManager object"""
 
     def cols_from_arg(concat_arg: str, client, table_obj: dict, query_str: str) -> list:
@@ -1503,7 +1513,7 @@ def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
     target_client = clients.get_data_client(mgr.get_connection_config(args.target_conn))
 
     # Get format: text, csv, json, table. Default is table
-    format = args.format if args.format else "table"
+    format = args.format if args.format else consts.FORMAT_TYPE_TABLE
 
     # Get random row arguments. Only in row validations these attributes can be present.
     # Bad coding here, but keeping it so as not to introduce a breaking change. See

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.4.0}/data_validation/combiner.py

@@ -25,17 +25,20 @@ import logging
 import ibis
 import ibis.expr.datatypes as dt
 
+from typing import TYPE_CHECKING
 from data_validation import consts
 
-DEFAULT_SOURCE = "source"
-DEFAULT_TARGET = "target"
+if TYPE_CHECKING:
+    from pandas import DataFrame
+    import ibis.expr.types.relations.table as IbisTable
+    from data_validation.metadata import RunMetadata, ValidationMetadata
 
 
 def generate_report(
     client,
-    run_metadata,
-    source,
-    target,
+    run_metadata: "RunMetadata",
+    source: "IbisTable",
+    target: "IbisTable",
     join_on_fields=(),
     is_value_comparison=False,
     verbose=False,
@@ -44,8 +47,7 @@ def generate_report(
 
     Args:
         client (ibis.client.Client): Ibis client used to combine results.
-        run_metadata (data_validation.metadata.RunMetadata):
-            Metadata about the run and validations.
+        run_metadata: Metadata about the run and validations.
         source (ibis.QUERY): Ibis query / table object.
         target (ibis.QUERY): Ibis query / table object.
         join_on_fields (Sequence[str]):
@@ -69,7 +71,7 @@ def generate_report(
     if source_names != target_names:
         raise ValueError(
             "Expected source and target to have same schema, got "
-            f"source: {source_names} target: {target_names}"
+            f"{consts.RESULT_TYPE_SOURCE}: {source_names}; {consts.RESULT_TYPE_TARGET}: {target_names}"
         )
     differences_pivot = _calculate_differences(
         source, target, join_on_fields, run_metadata.validations, is_value_comparison
@@ -87,13 +89,17 @@ def generate_report(
     target_df = client.execute(target_pivot)
 
     con = ibis.pandas.connect(
-        {"source": source_df, "differences": differences_df, "target": target_df}
+        {
+            consts.RESULT_TYPE_SOURCE: source_df,
+            "differences": differences_df,
+            consts.RESULT_TYPE_TARGET: target_df,
+        }
     )
     joined = _join_pivots(
         con.tables.source, con.tables.target, con.tables.differences, join_on_fields
     )
 
-    documented = _add_metadata(joined, run_metadata)
+    documented, run_metadata = _add_metadata(joined, run_metadata)
 
     if verbose:
         logging.debug("-- ** Combiner Query ** --")
@@ -112,15 +118,17 @@ def generate_report(
             first.get_table_name(consts.RESULT_TYPE_TARGET), inplace=True
         )
 
+    _get_summary(run_metadata, result_df, source_df, target_df)
+
     return result_df
 
 
 def _calculate_difference(
-    field_differences,
+    field_differences: "IbisTable",
     datatype: dt.DataType,
     target_type: dt.DataType,
-    validation,
-    is_value_comparison,
+    validation: "ValidationMetadata",
+    is_value_comparison: bool,
 ):
     pct_threshold = ibis.literal(validation.threshold)
     if datatype.is_timestamp() or datatype.is_date():
@@ -210,12 +218,16 @@ def _calculate_difference(
         difference.name("difference"),
         pct_difference.name("pct_difference"),
         pct_threshold.name("pct_threshold"),
-        validation_status.name("validation_status"),
+        validation_status.name(consts.VALIDATION_STATUS),
     )
 
 
 def _calculate_differences(
-    source, target, join_on_fields, validations, is_value_comparison
+    source: "IbisTable",
+    target: "IbisTable",
+    join_on_fields: tuple,
+    validations: "dict[ValidationMetadata]",
+    is_value_comparison: bool,
 ):
     """Calculate differences between source and target fields.
 
@@ -267,7 +279,12 @@ def _calculate_differences(
     return differences_pivot
 
 
-def _pivot_result(result, join_on_fields, validations, result_type):
+def _pivot_result(
+    result: "IbisTable",
+    join_on_fields: tuple,
+    validations: "dict[ValidationMetadata]",
+    result_type: str,
+):
     all_fields = frozenset(result.schema().names)
     validation_fields = (
         all_fields - frozenset(join_on_fields)
@@ -285,19 +302,21 @@ def _pivot_result(result, join_on_fields, validations, result_type):
                     ibis.literal("{")
                     + ibis.literal(", ").join(validation.primary_keys)
                     + ibis.literal("}")
-                ).name("primary_keys")
+                ).name(consts.CONFIG_PRIMARY_KEYS)
             else:
-                primary_keys = ibis.literal(None).cast("string").name("primary_keys")
+                primary_keys = (
+                    ibis.literal(None).cast("string").name(consts.CONFIG_PRIMARY_KEYS)
+                )
 
             pivots.append(
                 result.projection(
                     (
                         ibis.literal(field).name("validation_name"),
                         ibis.literal(validation.validation_type).name(
-                            "validation_type"
+                            consts.VALIDATION_TYPE
                         ),
                         ibis.literal(validation.aggregation_type).name(
-                            "aggregation_type"
+                            consts.AGGREGATION_TYPE
                         ),
                         ibis.literal(validation.get_table_name(result_type)).name(
                             "table_name"
@@ -333,7 +352,12 @@ def _as_json(expr):
     )
 
 
-def _join_pivots(source, target, differences, join_on_fields):
+def _join_pivots(
+    source: "IbisTable",
+    target: "IbisTable",
+    differences: "IbisTable",
+    join_on_fields: tuple,
+):
     if join_on_fields:
         join_values = []
         for field in join_on_fields:
@@ -346,62 +370,125 @@ def _join_pivots(source, target, differences, join_on_fields):
 
         group_by_columns = (
             ibis.literal("{") + ibis.literal(", ").join(join_values) + ibis.literal("}")
-        ).name("group_by_columns")
+        ).name(consts.GROUP_BY_COLUMNS)
     else:
-        group_by_columns = ibis.literal(None).cast("string").name("group_by_columns")
+        group_by_columns = (
+            ibis.literal(None).cast("string").name(consts.GROUP_BY_COLUMNS)
+        )
 
     join_keys = ("validation_name",) + join_on_fields
     source_difference = source.join(differences, join_keys, how="outer")[
         [source[field] for field in join_keys]
         + [
-            source["validation_type"],
-            source["aggregation_type"],
-            source["table_name"],
+            source[consts.VALIDATION_TYPE],
+            source[consts.AGGREGATION_TYPE],
+            source[consts.CONFIG_TABLE_NAME],
             source["column_name"],
-            source["primary_keys"],
+            source[consts.CONFIG_PRIMARY_KEYS],
             source["num_random_rows"],
             source["agg_value"],
             differences["difference"],
             differences["pct_difference"],
             differences["pct_threshold"],
-            differences["validation_status"],
+            differences[consts.VALIDATION_STATUS],
         ]
     ]
     joined = source_difference.join(target, join_keys, how="outer")[
         source_difference["validation_name"],
-        source_difference["validation_type"]
-        .fillna(target["validation_type"])
-        .name("validation_type"),
-        source_difference["aggregation_type"]
-        .fillna(target["aggregation_type"])
-        .name("aggregation_type"),
-        source_difference["table_name"].name("source_table_name"),
-        source_difference["column_name"].name("source_column_name"),
-        source_difference["agg_value"].name("source_agg_value"),
-        target["table_name"].name("target_table_name"),
-        target["column_name"].name("target_column_name"),
-        target["agg_value"].name("target_agg_value"),
+        source_difference[consts.VALIDATION_TYPE]
+        .fillna(target[consts.VALIDATION_TYPE])
+        .name(consts.VALIDATION_TYPE),
+        source_difference[consts.AGGREGATION_TYPE]
+        .fillna(target[consts.AGGREGATION_TYPE])
+        .name(consts.AGGREGATION_TYPE),
+        source_difference["table_name"].name(consts.SOURCE_TABLE_NAME),
+        source_difference["column_name"].name(consts.SOURCE_COLUMN_NAME),
+        source_difference["agg_value"].name(consts.SOURCE_AGG_VALUE),
+        target["table_name"].name(consts.TARGET_TABLE_NAME),
+        target["column_name"].name(consts.TARGET_COLUMN_NAME),
+        target["agg_value"].name(consts.TARGET_AGG_VALUE),
         group_by_columns,
-        source_difference["primary_keys"],
+        source_difference[consts.CONFIG_PRIMARY_KEYS],
         source_difference["num_random_rows"],
         source_difference["difference"],
         source_difference["pct_difference"],
         source_difference["pct_threshold"],
-        source_difference["validation_status"],
+        source_difference[consts.VALIDATION_STATUS],
     ]
     return joined
 
 
-def _add_metadata(joined, run_metadata):
+def _add_metadata(joined: "IbisTable", run_metadata: "RunMetadata"):
     # TODO: Add source and target queries to metadata
     run_metadata.end_time = datetime.datetime.now(datetime.timezone.utc)
 
     joined = joined[
         joined,
-        ibis.literal(run_metadata.run_id).name("run_id"),
-        ibis.literal(run_metadata.labels).name("labels"),
-        ibis.literal(run_metadata.start_time).name("start_time"),
-        ibis.literal(run_metadata.end_time).name("end_time"),
+        ibis.literal(run_metadata.run_id).name(consts.CONFIG_RUN_ID),
+        ibis.literal(run_metadata.labels).name(consts.CONFIG_LABELS),
+        ibis.literal(run_metadata.start_time).name(consts.CONFIG_START_TIME),
+        ibis.literal(run_metadata.end_time).name(consts.CONFIG_END_TIME),
     ]
 
-    return joined
+    return (joined, run_metadata)
+
+
+def _get_summary(
+    run_metadata: "RunMetadata",
+    result_df: "DataFrame",
+    source_df: "DataFrame",
+    target_df: "DataFrame",
+):
+    """Logs a summary report/stats of row validation results."""
+    try:
+        if result_df.loc[0, consts.VALIDATION_TYPE] == consts.ROW_VALIDATION:
+            # Vectorized calculations for all counts
+            success_condition = (
+                result_df[consts.VALIDATION_STATUS] == consts.VALIDATION_STATUS_SUCCESS
+            )
+            fail_condition = ~success_condition  # Invert success for fail condition
+
+            source_not_in_target = (
+                result_df[consts.SOURCE_AGG_VALUE].notnull()
+                & result_df[consts.TARGET_AGG_VALUE].isnull()
+            )
+            target_not_in_source = (
+                result_df[consts.SOURCE_AGG_VALUE].isnull()
+                & result_df[consts.TARGET_AGG_VALUE].notnull()
+            )
+            present_in_both_tables = (
+                result_df[consts.SOURCE_AGG_VALUE].notnull()
+                & result_df[consts.TARGET_AGG_VALUE].notnull()
+            )
+
+            logging.info(
+                {
+                    consts.CONFIG_RUN_ID: run_metadata.run_id,
+                    consts.CONFIG_START_TIME: run_metadata.start_time.strftime(
+                        "%Y-%m-%d %H:%M:%S %Z"
+                    ),
+                    consts.CONFIG_END_TIME: run_metadata.end_time.strftime(
+                        "%Y-%m-%d %H:%M:%S %Z"
+                    ),
+                    consts.TOTAL_SOURCE_ROWS: source_df.shape[0],
+                    consts.TOTAL_TARGET_ROWS: target_df.shape[0],
+                    consts.TOTAL_ROWS_VALIDATED: result_df.shape[0],
+                    # Using .sum() on boolean Series for much faster counting
+                    consts.TOTAL_ROWS_SUCCESS: success_condition.sum(),
+                    consts.TOTAL_ROWS_FAIL: fail_condition.sum(),
+                    consts.FAILED_SOURCE_NOT_IN_TARGET: (
+                        fail_condition & source_not_in_target
+                    ).sum(),
+                    consts.FAILED_TARGET_NOT_IN_SOURCE: (
+                        fail_condition & target_not_in_source
+                    ).sum(),
+                    consts.FAILED_PRESENT_IN_BOTH_TABLES: (
+                        fail_condition & present_in_both_tables
+                    ).sum(),
+                }
+            )
+    except Exception as e:
+        logging.warning(
+            f"Error while generating summary report of row validation results: {e}",
+            exc_info=True,
+        )

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.4.0}/data_validation/config_manager.py

@@ -459,7 +459,7 @@ class ConfigManager(object):
                 cols_filter_list = consts.COLUMN_FILTER_LIST
             # handler that display results either to output or in a file
             return TextResultHandler(
-                self._config.get(consts.CONFIG_FORMAT, "table"),
+                self._config.get(consts.CONFIG_FORMAT, consts.FORMAT_TYPE_TABLE),
                 self.filter_status,
                 cols_filter_list,
             )
@@ -486,7 +486,9 @@ class ConfigManager(object):
                 table_id=table_id,
                 credentials=credentials,
                 api_endpoint=api_endpoint,
-                text_format=self._config.get(consts.CONFIG_FORMAT, "table"),
+                text_format=self._config.get(
+                    consts.CONFIG_FORMAT, consts.FORMAT_TYPE_TABLE
+                ),
             )
         else:
             raise ValueError(f"Unknown ResultHandler Class: {result_type}")
@@ -874,7 +876,7 @@ class ConfigManager(object):
         target_column_ibis_type: dt.DataType,
         margin: int = 0,
     ) -> bool:
-        """Identifies Decimal columns that will cause problems in a Pandas Dataframe.
+        """Identifies numeric columns that will cause problems in a Pandas Dataframe.
 
         i.e. are of greater precision than a 64bit int/real can hold.
 
@@ -883,17 +885,23 @@ class ConfigManager(object):
         """
         return bool(
             (
-                isinstance(source_column_ibis_type, dt.Decimal)
-                and (
-                    source_column_ibis_type.precision is None
-                    or source_column_ibis_type.precision > (18 - margin)
+                (isinstance(source_column_ibis_type, dt.Int64) and margin > 0)
+                or (
+                    isinstance(source_column_ibis_type, dt.Decimal)
+                    and (
+                        source_column_ibis_type.precision is None
+                        or source_column_ibis_type.precision > (18 - margin)
+                    )
                 )
             )
             and (
-                isinstance(target_column_ibis_type, dt.Decimal)
-                and (
-                    target_column_ibis_type.precision is None
-                    or target_column_ibis_type.precision > (18 - margin)
+                (isinstance(target_column_ibis_type, dt.Int64) and margin > 0)
+                or (
+                    isinstance(target_column_ibis_type, dt.Decimal)
+                    and (
+                        target_column_ibis_type.precision is None
+                        or target_column_ibis_type.precision > (18 - margin)
+                    )
                 )
             )
         )

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.4.0}/data_validation/consts.py

@@ -56,6 +56,8 @@ CONFIG_CASE_INSENSITIVE_MATCH = "case_insensitive_match"
 CONFIG_ROW_CONCAT = "concat"
 CONFIG_ROW_HASH = "hash"
 CONFIG_RUN_ID = "run_id"
+CONFIG_START_TIME = "start_time"
+CONFIG_END_TIME = "end_time"
 CONFIG_SOURCE_COLUMN = "source_column"
 CONFIG_TARGET_COLUMN = "target_column"
 CONFIG_THRESHOLD = "threshold"
@@ -142,12 +144,21 @@ TARGET_AGG_VALUE = "target_agg_value"
 VALIDATION_STATUS = "validation_status"
 VALIDATION_STATUS_SUCCESS = "success"
 VALIDATION_STATUS_FAIL = "fail"
-# TODO: update if we start to support other statuses
 VALIDATION_STATUSES = [
     VALIDATION_STATUS_SUCCESS,
     VALIDATION_STATUS_FAIL,
 ]
 
+# Summary stats of Row Validation results
+TOTAL_SOURCE_ROWS = "total_source_rows"
+TOTAL_TARGET_ROWS = "total_target_rows"
+TOTAL_ROWS_VALIDATED = "total_rows_validated"
+TOTAL_ROWS_SUCCESS = "total_rows_success_validation_status"
+TOTAL_ROWS_FAIL = "total_rows_fail_validation_status"
+FAILED_SOURCE_NOT_IN_TARGET = "failed_rows_present_in_source_not_in_target"
+FAILED_TARGET_NOT_IN_SOURCE = "failed_rows_present_in_target_not_in_source"
+FAILED_PRESENT_IN_BOTH_TABLES = "failed_rows_present_in_both_source_and_target"
+
 # SQL Template Formatting
 # TODO: should this be managed in query_builder if that is the only place its used?
 COUNT_STAR = "{count_star}"
@@ -156,29 +167,36 @@ COUNT_STAR = "{count_star}"
 RESULT_TYPE_SOURCE = "source"
 RESULT_TYPE_TARGET = "target"
 
-FORMAT_TYPES = ["csv", "json", "table", "text"]
+FORMAT_TYPE_CSV = "csv"
+FORMAT_TYPE_JSON = "json"
+FORMAT_TYPE_MINIMAL = "minimal"
+FORMAT_TYPE_PYTHON = "python"
+FORMAT_TYPE_TABLE = "table"
+FORMAT_TYPE_TEXT = "text"
+FORMAT_TYPES = [FORMAT_TYPE_CSV, FORMAT_TYPE_JSON, FORMAT_TYPE_TABLE, FORMAT_TYPE_TEXT]
+RAW_QUERY_FORMAT_TYPES = [FORMAT_TYPE_MINIMAL, FORMAT_TYPE_PYTHON]
 
 # Text Result Handler column filter list
 COLUMN_FILTER_LIST = [
-    "aggregation_type",
-    "end_time",
-    "labels",
+    AGGREGATION_TYPE,
+    CONFIG_END_TIME,
+    CONFIG_LABELS,
     "pct_threshold",
-    "start_time",
-    "target_table_name",
-    "target_column_name",
+    CONFIG_START_TIME,
+    TARGET_TABLE_NAME,
+    TARGET_COLUMN_NAME,
     "difference",
-    "primary_keys",
-    "group_by_columns",
+    CONFIG_PRIMARY_KEYS,
+    GROUP_BY_COLUMNS,
     "num_random_rows",
 ]
 SCHEMA_VALIDATION_COLUMN_FILTER_LIST = [
-    "start_time",
-    "end_time",
-    "aggregation_type",
+    CONFIG_START_TIME,
+    CONFIG_END_TIME,
+    AGGREGATION_TYPE,
     "difference",
-    "primary_keys",
-    "group_by_columns",
+    CONFIG_PRIMARY_KEYS,
+    GROUP_BY_COLUMNS,
     "num_random_rows",
     "pct_threshold",
 ]

{google_pso_data_validator-7.2.0 → google_pso_data_validator-7.4.0}/data_validation/data_validation.py

@@ -353,7 +353,10 @@ class DataValidation(object):
                 target_df = futures[1].result()
 
             pandas_client = ibis.pandas.connect(
-                {combiner.DEFAULT_SOURCE: source_df, combiner.DEFAULT_TARGET: target_df}
+                {
+                    consts.RESULT_TYPE_SOURCE: source_df,
+                    consts.RESULT_TYPE_TARGET: target_df,
+                }
             )
 
             try:
@@ -362,8 +365,8 @@ class DataValidation(object):
                     combiner.generate_report,
                     pandas_client,
                     self.run_metadata,
-                    pandas_client.table(combiner.DEFAULT_SOURCE),
-                    pandas_client.table(combiner.DEFAULT_TARGET),
+                    pandas_client.table(consts.RESULT_TYPE_SOURCE),
+                    pandas_client.table(consts.RESULT_TYPE_TARGET),
                     join_on_fields=join_on_fields,
                     is_value_comparison=is_value_comparison,
                     verbose=self.verbose,