google-pso-data-validator 7.1.0__tar.gz → 7.3.0__tar.gz

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: google-pso-data-validator
-Version: 7.1.0
+Version: 7.3.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -38,6 +38,14 @@ Provides-Extra: apache-airflow
 Requires-Dist: 1.10.11; extra == "apache-airflow"
 Provides-Extra: pyspark
 Requires-Dist: 3.0.0; extra == "pyspark"
+Provides-Extra: develop
+Requires-Dist: black==22.3.0; extra == "develop"
+Requires-Dist: flake8; extra == "develop"
+Requires-Dist: freezegun; extra == "develop"
+Requires-Dist: pyfakefs==4.6.2; extra == "develop"
+Requires-Dist: pytest; extra == "develop"
+Requires-Dist: pytest-cov; extra == "develop"
+Requires-Dist: pytest-timeout; extra == "develop"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.3.0}/data_validation/__main__.py

@@ -570,7 +570,6 @@ def run_validations(args, config_managers):
     Args:
         config_managers (list[ConfigManager]): List of config manager instances.
     """
-    # TODO(issue/31): Add parallel execution logic
     for config_manager in config_managers:
         run_validation(config_manager, dry_run=args.dry_run, verbose=args.verbose)
 
@@ -640,6 +639,10 @@ def run_connections(args):
     """Run commands related to connection management."""
     if args.connect_cmd == "list":
         cli_tools.list_connections()
+    elif args.connect_cmd == "delete":
+        cli_tools.delete_connection(args.connection_name)
+    elif args.connect_cmd == "describe":
+        cli_tools.describe_connection(args.connection_name, args.output_format)
     elif args.connect_cmd == "add":
         conn = cli_tools.get_connection_config_from_args(args)
         # Test getting a client to validate connection details

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.3.0}/data_validation/cli_tools.py

@@ -52,12 +52,18 @@ import sys
 import uuid
 import os
 import math
-import re
 from argparse import Namespace
 from typing import Dict, List, Optional
 from yaml import Dumper, Loader, dump, load
 
-from data_validation import clients, consts, find_tables, state_manager, gcs_helper
+from data_validation import (
+    clients,
+    consts,
+    find_tables,
+    state_manager,
+    gcs_helper,
+    util,
+)
 from data_validation.validation_builder import list_to_sublists
 
 
@@ -468,6 +474,27 @@ def _configure_connection_parser(subparsers):
     )
     _configure_database_specific_parsers(add_parser)
 
+    delete_parser = connect_subparsers.add_parser(
+        "delete", help="Delete an existing connection"
+    )
+    delete_parser.add_argument(
+        "--connection-name", "-c", required=True, help="Name of connection to delete"
+    )
+    describe_parser = connect_subparsers.add_parser(
+        "describe", help="Describe an existing connection"
+    )
+    describe_parser.add_argument(
+        "--connection-name", "-c", required=True, help="Name of connection to describe"
+    )
+    describe_parser.add_argument(
+        "--format",
+        "-f",
+        dest="output_format",
+        choices=["json", "yaml"],
+        default="yaml",
+        help="Output format for the configuration (default: yaml)",
+    )
+
 
 def _configure_database_specific_parsers(parser):
     """Configure a separate subparser for each supported DB."""
@@ -1085,13 +1112,29 @@ def store_connection(connection_name, conn):
     mgr.create_connection(connection_name, conn)
 
 
+def delete_connection(connection_name):
+    """Delete the connection config under the given name."""
+    mgr = state_manager.StateManager()
+    mgr.delete_connection(connection_name)
+
+
 def list_connections():
     """List all saved connections."""
     mgr = state_manager.StateManager()
     connections = mgr.list_connections()
     for conn_name in connections:
         source_type = mgr.get_connection_config(conn_name).get("source_type")
-        logging.info(f"Connection Name: {conn_name} : {source_type}")
+        print(f"Connection Name: {conn_name}")
+        print(f"Source Type:     {source_type}\n")
+    return connections
+
+
+def describe_connection(connection_name, output_format):
+    """Return yaml connection details for a specific connection"""
+    mgr = state_manager.StateManager()
+    connection_details = mgr.describe_connection(connection_name, output_format)
+    print(connection_details)
+    return connection_details
 
 
 def get_connection(connection_name):
@@ -1165,31 +1208,29 @@ def get_filters(filter_value: str) -> List[Dict]:
     If only one filter is specified, it applies to both source and target
     For a doc on regular expression for filters see docs/internal/filters_regex.md
     """
-
-    single_filter = r"([^':]*('[^']*')*)*"
-    double_filter = (
-        r"(?P<source>" + single_filter + r"):(?P<target>" + single_filter + r")"
-    )
-    filter_config = []
-    if result := re.fullmatch(single_filter, filter_value):
-        if result.group(0) == "":
+    filters = util.split_not_in_quotes(filter_value, ":")
+    if len(filters) not in (1, 2):
+        raise argparse.ArgumentTypeError("Unable to parse filter arguments.")
+    filters = [_.strip() for _ in filters]
+    if len(filters) == 1:
+        if not filters[0]:
             raise argparse.ArgumentTypeError("Empty string not allowed in filter")
         filter_dict = {
             "type": "custom",
-            "source": result.group(0),
-            "target": result.group(0),
+            "source": filters[0],
+            "target": filters[0],
         }
-    elif result := re.fullmatch(double_filter, filter_value):
-        if result.group("source") == "" or result.group("target") == "":
+    elif len(filters) == 2:
+        if not filters[0] or not filters[1]:
             raise argparse.ArgumentTypeError("Empty string not allowed in filter")
         filter_dict = {
             "type": "custom",
-            "source": result.group("source"),
-            "target": result.group("target"),
+            "source": filters[0],
+            "target": filters[1],
         }
-    else:
-        raise argparse.ArgumentTypeError("Unable to parse filter arguments.")
-    filter_config.append(filter_dict)
+    filter_config = [
+        filter_dict,
+    ]
     return filter_config
 
 

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.3.0}/data_validation/combiner.py

@@ -25,17 +25,20 @@ import logging
 import ibis
 import ibis.expr.datatypes as dt
 
+from typing import TYPE_CHECKING
 from data_validation import consts
 
-DEFAULT_SOURCE = "source"
-DEFAULT_TARGET = "target"
+if TYPE_CHECKING:
+    from pandas import DataFrame
+    import ibis.expr.types.relations.table as IbisTable
+    from data_validation.metadata import RunMetadata, ValidationMetadata
 
 
 def generate_report(
     client,
-    run_metadata,
-    source,
-    target,
+    run_metadata: "RunMetadata",
+    source: "IbisTable",
+    target: "IbisTable",
     join_on_fields=(),
     is_value_comparison=False,
     verbose=False,
@@ -44,8 +47,7 @@ def generate_report(
 
     Args:
         client (ibis.client.Client): Ibis client used to combine results.
-        run_metadata (data_validation.metadata.RunMetadata):
-            Metadata about the run and validations.
+        run_metadata: Metadata about the run and validations.
         source (ibis.QUERY): Ibis query / table object.
         target (ibis.QUERY): Ibis query / table object.
         join_on_fields (Sequence[str]):
@@ -69,7 +71,7 @@ def generate_report(
     if source_names != target_names:
         raise ValueError(
             "Expected source and target to have same schema, got "
-            f"source: {source_names} target: {target_names}"
+            f"{consts.RESULT_TYPE_SOURCE}: {source_names}; {consts.RESULT_TYPE_TARGET}: {target_names}"
         )
     differences_pivot = _calculate_differences(
         source, target, join_on_fields, run_metadata.validations, is_value_comparison
@@ -87,13 +89,17 @@ def generate_report(
     target_df = client.execute(target_pivot)
 
     con = ibis.pandas.connect(
-        {"source": source_df, "differences": differences_df, "target": target_df}
+        {
+            consts.RESULT_TYPE_SOURCE: source_df,
+            "differences": differences_df,
+            consts.RESULT_TYPE_TARGET: target_df,
+        }
     )
     joined = _join_pivots(
         con.tables.source, con.tables.target, con.tables.differences, join_on_fields
     )
 
-    documented = _add_metadata(joined, run_metadata)
+    documented, run_metadata = _add_metadata(joined, run_metadata)
 
     if verbose:
         logging.debug("-- ** Combiner Query ** --")
@@ -112,20 +118,26 @@ def generate_report(
             first.get_table_name(consts.RESULT_TYPE_TARGET), inplace=True
         )
 
+    _get_summary(run_metadata, result_df, source_df, target_df)
+
     return result_df
 
 
 def _calculate_difference(
-    field_differences,
+    field_differences: "IbisTable",
     datatype: dt.DataType,
     target_type: dt.DataType,
-    validation,
-    is_value_comparison,
+    validation: "ValidationMetadata",
+    is_value_comparison: bool,
 ):
     pct_threshold = ibis.literal(validation.threshold)
     if datatype.is_timestamp() or datatype.is_date():
-        source_value = field_differences["differences_source_value"].epoch_seconds()
-        target_value = field_differences["differences_target_value"].epoch_seconds()
+        source_value = (
+            field_differences["differences_source_value"].epoch_seconds().cast("int64")
+        )
+        target_value = (
+            field_differences["differences_target_value"].epoch_seconds().cast("int64")
+        )
     elif datatype.is_boolean() or (target_type and target_type.is_boolean()):
         source_value = field_differences["differences_source_value"].cast("boolean")
         target_value = field_differences["differences_target_value"].cast("boolean")
@@ -206,12 +218,16 @@ def _calculate_difference(
         difference.name("difference"),
         pct_difference.name("pct_difference"),
         pct_threshold.name("pct_threshold"),
-        validation_status.name("validation_status"),
+        validation_status.name(consts.VALIDATION_STATUS),
     )
 
 
 def _calculate_differences(
-    source, target, join_on_fields, validations, is_value_comparison
+    source: "IbisTable",
+    target: "IbisTable",
+    join_on_fields: tuple,
+    validations: "dict[ValidationMetadata]",
+    is_value_comparison: bool,
 ):
     """Calculate differences between source and target fields.
 
@@ -263,7 +279,12 @@ def _calculate_differences(
     return differences_pivot
 
 
-def _pivot_result(result, join_on_fields, validations, result_type):
+def _pivot_result(
+    result: "IbisTable",
+    join_on_fields: tuple,
+    validations: "dict[ValidationMetadata]",
+    result_type: str,
+):
     all_fields = frozenset(result.schema().names)
     validation_fields = (
         all_fields - frozenset(join_on_fields)
@@ -281,19 +302,21 @@ def _pivot_result(result, join_on_fields, validations, result_type):
                     ibis.literal("{")
                     + ibis.literal(", ").join(validation.primary_keys)
                     + ibis.literal("}")
-                ).name("primary_keys")
+                ).name(consts.CONFIG_PRIMARY_KEYS)
             else:
-                primary_keys = ibis.literal(None).cast("string").name("primary_keys")
+                primary_keys = (
+                    ibis.literal(None).cast("string").name(consts.CONFIG_PRIMARY_KEYS)
+                )
 
             pivots.append(
                 result.projection(
                     (
                         ibis.literal(field).name("validation_name"),
                         ibis.literal(validation.validation_type).name(
-                            "validation_type"
+                            consts.VALIDATION_TYPE
                         ),
                         ibis.literal(validation.aggregation_type).name(
-                            "aggregation_type"
+                            consts.AGGREGATION_TYPE
                         ),
                         ibis.literal(validation.get_table_name(result_type)).name(
                             "table_name"
@@ -329,7 +352,12 @@ def _as_json(expr):
     )
 
 
-def _join_pivots(source, target, differences, join_on_fields):
+def _join_pivots(
+    source: "IbisTable",
+    target: "IbisTable",
+    differences: "IbisTable",
+    join_on_fields: tuple,
+):
     if join_on_fields:
         join_values = []
         for field in join_on_fields:
@@ -342,62 +370,125 @@ def _join_pivots(source, target, differences, join_on_fields):
 
         group_by_columns = (
             ibis.literal("{") + ibis.literal(", ").join(join_values) + ibis.literal("}")
-        ).name("group_by_columns")
+        ).name(consts.GROUP_BY_COLUMNS)
     else:
-        group_by_columns = ibis.literal(None).cast("string").name("group_by_columns")
+        group_by_columns = (
+            ibis.literal(None).cast("string").name(consts.GROUP_BY_COLUMNS)
+        )
 
     join_keys = ("validation_name",) + join_on_fields
     source_difference = source.join(differences, join_keys, how="outer")[
         [source[field] for field in join_keys]
         + [
-            source["validation_type"],
-            source["aggregation_type"],
-            source["table_name"],
+            source[consts.VALIDATION_TYPE],
+            source[consts.AGGREGATION_TYPE],
+            source[consts.CONFIG_TABLE_NAME],
             source["column_name"],
-            source["primary_keys"],
+            source[consts.CONFIG_PRIMARY_KEYS],
             source["num_random_rows"],
             source["agg_value"],
             differences["difference"],
             differences["pct_difference"],
             differences["pct_threshold"],
-            differences["validation_status"],
+            differences[consts.VALIDATION_STATUS],
         ]
     ]
     joined = source_difference.join(target, join_keys, how="outer")[
         source_difference["validation_name"],
-        source_difference["validation_type"]
-        .fillna(target["validation_type"])
-        .name("validation_type"),
-        source_difference["aggregation_type"]
-        .fillna(target["aggregation_type"])
-        .name("aggregation_type"),
-        source_difference["table_name"].name("source_table_name"),
-        source_difference["column_name"].name("source_column_name"),
-        source_difference["agg_value"].name("source_agg_value"),
-        target["table_name"].name("target_table_name"),
-        target["column_name"].name("target_column_name"),
-        target["agg_value"].name("target_agg_value"),
+        source_difference[consts.VALIDATION_TYPE]
+        .fillna(target[consts.VALIDATION_TYPE])
+        .name(consts.VALIDATION_TYPE),
+        source_difference[consts.AGGREGATION_TYPE]
+        .fillna(target[consts.AGGREGATION_TYPE])
+        .name(consts.AGGREGATION_TYPE),
+        source_difference["table_name"].name(consts.SOURCE_TABLE_NAME),
+        source_difference["column_name"].name(consts.SOURCE_COLUMN_NAME),
+        source_difference["agg_value"].name(consts.SOURCE_AGG_VALUE),
+        target["table_name"].name(consts.TARGET_TABLE_NAME),
+        target["column_name"].name(consts.TARGET_COLUMN_NAME),
+        target["agg_value"].name(consts.TARGET_AGG_VALUE),
         group_by_columns,
-        source_difference["primary_keys"],
+        source_difference[consts.CONFIG_PRIMARY_KEYS],
         source_difference["num_random_rows"],
         source_difference["difference"],
         source_difference["pct_difference"],
         source_difference["pct_threshold"],
-        source_difference["validation_status"],
+        source_difference[consts.VALIDATION_STATUS],
     ]
     return joined
 
 
-def _add_metadata(joined, run_metadata):
+def _add_metadata(joined: "IbisTable", run_metadata: "RunMetadata"):
     # TODO: Add source and target queries to metadata
     run_metadata.end_time = datetime.datetime.now(datetime.timezone.utc)
 
     joined = joined[
         joined,
-        ibis.literal(run_metadata.run_id).name("run_id"),
-        ibis.literal(run_metadata.labels).name("labels"),
-        ibis.literal(run_metadata.start_time).name("start_time"),
-        ibis.literal(run_metadata.end_time).name("end_time"),
+        ibis.literal(run_metadata.run_id).name(consts.CONFIG_RUN_ID),
+        ibis.literal(run_metadata.labels).name(consts.CONFIG_LABELS),
+        ibis.literal(run_metadata.start_time).name(consts.CONFIG_START_TIME),
+        ibis.literal(run_metadata.end_time).name(consts.CONFIG_END_TIME),
     ]
 
-    return joined
+    return (joined, run_metadata)
+
+
+def _get_summary(
+    run_metadata: "RunMetadata",
+    result_df: "DataFrame",
+    source_df: "DataFrame",
+    target_df: "DataFrame",
+):
+    """Logs a summary report/stats of row validation results."""
+    try:
+        if result_df.loc[0, consts.VALIDATION_TYPE] == consts.ROW_VALIDATION:
+            # Vectorized calculations for all counts
+            success_condition = (
+                result_df[consts.VALIDATION_STATUS] == consts.VALIDATION_STATUS_SUCCESS
+            )
+            fail_condition = ~success_condition  # Invert success for fail condition
+
+            source_not_in_target = (
+                result_df[consts.SOURCE_AGG_VALUE].notnull()
+                & result_df[consts.TARGET_AGG_VALUE].isnull()
+            )
+            target_not_in_source = (
+                result_df[consts.SOURCE_AGG_VALUE].isnull()
+                & result_df[consts.TARGET_AGG_VALUE].notnull()
+            )
+            present_in_both_tables = (
+                result_df[consts.SOURCE_AGG_VALUE].notnull()
+                & result_df[consts.TARGET_AGG_VALUE].notnull()
+            )
+
+            logging.info(
+                {
+                    consts.CONFIG_RUN_ID: run_metadata.run_id,
+                    consts.CONFIG_START_TIME: run_metadata.start_time.strftime(
+                        "%Y-%m-%d %H:%M:%S %Z"
+                    ),
+                    consts.CONFIG_END_TIME: run_metadata.end_time.strftime(
+                        "%Y-%m-%d %H:%M:%S %Z"
+                    ),
+                    consts.TOTAL_SOURCE_ROWS: source_df.shape[0],
+                    consts.TOTAL_TARGET_ROWS: target_df.shape[0],
+                    consts.TOTAL_ROWS_VALIDATED: result_df.shape[0],
+                    # Using .sum() on boolean Series for much faster counting
+                    consts.TOTAL_ROWS_SUCCESS: success_condition.sum(),
+                    consts.TOTAL_ROWS_FAIL: fail_condition.sum(),
+                    consts.FAILED_SOURCE_NOT_IN_TARGET: (
+                        fail_condition & source_not_in_target
+                    ).sum(),
+                    consts.FAILED_TARGET_NOT_IN_SOURCE: (
+                        fail_condition & target_not_in_source
+                    ).sum(),
+                    consts.FAILED_PRESENT_IN_BOTH_TABLES: (
+                        fail_condition & present_in_both_tables
+                    ).sum(),
+                }
+            )
+    except Exception as e:
+        logging.warning(
+            f"Error while generating summary report of row validation results: {e}",
+            exc_info=True,
+        )

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.3.0}/data_validation/config_manager.py

@@ -784,7 +784,9 @@ class ConfigManager(object):
         column_position: int,
     ) -> dict:
         """Append calculated field for length() or epoch_seconds(timestamp) for preprocessing before column validation aggregation."""
-        depth, cast_type = 0, None
+        depth = 0
+        cast_type = None
+        final_cast_type = None
         if any(_ in ["json", "!json"] for _ in [column_type, target_column_type]):
             # JSON data which needs casting to string before we apply a length function.
             pre_calculated_config = self.build_and_append_pre_agg_calc_config(
@@ -829,6 +831,10 @@ class ConfigManager(object):
                 depth = 1
 
             calc_func = "epoch_seconds"
+            if agg_type == consts.CONFIG_TYPE_SUM:
+                # It is possible to exceed int64 when summing epoch_seconds therefore cast to string.
+                # See issue 1391 for details.
+                final_cast_type = "string"
 
         elif column_type == "int32" or column_type == "!int32":
             calc_func = consts.CONFIG_CAST
@@ -856,6 +862,9 @@ class ConfigManager(object):
             ),
             consts.CONFIG_TYPE: agg_type,
         }
+        if final_cast_type:
+            # Adding to dict this way to avoid adding a lot of empty cast attributes.
+            aggregate_config[consts.CONFIG_CAST] = final_cast_type
 
         return aggregate_config
 
@@ -863,22 +872,28 @@ class ConfigManager(object):
         self,
         source_column_ibis_type: dt.DataType,
         target_column_ibis_type: dt.DataType,
+        margin: int = 0,
     ) -> bool:
-        """Identifies Decimal columns that will cause problems in a Pandas Dataframe, i.e.
-        is of greater precision than a 64bit int/real can hold."""
+        """Identifies Decimal columns that will cause problems in a Pandas Dataframe.
+
+        i.e. are of greater precision than a 64bit int/real can hold.
+
+        margin: Allows us to lower the precision threshold. This is helpful when summing column
+                values that are okay by themselves but cumulativaly could overflow a 64bit value.
+        """
         return bool(
             (
                 isinstance(source_column_ibis_type, dt.Decimal)
                 and (
                     source_column_ibis_type.precision is None
-                    or source_column_ibis_type.precision > 18
+                    or source_column_ibis_type.precision > (18 - margin)
                 )
             )
             and (
                 isinstance(target_column_ibis_type, dt.Decimal)
                 and (
                     target_column_ibis_type.precision is None
-                    or target_column_ibis_type.precision > 18
+                    or target_column_ibis_type.precision > (18 - margin)
                 )
             )
         )
@@ -1042,7 +1057,9 @@ class ConfigManager(object):
                     consts.CONFIG_TYPE: agg_type,
                 }
                 if self._decimal_column_too_big_for_pandas(
-                    source_column_ibis_type, target_column_ibis_type
+                    source_column_ibis_type,
+                    target_column_ibis_type,
+                    margin=(2 if agg_type == consts.CONFIG_TYPE_SUM else 0),
                 ):
                     aggregate_config[consts.CONFIG_CAST] = "string"
 

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.3.0}/data_validation/consts.py

@@ -56,6 +56,8 @@ CONFIG_CASE_INSENSITIVE_MATCH = "case_insensitive_match"
 CONFIG_ROW_CONCAT = "concat"
 CONFIG_ROW_HASH = "hash"
 CONFIG_RUN_ID = "run_id"
+CONFIG_START_TIME = "start_time"
+CONFIG_END_TIME = "end_time"
 CONFIG_SOURCE_COLUMN = "source_column"
 CONFIG_TARGET_COLUMN = "target_column"
 CONFIG_THRESHOLD = "threshold"
@@ -142,12 +144,21 @@ TARGET_AGG_VALUE = "target_agg_value"
 VALIDATION_STATUS = "validation_status"
 VALIDATION_STATUS_SUCCESS = "success"
 VALIDATION_STATUS_FAIL = "fail"
-# TODO: update if we start to support other statuses
 VALIDATION_STATUSES = [
     VALIDATION_STATUS_SUCCESS,
     VALIDATION_STATUS_FAIL,
 ]
 
+# Summary stats of Row Validation results
+TOTAL_SOURCE_ROWS = "total_source_rows"
+TOTAL_TARGET_ROWS = "total_target_rows"
+TOTAL_ROWS_VALIDATED = "total_rows_validated"
+TOTAL_ROWS_SUCCESS = "total_rows_success_validation_status"
+TOTAL_ROWS_FAIL = "total_rows_fail_validation_status"
+FAILED_SOURCE_NOT_IN_TARGET = "failed_rows_present_in_source_not_in_target"
+FAILED_TARGET_NOT_IN_SOURCE = "failed_rows_present_in_target_not_in_source"
+FAILED_PRESENT_IN_BOTH_TABLES = "failed_rows_present_in_both_source_and_target"
+
 # SQL Template Formatting
 # TODO: should this be managed in query_builder if that is the only place its used?
 COUNT_STAR = "{count_star}"
@@ -160,25 +171,25 @@ FORMAT_TYPES = ["csv", "json", "table", "text"]
 
 # Text Result Handler column filter list
 COLUMN_FILTER_LIST = [
-    "aggregation_type",
-    "end_time",
-    "labels",
+    AGGREGATION_TYPE,
+    CONFIG_END_TIME,
+    CONFIG_LABELS,
     "pct_threshold",
-    "start_time",
-    "target_table_name",
-    "target_column_name",
+    CONFIG_START_TIME,
+    TARGET_TABLE_NAME,
+    TARGET_COLUMN_NAME,
     "difference",
-    "primary_keys",
-    "group_by_columns",
+    CONFIG_PRIMARY_KEYS,
+    GROUP_BY_COLUMNS,
     "num_random_rows",
 ]
 SCHEMA_VALIDATION_COLUMN_FILTER_LIST = [
-    "start_time",
-    "end_time",
-    "aggregation_type",
+    CONFIG_START_TIME,
+    CONFIG_END_TIME,
+    AGGREGATION_TYPE,
     "difference",
-    "primary_keys",
-    "group_by_columns",
+    CONFIG_PRIMARY_KEYS,
+    GROUP_BY_COLUMNS,
     "num_random_rows",
     "pct_threshold",
 ]

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.3.0}/data_validation/data_validation.py

@@ -353,7 +353,10 @@ class DataValidation(object):
                 target_df = futures[1].result()
 
             pandas_client = ibis.pandas.connect(
-                {combiner.DEFAULT_SOURCE: source_df, combiner.DEFAULT_TARGET: target_df}
+                {
+                    consts.RESULT_TYPE_SOURCE: source_df,
+                    consts.RESULT_TYPE_TARGET: target_df,
+                }
             )
 
             try:
@@ -362,8 +365,8 @@ class DataValidation(object):
                     combiner.generate_report,
                     pandas_client,
                     self.run_metadata,
-                    pandas_client.table(combiner.DEFAULT_SOURCE),
-                    pandas_client.table(combiner.DEFAULT_TARGET),
+                    pandas_client.table(consts.RESULT_TYPE_SOURCE),
+                    pandas_client.table(consts.RESULT_TYPE_TARGET),
                     join_on_fields=join_on_fields,
                     is_value_comparison=is_value_comparison,
                     verbose=self.verbose,