google-pso-data-validator 7.7.0__tar.gz → 8.0.0__tar.gz

{google_pso_data_validator-7.7.0/google_pso_data_validator.egg-info → google_pso_data_validator-8.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 7.7.0
+Version: 8.0.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -32,7 +32,7 @@ Requires-Dist: pyarrow==14.0.1
 Requires-Dist: pydata-google-auth>=1.8.2
 Requires-Dist: PyMySQL>=1.1.1
 Requires-Dist: PyYAML>=6.0.2
-Requires-Dist: SQLAlchemy==1.4.49
+Requires-Dist: SQLAlchemy==2.0.41
 Requires-Dist: tabulate>=0.9.0
 Provides-Extra: apache-airflow
 Requires-Dist: 1.10.11; extra == "apache-airflow"
@@ -171,7 +171,9 @@ data-validation
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
-  [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
+  [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric.
+                        Please note that not all supported SQL engines give results from STDDV_SAMP (or engine specific equivalent) that
+                        are comparable across all other supported SQL engines. This option may produce unreliable results.
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
   [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
@@ -256,6 +258,8 @@ data-validation
                         See: *Calculated Fields* section for details
   --hash COLUMNS        Comma separated list of columns to hash or * for all columns
   --concat COLUMNS      Comma separated list of columns to concatenate or * for all columns (use if a common hash function is not available between databases)
+  --max-concat-columns INT, -mcc INT
+                        Maximum number of columns used in one --hash or --concat validation. When there are more columns in the validation, the validation will be split into multiple validations. There are engine specific defaults, so most users do not need to use this option unless they encounter errors.
   [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
                         Comma separated list of primary key columns, when not specified the value will be inferred
                         from the source or target table if available.  See *Primary Keys* section
@@ -288,8 +292,6 @@ data-validation
                         Row batch size used for random row filters (default 10,000).
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
-  [--trim-string-pks, -tsp]
-                        Trims string based primary key values, intended for use when one engine uses padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n)).
   [--case-insensitive-match, -cim]
                         Performs a case insensitive match by adding an UPPER() before comparison.
 ```
@@ -365,8 +367,6 @@ data-validation
                         Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
-  [--trim-string-pks, -tsp]
-                        Trims string based primary key values, intended for use when one engine uses padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n)).
   [--case-insensitive-match, -cim]
                         Performs a case insensitive match by adding an UPPER() before comparison.
 ```
@@ -454,7 +454,9 @@ data-validation
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
-  [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
+  [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric.
+                        Please note that not all supported SQL engines give results from STDDV_SAMP (or engine specific equivalent) that
+                        are comparable across all other supported SQL engines. This option may produce unreliable results.
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
   [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
@@ -525,6 +527,8 @@ data-validation
   --hash '*'            '*' to hash all columns.
   --concat COLUMNS      Comma separated list of columns to concatenate or * for all columns
                         (use if a common hash function is not available between databases)
+  --max-concat-columns INT, -mcc INT
+                        Maximum number of columns used in one --hash or --concat validation. When there are more columns in the validation, the validation will be split into multiple validations. There are engine specific defaults, so most users do not need to use this option unless they encounter errors.
   [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
                        Common column between source and target queries for join
   [--exclude-columns or -ec]
@@ -548,8 +552,6 @@ data-validation
                         Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
-  [--trim-string-pks, -tsp]
-                        Trims string based primary key values, intended for use when one engine uses padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n)).
   [--case-insensitive-match, -cim]
                         Performs a case insensitive match by adding an UPPER() before comparison.
 ```

{google_pso_data_validator-7.7.0 → google_pso_data_validator-8.0.0}/README.md

@@ -112,7 +112,9 @@ data-validation
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
-  [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
+  [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric.
+                        Please note that not all supported SQL engines give results from STDDV_SAMP (or engine specific equivalent) that
+                        are comparable across all other supported SQL engines. This option may produce unreliable results.
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
   [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
@@ -197,6 +199,8 @@ data-validation
                         See: *Calculated Fields* section for details
   --hash COLUMNS        Comma separated list of columns to hash or * for all columns
   --concat COLUMNS      Comma separated list of columns to concatenate or * for all columns (use if a common hash function is not available between databases)
+  --max-concat-columns INT, -mcc INT
+                        Maximum number of columns used in one --hash or --concat validation. When there are more columns in the validation, the validation will be split into multiple validations. There are engine specific defaults, so most users do not need to use this option unless they encounter errors.
   [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
                         Comma separated list of primary key columns, when not specified the value will be inferred
                         from the source or target table if available.  See *Primary Keys* section
@@ -229,8 +233,6 @@ data-validation
                         Row batch size used for random row filters (default 10,000).
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
-  [--trim-string-pks, -tsp]
-                        Trims string based primary key values, intended for use when one engine uses padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n)).
   [--case-insensitive-match, -cim]
                         Performs a case insensitive match by adding an UPPER() before comparison.
 ```
@@ -306,8 +308,6 @@ data-validation
                         Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
-  [--trim-string-pks, -tsp]
-                        Trims string based primary key values, intended for use when one engine uses padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n)).
   [--case-insensitive-match, -cim]
                         Performs a case insensitive match by adding an UPPER() before comparison.
 ```
@@ -395,7 +395,9 @@ data-validation
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
-  [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
+  [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric.
+                        Please note that not all supported SQL engines give results from STDDV_SAMP (or engine specific equivalent) that
+                        are comparable across all other supported SQL engines. This option may produce unreliable results.
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
   [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
@@ -466,6 +468,8 @@ data-validation
   --hash '*'            '*' to hash all columns.
   --concat COLUMNS      Comma separated list of columns to concatenate or * for all columns
                         (use if a common hash function is not available between databases)
+  --max-concat-columns INT, -mcc INT
+                        Maximum number of columns used in one --hash or --concat validation. When there are more columns in the validation, the validation will be split into multiple validations. There are engine specific defaults, so most users do not need to use this option unless they encounter errors.
   [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
                        Common column between source and target queries for join
   [--exclude-columns or -ec]
@@ -489,8 +493,6 @@ data-validation
                         Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
-  [--trim-string-pks, -tsp]
-                        Trims string based primary key values, intended for use when one engine uses padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n)).
   [--case-insensitive-match, -cim]
                         Performs a case insensitive match by adding an UPPER() before comparison.
 ```

{google_pso_data_validator-7.7.0 → google_pso_data_validator-8.0.0}/data_validation/__main__.py

@@ -108,12 +108,16 @@ def get_aggregate_config(args, config_manager: ConfigManager):
     if args.count:
         col_args = None if args.count == "*" else cli_tools.get_arg_list(args.count)
         aggregate_configs += config_manager.build_config_column_aggregates(
-            "count", col_args, args.exclude_columns, None, cast_to_bigint=cast_to_bigint
+            consts.CONFIG_TYPE_COUNT,
+            col_args,
+            args.exclude_columns,
+            None,
+            cast_to_bigint=cast_to_bigint,
         )
     if args.sum:
         col_args = None if args.sum == "*" else cli_tools.get_arg_list(args.sum)
         aggregate_configs += config_manager.build_config_column_aggregates(
-            "sum",
+            consts.CONFIG_TYPE_SUM,
             col_args,
             args.exclude_columns,
             supported_data_types,
@@ -122,7 +126,7 @@ def get_aggregate_config(args, config_manager: ConfigManager):
     if args.avg:
         col_args = None if args.avg == "*" else cli_tools.get_arg_list(args.avg)
         aggregate_configs += config_manager.build_config_column_aggregates(
-            "avg",
+            consts.CONFIG_TYPE_AVG,
             col_args,
             args.exclude_columns,
             supported_data_types,
@@ -131,7 +135,7 @@ def get_aggregate_config(args, config_manager: ConfigManager):
     if args.min:
         col_args = None if args.min == "*" else cli_tools.get_arg_list(args.min)
         aggregate_configs += config_manager.build_config_column_aggregates(
-            "min",
+            consts.CONFIG_TYPE_MIN,
             col_args,
             args.exclude_columns,
             supported_data_types + uuid_types,
@@ -140,7 +144,7 @@ def get_aggregate_config(args, config_manager: ConfigManager):
     if args.max:
         col_args = None if args.max == "*" else cli_tools.get_arg_list(args.max)
         aggregate_configs += config_manager.build_config_column_aggregates(
-            "max",
+            consts.CONFIG_TYPE_MAX,
             col_args,
             args.exclude_columns,
             supported_data_types + uuid_types,
@@ -149,7 +153,7 @@ def get_aggregate_config(args, config_manager: ConfigManager):
     if args.bit_xor:
         col_args = None if args.bit_xor == "*" else cli_tools.get_arg_list(args.bit_xor)
         aggregate_configs += config_manager.build_config_column_aggregates(
-            "bit_xor",
+            consts.CONFIG_TYPE_BIT_XOR,
             col_args,
             args.exclude_columns,
             supported_data_types,
@@ -158,7 +162,7 @@ def get_aggregate_config(args, config_manager: ConfigManager):
     if args.std:
         col_args = None if args.std == "*" else cli_tools.get_arg_list(args.std)
         aggregate_configs += config_manager.build_config_column_aggregates(
-            "std",
+            consts.CONFIG_TYPE_STD,
             col_args,
             args.exclude_columns,
             supported_data_types,
@@ -181,18 +185,14 @@ def _get_calculated_config(args, config_manager: ConfigManager) -> List[dict]:
             if config_manager.hash == "*"
             else cli_tools.get_arg_list(config_manager.hash)
         )
-        fields = config_manager.build_dependent_aliases(
-            "hash", col_list, args.exclude_columns
-        )
+        fields = config_manager.build_dependent_aliases("hash", col_list)
     elif config_manager.concat:
         col_list = (
             None
             if config_manager.concat == "*"
             else cli_tools.get_arg_list(config_manager.concat)
         )
-        fields = config_manager.build_dependent_aliases(
-            "concat", col_list, args.exclude_columns
-        )
+        fields = config_manager.build_dependent_aliases("concat", col_list)
 
     if len(fields) > 0:
         max_depth = max([x["depth"] for x in fields])
@@ -232,10 +232,7 @@ def _get_comparison_config(
         if args.comparison_fields == "*"
         else cli_tools.get_arg_list(args.comparison_fields)
     )
-    comparison_fields = config_manager.build_comp_fields(
-        col_list,
-        args.exclude_columns,
-    )
+    comparison_fields = config_manager.build_comp_fields(col_list, args.exclude_columns)
     # We can't have the PK columns in the comparison SQL twice therefore filter them out here if included.
     comparison_fields = [_ for _ in comparison_fields if _ not in primary_keys]
 
@@ -520,6 +517,11 @@ def run_validation(config_manager: ConfigManager, dry_run=False, verbose=False):
         if clients.is_sqlalchemy_backend(config_manager.target_client)
         else None
     )
+    # Though trims string based primary key value has been deprecated, some yaml files may still have that property set.
+    if config_manager.trim_string_pks():
+        logging.warning(
+            "Trim String Primary Keys has been deprecated, validation results may vary"
+        )
     with DataValidation(
         config_manager.config,
         validation_builder=None,

{google_pso_data_validator-7.7.0 → google_pso_data_validator-8.0.0}/data_validation/cli_tools.py

@@ -96,9 +96,11 @@ CONNECTION_SOURCE_FIELDS = {
         ["host", "Desired Oracle host"],
         ["port", "Oracle port to connect on"],
         ["user", "User used to connect"],
+        ["thick_mode", "Flag indicating thick_mode"],
         ["password", "Password for supplied user"],
         ["database", "Database to connect to"],
-        ["url", "Oracle SQLAlchemy connection URL"],
+        ["protocol", "Oracle networking protocol (default TPC)"],
+        ["connect_args", "(Optional) Additional connection argument mapping"],
     ],
     consts.SOURCE_TYPE_MSSQL: [
         ["host", "Desired SQL Server host (default localhost)"],
@@ -121,7 +123,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["password", "Password for authentication of user"],
         ["account", "Snowflake account to connect to"],
         ["database", "Database in snowflake to connect to"],
-        ["connect_args", "(Optional) Additional connection arg mapping"],
+        ["connect_args", "(Optional) Additional connection argument mapping"],
     ],
     consts.SOURCE_TYPE_POSTGRES: [
         ["host", "Desired PostgreSQL host."],
@@ -198,6 +200,16 @@ VALIDATE_SCHEMA_HELP_TEXT = "Run a schema validation"
 VALIDATE_CUSTOM_QUERY_HELP_TEXT = "Run a custom query validation"
 
 
+class deprecate_action(argparse.Action):
+    def __init__(self, option_strings, dest, nargs=0, **kwargs):
+        super().__init__(option_strings, dest, nargs=0, **kwargs)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        logging.warning(
+            f"Argument {option_string} is deprecated and may be removed in a future release"
+        )
+
+
 def _check_custom_query_args(parser: argparse.ArgumentParser, parsed_args: "Namespace"):
     # This is where we make additional checks if the arguments provided are what we expect
     # For example, only one of -tbls and custom query options can be provided
@@ -529,7 +541,12 @@ def _configure_database_specific_parsers(parser):
         for field_obj in CONNECTION_SOURCE_FIELDS[database]:
             arg_field = "--" + field_obj[0].replace("_", "-")
             help_txt = field_obj[1]
-            db_parser.add_argument(arg_field, help=help_txt)
+            if (
+                field_obj[0] == "thick_mode" and database == consts.SOURCE_TYPE_ORACLE
+            ):  # flag
+                db_parser.add_argument(arg_field, help=help_txt, action="store_true")
+            else:
+                db_parser.add_argument(arg_field, help=help_txt)
 
 
 def _configure_validate_parser(subparsers):
@@ -607,11 +624,9 @@ def _configure_row_parser(
     optional_arguments.add_argument(
         "--trim-string-pks",
         "-tsp",
-        action="store_true",
-        help=(
-            "Trims string based primary key values, intended for use when one engine uses "
-            "padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n))."
-        ),
+        action=deprecate_action,
+        default=argparse.SUPPRESS,
+        help=argparse.SUPPRESS,
     )
     optional_arguments.add_argument(
         "--case-insensitive-match",
@@ -626,9 +641,9 @@ def _configure_row_parser(
         "-mcc",
         type=int,
         help=(
-            "The maximum number of columns accepted by a --hash or --concat validation. When there are "
-            "more columns than this the validation will implicitly be split into multiple validations. "
-            "This option has engine specific defaults."
+            """Maximum number of columns used in one --hash or --concat validation. When there are more columns
+                in the validation, the validation will be split into multiple validations. There are engine specific
+                defaults, so most users do not need to use this option unless they encounter errors."""
         ),
     )
     # Generate-table-partitions and custom-query does not support random row
@@ -729,7 +744,10 @@ def _configure_column_parser(column_parser):
     optional_arguments.add_argument(
         "--std",
         "-std",
-        help="Comma separated list of columns for standard deviation 'col_a,col_b' or * for all columns",
+        help="""Comma separated list of columns for standard deviation 'col_a,col_b' or * for all columns.
+                Please note that not all supported SQL engines give results from STDDV_SAMP (or engine specific
+                equivalent) that are comparable across all other supported SQL engines. This option may produce
+                unreliable results.""",
     )
     optional_arguments.add_argument(
         "--grouped-columns",
@@ -925,7 +943,10 @@ def _configure_custom_query_column_parser(custom_query_column_parser):
     optional_arguments.add_argument(
         "--std",
         "-std",
-        help="Comma separated list of columns for standard deviation 'col_a,col_b' or * for all columns",
+        help="""Comma separated list of columns for standard deviation 'col_a,col_b' or * for all columns.
+                Please note that not all supported SQL engines give results from STDDV_SAMP (or engine specific
+                equivalent) that are comparable across all other supported SQL engines. This option may produce
+                unreliable results.""",
     )
     optional_arguments.add_argument(
         "--exclude-columns",
@@ -1490,26 +1511,49 @@ def _concat_column_count_configs(
     return return_list
 
 
+def _get_pre_build_configs_base_columns(
+    client, table_obj: dict, query_str: str
+) -> list:
+    """Return a list of base columns for the table/custom query, used for both columns="*" and --exclude-columns."""
+    if table_obj:
+        full_col_list = clients.get_ibis_table_schema(
+            client,
+            table_obj["schema_name"],
+            table_obj["table_name"],
+        ).names
+    else:
+        full_col_list = clients.get_ibis_query_schema(
+            client,
+            query_str,
+        ).names
+    return [_.casefold() for _ in full_col_list]
+
+
+def _get_pre_build_configs_cols_from_arg(
+    column_csv_arg: str, casefold_columns: list, exclude_columns: bool
+) -> Optional[list]:
+    if column_csv_arg is None:
+        return None
+
+    column_arg_list = [_.casefold() for _ in get_arg_list(column_csv_arg)]
+    if column_csv_arg == "*" and exclude_columns:
+        raise ValueError(
+            "Exclude columns flag cannot be present with '*' column specification"
+        )
+    elif column_csv_arg == "*" or exclude_columns:
+        # If validating with "*" or need to invert the list then we need to expand to count the columns.
+        if column_csv_arg == "*":
+            return casefold_columns
+
+        if exclude_columns:
+            return [col for col in casefold_columns if col not in column_arg_list]
+    else:
+        return column_arg_list
+
+
 def get_pre_build_configs(args: "Namespace", validate_cmd: str) -> List[Dict]:
     """Return a dict of configurations to build ConfigManager object"""
 
-    def cols_from_arg(concat_arg: str, client, table_obj: dict, query_str: str) -> list:
-        if concat_arg == "*":
-            # If validating with "*" then we need to expand to count the columns.
-            if table_obj:
-                return clients.get_ibis_table_schema(
-                    client,
-                    table_obj["schema_name"],
-                    table_obj["table_name"],
-                ).names
-            else:
-                return clients.get_ibis_query_schema(
-                    client,
-                    query_str,
-                ).names
-        else:
-            return get_arg_list(concat_arg)
-
     # validate_cmd will be set to 'row`, or 'Custom-query' if invoked by generate-table-partitions depending
     # on what is being partitioned. Otherwise validate_cmd will be set to None
     if validate_cmd is None:
@@ -1589,6 +1633,10 @@ def get_pre_build_configs(args: "Namespace", validate_cmd: str) -> List[Dict]:
             tables_list, source_client, target_client
         )
     for table_obj in tables_list:
+        casefold_source_columns = _get_pre_build_configs_base_columns(
+            source_client, table_obj, query_str
+        )
+
         pre_build_configs = {
             "config_type": config_type,
             consts.CONFIG_SOURCE_CONN_NAME: args.source_conn,
@@ -1604,9 +1652,6 @@ def get_pre_build_configs(args: "Namespace", validate_cmd: str) -> List[Dict]:
             "result_handler_config": result_handler_config,
             "filter_config": filter_config,
             consts.CONFIG_FILTER_STATUS: filter_status,
-            consts.CONFIG_TRIM_STRING_PKS: getattr(
-                args, consts.CONFIG_TRIM_STRING_PKS, False
-            ),
             consts.CONFIG_CASE_INSENSITIVE_MATCH: getattr(
                 args, consts.CONFIG_CASE_INSENSITIVE_MATCH, False
             ),
@@ -1615,18 +1660,25 @@ def get_pre_build_configs(args: "Namespace", validate_cmd: str) -> List[Dict]:
             consts.CONFIG_RUN_ID: getattr(args, consts.CONFIG_RUN_ID, None),
             "verbose": args.verbose,
         }
+
         if (
             pre_build_configs[consts.CONFIG_ROW_CONCAT]
             or pre_build_configs[consts.CONFIG_ROW_HASH]
         ):
             # Ensure we don't have too many columns for the engines involved.
-            cols = cols_from_arg(
+            cols = _get_pre_build_configs_cols_from_arg(
                 pre_build_configs[consts.CONFIG_ROW_HASH]
                 or pre_build_configs[consts.CONFIG_ROW_CONCAT],
-                source_client,
-                table_obj,
-                query_str,
+                casefold_source_columns,
+                args.exclude_columns,
             )
+            if args.exclude_columns:
+                # Put the column csv back into pre_build_configs because it has been inverted.
+                if pre_build_configs[consts.CONFIG_ROW_HASH]:
+                    pre_build_configs[consts.CONFIG_ROW_HASH] = ",".join(cols)
+                elif pre_build_configs[consts.CONFIG_ROW_CONCAT]:
+                    pre_build_configs[consts.CONFIG_ROW_CONCAT] = ",".join(cols)
+
             new_pre_build_configs = _concat_column_count_configs(
                 cols,
                 pre_build_configs,

{google_pso_data_validator-7.7.0 → google_pso_data_validator-8.0.0}/data_validation/clients.py

@@ -73,11 +73,11 @@ except Exception:
     msg = "pip install teradatasql (requires Teradata licensing)"
     teradata_connect = _raise_missing_client_error(msg)
 
-# Oracle requires cx_Oracle driver
+# Oracle requires python-oracldb driver
 try:
     from third_party.ibis.ibis_oracle.api import oracle_connect
 except Exception:
-    oracle_connect = _raise_missing_client_error("pip install cx_Oracle")
+    oracle_connect = _raise_missing_client_error("pip install oracledb")
 
 # Snowflake requires snowflake-connector-python and snowflake-sqlalchemy
 try:
@@ -90,7 +90,8 @@ except Exception:
 # DB2 requires ibm_db_sa
 try:
     from third_party.ibis.ibis_db2.api import db2_connect
-except Exception:
+except Exception as e:
+    logging.error(f"Exception {str(e)} while importing db2_connect")
     db2_connect = _raise_missing_client_error("pip install ibm_db_sa")
 
 
@@ -261,7 +262,7 @@ def list_tables(client, schema_name, tables_only=True):
         if tables_only and client.name != "pandas"
         else client.list_tables
     )
-    if client.name in ["db2", "mssql", "redshift", "snowflake", "pandas"]:
+    if client.name in ["db2", "redshift", "snowflake", "pandas"]:
         return fn()
     return fn(database=schema_name)
 
@@ -320,6 +321,14 @@ def get_data_client(connection_config):
                 key_path
             )
 
+    # Oracle no longer supports the url option, so check and warn users
+    if (
+        source_type == consts.SOURCE_TYPE_ORACLE
+        and "url" in decrypted_connection_config
+    ):
+        msg = f"Connection Configuration Error: url parameter no longer supported for {source_type}.\nRecreate connection with --connect-args parameter."
+        raise Exception(msg)
+
     if source_type not in CLIENT_LOOKUP:
         msg = 'ConfigurationError: Source type "{source_type}" is not supported'.format(
             source_type=source_type