google-pso-data-validator 7.5.2__tar.gz → 7.6.0__tar.gz

{google_pso_data_validator-7.5.2 → google_pso_data_validator-7.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 7.5.2
+Version: 7.6.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -172,9 +172,12 @@ data-validation
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--wildcard-include-string-len or -wis]
@@ -256,9 +259,12 @@ data-validation
                         from the source or target table if available.  See *Primary Keys* section
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--filters SOURCE_FILTER:TARGET_FILTER]
@@ -337,9 +343,12 @@ data-validation
   [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
                         Comma separated list of primary key columns, when not specified the value will be inferred
                         from the source or target table if available.  See *Primary Keys* section
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--parts-per-file INT], [-ppf INT]
@@ -383,9 +392,12 @@ data-validation
                         Comma separated list of tables in the form schema.table=target_schema.target_table. Or shorthand schema.* for all tables.
                         Target schema name and table name are optional.
                         e.g.: 'bigquery-public-data.new_york_citibike.citibike_trips'
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
@@ -443,9 +455,12 @@ data-validation
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
@@ -512,9 +527,12 @@ data-validation
                        Common column between source and target queries for join
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
@@ -624,20 +642,40 @@ The `--config-dir` flag will specify the directory with the YAML files to be exe
 
 ### Validation Reports
 
-The result handlers tell DVT where to store the results of
-each validation. The tool can write the results of a validation run to Google
-BigQuery or print to stdout (default). View the schema of the results
-table [here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/terraform/results_schema.json).
+The result handlers tell DVT where to store the results of each validation. The
+tool can write the results of a validation run to Google BigQuery, PostgreSQL
+or print to stdout (default). View the schema of the results table [here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/terraform/results_schema.json).
+
+To output to BigQuery or PostgreSQL, simply include the `-rh` flag during a validation run including
+the schema and table name for the results.
+
+BigQuery example by connection name:
+```shell
+data-validation validate column \
+  -sc bq_conn \
+  -tc bq_conn \
+  -tbls bigquery-public-data.new_york_citibike.citibike_trips \
+  -rh bq_conn.dataset.results_table \
+  -sa 'service-acct@project.iam.gserviceaccount.com'
+```
 
-To output to BigQuery, simply include the `-bqrh` flag during a validation run
-like so:
+BigQuery example by project name:
+```shell
+data-validation validate column \
+  -sc bq_conn \
+  -tc bq_conn \
+  -tbls bigquery-public-data.new_york_citibike.citibike_trips \
+  -rh bq-project-id.dataset.results_table \
+  -sa 'service-acct@project.iam.gserviceaccount.com'
 ```
-data-validation validate column
-  -sc bq_conn
-  -tc bq_conn
-  -tbls bigquery-public-data.new_york_citibike.citibike_trips
-  -bqrh project_id.dataset.table
-  -sa service-acct@project.iam.gserviceaccount.com
+
+PostgreSQL example:
+```shell
+data-validation validate column \
+  -sc ora_conn \
+  -tc pg_conn1 \
+  -tbls my_schema.some_table \
+  -rh pg_conn2.dvt_schema.results_table
 ```
 
 ### Ad Hoc SQL Exploration
@@ -670,7 +708,7 @@ target. The `find-tables` command:
 
 Note that our default value for the `score-cutoff` parameter is 1 and it seeks for identical matches. If no matches occur, reduce this value as deemed necessary. By using smaller numbers such as 0.7, 0.65 etc you can get more matches. For reference, we make use of [this jaro_similarity method](https://jamesturk.github.io/jellyfish/functions/#jaro-similarity) for the string comparison.
 
-```
+```shell
 data-validation find-tables --source-conn source --target-conn target \
     --allowed-schemas pso_data_validator \
     --score-cutoff 1

{google_pso_data_validator-7.5.2 → google_pso_data_validator-7.6.0}/README.md

@@ -113,9 +113,12 @@ data-validation
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--wildcard-include-string-len or -wis]
@@ -197,9 +200,12 @@ data-validation
                         from the source or target table if available.  See *Primary Keys* section
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--filters SOURCE_FILTER:TARGET_FILTER]
@@ -278,9 +284,12 @@ data-validation
   [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
                         Comma separated list of primary key columns, when not specified the value will be inferred
                         from the source or target table if available.  See *Primary Keys* section
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--parts-per-file INT], [-ppf INT]
@@ -324,9 +333,12 @@ data-validation
                         Comma separated list of tables in the form schema.table=target_schema.target_table. Or shorthand schema.* for all tables.
                         Target schema name and table name are optional.
                         e.g.: 'bigquery-public-data.new_york_citibike.citibike_trips'
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
@@ -384,9 +396,12 @@ data-validation
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
@@ -453,9 +468,12 @@ data-validation
                        Common column between source and target queries for join
   [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
-  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
-                        BigQuery destination for validation results. Defaults to stdout.
+  [--result-handler or -rh CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE]
+                        Specify a BigQuery or PostgreSQL connection name as destination for validation results.
+                        Also supports legacy BigQuery format BQ_PROJECT_ID.DATASET.TABLE.
                         See: *Validation Reports* section
+  [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
+                        This option has been deprecated and will be removed in a future release.
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
@@ -565,20 +583,40 @@ The `--config-dir` flag will specify the directory with the YAML files to be exe
 
 ### Validation Reports
 
-The result handlers tell DVT where to store the results of
-each validation. The tool can write the results of a validation run to Google
-BigQuery or print to stdout (default). View the schema of the results
-table [here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/terraform/results_schema.json).
+The result handlers tell DVT where to store the results of each validation. The
+tool can write the results of a validation run to Google BigQuery, PostgreSQL
+or print to stdout (default). View the schema of the results table [here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/terraform/results_schema.json).
+
+To output to BigQuery or PostgreSQL, simply include the `-rh` flag during a validation run including
+the schema and table name for the results.
+
+BigQuery example by connection name:
+```shell
+data-validation validate column \
+  -sc bq_conn \
+  -tc bq_conn \
+  -tbls bigquery-public-data.new_york_citibike.citibike_trips \
+  -rh bq_conn.dataset.results_table \
+  -sa 'service-acct@project.iam.gserviceaccount.com'
+```
 
-To output to BigQuery, simply include the `-bqrh` flag during a validation run
-like so:
+BigQuery example by project name:
+```shell
+data-validation validate column \
+  -sc bq_conn \
+  -tc bq_conn \
+  -tbls bigquery-public-data.new_york_citibike.citibike_trips \
+  -rh bq-project-id.dataset.results_table \
+  -sa 'service-acct@project.iam.gserviceaccount.com'
 ```
-data-validation validate column
-  -sc bq_conn
-  -tc bq_conn
-  -tbls bigquery-public-data.new_york_citibike.citibike_trips
-  -bqrh project_id.dataset.table
-  -sa service-acct@project.iam.gserviceaccount.com
+
+PostgreSQL example:
+```shell
+data-validation validate column \
+  -sc ora_conn \
+  -tc pg_conn1 \
+  -tbls my_schema.some_table \
+  -rh pg_conn2.dvt_schema.results_table
 ```
 
 ### Ad Hoc SQL Exploration
@@ -611,7 +649,7 @@ target. The `find-tables` command:
 
 Note that our default value for the `score-cutoff` parameter is 1 and it seeks for identical matches. If no matches occur, reduce this value as deemed necessary. By using smaller numbers such as 0.7, 0.65 etc you can get more matches. For reference, we make use of [this jaro_similarity method](https://jamesturk.github.io/jellyfish/functions/#jaro-similarity) for the string comparison.
 
-```
+```shell
 data-validation find-tables --source-conn source --target-conn target \
     --allowed-schemas pso_data_validator \
     --score-cutoff 1

{google_pso_data_validator-7.5.2 → google_pso_data_validator-7.6.0}/data_validation/__main__.py

@@ -530,35 +530,15 @@ def run_validation(config_manager: ConfigManager, dry_run=False, verbose=False):
     ) as validator:
 
         if dry_run:
-            sql_alchemy_clients = [
-                "mysql",
-                "oracle",
-                "postgres",
-                "db2",
-                "mssql",
-                "redshift",
-                "snowflake",
-            ]
-
-            source_query = validator.validation_builder.get_source_query().compile()
-            if config_manager.source_client.name in sql_alchemy_clients:
-                source_query = source_query.compile(
-                    config_manager.source_client.con.engine,
-                    compile_kwargs={"literal_binds": True},
-                )
-
-            target_query = validator.validation_builder.get_target_query().compile()
-            if config_manager.target_client.name in sql_alchemy_clients:
-                target_query = target_query.compile(
-                    config_manager.target_client.con.engine,
-                    compile_kwargs={"literal_binds": True},
-                )
-
             print(
                 json.dumps(
                     {
-                        "source_query": str(source_query),
-                        "target_query": str(target_query),
+                        "source_query": util.ibis_table_to_sql(
+                            validator.validation_builder.get_source_query()
+                        ),
+                        "target_query": util.ibis_table_to_sql(
+                            validator.validation_builder.get_target_query()
+                        ),
                     },
                     indent=4,
                 )

{google_pso_data_validator-7.5.2 → google_pso_data_validator-7.6.0}/data_validation/cli_tools.py

@@ -20,7 +20,7 @@ The Data Validator can be called either using:
 
 ex.
 Step 1) Store Connection to be used in validation
-data-validation connections add -c my_bq_conn BigQuery --project-id pso-kokoro-resources
+data-validation connections add -c my_bq_conn BigQuery --project-id pso-project
 
 Step 2) Run Validation using supplied connections
 data-validation validate column -sc my_bq_conn -tc my_bq_conn \
@@ -36,7 +36,7 @@ data-validation validate column \
 -sc my_bq_conn -tc my_bq_conn \
 -tbls bigquery-public-data.new_york_citibike.citibike_trips,bigquery-public-data.new_york_citibike.citibike_stations \
 --sum tripduration,start_station_name --count tripduration,start_station_name \
--bqrh pso-kokoro-resources.pso_data_validator.results
+-rh pso-project.pso_data_validator.results \
 -c ex_yaml.yaml
 
 command:
@@ -58,6 +58,7 @@ from yaml import Dumper, Loader, dump, load
 from data_validation import (
     clients,
     consts,
+    exceptions,
     find_tables,
     state_manager,
     gcs_helper,
@@ -70,7 +71,7 @@ if TYPE_CHECKING:
 
 
 CONNECTION_SOURCE_FIELDS = {
-    "BigQuery": [
+    consts.SOURCE_TYPE_BIGQUERY: [
         ["project_id", "GCP Project to use for BigQuery"],
         ["google_service_account_key_path", "(Optional) GCP SA Key Path"],
         [
@@ -78,7 +79,7 @@ CONNECTION_SOURCE_FIELDS = {
             '(Optional) GCP BigQuery API endpoint (e.g. "https://mybq.p.googleapis.com")',
         ],
     ],
-    "Teradata": [
+    consts.SOURCE_TYPE_TERADATA: [
         ["host", "Desired Teradata host"],
         ["port", "Teradata port to connect on"],
         ["user_name", "User used to connect"],
@@ -87,7 +88,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["use_no_lock_tables", "Use an access lock for queries (defaults to False)"],
         ["json_params", "(Optional) Additional teradatasql JSON string parameters"],
     ],
-    "Oracle": [
+    consts.SOURCE_TYPE_ORACLE: [
         ["host", "Desired Oracle host"],
         ["port", "Oracle port to connect on"],
         ["user", "User used to connect"],
@@ -95,7 +96,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["database", "Database to connect to"],
         ["url", "Oracle SQLAlchemy connection URL"],
     ],
-    "MSSQL": [
+    consts.SOURCE_TYPE_MSSQL: [
         ["host", "Desired SQL Server host (default localhost)"],
         ["port", "SQL Server port to connect on (default 1433)"],
         ["user", "User used to connect"],
@@ -104,35 +105,35 @@ CONNECTION_SOURCE_FIELDS = {
         ["query", "Connection query parameters"],
         ["url", "SQL Server SQLAlchemy connection URL"],
     ],
-    "MySQL": [
+    consts.SOURCE_TYPE_MYSQL: [
         ["host", "Desired MySQL host (default localhost)"],
         ["port", "MySQL port to connect on (default 3306)"],
         ["user", "User used to connect"],
         ["password", "Password for supplied user"],
         ["database", "Database to connect to (default master)"],
     ],
-    "Snowflake": [
+    consts.SOURCE_TYPE_SNOWFLAKE: [
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
         ["account", "Snowflake account to connect to"],
         ["database", "Database in snowflake to connect to"],
         ["connect_args", "(Optional) Additional connection arg mapping"],
     ],
-    "Postgres": [
+    consts.SOURCE_TYPE_POSTGRES: [
         ["host", "Desired PostgreSQL host."],
         ["port", "PostgreSQL port to connect on (e.g. 5432)"],
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
         ["database", "Database in PostgreSQL to connect to (default postgres)"],
     ],
-    "Redshift": [
+    consts.SOURCE_TYPE_REDSHIFT: [
         ["host", "Desired Redshift host."],
         ["port", "Redshift port to connect on (e.g. 5439)"],
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
         ["database", "Database in Redshift to connect to"],
     ],
-    "Spanner": [
+    consts.SOURCE_TYPE_SPANNER: [
         ["project_id", "GCP Project to use for Spanner"],
         ["instance_id", "ID of Spanner instance to connect to"],
         ["database_id", "ID of Spanner database (schema) to connect to"],
@@ -142,12 +143,12 @@ CONNECTION_SOURCE_FIELDS = {
             '(Optional) GCP Spanner API endpoint (e.g. "https://mycs.p.googleapis.com")',
         ],
     ],
-    "FileSystem": [
+    consts.SOURCE_TYPE_FILESYSTEM: [
         ["table_name", "Table name to use as reference for file data"],
         ["file_path", "The local, s3, or GCS file path to the data"],
         ["file_type", "The file type of the file. 'csv', 'orc', 'parquet' or 'json'"],
     ],
-    "Impala": [
+    consts.SOURCE_TYPE_IMPALA: [
         ["host", "Desired Impala host"],
         ["port", "Desired Impala port (10000 if not provided)"],
         ["database", "Desired Impala database (default if not provided)"],
@@ -175,7 +176,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["use_http_transport", "Boolean if HTTP proxy is provided (default is False)"],
         ["http_path", "URL path of HTTP proxy"],
     ],
-    "DB2": [
+    consts.SOURCE_TYPE_DB2: [
         ["host", "Desired DB2 host"],
         ["port", "Desired DB2 port (50000 if not provided)"],
         ["user", "Username to connect to"],
@@ -297,7 +298,7 @@ def _configure_partition_parser(subparsers):
         "-pn",
         required=True,
         help="Number of partitions into which the table should be split",
-        type=_check_positive,
+        type=_check_gt_one,
     )
     # User can provide tables or custom queries, but not both
     # However, Argparse does not support adding an argument_group to an argument_group or adding a
@@ -1012,7 +1013,15 @@ def _add_common_arguments(
 
     # Optional arguments
     optional_arguments.add_argument(
-        "--bq-result-handler", "-bqrh", help="BigQuery result handler config details"
+        "--bq-result-handler", "-bqrh", help=argparse.SUPPRESS
+    )
+    optional_arguments.add_argument(
+        "--result-handler",
+        "-rh",
+        help=(
+            "Result handler connection details. "
+            "CONNECTION_NAME.SCHEMA.TABLE or BQ_PROJECT_ID.DATASET.TABLE."
+        ),
     )
     optional_arguments.add_argument(
         "--labels", "-l", help="Key value pair labels for validation run"
@@ -1055,13 +1064,22 @@ def _add_common_arguments(
     )
 
 
-def _check_positive(value: int) -> int:
+def _check_positive(value: int, lower_bound: int = 1) -> int:
     ivalue = int(value)
-    if ivalue <= 0:
-        raise argparse.ArgumentTypeError("%s is an invalid positive int value" % value)
+    if ivalue < lower_bound:
+        if lower_bound == 1:
+            raise argparse.ArgumentTypeError(
+                f"{value} is an invalid positive int value"
+            )
+        else:
+            raise argparse.ArgumentTypeError(f"Value must be >= {lower_bound}: {value}")
     return ivalue
 
 
+def _check_gt_one(value: int) -> int:
+    return _check_positive(value, lower_bound=2)
+
+
 def check_no_yaml_files(partition_num: int, parts_per_file: int):
     """Check that number of yaml files generated is less than 10,001
     Will be invoked after all the arguments are processed."""
@@ -1244,7 +1262,7 @@ def get_filters(filter_value: str) -> List[Dict]:
     return filter_config
 
 
-def get_result_handler(rc_value: str, sa_file=None) -> dict:
+def _get_result_handler(rc_value: str, sa_file=None) -> dict:
     """Returns dict of result handler config. Backwards compatible for JSON input.
 
     rc_value (str): Result config argument specified.
@@ -1254,22 +1272,34 @@ def get_result_handler(rc_value: str, sa_file=None) -> dict:
     if len(config) != 2:
         raise ValueError(f"Unable to parse result handler config: `{rc_value}`")
 
-    # Check if the first part of the BQRH is a connection name.
+    # Check if the first part of the result handler is a connection name.
     mgr = state_manager.StateManager()
     connections = mgr.list_connections()
     if config[0] in connections:
-        # We received connection_name.bq_results_table
+        # We received connection_name.results_table.
         conn_from_file = get_connection(config[0])
-        result_handler = {
-            "type": "BigQuery",
-            consts.PROJECT_ID: conn_from_file["project_id"],
-            consts.TABLE_ID: config[1],
-            consts.API_ENDPOINT: conn_from_file.get("api_endpoint", None),
-        }
+        if conn_from_file[consts.SOURCE_TYPE] == consts.SOURCE_TYPE_BIGQUERY:
+            result_handler = {
+                consts.RH_TYPE: conn_from_file[consts.SOURCE_TYPE],
+                consts.PROJECT_ID: conn_from_file["project_id"],
+                consts.TABLE_ID: config[1],
+                consts.API_ENDPOINT: conn_from_file.get("api_endpoint", None),
+            }
+        elif conn_from_file[consts.SOURCE_TYPE] == consts.SOURCE_TYPE_POSTGRES:
+            result_handler = {
+                consts.RH_TYPE: conn_from_file[consts.SOURCE_TYPE],
+                consts.TABLE_ID: config[1],
+                consts.RH_CONN: conn_from_file,
+            }
+        # TODO Add filesytem handler too.
+        else:
+            raise exceptions.ResultHandlerException(
+                f"Unsupported result handler connection type: {conn_from_file[consts.SOURCE_TYPE]}"
+            )
     else:
-        # We received project_name.bq_results_table
+        # We received legacy format "bq-project-name.bq_results_table".
         result_handler = {
-            "type": "BigQuery",
+            consts.RH_TYPE: consts.SOURCE_TYPE_BIGQUERY,
             consts.PROJECT_ID: config[0],
             consts.TABLE_ID: config[1],
         }
@@ -1489,10 +1519,12 @@ def get_pre_build_configs(args: "Namespace", validate_cmd: str) -> List[Dict]:
     else:
         raise ValueError(f"Unknown Validation Type: {validate_cmd}")
 
+    # Cater for legacy -bqrh.
+    args.result_handler = args.result_handler or args.bq_result_handler
     # Get result handler config
-    if args.bq_result_handler:
-        result_handler_config = get_result_handler(
-            args.bq_result_handler, args.service_account
+    if args.result_handler:
+        result_handler_config = _get_result_handler(
+            args.result_handler, args.service_account
         )
     else:
         result_handler_config = None

{google_pso_data_validator-7.5.2 → google_pso_data_validator-7.6.0}/data_validation/clients.py

@@ -369,16 +369,16 @@ def get_max_in_list_size(client, in_list_over_expressions=False):
 
 
 CLIENT_LOOKUP = {
-    "BigQuery": get_bigquery_client,
-    "Impala": impala_connect,
-    "MySQL": ibis.mysql.connect,
-    "Oracle": oracle_connect,
-    "FileSystem": get_pandas_client,
-    "Postgres": ibis.postgres.connect,
-    "Redshift": redshift_connect,
-    "Teradata": teradata_connect,
-    "MSSQL": mssql_connect,
-    "Snowflake": snowflake_connect,
-    "Spanner": spanner_connect,
-    "DB2": db2_connect,
+    consts.SOURCE_TYPE_BIGQUERY: get_bigquery_client,
+    consts.SOURCE_TYPE_IMPALA: impala_connect,
+    consts.SOURCE_TYPE_MYSQL: ibis.mysql.connect,
+    consts.SOURCE_TYPE_ORACLE: oracle_connect,
+    consts.SOURCE_TYPE_FILESYSTEM: get_pandas_client,
+    consts.SOURCE_TYPE_POSTGRES: ibis.postgres.connect,
+    consts.SOURCE_TYPE_REDSHIFT: redshift_connect,
+    consts.SOURCE_TYPE_TERADATA: teradata_connect,
+    consts.SOURCE_TYPE_MSSQL: mssql_connect,
+    consts.SOURCE_TYPE_SNOWFLAKE: snowflake_connect,
+    consts.SOURCE_TYPE_SPANNER: spanner_connect,
+    consts.SOURCE_TYPE_DB2: db2_connect,
 }