google-pso-data-validator 4.5.0__tar.gz → 5.0.0__tar.gz

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: google-pso-data-validator
-Version: 4.5.0
+Version: 5.0.0
 Summary: A package to enable easy data validation
 Home-page: https://github.com/pypa/sampleproject
 Author: Dylan Hercher
@@ -172,7 +172,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -242,7 +242,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate row
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -322,7 +322,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate schema
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -369,7 +369,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
-  [--exclude-columns or -ec]   
+  [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
                         BigQuery destination for validation results. Defaults to stdout.
@@ -377,7 +377,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -439,7 +439,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -483,11 +483,10 @@ For example, this flag can be used as follows:
 
 Running DVT with YAML configuration files is the recommended approach if:
 * you want to customize the configuration for any given validation OR
-* you want to run DVT at scale (i.e. row validations across many partitions)
+* you want to run DVT at scale (i.e. run multiple validations sequentially or in parallel)
 
-We recommend generating YAML configs with the `--config-file <file-name>` flag when running a validation command. The validation 
-config file is saved to the GCS path specified by the `PSO_DV_CONFIG_HOME` env variable if that has been set;
-otherwise, it is saved to wherever the tool is run.
+We recommend generating YAML configs with the `--config-file <file-name>` flag when running a validation command, which supports
+GCS and local paths.
 
 You can use the `data-validation configs` command to run and view YAMLs.
 
@@ -500,7 +499,7 @@ data-validation (--verbose or -v) (--log-level or -ll) configs run
   [--dry-run or -dr]    If this flag is present, prints the source and target SQL generated in lieu of running the validation.
   [--kube-completions or -kc]
                         Flag to indicate usage in Kubernetes index completion mode.
-                        See *Scaling DVT to run 10's to 1000's of validations concurrently* section
+                        See *Scaling DVT* section
 ```
 
 ```
@@ -518,19 +517,17 @@ View the complete YAML file for a Grouped Column validation on the
 [Examples](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/examples.md#sample-yaml-config-grouped-column-validation) page.
 
 
-#### Scaling DVT to run 10's to 1000's of validations concurrently
+### Scaling DVT
 
-As described above, you can run multiple validation configs sequentially with the `--config-dir` flag. To optimize the validation speed for large tables, you can run DVT concurrently using GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Kubernetes](docs/internal/kubernetes_jobs.md) for a detailed overview.
+You can scale DVT for large table validations by running the tool in a distributed manner. To optimize the validation speed for large tables, you can use GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Distributed Jobs](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/internal/distributed_jobs.md) for a detailed overview.
 
-In order to validate partitions concurrently, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. The `--config-dir` flag will specify the directory with the YAML files to be executed in parallel. If you used `generate-table-partitions` to generate the YAMLs, this would be the directory where the partition files numbered `0000.yaml` to `<partition_num - 1>.yaml` are stored i.e (`my_config_dir/source_schema.source_table/`)
 
-In order to run DVT in a container, you have to build a Docker image - [see instructions here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/docker#readme). You will also need to set the `PSO_DV_CONFIG_HOME` environment variable to point to a GCS prefix where the connection configuration files are stored. In Cloud Run you can use the option `--set-env-vars` or `--update-env-vars` to pass [the environment variable](https://cloud.google.com/run/docs/configuring/services/environment-variables#setting). We recommend that you use the `bq-result-handler` to save your validation results to BigQuery. 
+We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started. 
 
-The Cloud Run and Kubernetes pods must run in a network with access to the database servers. Every Cloud Run job is associated with a [service account](https://cloud.google.com/run/docs/securing/service-identity). You need to ensure that this service account has access to Google Cloud Storage (to read connection configuration and YAML files) and BigQuery (to publish results). If you are using Kubernetes, you will need to use a service account with the same privileges as mentioned for Cloud Run. In Kubernetes, you will need to set up [workload identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) so the DVT container can impersonate the service account.
+When running DVT in a distributed fashion, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (i.e. a `--config-file` is provided instead), a warning is issued.
 
-In Cloud Run, the [job](https://cloud.google.com/run/docs/create-jobs) must be run as multiple, independent tasks with the task count set to the number of partitions generated. In Kubernetes, set the number of completions to the number of partitions generated - see [Kubernetes Parallel Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#parallel-jobs). The option `--kube-completions or -kc` tells DVT that many DVT containers are running in a Kubernetes cluster. Each DVT container only validates the specific partition YAML (based on the index assigned by Kubernetes control plane). If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (a `--config-file` is provided instead), a warning is issued.
+The `--config-dir` flag will specify the directory with the YAML files to be executed in parallel. If you used `generate-table-partitions` to generate the YAMLs, this would be the directory where the partition files numbered `0000.yaml` to `<partition_num - 1>.yaml` are stored i.e (`gs://my_config_dir/source_schema.source_table/`). When creating your Cloud Run Job, set the number of tasks equal to the number of table partitions so the task index matches the YAML file to be validated. When executed, each Cloud Run task will validate a partition in parallel.
 
-By default, each partition validation is retried up to 3 times if there is an error. In Kubernetes and Cloud Run, you can set the parallelism to the number you want. Keep in mind that if you are validating 1000's of partitions in parallel, you may find that setting the parallelism too high (say 100) may result in timeouts and slow down the validation.   
 
 ### Validation Reports
 
@@ -635,7 +632,8 @@ with calculated fields as shown in [this example](https://github.com/GoogleCloud
 
 Filters let you apply a WHERE statement to your validation query (ie. `SELECT *
 FROM table WHERE created_at > 30 days ago AND region_id = 71;`). The filter is
-written in the syntax of the given source.
+written in the syntax of the given source and must reference columns in the
+underlying table, not projected DVT expressions.
 
 Note that you are writing the query to execute, which does not have to match
 between source and target as long as the results can be expected to align. If

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.0.0}/README.md

@@ -124,7 +124,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -194,7 +194,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate row
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -274,7 +274,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate schema
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -321,7 +321,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
-  [--exclude-columns or -ec]   
+  [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
                         BigQuery destination for validation results. Defaults to stdout.
@@ -329,7 +329,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -391,7 +391,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -435,11 +435,10 @@ For example, this flag can be used as follows:
 
 Running DVT with YAML configuration files is the recommended approach if:
 * you want to customize the configuration for any given validation OR
-* you want to run DVT at scale (i.e. row validations across many partitions)
+* you want to run DVT at scale (i.e. run multiple validations sequentially or in parallel)
 
-We recommend generating YAML configs with the `--config-file <file-name>` flag when running a validation command. The validation 
-config file is saved to the GCS path specified by the `PSO_DV_CONFIG_HOME` env variable if that has been set;
-otherwise, it is saved to wherever the tool is run.
+We recommend generating YAML configs with the `--config-file <file-name>` flag when running a validation command, which supports
+GCS and local paths.
 
 You can use the `data-validation configs` command to run and view YAMLs.
 
@@ -452,7 +451,7 @@ data-validation (--verbose or -v) (--log-level or -ll) configs run
   [--dry-run or -dr]    If this flag is present, prints the source and target SQL generated in lieu of running the validation.
   [--kube-completions or -kc]
                         Flag to indicate usage in Kubernetes index completion mode.
-                        See *Scaling DVT to run 10's to 1000's of validations concurrently* section
+                        See *Scaling DVT* section
 ```
 
 ```
@@ -470,19 +469,17 @@ View the complete YAML file for a Grouped Column validation on the
 [Examples](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/examples.md#sample-yaml-config-grouped-column-validation) page.
 
 
-#### Scaling DVT to run 10's to 1000's of validations concurrently
+### Scaling DVT
 
-As described above, you can run multiple validation configs sequentially with the `--config-dir` flag. To optimize the validation speed for large tables, you can run DVT concurrently using GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Kubernetes](docs/internal/kubernetes_jobs.md) for a detailed overview.
+You can scale DVT for large table validations by running the tool in a distributed manner. To optimize the validation speed for large tables, you can use GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Distributed Jobs](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/internal/distributed_jobs.md) for a detailed overview.
 
-In order to validate partitions concurrently, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. The `--config-dir` flag will specify the directory with the YAML files to be executed in parallel. If you used `generate-table-partitions` to generate the YAMLs, this would be the directory where the partition files numbered `0000.yaml` to `<partition_num - 1>.yaml` are stored i.e (`my_config_dir/source_schema.source_table/`)
 
-In order to run DVT in a container, you have to build a Docker image - [see instructions here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/docker#readme). You will also need to set the `PSO_DV_CONFIG_HOME` environment variable to point to a GCS prefix where the connection configuration files are stored. In Cloud Run you can use the option `--set-env-vars` or `--update-env-vars` to pass [the environment variable](https://cloud.google.com/run/docs/configuring/services/environment-variables#setting). We recommend that you use the `bq-result-handler` to save your validation results to BigQuery. 
+We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started. 
 
-The Cloud Run and Kubernetes pods must run in a network with access to the database servers. Every Cloud Run job is associated with a [service account](https://cloud.google.com/run/docs/securing/service-identity). You need to ensure that this service account has access to Google Cloud Storage (to read connection configuration and YAML files) and BigQuery (to publish results). If you are using Kubernetes, you will need to use a service account with the same privileges as mentioned for Cloud Run. In Kubernetes, you will need to set up [workload identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) so the DVT container can impersonate the service account.
+When running DVT in a distributed fashion, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (i.e. a `--config-file` is provided instead), a warning is issued.
 
-In Cloud Run, the [job](https://cloud.google.com/run/docs/create-jobs) must be run as multiple, independent tasks with the task count set to the number of partitions generated. In Kubernetes, set the number of completions to the number of partitions generated - see [Kubernetes Parallel Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#parallel-jobs). The option `--kube-completions or -kc` tells DVT that many DVT containers are running in a Kubernetes cluster. Each DVT container only validates the specific partition YAML (based on the index assigned by Kubernetes control plane). If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (a `--config-file` is provided instead), a warning is issued.
+The `--config-dir` flag will specify the directory with the YAML files to be executed in parallel. If you used `generate-table-partitions` to generate the YAMLs, this would be the directory where the partition files numbered `0000.yaml` to `<partition_num - 1>.yaml` are stored i.e (`gs://my_config_dir/source_schema.source_table/`). When creating your Cloud Run Job, set the number of tasks equal to the number of table partitions so the task index matches the YAML file to be validated. When executed, each Cloud Run task will validate a partition in parallel.
 
-By default, each partition validation is retried up to 3 times if there is an error. In Kubernetes and Cloud Run, you can set the parallelism to the number you want. Keep in mind that if you are validating 1000's of partitions in parallel, you may find that setting the parallelism too high (say 100) may result in timeouts and slow down the validation.   
 
 ### Validation Reports
 
@@ -587,7 +584,8 @@ with calculated fields as shown in [this example](https://github.com/GoogleCloud
 
 Filters let you apply a WHERE statement to your validation query (ie. `SELECT *
 FROM table WHERE created_at > 30 days ago AND region_id = 71;`). The filter is
-written in the syntax of the given source.
+written in the syntax of the given source and must reference columns in the
+underlying table, not projected DVT expressions.
 
 Note that you are writing the query to execute, which does not have to match
 between source and target as long as the results can be expected to align. If

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.0.0}/data_validation/__main__.py

@@ -48,7 +48,10 @@ def _get_arg_config_file(args):
     """Return String YAML config file path."""
     if not args.config_file:
         raise ValueError("YAML Config File was not supplied.")
-
+    elif not args.config_file.endswith(".yaml"):
+        raise ValueError(
+            f"Invalid YAML config name: {args.config_file}. Provide YAML file extension."
+        )
     return args.config_file
 
 
@@ -56,7 +59,10 @@ def _get_arg_config_file_json(args):
     """Return String JSON config file path."""
     if not args.config_file_json:
         raise ValueError("JSON Config File was not supplied.")
-
+    elif not args.config_file_json.endswith(".json"):
+        raise ValueError(
+            f"Invalid JSON config name: {args.config_file_json}. Provide JSON file extension."
+        )
     return args.config_file_json
 
 
@@ -350,16 +356,17 @@ def config_runner(args):
             setattr(args, "config_dir", None)
             setattr(args, "config_file", config_file_path)
             config_managers = build_config_managers_from_yaml(args, config_file_path)
+            run_validations(args, config_managers)
         else:
             if args.kube_completions:
                 logging.warning(
                     "--kube-completions or -kc specified, however not running in Kubernetes Job completion, check your command line."
                 )
-            mgr = state_manager.StateManager(file_system_root_path=args.config_dir)
-            config_file_names = mgr.list_validations_in_dir(args.config_dir)
+            config_file_names = cli_tools.list_validations(config_dir=args.config_dir)
             config_managers = []
             for file in config_file_names:
-                config_managers.extend(build_config_managers_from_yaml(args, file))
+                config_managers = build_config_managers_from_yaml(args, file)
+                run_validations(args, config_managers)
     else:
         if args.kube_completions:
             logging.warning(
@@ -367,13 +374,12 @@ def config_runner(args):
             )
         config_file_path = _get_arg_config_file(args)
         config_managers = build_config_managers_from_yaml(args, config_file_path)
-
-    run_validations(args, config_managers)
+        run_validations(args, config_managers)
 
 
 def build_config_managers_from_yaml(args, config_file_path):
     """Returns List[ConfigManager] instances ready to be executed."""
-    if "config_dir" in args and args.config_dir:
+    if args.config_dir:
         yaml_configs = cli_tools.get_validation(config_file_path, args.config_dir)
     else:
         yaml_configs = cli_tools.get_validation(config_file_path)
@@ -616,7 +622,7 @@ def store_json_config_file(args, config_managers):
     """
     json_config = convert_config_to_json(config_managers)
     config_file_path = _get_arg_config_file_json(args)
-    cli_tools.store_validation_json(config_file_path, json_config)
+    cli_tools.store_validation(config_file_path, json_config)
 
 
 def partition_and_store_config_files(args: Namespace) -> None:
@@ -673,7 +679,7 @@ def run_validation_configs(args):
         config_runner(args)
     elif args.validation_config_cmd == "list":
         config_dir = args.config_dir or "./"
-        cli_tools.list_validations(config_dir=config_dir)
+        cli_tools.print_validations_in_dir(config_dir=config_dir)
     elif args.validation_config_cmd == "get":
         # Get and print yaml file config.
         yaml = cli_tools.get_validation(_get_arg_config_file(args))

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.0.0}/data_validation/cli_tools.py

@@ -49,10 +49,12 @@ import json
 import logging
 import sys
 import uuid
+import os
 from argparse import Namespace
 from typing import Dict, List
+from yaml import Dumper, Loader, dump, load
 
-from data_validation import clients, consts, state_manager
+from data_validation import clients, consts, state_manager, gcs_helper
 
 CONNECTION_SOURCE_FIELDS = {
     "BigQuery": [
@@ -1081,47 +1083,46 @@ def get_connection(connection_name):
     return mgr.get_connection_config(connection_name)
 
 
-def store_validation(validation_file_name, yaml_config):
-    """Store the validation YAML config under the given name."""
-    mgr = state_manager.StateManager()
-    mgr.create_validation_yaml(validation_file_name, yaml_config)
-
-
-def store_validation_json(validation_file_name, json_config):
-    """Store the validation JSON config under the given name."""
-    mgr = state_manager.StateManager()
-    mgr.create_validation_json(validation_file_name, json_config)
+def store_validation(validation_file_name, config, include_log=True):
+    """Store the validation config under the given name."""
+    validation_path = gcs_helper.get_validation_path(validation_file_name)
 
+    if validation_file_name.endswith(".yaml"):
+        config_str = dump(config, Dumper=Dumper)
+    elif validation_file_name.endswith("json"):
+        config_str = json.dumps(config)
+    else:
+        raise ValueError(f"Invalid validation file name: {validation_file_name}")
 
-def store_partition(target_file_path, yaml_config, target_folder_path=None):
-    """Store the partition YAML config under the given name."""
-    mgr = state_manager.StateManager(target_folder_path)
-    mgr.create_partition_yaml(target_file_path, yaml_config)
+    gcs_helper.write_file(validation_path, config_str, include_log=include_log)
 
 
-def get_validation(validation_name, config_dir=None):
-    """Return validation YAML for a specific connection."""
+def get_validation(name: str, config_dir: str = None):
+    """Return validation YAML config."""
     if config_dir:
-        mgr = state_manager.StateManager(file_system_root_path=config_dir)
-        return mgr.get_validation_config(validation_name, config_dir)
+        validation_path = os.path.join(config_dir, name)
     else:
-        if validation_name.startswith("gs://"):
-            obj_depth = len(validation_name.split("/"))
-            gcs_prefix = "/".join(validation_name.split("/")[: obj_depth - 1])
-            mgr = state_manager.StateManager(file_system_root_path=gcs_prefix)
-            return mgr.get_validation_config(
-                validation_name.split("/")[obj_depth - 1], gcs_prefix
-            )
-        else:
-            mgr = state_manager.StateManager()
-            return mgr.get_validation_config(validation_name)
+        validation_path = gcs_helper.get_validation_path(name)
+
+    validation_bytes = gcs_helper.read_file(validation_path)
+    return load(validation_bytes, Loader=Loader)
 
 
 def list_validations(config_dir="./"):
-    """List all saved validation YAMLs."""
-    mgr = state_manager.StateManager(file_system_root_path=config_dir)
-    validations = mgr.list_validations_in_dir(config_dir=config_dir)
+    """List all saved validation YAMLs in a directory."""
+    logging.info(f"Looking for validations in path {config_dir}")
+    if gcs_helper._is_gcs_path(config_dir):
+        if not config_dir.endswith("/"):
+            config_dir += "/"
+        files = gcs_helper.list_gcs_directory(config_dir)
+    else:
+        files = os.listdir(config_dir)
+
+    return [file_name for file_name in files if file_name.endswith(".yaml")]
+
 
+def print_validations_in_dir(config_dir="./"):
+    validations = list_validations(config_dir=config_dir)
     logging.info("Validation YAMLs found:")
     for validation_name in validations:
         logging.info(validation_name)

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.0.0}/data_validation/config_manager.py

@@ -14,19 +14,17 @@
 
 import copy
 import logging
-from typing import Optional, Union, TYPE_CHECKING
-import yaml
+from typing import TYPE_CHECKING, Optional, Union
 
 import google.oauth2.service_account
-
 import ibis.expr.datatypes as dt
+import yaml
 
-from data_validation import clients, consts, state_manager
+from data_validation import clients, consts, gcs_helper, state_manager
 from data_validation.result_handlers.bigquery import BigQueryResultHandler
 from data_validation.result_handlers.text import TextResultHandler
 from data_validation.validation_builder import ValidationBuilder
 
-
 if TYPE_CHECKING:
     import ibis.expr.types.TableExpr
 
@@ -322,9 +320,8 @@ class ConfigManager(object):
             allow_list = allow_list.replace(" ", "")
             full_allow_list.append(allow_list)
         if allow_list_file:
-            mgr = state_manager.StateManager(file_system_root_path=allow_list_file)
             try:
-                allow_list_yaml = mgr.read_file(allow_list_file)
+                allow_list_yaml = gcs_helper.read_file(allow_list_file)
             except FileNotFoundError as e:
                 raise ValueError(
                     "Cannot locate --allow-list-file: {allow_list_file}"
@@ -543,7 +540,7 @@ class ConfigManager(object):
             ].type()
             cast_type = (
                 "string"
-                if self._decimal_column_too_big_for_pandas(
+                if self._key_column_needs_casting_to_string(
                     source_ibis_type, target_ibis_type
                 )
                 else None
@@ -695,6 +692,19 @@ class ConfigManager(object):
             )
         )
 
+    def _key_column_needs_casting_to_string(
+        self,
+        source_column_ibis_type: dt.DataType,
+        target_column_ibis_type: dt.DataType,
+    ) -> bool:
+        return bool(
+            self._decimal_column_too_big_for_pandas(
+                source_column_ibis_type, target_column_ibis_type
+            )
+            or isinstance(source_column_ibis_type, dt.Binary)
+            or isinstance(target_column_ibis_type, dt.Binary)
+        )
+
     def build_config_column_aggregates(
         self, agg_type, arg_value, exclude_cols, supported_types, cast_to_bigint=False
     ):

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.0.0}/data_validation/consts.py

@@ -99,7 +99,7 @@ CONFIG_TYPES = [
 
 # State Manager Fields
 DEFAULT_ENV_DIRECTORY = "~/.config/google-pso-data-validator/"
-ENV_DIRECTORY_VAR = "PSO_DV_CONFIG_HOME"
+ENV_DIRECTORY_VAR = "PSO_DV_CONN_HOME"
 
 # Yaml File Config Fields
 YAML_RESULT_HANDLER = "result_handler"

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.0.0}/data_validation/data_validation.py

@@ -106,10 +106,15 @@ class DataValidation(object):
             raise ValueError("Primary Keys are required for Random Row Filters")
 
         # Filter for only first primary key (multi-pk filter not supported)
-        primary_key_info = self.config_manager.primary_keys[0]
+        source_pk_column = self.config_manager.primary_keys[0][
+            consts.CONFIG_SOURCE_COLUMN
+        ]
+        target_pk_column = self.config_manager.primary_keys[0][
+            consts.CONFIG_TARGET_COLUMN
+        ]
 
         randomRowBuilder = RandomRowBuilder(
-            [primary_key_info[consts.CONFIG_SOURCE_COLUMN]],
+            [source_pk_column],
             self.config_manager.random_row_batch_size(),
         )
 
@@ -128,24 +133,33 @@ class DataValidation(object):
                 self.validation_builder.source_builder,
             )
 
+        # Check if source table's primary key is BINARY, if so then
+        # force cast the id columns to STRING (HEX).
+        binary_conversion_required = False
+        if query[source_pk_column].type().is_binary():
+            binary_conversion_required = True
+            query = query.mutate(
+                **{source_pk_column: query[source_pk_column].cast("string")}
+            )
+
         random_rows = self.config_manager.source_client.execute(query)
         if len(random_rows) == 0:
             return
+
+        random_values = list(random_rows[source_pk_column])
+        if binary_conversion_required:
+            # For binary ids we have a list of hex strings for our IN list.
+            # Each of these needs to be cast back to binary.
+            random_values = [ibis.literal(_).cast("binary") for _ in random_values]
+
         filter_field = {
             consts.CONFIG_TYPE: consts.FILTER_TYPE_ISIN,
-            consts.CONFIG_FILTER_SOURCE_COLUMN: primary_key_info[
-                consts.CONFIG_FIELD_ALIAS
-            ],
-            consts.CONFIG_FILTER_SOURCE_VALUE: random_rows[
-                primary_key_info[consts.CONFIG_SOURCE_COLUMN]
-            ],
-            consts.CONFIG_FILTER_TARGET_COLUMN: primary_key_info[
-                consts.CONFIG_FIELD_ALIAS
-            ],
-            consts.CONFIG_FILTER_TARGET_VALUE: random_rows[
-                primary_key_info[consts.CONFIG_SOURCE_COLUMN]
-            ],
+            consts.CONFIG_FILTER_SOURCE_COLUMN: source_pk_column,
+            consts.CONFIG_FILTER_SOURCE_VALUE: random_values,
+            consts.CONFIG_FILTER_TARGET_COLUMN: target_pk_column,
+            consts.CONFIG_FILTER_TARGET_VALUE: random_values,
         }
+
         self.validation_builder.add_filter(filter_field)
 
     def query_too_large(self, rows_df, grouped_fields):