google-pso-data-validator 4.4.0__tar.gz → 5.0.0__tar.gz

{google-pso-data-validator-4.4.0 → google_pso_data_validator-5.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: google-pso-data-validator
-Version: 4.4.0
+Version: 5.0.0
 Summary: A package to enable easy data validation
 Home-page: https://github.com/pypa/sampleproject
 Author: Dylan Hercher
@@ -154,7 +154,8 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
-  [--exclude-columns or -ec]   Flag to indicate the list of columns provided should be excluded and not included.
+  [--exclude-columns or -ec]
+                        Flag to indicate the list of columns provided should be excluded and not included.
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
                         BigQuery destination for validation results. Defaults to stdout.
                         See: *Validation Reports* section
@@ -171,7 +172,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -179,8 +180,8 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         Float value. Maximum pct_difference allowed for validation to be considered a success. Defaults to 0.0
   [--labels or -l KEY1=VALUE1,KEY2=VALUE2]
                         Comma-separated key value pair labels for the run.
-  [--format or -fmt]    Format for stdout output. Supported formats are (text, csv, json, table).
-                        Defaults to table.
+  [--format or -fmt FORMAT]
+                        Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
 ```
@@ -241,14 +242,14 @@ data-validation (--verbose or -v) (--log-level or -ll) validate row
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
   [--labels or -l KEY1=VALUE1,KEY2=VALUE2]
                         Comma-separated key value pair labels for the run.
-  [--format or -fmt]    Format for stdout output. Supported formats are (text, csv, json, table).
-                        Defaults to table.
+  [--format or -fmt FORMAT]
+                        Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--use-random-row or -rr]
                         Finds a set of random rows of the first primary key supplied.
   [--random-row-batch-size or -rbs]
@@ -321,7 +322,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate schema
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -368,21 +369,22 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
-  [--exclude-columns or -ec]   Flag to indicate the list of columns provided should be excluded and not included.
+  [--exclude-columns or -ec]
+                        Flag to indicate the list of columns provided should be excluded and not included.
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
                         BigQuery destination for validation results. Defaults to stdout.
                         See: *Validation Reports* section
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
   [--labels or -l KEY1=VALUE1,KEY2=VALUE2]
                         Comma-separated key value pair labels for the run.
-  [--format or -fmt]    Format for stdout output. Supported formats are (text, csv, json, table).
-                        Defaults to table.
+  [--format or -fmt FORMAT]
+                        Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
 ```
@@ -437,14 +439,14 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
   [--labels or -l KEY1=VALUE1,KEY2=VALUE2]
                         Comma-separated key value pair labels for the run.
-  [--format or -fmt]    Format for stdout output. Supported formats are (text, csv, json, table).
-                        Defaults to table.
+  [--format or -fmt FORMAT]
+                        Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
 ```
@@ -481,59 +483,51 @@ For example, this flag can be used as follows:
 
 Running DVT with YAML configuration files is the recommended approach if:
 * you want to customize the configuration for any given validation OR
-* you want to run DVT at scale (i.e. row validations across many partitions)
-Nearly every validation command can take the argument `-c <file-name>` (the `generate-table-partitions` command takes  `-cdir <directory-name>`) where one or more yaml files are produced. These yaml files can be modified for custom validations.
+* you want to run DVT at scale (i.e. run multiple validations sequentially or in parallel)
 
-The following command creates a YAML file for the validation of the
-`new_york_citibike` table: `data-validation validate column -sc my_bq_conn -tc
-my_bq_conn -tbls bigquery-public-data.new_york_citibike.citibike_trips -c
-citibike.yaml`.
+We recommend generating YAML configs with the `--config-file <file-name>` flag when running a validation command, which supports
+GCS and local paths.
 
-The validation config file is saved to the GCS path specified by the `PSO_DV_CONFIG_HOME`
-env variable if that has been set; otherwise, it is saved to wherever the tool is run.
-
-You can now edit the YAML file if, for example, the `new_york_citibike` table is
-stored in datasets that have different names in the source and target systems.
-Once the file is updated and saved, the following command runs the
-validation:
+You can use the `data-validation configs` command to run and view YAMLs.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) configs run -c citibike.yaml
+data-validation (--verbose or -v) (--log-level or -ll) configs run
+  [--config-file or -c CONFIG_FILE]
+                        Path to YAML config file to run. Supports local and GCS paths.
+  [--config-dir or -cdir CONFIG_DIR]
+                        Directory path containing YAML configs to be run sequentially. Supports local and GCS paths.
+  [--dry-run or -dr]    If this flag is present, prints the source and target SQL generated in lieu of running the validation.
+  [--kube-completions or -kc]
+                        Flag to indicate usage in Kubernetes index completion mode.
+                        See *Scaling DVT* section
 ```
 
-You can also specify a directory to read from using the --config-dir (or -cdir) flag. All yaml files in that directory will be run as validations. Support both local and GCS paths. For example:
 ```
-data-validation configs run -cdir ./my-validations/
-
-OR
+data-validation configs list
+  [--config-dir or -cdir CONFIG_DIR]
+                        GCS or local directory from which to list validation YAML configs. Defaults to current local directory.
+```
 
-data-validation configs run -cdir gs://my-bucket/my-validations/
+```
+data-validation configs get
+  [--config-file or -c CONFIG_FILE] GCS or local path of validation YAML to print.
 ```
 
 View the complete YAML file for a Grouped Column validation on the
 [Examples](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/examples.md#sample-yaml-config-grouped-column-validation) page.
 
-You can view a list of all saved validation YAML files using `data-validation configs list`, and print a YAML config using `data-validation configs get -c citibike.yaml`.
-#### Scaling DVT to run 10's to 1000's of validations concurrently
-The above example `configs run -cdir` shows how you can run multiple validations with one command. If the directory used has been created by `generate-table-partitions`, you will have partition files numbered from `0000.yaml` to `<partno-1>.yaml`, where `<partno>` is the total number of partitions. These can be run as mentioned above and the partitions will be validated in order. This could take a long time if the number of partitions is large.
 
-DVT validations can be run concurrently (horizontal scaling) using GKE (Kubernetes Jobs) or Cloud Run Jobs. In order to run DVT in a container, you have to build a docker image, [see instructions](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/docker#readme). Set the `PSO_DV_CONFIG_HOME` environment variable to point to a GCS prefix where the connection configuration files are stored. In Cloud Run you can use the option `--set-env-vars` or `--update-env-vars` to pass [the environment variable](https://cloud.google.com/run/docs/configuring/services/environment-variables#setting). We recommend that you use the `bq-result-handler` to save your validation results. In order to validate partitions concurrently, run DVT in Kubernetes or Cloud Run as shown below:
-```
-data-validation (--verbose or -v) (--log-level or -ll) configs run
-  [--kube-completions or -kc] Specifies that validation is being run in Kubernetes or Cloud Run in indexed completion mode.
-              specifies to DVT that validation being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run.
-  --config-dir or -cdir GCS_DIRECTORY 
-              where GCS_DIRECTORY = CONFIG_DIR/SOURCE_SCHEMA.SOURCE_TABLE, where CONFIG_DIR, SOURCE_SCHEMA and SOURCE_TABLE are
-              the arguments provided to generate-table-partitions to generate the partition yamls. GCS_DIRECTORY is the directory
-              where the partition files numbered 0000.yaml to <partno-1>.yaml are stored.
-```
-If you are not familiar with Kubernetes or Cloud Run Jobs, review the [internal docs](docs/internal/kubernetes_jobs.md) for an overview of the orchestration of multiple validations.
+### Scaling DVT
+
+You can scale DVT for large table validations by running the tool in a distributed manner. To optimize the validation speed for large tables, you can use GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Distributed Jobs](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/internal/distributed_jobs.md) for a detailed overview.
+
+
+We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started. 
 
-The Cloud Run and Kubernetes pods must run in a network with access to the database servers. Every Cloud Run job is associated with a [service account](https://cloud.google.com/run/docs/securing/service-identity). You need to ensure that this service account has access to Google Cloud Storage (to read connection configuration and yaml files) and BigQuery (to publish results). If you are using Kubernetes, you will need to use a service account with the same privileges as mentioned for Cloud Run. In Kubernetes, you will need to set up [workload identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) so the DVT container can impersonate the service account.
+When running DVT in a distributed fashion, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (i.e. a `--config-file` is provided instead), a warning is issued.
 
-In Cloud Run, the [job](https://cloud.google.com/run/docs/create-jobs) must be run as multiple, independent tasks with the task count set to the number of partitions generated. In Kubernetes, set the number of completions to the number of partitions generated - see [Kubernetes Parallel Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#parallel-jobs). The option `--kube-completions or -kc` tells DVT that many DVT containers are running in a Kubernetes cluster. Each DVT container only validates the specific partition yaml (based on the index assigned by Kubernetes control plane). If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (a `--config-file` is provided instead), a warning is issued.
+The `--config-dir` flag will specify the directory with the YAML files to be executed in parallel. If you used `generate-table-partitions` to generate the YAMLs, this would be the directory where the partition files numbered `0000.yaml` to `<partition_num - 1>.yaml` are stored i.e (`gs://my_config_dir/source_schema.source_table/`). When creating your Cloud Run Job, set the number of tasks equal to the number of table partitions so the task index matches the YAML file to be validated. When executed, each Cloud Run task will validate a partition in parallel.
 
-By default, each partition validation is retried up to 3 times if there is an error. In Kubernetes and Cloud Run, you can set the parallelism to the number you want. Keep in mind that if you are validating 1000's of partitions in parallel, you may find that setting the parallelism too high (say 100) may result in timeouts and slow down the validation.   
 
 ### Validation Reports
 
@@ -638,7 +632,8 @@ with calculated fields as shown in [this example](https://github.com/GoogleCloud
 
 Filters let you apply a WHERE statement to your validation query (ie. `SELECT *
 FROM table WHERE created_at > 30 days ago AND region_id = 71;`). The filter is
-written in the syntax of the given source.
+written in the syntax of the given source and must reference columns in the
+underlying table, not projected DVT expressions.
 
 Note that you are writing the query to execute, which does not have to match
 between source and target as long as the results can be expected to align. If

{google-pso-data-validator-4.4.0 → google_pso_data_validator-5.0.0}/README.md

@@ -106,7 +106,8 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
-  [--exclude-columns or -ec]   Flag to indicate the list of columns provided should be excluded and not included.
+  [--exclude-columns or -ec]
+                        Flag to indicate the list of columns provided should be excluded and not included.
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
                         BigQuery destination for validation results. Defaults to stdout.
                         See: *Validation Reports* section
@@ -123,7 +124,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -131,8 +132,8 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         Float value. Maximum pct_difference allowed for validation to be considered a success. Defaults to 0.0
   [--labels or -l KEY1=VALUE1,KEY2=VALUE2]
                         Comma-separated key value pair labels for the run.
-  [--format or -fmt]    Format for stdout output. Supported formats are (text, csv, json, table).
-                        Defaults to table.
+  [--format or -fmt FORMAT]
+                        Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
 ```
@@ -193,14 +194,14 @@ data-validation (--verbose or -v) (--log-level or -ll) validate row
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
   [--labels or -l KEY1=VALUE1,KEY2=VALUE2]
                         Comma-separated key value pair labels for the run.
-  [--format or -fmt]    Format for stdout output. Supported formats are (text, csv, json, table).
-                        Defaults to table.
+  [--format or -fmt FORMAT]
+                        Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--use-random-row or -rr]
                         Finds a set of random rows of the first primary key supplied.
   [--random-row-batch-size or -rbs]
@@ -273,7 +274,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate schema
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -320,21 +321,22 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
-  [--exclude-columns or -ec]   Flag to indicate the list of columns provided should be excluded and not included.
+  [--exclude-columns or -ec]
+                        Flag to indicate the list of columns provided should be excluded and not included.
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
                         BigQuery destination for validation results. Defaults to stdout.
                         See: *Validation Reports* section
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
   [--labels or -l KEY1=VALUE1,KEY2=VALUE2]
                         Comma-separated key value pair labels for the run.
-  [--format or -fmt]    Format for stdout output. Supported formats are (text, csv, json, table).
-                        Defaults to table.
+  [--format or -fmt FORMAT]
+                        Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
 ```
@@ -389,14 +391,14 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
   [--labels or -l KEY1=VALUE1,KEY2=VALUE2]
                         Comma-separated key value pair labels for the run.
-  [--format or -fmt]    Format for stdout output. Supported formats are (text, csv, json, table).
-                        Defaults to table.
+  [--format or -fmt FORMAT]
+                        Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
 ```
@@ -433,59 +435,51 @@ For example, this flag can be used as follows:
 
 Running DVT with YAML configuration files is the recommended approach if:
 * you want to customize the configuration for any given validation OR
-* you want to run DVT at scale (i.e. row validations across many partitions)
-Nearly every validation command can take the argument `-c <file-name>` (the `generate-table-partitions` command takes  `-cdir <directory-name>`) where one or more yaml files are produced. These yaml files can be modified for custom validations.
+* you want to run DVT at scale (i.e. run multiple validations sequentially or in parallel)
 
-The following command creates a YAML file for the validation of the
-`new_york_citibike` table: `data-validation validate column -sc my_bq_conn -tc
-my_bq_conn -tbls bigquery-public-data.new_york_citibike.citibike_trips -c
-citibike.yaml`.
+We recommend generating YAML configs with the `--config-file <file-name>` flag when running a validation command, which supports
+GCS and local paths.
 
-The validation config file is saved to the GCS path specified by the `PSO_DV_CONFIG_HOME`
-env variable if that has been set; otherwise, it is saved to wherever the tool is run.
-
-You can now edit the YAML file if, for example, the `new_york_citibike` table is
-stored in datasets that have different names in the source and target systems.
-Once the file is updated and saved, the following command runs the
-validation:
+You can use the `data-validation configs` command to run and view YAMLs.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) configs run -c citibike.yaml
+data-validation (--verbose or -v) (--log-level or -ll) configs run
+  [--config-file or -c CONFIG_FILE]
+                        Path to YAML config file to run. Supports local and GCS paths.
+  [--config-dir or -cdir CONFIG_DIR]
+                        Directory path containing YAML configs to be run sequentially. Supports local and GCS paths.
+  [--dry-run or -dr]    If this flag is present, prints the source and target SQL generated in lieu of running the validation.
+  [--kube-completions or -kc]
+                        Flag to indicate usage in Kubernetes index completion mode.
+                        See *Scaling DVT* section
 ```
 
-You can also specify a directory to read from using the --config-dir (or -cdir) flag. All yaml files in that directory will be run as validations. Support both local and GCS paths. For example:
 ```
-data-validation configs run -cdir ./my-validations/
-
-OR
+data-validation configs list
+  [--config-dir or -cdir CONFIG_DIR]
+                        GCS or local directory from which to list validation YAML configs. Defaults to current local directory.
+```
 
-data-validation configs run -cdir gs://my-bucket/my-validations/
+```
+data-validation configs get
+  [--config-file or -c CONFIG_FILE] GCS or local path of validation YAML to print.
 ```
 
 View the complete YAML file for a Grouped Column validation on the
 [Examples](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/examples.md#sample-yaml-config-grouped-column-validation) page.
 
-You can view a list of all saved validation YAML files using `data-validation configs list`, and print a YAML config using `data-validation configs get -c citibike.yaml`.
-#### Scaling DVT to run 10's to 1000's of validations concurrently
-The above example `configs run -cdir` shows how you can run multiple validations with one command. If the directory used has been created by `generate-table-partitions`, you will have partition files numbered from `0000.yaml` to `<partno-1>.yaml`, where `<partno>` is the total number of partitions. These can be run as mentioned above and the partitions will be validated in order. This could take a long time if the number of partitions is large.
 
-DVT validations can be run concurrently (horizontal scaling) using GKE (Kubernetes Jobs) or Cloud Run Jobs. In order to run DVT in a container, you have to build a docker image, [see instructions](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/docker#readme). Set the `PSO_DV_CONFIG_HOME` environment variable to point to a GCS prefix where the connection configuration files are stored. In Cloud Run you can use the option `--set-env-vars` or `--update-env-vars` to pass [the environment variable](https://cloud.google.com/run/docs/configuring/services/environment-variables#setting). We recommend that you use the `bq-result-handler` to save your validation results. In order to validate partitions concurrently, run DVT in Kubernetes or Cloud Run as shown below:
-```
-data-validation (--verbose or -v) (--log-level or -ll) configs run
-  [--kube-completions or -kc] Specifies that validation is being run in Kubernetes or Cloud Run in indexed completion mode.
-              specifies to DVT that validation being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run.
-  --config-dir or -cdir GCS_DIRECTORY 
-              where GCS_DIRECTORY = CONFIG_DIR/SOURCE_SCHEMA.SOURCE_TABLE, where CONFIG_DIR, SOURCE_SCHEMA and SOURCE_TABLE are
-              the arguments provided to generate-table-partitions to generate the partition yamls. GCS_DIRECTORY is the directory
-              where the partition files numbered 0000.yaml to <partno-1>.yaml are stored.
-```
-If you are not familiar with Kubernetes or Cloud Run Jobs, review the [internal docs](docs/internal/kubernetes_jobs.md) for an overview of the orchestration of multiple validations.
+### Scaling DVT
+
+You can scale DVT for large table validations by running the tool in a distributed manner. To optimize the validation speed for large tables, you can use GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Distributed Jobs](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/internal/distributed_jobs.md) for a detailed overview.
+
+
+We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started. 
 
-The Cloud Run and Kubernetes pods must run in a network with access to the database servers. Every Cloud Run job is associated with a [service account](https://cloud.google.com/run/docs/securing/service-identity). You need to ensure that this service account has access to Google Cloud Storage (to read connection configuration and yaml files) and BigQuery (to publish results). If you are using Kubernetes, you will need to use a service account with the same privileges as mentioned for Cloud Run. In Kubernetes, you will need to set up [workload identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) so the DVT container can impersonate the service account.
+When running DVT in a distributed fashion, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (i.e. a `--config-file` is provided instead), a warning is issued.
 
-In Cloud Run, the [job](https://cloud.google.com/run/docs/create-jobs) must be run as multiple, independent tasks with the task count set to the number of partitions generated. In Kubernetes, set the number of completions to the number of partitions generated - see [Kubernetes Parallel Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#parallel-jobs). The option `--kube-completions or -kc` tells DVT that many DVT containers are running in a Kubernetes cluster. Each DVT container only validates the specific partition yaml (based on the index assigned by Kubernetes control plane). If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (a `--config-file` is provided instead), a warning is issued.
+The `--config-dir` flag will specify the directory with the YAML files to be executed in parallel. If you used `generate-table-partitions` to generate the YAMLs, this would be the directory where the partition files numbered `0000.yaml` to `<partition_num - 1>.yaml` are stored i.e (`gs://my_config_dir/source_schema.source_table/`). When creating your Cloud Run Job, set the number of tasks equal to the number of table partitions so the task index matches the YAML file to be validated. When executed, each Cloud Run task will validate a partition in parallel.
 
-By default, each partition validation is retried up to 3 times if there is an error. In Kubernetes and Cloud Run, you can set the parallelism to the number you want. Keep in mind that if you are validating 1000's of partitions in parallel, you may find that setting the parallelism too high (say 100) may result in timeouts and slow down the validation.   
 
 ### Validation Reports
 
@@ -590,7 +584,8 @@ with calculated fields as shown in [this example](https://github.com/GoogleCloud
 
 Filters let you apply a WHERE statement to your validation query (ie. `SELECT *
 FROM table WHERE created_at > 30 days ago AND region_id = 71;`). The filter is
-written in the syntax of the given source.
+written in the syntax of the given source and must reference columns in the
+underlying table, not projected DVT expressions.
 
 Note that you are writing the query to execute, which does not have to match
 between source and target as long as the results can be expected to align. If

{google-pso-data-validator-4.4.0 → google_pso_data_validator-5.0.0}/data_validation/__main__.py

@@ -48,7 +48,10 @@ def _get_arg_config_file(args):
     """Return String YAML config file path."""
     if not args.config_file:
         raise ValueError("YAML Config File was not supplied.")
-
+    elif not args.config_file.endswith(".yaml"):
+        raise ValueError(
+            f"Invalid YAML config name: {args.config_file}. Provide YAML file extension."
+        )
     return args.config_file
 
 
@@ -56,12 +59,15 @@ def _get_arg_config_file_json(args):
     """Return String JSON config file path."""
     if not args.config_file_json:
         raise ValueError("JSON Config File was not supplied.")
-
+    elif not args.config_file_json.endswith(".json"):
+        raise ValueError(
+            f"Invalid JSON config name: {args.config_file_json}. Provide JSON file extension."
+        )
     return args.config_file_json
 
 
 def get_aggregate_config(args, config_manager: ConfigManager):
-    """Return list of formated aggregation objects.
+    """Return list of formatted aggregation objects.
 
     Args:
         config_manager (ConfigManager): Validation config manager instance.
@@ -350,16 +356,17 @@ def config_runner(args):
             setattr(args, "config_dir", None)
             setattr(args, "config_file", config_file_path)
             config_managers = build_config_managers_from_yaml(args, config_file_path)
+            run_validations(args, config_managers)
         else:
             if args.kube_completions:
                 logging.warning(
                     "--kube-completions or -kc specified, however not running in Kubernetes Job completion, check your command line."
                 )
-            mgr = state_manager.StateManager(file_system_root_path=args.config_dir)
-            config_file_names = mgr.list_validations_in_dir(args.config_dir)
+            config_file_names = cli_tools.list_validations(config_dir=args.config_dir)
             config_managers = []
             for file in config_file_names:
-                config_managers.extend(build_config_managers_from_yaml(args, file))
+                config_managers = build_config_managers_from_yaml(args, file)
+                run_validations(args, config_managers)
     else:
         if args.kube_completions:
             logging.warning(
@@ -367,13 +374,12 @@ def config_runner(args):
             )
         config_file_path = _get_arg_config_file(args)
         config_managers = build_config_managers_from_yaml(args, config_file_path)
-
-    run_validations(args, config_managers)
+        run_validations(args, config_managers)
 
 
 def build_config_managers_from_yaml(args, config_file_path):
     """Returns List[ConfigManager] instances ready to be executed."""
-    if "config_dir" in args and args.config_dir:
+    if args.config_dir:
         yaml_configs = cli_tools.get_validation(config_file_path, args.config_dir)
     else:
         yaml_configs = cli_tools.get_validation(config_file_path)
@@ -616,7 +622,7 @@ def store_json_config_file(args, config_managers):
     """
     json_config = convert_config_to_json(config_managers)
     config_file_path = _get_arg_config_file_json(args)
-    cli_tools.store_validation_json(config_file_path, json_config)
+    cli_tools.store_validation(config_file_path, json_config)
 
 
 def partition_and_store_config_files(args: Namespace) -> None:
@@ -672,7 +678,8 @@ def run_validation_configs(args):
     if args.validation_config_cmd == "run":
         config_runner(args)
     elif args.validation_config_cmd == "list":
-        cli_tools.list_validations()
+        config_dir = args.config_dir or "./"
+        cli_tools.print_validations_in_dir(config_dir=config_dir)
     elif args.validation_config_cmd == "get":
         # Get and print yaml file config.
         yaml = cli_tools.get_validation(_get_arg_config_file(args))

{google-pso-data-validator-4.4.0 → google_pso_data_validator-5.0.0}/data_validation/cli_tools.py

@@ -49,10 +49,12 @@ import json
 import logging
 import sys
 import uuid
+import os
 from argparse import Namespace
 from typing import Dict, List
+from yaml import Dumper, Loader, dump, load
 
-from data_validation import clients, consts, state_manager
+from data_validation import clients, consts, state_manager, gcs_helper
 
 CONNECTION_SOURCE_FIELDS = {
     "BigQuery": [
@@ -328,7 +330,15 @@ def _configure_validation_config_parser(subparsers):
     configs_subparsers = validation_config_parser.add_subparsers(
         dest="validation_config_cmd"
     )
-    _ = configs_subparsers.add_parser("list", help="List your validation configs")
+    list_parser = configs_subparsers.add_parser(
+        "list", help="List your validation configs"
+    )
+    list_parser.add_argument(
+        "--config-dir",
+        "-cdir",
+        help="Directory path from which to list validation YAML configs.",
+    )
+
     run_parser = configs_subparsers.add_parser(
         "run", help="Run your validation configs"
     )
@@ -1073,47 +1083,46 @@ def get_connection(connection_name):
     return mgr.get_connection_config(connection_name)
 
 
-def store_validation(validation_file_name, yaml_config):
-    """Store the validation YAML config under the given name."""
-    mgr = state_manager.StateManager()
-    mgr.create_validation_yaml(validation_file_name, yaml_config)
+def store_validation(validation_file_name, config, include_log=True):
+    """Store the validation config under the given name."""
+    validation_path = gcs_helper.get_validation_path(validation_file_name)
 
+    if validation_file_name.endswith(".yaml"):
+        config_str = dump(config, Dumper=Dumper)
+    elif validation_file_name.endswith("json"):
+        config_str = json.dumps(config)
+    else:
+        raise ValueError(f"Invalid validation file name: {validation_file_name}")
 
-def store_validation_json(validation_file_name, json_config):
-    """Store the validation JSON config under the given name."""
-    mgr = state_manager.StateManager()
-    mgr.create_validation_json(validation_file_name, json_config)
+    gcs_helper.write_file(validation_path, config_str, include_log=include_log)
 
 
-def store_partition(target_file_path, yaml_config, target_folder_path=None):
-    """Store the partition YAML config under the given name."""
-    mgr = state_manager.StateManager(target_folder_path)
-    mgr.create_partition_yaml(target_file_path, yaml_config)
+def get_validation(name: str, config_dir: str = None):
+    """Return validation YAML config."""
+    if config_dir:
+        validation_path = os.path.join(config_dir, name)
+    else:
+        validation_path = gcs_helper.get_validation_path(name)
 
+    validation_bytes = gcs_helper.read_file(validation_path)
+    return load(validation_bytes, Loader=Loader)
 
-def get_validation(validation_name, config_dir=None):
-    """Return validation YAML for a specific connection."""
-    if config_dir:
-        mgr = state_manager.StateManager(file_system_root_path=config_dir)
-        return mgr.get_validation_config(validation_name, config_dir)
+
+def list_validations(config_dir="./"):
+    """List all saved validation YAMLs in a directory."""
+    logging.info(f"Looking for validations in path {config_dir}")
+    if gcs_helper._is_gcs_path(config_dir):
+        if not config_dir.endswith("/"):
+            config_dir += "/"
+        files = gcs_helper.list_gcs_directory(config_dir)
     else:
-        if validation_name.startswith("gs://"):
-            obj_depth = len(validation_name.split("/"))
-            gcs_prefix = "/".join(validation_name.split("/")[: obj_depth - 1])
-            mgr = state_manager.StateManager(file_system_root_path=gcs_prefix)
-            return mgr.get_validation_config(
-                validation_name.split("/")[obj_depth - 1], gcs_prefix
-            )
-        else:
-            mgr = state_manager.StateManager()
-            return mgr.get_validation_config(validation_name)
+        files = os.listdir(config_dir)
 
+    return [file_name for file_name in files if file_name.endswith(".yaml")]
 
-def list_validations():
-    """List all saved validation YAMLs."""
-    mgr = state_manager.StateManager()
-    validations = mgr.list_validations()
 
+def print_validations_in_dir(config_dir="./"):
+    validations = list_validations(config_dir=config_dir)
     logging.info("Validation YAMLs found:")
     for validation_name in validations:
         logging.info(validation_name)