google-pso-data-validator 4.5.0__tar.gz → 5.1.0__tar.gz

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.1.0}/PKG-INFO

@@ -1,11 +1,10 @@
 Metadata-Version: 2.1
 Name: google-pso-data-validator
-Version: 4.5.0
+Version: 5.1.0
 Summary: A package to enable easy data validation
-Home-page: https://github.com/pypa/sampleproject
-Author: Dylan Hercher
-Author-email: dhercher@google.com
-Classifier: Development Status :: 3 - Alpha
+Author: PSO DVT Engineering team
+Author-email: data-validator-eng@google.com
+Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.8
@@ -16,7 +15,7 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: attrs==23.1.0
-Requires-Dist: grpcio==1.53.0
+Requires-Dist: grpcio==1.53.2
 Requires-Dist: lazy-object-proxy==1.9.0
 Requires-Dist: marshmallow==3.19.0
 Requires-Dist: fsspec>=2022.8.2
@@ -24,7 +23,7 @@ Requires-Dist: google-api-python-client==2.91.0
 Requires-Dist: ibis-framework==5.1.0
 Requires-Dist: impyla==0.18.0
 Requires-Dist: SQLAlchemy==1.4.49
-Requires-Dist: PyMySQL==1.1.0
+Requires-Dist: PyMySQL==1.1.1
 Requires-Dist: psycopg2-binary==2.9.6
 Requires-Dist: PyYAML==6.0
 Requires-Dist: pandas==1.5.3
@@ -35,12 +34,11 @@ Requires-Dist: google-cloud-bigquery==3.11.3
 Requires-Dist: google-cloud-bigquery-storage==2.20.0
 Requires-Dist: google-cloud-spanner==3.36.0
 Requires-Dist: google-cloud-storage==2.10.0
-Requires-Dist: setuptools>=34.0.0
 Requires-Dist: jellyfish==1.0.0
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: Flask==2.3.2
 Requires-Dist: parsy==2.1
-Requires-Dist: google-cloud-secret-manager<=2.15.0
+Requires-Dist: google-cloud-secret-manager>=2.16.0
 Provides-Extra: apache-airflow
 Requires-Dist: 1.10.11; extra == "apache-airflow"
 Provides-Extra: pyspark
@@ -172,7 +170,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -242,7 +240,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate row
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -256,6 +254,8 @@ data-validation (--verbose or -v) (--log-level or -ll) validate row
                         Row batch size used for random row filters (default 10,000).
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
+  [--trim-string-pks, -tsp]
+                        Trims string based primary key values, intended for use when one engine uses padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n)).
 ```
 #### Generate Table Partitions for Large Table Row Validations
 
@@ -322,7 +322,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate schema
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -357,19 +357,19 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
                         Source sql query
                         Either --source-query or --source-query-file must be provided
   --source-query-file  SOURCE_QUERY_FILE, -sqf SOURCE_QUERY_FILE
-                        File containing the source sql commands
+                        File containing the source sql command. Supports GCS and local paths.
   --target-query TARGET_QUERY, -tq TARGET_QUERY
                         Target sql query
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
-                        File containing the target sql commands
+                        File containing the target sql command. Supports GCS and local paths.
   [--count COLUMNS]     Comma separated list of columns for count or * for all columns
   [--sum COLUMNS]       Comma separated list of columns for sum or * for all numeric
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
-  [--exclude-columns or -ec]   
+  [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
                         BigQuery destination for validation results. Defaults to stdout.
@@ -377,7 +377,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -419,12 +419,12 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
                         Source sql query
                         Either --source-query or --source-query-file must be provided
   --source-query-file SOURCE_QUERY_FILE, -sqf SOURCE_QUERY_FILE
-                        File containing the source sql commands
+                        File containing the source sql command. Supports GCS and local paths.
   --target-query TARGET_QUERY, -tq TARGET_QUERY
                         Target sql query
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
-                        File containing the target sql commands
+                        File containing the target sql command. Supports GCS and local paths.
   --comparison-fields or -comp-fields FIELDS
                         Comma separated list of columns to compare. Can either be a physical column or an alias
                         See: *Calculated Fields* section for details
@@ -439,7 +439,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -483,11 +483,10 @@ For example, this flag can be used as follows:
 
 Running DVT with YAML configuration files is the recommended approach if:
 * you want to customize the configuration for any given validation OR
-* you want to run DVT at scale (i.e. row validations across many partitions)
+* you want to run DVT at scale (i.e. run multiple validations sequentially or in parallel)
 
-We recommend generating YAML configs with the `--config-file <file-name>` flag when running a validation command. The validation 
-config file is saved to the GCS path specified by the `PSO_DV_CONFIG_HOME` env variable if that has been set;
-otherwise, it is saved to wherever the tool is run.
+We recommend generating YAML configs with the `--config-file <file-name>` flag when running a validation command, which supports
+GCS and local paths.
 
 You can use the `data-validation configs` command to run and view YAMLs.
 
@@ -500,7 +499,7 @@ data-validation (--verbose or -v) (--log-level or -ll) configs run
   [--dry-run or -dr]    If this flag is present, prints the source and target SQL generated in lieu of running the validation.
   [--kube-completions or -kc]
                         Flag to indicate usage in Kubernetes index completion mode.
-                        See *Scaling DVT to run 10's to 1000's of validations concurrently* section
+                        See *Scaling DVT* section
 ```
 
 ```
@@ -518,19 +517,17 @@ View the complete YAML file for a Grouped Column validation on the
 [Examples](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/examples.md#sample-yaml-config-grouped-column-validation) page.
 
 
-#### Scaling DVT to run 10's to 1000's of validations concurrently
+### Scaling DVT
 
-As described above, you can run multiple validation configs sequentially with the `--config-dir` flag. To optimize the validation speed for large tables, you can run DVT concurrently using GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Kubernetes](docs/internal/kubernetes_jobs.md) for a detailed overview.
+You can scale DVT for large table validations by running the tool in a distributed manner. To optimize the validation speed for large tables, you can use GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Distributed Jobs](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/internal/distributed_jobs.md) for a detailed overview.
 
-In order to validate partitions concurrently, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. The `--config-dir` flag will specify the directory with the YAML files to be executed in parallel. If you used `generate-table-partitions` to generate the YAMLs, this would be the directory where the partition files numbered `0000.yaml` to `<partition_num - 1>.yaml` are stored i.e (`my_config_dir/source_schema.source_table/`)
 
-In order to run DVT in a container, you have to build a Docker image - [see instructions here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/docker#readme). You will also need to set the `PSO_DV_CONFIG_HOME` environment variable to point to a GCS prefix where the connection configuration files are stored. In Cloud Run you can use the option `--set-env-vars` or `--update-env-vars` to pass [the environment variable](https://cloud.google.com/run/docs/configuring/services/environment-variables#setting). We recommend that you use the `bq-result-handler` to save your validation results to BigQuery. 
+We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started.
 
-The Cloud Run and Kubernetes pods must run in a network with access to the database servers. Every Cloud Run job is associated with a [service account](https://cloud.google.com/run/docs/securing/service-identity). You need to ensure that this service account has access to Google Cloud Storage (to read connection configuration and YAML files) and BigQuery (to publish results). If you are using Kubernetes, you will need to use a service account with the same privileges as mentioned for Cloud Run. In Kubernetes, you will need to set up [workload identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) so the DVT container can impersonate the service account.
+When running DVT in a distributed fashion, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (i.e. a `--config-file` is provided instead), a warning is issued.
 
-In Cloud Run, the [job](https://cloud.google.com/run/docs/create-jobs) must be run as multiple, independent tasks with the task count set to the number of partitions generated. In Kubernetes, set the number of completions to the number of partitions generated - see [Kubernetes Parallel Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#parallel-jobs). The option `--kube-completions or -kc` tells DVT that many DVT containers are running in a Kubernetes cluster. Each DVT container only validates the specific partition YAML (based on the index assigned by Kubernetes control plane). If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (a `--config-file` is provided instead), a warning is issued.
+The `--config-dir` flag will specify the directory with the YAML files to be executed in parallel. If you used `generate-table-partitions` to generate the YAMLs, this would be the directory where the partition files numbered `0000.yaml` to `<partition_num - 1>.yaml` are stored i.e (`gs://my_config_dir/source_schema.source_table/`). When creating your Cloud Run Job, set the number of tasks equal to the number of table partitions so the task index matches the YAML file to be validated. When executed, each Cloud Run task will validate a partition in parallel.
 
-By default, each partition validation is retried up to 3 times if there is an error. In Kubernetes and Cloud Run, you can set the parallelism to the number you want. Keep in mind that if you are validating 1000's of partitions in parallel, you may find that setting the parallelism too high (say 100) may result in timeouts and slow down the validation.   
 
 ### Validation Reports
 
@@ -635,7 +632,8 @@ with calculated fields as shown in [this example](https://github.com/GoogleCloud
 
 Filters let you apply a WHERE statement to your validation query (ie. `SELECT *
 FROM table WHERE created_at > 30 days ago AND region_id = 71;`). The filter is
-written in the syntax of the given source.
+written in the syntax of the given source and must reference columns in the
+underlying table, not projected DVT expressions.
 
 Note that you are writing the query to execute, which does not have to match
 between source and target as long as the results can be expected to align. If

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.1.0}/README.md

@@ -124,7 +124,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -194,7 +194,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate row
                         If target filter is not provided, the source filter will run on source and target tables.
                         See: *Filters* section
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -208,6 +208,8 @@ data-validation (--verbose or -v) (--log-level or -ll) validate row
                         Row batch size used for random row filters (default 10,000).
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
+  [--trim-string-pks, -tsp]
+                        Trims string based primary key values, intended for use when one engine uses padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n)).
 ```
 #### Generate Table Partitions for Large Table Row Validations
 
@@ -274,7 +276,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate schema
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -309,19 +311,19 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
                         Source sql query
                         Either --source-query or --source-query-file must be provided
   --source-query-file  SOURCE_QUERY_FILE, -sqf SOURCE_QUERY_FILE
-                        File containing the source sql commands
+                        File containing the source sql command. Supports GCS and local paths.
   --target-query TARGET_QUERY, -tq TARGET_QUERY
                         Target sql query
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
-                        File containing the target sql commands
+                        File containing the target sql command. Supports GCS and local paths.
   [--count COLUMNS]     Comma separated list of columns for count or * for all columns
   [--sum COLUMNS]       Comma separated list of columns for sum or * for all numeric
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric
   [--max COLUMNS]       Comma separated list of columns for max or * for all numeric
   [--avg COLUMNS]       Comma separated list of columns for avg or * for all numeric
   [--std COLUMNS]       Comma separated list of columns for stddev_samp or * for all numeric
-  [--exclude-columns or -ec]   
+  [--exclude-columns or -ec]
                         Flag to indicate the list of columns provided should be excluded and not included.
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
                         BigQuery destination for validation results. Defaults to stdout.
@@ -329,7 +331,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -371,12 +373,12 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
                         Source sql query
                         Either --source-query or --source-query-file must be provided
   --source-query-file SOURCE_QUERY_FILE, -sqf SOURCE_QUERY_FILE
-                        File containing the source sql commands
+                        File containing the source sql command. Supports GCS and local paths.
   --target-query TARGET_QUERY, -tq TARGET_QUERY
                         Target sql query
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
-                        File containing the target sql commands
+                        File containing the target sql command. Supports GCS and local paths.
   --comparison-fields or -comp-fields FIELDS
                         Comma separated list of columns to compare. Can either be a physical column or an alias
                         See: *Calculated Fields* section for details
@@ -391,7 +393,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
   [--config-file or -c CONFIG_FILE]
-                        YAML Config File Path to be used for storing validations and other features.
+                        YAML Config File Path to be used for storing validations and other features. Supports GCS and local paths.
                         See: *Running DVT with YAML Configuration Files* section
   [--config-file-json or -cj CONFIG_FILE_JSON]
                         JSON Config File Path to be used for storing validations only for application purposes.
@@ -435,11 +437,10 @@ For example, this flag can be used as follows:
 
 Running DVT with YAML configuration files is the recommended approach if:
 * you want to customize the configuration for any given validation OR
-* you want to run DVT at scale (i.e. row validations across many partitions)
+* you want to run DVT at scale (i.e. run multiple validations sequentially or in parallel)
 
-We recommend generating YAML configs with the `--config-file <file-name>` flag when running a validation command. The validation 
-config file is saved to the GCS path specified by the `PSO_DV_CONFIG_HOME` env variable if that has been set;
-otherwise, it is saved to wherever the tool is run.
+We recommend generating YAML configs with the `--config-file <file-name>` flag when running a validation command, which supports
+GCS and local paths.
 
 You can use the `data-validation configs` command to run and view YAMLs.
 
@@ -452,7 +453,7 @@ data-validation (--verbose or -v) (--log-level or -ll) configs run
   [--dry-run or -dr]    If this flag is present, prints the source and target SQL generated in lieu of running the validation.
   [--kube-completions or -kc]
                         Flag to indicate usage in Kubernetes index completion mode.
-                        See *Scaling DVT to run 10's to 1000's of validations concurrently* section
+                        See *Scaling DVT* section
 ```
 
 ```
@@ -470,19 +471,17 @@ View the complete YAML file for a Grouped Column validation on the
 [Examples](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/examples.md#sample-yaml-config-grouped-column-validation) page.
 
 
-#### Scaling DVT to run 10's to 1000's of validations concurrently
+### Scaling DVT
 
-As described above, you can run multiple validation configs sequentially with the `--config-dir` flag. To optimize the validation speed for large tables, you can run DVT concurrently using GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Kubernetes](docs/internal/kubernetes_jobs.md) for a detailed overview.
+You can scale DVT for large table validations by running the tool in a distributed manner. To optimize the validation speed for large tables, you can use GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Distributed Jobs](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/internal/distributed_jobs.md) for a detailed overview.
 
-In order to validate partitions concurrently, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. The `--config-dir` flag will specify the directory with the YAML files to be executed in parallel. If you used `generate-table-partitions` to generate the YAMLs, this would be the directory where the partition files numbered `0000.yaml` to `<partition_num - 1>.yaml` are stored i.e (`my_config_dir/source_schema.source_table/`)
 
-In order to run DVT in a container, you have to build a Docker image - [see instructions here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/docker#readme). You will also need to set the `PSO_DV_CONFIG_HOME` environment variable to point to a GCS prefix where the connection configuration files are stored. In Cloud Run you can use the option `--set-env-vars` or `--update-env-vars` to pass [the environment variable](https://cloud.google.com/run/docs/configuring/services/environment-variables#setting). We recommend that you use the `bq-result-handler` to save your validation results to BigQuery. 
+We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started.
 
-The Cloud Run and Kubernetes pods must run in a network with access to the database servers. Every Cloud Run job is associated with a [service account](https://cloud.google.com/run/docs/securing/service-identity). You need to ensure that this service account has access to Google Cloud Storage (to read connection configuration and YAML files) and BigQuery (to publish results). If you are using Kubernetes, you will need to use a service account with the same privileges as mentioned for Cloud Run. In Kubernetes, you will need to set up [workload identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) so the DVT container can impersonate the service account.
+When running DVT in a distributed fashion, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (i.e. a `--config-file` is provided instead), a warning is issued.
 
-In Cloud Run, the [job](https://cloud.google.com/run/docs/create-jobs) must be run as multiple, independent tasks with the task count set to the number of partitions generated. In Kubernetes, set the number of completions to the number of partitions generated - see [Kubernetes Parallel Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#parallel-jobs). The option `--kube-completions or -kc` tells DVT that many DVT containers are running in a Kubernetes cluster. Each DVT container only validates the specific partition YAML (based on the index assigned by Kubernetes control plane). If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (a `--config-file` is provided instead), a warning is issued.
+The `--config-dir` flag will specify the directory with the YAML files to be executed in parallel. If you used `generate-table-partitions` to generate the YAMLs, this would be the directory where the partition files numbered `0000.yaml` to `<partition_num - 1>.yaml` are stored i.e (`gs://my_config_dir/source_schema.source_table/`). When creating your Cloud Run Job, set the number of tasks equal to the number of table partitions so the task index matches the YAML file to be validated. When executed, each Cloud Run task will validate a partition in parallel.
 
-By default, each partition validation is retried up to 3 times if there is an error. In Kubernetes and Cloud Run, you can set the parallelism to the number you want. Keep in mind that if you are validating 1000's of partitions in parallel, you may find that setting the parallelism too high (say 100) may result in timeouts and slow down the validation.   
 
 ### Validation Reports
 
@@ -587,7 +586,8 @@ with calculated fields as shown in [this example](https://github.com/GoogleCloud
 
 Filters let you apply a WHERE statement to your validation query (ie. `SELECT *
 FROM table WHERE created_at > 30 days ago AND region_id = 71;`). The filter is
-written in the syntax of the given source.
+written in the syntax of the given source and must reference columns in the
+underlying table, not projected DVT expressions.
 
 Note that you are writing the query to execute, which does not have to match
 between source and target as long as the results can be expected to align. If

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.1.0}/data_validation/__main__.py

@@ -48,7 +48,10 @@ def _get_arg_config_file(args):
     """Return String YAML config file path."""
     if not args.config_file:
         raise ValueError("YAML Config File was not supplied.")
-
+    elif not args.config_file.endswith(".yaml"):
+        raise ValueError(
+            f"Invalid YAML config name: {args.config_file}. Provide YAML file extension."
+        )
     return args.config_file
 
 
@@ -56,7 +59,10 @@ def _get_arg_config_file_json(args):
     """Return String JSON config file path."""
     if not args.config_file_json:
         raise ValueError("JSON Config File was not supplied.")
-
+    elif not args.config_file_json.endswith(".json"):
+        raise ValueError(
+            f"Invalid JSON config name: {args.config_file_json}. Provide JSON file extension."
+        )
     return args.config_file_json
 
 
@@ -350,16 +356,17 @@ def config_runner(args):
             setattr(args, "config_dir", None)
             setattr(args, "config_file", config_file_path)
             config_managers = build_config_managers_from_yaml(args, config_file_path)
+            run_validations(args, config_managers)
         else:
             if args.kube_completions:
                 logging.warning(
                     "--kube-completions or -kc specified, however not running in Kubernetes Job completion, check your command line."
                 )
-            mgr = state_manager.StateManager(file_system_root_path=args.config_dir)
-            config_file_names = mgr.list_validations_in_dir(args.config_dir)
+            config_file_names = cli_tools.list_validations(config_dir=args.config_dir)
             config_managers = []
             for file in config_file_names:
-                config_managers.extend(build_config_managers_from_yaml(args, file))
+                config_managers = build_config_managers_from_yaml(args, file)
+                run_validations(args, config_managers)
     else:
         if args.kube_completions:
             logging.warning(
@@ -367,13 +374,12 @@ def config_runner(args):
             )
         config_file_path = _get_arg_config_file(args)
         config_managers = build_config_managers_from_yaml(args, config_file_path)
-
-    run_validations(args, config_managers)
+        run_validations(args, config_managers)
 
 
 def build_config_managers_from_yaml(args, config_file_path):
     """Returns List[ConfigManager] instances ready to be executed."""
-    if "config_dir" in args and args.config_dir:
+    if args.config_dir:
         yaml_configs = cli_tools.get_validation(config_file_path, args.config_dir)
     else:
         yaml_configs = cli_tools.get_validation(config_file_path)
@@ -616,7 +622,7 @@ def store_json_config_file(args, config_managers):
     """
     json_config = convert_config_to_json(config_managers)
     config_file_path = _get_arg_config_file_json(args)
-    cli_tools.store_validation_json(config_file_path, json_config)
+    cli_tools.store_validation(config_file_path, json_config)
 
 
 def partition_and_store_config_files(args: Namespace) -> None:
@@ -673,7 +679,7 @@ def run_validation_configs(args):
         config_runner(args)
     elif args.validation_config_cmd == "list":
         config_dir = args.config_dir or "./"
-        cli_tools.list_validations(config_dir=config_dir)
+        cli_tools.print_validations_in_dir(config_dir=config_dir)
     elif args.validation_config_cmd == "get":
         # Get and print yaml file config.
         yaml = cli_tools.get_validation(_get_arg_config_file(args))

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.1.0}/data_validation/cli_tools.py

@@ -49,10 +49,12 @@ import json
 import logging
 import sys
 import uuid
+import os
 from argparse import Namespace
 from typing import Dict, List
+from yaml import Dumper, Loader, dump, load
 
-from data_validation import clients, consts, state_manager
+from data_validation import clients, consts, state_manager, gcs_helper
 
 CONNECTION_SOURCE_FIELDS = {
     "BigQuery": [
@@ -482,6 +484,15 @@ def _configure_row_parser(row_parser):
         "-rbs",
         help="Row batch size used for random row filters (default 10,000).",
     )
+    optional_arguments.add_argument(
+        "--trim-string-pks",
+        "-tsp",
+        action="store_true",
+        help=(
+            "Trims string based primary key values, intended for use when one engine uses "
+            "padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n))."
+        ),
+    )
 
     # Group required arguments
     required_arguments = row_parser.add_argument_group("required arguments")
@@ -952,6 +963,18 @@ def _add_common_arguments(optional_arguments, required_arguments):
     )
 
 
+def _check_no_partitions(value: str) -> int:
+    """Check that number of partitions is between [2-10,000]
+    Using function to validate rather than choices as error message prints all choices.
+    """
+    if value.isdigit() and 2 <= int(value) <= 10000:
+        return int(value)
+    else:
+        raise argparse.ArgumentTypeError(
+            f"{value} is not valid for number of partitions, use number in range 2 to 10000"
+        )
+
+
 def _add_common_partition_arguments(optional_arguments, required_arguments):
     """Add all arguments common to get-partition command"""
 
@@ -976,10 +999,9 @@ def _add_common_partition_arguments(optional_arguments, required_arguments):
         "--partition-num",
         "-pn",
         required=True,
-        help="Number of partitions/config files to generate",
-        type=int,
-        choices=range(1, 1001),
-        metavar="[1-1000]",
+        help="Number of partitions/config files to generate, a number from 2 to 10,000",
+        type=_check_no_partitions,
+        metavar="[2-10000]",
     )
 
     # Optional arguments
@@ -1081,47 +1103,46 @@ def get_connection(connection_name):
     return mgr.get_connection_config(connection_name)
 
 
-def store_validation(validation_file_name, yaml_config):
-    """Store the validation YAML config under the given name."""
-    mgr = state_manager.StateManager()
-    mgr.create_validation_yaml(validation_file_name, yaml_config)
-
-
-def store_validation_json(validation_file_name, json_config):
-    """Store the validation JSON config under the given name."""
-    mgr = state_manager.StateManager()
-    mgr.create_validation_json(validation_file_name, json_config)
+def store_validation(validation_file_name, config, include_log=True):
+    """Store the validation config under the given name."""
+    validation_path = gcs_helper.get_validation_path(validation_file_name)
 
+    if validation_file_name.endswith(".yaml"):
+        config_str = dump(config, Dumper=Dumper)
+    elif validation_file_name.endswith("json"):
+        config_str = json.dumps(config)
+    else:
+        raise ValueError(f"Invalid validation file name: {validation_file_name}")
 
-def store_partition(target_file_path, yaml_config, target_folder_path=None):
-    """Store the partition YAML config under the given name."""
-    mgr = state_manager.StateManager(target_folder_path)
-    mgr.create_partition_yaml(target_file_path, yaml_config)
+    gcs_helper.write_file(validation_path, config_str, include_log=include_log)
 
 
-def get_validation(validation_name, config_dir=None):
-    """Return validation YAML for a specific connection."""
+def get_validation(name: str, config_dir: str = None):
+    """Return validation YAML config."""
     if config_dir:
-        mgr = state_manager.StateManager(file_system_root_path=config_dir)
-        return mgr.get_validation_config(validation_name, config_dir)
+        validation_path = os.path.join(config_dir, name)
     else:
-        if validation_name.startswith("gs://"):
-            obj_depth = len(validation_name.split("/"))
-            gcs_prefix = "/".join(validation_name.split("/")[: obj_depth - 1])
-            mgr = state_manager.StateManager(file_system_root_path=gcs_prefix)
-            return mgr.get_validation_config(
-                validation_name.split("/")[obj_depth - 1], gcs_prefix
-            )
-        else:
-            mgr = state_manager.StateManager()
-            return mgr.get_validation_config(validation_name)
+        validation_path = gcs_helper.get_validation_path(name)
+
+    validation_bytes = gcs_helper.read_file(validation_path)
+    return load(validation_bytes, Loader=Loader)
 
 
 def list_validations(config_dir="./"):
-    """List all saved validation YAMLs."""
-    mgr = state_manager.StateManager(file_system_root_path=config_dir)
-    validations = mgr.list_validations_in_dir(config_dir=config_dir)
+    """List all saved validation YAMLs in a directory."""
+    logging.info(f"Looking for validations in path {config_dir}")
+    if gcs_helper._is_gcs_path(config_dir):
+        if not config_dir.endswith("/"):
+            config_dir += "/"
+        files = gcs_helper.list_gcs_directory(config_dir)
+    else:
+        files = os.listdir(config_dir)
+
+    return [file_name for file_name in files if file_name.endswith(".yaml")]
+
 
+def print_validations_in_dir(config_dir="./"):
+    validations = list_validations(config_dir=config_dir)
     logging.info("Validation YAMLs found:")
     for validation_name in validations:
         logging.info(validation_name)
@@ -1386,6 +1407,7 @@ def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
             "result_handler_config": result_handler_config,
             "filter_config": filter_config,
             "filter_status": filter_status,
+            "trim_string_pks": getattr(args, "trim_string_pks", False),
             "verbose": args.verbose,
         }
         pre_build_configs_list.append(pre_build_configs)

{google-pso-data-validator-4.5.0 → google_pso_data_validator-5.1.0}/data_validation/clients.py

@@ -274,6 +274,23 @@ def get_max_column_length(client):
     return 128
 
 
+def get_max_in_list_size(client, in_list_over_expressions=False):
+    if client.name == "snowflake":
+        if in_list_over_expressions:
+            # This is a workaround for Snowflake limitation:
+            #   SQL compilation error: In-list contains more than 50 non-constant values
+            # getattr(..., "cast") expression above is looking for lists where the contents are casts and not simple literals.
+            return 50
+        else:
+            return 16000
+    elif is_oracle_client(client):
+        # This is a workaround for Oracle limitation:
+        #   ORA-01795: maximum number of expressions in a list is 1000
+        return 1000
+    else:
+        return None
+
+
 CLIENT_LOOKUP = {
     "BigQuery": get_bigquery_client,
     "Impala": impala_connect,