google-pso-data-validator 6.3.0__tar.gz → 7.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/PKG-INFO +21 -18
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/README.md +20 -17
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/__main__.py +77 -62
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/cli_tools.py +84 -37
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/clients.py +39 -12
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/config_manager.py +147 -41
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/consts.py +9 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/data_validation.py +33 -9
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/exceptions.py +5 -1
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/find_tables.py +13 -4
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/gcs_helper.py +4 -1
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/metadata.py +1 -3
- google_pso_data_validator-7.0.0/data_validation/raw_query.py +47 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/result_handlers/bigquery.py +13 -10
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/secret_manager.py +1 -1
- google_pso_data_validator-7.0.0/data_validation/util.py +24 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/PKG-INFO +21 -18
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/SOURCES.txt +4 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/setup.py +1 -1
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_addon/api.py +29 -1
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_addon/operations.py +12 -0
- google_pso_data_validator-7.0.0/third_party/ibis/ibis_biquery/api.py +15 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/__init__.py +16 -1
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/api.py +2 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_db2/__init__.py +23 -1
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_impala/api.py +12 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mssql/__init__.py +22 -1
- google_pso_data_validator-7.0.0/third_party/ibis/ibis_mysql/__init__.py +34 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_oracle/__init__.py +30 -2
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_postgres/client.py +54 -19
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_redshift/__init__.py +11 -2
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_snowflake/datatypes.py +11 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_teradata/__init__.py +24 -4
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/LICENSE +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/__init__.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/app.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/client_info.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/combiner.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/jellyfish_distance.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/partition_builder.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/query_builder/__init__.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/query_builder/partition_row_builder.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/query_builder/query_builder.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/query_builder/random_row_builder.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/result_handlers/__init__.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/result_handlers/text.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/schema_validation.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/state_manager.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/validation_builder.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/dependency_links.txt +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/entry_points.txt +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/requires.txt +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/top_level.txt +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/setup.cfg +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_addon/__init__.py +0 -0
- {google_pso_data_validator-6.3.0/third_party/ibis/ibis_impala → google_pso_data_validator-7.0.0/third_party/ibis/ibis_biquery}/__init__.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/client.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/compiler.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/datatypes.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/registry.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/tests/__init__.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/tests/conftest.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/to_pandas.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_db2/api.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_db2/compiler.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_db2/datatypes.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_db2/registry.py +0 -0
- {google_pso_data_validator-6.3.0/third_party/ibis/ibis_mysql → google_pso_data_validator-7.0.0/third_party/ibis/ibis_impala}/__init__.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mssql/api.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mssql/datatypes.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mssql/registry.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mysql/base_sql_compiler/__init__.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mysql/base_sql_compiler/select_builder.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mysql/compiler.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_oracle/api.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_oracle/compiler.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_oracle/datatypes.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_oracle/registry.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_postgres/__init__.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_postgres/datatypes.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_redshift/api.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_redshift/compiler.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_snowflake/__init__.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_snowflake/api.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_teradata/api.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_teradata/compiler.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_teradata/datatypes.py +0 -0
- {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_teradata/registry.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: google-pso-data-validator
|
|
3
|
-
Version:
|
|
3
|
+
Version: 7.0.0
|
|
4
4
|
Summary: A package to enable easy data validation
|
|
5
5
|
Author: PSO DVT Engineering team
|
|
6
6
|
Author-email: data-validator-eng@google.com
|
|
@@ -154,7 +154,7 @@ data-validation
|
|
|
154
154
|
[--std COLUMNS] Comma separated list of columns for stddev_samp or * for all numeric
|
|
155
155
|
[--exclude-columns or -ec]
|
|
156
156
|
Flag to indicate the list of columns provided should be excluded and not included.
|
|
157
|
-
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
|
|
157
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
158
158
|
BigQuery destination for validation results. Defaults to stdout.
|
|
159
159
|
See: *Validation Reports* section
|
|
160
160
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -197,17 +197,16 @@ In addition, please note that SHA256 is not a supported function on Teradata sys
|
|
|
197
197
|
If you wish to perform this comparison on Teradata you will need to
|
|
198
198
|
[deploy a UDF to perform the conversion](https://github.com/akuroda/teradata-udf-sha2/blob/master/src/sha256.c).)
|
|
199
199
|
|
|
200
|
-
Below is the command syntax for row validations. In order to run row level
|
|
201
|
-
|
|
202
|
-
the
|
|
203
|
-
|
|
200
|
+
Below is the command syntax for row validations. In order to run row level validations we require
|
|
201
|
+
unique columns to join row sets, which are either inferred from the source/target table or provided
|
|
202
|
+
via the `--primary-keys` flag, and either the `--hash`, `--concat` or `--comparison-fields` flags.
|
|
203
|
+
See *Primary Keys* section.
|
|
204
204
|
|
|
205
205
|
The `--comparison-fields` flag specifies the values (e.g. columns) whose raw values will be compared
|
|
206
206
|
based on the primary key join. The `--hash` flag will run a checksum across specified columns in
|
|
207
207
|
the table. This will include casting to string, sanitizing the data (ifnull, rtrim, upper), concatenating,
|
|
208
208
|
and finally hashing the row.
|
|
209
209
|
|
|
210
|
-
|
|
211
210
|
Under the hood, row validation uses
|
|
212
211
|
[Calculated Fields](https://github.com/GoogleCloudPlatform/professional-services-data-validator#calculated-fields) to
|
|
213
212
|
apply functions such as IFNULL() or RTRIM(). These can be edited in the YAML or JSON config file to customize your row validation.
|
|
@@ -229,16 +228,17 @@ data-validation
|
|
|
229
228
|
Comma separated list of tables in the form schema.table=target_schema.target_table
|
|
230
229
|
Target schema name and table name are optional.
|
|
231
230
|
i.e 'bigquery-public-data.new_york_citibike.citibike_trips'
|
|
232
|
-
--primary-keys or -pk PRIMARY_KEYS
|
|
233
|
-
Comma separated list of columns to use as primary keys. See *Primary Keys* section
|
|
234
231
|
--comparison-fields or -comp-fields FIELDS
|
|
235
232
|
Comma separated list of columns to compare. Can either be a physical column or an alias
|
|
236
233
|
See: *Calculated Fields* section for details
|
|
237
234
|
--hash COLUMNS Comma separated list of columns to hash or * for all columns
|
|
238
235
|
--concat COLUMNS Comma separated list of columns to concatenate or * for all columns (use if a common hash function is not available between databases)
|
|
236
|
+
[--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
|
|
237
|
+
Comma separated list of primary key columns, when not specified the value will be inferred
|
|
238
|
+
from the source or target table if available. See *Primary Keys* section
|
|
239
239
|
[--exclude-columns or -ec]
|
|
240
240
|
Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
|
|
241
|
-
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
|
|
241
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
242
242
|
BigQuery destination for validation results. Defaults to stdout.
|
|
243
243
|
See: *Validation Reports* section
|
|
244
244
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -303,8 +303,6 @@ data-validation
|
|
|
303
303
|
Either --tables-list or --source-query (or file) and --target-query (or file) must be provided
|
|
304
304
|
--target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
|
|
305
305
|
File containing the target sql command. Supports GCS and local paths.
|
|
306
|
-
--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS
|
|
307
|
-
Comma separated list of primary key columns 'col_a,col_b'. See *Primary Keys* section
|
|
308
306
|
--comparison-fields or -comp-fields FIELDS
|
|
309
307
|
Comma separated list of columns to compare. Can either be a physical column or an alias
|
|
310
308
|
See: *Calculated Fields* section for details
|
|
@@ -318,7 +316,10 @@ data-validation
|
|
|
318
316
|
--partition-num INT, -pn INT
|
|
319
317
|
Number of partitions into which the table should be split, e.g. 1000 or 10000
|
|
320
318
|
In case this value exceeds the row count of the source/target table, it will be decreased to max(source_row_count, target_row_count)
|
|
321
|
-
[--
|
|
319
|
+
[--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
|
|
320
|
+
Comma separated list of primary key columns, when not specified the value will be inferred
|
|
321
|
+
from the source or target table if available. See *Primary Keys* section
|
|
322
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
322
323
|
BigQuery destination for validation results. Defaults to stdout.
|
|
323
324
|
See: *Validation Reports* section
|
|
324
325
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -364,7 +365,7 @@ data-validation
|
|
|
364
365
|
Comma separated list of tables in the form schema.table=target_schema.target_table. Or shorthand schema.* for all tables.
|
|
365
366
|
Target schema name and table name are optional.
|
|
366
367
|
e.g.: 'bigquery-public-data.new_york_citibike.citibike_trips'
|
|
367
|
-
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
|
|
368
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
368
369
|
BigQuery destination for validation results. Defaults to stdout.
|
|
369
370
|
See: *Validation Reports* section
|
|
370
371
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -424,7 +425,7 @@ data-validation
|
|
|
424
425
|
[--std COLUMNS] Comma separated list of columns for stddev_samp or * for all numeric
|
|
425
426
|
[--exclude-columns or -ec]
|
|
426
427
|
Flag to indicate the list of columns provided should be excluded and not included.
|
|
427
|
-
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
|
|
428
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
428
429
|
BigQuery destination for validation results. Defaults to stdout.
|
|
429
430
|
See: *Validation Reports* section
|
|
430
431
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -489,11 +490,11 @@ data-validation
|
|
|
489
490
|
--hash '*' '*' to hash all columns.
|
|
490
491
|
--concat COLUMNS Comma separated list of columns to concatenate or * for all columns
|
|
491
492
|
(use if a common hash function is not available between databases)
|
|
492
|
-
--primary-
|
|
493
|
-
Common column between source and target
|
|
493
|
+
[--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
|
|
494
|
+
Common column between source and target queries for join
|
|
494
495
|
[--exclude-columns or -ec]
|
|
495
496
|
Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
|
|
496
|
-
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
|
|
497
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
497
498
|
BigQuery destination for validation results. Defaults to stdout.
|
|
498
499
|
See: *Validation Reports* section
|
|
499
500
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -720,6 +721,8 @@ In many cases, validations (e.g. count, min, max etc) produce one row per table.
|
|
|
720
721
|
and target table is to compare the value for each column in the source with the value of the column in the target.
|
|
721
722
|
`grouped-columns` validation and `validate row` produce multiple rows per table. Data Validation Tool needs one or more columns to uniquely identify each row so the source and target can be compared. Data Validation Tool refers to these columns as primary keys. These do not need to be primary keys in the table. The only requirement is that the keys uniquely identify the row in the results.
|
|
722
723
|
|
|
724
|
+
These columns are inferred, where possible, from the source/target table or can be provided via the `--primary-keys` flag.
|
|
725
|
+
|
|
723
726
|
### Grouped Columns
|
|
724
727
|
|
|
725
728
|
Grouped Columns contain the fields you want your aggregations to be broken out
|
|
@@ -113,7 +113,7 @@ data-validation
|
|
|
113
113
|
[--std COLUMNS] Comma separated list of columns for stddev_samp or * for all numeric
|
|
114
114
|
[--exclude-columns or -ec]
|
|
115
115
|
Flag to indicate the list of columns provided should be excluded and not included.
|
|
116
|
-
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
|
|
116
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
117
117
|
BigQuery destination for validation results. Defaults to stdout.
|
|
118
118
|
See: *Validation Reports* section
|
|
119
119
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -156,17 +156,16 @@ In addition, please note that SHA256 is not a supported function on Teradata sys
|
|
|
156
156
|
If you wish to perform this comparison on Teradata you will need to
|
|
157
157
|
[deploy a UDF to perform the conversion](https://github.com/akuroda/teradata-udf-sha2/blob/master/src/sha256.c).)
|
|
158
158
|
|
|
159
|
-
Below is the command syntax for row validations. In order to run row level
|
|
160
|
-
|
|
161
|
-
the
|
|
162
|
-
|
|
159
|
+
Below is the command syntax for row validations. In order to run row level validations we require
|
|
160
|
+
unique columns to join row sets, which are either inferred from the source/target table or provided
|
|
161
|
+
via the `--primary-keys` flag, and either the `--hash`, `--concat` or `--comparison-fields` flags.
|
|
162
|
+
See *Primary Keys* section.
|
|
163
163
|
|
|
164
164
|
The `--comparison-fields` flag specifies the values (e.g. columns) whose raw values will be compared
|
|
165
165
|
based on the primary key join. The `--hash` flag will run a checksum across specified columns in
|
|
166
166
|
the table. This will include casting to string, sanitizing the data (ifnull, rtrim, upper), concatenating,
|
|
167
167
|
and finally hashing the row.
|
|
168
168
|
|
|
169
|
-
|
|
170
169
|
Under the hood, row validation uses
|
|
171
170
|
[Calculated Fields](https://github.com/GoogleCloudPlatform/professional-services-data-validator#calculated-fields) to
|
|
172
171
|
apply functions such as IFNULL() or RTRIM(). These can be edited in the YAML or JSON config file to customize your row validation.
|
|
@@ -188,16 +187,17 @@ data-validation
|
|
|
188
187
|
Comma separated list of tables in the form schema.table=target_schema.target_table
|
|
189
188
|
Target schema name and table name are optional.
|
|
190
189
|
i.e 'bigquery-public-data.new_york_citibike.citibike_trips'
|
|
191
|
-
--primary-keys or -pk PRIMARY_KEYS
|
|
192
|
-
Comma separated list of columns to use as primary keys. See *Primary Keys* section
|
|
193
190
|
--comparison-fields or -comp-fields FIELDS
|
|
194
191
|
Comma separated list of columns to compare. Can either be a physical column or an alias
|
|
195
192
|
See: *Calculated Fields* section for details
|
|
196
193
|
--hash COLUMNS Comma separated list of columns to hash or * for all columns
|
|
197
194
|
--concat COLUMNS Comma separated list of columns to concatenate or * for all columns (use if a common hash function is not available between databases)
|
|
195
|
+
[--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
|
|
196
|
+
Comma separated list of primary key columns, when not specified the value will be inferred
|
|
197
|
+
from the source or target table if available. See *Primary Keys* section
|
|
198
198
|
[--exclude-columns or -ec]
|
|
199
199
|
Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
|
|
200
|
-
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
|
|
200
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
201
201
|
BigQuery destination for validation results. Defaults to stdout.
|
|
202
202
|
See: *Validation Reports* section
|
|
203
203
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -262,8 +262,6 @@ data-validation
|
|
|
262
262
|
Either --tables-list or --source-query (or file) and --target-query (or file) must be provided
|
|
263
263
|
--target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
|
|
264
264
|
File containing the target sql command. Supports GCS and local paths.
|
|
265
|
-
--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS
|
|
266
|
-
Comma separated list of primary key columns 'col_a,col_b'. See *Primary Keys* section
|
|
267
265
|
--comparison-fields or -comp-fields FIELDS
|
|
268
266
|
Comma separated list of columns to compare. Can either be a physical column or an alias
|
|
269
267
|
See: *Calculated Fields* section for details
|
|
@@ -277,7 +275,10 @@ data-validation
|
|
|
277
275
|
--partition-num INT, -pn INT
|
|
278
276
|
Number of partitions into which the table should be split, e.g. 1000 or 10000
|
|
279
277
|
In case this value exceeds the row count of the source/target table, it will be decreased to max(source_row_count, target_row_count)
|
|
280
|
-
[--
|
|
278
|
+
[--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
|
|
279
|
+
Comma separated list of primary key columns, when not specified the value will be inferred
|
|
280
|
+
from the source or target table if available. See *Primary Keys* section
|
|
281
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
281
282
|
BigQuery destination for validation results. Defaults to stdout.
|
|
282
283
|
See: *Validation Reports* section
|
|
283
284
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -323,7 +324,7 @@ data-validation
|
|
|
323
324
|
Comma separated list of tables in the form schema.table=target_schema.target_table. Or shorthand schema.* for all tables.
|
|
324
325
|
Target schema name and table name are optional.
|
|
325
326
|
e.g.: 'bigquery-public-data.new_york_citibike.citibike_trips'
|
|
326
|
-
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
|
|
327
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
327
328
|
BigQuery destination for validation results. Defaults to stdout.
|
|
328
329
|
See: *Validation Reports* section
|
|
329
330
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -383,7 +384,7 @@ data-validation
|
|
|
383
384
|
[--std COLUMNS] Comma separated list of columns for stddev_samp or * for all numeric
|
|
384
385
|
[--exclude-columns or -ec]
|
|
385
386
|
Flag to indicate the list of columns provided should be excluded and not included.
|
|
386
|
-
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
|
|
387
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
387
388
|
BigQuery destination for validation results. Defaults to stdout.
|
|
388
389
|
See: *Validation Reports* section
|
|
389
390
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -448,11 +449,11 @@ data-validation
|
|
|
448
449
|
--hash '*' '*' to hash all columns.
|
|
449
450
|
--concat COLUMNS Comma separated list of columns to concatenate or * for all columns
|
|
450
451
|
(use if a common hash function is not available between databases)
|
|
451
|
-
--primary-
|
|
452
|
-
Common column between source and target
|
|
452
|
+
[--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
|
|
453
|
+
Common column between source and target queries for join
|
|
453
454
|
[--exclude-columns or -ec]
|
|
454
455
|
Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
|
|
455
|
-
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
|
|
456
|
+
[--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
|
|
456
457
|
BigQuery destination for validation results. Defaults to stdout.
|
|
457
458
|
See: *Validation Reports* section
|
|
458
459
|
[--service-account or -sa PATH_TO_SA_KEY]
|
|
@@ -679,6 +680,8 @@ In many cases, validations (e.g. count, min, max etc) produce one row per table.
|
|
|
679
680
|
and target table is to compare the value for each column in the source with the value of the column in the target.
|
|
680
681
|
`grouped-columns` validation and `validate row` produce multiple rows per table. Data Validation Tool needs one or more columns to uniquely identify each row so the source and target can be compared. Data Validation Tool refers to these columns as primary keys. These do not need to be primary keys in the table. The only requirement is that the keys uniquely identify the row in the results.
|
|
681
682
|
|
|
683
|
+
These columns are inferred, where possible, from the source/target table or can be provided via the `--primary-keys` flag.
|
|
684
|
+
|
|
682
685
|
### Grouped Columns
|
|
683
686
|
|
|
684
687
|
Grouped Columns contain the fields you want your aggregations to be broken out
|
{google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/__main__.py
RENAMED
|
@@ -24,7 +24,10 @@ from data_validation import (
|
|
|
24
24
|
cli_tools,
|
|
25
25
|
clients,
|
|
26
26
|
consts,
|
|
27
|
+
exceptions,
|
|
28
|
+
raw_query,
|
|
27
29
|
state_manager,
|
|
30
|
+
util,
|
|
28
31
|
)
|
|
29
32
|
from data_validation.config_manager import ConfigManager
|
|
30
33
|
from data_validation.data_validation import DataValidation
|
|
@@ -89,6 +92,7 @@ def get_aggregate_config(args, config_manager: ConfigManager):
|
|
|
89
92
|
"!int64",
|
|
90
93
|
"!decimal",
|
|
91
94
|
]
|
|
95
|
+
uuid_types = ["uuid", "!uuid"]
|
|
92
96
|
|
|
93
97
|
if args.wildcard_include_string_len:
|
|
94
98
|
supported_data_types.extend(["string", "!string"])
|
|
@@ -127,7 +131,7 @@ def get_aggregate_config(args, config_manager: ConfigManager):
|
|
|
127
131
|
"min",
|
|
128
132
|
col_args,
|
|
129
133
|
args.exclude_columns,
|
|
130
|
-
supported_data_types,
|
|
134
|
+
supported_data_types + uuid_types,
|
|
131
135
|
cast_to_bigint=cast_to_bigint,
|
|
132
136
|
)
|
|
133
137
|
if args.max:
|
|
@@ -136,7 +140,7 @@ def get_aggregate_config(args, config_manager: ConfigManager):
|
|
|
136
140
|
"max",
|
|
137
141
|
col_args,
|
|
138
142
|
args.exclude_columns,
|
|
139
|
-
supported_data_types,
|
|
143
|
+
supported_data_types + uuid_types,
|
|
140
144
|
cast_to_bigint=cast_to_bigint,
|
|
141
145
|
)
|
|
142
146
|
if args.bit_xor:
|
|
@@ -217,22 +221,20 @@ def _get_calculated_config(args, config_manager: ConfigManager) -> List[dict]:
|
|
|
217
221
|
return calculated_configs
|
|
218
222
|
|
|
219
223
|
|
|
220
|
-
def _get_comparison_config(
|
|
224
|
+
def _get_comparison_config(
|
|
225
|
+
args, config_manager: ConfigManager, primary_keys: list
|
|
226
|
+
) -> List[dict]:
|
|
221
227
|
col_list = (
|
|
222
228
|
None
|
|
223
|
-
if
|
|
224
|
-
else cli_tools.get_arg_list(
|
|
229
|
+
if args.comparison_fields == "*"
|
|
230
|
+
else cli_tools.get_arg_list(args.comparison_fields)
|
|
225
231
|
)
|
|
226
232
|
comparison_fields = config_manager.build_comp_fields(
|
|
227
233
|
col_list,
|
|
228
234
|
args.exclude_columns,
|
|
229
235
|
)
|
|
230
236
|
# We can't have the PK columns in the comparison SQL twice therefore filter them out here if included.
|
|
231
|
-
comparison_fields = [
|
|
232
|
-
_
|
|
233
|
-
for _ in comparison_fields
|
|
234
|
-
if _ not in cli_tools.get_arg_list(args.primary_keys.casefold())
|
|
235
|
-
]
|
|
237
|
+
comparison_fields = [_ for _ in comparison_fields if _ not in primary_keys]
|
|
236
238
|
|
|
237
239
|
# As per #1190, add rstrip for Teradata string comparison fields
|
|
238
240
|
if (
|
|
@@ -312,18 +314,25 @@ def build_config_from_args(args: Namespace, config_manager: ConfigManager):
|
|
|
312
314
|
_get_calculated_config(args, config_manager)
|
|
313
315
|
)
|
|
314
316
|
|
|
315
|
-
# Append Comparison fields
|
|
316
|
-
if args.comparison_fields:
|
|
317
|
-
config_manager.append_comparison_fields(
|
|
318
|
-
_get_comparison_config(args, config_manager)
|
|
319
|
-
)
|
|
320
|
-
|
|
321
317
|
# Append primary_keys
|
|
322
318
|
primary_keys = cli_tools.get_arg_list(args.primary_keys)
|
|
319
|
+
if not primary_keys and config_manager.validation_type != consts.CUSTOM_QUERY:
|
|
320
|
+
primary_keys = config_manager.auto_list_primary_keys()
|
|
321
|
+
if not primary_keys:
|
|
322
|
+
raise ValueError(
|
|
323
|
+
"No primary keys were provided and neither the source or target tables have primary keys. Please include --primary-keys argument"
|
|
324
|
+
)
|
|
325
|
+
primary_keys = [_.casefold() for _ in primary_keys]
|
|
323
326
|
config_manager.append_primary_keys(
|
|
324
327
|
config_manager.build_column_configs(primary_keys)
|
|
325
328
|
)
|
|
326
329
|
|
|
330
|
+
# Append Comparison fields
|
|
331
|
+
if args.comparison_fields:
|
|
332
|
+
config_manager.append_comparison_fields(
|
|
333
|
+
_get_comparison_config(args, config_manager, primary_keys)
|
|
334
|
+
)
|
|
335
|
+
|
|
327
336
|
return config_manager
|
|
328
337
|
|
|
329
338
|
|
|
@@ -331,22 +340,25 @@ def build_config_managers_from_args(
|
|
|
331
340
|
args: Namespace, validate_cmd: str = None
|
|
332
341
|
) -> List[ConfigManager]:
|
|
333
342
|
"""Return a list of config managers ready to execute."""
|
|
334
|
-
configs = []
|
|
335
343
|
|
|
336
|
-
|
|
337
|
-
|
|
344
|
+
def _build_configs():
|
|
345
|
+
configs = []
|
|
346
|
+
|
|
347
|
+
# Get pre build configs to build ConfigManager objects
|
|
348
|
+
pre_build_configs_list = cli_tools.get_pre_build_configs(args, validate_cmd)
|
|
338
349
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
350
|
+
# Build a list of ConfigManager objects
|
|
351
|
+
for pre_build_configs in pre_build_configs_list:
|
|
352
|
+
config_manager = ConfigManager.build_config_manager(**pre_build_configs)
|
|
342
353
|
|
|
343
|
-
|
|
344
|
-
|
|
354
|
+
# Append post build configs to ConfigManager object
|
|
355
|
+
config_manager = build_config_from_args(args, config_manager)
|
|
345
356
|
|
|
346
|
-
|
|
347
|
-
|
|
357
|
+
# Append ConfigManager object to configs list
|
|
358
|
+
configs.append(config_manager)
|
|
359
|
+
return configs
|
|
348
360
|
|
|
349
|
-
return
|
|
361
|
+
return util.timed_call("Build config", _build_configs)
|
|
350
362
|
|
|
351
363
|
|
|
352
364
|
def config_runner(args):
|
|
@@ -388,9 +400,26 @@ def config_runner(args):
|
|
|
388
400
|
)
|
|
389
401
|
config_file_names = cli_tools.list_validations(config_dir=args.config_dir)
|
|
390
402
|
config_managers = []
|
|
403
|
+
errors = False
|
|
391
404
|
for file in config_file_names:
|
|
392
405
|
config_managers = build_config_managers_from_yaml(args, file)
|
|
393
|
-
|
|
406
|
+
try:
|
|
407
|
+
logging.info(
|
|
408
|
+
"Currently running the validation for YAML file: %s",
|
|
409
|
+
file,
|
|
410
|
+
)
|
|
411
|
+
run_validations(args, config_managers)
|
|
412
|
+
except Exception as e:
|
|
413
|
+
errors = True
|
|
414
|
+
logging.error(
|
|
415
|
+
"Error '%s' occurred while running config file %s. Skipping it for now.",
|
|
416
|
+
str(e),
|
|
417
|
+
file,
|
|
418
|
+
)
|
|
419
|
+
if errors:
|
|
420
|
+
raise exceptions.ValidationException(
|
|
421
|
+
"Some of the validations raised an exception"
|
|
422
|
+
)
|
|
394
423
|
else:
|
|
395
424
|
if args.kube_completions:
|
|
396
425
|
logging.warning(
|
|
@@ -429,20 +458,7 @@ def build_config_managers_from_yaml(args, config_file_path):
|
|
|
429
458
|
return config_managers
|
|
430
459
|
|
|
431
460
|
|
|
432
|
-
def
|
|
433
|
-
"""Return results of raw query for ad hoc usage."""
|
|
434
|
-
mgr = state_manager.StateManager()
|
|
435
|
-
client = clients.get_data_client(mgr.get_connection_config(args.conn))
|
|
436
|
-
cursor = client.raw_sql(args.query)
|
|
437
|
-
res = cursor.fetchall()
|
|
438
|
-
try:
|
|
439
|
-
cursor.close()
|
|
440
|
-
except Exception:
|
|
441
|
-
pass
|
|
442
|
-
return res
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
def convert_config_to_yaml(args, config_managers):
|
|
461
|
+
def convert_config_to_yaml(args, config_managers: list):
|
|
446
462
|
"""Return dict objects formatted for yaml validations.
|
|
447
463
|
|
|
448
464
|
Args:
|
|
@@ -462,7 +478,7 @@ def convert_config_to_yaml(args, config_managers):
|
|
|
462
478
|
return yaml_config
|
|
463
479
|
|
|
464
480
|
|
|
465
|
-
def convert_config_to_json(config_managers) -> dict:
|
|
481
|
+
def convert_config_to_json(config_managers: list) -> dict:
|
|
466
482
|
"""Return dict objects formatted for json validations.
|
|
467
483
|
JSON configs correspond to ConfigManager objects and therefore can only correspond to
|
|
468
484
|
one table validation.
|
|
@@ -482,7 +498,7 @@ def convert_config_to_json(config_managers) -> dict:
|
|
|
482
498
|
return json_config
|
|
483
499
|
|
|
484
500
|
|
|
485
|
-
def run_validation(config_manager, dry_run=False, verbose=False):
|
|
501
|
+
def run_validation(config_manager: ConfigManager, dry_run=False, verbose=False):
|
|
486
502
|
"""Run a single validation.
|
|
487
503
|
|
|
488
504
|
Args:
|
|
@@ -490,11 +506,24 @@ def run_validation(config_manager, dry_run=False, verbose=False):
|
|
|
490
506
|
dry_run (bool): Print source and target SQL to stdout in lieu of validation.
|
|
491
507
|
verbose (bool): Validation setting to log queries run.
|
|
492
508
|
"""
|
|
509
|
+
# Only use cached connection for SQLAlchemy backends that manage reconnects for us.
|
|
510
|
+
source_client = (
|
|
511
|
+
config_manager.source_client
|
|
512
|
+
if clients.is_sqlalchemy_backend(config_manager.source_client)
|
|
513
|
+
else None
|
|
514
|
+
)
|
|
515
|
+
target_client = (
|
|
516
|
+
config_manager.target_client
|
|
517
|
+
if clients.is_sqlalchemy_backend(config_manager.target_client)
|
|
518
|
+
else None
|
|
519
|
+
)
|
|
493
520
|
with DataValidation(
|
|
494
521
|
config_manager.config,
|
|
495
522
|
validation_builder=None,
|
|
496
523
|
result_handler=None,
|
|
497
524
|
verbose=verbose,
|
|
525
|
+
source_client=source_client,
|
|
526
|
+
target_client=target_client,
|
|
498
527
|
) as validator:
|
|
499
528
|
|
|
500
529
|
if dry_run:
|
|
@@ -543,23 +572,7 @@ def run_validations(args, config_managers):
|
|
|
543
572
|
"""
|
|
544
573
|
# TODO(issue/31): Add parallel execution logic
|
|
545
574
|
for config_manager in config_managers:
|
|
546
|
-
|
|
547
|
-
logging.info(
|
|
548
|
-
"Currently running the validation for YAML file: %s",
|
|
549
|
-
config_manager.config[consts.CONFIG_FILE],
|
|
550
|
-
)
|
|
551
|
-
try:
|
|
552
|
-
run_validation(
|
|
553
|
-
config_manager, dry_run=args.dry_run, verbose=args.verbose
|
|
554
|
-
)
|
|
555
|
-
except Exception as e:
|
|
556
|
-
logging.error(
|
|
557
|
-
"Error %s occurred while running config file %s. Skipping it for now.",
|
|
558
|
-
str(e),
|
|
559
|
-
config_manager.config[consts.CONFIG_FILE],
|
|
560
|
-
)
|
|
561
|
-
else:
|
|
562
|
-
run_validation(config_manager, dry_run=args.dry_run, verbose=args.verbose)
|
|
575
|
+
run_validation(config_manager, dry_run=args.dry_run, verbose=args.verbose)
|
|
563
576
|
|
|
564
577
|
|
|
565
578
|
def store_yaml_config_file(args, config_managers):
|
|
@@ -674,7 +687,9 @@ def main():
|
|
|
674
687
|
elif args.command == "find-tables":
|
|
675
688
|
print(find_tables_using_string_matching(args))
|
|
676
689
|
elif args.command == "query":
|
|
677
|
-
|
|
690
|
+
raw_query.print_raw_query_output(
|
|
691
|
+
raw_query.run_raw_query_against_connection(args)
|
|
692
|
+
)
|
|
678
693
|
elif args.command == "validate":
|
|
679
694
|
validate(args)
|
|
680
695
|
elif args.command == "generate-table-partitions":
|