google-pso-data-validator 6.3.0__tar.gz → 7.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/PKG-INFO +21 -18
  2. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/README.md +20 -17
  3. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/__main__.py +77 -62
  4. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/cli_tools.py +84 -37
  5. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/clients.py +39 -12
  6. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/config_manager.py +147 -41
  7. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/consts.py +9 -0
  8. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/data_validation.py +33 -9
  9. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/exceptions.py +5 -1
  10. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/find_tables.py +13 -4
  11. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/gcs_helper.py +4 -1
  12. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/metadata.py +1 -3
  13. google_pso_data_validator-7.0.0/data_validation/raw_query.py +47 -0
  14. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/result_handlers/bigquery.py +13 -10
  15. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/secret_manager.py +1 -1
  16. google_pso_data_validator-7.0.0/data_validation/util.py +24 -0
  17. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/PKG-INFO +21 -18
  18. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/SOURCES.txt +4 -0
  19. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/setup.py +1 -1
  20. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_addon/api.py +29 -1
  21. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_addon/operations.py +12 -0
  22. google_pso_data_validator-7.0.0/third_party/ibis/ibis_biquery/api.py +15 -0
  23. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/__init__.py +16 -1
  24. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/api.py +2 -0
  25. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_db2/__init__.py +23 -1
  26. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_impala/api.py +12 -0
  27. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mssql/__init__.py +22 -1
  28. google_pso_data_validator-7.0.0/third_party/ibis/ibis_mysql/__init__.py +34 -0
  29. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_oracle/__init__.py +30 -2
  30. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_postgres/client.py +54 -19
  31. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_redshift/__init__.py +11 -2
  32. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_snowflake/datatypes.py +11 -0
  33. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_teradata/__init__.py +24 -4
  34. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/LICENSE +0 -0
  35. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/__init__.py +0 -0
  36. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/app.py +0 -0
  37. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/client_info.py +0 -0
  38. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/combiner.py +0 -0
  39. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/jellyfish_distance.py +0 -0
  40. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/partition_builder.py +0 -0
  41. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/query_builder/__init__.py +0 -0
  42. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/query_builder/partition_row_builder.py +0 -0
  43. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/query_builder/query_builder.py +0 -0
  44. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/query_builder/random_row_builder.py +0 -0
  45. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/result_handlers/__init__.py +0 -0
  46. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/result_handlers/text.py +0 -0
  47. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/schema_validation.py +0 -0
  48. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/state_manager.py +0 -0
  49. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/data_validation/validation_builder.py +0 -0
  50. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/dependency_links.txt +0 -0
  51. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/entry_points.txt +0 -0
  52. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/requires.txt +0 -0
  53. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/google_pso_data_validator.egg-info/top_level.txt +0 -0
  54. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/setup.cfg +0 -0
  55. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_addon/__init__.py +0 -0
  56. {google_pso_data_validator-6.3.0/third_party/ibis/ibis_impala → google_pso_data_validator-7.0.0/third_party/ibis/ibis_biquery}/__init__.py +0 -0
  57. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/client.py +0 -0
  58. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/compiler.py +0 -0
  59. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/datatypes.py +0 -0
  60. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/registry.py +0 -0
  61. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/tests/__init__.py +0 -0
  62. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/tests/conftest.py +0 -0
  63. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_cloud_spanner/to_pandas.py +0 -0
  64. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_db2/api.py +0 -0
  65. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_db2/compiler.py +0 -0
  66. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_db2/datatypes.py +0 -0
  67. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_db2/registry.py +0 -0
  68. {google_pso_data_validator-6.3.0/third_party/ibis/ibis_mysql → google_pso_data_validator-7.0.0/third_party/ibis/ibis_impala}/__init__.py +0 -0
  69. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mssql/api.py +0 -0
  70. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mssql/datatypes.py +0 -0
  71. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mssql/registry.py +0 -0
  72. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mysql/base_sql_compiler/__init__.py +0 -0
  73. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mysql/base_sql_compiler/select_builder.py +0 -0
  74. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_mysql/compiler.py +0 -0
  75. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_oracle/api.py +0 -0
  76. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_oracle/compiler.py +0 -0
  77. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_oracle/datatypes.py +0 -0
  78. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_oracle/registry.py +0 -0
  79. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_postgres/__init__.py +0 -0
  80. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_postgres/datatypes.py +0 -0
  81. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_redshift/api.py +0 -0
  82. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_redshift/compiler.py +0 -0
  83. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_snowflake/__init__.py +0 -0
  84. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_snowflake/api.py +0 -0
  85. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_teradata/api.py +0 -0
  86. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_teradata/compiler.py +0 -0
  87. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_teradata/datatypes.py +0 -0
  88. {google_pso_data_validator-6.3.0 → google_pso_data_validator-7.0.0}/third_party/ibis/ibis_teradata/registry.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: google-pso-data-validator
3
- Version: 6.3.0
3
+ Version: 7.0.0
4
4
  Summary: A package to enable easy data validation
5
5
  Author: PSO DVT Engineering team
6
6
  Author-email: data-validator-eng@google.com
@@ -154,7 +154,7 @@ data-validation
154
154
  [--std COLUMNS] Comma separated list of columns for stddev_samp or * for all numeric
155
155
  [--exclude-columns or -ec]
156
156
  Flag to indicate the list of columns provided should be excluded and not included.
157
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
157
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
158
158
  BigQuery destination for validation results. Defaults to stdout.
159
159
  See: *Validation Reports* section
160
160
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -197,17 +197,16 @@ In addition, please note that SHA256 is not a supported function on Teradata sys
197
197
  If you wish to perform this comparison on Teradata you will need to
198
198
  [deploy a UDF to perform the conversion](https://github.com/akuroda/teradata-udf-sha2/blob/master/src/sha256.c).)
199
199
 
200
- Below is the command syntax for row validations. In order to run row level
201
- validations you need to pass a `--primary-key` flag which defines what field(s)
202
- the validation will be compared on, as well as either the `--comparison-fields` flag
203
- or the `--hash` flag. See *Primary Keys* section
200
+ Below is the command syntax for row validations. In order to run row level validations we require
201
+ unique columns to join row sets, which are either inferred from the source/target table or provided
202
+ via the `--primary-keys` flag, and either the `--hash`, `--concat` or `--comparison-fields` flags.
203
+ See *Primary Keys* section.
204
204
 
205
205
  The `--comparison-fields` flag specifies the values (e.g. columns) whose raw values will be compared
206
206
  based on the primary key join. The `--hash` flag will run a checksum across specified columns in
207
207
  the table. This will include casting to string, sanitizing the data (ifnull, rtrim, upper), concatenating,
208
208
  and finally hashing the row.
209
209
 
210
-
211
210
  Under the hood, row validation uses
212
211
  [Calculated Fields](https://github.com/GoogleCloudPlatform/professional-services-data-validator#calculated-fields) to
213
212
  apply functions such as IFNULL() or RTRIM(). These can be edited in the YAML or JSON config file to customize your row validation.
@@ -229,16 +228,17 @@ data-validation
229
228
  Comma separated list of tables in the form schema.table=target_schema.target_table
230
229
  Target schema name and table name are optional.
231
230
  i.e 'bigquery-public-data.new_york_citibike.citibike_trips'
232
- --primary-keys or -pk PRIMARY_KEYS
233
- Comma separated list of columns to use as primary keys. See *Primary Keys* section
234
231
  --comparison-fields or -comp-fields FIELDS
235
232
  Comma separated list of columns to compare. Can either be a physical column or an alias
236
233
  See: *Calculated Fields* section for details
237
234
  --hash COLUMNS Comma separated list of columns to hash or * for all columns
238
235
  --concat COLUMNS Comma separated list of columns to concatenate or * for all columns (use if a common hash function is not available between databases)
236
+ [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
237
+ Comma separated list of primary key columns, when not specified the value will be inferred
238
+ from the source or target table if available. See *Primary Keys* section
239
239
  [--exclude-columns or -ec]
240
240
  Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
241
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
241
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
242
242
  BigQuery destination for validation results. Defaults to stdout.
243
243
  See: *Validation Reports* section
244
244
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -303,8 +303,6 @@ data-validation
303
303
  Either --tables-list or --source-query (or file) and --target-query (or file) must be provided
304
304
  --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
305
305
  File containing the target sql command. Supports GCS and local paths.
306
- --primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS
307
- Comma separated list of primary key columns 'col_a,col_b'. See *Primary Keys* section
308
306
  --comparison-fields or -comp-fields FIELDS
309
307
  Comma separated list of columns to compare. Can either be a physical column or an alias
310
308
  See: *Calculated Fields* section for details
@@ -318,7 +316,10 @@ data-validation
318
316
  --partition-num INT, -pn INT
319
317
  Number of partitions into which the table should be split, e.g. 1000 or 10000
320
318
  In case this value exceeds the row count of the source/target table, it will be decreased to max(source_row_count, target_row_count)
321
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
319
+ [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
320
+ Comma separated list of primary key columns, when not specified the value will be inferred
321
+ from the source or target table if available. See *Primary Keys* section
322
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
322
323
  BigQuery destination for validation results. Defaults to stdout.
323
324
  See: *Validation Reports* section
324
325
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -364,7 +365,7 @@ data-validation
364
365
  Comma separated list of tables in the form schema.table=target_schema.target_table. Or shorthand schema.* for all tables.
365
366
  Target schema name and table name are optional.
366
367
  e.g.: 'bigquery-public-data.new_york_citibike.citibike_trips'
367
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
368
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
368
369
  BigQuery destination for validation results. Defaults to stdout.
369
370
  See: *Validation Reports* section
370
371
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -424,7 +425,7 @@ data-validation
424
425
  [--std COLUMNS] Comma separated list of columns for stddev_samp or * for all numeric
425
426
  [--exclude-columns or -ec]
426
427
  Flag to indicate the list of columns provided should be excluded and not included.
427
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
428
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
428
429
  BigQuery destination for validation results. Defaults to stdout.
429
430
  See: *Validation Reports* section
430
431
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -489,11 +490,11 @@ data-validation
489
490
  --hash '*' '*' to hash all columns.
490
491
  --concat COLUMNS Comma separated list of columns to concatenate or * for all columns
491
492
  (use if a common hash function is not available between databases)
492
- --primary-key or -pk JOIN_KEY
493
- Common column between source and target tables for join
493
+ [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
494
+ Common column between source and target queries for join
494
495
  [--exclude-columns or -ec]
495
496
  Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
496
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
497
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
497
498
  BigQuery destination for validation results. Defaults to stdout.
498
499
  See: *Validation Reports* section
499
500
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -720,6 +721,8 @@ In many cases, validations (e.g. count, min, max etc) produce one row per table.
720
721
  and target table is to compare the value for each column in the source with the value of the column in the target.
721
722
  `grouped-columns` validation and `validate row` produce multiple rows per table. Data Validation Tool needs one or more columns to uniquely identify each row so the source and target can be compared. Data Validation Tool refers to these columns as primary keys. These do not need to be primary keys in the table. The only requirement is that the keys uniquely identify the row in the results.
722
723
 
724
+ These columns are inferred, where possible, from the source/target table or can be provided via the `--primary-keys` flag.
725
+
723
726
  ### Grouped Columns
724
727
 
725
728
  Grouped Columns contain the fields you want your aggregations to be broken out
@@ -113,7 +113,7 @@ data-validation
113
113
  [--std COLUMNS] Comma separated list of columns for stddev_samp or * for all numeric
114
114
  [--exclude-columns or -ec]
115
115
  Flag to indicate the list of columns provided should be excluded and not included.
116
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
116
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
117
117
  BigQuery destination for validation results. Defaults to stdout.
118
118
  See: *Validation Reports* section
119
119
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -156,17 +156,16 @@ In addition, please note that SHA256 is not a supported function on Teradata sys
156
156
  If you wish to perform this comparison on Teradata you will need to
157
157
  [deploy a UDF to perform the conversion](https://github.com/akuroda/teradata-udf-sha2/blob/master/src/sha256.c).)
158
158
 
159
- Below is the command syntax for row validations. In order to run row level
160
- validations you need to pass a `--primary-key` flag which defines what field(s)
161
- the validation will be compared on, as well as either the `--comparison-fields` flag
162
- or the `--hash` flag. See *Primary Keys* section
159
+ Below is the command syntax for row validations. In order to run row level validations we require
160
+ unique columns to join row sets, which are either inferred from the source/target table or provided
161
+ via the `--primary-keys` flag, and either the `--hash`, `--concat` or `--comparison-fields` flags.
162
+ See *Primary Keys* section.
163
163
 
164
164
  The `--comparison-fields` flag specifies the values (e.g. columns) whose raw values will be compared
165
165
  based on the primary key join. The `--hash` flag will run a checksum across specified columns in
166
166
  the table. This will include casting to string, sanitizing the data (ifnull, rtrim, upper), concatenating,
167
167
  and finally hashing the row.
168
168
 
169
-
170
169
  Under the hood, row validation uses
171
170
  [Calculated Fields](https://github.com/GoogleCloudPlatform/professional-services-data-validator#calculated-fields) to
172
171
  apply functions such as IFNULL() or RTRIM(). These can be edited in the YAML or JSON config file to customize your row validation.
@@ -188,16 +187,17 @@ data-validation
188
187
  Comma separated list of tables in the form schema.table=target_schema.target_table
189
188
  Target schema name and table name are optional.
190
189
  i.e 'bigquery-public-data.new_york_citibike.citibike_trips'
191
- --primary-keys or -pk PRIMARY_KEYS
192
- Comma separated list of columns to use as primary keys. See *Primary Keys* section
193
190
  --comparison-fields or -comp-fields FIELDS
194
191
  Comma separated list of columns to compare. Can either be a physical column or an alias
195
192
  See: *Calculated Fields* section for details
196
193
  --hash COLUMNS Comma separated list of columns to hash or * for all columns
197
194
  --concat COLUMNS Comma separated list of columns to concatenate or * for all columns (use if a common hash function is not available between databases)
195
+ [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
196
+ Comma separated list of primary key columns, when not specified the value will be inferred
197
+ from the source or target table if available. See *Primary Keys* section
198
198
  [--exclude-columns or -ec]
199
199
  Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
200
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
200
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
201
201
  BigQuery destination for validation results. Defaults to stdout.
202
202
  See: *Validation Reports* section
203
203
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -262,8 +262,6 @@ data-validation
262
262
  Either --tables-list or --source-query (or file) and --target-query (or file) must be provided
263
263
  --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
264
264
  File containing the target sql command. Supports GCS and local paths.
265
- --primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS
266
- Comma separated list of primary key columns 'col_a,col_b'. See *Primary Keys* section
267
265
  --comparison-fields or -comp-fields FIELDS
268
266
  Comma separated list of columns to compare. Can either be a physical column or an alias
269
267
  See: *Calculated Fields* section for details
@@ -277,7 +275,10 @@ data-validation
277
275
  --partition-num INT, -pn INT
278
276
  Number of partitions into which the table should be split, e.g. 1000 or 10000
279
277
  In case this value exceeds the row count of the source/target table, it will be decreased to max(source_row_count, target_row_count)
280
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
278
+ [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
279
+ Comma separated list of primary key columns, when not specified the value will be inferred
280
+ from the source or target table if available. See *Primary Keys* section
281
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
281
282
  BigQuery destination for validation results. Defaults to stdout.
282
283
  See: *Validation Reports* section
283
284
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -323,7 +324,7 @@ data-validation
323
324
  Comma separated list of tables in the form schema.table=target_schema.target_table. Or shorthand schema.* for all tables.
324
325
  Target schema name and table name are optional.
325
326
  e.g.: 'bigquery-public-data.new_york_citibike.citibike_trips'
326
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
327
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
327
328
  BigQuery destination for validation results. Defaults to stdout.
328
329
  See: *Validation Reports* section
329
330
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -383,7 +384,7 @@ data-validation
383
384
  [--std COLUMNS] Comma separated list of columns for stddev_samp or * for all numeric
384
385
  [--exclude-columns or -ec]
385
386
  Flag to indicate the list of columns provided should be excluded and not included.
386
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
387
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
387
388
  BigQuery destination for validation results. Defaults to stdout.
388
389
  See: *Validation Reports* section
389
390
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -448,11 +449,11 @@ data-validation
448
449
  --hash '*' '*' to hash all columns.
449
450
  --concat COLUMNS Comma separated list of columns to concatenate or * for all columns
450
451
  (use if a common hash function is not available between databases)
451
- --primary-key or -pk JOIN_KEY
452
- Common column between source and target tables for join
452
+ [--primary-keys PRIMARY_KEYS, -pk PRIMARY_KEYS]
453
+ Common column between source and target queries for join
453
454
  [--exclude-columns or -ec]
454
455
  Flag to indicate the list of columns provided should be excluded from hash or concat instead of included.
455
- [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
456
+ [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE or CONNECTION_NAME.DATASET.TABLE]
456
457
  BigQuery destination for validation results. Defaults to stdout.
457
458
  See: *Validation Reports* section
458
459
  [--service-account or -sa PATH_TO_SA_KEY]
@@ -679,6 +680,8 @@ In many cases, validations (e.g. count, min, max etc) produce one row per table.
679
680
  and target table is to compare the value for each column in the source with the value of the column in the target.
680
681
  `grouped-columns` validation and `validate row` produce multiple rows per table. Data Validation Tool needs one or more columns to uniquely identify each row so the source and target can be compared. Data Validation Tool refers to these columns as primary keys. These do not need to be primary keys in the table. The only requirement is that the keys uniquely identify the row in the results.
681
682
 
683
+ These columns are inferred, where possible, from the source/target table or can be provided via the `--primary-keys` flag.
684
+
682
685
  ### Grouped Columns
683
686
 
684
687
  Grouped Columns contain the fields you want your aggregations to be broken out
@@ -24,7 +24,10 @@ from data_validation import (
24
24
  cli_tools,
25
25
  clients,
26
26
  consts,
27
+ exceptions,
28
+ raw_query,
27
29
  state_manager,
30
+ util,
28
31
  )
29
32
  from data_validation.config_manager import ConfigManager
30
33
  from data_validation.data_validation import DataValidation
@@ -89,6 +92,7 @@ def get_aggregate_config(args, config_manager: ConfigManager):
89
92
  "!int64",
90
93
  "!decimal",
91
94
  ]
95
+ uuid_types = ["uuid", "!uuid"]
92
96
 
93
97
  if args.wildcard_include_string_len:
94
98
  supported_data_types.extend(["string", "!string"])
@@ -127,7 +131,7 @@ def get_aggregate_config(args, config_manager: ConfigManager):
127
131
  "min",
128
132
  col_args,
129
133
  args.exclude_columns,
130
- supported_data_types,
134
+ supported_data_types + uuid_types,
131
135
  cast_to_bigint=cast_to_bigint,
132
136
  )
133
137
  if args.max:
@@ -136,7 +140,7 @@ def get_aggregate_config(args, config_manager: ConfigManager):
136
140
  "max",
137
141
  col_args,
138
142
  args.exclude_columns,
139
- supported_data_types,
143
+ supported_data_types + uuid_types,
140
144
  cast_to_bigint=cast_to_bigint,
141
145
  )
142
146
  if args.bit_xor:
@@ -217,22 +221,20 @@ def _get_calculated_config(args, config_manager: ConfigManager) -> List[dict]:
217
221
  return calculated_configs
218
222
 
219
223
 
220
- def _get_comparison_config(args, config_manager: ConfigManager) -> List[dict]:
224
+ def _get_comparison_config(
225
+ args, config_manager: ConfigManager, primary_keys: list
226
+ ) -> List[dict]:
221
227
  col_list = (
222
228
  None
223
- if config_manager.comparison_fields == "*"
224
- else cli_tools.get_arg_list(config_manager.comparison_fields)
229
+ if args.comparison_fields == "*"
230
+ else cli_tools.get_arg_list(args.comparison_fields)
225
231
  )
226
232
  comparison_fields = config_manager.build_comp_fields(
227
233
  col_list,
228
234
  args.exclude_columns,
229
235
  )
230
236
  # We can't have the PK columns in the comparison SQL twice therefore filter them out here if included.
231
- comparison_fields = [
232
- _
233
- for _ in comparison_fields
234
- if _ not in cli_tools.get_arg_list(args.primary_keys.casefold())
235
- ]
237
+ comparison_fields = [_ for _ in comparison_fields if _ not in primary_keys]
236
238
 
237
239
  # As per #1190, add rstrip for Teradata string comparison fields
238
240
  if (
@@ -312,18 +314,25 @@ def build_config_from_args(args: Namespace, config_manager: ConfigManager):
312
314
  _get_calculated_config(args, config_manager)
313
315
  )
314
316
 
315
- # Append Comparison fields
316
- if args.comparison_fields:
317
- config_manager.append_comparison_fields(
318
- _get_comparison_config(args, config_manager)
319
- )
320
-
321
317
  # Append primary_keys
322
318
  primary_keys = cli_tools.get_arg_list(args.primary_keys)
319
+ if not primary_keys and config_manager.validation_type != consts.CUSTOM_QUERY:
320
+ primary_keys = config_manager.auto_list_primary_keys()
321
+ if not primary_keys:
322
+ raise ValueError(
323
+ "No primary keys were provided and neither the source or target tables have primary keys. Please include --primary-keys argument"
324
+ )
325
+ primary_keys = [_.casefold() for _ in primary_keys]
323
326
  config_manager.append_primary_keys(
324
327
  config_manager.build_column_configs(primary_keys)
325
328
  )
326
329
 
330
+ # Append Comparison fields
331
+ if args.comparison_fields:
332
+ config_manager.append_comparison_fields(
333
+ _get_comparison_config(args, config_manager, primary_keys)
334
+ )
335
+
327
336
  return config_manager
328
337
 
329
338
 
@@ -331,22 +340,25 @@ def build_config_managers_from_args(
331
340
  args: Namespace, validate_cmd: str = None
332
341
  ) -> List[ConfigManager]:
333
342
  """Return a list of config managers ready to execute."""
334
- configs = []
335
343
 
336
- # Get pre build configs to build ConfigManager objects
337
- pre_build_configs_list = cli_tools.get_pre_build_configs(args, validate_cmd)
344
+ def _build_configs():
345
+ configs = []
346
+
347
+ # Get pre build configs to build ConfigManager objects
348
+ pre_build_configs_list = cli_tools.get_pre_build_configs(args, validate_cmd)
338
349
 
339
- # Build a list of ConfigManager objects
340
- for pre_build_configs in pre_build_configs_list:
341
- config_manager = ConfigManager.build_config_manager(**pre_build_configs)
350
+ # Build a list of ConfigManager objects
351
+ for pre_build_configs in pre_build_configs_list:
352
+ config_manager = ConfigManager.build_config_manager(**pre_build_configs)
342
353
 
343
- # Append post build configs to ConfigManager object
344
- config_manager = build_config_from_args(args, config_manager)
354
+ # Append post build configs to ConfigManager object
355
+ config_manager = build_config_from_args(args, config_manager)
345
356
 
346
- # Append ConfigManager object to configs list
347
- configs.append(config_manager)
357
+ # Append ConfigManager object to configs list
358
+ configs.append(config_manager)
359
+ return configs
348
360
 
349
- return configs
361
+ return util.timed_call("Build config", _build_configs)
350
362
 
351
363
 
352
364
  def config_runner(args):
@@ -388,9 +400,26 @@ def config_runner(args):
388
400
  )
389
401
  config_file_names = cli_tools.list_validations(config_dir=args.config_dir)
390
402
  config_managers = []
403
+ errors = False
391
404
  for file in config_file_names:
392
405
  config_managers = build_config_managers_from_yaml(args, file)
393
- run_validations(args, config_managers)
406
+ try:
407
+ logging.info(
408
+ "Currently running the validation for YAML file: %s",
409
+ file,
410
+ )
411
+ run_validations(args, config_managers)
412
+ except Exception as e:
413
+ errors = True
414
+ logging.error(
415
+ "Error '%s' occurred while running config file %s. Skipping it for now.",
416
+ str(e),
417
+ file,
418
+ )
419
+ if errors:
420
+ raise exceptions.ValidationException(
421
+ "Some of the validations raised an exception"
422
+ )
394
423
  else:
395
424
  if args.kube_completions:
396
425
  logging.warning(
@@ -429,20 +458,7 @@ def build_config_managers_from_yaml(args, config_file_path):
429
458
  return config_managers
430
459
 
431
460
 
432
- def run_raw_query_against_connection(args):
433
- """Return results of raw query for ad hoc usage."""
434
- mgr = state_manager.StateManager()
435
- client = clients.get_data_client(mgr.get_connection_config(args.conn))
436
- cursor = client.raw_sql(args.query)
437
- res = cursor.fetchall()
438
- try:
439
- cursor.close()
440
- except Exception:
441
- pass
442
- return res
443
-
444
-
445
- def convert_config_to_yaml(args, config_managers):
461
+ def convert_config_to_yaml(args, config_managers: list):
446
462
  """Return dict objects formatted for yaml validations.
447
463
 
448
464
  Args:
@@ -462,7 +478,7 @@ def convert_config_to_yaml(args, config_managers):
462
478
  return yaml_config
463
479
 
464
480
 
465
- def convert_config_to_json(config_managers) -> dict:
481
+ def convert_config_to_json(config_managers: list) -> dict:
466
482
  """Return dict objects formatted for json validations.
467
483
  JSON configs correspond to ConfigManager objects and therefore can only correspond to
468
484
  one table validation.
@@ -482,7 +498,7 @@ def convert_config_to_json(config_managers) -> dict:
482
498
  return json_config
483
499
 
484
500
 
485
- def run_validation(config_manager, dry_run=False, verbose=False):
501
+ def run_validation(config_manager: ConfigManager, dry_run=False, verbose=False):
486
502
  """Run a single validation.
487
503
 
488
504
  Args:
@@ -490,11 +506,24 @@ def run_validation(config_manager, dry_run=False, verbose=False):
490
506
  dry_run (bool): Print source and target SQL to stdout in lieu of validation.
491
507
  verbose (bool): Validation setting to log queries run.
492
508
  """
509
+ # Only use cached connection for SQLAlchemy backends that manage reconnects for us.
510
+ source_client = (
511
+ config_manager.source_client
512
+ if clients.is_sqlalchemy_backend(config_manager.source_client)
513
+ else None
514
+ )
515
+ target_client = (
516
+ config_manager.target_client
517
+ if clients.is_sqlalchemy_backend(config_manager.target_client)
518
+ else None
519
+ )
493
520
  with DataValidation(
494
521
  config_manager.config,
495
522
  validation_builder=None,
496
523
  result_handler=None,
497
524
  verbose=verbose,
525
+ source_client=source_client,
526
+ target_client=target_client,
498
527
  ) as validator:
499
528
 
500
529
  if dry_run:
@@ -543,23 +572,7 @@ def run_validations(args, config_managers):
543
572
  """
544
573
  # TODO(issue/31): Add parallel execution logic
545
574
  for config_manager in config_managers:
546
- if config_manager.config and consts.CONFIG_FILE in config_manager.config:
547
- logging.info(
548
- "Currently running the validation for YAML file: %s",
549
- config_manager.config[consts.CONFIG_FILE],
550
- )
551
- try:
552
- run_validation(
553
- config_manager, dry_run=args.dry_run, verbose=args.verbose
554
- )
555
- except Exception as e:
556
- logging.error(
557
- "Error %s occurred while running config file %s. Skipping it for now.",
558
- str(e),
559
- config_manager.config[consts.CONFIG_FILE],
560
- )
561
- else:
562
- run_validation(config_manager, dry_run=args.dry_run, verbose=args.verbose)
575
+ run_validation(config_manager, dry_run=args.dry_run, verbose=args.verbose)
563
576
 
564
577
 
565
578
  def store_yaml_config_file(args, config_managers):
@@ -674,7 +687,9 @@ def main():
674
687
  elif args.command == "find-tables":
675
688
  print(find_tables_using_string_matching(args))
676
689
  elif args.command == "query":
677
- print(run_raw_query_against_connection(args))
690
+ raw_query.print_raw_query_output(
691
+ raw_query.run_raw_query_against_connection(args)
692
+ )
678
693
  elif args.command == "validate":
679
694
  validate(args)
680
695
  elif args.command == "generate-table-partitions":