google-pso-data-validator 5.0.0__tar.gz → 5.1.0__tar.gz

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/PKG-INFO

@@ -1,11 +1,10 @@
 Metadata-Version: 2.1
 Name: google-pso-data-validator
-Version: 5.0.0
+Version: 5.1.0
 Summary: A package to enable easy data validation
-Home-page: https://github.com/pypa/sampleproject
-Author: Dylan Hercher
-Author-email: dhercher@google.com
-Classifier: Development Status :: 3 - Alpha
+Author: PSO DVT Engineering team
+Author-email: data-validator-eng@google.com
+Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.8
@@ -16,7 +15,7 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: attrs==23.1.0
-Requires-Dist: grpcio==1.53.0
+Requires-Dist: grpcio==1.53.2
 Requires-Dist: lazy-object-proxy==1.9.0
 Requires-Dist: marshmallow==3.19.0
 Requires-Dist: fsspec>=2022.8.2
@@ -24,7 +23,7 @@ Requires-Dist: google-api-python-client==2.91.0
 Requires-Dist: ibis-framework==5.1.0
 Requires-Dist: impyla==0.18.0
 Requires-Dist: SQLAlchemy==1.4.49
-Requires-Dist: PyMySQL==1.1.0
+Requires-Dist: PyMySQL==1.1.1
 Requires-Dist: psycopg2-binary==2.9.6
 Requires-Dist: PyYAML==6.0
 Requires-Dist: pandas==1.5.3
@@ -35,12 +34,11 @@ Requires-Dist: google-cloud-bigquery==3.11.3
 Requires-Dist: google-cloud-bigquery-storage==2.20.0
 Requires-Dist: google-cloud-spanner==3.36.0
 Requires-Dist: google-cloud-storage==2.10.0
-Requires-Dist: setuptools>=34.0.0
 Requires-Dist: jellyfish==1.0.0
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: Flask==2.3.2
 Requires-Dist: parsy==2.1
-Requires-Dist: google-cloud-secret-manager<=2.15.0
+Requires-Dist: google-cloud-secret-manager>=2.16.0
 Provides-Extra: apache-airflow
 Requires-Dist: 1.10.11; extra == "apache-airflow"
 Provides-Extra: pyspark
@@ -256,6 +254,8 @@ data-validation (--verbose or -v) (--log-level or -ll) validate row
                         Row batch size used for random row filters (default 10,000).
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
+  [--trim-string-pks, -tsp]
+                        Trims string based primary key values, intended for use when one engine uses padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n)).
 ```
 #### Generate Table Partitions for Large Table Row Validations
 
@@ -357,12 +357,12 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
                         Source sql query
                         Either --source-query or --source-query-file must be provided
   --source-query-file  SOURCE_QUERY_FILE, -sqf SOURCE_QUERY_FILE
-                        File containing the source sql commands
+                        File containing the source sql command. Supports GCS and local paths.
   --target-query TARGET_QUERY, -tq TARGET_QUERY
                         Target sql query
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
-                        File containing the target sql commands
+                        File containing the target sql command. Supports GCS and local paths.
   [--count COLUMNS]     Comma separated list of columns for count or * for all columns
   [--sum COLUMNS]       Comma separated list of columns for sum or * for all numeric
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric
@@ -419,12 +419,12 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
                         Source sql query
                         Either --source-query or --source-query-file must be provided
   --source-query-file SOURCE_QUERY_FILE, -sqf SOURCE_QUERY_FILE
-                        File containing the source sql commands
+                        File containing the source sql command. Supports GCS and local paths.
   --target-query TARGET_QUERY, -tq TARGET_QUERY
                         Target sql query
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
-                        File containing the target sql commands
+                        File containing the target sql command. Supports GCS and local paths.
   --comparison-fields or -comp-fields FIELDS
                         Comma separated list of columns to compare. Can either be a physical column or an alias
                         See: *Calculated Fields* section for details
@@ -522,7 +522,7 @@ View the complete YAML file for a Grouped Column validation on the
 You can scale DVT for large table validations by running the tool in a distributed manner. To optimize the validation speed for large tables, you can use GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Distributed Jobs](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/internal/distributed_jobs.md) for a detailed overview.
 
 
-We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started. 
+We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started.
 
 When running DVT in a distributed fashion, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (i.e. a `--config-file` is provided instead), a warning is issued.
 

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/README.md

@@ -208,6 +208,8 @@ data-validation (--verbose or -v) (--log-level or -ll) validate row
                         Row batch size used for random row filters (default 10,000).
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
+  [--trim-string-pks, -tsp]
+                        Trims string based primary key values, intended for use when one engine uses padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n)).
 ```
 #### Generate Table Partitions for Large Table Row Validations
 
@@ -309,12 +311,12 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
                         Source sql query
                         Either --source-query or --source-query-file must be provided
   --source-query-file  SOURCE_QUERY_FILE, -sqf SOURCE_QUERY_FILE
-                        File containing the source sql commands
+                        File containing the source sql command. Supports GCS and local paths.
   --target-query TARGET_QUERY, -tq TARGET_QUERY
                         Target sql query
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
-                        File containing the target sql commands
+                        File containing the target sql command. Supports GCS and local paths.
   [--count COLUMNS]     Comma separated list of columns for count or * for all columns
   [--sum COLUMNS]       Comma separated list of columns for sum or * for all numeric
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric
@@ -371,12 +373,12 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
                         Source sql query
                         Either --source-query or --source-query-file must be provided
   --source-query-file SOURCE_QUERY_FILE, -sqf SOURCE_QUERY_FILE
-                        File containing the source sql commands
+                        File containing the source sql command. Supports GCS and local paths.
   --target-query TARGET_QUERY, -tq TARGET_QUERY
                         Target sql query
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
-                        File containing the target sql commands
+                        File containing the target sql command. Supports GCS and local paths.
   --comparison-fields or -comp-fields FIELDS
                         Comma separated list of columns to compare. Can either be a physical column or an alias
                         See: *Calculated Fields* section for details
@@ -474,7 +476,7 @@ View the complete YAML file for a Grouped Column validation on the
 You can scale DVT for large table validations by running the tool in a distributed manner. To optimize the validation speed for large tables, you can use GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Distributed Jobs](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/internal/distributed_jobs.md) for a detailed overview.
 
 
-We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started. 
+We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started.
 
 When running DVT in a distributed fashion, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (i.e. a `--config-file` is provided instead), a warning is issued.
 

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/data_validation/cli_tools.py

@@ -484,6 +484,15 @@ def _configure_row_parser(row_parser):
         "-rbs",
         help="Row batch size used for random row filters (default 10,000).",
     )
+    optional_arguments.add_argument(
+        "--trim-string-pks",
+        "-tsp",
+        action="store_true",
+        help=(
+            "Trims string based primary key values, intended for use when one engine uses "
+            "padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n))."
+        ),
+    )
 
     # Group required arguments
     required_arguments = row_parser.add_argument_group("required arguments")
@@ -954,6 +963,18 @@ def _add_common_arguments(optional_arguments, required_arguments):
     )
 
 
+def _check_no_partitions(value: str) -> int:
+    """Check that number of partitions is between [2-10,000]
+    Using function to validate rather than choices as error message prints all choices.
+    """
+    if value.isdigit() and 2 <= int(value) <= 10000:
+        return int(value)
+    else:
+        raise argparse.ArgumentTypeError(
+            f"{value} is not valid for number of partitions, use number in range 2 to 10000"
+        )
+
+
 def _add_common_partition_arguments(optional_arguments, required_arguments):
     """Add all arguments common to get-partition command"""
 
@@ -978,10 +999,9 @@ def _add_common_partition_arguments(optional_arguments, required_arguments):
         "--partition-num",
         "-pn",
         required=True,
-        help="Number of partitions/config files to generate",
-        type=int,
-        choices=range(1, 1001),
-        metavar="[1-1000]",
+        help="Number of partitions/config files to generate, a number from 2 to 10,000",
+        type=_check_no_partitions,
+        metavar="[2-10000]",
     )
 
     # Optional arguments
@@ -1387,6 +1407,7 @@ def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
             "result_handler_config": result_handler_config,
             "filter_config": filter_config,
             "filter_status": filter_status,
+            "trim_string_pks": getattr(args, "trim_string_pks", False),
             "verbose": args.verbose,
         }
         pre_build_configs_list.append(pre_build_configs)

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/data_validation/clients.py

@@ -274,6 +274,23 @@ def get_max_column_length(client):
     return 128
 
 
+def get_max_in_list_size(client, in_list_over_expressions=False):
+    if client.name == "snowflake":
+        if in_list_over_expressions:
+            # This is a workaround for Snowflake limitation:
+            #   SQL compilation error: In-list contains more than 50 non-constant values
+            # getattr(..., "cast") expression above is looking for lists where the contents are casts and not simple literals.
+            return 50
+        else:
+            return 16000
+    elif is_oracle_client(client):
+        # This is a workaround for Oracle limitation:
+        #   ORA-01795: maximum number of expressions in a list is 1000
+        return 1000
+    else:
+        return None
+
+
 CLIENT_LOOKUP = {
     "BigQuery": get_bigquery_client,
     "Impala": impala_connect,

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/data_validation/config_manager.py

@@ -112,6 +112,10 @@ class ConfigManager(object):
         """Return number of random rows or None."""
         return self.random_row_batch_size() if self.use_random_rows() else None
 
+    def trim_string_pks(self):
+        """Return if the validation should trim string primary keys."""
+        return self._config.get(consts.CONFIG_TRIM_STRING_PKS) or False
+
     def process_in_memory(self):
         """Return whether to process in memory or on a remote platform."""
         return True
@@ -459,6 +463,7 @@ class ConfigManager(object):
         result_handler_config=None,
         filter_config=None,
         filter_status=None,
+        trim_string_pks=None,
         verbose=False,
     ):
         if isinstance(filter_config, dict):
@@ -487,6 +492,7 @@ class ConfigManager(object):
             consts.CONFIG_USE_RANDOM_ROWS: use_random_rows,
             consts.CONFIG_RANDOM_ROW_BATCH_SIZE: random_row_batch_size,
             consts.CONFIG_FILTER_STATUS: filter_status,
+            consts.CONFIG_TRIM_STRING_PKS: trim_string_pks,
         }
 
         return ConfigManager(
@@ -1034,8 +1040,7 @@ class ConfigManager(object):
         """Return query from input file"""
         query = ""
         try:
-            file = open(filename, "r")
-            query = file.read()
+            query = gcs_helper.read_file(filename, download_as_text=True)
             query = query.rstrip(";\n")
         except IOError:
             logging.error("Cannot read query file: ", filename)
@@ -1045,7 +1050,6 @@ class ConfigManager(object):
                 "Expected file with sql query, got empty file or file with white spaces. "
                 f"input file: {filename}"
             )
-        file.close()
         return query
 
     def get_query_from_inline(self, inline_query):

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/data_validation/consts.py

@@ -45,6 +45,7 @@ CONFIG_CALCULATED_TARGET_COLUMNS = "target_calculated_columns"
 CONFIG_USE_RANDOM_ROWS = "use_random_rows"
 CONFIG_RANDOM_ROW_BATCH_SIZE = "random_row_batch_size"
 CONFIG_PRIMARY_KEYS = "primary_keys"
+CONFIG_TRIM_STRING_PKS = "trim_string_pks"
 CONFIG_SOURCE_COLUMN = "source_column"
 CONFIG_TARGET_COLUMN = "target_column"
 CONFIG_THRESHOLD = "threshold"

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/data_validation/data_validation.py

@@ -142,6 +142,9 @@ class DataValidation(object):
                 **{source_pk_column: query[source_pk_column].cast("string")}
             )
 
+        if self.config_manager.trim_string_pks():
+            query = query.mutate(**{source_pk_column: query[source_pk_column].rstrip()})
+
         random_rows = self.config_manager.source_client.execute(query)
         if len(random_rows) == 0:
             return

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/data_validation/gcs_helper.py

@@ -50,12 +50,12 @@ def _get_gcs_file_path(gcs_file_path: str) -> str:
     return "".join(gcs_file_path[5:].split("/", 1)[1:])
 
 
-def _read_gcs_file(file_path: str) -> bytes:
+def _read_gcs_file(file_path: str, download_as_text: bool = False):
     gcs_bucket = get_gcs_bucket(file_path)
     blob = gcs_bucket.blob(_get_gcs_file_path(file_path))
     if not blob:
         raise ValueError(f"Invalid Cloud Storage Path: {file_path}")
-    return blob.download_as_bytes()
+    return blob.download_as_text() if download_as_text else blob.download_as_bytes()
 
 
 def _write_gcs_file(file_path: str, data: str):
@@ -64,9 +64,9 @@ def _write_gcs_file(file_path: str, data: str):
     blob.upload_from_string(data)
 
 
-def read_file(file_path: str):
+def read_file(file_path: str, download_as_text: bool = False):
     if _is_gcs_path(file_path):
-        return _read_gcs_file(file_path)
+        return _read_gcs_file(file_path, download_as_text)
     else:
         with open(file_path, "r") as f:
             return f.read()

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/data_validation/partition_builder.py

@@ -32,7 +32,6 @@ class PartitionBuilder:
         self.table_count = len(config_managers)
         self.args = args
         self.config_dir = self._get_arg_config_dir()
-        self.primary_keys = self._get_primary_keys()
 
     def _get_arg_config_dir(self) -> str:
         """Return String yaml config folder path."""
@@ -41,10 +40,6 @@ class PartitionBuilder:
 
         return self.args.config_dir
 
-    def _get_primary_keys(self) -> str:
-        """Return the Primary Keys"""
-        return cli_tools.get_arg_list(self.args.primary_keys)
-
     def _get_yaml_from_config(self, config_manager: ConfigManager) -> Dict:
         """Return dict objects formatted for yaml validations.
 
@@ -106,8 +101,13 @@ class PartitionBuilder:
         for config_manager in self.config_managers:  # For each pair of tables
             validation_builder = ValidationBuilder(config_manager)
 
+            source_pks, target_pks = [], []
+            for pk in config_manager.primary_keys:
+                source_pks.append(pk["source_column"])
+                target_pks.append(pk["target_column"])
+
             source_partition_row_builder = PartitionRowBuilder(
-                self.primary_keys,
+                source_pks,
                 config_manager.source_client,
                 config_manager.source_schema,
                 config_manager.source_table,
@@ -115,7 +115,7 @@ class PartitionBuilder:
             )
             source_table = source_partition_row_builder.query
             target_partition_row_builder = PartitionRowBuilder(
-                self.primary_keys,
+                target_pks,
                 config_manager.target_client,
                 config_manager.target_schema,
                 config_manager.target_table,
@@ -150,9 +150,9 @@ class PartitionBuilder:
             # First we number each row in the source table. Using row_number instead of ntile since it is
             # available on all platforms (Teradata does not support NTILE). For our purposes, it is likely
             # more efficient
-            window1 = ibis.window(order_by=self.primary_keys)
+            window1 = ibis.window(order_by=source_pks)
             row_number = (ibis.row_number().over(window1) + 1).name(consts.DVT_POS_COL)
-            dvt_keys = self.primary_keys.copy()
+            dvt_keys = source_pks.copy()
             dvt_keys.append(row_number)
             rownum_table = source_table.select(dvt_keys)
             # Rownum table is just the primary key columns in the source table along with
@@ -182,7 +182,7 @@ class PartitionBuilder:
                 )
                 > 0
             )
-            first_keys_table = rownum_table[cond].order_by(self.primary_keys)
+            first_keys_table = rownum_table[cond].order_by(source_pks)
 
             # Up until this point, we have built the table expression, have not executed the query yet.
             # The query is now executed to find the first element of each partition
@@ -219,13 +219,13 @@ class PartitionBuilder:
 
             filter_source_clause = less_than_value(
                 source_table,
-                self.primary_keys,
-                first_elements[1, : len(self.primary_keys)],
+                source_pks,
+                first_elements[1, : len(source_pks)],
             )
             filter_target_clause = less_than_value(
                 target_table,
-                self.primary_keys,
-                first_elements[1, : len(self.primary_keys)],
+                target_pks,
+                first_elements[1, : len(target_pks)],
             )
             source_where_list.append(
                 self._extract_where(
@@ -243,21 +243,21 @@ class PartitionBuilder:
             for i in range(1, first_elements.shape[0] - 1):
                 filter_source_clause = geq_value(
                     source_table,
-                    self.primary_keys,
-                    first_elements[i, : len(self.primary_keys)],
+                    source_pks,
+                    first_elements[i, : len(source_pks)],
                 ) & less_than_value(
                     source_table,
-                    self.primary_keys,
-                    first_elements[i + 1, : len(self.primary_keys)],
+                    source_pks,
+                    first_elements[i + 1, : len(source_pks)],
                 )
                 filter_target_clause = geq_value(
                     target_table,
-                    self.primary_keys,
-                    first_elements[i, : len(self.primary_keys)],
+                    target_pks,
+                    first_elements[i, : len(target_pks)],
                 ) & less_than_value(
                     target_table,
-                    self.primary_keys,
-                    first_elements[i + 1, : len(self.primary_keys)],
+                    target_pks,
+                    first_elements[i + 1, : len(target_pks)],
                 )
                 source_where_list.append(
                     self._extract_where(
@@ -273,13 +273,13 @@ class PartitionBuilder:
                 )
             filter_source_clause = geq_value(
                 source_table,
-                self.primary_keys,
-                first_elements[len(first_elements) - 1, : len(self.primary_keys)],
+                source_pks,
+                first_elements[len(first_elements) - 1, : len(source_pks)],
             )
             filter_target_clause = geq_value(
                 target_table,
-                self.primary_keys,
-                first_elements[len(first_elements) - 1, : len(self.primary_keys)],
+                target_pks,
+                first_elements[len(first_elements) - 1, : len(target_pks)],
             )
             source_where_list.append(
                 self._extract_where(

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/data_validation/query_builder/query_builder.py

@@ -179,6 +179,10 @@ class FilterField(object):
         """
         return FilterField(None, left=expr)
 
+    @staticmethod
+    def or_(field_list: list):
+        return FilterField(ibis.or_, left=field_list)
+
     def compile(self, ibis_table):
         if self.expr is None:
             return operations.compile_raw_sql(ibis_table, self.left)
@@ -189,11 +193,16 @@ class FilterField(object):
         if self.right_field:
             self.right = ibis_table[self.right_field]
 
-        return self.expr(self.left, self.right)
+        if self.expr == ibis.or_:
+            return self.expr(*[_.compile(ibis_table) for _ in self.left])
+        else:
+            return self.expr(self.left, self.right)
 
 
 class ComparisonField(object):
-    def __init__(self, field_name, alias=None, cast=None):
+    def __init__(
+        self, field_name: str, alias: str = None, cast: str = None, trim: bool = None
+    ):
         """A representation of a comparison field used to build a query.
 
         Args:
@@ -204,6 +213,7 @@ class ComparisonField(object):
         self.field_name = field_name
         self.alias = alias
         self.cast = cast
+        self.trim = trim
 
     def compile(self, ibis_table):
         # Fields are supplied on compile or on build
@@ -211,6 +221,8 @@ class ComparisonField(object):
         alias = self.alias or self.field_name
         if self.cast:
             comparison_field = comparison_field.force_cast(self.cast)
+        elif self.trim and comparison_field.type().is_string():
+            comparison_field = comparison_field.rstrip()
         comparison_field = comparison_field.name(alias)
 
         return comparison_field

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/data_validation/validation_builder.py

@@ -15,6 +15,7 @@ import logging
 from copy import deepcopy
 
 from data_validation import consts, metadata
+from data_validation.clients import get_max_in_list_size
 from data_validation.query_builder.query_builder import (
     AggregateField,
     CalculatedField,
@@ -25,6 +26,11 @@ from data_validation.query_builder.query_builder import (
 )
 
 
+def list_to_sublists(id_list: list, max_size: int) -> list:
+    """Return a list of items as a list of lists based on a max list length of max_size."""
+    return [id_list[_ : _ + max_size] for _ in range(0, len(id_list), max_size)]
+
+
 class ValidationBuilder(object):
     def __init__(self, config_manager):
         """Initialize a ValidationBuilder client which supplies the
@@ -60,6 +66,27 @@ class ValidationBuilder(object):
         self.add_primary_keys()
         self.add_query_limit()
 
+    def _construct_isin_filter(
+        self, client, column_name: str, in_list: list
+    ) -> FilterField:
+        """Return a FilterField object that is either isin(...) or (isin(...) OR isin(...) OR...)."""
+
+        max_in_list_size = get_max_in_list_size(
+            client,
+            in_list_over_expressions=bool(
+                in_list and getattr(in_list[0], "cast", None)
+            ),
+        )
+        if max_in_list_size and len(in_list) > max_in_list_size:
+            source_filters = [
+                FilterField.isin(column_name, _)
+                for _ in list_to_sublists(in_list, max_in_list_size)
+            ]
+            return FilterField.or_(source_filters)
+
+        else:
+            return FilterField.isin(column_name, in_list)
+
     def clone(self):
         cloned_builder = ValidationBuilder(self.config_manager)
 
@@ -237,12 +264,13 @@ class ValidationBuilder(object):
         # grab calc field metadata
         alias = primary_key.get(consts.CONFIG_FIELD_ALIAS)
         cast = primary_key.get(consts.CONFIG_CAST)
+        trim = self.config_manager.trim_string_pks()
         # check if valid calc field and return correct object
         source_field = ComparisonField(
-            field_name=source_field_name, alias=alias, cast=cast
+            field_name=source_field_name, alias=alias, cast=cast, trim=trim
         )
         target_field = ComparisonField(
-            field_name=target_field_name, alias=alias, cast=cast
+            field_name=target_field_name, alias=alias, cast=cast, trim=trim
         )
         self.source_builder.add_comparison_field(source_field)
         self.target_builder.add_comparison_field(target_field)
@@ -271,11 +299,13 @@ class ValidationBuilder(object):
                 filter_field[consts.CONFIG_FILTER_TARGET_VALUE],
             )
         elif filter_field[consts.CONFIG_TYPE] == consts.FILTER_TYPE_ISIN:
-            source_filter = FilterField.isin(
+            source_filter = self._construct_isin_filter(
+                self.source_client,
                 filter_field[consts.CONFIG_FILTER_SOURCE_COLUMN],
                 filter_field[consts.CONFIG_FILTER_SOURCE_VALUE],
             )
-            target_filter = FilterField.isin(
+            target_filter = self._construct_isin_filter(
+                self.target_client,
                 filter_field[consts.CONFIG_FILTER_TARGET_COLUMN],
                 filter_field[consts.CONFIG_FILTER_TARGET_VALUE],
             )

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/google_pso_data_validator.egg-info/PKG-INFO

@@ -1,11 +1,10 @@
 Metadata-Version: 2.1
 Name: google-pso-data-validator
-Version: 5.0.0
+Version: 5.1.0
 Summary: A package to enable easy data validation
-Home-page: https://github.com/pypa/sampleproject
-Author: Dylan Hercher
-Author-email: dhercher@google.com
-Classifier: Development Status :: 3 - Alpha
+Author: PSO DVT Engineering team
+Author-email: data-validator-eng@google.com
+Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.8
@@ -16,7 +15,7 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: attrs==23.1.0
-Requires-Dist: grpcio==1.53.0
+Requires-Dist: grpcio==1.53.2
 Requires-Dist: lazy-object-proxy==1.9.0
 Requires-Dist: marshmallow==3.19.0
 Requires-Dist: fsspec>=2022.8.2
@@ -24,7 +23,7 @@ Requires-Dist: google-api-python-client==2.91.0
 Requires-Dist: ibis-framework==5.1.0
 Requires-Dist: impyla==0.18.0
 Requires-Dist: SQLAlchemy==1.4.49
-Requires-Dist: PyMySQL==1.1.0
+Requires-Dist: PyMySQL==1.1.1
 Requires-Dist: psycopg2-binary==2.9.6
 Requires-Dist: PyYAML==6.0
 Requires-Dist: pandas==1.5.3
@@ -35,12 +34,11 @@ Requires-Dist: google-cloud-bigquery==3.11.3
 Requires-Dist: google-cloud-bigquery-storage==2.20.0
 Requires-Dist: google-cloud-spanner==3.36.0
 Requires-Dist: google-cloud-storage==2.10.0
-Requires-Dist: setuptools>=34.0.0
 Requires-Dist: jellyfish==1.0.0
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: Flask==2.3.2
 Requires-Dist: parsy==2.1
-Requires-Dist: google-cloud-secret-manager<=2.15.0
+Requires-Dist: google-cloud-secret-manager>=2.16.0
 Provides-Extra: apache-airflow
 Requires-Dist: 1.10.11; extra == "apache-airflow"
 Provides-Extra: pyspark
@@ -256,6 +254,8 @@ data-validation (--verbose or -v) (--log-level or -ll) validate row
                         Row batch size used for random row filters (default 10,000).
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
+  [--trim-string-pks, -tsp]
+                        Trims string based primary key values, intended for use when one engine uses padded string semantics (e.g. CHAR(n)) and the other does not (e.g. VARCHAR(n)).
 ```
 #### Generate Table Partitions for Large Table Row Validations
 
@@ -357,12 +357,12 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query col
                         Source sql query
                         Either --source-query or --source-query-file must be provided
   --source-query-file  SOURCE_QUERY_FILE, -sqf SOURCE_QUERY_FILE
-                        File containing the source sql commands
+                        File containing the source sql command. Supports GCS and local paths.
   --target-query TARGET_QUERY, -tq TARGET_QUERY
                         Target sql query
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
-                        File containing the target sql commands
+                        File containing the target sql command. Supports GCS and local paths.
   [--count COLUMNS]     Comma separated list of columns for count or * for all columns
   [--sum COLUMNS]       Comma separated list of columns for sum or * for all numeric
   [--min COLUMNS]       Comma separated list of columns for min or * for all numeric
@@ -419,12 +419,12 @@ data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
                         Source sql query
                         Either --source-query or --source-query-file must be provided
   --source-query-file SOURCE_QUERY_FILE, -sqf SOURCE_QUERY_FILE
-                        File containing the source sql commands
+                        File containing the source sql command. Supports GCS and local paths.
   --target-query TARGET_QUERY, -tq TARGET_QUERY
                         Target sql query
                         Either --target-query or --target-query-file must be provided
   --target-query-file TARGET_QUERY_FILE, -tqf TARGET_QUERY_FILE
-                        File containing the target sql commands
+                        File containing the target sql command. Supports GCS and local paths.
   --comparison-fields or -comp-fields FIELDS
                         Comma separated list of columns to compare. Can either be a physical column or an alias
                         See: *Calculated Fields* section for details
@@ -522,7 +522,7 @@ View the complete YAML file for a Grouped Column validation on the
 You can scale DVT for large table validations by running the tool in a distributed manner. To optimize the validation speed for large tables, you can use GKE Jobs ([Google Kubernetes Jobs](https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-workloads-overview#batch_jobs)) or [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs). If you are not familiar with Kubernetes or Cloud Run Jobs, see [Scaling DVT with Distributed Jobs](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/internal/distributed_jobs.md) for a detailed overview.
 
 
-We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started. 
+We recommend first generating table partitions with the `generate-table-partitions` command for your large tables. Then, use Cloud Run or GKE to distribute validating each chunk in parallel. See the [Cloud Run Jobs Quickstart sample](https://github.com/GoogleCloudPlatform/professional-services-data-validator/tree/develop/samples/cloud_run_jobs) to get started.
 
 When running DVT in a distributed fashion, both the `--kube-completions` and `--config-dir` flags are required. The `--kube-completions` flag specifies that the validation is being run in indexed completion mode in Kubernetes or as multiple independent tasks in Cloud Run. If the `-kc` option is used and you are not running in indexed mode, you will receive a warning and the container will process all the validations sequentially. If the `-kc` option is used and a config directory is not provided (i.e. a `--config-file` is provided instead), a warning is issued.
 

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/google_pso_data_validator.egg-info/requires.txt

@@ -1,5 +1,5 @@
 attrs==23.1.0
-grpcio==1.53.0
+grpcio==1.53.2
 lazy-object-proxy==1.9.0
 marshmallow==3.19.0
 fsspec>=2022.8.2
@@ -7,7 +7,7 @@ google-api-python-client==2.91.0
 ibis-framework==5.1.0
 impyla==0.18.0
 SQLAlchemy==1.4.49
-PyMySQL==1.1.0
+PyMySQL==1.1.1
 psycopg2-binary==2.9.6
 PyYAML==6.0
 pandas==1.5.3
@@ -18,12 +18,11 @@ google-cloud-bigquery==3.11.3
 google-cloud-bigquery-storage==2.20.0
 google-cloud-spanner==3.36.0
 google-cloud-storage==2.10.0
-setuptools>=34.0.0
 jellyfish==1.0.0
 tabulate==0.9.0
 Flask==2.3.2
 parsy==2.1
-google-cloud-secret-manager<=2.15.0
+google-cloud-secret-manager>=2.16.0
 
 [apache-airflow]
 1.10.11

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/setup.py

@@ -18,8 +18,8 @@ import setuptools
 
 name = "google-pso-data-validator"
 description = "A package to enable easy data validation"
-version = "5.0.0"
-release_status = "Development Status :: 3 - Alpha"
+version = "5.1.0"
+release_status = "Development Status :: 4 - Beta"
 
 with open("README.md", "r") as fh:
     long_description = fh.read()
@@ -27,7 +27,7 @@ with open("README.md", "r") as fh:
 dependencies = [
     # Dependency corrections from our requirements
     "attrs==23.1.0",
-    "grpcio==1.53.0",
+    "grpcio==1.53.2",
     "lazy-object-proxy==1.9.0",
     "marshmallow==3.19.0",
     # Core dependencies
@@ -36,7 +36,7 @@ dependencies = [
     "ibis-framework==5.1.0",
     "impyla==0.18.0",
     "SQLAlchemy==1.4.49",
-    "PyMySQL==1.1.0",
+    "PyMySQL==1.1.1",
     "psycopg2-binary==2.9.6",
     "PyYAML==6.0",
     "pandas==1.5.3",
@@ -47,12 +47,11 @@ dependencies = [
     "google-cloud-bigquery-storage==2.20.0",
     "google-cloud-spanner==3.36.0",
     "google-cloud-storage==2.10.0",
-    "setuptools>=34.0.0",
     "jellyfish==1.0.0",
     "tabulate==0.9.0",
     "Flask==2.3.2",
     "parsy==2.1",
-    "google-cloud-secret-manager<=2.15.0",
+    "google-cloud-secret-manager>=2.16.0",
 ]
 
 extras_require = {
@@ -74,11 +73,10 @@ setuptools.setup(
     name=name,
     description=description,
     version=version,
-    author="Dylan Hercher",
-    author_email="dhercher@google.com",
+    author="PSO DVT Engineering team",
+    author_email="data-validator-eng@google.com",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    url="https://github.com/pypa/sampleproject",
     packages=packages,
     classifiers=[
         release_status,

{google_pso_data_validator-5.0.0 → google_pso_data_validator-5.1.0}/third_party/ibis/ibis_impala/api.py

@@ -197,6 +197,10 @@ def _get_schema_using_query(self, query):
     # Removing LIMIT 0 around query since it returns no results in Hive
     cur = self.raw_sql(f"SELECT * FROM ({query}) t0 LIMIT 1")
     cur.fetchall()
+    cur.description = [
+        (description[0].replace("t0.", "", 1), *description[1:])
+        for description in cur.description
+    ]
     ibis_fields = self._adapt_types(cur.description)
     cur.release()