google-pso-data-validator 6.3.0__tar.gz → 6.4.0__tar.gz

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: google-pso-data-validator
-Version: 6.3.0
+Version: 6.4.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/data_validation/__main__.py

@@ -24,7 +24,9 @@ from data_validation import (
     cli_tools,
     clients,
     consts,
+    exceptions,
     state_manager,
+    util,
 )
 from data_validation.config_manager import ConfigManager
 from data_validation.data_validation import DataValidation
@@ -220,8 +222,8 @@ def _get_calculated_config(args, config_manager: ConfigManager) -> List[dict]:
 def _get_comparison_config(args, config_manager: ConfigManager) -> List[dict]:
     col_list = (
         None
-        if config_manager.comparison_fields == "*"
-        else cli_tools.get_arg_list(config_manager.comparison_fields)
+        if args.comparison_fields == "*"
+        else cli_tools.get_arg_list(args.comparison_fields)
     )
     comparison_fields = config_manager.build_comp_fields(
         col_list,
@@ -331,22 +333,25 @@ def build_config_managers_from_args(
     args: Namespace, validate_cmd: str = None
 ) -> List[ConfigManager]:
     """Return a list of config managers ready to execute."""
-    configs = []
 
-    # Get pre build configs to build ConfigManager objects
-    pre_build_configs_list = cli_tools.get_pre_build_configs(args, validate_cmd)
+    def _build_configs():
+        configs = []
 
-    # Build a list of ConfigManager objects
-    for pre_build_configs in pre_build_configs_list:
-        config_manager = ConfigManager.build_config_manager(**pre_build_configs)
+        # Get pre build configs to build ConfigManager objects
+        pre_build_configs_list = cli_tools.get_pre_build_configs(args, validate_cmd)
 
-        # Append post build configs to ConfigManager object
-        config_manager = build_config_from_args(args, config_manager)
+        # Build a list of ConfigManager objects
+        for pre_build_configs in pre_build_configs_list:
+            config_manager = ConfigManager.build_config_manager(**pre_build_configs)
 
-        # Append ConfigManager object to configs list
-        configs.append(config_manager)
+            # Append post build configs to ConfigManager object
+            config_manager = build_config_from_args(args, config_manager)
 
-    return configs
+            # Append ConfigManager object to configs list
+            configs.append(config_manager)
+        return configs
+
+    return util.timed_call("Build config", _build_configs)
 
 
 def config_runner(args):
@@ -388,9 +393,26 @@ def config_runner(args):
                 )
             config_file_names = cli_tools.list_validations(config_dir=args.config_dir)
             config_managers = []
+            errors = False
             for file in config_file_names:
                 config_managers = build_config_managers_from_yaml(args, file)
-                run_validations(args, config_managers)
+                try:
+                    logging.info(
+                        "Currently running the validation for YAML file: %s",
+                        file,
+                    )
+                    run_validations(args, config_managers)
+                except Exception as e:
+                    errors = True
+                    logging.error(
+                        "Error '%s' occurred while running config file %s. Skipping it for now.",
+                        str(e),
+                        file,
+                    )
+            if errors:
+                raise exceptions.ValidationException(
+                    "Some of the validations raised an exception"
+                )
     else:
         if args.kube_completions:
             logging.warning(
@@ -543,23 +565,7 @@ def run_validations(args, config_managers):
     """
     # TODO(issue/31): Add parallel execution logic
     for config_manager in config_managers:
-        if config_manager.config and consts.CONFIG_FILE in config_manager.config:
-            logging.info(
-                "Currently running the validation for YAML file: %s",
-                config_manager.config[consts.CONFIG_FILE],
-            )
-            try:
-                run_validation(
-                    config_manager, dry_run=args.dry_run, verbose=args.verbose
-                )
-            except Exception as e:
-                logging.error(
-                    "Error %s occurred while running config file %s. Skipping it for now.",
-                    str(e),
-                    config_manager.config[consts.CONFIG_FILE],
-                )
-        else:
-            run_validation(config_manager, dry_run=args.dry_run, verbose=args.verbose)
+        run_validation(config_manager, dry_run=args.dry_run, verbose=args.verbose)
 
 
 def store_yaml_config_file(args, config_managers):

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/data_validation/cli_tools.py

@@ -990,6 +990,12 @@ def _add_common_arguments(
         # TODO: update if we start to support other statuses
         help="Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned",
     )
+    optional_arguments.add_argument(
+        "--run-id",
+        "-rid",
+        default=None,
+        help="Set a string for the run_id, if None is input then a randomly generated UUID will be used, which is the default behaviour",
+    )
 
 
 def _check_positive(value: int) -> int:
@@ -1407,8 +1413,8 @@ def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
         result_handler_config = None
 
     # Set filter_config and threshold. Not supported in case of schema validation
-    filter_config = getattr(args, "filters", [])
-    threshold = getattr(args, "threshold", 0.0)
+    filter_config = getattr(args, consts.CONFIG_FILTERS, [])
+    threshold = getattr(args, consts.CONFIG_THRESHOLD, 0.0)
 
     # Get labels
     if args.labels is None:
@@ -1425,8 +1431,10 @@ def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
     format = args.format if args.format else "table"
 
     # Get random row arguments. Only in row validations these attributes can be present.
+    # Bad coding here, but keeping it so as not to introduce a breaking change. See
+    # consts.py Line 17 for a more detailed explanation.
     use_random_rows = getattr(args, "use_random_row", False)
-    random_row_batch_size = getattr(args, "random_row_batch_size", None)
+    random_row_batch_size = getattr(args, consts.CONFIG_RANDOM_ROW_BATCH_SIZE, None)
 
     # Get table list. Not supported in case of custom query validation
     is_filesystem = source_client._source_type == "FileSystem"
@@ -1459,23 +1467,28 @@ def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
     for table_obj in tables_list:
         pre_build_configs = {
             "config_type": config_type,
-            "source_conn_name": args.source_conn,
-            "target_conn_name": args.target_conn,
+            consts.CONFIG_SOURCE_CONN_NAME: args.source_conn,
+            consts.CONFIG_TARGET_CONN_NAME: args.target_conn,
             "table_obj": table_obj,
-            "labels": labels,
-            "threshold": threshold,
-            "format": format,
-            "use_random_rows": use_random_rows,
-            "random_row_batch_size": random_row_batch_size,
+            consts.CONFIG_LABELS: labels,
+            consts.CONFIG_THRESHOLD: threshold,
+            consts.CONFIG_FORMAT: format,
+            consts.CONFIG_USE_RANDOM_ROWS: use_random_rows,
+            consts.CONFIG_RANDOM_ROW_BATCH_SIZE: random_row_batch_size,
             "source_client": source_client,
             "target_client": target_client,
             "result_handler_config": result_handler_config,
             "filter_config": filter_config,
-            "filter_status": filter_status,
-            "trim_string_pks": getattr(args, "trim_string_pks", False),
-            "case_insensitive_match": getattr(args, "case_insensitive_match", False),
+            consts.CONFIG_FILTER_STATUS: filter_status,
+            consts.CONFIG_TRIM_STRING_PKS: getattr(
+                args, consts.CONFIG_TRIM_STRING_PKS, False
+            ),
+            consts.CONFIG_CASE_INSENSITIVE_MATCH: getattr(
+                args, consts.CONFIG_CASE_INSENSITIVE_MATCH, False
+            ),
             consts.CONFIG_ROW_CONCAT: getattr(args, consts.CONFIG_ROW_CONCAT, None),
             consts.CONFIG_ROW_HASH: getattr(args, consts.CONFIG_ROW_HASH, None),
+            consts.CONFIG_RUN_ID: getattr(args, consts.CONFIG_RUN_ID, None),
             "verbose": args.verbose,
         }
         if (

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/data_validation/config_manager.py

@@ -245,6 +245,11 @@ class ConfigManager(object):
         """Return field from Config"""
         return self._config.get(consts.CONFIG_ROW_HASH, [])
 
+    @property
+    def run_id(self):
+        """Return field from Config"""
+        return self._config.get(consts.CONFIG_RUN_ID, None)
+
     @property
     def filters(self):
         """Return Filters from Config"""
@@ -504,6 +509,7 @@ class ConfigManager(object):
         case_insensitive_match=None,
         concat=None,
         hash=None,
+        run_id=None,
         verbose=False,
     ):
         if isinstance(filter_config, dict):
@@ -536,6 +542,7 @@ class ConfigManager(object):
             consts.CONFIG_CASE_INSENSITIVE_MATCH: case_insensitive_match,
             consts.CONFIG_ROW_CONCAT: concat,
             consts.CONFIG_ROW_HASH: hash,
+            consts.CONFIG_RUN_ID: run_id,
         }
 
         return ConfigManager(
@@ -1160,7 +1167,7 @@ class ConfigManager(object):
                     col_names.append(col)
         return col_names
 
-    def build_comp_fields(self, col_list: list, exclude_cols: bool = False) -> list:
+    def build_comp_fields(self, col_list: list, exclude_cols: bool = False) -> dict:
         """This is a utility function processing comp-fields values like we do for hash/concat."""
         source_table = self.get_source_ibis_calculated_table()
         casefold_source_columns = {_.casefold(): str(_) for _ in source_table.columns}

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/data_validation/consts.py

@@ -14,6 +14,12 @@
 
 
 # Configuration Fields
+# These are all keys into the config dict. Where these values come from the command line, these would
+# match with the command line argument - so --secret-manager-type value would be inserted with the
+# 'secret_manager_type' key.
+# This is true for the most part, except where a typo gets introduced, i.e. --use-random-row became use_random_rows
+# Fixing this would be a breaking change. A future cleanup to prevent this would be to use
+# f'--{config_constant.repl('_','-')}' as the name argument to argparse.
 SOURCE_TYPE = "source_type"
 SECRET_MANAGER_TYPE = "secret_manager_type"
 SECRET_MANAGER_PROJECT_ID = "secret_manager_project_id"
@@ -49,6 +55,7 @@ CONFIG_TRIM_STRING_PKS = "trim_string_pks"
 CONFIG_CASE_INSENSITIVE_MATCH = "case_insensitive_match"
 CONFIG_ROW_CONCAT = "concat"
 CONFIG_ROW_HASH = "hash"
+CONFIG_RUN_ID = "run_id"
 CONFIG_SOURCE_COLUMN = "source_column"
 CONFIG_TARGET_COLUMN = "target_column"
 CONFIG_THRESHOLD = "threshold"

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/data_validation/data_validation.py

@@ -16,11 +16,11 @@ import json
 import logging
 import warnings
 from concurrent.futures import ThreadPoolExecutor
-
 import ibis.backends.pandas
 import pandas
+import uuid
 
-from data_validation import combiner, consts, metadata
+from data_validation import combiner, consts, metadata, util
 from data_validation.config_manager import ConfigManager
 from data_validation.query_builder.random_row_builder import RandomRowBuilder
 from data_validation.schema_validation import SchemaValidation
@@ -63,6 +63,9 @@ class DataValidation(object):
         self.run_metadata = metadata.RunMetadata()
         self.run_metadata.labels = self.config_manager.labels
 
+        # Use a generated uuid for the run_id if None was supplied via config
+        self.run_metadata.run_id = self.config_manager.run_id or str(uuid.uuid4())
+
         # Initialize Validation Builder if None was supplied
         self.validation_builder = validation_builder or ValidationBuilder(
             self.config_manager
@@ -88,7 +91,7 @@ class DataValidation(object):
         """Execute Queries and Store Results"""
         # Apply random row filter before validations run
         if self.config_manager.use_random_rows():
-            self._add_random_row_filter()
+            util.timed_call("Random row filter", self._add_random_row_filter)
 
         # Run correct execution for the given validation type
         if self.config_manager.validation_type == consts.ROW_VALIDATION:
@@ -98,7 +101,9 @@ class DataValidation(object):
             )
         elif self.config_manager.validation_type == consts.SCHEMA_VALIDATION:
             """Perform only schema validation"""
-            result_df = self.schema_validator.execute()
+            result_df = util.timed_call(
+                "Schema validation", self.schema_validator.execute
+            )
         else:
             result_df = self._execute_validation(
                 self.validation_builder, process_in_memory=True
@@ -289,6 +294,7 @@ class DataValidation(object):
 
     def _execute_validation(self, validation_builder, process_in_memory=True):
         """Execute Against a Supplied Validation Builder"""
+
         self.run_metadata.validations = validation_builder.get_metadata()
 
         source_query = validation_builder.get_source_query()
@@ -319,12 +325,18 @@ class DataValidation(object):
                 # Submit the two query network calls concurrently
                 futures.append(
                     executor.submit(
-                        self.config_manager.source_client.execute, source_query
+                        util.timed_call,
+                        "Source query",
+                        self.config_manager.source_client.execute,
+                        source_query,
                     )
                 )
                 futures.append(
                     executor.submit(
-                        self.config_manager.target_client.execute, target_query
+                        util.timed_call,
+                        "Target query",
+                        self.config_manager.target_client.execute,
+                        target_query,
                     )
                 )
                 source_df = futures[0].result()
@@ -335,7 +347,9 @@ class DataValidation(object):
             )
 
             try:
-                result_df = combiner.generate_report(
+                result_df = util.timed_call(
+                    "Generate report",
+                    combiner.generate_report,
                     pandas_client,
                     self.run_metadata,
                     pandas_client.table(combiner.DEFAULT_SOURCE),

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/data_validation/exceptions.py

@@ -17,5 +17,9 @@ class DataClientConnectionFailure(Exception):
     pass
 
 
-class SchemaValidationException(Exception):
+class ValidationException(Exception):
+    pass
+
+
+class SchemaValidationException(ValidationException):
     pass

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/data_validation/metadata.py

@@ -14,11 +14,9 @@
 
 """Metadata classes with data about the validation run."""
 
-
 import dataclasses
 import datetime
 import typing
-import uuid
 
 from data_validation import consts
 
@@ -64,7 +62,7 @@ class ValidationMetadata(object):
 
 @dataclasses.dataclass
 class RunMetadata(object):
-    run_id: str = dataclasses.field(default_factory=lambda: str(uuid.uuid4()))
+    run_id: str = dataclasses.field(default_factory=str)
     validations: dict = dataclasses.field(default_factory=dict)
     labels: list = dataclasses.field(default_factory=list)
     start_time: typing.Optional[datetime.datetime] = dataclasses.field(

google_pso_data_validator-6.4.0/data_validation/util.py

@@ -0,0 +1,24 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+
+
+def timed_call(log_txt, fn, *args, **kwargs):
+    t0 = time.time()
+    result = fn(*args, **kwargs)
+    elapsed = time.time() - t0
+    logging.debug(f"{log_txt} elapsed: {round(elapsed,2)}s")
+    return result

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/google_pso_data_validator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: google-pso-data-validator
-Version: 6.3.0
+Version: 6.4.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/google_pso_data_validator.egg-info/SOURCES.txt

@@ -21,6 +21,7 @@ data_validation/partition_builder.py
 data_validation/schema_validation.py
 data_validation/secret_manager.py
 data_validation/state_manager.py
+data_validation/util.py
 data_validation/validation_builder.py
 data_validation/query_builder/__init__.py
 data_validation/query_builder/partition_row_builder.py

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/setup.py

@@ -18,7 +18,7 @@ import setuptools
 
 name = "google-pso-data-validator"
 description = "A package to enable easy data validation"
-version = "6.3.0"
+version = "6.4.0"
 release_status = "Development Status :: 4 - Beta"
 
 with open("README.md", "r") as fh:

{google_pso_data_validator-6.3.0 → google_pso_data_validator-6.4.0}/third_party/ibis/ibis_oracle/__init__.py

@@ -108,6 +108,16 @@ class Backend(BaseAlchemyBackend):
             sa_url,
             poolclass=sa.pool.StaticPool,
             arraysize=self.arraysize,
+            # The hardcoding of 128 below is not great but is the simplest way of dealing with:
+            #   https://github.com/GoogleCloudPlatform/professional-services-data-validator/issues/1250
+            # There is a quirk of SQLAlchemy behaviour that is discussed in comments of issue 1250 that
+            # we cannot easily deal with, even if we "fixed" it it could be months or years before we can
+            # take advantage of it.
+            # We manage the max length of identifiers generated by DVT in clients.get_max_column_length()
+            # and in all other cases re-use existing column names (i.e. we know the length is already fine).
+            # Therefore the ugly hardcoding of 128 kicks the can down the road and unblocks a customer
+            # who is working with Oracle 11g and a max identifier length of 30.
+            max_identifier_length=128,
         )
         try:
             # Identify the session in Oracle as DVT, no-op if this fails.