google-pso-data-validator 8.5.2__tar.gz → 8.6.0__tar.gz

{google_pso_data_validator-8.5.2/google_pso_data_validator.egg-info → google_pso_data_validator-8.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 8.5.2
+Version: 8.6.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -39,7 +39,7 @@ Requires-Dist: 1.10.11; extra == "apache-airflow"
 Provides-Extra: pyspark
 Requires-Dist: 3.0.0; extra == "pyspark"
 Provides-Extra: develop
-Requires-Dist: black==22.3.0; extra == "develop"
+Requires-Dist: black==26.1.0; extra == "develop"
 Requires-Dist: flake8; extra == "develop"
 Requires-Dist: freezegun; extra == "develop"
 Requires-Dist: pyfakefs; extra == "develop"
@@ -719,17 +719,27 @@ target. The `find-tables` command:
 -   Pulls all tables in the source (applying a supplied `allowed-schemas` filter)
 -   Pulls all tables from the target
 -   Uses Jaro Similarity algorithm to match tables
+-   Optionally translates source schema names using mappings supplied in `allowed-schemas` before matching
 -   Finally, it prints a JSON list of tables which can be a reference for the
     validation run config.
 
 Note that our default value for the `score-cutoff` parameter is 1 and it seeks for identical matches. If no matches occur, reduce this value as deemed necessary. By using smaller numbers such as 0.7, 0.65 etc you can get more matches. For reference, we make use of [this jaro_similarity method](https://jamesturk.github.io/jellyfish/functions/#jaro-similarity) for the string comparison.
 
+
 ```shell
 data-validation find-tables --source-conn source --target-conn target \
     --allowed-schemas pso_data_validator \
     --score-cutoff 1
 ```
 
+If your source and target environments have different schema names (e.g., `prod_raw` vs `dwh_raw`), you can provide a mapping directly in the `--allowed-schemas` flag using the `source_schema=target_schema` syntax.
+
+```shell
+data-validation find-tables --source-conn source --target-conn target \
+    --allowed-schemas prod=dwh \
+    --score-cutoff 1
+```
+
 ### Using Beta CLI Features
 
 There may be occasions we want to release a new CLI feature under a Beta flag.

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/README.md

@@ -659,17 +659,27 @@ target. The `find-tables` command:
 -   Pulls all tables in the source (applying a supplied `allowed-schemas` filter)
 -   Pulls all tables from the target
 -   Uses Jaro Similarity algorithm to match tables
+-   Optionally translates source schema names using mappings supplied in `allowed-schemas` before matching
 -   Finally, it prints a JSON list of tables which can be a reference for the
     validation run config.
 
 Note that our default value for the `score-cutoff` parameter is 1 and it seeks for identical matches. If no matches occur, reduce this value as deemed necessary. By using smaller numbers such as 0.7, 0.65 etc you can get more matches. For reference, we make use of [this jaro_similarity method](https://jamesturk.github.io/jellyfish/functions/#jaro-similarity) for the string comparison.
 
+
 ```shell
 data-validation find-tables --source-conn source --target-conn target \
     --allowed-schemas pso_data_validator \
     --score-cutoff 1
 ```
 
+If your source and target environments have different schema names (e.g., `prod_raw` vs `dwh_raw`), you can provide a mapping directly in the `--allowed-schemas` flag using the `source_schema=target_schema` syntax.
+
+```shell
+data-validation find-tables --source-conn source --target-conn target \
+    --allowed-schemas prod=dwh \
+    --score-cutoff 1
+```
+
 ### Using Beta CLI Features
 
 There may be occasions we want to release a new CLI feature under a Beta flag.

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/__init__.py

@@ -14,6 +14,5 @@
 
 from importlib.metadata import version
 
-
 __version__ = version("google-pso-data-validator")
 __all__ = ["__version__"]

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/cli_tools.py

@@ -404,7 +404,9 @@ def _configure_find_tables(subparsers):
         "--target-conn", "-tc", help="Target connection name."
     )
     find_tables_parser.add_argument(
-        "--allowed-schemas", "-as", help="List of source schemas to match."
+        "--allowed-schemas",
+        "-as",
+        help="List of source schemas to match. Can include mappings like 'source_schema=target_schema'.",
     )
     find_tables_parser.add_argument(
         "--include-views",

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/client_info.py

@@ -16,7 +16,6 @@ from google.api_core import client_info as http_client_info
 
 import data_validation
 
-
 APPLICATION_NAME = "google-pso-tool/data-validator"
 USER_AGENT = "{}/{}".format(APPLICATION_NAME, data_validation.__version__)
 

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/clients.py

@@ -342,10 +342,10 @@ def get_data_client(connection_config):
             consts.GOOGLE_SERVICE_ACCOUNT_KEY_PATH
         )
         if key_path:
-            decrypted_connection_config[
-                "credentials"
-            ] = google.oauth2.service_account.Credentials.from_service_account_file(
-                key_path
+            decrypted_connection_config["credentials"] = (
+                google.oauth2.service_account.Credentials.from_service_account_file(
+                    key_path
+                )
             )
 
     if source_type not in CLIENT_LOOKUP:

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/config_manager.py

@@ -667,6 +667,14 @@ class ConfigManager(object):
                 isinstance(source_type, dt.UUID) or isinstance(target_type, dt.UUID)
             )
 
+    def _is_db2_blob(self, source_column_name: str, target_column_name: str) -> bool:
+        """Returns True when either source or target column is Db2 (LUW and z/OS) BLOB data type."""
+        return self._is_raw_data_type(
+            "db2", source_column_name, target_column_name, ["BLOB"]
+        ) or self._is_raw_data_type(
+            "db2_zos", source_column_name, target_column_name, ["BLOB"]
+        )
+
     def _is_db2_xml(self, source_column_name: str, target_column_name: str) -> bool:
         """Returns True when either source or target column is Oracle LOB data type."""
         return self._is_raw_data_type(
@@ -1351,6 +1359,14 @@ class ConfigManager(object):
                 )
                 result_source_columns.pop(source_column)
                 result_target_columns.pop(target_column)
+            if self._is_db2_blob(source_column, target_column):
+                # Db2 BLOB is incompatible with Db2s own HEX function.
+                # Users would need to use a custom query to a custom UDF to row validate these.
+                logging.info(
+                    f"Skipping column {source_column} due to Db2 BLOB data type"
+                )
+                result_source_columns.pop(source_column)
+                result_target_columns.pop(target_column)
 
         return result_source_columns, result_target_columns
 

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/find_tables.py

@@ -28,26 +28,37 @@ if TYPE_CHECKING:
     import ibis
 
 
-def _compare_match_tables(source_table_map, target_table_map, score_cutoff=0.8) -> list:
+def _compare_match_tables(
+    source_table_map: dict,
+    target_table_map: dict,
+    score_cutoff: float = 0.8,
+    schema_map: dict = None,
+) -> list:
     """Return dict config object from matching tables."""
     # TODO(dhercher): evaluate if improved comparison and score cutoffs should be used.
     table_configs = []
+    schema_map = schema_map or {}
 
     target_keys = target_table_map.keys()
     for source_key in source_table_map:
+        source_schema = source_table_map[source_key][consts.CONFIG_SCHEMA_NAME]
+        source_table = source_table_map[source_key][consts.CONFIG_TABLE_NAME]
+
+        # Apply schema mapping if it exists, when there is a mapping
+        # lookup_key is {mapped_schema}.{source_name} which should then
+        # find a match in target_keys.
+        lookup_schema = schema_map.get(source_schema, source_schema)
+        lookup_key = f"{lookup_schema}.{source_table}".casefold()
+
         target_key = jellyfish_distance.extract_closest_match(
-            source_key, target_keys, score_cutoff=score_cutoff
+            lookup_key, target_keys, score_cutoff=score_cutoff
         )
         if target_key is None:
             continue
 
         table_config = {
-            consts.CONFIG_SCHEMA_NAME: source_table_map[source_key][
-                consts.CONFIG_SCHEMA_NAME
-            ],
-            consts.CONFIG_TABLE_NAME: source_table_map[source_key][
-                consts.CONFIG_TABLE_NAME
-            ],
+            consts.CONFIG_SCHEMA_NAME: source_schema,
+            consts.CONFIG_TABLE_NAME: source_table,
             consts.CONFIG_TARGET_SCHEMA_NAME: target_table_map[target_key][
                 consts.CONFIG_SCHEMA_NAME
             ],
@@ -95,6 +106,7 @@ def get_mapped_table_configs(
     allowed_schemas: list = None,
     include_views: bool = False,
     score_cutoff: int = 1,
+    schema_map: dict = None,
 ) -> list:
     """Get table list from each client and match them together into a single list of dicts."""
     source_table_map = _get_table_map(
@@ -102,7 +114,10 @@ def get_mapped_table_configs(
     )
     target_table_map = _get_table_map(target_client, include_views=include_views)
     return _compare_match_tables(
-        source_table_map, target_table_map, score_cutoff=score_cutoff
+        source_table_map,
+        target_table_map,
+        score_cutoff=score_cutoff,
+        schema_map=schema_map,
     )
 
 
@@ -114,13 +129,22 @@ def find_tables_using_string_matching(args) -> str:
     source_client = clients.get_data_client(mgr.get_connection_config(args.source_conn))
     target_client = clients.get_data_client(mgr.get_connection_config(args.target_conn))
 
-    allowed_schemas = cli_tools.get_arg_list(args.allowed_schemas)
+    allowed_schemas_raw = cli_tools.get_arg_list(args.allowed_schemas) or []
+    allowed_schemas = []
+    schema_map = {}
+    for schema in allowed_schemas_raw:
+        if "=" in schema:
+            schema, target_schema = schema.split("=", 1)
+            schema_map[schema] = target_schema
+        allowed_schemas.append(schema)
+
     table_configs = get_mapped_table_configs(
         source_client,
         target_client,
         allowed_schemas=allowed_schemas,
         include_views=args.include_views,
         score_cutoff=score_cutoff,
+        schema_map=schema_map,
     )
     return json.dumps(table_configs)
 

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/gcs_helper.py

@@ -19,7 +19,6 @@ from typing import List
 from google.cloud import storage
 from data_validation import client_info
 
-
 WRITE_SUCCESS_STRING = "Success! Config output written to"
 DELETE_SUCCESS_STRING = "Successfully deleted"
 

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/query_builder/random_row_builder.py

@@ -19,7 +19,6 @@ import ibis
 from data_validation import clients
 from data_validation.query_builder.query_builder import QueryBuilder
 
-
 # Adding new data sources should be done by adding the Backend name here
 RANDOM_SORT_SUPPORTS = [
     "bigquery",

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/result_handlers/base_backend.py

@@ -20,7 +20,6 @@ import ibis
 from data_validation import consts, util
 from data_validation.result_handlers import text as text_handler
 
-
 if TYPE_CHECKING:
     from pandas import DataFrame
 

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/result_handlers/bigquery.py

@@ -23,7 +23,6 @@ import google.oauth2.service_account
 from data_validation import clients, consts, exceptions, util
 from data_validation.result_handlers.base_backend import BaseBackendResultHandler
 
-
 BQRH_WRITE_MESSAGE = "Results written to BigQuery"
 BQRH_NO_WRITE_MESSAGE = "No results to write to BigQuery"
 

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/result_handlers/postgres.py

@@ -28,7 +28,6 @@ from data_validation.result_handlers.base_backend import (
     RH_NO_WRITE_MESSAGE,
 )
 
-
 if TYPE_CHECKING:
     from pandas import DataFrame
     from ibis.backends.base import BaseBackend

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/schema_validation.py

@@ -20,7 +20,6 @@ import re
 
 from data_validation import metadata, consts, clients, exceptions
 
-
 # Check for decimal data type with precision and/or scale. Permits hyphen in p/s for value ranges.
 DECIMAL_PRECISION_SCALE_PATTERN = re.compile(
     r"([!]?decimal)\(([0-9\-]+)(?:,[ ]*([0-9\-]+))?\)", re.I
@@ -187,7 +186,10 @@ def schema_validation_matching(
             )
         else:
             # Target data type mismatch
-            (higher_precision, lower_precision,) = parse_n_validate_datatypes(
+            (
+                higher_precision,
+                lower_precision,
+            ) = parse_n_validate_datatypes(
                 string_val(source_field_type), string_val(target_field_type)
             )
             if higher_precision:

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/data_validation/util.py

@@ -16,6 +16,7 @@ import json
 import logging
 import re
 import time
+import uuid
 
 from data_validation import clients, exceptions
 
@@ -87,3 +88,19 @@ def ibis_table_to_sql(ibis_table: "IbisTable", alchemy_client: "BaseBackend") ->
     else:
         sql_string = str(ibis_table.compile())
     return sql_string
+
+
+def dvt_temp_object_name(prefix: str = "dvt_temp") -> str:
+    """Generate a random name for when DVT needs to create a temporary object.
+
+    Args:
+        prefix: The prefix to use for the temporary object name.
+
+    Returns:
+        A lower case random name for when DVT needs to create a temporary object.
+    """
+    if not isinstance(prefix, str) or not re.match(r"^[a-zA-Z0-9_]+$", prefix):
+        raise exceptions.ValidationException(
+            f"Invalid prefix: '{prefix}'. Only alphanumeric and underscore characters are allowed."
+        )
+    return f"{prefix}_{uuid.uuid4().hex[:8].lower()}"

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0/google_pso_data_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 8.5.2
+Version: 8.6.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -39,7 +39,7 @@ Requires-Dist: 1.10.11; extra == "apache-airflow"
 Provides-Extra: pyspark
 Requires-Dist: 3.0.0; extra == "pyspark"
 Provides-Extra: develop
-Requires-Dist: black==22.3.0; extra == "develop"
+Requires-Dist: black==26.1.0; extra == "develop"
 Requires-Dist: flake8; extra == "develop"
 Requires-Dist: freezegun; extra == "develop"
 Requires-Dist: pyfakefs; extra == "develop"
@@ -719,17 +719,27 @@ target. The `find-tables` command:
 -   Pulls all tables in the source (applying a supplied `allowed-schemas` filter)
 -   Pulls all tables from the target
 -   Uses Jaro Similarity algorithm to match tables
+-   Optionally translates source schema names using mappings supplied in `allowed-schemas` before matching
 -   Finally, it prints a JSON list of tables which can be a reference for the
     validation run config.
 
 Note that our default value for the `score-cutoff` parameter is 1 and it seeks for identical matches. If no matches occur, reduce this value as deemed necessary. By using smaller numbers such as 0.7, 0.65 etc you can get more matches. For reference, we make use of [this jaro_similarity method](https://jamesturk.github.io/jellyfish/functions/#jaro-similarity) for the string comparison.
 
+
 ```shell
 data-validation find-tables --source-conn source --target-conn target \
     --allowed-schemas pso_data_validator \
     --score-cutoff 1
 ```
 
+If your source and target environments have different schema names (e.g., `prod_raw` vs `dwh_raw`), you can provide a mapping directly in the `--allowed-schemas` flag using the `source_schema=target_schema` syntax.
+
+```shell
+data-validation find-tables --source-conn source --target-conn target \
+    --allowed-schemas prod=dwh \
+    --score-cutoff 1
+```
+
 ### Using Beta CLI Features
 
 There may be occasions we want to release a new CLI feature under a Beta flag.

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/google_pso_data_validator.egg-info/requires.txt

@@ -23,7 +23,7 @@ tabulate>=0.9.0
 1.10.11
 
 [develop]
-black==22.3.0
+black==26.1.0
 flake8
 freezegun
 pyfakefs

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/setup.py

@@ -18,7 +18,7 @@ import setuptools
 
 name = "google-pso-data-validator"
 description = "A package to enable easy data validation"
-version = "8.5.2"
+version = "8.6.0"
 release_status = "Development Status :: 4 - Beta"
 
 with open("README.md", "r") as fh:
@@ -51,7 +51,7 @@ extras_require = {
     "apache-airflow": "1.10.11",
     "pyspark": "3.0.0",
     "develop": [
-        "black==22.3.0",
+        "black==26.1.0",
         "flake8",
         "freezegun",
         "pyfakefs",

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/third_party/ibis/ibis_addon/api.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 from typing import Iterable, Optional
+import functools
+import inspect
 
 from ibis.backends.base.sql.alchemy.datatypes import to_sqla_type
 import ibis.expr.datatypes as dt
@@ -174,6 +176,9 @@ def db2_type_string_length(
         return 19
     elif ibis_column.is_integer():
         return ibis_integer_string_length(ibis_column)
+    elif ibis_column.is_binary() and raw_data_type:
+        # Position 1 in raw_types is the data length.
+        return raw_data_type[1]
     else:
         return None
 
@@ -196,3 +201,42 @@ def ibis_integer_string_length(ibis_column: dt.DataType) -> Optional[int]:
 
 
 Value.force_cast = force_cast
+
+
+def cache_generator_results(func):
+    """Decorator to cache generator results at the instance level."""
+    # Pre-compute the function signature to avoid doing this on every call
+    sig = inspect.signature(func)
+
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        # Create an instance-level cache if it doesn't already exist.
+        # This prevents instances from sharing cache state and avoids memory leaks.
+        if not hasattr(self, "_generator_cache"):
+            self._generator_cache = {}
+
+        # Bind the provided args and kwargs to the function's signature.
+        # This creates a unified representation of the arguments, avoiding cache misses
+        # when identical values are passed differently (e.g., positional vs. keyword).
+        bound_args = sig.bind(self, *args, **kwargs)
+        bound_args.apply_defaults()
+
+        # Remove 'self' from the arguments since the cache is already scoped to the instance
+        arguments = bound_args.arguments.copy()
+        arguments.pop("self", None)
+
+        # Ensure only supported types are used to prevent frozenset failures on unhashable types
+        for v in arguments.values():
+            assert isinstance(
+                v, (str, int, type(None))
+            ), f"Unsupported argument type in cache_generator_results: {type(v)}"
+
+        # Generate a unique cache key based on the function name and the canonical arguments
+        key = (func.__name__, frozenset(arguments.items()))
+        if key not in self._generator_cache:
+            # Fully evaluate the generator to a list to store it in the cache
+            self._generator_cache[key] = list(func(self, *args, **kwargs))
+
+        yield from self._generator_cache[key]
+
+    return wrapper

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/third_party/ibis/ibis_addon/operations.py

@@ -22,6 +22,7 @@ extended its own registry.  Eventually this can potentially be pushed to
 Ibis as an override, though it would not apply for Pandas and other
 non-textual languages.
 """
+
 import datetime
 import dateutil
 import numpy as np
@@ -508,19 +509,19 @@ if OracleExprTranslator:
         ops.StringLength
     ]
 
-PostgreSQLExprTranslator._registry[
-    ops.HashBytes
-] = postgres_registry.sa_format_hashbytes
+PostgreSQLExprTranslator._registry[ops.HashBytes] = (
+    postgres_registry.sa_format_hashbytes
+)
 PostgreSQLExprTranslator._registry[RawSQL] = sa_format_raw_sql
 PostgreSQLExprTranslator._registry[ToChar] = sa_format_to_char
 PostgreSQLExprTranslator._registry[ops.Cast] = postgres_registry.sa_cast_postgres
 PostgreSQLExprTranslator._registry[BinaryLength] = sa_format_binary_length
-PostgreSQLExprTranslator._registry[
-    ops.ExtractEpochSeconds
-] = postgres_registry.sa_epoch_seconds
-PostgreSQLExprTranslator._registry[
-    PaddedCharLength
-] = postgres_registry.sa_format_postgres_padded_char_length
+PostgreSQLExprTranslator._registry[ops.ExtractEpochSeconds] = (
+    postgres_registry.sa_epoch_seconds
+)
+PostgreSQLExprTranslator._registry[PaddedCharLength] = (
+    postgres_registry.sa_format_postgres_padded_char_length
+)
 
 
 MsSqlExprTranslator._registry[ops.HashBytes] = mssql_registry.sa_format_hashbytes
@@ -568,9 +569,9 @@ if TeradataExprTranslator:
     TeradataExprTranslator._registry[RawSQL] = format_raw_sql
     TeradataExprTranslator._registry[ops.HashBytes] = format_hashbytes_teradata
     TeradataExprTranslator._registry[BinaryLength] = sa_format_binary_length
-    TeradataExprTranslator._registry[
-        PaddedCharLength
-    ] = TeradataExprTranslator._registry[ops.StringLength]
+    TeradataExprTranslator._registry[PaddedCharLength] = (
+        TeradataExprTranslator._registry[ops.StringLength]
+    )
 
 if SnowflakeExprTranslator:
     SnowflakeExprTranslator._registry[ops.Cast] = sa_cast_snowflake
@@ -583,10 +584,10 @@ if SnowflakeExprTranslator:
     SnowflakeExprTranslator._registry[ops.RStrip] = _sa_whitespace_rstrip
 
 if SybaseExprTranslator:
-    SybaseExprTranslator._registry[
-        BinaryLength
-    ] = mssql_registry.sa_format_binary_length
+    SybaseExprTranslator._registry[BinaryLength] = (
+        mssql_registry.sa_format_binary_length
+    )
     SybaseExprTranslator._registry[RawSQL] = sa_format_raw_sql
-    SybaseExprTranslator._registry[
-        PaddedCharLength
-    ] = mssql_registry.sa_format_string_length
+    SybaseExprTranslator._registry[PaddedCharLength] = (
+        mssql_registry.sa_format_string_length
+    )

{google_pso_data_validator-8.5.2 → google_pso_data_validator-8.6.0}/third_party/ibis/ibis_cloud_spanner/compiler.py

@@ -19,7 +19,6 @@ from ibis.backends.bigquery import compiler as bigquery_compiler
 from ibis.backends.bigquery import rewrites
 from third_party.ibis.ibis_cloud_spanner import registry
 
-
 _NAME_REGEX = re.compile(r"[_A-Za-z][A-Za-z_0-9]*")