google-pso-data-validator 8.2.0__tar.gz → 8.4.0__tar.gz

{google_pso_data_validator-8.2.0/google_pso_data_validator.egg-info → google_pso_data_validator-8.4.0}/PKG-INFO

@@ -1,21 +1,21 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 8.2.0
+Version: 8.4.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: Flask>=2.2
-Requires-Dist: fsspec!=2025.3.1
+Requires-Dist: fsspec
 Requires-Dist: google-api-python-client>=2.144.0
 Requires-Dist: google-cloud-bigquery>=3.25.0
 Requires-Dist: google-cloud-bigquery-storage>=2.26.0
@@ -25,10 +25,10 @@ Requires-Dist: google-cloud-storage>=2.18.2
 Requires-Dist: ibis-framework==5.1.0
 Requires-Dist: impyla>=0.19.0
 Requires-Dist: jellyfish>=1.1.0
-Requires-Dist: pandas==2.0.3
+Requires-Dist: pandas
 Requires-Dist: parsy>=2.1
 Requires-Dist: psycopg2-binary>=2.9.9
-Requires-Dist: pyarrow==14.0.1
+Requires-Dist: pyarrow
 Requires-Dist: pydata-google-auth>=1.8.2
 Requires-Dist: PyMySQL>=1.1.1
 Requires-Dist: PyYAML>=6.0.2
@@ -45,6 +45,7 @@ Requires-Dist: freezegun; extra == "develop"
 Requires-Dist: pyfakefs; extra == "develop"
 Requires-Dist: pytest; extra == "develop"
 Requires-Dist: pytest-cov; extra == "develop"
+Requires-Dist: pytest-rerunfailures; extra == "develop"
 Requires-Dist: pytest-timeout; extra == "develop"
 Dynamic: author
 Dynamic: author-email
@@ -92,8 +93,9 @@ DVT supports the following connection types:
 *   [Postgres](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#postgres)
 *   [Redshift](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#redshift)
 *   [Spanner](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#google-spanner)
-*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 *   [Snowflake](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#snowflake)
+*   [Sybase ASE](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#sybase)
+*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 
 The [Connections](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md) page provides details about how to create
 and list connections for the validation tool.
@@ -340,7 +342,7 @@ data-validation
                         Directory Path to store YAML Config Files
                         GCS: Provide a full gs:// path of the target directory. Eg: `gs://<BUCKET>/partitions_dir`
                         Local: Provide a relative path of the target directory. Eg: `partitions_dir`
-                        If invoked with -tbls parameter, the validations are stored in a directory named <schema>.<table>, otherwise the directory is named `custom.<random_string>`
+                        For custom queries, the yaml files are stored in CONFIG_DIR and, for tables the yamls are stored in a directory named <source_schema>.<table> within CONFIG_DIR.
   --partition-num INT, -pn INT
                         Number of partitions into which the table should be split, e.g. 1000 or 10000
                         In case this value exceeds the row count of the source/target table, it will be decreased to max(source_row_count, target_row_count)
@@ -653,9 +655,8 @@ The `--config-dir` flag will specify the directory with the YAML files to be exe
 
 ### Validation Reports
 
-The result handlers tell DVT where to store the results of each validation. The
-tool can write the results of a validation run to Google BigQuery, PostgreSQL
-or print to stdout (default). View the schema of the results table [here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/terraform/results_schema.json).
+The result handlers tell DVT where to store the results of each validation. The tool can write the results of a validation run to Google BigQuery, PostgreSQL or print to stdout (default). See [result handler setup](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/installation.md#result-handler-setup) for pre-requisites. View the schema of the BigQuery results table [here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/terraform/results_schema.json).
+
 
 To output to BigQuery or PostgreSQL, simply include the `-rh` flag during a validation run including
 the schema and table name for the results.

{google_pso_data_validator-8.2.0 → google_pso_data_validator-8.4.0}/README.md

@@ -33,8 +33,9 @@ DVT supports the following connection types:
 *   [Postgres](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#postgres)
 *   [Redshift](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#redshift)
 *   [Spanner](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#google-spanner)
-*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 *   [Snowflake](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#snowflake)
+*   [Sybase ASE](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#sybase)
+*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 
 The [Connections](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md) page provides details about how to create
 and list connections for the validation tool.
@@ -281,7 +282,7 @@ data-validation
                         Directory Path to store YAML Config Files
                         GCS: Provide a full gs:// path of the target directory. Eg: `gs://<BUCKET>/partitions_dir`
                         Local: Provide a relative path of the target directory. Eg: `partitions_dir`
-                        If invoked with -tbls parameter, the validations are stored in a directory named <schema>.<table>, otherwise the directory is named `custom.<random_string>`
+                        For custom queries, the yaml files are stored in CONFIG_DIR and, for tables the yamls are stored in a directory named <source_schema>.<table> within CONFIG_DIR.
   --partition-num INT, -pn INT
                         Number of partitions into which the table should be split, e.g. 1000 or 10000
                         In case this value exceeds the row count of the source/target table, it will be decreased to max(source_row_count, target_row_count)
@@ -594,9 +595,8 @@ The `--config-dir` flag will specify the directory with the YAML files to be exe
 
 ### Validation Reports
 
-The result handlers tell DVT where to store the results of each validation. The
-tool can write the results of a validation run to Google BigQuery, PostgreSQL
-or print to stdout (default). View the schema of the results table [here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/terraform/results_schema.json).
+The result handlers tell DVT where to store the results of each validation. The tool can write the results of a validation run to Google BigQuery, PostgreSQL or print to stdout (default). See [result handler setup](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/installation.md#result-handler-setup) for pre-requisites. View the schema of the BigQuery results table [here](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/terraform/results_schema.json).
+
 
 To output to BigQuery or PostgreSQL, simply include the `-rh` flag during a validation run including
 the schema and table name for the results.

{google_pso_data_validator-8.2.0 → google_pso_data_validator-8.4.0}/data_validation/__main__.py

@@ -524,8 +524,8 @@ def run_validation(config_manager: ConfigManager, dry_run=False, verbose=False):
         validation_builder=None,
         result_handler=None,
         verbose=verbose,
-        source_client=source_client,
-        target_client=target_client,
+        cached_source_client=source_client,
+        cached_target_client=target_client,
     ) as validator:
 
         if dry_run:
@@ -533,10 +533,12 @@ def run_validation(config_manager: ConfigManager, dry_run=False, verbose=False):
                 json.dumps(
                     {
                         "source_query": util.ibis_table_to_sql(
-                            validator.validation_builder.get_source_query()
+                            validator.validation_builder.get_source_query(),
+                            source_client,
                         ),
                         "target_query": util.ibis_table_to_sql(
-                            validator.validation_builder.get_target_query()
+                            validator.validation_builder.get_target_query(),
+                            target_client,
                         ),
                     },
                     indent=4,
@@ -594,6 +596,7 @@ def partition_and_store_config_files(args: "Namespace") -> None:
         setattr(args, "custom_query_type", "row")
         config_managers = build_config_managers_from_args(args, consts.CUSTOM_QUERY)
     partition_builder = PartitionBuilder(config_managers, args)
+    partition_builder.check_partition_configs()
     partition_builder.partition_configs()
 
 

{google_pso_data_validator-8.2.0 → google_pso_data_validator-8.4.0}/data_validation/cli_tools.py

@@ -73,6 +73,10 @@ if TYPE_CHECKING:
 CONNECTION_SOURCE_FIELDS = {
     consts.SOURCE_TYPE_BIGQUERY: [
         ["project_id", "GCP Project to use for BigQuery"],
+        [
+            "client_project_id",
+            "(Optional) BigQuery job/billing project (can differ from data project)",
+        ],
         ["google_service_account_key_path", "(Optional) GCP SA Key Path"],
         [
             "api_endpoint",
@@ -84,7 +88,7 @@ CONNECTION_SOURCE_FIELDS = {
         ],
     ],
     consts.SOURCE_TYPE_TERADATA: [
-        ["host", "Desired Teradata host"],
+        ["host", "Teradata host"],
         ["port", "Teradata port to connect on"],
         ["user_name", "User used to connect"],
         ["password", "Password for supplied user"],
@@ -93,7 +97,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["json_params", "(Optional) Additional teradatasql JSON string parameters"],
     ],
     consts.SOURCE_TYPE_ORACLE: [
-        ["host", "Desired Oracle host"],
+        ["host", "Oracle host"],
         ["port", "Oracle port to connect on"],
         ["user", "User used to connect"],
         ["thick_mode", "Flag indicating thick_mode"],
@@ -103,7 +107,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["connect_args", "(Optional) Additional connection argument mapping"],
     ],
     consts.SOURCE_TYPE_MSSQL: [
-        ["host", "Desired SQL Server host (default localhost)"],
+        ["host", "SQL Server host (default localhost)"],
         ["port", "SQL Server port to connect on (default 1433)"],
         ["user", "User used to connect"],
         ["password", "Password for supplied user"],
@@ -112,7 +116,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["url", "SQL Server SQLAlchemy connection URL"],
     ],
     consts.SOURCE_TYPE_MYSQL: [
-        ["host", "Desired MySQL host (default localhost)"],
+        ["host", "MySQL host (default localhost)"],
         ["port", "MySQL port to connect on (default 3306)"],
         ["user", "User used to connect"],
         ["password", "Password for supplied user"],
@@ -126,14 +130,14 @@ CONNECTION_SOURCE_FIELDS = {
         ["connect_args", "(Optional) Additional connection argument mapping"],
     ],
     consts.SOURCE_TYPE_POSTGRES: [
-        ["host", "Desired PostgreSQL host."],
+        ["host", "PostgreSQL host."],
         ["port", "PostgreSQL port to connect on (e.g. 5432)"],
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
         ["database", "Database in PostgreSQL to connect to (default postgres)"],
     ],
     consts.SOURCE_TYPE_REDSHIFT: [
-        ["host", "Desired Redshift host."],
+        ["host", "Redshift host."],
         ["port", "Redshift port to connect on (e.g. 5439)"],
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
@@ -149,16 +153,26 @@ CONNECTION_SOURCE_FIELDS = {
             '(Optional) GCP Spanner API endpoint (e.g. "https://spanner-mypsc.p.googleapis.com")',
         ],
     ],
+    consts.SOURCE_TYPE_SYBASE: [
+        ["host", "Sybase host (default localhost)"],
+        ["port", "Sybase port to connect on (default 5000)"],
+        ["user", "User used to connect"],
+        ["password", "Password for supplied user"],
+        ["database", "Database to connect to (default master)"],
+        ["odbc_driver", "ODBC driver name"],
+        ["query", "Connection query parameters"],
+        ["url", "Sybase SQLAlchemy connection URL"],
+    ],
     consts.SOURCE_TYPE_FILESYSTEM: [
         ["table_name", "Table name to use as reference for file data"],
         ["file_path", "The local, s3, or GCS file path to the data"],
         ["file_type", "The file type of the file. 'csv', 'orc', 'parquet' or 'json'"],
     ],
     consts.SOURCE_TYPE_IMPALA: [
-        ["host", "Desired Impala host"],
-        ["port", "Desired Impala port (10000 if not provided)"],
-        ["database", "Desired Impala database (default if not provided)"],
-        ["auth_mechanism", "Desired Impala auth mechanism (PLAIN if not provided)"],
+        ["host", "Impala host"],
+        ["port", "Impala port (10000 if not provided)"],
+        ["database", "Impala database (default if not provided)"],
+        ["auth_mechanism", "Impala auth mechanism (PLAIN if not provided)"],
         [
             "kerberos_service_name",
             "Desired Kerberos service name ('impala' if not provided)",
@@ -183,8 +197,8 @@ CONNECTION_SOURCE_FIELDS = {
         ["http_path", "URL path of HTTP proxy"],
     ],
     consts.SOURCE_TYPE_DB2: [
-        ["host", "Desired DB2 host"],
-        ["port", "Desired DB2 port (50000 if not provided)"],
+        ["host", "DB2 host"],
+        ["port", "DB2 port (50000 if not provided)"],
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
         ["database", "Database in DB2 to connect to"],
@@ -1308,21 +1322,16 @@ def _get_result_handler(rc_value: str, sa_file=None) -> dict:
     if config[0] in connections:
         # We received connection_name.results_table.
         conn_from_file = get_connection(config[0])
-        if conn_from_file[consts.SOURCE_TYPE] == consts.SOURCE_TYPE_BIGQUERY:
-            result_handler = {
-                consts.RH_TYPE: conn_from_file[consts.SOURCE_TYPE],
-                consts.PROJECT_ID: conn_from_file["project_id"],
-                consts.TABLE_ID: config[1],
-                consts.API_ENDPOINT: conn_from_file.get("api_endpoint", None),
-                consts.STORAGE_API_ENDPOINT: conn_from_file.get(
-                    "storage_api_endpoint", None
-                ),
-            }
-        elif conn_from_file[consts.SOURCE_TYPE] == consts.SOURCE_TYPE_POSTGRES:
+        if conn_from_file[consts.SOURCE_TYPE] in [
+            consts.SOURCE_TYPE_BIGQUERY,
+            consts.SOURCE_TYPE_POSTGRES,
+        ]:
             result_handler = {
                 consts.RH_TYPE: conn_from_file[consts.SOURCE_TYPE],
                 consts.TABLE_ID: config[1],
-                consts.RH_CONN: conn_from_file,
+                # Only store the connection name in the result handler to avoid accidentally
+                # storing credentials in config files.
+                consts.RH_CONN: config[0],
             }
         # TODO Add filesytem handler too.
         else:

{google_pso_data_validator-8.2.0 → google_pso_data_validator-8.4.0}/data_validation/clients.py

@@ -16,7 +16,7 @@
 from contextlib import contextmanager
 import copy
 import logging
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 import warnings
 
 import google.oauth2.service_account
@@ -56,6 +56,7 @@ IBIS_ALCHEMY_BACKENDS = [
     "mssql",
     "redshift",
     "snowflake",
+    "sybase",
 ]
 
 
@@ -93,19 +94,32 @@ try:
 except ImportError:
     db2_connect = _raise_missing_client_error("pip install ibm_db_sa")
 
+# Sybase requires sqlalchemy_sybase package.
+try:
+    from third_party.ibis.ibis_sybase.api import sybase_connect
+except ImportError:
+    sybase_connect = _raise_missing_client_error("pip install sqlalchemy_sybase")
+
 
 def get_google_bigquery_client(
-    project_id: str, credentials=None, api_endpoint: str = None
+    project_id: str,
+    credentials=None,
+    api_endpoint: Optional[str] = None,
+    quota_project_id: Optional[str] = None,
 ):
     info = client_info.get_http_client_info()
     job_config = bigquery.QueryJobConfig(
         connection_properties=[bigquery.ConnectionProperty("time_zone", "UTC")]
     )
+    effective_project = quota_project_id or project_id
     options = None
-    if api_endpoint:
-        options = client_options.ClientOptions(api_endpoint=api_endpoint)
+    if api_endpoint or quota_project_id:
+        options = client_options.ClientOptions(
+            api_endpoint=api_endpoint,
+            quota_project_id=quota_project_id if quota_project_id else None,
+        )
     return bigquery.Client(
-        project=project_id,
+        project=effective_project,
         client_info=info,
         credentials=credentials,
         default_query_job_config=job_config,
@@ -113,10 +127,17 @@ def get_google_bigquery_client(
     )
 
 
-def _get_google_bqstorage_client(credentials=None, api_endpoint: str = None):
+def _get_google_bqstorage_client(
+    credentials=None,
+    api_endpoint: Optional[str] = None,
+    quota_project_id: Optional[str] = None,
+):
     options = None
-    if api_endpoint:
-        options = client_options.ClientOptions(api_endpoint=api_endpoint)
+    if api_endpoint or quota_project_id:
+        options = client_options.ClientOptions(
+            api_endpoint=api_endpoint,
+            quota_project_id=quota_project_id if quota_project_id else None,
+        )
     from google.cloud import bigquery_storage_v1 as bigquery_storage
 
     return bigquery_storage.BigQueryReadClient(
@@ -129,20 +150,26 @@ def get_bigquery_client(
     project_id: str,
     dataset_id: str = "",
     credentials=None,
-    api_endpoint: str = None,
-    storage_api_endpoint: str = None,
+    api_endpoint: Optional[str] = None,
+    storage_api_endpoint: Optional[str] = None,
+    client_project_id: Optional[str] = None,
 ):
     google_client = get_google_bigquery_client(
-        project_id, credentials=credentials, api_endpoint=api_endpoint
+        project_id,
+        credentials=credentials,
+        api_endpoint=api_endpoint,
+        quota_project_id=client_project_id,
     )
     bqstorage_client = None
     if storage_api_endpoint:
         bqstorage_client = _get_google_bqstorage_client(
-            credentials=credentials, api_endpoint=storage_api_endpoint
+            credentials=credentials,
+            api_endpoint=storage_api_endpoint,
+            quota_project_id=client_project_id,
         )
 
     return bigquery_connect(
-        project_id=project_id,
+        project_id=project_id or client_project_id,
         dataset_id=dataset_id,
         credentials=credentials,
         bigquery_client=google_client,
@@ -202,6 +229,7 @@ def get_ibis_table(client, schema_name, table_name, database_name=None):
         "db2",
         "mssql",
         "redshift",
+        "sybase",
     ]:
         return client.table(table_name, database=database_name, schema=schema_name)
     elif client.name == "pandas":
@@ -400,5 +428,6 @@ CLIENT_LOOKUP = {
     consts.SOURCE_TYPE_MSSQL: mssql_connect,
     consts.SOURCE_TYPE_SNOWFLAKE: snowflake_connect,
     consts.SOURCE_TYPE_SPANNER: spanner_connect,
+    consts.SOURCE_TYPE_SYBASE: sybase_connect,
     consts.SOURCE_TYPE_DB2: db2_connect,
 }

{google_pso_data_validator-8.2.0 → google_pso_data_validator-8.4.0}/data_validation/config_manager.py

@@ -71,7 +71,7 @@ class ConfigManager(object):
         """Return config object."""
         return self._config
 
-    def get_source_connection(self):
+    def get_source_connection(self) -> dict:
         """Return source connection object."""
         if not self._source_conn:
             if self._config.get(consts.CONFIG_SOURCE_CONN):
@@ -82,7 +82,7 @@ class ConfigManager(object):
 
         return self._source_conn
 
-    def get_target_connection(self):
+    def get_target_connection(self) -> dict:
         """Return target connection object."""
         if not self._target_conn:
             if self._config.get(consts.CONFIG_TARGET_CONN):
@@ -671,20 +671,38 @@ class ConfigManager(object):
 
         This is because SQL Server text does not support the len() function to get the length in characters.
         Instead we must fall back to using ByteLength."""
+        return self._is_sql_server_type(
+            source_column_name, target_column_name, ["text", "ntext"]
+        )
+
+    def _is_sql_server_image(
+        self, source_column_name: str, target_column_name: str
+    ) -> bool:
+        """Returns True when either source or target column is of SQL Server image data type.
+
+        This is because SQL Server image is deprecated and not supported by a number of SQL functions.
+        """
+        return self._is_sql_server_type(
+            source_column_name, target_column_name, ["image"]
+        )
+
+    def _is_sql_server_type(
+        self, source_column_name: str, target_column_name: str, type_list: List[str]
+    ) -> bool:
+        """Returns True when either source or target column is of a SQL Server type listed in type_list."""
 
         raw_source_types = self.get_source_raw_data_types()
         raw_target_types = self.get_target_raw_data_types()
-        text_types = ("text", "ntext")
         return bool(
             (
                 self.source_client.name == "mssql"
                 and raw_source_types
-                and raw_source_types.get(source_column_name, [None])[0] in text_types
+                and raw_source_types.get(source_column_name, [None])[0] in type_list
             )
             or (
                 self.target_client.name == "mssql"
                 and raw_target_types
-                and raw_target_types.get(target_column_name, [None])[0] in text_types
+                and raw_target_types.get(target_column_name, [None])[0] in type_list
             )
         )
 
@@ -1075,6 +1093,11 @@ class ConfigManager(object):
                         f"Skipping {agg_type} on {column} due to data type: {column_type}"
                     )
                 continue
+            elif self._is_sql_server_image(column, column):
+                logging.info(
+                    f"Skipping {agg_type} on {column} due to SQL Server image data type"
+                )
+                continue
 
             if require_pre_agg_calc_field(
                 column_type, target_column_type, agg_type, cast_to_bigint
@@ -1252,8 +1275,25 @@ class ConfigManager(object):
             raise ValueError(
                 "Exclude columns flag cannot be present with column list '*'"
             )
+
         return casefold_columns
 
+    def _filter_columns_by_data_type(
+        self, source_columns: dict, target_columns: dict
+    ) -> tuple:
+        """Filter out columns with a data type that is incompatible with DVT."""
+        result_source_columns = source_columns.copy()
+        result_target_columns = target_columns.copy()
+        for source_column, target_column in zip(source_columns, target_columns):
+            if self._is_sql_server_image(source_column, target_column):
+                logging.info(
+                    f"Skipping column {source_column} due to SQL Server image data type"
+                )
+                result_source_columns.pop(source_column)
+                result_target_columns.pop(target_column)
+
+        return result_source_columns, result_target_columns
+
     def build_dependent_aliases(self, calc_type: str, col_list=None) -> List[Dict]:
         """This is a utility function for determining the required depth of all fields"""
         source_table = self.get_source_ibis_calculated_table()
@@ -1269,6 +1309,13 @@ class ConfigManager(object):
             casefold_target_columns, col_list
         )
 
+        (
+            casefold_source_columns,
+            casefold_target_columns,
+        ) = self._filter_columns_by_data_type(
+            casefold_source_columns, casefold_target_columns
+        )
+
         column_aliases = {}
         col_names = []
         for i, calc in enumerate(self._get_order_of_operations(calc_type)):
@@ -1325,6 +1372,9 @@ class ConfigManager(object):
         casefold_source_columns = self._filter_columns_by_column_list(
             casefold_source_columns, col_list, exclude_cols=exclude_cols
         )
+        casefold_source_columns, _ = self._filter_columns_by_data_type(
+            casefold_source_columns, casefold_source_columns
+        )
         return casefold_source_columns
 
     def auto_list_primary_keys(self) -> list:

{google_pso_data_validator-8.2.0 → google_pso_data_validator-8.4.0}/data_validation/consts.py

@@ -139,6 +139,7 @@ SOURCE_TYPE_POSTGRES = "Postgres"
 SOURCE_TYPE_REDSHIFT = "Redshift"
 SOURCE_TYPE_SNOWFLAKE = "Snowflake"
 SOURCE_TYPE_SPANNER = "Spanner"
+SOURCE_TYPE_SYBASE = "Sybase"
 SOURCE_TYPE_TERADATA = "Teradata"
 
 # BigQuery Result Handler Configs
@@ -149,6 +150,7 @@ TABLE_ID = "table_id"
 GOOGLE_SERVICE_ACCOUNT_KEY_PATH = "google_service_account_key_path"
 API_ENDPOINT = "api_endpoint"
 STORAGE_API_ENDPOINT = "storage_api_endpoint"
+CLIENT_PROJECT_ID = "client_project_id"
 
 # Result Handler Output Table Fields
 VALIDATION_TYPE = "validation_type"
@@ -262,3 +264,6 @@ CALC_FIELD_LENGTH = "length"
 CALC_FIELD_PADDED_CHAR_LENGTH = "padded_char_length"
 CALC_FIELD_RSTRIP = "rstrip"
 CALC_FIELD_UPPER = "upper"
+
+# Backends that do not support window functions.
+NO_WINDOW_FUNCTION_SUPPORT = [SOURCE_TYPE_SYBASE]

{google_pso_data_validator-8.2.0 → google_pso_data_validator-8.4.0}/data_validation/data_validation.py

@@ -16,9 +16,11 @@ import json
 import logging
 import warnings
 from concurrent.futures import ThreadPoolExecutor
-import ibis.backends.pandas
-import pandas
 import uuid
+from typing import TYPE_CHECKING
+
+import ibis
+import pandas
 
 from data_validation import combiner, consts, metadata, util
 from data_validation.config_manager import ConfigManager
@@ -26,6 +28,9 @@ from data_validation.query_builder.random_row_builder import RandomRowBuilder
 from data_validation.schema_validation import SchemaValidation
 from data_validation.validation_builder import ValidationBuilder
 
+if TYPE_CHECKING:
+    from ibis.backends.base import BaseBackend
+
 """ The DataValidation class is where the code becomes source/target aware
 
     The class builds specific source and target clients and is likely where someone would go to
@@ -43,8 +48,8 @@ class DataValidation(object):
         schema_validator=None,
         result_handler=None,
         verbose=False,
-        source_client: ibis.backends.base.BaseBackend = None,
-        target_client: ibis.backends.base.BaseBackend = None,
+        cached_source_client: "BaseBackend" = None,
+        cached_target_client: "BaseBackend" = None,
     ):
         """Initialize a DataValidation client
 
@@ -54,19 +59,21 @@ class DataValidation(object):
             schema_validator (SchemaValidation): Optional instance of a SchemaValidation.
             result_handler (ResultHandler): Optional instance of as ResultHandler client.
             verbose (bool): If verbose, the Data Validation client will print the queries run.
-            source_client: Optional client to avoid unnecessary connections,
-            target_client: Optional client to avoid unnecessary connections,
+            cached_source_client: Optional client to avoid unnecessary connections,
+            cached_target_client: Optional client to avoid unnecessary connections,
         """
         self.verbose = verbose
-        self._fresh_connections = not bool(source_client and target_client)
+        self._fresh_connections = not bool(
+            cached_source_client and cached_target_client
+        )
 
         # Data Client Management
         self.config = config
 
         self.config_manager = ConfigManager(
             config,
-            source_client=source_client,
-            target_client=target_client,
+            source_client=cached_source_client,
+            target_client=cached_target_client,
             verbose=self.verbose,
         )
 

{google_pso_data_validator-8.2.0 → google_pso_data_validator-8.4.0}/data_validation/exceptions.py

@@ -27,3 +27,7 @@ class ResultHandlerException(Exception):
 
 class SchemaValidationException(ValidationException):
     pass
+
+
+class PartitionBuilderException(ValidationException):
+    pass

{google_pso_data_validator-8.2.0 → google_pso_data_validator-8.4.0}/data_validation/find_tables.py

@@ -60,15 +60,9 @@ def _compare_match_tables(source_table_map, target_table_map, score_cutoff=0.8)
     return table_configs
 
 
-def _get_table_map(
-    client: "ibis.backends.base.BaseBackend", allowed_schemas=None, include_views=False
-):
-    """Return dict with searchable keys for table matching."""
+def _get_table_map_from_obj_list(table_objs: list) -> dict:
+    """Convert list of schema, table tuples into dict with searchable keys for table matching."""
     table_map = {}
-    table_objs = clients.get_all_tables(
-        client, allowed_schemas=allowed_schemas, tables_only=(not include_views)
-    )
-
     for table_obj in table_objs:
         table_key = ".".join([t for t in table_obj if t])
         table_map[table_key] = {
@@ -76,9 +70,25 @@ def _get_table_map(
             consts.CONFIG_TABLE_NAME: table_obj[1],
         }
 
+    # Post process table_map and lower case table_keys, if possible.
+    for table_key in [_ for _ in table_map if _ != _.casefold()]:
+        if table_key.casefold() not in table_map:
+            # Only lower case the key if there isn't one already.
+            table_map[table_key.casefold()] = table_map.pop(table_key)
+
     return table_map
 
 
+def _get_table_map(
+    client: "ibis.backends.base.BaseBackend", allowed_schemas=None, include_views=False
+) -> dict:
+    """Return dict with searchable keys for table matching."""
+    table_objs = clients.get_all_tables(
+        client, allowed_schemas=allowed_schemas, tables_only=(not include_views)
+    )
+    return _get_table_map_from_obj_list(table_objs)
+
+
 def get_mapped_table_configs(
     source_client: "ibis.backends.base.BaseBackend",
     target_client: "ibis.backends.base.BaseBackend",