google-pso-data-validator 8.3.0__tar.gz → 8.5.0__tar.gz

{google_pso_data_validator-8.3.0/google_pso_data_validator.egg-info → google_pso_data_validator-8.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 8.3.0
+Version: 8.5.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -28,7 +28,7 @@ Requires-Dist: jellyfish>=1.1.0
 Requires-Dist: pandas
 Requires-Dist: parsy>=2.1
 Requires-Dist: psycopg2-binary>=2.9.9
-Requires-Dist: pyarrow==14.0.1
+Requires-Dist: pyarrow
 Requires-Dist: pydata-google-auth>=1.8.2
 Requires-Dist: PyMySQL>=1.1.1
 Requires-Dist: PyYAML>=6.0.2
@@ -45,6 +45,7 @@ Requires-Dist: freezegun; extra == "develop"
 Requires-Dist: pyfakefs; extra == "develop"
 Requires-Dist: pytest; extra == "develop"
 Requires-Dist: pytest-cov; extra == "develop"
+Requires-Dist: pytest-rerunfailures; extra == "develop"
 Requires-Dist: pytest-timeout; extra == "develop"
 Dynamic: author
 Dynamic: author-email
@@ -92,8 +93,9 @@ DVT supports the following connection types:
 *   [Postgres](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#postgres)
 *   [Redshift](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#redshift)
 *   [Spanner](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#google-spanner)
-*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 *   [Snowflake](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#snowflake)
+*   [Sybase ASE](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#sybase)
+*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 
 The [Connections](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md) page provides details about how to create
 and list connections for the validation tool.
@@ -552,6 +554,10 @@ data-validation
                         Comma-separated key value pair labels for the run.
   [--format or -fmt FORMAT]
                         Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
+  [--use-random-row or -rr]
+                        Finds a set of random rows of the first primary key supplied.
+  [--random-row-batch-size or -rbs]
+                        Row batch size used for random row filters (default 10,000).
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
   [--case-insensitive-match, -cim]

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0}/README.md

@@ -33,8 +33,9 @@ DVT supports the following connection types:
 *   [Postgres](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#postgres)
 *   [Redshift](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#redshift)
 *   [Spanner](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#google-spanner)
-*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 *   [Snowflake](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#snowflake)
+*   [Sybase ASE](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#sybase)
+*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 
 The [Connections](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md) page provides details about how to create
 and list connections for the validation tool.
@@ -493,6 +494,10 @@ data-validation
                         Comma-separated key value pair labels for the run.
   [--format or -fmt FORMAT]
                         Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
+  [--use-random-row or -rr]
+                        Finds a set of random rows of the first primary key supplied.
+  [--random-row-batch-size or -rbs]
+                        Row batch size used for random row filters (default 10,000).
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
   [--case-insensitive-match, -cim]

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0}/data_validation/__main__.py

@@ -596,6 +596,7 @@ def partition_and_store_config_files(args: "Namespace") -> None:
         setattr(args, "custom_query_type", "row")
         config_managers = build_config_managers_from_args(args, consts.CUSTOM_QUERY)
     partition_builder = PartitionBuilder(config_managers, args)
+    partition_builder.check_partition_configs()
     partition_builder.partition_configs()
 
 

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0}/data_validation/cli_tools.py

@@ -73,6 +73,10 @@ if TYPE_CHECKING:
 CONNECTION_SOURCE_FIELDS = {
     consts.SOURCE_TYPE_BIGQUERY: [
         ["project_id", "GCP Project to use for BigQuery"],
+        [
+            "client_project_id",
+            "(Optional) BigQuery job/billing project (can differ from data project)",
+        ],
         ["google_service_account_key_path", "(Optional) GCP SA Key Path"],
         [
             "api_endpoint",
@@ -84,7 +88,7 @@ CONNECTION_SOURCE_FIELDS = {
         ],
     ],
     consts.SOURCE_TYPE_TERADATA: [
-        ["host", "Desired Teradata host"],
+        ["host", "Teradata host"],
         ["port", "Teradata port to connect on"],
         ["user_name", "User used to connect"],
         ["password", "Password for supplied user"],
@@ -93,7 +97,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["json_params", "(Optional) Additional teradatasql JSON string parameters"],
     ],
     consts.SOURCE_TYPE_ORACLE: [
-        ["host", "Desired Oracle host"],
+        ["host", "Oracle host"],
         ["port", "Oracle port to connect on"],
         ["user", "User used to connect"],
         ["thick_mode", "Flag indicating thick_mode"],
@@ -103,7 +107,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["connect_args", "(Optional) Additional connection argument mapping"],
     ],
     consts.SOURCE_TYPE_MSSQL: [
-        ["host", "Desired SQL Server host (default localhost)"],
+        ["host", "SQL Server host (default localhost)"],
         ["port", "SQL Server port to connect on (default 1433)"],
         ["user", "User used to connect"],
         ["password", "Password for supplied user"],
@@ -112,7 +116,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["url", "SQL Server SQLAlchemy connection URL"],
     ],
     consts.SOURCE_TYPE_MYSQL: [
-        ["host", "Desired MySQL host (default localhost)"],
+        ["host", "MySQL host (default localhost)"],
         ["port", "MySQL port to connect on (default 3306)"],
         ["user", "User used to connect"],
         ["password", "Password for supplied user"],
@@ -123,17 +127,19 @@ CONNECTION_SOURCE_FIELDS = {
         ["password", "Password for authentication of user"],
         ["account", "Snowflake account to connect to"],
         ["database", "Database in snowflake to connect to"],
+        ["warehouse", "Warehouse in snowflake to connect to"],
+        ["role", "Role in snowflake to connect as"],
         ["connect_args", "(Optional) Additional connection argument mapping"],
     ],
     consts.SOURCE_TYPE_POSTGRES: [
-        ["host", "Desired PostgreSQL host."],
+        ["host", "PostgreSQL host."],
         ["port", "PostgreSQL port to connect on (e.g. 5432)"],
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
         ["database", "Database in PostgreSQL to connect to (default postgres)"],
     ],
     consts.SOURCE_TYPE_REDSHIFT: [
-        ["host", "Desired Redshift host."],
+        ["host", "Redshift host."],
         ["port", "Redshift port to connect on (e.g. 5439)"],
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
@@ -149,16 +155,26 @@ CONNECTION_SOURCE_FIELDS = {
             '(Optional) GCP Spanner API endpoint (e.g. "https://spanner-mypsc.p.googleapis.com")',
         ],
     ],
+    consts.SOURCE_TYPE_SYBASE: [
+        ["host", "Sybase host (default localhost)"],
+        ["port", "Sybase port to connect on (default 5000)"],
+        ["user", "User used to connect"],
+        ["password", "Password for supplied user"],
+        ["database", "Database to connect to (default master)"],
+        ["odbc_driver", "ODBC driver name"],
+        ["query", "Connection query parameters"],
+        ["url", "Sybase SQLAlchemy connection URL"],
+    ],
     consts.SOURCE_TYPE_FILESYSTEM: [
         ["table_name", "Table name to use as reference for file data"],
         ["file_path", "The local, s3, or GCS file path to the data"],
         ["file_type", "The file type of the file. 'csv', 'orc', 'parquet' or 'json'"],
     ],
     consts.SOURCE_TYPE_IMPALA: [
-        ["host", "Desired Impala host"],
-        ["port", "Desired Impala port (10000 if not provided)"],
-        ["database", "Desired Impala database (default if not provided)"],
-        ["auth_mechanism", "Desired Impala auth mechanism (PLAIN if not provided)"],
+        ["host", "Impala host"],
+        ["port", "Impala port (10000 if not provided)"],
+        ["database", "Impala database (default if not provided)"],
+        ["auth_mechanism", "Impala auth mechanism (PLAIN if not provided)"],
         [
             "kerberos_service_name",
             "Desired Kerberos service name ('impala' if not provided)",
@@ -183,8 +199,8 @@ CONNECTION_SOURCE_FIELDS = {
         ["http_path", "URL path of HTTP proxy"],
     ],
     consts.SOURCE_TYPE_DB2: [
-        ["host", "Desired DB2 host"],
-        ["port", "Desired DB2 port (50000 if not provided)"],
+        ["host", "DB2 host"],
+        ["port", "DB2 port (50000 if not provided)"],
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
         ["database", "Database in DB2 to connect to"],
@@ -646,8 +662,8 @@ def _configure_row_parser(
                 defaults, so most users do not need to use this option unless they encounter errors."""
         ),
     )
-    # Generate-table-partitions and custom-query does not support random row
-    if not (is_generate_partitions or is_custom_query):
+    # Generate-table-partitions does not support random row
+    if not is_generate_partitions:
         optional_arguments.add_argument(
             "--use-random-row",
             "-rr",
@@ -659,17 +675,18 @@ def _configure_row_parser(
             "-rbs",
             help="Row batch size used for random row filters (default 10,000).",
         )
-        # Generate table partitions follows a new argument spec where either the table names or queries can be provided, but not both.
-        # that is specified in configure_partition_parser. If we use the same spec for row and column validation, the custom query commands
-        # may get subsumed by validate and validate commands by specifying tables name or queries. Until this -tbls will be
-        # a required argument for validate row, validate column and validate schema.
-        required_arguments.add_argument(
-            "--tables-list",
-            "-tbls",
-            default=None,
-            required=True,
-            help="Comma separated tables list in the form 'schema.table=target_schema.target_table'",
-        )
+        if not is_custom_query:
+            # Generate table partitions follows a new argument spec where either the table names or queries can be provided, but not both.
+            # that is specified in configure_partition_parser. If we use the same spec for row and column validation, the custom query commands
+            # may get subsumed by validate and validate commands by specifying tables name or queries. Until this -tbls will be
+            # a required argument for validate row, validate column and validate schema.
+            required_arguments.add_argument(
+                "--tables-list",
+                "-tbls",
+                default=None,
+                required=True,
+                help="Comma separated tables list in the form 'schema.table=target_schema.target_table'",
+            )
 
     # Group for mutually exclusive required arguments. Either must be supplied
     mutually_exclusive_arguments = required_arguments.add_mutually_exclusive_group(

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0}/data_validation/clients.py

@@ -16,7 +16,7 @@
 from contextlib import contextmanager
 import copy
 import logging
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 import warnings
 
 import google.oauth2.service_account
@@ -56,6 +56,7 @@ IBIS_ALCHEMY_BACKENDS = [
     "mssql",
     "redshift",
     "snowflake",
+    "sybase",
 ]
 
 
@@ -93,19 +94,32 @@ try:
 except ImportError:
     db2_connect = _raise_missing_client_error("pip install ibm_db_sa")
 
+# Sybase requires sqlalchemy_sybase package.
+try:
+    from third_party.ibis.ibis_sybase.api import sybase_connect
+except ImportError:
+    sybase_connect = _raise_missing_client_error("pip install sqlalchemy_sybase")
+
 
 def get_google_bigquery_client(
-    project_id: str, credentials=None, api_endpoint: str = None
+    project_id: str,
+    credentials=None,
+    api_endpoint: Optional[str] = None,
+    quota_project_id: Optional[str] = None,
 ):
     info = client_info.get_http_client_info()
     job_config = bigquery.QueryJobConfig(
         connection_properties=[bigquery.ConnectionProperty("time_zone", "UTC")]
     )
+    effective_project = quota_project_id or project_id
     options = None
-    if api_endpoint:
-        options = client_options.ClientOptions(api_endpoint=api_endpoint)
+    if api_endpoint or quota_project_id:
+        options = client_options.ClientOptions(
+            api_endpoint=api_endpoint,
+            quota_project_id=quota_project_id if quota_project_id else None,
+        )
     return bigquery.Client(
-        project=project_id,
+        project=effective_project,
         client_info=info,
         credentials=credentials,
         default_query_job_config=job_config,
@@ -113,10 +127,17 @@ def get_google_bigquery_client(
     )
 
 
-def _get_google_bqstorage_client(credentials=None, api_endpoint: str = None):
+def _get_google_bqstorage_client(
+    credentials=None,
+    api_endpoint: Optional[str] = None,
+    quota_project_id: Optional[str] = None,
+):
     options = None
-    if api_endpoint:
-        options = client_options.ClientOptions(api_endpoint=api_endpoint)
+    if api_endpoint or quota_project_id:
+        options = client_options.ClientOptions(
+            api_endpoint=api_endpoint,
+            quota_project_id=quota_project_id if quota_project_id else None,
+        )
     from google.cloud import bigquery_storage_v1 as bigquery_storage
 
     return bigquery_storage.BigQueryReadClient(
@@ -129,20 +150,26 @@ def get_bigquery_client(
     project_id: str,
     dataset_id: str = "",
     credentials=None,
-    api_endpoint: str = None,
-    storage_api_endpoint: str = None,
+    api_endpoint: Optional[str] = None,
+    storage_api_endpoint: Optional[str] = None,
+    client_project_id: Optional[str] = None,
 ):
     google_client = get_google_bigquery_client(
-        project_id, credentials=credentials, api_endpoint=api_endpoint
+        project_id,
+        credentials=credentials,
+        api_endpoint=api_endpoint,
+        quota_project_id=client_project_id,
     )
     bqstorage_client = None
     if storage_api_endpoint:
         bqstorage_client = _get_google_bqstorage_client(
-            credentials=credentials, api_endpoint=storage_api_endpoint
+            credentials=credentials,
+            api_endpoint=storage_api_endpoint,
+            quota_project_id=client_project_id,
         )
 
     return bigquery_connect(
-        project_id=project_id,
+        project_id=project_id or client_project_id,
         dataset_id=dataset_id,
         credentials=credentials,
         bigquery_client=google_client,
@@ -202,6 +229,7 @@ def get_ibis_table(client, schema_name, table_name, database_name=None):
         "db2",
         "mssql",
         "redshift",
+        "sybase",
     ]:
         return client.table(table_name, database=database_name, schema=schema_name)
     elif client.name == "pandas":
@@ -261,7 +289,7 @@ def list_tables(client, schema_name, tables_only=True):
         if tables_only and client.name != "pandas"
         else client.list_tables
     )
-    if client.name in ["db2", "redshift", "snowflake", "pandas"]:
+    if client.name in ["redshift", "snowflake", "pandas"]:
         return fn()
     return fn(database=schema_name)
 
@@ -400,5 +428,6 @@ CLIENT_LOOKUP = {
     consts.SOURCE_TYPE_MSSQL: mssql_connect,
     consts.SOURCE_TYPE_SNOWFLAKE: snowflake_connect,
     consts.SOURCE_TYPE_SPANNER: spanner_connect,
+    consts.SOURCE_TYPE_SYBASE: sybase_connect,
     consts.SOURCE_TYPE_DB2: db2_connect,
 }

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0}/data_validation/config_manager.py

@@ -23,7 +23,10 @@ from data_validation import clients, consts, gcs_helper, state_manager
 from data_validation.result_handlers.factory import build_result_handler
 from data_validation.validation_builder import ValidationBuilder
 
+from third_party.ibis.ibis_addon.api import db2_type_string_length
+
 if TYPE_CHECKING:
+    from ibis.backends.base import BaseBackend
     import ibis.expr.types.Table
 
 
@@ -71,7 +74,7 @@ class ConfigManager(object):
         """Return config object."""
         return self._config
 
-    def get_source_connection(self):
+    def get_source_connection(self) -> dict:
         """Return source connection object."""
         if not self._source_conn:
             if self._config.get(consts.CONFIG_SOURCE_CONN):
@@ -82,7 +85,7 @@ class ConfigManager(object):
 
         return self._source_conn
 
-    def get_target_connection(self):
+    def get_target_connection(self) -> dict:
         """Return target connection object."""
         if not self._target_conn:
             if self._config.get(consts.CONFIG_TARGET_CONN):
@@ -1160,6 +1163,8 @@ class ConfigManager(object):
             calculated_config.update(custom_params)
         elif calc_type == consts.CONFIG_CAST and custom_params:
             calculated_config[consts.CONFIG_DEFAULT_CAST] = custom_params
+        elif calc_type == consts.CALC_FIELD_IFNULL and custom_params:
+            calculated_config.update(custom_params)
 
         return calculated_config
 
@@ -1294,7 +1299,45 @@ class ConfigManager(object):
 
         return result_source_columns, result_target_columns
 
-    def build_dependent_aliases(self, calc_type: str, col_list=None) -> List[Dict]:
+    def _string_column_ifnull_limits(
+        self,
+        client: "BaseBackend",
+        casefold_columns: dict,
+        ibis_table: "ibis.expr.types.Table",
+        raw_types: dict,
+        string_column_ifnull_limits: dict,
+    ):
+        """Defines IfNull replacement token length limits (due to Db2 limitations).
+
+        This is because in Db2 a coalesce of a column cannot exceed the length of the initial argument resulting in:
+            String data right truncation. SQLSTATE=22001
+
+        We need to ensure both source and target systems use the same replacement token therefore these limits are used for both.
+
+        Mutates string_column_ifnull_limits to contain a dict of column names with replacement token length limits:
+            {"col1": 10, "col2": 12, "col_unlimited": None}
+        """
+        if not casefold_columns or client.name != "db2":
+            return {}
+        table_schema = {k: v for k, v in ibis_table.schema().items()}
+        for casefold_column, column in casefold_columns.items():
+            ifnull_limit = db2_type_string_length(
+                table_schema[casefold_column], raw_types.get(column, [])
+            )
+            if (
+                not string_column_ifnull_limits.get(casefold_column, None)
+                # If the current limit is shorter than an existing one then overwrite the value.
+                or (
+                    ifnull_limit
+                    and ifnull_limit < string_column_ifnull_limits[casefold_column]
+                )
+            ):
+                string_column_ifnull_limits[casefold_column] = ifnull_limit
+        return string_column_ifnull_limits
+
+    def build_dependent_aliases(
+        self, calc_type: str, col_list: Optional[list] = None
+    ) -> List[Dict]:
         """This is a utility function for determining the required depth of all fields"""
         source_table = self.get_source_ibis_calculated_table()
         target_table = self.get_target_ibis_calculated_table()
@@ -1318,6 +1361,23 @@ class ConfigManager(object):
 
         column_aliases = {}
         col_names = []
+        # Get any ifnull token limits from either the source or target system.
+        # Most engines have no requirement for limits and will return an empty dict.
+        string_column_ifnull_limits = {}
+        self._string_column_ifnull_limits(
+            self.source_client,
+            casefold_source_columns,
+            source_table,
+            self.get_source_raw_data_types(),
+            string_column_ifnull_limits,
+        )
+        self._string_column_ifnull_limits(
+            self.target_client,
+            casefold_target_columns,
+            target_table,
+            self.get_target_raw_data_types(),
+            string_column_ifnull_limits,
+        )
         for i, calc in enumerate(self._get_order_of_operations(calc_type)):
             if i == 0:
                 previous_level = [x for x in casefold_source_columns.keys()]
@@ -1331,7 +1391,7 @@ class ConfigManager(object):
                 col["calc_type"] = calc
                 col["depth"] = i
                 name = col["name"]
-                # need to capture all aliases at the previous level. probably name concat__all
+                # Need to capture all aliases at the previous level. probably name concat__all
                 column_aliases[name] = i
                 col_names.append(col)
             else:
@@ -1343,6 +1403,16 @@ class ConfigManager(object):
                     col["name"] = self._prefix_calc_col_name(column, calc, j)
                     col["calc_type"] = calc
                     col["depth"] = i
+                    if (
+                        calc == consts.CALC_FIELD_IFNULL
+                        and string_column_ifnull_limits.get(column)
+                    ):
+                        # Trim the replacement token to the max length allowed for the column.
+                        col["calc_params"] = {
+                            consts.CALC_FIELD_IFNULL_DEFAULT: consts.CALC_FIELD_IFNULL_DEFAULT_STRING[
+                                0 : string_column_ifnull_limits[column]
+                            ]
+                        }
 
                     if i == 0:
                         # If depth 0, get raw column name with correct casing
@@ -1363,6 +1433,11 @@ class ConfigManager(object):
                     name = col["name"]
                     column_aliases[name] = i
                     col_names.append(col)
+                    if string_column_ifnull_limits.get(column):
+                        # Keep track of column limits as we move through aliased expressions.
+                        string_column_ifnull_limits[name] = string_column_ifnull_limits[
+                            column
+                        ]
         return col_names
 
     def build_comp_fields(self, col_list: list, exclude_cols: bool) -> dict:

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0}/data_validation/consts.py

@@ -139,6 +139,7 @@ SOURCE_TYPE_POSTGRES = "Postgres"
 SOURCE_TYPE_REDSHIFT = "Redshift"
 SOURCE_TYPE_SNOWFLAKE = "Snowflake"
 SOURCE_TYPE_SPANNER = "Spanner"
+SOURCE_TYPE_SYBASE = "Sybase"
 SOURCE_TYPE_TERADATA = "Teradata"
 
 # BigQuery Result Handler Configs
@@ -149,6 +150,7 @@ TABLE_ID = "table_id"
 GOOGLE_SERVICE_ACCOUNT_KEY_PATH = "google_service_account_key_path"
 API_ENDPOINT = "api_endpoint"
 STORAGE_API_ENDPOINT = "storage_api_endpoint"
+CLIENT_PROJECT_ID = "client_project_id"
 
 # Result Handler Output Table Fields
 VALIDATION_TYPE = "validation_type"
@@ -258,7 +260,12 @@ CALC_FIELD_BYTE_LENGTH = "byte_length"
 CALC_FIELD_EPOCH_SECONDS = "epoch_seconds"
 CALC_FIELD_HASH = "hash"
 CALC_FIELD_IFNULL = "ifnull"
+CALC_FIELD_IFNULL_DEFAULT = "default_null_string"
+CALC_FIELD_IFNULL_DEFAULT_STRING = "DEFAULT_REPLACEMENT_STRING"
 CALC_FIELD_LENGTH = "length"
 CALC_FIELD_PADDED_CHAR_LENGTH = "padded_char_length"
 CALC_FIELD_RSTRIP = "rstrip"
 CALC_FIELD_UPPER = "upper"
+
+# Backends that do not support window functions.
+NO_WINDOW_FUNCTION_SUPPORT = [SOURCE_TYPE_SYBASE]

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0}/data_validation/exceptions.py

@@ -27,3 +27,7 @@ class ResultHandlerException(Exception):
 
 class SchemaValidationException(ValidationException):
     pass
+
+
+class PartitionBuilderException(ValidationException):
+    pass

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0}/data_validation/partition_builder.py

@@ -20,7 +20,7 @@ import re
 import datetime
 from typing import List, Dict, TYPE_CHECKING
 
-from data_validation import cli_tools, consts, util
+from data_validation import cli_tools, consts, exceptions, util
 from data_validation.config_manager import ConfigManager
 from data_validation.query_builder.partition_row_builder import PartitionRowBuilder
 from data_validation.validation_builder import ValidationBuilder
@@ -105,6 +105,15 @@ class PartitionBuilder:
         yaml_configs_list = self._add_partition_filters(partition_filters)
         self._store_partitions(yaml_configs_list)
 
+    def check_partition_configs(self) -> None:
+        for config_manager in self.config_managers:
+            source_type = config_manager.get_source_connection()[consts.SOURCE_TYPE]
+            if source_type in consts.NO_WINDOW_FUNCTION_SUPPORT:
+                raise exceptions.PartitionBuilderException(
+                    f"Source client {source_type} does not support window functions, "
+                    "please use this connection as a target connection for generate-table-partitions."
+                )
+
     @staticmethod
     def _extract_where(table_expr: "IbisTable", client: "BaseBackend") -> str:
         """Given a ibis table expression with a filter (i.e. WHERE) clause, this function extracts the

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0}/data_validation/query_builder/query_builder.py

@@ -341,7 +341,10 @@ class CalculatedField(object):
     @staticmethod
     def ifnull(config, fields):
         default_null_string = ibis.literal(
-            config.get("default_null_string", "DEFAULT_REPLACEMENT_STRING")
+            config.get(
+                consts.CALC_FIELD_IFNULL_DEFAULT,
+                consts.CALC_FIELD_IFNULL_DEFAULT_STRING,
+            )
         )
         fields = [fields[0], default_null_string]
         return CalculatedField(

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0}/data_validation/query_builder/random_row_builder.py

@@ -30,9 +30,10 @@ RANDOM_SORT_SUPPORTS = [
     "mssql",
     "db2",
     "mysql",
-    "spanner",
     "redshift",
     "snowflake",
+    "spanner",
+    "sybase",
 ]
 
 

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0/google_pso_data_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 8.3.0
+Version: 8.5.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -28,7 +28,7 @@ Requires-Dist: jellyfish>=1.1.0
 Requires-Dist: pandas
 Requires-Dist: parsy>=2.1
 Requires-Dist: psycopg2-binary>=2.9.9
-Requires-Dist: pyarrow==14.0.1
+Requires-Dist: pyarrow
 Requires-Dist: pydata-google-auth>=1.8.2
 Requires-Dist: PyMySQL>=1.1.1
 Requires-Dist: PyYAML>=6.0.2
@@ -45,6 +45,7 @@ Requires-Dist: freezegun; extra == "develop"
 Requires-Dist: pyfakefs; extra == "develop"
 Requires-Dist: pytest; extra == "develop"
 Requires-Dist: pytest-cov; extra == "develop"
+Requires-Dist: pytest-rerunfailures; extra == "develop"
 Requires-Dist: pytest-timeout; extra == "develop"
 Dynamic: author
 Dynamic: author-email
@@ -92,8 +93,9 @@ DVT supports the following connection types:
 *   [Postgres](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#postgres)
 *   [Redshift](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#redshift)
 *   [Spanner](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#google-spanner)
-*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 *   [Snowflake](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#snowflake)
+*   [Sybase ASE](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#sybase)
+*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 
 The [Connections](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md) page provides details about how to create
 and list connections for the validation tool.
@@ -552,6 +554,10 @@ data-validation
                         Comma-separated key value pair labels for the run.
   [--format or -fmt FORMAT]
                         Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
+  [--use-random-row or -rr]
+                        Finds a set of random rows of the first primary key supplied.
+  [--random-row-batch-size or -rbs]
+                        Row batch size used for random row filters (default 10,000).
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
   [--case-insensitive-match, -cim]

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.5.0}/google_pso_data_validator.egg-info/SOURCES.txt

@@ -85,6 +85,11 @@ third_party/ibis/ibis_redshift/compiler.py
 third_party/ibis/ibis_snowflake/__init__.py
 third_party/ibis/ibis_snowflake/api.py
 third_party/ibis/ibis_snowflake/datatypes.py
+third_party/ibis/ibis_sybase/__init__.py
+third_party/ibis/ibis_sybase/api.py
+third_party/ibis/ibis_sybase/compiler.py
+third_party/ibis/ibis_sybase/datatypes.py
+third_party/ibis/ibis_sybase/registry.py
 third_party/ibis/ibis_teradata/__init__.py
 third_party/ibis/ibis_teradata/api.py
 third_party/ibis/ibis_teradata/compiler.py