google-pso-data-validator 8.3.0__tar.gz → 8.4.0__tar.gz

{google_pso_data_validator-8.3.0/google_pso_data_validator.egg-info → google_pso_data_validator-8.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 8.3.0
+Version: 8.4.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -28,7 +28,7 @@ Requires-Dist: jellyfish>=1.1.0
 Requires-Dist: pandas
 Requires-Dist: parsy>=2.1
 Requires-Dist: psycopg2-binary>=2.9.9
-Requires-Dist: pyarrow==14.0.1
+Requires-Dist: pyarrow
 Requires-Dist: pydata-google-auth>=1.8.2
 Requires-Dist: PyMySQL>=1.1.1
 Requires-Dist: PyYAML>=6.0.2
@@ -45,6 +45,7 @@ Requires-Dist: freezegun; extra == "develop"
 Requires-Dist: pyfakefs; extra == "develop"
 Requires-Dist: pytest; extra == "develop"
 Requires-Dist: pytest-cov; extra == "develop"
+Requires-Dist: pytest-rerunfailures; extra == "develop"
 Requires-Dist: pytest-timeout; extra == "develop"
 Dynamic: author
 Dynamic: author-email
@@ -92,8 +93,9 @@ DVT supports the following connection types:
 *   [Postgres](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#postgres)
 *   [Redshift](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#redshift)
 *   [Spanner](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#google-spanner)
-*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 *   [Snowflake](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#snowflake)
+*   [Sybase ASE](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#sybase)
+*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 
 The [Connections](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md) page provides details about how to create
 and list connections for the validation tool.

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/README.md

@@ -33,8 +33,9 @@ DVT supports the following connection types:
 *   [Postgres](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#postgres)
 *   [Redshift](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#redshift)
 *   [Spanner](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#google-spanner)
-*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 *   [Snowflake](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#snowflake)
+*   [Sybase ASE](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#sybase)
+*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 
 The [Connections](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md) page provides details about how to create
 and list connections for the validation tool.

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/data_validation/__main__.py

@@ -596,6 +596,7 @@ def partition_and_store_config_files(args: "Namespace") -> None:
         setattr(args, "custom_query_type", "row")
         config_managers = build_config_managers_from_args(args, consts.CUSTOM_QUERY)
     partition_builder = PartitionBuilder(config_managers, args)
+    partition_builder.check_partition_configs()
     partition_builder.partition_configs()
 
 

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/data_validation/cli_tools.py

@@ -73,6 +73,10 @@ if TYPE_CHECKING:
 CONNECTION_SOURCE_FIELDS = {
     consts.SOURCE_TYPE_BIGQUERY: [
         ["project_id", "GCP Project to use for BigQuery"],
+        [
+            "client_project_id",
+            "(Optional) BigQuery job/billing project (can differ from data project)",
+        ],
         ["google_service_account_key_path", "(Optional) GCP SA Key Path"],
         [
             "api_endpoint",
@@ -84,7 +88,7 @@ CONNECTION_SOURCE_FIELDS = {
         ],
     ],
     consts.SOURCE_TYPE_TERADATA: [
-        ["host", "Desired Teradata host"],
+        ["host", "Teradata host"],
         ["port", "Teradata port to connect on"],
         ["user_name", "User used to connect"],
         ["password", "Password for supplied user"],
@@ -93,7 +97,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["json_params", "(Optional) Additional teradatasql JSON string parameters"],
     ],
     consts.SOURCE_TYPE_ORACLE: [
-        ["host", "Desired Oracle host"],
+        ["host", "Oracle host"],
         ["port", "Oracle port to connect on"],
         ["user", "User used to connect"],
         ["thick_mode", "Flag indicating thick_mode"],
@@ -103,7 +107,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["connect_args", "(Optional) Additional connection argument mapping"],
     ],
     consts.SOURCE_TYPE_MSSQL: [
-        ["host", "Desired SQL Server host (default localhost)"],
+        ["host", "SQL Server host (default localhost)"],
         ["port", "SQL Server port to connect on (default 1433)"],
         ["user", "User used to connect"],
         ["password", "Password for supplied user"],
@@ -112,7 +116,7 @@ CONNECTION_SOURCE_FIELDS = {
         ["url", "SQL Server SQLAlchemy connection URL"],
     ],
     consts.SOURCE_TYPE_MYSQL: [
-        ["host", "Desired MySQL host (default localhost)"],
+        ["host", "MySQL host (default localhost)"],
         ["port", "MySQL port to connect on (default 3306)"],
         ["user", "User used to connect"],
         ["password", "Password for supplied user"],
@@ -126,14 +130,14 @@ CONNECTION_SOURCE_FIELDS = {
         ["connect_args", "(Optional) Additional connection argument mapping"],
     ],
     consts.SOURCE_TYPE_POSTGRES: [
-        ["host", "Desired PostgreSQL host."],
+        ["host", "PostgreSQL host."],
         ["port", "PostgreSQL port to connect on (e.g. 5432)"],
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
         ["database", "Database in PostgreSQL to connect to (default postgres)"],
     ],
     consts.SOURCE_TYPE_REDSHIFT: [
-        ["host", "Desired Redshift host."],
+        ["host", "Redshift host."],
         ["port", "Redshift port to connect on (e.g. 5439)"],
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
@@ -149,16 +153,26 @@ CONNECTION_SOURCE_FIELDS = {
             '(Optional) GCP Spanner API endpoint (e.g. "https://spanner-mypsc.p.googleapis.com")',
         ],
     ],
+    consts.SOURCE_TYPE_SYBASE: [
+        ["host", "Sybase host (default localhost)"],
+        ["port", "Sybase port to connect on (default 5000)"],
+        ["user", "User used to connect"],
+        ["password", "Password for supplied user"],
+        ["database", "Database to connect to (default master)"],
+        ["odbc_driver", "ODBC driver name"],
+        ["query", "Connection query parameters"],
+        ["url", "Sybase SQLAlchemy connection URL"],
+    ],
     consts.SOURCE_TYPE_FILESYSTEM: [
         ["table_name", "Table name to use as reference for file data"],
         ["file_path", "The local, s3, or GCS file path to the data"],
         ["file_type", "The file type of the file. 'csv', 'orc', 'parquet' or 'json'"],
     ],
     consts.SOURCE_TYPE_IMPALA: [
-        ["host", "Desired Impala host"],
-        ["port", "Desired Impala port (10000 if not provided)"],
-        ["database", "Desired Impala database (default if not provided)"],
-        ["auth_mechanism", "Desired Impala auth mechanism (PLAIN if not provided)"],
+        ["host", "Impala host"],
+        ["port", "Impala port (10000 if not provided)"],
+        ["database", "Impala database (default if not provided)"],
+        ["auth_mechanism", "Impala auth mechanism (PLAIN if not provided)"],
         [
             "kerberos_service_name",
             "Desired Kerberos service name ('impala' if not provided)",
@@ -183,8 +197,8 @@ CONNECTION_SOURCE_FIELDS = {
         ["http_path", "URL path of HTTP proxy"],
     ],
     consts.SOURCE_TYPE_DB2: [
-        ["host", "Desired DB2 host"],
-        ["port", "Desired DB2 port (50000 if not provided)"],
+        ["host", "DB2 host"],
+        ["port", "DB2 port (50000 if not provided)"],
         ["user", "Username to connect to"],
         ["password", "Password for authentication of user"],
         ["database", "Database in DB2 to connect to"],

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/data_validation/clients.py

@@ -16,7 +16,7 @@
 from contextlib import contextmanager
 import copy
 import logging
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 import warnings
 
 import google.oauth2.service_account
@@ -56,6 +56,7 @@ IBIS_ALCHEMY_BACKENDS = [
     "mssql",
     "redshift",
     "snowflake",
+    "sybase",
 ]
 
 
@@ -93,19 +94,32 @@ try:
 except ImportError:
     db2_connect = _raise_missing_client_error("pip install ibm_db_sa")
 
+# Sybase requires sqlalchemy_sybase package.
+try:
+    from third_party.ibis.ibis_sybase.api import sybase_connect
+except ImportError:
+    sybase_connect = _raise_missing_client_error("pip install sqlalchemy_sybase")
+
 
 def get_google_bigquery_client(
-    project_id: str, credentials=None, api_endpoint: str = None
+    project_id: str,
+    credentials=None,
+    api_endpoint: Optional[str] = None,
+    quota_project_id: Optional[str] = None,
 ):
     info = client_info.get_http_client_info()
     job_config = bigquery.QueryJobConfig(
         connection_properties=[bigquery.ConnectionProperty("time_zone", "UTC")]
     )
+    effective_project = quota_project_id or project_id
     options = None
-    if api_endpoint:
-        options = client_options.ClientOptions(api_endpoint=api_endpoint)
+    if api_endpoint or quota_project_id:
+        options = client_options.ClientOptions(
+            api_endpoint=api_endpoint,
+            quota_project_id=quota_project_id if quota_project_id else None,
+        )
     return bigquery.Client(
-        project=project_id,
+        project=effective_project,
         client_info=info,
         credentials=credentials,
         default_query_job_config=job_config,
@@ -113,10 +127,17 @@ def get_google_bigquery_client(
     )
 
 
-def _get_google_bqstorage_client(credentials=None, api_endpoint: str = None):
+def _get_google_bqstorage_client(
+    credentials=None,
+    api_endpoint: Optional[str] = None,
+    quota_project_id: Optional[str] = None,
+):
     options = None
-    if api_endpoint:
-        options = client_options.ClientOptions(api_endpoint=api_endpoint)
+    if api_endpoint or quota_project_id:
+        options = client_options.ClientOptions(
+            api_endpoint=api_endpoint,
+            quota_project_id=quota_project_id if quota_project_id else None,
+        )
     from google.cloud import bigquery_storage_v1 as bigquery_storage
 
     return bigquery_storage.BigQueryReadClient(
@@ -129,20 +150,26 @@ def get_bigquery_client(
     project_id: str,
     dataset_id: str = "",
     credentials=None,
-    api_endpoint: str = None,
-    storage_api_endpoint: str = None,
+    api_endpoint: Optional[str] = None,
+    storage_api_endpoint: Optional[str] = None,
+    client_project_id: Optional[str] = None,
 ):
     google_client = get_google_bigquery_client(
-        project_id, credentials=credentials, api_endpoint=api_endpoint
+        project_id,
+        credentials=credentials,
+        api_endpoint=api_endpoint,
+        quota_project_id=client_project_id,
     )
     bqstorage_client = None
     if storage_api_endpoint:
         bqstorage_client = _get_google_bqstorage_client(
-            credentials=credentials, api_endpoint=storage_api_endpoint
+            credentials=credentials,
+            api_endpoint=storage_api_endpoint,
+            quota_project_id=client_project_id,
         )
 
     return bigquery_connect(
-        project_id=project_id,
+        project_id=project_id or client_project_id,
         dataset_id=dataset_id,
         credentials=credentials,
         bigquery_client=google_client,
@@ -202,6 +229,7 @@ def get_ibis_table(client, schema_name, table_name, database_name=None):
         "db2",
         "mssql",
         "redshift",
+        "sybase",
     ]:
         return client.table(table_name, database=database_name, schema=schema_name)
     elif client.name == "pandas":
@@ -400,5 +428,6 @@ CLIENT_LOOKUP = {
     consts.SOURCE_TYPE_MSSQL: mssql_connect,
     consts.SOURCE_TYPE_SNOWFLAKE: snowflake_connect,
     consts.SOURCE_TYPE_SPANNER: spanner_connect,
+    consts.SOURCE_TYPE_SYBASE: sybase_connect,
     consts.SOURCE_TYPE_DB2: db2_connect,
 }

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/data_validation/config_manager.py

@@ -71,7 +71,7 @@ class ConfigManager(object):
         """Return config object."""
         return self._config
 
-    def get_source_connection(self):
+    def get_source_connection(self) -> dict:
         """Return source connection object."""
         if not self._source_conn:
             if self._config.get(consts.CONFIG_SOURCE_CONN):
@@ -82,7 +82,7 @@ class ConfigManager(object):
 
         return self._source_conn
 
-    def get_target_connection(self):
+    def get_target_connection(self) -> dict:
         """Return target connection object."""
         if not self._target_conn:
             if self._config.get(consts.CONFIG_TARGET_CONN):

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/data_validation/consts.py

@@ -139,6 +139,7 @@ SOURCE_TYPE_POSTGRES = "Postgres"
 SOURCE_TYPE_REDSHIFT = "Redshift"
 SOURCE_TYPE_SNOWFLAKE = "Snowflake"
 SOURCE_TYPE_SPANNER = "Spanner"
+SOURCE_TYPE_SYBASE = "Sybase"
 SOURCE_TYPE_TERADATA = "Teradata"
 
 # BigQuery Result Handler Configs
@@ -149,6 +150,7 @@ TABLE_ID = "table_id"
 GOOGLE_SERVICE_ACCOUNT_KEY_PATH = "google_service_account_key_path"
 API_ENDPOINT = "api_endpoint"
 STORAGE_API_ENDPOINT = "storage_api_endpoint"
+CLIENT_PROJECT_ID = "client_project_id"
 
 # Result Handler Output Table Fields
 VALIDATION_TYPE = "validation_type"
@@ -262,3 +264,6 @@ CALC_FIELD_LENGTH = "length"
 CALC_FIELD_PADDED_CHAR_LENGTH = "padded_char_length"
 CALC_FIELD_RSTRIP = "rstrip"
 CALC_FIELD_UPPER = "upper"
+
+# Backends that do not support window functions.
+NO_WINDOW_FUNCTION_SUPPORT = [SOURCE_TYPE_SYBASE]

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/data_validation/exceptions.py

@@ -27,3 +27,7 @@ class ResultHandlerException(Exception):
 
 class SchemaValidationException(ValidationException):
     pass
+
+
+class PartitionBuilderException(ValidationException):
+    pass

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/data_validation/partition_builder.py

@@ -20,7 +20,7 @@ import re
 import datetime
 from typing import List, Dict, TYPE_CHECKING
 
-from data_validation import cli_tools, consts, util
+from data_validation import cli_tools, consts, exceptions, util
 from data_validation.config_manager import ConfigManager
 from data_validation.query_builder.partition_row_builder import PartitionRowBuilder
 from data_validation.validation_builder import ValidationBuilder
@@ -105,6 +105,15 @@ class PartitionBuilder:
         yaml_configs_list = self._add_partition_filters(partition_filters)
         self._store_partitions(yaml_configs_list)
 
+    def check_partition_configs(self) -> None:
+        for config_manager in self.config_managers:
+            source_type = config_manager.get_source_connection()[consts.SOURCE_TYPE]
+            if source_type in consts.NO_WINDOW_FUNCTION_SUPPORT:
+                raise exceptions.PartitionBuilderException(
+                    f"Source client {source_type} does not support window functions, "
+                    "please use this connection as a target connection for generate-table-partitions."
+                )
+
     @staticmethod
     def _extract_where(table_expr: "IbisTable", client: "BaseBackend") -> str:
         """Given a ibis table expression with a filter (i.e. WHERE) clause, this function extracts the

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/data_validation/query_builder/random_row_builder.py

@@ -30,9 +30,10 @@ RANDOM_SORT_SUPPORTS = [
     "mssql",
     "db2",
     "mysql",
-    "spanner",
     "redshift",
     "snowflake",
+    "spanner",
+    "sybase",
 ]
 
 

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0/google_pso_data_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 8.3.0
+Version: 8.4.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -28,7 +28,7 @@ Requires-Dist: jellyfish>=1.1.0
 Requires-Dist: pandas
 Requires-Dist: parsy>=2.1
 Requires-Dist: psycopg2-binary>=2.9.9
-Requires-Dist: pyarrow==14.0.1
+Requires-Dist: pyarrow
 Requires-Dist: pydata-google-auth>=1.8.2
 Requires-Dist: PyMySQL>=1.1.1
 Requires-Dist: PyYAML>=6.0.2
@@ -45,6 +45,7 @@ Requires-Dist: freezegun; extra == "develop"
 Requires-Dist: pyfakefs; extra == "develop"
 Requires-Dist: pytest; extra == "develop"
 Requires-Dist: pytest-cov; extra == "develop"
+Requires-Dist: pytest-rerunfailures; extra == "develop"
 Requires-Dist: pytest-timeout; extra == "develop"
 Dynamic: author
 Dynamic: author-email
@@ -92,8 +93,9 @@ DVT supports the following connection types:
 *   [Postgres](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#postgres)
 *   [Redshift](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#redshift)
 *   [Spanner](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#google-spanner)
-*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 *   [Snowflake](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#snowflake)
+*   [Sybase ASE](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#sybase)
+*   [Teradata](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md#teradata)
 
 The [Connections](https://github.com/GoogleCloudPlatform/professional-services-data-validator/blob/develop/docs/connections.md) page provides details about how to create
 and list connections for the validation tool.

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/google_pso_data_validator.egg-info/SOURCES.txt

@@ -85,6 +85,11 @@ third_party/ibis/ibis_redshift/compiler.py
 third_party/ibis/ibis_snowflake/__init__.py
 third_party/ibis/ibis_snowflake/api.py
 third_party/ibis/ibis_snowflake/datatypes.py
+third_party/ibis/ibis_sybase/__init__.py
+third_party/ibis/ibis_sybase/api.py
+third_party/ibis/ibis_sybase/compiler.py
+third_party/ibis/ibis_sybase/datatypes.py
+third_party/ibis/ibis_sybase/registry.py
 third_party/ibis/ibis_teradata/__init__.py
 third_party/ibis/ibis_teradata/api.py
 third_party/ibis/ibis_teradata/compiler.py

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/google_pso_data_validator.egg-info/requires.txt

@@ -12,7 +12,7 @@ jellyfish>=1.1.0
 pandas
 parsy>=2.1
 psycopg2-binary>=2.9.9
-pyarrow==14.0.1
+pyarrow
 pydata-google-auth>=1.8.2
 PyMySQL>=1.1.1
 PyYAML>=6.0.2
@@ -29,6 +29,7 @@ freezegun
 pyfakefs
 pytest
 pytest-cov
+pytest-rerunfailures
 pytest-timeout
 
 [pyspark]

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/setup.py

@@ -18,7 +18,7 @@ import setuptools
 
 name = "google-pso-data-validator"
 description = "A package to enable easy data validation"
-version = "8.3.0"
+version = "8.4.0"
 release_status = "Development Status :: 4 - Beta"
 
 with open("README.md", "r") as fh:
@@ -39,7 +39,7 @@ dependencies = [
     "pandas",
     "parsy>=2.1",
     "psycopg2-binary>=2.9.9",
-    "pyarrow==14.0.1",  # ibis-framework 7.1.0 depends on pyarrow<15 and >=2
+    "pyarrow",
     "pydata-google-auth>=1.8.2",
     "PyMySQL>=1.1.1",
     "PyYAML>=6.0.2",
@@ -57,6 +57,7 @@ extras_require = {
         "pyfakefs",
         "pytest",
         "pytest-cov",
+        "pytest-rerunfailures",
         "pytest-timeout",
     ],
 }

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/third_party/ibis/ibis_addon/operations.py

@@ -92,6 +92,12 @@ try:
 except Exception:
     SnowflakeExprTranslator = None
 
+# Sybase requires sqlalchemy_sybase package.
+try:
+    from third_party.ibis.ibis_sybase.compiler import SybaseExprTranslator
+except Exception:
+    SybaseExprTranslator = None
+
 
 # Cast of datetime64 NaT to int64 and then in seconds results in the value below.
 # We need to use this value in the datetime.date simulation of the datetime64 behaviour.
@@ -591,3 +597,12 @@ if SnowflakeExprTranslator:
     SnowflakeExprTranslator._registry[ops.RandomScalar] = sa_format_random
     SnowflakeExprTranslator._registry[BinaryLength] = sa_format_binary_length
     SnowflakeExprTranslator._registry[ops.RStrip] = _sa_whitespace_rstrip
+
+if SybaseExprTranslator:
+    SybaseExprTranslator._registry[
+        BinaryLength
+    ] = mssql_registry.sa_format_binary_length
+    SybaseExprTranslator._registry[RawSQL] = sa_format_raw_sql
+    SybaseExprTranslator._registry[
+        PaddedCharLength
+    ] = mssql_registry.sa_format_string_length

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/third_party/ibis/ibis_bigquery/__init__.py

@@ -22,12 +22,12 @@ from pydata_google_auth import cache
 from ibis.backends.bigquery import (
     Backend as BigQueryBackend,
     _create_client_info,
-    parse_project_and_dataset,
     CLIENT_ID,
     CLIENT_SECRET,
     EXTERNAL_DATA_SCOPES,
     SCOPES,
 )
+from ibis.backends.bigquery.client import parse_project_and_dataset
 
 if TYPE_CHECKING:
     import google.cloud.bigquery_storage_v1
@@ -53,9 +53,12 @@ class Backend(BigQueryBackend):
         bqstorage_client: "google.cloud.bigquery_storage_v1.BigQueryReadClient" = None,
     ):
         """Copy of Ibis v5 BigQuery do_connect() customized for DVT, see original method for docs."""
-        default_project_id = ""
+        client_project_id = (
+            bigquery_client.project if bigquery_client is not None else None
+        )
+        default_project_id = None
 
-        if credentials is None:
+        if bigquery_client is None and credentials is None:
             scopes = SCOPES
             if auth_external_data:
                 scopes = EXTERNAL_DATA_SCOPES
@@ -83,14 +86,11 @@ class Backend(BigQueryBackend):
                 credentials_cache=credentials_cache,
                 use_local_webserver=auth_local_webserver,
             )
+            project_id = project_id or default_project_id
 
-        project_id = project_id or default_project_id
-
-        (
-            self.data_project,
-            self.billing_project,
-            self.dataset,
-        ) = parse_project_and_dataset(project_id, dataset_id)
+        self.billing_project = client_project_id or project_id
+        self.data_project = project_id or self.billing_project
+        self.dataset = None
 
         if bigquery_client is None:
             self.client = bq.Client(
@@ -133,6 +133,16 @@ class Backend(BigQueryBackend):
             query_result._project = orig_project
         return arrow_obj
 
+    def _parse_project_and_dataset(self, dataset) -> tuple[str, str]:
+        """Copied from Ibis code and modified to use self.data_project."""
+        if not dataset and not self.dataset:
+            raise ValueError("Unable to determine BigQuery dataset.")
+        project, _, dataset = parse_project_and_dataset(
+            self.data_project,
+            dataset or f"{self.data_project}.{self.dataset}",
+        )
+        return project, dataset
+
     def list_primary_key_columns(self, database: str, table: str) -> list:
         """Return a list of primary key column names."""
         # TODO: Related to issue-1253, it's not clear if this is possible, we should revisit if it becomes a requirement.

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/third_party/ibis/ibis_mssql/registry.py

@@ -51,7 +51,7 @@ def sa_table_column(t, op):
 
 
 def strftime(translator, op):
-    """Use MS SQL CONVERT() in place of STRFTIME().
+    """Use SQL Server CONVERT() in place of STRFTIME().
 
     This is pretty restrictive due to the limited styles offered by SQL Server,
     we've just covered off the generic formats used when casting date based columns

{google_pso_data_validator-8.3.0 → google_pso_data_validator-8.4.0}/third_party/ibis/ibis_oracle/datatypes.py

@@ -91,7 +91,7 @@ _type_mapping = {
 # _handle_failed_column_type_inference().
 # This method has a flaw that is doesn't prefix the table name with the schema
 # which causes a table not found exception.
-# I (nj@2024-02-15) felt the simplest change was to add LONG RAW to SQL Alchemy
+# I (nj@2024-02-15) felt the simplest change was to add LONG RAW to SQLAlchemy
 # as a RAW variant, as below.
 if "LONG RAW" not in OracleDialect_oracledb.ischema_names:
     OracleDialect_oracledb.ischema_names["LONG RAW"] = oracle.RAW