google-cloud-bigquery 3.23.0__tar.gz → 3.24.0__tar.gz

{google-cloud-bigquery-3.23.0/google_cloud_bigquery.egg-info → google-cloud-bigquery-3.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: google-cloud-bigquery
-Version: 3.23.0
+Version: 3.24.0
 Summary: Google BigQuery API client library
 Home-page: https://github.com/googleapis/python-bigquery
 Author: Google LLC

{google-cloud-bigquery-3.23.0 → google-cloud-bigquery-3.24.0}/google/cloud/bigquery/_helpers.py

@@ -33,6 +33,8 @@ from google.cloud._helpers import _to_bytes
 from google.auth import credentials as ga_credentials  # type: ignore
 from google.api_core import client_options as client_options_lib
 
+TimeoutType = Union[float, None]
+
 _RFC3339_MICROS_NO_ZULU = "%Y-%m-%dT%H:%M:%S.%f"
 _TIMEONLY_WO_MICROS = "%H:%M:%S"
 _TIMEONLY_W_MICROS = "%H:%M:%S.%f"

{google-cloud-bigquery-3.23.0 → google-cloud-bigquery-3.24.0}/google/cloud/bigquery/_job_helpers.py

@@ -39,7 +39,7 @@ import copy
 import functools
 import os
 import uuid
-from typing import Any, Dict, TYPE_CHECKING, Optional
+from typing import Any, Dict, Optional, TYPE_CHECKING, Union
 
 import google.api_core.exceptions as core_exceptions
 from google.api_core import retry as retries
@@ -47,6 +47,7 @@ from google.api_core import retry as retries
 from google.cloud.bigquery import job
 import google.cloud.bigquery.query
 from google.cloud.bigquery import table
+from google.cloud.bigquery.retry import POLLING_DEFAULT_VALUE
 
 # Avoid circular imports
 if TYPE_CHECKING:  # pragma: NO COVER
@@ -328,7 +329,7 @@ def query_and_wait(
     location: Optional[str],
     project: str,
     api_timeout: Optional[float] = None,
-    wait_timeout: Optional[float] = None,
+    wait_timeout: Optional[Union[float, object]] = POLLING_DEFAULT_VALUE,
     retry: Optional[retries.Retry],
     job_retry: Optional[retries.Retry],
     page_size: Optional[int] = None,
@@ -364,10 +365,12 @@ def query_and_wait(
         api_timeout (Optional[float]):
             The number of seconds to wait for the underlying HTTP transport
             before using ``retry``.
-        wait_timeout (Optional[float]):
+        wait_timeout (Optional[Union[float, object]]):
             The number of seconds to wait for the query to finish. If the
             query doesn't finish before this timeout, the client attempts
-            to cancel the query.
+            to cancel the query. If unset, the underlying Client.get_job() API
+            call has timeout, but we still wait indefinitely for the job to
+            finish.
         retry (Optional[google.api_core.retry.Retry]):
             How to retry the RPC.  This only applies to making RPC
             calls.  It isn't used to retry failed jobs.  This has
@@ -545,7 +548,7 @@ def _supported_by_jobs_query(request_body: Dict[str, Any]) -> bool:
 def _wait_or_cancel(
     job: job.QueryJob,
     api_timeout: Optional[float],
-    wait_timeout: Optional[float],
+    wait_timeout: Optional[Union[object, float]],
     retry: Optional[retries.Retry],
     page_size: Optional[int],
     max_results: Optional[int],

{google-cloud-bigquery-3.23.0 → google-cloud-bigquery-3.24.0}/google/cloud/bigquery/client.py

@@ -82,6 +82,7 @@ from google.cloud.bigquery._helpers import _DEFAULT_HOST_TEMPLATE
 from google.cloud.bigquery._helpers import _DEFAULT_UNIVERSE
 from google.cloud.bigquery._helpers import _validate_universe
 from google.cloud.bigquery._helpers import _get_client_universe
+from google.cloud.bigquery._helpers import TimeoutType
 from google.cloud.bigquery._job_helpers import make_job_id as _make_job_id
 from google.cloud.bigquery.dataset import Dataset
 from google.cloud.bigquery.dataset import DatasetListItem
@@ -107,6 +108,8 @@ from google.cloud.bigquery.retry import (
     DEFAULT_JOB_RETRY,
     DEFAULT_RETRY,
     DEFAULT_TIMEOUT,
+    DEFAULT_GET_JOB_TIMEOUT,
+    POLLING_DEFAULT_VALUE,
 )
 from google.cloud.bigquery.routine import Routine
 from google.cloud.bigquery.routine import RoutineReference
@@ -123,7 +126,6 @@ pandas = (
     _versions_helpers.PANDAS_VERSIONS.try_import()
 )  # mypy check fails because pandas import is outside module, there are type: ignore comments related to this
 
-TimeoutType = Union[float, None]
 ResumableTimeoutType = Union[
     None, float, Tuple[float, float]
 ]  # for resumable media methods
@@ -1962,6 +1964,7 @@ class Client(ClientWithProject):
         timeout_ms: Optional[int] = None,
         location: Optional[str] = None,
         timeout: TimeoutType = DEFAULT_TIMEOUT,
+        page_size: int = 0,
     ) -> _QueryResults:
         """Get the query results object for a query job.
 
@@ -1980,13 +1983,16 @@ class Client(ClientWithProject):
                 before using ``retry``. If set, this connection timeout may be
                 increased to a minimum value. This prevents retries on what
                 would otherwise be a successful response.
+            page_size (int):
+                Maximum number of rows in a single response. See maxResults in
+                the jobs.getQueryResults REST API.
 
         Returns:
             google.cloud.bigquery.query._QueryResults:
                 A new ``_QueryResults`` instance.
         """
 
-        extra_params: Dict[str, Any] = {"maxResults": 0}
+        extra_params: Dict[str, Any] = {"maxResults": page_size}
 
         if timeout is not None:
             if not isinstance(timeout, (int, float)):
@@ -1994,6 +2000,9 @@ class Client(ClientWithProject):
             else:
                 timeout = max(timeout, _MIN_GET_QUERY_RESULTS_TIMEOUT)
 
+        if page_size > 0:
+            extra_params["formatOptions.useInt64Timestamp"] = True
+
         if project is None:
             project = self.project
 
@@ -2139,7 +2148,7 @@ class Client(ClientWithProject):
         project: Optional[str] = None,
         location: Optional[str] = None,
         retry: retries.Retry = DEFAULT_RETRY,
-        timeout: TimeoutType = DEFAULT_TIMEOUT,
+        timeout: TimeoutType = DEFAULT_GET_JOB_TIMEOUT,
     ) -> Union[job.LoadJob, job.CopyJob, job.ExtractJob, job.QueryJob, job.UnknownJob]:
         """Fetch a job for the project associated with this client.
 
@@ -3503,7 +3512,7 @@ class Client(ClientWithProject):
         location: Optional[str] = None,
         project: Optional[str] = None,
         api_timeout: TimeoutType = DEFAULT_TIMEOUT,
-        wait_timeout: TimeoutType = None,
+        wait_timeout: Union[Optional[float], object] = POLLING_DEFAULT_VALUE,
         retry: retries.Retry = DEFAULT_RETRY,
         job_retry: retries.Retry = DEFAULT_JOB_RETRY,
         page_size: Optional[int] = None,
@@ -3537,10 +3546,12 @@ class Client(ClientWithProject):
             api_timeout (Optional[float]):
                 The number of seconds to wait for the underlying HTTP transport
                 before using ``retry``.
-            wait_timeout (Optional[float]):
+            wait_timeout (Optional[Union[float, object]]):
                 The number of seconds to wait for the query to finish. If the
                 query doesn't finish before this timeout, the client attempts
-                to cancel the query.
+                to cancel the query. If unset, the underlying REST API calls
+                have timeouts, but we still wait indefinitely for the job to
+                finish.
             retry (Optional[google.api_core.retry.Retry]):
                 How to retry the RPC.  This only applies to making RPC
                 calls.  It isn't used to retry failed jobs.  This has
@@ -4127,11 +4138,6 @@ class Client(ClientWithProject):
         if start_index is not None:
             params["startIndex"] = start_index
 
-        # We don't call jobs.query with a page size, so if the user explicitly
-        # requests a certain size, invalidate the cache.
-        if page_size is not None:
-            first_page_response = None
-
         params["formatOptions.useInt64Timestamp"] = True
         row_iterator = RowIterator(
             client=self,

{google-cloud-bigquery-3.23.0 → google-cloud-bigquery-3.24.0}/google/cloud/bigquery/format_options.py

@@ -105,6 +105,21 @@ class ParquetOptions:
     def enable_list_inference(self, value: bool) -> None:
         self._properties["enableListInference"] = value
 
+    @property
+    def map_target_type(self) -> str:
+        """Indicates whether to simplify the representation of parquet maps to only show keys and values."""
+
+        return self._properties.get("mapTargetType")
+
+    @map_target_type.setter
+    def map_target_type(self, value: str) -> None:
+        """Sets the map target type.
+
+        Args:
+          value: The map target type (eg ARRAY_OF_STRUCT).
+        """
+        self._properties["mapTargetType"] = value
+
     @classmethod
     def from_api_repr(cls, resource: Dict[str, bool]) -> "ParquetOptions":
         """Factory: construct an instance from a resource dict.

{google-cloud-bigquery-3.23.0 → google-cloud-bigquery-3.24.0}/google/cloud/bigquery/job/base.py

@@ -26,8 +26,11 @@ from google.api_core import exceptions
 import google.api_core.future.polling
 
 from google.cloud.bigquery import _helpers
-from google.cloud.bigquery.retry import DEFAULT_RETRY
 from google.cloud.bigquery._helpers import _int_or_none
+from google.cloud.bigquery.retry import (
+    DEFAULT_GET_JOB_TIMEOUT,
+    DEFAULT_RETRY,
+)
 
 
 _DONE_STATE = "DONE"
@@ -801,7 +804,7 @@ class _AsyncJob(google.api_core.future.polling.PollingFuture):
         self,
         client=None,
         retry: "retries.Retry" = DEFAULT_RETRY,
-        timeout: Optional[float] = None,
+        timeout: Optional[float] = DEFAULT_GET_JOB_TIMEOUT,
     ):
         """API call:  refresh job properties via a GET request.
 
@@ -820,22 +823,14 @@ class _AsyncJob(google.api_core.future.polling.PollingFuture):
         """
         client = self._require_client(client)
 
-        extra_params = {}
-        if self.location:
-            extra_params["location"] = self.location
-        span_attributes = {"path": self.path}
-
-        api_response = client._call_api(
-            retry,
-            span_name="BigQuery.job.reload",
-            span_attributes=span_attributes,
-            job_ref=self,
-            method="GET",
-            path=self.path,
-            query_params=extra_params,
+        got_job = client.get_job(
+            self,
+            project=self.project,
+            location=self.location,
+            retry=retry,
             timeout=timeout,
         )
-        self._set_properties(api_response)
+        self._set_properties(got_job._properties)
 
     def cancel(
         self,
@@ -913,7 +908,7 @@ class _AsyncJob(google.api_core.future.polling.PollingFuture):
     def done(
         self,
         retry: "retries.Retry" = DEFAULT_RETRY,
-        timeout: Optional[float] = None,
+        timeout: Optional[float] = DEFAULT_GET_JOB_TIMEOUT,
         reload: bool = True,
     ) -> bool:
         """Checks if the job is complete.

{google-cloud-bigquery-3.23.0 → google-cloud-bigquery-3.24.0}/google/cloud/bigquery/job/query.py

@@ -40,7 +40,11 @@ from google.cloud.bigquery.query import (
     StructQueryParameter,
     UDFResource,
 )
-from google.cloud.bigquery.retry import DEFAULT_RETRY, DEFAULT_JOB_RETRY
+from google.cloud.bigquery.retry import (
+    DEFAULT_RETRY,
+    DEFAULT_JOB_RETRY,
+    POLLING_DEFAULT_VALUE,
+)
 from google.cloud.bigquery.routine import RoutineReference
 from google.cloud.bigquery.schema import SchemaField
 from google.cloud.bigquery.table import _EmptyRowIterator
@@ -1381,7 +1385,10 @@ class QueryJob(_AsyncJob):
             raise
 
     def _reload_query_results(
-        self, retry: "retries.Retry" = DEFAULT_RETRY, timeout: Optional[float] = None
+        self,
+        retry: "retries.Retry" = DEFAULT_RETRY,
+        timeout: Optional[float] = None,
+        page_size: int = 0,
     ):
         """Refresh the cached query results unless already cached and complete.
 
@@ -1391,6 +1398,9 @@ class QueryJob(_AsyncJob):
             timeout (Optional[float]):
                 The number of seconds to wait for the underlying HTTP transport
                 before using ``retry``.
+            page_size (int):
+                Maximum number of rows in a single response. See maxResults in
+                the jobs.getQueryResults REST API.
         """
         # Optimization: avoid a call to jobs.getQueryResults if it's already
         # been fetched, e.g. from jobs.query first page of results.
@@ -1421,7 +1431,14 @@ class QueryJob(_AsyncJob):
 
         # If an explicit timeout is not given, fall back to the transport timeout
         # stored in _blocking_poll() in the process of polling for job completion.
-        transport_timeout = timeout if timeout is not None else self._transport_timeout
+        if timeout is not None:
+            transport_timeout = timeout
+        else:
+            transport_timeout = self._transport_timeout
+
+            # Handle PollingJob._DEFAULT_VALUE.
+            if not isinstance(transport_timeout, (float, int)):
+                transport_timeout = None
 
         self._query_results = self._client._get_query_results(
             self.job_id,
@@ -1430,6 +1447,7 @@ class QueryJob(_AsyncJob):
             timeout_ms=timeout_ms,
             location=self.location,
             timeout=transport_timeout,
+            page_size=page_size,
         )
 
     def result(  # type: ignore  # (incompatible with supertype)
@@ -1437,7 +1455,7 @@ class QueryJob(_AsyncJob):
         page_size: Optional[int] = None,
         max_results: Optional[int] = None,
         retry: Optional[retries.Retry] = DEFAULT_RETRY,
-        timeout: Optional[float] = None,
+        timeout: Optional[Union[float, object]] = POLLING_DEFAULT_VALUE,
         start_index: Optional[int] = None,
         job_retry: Optional[retries.Retry] = DEFAULT_JOB_RETRY,
     ) -> Union["RowIterator", _EmptyRowIterator]:
@@ -1457,11 +1475,14 @@ class QueryJob(_AsyncJob):
                 is ``DONE``, retrying is aborted early even if the
                 results are not available, as this will not change
                 anymore.
-            timeout (Optional[float]):
+            timeout (Optional[Union[float, \
+                google.api_core.future.polling.PollingFuture._DEFAULT_VALUE, \
+            ]]):
                 The number of seconds to wait for the underlying HTTP transport
-                before using ``retry``.
-                If multiple requests are made under the hood, ``timeout``
-                applies to each individual request.
+                before using ``retry``. If ``None``, wait indefinitely
+                unless an error is returned. If unset, only the
+                underlying API calls have their default timeouts, but we still
+                wait indefinitely for the job to finish.
             start_index (Optional[int]):
                 The zero-based index of the starting row to read.
             job_retry (Optional[google.api_core.retry.Retry]):
@@ -1507,6 +1528,27 @@ class QueryJob(_AsyncJob):
                 # Intentionally omit job_id and query_id since this doesn't
                 # actually correspond to a finished query job.
             )
+
+        # Setting max_results should be equivalent to setting page_size with
+        # regards to allowing the user to tune how many results to download
+        # while we wait for the query to finish. See internal issue:
+        # 344008814.
+        if page_size is None and max_results is not None:
+            page_size = max_results
+
+        # When timeout has default sentinel value ``object()``, do not pass
+        # anything to invoke default timeouts in subsequent calls.
+        done_kwargs: Dict[str, Union[_helpers.TimeoutType, object]] = {}
+        reload_query_results_kwargs: Dict[str, Union[_helpers.TimeoutType, object]] = {}
+        list_rows_kwargs: Dict[str, Union[_helpers.TimeoutType, object]] = {}
+        if type(timeout) is not object:
+            done_kwargs["timeout"] = timeout
+            list_rows_kwargs["timeout"] = timeout
+            reload_query_results_kwargs["timeout"] = timeout
+
+        if page_size is not None:
+            reload_query_results_kwargs["page_size"] = page_size
+
         try:
             retry_do_query = getattr(self, "_retry_do_query", None)
             if retry_do_query is not None:
@@ -1543,12 +1585,17 @@ class QueryJob(_AsyncJob):
                     self._retry_do_query = retry_do_query
                     self._job_retry = job_retry
 
+                # If the job hasn't been created, create it now. Related:
+                # https://github.com/googleapis/python-bigquery/issues/1940
+                if self.state is None:
+                    self._begin(retry=retry, **done_kwargs)
+
                 # Refresh the job status with jobs.get because some of the
                 # exceptions thrown by jobs.getQueryResults like timeout and
                 # rateLimitExceeded errors are ambiguous. We want to know if
                 # the query job failed and not just the call to
                 # jobs.getQueryResults.
-                if self.done(retry=retry, timeout=timeout):
+                if self.done(retry=retry, **done_kwargs):
                     # If it's already failed, we might as well stop.
                     job_failed_exception = self.exception()
                     if job_failed_exception is not None:
@@ -1585,14 +1632,16 @@ class QueryJob(_AsyncJob):
                         # response from the REST API. This ensures we aren't
                         # making any extra API calls if the previous loop
                         # iteration fetched the finished job.
-                        self._reload_query_results(retry=retry, timeout=timeout)
+                        self._reload_query_results(
+                            retry=retry, **reload_query_results_kwargs
+                        )
                         return True
 
                 # Call jobs.getQueryResults with max results set to 0 just to
                 # wait for the query to finish. Unlike most methods,
                 # jobs.getQueryResults hangs as long as it can to ensure we
                 # know when the query has finished as soon as possible.
-                self._reload_query_results(retry=retry, timeout=timeout)
+                self._reload_query_results(retry=retry, **reload_query_results_kwargs)
 
                 # Even if the query is finished now according to
                 # jobs.getQueryResults, we'll want to reload the job status if
@@ -1665,8 +1714,9 @@ class QueryJob(_AsyncJob):
         # We know that there's at least 1 row, so only treat the response from
         # jobs.getQueryResults / jobs.query as the first page of the
         # RowIterator response if there are any rows in it. This prevents us
-        # from stopping the iteration early because we're missing rows and
-        # there's no next page token.
+        # from stopping the iteration early in the cases where we set
+        # maxResults=0. In that case, we're missing rows and there's no next
+        # page token.
         first_page_response = self._query_results._properties
         if "rows" not in first_page_response:
             first_page_response = None
@@ -1682,10 +1732,10 @@ class QueryJob(_AsyncJob):
             max_results=max_results,
             start_index=start_index,
             retry=retry,
-            timeout=timeout,
             query_id=self.query_id,
             first_page_response=first_page_response,
             num_dml_affected_rows=self._query_results.num_dml_affected_rows,
+            **list_rows_kwargs,
         )
         rows._preserve_order = _contains_order_by(self.query)
         return rows

{google-cloud-bigquery-3.23.0 → google-cloud-bigquery-3.24.0}/google/cloud/bigquery/retry.py

@@ -14,6 +14,7 @@
 
 from google.api_core import exceptions
 from google.api_core import retry
+import google.api_core.future.polling
 from google.auth import exceptions as auth_exceptions  # type: ignore
 import requests.exceptions
 
@@ -119,6 +120,14 @@ def _job_should_retry(exc):
     if isinstance(exc, exceptions.RetryError):
         exc = exc.cause
 
+    # Per https://github.com/googleapis/python-bigquery/issues/1929, sometimes
+    # retriable errors make their way here. Because of the separate
+    # `restart_query_job` logic to make sure we aren't restarting non-failed
+    # jobs, it should be safe to continue and not totally fail our attempt at
+    # waiting for the query to complete.
+    if _should_retry(exc):
+        return True
+
     if not hasattr(exc, "errors") or len(exc.errors) == 0:
         return False
 
@@ -132,3 +141,13 @@ DEFAULT_JOB_RETRY = retry.Retry(
 """
 The default job retry object.
 """
+
+DEFAULT_GET_JOB_TIMEOUT = 128
+"""
+Default timeout for Client.get_job().
+"""
+
+POLLING_DEFAULT_VALUE = google.api_core.future.polling.PollingFuture._DEFAULT_VALUE
+"""
+Default value defined in google.api_core.future.polling.PollingFuture.
+"""

{google-cloud-bigquery-3.23.0 → google-cloud-bigquery-3.24.0}/google/cloud/bigquery/table.py

@@ -108,7 +108,17 @@ _RANGE_PYARROW_WARNING = (
 
 # How many of the total rows need to be downloaded already for us to skip
 # calling the BQ Storage API?
-ALMOST_COMPLETELY_CACHED_RATIO = 0.333
+#
+# In microbenchmarks on 2024-05-21, I (tswast@) measure that at about 2 MB of
+# remaining results, it's faster to use the BQ Storage Read API to download
+# the results than use jobs.getQueryResults. Since we don't have a good way to
+# know the remaining bytes, we estimate by remaining number of rows.
+#
+# Except when rows themselves are larger, I observe that the a single page of
+# results will be around 10 MB. Therefore, the proportion of rows already
+# downloaded should be 10 (first page) / 12 (all results) or less for it to be
+# worth it to make a call to jobs.getQueryResults.
+ALMOST_COMPLETELY_CACHED_RATIO = 0.833333
 
 
 def _reference_getter(table):
@@ -1714,6 +1724,10 @@ class RowIterator(HTTPIterator):
         try:
             _versions_helpers.BQ_STORAGE_VERSIONS.try_import(raise_if_error=True)
         except bq_exceptions.BigQueryStorageNotFoundError:
+            warnings.warn(
+                "BigQuery Storage module not found, fetch data with the REST "
+                "endpoint instead."
+            )
             return False
         except bq_exceptions.LegacyBigQueryStorageError as exc:
             warnings.warn(str(exc))
@@ -1745,7 +1759,7 @@ class RowIterator(HTTPIterator):
         if self._page_size is not None:
             if self.page_number and "startIndex" in params:
                 del params["startIndex"]
-            params["maxResults"] = self._page_size
+
         return self.api_request(
             method=self._HTTP_METHOD, path=self.path, query_params=params
         )

{google-cloud-bigquery-3.23.0 → google-cloud-bigquery-3.24.0}/google/cloud/bigquery/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.23.0"
+__version__ = "3.24.0"

{google-cloud-bigquery-3.23.0 → google-cloud-bigquery-3.24.0/google_cloud_bigquery.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: google-cloud-bigquery
-Version: 3.23.0
+Version: 3.24.0
 Summary: Google BigQuery API client library
 Home-page: https://github.com/googleapis/python-bigquery
 Author: Google LLC