google-cloud-bigquery 3.35.0__tar.gz → 3.36.0__tar.gz

{google_cloud_bigquery-3.35.0/google_cloud_bigquery.egg-info → google_cloud_bigquery-3.36.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-cloud-bigquery
-Version: 3.35.0
+Version: 3.36.0
 Summary: Google BigQuery API client library
 Author-email: Google LLC <googleapis-packages@google.com>
 License: Apache 2.0
@@ -84,7 +84,7 @@ processing power of Google's infrastructure.
 .. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-bigquery.svg
    :target: https://pypi.org/project/google-cloud-bigquery/
 .. _BigQuery: https://cloud.google.com/bigquery/what-is-bigquery
-.. _Client Library Documentation: https://googleapis.dev/python/bigquery/latest
+.. _Client Library Documentation: https://cloud.google.com/python/docs/reference/bigquery/latest/summary_overview
 .. _Product Documentation: https://cloud.google.com/bigquery/docs/reference/v2/
 
 Quick Start

{google_cloud_bigquery-3.35.0 → google_cloud_bigquery-3.36.0}/README.rst

@@ -18,7 +18,7 @@ processing power of Google's infrastructure.
 .. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-bigquery.svg
    :target: https://pypi.org/project/google-cloud-bigquery/
 .. _BigQuery: https://cloud.google.com/bigquery/what-is-bigquery
-.. _Client Library Documentation: https://googleapis.dev/python/bigquery/latest
+.. _Client Library Documentation: https://cloud.google.com/python/docs/reference/bigquery/latest/summary_overview
 .. _Product Documentation: https://cloud.google.com/bigquery/docs/reference/v2/
 
 Quick Start

{google_cloud_bigquery-3.35.0 → google_cloud_bigquery-3.36.0}/google/cloud/bigquery/_job_helpers.py

@@ -35,17 +35,22 @@ we still need a separate job_retry object because there are different
 predicates where it is safe to generate a new query ID.
 """
 
+from __future__ import annotations
+
 import copy
+import dataclasses
+import datetime
 import functools
 import uuid
 import textwrap
-from typing import Any, Dict, Optional, TYPE_CHECKING, Union
+from typing import Any, Callable, Dict, Optional, TYPE_CHECKING, Union
 import warnings
 
 import google.api_core.exceptions as core_exceptions
 from google.api_core import retry as retries
 
 from google.cloud.bigquery import job
+import google.cloud.bigquery.job.query
 import google.cloud.bigquery.query
 from google.cloud.bigquery import table
 import google.cloud.bigquery.retry
@@ -116,14 +121,21 @@ def query_jobs_insert(
     retry: Optional[retries.Retry],
     timeout: Optional[float],
     job_retry: Optional[retries.Retry],
+    *,
+    callback: Callable = lambda _: None,
 ) -> job.QueryJob:
     """Initiate a query using jobs.insert.
 
     See: https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/insert
+
+    Args:
+        callback (Callable):
+            A callback function used by bigframes to report query progress.
     """
     job_id_given = job_id is not None
     job_id_save = job_id
     job_config_save = job_config
+    query_sent_factory = QuerySentEventFactory()
 
     def do_query():
         # Make a copy now, so that original doesn't get changed by the process
@@ -136,6 +148,16 @@ def query_jobs_insert(
 
         try:
             query_job._begin(retry=retry, timeout=timeout)
+            if job_config is not None and not job_config.dry_run:
+                callback(
+                    query_sent_factory(
+                        query=query,
+                        billing_project=query_job.project,
+                        location=query_job.location,
+                        job_id=query_job.job_id,
+                        request_id=None,
+                    )
+                )
         except core_exceptions.Conflict as create_exc:
             # The thought is if someone is providing their own job IDs and they get
             # their job ID generation wrong, this could end up returning results for
@@ -396,6 +418,7 @@ def query_and_wait(
     job_retry: Optional[retries.Retry],
     page_size: Optional[int] = None,
     max_results: Optional[int] = None,
+    callback: Callable = lambda _: None,
 ) -> table.RowIterator:
     """Run the query, wait for it to finish, and return the results.
 
@@ -415,9 +438,8 @@ def query_and_wait(
         location (Optional[str]):
             Location where to run the job. Must match the location of the
             table used in the query as well as the destination table.
-        project (Optional[str]):
-            Project ID of the project of where to run the job. Defaults
-            to the client's project.
+        project (str):
+            Project ID of the project of where to run the job.
         api_timeout (Optional[float]):
             The number of seconds to wait for the underlying HTTP transport
             before using ``retry``.
@@ -441,6 +463,8 @@ def query_and_wait(
             request. Non-positive values are ignored.
         max_results (Optional[int]):
             The maximum total number of rows from this request.
+        callback (Callable):
+            A callback function used by bigframes to report query progress.
 
     Returns:
         google.cloud.bigquery.table.RowIterator:
@@ -479,12 +503,14 @@ def query_and_wait(
                 retry=retry,
                 timeout=api_timeout,
                 job_retry=job_retry,
+                callback=callback,
             ),
             api_timeout=api_timeout,
             wait_timeout=wait_timeout,
             retry=retry,
             page_size=page_size,
             max_results=max_results,
+            callback=callback,
         )
 
     path = _to_query_path(project)
@@ -496,10 +522,24 @@ def query_and_wait(
     if client.default_job_creation_mode:
         request_body["jobCreationMode"] = client.default_job_creation_mode
 
+    query_sent_factory = QuerySentEventFactory()
+
     def do_query():
-        request_body["requestId"] = make_job_id()
+        request_id = make_job_id()
+        request_body["requestId"] = request_id
         span_attributes = {"path": path}
 
+        if "dryRun" not in request_body:
+            callback(
+                query_sent_factory(
+                    query=query,
+                    billing_project=project,
+                    location=location,
+                    job_id=None,
+                    request_id=request_id,
+                )
+            )
+
         # For easier testing, handle the retries ourselves.
         if retry is not None:
             response = retry(client._call_api)(
@@ -542,8 +582,25 @@ def query_and_wait(
                 retry=retry,
                 page_size=page_size,
                 max_results=max_results,
+                callback=callback,
             )
 
+        if "dryRun" not in request_body:
+            callback(
+                QueryFinishedEvent(
+                    billing_project=project,
+                    location=query_results.location,
+                    query_id=query_results.query_id,
+                    job_id=query_results.job_id,
+                    total_rows=query_results.total_rows,
+                    total_bytes_processed=query_results.total_bytes_processed,
+                    slot_millis=query_results.slot_millis,
+                    destination=None,
+                    created=query_results.created,
+                    started=query_results.started,
+                    ended=query_results.ended,
+                )
+            )
         return table.RowIterator(
             client=client,
             api_request=functools.partial(client._call_api, retry, timeout=api_timeout),
@@ -561,6 +618,9 @@ def query_and_wait(
             query=query,
             total_bytes_processed=query_results.total_bytes_processed,
             slot_millis=query_results.slot_millis,
+            created=query_results.created,
+            started=query_results.started,
+            ended=query_results.ended,
         )
 
     if job_retry is not None:
@@ -611,6 +671,8 @@ def _wait_or_cancel(
     retry: Optional[retries.Retry],
     page_size: Optional[int],
     max_results: Optional[int],
+    *,
+    callback: Callable = lambda _: None,
 ) -> table.RowIterator:
     """Wait for a job to complete and return the results.
 
@@ -618,12 +680,43 @@ def _wait_or_cancel(
     the job.
     """
     try:
-        return job.result(
+        if not job.dry_run:
+            callback(
+                QueryReceivedEvent(
+                    billing_project=job.project,
+                    location=job.location,
+                    job_id=job.job_id,
+                    statement_type=job.statement_type,
+                    state=job.state,
+                    query_plan=job.query_plan,
+                    created=job.created,
+                    started=job.started,
+                    ended=job.ended,
+                )
+            )
+        query_results = job.result(
             page_size=page_size,
             max_results=max_results,
             retry=retry,
             timeout=wait_timeout,
         )
+        if not job.dry_run:
+            callback(
+                QueryFinishedEvent(
+                    billing_project=job.project,
+                    location=query_results.location,
+                    query_id=query_results.query_id,
+                    job_id=query_results.job_id,
+                    total_rows=query_results.total_rows,
+                    total_bytes_processed=query_results.total_bytes_processed,
+                    slot_millis=query_results.slot_millis,
+                    destination=job.destination,
+                    created=job.created,
+                    started=job.started,
+                    ended=job.ended,
+                )
+            )
+        return query_results
     except Exception:
         # Attempt to cancel the job since we can't return the results.
         try:
@@ -632,3 +725,62 @@ def _wait_or_cancel(
             # Don't eat the original exception if cancel fails.
             pass
         raise
+
+
+@dataclasses.dataclass(frozen=True)
+class QueryFinishedEvent:
+    """Query finished successfully."""
+
+    billing_project: Optional[str]
+    location: Optional[str]
+    query_id: Optional[str]
+    job_id: Optional[str]
+    destination: Optional[table.TableReference]
+    total_rows: Optional[int]
+    total_bytes_processed: Optional[int]
+    slot_millis: Optional[int]
+    created: Optional[datetime.datetime]
+    started: Optional[datetime.datetime]
+    ended: Optional[datetime.datetime]
+
+
+@dataclasses.dataclass(frozen=True)
+class QueryReceivedEvent:
+    """Query received and acknowledged by the BigQuery API."""
+
+    billing_project: Optional[str]
+    location: Optional[str]
+    job_id: Optional[str]
+    statement_type: Optional[str]
+    state: Optional[str]
+    query_plan: Optional[list[google.cloud.bigquery.job.query.QueryPlanEntry]]
+    created: Optional[datetime.datetime]
+    started: Optional[datetime.datetime]
+    ended: Optional[datetime.datetime]
+
+
+@dataclasses.dataclass(frozen=True)
+class QuerySentEvent:
+    """Query sent to BigQuery."""
+
+    query: str
+    billing_project: Optional[str]
+    location: Optional[str]
+    job_id: Optional[str]
+    request_id: Optional[str]
+
+
+class QueryRetryEvent(QuerySentEvent):
+    """Query sent another time because the previous attempt failed."""
+
+
+class QuerySentEventFactory:
+    """Creates a QuerySentEvent first, then QueryRetryEvent after that."""
+
+    def __init__(self):
+        self._event_constructor = QuerySentEvent
+
+    def __call__(self, **kwargs):
+        result = self._event_constructor(**kwargs)
+        self._event_constructor = QueryRetryEvent
+        return result

{google_cloud_bigquery-3.35.0 → google_cloud_bigquery-3.36.0}/google/cloud/bigquery/client.py

@@ -15,6 +15,7 @@
 """Client for interacting with the Google BigQuery API."""
 
 from __future__ import absolute_import
+from __future__ import annotations
 from __future__ import division
 
 from collections import abc as collections_abc
@@ -31,6 +32,7 @@ import tempfile
 import typing
 from typing import (
     Any,
+    Callable,
     Dict,
     IO,
     Iterable,
@@ -3633,8 +3635,8 @@ class Client(ClientWithProject):
                 rate-limit-exceeded errors.  Passing ``None`` disables
                 job retry. Not all jobs can be retried.
             page_size (Optional[int]):
-                The maximum number of rows in each page of results from this
-                request. Non-positive values are ignored.
+                The maximum number of rows in each page of results from the
+                initial jobs.query request. Non-positive values are ignored.
             max_results (Optional[int]):
                 The maximum total number of rows from this request.
 
@@ -3656,6 +3658,39 @@ class Client(ClientWithProject):
                 :class:`~google.cloud.bigquery.job.QueryJobConfig`
                 class.
         """
+        return self._query_and_wait_bigframes(
+            query,
+            job_config=job_config,
+            location=location,
+            project=project,
+            api_timeout=api_timeout,
+            wait_timeout=wait_timeout,
+            retry=retry,
+            job_retry=job_retry,
+            page_size=page_size,
+            max_results=max_results,
+        )
+
+    def _query_and_wait_bigframes(
+        self,
+        query,
+        *,
+        job_config: Optional[QueryJobConfig] = None,
+        location: Optional[str] = None,
+        project: Optional[str] = None,
+        api_timeout: TimeoutType = DEFAULT_TIMEOUT,
+        wait_timeout: Union[Optional[float], object] = POLLING_DEFAULT_VALUE,
+        retry: retries.Retry = DEFAULT_RETRY,
+        job_retry: retries.Retry = DEFAULT_JOB_RETRY,
+        page_size: Optional[int] = None,
+        max_results: Optional[int] = None,
+        callback: Callable = lambda _: None,
+    ) -> RowIterator:
+        """See query_and_wait.
+
+        This method has an extra callback parameter, which is used by bigframes
+        to create better progress bars.
+        """
         if project is None:
             project = self.project
 
@@ -3681,6 +3716,7 @@ class Client(ClientWithProject):
             job_retry=job_retry,
             page_size=page_size,
             max_results=max_results,
+            callback=callback,
         )
 
     def insert_rows(
@@ -4145,6 +4181,9 @@ class Client(ClientWithProject):
         query: Optional[str] = None,
         total_bytes_processed: Optional[int] = None,
         slot_millis: Optional[int] = None,
+        created: Optional[datetime.datetime] = None,
+        started: Optional[datetime.datetime] = None,
+        ended: Optional[datetime.datetime] = None,
     ) -> RowIterator:
         """List the rows of a completed query.
         See
@@ -4198,6 +4237,12 @@ class Client(ClientWithProject):
                 total bytes processed from job statistics, if present.
             slot_millis (Optional[int]):
                 Number of slot ms the user is actually billed for.
+            created (Optional[datetime.datetime]):
+                Datetime at which the job was created.
+            started (Optional[datetime.datetime]):
+                Datetime at which the job was started.
+            ended (Optional[datetime.datetime]):
+                Datetime at which the job finished.
 
         Returns:
             google.cloud.bigquery.table.RowIterator:
@@ -4238,6 +4283,9 @@ class Client(ClientWithProject):
             query=query,
             total_bytes_processed=total_bytes_processed,
             slot_millis=slot_millis,
+            created=created,
+            started=started,
+            ended=ended,
         )
         return row_iterator
 

{google_cloud_bigquery-3.35.0 → google_cloud_bigquery-3.36.0}/google/cloud/bigquery/job/query.py

@@ -1550,6 +1550,8 @@ class QueryJob(_AsyncJob):
             return _EmptyRowIterator(
                 project=self.project,
                 location=self.location,
+                schema=self.schema,
+                total_bytes_processed=self.total_bytes_processed,
                 # Intentionally omit job_id and query_id since this doesn't
                 # actually correspond to a finished query job.
             )
@@ -1737,7 +1739,11 @@ class QueryJob(_AsyncJob):
                 project=self.project,
                 job_id=self.job_id,
                 query_id=self.query_id,
+                schema=self.schema,
                 num_dml_affected_rows=self._query_results.num_dml_affected_rows,
+                query=self.query,
+                total_bytes_processed=self.total_bytes_processed,
+                slot_millis=self.slot_millis,
             )
 
         # We know that there's at least 1 row, so only treat the response from
@@ -1767,6 +1773,9 @@ class QueryJob(_AsyncJob):
             query=self.query,
             total_bytes_processed=self.total_bytes_processed,
             slot_millis=self.slot_millis,
+            created=self.created,
+            started=self.started,
+            ended=self.ended,
             **list_rows_kwargs,
         )
         rows._preserve_order = _contains_order_by(self.query)

{google_cloud_bigquery-3.35.0 → google_cloud_bigquery-3.36.0}/google/cloud/bigquery/query.py

@@ -1228,11 +1228,18 @@ class _QueryResults(object):
 
         See:
         https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.job_reference
+        or https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.location
 
         Returns:
             str: Job ID of the query job.
         """
-        return self._properties.get("jobReference", {}).get("location")
+        location = self._properties.get("jobReference", {}).get("location")
+
+        # Sometimes there's no job, but we still want to get the location
+        # information. Prefer the value from job for backwards compatibilitity.
+        if not location:
+            location = self._properties.get("location")
+        return location
 
     @property
     def query_id(self) -> Optional[str]:
@@ -1287,7 +1294,7 @@ class _QueryResults(object):
         """Total number of slot ms the user is actually billed for.
 
         See:
-        https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.slot_millis
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.total_slot_ms
 
         Returns:
             Optional[int]: Count generated on the server (None until set by the server).
@@ -1310,6 +1317,56 @@ class _QueryResults(object):
         if num_dml_affected_rows is not None:
             return int(num_dml_affected_rows)
 
+    @property
+    def created(self):
+        """Creation time of this query.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.creation_time
+
+        Returns:
+            Optional[datetime.datetime]:
+                the creation time (None until set from the server).
+        """
+        millis = self._properties.get("creationTime")
+        if millis is not None:
+            return _helpers._datetime_from_microseconds(int(millis) * 1000.0)
+
+    @property
+    def started(self):
+        """Start time of this query.
+
+        This field will be present when the query transitions from the
+        PENDING state to either RUNNING or DONE.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.start_time
+
+        Returns:
+            Optional[datetime.datetime]:
+                the start time (None until set from the server).
+        """
+        millis = self._properties.get("startTime")
+        if millis is not None:
+            return _helpers._datetime_from_microseconds(int(millis) * 1000.0)
+
+    @property
+    def ended(self):
+        """End time of this query.
+
+        This field will be present whenever a query is in the DONE state.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#body.QueryResponse.FIELDS.end_time
+
+        Returns:
+            Optional[datetime.datetime]:
+                the end time (None until set from the server).
+        """
+        millis = self._properties.get("endTime")
+        if millis is not None:
+            return _helpers._datetime_from_microseconds(int(millis) * 1000.0)
+
     @property
     def rows(self):
         """Query results.

{google_cloud_bigquery-3.35.0 → google_cloud_bigquery-3.36.0}/google/cloud/bigquery/retry.py

@@ -124,6 +124,8 @@ job_retry_reasons = (
     "rateLimitExceeded",
     "backendError",
     "internalError",
+    "jobBackendError",
+    "jobInternalError",
     "jobRateLimitExceeded",
 )
 

{google_cloud_bigquery-3.35.0 → google_cloud_bigquery-3.36.0}/google/cloud/bigquery/table.py

@@ -1788,7 +1788,15 @@ class RowIterator(HTTPIterator):
         query (Optional[str]):
             The query text used.
         total_bytes_processed (Optional[int]):
-            total bytes processed from job statistics, if present.
+            If representing query results, the total bytes processed by the associated query.
+        slot_millis (Optional[int]):
+            If representing query results, the number of slot ms billed for the associated query.
+        created (Optional[datetime.datetime]):
+            If representing query results, the creation time of the associated query.
+        started (Optional[datetime.datetime]):
+            If representing query results, the start time of the associated query.
+        ended (Optional[datetime.datetime]):
+            If representing query results, the end time of the associated query.
     """
 
     def __init__(
@@ -1813,6 +1821,9 @@ class RowIterator(HTTPIterator):
         query: Optional[str] = None,
         total_bytes_processed: Optional[int] = None,
         slot_millis: Optional[int] = None,
+        created: Optional[datetime.datetime] = None,
+        started: Optional[datetime.datetime] = None,
+        ended: Optional[datetime.datetime] = None,
     ):
         super(RowIterator, self).__init__(
             client,
@@ -1826,7 +1837,7 @@ class RowIterator(HTTPIterator):
             page_start=_rows_page_start,
             next_token="pageToken",
         )
-        schema = _to_schema_fields(schema)
+        schema = _to_schema_fields(schema) if schema else ()
         self._field_to_index = _helpers._field_to_index_mapping(schema)
         self._page_size = page_size
         self._preserve_order = False
@@ -1843,6 +1854,9 @@ class RowIterator(HTTPIterator):
         self._query = query
         self._total_bytes_processed = total_bytes_processed
         self._slot_millis = slot_millis
+        self._job_created = created
+        self._job_started = started
+        self._job_ended = ended
 
     @property
     def _billing_project(self) -> Optional[str]:
@@ -1905,6 +1919,21 @@ class RowIterator(HTTPIterator):
         """Number of slot ms the user is actually billed for."""
         return self._slot_millis
 
+    @property
+    def created(self) -> Optional[datetime.datetime]:
+        """If representing query results, the creation time of the associated query."""
+        return self._job_created
+
+    @property
+    def started(self) -> Optional[datetime.datetime]:
+        """If representing query results, the start time of the associated query."""
+        return self._job_started
+
+    @property
+    def ended(self) -> Optional[datetime.datetime]:
+        """If representing query results, the end time of the associated query."""
+        return self._job_ended
+
     def _is_almost_completely_cached(self):
         """Check if all results are completely cached.
 
@@ -2888,7 +2917,6 @@ class _EmptyRowIterator(RowIterator):
     statements.
     """
 
-    schema = ()
     pages = ()
     total_rows = 0
 

{google_cloud_bigquery-3.35.0 → google_cloud_bigquery-3.36.0}/google/cloud/bigquery/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.35.0"
+__version__ = "3.36.0"

{google_cloud_bigquery-3.35.0 → google_cloud_bigquery-3.36.0/google_cloud_bigquery.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-cloud-bigquery
-Version: 3.35.0
+Version: 3.36.0
 Summary: Google BigQuery API client library
 Author-email: Google LLC <googleapis-packages@google.com>
 License: Apache 2.0
@@ -84,7 +84,7 @@ processing power of Google's infrastructure.
 .. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-bigquery.svg
    :target: https://pypi.org/project/google-cloud-bigquery/
 .. _BigQuery: https://cloud.google.com/bigquery/what-is-bigquery
-.. _Client Library Documentation: https://googleapis.dev/python/bigquery/latest
+.. _Client Library Documentation: https://cloud.google.com/python/docs/reference/bigquery/latest/summary_overview
 .. _Product Documentation: https://cloud.google.com/bigquery/docs/reference/v2/
 
 Quick Start

{google_cloud_bigquery-3.35.0 → google_cloud_bigquery-3.36.0}/google_cloud_bigquery.egg-info/SOURCES.txt

@@ -92,10 +92,14 @@ tests/unit/helpers.py
 tests/unit/test__helpers.py
 tests/unit/test__http.py
 tests/unit/test__job_helpers.py
+tests/unit/test__job_helpers_retry.py
 tests/unit/test__pandas_helpers.py
 tests/unit/test__pyarrow_helpers.py
 tests/unit/test__versions_helpers.py
 tests/unit/test_client.py
+tests/unit/test_client_bigframes.py
+tests/unit/test_client_resumable_media_upload.py
+tests/unit/test_client_retry.py
 tests/unit/test_create_dataset.py
 tests/unit/test_dataset.py
 tests/unit/test_dbapi__helpers.py
@@ -131,6 +135,7 @@ tests/unit/_helpers/test_data_frame_cell_data_parser.py
 tests/unit/_helpers/test_scalar_query_param_parser.py
 tests/unit/job/__init__.py
 tests/unit/job/helpers.py
+tests/unit/job/test_async_job_retry.py
 tests/unit/job/test_base.py
 tests/unit/job/test_copy.py
 tests/unit/job/test_extract.py
@@ -138,6 +143,7 @@ tests/unit/job/test_load.py
 tests/unit/job/test_load_config.py
 tests/unit/job/test_query.py
 tests/unit/job/test_query_config.py
+tests/unit/job/test_query_job_retry.py
 tests/unit/job/test_query_pandas.py
 tests/unit/job/test_query_stats.py
 tests/unit/line_arg_parser/__init__.py