google-cloud-bigquery 3.43.0__tar.gz → 3.45.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {google_cloud_bigquery-3.43.0/google_cloud_bigquery.egg-info → google_cloud_bigquery-3.45.0}/PKG-INFO +1 -1
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/__init__.py +9 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/_job_helpers.py +19 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/client.py +34 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/enums.py +17 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/table.py +274 -38
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/version.py +1 -1
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0/google_cloud_bigquery.egg-info}/PKG-INFO +1 -1
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google_cloud_bigquery.egg-info/SOURCES.txt +1 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/test_arrow.py +51 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test__job_helpers.py +1 -1
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_client.py +51 -6
- google_cloud_bigquery-3.45.0/tests/unit/test_query_results_format_arrow.py +573 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_table.py +386 -50
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/LICENSE +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/MANIFEST.in +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/README.rst +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/_http.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/_pandas_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/_pyarrow_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/_string_references.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/_tqdm_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/_versions_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/dataset.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/dbapi/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/dbapi/_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/dbapi/connection.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/dbapi/cursor.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/dbapi/exceptions.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/dbapi/types.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/encryption_configuration.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/exceptions.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/external_config.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/format_options.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/iam.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/job/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/job/base.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/job/copy_.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/job/extract.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/job/load.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/job/query.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/magics/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/magics/line_arg_parser/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/magics/line_arg_parser/exceptions.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/magics/line_arg_parser/lexer.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/magics/line_arg_parser/parser.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/magics/line_arg_parser/visitors.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/magics/magics.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/model.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/opentelemetry_tracing.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/py.typed +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/query.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/routine/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/routine/routine.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/schema.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/standard_sql.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery_v2/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery_v2/types/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery_v2/types/encryption_config.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery_v2/types/model.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery_v2/types/model_reference.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery_v2/types/standard_sql.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery_v2/types/table_reference.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google_cloud_bigquery.egg-info/dependency_links.txt +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google_cloud_bigquery.egg-info/requires.txt +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google_cloud_bigquery.egg-info/top_level.txt +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/pyproject.toml +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/setup.cfg +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/setup.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/characters.json +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/characters.jsonl +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/colors.avro +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/numeric_38_12.parquet +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/people.csv +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/pico.csv +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/pico_schema.json +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/scalars.csv +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/scalars.jsonl +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/scalars_extreme.jsonl +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/scalars_schema.json +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/scalars_schema_csv.json +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/data/schema.json +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/scrub_datasets.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/conftest.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/test_client.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/test_job_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/test_list_rows.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/test_magics.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/test_pandas.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/test_query.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/test_ssl_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/system/test_structs.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/_helpers/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/_helpers/test_cell_data_parser.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/_helpers/test_data_frame_cell_data_parser.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/_helpers/test_scalar_query_param_parser.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/conftest.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/test_async_job_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/test_base.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/test_copy.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/test_extract.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/test_load.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/test_load_config.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/test_query.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/test_query_config.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/test_query_job_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/test_query_pandas.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/job/test_query_stats.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/line_arg_parser/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/line_arg_parser/test_lexer.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/line_arg_parser/test_parser.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/line_arg_parser/test_visitors.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/model/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/model/test_model.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/model/test_model_reference.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/routine/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/routine/test_external_runtime_options.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/routine/test_remote_function_options.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/routine/test_routine.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/routine/test_routine_argument.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/routine/test_routine_reference.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test__helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test__http.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test__job_helpers_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test__pandas_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test__pyarrow_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test__versions_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_client_bigframes.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_client_resumable_media_upload.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_client_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_create_dataset.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_dataset.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_dbapi__helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_dbapi_connection.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_dbapi_cursor.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_dbapi_types.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_delete_dataset.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_encryption_configuration.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_external_config.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_format_options.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_job_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_legacy_types.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_list_datasets.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_list_jobs.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_list_models.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_list_projects.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_list_routines.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_list_tables.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_magics.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_opentelemetry_tracing.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_packaging.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_query.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_schema.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_signature_compatibility.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_standard_sql_types.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_string_references.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_table_arrow.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/tests/unit/test_table_pandas.py +0 -0
{google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/__init__.py
RENAMED
|
@@ -30,6 +30,8 @@ The main concepts with this API are:
|
|
|
30
30
|
import sys
|
|
31
31
|
import warnings
|
|
32
32
|
|
|
33
|
+
import google.api_core as api_core
|
|
34
|
+
|
|
33
35
|
from google.cloud.bigquery import version as bigquery_version
|
|
34
36
|
|
|
35
37
|
__version__ = bigquery_version.__version__
|
|
@@ -57,6 +59,8 @@ from google.cloud.bigquery.external_config import ExternalSourceFormat
|
|
|
57
59
|
from google.cloud.bigquery.external_config import HivePartitioningOptions
|
|
58
60
|
from google.cloud.bigquery.format_options import AvroOptions
|
|
59
61
|
from google.cloud.bigquery.format_options import ParquetOptions
|
|
62
|
+
from google.cloud.bigquery.enums import QueryResultsCompressionCodec
|
|
63
|
+
from google.cloud.bigquery.enums import QueryResultsFormat
|
|
60
64
|
from google.cloud.bigquery.job.base import SessionInfo
|
|
61
65
|
from google.cloud.bigquery.job import Compression
|
|
62
66
|
from google.cloud.bigquery.job import CopyJob
|
|
@@ -132,6 +136,9 @@ if sys.version_info < (3, 10): # pragma: NO COVER
|
|
|
132
136
|
FutureWarning,
|
|
133
137
|
)
|
|
134
138
|
|
|
139
|
+
api_core.check_python_version(__name__)
|
|
140
|
+
api_core.check_dependency_versions(__name__)
|
|
141
|
+
|
|
135
142
|
__all__ = [
|
|
136
143
|
"__version__",
|
|
137
144
|
"Client",
|
|
@@ -216,6 +223,8 @@ __all__ = [
|
|
|
216
223
|
"KeyResultStatementKind",
|
|
217
224
|
"OperationType",
|
|
218
225
|
"QueryPriority",
|
|
226
|
+
"QueryResultsCompressionCodec",
|
|
227
|
+
"QueryResultsFormat",
|
|
219
228
|
"RoutineType",
|
|
220
229
|
"SchemaUpdateOption",
|
|
221
230
|
"SourceFormat",
|
{google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/_job_helpers.py
RENAMED
|
@@ -430,6 +430,8 @@ def query_and_wait(
|
|
|
430
430
|
job_retry: Optional[retries.Retry],
|
|
431
431
|
page_size: Optional[int] = None,
|
|
432
432
|
max_results: Optional[int] = None,
|
|
433
|
+
query_results_format: Optional[str] = None,
|
|
434
|
+
compression_codec: Optional[str] = None,
|
|
433
435
|
callback: Callable = lambda _: None,
|
|
434
436
|
) -> table.RowIterator:
|
|
435
437
|
"""Run the query, wait for it to finish, and return the results.
|
|
@@ -475,6 +477,10 @@ def query_and_wait(
|
|
|
475
477
|
request. Non-positive values are ignored.
|
|
476
478
|
max_results (Optional[int]):
|
|
477
479
|
The maximum total number of rows from this request.
|
|
480
|
+
query_results_format (Optional[Union[str, google.cloud.bigquery.enums.QueryResultsFormat]]):
|
|
481
|
+
[Beta] The format for query results (e.g. "ARROW" or :class:`~google.cloud.bigquery.enums.QueryResultsFormat.ARROW`).
|
|
482
|
+
compression_codec (Optional[Union[str, google.cloud.bigquery.enums.QueryResultsCompressionCodec]]):
|
|
483
|
+
[Beta] Compression codec for Arrow serialization (e.g. "LZ4_FRAME" or :class:`~google.cloud.bigquery.enums.QueryResultsCompressionCodec.LZ4_FRAME`).
|
|
478
484
|
callback (Callable):
|
|
479
485
|
A callback function used by bigframes to report query progress.
|
|
480
486
|
|
|
@@ -499,6 +505,13 @@ def query_and_wait(
|
|
|
499
505
|
request_body = _to_query_request(
|
|
500
506
|
query=query, job_config=job_config, location=location, timeout=api_timeout
|
|
501
507
|
)
|
|
508
|
+
if query_results_format is not None:
|
|
509
|
+
request_body["queryResultsFormat"] = query_results_format
|
|
510
|
+
if compression_codec is not None:
|
|
511
|
+
request_body.setdefault("formatOptions", {})
|
|
512
|
+
request_body["formatOptions"]["arrowSerializationOptions"] = {
|
|
513
|
+
"bufferCompression": compression_codec
|
|
514
|
+
}
|
|
502
515
|
|
|
503
516
|
# Some API parameters aren't supported by the jobs.query API. In these
|
|
504
517
|
# cases, fallback to a jobs.insert call.
|
|
@@ -522,6 +535,7 @@ def query_and_wait(
|
|
|
522
535
|
retry=retry,
|
|
523
536
|
page_size=page_size,
|
|
524
537
|
max_results=max_results,
|
|
538
|
+
query_results_format=query_results_format,
|
|
525
539
|
callback=callback,
|
|
526
540
|
)
|
|
527
541
|
|
|
@@ -594,6 +608,7 @@ def query_and_wait(
|
|
|
594
608
|
retry=retry,
|
|
595
609
|
page_size=page_size,
|
|
596
610
|
max_results=max_results,
|
|
611
|
+
query_results_format=query_results_format,
|
|
597
612
|
callback=callback,
|
|
598
613
|
)
|
|
599
614
|
|
|
@@ -633,6 +648,7 @@ def query_and_wait(
|
|
|
633
648
|
created=query_results.created,
|
|
634
649
|
started=query_results.started,
|
|
635
650
|
ended=query_results.ended,
|
|
651
|
+
query_results_format=query_results_format,
|
|
636
652
|
)
|
|
637
653
|
|
|
638
654
|
if job_retry is not None:
|
|
@@ -673,6 +689,7 @@ def _supported_by_jobs_query(request_body: Dict[str, Any]) -> bool:
|
|
|
673
689
|
"jobTimeoutMs",
|
|
674
690
|
"reservation",
|
|
675
691
|
"maxSlots",
|
|
692
|
+
"queryResultsFormat",
|
|
676
693
|
}
|
|
677
694
|
|
|
678
695
|
unsupported_keys = request_keys - keys_allowlist
|
|
@@ -687,6 +704,7 @@ def _wait_or_cancel(
|
|
|
687
704
|
page_size: Optional[int],
|
|
688
705
|
max_results: Optional[int],
|
|
689
706
|
*,
|
|
707
|
+
query_results_format: Optional[str] = None,
|
|
690
708
|
callback: Callable = lambda _: None,
|
|
691
709
|
) -> table.RowIterator:
|
|
692
710
|
"""Wait for a job to complete and return the results.
|
|
@@ -731,6 +749,7 @@ def _wait_or_cancel(
|
|
|
731
749
|
ended=job.ended,
|
|
732
750
|
)
|
|
733
751
|
)
|
|
752
|
+
query_results._query_results_format = query_results_format
|
|
734
753
|
return query_results
|
|
735
754
|
except Exception:
|
|
736
755
|
# Attempt to cancel the job since we can't return the results.
|
{google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/client.py
RENAMED
|
@@ -163,6 +163,16 @@ _LIST_ROWS_FROM_QUERY_RESULTS_FIELDS = "jobReference,totalRows,pageToken,rows"
|
|
|
163
163
|
# https://github.com/googleapis/python-bigquery/issues/438
|
|
164
164
|
_MIN_GET_QUERY_RESULTS_TIMEOUT = 120
|
|
165
165
|
|
|
166
|
+
_LOAD_TABLE_FROM_DATAFRAME_DEPRECATED = (
|
|
167
|
+
"Loading DataFrames via google-cloud-bigquery is deprecated. "
|
|
168
|
+
"For direct, optimized loading, please call 'pandas_gbq.to_gbq()' directly."
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
_INSERT_ROWS_FROM_DATAFRAME_DEPRECATED = (
|
|
172
|
+
"Inserting rows from DataFrames via google-cloud-bigquery is deprecated. "
|
|
173
|
+
"For direct, optimized access, please call 'pandas_gbq.to_gbq()' directly."
|
|
174
|
+
)
|
|
175
|
+
|
|
166
176
|
TIMEOUT_HEADER = "X-Server-Timeout"
|
|
167
177
|
|
|
168
178
|
|
|
@@ -2830,6 +2840,12 @@ class Client(ClientWithProject):
|
|
|
2830
2840
|
If ``job_config`` is not an instance of
|
|
2831
2841
|
:class:`~google.cloud.bigquery.job.LoadJobConfig` class.
|
|
2832
2842
|
"""
|
|
2843
|
+
warnings.warn(
|
|
2844
|
+
_LOAD_TABLE_FROM_DATAFRAME_DEPRECATED,
|
|
2845
|
+
PendingDeprecationWarning,
|
|
2846
|
+
stacklevel=2,
|
|
2847
|
+
)
|
|
2848
|
+
|
|
2833
2849
|
job_id = _make_job_id(job_id, job_id_prefix)
|
|
2834
2850
|
|
|
2835
2851
|
if job_config is not None:
|
|
@@ -3649,6 +3665,8 @@ class Client(ClientWithProject):
|
|
|
3649
3665
|
job_retry: retries.Retry = DEFAULT_JOB_RETRY,
|
|
3650
3666
|
page_size: Optional[int] = None,
|
|
3651
3667
|
max_results: Optional[int] = None,
|
|
3668
|
+
query_results_format: Optional[str] = None,
|
|
3669
|
+
compression_codec: Optional[str] = None,
|
|
3652
3670
|
) -> RowIterator:
|
|
3653
3671
|
"""Run the query, wait for it to finish, and return the results.
|
|
3654
3672
|
|
|
@@ -3696,6 +3714,10 @@ class Client(ClientWithProject):
|
|
|
3696
3714
|
by this parameter.
|
|
3697
3715
|
max_results (Optional[int]):
|
|
3698
3716
|
The maximum total number of rows from this request.
|
|
3717
|
+
query_results_format (Optional[Union[str, google.cloud.bigquery.enums.QueryResultsFormat]]):
|
|
3718
|
+
[Beta] The format for query results (e.g. "ARROW" or :class:`~google.cloud.bigquery.enums.QueryResultsFormat.ARROW`).
|
|
3719
|
+
compression_codec (Optional[Union[str, google.cloud.bigquery.enums.QueryResultsCompressionCodec]]):
|
|
3720
|
+
[Beta] Compression codec for Arrow serialization (e.g. "LZ4_FRAME" or :class:`~google.cloud.bigquery.enums.QueryResultsCompressionCodec.LZ4_FRAME`).
|
|
3699
3721
|
|
|
3700
3722
|
Returns:
|
|
3701
3723
|
google.cloud.bigquery.table.RowIterator:
|
|
@@ -3726,6 +3748,8 @@ class Client(ClientWithProject):
|
|
|
3726
3748
|
job_retry=job_retry,
|
|
3727
3749
|
page_size=page_size,
|
|
3728
3750
|
max_results=max_results,
|
|
3751
|
+
query_results_format=query_results_format,
|
|
3752
|
+
compression_codec=compression_codec,
|
|
3729
3753
|
)
|
|
3730
3754
|
|
|
3731
3755
|
def _query_and_wait_bigframes(
|
|
@@ -3741,6 +3765,8 @@ class Client(ClientWithProject):
|
|
|
3741
3765
|
job_retry: retries.Retry = DEFAULT_JOB_RETRY,
|
|
3742
3766
|
page_size: Optional[int] = None,
|
|
3743
3767
|
max_results: Optional[int] = None,
|
|
3768
|
+
query_results_format: Optional[str] = None,
|
|
3769
|
+
compression_codec: Optional[str] = None,
|
|
3744
3770
|
callback: Callable = lambda _: None,
|
|
3745
3771
|
) -> RowIterator:
|
|
3746
3772
|
"""See query_and_wait.
|
|
@@ -3773,6 +3799,8 @@ class Client(ClientWithProject):
|
|
|
3773
3799
|
job_retry=job_retry,
|
|
3774
3800
|
page_size=page_size,
|
|
3775
3801
|
max_results=max_results,
|
|
3802
|
+
query_results_format=query_results_format,
|
|
3803
|
+
compression_codec=compression_codec,
|
|
3776
3804
|
callback=callback,
|
|
3777
3805
|
)
|
|
3778
3806
|
|
|
@@ -3900,6 +3928,12 @@ class Client(ClientWithProject):
|
|
|
3900
3928
|
Raises:
|
|
3901
3929
|
ValueError: if table's schema is not set
|
|
3902
3930
|
"""
|
|
3931
|
+
warnings.warn(
|
|
3932
|
+
_INSERT_ROWS_FROM_DATAFRAME_DEPRECATED,
|
|
3933
|
+
PendingDeprecationWarning,
|
|
3934
|
+
stacklevel=2,
|
|
3935
|
+
)
|
|
3936
|
+
|
|
3903
3937
|
insert_results = []
|
|
3904
3938
|
|
|
3905
3939
|
chunk_count = int(math.ceil(len(dataframe) / chunk_size))
|
{google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/enums.py
RENAMED
|
@@ -495,3 +495,20 @@ class TimestampPrecision(enum.Enum):
|
|
|
495
495
|
"""
|
|
496
496
|
For TIMESTAMP type with picosecond precision.
|
|
497
497
|
"""
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
class QueryResultsFormat(str, enum.Enum):
|
|
501
|
+
"""[Beta] Format for query results response."""
|
|
502
|
+
|
|
503
|
+
ARROW = "ARROW"
|
|
504
|
+
"""Specifies Apache Arrow format for query results."""
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
class QueryResultsCompressionCodec(str, enum.Enum):
|
|
508
|
+
"""[Beta] Compression codec for Arrow query results serialization."""
|
|
509
|
+
|
|
510
|
+
LZ4_FRAME = "LZ4_FRAME"
|
|
511
|
+
"""Specifies LZ4_FRAME compression codec."""
|
|
512
|
+
|
|
513
|
+
ZSTD = "ZSTD"
|
|
514
|
+
"""Specifies ZSTD compression codec."""
|
{google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.45.0}/google/cloud/bigquery/table.py
RENAMED
|
@@ -16,14 +16,15 @@
|
|
|
16
16
|
|
|
17
17
|
from __future__ import absolute_import
|
|
18
18
|
|
|
19
|
+
import base64
|
|
19
20
|
import copy
|
|
20
21
|
import datetime
|
|
21
22
|
import functools
|
|
23
|
+
import logging
|
|
22
24
|
import operator
|
|
23
25
|
import typing
|
|
24
|
-
from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union, Sequence
|
|
25
|
-
|
|
26
26
|
import warnings
|
|
27
|
+
from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple, Union
|
|
27
28
|
|
|
28
29
|
try:
|
|
29
30
|
import pandas # type: ignore
|
|
@@ -56,33 +57,37 @@ else:
|
|
|
56
57
|
_read_wkt = wkt.loads
|
|
57
58
|
|
|
58
59
|
import google.api_core.exceptions
|
|
59
|
-
from google.api_core.page_iterator import HTTPIterator
|
|
60
|
-
|
|
61
60
|
import google.cloud._helpers # type: ignore
|
|
62
|
-
from google.
|
|
63
|
-
from google.cloud.bigquery import
|
|
64
|
-
|
|
61
|
+
from google.api_core.page_iterator import HTTPIterator
|
|
62
|
+
from google.cloud.bigquery import (
|
|
63
|
+
_helpers,
|
|
64
|
+
_pandas_helpers,
|
|
65
|
+
_string_references,
|
|
66
|
+
_versions_helpers,
|
|
67
|
+
external_config,
|
|
68
|
+
)
|
|
65
69
|
from google.cloud.bigquery import exceptions as bq_exceptions
|
|
70
|
+
from google.cloud.bigquery import schema as _schema
|
|
66
71
|
from google.cloud.bigquery._tqdm_helpers import get_progress_bar
|
|
67
72
|
from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
|
|
68
|
-
from google.cloud.bigquery.enums import DefaultPandasDTypes
|
|
73
|
+
from google.cloud.bigquery.enums import DefaultPandasDTypes, QueryResultsFormat
|
|
69
74
|
from google.cloud.bigquery.external_config import ExternalConfig
|
|
70
|
-
from google.cloud.bigquery import
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
from google.cloud.bigquery import _string_references
|
|
75
|
+
from google.cloud.bigquery.schema import (
|
|
76
|
+
_build_schema_resource,
|
|
77
|
+
_parse_schema_resource,
|
|
78
|
+
_to_schema_fields,
|
|
79
|
+
)
|
|
76
80
|
|
|
77
81
|
if typing.TYPE_CHECKING: # pragma: NO COVER
|
|
78
82
|
# Unconditionally import optional dependencies again to tell pytype that
|
|
79
83
|
# they are not None, avoiding false "no attribute" errors.
|
|
84
|
+
import geopandas # type: ignore
|
|
80
85
|
import pandas
|
|
81
86
|
import pyarrow
|
|
82
|
-
import geopandas # type: ignore
|
|
83
87
|
from google.cloud import bigquery_storage # type: ignore
|
|
84
88
|
from google.cloud.bigquery.dataset import DatasetReference
|
|
85
89
|
|
|
90
|
+
_LOGGER = logging.getLogger(__name__)
|
|
86
91
|
|
|
87
92
|
_NO_GEOPANDAS_ERROR = (
|
|
88
93
|
"The geopandas library is not installed, please install "
|
|
@@ -110,6 +115,22 @@ _RANGE_PYARROW_WARNING = (
|
|
|
110
115
|
"pyarrow >= 10.0.1."
|
|
111
116
|
)
|
|
112
117
|
|
|
118
|
+
_TO_DATAFRAME_DEPRECATED = (
|
|
119
|
+
"Retrieving DataFrames via google-cloud-bigquery is deprecated. "
|
|
120
|
+
"For direct, optimized access, please call 'pandas_gbq.read_gbq()' directly."
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
_TO_ARROW_DEPRECATED = (
|
|
124
|
+
"Retrieving PyArrow Tables via google-cloud-bigquery is deprecated. "
|
|
125
|
+
"For direct, optimized access, please call 'pandas_gbq.arrow.read_bigquery_table()' "
|
|
126
|
+
"or 'pandas_gbq.arrow.read_bigquery_query()' directly."
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
_CANNOT_ITERATE_ARROW_ERROR = (
|
|
130
|
+
"Cannot iterate over non-arrow results when queryResultsFormat is ARROW. "
|
|
131
|
+
"Use to_arrow_iterable() or to_arrow() instead."
|
|
132
|
+
)
|
|
133
|
+
|
|
113
134
|
# How many of the total rows need to be downloaded already for us to skip
|
|
114
135
|
# calling the BQ Storage API?
|
|
115
136
|
#
|
|
@@ -164,6 +185,18 @@ def _view_use_legacy_sql_getter(
|
|
|
164
185
|
return None # explicit return statement to appease mypy
|
|
165
186
|
|
|
166
187
|
|
|
188
|
+
def _disallow_when_arrow(func):
|
|
189
|
+
"""Decorator that disallows iteration/access when queryResultsFormat is ARROW."""
|
|
190
|
+
|
|
191
|
+
@functools.wraps(func)
|
|
192
|
+
def wrapper(self, *args, **kwargs):
|
|
193
|
+
if self._query_results_format == QueryResultsFormat.ARROW.value:
|
|
194
|
+
raise ValueError(_CANNOT_ITERATE_ARROW_ERROR)
|
|
195
|
+
return func(self, *args, **kwargs)
|
|
196
|
+
|
|
197
|
+
return wrapper
|
|
198
|
+
|
|
199
|
+
|
|
167
200
|
class _TableBase:
|
|
168
201
|
"""Base class for Table-related classes with common functionality."""
|
|
169
202
|
|
|
@@ -1912,6 +1945,7 @@ class RowIterator(HTTPIterator):
|
|
|
1912
1945
|
created: Optional[datetime.datetime] = None,
|
|
1913
1946
|
started: Optional[datetime.datetime] = None,
|
|
1914
1947
|
ended: Optional[datetime.datetime] = None,
|
|
1948
|
+
query_results_format: Optional[str] = None,
|
|
1915
1949
|
):
|
|
1916
1950
|
super(RowIterator, self).__init__(
|
|
1917
1951
|
client,
|
|
@@ -1945,6 +1979,20 @@ class RowIterator(HTTPIterator):
|
|
|
1945
1979
|
self._job_created = created
|
|
1946
1980
|
self._job_started = started
|
|
1947
1981
|
self._job_ended = ended
|
|
1982
|
+
self._query_results_format = query_results_format
|
|
1983
|
+
|
|
1984
|
+
@property
|
|
1985
|
+
@_disallow_when_arrow
|
|
1986
|
+
def pages(self):
|
|
1987
|
+
return super().pages
|
|
1988
|
+
|
|
1989
|
+
@_disallow_when_arrow
|
|
1990
|
+
def __iter__(self):
|
|
1991
|
+
return super().__iter__()
|
|
1992
|
+
|
|
1993
|
+
@_disallow_when_arrow
|
|
1994
|
+
def __next__(self):
|
|
1995
|
+
return super().__next__()
|
|
1948
1996
|
|
|
1949
1997
|
@property
|
|
1950
1998
|
def _billing_project(self) -> Optional[str]:
|
|
@@ -2226,6 +2274,12 @@ class RowIterator(HTTPIterator):
|
|
|
2226
2274
|
|
|
2227
2275
|
.. versionadded:: 2.31.0
|
|
2228
2276
|
"""
|
|
2277
|
+
if self._query_results_format == QueryResultsFormat.ARROW.value:
|
|
2278
|
+
return self._download_arrow_from_job_id(
|
|
2279
|
+
bqstorage_client=bqstorage_client,
|
|
2280
|
+
timeout=timeout,
|
|
2281
|
+
)
|
|
2282
|
+
|
|
2229
2283
|
self._maybe_warn_max_results(bqstorage_client)
|
|
2230
2284
|
|
|
2231
2285
|
bqstorage_download = functools.partial(
|
|
@@ -2251,6 +2305,106 @@ class RowIterator(HTTPIterator):
|
|
|
2251
2305
|
bqstorage_client=bqstorage_client,
|
|
2252
2306
|
)
|
|
2253
2307
|
|
|
2308
|
+
def _download_arrow_from_job_id(
|
|
2309
|
+
self,
|
|
2310
|
+
bqstorage_client: Optional["bigquery_storage.BigQueryReadClient"] = None,
|
|
2311
|
+
timeout: Optional[float] = None,
|
|
2312
|
+
) -> Iterator["pyarrow.RecordBatch"]:
|
|
2313
|
+
"""Yield Arrow record batches for query results formatted as ARROW.
|
|
2314
|
+
|
|
2315
|
+
First processes inline Arrow data in the initial page response (if present),
|
|
2316
|
+
updating row offset and total row counts. If more rows remain, streams
|
|
2317
|
+
remaining batches using the BigQuery Read API default job stream
|
|
2318
|
+
(``projects/.../locations/.../jobs/.../streams/_default``).
|
|
2319
|
+
|
|
2320
|
+
Args:
|
|
2321
|
+
bqstorage_client (Optional[bigquery_storage.BigQueryReadClient]):
|
|
2322
|
+
Client for BigQuery Storage Read API. If None, one will be created.
|
|
2323
|
+
timeout (Optional[float]):
|
|
2324
|
+
Timeout in seconds for read operations.
|
|
2325
|
+
|
|
2326
|
+
Yields:
|
|
2327
|
+
pyarrow.RecordBatch: Record batches generated from the query result.
|
|
2328
|
+
"""
|
|
2329
|
+
if pyarrow is None:
|
|
2330
|
+
raise ValueError(_NO_PYARROW_ERROR)
|
|
2331
|
+
|
|
2332
|
+
# Step 1: Ensure BigQuery Read API client is available upfront.
|
|
2333
|
+
if bqstorage_client is None:
|
|
2334
|
+
if self.client is None:
|
|
2335
|
+
raise ValueError("RowIterator client is None.")
|
|
2336
|
+
bqstorage_client = self.client._ensure_bqstorage_client()
|
|
2337
|
+
if bqstorage_client is None:
|
|
2338
|
+
raise ValueError(
|
|
2339
|
+
"The google-cloud-bigquery-storage library is required to read Arrow results."
|
|
2340
|
+
)
|
|
2341
|
+
|
|
2342
|
+
offset = 0
|
|
2343
|
+
pa_schema = None
|
|
2344
|
+
total_rows = self.total_rows
|
|
2345
|
+
job_complete = False
|
|
2346
|
+
|
|
2347
|
+
# Step 2: Process initial inline Arrow response from jobs.query if available.
|
|
2348
|
+
if self._first_page_response:
|
|
2349
|
+
first_page = self._first_page_response
|
|
2350
|
+
self._first_page_response = None
|
|
2351
|
+
|
|
2352
|
+
job_complete = bool(first_page.get("jobComplete", False))
|
|
2353
|
+
if job_complete:
|
|
2354
|
+
total_rows = int(first_page.get("totalRows", 0))
|
|
2355
|
+
|
|
2356
|
+
arrow_schema_json = first_page.get("arrowSchema")
|
|
2357
|
+
if isinstance(arrow_schema_json, dict):
|
|
2358
|
+
schema_bytes = arrow_schema_json.get("serializedSchema")
|
|
2359
|
+
if schema_bytes:
|
|
2360
|
+
if isinstance(schema_bytes, str):
|
|
2361
|
+
schema_bytes = base64.b64decode(schema_bytes)
|
|
2362
|
+
pa_schema = pyarrow.ipc.read_schema(pyarrow.py_buffer(schema_bytes))
|
|
2363
|
+
|
|
2364
|
+
arrow_batch_json = first_page.get("arrowRecordBatch")
|
|
2365
|
+
if isinstance(arrow_batch_json, dict) and pa_schema is not None:
|
|
2366
|
+
batch_bytes = arrow_batch_json.get("serializedRecordBatch")
|
|
2367
|
+
if batch_bytes:
|
|
2368
|
+
if isinstance(batch_bytes, str):
|
|
2369
|
+
batch_bytes = base64.b64decode(batch_bytes)
|
|
2370
|
+
batch = pyarrow.ipc.read_record_batch(
|
|
2371
|
+
pyarrow.py_buffer(batch_bytes),
|
|
2372
|
+
pa_schema,
|
|
2373
|
+
)
|
|
2374
|
+
offset += batch.num_rows
|
|
2375
|
+
yield batch
|
|
2376
|
+
|
|
2377
|
+
# Step 3: Return early if all results were delivered in the first page response.
|
|
2378
|
+
if job_complete and offset >= total_rows:
|
|
2379
|
+
return
|
|
2380
|
+
|
|
2381
|
+
# Step 4: Stream remaining Arrow record batches from the job default stream.
|
|
2382
|
+
project = self._project or (self.client.project if self.client else None)
|
|
2383
|
+
location = self._location or (self.client.location if self.client else None)
|
|
2384
|
+
stream_name = f"projects/{project}/locations/{location}/jobs/{self._job_id}/streams/_default"
|
|
2385
|
+
reader = bqstorage_client.read_rows(stream_name, offset=offset, timeout=timeout)
|
|
2386
|
+
for response in reader:
|
|
2387
|
+
if (
|
|
2388
|
+
response.arrow_schema
|
|
2389
|
+
and response.arrow_schema.serialized_schema
|
|
2390
|
+
and pa_schema is None
|
|
2391
|
+
):
|
|
2392
|
+
pa_schema = pyarrow.ipc.read_schema(
|
|
2393
|
+
pyarrow.py_buffer(response.arrow_schema.serialized_schema)
|
|
2394
|
+
)
|
|
2395
|
+
if (
|
|
2396
|
+
response.arrow_record_batch
|
|
2397
|
+
and response.arrow_record_batch.serialized_record_batch
|
|
2398
|
+
and pa_schema is not None
|
|
2399
|
+
):
|
|
2400
|
+
batch = pyarrow.ipc.read_record_batch(
|
|
2401
|
+
pyarrow.py_buffer(
|
|
2402
|
+
response.arrow_record_batch.serialized_record_batch
|
|
2403
|
+
),
|
|
2404
|
+
pa_schema,
|
|
2405
|
+
)
|
|
2406
|
+
yield batch
|
|
2407
|
+
|
|
2254
2408
|
# If changing the signature of this method, make sure to apply the same
|
|
2255
2409
|
# changes to job.QueryJob.to_arrow()
|
|
2256
2410
|
def to_arrow(
|
|
@@ -2316,6 +2470,12 @@ class RowIterator(HTTPIterator):
|
|
|
2316
2470
|
|
|
2317
2471
|
.. versionadded:: 1.17.0
|
|
2318
2472
|
"""
|
|
2473
|
+
warnings.warn(
|
|
2474
|
+
_TO_ARROW_DEPRECATED,
|
|
2475
|
+
PendingDeprecationWarning,
|
|
2476
|
+
stacklevel=2,
|
|
2477
|
+
)
|
|
2478
|
+
|
|
2319
2479
|
if pyarrow is None:
|
|
2320
2480
|
raise ValueError(_NO_PYARROW_ERROR)
|
|
2321
2481
|
|
|
@@ -2357,7 +2517,10 @@ class RowIterator(HTTPIterator):
|
|
|
2357
2517
|
# but mypy cannot infer this correlation. We ignore the union-attr error here.
|
|
2358
2518
|
bqstorage_client._transport.close() # type: ignore[union-attr]
|
|
2359
2519
|
|
|
2360
|
-
if record_batches and
|
|
2520
|
+
if record_batches and (
|
|
2521
|
+
bqstorage_client is not None
|
|
2522
|
+
or self._query_results_format == QueryResultsFormat.ARROW.value
|
|
2523
|
+
):
|
|
2361
2524
|
return pyarrow.Table.from_batches(record_batches)
|
|
2362
2525
|
else:
|
|
2363
2526
|
# No records (not record_batches), use schema based on BigQuery schema
|
|
@@ -2438,6 +2601,24 @@ class RowIterator(HTTPIterator):
|
|
|
2438
2601
|
if dtypes is None:
|
|
2439
2602
|
dtypes = {}
|
|
2440
2603
|
|
|
2604
|
+
if self._query_results_format == QueryResultsFormat.ARROW.value:
|
|
2605
|
+
|
|
2606
|
+
def _batch_to_dataframe(batch):
|
|
2607
|
+
df = batch.to_pandas()
|
|
2608
|
+
if dtypes:
|
|
2609
|
+
for col, dtype in dtypes.items():
|
|
2610
|
+
if col in df.columns:
|
|
2611
|
+
df[col] = df[col].astype(dtype)
|
|
2612
|
+
return df
|
|
2613
|
+
|
|
2614
|
+
return (
|
|
2615
|
+
_batch_to_dataframe(batch)
|
|
2616
|
+
for batch in self.to_arrow_iterable(
|
|
2617
|
+
bqstorage_client=bqstorage_client,
|
|
2618
|
+
timeout=timeout,
|
|
2619
|
+
)
|
|
2620
|
+
)
|
|
2621
|
+
|
|
2441
2622
|
self._maybe_warn_max_results(bqstorage_client)
|
|
2442
2623
|
|
|
2443
2624
|
column_names = [field.name for field in self._schema]
|
|
@@ -2709,6 +2890,12 @@ class RowIterator(HTTPIterator):
|
|
|
2709
2890
|
is not supported dtype.
|
|
2710
2891
|
|
|
2711
2892
|
"""
|
|
2893
|
+
warnings.warn(
|
|
2894
|
+
_TO_DATAFRAME_DEPRECATED,
|
|
2895
|
+
PendingDeprecationWarning,
|
|
2896
|
+
stacklevel=2,
|
|
2897
|
+
)
|
|
2898
|
+
|
|
2712
2899
|
_pandas_helpers.verify_pandas_imports()
|
|
2713
2900
|
|
|
2714
2901
|
if geography_as_object and shapely is None:
|
|
@@ -2797,16 +2984,42 @@ class RowIterator(HTTPIterator):
|
|
|
2797
2984
|
|
|
2798
2985
|
self._maybe_warn_max_results(bqstorage_client)
|
|
2799
2986
|
|
|
2800
|
-
if
|
|
2987
|
+
if (
|
|
2988
|
+
self._query_results_format != QueryResultsFormat.ARROW.value
|
|
2989
|
+
and not self._should_use_bqstorage(
|
|
2990
|
+
bqstorage_client, create_bqstorage_client
|
|
2991
|
+
)
|
|
2992
|
+
):
|
|
2801
2993
|
create_bqstorage_client = False
|
|
2802
2994
|
bqstorage_client = None
|
|
2803
2995
|
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2996
|
+
if _versions_helpers.PANDAS_GBQ_VERSIONS.is_delegation_supported:
|
|
2997
|
+
try:
|
|
2998
|
+
client_info = getattr(
|
|
2999
|
+
getattr(self.client, "_connection", None), "_client_info", None
|
|
3000
|
+
)
|
|
3001
|
+
if client_info:
|
|
3002
|
+
ua = getattr(client_info, "user_agent", None) or ""
|
|
3003
|
+
if "pandas-gbq" not in ua:
|
|
3004
|
+
version = (
|
|
3005
|
+
_versions_helpers.PANDAS_GBQ_VERSIONS.installed_version
|
|
3006
|
+
)
|
|
3007
|
+
client_info.user_agent = f"{ua} pandas-gbq/{version}".strip()
|
|
3008
|
+
except Exception as exc:
|
|
3009
|
+
_LOGGER.debug("Failed to update telemetry user-agent: %s", exc)
|
|
3010
|
+
|
|
3011
|
+
with warnings.catch_warnings():
|
|
3012
|
+
warnings.filterwarnings(
|
|
3013
|
+
"ignore",
|
|
3014
|
+
category=PendingDeprecationWarning,
|
|
3015
|
+
message="Retrieving PyArrow Tables.*",
|
|
3016
|
+
)
|
|
3017
|
+
record_batch = self.to_arrow(
|
|
3018
|
+
progress_bar_type=progress_bar_type,
|
|
3019
|
+
bqstorage_client=bqstorage_client,
|
|
3020
|
+
create_bqstorage_client=create_bqstorage_client,
|
|
3021
|
+
timeout=timeout,
|
|
3022
|
+
)
|
|
2810
3023
|
|
|
2811
3024
|
# Default date dtype is `db_dtypes.DateDtype()` that could cause out of bounds error,
|
|
2812
3025
|
# when pyarrow converts date values to nanosecond precision. To avoid the error, we
|
|
@@ -3009,18 +3222,24 @@ class RowIterator(HTTPIterator):
|
|
|
3009
3222
|
"one to use to create a GeoDataFrame"
|
|
3010
3223
|
)
|
|
3011
3224
|
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3225
|
+
with warnings.catch_warnings():
|
|
3226
|
+
warnings.filterwarnings(
|
|
3227
|
+
"ignore",
|
|
3228
|
+
category=PendingDeprecationWarning,
|
|
3229
|
+
message="Retrieving DataFrames via google-cloud-bigquery is deprecated.*",
|
|
3230
|
+
)
|
|
3231
|
+
df = self.to_dataframe(
|
|
3232
|
+
bqstorage_client,
|
|
3233
|
+
dtypes,
|
|
3234
|
+
progress_bar_type,
|
|
3235
|
+
create_bqstorage_client,
|
|
3236
|
+
geography_as_object=True,
|
|
3237
|
+
bool_dtype=bool_dtype,
|
|
3238
|
+
int_dtype=int_dtype,
|
|
3239
|
+
float_dtype=float_dtype,
|
|
3240
|
+
string_dtype=string_dtype,
|
|
3241
|
+
timeout=timeout,
|
|
3242
|
+
)
|
|
3024
3243
|
|
|
3025
3244
|
return geopandas.GeoDataFrame(
|
|
3026
3245
|
df, crs=_COORDINATE_REFERENCE_SYSTEM, geometry=geography_column
|
|
@@ -3048,6 +3267,14 @@ class _EmptyRowIterator(RowIterator):
|
|
|
3048
3267
|
)
|
|
3049
3268
|
self._total_rows = 0
|
|
3050
3269
|
|
|
3270
|
+
@_disallow_when_arrow
|
|
3271
|
+
def __iter__(self):
|
|
3272
|
+
return iter(())
|
|
3273
|
+
|
|
3274
|
+
@_disallow_when_arrow
|
|
3275
|
+
def __next__(self):
|
|
3276
|
+
raise StopIteration
|
|
3277
|
+
|
|
3051
3278
|
def to_arrow(
|
|
3052
3279
|
self,
|
|
3053
3280
|
progress_bar_type=None,
|
|
@@ -3068,6 +3295,11 @@ class _EmptyRowIterator(RowIterator):
|
|
|
3068
3295
|
"""
|
|
3069
3296
|
if pyarrow is None:
|
|
3070
3297
|
raise ValueError(_NO_PYARROW_ERROR)
|
|
3298
|
+
warnings.warn(
|
|
3299
|
+
_TO_ARROW_DEPRECATED,
|
|
3300
|
+
PendingDeprecationWarning,
|
|
3301
|
+
stacklevel=2,
|
|
3302
|
+
)
|
|
3071
3303
|
return pyarrow.Table.from_arrays(())
|
|
3072
3304
|
|
|
3073
3305
|
def to_dataframe(
|
|
@@ -3114,6 +3346,11 @@ class _EmptyRowIterator(RowIterator):
|
|
|
3114
3346
|
Returns:
|
|
3115
3347
|
pandas.DataFrame: An empty :class:`~pandas.DataFrame`.
|
|
3116
3348
|
"""
|
|
3349
|
+
warnings.warn(
|
|
3350
|
+
_TO_DATAFRAME_DEPRECATED,
|
|
3351
|
+
PendingDeprecationWarning,
|
|
3352
|
+
stacklevel=2,
|
|
3353
|
+
)
|
|
3117
3354
|
_pandas_helpers.verify_pandas_imports()
|
|
3118
3355
|
return pandas.DataFrame()
|
|
3119
3356
|
|
|
@@ -3219,11 +3456,10 @@ class _EmptyRowIterator(RowIterator):
|
|
|
3219
3456
|
Returns:
|
|
3220
3457
|
An iterator yielding a single empty :class:`~pyarrow.RecordBatch`.
|
|
3221
3458
|
"""
|
|
3459
|
+
if pyarrow is None:
|
|
3460
|
+
raise ValueError(_NO_PYARROW_ERROR)
|
|
3222
3461
|
return iter((pyarrow.record_batch([]),))
|
|
3223
3462
|
|
|
3224
|
-
def __iter__(self):
|
|
3225
|
-
return iter(())
|
|
3226
|
-
|
|
3227
3463
|
|
|
3228
3464
|
class PartitionRange(object):
|
|
3229
3465
|
"""Definition of the ranges for range partitioning.
|