google-cloud-bigquery 3.43.0__tar.gz → 3.44.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {google_cloud_bigquery-3.43.0/google_cloud_bigquery.egg-info → google_cloud_bigquery-3.44.0}/PKG-INFO +1 -1
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/__init__.py +9 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/_job_helpers.py +19 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/client.py +34 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/enums.py +17 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/table.py +257 -38
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/version.py +1 -1
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0/google_cloud_bigquery.egg-info}/PKG-INFO +1 -1
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google_cloud_bigquery.egg-info/SOURCES.txt +1 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/test_arrow.py +51 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test__job_helpers.py +1 -1
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_client.py +40 -1
- google_cloud_bigquery-3.44.0/tests/unit/test_query_results_format_arrow.py +573 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_table.py +122 -50
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/LICENSE +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/MANIFEST.in +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/README.rst +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/_http.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/_pandas_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/_pyarrow_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/_string_references.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/_tqdm_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/_versions_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/dataset.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/dbapi/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/dbapi/_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/dbapi/connection.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/dbapi/cursor.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/dbapi/exceptions.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/dbapi/types.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/encryption_configuration.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/exceptions.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/external_config.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/format_options.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/iam.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/job/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/job/base.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/job/copy_.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/job/extract.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/job/load.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/job/query.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/magics/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/magics/line_arg_parser/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/magics/line_arg_parser/exceptions.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/magics/line_arg_parser/lexer.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/magics/line_arg_parser/parser.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/magics/line_arg_parser/visitors.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/magics/magics.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/model.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/opentelemetry_tracing.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/py.typed +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/query.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/routine/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/routine/routine.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/schema.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/standard_sql.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery_v2/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery_v2/types/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery_v2/types/encryption_config.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery_v2/types/model.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery_v2/types/model_reference.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery_v2/types/standard_sql.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery_v2/types/table_reference.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google_cloud_bigquery.egg-info/dependency_links.txt +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google_cloud_bigquery.egg-info/requires.txt +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google_cloud_bigquery.egg-info/top_level.txt +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/pyproject.toml +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/setup.cfg +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/setup.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/characters.json +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/characters.jsonl +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/colors.avro +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/numeric_38_12.parquet +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/people.csv +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/pico.csv +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/pico_schema.json +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/scalars.csv +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/scalars.jsonl +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/scalars_extreme.jsonl +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/scalars_schema.json +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/scalars_schema_csv.json +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/data/schema.json +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/scrub_datasets.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/conftest.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/test_client.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/test_job_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/test_list_rows.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/test_magics.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/test_pandas.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/test_query.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/test_ssl_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/system/test_structs.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/_helpers/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/_helpers/test_cell_data_parser.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/_helpers/test_data_frame_cell_data_parser.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/_helpers/test_scalar_query_param_parser.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/conftest.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/test_async_job_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/test_base.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/test_copy.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/test_extract.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/test_load.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/test_load_config.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/test_query.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/test_query_config.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/test_query_job_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/test_query_pandas.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/job/test_query_stats.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/line_arg_parser/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/line_arg_parser/test_lexer.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/line_arg_parser/test_parser.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/line_arg_parser/test_visitors.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/model/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/model/test_model.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/model/test_model_reference.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/routine/__init__.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/routine/test_external_runtime_options.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/routine/test_remote_function_options.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/routine/test_routine.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/routine/test_routine_argument.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/routine/test_routine_reference.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test__helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test__http.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test__job_helpers_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test__pandas_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test__pyarrow_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test__versions_helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_client_bigframes.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_client_resumable_media_upload.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_client_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_create_dataset.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_dataset.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_dbapi__helpers.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_dbapi_connection.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_dbapi_cursor.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_dbapi_types.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_delete_dataset.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_encryption_configuration.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_external_config.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_format_options.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_job_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_legacy_types.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_list_datasets.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_list_jobs.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_list_models.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_list_projects.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_list_routines.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_list_tables.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_magics.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_opentelemetry_tracing.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_packaging.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_query.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_retry.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_schema.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_signature_compatibility.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_standard_sql_types.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_string_references.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_table_arrow.py +0 -0
- {google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/tests/unit/test_table_pandas.py +0 -0
{google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/__init__.py
RENAMED
|
@@ -30,6 +30,8 @@ The main concepts with this API are:
|
|
|
30
30
|
import sys
|
|
31
31
|
import warnings
|
|
32
32
|
|
|
33
|
+
import google.api_core as api_core
|
|
34
|
+
|
|
33
35
|
from google.cloud.bigquery import version as bigquery_version
|
|
34
36
|
|
|
35
37
|
__version__ = bigquery_version.__version__
|
|
@@ -57,6 +59,8 @@ from google.cloud.bigquery.external_config import ExternalSourceFormat
|
|
|
57
59
|
from google.cloud.bigquery.external_config import HivePartitioningOptions
|
|
58
60
|
from google.cloud.bigquery.format_options import AvroOptions
|
|
59
61
|
from google.cloud.bigquery.format_options import ParquetOptions
|
|
62
|
+
from google.cloud.bigquery.enums import QueryResultsCompressionCodec
|
|
63
|
+
from google.cloud.bigquery.enums import QueryResultsFormat
|
|
60
64
|
from google.cloud.bigquery.job.base import SessionInfo
|
|
61
65
|
from google.cloud.bigquery.job import Compression
|
|
62
66
|
from google.cloud.bigquery.job import CopyJob
|
|
@@ -132,6 +136,9 @@ if sys.version_info < (3, 10): # pragma: NO COVER
|
|
|
132
136
|
FutureWarning,
|
|
133
137
|
)
|
|
134
138
|
|
|
139
|
+
api_core.check_python_version(__name__)
|
|
140
|
+
api_core.check_dependency_versions(__name__)
|
|
141
|
+
|
|
135
142
|
__all__ = [
|
|
136
143
|
"__version__",
|
|
137
144
|
"Client",
|
|
@@ -216,6 +223,8 @@ __all__ = [
|
|
|
216
223
|
"KeyResultStatementKind",
|
|
217
224
|
"OperationType",
|
|
218
225
|
"QueryPriority",
|
|
226
|
+
"QueryResultsCompressionCodec",
|
|
227
|
+
"QueryResultsFormat",
|
|
219
228
|
"RoutineType",
|
|
220
229
|
"SchemaUpdateOption",
|
|
221
230
|
"SourceFormat",
|
{google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/_job_helpers.py
RENAMED
|
@@ -430,6 +430,8 @@ def query_and_wait(
|
|
|
430
430
|
job_retry: Optional[retries.Retry],
|
|
431
431
|
page_size: Optional[int] = None,
|
|
432
432
|
max_results: Optional[int] = None,
|
|
433
|
+
query_results_format: Optional[str] = None,
|
|
434
|
+
compression_codec: Optional[str] = None,
|
|
433
435
|
callback: Callable = lambda _: None,
|
|
434
436
|
) -> table.RowIterator:
|
|
435
437
|
"""Run the query, wait for it to finish, and return the results.
|
|
@@ -475,6 +477,10 @@ def query_and_wait(
|
|
|
475
477
|
request. Non-positive values are ignored.
|
|
476
478
|
max_results (Optional[int]):
|
|
477
479
|
The maximum total number of rows from this request.
|
|
480
|
+
query_results_format (Optional[Union[str, google.cloud.bigquery.enums.QueryResultsFormat]]):
|
|
481
|
+
[Beta] The format for query results (e.g. "ARROW" or :class:`~google.cloud.bigquery.enums.QueryResultsFormat.ARROW`).
|
|
482
|
+
compression_codec (Optional[Union[str, google.cloud.bigquery.enums.QueryResultsCompressionCodec]]):
|
|
483
|
+
[Beta] Compression codec for Arrow serialization (e.g. "LZ4_FRAME" or :class:`~google.cloud.bigquery.enums.QueryResultsCompressionCodec.LZ4_FRAME`).
|
|
478
484
|
callback (Callable):
|
|
479
485
|
A callback function used by bigframes to report query progress.
|
|
480
486
|
|
|
@@ -499,6 +505,13 @@ def query_and_wait(
|
|
|
499
505
|
request_body = _to_query_request(
|
|
500
506
|
query=query, job_config=job_config, location=location, timeout=api_timeout
|
|
501
507
|
)
|
|
508
|
+
if query_results_format is not None:
|
|
509
|
+
request_body["queryResultsFormat"] = query_results_format
|
|
510
|
+
if compression_codec is not None:
|
|
511
|
+
request_body.setdefault("formatOptions", {})
|
|
512
|
+
request_body["formatOptions"]["arrowSerializationOptions"] = {
|
|
513
|
+
"bufferCompression": compression_codec
|
|
514
|
+
}
|
|
502
515
|
|
|
503
516
|
# Some API parameters aren't supported by the jobs.query API. In these
|
|
504
517
|
# cases, fallback to a jobs.insert call.
|
|
@@ -522,6 +535,7 @@ def query_and_wait(
|
|
|
522
535
|
retry=retry,
|
|
523
536
|
page_size=page_size,
|
|
524
537
|
max_results=max_results,
|
|
538
|
+
query_results_format=query_results_format,
|
|
525
539
|
callback=callback,
|
|
526
540
|
)
|
|
527
541
|
|
|
@@ -594,6 +608,7 @@ def query_and_wait(
|
|
|
594
608
|
retry=retry,
|
|
595
609
|
page_size=page_size,
|
|
596
610
|
max_results=max_results,
|
|
611
|
+
query_results_format=query_results_format,
|
|
597
612
|
callback=callback,
|
|
598
613
|
)
|
|
599
614
|
|
|
@@ -633,6 +648,7 @@ def query_and_wait(
|
|
|
633
648
|
created=query_results.created,
|
|
634
649
|
started=query_results.started,
|
|
635
650
|
ended=query_results.ended,
|
|
651
|
+
query_results_format=query_results_format,
|
|
636
652
|
)
|
|
637
653
|
|
|
638
654
|
if job_retry is not None:
|
|
@@ -673,6 +689,7 @@ def _supported_by_jobs_query(request_body: Dict[str, Any]) -> bool:
|
|
|
673
689
|
"jobTimeoutMs",
|
|
674
690
|
"reservation",
|
|
675
691
|
"maxSlots",
|
|
692
|
+
"queryResultsFormat",
|
|
676
693
|
}
|
|
677
694
|
|
|
678
695
|
unsupported_keys = request_keys - keys_allowlist
|
|
@@ -687,6 +704,7 @@ def _wait_or_cancel(
|
|
|
687
704
|
page_size: Optional[int],
|
|
688
705
|
max_results: Optional[int],
|
|
689
706
|
*,
|
|
707
|
+
query_results_format: Optional[str] = None,
|
|
690
708
|
callback: Callable = lambda _: None,
|
|
691
709
|
) -> table.RowIterator:
|
|
692
710
|
"""Wait for a job to complete and return the results.
|
|
@@ -731,6 +749,7 @@ def _wait_or_cancel(
|
|
|
731
749
|
ended=job.ended,
|
|
732
750
|
)
|
|
733
751
|
)
|
|
752
|
+
query_results._query_results_format = query_results_format
|
|
734
753
|
return query_results
|
|
735
754
|
except Exception:
|
|
736
755
|
# Attempt to cancel the job since we can't return the results.
|
{google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/client.py
RENAMED
|
@@ -163,6 +163,16 @@ _LIST_ROWS_FROM_QUERY_RESULTS_FIELDS = "jobReference,totalRows,pageToken,rows"
|
|
|
163
163
|
# https://github.com/googleapis/python-bigquery/issues/438
|
|
164
164
|
_MIN_GET_QUERY_RESULTS_TIMEOUT = 120
|
|
165
165
|
|
|
166
|
+
_LOAD_TABLE_FROM_DATAFRAME_DEPRECATED = (
|
|
167
|
+
"Loading DataFrames via google-cloud-bigquery is deprecated. "
|
|
168
|
+
"For direct, optimized loading, please call 'pandas_gbq.to_gbq()' directly."
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
_INSERT_ROWS_FROM_DATAFRAME_DEPRECATED = (
|
|
172
|
+
"Inserting rows from DataFrames via google-cloud-bigquery is deprecated. "
|
|
173
|
+
"For direct, optimized access, please call 'pandas_gbq.to_gbq()' directly."
|
|
174
|
+
)
|
|
175
|
+
|
|
166
176
|
TIMEOUT_HEADER = "X-Server-Timeout"
|
|
167
177
|
|
|
168
178
|
|
|
@@ -2830,6 +2840,12 @@ class Client(ClientWithProject):
|
|
|
2830
2840
|
If ``job_config`` is not an instance of
|
|
2831
2841
|
:class:`~google.cloud.bigquery.job.LoadJobConfig` class.
|
|
2832
2842
|
"""
|
|
2843
|
+
warnings.warn(
|
|
2844
|
+
_LOAD_TABLE_FROM_DATAFRAME_DEPRECATED,
|
|
2845
|
+
PendingDeprecationWarning,
|
|
2846
|
+
stacklevel=2,
|
|
2847
|
+
)
|
|
2848
|
+
|
|
2833
2849
|
job_id = _make_job_id(job_id, job_id_prefix)
|
|
2834
2850
|
|
|
2835
2851
|
if job_config is not None:
|
|
@@ -3649,6 +3665,8 @@ class Client(ClientWithProject):
|
|
|
3649
3665
|
job_retry: retries.Retry = DEFAULT_JOB_RETRY,
|
|
3650
3666
|
page_size: Optional[int] = None,
|
|
3651
3667
|
max_results: Optional[int] = None,
|
|
3668
|
+
query_results_format: Optional[str] = None,
|
|
3669
|
+
compression_codec: Optional[str] = None,
|
|
3652
3670
|
) -> RowIterator:
|
|
3653
3671
|
"""Run the query, wait for it to finish, and return the results.
|
|
3654
3672
|
|
|
@@ -3696,6 +3714,10 @@ class Client(ClientWithProject):
|
|
|
3696
3714
|
by this parameter.
|
|
3697
3715
|
max_results (Optional[int]):
|
|
3698
3716
|
The maximum total number of rows from this request.
|
|
3717
|
+
query_results_format (Optional[Union[str, google.cloud.bigquery.enums.QueryResultsFormat]]):
|
|
3718
|
+
[Beta] The format for query results (e.g. "ARROW" or :class:`~google.cloud.bigquery.enums.QueryResultsFormat.ARROW`).
|
|
3719
|
+
compression_codec (Optional[Union[str, google.cloud.bigquery.enums.QueryResultsCompressionCodec]]):
|
|
3720
|
+
[Beta] Compression codec for Arrow serialization (e.g. "LZ4_FRAME" or :class:`~google.cloud.bigquery.enums.QueryResultsCompressionCodec.LZ4_FRAME`).
|
|
3699
3721
|
|
|
3700
3722
|
Returns:
|
|
3701
3723
|
google.cloud.bigquery.table.RowIterator:
|
|
@@ -3726,6 +3748,8 @@ class Client(ClientWithProject):
|
|
|
3726
3748
|
job_retry=job_retry,
|
|
3727
3749
|
page_size=page_size,
|
|
3728
3750
|
max_results=max_results,
|
|
3751
|
+
query_results_format=query_results_format,
|
|
3752
|
+
compression_codec=compression_codec,
|
|
3729
3753
|
)
|
|
3730
3754
|
|
|
3731
3755
|
def _query_and_wait_bigframes(
|
|
@@ -3741,6 +3765,8 @@ class Client(ClientWithProject):
|
|
|
3741
3765
|
job_retry: retries.Retry = DEFAULT_JOB_RETRY,
|
|
3742
3766
|
page_size: Optional[int] = None,
|
|
3743
3767
|
max_results: Optional[int] = None,
|
|
3768
|
+
query_results_format: Optional[str] = None,
|
|
3769
|
+
compression_codec: Optional[str] = None,
|
|
3744
3770
|
callback: Callable = lambda _: None,
|
|
3745
3771
|
) -> RowIterator:
|
|
3746
3772
|
"""See query_and_wait.
|
|
@@ -3773,6 +3799,8 @@ class Client(ClientWithProject):
|
|
|
3773
3799
|
job_retry=job_retry,
|
|
3774
3800
|
page_size=page_size,
|
|
3775
3801
|
max_results=max_results,
|
|
3802
|
+
query_results_format=query_results_format,
|
|
3803
|
+
compression_codec=compression_codec,
|
|
3776
3804
|
callback=callback,
|
|
3777
3805
|
)
|
|
3778
3806
|
|
|
@@ -3900,6 +3928,12 @@ class Client(ClientWithProject):
|
|
|
3900
3928
|
Raises:
|
|
3901
3929
|
ValueError: if table's schema is not set
|
|
3902
3930
|
"""
|
|
3931
|
+
warnings.warn(
|
|
3932
|
+
_INSERT_ROWS_FROM_DATAFRAME_DEPRECATED,
|
|
3933
|
+
PendingDeprecationWarning,
|
|
3934
|
+
stacklevel=2,
|
|
3935
|
+
)
|
|
3936
|
+
|
|
3903
3937
|
insert_results = []
|
|
3904
3938
|
|
|
3905
3939
|
chunk_count = int(math.ceil(len(dataframe) / chunk_size))
|
{google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/enums.py
RENAMED
|
@@ -495,3 +495,20 @@ class TimestampPrecision(enum.Enum):
|
|
|
495
495
|
"""
|
|
496
496
|
For TIMESTAMP type with picosecond precision.
|
|
497
497
|
"""
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
class QueryResultsFormat(str, enum.Enum):
|
|
501
|
+
"""[Beta] Format for query results response."""
|
|
502
|
+
|
|
503
|
+
ARROW = "ARROW"
|
|
504
|
+
"""Specifies Apache Arrow format for query results."""
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
class QueryResultsCompressionCodec(str, enum.Enum):
|
|
508
|
+
"""[Beta] Compression codec for Arrow query results serialization."""
|
|
509
|
+
|
|
510
|
+
LZ4_FRAME = "LZ4_FRAME"
|
|
511
|
+
"""Specifies LZ4_FRAME compression codec."""
|
|
512
|
+
|
|
513
|
+
ZSTD = "ZSTD"
|
|
514
|
+
"""Specifies ZSTD compression codec."""
|
{google_cloud_bigquery-3.43.0 → google_cloud_bigquery-3.44.0}/google/cloud/bigquery/table.py
RENAMED
|
@@ -16,14 +16,14 @@
|
|
|
16
16
|
|
|
17
17
|
from __future__ import absolute_import
|
|
18
18
|
|
|
19
|
+
import base64
|
|
19
20
|
import copy
|
|
20
21
|
import datetime
|
|
21
22
|
import functools
|
|
22
23
|
import operator
|
|
23
24
|
import typing
|
|
24
|
-
from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union, Sequence
|
|
25
|
-
|
|
26
25
|
import warnings
|
|
26
|
+
from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple, Union
|
|
27
27
|
|
|
28
28
|
try:
|
|
29
29
|
import pandas # type: ignore
|
|
@@ -56,30 +56,33 @@ else:
|
|
|
56
56
|
_read_wkt = wkt.loads
|
|
57
57
|
|
|
58
58
|
import google.api_core.exceptions
|
|
59
|
-
from google.api_core.page_iterator import HTTPIterator
|
|
60
|
-
|
|
61
59
|
import google.cloud._helpers # type: ignore
|
|
62
|
-
from google.
|
|
63
|
-
from google.cloud.bigquery import
|
|
64
|
-
|
|
60
|
+
from google.api_core.page_iterator import HTTPIterator
|
|
61
|
+
from google.cloud.bigquery import (
|
|
62
|
+
_helpers,
|
|
63
|
+
_pandas_helpers,
|
|
64
|
+
_string_references,
|
|
65
|
+
_versions_helpers,
|
|
66
|
+
external_config,
|
|
67
|
+
)
|
|
65
68
|
from google.cloud.bigquery import exceptions as bq_exceptions
|
|
69
|
+
from google.cloud.bigquery import schema as _schema
|
|
66
70
|
from google.cloud.bigquery._tqdm_helpers import get_progress_bar
|
|
67
71
|
from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
|
|
68
|
-
from google.cloud.bigquery.enums import DefaultPandasDTypes
|
|
72
|
+
from google.cloud.bigquery.enums import DefaultPandasDTypes, QueryResultsFormat
|
|
69
73
|
from google.cloud.bigquery.external_config import ExternalConfig
|
|
70
|
-
from google.cloud.bigquery import
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
from google.cloud.bigquery import _string_references
|
|
74
|
+
from google.cloud.bigquery.schema import (
|
|
75
|
+
_build_schema_resource,
|
|
76
|
+
_parse_schema_resource,
|
|
77
|
+
_to_schema_fields,
|
|
78
|
+
)
|
|
76
79
|
|
|
77
80
|
if typing.TYPE_CHECKING: # pragma: NO COVER
|
|
78
81
|
# Unconditionally import optional dependencies again to tell pytype that
|
|
79
82
|
# they are not None, avoiding false "no attribute" errors.
|
|
83
|
+
import geopandas # type: ignore
|
|
80
84
|
import pandas
|
|
81
85
|
import pyarrow
|
|
82
|
-
import geopandas # type: ignore
|
|
83
86
|
from google.cloud import bigquery_storage # type: ignore
|
|
84
87
|
from google.cloud.bigquery.dataset import DatasetReference
|
|
85
88
|
|
|
@@ -110,6 +113,22 @@ _RANGE_PYARROW_WARNING = (
|
|
|
110
113
|
"pyarrow >= 10.0.1."
|
|
111
114
|
)
|
|
112
115
|
|
|
116
|
+
_TO_DATAFRAME_DEPRECATED = (
|
|
117
|
+
"Retrieving DataFrames via google-cloud-bigquery is deprecated. "
|
|
118
|
+
"For direct, optimized access, please call 'pandas_gbq.read_gbq()' directly."
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
_TO_ARROW_DEPRECATED = (
|
|
122
|
+
"Retrieving PyArrow Tables via google-cloud-bigquery is deprecated. "
|
|
123
|
+
"For direct, optimized access, please call 'pandas_gbq.arrow.read_bigquery_table()' "
|
|
124
|
+
"or 'pandas_gbq.arrow.read_bigquery_query()' directly."
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
_CANNOT_ITERATE_ARROW_ERROR = (
|
|
128
|
+
"Cannot iterate over non-arrow results when queryResultsFormat is ARROW. "
|
|
129
|
+
"Use to_arrow_iterable() or to_arrow() instead."
|
|
130
|
+
)
|
|
131
|
+
|
|
113
132
|
# How many of the total rows need to be downloaded already for us to skip
|
|
114
133
|
# calling the BQ Storage API?
|
|
115
134
|
#
|
|
@@ -164,6 +183,18 @@ def _view_use_legacy_sql_getter(
|
|
|
164
183
|
return None # explicit return statement to appease mypy
|
|
165
184
|
|
|
166
185
|
|
|
186
|
+
def _disallow_when_arrow(func):
|
|
187
|
+
"""Decorator that disallows iteration/access when queryResultsFormat is ARROW."""
|
|
188
|
+
|
|
189
|
+
@functools.wraps(func)
|
|
190
|
+
def wrapper(self, *args, **kwargs):
|
|
191
|
+
if self._query_results_format == QueryResultsFormat.ARROW.value:
|
|
192
|
+
raise ValueError(_CANNOT_ITERATE_ARROW_ERROR)
|
|
193
|
+
return func(self, *args, **kwargs)
|
|
194
|
+
|
|
195
|
+
return wrapper
|
|
196
|
+
|
|
197
|
+
|
|
167
198
|
class _TableBase:
|
|
168
199
|
"""Base class for Table-related classes with common functionality."""
|
|
169
200
|
|
|
@@ -1912,6 +1943,7 @@ class RowIterator(HTTPIterator):
|
|
|
1912
1943
|
created: Optional[datetime.datetime] = None,
|
|
1913
1944
|
started: Optional[datetime.datetime] = None,
|
|
1914
1945
|
ended: Optional[datetime.datetime] = None,
|
|
1946
|
+
query_results_format: Optional[str] = None,
|
|
1915
1947
|
):
|
|
1916
1948
|
super(RowIterator, self).__init__(
|
|
1917
1949
|
client,
|
|
@@ -1945,6 +1977,20 @@ class RowIterator(HTTPIterator):
|
|
|
1945
1977
|
self._job_created = created
|
|
1946
1978
|
self._job_started = started
|
|
1947
1979
|
self._job_ended = ended
|
|
1980
|
+
self._query_results_format = query_results_format
|
|
1981
|
+
|
|
1982
|
+
@property
|
|
1983
|
+
@_disallow_when_arrow
|
|
1984
|
+
def pages(self):
|
|
1985
|
+
return super().pages
|
|
1986
|
+
|
|
1987
|
+
@_disallow_when_arrow
|
|
1988
|
+
def __iter__(self):
|
|
1989
|
+
return super().__iter__()
|
|
1990
|
+
|
|
1991
|
+
@_disallow_when_arrow
|
|
1992
|
+
def __next__(self):
|
|
1993
|
+
return super().__next__()
|
|
1948
1994
|
|
|
1949
1995
|
@property
|
|
1950
1996
|
def _billing_project(self) -> Optional[str]:
|
|
@@ -2226,6 +2272,12 @@ class RowIterator(HTTPIterator):
|
|
|
2226
2272
|
|
|
2227
2273
|
.. versionadded:: 2.31.0
|
|
2228
2274
|
"""
|
|
2275
|
+
if self._query_results_format == QueryResultsFormat.ARROW.value:
|
|
2276
|
+
return self._download_arrow_from_job_id(
|
|
2277
|
+
bqstorage_client=bqstorage_client,
|
|
2278
|
+
timeout=timeout,
|
|
2279
|
+
)
|
|
2280
|
+
|
|
2229
2281
|
self._maybe_warn_max_results(bqstorage_client)
|
|
2230
2282
|
|
|
2231
2283
|
bqstorage_download = functools.partial(
|
|
@@ -2251,6 +2303,106 @@ class RowIterator(HTTPIterator):
|
|
|
2251
2303
|
bqstorage_client=bqstorage_client,
|
|
2252
2304
|
)
|
|
2253
2305
|
|
|
2306
|
+
def _download_arrow_from_job_id(
|
|
2307
|
+
self,
|
|
2308
|
+
bqstorage_client: Optional["bigquery_storage.BigQueryReadClient"] = None,
|
|
2309
|
+
timeout: Optional[float] = None,
|
|
2310
|
+
) -> Iterator["pyarrow.RecordBatch"]:
|
|
2311
|
+
"""Yield Arrow record batches for query results formatted as ARROW.
|
|
2312
|
+
|
|
2313
|
+
First processes inline Arrow data in the initial page response (if present),
|
|
2314
|
+
updating row offset and total row counts. If more rows remain, streams
|
|
2315
|
+
remaining batches using the BigQuery Read API default job stream
|
|
2316
|
+
(``projects/.../locations/.../jobs/.../streams/_default``).
|
|
2317
|
+
|
|
2318
|
+
Args:
|
|
2319
|
+
bqstorage_client (Optional[bigquery_storage.BigQueryReadClient]):
|
|
2320
|
+
Client for BigQuery Storage Read API. If None, one will be created.
|
|
2321
|
+
timeout (Optional[float]):
|
|
2322
|
+
Timeout in seconds for read operations.
|
|
2323
|
+
|
|
2324
|
+
Yields:
|
|
2325
|
+
pyarrow.RecordBatch: Record batches generated from the query result.
|
|
2326
|
+
"""
|
|
2327
|
+
if pyarrow is None:
|
|
2328
|
+
raise ValueError(_NO_PYARROW_ERROR)
|
|
2329
|
+
|
|
2330
|
+
# Step 1: Ensure BigQuery Read API client is available upfront.
|
|
2331
|
+
if bqstorage_client is None:
|
|
2332
|
+
if self.client is None:
|
|
2333
|
+
raise ValueError("RowIterator client is None.")
|
|
2334
|
+
bqstorage_client = self.client._ensure_bqstorage_client()
|
|
2335
|
+
if bqstorage_client is None:
|
|
2336
|
+
raise ValueError(
|
|
2337
|
+
"The google-cloud-bigquery-storage library is required to read Arrow results."
|
|
2338
|
+
)
|
|
2339
|
+
|
|
2340
|
+
offset = 0
|
|
2341
|
+
pa_schema = None
|
|
2342
|
+
total_rows = self.total_rows
|
|
2343
|
+
job_complete = False
|
|
2344
|
+
|
|
2345
|
+
# Step 2: Process initial inline Arrow response from jobs.query if available.
|
|
2346
|
+
if self._first_page_response:
|
|
2347
|
+
first_page = self._first_page_response
|
|
2348
|
+
self._first_page_response = None
|
|
2349
|
+
|
|
2350
|
+
job_complete = bool(first_page.get("jobComplete", False))
|
|
2351
|
+
if job_complete:
|
|
2352
|
+
total_rows = int(first_page.get("totalRows", 0))
|
|
2353
|
+
|
|
2354
|
+
arrow_schema_json = first_page.get("arrowSchema")
|
|
2355
|
+
if isinstance(arrow_schema_json, dict):
|
|
2356
|
+
schema_bytes = arrow_schema_json.get("serializedSchema")
|
|
2357
|
+
if schema_bytes:
|
|
2358
|
+
if isinstance(schema_bytes, str):
|
|
2359
|
+
schema_bytes = base64.b64decode(schema_bytes)
|
|
2360
|
+
pa_schema = pyarrow.ipc.read_schema(pyarrow.py_buffer(schema_bytes))
|
|
2361
|
+
|
|
2362
|
+
arrow_batch_json = first_page.get("arrowRecordBatch")
|
|
2363
|
+
if isinstance(arrow_batch_json, dict) and pa_schema is not None:
|
|
2364
|
+
batch_bytes = arrow_batch_json.get("serializedRecordBatch")
|
|
2365
|
+
if batch_bytes:
|
|
2366
|
+
if isinstance(batch_bytes, str):
|
|
2367
|
+
batch_bytes = base64.b64decode(batch_bytes)
|
|
2368
|
+
batch = pyarrow.ipc.read_record_batch(
|
|
2369
|
+
pyarrow.py_buffer(batch_bytes),
|
|
2370
|
+
pa_schema,
|
|
2371
|
+
)
|
|
2372
|
+
offset += batch.num_rows
|
|
2373
|
+
yield batch
|
|
2374
|
+
|
|
2375
|
+
# Step 3: Return early if all results were delivered in the first page response.
|
|
2376
|
+
if job_complete and offset >= total_rows:
|
|
2377
|
+
return
|
|
2378
|
+
|
|
2379
|
+
# Step 4: Stream remaining Arrow record batches from the job default stream.
|
|
2380
|
+
project = self._project or (self.client.project if self.client else None)
|
|
2381
|
+
location = self._location or (self.client.location if self.client else None)
|
|
2382
|
+
stream_name = f"projects/{project}/locations/{location}/jobs/{self._job_id}/streams/_default"
|
|
2383
|
+
reader = bqstorage_client.read_rows(stream_name, offset=offset, timeout=timeout)
|
|
2384
|
+
for response in reader:
|
|
2385
|
+
if (
|
|
2386
|
+
response.arrow_schema
|
|
2387
|
+
and response.arrow_schema.serialized_schema
|
|
2388
|
+
and pa_schema is None
|
|
2389
|
+
):
|
|
2390
|
+
pa_schema = pyarrow.ipc.read_schema(
|
|
2391
|
+
pyarrow.py_buffer(response.arrow_schema.serialized_schema)
|
|
2392
|
+
)
|
|
2393
|
+
if (
|
|
2394
|
+
response.arrow_record_batch
|
|
2395
|
+
and response.arrow_record_batch.serialized_record_batch
|
|
2396
|
+
and pa_schema is not None
|
|
2397
|
+
):
|
|
2398
|
+
batch = pyarrow.ipc.read_record_batch(
|
|
2399
|
+
pyarrow.py_buffer(
|
|
2400
|
+
response.arrow_record_batch.serialized_record_batch
|
|
2401
|
+
),
|
|
2402
|
+
pa_schema,
|
|
2403
|
+
)
|
|
2404
|
+
yield batch
|
|
2405
|
+
|
|
2254
2406
|
# If changing the signature of this method, make sure to apply the same
|
|
2255
2407
|
# changes to job.QueryJob.to_arrow()
|
|
2256
2408
|
def to_arrow(
|
|
@@ -2316,6 +2468,12 @@ class RowIterator(HTTPIterator):
|
|
|
2316
2468
|
|
|
2317
2469
|
.. versionadded:: 1.17.0
|
|
2318
2470
|
"""
|
|
2471
|
+
warnings.warn(
|
|
2472
|
+
_TO_ARROW_DEPRECATED,
|
|
2473
|
+
PendingDeprecationWarning,
|
|
2474
|
+
stacklevel=2,
|
|
2475
|
+
)
|
|
2476
|
+
|
|
2319
2477
|
if pyarrow is None:
|
|
2320
2478
|
raise ValueError(_NO_PYARROW_ERROR)
|
|
2321
2479
|
|
|
@@ -2357,7 +2515,10 @@ class RowIterator(HTTPIterator):
|
|
|
2357
2515
|
# but mypy cannot infer this correlation. We ignore the union-attr error here.
|
|
2358
2516
|
bqstorage_client._transport.close() # type: ignore[union-attr]
|
|
2359
2517
|
|
|
2360
|
-
if record_batches and
|
|
2518
|
+
if record_batches and (
|
|
2519
|
+
bqstorage_client is not None
|
|
2520
|
+
or self._query_results_format == QueryResultsFormat.ARROW.value
|
|
2521
|
+
):
|
|
2361
2522
|
return pyarrow.Table.from_batches(record_batches)
|
|
2362
2523
|
else:
|
|
2363
2524
|
# No records (not record_batches), use schema based on BigQuery schema
|
|
@@ -2438,6 +2599,24 @@ class RowIterator(HTTPIterator):
|
|
|
2438
2599
|
if dtypes is None:
|
|
2439
2600
|
dtypes = {}
|
|
2440
2601
|
|
|
2602
|
+
if self._query_results_format == QueryResultsFormat.ARROW.value:
|
|
2603
|
+
|
|
2604
|
+
def _batch_to_dataframe(batch):
|
|
2605
|
+
df = batch.to_pandas()
|
|
2606
|
+
if dtypes:
|
|
2607
|
+
for col, dtype in dtypes.items():
|
|
2608
|
+
if col in df.columns:
|
|
2609
|
+
df[col] = df[col].astype(dtype)
|
|
2610
|
+
return df
|
|
2611
|
+
|
|
2612
|
+
return (
|
|
2613
|
+
_batch_to_dataframe(batch)
|
|
2614
|
+
for batch in self.to_arrow_iterable(
|
|
2615
|
+
bqstorage_client=bqstorage_client,
|
|
2616
|
+
timeout=timeout,
|
|
2617
|
+
)
|
|
2618
|
+
)
|
|
2619
|
+
|
|
2441
2620
|
self._maybe_warn_max_results(bqstorage_client)
|
|
2442
2621
|
|
|
2443
2622
|
column_names = [field.name for field in self._schema]
|
|
@@ -2709,6 +2888,12 @@ class RowIterator(HTTPIterator):
|
|
|
2709
2888
|
is not supported dtype.
|
|
2710
2889
|
|
|
2711
2890
|
"""
|
|
2891
|
+
warnings.warn(
|
|
2892
|
+
_TO_DATAFRAME_DEPRECATED,
|
|
2893
|
+
PendingDeprecationWarning,
|
|
2894
|
+
stacklevel=2,
|
|
2895
|
+
)
|
|
2896
|
+
|
|
2712
2897
|
_pandas_helpers.verify_pandas_imports()
|
|
2713
2898
|
|
|
2714
2899
|
if geography_as_object and shapely is None:
|
|
@@ -2797,16 +2982,27 @@ class RowIterator(HTTPIterator):
|
|
|
2797
2982
|
|
|
2798
2983
|
self._maybe_warn_max_results(bqstorage_client)
|
|
2799
2984
|
|
|
2800
|
-
if
|
|
2985
|
+
if (
|
|
2986
|
+
self._query_results_format != QueryResultsFormat.ARROW.value
|
|
2987
|
+
and not self._should_use_bqstorage(
|
|
2988
|
+
bqstorage_client, create_bqstorage_client
|
|
2989
|
+
)
|
|
2990
|
+
):
|
|
2801
2991
|
create_bqstorage_client = False
|
|
2802
2992
|
bqstorage_client = None
|
|
2803
2993
|
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2994
|
+
with warnings.catch_warnings():
|
|
2995
|
+
warnings.filterwarnings(
|
|
2996
|
+
"ignore",
|
|
2997
|
+
category=PendingDeprecationWarning,
|
|
2998
|
+
message="Retrieving PyArrow Tables.*",
|
|
2999
|
+
)
|
|
3000
|
+
record_batch = self.to_arrow(
|
|
3001
|
+
progress_bar_type=progress_bar_type,
|
|
3002
|
+
bqstorage_client=bqstorage_client,
|
|
3003
|
+
create_bqstorage_client=create_bqstorage_client,
|
|
3004
|
+
timeout=timeout,
|
|
3005
|
+
)
|
|
2810
3006
|
|
|
2811
3007
|
# Default date dtype is `db_dtypes.DateDtype()` that could cause out of bounds error,
|
|
2812
3008
|
# when pyarrow converts date values to nanosecond precision. To avoid the error, we
|
|
@@ -3009,18 +3205,24 @@ class RowIterator(HTTPIterator):
|
|
|
3009
3205
|
"one to use to create a GeoDataFrame"
|
|
3010
3206
|
)
|
|
3011
3207
|
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3208
|
+
with warnings.catch_warnings():
|
|
3209
|
+
warnings.filterwarnings(
|
|
3210
|
+
"ignore",
|
|
3211
|
+
category=PendingDeprecationWarning,
|
|
3212
|
+
message="Retrieving DataFrames via google-cloud-bigquery is deprecated.*",
|
|
3213
|
+
)
|
|
3214
|
+
df = self.to_dataframe(
|
|
3215
|
+
bqstorage_client,
|
|
3216
|
+
dtypes,
|
|
3217
|
+
progress_bar_type,
|
|
3218
|
+
create_bqstorage_client,
|
|
3219
|
+
geography_as_object=True,
|
|
3220
|
+
bool_dtype=bool_dtype,
|
|
3221
|
+
int_dtype=int_dtype,
|
|
3222
|
+
float_dtype=float_dtype,
|
|
3223
|
+
string_dtype=string_dtype,
|
|
3224
|
+
timeout=timeout,
|
|
3225
|
+
)
|
|
3024
3226
|
|
|
3025
3227
|
return geopandas.GeoDataFrame(
|
|
3026
3228
|
df, crs=_COORDINATE_REFERENCE_SYSTEM, geometry=geography_column
|
|
@@ -3048,6 +3250,14 @@ class _EmptyRowIterator(RowIterator):
|
|
|
3048
3250
|
)
|
|
3049
3251
|
self._total_rows = 0
|
|
3050
3252
|
|
|
3253
|
+
@_disallow_when_arrow
|
|
3254
|
+
def __iter__(self):
|
|
3255
|
+
return iter(())
|
|
3256
|
+
|
|
3257
|
+
@_disallow_when_arrow
|
|
3258
|
+
def __next__(self):
|
|
3259
|
+
raise StopIteration
|
|
3260
|
+
|
|
3051
3261
|
def to_arrow(
|
|
3052
3262
|
self,
|
|
3053
3263
|
progress_bar_type=None,
|
|
@@ -3068,6 +3278,11 @@ class _EmptyRowIterator(RowIterator):
|
|
|
3068
3278
|
"""
|
|
3069
3279
|
if pyarrow is None:
|
|
3070
3280
|
raise ValueError(_NO_PYARROW_ERROR)
|
|
3281
|
+
warnings.warn(
|
|
3282
|
+
_TO_ARROW_DEPRECATED,
|
|
3283
|
+
PendingDeprecationWarning,
|
|
3284
|
+
stacklevel=2,
|
|
3285
|
+
)
|
|
3071
3286
|
return pyarrow.Table.from_arrays(())
|
|
3072
3287
|
|
|
3073
3288
|
def to_dataframe(
|
|
@@ -3114,6 +3329,11 @@ class _EmptyRowIterator(RowIterator):
|
|
|
3114
3329
|
Returns:
|
|
3115
3330
|
pandas.DataFrame: An empty :class:`~pandas.DataFrame`.
|
|
3116
3331
|
"""
|
|
3332
|
+
warnings.warn(
|
|
3333
|
+
_TO_DATAFRAME_DEPRECATED,
|
|
3334
|
+
PendingDeprecationWarning,
|
|
3335
|
+
stacklevel=2,
|
|
3336
|
+
)
|
|
3117
3337
|
_pandas_helpers.verify_pandas_imports()
|
|
3118
3338
|
return pandas.DataFrame()
|
|
3119
3339
|
|
|
@@ -3219,11 +3439,10 @@ class _EmptyRowIterator(RowIterator):
|
|
|
3219
3439
|
Returns:
|
|
3220
3440
|
An iterator yielding a single empty :class:`~pyarrow.RecordBatch`.
|
|
3221
3441
|
"""
|
|
3442
|
+
if pyarrow is None:
|
|
3443
|
+
raise ValueError(_NO_PYARROW_ERROR)
|
|
3222
3444
|
return iter((pyarrow.record_batch([]),))
|
|
3223
3445
|
|
|
3224
|
-
def __iter__(self):
|
|
3225
|
-
return iter(())
|
|
3226
|
-
|
|
3227
3446
|
|
|
3228
3447
|
class PartitionRange(object):
|
|
3229
3448
|
"""Definition of the ranges for range partitioning.
|