google-cloud-bigquery 3.29.0__tar.gz → 3.30.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {google_cloud_bigquery-3.29.0/google_cloud_bigquery.egg-info → google_cloud_bigquery-3.30.0}/PKG-INFO +4 -1
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/_helpers.py +2 -2
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/_job_helpers.py +25 -2
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/_pandas_helpers.py +84 -15
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/_pyarrow_helpers.py +6 -1
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/enums.py +44 -1
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/retry.py +54 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/schema.py +89 -27
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/table.py +86 -6
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/version.py +1 -1
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0/google_cloud_bigquery.egg-info}/PKG-INFO +4 -1
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google_cloud_bigquery.egg-info/requires.txt +7 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/pyproject.toml +3 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/system/test_client.py +42 -2
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/system/test_pandas.py +1 -1
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/job/test_load.py +1 -1
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test__pandas_helpers.py +147 -11
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_client.py +207 -14
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_schema.py +173 -66
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_table.py +199 -1
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/LICENSE +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/MANIFEST.in +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/README.rst +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/_http.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/_tqdm_helpers.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/_versions_helpers.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/client.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/dataset.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/dbapi/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/dbapi/_helpers.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/dbapi/connection.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/dbapi/cursor.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/dbapi/exceptions.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/dbapi/types.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/encryption_configuration.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/exceptions.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/external_config.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/format_options.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/iam.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/job/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/job/base.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/job/copy_.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/job/extract.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/job/load.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/job/query.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/magics/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/magics/line_arg_parser/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/magics/line_arg_parser/exceptions.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/magics/line_arg_parser/lexer.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/magics/line_arg_parser/parser.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/magics/line_arg_parser/visitors.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/magics/magics.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/model.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/opentelemetry_tracing.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/py.typed +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/query.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/routine/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/routine/routine.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/standard_sql.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery_v2/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery_v2/types/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery_v2/types/encryption_config.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery_v2/types/model.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery_v2/types/model_reference.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery_v2/types/standard_sql.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery_v2/types/table_reference.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google_cloud_bigquery.egg-info/SOURCES.txt +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google_cloud_bigquery.egg-info/dependency_links.txt +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google_cloud_bigquery.egg-info/top_level.txt +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/setup.cfg +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/setup.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/data/characters.json +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/data/characters.jsonl +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/data/colors.avro +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/data/numeric_38_12.parquet +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/data/people.csv +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/data/scalars.csv +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/data/scalars.jsonl +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/data/scalars_extreme.jsonl +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/data/scalars_schema.json +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/data/scalars_schema_csv.json +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/data/schema.json +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/scrub_datasets.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/system/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/system/conftest.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/system/helpers.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/system/test_arrow.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/system/test_job_retry.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/system/test_list_rows.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/system/test_magics.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/system/test_query.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/system/test_structs.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/_helpers/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/_helpers/test_from_json.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/conftest.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/helpers.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/job/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/job/helpers.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/job/test_base.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/job/test_copy.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/job/test_extract.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/job/test_load_config.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/job/test_query.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/job/test_query_config.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/job/test_query_pandas.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/job/test_query_stats.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/line_arg_parser/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/line_arg_parser/test_lexer.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/line_arg_parser/test_parser.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/line_arg_parser/test_visitors.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/model/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/model/test_model.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/model/test_model_reference.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/routine/__init__.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/routine/test_remote_function_options.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/routine/test_routine.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/routine/test_routine_argument.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/routine/test_routine_reference.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test__helpers.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test__http.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test__job_helpers.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test__pyarrow_helpers.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test__versions_helpers.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_create_dataset.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_dataset.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_dbapi__helpers.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_dbapi_connection.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_dbapi_cursor.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_dbapi_types.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_delete_dataset.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_encryption_configuration.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_external_config.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_format_options.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_job_retry.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_legacy_types.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_list_datasets.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_list_jobs.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_list_models.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_list_projects.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_list_routines.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_list_tables.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_magics.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_opentelemetry_tracing.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_packaging.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_query.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_retry.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_signature_compatibility.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_standard_sql_types.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_table_arrow.py +0 -0
- {google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/tests/unit/test_table_pandas.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: google-cloud-bigquery
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.30.0
|
|
4
4
|
Summary: Google BigQuery API client library
|
|
5
5
|
Author-email: Google LLC <googleapis-packages@google.com>
|
|
6
6
|
License: Apache 2.0
|
|
@@ -35,6 +35,9 @@ Requires-Dist: grpcio<2.0dev,>=1.49.1; python_version >= "3.11" and extra == "bq
|
|
|
35
35
|
Requires-Dist: pyarrow>=3.0.0; extra == "bqstorage"
|
|
36
36
|
Provides-Extra: pandas
|
|
37
37
|
Requires-Dist: pandas>=1.1.0; extra == "pandas"
|
|
38
|
+
Requires-Dist: pandas-gbq>=0.26.1; python_version >= "3.8" and extra == "pandas"
|
|
39
|
+
Requires-Dist: grpcio<2.0dev,>=1.47.0; extra == "pandas"
|
|
40
|
+
Requires-Dist: grpcio<2.0dev,>=1.49.1; python_version >= "3.11" and extra == "pandas"
|
|
38
41
|
Requires-Dist: pyarrow>=3.0.0; extra == "pandas"
|
|
39
42
|
Requires-Dist: db-dtypes<2.0.0dev,>=0.3.0; extra == "pandas"
|
|
40
43
|
Requires-Dist: importlib_metadata>=1.0.0; python_version < "3.8" and extra == "pandas"
|
{google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/_helpers.py
RENAMED
|
@@ -978,11 +978,11 @@ def _build_resource_from_properties(obj, filter_fields):
|
|
|
978
978
|
"""
|
|
979
979
|
partial = {}
|
|
980
980
|
for filter_field in filter_fields:
|
|
981
|
-
api_field = obj._PROPERTY_TO_API_FIELD
|
|
981
|
+
api_field = _get_sub_prop(obj._PROPERTY_TO_API_FIELD, filter_field)
|
|
982
982
|
if api_field is None and filter_field not in obj._properties:
|
|
983
983
|
raise ValueError("No property %s" % filter_field)
|
|
984
984
|
elif api_field is not None:
|
|
985
|
-
partial
|
|
985
|
+
_set_sub_prop(partial, api_field, _get_sub_prop(obj._properties, api_field))
|
|
986
986
|
else:
|
|
987
987
|
# allows properties that are not defined in the library
|
|
988
988
|
# and properties that have the same name as API resource key
|
{google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/_job_helpers.py
RENAMED
|
@@ -47,6 +47,7 @@ from google.api_core import retry as retries
|
|
|
47
47
|
from google.cloud.bigquery import job
|
|
48
48
|
import google.cloud.bigquery.query
|
|
49
49
|
from google.cloud.bigquery import table
|
|
50
|
+
import google.cloud.bigquery.retry
|
|
50
51
|
from google.cloud.bigquery.retry import POLLING_DEFAULT_VALUE
|
|
51
52
|
|
|
52
53
|
# Avoid circular imports
|
|
@@ -142,12 +143,28 @@ def query_jobs_insert(
|
|
|
142
143
|
raise create_exc
|
|
143
144
|
|
|
144
145
|
try:
|
|
146
|
+
# Sometimes we get a 404 after a Conflict. In this case, we
|
|
147
|
+
# have pretty high confidence that by retrying the 404, we'll
|
|
148
|
+
# (hopefully) eventually recover the job.
|
|
149
|
+
# https://github.com/googleapis/python-bigquery/issues/2134
|
|
150
|
+
#
|
|
151
|
+
# Allow users who want to completely disable retries to
|
|
152
|
+
# continue to do so by setting retry to None.
|
|
153
|
+
get_job_retry = retry
|
|
154
|
+
if retry is not None:
|
|
155
|
+
# TODO(tswast): Amend the user's retry object with allowing
|
|
156
|
+
# 404 to retry when there's a public way to do so.
|
|
157
|
+
# https://github.com/googleapis/python-api-core/issues/796
|
|
158
|
+
get_job_retry = (
|
|
159
|
+
google.cloud.bigquery.retry._DEFAULT_GET_JOB_CONFLICT_RETRY
|
|
160
|
+
)
|
|
161
|
+
|
|
145
162
|
query_job = client.get_job(
|
|
146
163
|
job_id,
|
|
147
164
|
project=project,
|
|
148
165
|
location=location,
|
|
149
|
-
retry=
|
|
150
|
-
timeout=
|
|
166
|
+
retry=get_job_retry,
|
|
167
|
+
timeout=google.cloud.bigquery.retry.DEFAULT_GET_JOB_TIMEOUT,
|
|
151
168
|
)
|
|
152
169
|
except core_exceptions.GoogleAPIError: # (includes RetryError)
|
|
153
170
|
raise
|
|
@@ -156,7 +173,13 @@ def query_jobs_insert(
|
|
|
156
173
|
else:
|
|
157
174
|
return query_job
|
|
158
175
|
|
|
176
|
+
# Allow users who want to completely disable retries to
|
|
177
|
+
# continue to do so by setting job_retry to None.
|
|
178
|
+
if job_retry is not None:
|
|
179
|
+
do_query = google.cloud.bigquery.retry._DEFAULT_QUERY_JOB_INSERT_RETRY(do_query)
|
|
180
|
+
|
|
159
181
|
future = do_query()
|
|
182
|
+
|
|
160
183
|
# The future might be in a failed state now, but if it's
|
|
161
184
|
# unrecoverable, we'll find out when we ask for it's result, at which
|
|
162
185
|
# point, we may retry.
|
|
@@ -12,7 +12,12 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
"""Shared helper functions for connecting BigQuery and pandas.
|
|
15
|
+
"""Shared helper functions for connecting BigQuery and pandas.
|
|
16
|
+
|
|
17
|
+
NOTE: This module is DEPRECATED. Please make updates in the pandas-gbq package,
|
|
18
|
+
instead. See: go/pandas-gbq-and-bigframes-redundancy and
|
|
19
|
+
https://github.com/googleapis/python-bigquery-pandas/blob/main/pandas_gbq/schema/pandas_to_bigquery.py
|
|
20
|
+
"""
|
|
16
21
|
|
|
17
22
|
import concurrent.futures
|
|
18
23
|
from datetime import datetime
|
|
@@ -20,6 +25,7 @@ import functools
|
|
|
20
25
|
from itertools import islice
|
|
21
26
|
import logging
|
|
22
27
|
import queue
|
|
28
|
+
import threading
|
|
23
29
|
import warnings
|
|
24
30
|
from typing import Any, Union, Optional, Callable, Generator, List
|
|
25
31
|
|
|
@@ -39,6 +45,16 @@ except ImportError as exc:
|
|
|
39
45
|
else:
|
|
40
46
|
import numpy
|
|
41
47
|
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
import pandas_gbq.schema.pandas_to_bigquery # type: ignore
|
|
51
|
+
|
|
52
|
+
pandas_gbq_import_exception = None
|
|
53
|
+
except ImportError as exc:
|
|
54
|
+
pandas_gbq = None
|
|
55
|
+
pandas_gbq_import_exception = exc
|
|
56
|
+
|
|
57
|
+
|
|
42
58
|
try:
|
|
43
59
|
import db_dtypes # type: ignore
|
|
44
60
|
|
|
@@ -119,6 +135,21 @@ class _DownloadState(object):
|
|
|
119
135
|
# be an atomic operation in the Python language definition (enforced by
|
|
120
136
|
# the global interpreter lock).
|
|
121
137
|
self.done = False
|
|
138
|
+
# To assist with testing and understanding the behavior of the
|
|
139
|
+
# download, use this object as shared state to track how many worker
|
|
140
|
+
# threads have started and have gracefully shutdown.
|
|
141
|
+
self._started_workers_lock = threading.Lock()
|
|
142
|
+
self.started_workers = 0
|
|
143
|
+
self._finished_workers_lock = threading.Lock()
|
|
144
|
+
self.finished_workers = 0
|
|
145
|
+
|
|
146
|
+
def start(self):
|
|
147
|
+
with self._started_workers_lock:
|
|
148
|
+
self.started_workers += 1
|
|
149
|
+
|
|
150
|
+
def finish(self):
|
|
151
|
+
with self._finished_workers_lock:
|
|
152
|
+
self.finished_workers += 1
|
|
122
153
|
|
|
123
154
|
|
|
124
155
|
BQ_FIELD_TYPE_TO_ARROW_FIELD_METADATA = {
|
|
@@ -429,6 +460,10 @@ def _first_array_valid(series):
|
|
|
429
460
|
def dataframe_to_bq_schema(dataframe, bq_schema):
|
|
430
461
|
"""Convert a pandas DataFrame schema to a BigQuery schema.
|
|
431
462
|
|
|
463
|
+
DEPRECATED: Use
|
|
464
|
+
pandas_gbq.schema.pandas_to_bigquery.dataframe_to_bigquery_fields(),
|
|
465
|
+
instead. See: go/pandas-gbq-and-bigframes-redundancy.
|
|
466
|
+
|
|
432
467
|
Args:
|
|
433
468
|
dataframe (pandas.DataFrame):
|
|
434
469
|
DataFrame for which the client determines the BigQuery schema.
|
|
@@ -444,6 +479,20 @@ def dataframe_to_bq_schema(dataframe, bq_schema):
|
|
|
444
479
|
The automatically determined schema. Returns None if the type of
|
|
445
480
|
any column cannot be determined.
|
|
446
481
|
"""
|
|
482
|
+
if pandas_gbq is None:
|
|
483
|
+
warnings.warn(
|
|
484
|
+
"Loading pandas DataFrame into BigQuery will require pandas-gbq "
|
|
485
|
+
"package version 0.26.1 or greater in the future. "
|
|
486
|
+
f"Tried to import pandas-gbq and got: {pandas_gbq_import_exception}",
|
|
487
|
+
category=FutureWarning,
|
|
488
|
+
)
|
|
489
|
+
else:
|
|
490
|
+
return pandas_gbq.schema.pandas_to_bigquery.dataframe_to_bigquery_fields(
|
|
491
|
+
dataframe,
|
|
492
|
+
override_bigquery_fields=bq_schema,
|
|
493
|
+
index=True,
|
|
494
|
+
)
|
|
495
|
+
|
|
447
496
|
if bq_schema:
|
|
448
497
|
bq_schema = schema._to_schema_fields(bq_schema)
|
|
449
498
|
bq_schema_index = {field.name: field for field in bq_schema}
|
|
@@ -786,20 +835,35 @@ def _bqstorage_page_to_dataframe(column_names, dtypes, page):
|
|
|
786
835
|
def _download_table_bqstorage_stream(
|
|
787
836
|
download_state, bqstorage_client, session, stream, worker_queue, page_to_item
|
|
788
837
|
):
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
# https://github.com/googleapis/python-bigquery-storage/issues/229
|
|
793
|
-
if _versions_helpers.BQ_STORAGE_VERSIONS.is_read_session_optional:
|
|
794
|
-
rowstream = reader.rows()
|
|
795
|
-
else:
|
|
796
|
-
rowstream = reader.rows(session)
|
|
838
|
+
download_state.start()
|
|
839
|
+
try:
|
|
840
|
+
reader = bqstorage_client.read_rows(stream.name)
|
|
797
841
|
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
842
|
+
# Avoid deprecation warnings for passing in unnecessary read session.
|
|
843
|
+
# https://github.com/googleapis/python-bigquery-storage/issues/229
|
|
844
|
+
if _versions_helpers.BQ_STORAGE_VERSIONS.is_read_session_optional:
|
|
845
|
+
rowstream = reader.rows()
|
|
846
|
+
else:
|
|
847
|
+
rowstream = reader.rows(session)
|
|
848
|
+
|
|
849
|
+
for page in rowstream.pages:
|
|
850
|
+
item = page_to_item(page)
|
|
851
|
+
|
|
852
|
+
# Make sure we set a timeout on put() so that we give the worker
|
|
853
|
+
# thread opportunities to shutdown gracefully, for example if the
|
|
854
|
+
# parent thread shuts down or the parent generator object which
|
|
855
|
+
# collects rows from all workers goes out of scope. See:
|
|
856
|
+
# https://github.com/googleapis/python-bigquery/issues/2032
|
|
857
|
+
while True:
|
|
858
|
+
if download_state.done:
|
|
859
|
+
return
|
|
860
|
+
try:
|
|
861
|
+
worker_queue.put(item, timeout=_PROGRESS_INTERVAL)
|
|
862
|
+
break
|
|
863
|
+
except queue.Full:
|
|
864
|
+
continue
|
|
865
|
+
finally:
|
|
866
|
+
download_state.finish()
|
|
803
867
|
|
|
804
868
|
|
|
805
869
|
def _nowait(futures):
|
|
@@ -825,6 +889,7 @@ def _download_table_bqstorage(
|
|
|
825
889
|
page_to_item: Optional[Callable] = None,
|
|
826
890
|
max_queue_size: Any = _MAX_QUEUE_SIZE_DEFAULT,
|
|
827
891
|
max_stream_count: Optional[int] = None,
|
|
892
|
+
download_state: Optional[_DownloadState] = None,
|
|
828
893
|
) -> Generator[Any, None, None]:
|
|
829
894
|
"""Downloads a BigQuery table using the BigQuery Storage API.
|
|
830
895
|
|
|
@@ -852,6 +917,9 @@ def _download_table_bqstorage(
|
|
|
852
917
|
is True, the requested streams are limited to 1 regardless of the
|
|
853
918
|
`max_stream_count` value. If 0 or None, then the number of
|
|
854
919
|
requested streams will be unbounded. Defaults to None.
|
|
920
|
+
download_state (Optional[_DownloadState]):
|
|
921
|
+
A threadsafe state object which can be used to observe the
|
|
922
|
+
behavior of the worker threads created by this method.
|
|
855
923
|
|
|
856
924
|
Yields:
|
|
857
925
|
pandas.DataFrame: Pandas DataFrames, one for each chunk of data
|
|
@@ -910,7 +978,8 @@ def _download_table_bqstorage(
|
|
|
910
978
|
|
|
911
979
|
# Use _DownloadState to notify worker threads when to quit.
|
|
912
980
|
# See: https://stackoverflow.com/a/29237343/101923
|
|
913
|
-
download_state
|
|
981
|
+
if download_state is None:
|
|
982
|
+
download_state = _DownloadState()
|
|
914
983
|
|
|
915
984
|
# Create a queue to collect frames as they are created in each thread.
|
|
916
985
|
#
|
|
@@ -12,7 +12,12 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
"""Shared helper functions for connecting BigQuery and pyarrow.
|
|
15
|
+
"""Shared helper functions for connecting BigQuery and pyarrow.
|
|
16
|
+
|
|
17
|
+
NOTE: This module is DEPRECATED. Please make updates in the pandas-gbq package,
|
|
18
|
+
instead. See: go/pandas-gbq-and-bigframes-redundancy and
|
|
19
|
+
https://github.com/googleapis/python-bigquery-pandas/blob/main/pandas_gbq/schema/pyarrow_to_bigquery.py
|
|
20
|
+
"""
|
|
16
21
|
|
|
17
22
|
from typing import Any
|
|
18
23
|
|
{google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/enums.py
RENAMED
|
@@ -246,6 +246,11 @@ class KeyResultStatementKind:
|
|
|
246
246
|
|
|
247
247
|
|
|
248
248
|
class StandardSqlTypeNames(str, enum.Enum):
|
|
249
|
+
"""Enum of allowed SQL type names in schema.SchemaField.
|
|
250
|
+
|
|
251
|
+
Datatype used in GoogleSQL.
|
|
252
|
+
"""
|
|
253
|
+
|
|
249
254
|
def _generate_next_value_(name, start, count, last_values):
|
|
250
255
|
return name
|
|
251
256
|
|
|
@@ -267,6 +272,9 @@ class StandardSqlTypeNames(str, enum.Enum):
|
|
|
267
272
|
ARRAY = enum.auto()
|
|
268
273
|
STRUCT = enum.auto()
|
|
269
274
|
RANGE = enum.auto()
|
|
275
|
+
# NOTE: FOREIGN acts as a wrapper for data types
|
|
276
|
+
# not natively understood by BigQuery unless translated
|
|
277
|
+
FOREIGN = enum.auto()
|
|
270
278
|
|
|
271
279
|
|
|
272
280
|
class EntityTypes(str, enum.Enum):
|
|
@@ -285,7 +293,10 @@ class EntityTypes(str, enum.Enum):
|
|
|
285
293
|
# See also: https://cloud.google.com/bigquery/data-types#legacy_sql_data_types
|
|
286
294
|
# and https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
|
|
287
295
|
class SqlTypeNames(str, enum.Enum):
|
|
288
|
-
"""Enum of allowed SQL type names in schema.SchemaField.
|
|
296
|
+
"""Enum of allowed SQL type names in schema.SchemaField.
|
|
297
|
+
|
|
298
|
+
Datatype used in Legacy SQL.
|
|
299
|
+
"""
|
|
289
300
|
|
|
290
301
|
STRING = "STRING"
|
|
291
302
|
BYTES = "BYTES"
|
|
@@ -306,6 +317,9 @@ class SqlTypeNames(str, enum.Enum):
|
|
|
306
317
|
DATETIME = "DATETIME"
|
|
307
318
|
INTERVAL = "INTERVAL" # NOTE: not available in legacy types
|
|
308
319
|
RANGE = "RANGE" # NOTE: not available in legacy types
|
|
320
|
+
# NOTE: FOREIGN acts as a wrapper for data types
|
|
321
|
+
# not natively understood by BigQuery unless translated
|
|
322
|
+
FOREIGN = "FOREIGN"
|
|
309
323
|
|
|
310
324
|
|
|
311
325
|
class WriteDisposition(object):
|
|
@@ -344,3 +358,32 @@ class DeterminismLevel:
|
|
|
344
358
|
|
|
345
359
|
NOT_DETERMINISTIC = "NOT_DETERMINISTIC"
|
|
346
360
|
"""The UDF is not deterministic."""
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
class RoundingMode(str, enum.Enum):
|
|
364
|
+
"""Rounding mode options that can be used when storing NUMERIC or BIGNUMERIC
|
|
365
|
+
values.
|
|
366
|
+
|
|
367
|
+
ROUNDING_MODE_UNSPECIFIED: will default to using ROUND_HALF_AWAY_FROM_ZERO.
|
|
368
|
+
|
|
369
|
+
ROUND_HALF_AWAY_FROM_ZERO: rounds half values away from zero when applying
|
|
370
|
+
precision and scale upon writing of NUMERIC and BIGNUMERIC values.
|
|
371
|
+
For Scale: 0
|
|
372
|
+
* 1.1, 1.2, 1.3, 1.4 => 1
|
|
373
|
+
* 1.5, 1.6, 1.7, 1.8, 1.9 => 2
|
|
374
|
+
|
|
375
|
+
ROUND_HALF_EVEN: rounds half values to the nearest even value when applying
|
|
376
|
+
precision and scale upon writing of NUMERIC and BIGNUMERIC values.
|
|
377
|
+
For Scale: 0
|
|
378
|
+
* 1.1, 1.2, 1.3, 1.4 => 1
|
|
379
|
+
* 1.5 => 2
|
|
380
|
+
* 1.6, 1.7, 1.8, 1.9 => 2
|
|
381
|
+
* 2.5 => 2
|
|
382
|
+
"""
|
|
383
|
+
|
|
384
|
+
def _generate_next_value_(name, start, count, last_values):
|
|
385
|
+
return name
|
|
386
|
+
|
|
387
|
+
ROUNDING_MODE_UNSPECIFIED = enum.auto()
|
|
388
|
+
ROUND_HALF_AWAY_FROM_ZERO = enum.auto()
|
|
389
|
+
ROUND_HALF_EVEN = enum.auto()
|
{google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/retry.py
RENAMED
|
@@ -82,6 +82,32 @@ on ``DEFAULT_RETRY``. For example, to change the deadline to 30 seconds,
|
|
|
82
82
|
pass ``retry=bigquery.DEFAULT_RETRY.with_deadline(30)``.
|
|
83
83
|
"""
|
|
84
84
|
|
|
85
|
+
|
|
86
|
+
def _should_retry_get_job_conflict(exc):
|
|
87
|
+
"""Predicate for determining when to retry a jobs.get call after a conflict error.
|
|
88
|
+
|
|
89
|
+
Sometimes we get a 404 after a Conflict. In this case, we
|
|
90
|
+
have pretty high confidence that by retrying the 404, we'll
|
|
91
|
+
(hopefully) eventually recover the job.
|
|
92
|
+
https://github.com/googleapis/python-bigquery/issues/2134
|
|
93
|
+
|
|
94
|
+
Note: we may be able to extend this to user-specified predicates
|
|
95
|
+
after https://github.com/googleapis/python-api-core/issues/796
|
|
96
|
+
to tweak existing Retry object predicates.
|
|
97
|
+
"""
|
|
98
|
+
return isinstance(exc, exceptions.NotFound) or _should_retry(exc)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# Pick a deadline smaller than our other deadlines since we want to timeout
|
|
102
|
+
# before those expire.
|
|
103
|
+
_DEFAULT_GET_JOB_CONFLICT_DEADLINE = _DEFAULT_RETRY_DEADLINE / 3.0
|
|
104
|
+
_DEFAULT_GET_JOB_CONFLICT_RETRY = retry.Retry(
|
|
105
|
+
predicate=_should_retry_get_job_conflict,
|
|
106
|
+
deadline=_DEFAULT_GET_JOB_CONFLICT_DEADLINE,
|
|
107
|
+
)
|
|
108
|
+
"""Private, may be removed in future."""
|
|
109
|
+
|
|
110
|
+
|
|
85
111
|
# Note: Take care when updating DEFAULT_TIMEOUT to anything but None. We
|
|
86
112
|
# briefly had a default timeout, but even setting it at more than twice the
|
|
87
113
|
# theoretical server-side default timeout of 2 minutes was not enough for
|
|
@@ -142,6 +168,34 @@ DEFAULT_JOB_RETRY = retry.Retry(
|
|
|
142
168
|
The default job retry object.
|
|
143
169
|
"""
|
|
144
170
|
|
|
171
|
+
|
|
172
|
+
def _query_job_insert_should_retry(exc):
|
|
173
|
+
# Per https://github.com/googleapis/python-bigquery/issues/2134, sometimes
|
|
174
|
+
# we get a 404 error. In this case, if we get this far, assume that the job
|
|
175
|
+
# doesn't actually exist and try again. We can't add 404 to the default
|
|
176
|
+
# job_retry because that happens for errors like "this table does not
|
|
177
|
+
# exist", which probably won't resolve with a retry.
|
|
178
|
+
if isinstance(exc, exceptions.RetryError):
|
|
179
|
+
exc = exc.cause
|
|
180
|
+
|
|
181
|
+
if isinstance(exc, exceptions.NotFound):
|
|
182
|
+
message = exc.message
|
|
183
|
+
# Don't try to retry table/dataset not found, just job not found.
|
|
184
|
+
# The URL contains jobs, so use whitespace to disambiguate.
|
|
185
|
+
return message is not None and " job" in message.lower()
|
|
186
|
+
|
|
187
|
+
return _job_should_retry(exc)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
_DEFAULT_QUERY_JOB_INSERT_RETRY = retry.Retry(
|
|
191
|
+
predicate=_query_job_insert_should_retry,
|
|
192
|
+
# jobs.insert doesn't wait for the job to complete, so we don't need the
|
|
193
|
+
# long _DEFAULT_JOB_DEADLINE for this part.
|
|
194
|
+
deadline=_DEFAULT_RETRY_DEADLINE,
|
|
195
|
+
)
|
|
196
|
+
"""Private, may be removed in future."""
|
|
197
|
+
|
|
198
|
+
|
|
145
199
|
DEFAULT_GET_JOB_TIMEOUT = 128
|
|
146
200
|
"""
|
|
147
201
|
Default timeout for Client.get_job().
|
{google_cloud_bigquery-3.29.0 → google_cloud_bigquery-3.30.0}/google/cloud/bigquery/schema.py
RENAMED
|
@@ -15,21 +15,21 @@
|
|
|
15
15
|
"""Schemas for BigQuery tables / queries."""
|
|
16
16
|
|
|
17
17
|
from __future__ import annotations
|
|
18
|
-
import collections
|
|
19
18
|
import enum
|
|
20
19
|
import typing
|
|
21
|
-
from typing import Any, cast, Dict, Iterable, Optional, Union
|
|
20
|
+
from typing import Any, cast, Dict, Iterable, Optional, Union, Sequence
|
|
22
21
|
|
|
23
22
|
from google.cloud.bigquery import _helpers
|
|
24
23
|
from google.cloud.bigquery import standard_sql
|
|
24
|
+
from google.cloud.bigquery import enums
|
|
25
25
|
from google.cloud.bigquery.enums import StandardSqlTypeNames
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
_STRUCT_TYPES = ("RECORD", "STRUCT")
|
|
29
29
|
|
|
30
30
|
# SQL types reference:
|
|
31
|
-
# https://cloud.google.com/bigquery/data-types#legacy_sql_data_types
|
|
32
|
-
# https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
|
|
31
|
+
# LEGACY SQL: https://cloud.google.com/bigquery/data-types#legacy_sql_data_types
|
|
32
|
+
# GoogleSQL: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
|
|
33
33
|
LEGACY_TO_STANDARD_TYPES = {
|
|
34
34
|
"STRING": StandardSqlTypeNames.STRING,
|
|
35
35
|
"BYTES": StandardSqlTypeNames.BYTES,
|
|
@@ -48,6 +48,7 @@ LEGACY_TO_STANDARD_TYPES = {
|
|
|
48
48
|
"DATE": StandardSqlTypeNames.DATE,
|
|
49
49
|
"TIME": StandardSqlTypeNames.TIME,
|
|
50
50
|
"DATETIME": StandardSqlTypeNames.DATETIME,
|
|
51
|
+
"FOREIGN": StandardSqlTypeNames.FOREIGN,
|
|
51
52
|
# no direct conversion from ARRAY, the latter is represented by mode="REPEATED"
|
|
52
53
|
}
|
|
53
54
|
"""String names of the legacy SQL types to integer codes of Standard SQL standard_sql."""
|
|
@@ -166,6 +167,35 @@ class SchemaField(object):
|
|
|
166
167
|
the type is RANGE, this field is required. Possible values for the
|
|
167
168
|
field element type of a RANGE include `DATE`, `DATETIME` and
|
|
168
169
|
`TIMESTAMP`.
|
|
170
|
+
|
|
171
|
+
rounding_mode: Union[enums.RoundingMode, str, None]
|
|
172
|
+
Specifies the rounding mode to be used when storing values of
|
|
173
|
+
NUMERIC and BIGNUMERIC type.
|
|
174
|
+
|
|
175
|
+
Unspecified will default to using ROUND_HALF_AWAY_FROM_ZERO.
|
|
176
|
+
ROUND_HALF_AWAY_FROM_ZERO rounds half values away from zero
|
|
177
|
+
when applying precision and scale upon writing of NUMERIC and BIGNUMERIC
|
|
178
|
+
values.
|
|
179
|
+
|
|
180
|
+
For Scale: 0
|
|
181
|
+
1.1, 1.2, 1.3, 1.4 => 1
|
|
182
|
+
1.5, 1.6, 1.7, 1.8, 1.9 => 2
|
|
183
|
+
|
|
184
|
+
ROUND_HALF_EVEN rounds half values to the nearest even value
|
|
185
|
+
when applying precision and scale upon writing of NUMERIC and BIGNUMERIC
|
|
186
|
+
values.
|
|
187
|
+
|
|
188
|
+
For Scale: 0
|
|
189
|
+
1.1, 1.2, 1.3, 1.4 => 1
|
|
190
|
+
1.5 => 2
|
|
191
|
+
1.6, 1.7, 1.8, 1.9 => 2
|
|
192
|
+
2.5 => 2
|
|
193
|
+
|
|
194
|
+
foreign_type_definition: Optional[str]
|
|
195
|
+
Definition of the foreign data type.
|
|
196
|
+
|
|
197
|
+
Only valid for top-level schema fields (not nested fields).
|
|
198
|
+
If the type is FOREIGN, this field is required.
|
|
169
199
|
"""
|
|
170
200
|
|
|
171
201
|
def __init__(
|
|
@@ -181,11 +211,14 @@ class SchemaField(object):
|
|
|
181
211
|
scale: Union[int, _DefaultSentinel] = _DEFAULT_VALUE,
|
|
182
212
|
max_length: Union[int, _DefaultSentinel] = _DEFAULT_VALUE,
|
|
183
213
|
range_element_type: Union[FieldElementType, str, None] = None,
|
|
214
|
+
rounding_mode: Union[enums.RoundingMode, str, None] = None,
|
|
215
|
+
foreign_type_definition: Optional[str] = None,
|
|
184
216
|
):
|
|
185
217
|
self._properties: Dict[str, Any] = {
|
|
186
218
|
"name": name,
|
|
187
219
|
"type": field_type,
|
|
188
220
|
}
|
|
221
|
+
self._properties["name"] = name
|
|
189
222
|
if mode is not None:
|
|
190
223
|
self._properties["mode"] = mode.upper()
|
|
191
224
|
if description is not _DEFAULT_VALUE:
|
|
@@ -206,6 +239,11 @@ class SchemaField(object):
|
|
|
206
239
|
self._properties["rangeElementType"] = {"type": range_element_type}
|
|
207
240
|
if isinstance(range_element_type, FieldElementType):
|
|
208
241
|
self._properties["rangeElementType"] = range_element_type.to_api_repr()
|
|
242
|
+
if rounding_mode is not None:
|
|
243
|
+
self._properties["roundingMode"] = rounding_mode
|
|
244
|
+
if foreign_type_definition is not None:
|
|
245
|
+
self._properties["foreignTypeDefinition"] = foreign_type_definition
|
|
246
|
+
|
|
209
247
|
if fields: # Don't set the property if it's not set.
|
|
210
248
|
self._properties["fields"] = [field.to_api_repr() for field in fields]
|
|
211
249
|
|
|
@@ -304,6 +342,22 @@ class SchemaField(object):
|
|
|
304
342
|
ret = self._properties.get("rangeElementType")
|
|
305
343
|
return FieldElementType.from_api_repr(ret)
|
|
306
344
|
|
|
345
|
+
@property
|
|
346
|
+
def rounding_mode(self):
|
|
347
|
+
"""Enum that specifies the rounding mode to be used when storing values of
|
|
348
|
+
NUMERIC and BIGNUMERIC type.
|
|
349
|
+
"""
|
|
350
|
+
return self._properties.get("roundingMode")
|
|
351
|
+
|
|
352
|
+
@property
|
|
353
|
+
def foreign_type_definition(self):
|
|
354
|
+
"""Definition of the foreign data type.
|
|
355
|
+
|
|
356
|
+
Only valid for top-level schema fields (not nested fields).
|
|
357
|
+
If the type is FOREIGN, this field is required.
|
|
358
|
+
"""
|
|
359
|
+
return self._properties.get("foreignTypeDefinition")
|
|
360
|
+
|
|
307
361
|
@property
|
|
308
362
|
def fields(self):
|
|
309
363
|
"""Optional[tuple]: Subfields contained in this field.
|
|
@@ -434,6 +488,8 @@ def _parse_schema_resource(info):
|
|
|
434
488
|
Optional[Sequence[google.cloud.bigquery.schema.SchemaField`]:
|
|
435
489
|
A list of parsed fields, or ``None`` if no "fields" key found.
|
|
436
490
|
"""
|
|
491
|
+
if isinstance(info, list):
|
|
492
|
+
return [SchemaField.from_api_repr(f) for f in info]
|
|
437
493
|
return [SchemaField.from_api_repr(f) for f in info.get("fields", ())]
|
|
438
494
|
|
|
439
495
|
|
|
@@ -446,40 +502,46 @@ def _build_schema_resource(fields):
|
|
|
446
502
|
Returns:
|
|
447
503
|
Sequence[Dict]: Mappings describing the schema of the supplied fields.
|
|
448
504
|
"""
|
|
449
|
-
|
|
505
|
+
if isinstance(fields, Sequence):
|
|
506
|
+
# Input is a Sequence (e.g. a list): Process and return a list of SchemaFields
|
|
507
|
+
return [field.to_api_repr() for field in fields]
|
|
508
|
+
|
|
509
|
+
else:
|
|
510
|
+
raise TypeError("Schema must be a Sequence (e.g. a list) or None.")
|
|
450
511
|
|
|
451
512
|
|
|
452
513
|
def _to_schema_fields(schema):
|
|
453
|
-
"""
|
|
514
|
+
"""Coerces schema to a list of SchemaField instances while
|
|
515
|
+
preserving the original structure as much as possible.
|
|
454
516
|
|
|
455
517
|
Args:
|
|
456
|
-
schema(Sequence[Union[ \
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
518
|
+
schema (Sequence[Union[ \
|
|
519
|
+
:class:`~google.cloud.bigquery.schema.SchemaField`, \
|
|
520
|
+
Mapping[str, Any] \
|
|
521
|
+
]
|
|
522
|
+
]
|
|
523
|
+
)::
|
|
524
|
+
Table schema to convert. Can be a list of SchemaField
|
|
525
|
+
objects or mappings.
|
|
463
526
|
|
|
464
527
|
Returns:
|
|
465
|
-
|
|
528
|
+
A list of SchemaField objects.
|
|
466
529
|
|
|
467
530
|
Raises:
|
|
468
|
-
|
|
469
|
-
sequence is not a :class:`~google.cloud.bigquery.schema.SchemaField`
|
|
470
|
-
instance or a compatible mapping representation of the field.
|
|
531
|
+
TypeError: If schema is not a Sequence.
|
|
471
532
|
"""
|
|
472
|
-
for field in schema:
|
|
473
|
-
if not isinstance(field, (SchemaField, collections.abc.Mapping)):
|
|
474
|
-
raise ValueError(
|
|
475
|
-
"Schema items must either be fields or compatible "
|
|
476
|
-
"mapping representations."
|
|
477
|
-
)
|
|
478
533
|
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
534
|
+
if isinstance(schema, Sequence):
|
|
535
|
+
# Input is a Sequence (e.g. a list): Process and return a list of SchemaFields
|
|
536
|
+
return [
|
|
537
|
+
field
|
|
538
|
+
if isinstance(field, SchemaField)
|
|
539
|
+
else SchemaField.from_api_repr(field)
|
|
540
|
+
for field in schema
|
|
541
|
+
]
|
|
542
|
+
|
|
543
|
+
else:
|
|
544
|
+
raise TypeError("Schema must be a Sequence (e.g. a list) or None.")
|
|
483
545
|
|
|
484
546
|
|
|
485
547
|
class PolicyTagList(object):
|