google-cloud-bigquery 3.31.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- google/cloud/bigquery/__init__.py +249 -0
- google/cloud/bigquery/_helpers.py +1102 -0
- google/cloud/bigquery/_http.py +47 -0
- google/cloud/bigquery/_job_helpers.py +600 -0
- google/cloud/bigquery/_pandas_helpers.py +1181 -0
- google/cloud/bigquery/_pyarrow_helpers.py +147 -0
- google/cloud/bigquery/_tqdm_helpers.py +137 -0
- google/cloud/bigquery/_versions_helpers.py +264 -0
- google/cloud/bigquery/client.py +4406 -0
- google/cloud/bigquery/dataset.py +1076 -0
- google/cloud/bigquery/dbapi/__init__.py +87 -0
- google/cloud/bigquery/dbapi/_helpers.py +522 -0
- google/cloud/bigquery/dbapi/connection.py +128 -0
- google/cloud/bigquery/dbapi/cursor.py +586 -0
- google/cloud/bigquery/dbapi/exceptions.py +58 -0
- google/cloud/bigquery/dbapi/types.py +96 -0
- google/cloud/bigquery/encryption_configuration.py +84 -0
- google/cloud/bigquery/enums.py +389 -0
- google/cloud/bigquery/exceptions.py +35 -0
- google/cloud/bigquery/external_config.py +1188 -0
- google/cloud/bigquery/format_options.py +147 -0
- google/cloud/bigquery/iam.py +38 -0
- google/cloud/bigquery/job/__init__.py +87 -0
- google/cloud/bigquery/job/base.py +1116 -0
- google/cloud/bigquery/job/copy_.py +282 -0
- google/cloud/bigquery/job/extract.py +271 -0
- google/cloud/bigquery/job/load.py +985 -0
- google/cloud/bigquery/job/query.py +2498 -0
- google/cloud/bigquery/magics/__init__.py +20 -0
- google/cloud/bigquery/magics/line_arg_parser/__init__.py +34 -0
- google/cloud/bigquery/magics/line_arg_parser/exceptions.py +25 -0
- google/cloud/bigquery/magics/line_arg_parser/lexer.py +200 -0
- google/cloud/bigquery/magics/line_arg_parser/parser.py +484 -0
- google/cloud/bigquery/magics/line_arg_parser/visitors.py +159 -0
- google/cloud/bigquery/magics/magics.py +776 -0
- google/cloud/bigquery/model.py +517 -0
- google/cloud/bigquery/opentelemetry_tracing.py +164 -0
- google/cloud/bigquery/py.typed +2 -0
- google/cloud/bigquery/query.py +1344 -0
- google/cloud/bigquery/retry.py +207 -0
- google/cloud/bigquery/routine/__init__.py +33 -0
- google/cloud/bigquery/routine/routine.py +744 -0
- google/cloud/bigquery/schema.py +896 -0
- google/cloud/bigquery/standard_sql.py +389 -0
- google/cloud/bigquery/table.py +3594 -0
- google/cloud/bigquery/version.py +15 -0
- google/cloud/bigquery_v2/__init__.py +56 -0
- google/cloud/bigquery_v2/types/__init__.py +54 -0
- google/cloud/bigquery_v2/types/encryption_config.py +48 -0
- google/cloud/bigquery_v2/types/model.py +1994 -0
- google/cloud/bigquery_v2/types/model_reference.py +57 -0
- google/cloud/bigquery_v2/types/standard_sql.py +156 -0
- google/cloud/bigquery_v2/types/table_reference.py +80 -0
- google_cloud_bigquery-3.31.0.dist-info/LICENSE +202 -0
- google_cloud_bigquery-3.31.0.dist-info/METADATA +203 -0
- google_cloud_bigquery-3.31.0.dist-info/RECORD +58 -0
- google_cloud_bigquery-3.31.0.dist-info/WHEEL +5 -0
- google_cloud_bigquery-3.31.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,985 @@
|
|
|
1
|
+
# Copyright 2015 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Classes for load jobs."""
|
|
16
|
+
|
|
17
|
+
import typing
|
|
18
|
+
from typing import FrozenSet, List, Iterable, Optional
|
|
19
|
+
|
|
20
|
+
from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
|
|
21
|
+
from google.cloud.bigquery.external_config import HivePartitioningOptions
|
|
22
|
+
from google.cloud.bigquery.format_options import ParquetOptions
|
|
23
|
+
from google.cloud.bigquery import _helpers
|
|
24
|
+
from google.cloud.bigquery.schema import SchemaField
|
|
25
|
+
from google.cloud.bigquery.schema import _to_schema_fields
|
|
26
|
+
from google.cloud.bigquery.table import RangePartitioning
|
|
27
|
+
from google.cloud.bigquery.table import TableReference
|
|
28
|
+
from google.cloud.bigquery.table import TimePartitioning
|
|
29
|
+
from google.cloud.bigquery.job.base import _AsyncJob
|
|
30
|
+
from google.cloud.bigquery.job.base import _JobConfig
|
|
31
|
+
from google.cloud.bigquery.job.base import _JobReference
|
|
32
|
+
from google.cloud.bigquery.query import ConnectionProperty
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ColumnNameCharacterMap:
|
|
36
|
+
"""Indicates the character map used for column names.
|
|
37
|
+
|
|
38
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#columnnamecharactermap
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED = "COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED"
|
|
42
|
+
"""Unspecified column name character map."""
|
|
43
|
+
|
|
44
|
+
STRICT = "STRICT"
|
|
45
|
+
"""Support flexible column name and reject invalid column names."""
|
|
46
|
+
|
|
47
|
+
V1 = "V1"
|
|
48
|
+
""" Support alphanumeric + underscore characters and names must start with
|
|
49
|
+
a letter or underscore. Invalid column names will be normalized."""
|
|
50
|
+
|
|
51
|
+
V2 = "V2"
|
|
52
|
+
"""Support flexible column name. Invalid column names will be normalized."""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class LoadJobConfig(_JobConfig):
|
|
56
|
+
"""Configuration options for load jobs.
|
|
57
|
+
|
|
58
|
+
Set properties on the constructed configuration by using the property name
|
|
59
|
+
as the name of a keyword argument. Values which are unset or :data:`None`
|
|
60
|
+
use the BigQuery REST API default values. See the `BigQuery REST API
|
|
61
|
+
reference documentation
|
|
62
|
+
<https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad>`_
|
|
63
|
+
for a list of default values.
|
|
64
|
+
|
|
65
|
+
Required options differ based on the
|
|
66
|
+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` value.
|
|
67
|
+
For example, the BigQuery API's default value for
|
|
68
|
+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` is ``"CSV"``.
|
|
69
|
+
When loading a CSV file, either
|
|
70
|
+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.schema` must be set or
|
|
71
|
+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.autodetect` must be set to
|
|
72
|
+
:data:`True`.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
def __init__(self, **kwargs) -> None:
|
|
76
|
+
super(LoadJobConfig, self).__init__("load", **kwargs)
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def allow_jagged_rows(self):
|
|
80
|
+
"""Optional[bool]: Allow missing trailing optional columns (CSV only).
|
|
81
|
+
|
|
82
|
+
See:
|
|
83
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.allow_jagged_rows
|
|
84
|
+
"""
|
|
85
|
+
return self._get_sub_prop("allowJaggedRows")
|
|
86
|
+
|
|
87
|
+
@allow_jagged_rows.setter
|
|
88
|
+
def allow_jagged_rows(self, value):
|
|
89
|
+
self._set_sub_prop("allowJaggedRows", value)
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def allow_quoted_newlines(self):
|
|
93
|
+
"""Optional[bool]: Allow quoted data containing newline characters (CSV only).
|
|
94
|
+
|
|
95
|
+
See:
|
|
96
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.allow_quoted_newlines
|
|
97
|
+
"""
|
|
98
|
+
return self._get_sub_prop("allowQuotedNewlines")
|
|
99
|
+
|
|
100
|
+
@allow_quoted_newlines.setter
|
|
101
|
+
def allow_quoted_newlines(self, value):
|
|
102
|
+
self._set_sub_prop("allowQuotedNewlines", value)
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def autodetect(self):
|
|
106
|
+
"""Optional[bool]: Automatically infer the schema from a sample of the data.
|
|
107
|
+
|
|
108
|
+
See:
|
|
109
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.autodetect
|
|
110
|
+
"""
|
|
111
|
+
return self._get_sub_prop("autodetect")
|
|
112
|
+
|
|
113
|
+
@autodetect.setter
|
|
114
|
+
def autodetect(self, value):
|
|
115
|
+
self._set_sub_prop("autodetect", value)
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def clustering_fields(self):
|
|
119
|
+
"""Optional[List[str]]: Fields defining clustering for the table
|
|
120
|
+
|
|
121
|
+
(Defaults to :data:`None`).
|
|
122
|
+
|
|
123
|
+
Clustering fields are immutable after table creation.
|
|
124
|
+
|
|
125
|
+
.. note::
|
|
126
|
+
|
|
127
|
+
BigQuery supports clustering for both partitioned and
|
|
128
|
+
non-partitioned tables.
|
|
129
|
+
"""
|
|
130
|
+
prop = self._get_sub_prop("clustering")
|
|
131
|
+
if prop is not None:
|
|
132
|
+
return list(prop.get("fields", ()))
|
|
133
|
+
|
|
134
|
+
@clustering_fields.setter
|
|
135
|
+
def clustering_fields(self, value):
|
|
136
|
+
"""Optional[List[str]]: Fields defining clustering for the table
|
|
137
|
+
|
|
138
|
+
(Defaults to :data:`None`).
|
|
139
|
+
"""
|
|
140
|
+
if value is not None:
|
|
141
|
+
self._set_sub_prop("clustering", {"fields": value})
|
|
142
|
+
else:
|
|
143
|
+
self._del_sub_prop("clustering")
|
|
144
|
+
|
|
145
|
+
@property
|
|
146
|
+
def connection_properties(self) -> List[ConnectionProperty]:
|
|
147
|
+
"""Connection properties.
|
|
148
|
+
|
|
149
|
+
See
|
|
150
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.connection_properties
|
|
151
|
+
|
|
152
|
+
.. versionadded:: 3.7.0
|
|
153
|
+
"""
|
|
154
|
+
resource = self._get_sub_prop("connectionProperties", [])
|
|
155
|
+
return [ConnectionProperty.from_api_repr(prop) for prop in resource]
|
|
156
|
+
|
|
157
|
+
@connection_properties.setter
|
|
158
|
+
def connection_properties(self, value: Iterable[ConnectionProperty]):
|
|
159
|
+
self._set_sub_prop(
|
|
160
|
+
"connectionProperties",
|
|
161
|
+
[prop.to_api_repr() for prop in value],
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
@property
|
|
165
|
+
def create_disposition(self):
|
|
166
|
+
"""Optional[google.cloud.bigquery.job.CreateDisposition]: Specifies behavior
|
|
167
|
+
for creating tables.
|
|
168
|
+
|
|
169
|
+
See:
|
|
170
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition
|
|
171
|
+
"""
|
|
172
|
+
return self._get_sub_prop("createDisposition")
|
|
173
|
+
|
|
174
|
+
@create_disposition.setter
|
|
175
|
+
def create_disposition(self, value):
|
|
176
|
+
self._set_sub_prop("createDisposition", value)
|
|
177
|
+
|
|
178
|
+
@property
|
|
179
|
+
def create_session(self) -> Optional[bool]:
|
|
180
|
+
"""[Preview] If :data:`True`, creates a new session, where
|
|
181
|
+
:attr:`~google.cloud.bigquery.job.LoadJob.session_info` will contain a
|
|
182
|
+
random server generated session id.
|
|
183
|
+
|
|
184
|
+
If :data:`False`, runs load job with an existing ``session_id`` passed in
|
|
185
|
+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.connection_properties`,
|
|
186
|
+
otherwise runs load job in non-session mode.
|
|
187
|
+
|
|
188
|
+
See
|
|
189
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_session
|
|
190
|
+
|
|
191
|
+
.. versionadded:: 3.7.0
|
|
192
|
+
"""
|
|
193
|
+
return self._get_sub_prop("createSession")
|
|
194
|
+
|
|
195
|
+
@create_session.setter
|
|
196
|
+
def create_session(self, value: Optional[bool]):
|
|
197
|
+
self._set_sub_prop("createSession", value)
|
|
198
|
+
|
|
199
|
+
@property
|
|
200
|
+
def decimal_target_types(self) -> Optional[FrozenSet[str]]:
|
|
201
|
+
"""Possible SQL data types to which the source decimal values are converted.
|
|
202
|
+
|
|
203
|
+
See:
|
|
204
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.decimal_target_types
|
|
205
|
+
|
|
206
|
+
.. versionadded:: 2.21.0
|
|
207
|
+
"""
|
|
208
|
+
prop = self._get_sub_prop("decimalTargetTypes")
|
|
209
|
+
if prop is not None:
|
|
210
|
+
prop = frozenset(prop)
|
|
211
|
+
return prop
|
|
212
|
+
|
|
213
|
+
@decimal_target_types.setter
|
|
214
|
+
def decimal_target_types(self, value: Optional[Iterable[str]]):
|
|
215
|
+
if value is not None:
|
|
216
|
+
self._set_sub_prop("decimalTargetTypes", list(value))
|
|
217
|
+
else:
|
|
218
|
+
self._del_sub_prop("decimalTargetTypes")
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def destination_encryption_configuration(self):
|
|
222
|
+
"""Optional[google.cloud.bigquery.encryption_configuration.EncryptionConfiguration]: Custom
|
|
223
|
+
encryption configuration for the destination table.
|
|
224
|
+
|
|
225
|
+
Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
|
|
226
|
+
if using default encryption.
|
|
227
|
+
|
|
228
|
+
See:
|
|
229
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.destination_encryption_configuration
|
|
230
|
+
"""
|
|
231
|
+
prop = self._get_sub_prop("destinationEncryptionConfiguration")
|
|
232
|
+
if prop is not None:
|
|
233
|
+
prop = EncryptionConfiguration.from_api_repr(prop)
|
|
234
|
+
return prop
|
|
235
|
+
|
|
236
|
+
@destination_encryption_configuration.setter
|
|
237
|
+
def destination_encryption_configuration(self, value):
|
|
238
|
+
api_repr = value
|
|
239
|
+
if value is not None:
|
|
240
|
+
api_repr = value.to_api_repr()
|
|
241
|
+
self._set_sub_prop("destinationEncryptionConfiguration", api_repr)
|
|
242
|
+
else:
|
|
243
|
+
self._del_sub_prop("destinationEncryptionConfiguration")
|
|
244
|
+
|
|
245
|
+
@property
|
|
246
|
+
def destination_table_description(self):
|
|
247
|
+
"""Optional[str]: Description of the destination table.
|
|
248
|
+
|
|
249
|
+
See:
|
|
250
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.description
|
|
251
|
+
"""
|
|
252
|
+
prop = self._get_sub_prop("destinationTableProperties")
|
|
253
|
+
if prop is not None:
|
|
254
|
+
return prop["description"]
|
|
255
|
+
|
|
256
|
+
@destination_table_description.setter
|
|
257
|
+
def destination_table_description(self, value):
|
|
258
|
+
keys = [self._job_type, "destinationTableProperties", "description"]
|
|
259
|
+
if value is not None:
|
|
260
|
+
_helpers._set_sub_prop(self._properties, keys, value)
|
|
261
|
+
else:
|
|
262
|
+
_helpers._del_sub_prop(self._properties, keys)
|
|
263
|
+
|
|
264
|
+
@property
|
|
265
|
+
def destination_table_friendly_name(self):
|
|
266
|
+
"""Optional[str]: Name given to destination table.
|
|
267
|
+
|
|
268
|
+
See:
|
|
269
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.friendly_name
|
|
270
|
+
"""
|
|
271
|
+
prop = self._get_sub_prop("destinationTableProperties")
|
|
272
|
+
if prop is not None:
|
|
273
|
+
return prop["friendlyName"]
|
|
274
|
+
|
|
275
|
+
@destination_table_friendly_name.setter
|
|
276
|
+
def destination_table_friendly_name(self, value):
|
|
277
|
+
keys = [self._job_type, "destinationTableProperties", "friendlyName"]
|
|
278
|
+
if value is not None:
|
|
279
|
+
_helpers._set_sub_prop(self._properties, keys, value)
|
|
280
|
+
else:
|
|
281
|
+
_helpers._del_sub_prop(self._properties, keys)
|
|
282
|
+
|
|
283
|
+
@property
|
|
284
|
+
def encoding(self):
|
|
285
|
+
"""Optional[google.cloud.bigquery.job.Encoding]: The character encoding of the
|
|
286
|
+
data.
|
|
287
|
+
|
|
288
|
+
See:
|
|
289
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.encoding
|
|
290
|
+
"""
|
|
291
|
+
return self._get_sub_prop("encoding")
|
|
292
|
+
|
|
293
|
+
@encoding.setter
|
|
294
|
+
def encoding(self, value):
|
|
295
|
+
self._set_sub_prop("encoding", value)
|
|
296
|
+
|
|
297
|
+
@property
|
|
298
|
+
def field_delimiter(self):
|
|
299
|
+
"""Optional[str]: The separator for fields in a CSV file.
|
|
300
|
+
|
|
301
|
+
See:
|
|
302
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.field_delimiter
|
|
303
|
+
"""
|
|
304
|
+
return self._get_sub_prop("fieldDelimiter")
|
|
305
|
+
|
|
306
|
+
@field_delimiter.setter
|
|
307
|
+
def field_delimiter(self, value):
|
|
308
|
+
self._set_sub_prop("fieldDelimiter", value)
|
|
309
|
+
|
|
310
|
+
@property
|
|
311
|
+
def hive_partitioning(self):
|
|
312
|
+
"""Optional[:class:`~.external_config.HivePartitioningOptions`]: [Beta] When set, \
|
|
313
|
+
it configures hive partitioning support.
|
|
314
|
+
|
|
315
|
+
.. note::
|
|
316
|
+
**Experimental**. This feature is experimental and might change or
|
|
317
|
+
have limited support.
|
|
318
|
+
|
|
319
|
+
See:
|
|
320
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.hive_partitioning_options
|
|
321
|
+
"""
|
|
322
|
+
prop = self._get_sub_prop("hivePartitioningOptions")
|
|
323
|
+
if prop is None:
|
|
324
|
+
return None
|
|
325
|
+
return HivePartitioningOptions.from_api_repr(prop)
|
|
326
|
+
|
|
327
|
+
@hive_partitioning.setter
|
|
328
|
+
def hive_partitioning(self, value):
|
|
329
|
+
if value is not None:
|
|
330
|
+
if isinstance(value, HivePartitioningOptions):
|
|
331
|
+
value = value.to_api_repr()
|
|
332
|
+
else:
|
|
333
|
+
raise TypeError("Expected a HivePartitioningOptions instance or None.")
|
|
334
|
+
|
|
335
|
+
self._set_sub_prop("hivePartitioningOptions", value)
|
|
336
|
+
|
|
337
|
+
@property
|
|
338
|
+
def ignore_unknown_values(self):
|
|
339
|
+
"""Optional[bool]: Ignore extra values not represented in the table schema.
|
|
340
|
+
|
|
341
|
+
See:
|
|
342
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.ignore_unknown_values
|
|
343
|
+
"""
|
|
344
|
+
return self._get_sub_prop("ignoreUnknownValues")
|
|
345
|
+
|
|
346
|
+
@ignore_unknown_values.setter
|
|
347
|
+
def ignore_unknown_values(self, value):
|
|
348
|
+
self._set_sub_prop("ignoreUnknownValues", value)
|
|
349
|
+
|
|
350
|
+
@property
|
|
351
|
+
def json_extension(self):
|
|
352
|
+
"""Optional[str]: The extension to use for writing JSON data to BigQuery. Only supports GeoJSON currently.
|
|
353
|
+
|
|
354
|
+
See: https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.json_extension
|
|
355
|
+
|
|
356
|
+
"""
|
|
357
|
+
return self._get_sub_prop("jsonExtension")
|
|
358
|
+
|
|
359
|
+
@json_extension.setter
|
|
360
|
+
def json_extension(self, value):
|
|
361
|
+
self._set_sub_prop("jsonExtension", value)
|
|
362
|
+
|
|
363
|
+
@property
|
|
364
|
+
def max_bad_records(self):
|
|
365
|
+
"""Optional[int]: Number of invalid rows to ignore.
|
|
366
|
+
|
|
367
|
+
See:
|
|
368
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.max_bad_records
|
|
369
|
+
"""
|
|
370
|
+
return _helpers._int_or_none(self._get_sub_prop("maxBadRecords"))
|
|
371
|
+
|
|
372
|
+
@max_bad_records.setter
|
|
373
|
+
def max_bad_records(self, value):
|
|
374
|
+
self._set_sub_prop("maxBadRecords", value)
|
|
375
|
+
|
|
376
|
+
@property
|
|
377
|
+
def null_marker(self):
|
|
378
|
+
"""Optional[str]: Represents a null value (CSV only).
|
|
379
|
+
|
|
380
|
+
See:
|
|
381
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.null_marker
|
|
382
|
+
"""
|
|
383
|
+
return self._get_sub_prop("nullMarker")
|
|
384
|
+
|
|
385
|
+
@null_marker.setter
|
|
386
|
+
def null_marker(self, value):
|
|
387
|
+
self._set_sub_prop("nullMarker", value)
|
|
388
|
+
|
|
389
|
+
@property
|
|
390
|
+
def preserve_ascii_control_characters(self):
|
|
391
|
+
"""Optional[bool]: Preserves the embedded ASCII control characters when sourceFormat is set to CSV.
|
|
392
|
+
|
|
393
|
+
See:
|
|
394
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.preserve_ascii_control_characters
|
|
395
|
+
"""
|
|
396
|
+
return self._get_sub_prop("preserveAsciiControlCharacters")
|
|
397
|
+
|
|
398
|
+
@preserve_ascii_control_characters.setter
|
|
399
|
+
def preserve_ascii_control_characters(self, value):
|
|
400
|
+
self._set_sub_prop("preserveAsciiControlCharacters", bool(value))
|
|
401
|
+
|
|
402
|
+
@property
|
|
403
|
+
def projection_fields(self) -> Optional[List[str]]:
|
|
404
|
+
"""Optional[List[str]]: If
|
|
405
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.source_format` is set to
|
|
406
|
+
"DATASTORE_BACKUP", indicates which entity properties to load into
|
|
407
|
+
BigQuery from a Cloud Datastore backup.
|
|
408
|
+
|
|
409
|
+
Property names are case sensitive and must be top-level properties. If
|
|
410
|
+
no properties are specified, BigQuery loads all properties. If any
|
|
411
|
+
named property isn't found in the Cloud Datastore backup, an invalid
|
|
412
|
+
error is returned in the job result.
|
|
413
|
+
|
|
414
|
+
See:
|
|
415
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.projection_fields
|
|
416
|
+
"""
|
|
417
|
+
return self._get_sub_prop("projectionFields")
|
|
418
|
+
|
|
419
|
+
@projection_fields.setter
|
|
420
|
+
def projection_fields(self, value: Optional[List[str]]):
|
|
421
|
+
self._set_sub_prop("projectionFields", value)
|
|
422
|
+
|
|
423
|
+
@property
|
|
424
|
+
def quote_character(self):
|
|
425
|
+
"""Optional[str]: Character used to quote data sections (CSV only).
|
|
426
|
+
|
|
427
|
+
See:
|
|
428
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.quote
|
|
429
|
+
"""
|
|
430
|
+
return self._get_sub_prop("quote")
|
|
431
|
+
|
|
432
|
+
@quote_character.setter
|
|
433
|
+
def quote_character(self, value):
|
|
434
|
+
self._set_sub_prop("quote", value)
|
|
435
|
+
|
|
436
|
+
@property
|
|
437
|
+
def range_partitioning(self):
|
|
438
|
+
"""Optional[google.cloud.bigquery.table.RangePartitioning]:
|
|
439
|
+
Configures range-based partitioning for destination table.
|
|
440
|
+
|
|
441
|
+
.. note::
|
|
442
|
+
**Beta**. The integer range partitioning feature is in a
|
|
443
|
+
pre-release state and might change or have limited support.
|
|
444
|
+
|
|
445
|
+
Only specify at most one of
|
|
446
|
+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
|
|
447
|
+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
|
|
448
|
+
|
|
449
|
+
Raises:
|
|
450
|
+
ValueError:
|
|
451
|
+
If the value is not
|
|
452
|
+
:class:`~google.cloud.bigquery.table.RangePartitioning` or
|
|
453
|
+
:data:`None`.
|
|
454
|
+
"""
|
|
455
|
+
resource = self._get_sub_prop("rangePartitioning")
|
|
456
|
+
if resource is not None:
|
|
457
|
+
return RangePartitioning(_properties=resource)
|
|
458
|
+
|
|
459
|
+
@range_partitioning.setter
|
|
460
|
+
def range_partitioning(self, value):
|
|
461
|
+
resource = value
|
|
462
|
+
if isinstance(value, RangePartitioning):
|
|
463
|
+
resource = value._properties
|
|
464
|
+
elif value is not None:
|
|
465
|
+
raise ValueError(
|
|
466
|
+
"Expected value to be RangePartitioning or None, got {}.".format(value)
|
|
467
|
+
)
|
|
468
|
+
self._set_sub_prop("rangePartitioning", resource)
|
|
469
|
+
|
|
470
|
+
@property
|
|
471
|
+
def reference_file_schema_uri(self):
|
|
472
|
+
"""Optional[str]:
|
|
473
|
+
When creating an external table, the user can provide a reference file with the
|
|
474
|
+
table schema. This is enabled for the following formats:
|
|
475
|
+
|
|
476
|
+
AVRO, PARQUET, ORC
|
|
477
|
+
"""
|
|
478
|
+
return self._get_sub_prop("referenceFileSchemaUri")
|
|
479
|
+
|
|
480
|
+
@reference_file_schema_uri.setter
|
|
481
|
+
def reference_file_schema_uri(self, value):
|
|
482
|
+
return self._set_sub_prop("referenceFileSchemaUri", value)
|
|
483
|
+
|
|
484
|
+
@property
|
|
485
|
+
def schema(self):
|
|
486
|
+
"""Optional[Sequence[Union[ \
|
|
487
|
+
:class:`~google.cloud.bigquery.schema.SchemaField`, \
|
|
488
|
+
Mapping[str, Any] \
|
|
489
|
+
]]]: Schema of the destination table.
|
|
490
|
+
|
|
491
|
+
See:
|
|
492
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.schema
|
|
493
|
+
"""
|
|
494
|
+
schema = _helpers._get_sub_prop(self._properties, ["load", "schema", "fields"])
|
|
495
|
+
if schema is None:
|
|
496
|
+
return
|
|
497
|
+
return [SchemaField.from_api_repr(field) for field in schema]
|
|
498
|
+
|
|
499
|
+
@schema.setter
|
|
500
|
+
def schema(self, value):
|
|
501
|
+
if value is None:
|
|
502
|
+
self._del_sub_prop("schema")
|
|
503
|
+
return
|
|
504
|
+
|
|
505
|
+
value = _to_schema_fields(value)
|
|
506
|
+
|
|
507
|
+
_helpers._set_sub_prop(
|
|
508
|
+
self._properties,
|
|
509
|
+
["load", "schema", "fields"],
|
|
510
|
+
[field.to_api_repr() for field in value],
|
|
511
|
+
)
|
|
512
|
+
|
|
513
|
+
@property
|
|
514
|
+
def schema_update_options(self):
|
|
515
|
+
"""Optional[List[google.cloud.bigquery.job.SchemaUpdateOption]]: Specifies
|
|
516
|
+
updates to the destination table schema to allow as a side effect of
|
|
517
|
+
the load job.
|
|
518
|
+
"""
|
|
519
|
+
return self._get_sub_prop("schemaUpdateOptions")
|
|
520
|
+
|
|
521
|
+
@schema_update_options.setter
|
|
522
|
+
def schema_update_options(self, values):
|
|
523
|
+
self._set_sub_prop("schemaUpdateOptions", values)
|
|
524
|
+
|
|
525
|
+
@property
|
|
526
|
+
def skip_leading_rows(self):
|
|
527
|
+
"""Optional[int]: Number of rows to skip when reading data (CSV only).
|
|
528
|
+
|
|
529
|
+
See:
|
|
530
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.skip_leading_rows
|
|
531
|
+
"""
|
|
532
|
+
return _helpers._int_or_none(self._get_sub_prop("skipLeadingRows"))
|
|
533
|
+
|
|
534
|
+
@skip_leading_rows.setter
|
|
535
|
+
def skip_leading_rows(self, value):
|
|
536
|
+
self._set_sub_prop("skipLeadingRows", str(value))
|
|
537
|
+
|
|
538
|
+
@property
|
|
539
|
+
def source_format(self):
|
|
540
|
+
"""Optional[google.cloud.bigquery.job.SourceFormat]: File format of the data.
|
|
541
|
+
|
|
542
|
+
See:
|
|
543
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_format
|
|
544
|
+
"""
|
|
545
|
+
return self._get_sub_prop("sourceFormat")
|
|
546
|
+
|
|
547
|
+
@source_format.setter
|
|
548
|
+
def source_format(self, value):
|
|
549
|
+
self._set_sub_prop("sourceFormat", value)
|
|
550
|
+
|
|
551
|
+
@property
|
|
552
|
+
def time_partitioning(self):
|
|
553
|
+
"""Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies time-based
|
|
554
|
+
partitioning for the destination table.
|
|
555
|
+
|
|
556
|
+
Only specify at most one of
|
|
557
|
+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
|
|
558
|
+
:attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
|
|
559
|
+
"""
|
|
560
|
+
prop = self._get_sub_prop("timePartitioning")
|
|
561
|
+
if prop is not None:
|
|
562
|
+
prop = TimePartitioning.from_api_repr(prop)
|
|
563
|
+
return prop
|
|
564
|
+
|
|
565
|
+
@time_partitioning.setter
|
|
566
|
+
def time_partitioning(self, value):
|
|
567
|
+
api_repr = value
|
|
568
|
+
if value is not None:
|
|
569
|
+
api_repr = value.to_api_repr()
|
|
570
|
+
self._set_sub_prop("timePartitioning", api_repr)
|
|
571
|
+
else:
|
|
572
|
+
self._del_sub_prop("timePartitioning")
|
|
573
|
+
|
|
574
|
+
@property
|
|
575
|
+
def use_avro_logical_types(self):
|
|
576
|
+
"""Optional[bool]: For loads of Avro data, governs whether Avro logical types are
|
|
577
|
+
converted to their corresponding BigQuery types (e.g. TIMESTAMP) rather than
|
|
578
|
+
raw types (e.g. INTEGER).
|
|
579
|
+
"""
|
|
580
|
+
return self._get_sub_prop("useAvroLogicalTypes")
|
|
581
|
+
|
|
582
|
+
@use_avro_logical_types.setter
|
|
583
|
+
def use_avro_logical_types(self, value):
|
|
584
|
+
self._set_sub_prop("useAvroLogicalTypes", bool(value))
|
|
585
|
+
|
|
586
|
+
@property
|
|
587
|
+
def write_disposition(self):
|
|
588
|
+
"""Optional[google.cloud.bigquery.job.WriteDisposition]: Action that occurs if
|
|
589
|
+
the destination table already exists.
|
|
590
|
+
|
|
591
|
+
See:
|
|
592
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.write_disposition
|
|
593
|
+
"""
|
|
594
|
+
return self._get_sub_prop("writeDisposition")
|
|
595
|
+
|
|
596
|
+
@write_disposition.setter
|
|
597
|
+
def write_disposition(self, value):
|
|
598
|
+
self._set_sub_prop("writeDisposition", value)
|
|
599
|
+
|
|
600
|
+
@property
|
|
601
|
+
def parquet_options(self):
|
|
602
|
+
"""Optional[google.cloud.bigquery.format_options.ParquetOptions]: Additional
|
|
603
|
+
properties to set if ``sourceFormat`` is set to PARQUET.
|
|
604
|
+
|
|
605
|
+
See:
|
|
606
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.parquet_options
|
|
607
|
+
"""
|
|
608
|
+
prop = self._get_sub_prop("parquetOptions")
|
|
609
|
+
if prop is not None:
|
|
610
|
+
prop = ParquetOptions.from_api_repr(prop)
|
|
611
|
+
return prop
|
|
612
|
+
|
|
613
|
+
@parquet_options.setter
|
|
614
|
+
def parquet_options(self, value):
|
|
615
|
+
if value is not None:
|
|
616
|
+
self._set_sub_prop("parquetOptions", value.to_api_repr())
|
|
617
|
+
else:
|
|
618
|
+
self._del_sub_prop("parquetOptions")
|
|
619
|
+
|
|
620
|
+
@property
|
|
621
|
+
def column_name_character_map(self) -> str:
|
|
622
|
+
"""Optional[google.cloud.bigquery.job.ColumnNameCharacterMap]:
|
|
623
|
+
Character map supported for column names in CSV/Parquet loads. Defaults
|
|
624
|
+
to STRICT and can be overridden by Project Config Service. Using this
|
|
625
|
+
option with unsupported load formats will result in an error.
|
|
626
|
+
|
|
627
|
+
See
|
|
628
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.column_name_character_map
|
|
629
|
+
"""
|
|
630
|
+
return self._get_sub_prop(
|
|
631
|
+
"columnNameCharacterMap",
|
|
632
|
+
ColumnNameCharacterMap.COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED,
|
|
633
|
+
)
|
|
634
|
+
|
|
635
|
+
@column_name_character_map.setter
|
|
636
|
+
def column_name_character_map(self, value: Optional[str]):
|
|
637
|
+
if value is None:
|
|
638
|
+
value = ColumnNameCharacterMap.COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED
|
|
639
|
+
self._set_sub_prop("columnNameCharacterMap", value)
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
class LoadJob(_AsyncJob):
|
|
643
|
+
"""Asynchronous job for loading data into a table.
|
|
644
|
+
|
|
645
|
+
Can load from Google Cloud Storage URIs or from a file.
|
|
646
|
+
|
|
647
|
+
Args:
|
|
648
|
+
job_id (str): the job's ID
|
|
649
|
+
|
|
650
|
+
source_uris (Optional[Sequence[str]]):
|
|
651
|
+
URIs of one or more data files to be loaded. See
|
|
652
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_uris
|
|
653
|
+
for supported URI formats. Pass None for jobs that load from a file.
|
|
654
|
+
|
|
655
|
+
destination (google.cloud.bigquery.table.TableReference): reference to table into which data is to be loaded.
|
|
656
|
+
|
|
657
|
+
client (google.cloud.bigquery.client.Client):
|
|
658
|
+
A client which holds credentials and project configuration
|
|
659
|
+
for the dataset (which requires a project).
|
|
660
|
+
"""
|
|
661
|
+
|
|
662
|
+
_JOB_TYPE = "load"
|
|
663
|
+
_CONFIG_CLASS = LoadJobConfig
|
|
664
|
+
|
|
665
|
+
def __init__(self, job_id, source_uris, destination, client, job_config=None):
|
|
666
|
+
super(LoadJob, self).__init__(job_id, client)
|
|
667
|
+
|
|
668
|
+
if job_config is not None:
|
|
669
|
+
self._properties["configuration"] = job_config._properties
|
|
670
|
+
|
|
671
|
+
if source_uris is not None:
|
|
672
|
+
_helpers._set_sub_prop(
|
|
673
|
+
self._properties, ["configuration", "load", "sourceUris"], source_uris
|
|
674
|
+
)
|
|
675
|
+
|
|
676
|
+
if destination is not None:
|
|
677
|
+
_helpers._set_sub_prop(
|
|
678
|
+
self._properties,
|
|
679
|
+
["configuration", "load", "destinationTable"],
|
|
680
|
+
destination.to_api_repr(),
|
|
681
|
+
)
|
|
682
|
+
|
|
683
|
+
@property
|
|
684
|
+
def configuration(self) -> LoadJobConfig:
|
|
685
|
+
"""The configuration for this load job."""
|
|
686
|
+
return typing.cast(LoadJobConfig, super().configuration)
|
|
687
|
+
|
|
688
|
+
@property
|
|
689
|
+
def destination(self):
|
|
690
|
+
"""google.cloud.bigquery.table.TableReference: table where loaded rows are written
|
|
691
|
+
|
|
692
|
+
See:
|
|
693
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.destination_table
|
|
694
|
+
"""
|
|
695
|
+
dest_config = _helpers._get_sub_prop(
|
|
696
|
+
self._properties, ["configuration", "load", "destinationTable"]
|
|
697
|
+
)
|
|
698
|
+
return TableReference.from_api_repr(dest_config)
|
|
699
|
+
|
|
700
|
+
@property
|
|
701
|
+
def source_uris(self):
|
|
702
|
+
"""Optional[Sequence[str]]: URIs of data files to be loaded. See
|
|
703
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_uris
|
|
704
|
+
for supported URI formats. None for jobs that load from a file.
|
|
705
|
+
"""
|
|
706
|
+
return _helpers._get_sub_prop(
|
|
707
|
+
self._properties, ["configuration", "load", "sourceUris"]
|
|
708
|
+
)
|
|
709
|
+
|
|
710
|
+
@property
|
|
711
|
+
def allow_jagged_rows(self):
|
|
712
|
+
"""See
|
|
713
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.allow_jagged_rows`.
|
|
714
|
+
"""
|
|
715
|
+
return self.configuration.allow_jagged_rows
|
|
716
|
+
|
|
717
|
+
@property
|
|
718
|
+
def allow_quoted_newlines(self):
|
|
719
|
+
"""See
|
|
720
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.allow_quoted_newlines`.
|
|
721
|
+
"""
|
|
722
|
+
return self.configuration.allow_quoted_newlines
|
|
723
|
+
|
|
724
|
+
@property
|
|
725
|
+
def autodetect(self):
|
|
726
|
+
"""See
|
|
727
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.autodetect`.
|
|
728
|
+
"""
|
|
729
|
+
return self.configuration.autodetect
|
|
730
|
+
|
|
731
|
+
@property
|
|
732
|
+
def connection_properties(self) -> List[ConnectionProperty]:
|
|
733
|
+
"""See
|
|
734
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.connection_properties`.
|
|
735
|
+
|
|
736
|
+
.. versionadded:: 3.7.0
|
|
737
|
+
"""
|
|
738
|
+
return self.configuration.connection_properties
|
|
739
|
+
|
|
740
|
+
@property
|
|
741
|
+
def create_disposition(self):
|
|
742
|
+
"""See
|
|
743
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.create_disposition`.
|
|
744
|
+
"""
|
|
745
|
+
return self.configuration.create_disposition
|
|
746
|
+
|
|
747
|
+
@property
|
|
748
|
+
def create_session(self) -> Optional[bool]:
|
|
749
|
+
"""See
|
|
750
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.create_session`.
|
|
751
|
+
|
|
752
|
+
.. versionadded:: 3.7.0
|
|
753
|
+
"""
|
|
754
|
+
return self.configuration.create_session
|
|
755
|
+
|
|
756
|
+
@property
|
|
757
|
+
def encoding(self):
|
|
758
|
+
"""See
|
|
759
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.encoding`.
|
|
760
|
+
"""
|
|
761
|
+
return self.configuration.encoding
|
|
762
|
+
|
|
763
|
+
@property
|
|
764
|
+
def field_delimiter(self):
|
|
765
|
+
"""See
|
|
766
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.field_delimiter`.
|
|
767
|
+
"""
|
|
768
|
+
return self.configuration.field_delimiter
|
|
769
|
+
|
|
770
|
+
@property
|
|
771
|
+
def ignore_unknown_values(self):
|
|
772
|
+
"""See
|
|
773
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.ignore_unknown_values`.
|
|
774
|
+
"""
|
|
775
|
+
return self.configuration.ignore_unknown_values
|
|
776
|
+
|
|
777
|
+
@property
|
|
778
|
+
def max_bad_records(self):
|
|
779
|
+
"""See
|
|
780
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.max_bad_records`.
|
|
781
|
+
"""
|
|
782
|
+
return self.configuration.max_bad_records
|
|
783
|
+
|
|
784
|
+
@property
|
|
785
|
+
def null_marker(self):
|
|
786
|
+
"""See
|
|
787
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.null_marker`.
|
|
788
|
+
"""
|
|
789
|
+
return self.configuration.null_marker
|
|
790
|
+
|
|
791
|
+
@property
|
|
792
|
+
def quote_character(self):
|
|
793
|
+
"""See
|
|
794
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.quote_character`.
|
|
795
|
+
"""
|
|
796
|
+
return self.configuration.quote_character
|
|
797
|
+
|
|
798
|
+
@property
|
|
799
|
+
def reference_file_schema_uri(self):
|
|
800
|
+
"""See:
|
|
801
|
+
attr:`google.cloud.bigquery.job.LoadJobConfig.reference_file_schema_uri`.
|
|
802
|
+
"""
|
|
803
|
+
return self.configuration.reference_file_schema_uri
|
|
804
|
+
|
|
805
|
+
@property
|
|
806
|
+
def skip_leading_rows(self):
|
|
807
|
+
"""See
|
|
808
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.skip_leading_rows`.
|
|
809
|
+
"""
|
|
810
|
+
return self.configuration.skip_leading_rows
|
|
811
|
+
|
|
812
|
+
@property
|
|
813
|
+
def source_format(self):
|
|
814
|
+
"""See
|
|
815
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.source_format`.
|
|
816
|
+
"""
|
|
817
|
+
return self.configuration.source_format
|
|
818
|
+
|
|
819
|
+
@property
|
|
820
|
+
def write_disposition(self):
|
|
821
|
+
"""See
|
|
822
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.write_disposition`.
|
|
823
|
+
"""
|
|
824
|
+
return self.configuration.write_disposition
|
|
825
|
+
|
|
826
|
+
@property
|
|
827
|
+
def schema(self):
|
|
828
|
+
"""See
|
|
829
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.schema`.
|
|
830
|
+
"""
|
|
831
|
+
return self.configuration.schema
|
|
832
|
+
|
|
833
|
+
@property
|
|
834
|
+
def destination_encryption_configuration(self):
|
|
835
|
+
"""google.cloud.bigquery.encryption_configuration.EncryptionConfiguration: Custom
|
|
836
|
+
encryption configuration for the destination table.
|
|
837
|
+
|
|
838
|
+
Custom encryption configuration (e.g., Cloud KMS keys)
|
|
839
|
+
or :data:`None` if using default encryption.
|
|
840
|
+
|
|
841
|
+
See
|
|
842
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.destination_encryption_configuration`.
|
|
843
|
+
"""
|
|
844
|
+
return self.configuration.destination_encryption_configuration
|
|
845
|
+
|
|
846
|
+
@property
|
|
847
|
+
def destination_table_description(self):
|
|
848
|
+
"""Optional[str] name given to destination table.
|
|
849
|
+
|
|
850
|
+
See:
|
|
851
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.description
|
|
852
|
+
"""
|
|
853
|
+
return self.configuration.destination_table_description
|
|
854
|
+
|
|
855
|
+
@property
|
|
856
|
+
def destination_table_friendly_name(self):
|
|
857
|
+
"""Optional[str] name given to destination table.
|
|
858
|
+
|
|
859
|
+
See:
|
|
860
|
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.friendly_name
|
|
861
|
+
"""
|
|
862
|
+
return self.configuration.destination_table_friendly_name
|
|
863
|
+
|
|
864
|
+
@property
|
|
865
|
+
def range_partitioning(self):
|
|
866
|
+
"""See
|
|
867
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
|
|
868
|
+
"""
|
|
869
|
+
return self.configuration.range_partitioning
|
|
870
|
+
|
|
871
|
+
@property
|
|
872
|
+
def time_partitioning(self):
|
|
873
|
+
"""See
|
|
874
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.time_partitioning`.
|
|
875
|
+
"""
|
|
876
|
+
return self.configuration.time_partitioning
|
|
877
|
+
|
|
878
|
+
@property
|
|
879
|
+
def use_avro_logical_types(self):
|
|
880
|
+
"""See
|
|
881
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.use_avro_logical_types`.
|
|
882
|
+
"""
|
|
883
|
+
return self.configuration.use_avro_logical_types
|
|
884
|
+
|
|
885
|
+
@property
|
|
886
|
+
def clustering_fields(self):
|
|
887
|
+
"""See
|
|
888
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.clustering_fields`.
|
|
889
|
+
"""
|
|
890
|
+
return self.configuration.clustering_fields
|
|
891
|
+
|
|
892
|
+
@property
|
|
893
|
+
def schema_update_options(self):
|
|
894
|
+
"""See
|
|
895
|
+
:attr:`google.cloud.bigquery.job.LoadJobConfig.schema_update_options`.
|
|
896
|
+
"""
|
|
897
|
+
return self.configuration.schema_update_options
|
|
898
|
+
|
|
899
|
+
@property
|
|
900
|
+
def input_file_bytes(self):
|
|
901
|
+
"""Count of bytes loaded from source files.
|
|
902
|
+
|
|
903
|
+
Returns:
|
|
904
|
+
Optional[int]: the count (None until set from the server).
|
|
905
|
+
|
|
906
|
+
Raises:
|
|
907
|
+
ValueError: for invalid value types.
|
|
908
|
+
"""
|
|
909
|
+
return _helpers._int_or_none(
|
|
910
|
+
_helpers._get_sub_prop(
|
|
911
|
+
self._properties, ["statistics", "load", "inputFileBytes"]
|
|
912
|
+
)
|
|
913
|
+
)
|
|
914
|
+
|
|
915
|
+
@property
|
|
916
|
+
def input_files(self):
|
|
917
|
+
"""Count of source files.
|
|
918
|
+
|
|
919
|
+
Returns:
|
|
920
|
+
Optional[int]: the count (None until set from the server).
|
|
921
|
+
"""
|
|
922
|
+
return _helpers._int_or_none(
|
|
923
|
+
_helpers._get_sub_prop(
|
|
924
|
+
self._properties, ["statistics", "load", "inputFiles"]
|
|
925
|
+
)
|
|
926
|
+
)
|
|
927
|
+
|
|
928
|
+
@property
|
|
929
|
+
def output_bytes(self):
|
|
930
|
+
"""Count of bytes saved to destination table.
|
|
931
|
+
|
|
932
|
+
Returns:
|
|
933
|
+
Optional[int]: the count (None until set from the server).
|
|
934
|
+
"""
|
|
935
|
+
return _helpers._int_or_none(
|
|
936
|
+
_helpers._get_sub_prop(
|
|
937
|
+
self._properties, ["statistics", "load", "outputBytes"]
|
|
938
|
+
)
|
|
939
|
+
)
|
|
940
|
+
|
|
941
|
+
@property
|
|
942
|
+
def output_rows(self):
|
|
943
|
+
"""Count of rows saved to destination table.
|
|
944
|
+
|
|
945
|
+
Returns:
|
|
946
|
+
Optional[int]: the count (None until set from the server).
|
|
947
|
+
"""
|
|
948
|
+
return _helpers._int_or_none(
|
|
949
|
+
_helpers._get_sub_prop(
|
|
950
|
+
self._properties, ["statistics", "load", "outputRows"]
|
|
951
|
+
)
|
|
952
|
+
)
|
|
953
|
+
|
|
954
|
+
def to_api_repr(self):
|
|
955
|
+
"""Generate a resource for :meth:`_begin`."""
|
|
956
|
+
# Exclude statistics, if set.
|
|
957
|
+
return {
|
|
958
|
+
"jobReference": self._properties["jobReference"],
|
|
959
|
+
"configuration": self._properties["configuration"],
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
@classmethod
|
|
963
|
+
def from_api_repr(cls, resource: dict, client) -> "LoadJob":
|
|
964
|
+
"""Factory: construct a job given its API representation
|
|
965
|
+
|
|
966
|
+
.. note::
|
|
967
|
+
|
|
968
|
+
This method assumes that the project found in the resource matches
|
|
969
|
+
the client's project.
|
|
970
|
+
|
|
971
|
+
Args:
|
|
972
|
+
resource (Dict): dataset job representation returned from the API
|
|
973
|
+
|
|
974
|
+
client (google.cloud.bigquery.client.Client):
|
|
975
|
+
Client which holds credentials and project
|
|
976
|
+
configuration for the dataset.
|
|
977
|
+
|
|
978
|
+
Returns:
|
|
979
|
+
google.cloud.bigquery.job.LoadJob: Job parsed from ``resource``.
|
|
980
|
+
"""
|
|
981
|
+
cls._check_resource_config(resource)
|
|
982
|
+
job_ref = _JobReference._from_api_repr(resource["jobReference"])
|
|
983
|
+
job = cls(job_ref, None, None, client)
|
|
984
|
+
job._set_properties(resource)
|
|
985
|
+
return job
|