google-cloud-bigquery 3.31.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. google/cloud/bigquery/__init__.py +249 -0
  2. google/cloud/bigquery/_helpers.py +1102 -0
  3. google/cloud/bigquery/_http.py +47 -0
  4. google/cloud/bigquery/_job_helpers.py +600 -0
  5. google/cloud/bigquery/_pandas_helpers.py +1181 -0
  6. google/cloud/bigquery/_pyarrow_helpers.py +147 -0
  7. google/cloud/bigquery/_tqdm_helpers.py +137 -0
  8. google/cloud/bigquery/_versions_helpers.py +264 -0
  9. google/cloud/bigquery/client.py +4406 -0
  10. google/cloud/bigquery/dataset.py +1076 -0
  11. google/cloud/bigquery/dbapi/__init__.py +87 -0
  12. google/cloud/bigquery/dbapi/_helpers.py +522 -0
  13. google/cloud/bigquery/dbapi/connection.py +128 -0
  14. google/cloud/bigquery/dbapi/cursor.py +586 -0
  15. google/cloud/bigquery/dbapi/exceptions.py +58 -0
  16. google/cloud/bigquery/dbapi/types.py +96 -0
  17. google/cloud/bigquery/encryption_configuration.py +84 -0
  18. google/cloud/bigquery/enums.py +389 -0
  19. google/cloud/bigquery/exceptions.py +35 -0
  20. google/cloud/bigquery/external_config.py +1188 -0
  21. google/cloud/bigquery/format_options.py +147 -0
  22. google/cloud/bigquery/iam.py +38 -0
  23. google/cloud/bigquery/job/__init__.py +87 -0
  24. google/cloud/bigquery/job/base.py +1116 -0
  25. google/cloud/bigquery/job/copy_.py +282 -0
  26. google/cloud/bigquery/job/extract.py +271 -0
  27. google/cloud/bigquery/job/load.py +985 -0
  28. google/cloud/bigquery/job/query.py +2498 -0
  29. google/cloud/bigquery/magics/__init__.py +20 -0
  30. google/cloud/bigquery/magics/line_arg_parser/__init__.py +34 -0
  31. google/cloud/bigquery/magics/line_arg_parser/exceptions.py +25 -0
  32. google/cloud/bigquery/magics/line_arg_parser/lexer.py +200 -0
  33. google/cloud/bigquery/magics/line_arg_parser/parser.py +484 -0
  34. google/cloud/bigquery/magics/line_arg_parser/visitors.py +159 -0
  35. google/cloud/bigquery/magics/magics.py +776 -0
  36. google/cloud/bigquery/model.py +517 -0
  37. google/cloud/bigquery/opentelemetry_tracing.py +164 -0
  38. google/cloud/bigquery/py.typed +2 -0
  39. google/cloud/bigquery/query.py +1344 -0
  40. google/cloud/bigquery/retry.py +207 -0
  41. google/cloud/bigquery/routine/__init__.py +33 -0
  42. google/cloud/bigquery/routine/routine.py +744 -0
  43. google/cloud/bigquery/schema.py +896 -0
  44. google/cloud/bigquery/standard_sql.py +389 -0
  45. google/cloud/bigquery/table.py +3594 -0
  46. google/cloud/bigquery/version.py +15 -0
  47. google/cloud/bigquery_v2/__init__.py +56 -0
  48. google/cloud/bigquery_v2/types/__init__.py +54 -0
  49. google/cloud/bigquery_v2/types/encryption_config.py +48 -0
  50. google/cloud/bigquery_v2/types/model.py +1994 -0
  51. google/cloud/bigquery_v2/types/model_reference.py +57 -0
  52. google/cloud/bigquery_v2/types/standard_sql.py +156 -0
  53. google/cloud/bigquery_v2/types/table_reference.py +80 -0
  54. google_cloud_bigquery-3.31.0.dist-info/LICENSE +202 -0
  55. google_cloud_bigquery-3.31.0.dist-info/METADATA +203 -0
  56. google_cloud_bigquery-3.31.0.dist-info/RECORD +58 -0
  57. google_cloud_bigquery-3.31.0.dist-info/WHEEL +5 -0
  58. google_cloud_bigquery-3.31.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,985 @@
1
+ # Copyright 2015 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Classes for load jobs."""
16
+
17
+ import typing
18
+ from typing import FrozenSet, List, Iterable, Optional
19
+
20
+ from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
21
+ from google.cloud.bigquery.external_config import HivePartitioningOptions
22
+ from google.cloud.bigquery.format_options import ParquetOptions
23
+ from google.cloud.bigquery import _helpers
24
+ from google.cloud.bigquery.schema import SchemaField
25
+ from google.cloud.bigquery.schema import _to_schema_fields
26
+ from google.cloud.bigquery.table import RangePartitioning
27
+ from google.cloud.bigquery.table import TableReference
28
+ from google.cloud.bigquery.table import TimePartitioning
29
+ from google.cloud.bigquery.job.base import _AsyncJob
30
+ from google.cloud.bigquery.job.base import _JobConfig
31
+ from google.cloud.bigquery.job.base import _JobReference
32
+ from google.cloud.bigquery.query import ConnectionProperty
33
+
34
+
35
+ class ColumnNameCharacterMap:
36
+ """Indicates the character map used for column names.
37
+
38
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#columnnamecharactermap
39
+ """
40
+
41
+ COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED = "COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED"
42
+ """Unspecified column name character map."""
43
+
44
+ STRICT = "STRICT"
45
+ """Support flexible column name and reject invalid column names."""
46
+
47
+ V1 = "V1"
48
+ """ Support alphanumeric + underscore characters and names must start with
49
+ a letter or underscore. Invalid column names will be normalized."""
50
+
51
+ V2 = "V2"
52
+ """Support flexible column name. Invalid column names will be normalized."""
53
+
54
+
55
+ class LoadJobConfig(_JobConfig):
56
+ """Configuration options for load jobs.
57
+
58
+ Set properties on the constructed configuration by using the property name
59
+ as the name of a keyword argument. Values which are unset or :data:`None`
60
+ use the BigQuery REST API default values. See the `BigQuery REST API
61
+ reference documentation
62
+ <https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad>`_
63
+ for a list of default values.
64
+
65
+ Required options differ based on the
66
+ :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` value.
67
+ For example, the BigQuery API's default value for
68
+ :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` is ``"CSV"``.
69
+ When loading a CSV file, either
70
+ :attr:`~google.cloud.bigquery.job.LoadJobConfig.schema` must be set or
71
+ :attr:`~google.cloud.bigquery.job.LoadJobConfig.autodetect` must be set to
72
+ :data:`True`.
73
+ """
74
+
75
+ def __init__(self, **kwargs) -> None:
76
+ super(LoadJobConfig, self).__init__("load", **kwargs)
77
+
78
+ @property
79
+ def allow_jagged_rows(self):
80
+ """Optional[bool]: Allow missing trailing optional columns (CSV only).
81
+
82
+ See:
83
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.allow_jagged_rows
84
+ """
85
+ return self._get_sub_prop("allowJaggedRows")
86
+
87
+ @allow_jagged_rows.setter
88
+ def allow_jagged_rows(self, value):
89
+ self._set_sub_prop("allowJaggedRows", value)
90
+
91
+ @property
92
+ def allow_quoted_newlines(self):
93
+ """Optional[bool]: Allow quoted data containing newline characters (CSV only).
94
+
95
+ See:
96
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.allow_quoted_newlines
97
+ """
98
+ return self._get_sub_prop("allowQuotedNewlines")
99
+
100
+ @allow_quoted_newlines.setter
101
+ def allow_quoted_newlines(self, value):
102
+ self._set_sub_prop("allowQuotedNewlines", value)
103
+
104
+ @property
105
+ def autodetect(self):
106
+ """Optional[bool]: Automatically infer the schema from a sample of the data.
107
+
108
+ See:
109
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.autodetect
110
+ """
111
+ return self._get_sub_prop("autodetect")
112
+
113
+ @autodetect.setter
114
+ def autodetect(self, value):
115
+ self._set_sub_prop("autodetect", value)
116
+
117
+ @property
118
+ def clustering_fields(self):
119
+ """Optional[List[str]]: Fields defining clustering for the table
120
+
121
+ (Defaults to :data:`None`).
122
+
123
+ Clustering fields are immutable after table creation.
124
+
125
+ .. note::
126
+
127
+ BigQuery supports clustering for both partitioned and
128
+ non-partitioned tables.
129
+ """
130
+ prop = self._get_sub_prop("clustering")
131
+ if prop is not None:
132
+ return list(prop.get("fields", ()))
133
+
134
+ @clustering_fields.setter
135
+ def clustering_fields(self, value):
136
+ """Optional[List[str]]: Fields defining clustering for the table
137
+
138
+ (Defaults to :data:`None`).
139
+ """
140
+ if value is not None:
141
+ self._set_sub_prop("clustering", {"fields": value})
142
+ else:
143
+ self._del_sub_prop("clustering")
144
+
145
+ @property
146
+ def connection_properties(self) -> List[ConnectionProperty]:
147
+ """Connection properties.
148
+
149
+ See
150
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.connection_properties
151
+
152
+ .. versionadded:: 3.7.0
153
+ """
154
+ resource = self._get_sub_prop("connectionProperties", [])
155
+ return [ConnectionProperty.from_api_repr(prop) for prop in resource]
156
+
157
+ @connection_properties.setter
158
+ def connection_properties(self, value: Iterable[ConnectionProperty]):
159
+ self._set_sub_prop(
160
+ "connectionProperties",
161
+ [prop.to_api_repr() for prop in value],
162
+ )
163
+
164
+ @property
165
+ def create_disposition(self):
166
+ """Optional[google.cloud.bigquery.job.CreateDisposition]: Specifies behavior
167
+ for creating tables.
168
+
169
+ See:
170
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition
171
+ """
172
+ return self._get_sub_prop("createDisposition")
173
+
174
+ @create_disposition.setter
175
+ def create_disposition(self, value):
176
+ self._set_sub_prop("createDisposition", value)
177
+
178
+ @property
179
+ def create_session(self) -> Optional[bool]:
180
+ """[Preview] If :data:`True`, creates a new session, where
181
+ :attr:`~google.cloud.bigquery.job.LoadJob.session_info` will contain a
182
+ random server generated session id.
183
+
184
+ If :data:`False`, runs load job with an existing ``session_id`` passed in
185
+ :attr:`~google.cloud.bigquery.job.LoadJobConfig.connection_properties`,
186
+ otherwise runs load job in non-session mode.
187
+
188
+ See
189
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_session
190
+
191
+ .. versionadded:: 3.7.0
192
+ """
193
+ return self._get_sub_prop("createSession")
194
+
195
+ @create_session.setter
196
+ def create_session(self, value: Optional[bool]):
197
+ self._set_sub_prop("createSession", value)
198
+
199
+ @property
200
+ def decimal_target_types(self) -> Optional[FrozenSet[str]]:
201
+ """Possible SQL data types to which the source decimal values are converted.
202
+
203
+ See:
204
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.decimal_target_types
205
+
206
+ .. versionadded:: 2.21.0
207
+ """
208
+ prop = self._get_sub_prop("decimalTargetTypes")
209
+ if prop is not None:
210
+ prop = frozenset(prop)
211
+ return prop
212
+
213
+ @decimal_target_types.setter
214
+ def decimal_target_types(self, value: Optional[Iterable[str]]):
215
+ if value is not None:
216
+ self._set_sub_prop("decimalTargetTypes", list(value))
217
+ else:
218
+ self._del_sub_prop("decimalTargetTypes")
219
+
220
+ @property
221
+ def destination_encryption_configuration(self):
222
+ """Optional[google.cloud.bigquery.encryption_configuration.EncryptionConfiguration]: Custom
223
+ encryption configuration for the destination table.
224
+
225
+ Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
226
+ if using default encryption.
227
+
228
+ See:
229
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.destination_encryption_configuration
230
+ """
231
+ prop = self._get_sub_prop("destinationEncryptionConfiguration")
232
+ if prop is not None:
233
+ prop = EncryptionConfiguration.from_api_repr(prop)
234
+ return prop
235
+
236
+ @destination_encryption_configuration.setter
237
+ def destination_encryption_configuration(self, value):
238
+ api_repr = value
239
+ if value is not None:
240
+ api_repr = value.to_api_repr()
241
+ self._set_sub_prop("destinationEncryptionConfiguration", api_repr)
242
+ else:
243
+ self._del_sub_prop("destinationEncryptionConfiguration")
244
+
245
+ @property
246
+ def destination_table_description(self):
247
+ """Optional[str]: Description of the destination table.
248
+
249
+ See:
250
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.description
251
+ """
252
+ prop = self._get_sub_prop("destinationTableProperties")
253
+ if prop is not None:
254
+ return prop["description"]
255
+
256
+ @destination_table_description.setter
257
+ def destination_table_description(self, value):
258
+ keys = [self._job_type, "destinationTableProperties", "description"]
259
+ if value is not None:
260
+ _helpers._set_sub_prop(self._properties, keys, value)
261
+ else:
262
+ _helpers._del_sub_prop(self._properties, keys)
263
+
264
+ @property
265
+ def destination_table_friendly_name(self):
266
+ """Optional[str]: Name given to destination table.
267
+
268
+ See:
269
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.friendly_name
270
+ """
271
+ prop = self._get_sub_prop("destinationTableProperties")
272
+ if prop is not None:
273
+ return prop["friendlyName"]
274
+
275
+ @destination_table_friendly_name.setter
276
+ def destination_table_friendly_name(self, value):
277
+ keys = [self._job_type, "destinationTableProperties", "friendlyName"]
278
+ if value is not None:
279
+ _helpers._set_sub_prop(self._properties, keys, value)
280
+ else:
281
+ _helpers._del_sub_prop(self._properties, keys)
282
+
283
+ @property
284
+ def encoding(self):
285
+ """Optional[google.cloud.bigquery.job.Encoding]: The character encoding of the
286
+ data.
287
+
288
+ See:
289
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.encoding
290
+ """
291
+ return self._get_sub_prop("encoding")
292
+
293
+ @encoding.setter
294
+ def encoding(self, value):
295
+ self._set_sub_prop("encoding", value)
296
+
297
+ @property
298
+ def field_delimiter(self):
299
+ """Optional[str]: The separator for fields in a CSV file.
300
+
301
+ See:
302
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.field_delimiter
303
+ """
304
+ return self._get_sub_prop("fieldDelimiter")
305
+
306
+ @field_delimiter.setter
307
+ def field_delimiter(self, value):
308
+ self._set_sub_prop("fieldDelimiter", value)
309
+
310
+ @property
311
+ def hive_partitioning(self):
312
+ """Optional[:class:`~.external_config.HivePartitioningOptions`]: [Beta] When set, \
313
+ it configures hive partitioning support.
314
+
315
+ .. note::
316
+ **Experimental**. This feature is experimental and might change or
317
+ have limited support.
318
+
319
+ See:
320
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.hive_partitioning_options
321
+ """
322
+ prop = self._get_sub_prop("hivePartitioningOptions")
323
+ if prop is None:
324
+ return None
325
+ return HivePartitioningOptions.from_api_repr(prop)
326
+
327
+ @hive_partitioning.setter
328
+ def hive_partitioning(self, value):
329
+ if value is not None:
330
+ if isinstance(value, HivePartitioningOptions):
331
+ value = value.to_api_repr()
332
+ else:
333
+ raise TypeError("Expected a HivePartitioningOptions instance or None.")
334
+
335
+ self._set_sub_prop("hivePartitioningOptions", value)
336
+
337
+ @property
338
+ def ignore_unknown_values(self):
339
+ """Optional[bool]: Ignore extra values not represented in the table schema.
340
+
341
+ See:
342
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.ignore_unknown_values
343
+ """
344
+ return self._get_sub_prop("ignoreUnknownValues")
345
+
346
+ @ignore_unknown_values.setter
347
+ def ignore_unknown_values(self, value):
348
+ self._set_sub_prop("ignoreUnknownValues", value)
349
+
350
+ @property
351
+ def json_extension(self):
352
+ """Optional[str]: The extension to use for writing JSON data to BigQuery. Only supports GeoJSON currently.
353
+
354
+ See: https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.json_extension
355
+
356
+ """
357
+ return self._get_sub_prop("jsonExtension")
358
+
359
+ @json_extension.setter
360
+ def json_extension(self, value):
361
+ self._set_sub_prop("jsonExtension", value)
362
+
363
+ @property
364
+ def max_bad_records(self):
365
+ """Optional[int]: Number of invalid rows to ignore.
366
+
367
+ See:
368
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.max_bad_records
369
+ """
370
+ return _helpers._int_or_none(self._get_sub_prop("maxBadRecords"))
371
+
372
+ @max_bad_records.setter
373
+ def max_bad_records(self, value):
374
+ self._set_sub_prop("maxBadRecords", value)
375
+
376
+ @property
377
+ def null_marker(self):
378
+ """Optional[str]: Represents a null value (CSV only).
379
+
380
+ See:
381
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.null_marker
382
+ """
383
+ return self._get_sub_prop("nullMarker")
384
+
385
+ @null_marker.setter
386
+ def null_marker(self, value):
387
+ self._set_sub_prop("nullMarker", value)
388
+
389
+ @property
390
+ def preserve_ascii_control_characters(self):
391
+ """Optional[bool]: Preserves the embedded ASCII control characters when sourceFormat is set to CSV.
392
+
393
+ See:
394
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.preserve_ascii_control_characters
395
+ """
396
+ return self._get_sub_prop("preserveAsciiControlCharacters")
397
+
398
+ @preserve_ascii_control_characters.setter
399
+ def preserve_ascii_control_characters(self, value):
400
+ self._set_sub_prop("preserveAsciiControlCharacters", bool(value))
401
+
402
+ @property
403
+ def projection_fields(self) -> Optional[List[str]]:
404
+ """Optional[List[str]]: If
405
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.source_format` is set to
406
+ "DATASTORE_BACKUP", indicates which entity properties to load into
407
+ BigQuery from a Cloud Datastore backup.
408
+
409
+ Property names are case sensitive and must be top-level properties. If
410
+ no properties are specified, BigQuery loads all properties. If any
411
+ named property isn't found in the Cloud Datastore backup, an invalid
412
+ error is returned in the job result.
413
+
414
+ See:
415
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.projection_fields
416
+ """
417
+ return self._get_sub_prop("projectionFields")
418
+
419
+ @projection_fields.setter
420
+ def projection_fields(self, value: Optional[List[str]]):
421
+ self._set_sub_prop("projectionFields", value)
422
+
423
+ @property
424
+ def quote_character(self):
425
+ """Optional[str]: Character used to quote data sections (CSV only).
426
+
427
+ See:
428
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.quote
429
+ """
430
+ return self._get_sub_prop("quote")
431
+
432
+ @quote_character.setter
433
+ def quote_character(self, value):
434
+ self._set_sub_prop("quote", value)
435
+
436
+ @property
437
+ def range_partitioning(self):
438
+ """Optional[google.cloud.bigquery.table.RangePartitioning]:
439
+ Configures range-based partitioning for destination table.
440
+
441
+ .. note::
442
+ **Beta**. The integer range partitioning feature is in a
443
+ pre-release state and might change or have limited support.
444
+
445
+ Only specify at most one of
446
+ :attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
447
+ :attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
448
+
449
+ Raises:
450
+ ValueError:
451
+ If the value is not
452
+ :class:`~google.cloud.bigquery.table.RangePartitioning` or
453
+ :data:`None`.
454
+ """
455
+ resource = self._get_sub_prop("rangePartitioning")
456
+ if resource is not None:
457
+ return RangePartitioning(_properties=resource)
458
+
459
+ @range_partitioning.setter
460
+ def range_partitioning(self, value):
461
+ resource = value
462
+ if isinstance(value, RangePartitioning):
463
+ resource = value._properties
464
+ elif value is not None:
465
+ raise ValueError(
466
+ "Expected value to be RangePartitioning or None, got {}.".format(value)
467
+ )
468
+ self._set_sub_prop("rangePartitioning", resource)
469
+
470
+ @property
471
+ def reference_file_schema_uri(self):
472
+ """Optional[str]:
473
+ When creating an external table, the user can provide a reference file with the
474
+ table schema. This is enabled for the following formats:
475
+
476
+ AVRO, PARQUET, ORC
477
+ """
478
+ return self._get_sub_prop("referenceFileSchemaUri")
479
+
480
+ @reference_file_schema_uri.setter
481
+ def reference_file_schema_uri(self, value):
482
+ return self._set_sub_prop("referenceFileSchemaUri", value)
483
+
484
+ @property
485
+ def schema(self):
486
+ """Optional[Sequence[Union[ \
487
+ :class:`~google.cloud.bigquery.schema.SchemaField`, \
488
+ Mapping[str, Any] \
489
+ ]]]: Schema of the destination table.
490
+
491
+ See:
492
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.schema
493
+ """
494
+ schema = _helpers._get_sub_prop(self._properties, ["load", "schema", "fields"])
495
+ if schema is None:
496
+ return
497
+ return [SchemaField.from_api_repr(field) for field in schema]
498
+
499
+ @schema.setter
500
+ def schema(self, value):
501
+ if value is None:
502
+ self._del_sub_prop("schema")
503
+ return
504
+
505
+ value = _to_schema_fields(value)
506
+
507
+ _helpers._set_sub_prop(
508
+ self._properties,
509
+ ["load", "schema", "fields"],
510
+ [field.to_api_repr() for field in value],
511
+ )
512
+
513
+ @property
514
+ def schema_update_options(self):
515
+ """Optional[List[google.cloud.bigquery.job.SchemaUpdateOption]]: Specifies
516
+ updates to the destination table schema to allow as a side effect of
517
+ the load job.
518
+ """
519
+ return self._get_sub_prop("schemaUpdateOptions")
520
+
521
+ @schema_update_options.setter
522
+ def schema_update_options(self, values):
523
+ self._set_sub_prop("schemaUpdateOptions", values)
524
+
525
+ @property
526
+ def skip_leading_rows(self):
527
+ """Optional[int]: Number of rows to skip when reading data (CSV only).
528
+
529
+ See:
530
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.skip_leading_rows
531
+ """
532
+ return _helpers._int_or_none(self._get_sub_prop("skipLeadingRows"))
533
+
534
+ @skip_leading_rows.setter
535
+ def skip_leading_rows(self, value):
536
+ self._set_sub_prop("skipLeadingRows", str(value))
537
+
538
+ @property
539
+ def source_format(self):
540
+ """Optional[google.cloud.bigquery.job.SourceFormat]: File format of the data.
541
+
542
+ See:
543
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_format
544
+ """
545
+ return self._get_sub_prop("sourceFormat")
546
+
547
+ @source_format.setter
548
+ def source_format(self, value):
549
+ self._set_sub_prop("sourceFormat", value)
550
+
551
+ @property
552
+ def time_partitioning(self):
553
+ """Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies time-based
554
+ partitioning for the destination table.
555
+
556
+ Only specify at most one of
557
+ :attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
558
+ :attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
559
+ """
560
+ prop = self._get_sub_prop("timePartitioning")
561
+ if prop is not None:
562
+ prop = TimePartitioning.from_api_repr(prop)
563
+ return prop
564
+
565
+ @time_partitioning.setter
566
+ def time_partitioning(self, value):
567
+ api_repr = value
568
+ if value is not None:
569
+ api_repr = value.to_api_repr()
570
+ self._set_sub_prop("timePartitioning", api_repr)
571
+ else:
572
+ self._del_sub_prop("timePartitioning")
573
+
574
+ @property
575
+ def use_avro_logical_types(self):
576
+ """Optional[bool]: For loads of Avro data, governs whether Avro logical types are
577
+ converted to their corresponding BigQuery types (e.g. TIMESTAMP) rather than
578
+ raw types (e.g. INTEGER).
579
+ """
580
+ return self._get_sub_prop("useAvroLogicalTypes")
581
+
582
+ @use_avro_logical_types.setter
583
+ def use_avro_logical_types(self, value):
584
+ self._set_sub_prop("useAvroLogicalTypes", bool(value))
585
+
586
+ @property
587
+ def write_disposition(self):
588
+ """Optional[google.cloud.bigquery.job.WriteDisposition]: Action that occurs if
589
+ the destination table already exists.
590
+
591
+ See:
592
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.write_disposition
593
+ """
594
+ return self._get_sub_prop("writeDisposition")
595
+
596
+ @write_disposition.setter
597
+ def write_disposition(self, value):
598
+ self._set_sub_prop("writeDisposition", value)
599
+
600
+ @property
601
+ def parquet_options(self):
602
+ """Optional[google.cloud.bigquery.format_options.ParquetOptions]: Additional
603
+ properties to set if ``sourceFormat`` is set to PARQUET.
604
+
605
+ See:
606
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.parquet_options
607
+ """
608
+ prop = self._get_sub_prop("parquetOptions")
609
+ if prop is not None:
610
+ prop = ParquetOptions.from_api_repr(prop)
611
+ return prop
612
+
613
+ @parquet_options.setter
614
+ def parquet_options(self, value):
615
+ if value is not None:
616
+ self._set_sub_prop("parquetOptions", value.to_api_repr())
617
+ else:
618
+ self._del_sub_prop("parquetOptions")
619
+
620
+ @property
621
+ def column_name_character_map(self) -> str:
622
+ """Optional[google.cloud.bigquery.job.ColumnNameCharacterMap]:
623
+ Character map supported for column names in CSV/Parquet loads. Defaults
624
+ to STRICT and can be overridden by Project Config Service. Using this
625
+ option with unsupported load formats will result in an error.
626
+
627
+ See
628
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.column_name_character_map
629
+ """
630
+ return self._get_sub_prop(
631
+ "columnNameCharacterMap",
632
+ ColumnNameCharacterMap.COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED,
633
+ )
634
+
635
+ @column_name_character_map.setter
636
+ def column_name_character_map(self, value: Optional[str]):
637
+ if value is None:
638
+ value = ColumnNameCharacterMap.COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED
639
+ self._set_sub_prop("columnNameCharacterMap", value)
640
+
641
+
642
+ class LoadJob(_AsyncJob):
643
+ """Asynchronous job for loading data into a table.
644
+
645
+ Can load from Google Cloud Storage URIs or from a file.
646
+
647
+ Args:
648
+ job_id (str): the job's ID
649
+
650
+ source_uris (Optional[Sequence[str]]):
651
+ URIs of one or more data files to be loaded. See
652
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_uris
653
+ for supported URI formats. Pass None for jobs that load from a file.
654
+
655
+ destination (google.cloud.bigquery.table.TableReference): reference to table into which data is to be loaded.
656
+
657
+ client (google.cloud.bigquery.client.Client):
658
+ A client which holds credentials and project configuration
659
+ for the dataset (which requires a project).
660
+ """
661
+
662
+ _JOB_TYPE = "load"
663
+ _CONFIG_CLASS = LoadJobConfig
664
+
665
+ def __init__(self, job_id, source_uris, destination, client, job_config=None):
666
+ super(LoadJob, self).__init__(job_id, client)
667
+
668
+ if job_config is not None:
669
+ self._properties["configuration"] = job_config._properties
670
+
671
+ if source_uris is not None:
672
+ _helpers._set_sub_prop(
673
+ self._properties, ["configuration", "load", "sourceUris"], source_uris
674
+ )
675
+
676
+ if destination is not None:
677
+ _helpers._set_sub_prop(
678
+ self._properties,
679
+ ["configuration", "load", "destinationTable"],
680
+ destination.to_api_repr(),
681
+ )
682
+
683
+ @property
684
+ def configuration(self) -> LoadJobConfig:
685
+ """The configuration for this load job."""
686
+ return typing.cast(LoadJobConfig, super().configuration)
687
+
688
+ @property
689
+ def destination(self):
690
+ """google.cloud.bigquery.table.TableReference: table where loaded rows are written
691
+
692
+ See:
693
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.destination_table
694
+ """
695
+ dest_config = _helpers._get_sub_prop(
696
+ self._properties, ["configuration", "load", "destinationTable"]
697
+ )
698
+ return TableReference.from_api_repr(dest_config)
699
+
700
+ @property
701
+ def source_uris(self):
702
+ """Optional[Sequence[str]]: URIs of data files to be loaded. See
703
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_uris
704
+ for supported URI formats. None for jobs that load from a file.
705
+ """
706
+ return _helpers._get_sub_prop(
707
+ self._properties, ["configuration", "load", "sourceUris"]
708
+ )
709
+
710
+ @property
711
+ def allow_jagged_rows(self):
712
+ """See
713
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.allow_jagged_rows`.
714
+ """
715
+ return self.configuration.allow_jagged_rows
716
+
717
+ @property
718
+ def allow_quoted_newlines(self):
719
+ """See
720
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.allow_quoted_newlines`.
721
+ """
722
+ return self.configuration.allow_quoted_newlines
723
+
724
+ @property
725
+ def autodetect(self):
726
+ """See
727
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.autodetect`.
728
+ """
729
+ return self.configuration.autodetect
730
+
731
+ @property
732
+ def connection_properties(self) -> List[ConnectionProperty]:
733
+ """See
734
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.connection_properties`.
735
+
736
+ .. versionadded:: 3.7.0
737
+ """
738
+ return self.configuration.connection_properties
739
+
740
+ @property
741
+ def create_disposition(self):
742
+ """See
743
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.create_disposition`.
744
+ """
745
+ return self.configuration.create_disposition
746
+
747
+ @property
748
+ def create_session(self) -> Optional[bool]:
749
+ """See
750
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.create_session`.
751
+
752
+ .. versionadded:: 3.7.0
753
+ """
754
+ return self.configuration.create_session
755
+
756
+ @property
757
+ def encoding(self):
758
+ """See
759
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.encoding`.
760
+ """
761
+ return self.configuration.encoding
762
+
763
+ @property
764
+ def field_delimiter(self):
765
+ """See
766
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.field_delimiter`.
767
+ """
768
+ return self.configuration.field_delimiter
769
+
770
+ @property
771
+ def ignore_unknown_values(self):
772
+ """See
773
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.ignore_unknown_values`.
774
+ """
775
+ return self.configuration.ignore_unknown_values
776
+
777
+ @property
778
+ def max_bad_records(self):
779
+ """See
780
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.max_bad_records`.
781
+ """
782
+ return self.configuration.max_bad_records
783
+
784
+ @property
785
+ def null_marker(self):
786
+ """See
787
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.null_marker`.
788
+ """
789
+ return self.configuration.null_marker
790
+
791
+ @property
792
+ def quote_character(self):
793
+ """See
794
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.quote_character`.
795
+ """
796
+ return self.configuration.quote_character
797
+
798
+ @property
799
+ def reference_file_schema_uri(self):
800
+ """See:
801
+ attr:`google.cloud.bigquery.job.LoadJobConfig.reference_file_schema_uri`.
802
+ """
803
+ return self.configuration.reference_file_schema_uri
804
+
805
+ @property
806
+ def skip_leading_rows(self):
807
+ """See
808
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.skip_leading_rows`.
809
+ """
810
+ return self.configuration.skip_leading_rows
811
+
812
+ @property
813
+ def source_format(self):
814
+ """See
815
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.source_format`.
816
+ """
817
+ return self.configuration.source_format
818
+
819
+ @property
820
+ def write_disposition(self):
821
+ """See
822
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.write_disposition`.
823
+ """
824
+ return self.configuration.write_disposition
825
+
826
+ @property
827
+ def schema(self):
828
+ """See
829
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.schema`.
830
+ """
831
+ return self.configuration.schema
832
+
833
+ @property
834
+ def destination_encryption_configuration(self):
835
+ """google.cloud.bigquery.encryption_configuration.EncryptionConfiguration: Custom
836
+ encryption configuration for the destination table.
837
+
838
+ Custom encryption configuration (e.g., Cloud KMS keys)
839
+ or :data:`None` if using default encryption.
840
+
841
+ See
842
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.destination_encryption_configuration`.
843
+ """
844
+ return self.configuration.destination_encryption_configuration
845
+
846
+ @property
847
+ def destination_table_description(self):
848
+ """Optional[str] name given to destination table.
849
+
850
+ See:
851
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.description
852
+ """
853
+ return self.configuration.destination_table_description
854
+
855
+ @property
856
+ def destination_table_friendly_name(self):
857
+ """Optional[str] name given to destination table.
858
+
859
+ See:
860
+ https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.friendly_name
861
+ """
862
+ return self.configuration.destination_table_friendly_name
863
+
864
+ @property
865
+ def range_partitioning(self):
866
+ """See
867
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
868
+ """
869
+ return self.configuration.range_partitioning
870
+
871
+ @property
872
+ def time_partitioning(self):
873
+ """See
874
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.time_partitioning`.
875
+ """
876
+ return self.configuration.time_partitioning
877
+
878
+ @property
879
+ def use_avro_logical_types(self):
880
+ """See
881
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.use_avro_logical_types`.
882
+ """
883
+ return self.configuration.use_avro_logical_types
884
+
885
+ @property
886
+ def clustering_fields(self):
887
+ """See
888
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.clustering_fields`.
889
+ """
890
+ return self.configuration.clustering_fields
891
+
892
+ @property
893
+ def schema_update_options(self):
894
+ """See
895
+ :attr:`google.cloud.bigquery.job.LoadJobConfig.schema_update_options`.
896
+ """
897
+ return self.configuration.schema_update_options
898
+
899
+ @property
900
+ def input_file_bytes(self):
901
+ """Count of bytes loaded from source files.
902
+
903
+ Returns:
904
+ Optional[int]: the count (None until set from the server).
905
+
906
+ Raises:
907
+ ValueError: for invalid value types.
908
+ """
909
+ return _helpers._int_or_none(
910
+ _helpers._get_sub_prop(
911
+ self._properties, ["statistics", "load", "inputFileBytes"]
912
+ )
913
+ )
914
+
915
+ @property
916
+ def input_files(self):
917
+ """Count of source files.
918
+
919
+ Returns:
920
+ Optional[int]: the count (None until set from the server).
921
+ """
922
+ return _helpers._int_or_none(
923
+ _helpers._get_sub_prop(
924
+ self._properties, ["statistics", "load", "inputFiles"]
925
+ )
926
+ )
927
+
928
+ @property
929
+ def output_bytes(self):
930
+ """Count of bytes saved to destination table.
931
+
932
+ Returns:
933
+ Optional[int]: the count (None until set from the server).
934
+ """
935
+ return _helpers._int_or_none(
936
+ _helpers._get_sub_prop(
937
+ self._properties, ["statistics", "load", "outputBytes"]
938
+ )
939
+ )
940
+
941
+ @property
942
+ def output_rows(self):
943
+ """Count of rows saved to destination table.
944
+
945
+ Returns:
946
+ Optional[int]: the count (None until set from the server).
947
+ """
948
+ return _helpers._int_or_none(
949
+ _helpers._get_sub_prop(
950
+ self._properties, ["statistics", "load", "outputRows"]
951
+ )
952
+ )
953
+
954
+ def to_api_repr(self):
955
+ """Generate a resource for :meth:`_begin`."""
956
+ # Exclude statistics, if set.
957
+ return {
958
+ "jobReference": self._properties["jobReference"],
959
+ "configuration": self._properties["configuration"],
960
+ }
961
+
962
+ @classmethod
963
+ def from_api_repr(cls, resource: dict, client) -> "LoadJob":
964
+ """Factory: construct a job given its API representation
965
+
966
+ .. note::
967
+
968
+ This method assumes that the project found in the resource matches
969
+ the client's project.
970
+
971
+ Args:
972
+ resource (Dict): dataset job representation returned from the API
973
+
974
+ client (google.cloud.bigquery.client.Client):
975
+ Client which holds credentials and project
976
+ configuration for the dataset.
977
+
978
+ Returns:
979
+ google.cloud.bigquery.job.LoadJob: Job parsed from ``resource``.
980
+ """
981
+ cls._check_resource_config(resource)
982
+ job_ref = _JobReference._from_api_repr(resource["jobReference"])
983
+ job = cls(job_ref, None, None, client)
984
+ job._set_properties(resource)
985
+ return job