google-cloud-bigtable 2.30.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. google/cloud/bigtable/__init__.py +25 -0
  2. google/cloud/bigtable/app_profile.py +377 -0
  3. google/cloud/bigtable/backup.py +490 -0
  4. google/cloud/bigtable/batcher.py +414 -0
  5. google/cloud/bigtable/client.py +475 -0
  6. google/cloud/bigtable/cluster.py +541 -0
  7. google/cloud/bigtable/column_family.py +362 -0
  8. google/cloud/bigtable/data/__init__.py +103 -0
  9. google/cloud/bigtable/data/_async/__init__.py +25 -0
  10. google/cloud/bigtable/data/_async/_mutate_rows.py +226 -0
  11. google/cloud/bigtable/data/_async/_read_rows.py +363 -0
  12. google/cloud/bigtable/data/_async/client.py +1522 -0
  13. google/cloud/bigtable/data/_async/mutations_batcher.py +534 -0
  14. google/cloud/bigtable/data/_cross_sync/__init__.py +20 -0
  15. google/cloud/bigtable/data/_cross_sync/_decorators.py +441 -0
  16. google/cloud/bigtable/data/_cross_sync/_mapping_meta.py +64 -0
  17. google/cloud/bigtable/data/_cross_sync/cross_sync.py +334 -0
  18. google/cloud/bigtable/data/_helpers.py +249 -0
  19. google/cloud/bigtable/data/_sync_autogen/_mutate_rows.py +182 -0
  20. google/cloud/bigtable/data/_sync_autogen/_read_rows.py +304 -0
  21. google/cloud/bigtable/data/_sync_autogen/client.py +1293 -0
  22. google/cloud/bigtable/data/_sync_autogen/mutations_batcher.py +449 -0
  23. google/cloud/bigtable/data/exceptions.py +340 -0
  24. google/cloud/bigtable/data/execute_query/__init__.py +43 -0
  25. google/cloud/bigtable/data/execute_query/_async/__init__.py +13 -0
  26. google/cloud/bigtable/data/execute_query/_async/execute_query_iterator.py +291 -0
  27. google/cloud/bigtable/data/execute_query/_byte_cursor.py +123 -0
  28. google/cloud/bigtable/data/execute_query/_checksum.py +43 -0
  29. google/cloud/bigtable/data/execute_query/_parameters_formatting.py +155 -0
  30. google/cloud/bigtable/data/execute_query/_query_result_parsing_utils.py +134 -0
  31. google/cloud/bigtable/data/execute_query/_reader.py +122 -0
  32. google/cloud/bigtable/data/execute_query/_sync_autogen/execute_query_iterator.py +239 -0
  33. google/cloud/bigtable/data/execute_query/metadata.py +399 -0
  34. google/cloud/bigtable/data/execute_query/values.py +123 -0
  35. google/cloud/bigtable/data/mutations.py +380 -0
  36. google/cloud/bigtable/data/read_modify_write_rules.py +112 -0
  37. google/cloud/bigtable/data/read_rows_query.py +536 -0
  38. google/cloud/bigtable/data/row.py +535 -0
  39. google/cloud/bigtable/data/row_filters.py +968 -0
  40. google/cloud/bigtable/encryption_info.py +64 -0
  41. google/cloud/bigtable/enums.py +223 -0
  42. google/cloud/bigtable/error.py +64 -0
  43. google/cloud/bigtable/gapic_version.py +16 -0
  44. google/cloud/bigtable/helpers.py +31 -0
  45. google/cloud/bigtable/instance.py +789 -0
  46. google/cloud/bigtable/policy.py +255 -0
  47. google/cloud/bigtable/row.py +1267 -0
  48. google/cloud/bigtable/row_data.py +380 -0
  49. google/cloud/bigtable/row_filters.py +838 -0
  50. google/cloud/bigtable/row_merger.py +250 -0
  51. google/cloud/bigtable/row_set.py +213 -0
  52. google/cloud/bigtable/table.py +1409 -0
  53. google/cloud/bigtable_admin/__init__.py +410 -0
  54. google/cloud/bigtable_admin/gapic_version.py +16 -0
  55. google/cloud/bigtable_admin/py.typed +2 -0
  56. google/cloud/bigtable_admin_v2/__init__.py +250 -0
  57. google/cloud/bigtable_admin_v2/gapic_metadata.json +962 -0
  58. google/cloud/bigtable_admin_v2/gapic_version.py +16 -0
  59. google/cloud/bigtable_admin_v2/py.typed +2 -0
  60. google/cloud/bigtable_admin_v2/services/__init__.py +15 -0
  61. google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/__init__.py +22 -0
  62. google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py +3465 -0
  63. google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py +3944 -0
  64. google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/pagers.py +681 -0
  65. google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/__init__.py +40 -0
  66. google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/base.py +751 -0
  67. google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/grpc.py +1249 -0
  68. google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/grpc_asyncio.py +1579 -0
  69. google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/rest.py +6820 -0
  70. google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/rest_base.py +1746 -0
  71. google/cloud/bigtable_admin_v2/services/bigtable_table_admin/__init__.py +22 -0
  72. google/cloud/bigtable_admin_v2/services/bigtable_table_admin/async_client.py +3472 -0
  73. google/cloud/bigtable_admin_v2/services/bigtable_table_admin/client.py +3949 -0
  74. google/cloud/bigtable_admin_v2/services/bigtable_table_admin/pagers.py +669 -0
  75. google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/__init__.py +40 -0
  76. google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/base.py +708 -0
  77. google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/grpc.py +1257 -0
  78. google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/grpc_asyncio.py +1549 -0
  79. google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/rest.py +6609 -0
  80. google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/rest_base.py +1714 -0
  81. google/cloud/bigtable_admin_v2/types/__init__.py +248 -0
  82. google/cloud/bigtable_admin_v2/types/bigtable_instance_admin.py +1364 -0
  83. google/cloud/bigtable_admin_v2/types/bigtable_table_admin.py +1715 -0
  84. google/cloud/bigtable_admin_v2/types/common.py +81 -0
  85. google/cloud/bigtable_admin_v2/types/instance.py +819 -0
  86. google/cloud/bigtable_admin_v2/types/table.py +1028 -0
  87. google/cloud/bigtable_admin_v2/types/types.py +776 -0
  88. google/cloud/bigtable_v2/__init__.py +136 -0
  89. google/cloud/bigtable_v2/gapic_metadata.json +193 -0
  90. google/cloud/bigtable_v2/gapic_version.py +16 -0
  91. google/cloud/bigtable_v2/py.typed +2 -0
  92. google/cloud/bigtable_v2/services/__init__.py +15 -0
  93. google/cloud/bigtable_v2/services/bigtable/__init__.py +22 -0
  94. google/cloud/bigtable_v2/services/bigtable/async_client.py +1741 -0
  95. google/cloud/bigtable_v2/services/bigtable/client.py +2194 -0
  96. google/cloud/bigtable_v2/services/bigtable/transports/__init__.py +38 -0
  97. google/cloud/bigtable_v2/services/bigtable/transports/base.py +343 -0
  98. google/cloud/bigtable_v2/services/bigtable/transports/grpc.py +660 -0
  99. google/cloud/bigtable_v2/services/bigtable/transports/grpc_asyncio.py +762 -0
  100. google/cloud/bigtable_v2/services/bigtable/transports/rest.py +2489 -0
  101. google/cloud/bigtable_v2/services/bigtable/transports/rest_base.py +711 -0
  102. google/cloud/bigtable_v2/types/__init__.py +138 -0
  103. google/cloud/bigtable_v2/types/bigtable.py +1531 -0
  104. google/cloud/bigtable_v2/types/data.py +1612 -0
  105. google/cloud/bigtable_v2/types/feature_flags.py +119 -0
  106. google/cloud/bigtable_v2/types/request_stats.py +171 -0
  107. google/cloud/bigtable_v2/types/response_params.py +64 -0
  108. google/cloud/bigtable_v2/types/types.py +579 -0
  109. google_cloud_bigtable-2.30.0.data/scripts/fixup_bigtable_admin_v2_keywords.py +233 -0
  110. google_cloud_bigtable-2.30.0.data/scripts/fixup_bigtable_v2_keywords.py +186 -0
  111. google_cloud_bigtable-2.30.0.dist-info/LICENSE +202 -0
  112. google_cloud_bigtable-2.30.0.dist-info/METADATA +154 -0
  113. google_cloud_bigtable-2.30.0.dist-info/RECORD +115 -0
  114. google_cloud_bigtable-2.30.0.dist-info/WHEEL +5 -0
  115. google_cloud_bigtable-2.30.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1293 @@
1
+ # Copyright 2024 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+
17
+ # This file is automatically generated by CrossSync. Do not edit manually.
18
+
19
+ from __future__ import annotations
20
+ from typing import cast, Any, Optional, Set, Sequence, TYPE_CHECKING
21
+ import time
22
+ import warnings
23
+ import random
24
+ import os
25
+ import concurrent.futures
26
+ from functools import partial
27
+ from grpc import Channel
28
+ from google.cloud.bigtable.data.execute_query.values import ExecuteQueryValueType
29
+ from google.cloud.bigtable.data.execute_query.metadata import (
30
+ SqlType,
31
+ _pb_metadata_to_metadata_types,
32
+ )
33
+ from google.cloud.bigtable.data.execute_query._parameters_formatting import (
34
+ _format_execute_query_params,
35
+ _to_param_types,
36
+ )
37
+ from google.cloud.bigtable_v2.services.bigtable.transports.base import (
38
+ DEFAULT_CLIENT_INFO,
39
+ )
40
+ from google.cloud.bigtable_v2.types.bigtable import PingAndWarmRequest
41
+ from google.cloud.client import ClientWithProject
42
+ from google.cloud.environment_vars import BIGTABLE_EMULATOR
43
+ from google.api_core import retry as retries
44
+ from google.api_core.exceptions import DeadlineExceeded
45
+ from google.api_core.exceptions import ServiceUnavailable
46
+ from google.api_core.exceptions import Aborted
47
+ import google.auth.credentials
48
+ import google.auth._default
49
+ from google.api_core import client_options as client_options_lib
50
+ from google.cloud.bigtable.client import _DEFAULT_BIGTABLE_EMULATOR_CLIENT
51
+ from google.cloud.bigtable.data.row import Row
52
+ from google.cloud.bigtable.data.read_rows_query import ReadRowsQuery
53
+ from google.cloud.bigtable.data.exceptions import FailedQueryShardError
54
+ from google.cloud.bigtable.data.exceptions import ShardedReadRowsExceptionGroup
55
+ from google.cloud.bigtable.data._helpers import TABLE_DEFAULT, _align_timeouts
56
+ from google.cloud.bigtable.data._helpers import _WarmedInstanceKey
57
+ from google.cloud.bigtable.data._helpers import _CONCURRENCY_LIMIT
58
+ from google.cloud.bigtable.data._helpers import _retry_exception_factory
59
+ from google.cloud.bigtable.data._helpers import _validate_timeouts
60
+ from google.cloud.bigtable.data._helpers import _get_error_type
61
+ from google.cloud.bigtable.data._helpers import _get_retryable_errors
62
+ from google.cloud.bigtable.data._helpers import _get_timeouts
63
+ from google.cloud.bigtable.data._helpers import _attempt_timeout_generator
64
+ from google.cloud.bigtable.data.mutations import Mutation, RowMutationEntry
65
+ from google.cloud.bigtable.data.read_modify_write_rules import ReadModifyWriteRule
66
+ from google.cloud.bigtable.data.row_filters import RowFilter
67
+ from google.cloud.bigtable.data.row_filters import StripValueTransformerFilter
68
+ from google.cloud.bigtable.data.row_filters import CellsRowLimitFilter
69
+ from google.cloud.bigtable.data.row_filters import RowFilterChain
70
+ from google.cloud.bigtable.data._cross_sync import CrossSync
71
+ from typing import Iterable
72
+ from grpc import insecure_channel
73
+ from grpc import intercept_channel
74
+ from google.cloud.bigtable_v2.services.bigtable.transports import (
75
+ BigtableGrpcTransport as TransportType,
76
+ )
77
+ from google.cloud.bigtable.data._sync_autogen.mutations_batcher import _MB_SIZE
78
+
79
+ if TYPE_CHECKING:
80
+ from google.cloud.bigtable.data._helpers import RowKeySamples
81
+ from google.cloud.bigtable.data._helpers import ShardedQuery
82
+ from google.cloud.bigtable.data._sync_autogen.mutations_batcher import (
83
+ MutationsBatcher,
84
+ )
85
+ from google.cloud.bigtable.data.execute_query._sync_autogen.execute_query_iterator import (
86
+ ExecuteQueryIterator,
87
+ )
88
+
89
+
90
+ @CrossSync._Sync_Impl.add_mapping_decorator("DataClient")
91
+ class BigtableDataClient(ClientWithProject):
92
+ def __init__(
93
+ self,
94
+ *,
95
+ project: str | None = None,
96
+ credentials: google.auth.credentials.Credentials | None = None,
97
+ client_options: dict[str, Any]
98
+ | "google.api_core.client_options.ClientOptions"
99
+ | None = None,
100
+ **kwargs,
101
+ ):
102
+ """Create a client instance for the Bigtable Data API
103
+
104
+
105
+
106
+ Args:
107
+ project: the project which the client acts on behalf of.
108
+ If not passed, falls back to the default inferred
109
+ from the environment.
110
+ credentials:
111
+ Thehe OAuth2 Credentials to use for this
112
+ client. If not passed (and if no ``_http`` object is
113
+ passed), falls back to the default inferred from the
114
+ environment.
115
+ client_options:
116
+ Client options used to set user options
117
+ on the client. API Endpoint should be set through client_options.
118
+ Raises:
119
+ """
120
+ if "pool_size" in kwargs:
121
+ warnings.warn("pool_size no longer supported")
122
+ self.client_info = DEFAULT_CLIENT_INFO
123
+ self.client_info.client_library_version = self._client_version()
124
+ if type(client_options) is dict:
125
+ client_options = client_options_lib.from_dict(client_options)
126
+ client_options = cast(
127
+ Optional[client_options_lib.ClientOptions], client_options
128
+ )
129
+ custom_channel = None
130
+ self._emulator_host = os.getenv(BIGTABLE_EMULATOR)
131
+ if self._emulator_host is not None:
132
+ warnings.warn(
133
+ "Connecting to Bigtable emulator at {}".format(self._emulator_host),
134
+ RuntimeWarning,
135
+ stacklevel=2,
136
+ )
137
+ custom_channel = insecure_channel(self._emulator_host)
138
+ if credentials is None:
139
+ credentials = google.auth.credentials.AnonymousCredentials()
140
+ if project is None:
141
+ project = _DEFAULT_BIGTABLE_EMULATOR_CLIENT
142
+ ClientWithProject.__init__(
143
+ self,
144
+ credentials=credentials,
145
+ project=project,
146
+ client_options=client_options,
147
+ )
148
+ self._gapic_client = CrossSync._Sync_Impl.GapicClient(
149
+ credentials=credentials,
150
+ client_options=client_options,
151
+ client_info=self.client_info,
152
+ transport=lambda *args, **kwargs: TransportType(
153
+ *args, **kwargs, channel=custom_channel
154
+ ),
155
+ )
156
+ self._is_closed = CrossSync._Sync_Impl.Event()
157
+ self.transport = cast(TransportType, self._gapic_client.transport)
158
+ self._active_instances: Set[_WarmedInstanceKey] = set()
159
+ self._instance_owners: dict[_WarmedInstanceKey, Set[int]] = {}
160
+ self._channel_init_time = time.monotonic()
161
+ self._channel_refresh_task: CrossSync._Sync_Impl.Task[None] | None = None
162
+ self._executor = (
163
+ concurrent.futures.ThreadPoolExecutor()
164
+ if not CrossSync._Sync_Impl.is_async
165
+ else None
166
+ )
167
+ if self._emulator_host is None:
168
+ try:
169
+ self._start_background_channel_refresh()
170
+ except RuntimeError:
171
+ warnings.warn(
172
+ f"{self.__class__.__name__} should be started in an asyncio event loop. Channel refresh will not be started",
173
+ RuntimeWarning,
174
+ stacklevel=2,
175
+ )
176
+
177
+ @staticmethod
178
+ def _client_version() -> str:
179
+ """Helper function to return the client version string for this client"""
180
+ version_str = f"{google.cloud.bigtable.__version__}-data"
181
+ return version_str
182
+
183
+ def _start_background_channel_refresh(self) -> None:
184
+ """Starts a background task to ping and warm grpc channel
185
+
186
+ Raises:
187
+ None"""
188
+ if (
189
+ not self._channel_refresh_task
190
+ and (not self._emulator_host)
191
+ and (not self._is_closed.is_set())
192
+ ):
193
+ CrossSync._Sync_Impl.verify_async_event_loop()
194
+ self._channel_refresh_task = CrossSync._Sync_Impl.create_task(
195
+ self._manage_channel,
196
+ sync_executor=self._executor,
197
+ task_name=f"{self.__class__.__name__} channel refresh",
198
+ )
199
+
200
+ def close(self, timeout: float | None = 2.0):
201
+ """Cancel all background tasks"""
202
+ self._is_closed.set()
203
+ if self._channel_refresh_task is not None:
204
+ self._channel_refresh_task.cancel()
205
+ CrossSync._Sync_Impl.wait([self._channel_refresh_task], timeout=timeout)
206
+ self.transport.close()
207
+ if self._executor:
208
+ self._executor.shutdown(wait=False)
209
+ self._channel_refresh_task = None
210
+
211
+ def _ping_and_warm_instances(
212
+ self,
213
+ instance_key: _WarmedInstanceKey | None = None,
214
+ channel: Channel | None = None,
215
+ ) -> list[BaseException | None]:
216
+ """Prepares the backend for requests on a channel
217
+
218
+ Pings each Bigtable instance registered in `_active_instances` on the client
219
+
220
+ Args:
221
+ instance_key: if provided, only warm the instance associated with the key
222
+ channel: grpc channel to warm. If none, warms `self.transport.grpc_channel`
223
+ Returns:
224
+ list[BaseException | None]: sequence of results or exceptions from the ping requests
225
+ """
226
+ channel = channel or self.transport.grpc_channel
227
+ instance_list = (
228
+ [instance_key] if instance_key is not None else self._active_instances
229
+ )
230
+ ping_rpc = channel.unary_unary(
231
+ "/google.bigtable.v2.Bigtable/PingAndWarm",
232
+ request_serializer=PingAndWarmRequest.serialize,
233
+ )
234
+ partial_list = [
235
+ partial(
236
+ ping_rpc,
237
+ request={"name": instance_name, "app_profile_id": app_profile_id},
238
+ metadata=[
239
+ (
240
+ "x-goog-request-params",
241
+ f"name={instance_name}&app_profile_id={app_profile_id}",
242
+ )
243
+ ],
244
+ wait_for_ready=True,
245
+ )
246
+ for (instance_name, table_name, app_profile_id) in instance_list
247
+ ]
248
+ result_list = CrossSync._Sync_Impl.gather_partials(
249
+ partial_list, return_exceptions=True, sync_executor=self._executor
250
+ )
251
+ return [r or None for r in result_list]
252
+
253
+ def _manage_channel(
254
+ self,
255
+ refresh_interval_min: float = 60 * 35,
256
+ refresh_interval_max: float = 60 * 45,
257
+ grace_period: float = 60 * 10,
258
+ ) -> None:
259
+ """Background task that periodically refreshes and warms a grpc channel
260
+
261
+ The backend will automatically close channels after 60 minutes, so
262
+ `refresh_interval` + `grace_period` should be < 60 minutes
263
+
264
+ Runs continuously until the client is closed
265
+
266
+ Args:
267
+ refresh_interval_min: minimum interval before initiating refresh
268
+ process in seconds. Actual interval will be a random value
269
+ between `refresh_interval_min` and `refresh_interval_max`
270
+ refresh_interval_max: maximum interval before initiating refresh
271
+ process in seconds. Actual interval will be a random value
272
+ between `refresh_interval_min` and `refresh_interval_max`
273
+ grace_period: time to allow previous channel to serve existing
274
+ requests before closing, in seconds"""
275
+ first_refresh = self._channel_init_time + random.uniform(
276
+ refresh_interval_min, refresh_interval_max
277
+ )
278
+ next_sleep = max(first_refresh - time.monotonic(), 0)
279
+ if next_sleep > 0:
280
+ self._ping_and_warm_instances(channel=self.transport.grpc_channel)
281
+ while not self._is_closed.is_set():
282
+ CrossSync._Sync_Impl.event_wait(
283
+ self._is_closed, next_sleep, async_break_early=False
284
+ )
285
+ if self._is_closed.is_set():
286
+ break
287
+ start_timestamp = time.monotonic()
288
+ old_channel = self.transport.grpc_channel
289
+ new_channel = self.transport.create_channel()
290
+ new_channel = intercept_channel(new_channel, self.transport._interceptor)
291
+ self._ping_and_warm_instances(channel=new_channel)
292
+ self.transport._grpc_channel = new_channel
293
+ self.transport._logged_channel = new_channel
294
+ self.transport._stubs = {}
295
+ self.transport._prep_wrapped_messages(self.client_info)
296
+ if grace_period:
297
+ self._is_closed.wait(grace_period)
298
+ old_channel.close()
299
+ next_refresh = random.uniform(refresh_interval_min, refresh_interval_max)
300
+ next_sleep = max(next_refresh - (time.monotonic() - start_timestamp), 0)
301
+
302
+ def _register_instance(
303
+ self, instance_id: str, owner: Table | ExecuteQueryIterator
304
+ ) -> None:
305
+ """Registers an instance with the client, and warms the channel for the instance
306
+ The client will periodically refresh grpc channel used to make
307
+ requests, and new channels will be warmed for each registered instance
308
+ Channels will not be refreshed unless at least one instance is registered
309
+
310
+ Args:
311
+ instance_id: id of the instance to register.
312
+ owner: table that owns the instance. Owners will be tracked in
313
+ _instance_owners, and instances will only be unregistered when all
314
+ owners call _remove_instance_registration"""
315
+ instance_name = self._gapic_client.instance_path(self.project, instance_id)
316
+ instance_key = _WarmedInstanceKey(
317
+ instance_name, owner.table_name, owner.app_profile_id
318
+ )
319
+ self._instance_owners.setdefault(instance_key, set()).add(id(owner))
320
+ if instance_key not in self._active_instances:
321
+ self._active_instances.add(instance_key)
322
+ if self._channel_refresh_task:
323
+ self._ping_and_warm_instances(instance_key)
324
+ else:
325
+ self._start_background_channel_refresh()
326
+
327
+ def _remove_instance_registration(
328
+ self, instance_id: str, owner: Table | "ExecuteQueryIterator"
329
+ ) -> bool:
330
+ """Removes an instance from the client's registered instances, to prevent
331
+ warming new channels for the instance
332
+
333
+ If instance_id is not registered, or is still in use by other tables, returns False
334
+
335
+ Args:
336
+ instance_id: id of the instance to remove
337
+ owner: table that owns the instance. Owners will be tracked in
338
+ _instance_owners, and instances will only be unregistered when all
339
+ owners call _remove_instance_registration
340
+ Returns:
341
+ bool: True if instance was removed, else False"""
342
+ instance_name = self._gapic_client.instance_path(self.project, instance_id)
343
+ instance_key = _WarmedInstanceKey(
344
+ instance_name, owner.table_name, owner.app_profile_id
345
+ )
346
+ owner_list = self._instance_owners.get(instance_key, set())
347
+ try:
348
+ owner_list.remove(id(owner))
349
+ if len(owner_list) == 0:
350
+ self._active_instances.remove(instance_key)
351
+ return True
352
+ except KeyError:
353
+ return False
354
+
355
+ def get_table(self, instance_id: str, table_id: str, *args, **kwargs) -> Table:
356
+ """Returns a table instance for making data API requests. All arguments are passed
357
+ directly to the Table constructor.
358
+
359
+
360
+
361
+ Args:
362
+ instance_id: The Bigtable instance ID to associate with this client.
363
+ instance_id is combined with the client's project to fully
364
+ specify the instance
365
+ table_id: The ID of the table. table_id is combined with the
366
+ instance_id and the client's project to fully specify the table
367
+ app_profile_id: The app profile to associate with requests.
368
+ https://cloud.google.com/bigtable/docs/app-profiles
369
+ default_read_rows_operation_timeout: The default timeout for read rows
370
+ operations, in seconds. If not set, defaults to 600 seconds (10 minutes)
371
+ default_read_rows_attempt_timeout: The default timeout for individual
372
+ read rows rpc requests, in seconds. If not set, defaults to 20 seconds
373
+ default_mutate_rows_operation_timeout: The default timeout for mutate rows
374
+ operations, in seconds. If not set, defaults to 600 seconds (10 minutes)
375
+ default_mutate_rows_attempt_timeout: The default timeout for individual
376
+ mutate rows rpc requests, in seconds. If not set, defaults to 60 seconds
377
+ default_operation_timeout: The default timeout for all other operations, in
378
+ seconds. If not set, defaults to 60 seconds
379
+ default_attempt_timeout: The default timeout for all other individual rpc
380
+ requests, in seconds. If not set, defaults to 20 seconds
381
+ default_read_rows_retryable_errors: a list of errors that will be retried
382
+ if encountered during read_rows and related operations.
383
+ Defaults to 4 (DeadlineExceeded), 14 (ServiceUnavailable), and 10 (Aborted)
384
+ default_mutate_rows_retryable_errors: a list of errors that will be retried
385
+ if encountered during mutate_rows and related operations.
386
+ Defaults to 4 (DeadlineExceeded) and 14 (ServiceUnavailable)
387
+ default_retryable_errors: a list of errors that will be retried if
388
+ encountered during all other operations.
389
+ Defaults to 4 (DeadlineExceeded) and 14 (ServiceUnavailable)
390
+ Returns:
391
+ Table: a table instance for making data API requests
392
+ Raises:
393
+ None"""
394
+ return Table(self, instance_id, table_id, *args, **kwargs)
395
+
396
+ def execute_query(
397
+ self,
398
+ query: str,
399
+ instance_id: str,
400
+ *,
401
+ parameters: dict[str, ExecuteQueryValueType] | None = None,
402
+ parameter_types: dict[str, SqlType.Type] | None = None,
403
+ app_profile_id: str | None = None,
404
+ operation_timeout: float = 600,
405
+ attempt_timeout: float | None = 20,
406
+ retryable_errors: Sequence[type[Exception]] = (
407
+ DeadlineExceeded,
408
+ ServiceUnavailable,
409
+ Aborted,
410
+ ),
411
+ prepare_operation_timeout: float = 60,
412
+ prepare_attempt_timeout: float | None = 20,
413
+ prepare_retryable_errors: Sequence[type[Exception]] = (
414
+ DeadlineExceeded,
415
+ ServiceUnavailable,
416
+ ),
417
+ ) -> "ExecuteQueryIterator":
418
+ """Executes an SQL query on an instance.
419
+ Returns an iterator to asynchronously stream back columns from selected rows.
420
+
421
+ Failed requests within operation_timeout will be retried based on the
422
+ retryable_errors list until operation_timeout is reached.
423
+
424
+ Note that this makes two requests, one to ``PrepareQuery`` and one to ``ExecuteQuery``.
425
+ These have separate retry configurations. ``ExecuteQuery`` is where the bulk of the
426
+ work happens.
427
+
428
+ Args:
429
+ query: Query to be run on Bigtable instance. The query can use ``@param``
430
+ placeholders to use parameter interpolation on the server. Values for all
431
+ parameters should be provided in ``parameters``. Types of parameters are
432
+ inferred but should be provided in ``parameter_types`` if the inference is
433
+ not possible (i.e. when value can be None, an empty list or an empty dict).
434
+ instance_id: The Bigtable instance ID to perform the query on.
435
+ instance_id is combined with the client's project to fully
436
+ specify the instance.
437
+ parameters: Dictionary with values for all parameters used in the ``query``.
438
+ parameter_types: Dictionary with types of parameters used in the ``query``.
439
+ Required to contain entries only for parameters whose type cannot be
440
+ detected automatically (i.e. the value can be None, an empty list or
441
+ an empty dict).
442
+ app_profile_id: The app profile to associate with requests.
443
+ https://cloud.google.com/bigtable/docs/app-profiles
444
+ operation_timeout: the time budget for the entire executeQuery operation, in seconds.
445
+ Failed requests will be retried within the budget.
446
+ Defaults to 600 seconds.
447
+ attempt_timeout: the time budget for an individual executeQuery network request, in seconds.
448
+ If it takes longer than this time to complete, the request will be cancelled with
449
+ a DeadlineExceeded exception, and a retry will be attempted.
450
+ Defaults to the 20 seconds.
451
+ If None, defaults to operation_timeout.
452
+ retryable_errors: a list of errors that will be retried if encountered during executeQuery.
453
+ Defaults to 4 (DeadlineExceeded), 14 (ServiceUnavailable), and 10 (Aborted)
454
+ prepare_operation_timeout: the time budget for the entire prepareQuery operation, in seconds.
455
+ Failed requests will be retried within the budget.
456
+ Defaults to 60 seconds.
457
+ prepare_attempt_timeout: the time budget for an individual prepareQuery network request, in seconds.
458
+ If it takes longer than this time to complete, the request will be cancelled with
459
+ a DeadlineExceeded exception, and a retry will be attempted.
460
+ Defaults to the 20 seconds.
461
+ If None, defaults to prepare_operation_timeout.
462
+ prepare_retryable_errors: a list of errors that will be retried if encountered during prepareQuery.
463
+ Defaults to 4 (DeadlineExceeded) and 14 (ServiceUnavailable)
464
+ Returns:
465
+ ExecuteQueryIterator: an asynchronous iterator that yields rows returned by the query
466
+ Raises:
467
+ google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
468
+ will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
469
+ from any retries that failed
470
+ google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
471
+ google.cloud.bigtable.data.exceptions.ParameterTypeInferenceFailed: Raised if
472
+ a parameter is passed without an explicit type, and the type cannot be infered
473
+ """
474
+ instance_name = self._gapic_client.instance_path(self.project, instance_id)
475
+ converted_param_types = _to_param_types(parameters, parameter_types)
476
+ prepare_request = {
477
+ "instance_name": instance_name,
478
+ "query": query,
479
+ "app_profile_id": app_profile_id,
480
+ "param_types": converted_param_types,
481
+ "proto_format": {},
482
+ }
483
+ prepare_predicate = retries.if_exception_type(
484
+ *[_get_error_type(e) for e in prepare_retryable_errors]
485
+ )
486
+ (prepare_operation_timeout, prepare_attempt_timeout) = _align_timeouts(
487
+ prepare_operation_timeout, prepare_attempt_timeout
488
+ )
489
+ prepare_sleep_generator = retries.exponential_sleep_generator(0.01, 2, 60)
490
+ target = partial(
491
+ self._gapic_client.prepare_query,
492
+ request=prepare_request,
493
+ timeout=prepare_attempt_timeout,
494
+ retry=None,
495
+ )
496
+ prepare_result = CrossSync._Sync_Impl.retry_target(
497
+ target,
498
+ prepare_predicate,
499
+ prepare_sleep_generator,
500
+ prepare_operation_timeout,
501
+ exception_factory=_retry_exception_factory,
502
+ )
503
+ prepare_metadata = _pb_metadata_to_metadata_types(prepare_result.metadata)
504
+ retryable_excs = [_get_error_type(e) for e in retryable_errors]
505
+ pb_params = _format_execute_query_params(parameters, parameter_types)
506
+ request_body = {
507
+ "instance_name": instance_name,
508
+ "app_profile_id": app_profile_id,
509
+ "prepared_query": prepare_result.prepared_query,
510
+ "params": pb_params,
511
+ }
512
+ (operation_timeout, attempt_timeout) = _align_timeouts(
513
+ operation_timeout, attempt_timeout
514
+ )
515
+ return CrossSync._Sync_Impl.ExecuteQueryIterator(
516
+ self,
517
+ instance_id,
518
+ app_profile_id,
519
+ request_body,
520
+ prepare_metadata,
521
+ attempt_timeout,
522
+ operation_timeout,
523
+ retryable_excs=retryable_excs,
524
+ )
525
+
526
+ def __enter__(self):
527
+ self._start_background_channel_refresh()
528
+ return self
529
+
530
+ def __exit__(self, exc_type, exc_val, exc_tb):
531
+ self.close()
532
+ self._gapic_client.__exit__(exc_type, exc_val, exc_tb)
533
+
534
+
535
+ @CrossSync._Sync_Impl.add_mapping_decorator("Table")
536
+ class Table:
537
+ """
538
+ Main Data API surface
539
+
540
+ Table object maintains table_id, and app_profile_id context, and passes them with
541
+ each call
542
+ """
543
+
544
+ def __init__(
545
+ self,
546
+ client: BigtableDataClient,
547
+ instance_id: str,
548
+ table_id: str,
549
+ app_profile_id: str | None = None,
550
+ *,
551
+ default_read_rows_operation_timeout: float = 600,
552
+ default_read_rows_attempt_timeout: float | None = 20,
553
+ default_mutate_rows_operation_timeout: float = 600,
554
+ default_mutate_rows_attempt_timeout: float | None = 60,
555
+ default_operation_timeout: float = 60,
556
+ default_attempt_timeout: float | None = 20,
557
+ default_read_rows_retryable_errors: Sequence[type[Exception]] = (
558
+ DeadlineExceeded,
559
+ ServiceUnavailable,
560
+ Aborted,
561
+ ),
562
+ default_mutate_rows_retryable_errors: Sequence[type[Exception]] = (
563
+ DeadlineExceeded,
564
+ ServiceUnavailable,
565
+ ),
566
+ default_retryable_errors: Sequence[type[Exception]] = (
567
+ DeadlineExceeded,
568
+ ServiceUnavailable,
569
+ ),
570
+ ):
571
+ """Initialize a Table instance
572
+
573
+
574
+
575
+ Args:
576
+ instance_id: The Bigtable instance ID to associate with this client.
577
+ instance_id is combined with the client's project to fully
578
+ specify the instance
579
+ table_id: The ID of the table. table_id is combined with the
580
+ instance_id and the client's project to fully specify the table
581
+ app_profile_id: The app profile to associate with requests.
582
+ https://cloud.google.com/bigtable/docs/app-profiles
583
+ default_read_rows_operation_timeout: The default timeout for read rows
584
+ operations, in seconds. If not set, defaults to 600 seconds (10 minutes)
585
+ default_read_rows_attempt_timeout: The default timeout for individual
586
+ read rows rpc requests, in seconds. If not set, defaults to 20 seconds
587
+ default_mutate_rows_operation_timeout: The default timeout for mutate rows
588
+ operations, in seconds. If not set, defaults to 600 seconds (10 minutes)
589
+ default_mutate_rows_attempt_timeout: The default timeout for individual
590
+ mutate rows rpc requests, in seconds. If not set, defaults to 60 seconds
591
+ default_operation_timeout: The default timeout for all other operations, in
592
+ seconds. If not set, defaults to 60 seconds
593
+ default_attempt_timeout: The default timeout for all other individual rpc
594
+ requests, in seconds. If not set, defaults to 20 seconds
595
+ default_read_rows_retryable_errors: a list of errors that will be retried
596
+ if encountered during read_rows and related operations.
597
+ Defaults to 4 (DeadlineExceeded), 14 (ServiceUnavailable), and 10 (Aborted)
598
+ default_mutate_rows_retryable_errors: a list of errors that will be retried
599
+ if encountered during mutate_rows and related operations.
600
+ Defaults to 4 (DeadlineExceeded) and 14 (ServiceUnavailable)
601
+ default_retryable_errors: a list of errors that will be retried if
602
+ encountered during all other operations.
603
+ Defaults to 4 (DeadlineExceeded) and 14 (ServiceUnavailable)
604
+ Raises:
605
+ None"""
606
+ _validate_timeouts(
607
+ default_operation_timeout, default_attempt_timeout, allow_none=True
608
+ )
609
+ _validate_timeouts(
610
+ default_read_rows_operation_timeout,
611
+ default_read_rows_attempt_timeout,
612
+ allow_none=True,
613
+ )
614
+ _validate_timeouts(
615
+ default_mutate_rows_operation_timeout,
616
+ default_mutate_rows_attempt_timeout,
617
+ allow_none=True,
618
+ )
619
+ self.client = client
620
+ self.instance_id = instance_id
621
+ self.instance_name = self.client._gapic_client.instance_path(
622
+ self.client.project, instance_id
623
+ )
624
+ self.table_id = table_id
625
+ self.table_name = self.client._gapic_client.table_path(
626
+ self.client.project, instance_id, table_id
627
+ )
628
+ self.app_profile_id = app_profile_id
629
+ self.default_operation_timeout = default_operation_timeout
630
+ self.default_attempt_timeout = default_attempt_timeout
631
+ self.default_read_rows_operation_timeout = default_read_rows_operation_timeout
632
+ self.default_read_rows_attempt_timeout = default_read_rows_attempt_timeout
633
+ self.default_mutate_rows_operation_timeout = (
634
+ default_mutate_rows_operation_timeout
635
+ )
636
+ self.default_mutate_rows_attempt_timeout = default_mutate_rows_attempt_timeout
637
+ self.default_read_rows_retryable_errors = (
638
+ default_read_rows_retryable_errors or ()
639
+ )
640
+ self.default_mutate_rows_retryable_errors = (
641
+ default_mutate_rows_retryable_errors or ()
642
+ )
643
+ self.default_retryable_errors = default_retryable_errors or ()
644
+ try:
645
+ self._register_instance_future = CrossSync._Sync_Impl.create_task(
646
+ self.client._register_instance,
647
+ self.instance_id,
648
+ self,
649
+ sync_executor=self.client._executor,
650
+ )
651
+ except RuntimeError as e:
652
+ raise RuntimeError(
653
+ f"{self.__class__.__name__} must be created within an async event loop context."
654
+ ) from e
655
+
656
+ def read_rows_stream(
657
+ self,
658
+ query: ReadRowsQuery,
659
+ *,
660
+ operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
661
+ attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
662
+ retryable_errors: Sequence[type[Exception]]
663
+ | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
664
+ ) -> Iterable[Row]:
665
+ """Read a set of rows from the table, based on the specified query.
666
+ Returns an iterator to asynchronously stream back row data.
667
+
668
+ Failed requests within operation_timeout will be retried based on the
669
+ retryable_errors list until operation_timeout is reached.
670
+
671
+ Args:
672
+ query: contains details about which rows to return
673
+ operation_timeout: the time budget for the entire operation, in seconds.
674
+ Failed requests will be retried within the budget.
675
+ Defaults to the Table's default_read_rows_operation_timeout
676
+ attempt_timeout: the time budget for an individual network request, in seconds.
677
+ If it takes longer than this time to complete, the request will be cancelled with
678
+ a DeadlineExceeded exception, and a retry will be attempted.
679
+ Defaults to the Table's default_read_rows_attempt_timeout.
680
+ If None, defaults to operation_timeout.
681
+ retryable_errors: a list of errors that will be retried if encountered.
682
+ Defaults to the Table's default_read_rows_retryable_errors
683
+ Returns:
684
+ Iterable[Row]: an asynchronous iterator that yields rows returned by the query
685
+ Raises:
686
+ google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
687
+ will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
688
+ from any retries that failed
689
+ google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
690
+ """
691
+ (operation_timeout, attempt_timeout) = _get_timeouts(
692
+ operation_timeout, attempt_timeout, self
693
+ )
694
+ retryable_excs = _get_retryable_errors(retryable_errors, self)
695
+ row_merger = CrossSync._Sync_Impl._ReadRowsOperation(
696
+ query,
697
+ self,
698
+ operation_timeout=operation_timeout,
699
+ attempt_timeout=attempt_timeout,
700
+ retryable_exceptions=retryable_excs,
701
+ )
702
+ return row_merger.start_operation()
703
+
704
+ def read_rows(
705
+ self,
706
+ query: ReadRowsQuery,
707
+ *,
708
+ operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
709
+ attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
710
+ retryable_errors: Sequence[type[Exception]]
711
+ | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
712
+ ) -> list[Row]:
713
+ """Read a set of rows from the table, based on the specified query.
714
+ Retruns results as a list of Row objects when the request is complete.
715
+ For streamed results, use read_rows_stream.
716
+
717
+ Failed requests within operation_timeout will be retried based on the
718
+ retryable_errors list until operation_timeout is reached.
719
+
720
+ Args:
721
+ query: contains details about which rows to return
722
+ operation_timeout: the time budget for the entire operation, in seconds.
723
+ Failed requests will be retried within the budget.
724
+ Defaults to the Table's default_read_rows_operation_timeout
725
+ attempt_timeout: the time budget for an individual network request, in seconds.
726
+ If it takes longer than this time to complete, the request will be cancelled with
727
+ a DeadlineExceeded exception, and a retry will be attempted.
728
+ Defaults to the Table's default_read_rows_attempt_timeout.
729
+ If None, defaults to operation_timeout.
730
+ If None, defaults to the Table's default_read_rows_attempt_timeout,
731
+ or the operation_timeout if that is also None.
732
+ retryable_errors: a list of errors that will be retried if encountered.
733
+ Defaults to the Table's default_read_rows_retryable_errors.
734
+ Returns:
735
+ list[Row]: a list of Rows returned by the query
736
+ Raises:
737
+ google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
738
+ will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
739
+ from any retries that failed
740
+ google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
741
+ """
742
+ row_generator = self.read_rows_stream(
743
+ query,
744
+ operation_timeout=operation_timeout,
745
+ attempt_timeout=attempt_timeout,
746
+ retryable_errors=retryable_errors,
747
+ )
748
+ return [row for row in row_generator]
749
+
750
+ def read_row(
751
+ self,
752
+ row_key: str | bytes,
753
+ *,
754
+ row_filter: RowFilter | None = None,
755
+ operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
756
+ attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
757
+ retryable_errors: Sequence[type[Exception]]
758
+ | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
759
+ ) -> Row | None:
760
+ """Read a single row from the table, based on the specified key.
761
+
762
+ Failed requests within operation_timeout will be retried based on the
763
+ retryable_errors list until operation_timeout is reached.
764
+
765
+ Args:
766
+ query: contains details about which rows to return
767
+ operation_timeout: the time budget for the entire operation, in seconds.
768
+ Failed requests will be retried within the budget.
769
+ Defaults to the Table's default_read_rows_operation_timeout
770
+ attempt_timeout: the time budget for an individual network request, in seconds.
771
+ If it takes longer than this time to complete, the request will be cancelled with
772
+ a DeadlineExceeded exception, and a retry will be attempted.
773
+ Defaults to the Table's default_read_rows_attempt_timeout.
774
+ If None, defaults to operation_timeout.
775
+ retryable_errors: a list of errors that will be retried if encountered.
776
+ Defaults to the Table's default_read_rows_retryable_errors.
777
+ Returns:
778
+ Row | None: a Row object if the row exists, otherwise None
779
+ Raises:
780
+ google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
781
+ will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
782
+ from any retries that failed
783
+ google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
784
+ """
785
+ if row_key is None:
786
+ raise ValueError("row_key must be string or bytes")
787
+ query = ReadRowsQuery(row_keys=row_key, row_filter=row_filter, limit=1)
788
+ results = self.read_rows(
789
+ query,
790
+ operation_timeout=operation_timeout,
791
+ attempt_timeout=attempt_timeout,
792
+ retryable_errors=retryable_errors,
793
+ )
794
+ if len(results) == 0:
795
+ return None
796
+ return results[0]
797
+
798
+ def read_rows_sharded(
799
+ self,
800
+ sharded_query: ShardedQuery,
801
+ *,
802
+ operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
803
+ attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
804
+ retryable_errors: Sequence[type[Exception]]
805
+ | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
806
+ ) -> list[Row]:
807
+ """Runs a sharded query in parallel, then return the results in a single list.
808
+ Results will be returned in the order of the input queries.
809
+
810
+ This function is intended to be run on the results on a query.shard() call.
811
+ For example::
812
+
813
+ table_shard_keys = await table.sample_row_keys()
814
+ query = ReadRowsQuery(...)
815
+ shard_queries = query.shard(table_shard_keys)
816
+ results = await table.read_rows_sharded(shard_queries)
817
+
818
+ Args:
819
+ sharded_query: a sharded query to execute
820
+ operation_timeout: the time budget for the entire operation, in seconds.
821
+ Failed requests will be retried within the budget.
822
+ Defaults to the Table's default_read_rows_operation_timeout
823
+ attempt_timeout: the time budget for an individual network request, in seconds.
824
+ If it takes longer than this time to complete, the request will be cancelled with
825
+ a DeadlineExceeded exception, and a retry will be attempted.
826
+ Defaults to the Table's default_read_rows_attempt_timeout.
827
+ If None, defaults to operation_timeout.
828
+ retryable_errors: a list of errors that will be retried if encountered.
829
+ Defaults to the Table's default_read_rows_retryable_errors.
830
+ Returns:
831
+ list[Row]: a list of Rows returned by the query
832
+ Raises:
833
+ ShardedReadRowsExceptionGroup: if any of the queries failed
834
+ ValueError: if the query_list is empty"""
835
+ if not sharded_query:
836
+ raise ValueError("empty sharded_query")
837
+ (operation_timeout, attempt_timeout) = _get_timeouts(
838
+ operation_timeout, attempt_timeout, self
839
+ )
840
+ rpc_timeout_generator = _attempt_timeout_generator(
841
+ operation_timeout, operation_timeout
842
+ )
843
+ concurrency_sem = CrossSync._Sync_Impl.Semaphore(_CONCURRENCY_LIMIT)
844
+
845
+ def read_rows_with_semaphore(query):
846
+ with concurrency_sem:
847
+ shard_timeout = next(rpc_timeout_generator)
848
+ if shard_timeout <= 0:
849
+ raise DeadlineExceeded(
850
+ "Operation timeout exceeded before starting query"
851
+ )
852
+ return self.read_rows(
853
+ query,
854
+ operation_timeout=shard_timeout,
855
+ attempt_timeout=min(attempt_timeout, shard_timeout),
856
+ retryable_errors=retryable_errors,
857
+ )
858
+
859
+ routine_list = [
860
+ partial(read_rows_with_semaphore, query) for query in sharded_query
861
+ ]
862
+ batch_result = CrossSync._Sync_Impl.gather_partials(
863
+ routine_list, return_exceptions=True, sync_executor=self.client._executor
864
+ )
865
+ error_dict = {}
866
+ shard_idx = 0
867
+ results_list = []
868
+ for result in batch_result:
869
+ if isinstance(result, Exception):
870
+ error_dict[shard_idx] = result
871
+ elif isinstance(result, BaseException):
872
+ raise result
873
+ else:
874
+ results_list.extend(result)
875
+ shard_idx += 1
876
+ if error_dict:
877
+ raise ShardedReadRowsExceptionGroup(
878
+ [
879
+ FailedQueryShardError(idx, sharded_query[idx], e)
880
+ for (idx, e) in error_dict.items()
881
+ ],
882
+ results_list,
883
+ len(sharded_query),
884
+ )
885
+ return results_list
886
+
887
+ def row_exists(
888
+ self,
889
+ row_key: str | bytes,
890
+ *,
891
+ operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
892
+ attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
893
+ retryable_errors: Sequence[type[Exception]]
894
+ | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
895
+ ) -> bool:
896
+ """Return a boolean indicating whether the specified row exists in the table.
897
+ uses the filters: chain(limit cells per row = 1, strip value)
898
+
899
+ Args:
900
+ row_key: the key of the row to check
901
+ operation_timeout: the time budget for the entire operation, in seconds.
902
+ Failed requests will be retried within the budget.
903
+ Defaults to the Table's default_read_rows_operation_timeout
904
+ attempt_timeout: the time budget for an individual network request, in seconds.
905
+ If it takes longer than this time to complete, the request will be cancelled with
906
+ a DeadlineExceeded exception, and a retry will be attempted.
907
+ Defaults to the Table's default_read_rows_attempt_timeout.
908
+ If None, defaults to operation_timeout.
909
+ retryable_errors: a list of errors that will be retried if encountered.
910
+ Defaults to the Table's default_read_rows_retryable_errors.
911
+ Returns:
912
+ bool: a bool indicating whether the row exists
913
+ Raises:
914
+ google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
915
+ will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
916
+ from any retries that failed
917
+ google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
918
+ """
919
+ if row_key is None:
920
+ raise ValueError("row_key must be string or bytes")
921
+ strip_filter = StripValueTransformerFilter(flag=True)
922
+ limit_filter = CellsRowLimitFilter(1)
923
+ chain_filter = RowFilterChain(filters=[limit_filter, strip_filter])
924
+ query = ReadRowsQuery(row_keys=row_key, limit=1, row_filter=chain_filter)
925
+ results = self.read_rows(
926
+ query,
927
+ operation_timeout=operation_timeout,
928
+ attempt_timeout=attempt_timeout,
929
+ retryable_errors=retryable_errors,
930
+ )
931
+ return len(results) > 0
932
+
933
+ def sample_row_keys(
934
+ self,
935
+ *,
936
+ operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
937
+ attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
938
+ retryable_errors: Sequence[type[Exception]]
939
+ | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
940
+ ) -> RowKeySamples:
941
+ """Return a set of RowKeySamples that delimit contiguous sections of the table of
942
+ approximately equal size
943
+
944
+ RowKeySamples output can be used with ReadRowsQuery.shard() to create a sharded query that
945
+ can be parallelized across multiple backend nodes read_rows and read_rows_stream
946
+ requests will call sample_row_keys internally for this purpose when sharding is enabled
947
+
948
+ RowKeySamples is simply a type alias for list[tuple[bytes, int]]; a list of
949
+ row_keys, along with offset positions in the table
950
+
951
+ Args:
952
+ operation_timeout: the time budget for the entire operation, in seconds.
953
+ Failed requests will be retried within the budget.i
954
+ Defaults to the Table's default_operation_timeout
955
+ attempt_timeout: the time budget for an individual network request, in seconds.
956
+ If it takes longer than this time to complete, the request will be cancelled with
957
+ a DeadlineExceeded exception, and a retry will be attempted.
958
+ Defaults to the Table's default_attempt_timeout.
959
+ If None, defaults to operation_timeout.
960
+ retryable_errors: a list of errors that will be retried if encountered.
961
+ Defaults to the Table's default_retryable_errors.
962
+ Returns:
963
+ RowKeySamples: a set of RowKeySamples the delimit contiguous sections of the table
964
+ Raises:
965
+ google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
966
+ will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
967
+ from any retries that failed
968
+ google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
969
+ """
970
+ (operation_timeout, attempt_timeout) = _get_timeouts(
971
+ operation_timeout, attempt_timeout, self
972
+ )
973
+ attempt_timeout_gen = _attempt_timeout_generator(
974
+ attempt_timeout, operation_timeout
975
+ )
976
+ retryable_excs = _get_retryable_errors(retryable_errors, self)
977
+ predicate = retries.if_exception_type(*retryable_excs)
978
+ sleep_generator = retries.exponential_sleep_generator(0.01, 2, 60)
979
+
980
+ def execute_rpc():
981
+ results = self.client._gapic_client.sample_row_keys(
982
+ table_name=self.table_name,
983
+ app_profile_id=self.app_profile_id,
984
+ timeout=next(attempt_timeout_gen),
985
+ retry=None,
986
+ )
987
+ return [(s.row_key, s.offset_bytes) for s in results]
988
+
989
+ return CrossSync._Sync_Impl.retry_target(
990
+ execute_rpc,
991
+ predicate,
992
+ sleep_generator,
993
+ operation_timeout,
994
+ exception_factory=_retry_exception_factory,
995
+ )
996
+
997
+ def mutations_batcher(
998
+ self,
999
+ *,
1000
+ flush_interval: float | None = 5,
1001
+ flush_limit_mutation_count: int | None = 1000,
1002
+ flush_limit_bytes: int = 20 * _MB_SIZE,
1003
+ flow_control_max_mutation_count: int = 100000,
1004
+ flow_control_max_bytes: int = 100 * _MB_SIZE,
1005
+ batch_operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
1006
+ batch_attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
1007
+ batch_retryable_errors: Sequence[type[Exception]]
1008
+ | TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
1009
+ ) -> "MutationsBatcher":
1010
+ """Returns a new mutations batcher instance.
1011
+
1012
+ Can be used to iteratively add mutations that are flushed as a group,
1013
+ to avoid excess network calls
1014
+
1015
+ Args:
1016
+ flush_interval: Automatically flush every flush_interval seconds. If None,
1017
+ a table default will be used
1018
+ flush_limit_mutation_count: Flush immediately after flush_limit_mutation_count
1019
+ mutations are added across all entries. If None, this limit is ignored.
1020
+ flush_limit_bytes: Flush immediately after flush_limit_bytes bytes are added.
1021
+ flow_control_max_mutation_count: Maximum number of inflight mutations.
1022
+ flow_control_max_bytes: Maximum number of inflight bytes.
1023
+ batch_operation_timeout: timeout for each mutate_rows operation, in seconds.
1024
+ Defaults to the Table's default_mutate_rows_operation_timeout
1025
+ batch_attempt_timeout: timeout for each individual request, in seconds.
1026
+ Defaults to the Table's default_mutate_rows_attempt_timeout.
1027
+ If None, defaults to batch_operation_timeout.
1028
+ batch_retryable_errors: a list of errors that will be retried if encountered.
1029
+ Defaults to the Table's default_mutate_rows_retryable_errors.
1030
+ Returns:
1031
+ MutationsBatcher: a MutationsBatcher context manager that can batch requests
1032
+ """
1033
+ return CrossSync._Sync_Impl.MutationsBatcher(
1034
+ self,
1035
+ flush_interval=flush_interval,
1036
+ flush_limit_mutation_count=flush_limit_mutation_count,
1037
+ flush_limit_bytes=flush_limit_bytes,
1038
+ flow_control_max_mutation_count=flow_control_max_mutation_count,
1039
+ flow_control_max_bytes=flow_control_max_bytes,
1040
+ batch_operation_timeout=batch_operation_timeout,
1041
+ batch_attempt_timeout=batch_attempt_timeout,
1042
+ batch_retryable_errors=batch_retryable_errors,
1043
+ )
1044
+
1045
+ def mutate_row(
1046
+ self,
1047
+ row_key: str | bytes,
1048
+ mutations: list[Mutation] | Mutation,
1049
+ *,
1050
+ operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
1051
+ attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
1052
+ retryable_errors: Sequence[type[Exception]]
1053
+ | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
1054
+ ):
1055
+ """Mutates a row atomically.
1056
+
1057
+ Cells already present in the row are left unchanged unless explicitly changed
1058
+ by ``mutation``.
1059
+
1060
+ Idempotent operations (i.e, all mutations have an explicit timestamp) will be
1061
+ retried on server failure. Non-idempotent operations will not.
1062
+
1063
+ Args:
1064
+ row_key: the row to apply mutations to
1065
+ mutations: the set of mutations to apply to the row
1066
+ operation_timeout: the time budget for the entire operation, in seconds.
1067
+ Failed requests will be retried within the budget.
1068
+ Defaults to the Table's default_operation_timeout
1069
+ attempt_timeout: the time budget for an individual network request, in seconds.
1070
+ If it takes longer than this time to complete, the request will be cancelled with
1071
+ a DeadlineExceeded exception, and a retry will be attempted.
1072
+ Defaults to the Table's default_attempt_timeout.
1073
+ If None, defaults to operation_timeout.
1074
+ retryable_errors: a list of errors that will be retried if encountered.
1075
+ Only idempotent mutations will be retried. Defaults to the Table's
1076
+ default_retryable_errors.
1077
+ Raises:
1078
+ google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
1079
+ will be chained with a RetryExceptionGroup containing all
1080
+ GoogleAPIError exceptions from any retries that failed
1081
+ google.api_core.exceptions.GoogleAPIError: raised on non-idempotent operations that cannot be
1082
+ safely retried.
1083
+ ValueError: if invalid arguments are provided"""
1084
+ (operation_timeout, attempt_timeout) = _get_timeouts(
1085
+ operation_timeout, attempt_timeout, self
1086
+ )
1087
+ if not mutations:
1088
+ raise ValueError("No mutations provided")
1089
+ mutations_list = mutations if isinstance(mutations, list) else [mutations]
1090
+ if all((mutation.is_idempotent() for mutation in mutations_list)):
1091
+ predicate = retries.if_exception_type(
1092
+ *_get_retryable_errors(retryable_errors, self)
1093
+ )
1094
+ else:
1095
+ predicate = retries.if_exception_type()
1096
+ sleep_generator = retries.exponential_sleep_generator(0.01, 2, 60)
1097
+ target = partial(
1098
+ self.client._gapic_client.mutate_row,
1099
+ row_key=row_key.encode("utf-8") if isinstance(row_key, str) else row_key,
1100
+ mutations=[mutation._to_pb() for mutation in mutations_list],
1101
+ table_name=self.table_name,
1102
+ app_profile_id=self.app_profile_id,
1103
+ timeout=attempt_timeout,
1104
+ retry=None,
1105
+ )
1106
+ return CrossSync._Sync_Impl.retry_target(
1107
+ target,
1108
+ predicate,
1109
+ sleep_generator,
1110
+ operation_timeout,
1111
+ exception_factory=_retry_exception_factory,
1112
+ )
1113
+
1114
+ def bulk_mutate_rows(
1115
+ self,
1116
+ mutation_entries: list[RowMutationEntry],
1117
+ *,
1118
+ operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
1119
+ attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
1120
+ retryable_errors: Sequence[type[Exception]]
1121
+ | TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
1122
+ ):
1123
+ """Applies mutations for multiple rows in a single batched request.
1124
+
1125
+ Each individual RowMutationEntry is applied atomically, but separate entries
1126
+ may be applied in arbitrary order (even for entries targetting the same row)
1127
+ In total, the row_mutations can contain at most 100000 individual mutations
1128
+ across all entries
1129
+
1130
+ Idempotent entries (i.e., entries with mutations with explicit timestamps)
1131
+ will be retried on failure. Non-idempotent will not, and will reported in a
1132
+ raised exception group
1133
+
1134
+ Args:
1135
+ mutation_entries: the batches of mutations to apply
1136
+ Each entry will be applied atomically, but entries will be applied
1137
+ in arbitrary order
1138
+ operation_timeout: the time budget for the entire operation, in seconds.
1139
+ Failed requests will be retried within the budget.
1140
+ Defaults to the Table's default_mutate_rows_operation_timeout
1141
+ attempt_timeout: the time budget for an individual network request, in seconds.
1142
+ If it takes longer than this time to complete, the request will be cancelled with
1143
+ a DeadlineExceeded exception, and a retry will be attempted.
1144
+ Defaults to the Table's default_mutate_rows_attempt_timeout.
1145
+ If None, defaults to operation_timeout.
1146
+ retryable_errors: a list of errors that will be retried if encountered.
1147
+ Defaults to the Table's default_mutate_rows_retryable_errors
1148
+ Raises:
1149
+ MutationsExceptionGroup: if one or more mutations fails
1150
+ Contains details about any failed entries in .exceptions
1151
+ ValueError: if invalid arguments are provided"""
1152
+ (operation_timeout, attempt_timeout) = _get_timeouts(
1153
+ operation_timeout, attempt_timeout, self
1154
+ )
1155
+ retryable_excs = _get_retryable_errors(retryable_errors, self)
1156
+ operation = CrossSync._Sync_Impl._MutateRowsOperation(
1157
+ self.client._gapic_client,
1158
+ self,
1159
+ mutation_entries,
1160
+ operation_timeout,
1161
+ attempt_timeout,
1162
+ retryable_exceptions=retryable_excs,
1163
+ )
1164
+ operation.start()
1165
+
1166
+ def check_and_mutate_row(
1167
+ self,
1168
+ row_key: str | bytes,
1169
+ predicate: RowFilter | None,
1170
+ *,
1171
+ true_case_mutations: Mutation | list[Mutation] | None = None,
1172
+ false_case_mutations: Mutation | list[Mutation] | None = None,
1173
+ operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
1174
+ ) -> bool:
1175
+ """Mutates a row atomically based on the output of a predicate filter
1176
+
1177
+ Non-idempotent operation: will not be retried
1178
+
1179
+ Args:
1180
+ row_key: the key of the row to mutate
1181
+ predicate: the filter to be applied to the contents of the specified row.
1182
+ Depending on whether or not any results are yielded,
1183
+ either true_case_mutations or false_case_mutations will be executed.
1184
+ If None, checks that the row contains any values at all.
1185
+ true_case_mutations:
1186
+ Changes to be atomically applied to the specified row if
1187
+ predicate yields at least one cell when
1188
+ applied to row_key. Entries are applied in order,
1189
+ meaning that earlier mutations can be masked by later
1190
+ ones. Must contain at least one entry if
1191
+ false_case_mutations is empty, and at most 100000.
1192
+ false_case_mutations:
1193
+ Changes to be atomically applied to the specified row if
1194
+ predicate_filter does not yield any cells when
1195
+ applied to row_key. Entries are applied in order,
1196
+ meaning that earlier mutations can be masked by later
1197
+ ones. Must contain at least one entry if
1198
+ `true_case_mutations` is empty, and at most 100000.
1199
+ operation_timeout: the time budget for the entire operation, in seconds.
1200
+ Failed requests will not be retried. Defaults to the Table's default_operation_timeout
1201
+ Returns:
1202
+ bool indicating whether the predicate was true or false
1203
+ Raises:
1204
+ google.api_core.exceptions.GoogleAPIError: exceptions from grpc call"""
1205
+ (operation_timeout, _) = _get_timeouts(operation_timeout, None, self)
1206
+ if true_case_mutations is not None and (
1207
+ not isinstance(true_case_mutations, list)
1208
+ ):
1209
+ true_case_mutations = [true_case_mutations]
1210
+ true_case_list = [m._to_pb() for m in true_case_mutations or []]
1211
+ if false_case_mutations is not None and (
1212
+ not isinstance(false_case_mutations, list)
1213
+ ):
1214
+ false_case_mutations = [false_case_mutations]
1215
+ false_case_list = [m._to_pb() for m in false_case_mutations or []]
1216
+ result = self.client._gapic_client.check_and_mutate_row(
1217
+ true_mutations=true_case_list,
1218
+ false_mutations=false_case_list,
1219
+ predicate_filter=predicate._to_pb() if predicate is not None else None,
1220
+ row_key=row_key.encode("utf-8") if isinstance(row_key, str) else row_key,
1221
+ table_name=self.table_name,
1222
+ app_profile_id=self.app_profile_id,
1223
+ timeout=operation_timeout,
1224
+ retry=None,
1225
+ )
1226
+ return result.predicate_matched
1227
+
1228
+ def read_modify_write_row(
1229
+ self,
1230
+ row_key: str | bytes,
1231
+ rules: ReadModifyWriteRule | list[ReadModifyWriteRule],
1232
+ *,
1233
+ operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
1234
+ ) -> Row:
1235
+ """Reads and modifies a row atomically according to input ReadModifyWriteRules,
1236
+ and returns the contents of all modified cells
1237
+
1238
+ The new value for the timestamp is the greater of the existing timestamp or
1239
+ the current server time.
1240
+
1241
+ Non-idempotent operation: will not be retried
1242
+
1243
+ Args:
1244
+ row_key: the key of the row to apply read/modify/write rules to
1245
+ rules: A rule or set of rules to apply to the row.
1246
+ Rules are applied in order, meaning that earlier rules will affect the
1247
+ results of later ones.
1248
+ operation_timeout: the time budget for the entire operation, in seconds.
1249
+ Failed requests will not be retried.
1250
+ Defaults to the Table's default_operation_timeout.
1251
+ Returns:
1252
+ Row: a Row containing cell data that was modified as part of the operation
1253
+ Raises:
1254
+ google.api_core.exceptions.GoogleAPIError: exceptions from grpc call
1255
+ ValueError: if invalid arguments are provided"""
1256
+ (operation_timeout, _) = _get_timeouts(operation_timeout, None, self)
1257
+ if operation_timeout <= 0:
1258
+ raise ValueError("operation_timeout must be greater than 0")
1259
+ if rules is not None and (not isinstance(rules, list)):
1260
+ rules = [rules]
1261
+ if not rules:
1262
+ raise ValueError("rules must contain at least one item")
1263
+ result = self.client._gapic_client.read_modify_write_row(
1264
+ rules=[rule._to_pb() for rule in rules],
1265
+ row_key=row_key.encode("utf-8") if isinstance(row_key, str) else row_key,
1266
+ table_name=self.table_name,
1267
+ app_profile_id=self.app_profile_id,
1268
+ timeout=operation_timeout,
1269
+ retry=None,
1270
+ )
1271
+ return Row._from_pb(result.row)
1272
+
1273
+ def close(self):
1274
+ """Called to close the Table instance and release any resources held by it."""
1275
+ if self._register_instance_future:
1276
+ self._register_instance_future.cancel()
1277
+ self.client._remove_instance_registration(self.instance_id, self)
1278
+
1279
+ def __enter__(self):
1280
+ """Implement async context manager protocol
1281
+
1282
+ Ensure registration task has time to run, so that
1283
+ grpc channels will be warmed for the specified instance"""
1284
+ if self._register_instance_future:
1285
+ self._register_instance_future
1286
+ return self
1287
+
1288
+ def __exit__(self, exc_type, exc_val, exc_tb):
1289
+ """Implement async context manager protocol
1290
+
1291
+ Unregister this instance with the client, so that
1292
+ grpc channels will no longer be warmed"""
1293
+ self.close()