google-cloud-bigtable 2.30.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- google/cloud/bigtable/__init__.py +25 -0
- google/cloud/bigtable/app_profile.py +377 -0
- google/cloud/bigtable/backup.py +490 -0
- google/cloud/bigtable/batcher.py +414 -0
- google/cloud/bigtable/client.py +475 -0
- google/cloud/bigtable/cluster.py +541 -0
- google/cloud/bigtable/column_family.py +362 -0
- google/cloud/bigtable/data/__init__.py +103 -0
- google/cloud/bigtable/data/_async/__init__.py +25 -0
- google/cloud/bigtable/data/_async/_mutate_rows.py +226 -0
- google/cloud/bigtable/data/_async/_read_rows.py +363 -0
- google/cloud/bigtable/data/_async/client.py +1522 -0
- google/cloud/bigtable/data/_async/mutations_batcher.py +534 -0
- google/cloud/bigtable/data/_cross_sync/__init__.py +20 -0
- google/cloud/bigtable/data/_cross_sync/_decorators.py +441 -0
- google/cloud/bigtable/data/_cross_sync/_mapping_meta.py +64 -0
- google/cloud/bigtable/data/_cross_sync/cross_sync.py +334 -0
- google/cloud/bigtable/data/_helpers.py +249 -0
- google/cloud/bigtable/data/_sync_autogen/_mutate_rows.py +182 -0
- google/cloud/bigtable/data/_sync_autogen/_read_rows.py +304 -0
- google/cloud/bigtable/data/_sync_autogen/client.py +1293 -0
- google/cloud/bigtable/data/_sync_autogen/mutations_batcher.py +449 -0
- google/cloud/bigtable/data/exceptions.py +340 -0
- google/cloud/bigtable/data/execute_query/__init__.py +43 -0
- google/cloud/bigtable/data/execute_query/_async/__init__.py +13 -0
- google/cloud/bigtable/data/execute_query/_async/execute_query_iterator.py +291 -0
- google/cloud/bigtable/data/execute_query/_byte_cursor.py +123 -0
- google/cloud/bigtable/data/execute_query/_checksum.py +43 -0
- google/cloud/bigtable/data/execute_query/_parameters_formatting.py +155 -0
- google/cloud/bigtable/data/execute_query/_query_result_parsing_utils.py +134 -0
- google/cloud/bigtable/data/execute_query/_reader.py +122 -0
- google/cloud/bigtable/data/execute_query/_sync_autogen/execute_query_iterator.py +239 -0
- google/cloud/bigtable/data/execute_query/metadata.py +399 -0
- google/cloud/bigtable/data/execute_query/values.py +123 -0
- google/cloud/bigtable/data/mutations.py +380 -0
- google/cloud/bigtable/data/read_modify_write_rules.py +112 -0
- google/cloud/bigtable/data/read_rows_query.py +536 -0
- google/cloud/bigtable/data/row.py +535 -0
- google/cloud/bigtable/data/row_filters.py +968 -0
- google/cloud/bigtable/encryption_info.py +64 -0
- google/cloud/bigtable/enums.py +223 -0
- google/cloud/bigtable/error.py +64 -0
- google/cloud/bigtable/gapic_version.py +16 -0
- google/cloud/bigtable/helpers.py +31 -0
- google/cloud/bigtable/instance.py +789 -0
- google/cloud/bigtable/policy.py +255 -0
- google/cloud/bigtable/row.py +1267 -0
- google/cloud/bigtable/row_data.py +380 -0
- google/cloud/bigtable/row_filters.py +838 -0
- google/cloud/bigtable/row_merger.py +250 -0
- google/cloud/bigtable/row_set.py +213 -0
- google/cloud/bigtable/table.py +1409 -0
- google/cloud/bigtable_admin/__init__.py +410 -0
- google/cloud/bigtable_admin/gapic_version.py +16 -0
- google/cloud/bigtable_admin/py.typed +2 -0
- google/cloud/bigtable_admin_v2/__init__.py +250 -0
- google/cloud/bigtable_admin_v2/gapic_metadata.json +962 -0
- google/cloud/bigtable_admin_v2/gapic_version.py +16 -0
- google/cloud/bigtable_admin_v2/py.typed +2 -0
- google/cloud/bigtable_admin_v2/services/__init__.py +15 -0
- google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/__init__.py +22 -0
- google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py +3465 -0
- google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py +3944 -0
- google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/pagers.py +681 -0
- google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/__init__.py +40 -0
- google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/base.py +751 -0
- google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/grpc.py +1249 -0
- google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/grpc_asyncio.py +1579 -0
- google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/rest.py +6820 -0
- google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/transports/rest_base.py +1746 -0
- google/cloud/bigtable_admin_v2/services/bigtable_table_admin/__init__.py +22 -0
- google/cloud/bigtable_admin_v2/services/bigtable_table_admin/async_client.py +3472 -0
- google/cloud/bigtable_admin_v2/services/bigtable_table_admin/client.py +3949 -0
- google/cloud/bigtable_admin_v2/services/bigtable_table_admin/pagers.py +669 -0
- google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/__init__.py +40 -0
- google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/base.py +708 -0
- google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/grpc.py +1257 -0
- google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/grpc_asyncio.py +1549 -0
- google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/rest.py +6609 -0
- google/cloud/bigtable_admin_v2/services/bigtable_table_admin/transports/rest_base.py +1714 -0
- google/cloud/bigtable_admin_v2/types/__init__.py +248 -0
- google/cloud/bigtable_admin_v2/types/bigtable_instance_admin.py +1364 -0
- google/cloud/bigtable_admin_v2/types/bigtable_table_admin.py +1715 -0
- google/cloud/bigtable_admin_v2/types/common.py +81 -0
- google/cloud/bigtable_admin_v2/types/instance.py +819 -0
- google/cloud/bigtable_admin_v2/types/table.py +1028 -0
- google/cloud/bigtable_admin_v2/types/types.py +776 -0
- google/cloud/bigtable_v2/__init__.py +136 -0
- google/cloud/bigtable_v2/gapic_metadata.json +193 -0
- google/cloud/bigtable_v2/gapic_version.py +16 -0
- google/cloud/bigtable_v2/py.typed +2 -0
- google/cloud/bigtable_v2/services/__init__.py +15 -0
- google/cloud/bigtable_v2/services/bigtable/__init__.py +22 -0
- google/cloud/bigtable_v2/services/bigtable/async_client.py +1741 -0
- google/cloud/bigtable_v2/services/bigtable/client.py +2194 -0
- google/cloud/bigtable_v2/services/bigtable/transports/__init__.py +38 -0
- google/cloud/bigtable_v2/services/bigtable/transports/base.py +343 -0
- google/cloud/bigtable_v2/services/bigtable/transports/grpc.py +660 -0
- google/cloud/bigtable_v2/services/bigtable/transports/grpc_asyncio.py +762 -0
- google/cloud/bigtable_v2/services/bigtable/transports/rest.py +2489 -0
- google/cloud/bigtable_v2/services/bigtable/transports/rest_base.py +711 -0
- google/cloud/bigtable_v2/types/__init__.py +138 -0
- google/cloud/bigtable_v2/types/bigtable.py +1531 -0
- google/cloud/bigtable_v2/types/data.py +1612 -0
- google/cloud/bigtable_v2/types/feature_flags.py +119 -0
- google/cloud/bigtable_v2/types/request_stats.py +171 -0
- google/cloud/bigtable_v2/types/response_params.py +64 -0
- google/cloud/bigtable_v2/types/types.py +579 -0
- google_cloud_bigtable-2.30.0.data/scripts/fixup_bigtable_admin_v2_keywords.py +233 -0
- google_cloud_bigtable-2.30.0.data/scripts/fixup_bigtable_v2_keywords.py +186 -0
- google_cloud_bigtable-2.30.0.dist-info/LICENSE +202 -0
- google_cloud_bigtable-2.30.0.dist-info/METADATA +154 -0
- google_cloud_bigtable-2.30.0.dist-info/RECORD +115 -0
- google_cloud_bigtable-2.30.0.dist-info/WHEEL +5 -0
- google_cloud_bigtable-2.30.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1293 @@
|
|
|
1
|
+
# Copyright 2024 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
#
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# This file is automatically generated by CrossSync. Do not edit manually.
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
from typing import cast, Any, Optional, Set, Sequence, TYPE_CHECKING
|
|
21
|
+
import time
|
|
22
|
+
import warnings
|
|
23
|
+
import random
|
|
24
|
+
import os
|
|
25
|
+
import concurrent.futures
|
|
26
|
+
from functools import partial
|
|
27
|
+
from grpc import Channel
|
|
28
|
+
from google.cloud.bigtable.data.execute_query.values import ExecuteQueryValueType
|
|
29
|
+
from google.cloud.bigtable.data.execute_query.metadata import (
|
|
30
|
+
SqlType,
|
|
31
|
+
_pb_metadata_to_metadata_types,
|
|
32
|
+
)
|
|
33
|
+
from google.cloud.bigtable.data.execute_query._parameters_formatting import (
|
|
34
|
+
_format_execute_query_params,
|
|
35
|
+
_to_param_types,
|
|
36
|
+
)
|
|
37
|
+
from google.cloud.bigtable_v2.services.bigtable.transports.base import (
|
|
38
|
+
DEFAULT_CLIENT_INFO,
|
|
39
|
+
)
|
|
40
|
+
from google.cloud.bigtable_v2.types.bigtable import PingAndWarmRequest
|
|
41
|
+
from google.cloud.client import ClientWithProject
|
|
42
|
+
from google.cloud.environment_vars import BIGTABLE_EMULATOR
|
|
43
|
+
from google.api_core import retry as retries
|
|
44
|
+
from google.api_core.exceptions import DeadlineExceeded
|
|
45
|
+
from google.api_core.exceptions import ServiceUnavailable
|
|
46
|
+
from google.api_core.exceptions import Aborted
|
|
47
|
+
import google.auth.credentials
|
|
48
|
+
import google.auth._default
|
|
49
|
+
from google.api_core import client_options as client_options_lib
|
|
50
|
+
from google.cloud.bigtable.client import _DEFAULT_BIGTABLE_EMULATOR_CLIENT
|
|
51
|
+
from google.cloud.bigtable.data.row import Row
|
|
52
|
+
from google.cloud.bigtable.data.read_rows_query import ReadRowsQuery
|
|
53
|
+
from google.cloud.bigtable.data.exceptions import FailedQueryShardError
|
|
54
|
+
from google.cloud.bigtable.data.exceptions import ShardedReadRowsExceptionGroup
|
|
55
|
+
from google.cloud.bigtable.data._helpers import TABLE_DEFAULT, _align_timeouts
|
|
56
|
+
from google.cloud.bigtable.data._helpers import _WarmedInstanceKey
|
|
57
|
+
from google.cloud.bigtable.data._helpers import _CONCURRENCY_LIMIT
|
|
58
|
+
from google.cloud.bigtable.data._helpers import _retry_exception_factory
|
|
59
|
+
from google.cloud.bigtable.data._helpers import _validate_timeouts
|
|
60
|
+
from google.cloud.bigtable.data._helpers import _get_error_type
|
|
61
|
+
from google.cloud.bigtable.data._helpers import _get_retryable_errors
|
|
62
|
+
from google.cloud.bigtable.data._helpers import _get_timeouts
|
|
63
|
+
from google.cloud.bigtable.data._helpers import _attempt_timeout_generator
|
|
64
|
+
from google.cloud.bigtable.data.mutations import Mutation, RowMutationEntry
|
|
65
|
+
from google.cloud.bigtable.data.read_modify_write_rules import ReadModifyWriteRule
|
|
66
|
+
from google.cloud.bigtable.data.row_filters import RowFilter
|
|
67
|
+
from google.cloud.bigtable.data.row_filters import StripValueTransformerFilter
|
|
68
|
+
from google.cloud.bigtable.data.row_filters import CellsRowLimitFilter
|
|
69
|
+
from google.cloud.bigtable.data.row_filters import RowFilterChain
|
|
70
|
+
from google.cloud.bigtable.data._cross_sync import CrossSync
|
|
71
|
+
from typing import Iterable
|
|
72
|
+
from grpc import insecure_channel
|
|
73
|
+
from grpc import intercept_channel
|
|
74
|
+
from google.cloud.bigtable_v2.services.bigtable.transports import (
|
|
75
|
+
BigtableGrpcTransport as TransportType,
|
|
76
|
+
)
|
|
77
|
+
from google.cloud.bigtable.data._sync_autogen.mutations_batcher import _MB_SIZE
|
|
78
|
+
|
|
79
|
+
if TYPE_CHECKING:
|
|
80
|
+
from google.cloud.bigtable.data._helpers import RowKeySamples
|
|
81
|
+
from google.cloud.bigtable.data._helpers import ShardedQuery
|
|
82
|
+
from google.cloud.bigtable.data._sync_autogen.mutations_batcher import (
|
|
83
|
+
MutationsBatcher,
|
|
84
|
+
)
|
|
85
|
+
from google.cloud.bigtable.data.execute_query._sync_autogen.execute_query_iterator import (
|
|
86
|
+
ExecuteQueryIterator,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@CrossSync._Sync_Impl.add_mapping_decorator("DataClient")
|
|
91
|
+
class BigtableDataClient(ClientWithProject):
|
|
92
|
+
def __init__(
|
|
93
|
+
self,
|
|
94
|
+
*,
|
|
95
|
+
project: str | None = None,
|
|
96
|
+
credentials: google.auth.credentials.Credentials | None = None,
|
|
97
|
+
client_options: dict[str, Any]
|
|
98
|
+
| "google.api_core.client_options.ClientOptions"
|
|
99
|
+
| None = None,
|
|
100
|
+
**kwargs,
|
|
101
|
+
):
|
|
102
|
+
"""Create a client instance for the Bigtable Data API
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
Args:
|
|
107
|
+
project: the project which the client acts on behalf of.
|
|
108
|
+
If not passed, falls back to the default inferred
|
|
109
|
+
from the environment.
|
|
110
|
+
credentials:
|
|
111
|
+
Thehe OAuth2 Credentials to use for this
|
|
112
|
+
client. If not passed (and if no ``_http`` object is
|
|
113
|
+
passed), falls back to the default inferred from the
|
|
114
|
+
environment.
|
|
115
|
+
client_options:
|
|
116
|
+
Client options used to set user options
|
|
117
|
+
on the client. API Endpoint should be set through client_options.
|
|
118
|
+
Raises:
|
|
119
|
+
"""
|
|
120
|
+
if "pool_size" in kwargs:
|
|
121
|
+
warnings.warn("pool_size no longer supported")
|
|
122
|
+
self.client_info = DEFAULT_CLIENT_INFO
|
|
123
|
+
self.client_info.client_library_version = self._client_version()
|
|
124
|
+
if type(client_options) is dict:
|
|
125
|
+
client_options = client_options_lib.from_dict(client_options)
|
|
126
|
+
client_options = cast(
|
|
127
|
+
Optional[client_options_lib.ClientOptions], client_options
|
|
128
|
+
)
|
|
129
|
+
custom_channel = None
|
|
130
|
+
self._emulator_host = os.getenv(BIGTABLE_EMULATOR)
|
|
131
|
+
if self._emulator_host is not None:
|
|
132
|
+
warnings.warn(
|
|
133
|
+
"Connecting to Bigtable emulator at {}".format(self._emulator_host),
|
|
134
|
+
RuntimeWarning,
|
|
135
|
+
stacklevel=2,
|
|
136
|
+
)
|
|
137
|
+
custom_channel = insecure_channel(self._emulator_host)
|
|
138
|
+
if credentials is None:
|
|
139
|
+
credentials = google.auth.credentials.AnonymousCredentials()
|
|
140
|
+
if project is None:
|
|
141
|
+
project = _DEFAULT_BIGTABLE_EMULATOR_CLIENT
|
|
142
|
+
ClientWithProject.__init__(
|
|
143
|
+
self,
|
|
144
|
+
credentials=credentials,
|
|
145
|
+
project=project,
|
|
146
|
+
client_options=client_options,
|
|
147
|
+
)
|
|
148
|
+
self._gapic_client = CrossSync._Sync_Impl.GapicClient(
|
|
149
|
+
credentials=credentials,
|
|
150
|
+
client_options=client_options,
|
|
151
|
+
client_info=self.client_info,
|
|
152
|
+
transport=lambda *args, **kwargs: TransportType(
|
|
153
|
+
*args, **kwargs, channel=custom_channel
|
|
154
|
+
),
|
|
155
|
+
)
|
|
156
|
+
self._is_closed = CrossSync._Sync_Impl.Event()
|
|
157
|
+
self.transport = cast(TransportType, self._gapic_client.transport)
|
|
158
|
+
self._active_instances: Set[_WarmedInstanceKey] = set()
|
|
159
|
+
self._instance_owners: dict[_WarmedInstanceKey, Set[int]] = {}
|
|
160
|
+
self._channel_init_time = time.monotonic()
|
|
161
|
+
self._channel_refresh_task: CrossSync._Sync_Impl.Task[None] | None = None
|
|
162
|
+
self._executor = (
|
|
163
|
+
concurrent.futures.ThreadPoolExecutor()
|
|
164
|
+
if not CrossSync._Sync_Impl.is_async
|
|
165
|
+
else None
|
|
166
|
+
)
|
|
167
|
+
if self._emulator_host is None:
|
|
168
|
+
try:
|
|
169
|
+
self._start_background_channel_refresh()
|
|
170
|
+
except RuntimeError:
|
|
171
|
+
warnings.warn(
|
|
172
|
+
f"{self.__class__.__name__} should be started in an asyncio event loop. Channel refresh will not be started",
|
|
173
|
+
RuntimeWarning,
|
|
174
|
+
stacklevel=2,
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
@staticmethod
|
|
178
|
+
def _client_version() -> str:
|
|
179
|
+
"""Helper function to return the client version string for this client"""
|
|
180
|
+
version_str = f"{google.cloud.bigtable.__version__}-data"
|
|
181
|
+
return version_str
|
|
182
|
+
|
|
183
|
+
def _start_background_channel_refresh(self) -> None:
|
|
184
|
+
"""Starts a background task to ping and warm grpc channel
|
|
185
|
+
|
|
186
|
+
Raises:
|
|
187
|
+
None"""
|
|
188
|
+
if (
|
|
189
|
+
not self._channel_refresh_task
|
|
190
|
+
and (not self._emulator_host)
|
|
191
|
+
and (not self._is_closed.is_set())
|
|
192
|
+
):
|
|
193
|
+
CrossSync._Sync_Impl.verify_async_event_loop()
|
|
194
|
+
self._channel_refresh_task = CrossSync._Sync_Impl.create_task(
|
|
195
|
+
self._manage_channel,
|
|
196
|
+
sync_executor=self._executor,
|
|
197
|
+
task_name=f"{self.__class__.__name__} channel refresh",
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
def close(self, timeout: float | None = 2.0):
|
|
201
|
+
"""Cancel all background tasks"""
|
|
202
|
+
self._is_closed.set()
|
|
203
|
+
if self._channel_refresh_task is not None:
|
|
204
|
+
self._channel_refresh_task.cancel()
|
|
205
|
+
CrossSync._Sync_Impl.wait([self._channel_refresh_task], timeout=timeout)
|
|
206
|
+
self.transport.close()
|
|
207
|
+
if self._executor:
|
|
208
|
+
self._executor.shutdown(wait=False)
|
|
209
|
+
self._channel_refresh_task = None
|
|
210
|
+
|
|
211
|
+
def _ping_and_warm_instances(
|
|
212
|
+
self,
|
|
213
|
+
instance_key: _WarmedInstanceKey | None = None,
|
|
214
|
+
channel: Channel | None = None,
|
|
215
|
+
) -> list[BaseException | None]:
|
|
216
|
+
"""Prepares the backend for requests on a channel
|
|
217
|
+
|
|
218
|
+
Pings each Bigtable instance registered in `_active_instances` on the client
|
|
219
|
+
|
|
220
|
+
Args:
|
|
221
|
+
instance_key: if provided, only warm the instance associated with the key
|
|
222
|
+
channel: grpc channel to warm. If none, warms `self.transport.grpc_channel`
|
|
223
|
+
Returns:
|
|
224
|
+
list[BaseException | None]: sequence of results or exceptions from the ping requests
|
|
225
|
+
"""
|
|
226
|
+
channel = channel or self.transport.grpc_channel
|
|
227
|
+
instance_list = (
|
|
228
|
+
[instance_key] if instance_key is not None else self._active_instances
|
|
229
|
+
)
|
|
230
|
+
ping_rpc = channel.unary_unary(
|
|
231
|
+
"/google.bigtable.v2.Bigtable/PingAndWarm",
|
|
232
|
+
request_serializer=PingAndWarmRequest.serialize,
|
|
233
|
+
)
|
|
234
|
+
partial_list = [
|
|
235
|
+
partial(
|
|
236
|
+
ping_rpc,
|
|
237
|
+
request={"name": instance_name, "app_profile_id": app_profile_id},
|
|
238
|
+
metadata=[
|
|
239
|
+
(
|
|
240
|
+
"x-goog-request-params",
|
|
241
|
+
f"name={instance_name}&app_profile_id={app_profile_id}",
|
|
242
|
+
)
|
|
243
|
+
],
|
|
244
|
+
wait_for_ready=True,
|
|
245
|
+
)
|
|
246
|
+
for (instance_name, table_name, app_profile_id) in instance_list
|
|
247
|
+
]
|
|
248
|
+
result_list = CrossSync._Sync_Impl.gather_partials(
|
|
249
|
+
partial_list, return_exceptions=True, sync_executor=self._executor
|
|
250
|
+
)
|
|
251
|
+
return [r or None for r in result_list]
|
|
252
|
+
|
|
253
|
+
def _manage_channel(
|
|
254
|
+
self,
|
|
255
|
+
refresh_interval_min: float = 60 * 35,
|
|
256
|
+
refresh_interval_max: float = 60 * 45,
|
|
257
|
+
grace_period: float = 60 * 10,
|
|
258
|
+
) -> None:
|
|
259
|
+
"""Background task that periodically refreshes and warms a grpc channel
|
|
260
|
+
|
|
261
|
+
The backend will automatically close channels after 60 minutes, so
|
|
262
|
+
`refresh_interval` + `grace_period` should be < 60 minutes
|
|
263
|
+
|
|
264
|
+
Runs continuously until the client is closed
|
|
265
|
+
|
|
266
|
+
Args:
|
|
267
|
+
refresh_interval_min: minimum interval before initiating refresh
|
|
268
|
+
process in seconds. Actual interval will be a random value
|
|
269
|
+
between `refresh_interval_min` and `refresh_interval_max`
|
|
270
|
+
refresh_interval_max: maximum interval before initiating refresh
|
|
271
|
+
process in seconds. Actual interval will be a random value
|
|
272
|
+
between `refresh_interval_min` and `refresh_interval_max`
|
|
273
|
+
grace_period: time to allow previous channel to serve existing
|
|
274
|
+
requests before closing, in seconds"""
|
|
275
|
+
first_refresh = self._channel_init_time + random.uniform(
|
|
276
|
+
refresh_interval_min, refresh_interval_max
|
|
277
|
+
)
|
|
278
|
+
next_sleep = max(first_refresh - time.monotonic(), 0)
|
|
279
|
+
if next_sleep > 0:
|
|
280
|
+
self._ping_and_warm_instances(channel=self.transport.grpc_channel)
|
|
281
|
+
while not self._is_closed.is_set():
|
|
282
|
+
CrossSync._Sync_Impl.event_wait(
|
|
283
|
+
self._is_closed, next_sleep, async_break_early=False
|
|
284
|
+
)
|
|
285
|
+
if self._is_closed.is_set():
|
|
286
|
+
break
|
|
287
|
+
start_timestamp = time.monotonic()
|
|
288
|
+
old_channel = self.transport.grpc_channel
|
|
289
|
+
new_channel = self.transport.create_channel()
|
|
290
|
+
new_channel = intercept_channel(new_channel, self.transport._interceptor)
|
|
291
|
+
self._ping_and_warm_instances(channel=new_channel)
|
|
292
|
+
self.transport._grpc_channel = new_channel
|
|
293
|
+
self.transport._logged_channel = new_channel
|
|
294
|
+
self.transport._stubs = {}
|
|
295
|
+
self.transport._prep_wrapped_messages(self.client_info)
|
|
296
|
+
if grace_period:
|
|
297
|
+
self._is_closed.wait(grace_period)
|
|
298
|
+
old_channel.close()
|
|
299
|
+
next_refresh = random.uniform(refresh_interval_min, refresh_interval_max)
|
|
300
|
+
next_sleep = max(next_refresh - (time.monotonic() - start_timestamp), 0)
|
|
301
|
+
|
|
302
|
+
def _register_instance(
|
|
303
|
+
self, instance_id: str, owner: Table | ExecuteQueryIterator
|
|
304
|
+
) -> None:
|
|
305
|
+
"""Registers an instance with the client, and warms the channel for the instance
|
|
306
|
+
The client will periodically refresh grpc channel used to make
|
|
307
|
+
requests, and new channels will be warmed for each registered instance
|
|
308
|
+
Channels will not be refreshed unless at least one instance is registered
|
|
309
|
+
|
|
310
|
+
Args:
|
|
311
|
+
instance_id: id of the instance to register.
|
|
312
|
+
owner: table that owns the instance. Owners will be tracked in
|
|
313
|
+
_instance_owners, and instances will only be unregistered when all
|
|
314
|
+
owners call _remove_instance_registration"""
|
|
315
|
+
instance_name = self._gapic_client.instance_path(self.project, instance_id)
|
|
316
|
+
instance_key = _WarmedInstanceKey(
|
|
317
|
+
instance_name, owner.table_name, owner.app_profile_id
|
|
318
|
+
)
|
|
319
|
+
self._instance_owners.setdefault(instance_key, set()).add(id(owner))
|
|
320
|
+
if instance_key not in self._active_instances:
|
|
321
|
+
self._active_instances.add(instance_key)
|
|
322
|
+
if self._channel_refresh_task:
|
|
323
|
+
self._ping_and_warm_instances(instance_key)
|
|
324
|
+
else:
|
|
325
|
+
self._start_background_channel_refresh()
|
|
326
|
+
|
|
327
|
+
def _remove_instance_registration(
|
|
328
|
+
self, instance_id: str, owner: Table | "ExecuteQueryIterator"
|
|
329
|
+
) -> bool:
|
|
330
|
+
"""Removes an instance from the client's registered instances, to prevent
|
|
331
|
+
warming new channels for the instance
|
|
332
|
+
|
|
333
|
+
If instance_id is not registered, or is still in use by other tables, returns False
|
|
334
|
+
|
|
335
|
+
Args:
|
|
336
|
+
instance_id: id of the instance to remove
|
|
337
|
+
owner: table that owns the instance. Owners will be tracked in
|
|
338
|
+
_instance_owners, and instances will only be unregistered when all
|
|
339
|
+
owners call _remove_instance_registration
|
|
340
|
+
Returns:
|
|
341
|
+
bool: True if instance was removed, else False"""
|
|
342
|
+
instance_name = self._gapic_client.instance_path(self.project, instance_id)
|
|
343
|
+
instance_key = _WarmedInstanceKey(
|
|
344
|
+
instance_name, owner.table_name, owner.app_profile_id
|
|
345
|
+
)
|
|
346
|
+
owner_list = self._instance_owners.get(instance_key, set())
|
|
347
|
+
try:
|
|
348
|
+
owner_list.remove(id(owner))
|
|
349
|
+
if len(owner_list) == 0:
|
|
350
|
+
self._active_instances.remove(instance_key)
|
|
351
|
+
return True
|
|
352
|
+
except KeyError:
|
|
353
|
+
return False
|
|
354
|
+
|
|
355
|
+
def get_table(self, instance_id: str, table_id: str, *args, **kwargs) -> Table:
|
|
356
|
+
"""Returns a table instance for making data API requests. All arguments are passed
|
|
357
|
+
directly to the Table constructor.
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
Args:
|
|
362
|
+
instance_id: The Bigtable instance ID to associate with this client.
|
|
363
|
+
instance_id is combined with the client's project to fully
|
|
364
|
+
specify the instance
|
|
365
|
+
table_id: The ID of the table. table_id is combined with the
|
|
366
|
+
instance_id and the client's project to fully specify the table
|
|
367
|
+
app_profile_id: The app profile to associate with requests.
|
|
368
|
+
https://cloud.google.com/bigtable/docs/app-profiles
|
|
369
|
+
default_read_rows_operation_timeout: The default timeout for read rows
|
|
370
|
+
operations, in seconds. If not set, defaults to 600 seconds (10 minutes)
|
|
371
|
+
default_read_rows_attempt_timeout: The default timeout for individual
|
|
372
|
+
read rows rpc requests, in seconds. If not set, defaults to 20 seconds
|
|
373
|
+
default_mutate_rows_operation_timeout: The default timeout for mutate rows
|
|
374
|
+
operations, in seconds. If not set, defaults to 600 seconds (10 minutes)
|
|
375
|
+
default_mutate_rows_attempt_timeout: The default timeout for individual
|
|
376
|
+
mutate rows rpc requests, in seconds. If not set, defaults to 60 seconds
|
|
377
|
+
default_operation_timeout: The default timeout for all other operations, in
|
|
378
|
+
seconds. If not set, defaults to 60 seconds
|
|
379
|
+
default_attempt_timeout: The default timeout for all other individual rpc
|
|
380
|
+
requests, in seconds. If not set, defaults to 20 seconds
|
|
381
|
+
default_read_rows_retryable_errors: a list of errors that will be retried
|
|
382
|
+
if encountered during read_rows and related operations.
|
|
383
|
+
Defaults to 4 (DeadlineExceeded), 14 (ServiceUnavailable), and 10 (Aborted)
|
|
384
|
+
default_mutate_rows_retryable_errors: a list of errors that will be retried
|
|
385
|
+
if encountered during mutate_rows and related operations.
|
|
386
|
+
Defaults to 4 (DeadlineExceeded) and 14 (ServiceUnavailable)
|
|
387
|
+
default_retryable_errors: a list of errors that will be retried if
|
|
388
|
+
encountered during all other operations.
|
|
389
|
+
Defaults to 4 (DeadlineExceeded) and 14 (ServiceUnavailable)
|
|
390
|
+
Returns:
|
|
391
|
+
Table: a table instance for making data API requests
|
|
392
|
+
Raises:
|
|
393
|
+
None"""
|
|
394
|
+
return Table(self, instance_id, table_id, *args, **kwargs)
|
|
395
|
+
|
|
396
|
+
def execute_query(
|
|
397
|
+
self,
|
|
398
|
+
query: str,
|
|
399
|
+
instance_id: str,
|
|
400
|
+
*,
|
|
401
|
+
parameters: dict[str, ExecuteQueryValueType] | None = None,
|
|
402
|
+
parameter_types: dict[str, SqlType.Type] | None = None,
|
|
403
|
+
app_profile_id: str | None = None,
|
|
404
|
+
operation_timeout: float = 600,
|
|
405
|
+
attempt_timeout: float | None = 20,
|
|
406
|
+
retryable_errors: Sequence[type[Exception]] = (
|
|
407
|
+
DeadlineExceeded,
|
|
408
|
+
ServiceUnavailable,
|
|
409
|
+
Aborted,
|
|
410
|
+
),
|
|
411
|
+
prepare_operation_timeout: float = 60,
|
|
412
|
+
prepare_attempt_timeout: float | None = 20,
|
|
413
|
+
prepare_retryable_errors: Sequence[type[Exception]] = (
|
|
414
|
+
DeadlineExceeded,
|
|
415
|
+
ServiceUnavailable,
|
|
416
|
+
),
|
|
417
|
+
) -> "ExecuteQueryIterator":
|
|
418
|
+
"""Executes an SQL query on an instance.
|
|
419
|
+
Returns an iterator to asynchronously stream back columns from selected rows.
|
|
420
|
+
|
|
421
|
+
Failed requests within operation_timeout will be retried based on the
|
|
422
|
+
retryable_errors list until operation_timeout is reached.
|
|
423
|
+
|
|
424
|
+
Note that this makes two requests, one to ``PrepareQuery`` and one to ``ExecuteQuery``.
|
|
425
|
+
These have separate retry configurations. ``ExecuteQuery`` is where the bulk of the
|
|
426
|
+
work happens.
|
|
427
|
+
|
|
428
|
+
Args:
|
|
429
|
+
query: Query to be run on Bigtable instance. The query can use ``@param``
|
|
430
|
+
placeholders to use parameter interpolation on the server. Values for all
|
|
431
|
+
parameters should be provided in ``parameters``. Types of parameters are
|
|
432
|
+
inferred but should be provided in ``parameter_types`` if the inference is
|
|
433
|
+
not possible (i.e. when value can be None, an empty list or an empty dict).
|
|
434
|
+
instance_id: The Bigtable instance ID to perform the query on.
|
|
435
|
+
instance_id is combined with the client's project to fully
|
|
436
|
+
specify the instance.
|
|
437
|
+
parameters: Dictionary with values for all parameters used in the ``query``.
|
|
438
|
+
parameter_types: Dictionary with types of parameters used in the ``query``.
|
|
439
|
+
Required to contain entries only for parameters whose type cannot be
|
|
440
|
+
detected automatically (i.e. the value can be None, an empty list or
|
|
441
|
+
an empty dict).
|
|
442
|
+
app_profile_id: The app profile to associate with requests.
|
|
443
|
+
https://cloud.google.com/bigtable/docs/app-profiles
|
|
444
|
+
operation_timeout: the time budget for the entire executeQuery operation, in seconds.
|
|
445
|
+
Failed requests will be retried within the budget.
|
|
446
|
+
Defaults to 600 seconds.
|
|
447
|
+
attempt_timeout: the time budget for an individual executeQuery network request, in seconds.
|
|
448
|
+
If it takes longer than this time to complete, the request will be cancelled with
|
|
449
|
+
a DeadlineExceeded exception, and a retry will be attempted.
|
|
450
|
+
Defaults to the 20 seconds.
|
|
451
|
+
If None, defaults to operation_timeout.
|
|
452
|
+
retryable_errors: a list of errors that will be retried if encountered during executeQuery.
|
|
453
|
+
Defaults to 4 (DeadlineExceeded), 14 (ServiceUnavailable), and 10 (Aborted)
|
|
454
|
+
prepare_operation_timeout: the time budget for the entire prepareQuery operation, in seconds.
|
|
455
|
+
Failed requests will be retried within the budget.
|
|
456
|
+
Defaults to 60 seconds.
|
|
457
|
+
prepare_attempt_timeout: the time budget for an individual prepareQuery network request, in seconds.
|
|
458
|
+
If it takes longer than this time to complete, the request will be cancelled with
|
|
459
|
+
a DeadlineExceeded exception, and a retry will be attempted.
|
|
460
|
+
Defaults to the 20 seconds.
|
|
461
|
+
If None, defaults to prepare_operation_timeout.
|
|
462
|
+
prepare_retryable_errors: a list of errors that will be retried if encountered during prepareQuery.
|
|
463
|
+
Defaults to 4 (DeadlineExceeded) and 14 (ServiceUnavailable)
|
|
464
|
+
Returns:
|
|
465
|
+
ExecuteQueryIterator: an asynchronous iterator that yields rows returned by the query
|
|
466
|
+
Raises:
|
|
467
|
+
google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
|
|
468
|
+
will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
|
|
469
|
+
from any retries that failed
|
|
470
|
+
google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
|
|
471
|
+
google.cloud.bigtable.data.exceptions.ParameterTypeInferenceFailed: Raised if
|
|
472
|
+
a parameter is passed without an explicit type, and the type cannot be infered
|
|
473
|
+
"""
|
|
474
|
+
instance_name = self._gapic_client.instance_path(self.project, instance_id)
|
|
475
|
+
converted_param_types = _to_param_types(parameters, parameter_types)
|
|
476
|
+
prepare_request = {
|
|
477
|
+
"instance_name": instance_name,
|
|
478
|
+
"query": query,
|
|
479
|
+
"app_profile_id": app_profile_id,
|
|
480
|
+
"param_types": converted_param_types,
|
|
481
|
+
"proto_format": {},
|
|
482
|
+
}
|
|
483
|
+
prepare_predicate = retries.if_exception_type(
|
|
484
|
+
*[_get_error_type(e) for e in prepare_retryable_errors]
|
|
485
|
+
)
|
|
486
|
+
(prepare_operation_timeout, prepare_attempt_timeout) = _align_timeouts(
|
|
487
|
+
prepare_operation_timeout, prepare_attempt_timeout
|
|
488
|
+
)
|
|
489
|
+
prepare_sleep_generator = retries.exponential_sleep_generator(0.01, 2, 60)
|
|
490
|
+
target = partial(
|
|
491
|
+
self._gapic_client.prepare_query,
|
|
492
|
+
request=prepare_request,
|
|
493
|
+
timeout=prepare_attempt_timeout,
|
|
494
|
+
retry=None,
|
|
495
|
+
)
|
|
496
|
+
prepare_result = CrossSync._Sync_Impl.retry_target(
|
|
497
|
+
target,
|
|
498
|
+
prepare_predicate,
|
|
499
|
+
prepare_sleep_generator,
|
|
500
|
+
prepare_operation_timeout,
|
|
501
|
+
exception_factory=_retry_exception_factory,
|
|
502
|
+
)
|
|
503
|
+
prepare_metadata = _pb_metadata_to_metadata_types(prepare_result.metadata)
|
|
504
|
+
retryable_excs = [_get_error_type(e) for e in retryable_errors]
|
|
505
|
+
pb_params = _format_execute_query_params(parameters, parameter_types)
|
|
506
|
+
request_body = {
|
|
507
|
+
"instance_name": instance_name,
|
|
508
|
+
"app_profile_id": app_profile_id,
|
|
509
|
+
"prepared_query": prepare_result.prepared_query,
|
|
510
|
+
"params": pb_params,
|
|
511
|
+
}
|
|
512
|
+
(operation_timeout, attempt_timeout) = _align_timeouts(
|
|
513
|
+
operation_timeout, attempt_timeout
|
|
514
|
+
)
|
|
515
|
+
return CrossSync._Sync_Impl.ExecuteQueryIterator(
|
|
516
|
+
self,
|
|
517
|
+
instance_id,
|
|
518
|
+
app_profile_id,
|
|
519
|
+
request_body,
|
|
520
|
+
prepare_metadata,
|
|
521
|
+
attempt_timeout,
|
|
522
|
+
operation_timeout,
|
|
523
|
+
retryable_excs=retryable_excs,
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
def __enter__(self):
|
|
527
|
+
self._start_background_channel_refresh()
|
|
528
|
+
return self
|
|
529
|
+
|
|
530
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
531
|
+
self.close()
|
|
532
|
+
self._gapic_client.__exit__(exc_type, exc_val, exc_tb)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
@CrossSync._Sync_Impl.add_mapping_decorator("Table")
|
|
536
|
+
class Table:
|
|
537
|
+
"""
|
|
538
|
+
Main Data API surface
|
|
539
|
+
|
|
540
|
+
Table object maintains table_id, and app_profile_id context, and passes them with
|
|
541
|
+
each call
|
|
542
|
+
"""
|
|
543
|
+
|
|
544
|
+
def __init__(
|
|
545
|
+
self,
|
|
546
|
+
client: BigtableDataClient,
|
|
547
|
+
instance_id: str,
|
|
548
|
+
table_id: str,
|
|
549
|
+
app_profile_id: str | None = None,
|
|
550
|
+
*,
|
|
551
|
+
default_read_rows_operation_timeout: float = 600,
|
|
552
|
+
default_read_rows_attempt_timeout: float | None = 20,
|
|
553
|
+
default_mutate_rows_operation_timeout: float = 600,
|
|
554
|
+
default_mutate_rows_attempt_timeout: float | None = 60,
|
|
555
|
+
default_operation_timeout: float = 60,
|
|
556
|
+
default_attempt_timeout: float | None = 20,
|
|
557
|
+
default_read_rows_retryable_errors: Sequence[type[Exception]] = (
|
|
558
|
+
DeadlineExceeded,
|
|
559
|
+
ServiceUnavailable,
|
|
560
|
+
Aborted,
|
|
561
|
+
),
|
|
562
|
+
default_mutate_rows_retryable_errors: Sequence[type[Exception]] = (
|
|
563
|
+
DeadlineExceeded,
|
|
564
|
+
ServiceUnavailable,
|
|
565
|
+
),
|
|
566
|
+
default_retryable_errors: Sequence[type[Exception]] = (
|
|
567
|
+
DeadlineExceeded,
|
|
568
|
+
ServiceUnavailable,
|
|
569
|
+
),
|
|
570
|
+
):
|
|
571
|
+
"""Initialize a Table instance
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
Args:
|
|
576
|
+
instance_id: The Bigtable instance ID to associate with this client.
|
|
577
|
+
instance_id is combined with the client's project to fully
|
|
578
|
+
specify the instance
|
|
579
|
+
table_id: The ID of the table. table_id is combined with the
|
|
580
|
+
instance_id and the client's project to fully specify the table
|
|
581
|
+
app_profile_id: The app profile to associate with requests.
|
|
582
|
+
https://cloud.google.com/bigtable/docs/app-profiles
|
|
583
|
+
default_read_rows_operation_timeout: The default timeout for read rows
|
|
584
|
+
operations, in seconds. If not set, defaults to 600 seconds (10 minutes)
|
|
585
|
+
default_read_rows_attempt_timeout: The default timeout for individual
|
|
586
|
+
read rows rpc requests, in seconds. If not set, defaults to 20 seconds
|
|
587
|
+
default_mutate_rows_operation_timeout: The default timeout for mutate rows
|
|
588
|
+
operations, in seconds. If not set, defaults to 600 seconds (10 minutes)
|
|
589
|
+
default_mutate_rows_attempt_timeout: The default timeout for individual
|
|
590
|
+
mutate rows rpc requests, in seconds. If not set, defaults to 60 seconds
|
|
591
|
+
default_operation_timeout: The default timeout for all other operations, in
|
|
592
|
+
seconds. If not set, defaults to 60 seconds
|
|
593
|
+
default_attempt_timeout: The default timeout for all other individual rpc
|
|
594
|
+
requests, in seconds. If not set, defaults to 20 seconds
|
|
595
|
+
default_read_rows_retryable_errors: a list of errors that will be retried
|
|
596
|
+
if encountered during read_rows and related operations.
|
|
597
|
+
Defaults to 4 (DeadlineExceeded), 14 (ServiceUnavailable), and 10 (Aborted)
|
|
598
|
+
default_mutate_rows_retryable_errors: a list of errors that will be retried
|
|
599
|
+
if encountered during mutate_rows and related operations.
|
|
600
|
+
Defaults to 4 (DeadlineExceeded) and 14 (ServiceUnavailable)
|
|
601
|
+
default_retryable_errors: a list of errors that will be retried if
|
|
602
|
+
encountered during all other operations.
|
|
603
|
+
Defaults to 4 (DeadlineExceeded) and 14 (ServiceUnavailable)
|
|
604
|
+
Raises:
|
|
605
|
+
None"""
|
|
606
|
+
_validate_timeouts(
|
|
607
|
+
default_operation_timeout, default_attempt_timeout, allow_none=True
|
|
608
|
+
)
|
|
609
|
+
_validate_timeouts(
|
|
610
|
+
default_read_rows_operation_timeout,
|
|
611
|
+
default_read_rows_attempt_timeout,
|
|
612
|
+
allow_none=True,
|
|
613
|
+
)
|
|
614
|
+
_validate_timeouts(
|
|
615
|
+
default_mutate_rows_operation_timeout,
|
|
616
|
+
default_mutate_rows_attempt_timeout,
|
|
617
|
+
allow_none=True,
|
|
618
|
+
)
|
|
619
|
+
self.client = client
|
|
620
|
+
self.instance_id = instance_id
|
|
621
|
+
self.instance_name = self.client._gapic_client.instance_path(
|
|
622
|
+
self.client.project, instance_id
|
|
623
|
+
)
|
|
624
|
+
self.table_id = table_id
|
|
625
|
+
self.table_name = self.client._gapic_client.table_path(
|
|
626
|
+
self.client.project, instance_id, table_id
|
|
627
|
+
)
|
|
628
|
+
self.app_profile_id = app_profile_id
|
|
629
|
+
self.default_operation_timeout = default_operation_timeout
|
|
630
|
+
self.default_attempt_timeout = default_attempt_timeout
|
|
631
|
+
self.default_read_rows_operation_timeout = default_read_rows_operation_timeout
|
|
632
|
+
self.default_read_rows_attempt_timeout = default_read_rows_attempt_timeout
|
|
633
|
+
self.default_mutate_rows_operation_timeout = (
|
|
634
|
+
default_mutate_rows_operation_timeout
|
|
635
|
+
)
|
|
636
|
+
self.default_mutate_rows_attempt_timeout = default_mutate_rows_attempt_timeout
|
|
637
|
+
self.default_read_rows_retryable_errors = (
|
|
638
|
+
default_read_rows_retryable_errors or ()
|
|
639
|
+
)
|
|
640
|
+
self.default_mutate_rows_retryable_errors = (
|
|
641
|
+
default_mutate_rows_retryable_errors or ()
|
|
642
|
+
)
|
|
643
|
+
self.default_retryable_errors = default_retryable_errors or ()
|
|
644
|
+
try:
|
|
645
|
+
self._register_instance_future = CrossSync._Sync_Impl.create_task(
|
|
646
|
+
self.client._register_instance,
|
|
647
|
+
self.instance_id,
|
|
648
|
+
self,
|
|
649
|
+
sync_executor=self.client._executor,
|
|
650
|
+
)
|
|
651
|
+
except RuntimeError as e:
|
|
652
|
+
raise RuntimeError(
|
|
653
|
+
f"{self.__class__.__name__} must be created within an async event loop context."
|
|
654
|
+
) from e
|
|
655
|
+
|
|
656
|
+
def read_rows_stream(
|
|
657
|
+
self,
|
|
658
|
+
query: ReadRowsQuery,
|
|
659
|
+
*,
|
|
660
|
+
operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
661
|
+
attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
662
|
+
retryable_errors: Sequence[type[Exception]]
|
|
663
|
+
| TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
664
|
+
) -> Iterable[Row]:
|
|
665
|
+
"""Read a set of rows from the table, based on the specified query.
|
|
666
|
+
Returns an iterator to asynchronously stream back row data.
|
|
667
|
+
|
|
668
|
+
Failed requests within operation_timeout will be retried based on the
|
|
669
|
+
retryable_errors list until operation_timeout is reached.
|
|
670
|
+
|
|
671
|
+
Args:
|
|
672
|
+
query: contains details about which rows to return
|
|
673
|
+
operation_timeout: the time budget for the entire operation, in seconds.
|
|
674
|
+
Failed requests will be retried within the budget.
|
|
675
|
+
Defaults to the Table's default_read_rows_operation_timeout
|
|
676
|
+
attempt_timeout: the time budget for an individual network request, in seconds.
|
|
677
|
+
If it takes longer than this time to complete, the request will be cancelled with
|
|
678
|
+
a DeadlineExceeded exception, and a retry will be attempted.
|
|
679
|
+
Defaults to the Table's default_read_rows_attempt_timeout.
|
|
680
|
+
If None, defaults to operation_timeout.
|
|
681
|
+
retryable_errors: a list of errors that will be retried if encountered.
|
|
682
|
+
Defaults to the Table's default_read_rows_retryable_errors
|
|
683
|
+
Returns:
|
|
684
|
+
Iterable[Row]: an asynchronous iterator that yields rows returned by the query
|
|
685
|
+
Raises:
|
|
686
|
+
google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
|
|
687
|
+
will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
|
|
688
|
+
from any retries that failed
|
|
689
|
+
google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
|
|
690
|
+
"""
|
|
691
|
+
(operation_timeout, attempt_timeout) = _get_timeouts(
|
|
692
|
+
operation_timeout, attempt_timeout, self
|
|
693
|
+
)
|
|
694
|
+
retryable_excs = _get_retryable_errors(retryable_errors, self)
|
|
695
|
+
row_merger = CrossSync._Sync_Impl._ReadRowsOperation(
|
|
696
|
+
query,
|
|
697
|
+
self,
|
|
698
|
+
operation_timeout=operation_timeout,
|
|
699
|
+
attempt_timeout=attempt_timeout,
|
|
700
|
+
retryable_exceptions=retryable_excs,
|
|
701
|
+
)
|
|
702
|
+
return row_merger.start_operation()
|
|
703
|
+
|
|
704
|
+
def read_rows(
|
|
705
|
+
self,
|
|
706
|
+
query: ReadRowsQuery,
|
|
707
|
+
*,
|
|
708
|
+
operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
709
|
+
attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
710
|
+
retryable_errors: Sequence[type[Exception]]
|
|
711
|
+
| TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
712
|
+
) -> list[Row]:
|
|
713
|
+
"""Read a set of rows from the table, based on the specified query.
|
|
714
|
+
Retruns results as a list of Row objects when the request is complete.
|
|
715
|
+
For streamed results, use read_rows_stream.
|
|
716
|
+
|
|
717
|
+
Failed requests within operation_timeout will be retried based on the
|
|
718
|
+
retryable_errors list until operation_timeout is reached.
|
|
719
|
+
|
|
720
|
+
Args:
|
|
721
|
+
query: contains details about which rows to return
|
|
722
|
+
operation_timeout: the time budget for the entire operation, in seconds.
|
|
723
|
+
Failed requests will be retried within the budget.
|
|
724
|
+
Defaults to the Table's default_read_rows_operation_timeout
|
|
725
|
+
attempt_timeout: the time budget for an individual network request, in seconds.
|
|
726
|
+
If it takes longer than this time to complete, the request will be cancelled with
|
|
727
|
+
a DeadlineExceeded exception, and a retry will be attempted.
|
|
728
|
+
Defaults to the Table's default_read_rows_attempt_timeout.
|
|
729
|
+
If None, defaults to operation_timeout.
|
|
730
|
+
If None, defaults to the Table's default_read_rows_attempt_timeout,
|
|
731
|
+
or the operation_timeout if that is also None.
|
|
732
|
+
retryable_errors: a list of errors that will be retried if encountered.
|
|
733
|
+
Defaults to the Table's default_read_rows_retryable_errors.
|
|
734
|
+
Returns:
|
|
735
|
+
list[Row]: a list of Rows returned by the query
|
|
736
|
+
Raises:
|
|
737
|
+
google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
|
|
738
|
+
will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
|
|
739
|
+
from any retries that failed
|
|
740
|
+
google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
|
|
741
|
+
"""
|
|
742
|
+
row_generator = self.read_rows_stream(
|
|
743
|
+
query,
|
|
744
|
+
operation_timeout=operation_timeout,
|
|
745
|
+
attempt_timeout=attempt_timeout,
|
|
746
|
+
retryable_errors=retryable_errors,
|
|
747
|
+
)
|
|
748
|
+
return [row for row in row_generator]
|
|
749
|
+
|
|
750
|
+
def read_row(
|
|
751
|
+
self,
|
|
752
|
+
row_key: str | bytes,
|
|
753
|
+
*,
|
|
754
|
+
row_filter: RowFilter | None = None,
|
|
755
|
+
operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
756
|
+
attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
757
|
+
retryable_errors: Sequence[type[Exception]]
|
|
758
|
+
| TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
759
|
+
) -> Row | None:
|
|
760
|
+
"""Read a single row from the table, based on the specified key.
|
|
761
|
+
|
|
762
|
+
Failed requests within operation_timeout will be retried based on the
|
|
763
|
+
retryable_errors list until operation_timeout is reached.
|
|
764
|
+
|
|
765
|
+
Args:
|
|
766
|
+
query: contains details about which rows to return
|
|
767
|
+
operation_timeout: the time budget for the entire operation, in seconds.
|
|
768
|
+
Failed requests will be retried within the budget.
|
|
769
|
+
Defaults to the Table's default_read_rows_operation_timeout
|
|
770
|
+
attempt_timeout: the time budget for an individual network request, in seconds.
|
|
771
|
+
If it takes longer than this time to complete, the request will be cancelled with
|
|
772
|
+
a DeadlineExceeded exception, and a retry will be attempted.
|
|
773
|
+
Defaults to the Table's default_read_rows_attempt_timeout.
|
|
774
|
+
If None, defaults to operation_timeout.
|
|
775
|
+
retryable_errors: a list of errors that will be retried if encountered.
|
|
776
|
+
Defaults to the Table's default_read_rows_retryable_errors.
|
|
777
|
+
Returns:
|
|
778
|
+
Row | None: a Row object if the row exists, otherwise None
|
|
779
|
+
Raises:
|
|
780
|
+
google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
|
|
781
|
+
will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
|
|
782
|
+
from any retries that failed
|
|
783
|
+
google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
|
|
784
|
+
"""
|
|
785
|
+
if row_key is None:
|
|
786
|
+
raise ValueError("row_key must be string or bytes")
|
|
787
|
+
query = ReadRowsQuery(row_keys=row_key, row_filter=row_filter, limit=1)
|
|
788
|
+
results = self.read_rows(
|
|
789
|
+
query,
|
|
790
|
+
operation_timeout=operation_timeout,
|
|
791
|
+
attempt_timeout=attempt_timeout,
|
|
792
|
+
retryable_errors=retryable_errors,
|
|
793
|
+
)
|
|
794
|
+
if len(results) == 0:
|
|
795
|
+
return None
|
|
796
|
+
return results[0]
|
|
797
|
+
|
|
798
|
+
def read_rows_sharded(
|
|
799
|
+
self,
|
|
800
|
+
sharded_query: ShardedQuery,
|
|
801
|
+
*,
|
|
802
|
+
operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
803
|
+
attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
804
|
+
retryable_errors: Sequence[type[Exception]]
|
|
805
|
+
| TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
806
|
+
) -> list[Row]:
|
|
807
|
+
"""Runs a sharded query in parallel, then return the results in a single list.
|
|
808
|
+
Results will be returned in the order of the input queries.
|
|
809
|
+
|
|
810
|
+
This function is intended to be run on the results on a query.shard() call.
|
|
811
|
+
For example::
|
|
812
|
+
|
|
813
|
+
table_shard_keys = await table.sample_row_keys()
|
|
814
|
+
query = ReadRowsQuery(...)
|
|
815
|
+
shard_queries = query.shard(table_shard_keys)
|
|
816
|
+
results = await table.read_rows_sharded(shard_queries)
|
|
817
|
+
|
|
818
|
+
Args:
|
|
819
|
+
sharded_query: a sharded query to execute
|
|
820
|
+
operation_timeout: the time budget for the entire operation, in seconds.
|
|
821
|
+
Failed requests will be retried within the budget.
|
|
822
|
+
Defaults to the Table's default_read_rows_operation_timeout
|
|
823
|
+
attempt_timeout: the time budget for an individual network request, in seconds.
|
|
824
|
+
If it takes longer than this time to complete, the request will be cancelled with
|
|
825
|
+
a DeadlineExceeded exception, and a retry will be attempted.
|
|
826
|
+
Defaults to the Table's default_read_rows_attempt_timeout.
|
|
827
|
+
If None, defaults to operation_timeout.
|
|
828
|
+
retryable_errors: a list of errors that will be retried if encountered.
|
|
829
|
+
Defaults to the Table's default_read_rows_retryable_errors.
|
|
830
|
+
Returns:
|
|
831
|
+
list[Row]: a list of Rows returned by the query
|
|
832
|
+
Raises:
|
|
833
|
+
ShardedReadRowsExceptionGroup: if any of the queries failed
|
|
834
|
+
ValueError: if the query_list is empty"""
|
|
835
|
+
if not sharded_query:
|
|
836
|
+
raise ValueError("empty sharded_query")
|
|
837
|
+
(operation_timeout, attempt_timeout) = _get_timeouts(
|
|
838
|
+
operation_timeout, attempt_timeout, self
|
|
839
|
+
)
|
|
840
|
+
rpc_timeout_generator = _attempt_timeout_generator(
|
|
841
|
+
operation_timeout, operation_timeout
|
|
842
|
+
)
|
|
843
|
+
concurrency_sem = CrossSync._Sync_Impl.Semaphore(_CONCURRENCY_LIMIT)
|
|
844
|
+
|
|
845
|
+
def read_rows_with_semaphore(query):
|
|
846
|
+
with concurrency_sem:
|
|
847
|
+
shard_timeout = next(rpc_timeout_generator)
|
|
848
|
+
if shard_timeout <= 0:
|
|
849
|
+
raise DeadlineExceeded(
|
|
850
|
+
"Operation timeout exceeded before starting query"
|
|
851
|
+
)
|
|
852
|
+
return self.read_rows(
|
|
853
|
+
query,
|
|
854
|
+
operation_timeout=shard_timeout,
|
|
855
|
+
attempt_timeout=min(attempt_timeout, shard_timeout),
|
|
856
|
+
retryable_errors=retryable_errors,
|
|
857
|
+
)
|
|
858
|
+
|
|
859
|
+
routine_list = [
|
|
860
|
+
partial(read_rows_with_semaphore, query) for query in sharded_query
|
|
861
|
+
]
|
|
862
|
+
batch_result = CrossSync._Sync_Impl.gather_partials(
|
|
863
|
+
routine_list, return_exceptions=True, sync_executor=self.client._executor
|
|
864
|
+
)
|
|
865
|
+
error_dict = {}
|
|
866
|
+
shard_idx = 0
|
|
867
|
+
results_list = []
|
|
868
|
+
for result in batch_result:
|
|
869
|
+
if isinstance(result, Exception):
|
|
870
|
+
error_dict[shard_idx] = result
|
|
871
|
+
elif isinstance(result, BaseException):
|
|
872
|
+
raise result
|
|
873
|
+
else:
|
|
874
|
+
results_list.extend(result)
|
|
875
|
+
shard_idx += 1
|
|
876
|
+
if error_dict:
|
|
877
|
+
raise ShardedReadRowsExceptionGroup(
|
|
878
|
+
[
|
|
879
|
+
FailedQueryShardError(idx, sharded_query[idx], e)
|
|
880
|
+
for (idx, e) in error_dict.items()
|
|
881
|
+
],
|
|
882
|
+
results_list,
|
|
883
|
+
len(sharded_query),
|
|
884
|
+
)
|
|
885
|
+
return results_list
|
|
886
|
+
|
|
887
|
+
def row_exists(
|
|
888
|
+
self,
|
|
889
|
+
row_key: str | bytes,
|
|
890
|
+
*,
|
|
891
|
+
operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
892
|
+
attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
893
|
+
retryable_errors: Sequence[type[Exception]]
|
|
894
|
+
| TABLE_DEFAULT = TABLE_DEFAULT.READ_ROWS,
|
|
895
|
+
) -> bool:
|
|
896
|
+
"""Return a boolean indicating whether the specified row exists in the table.
|
|
897
|
+
uses the filters: chain(limit cells per row = 1, strip value)
|
|
898
|
+
|
|
899
|
+
Args:
|
|
900
|
+
row_key: the key of the row to check
|
|
901
|
+
operation_timeout: the time budget for the entire operation, in seconds.
|
|
902
|
+
Failed requests will be retried within the budget.
|
|
903
|
+
Defaults to the Table's default_read_rows_operation_timeout
|
|
904
|
+
attempt_timeout: the time budget for an individual network request, in seconds.
|
|
905
|
+
If it takes longer than this time to complete, the request will be cancelled with
|
|
906
|
+
a DeadlineExceeded exception, and a retry will be attempted.
|
|
907
|
+
Defaults to the Table's default_read_rows_attempt_timeout.
|
|
908
|
+
If None, defaults to operation_timeout.
|
|
909
|
+
retryable_errors: a list of errors that will be retried if encountered.
|
|
910
|
+
Defaults to the Table's default_read_rows_retryable_errors.
|
|
911
|
+
Returns:
|
|
912
|
+
bool: a bool indicating whether the row exists
|
|
913
|
+
Raises:
|
|
914
|
+
google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
|
|
915
|
+
will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
|
|
916
|
+
from any retries that failed
|
|
917
|
+
google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
|
|
918
|
+
"""
|
|
919
|
+
if row_key is None:
|
|
920
|
+
raise ValueError("row_key must be string or bytes")
|
|
921
|
+
strip_filter = StripValueTransformerFilter(flag=True)
|
|
922
|
+
limit_filter = CellsRowLimitFilter(1)
|
|
923
|
+
chain_filter = RowFilterChain(filters=[limit_filter, strip_filter])
|
|
924
|
+
query = ReadRowsQuery(row_keys=row_key, limit=1, row_filter=chain_filter)
|
|
925
|
+
results = self.read_rows(
|
|
926
|
+
query,
|
|
927
|
+
operation_timeout=operation_timeout,
|
|
928
|
+
attempt_timeout=attempt_timeout,
|
|
929
|
+
retryable_errors=retryable_errors,
|
|
930
|
+
)
|
|
931
|
+
return len(results) > 0
|
|
932
|
+
|
|
933
|
+
def sample_row_keys(
|
|
934
|
+
self,
|
|
935
|
+
*,
|
|
936
|
+
operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
|
|
937
|
+
attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
|
|
938
|
+
retryable_errors: Sequence[type[Exception]]
|
|
939
|
+
| TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
|
|
940
|
+
) -> RowKeySamples:
|
|
941
|
+
"""Return a set of RowKeySamples that delimit contiguous sections of the table of
|
|
942
|
+
approximately equal size
|
|
943
|
+
|
|
944
|
+
RowKeySamples output can be used with ReadRowsQuery.shard() to create a sharded query that
|
|
945
|
+
can be parallelized across multiple backend nodes read_rows and read_rows_stream
|
|
946
|
+
requests will call sample_row_keys internally for this purpose when sharding is enabled
|
|
947
|
+
|
|
948
|
+
RowKeySamples is simply a type alias for list[tuple[bytes, int]]; a list of
|
|
949
|
+
row_keys, along with offset positions in the table
|
|
950
|
+
|
|
951
|
+
Args:
|
|
952
|
+
operation_timeout: the time budget for the entire operation, in seconds.
|
|
953
|
+
Failed requests will be retried within the budget.i
|
|
954
|
+
Defaults to the Table's default_operation_timeout
|
|
955
|
+
attempt_timeout: the time budget for an individual network request, in seconds.
|
|
956
|
+
If it takes longer than this time to complete, the request will be cancelled with
|
|
957
|
+
a DeadlineExceeded exception, and a retry will be attempted.
|
|
958
|
+
Defaults to the Table's default_attempt_timeout.
|
|
959
|
+
If None, defaults to operation_timeout.
|
|
960
|
+
retryable_errors: a list of errors that will be retried if encountered.
|
|
961
|
+
Defaults to the Table's default_retryable_errors.
|
|
962
|
+
Returns:
|
|
963
|
+
RowKeySamples: a set of RowKeySamples the delimit contiguous sections of the table
|
|
964
|
+
Raises:
|
|
965
|
+
google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
|
|
966
|
+
will be chained with a RetryExceptionGroup containing GoogleAPIError exceptions
|
|
967
|
+
from any retries that failed
|
|
968
|
+
google.api_core.exceptions.GoogleAPIError: raised if the request encounters an unrecoverable error
|
|
969
|
+
"""
|
|
970
|
+
(operation_timeout, attempt_timeout) = _get_timeouts(
|
|
971
|
+
operation_timeout, attempt_timeout, self
|
|
972
|
+
)
|
|
973
|
+
attempt_timeout_gen = _attempt_timeout_generator(
|
|
974
|
+
attempt_timeout, operation_timeout
|
|
975
|
+
)
|
|
976
|
+
retryable_excs = _get_retryable_errors(retryable_errors, self)
|
|
977
|
+
predicate = retries.if_exception_type(*retryable_excs)
|
|
978
|
+
sleep_generator = retries.exponential_sleep_generator(0.01, 2, 60)
|
|
979
|
+
|
|
980
|
+
def execute_rpc():
|
|
981
|
+
results = self.client._gapic_client.sample_row_keys(
|
|
982
|
+
table_name=self.table_name,
|
|
983
|
+
app_profile_id=self.app_profile_id,
|
|
984
|
+
timeout=next(attempt_timeout_gen),
|
|
985
|
+
retry=None,
|
|
986
|
+
)
|
|
987
|
+
return [(s.row_key, s.offset_bytes) for s in results]
|
|
988
|
+
|
|
989
|
+
return CrossSync._Sync_Impl.retry_target(
|
|
990
|
+
execute_rpc,
|
|
991
|
+
predicate,
|
|
992
|
+
sleep_generator,
|
|
993
|
+
operation_timeout,
|
|
994
|
+
exception_factory=_retry_exception_factory,
|
|
995
|
+
)
|
|
996
|
+
|
|
997
|
+
def mutations_batcher(
|
|
998
|
+
self,
|
|
999
|
+
*,
|
|
1000
|
+
flush_interval: float | None = 5,
|
|
1001
|
+
flush_limit_mutation_count: int | None = 1000,
|
|
1002
|
+
flush_limit_bytes: int = 20 * _MB_SIZE,
|
|
1003
|
+
flow_control_max_mutation_count: int = 100000,
|
|
1004
|
+
flow_control_max_bytes: int = 100 * _MB_SIZE,
|
|
1005
|
+
batch_operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
|
|
1006
|
+
batch_attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
|
|
1007
|
+
batch_retryable_errors: Sequence[type[Exception]]
|
|
1008
|
+
| TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
|
|
1009
|
+
) -> "MutationsBatcher":
|
|
1010
|
+
"""Returns a new mutations batcher instance.
|
|
1011
|
+
|
|
1012
|
+
Can be used to iteratively add mutations that are flushed as a group,
|
|
1013
|
+
to avoid excess network calls
|
|
1014
|
+
|
|
1015
|
+
Args:
|
|
1016
|
+
flush_interval: Automatically flush every flush_interval seconds. If None,
|
|
1017
|
+
a table default will be used
|
|
1018
|
+
flush_limit_mutation_count: Flush immediately after flush_limit_mutation_count
|
|
1019
|
+
mutations are added across all entries. If None, this limit is ignored.
|
|
1020
|
+
flush_limit_bytes: Flush immediately after flush_limit_bytes bytes are added.
|
|
1021
|
+
flow_control_max_mutation_count: Maximum number of inflight mutations.
|
|
1022
|
+
flow_control_max_bytes: Maximum number of inflight bytes.
|
|
1023
|
+
batch_operation_timeout: timeout for each mutate_rows operation, in seconds.
|
|
1024
|
+
Defaults to the Table's default_mutate_rows_operation_timeout
|
|
1025
|
+
batch_attempt_timeout: timeout for each individual request, in seconds.
|
|
1026
|
+
Defaults to the Table's default_mutate_rows_attempt_timeout.
|
|
1027
|
+
If None, defaults to batch_operation_timeout.
|
|
1028
|
+
batch_retryable_errors: a list of errors that will be retried if encountered.
|
|
1029
|
+
Defaults to the Table's default_mutate_rows_retryable_errors.
|
|
1030
|
+
Returns:
|
|
1031
|
+
MutationsBatcher: a MutationsBatcher context manager that can batch requests
|
|
1032
|
+
"""
|
|
1033
|
+
return CrossSync._Sync_Impl.MutationsBatcher(
|
|
1034
|
+
self,
|
|
1035
|
+
flush_interval=flush_interval,
|
|
1036
|
+
flush_limit_mutation_count=flush_limit_mutation_count,
|
|
1037
|
+
flush_limit_bytes=flush_limit_bytes,
|
|
1038
|
+
flow_control_max_mutation_count=flow_control_max_mutation_count,
|
|
1039
|
+
flow_control_max_bytes=flow_control_max_bytes,
|
|
1040
|
+
batch_operation_timeout=batch_operation_timeout,
|
|
1041
|
+
batch_attempt_timeout=batch_attempt_timeout,
|
|
1042
|
+
batch_retryable_errors=batch_retryable_errors,
|
|
1043
|
+
)
|
|
1044
|
+
|
|
1045
|
+
def mutate_row(
|
|
1046
|
+
self,
|
|
1047
|
+
row_key: str | bytes,
|
|
1048
|
+
mutations: list[Mutation] | Mutation,
|
|
1049
|
+
*,
|
|
1050
|
+
operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
|
|
1051
|
+
attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
|
|
1052
|
+
retryable_errors: Sequence[type[Exception]]
|
|
1053
|
+
| TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
|
|
1054
|
+
):
|
|
1055
|
+
"""Mutates a row atomically.
|
|
1056
|
+
|
|
1057
|
+
Cells already present in the row are left unchanged unless explicitly changed
|
|
1058
|
+
by ``mutation``.
|
|
1059
|
+
|
|
1060
|
+
Idempotent operations (i.e, all mutations have an explicit timestamp) will be
|
|
1061
|
+
retried on server failure. Non-idempotent operations will not.
|
|
1062
|
+
|
|
1063
|
+
Args:
|
|
1064
|
+
row_key: the row to apply mutations to
|
|
1065
|
+
mutations: the set of mutations to apply to the row
|
|
1066
|
+
operation_timeout: the time budget for the entire operation, in seconds.
|
|
1067
|
+
Failed requests will be retried within the budget.
|
|
1068
|
+
Defaults to the Table's default_operation_timeout
|
|
1069
|
+
attempt_timeout: the time budget for an individual network request, in seconds.
|
|
1070
|
+
If it takes longer than this time to complete, the request will be cancelled with
|
|
1071
|
+
a DeadlineExceeded exception, and a retry will be attempted.
|
|
1072
|
+
Defaults to the Table's default_attempt_timeout.
|
|
1073
|
+
If None, defaults to operation_timeout.
|
|
1074
|
+
retryable_errors: a list of errors that will be retried if encountered.
|
|
1075
|
+
Only idempotent mutations will be retried. Defaults to the Table's
|
|
1076
|
+
default_retryable_errors.
|
|
1077
|
+
Raises:
|
|
1078
|
+
google.api_core.exceptions.DeadlineExceeded: raised after operation timeout
|
|
1079
|
+
will be chained with a RetryExceptionGroup containing all
|
|
1080
|
+
GoogleAPIError exceptions from any retries that failed
|
|
1081
|
+
google.api_core.exceptions.GoogleAPIError: raised on non-idempotent operations that cannot be
|
|
1082
|
+
safely retried.
|
|
1083
|
+
ValueError: if invalid arguments are provided"""
|
|
1084
|
+
(operation_timeout, attempt_timeout) = _get_timeouts(
|
|
1085
|
+
operation_timeout, attempt_timeout, self
|
|
1086
|
+
)
|
|
1087
|
+
if not mutations:
|
|
1088
|
+
raise ValueError("No mutations provided")
|
|
1089
|
+
mutations_list = mutations if isinstance(mutations, list) else [mutations]
|
|
1090
|
+
if all((mutation.is_idempotent() for mutation in mutations_list)):
|
|
1091
|
+
predicate = retries.if_exception_type(
|
|
1092
|
+
*_get_retryable_errors(retryable_errors, self)
|
|
1093
|
+
)
|
|
1094
|
+
else:
|
|
1095
|
+
predicate = retries.if_exception_type()
|
|
1096
|
+
sleep_generator = retries.exponential_sleep_generator(0.01, 2, 60)
|
|
1097
|
+
target = partial(
|
|
1098
|
+
self.client._gapic_client.mutate_row,
|
|
1099
|
+
row_key=row_key.encode("utf-8") if isinstance(row_key, str) else row_key,
|
|
1100
|
+
mutations=[mutation._to_pb() for mutation in mutations_list],
|
|
1101
|
+
table_name=self.table_name,
|
|
1102
|
+
app_profile_id=self.app_profile_id,
|
|
1103
|
+
timeout=attempt_timeout,
|
|
1104
|
+
retry=None,
|
|
1105
|
+
)
|
|
1106
|
+
return CrossSync._Sync_Impl.retry_target(
|
|
1107
|
+
target,
|
|
1108
|
+
predicate,
|
|
1109
|
+
sleep_generator,
|
|
1110
|
+
operation_timeout,
|
|
1111
|
+
exception_factory=_retry_exception_factory,
|
|
1112
|
+
)
|
|
1113
|
+
|
|
1114
|
+
def bulk_mutate_rows(
|
|
1115
|
+
self,
|
|
1116
|
+
mutation_entries: list[RowMutationEntry],
|
|
1117
|
+
*,
|
|
1118
|
+
operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
|
|
1119
|
+
attempt_timeout: float | None | TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
|
|
1120
|
+
retryable_errors: Sequence[type[Exception]]
|
|
1121
|
+
| TABLE_DEFAULT = TABLE_DEFAULT.MUTATE_ROWS,
|
|
1122
|
+
):
|
|
1123
|
+
"""Applies mutations for multiple rows in a single batched request.
|
|
1124
|
+
|
|
1125
|
+
Each individual RowMutationEntry is applied atomically, but separate entries
|
|
1126
|
+
may be applied in arbitrary order (even for entries targetting the same row)
|
|
1127
|
+
In total, the row_mutations can contain at most 100000 individual mutations
|
|
1128
|
+
across all entries
|
|
1129
|
+
|
|
1130
|
+
Idempotent entries (i.e., entries with mutations with explicit timestamps)
|
|
1131
|
+
will be retried on failure. Non-idempotent will not, and will reported in a
|
|
1132
|
+
raised exception group
|
|
1133
|
+
|
|
1134
|
+
Args:
|
|
1135
|
+
mutation_entries: the batches of mutations to apply
|
|
1136
|
+
Each entry will be applied atomically, but entries will be applied
|
|
1137
|
+
in arbitrary order
|
|
1138
|
+
operation_timeout: the time budget for the entire operation, in seconds.
|
|
1139
|
+
Failed requests will be retried within the budget.
|
|
1140
|
+
Defaults to the Table's default_mutate_rows_operation_timeout
|
|
1141
|
+
attempt_timeout: the time budget for an individual network request, in seconds.
|
|
1142
|
+
If it takes longer than this time to complete, the request will be cancelled with
|
|
1143
|
+
a DeadlineExceeded exception, and a retry will be attempted.
|
|
1144
|
+
Defaults to the Table's default_mutate_rows_attempt_timeout.
|
|
1145
|
+
If None, defaults to operation_timeout.
|
|
1146
|
+
retryable_errors: a list of errors that will be retried if encountered.
|
|
1147
|
+
Defaults to the Table's default_mutate_rows_retryable_errors
|
|
1148
|
+
Raises:
|
|
1149
|
+
MutationsExceptionGroup: if one or more mutations fails
|
|
1150
|
+
Contains details about any failed entries in .exceptions
|
|
1151
|
+
ValueError: if invalid arguments are provided"""
|
|
1152
|
+
(operation_timeout, attempt_timeout) = _get_timeouts(
|
|
1153
|
+
operation_timeout, attempt_timeout, self
|
|
1154
|
+
)
|
|
1155
|
+
retryable_excs = _get_retryable_errors(retryable_errors, self)
|
|
1156
|
+
operation = CrossSync._Sync_Impl._MutateRowsOperation(
|
|
1157
|
+
self.client._gapic_client,
|
|
1158
|
+
self,
|
|
1159
|
+
mutation_entries,
|
|
1160
|
+
operation_timeout,
|
|
1161
|
+
attempt_timeout,
|
|
1162
|
+
retryable_exceptions=retryable_excs,
|
|
1163
|
+
)
|
|
1164
|
+
operation.start()
|
|
1165
|
+
|
|
1166
|
+
def check_and_mutate_row(
|
|
1167
|
+
self,
|
|
1168
|
+
row_key: str | bytes,
|
|
1169
|
+
predicate: RowFilter | None,
|
|
1170
|
+
*,
|
|
1171
|
+
true_case_mutations: Mutation | list[Mutation] | None = None,
|
|
1172
|
+
false_case_mutations: Mutation | list[Mutation] | None = None,
|
|
1173
|
+
operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
|
|
1174
|
+
) -> bool:
|
|
1175
|
+
"""Mutates a row atomically based on the output of a predicate filter
|
|
1176
|
+
|
|
1177
|
+
Non-idempotent operation: will not be retried
|
|
1178
|
+
|
|
1179
|
+
Args:
|
|
1180
|
+
row_key: the key of the row to mutate
|
|
1181
|
+
predicate: the filter to be applied to the contents of the specified row.
|
|
1182
|
+
Depending on whether or not any results are yielded,
|
|
1183
|
+
either true_case_mutations or false_case_mutations will be executed.
|
|
1184
|
+
If None, checks that the row contains any values at all.
|
|
1185
|
+
true_case_mutations:
|
|
1186
|
+
Changes to be atomically applied to the specified row if
|
|
1187
|
+
predicate yields at least one cell when
|
|
1188
|
+
applied to row_key. Entries are applied in order,
|
|
1189
|
+
meaning that earlier mutations can be masked by later
|
|
1190
|
+
ones. Must contain at least one entry if
|
|
1191
|
+
false_case_mutations is empty, and at most 100000.
|
|
1192
|
+
false_case_mutations:
|
|
1193
|
+
Changes to be atomically applied to the specified row if
|
|
1194
|
+
predicate_filter does not yield any cells when
|
|
1195
|
+
applied to row_key. Entries are applied in order,
|
|
1196
|
+
meaning that earlier mutations can be masked by later
|
|
1197
|
+
ones. Must contain at least one entry if
|
|
1198
|
+
`true_case_mutations` is empty, and at most 100000.
|
|
1199
|
+
operation_timeout: the time budget for the entire operation, in seconds.
|
|
1200
|
+
Failed requests will not be retried. Defaults to the Table's default_operation_timeout
|
|
1201
|
+
Returns:
|
|
1202
|
+
bool indicating whether the predicate was true or false
|
|
1203
|
+
Raises:
|
|
1204
|
+
google.api_core.exceptions.GoogleAPIError: exceptions from grpc call"""
|
|
1205
|
+
(operation_timeout, _) = _get_timeouts(operation_timeout, None, self)
|
|
1206
|
+
if true_case_mutations is not None and (
|
|
1207
|
+
not isinstance(true_case_mutations, list)
|
|
1208
|
+
):
|
|
1209
|
+
true_case_mutations = [true_case_mutations]
|
|
1210
|
+
true_case_list = [m._to_pb() for m in true_case_mutations or []]
|
|
1211
|
+
if false_case_mutations is not None and (
|
|
1212
|
+
not isinstance(false_case_mutations, list)
|
|
1213
|
+
):
|
|
1214
|
+
false_case_mutations = [false_case_mutations]
|
|
1215
|
+
false_case_list = [m._to_pb() for m in false_case_mutations or []]
|
|
1216
|
+
result = self.client._gapic_client.check_and_mutate_row(
|
|
1217
|
+
true_mutations=true_case_list,
|
|
1218
|
+
false_mutations=false_case_list,
|
|
1219
|
+
predicate_filter=predicate._to_pb() if predicate is not None else None,
|
|
1220
|
+
row_key=row_key.encode("utf-8") if isinstance(row_key, str) else row_key,
|
|
1221
|
+
table_name=self.table_name,
|
|
1222
|
+
app_profile_id=self.app_profile_id,
|
|
1223
|
+
timeout=operation_timeout,
|
|
1224
|
+
retry=None,
|
|
1225
|
+
)
|
|
1226
|
+
return result.predicate_matched
|
|
1227
|
+
|
|
1228
|
+
def read_modify_write_row(
|
|
1229
|
+
self,
|
|
1230
|
+
row_key: str | bytes,
|
|
1231
|
+
rules: ReadModifyWriteRule | list[ReadModifyWriteRule],
|
|
1232
|
+
*,
|
|
1233
|
+
operation_timeout: float | TABLE_DEFAULT = TABLE_DEFAULT.DEFAULT,
|
|
1234
|
+
) -> Row:
|
|
1235
|
+
"""Reads and modifies a row atomically according to input ReadModifyWriteRules,
|
|
1236
|
+
and returns the contents of all modified cells
|
|
1237
|
+
|
|
1238
|
+
The new value for the timestamp is the greater of the existing timestamp or
|
|
1239
|
+
the current server time.
|
|
1240
|
+
|
|
1241
|
+
Non-idempotent operation: will not be retried
|
|
1242
|
+
|
|
1243
|
+
Args:
|
|
1244
|
+
row_key: the key of the row to apply read/modify/write rules to
|
|
1245
|
+
rules: A rule or set of rules to apply to the row.
|
|
1246
|
+
Rules are applied in order, meaning that earlier rules will affect the
|
|
1247
|
+
results of later ones.
|
|
1248
|
+
operation_timeout: the time budget for the entire operation, in seconds.
|
|
1249
|
+
Failed requests will not be retried.
|
|
1250
|
+
Defaults to the Table's default_operation_timeout.
|
|
1251
|
+
Returns:
|
|
1252
|
+
Row: a Row containing cell data that was modified as part of the operation
|
|
1253
|
+
Raises:
|
|
1254
|
+
google.api_core.exceptions.GoogleAPIError: exceptions from grpc call
|
|
1255
|
+
ValueError: if invalid arguments are provided"""
|
|
1256
|
+
(operation_timeout, _) = _get_timeouts(operation_timeout, None, self)
|
|
1257
|
+
if operation_timeout <= 0:
|
|
1258
|
+
raise ValueError("operation_timeout must be greater than 0")
|
|
1259
|
+
if rules is not None and (not isinstance(rules, list)):
|
|
1260
|
+
rules = [rules]
|
|
1261
|
+
if not rules:
|
|
1262
|
+
raise ValueError("rules must contain at least one item")
|
|
1263
|
+
result = self.client._gapic_client.read_modify_write_row(
|
|
1264
|
+
rules=[rule._to_pb() for rule in rules],
|
|
1265
|
+
row_key=row_key.encode("utf-8") if isinstance(row_key, str) else row_key,
|
|
1266
|
+
table_name=self.table_name,
|
|
1267
|
+
app_profile_id=self.app_profile_id,
|
|
1268
|
+
timeout=operation_timeout,
|
|
1269
|
+
retry=None,
|
|
1270
|
+
)
|
|
1271
|
+
return Row._from_pb(result.row)
|
|
1272
|
+
|
|
1273
|
+
def close(self):
|
|
1274
|
+
"""Called to close the Table instance and release any resources held by it."""
|
|
1275
|
+
if self._register_instance_future:
|
|
1276
|
+
self._register_instance_future.cancel()
|
|
1277
|
+
self.client._remove_instance_registration(self.instance_id, self)
|
|
1278
|
+
|
|
1279
|
+
def __enter__(self):
|
|
1280
|
+
"""Implement async context manager protocol
|
|
1281
|
+
|
|
1282
|
+
Ensure registration task has time to run, so that
|
|
1283
|
+
grpc channels will be warmed for the specified instance"""
|
|
1284
|
+
if self._register_instance_future:
|
|
1285
|
+
self._register_instance_future
|
|
1286
|
+
return self
|
|
1287
|
+
|
|
1288
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
1289
|
+
"""Implement async context manager protocol
|
|
1290
|
+
|
|
1291
|
+
Unregister this instance with the client, so that
|
|
1292
|
+
grpc channels will no longer be warmed"""
|
|
1293
|
+
self.close()
|