tikv-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tikv/__init__.py ADDED
@@ -0,0 +1,44 @@
1
+ """A pure-Python client for TiKV's raw key-value API.
2
+
3
+ Blocking::
4
+
5
+ import tikv
6
+
7
+ with tikv.RawClient(["127.0.0.1:2379"]) as client:
8
+ client.put(b"greeting", b"hello")
9
+ client.get(b"greeting") # b"hello"
10
+
11
+ asyncio::
12
+
13
+ async with tikv.AsyncRawClient(["127.0.0.1:2379"]) as client:
14
+ await client.put(b"greeting", b"hello")
15
+ """
16
+
17
+ from ._async._client import AsyncRawClient
18
+ from ._errors import (
19
+ PDError,
20
+ RegionError,
21
+ ServerError,
22
+ TiKVError,
23
+ TimeoutError,
24
+ UndeterminedError,
25
+ )
26
+ from ._rpc import TLSConfig
27
+ from ._sync._client import RawClient
28
+ from ._types import CasResult, Checksum
29
+
30
+ __all__ = [
31
+ "AsyncRawClient",
32
+ "CasResult",
33
+ "Checksum",
34
+ "PDError",
35
+ "RawClient",
36
+ "RegionError",
37
+ "ServerError",
38
+ "TLSConfig",
39
+ "TiKVError",
40
+ "TimeoutError",
41
+ "UndeterminedError",
42
+ ]
43
+
44
+ __version__ = "0.1.0"
@@ -0,0 +1 @@
1
+ """asyncio implementation; the source of truth for ``_sync``."""
tikv/_async/_client.py ADDED
@@ -0,0 +1,584 @@
1
+ """The raw key-value client."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import AsyncIterator, Callable, Iterable, Mapping, Sequence
7
+ from datetime import timedelta
8
+ from types import TracebackType
9
+ from typing import TypeVar
10
+
11
+ import grpc
12
+ from google.protobuf.message import Message
13
+
14
+ from .._backoff import Backoff, Wait, describe
15
+ from .._errors import RegionError, ServerError, TiKVError, UndeterminedError
16
+ from .._keys import as_key, as_value, chunked, key_range, min_end, ttl_seconds
17
+ from .._proto import kvrpcpb_pb2 as kvrpcpb
18
+ from .._region import MAX_KEY, Region, RegionCache, describe_region_error, on_region_error
19
+ from .._rpc import (
20
+ RAW_BATCH_DELETE,
21
+ RAW_BATCH_GET,
22
+ RAW_BATCH_PUT,
23
+ RAW_CHECKSUM,
24
+ RAW_COMPARE_AND_SWAP,
25
+ RAW_DELETE,
26
+ RAW_DELETE_RANGE,
27
+ RAW_GET,
28
+ RAW_GET_KEY_TTL,
29
+ RAW_PUT,
30
+ RAW_SCAN,
31
+ Method,
32
+ Response,
33
+ TLSConfig,
34
+ is_retryable,
35
+ )
36
+ from .._types import CasResult, Checksum
37
+ from ._compat import map_concurrently, sleep
38
+ from ._pd import PDClient
39
+ from ._transport import Transport
40
+
41
+ T = TypeVar("T")
42
+
43
+ # Per-request limits when splitting batches, following client-go.
44
+ BATCH_MAX_KEYS = 512
45
+ BATCH_MAX_BYTES = 1024 * 1024
46
+ # Keys fetched per RawScan request when the caller does not set a limit.
47
+ SCAN_PAGE_SIZE = 512
48
+ # Regions fetched from PD in one go when walking a key range.
49
+ REGION_PREFETCH = 64
50
+
51
+
52
+ class AsyncRawClient:
53
+ """A client for TiKV's raw (non-transactional) key-value API.
54
+
55
+ Keys and values are :class:`bytes`. Each single-key operation is atomic;
56
+ batch operations are atomic per region but not across regions.
57
+
58
+ ::
59
+
60
+ async with AsyncRawClient(["127.0.0.1:2379"]) as client:
61
+ await client.put(b"greeting", b"hello")
62
+ assert await client.get(b"greeting") == b"hello"
63
+
64
+ :param pd_endpoints: Address of one or more PD servers, e.g.
65
+ ``"pd-0:2379"``. The rest of the cluster is discovered from them.
66
+ :param timeout: Time budget in seconds for each operation, including all
67
+ retries. Scans apply it to each page they fetch.
68
+ :param atomic: Route every write through TiKV's single-row transaction
69
+ path. Required by :meth:`compare_and_swap`; every client writing the
70
+ same keys must agree on this setting.
71
+ :param tls: Certificates for mutual TLS, if the cluster requires it.
72
+
73
+ One client can serve a whole application: it is safe to use from many
74
+ threads (blocking flavor) or tasks (asyncio flavor) at once, and it keeps
75
+ one connection per server. Close it with :meth:`close` or use it as a
76
+ context manager.
77
+ """
78
+
79
+ def __init__(
80
+ self,
81
+ pd_endpoints: str | Iterable[str],
82
+ *,
83
+ timeout: float = 10.0,
84
+ atomic: bool = False,
85
+ tls: TLSConfig | None = None,
86
+ ) -> None:
87
+ endpoints = [pd_endpoints] if isinstance(pd_endpoints, str) else list(pd_endpoints)
88
+ if not endpoints:
89
+ raise ValueError("at least one PD endpoint is required")
90
+ if not timeout > 0:
91
+ raise ValueError(f"timeout must be positive, got {timeout!r}")
92
+ self.timeout = timeout
93
+ self.atomic = atomic
94
+ self._endpoints = endpoints
95
+ self._cache = RegionCache()
96
+ self._transport = Transport(tls.credentials() if tls else None)
97
+ self._pd = PDClient(endpoints, self._transport, self._cache)
98
+ self._closed = False
99
+
100
+ def __repr__(self) -> str:
101
+ return f"{type(self).__name__}({self._endpoints!r}, timeout={self.timeout!r}, atomic={self.atomic!r})"
102
+
103
+ async def __aenter__(self) -> AsyncRawClient:
104
+ return self
105
+
106
+ async def __aexit__(
107
+ self,
108
+ exc_type: type[BaseException] | None,
109
+ exc: BaseException | None,
110
+ traceback: TracebackType | None,
111
+ ) -> None:
112
+ await self.close()
113
+
114
+ async def close(self) -> None:
115
+ """Close all connections. The client cannot be used afterwards."""
116
+ if not self._closed:
117
+ self._closed = True
118
+ await self._transport.close()
119
+ self._cache.clear()
120
+
121
+ # -- Reads ---------------------------------------------------------------
122
+
123
+ async def get(self, key: bytes) -> bytes | None:
124
+ """Return the value of *key*, or ``None`` if it does not exist."""
125
+ request = kvrpcpb.RawGetRequest(key=as_key(key))
126
+ response = await self._execute(request.key, RAW_GET, request)
127
+ return None if response.not_found else response.value
128
+
129
+ async def get_many(self, keys: Iterable[bytes]) -> dict[bytes, bytes]:
130
+ """Return a dict of the given keys that exist, mapped to their values.
131
+
132
+ Missing keys are left out. The dict follows the order of *keys*.
133
+ """
134
+ wanted = list(dict.fromkeys(map(as_key, keys)))
135
+ responses = await self._execute_batch(
136
+ sorted(wanted),
137
+ key=_identity,
138
+ method=RAW_BATCH_GET,
139
+ build=lambda chunk: kvrpcpb.RawBatchGetRequest(keys=chunk),
140
+ )
141
+ found = {pair.key: pair.value for response in responses for pair in _checked(response.pairs)}
142
+ return {key: found[key] for key in wanted if key in found}
143
+
144
+ async def ttl(self, key: bytes) -> int | None:
145
+ """Return the seconds until *key* expires, or ``None`` if it never does.
146
+
147
+ :raises KeyError: if *key* does not exist.
148
+ """
149
+ request = kvrpcpb.RawGetKeyTTLRequest(key=as_key(key))
150
+ response = await self._execute(request.key, RAW_GET_KEY_TTL, request)
151
+ if response.not_found:
152
+ raise KeyError(key)
153
+ return response.ttl or None
154
+
155
+ async def scan(
156
+ self,
157
+ start: bytes | None = None,
158
+ end: bytes | None = None,
159
+ *,
160
+ prefix: bytes | None = None,
161
+ limit: int | None = None,
162
+ reverse: bool = False,
163
+ ) -> AsyncIterator[tuple[bytes, bytes]]:
164
+ """Iterate over ``(key, value)`` pairs in key order.
165
+
166
+ Covers keys in ``[start, end)``, or every key starting with *prefix*.
167
+ ``None`` bounds are open-ended. Results are fetched lazily, a page at
168
+ a time, so scanning a large range uses little memory::
169
+
170
+ async for key, value in client.scan(prefix=b"user:"):
171
+ ...
172
+
173
+ :param limit: Stop after this many pairs.
174
+ :param reverse: Iterate from the largest key down.
175
+ """
176
+ async for pair in self._scan(start, end, prefix, limit, reverse, key_only=False):
177
+ yield pair.key, pair.value
178
+
179
+ async def scan_keys(
180
+ self,
181
+ start: bytes | None = None,
182
+ end: bytes | None = None,
183
+ *,
184
+ prefix: bytes | None = None,
185
+ limit: int | None = None,
186
+ reverse: bool = False,
187
+ ) -> AsyncIterator[bytes]:
188
+ """Like :meth:`scan`, but yields only keys, without fetching values."""
189
+ async for pair in self._scan(start, end, prefix, limit, reverse, key_only=True):
190
+ yield pair.key
191
+
192
+ async def checksum(
193
+ self,
194
+ start: bytes | None = None,
195
+ end: bytes | None = None,
196
+ *,
197
+ prefix: bytes | None = None,
198
+ ) -> Checksum:
199
+ """Fingerprint the keys in ``[start, end)``, or those starting with *prefix*.
200
+
201
+ The servers compute it in place, so no data crosses the network.
202
+ """
203
+ bounds = key_range(start, end, prefix)
204
+ if bounds is None:
205
+ return Checksum(0, 0, 0)
206
+ responses = await self._execute_range(
207
+ *bounds,
208
+ method=RAW_CHECKSUM,
209
+ build=lambda lo, hi: kvrpcpb.RawChecksumRequest(
210
+ ranges=[kvrpcpb.KeyRange(start_key=lo, end_key=hi)]
211
+ ),
212
+ )
213
+ crc64_xor = keys = size = 0
214
+ for response in responses:
215
+ crc64_xor ^= response.checksum
216
+ keys += response.total_kvs
217
+ size += response.total_bytes
218
+ return Checksum(crc64_xor, keys, size)
219
+
220
+ # -- Writes --------------------------------------------------------------
221
+
222
+ async def put(self, key: bytes, value: bytes, *, ttl: float | timedelta | None = None) -> None:
223
+ """Set *key* to *value*.
224
+
225
+ :param ttl: Expire the key after this many seconds (rounded up to a
226
+ whole second) or this :class:`~datetime.timedelta`. Requires
227
+ ``storage.enable-ttl`` on the TiKV servers.
228
+ """
229
+ request = kvrpcpb.RawPutRequest(
230
+ key=as_key(key), value=as_value(value), ttl=ttl_seconds(ttl), for_cas=self.atomic
231
+ )
232
+ await self._execute(request.key, RAW_PUT, request)
233
+
234
+ async def put_many(
235
+ self,
236
+ items: Mapping[bytes, bytes] | Iterable[tuple[bytes, bytes]],
237
+ *,
238
+ ttl: float | timedelta | None = None,
239
+ ) -> None:
240
+ """Set many keys at once, from a mapping or ``(key, value)`` pairs.
241
+
242
+ Writes are atomic per region but not across regions.
243
+
244
+ :param ttl: Expiry applied to every key; see :meth:`put`.
245
+ """
246
+ entries = sorted((as_key(k), as_value(v)) for k, v in dict(items).items())
247
+ ttls = [ttl_seconds(ttl)] if ttl is not None else []
248
+
249
+ def build(chunk: list[tuple[bytes, bytes]]) -> kvrpcpb.RawBatchPutRequest:
250
+ return kvrpcpb.RawBatchPutRequest(
251
+ pairs=[kvrpcpb.KvPair(key=k, value=v) for k, v in chunk], ttls=ttls, for_cas=self.atomic
252
+ )
253
+
254
+ await self._execute_batch(entries, key=_first, method=RAW_BATCH_PUT, build=build, size=_pair_size)
255
+
256
+ async def delete(self, key: bytes) -> None:
257
+ """Delete *key*. Deleting a missing key is not an error."""
258
+ request = kvrpcpb.RawDeleteRequest(key=as_key(key), for_cas=self.atomic)
259
+ await self._execute(request.key, RAW_DELETE, request)
260
+
261
+ async def delete_many(self, keys: Iterable[bytes]) -> None:
262
+ """Delete many keys at once. Missing keys are ignored."""
263
+ await self._execute_batch(
264
+ sorted(set(map(as_key, keys))),
265
+ key=_identity,
266
+ method=RAW_BATCH_DELETE,
267
+ build=lambda chunk: kvrpcpb.RawBatchDeleteRequest(keys=chunk, for_cas=self.atomic),
268
+ )
269
+
270
+ async def delete_range(
271
+ self,
272
+ start: bytes | None = None,
273
+ end: bytes | None = None,
274
+ *,
275
+ prefix: bytes | None = None,
276
+ ) -> None:
277
+ """Delete every key in ``[start, end)``, or every key starting with *prefix*.
278
+
279
+ ``end=None`` deletes through the end of the key space. To guard
280
+ against accidents, *start* or a non-empty *prefix* is required:
281
+ wiping everything takes an explicit ``delete_range(b"")``.
282
+ """
283
+ if self.atomic:
284
+ raise TiKVError("delete_range is not supported by clients in atomic mode")
285
+ if prefix is None and start is None:
286
+ raise TypeError("delete_range() requires start or prefix")
287
+ if prefix is not None and not prefix:
288
+ raise ValueError('an empty prefix matches every key; use delete_range(b"") to wipe them all')
289
+ bounds = key_range(start, end, prefix)
290
+ if bounds is not None:
291
+ await self._execute_range(
292
+ *bounds,
293
+ method=RAW_DELETE_RANGE,
294
+ build=lambda lo, hi: kvrpcpb.RawDeleteRangeRequest(start_key=lo, end_key=hi),
295
+ )
296
+
297
+ async def compare_and_swap(
298
+ self,
299
+ key: bytes,
300
+ expected: bytes | None,
301
+ new: bytes,
302
+ *,
303
+ ttl: float | timedelta | None = None,
304
+ ) -> CasResult:
305
+ """Set *key* to *new* only if its current value is *expected*.
306
+
307
+ Pass ``expected=None`` to require that *key* does not exist yet.
308
+ Requires a client created with ``atomic=True``::
309
+
310
+ swapped, previous = await client.compare_and_swap(b"lock", None, b"owner-1")
311
+
312
+ :raises UndeterminedError: if the request failed in flight, so the
313
+ swap may or may not have happened.
314
+ """
315
+ if not self.atomic:
316
+ raise TiKVError("compare_and_swap requires a client created with atomic=True")
317
+ request = kvrpcpb.RawCASRequest(
318
+ key=as_key(key),
319
+ value=as_value(new),
320
+ previous_not_exist=expected is None,
321
+ previous_value=b"" if expected is None else as_value(expected),
322
+ ttl=ttl_seconds(ttl),
323
+ )
324
+ response = await self._execute(request.key, RAW_COMPARE_AND_SWAP, request, idempotent=False)
325
+ return CasResult(response.succeed, None if response.previous_not_exist else response.previous_value)
326
+
327
+ # -- Request routing -----------------------------------------------------
328
+
329
+ async def _execute(
330
+ self, key: bytes, method: Method[Response], request: Message, *, idempotent: bool = True
331
+ ) -> Response:
332
+ """Send a single-key request to the region owning *key*."""
333
+ backoff = Backoff(self.timeout)
334
+ while True:
335
+ region = await self._locate(key, backoff)
336
+ response = await self._send(region, method, request, backoff, idempotent=idempotent)
337
+ if response is not None:
338
+ return response
339
+
340
+ async def _execute_batch(
341
+ self,
342
+ items: Sequence[T],
343
+ *,
344
+ key: Callable[[T], bytes],
345
+ method: Method[Response],
346
+ build: Callable[[list[T]], Message],
347
+ size: Callable[[T], int] | None = None,
348
+ backoff: Backoff | None = None,
349
+ ) -> list[Response]:
350
+ """Split key-sorted *items* by region and send one request per chunk."""
351
+ backoff = backoff or Backoff(self.timeout)
352
+ batches = await self._plan_batches(items, key, size, backoff)
353
+
354
+ async def run(batch: tuple[Region, list[T]]) -> list[Response]:
355
+ region, chunk = batch
356
+ retry = backoff.fork()
357
+ while True:
358
+ response = await self._send(region, method, build(chunk), retry)
359
+ if response is not None:
360
+ return [response]
361
+ # The region changed under us: re-route this chunk.
362
+ replanned = await self._plan_batches(chunk, key, size, retry)
363
+ if len(replanned) > 1: # it now spans several regions
364
+ return await self._execute_batch(
365
+ chunk, key=key, method=method, build=build, size=size, backoff=retry
366
+ )
367
+ region, chunk = replanned[0]
368
+
369
+ results = await map_concurrently(run, batches)
370
+ return [response for group in results for response in group]
371
+
372
+ async def _plan_batches(
373
+ self, items: Sequence[T], key: Callable[[T], bytes], size: Callable[[T], int] | None, backoff: Backoff
374
+ ) -> list[tuple[Region, list[T]]]:
375
+ """Group key-sorted *items* by region, in chunks small enough for one request."""
376
+ batches: list[tuple[Region, list[T]]] = []
377
+ i = 0
378
+ while i < len(items):
379
+ region = await self._locate(key(items[i]), backoff, until=key(items[-1]) + b"\0")
380
+ j = i + 1
381
+ while j < len(items) and region.contains(key(items[j])):
382
+ j += 1
383
+ for chunk in chunked(items[i:j], max_count=BATCH_MAX_KEYS, max_bytes=BATCH_MAX_BYTES, size=size):
384
+ batches.append((region, chunk))
385
+ i = j
386
+ await self._resolve_stores([region for region, _ in batches], backoff)
387
+ return batches
388
+
389
+ async def _execute_range(
390
+ self,
391
+ start: bytes,
392
+ end: bytes,
393
+ *,
394
+ method: Method[Response],
395
+ build: Callable[[bytes, bytes], Message],
396
+ backoff: Backoff | None = None,
397
+ ) -> list[Response]:
398
+ """Send one request per region overlapping ``[start, end)``."""
399
+ backoff = backoff or Backoff(self.timeout)
400
+ spans = await self._plan_spans(start, end, backoff)
401
+
402
+ async def run(span: tuple[Region, bytes, bytes]) -> list[Response]:
403
+ region, lo, hi = span
404
+ retry = backoff.fork()
405
+ while True:
406
+ response = await self._send(region, method, build(lo, hi), retry)
407
+ if response is not None:
408
+ return [response]
409
+ replanned = await self._plan_spans(lo, hi, retry)
410
+ if len(replanned) > 1:
411
+ return await self._execute_range(lo, hi, method=method, build=build, backoff=retry)
412
+ region, lo, hi = replanned[0]
413
+
414
+ results = await map_concurrently(run, spans)
415
+ return [response for group in results for response in group]
416
+
417
+ async def _plan_spans(
418
+ self, start: bytes, end: bytes, backoff: Backoff
419
+ ) -> list[tuple[Region, bytes, bytes]]:
420
+ """Split ``[start, end)`` at region boundaries."""
421
+ spans: list[tuple[Region, bytes, bytes]] = []
422
+ cursor = start
423
+ while True:
424
+ region = await self._locate(cursor, backoff, until=end)
425
+ spans.append((region, cursor, min_end(end, region.end)))
426
+ if not region.end or (end and region.end >= end):
427
+ break
428
+ cursor = region.end
429
+ await self._resolve_stores([region for region, _, _ in spans], backoff)
430
+ return spans
431
+
432
+ async def _scan(
433
+ self,
434
+ start: bytes | None,
435
+ end: bytes | None,
436
+ prefix: bytes | None,
437
+ limit: int | None,
438
+ reverse: bool,
439
+ *,
440
+ key_only: bool,
441
+ ) -> AsyncIterator[kvrpcpb.KvPair]:
442
+ if limit is not None and limit < 0:
443
+ raise ValueError(f"limit must not be negative, got {limit!r}")
444
+ bounds = key_range(start, end, prefix)
445
+ if bounds is None:
446
+ return
447
+ lower, upper = bounds
448
+ remaining = math.inf if limit is None else limit
449
+ # Forward scans move a lower bound up; reverse scans move an upper bound down.
450
+ cursor = (upper or MAX_KEY) if reverse else lower
451
+
452
+ while remaining > 0:
453
+ page = int(min(remaining, SCAN_PAGE_SIZE))
454
+ backoff = Backoff(self.timeout)
455
+ while True:
456
+ if reverse:
457
+ region = await self._locate_end(cursor, backoff)
458
+ request = kvrpcpb.RawScanRequest(
459
+ start_key=cursor, end_key=max(lower, region.start), reverse=True
460
+ )
461
+ else:
462
+ region = await self._locate(cursor, backoff)
463
+ request = kvrpcpb.RawScanRequest(start_key=cursor, end_key=min_end(upper, region.end))
464
+ request.limit = page
465
+ request.key_only = key_only
466
+ response = await self._send(region, RAW_SCAN, request, backoff)
467
+ if response is not None:
468
+ break
469
+
470
+ for pair in _checked(response.kvs):
471
+ yield pair
472
+ remaining -= len(response.kvs)
473
+
474
+ if len(response.kvs) == page: # the region may hold more
475
+ last = response.kvs[-1].key
476
+ if reverse:
477
+ if last <= lower:
478
+ return
479
+ cursor = last
480
+ else:
481
+ cursor = last + b"\0"
482
+ if upper and cursor >= upper:
483
+ return
484
+ elif reverse:
485
+ if region.start <= lower:
486
+ return
487
+ cursor = region.start
488
+ else:
489
+ if not region.end or (upper and region.end >= upper):
490
+ return
491
+ cursor = region.end
492
+
493
+ async def _locate(self, key: bytes, backoff: Backoff, *, until: bytes | None = None) -> Region:
494
+ """The region containing *key*.
495
+
496
+ On a cache miss, *until* lets PD return every region up to that key at
497
+ once, which saves round trips when walking a range or a sorted batch.
498
+ """
499
+ region = self._cache.find(key)
500
+ if region is not None:
501
+ return region
502
+ if until is None or (until and until <= key):
503
+ return await self._pd.region(key, backoff)
504
+ regions = await self._pd.scan_regions(key, until, REGION_PREFETCH, backoff)
505
+ if regions and regions[0].contains(key):
506
+ return regions[0]
507
+ return await self._pd.region(key, backoff)
508
+
509
+ async def _resolve_stores(self, regions: Iterable[Region], backoff: Backoff) -> None:
510
+ """Look up leader store addresses once, before fanning out requests."""
511
+ for store_id in sorted({r.leader.store_id for r in regions if r.leader is not None}):
512
+ await self._pd.store_address(store_id, backoff)
513
+
514
+ async def _locate_end(self, key: bytes, backoff: Backoff) -> Region:
515
+ """The region in which *key* is a valid exclusive upper bound."""
516
+ region = self._cache.find_end(key)
517
+ if region is not None:
518
+ return region
519
+ return await self._pd.region_by_end(key, backoff)
520
+
521
+ async def _send(
522
+ self,
523
+ region: Region,
524
+ method: Method[Response],
525
+ request: Message,
526
+ backoff: Backoff,
527
+ *,
528
+ idempotent: bool = True,
529
+ ) -> Response | None:
530
+ """Send *request* to the leader of *region*.
531
+
532
+ Returns ``None`` when the request must be re-routed, after updating
533
+ the region cache and backing off as needed.
534
+ """
535
+ if self._closed:
536
+ raise TiKVError("the client is closed")
537
+ if region.leader is None: # PD knows no leader: an election is under way
538
+ self._cache.invalidate(region)
539
+ await sleep(backoff.delay(Wait.REGION_SCHEDULING, RegionError(f"{region} has no leader")))
540
+ return None
541
+
542
+ store_id = region.leader.store_id
543
+ address = await self._pd.store_address(store_id, backoff)
544
+ request.context.CopyFrom(region.context()) # type: ignore[attr-defined]
545
+ try:
546
+ response = await self._transport.call(address, method, request, backoff.remaining())
547
+ except grpc.RpcError as error:
548
+ if not is_retryable(error):
549
+ raise ServerError(f"{method.path} failed: {describe(error)}") from error
550
+ if not idempotent:
551
+ raise UndeterminedError(f"{method.path} to {address} failed: {describe(error)}") from error
552
+ self._cache.invalidate(region)
553
+ self._cache.forget_store(store_id)
554
+ await sleep(backoff.delay(Wait.TIKV_RPC, error))
555
+ return None
556
+
557
+ if response.HasField("region_error"):
558
+ region_error = response.region_error # type: ignore[attr-defined]
559
+ wait, hint = on_region_error(self._cache, region, region_error, idempotent=idempotent)
560
+ cause = RegionError(describe_region_error(region, region_error))
561
+ await sleep(backoff.delay(wait, cause, hint=hint))
562
+ return None
563
+ if getattr(response, "error", ""):
564
+ raise ServerError(response.error) # type: ignore[attr-defined]
565
+ return response
566
+
567
+
568
+ def _checked(pairs: Iterable[kvrpcpb.KvPair]) -> Iterable[kvrpcpb.KvPair]:
569
+ for pair in pairs:
570
+ if pair.HasField("error"):
571
+ raise ServerError(pair.error.abort or pair.error.retryable or str(pair.error))
572
+ return pairs
573
+
574
+
575
+ def _identity(key: bytes) -> bytes:
576
+ return key
577
+
578
+
579
+ def _first(pair: tuple[bytes, bytes]) -> bytes:
580
+ return pair[0]
581
+
582
+
583
+ def _pair_size(pair: tuple[bytes, bytes]) -> int:
584
+ return len(pair[0]) + len(pair[1])
tikv/_async/_compat.py ADDED
@@ -0,0 +1,53 @@
1
+ """Concurrency primitives for the asyncio flavor of the client.
2
+
3
+ ``_sync/_compat.py`` provides the same names for the blocking flavor; the rest
4
+ of ``_sync`` is generated from ``_async`` by ``scripts/unasync.py``.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import asyncio
10
+ from collections.abc import Awaitable, Callable, Sequence
11
+ from typing import TypeVar
12
+
13
+ import grpc
14
+ import grpc.aio
15
+
16
+ T = TypeVar("T")
17
+ R = TypeVar("R")
18
+
19
+ Lock = asyncio.Lock
20
+ sleep = asyncio.sleep
21
+ Channel = grpc.aio.Channel
22
+
23
+ # Upper bound on requests in flight for one multi-region operation.
24
+ MAX_CONCURRENCY = 16
25
+
26
+
27
+ async def map_concurrently(fn: Callable[[T], Awaitable[R]], items: Sequence[T]) -> list[R]:
28
+ """Run ``fn`` on every item concurrently and return results in order."""
29
+ if len(items) <= 1:
30
+ return [await fn(item) for item in items]
31
+ semaphore = asyncio.Semaphore(MAX_CONCURRENCY)
32
+
33
+ async def bounded(item: T) -> R:
34
+ async with semaphore:
35
+ return await fn(item)
36
+
37
+ tasks = [asyncio.ensure_future(bounded(item)) for item in items]
38
+ try:
39
+ return list(await asyncio.gather(*tasks))
40
+ except BaseException:
41
+ for task in tasks:
42
+ task.cancel()
43
+ raise
44
+
45
+
46
+ def open_channel(
47
+ target: str,
48
+ credentials: grpc.ChannelCredentials | None,
49
+ options: Sequence[tuple[str, object]],
50
+ ) -> grpc.aio.Channel:
51
+ if credentials is None:
52
+ return grpc.aio.insecure_channel(target, options=options)
53
+ return grpc.aio.secure_channel(target, credentials, options=options)