tikv-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tikv/_sync/_compat.py ADDED
@@ -0,0 +1,43 @@
1
+ """Concurrency primitives for the blocking flavor of the client.
2
+
3
+ Mirrors ``_async/_compat.py``. Unlike the rest of ``_sync``, this module is
4
+ written by hand.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import threading
10
+ import time
11
+ from collections.abc import Callable, Sequence
12
+ from concurrent.futures import ThreadPoolExecutor
13
+ from typing import TypeVar
14
+
15
+ import grpc
16
+
17
+ T = TypeVar("T")
18
+ R = TypeVar("R")
19
+
20
+ Lock = threading.Lock
21
+ sleep = time.sleep
22
+ Channel = grpc.Channel
23
+
24
+ # Upper bound on requests in flight for one multi-region operation.
25
+ MAX_CONCURRENCY = 16
26
+
27
+
28
+ def map_concurrently(fn: Callable[[T], R], items: Sequence[T]) -> list[R]:
29
+ """Run ``fn`` on every item concurrently and return results in order."""
30
+ if len(items) <= 1:
31
+ return [fn(item) for item in items]
32
+ with ThreadPoolExecutor(max_workers=min(len(items), MAX_CONCURRENCY)) as pool:
33
+ return list(pool.map(fn, items))
34
+
35
+
36
+ def open_channel(
37
+ target: str,
38
+ credentials: grpc.ChannelCredentials | None,
39
+ options: Sequence[tuple[str, object]],
40
+ ) -> grpc.Channel:
41
+ if credentials is None:
42
+ return grpc.insecure_channel(target, options=options)
43
+ return grpc.secure_channel(target, credentials, options=options)
tikv/_sync/_pd.py ADDED
@@ -0,0 +1,168 @@
1
+ # Generated from _async/_pd.py by scripts/unasync.py. DO NOT EDIT.
2
+ """Client for the Placement Driver (PD), TiKV's cluster metadata service."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from collections.abc import Iterable
7
+
8
+ import grpc
9
+ from google.protobuf.message import Message
10
+
11
+ from .._backoff import Backoff, Wait, describe, log
12
+ from .._errors import PDError
13
+ from .._proto import pdpb_pb2 as pdpb
14
+ from .._region import Region, RegionCache, make_region
15
+ from .._rpc import (
16
+ PD_GET_MEMBERS,
17
+ PD_GET_PREV_REGION,
18
+ PD_GET_REGION,
19
+ PD_GET_STORE,
20
+ PD_SCAN_REGIONS,
21
+ Method,
22
+ Response,
23
+ is_retryable,
24
+ strip_scheme,
25
+ )
26
+ from ._compat import Lock, sleep
27
+ from ._transport import Transport
28
+
29
+ CALLER_ID = "tikv-python"
30
+
31
+ # PD answers metadata queries quickly; a slower reply means a sick leader,
32
+ # so give up on it early and rediscover instead of burning the whole budget.
33
+ RPC_TIMEOUT = 3.0
34
+
35
+
36
+ class PDClient:
37
+ """Finds the PD leader and asks it where regions and stores are."""
38
+
39
+ def __init__(self, endpoints: Iterable[str], transport: Transport, cache: RegionCache) -> None:
40
+ self._endpoints = list(dict.fromkeys(map(strip_scheme, endpoints)))
41
+ self._transport = transport
42
+ self._cache = cache
43
+ self._lock = Lock()
44
+ self._leader: str | None = None
45
+ self.cluster_id = 0
46
+
47
+ def region(self, key: bytes, backoff: Backoff) -> Region:
48
+ """The region containing *key*."""
49
+ while True:
50
+ region = self._fetch_region(PD_GET_REGION, key, backoff)
51
+ if region is not None:
52
+ return region
53
+ # PD has no region for this key yet, e.g. right after bootstrap.
54
+ sleep(backoff.delay(Wait.REGION_MISS, PDError(f"no region found for key {key!r}")))
55
+
56
+ def region_by_end(self, key: bytes, backoff: Backoff) -> Region:
57
+ """The region in which *key* is a valid exclusive upper bound."""
58
+ while True:
59
+ region = self.region(key, backoff)
60
+ if region.start != key or not key:
61
+ return region
62
+ # *key* opens this region, so the keys below it live in the one
63
+ # before. GetPrevRegion returns exactly that (like client-go).
64
+ previous = self._fetch_region(PD_GET_PREV_REGION, key, backoff)
65
+ if previous is not None:
66
+ return previous
67
+ # The regions merged in between: start over.
68
+ sleep(backoff.delay(Wait.REGION_MISS, PDError(f"no region found before key {key!r}")))
69
+
70
+ def _fetch_region(
71
+ self, method: Method[pdpb.GetRegionResponse], key: bytes, backoff: Backoff
72
+ ) -> Region | None:
73
+ response = self._call(method, pdpb.GetRegionRequest(region_key=key), backoff)
74
+ if not response.HasField("region"):
75
+ return None
76
+ region = make_region(response.region, response.leader)
77
+ self._cache.put(region)
78
+ return region
79
+
80
+ def scan_regions(self, start: bytes, end: bytes, limit: int, backoff: Backoff) -> list[Region]:
81
+ """Up to *limit* consecutive regions overlapping ``[start, end)``."""
82
+ request = pdpb.ScanRegionsRequest(start_key=start, end_key=end, limit=limit)
83
+ response = self._call(PD_SCAN_REGIONS, request, backoff)
84
+ if response.regions:
85
+ regions = [make_region(r.region, r.leader) for r in response.regions]
86
+ else: # older PD versions only fill the legacy fields
87
+ regions = [
88
+ make_region(m, leader)
89
+ for m, leader in zip(response.region_metas, response.leaders, strict=False)
90
+ ]
91
+ for region in regions:
92
+ self._cache.put(region)
93
+ return regions
94
+
95
+ def store_address(self, store_id: int, backoff: Backoff) -> str:
96
+ """The client-facing address of TiKV store *store_id*."""
97
+ address = self._cache.store_address(store_id)
98
+ if address is None:
99
+ response = self._call(PD_GET_STORE, pdpb.GetStoreRequest(store_id=store_id), backoff)
100
+ address = strip_scheme(response.store.address)
101
+ self._cache.set_store_address(store_id, address)
102
+ return address
103
+
104
+ def _call(self, method: Method[Response], request: Message, backoff: Backoff) -> Response:
105
+ """Send *request* to the PD leader, following leader changes."""
106
+ while True:
107
+ leader = self._leader_address(backoff)
108
+ request.header.cluster_id = self.cluster_id # type: ignore[attr-defined]
109
+ request.header.caller_id = CALLER_ID # type: ignore[attr-defined]
110
+ timeout = min(RPC_TIMEOUT, backoff.remaining())
111
+ try:
112
+ response = self._transport.call(leader, method, request, timeout)
113
+ except grpc.RpcError as error:
114
+ if not is_retryable(error):
115
+ raise PDError(f"{method.path} failed: {describe(error)}") from error
116
+ self._leader = None # the leader may have moved; rediscover it
117
+ sleep(backoff.delay(Wait.PD_RPC, error))
118
+ continue
119
+
120
+ header_error = response.header.error # type: ignore[attr-defined]
121
+ if header_error.type == pdpb.OK:
122
+ return response
123
+ failure = PDError(
124
+ f"{method.path} failed: {pdpb.ErrorType.Name(header_error.type)}: {header_error.message}"
125
+ )
126
+ if header_error.type != pdpb.REGION_NOT_FOUND:
127
+ raise failure
128
+ sleep(backoff.delay(Wait.REGION_MISS, failure))
129
+
130
+ def _leader_address(self, backoff: Backoff) -> str:
131
+ leader = self._leader
132
+ if leader is not None:
133
+ return leader
134
+ with self._lock:
135
+ while self._leader is None:
136
+ self._discover(backoff)
137
+ return self._leader
138
+
139
+ def _discover(self, backoff: Backoff) -> None:
140
+ """Ask the known PD members who the leader is."""
141
+ cause: Exception = PDError("no PD endpoint answered")
142
+ rejected: list[str] = []
143
+ for endpoint in list(self._endpoints):
144
+ timeout = min(RPC_TIMEOUT, backoff.remaining())
145
+ try:
146
+ response = self._transport.call(
147
+ endpoint, PD_GET_MEMBERS, pdpb.GetMembersRequest(), timeout
148
+ )
149
+ except grpc.RpcError as error:
150
+ if not is_retryable(error):
151
+ rejected.append(f"{endpoint} is not a PD server: {describe(error)}")
152
+ cause = error
153
+ continue
154
+ if not response.leader.client_urls:
155
+ cause = PDError(f"PD at {endpoint} reports no leader (election in progress?)")
156
+ continue
157
+ # Learn about every member so we survive losing the seed endpoints.
158
+ for member in response.members:
159
+ for url in member.client_urls:
160
+ if strip_scheme(url) not in self._endpoints:
161
+ self._endpoints.append(strip_scheme(url))
162
+ self.cluster_id = response.header.cluster_id
163
+ self._leader = strip_scheme(response.leader.client_urls[0])
164
+ log.debug("PD leader is %s (cluster %d)", self._leader, self.cluster_id)
165
+ return
166
+ if len(rejected) == len(self._endpoints): # retrying cannot help
167
+ raise PDError("; ".join(rejected)) from cause
168
+ sleep(backoff.delay(Wait.PD_RPC, cause))
@@ -0,0 +1,53 @@
1
+ # Generated from _async/_transport.py by scripts/unasync.py. DO NOT EDIT.
2
+ """A pool of gRPC channels, one per server address."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import threading
7
+ from typing import Any
8
+
9
+ import grpc
10
+ from google.protobuf.message import Message
11
+
12
+ from .._rpc import CHANNEL_OPTIONS, Method, Response
13
+ from ._compat import Channel, open_channel
14
+
15
+
16
+ class Transport:
17
+ """Sends unary requests to PD and TiKV servers over shared channels."""
18
+
19
+ def __init__(self, credentials: grpc.ChannelCredentials | None = None) -> None:
20
+ self._credentials = credentials
21
+ self._lock = threading.Lock() # never held across an await
22
+ self._channels: dict[str, Channel] = {}
23
+ self._stubs: dict[tuple[str, str], Any] = {}
24
+
25
+ def call(
26
+ self, address: str, method: Method[Response], request: Message, timeout: float
27
+ ) -> Response:
28
+ stub = self._stubs.get((address, method.path))
29
+ if stub is None:
30
+ stub = self._channel(address).unary_unary(
31
+ method.path,
32
+ request_serializer=type(request).SerializeToString,
33
+ response_deserializer=method.response.FromString,
34
+ )
35
+ self._stubs[address, method.path] = stub
36
+ response: Response = stub(request, timeout=timeout)
37
+ return response
38
+
39
+ def _channel(self, address: str) -> Channel:
40
+ with self._lock:
41
+ channel = self._channels.get(address)
42
+ if channel is None:
43
+ channel = open_channel(address, self._credentials, CHANNEL_OPTIONS)
44
+ self._channels[address] = channel
45
+ return channel
46
+
47
+ def close(self) -> None:
48
+ with self._lock:
49
+ channels = list(self._channels.values())
50
+ self._channels.clear()
51
+ self._stubs.clear()
52
+ for channel in channels:
53
+ channel.close()
tikv/_types.py ADDED
@@ -0,0 +1,34 @@
1
+ """Public result types."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import NamedTuple
6
+
7
+
8
+ class Checksum(NamedTuple):
9
+ """Outcome of :meth:`RawClient.checksum`: a fingerprint of a key range.
10
+
11
+ Equal ranges have equal checksums, which makes this handy for verifying
12
+ copies and migrations without reading the data back.
13
+ """
14
+
15
+ #: CRC64 of every key-value pair, XOR-ed together.
16
+ crc64_xor: int
17
+ #: Number of keys in the range.
18
+ keys: int
19
+ #: Total size of the keys and values, in bytes.
20
+ size: int
21
+
22
+
23
+ class CasResult(NamedTuple):
24
+ """Outcome of :meth:`RawClient.compare_and_swap`.
25
+
26
+ Unpacks naturally::
27
+
28
+ swapped, previous = client.compare_and_swap(key, b"old", b"new")
29
+ """
30
+
31
+ #: Whether the new value was written.
32
+ swapped: bool
33
+ #: The value before the operation, or ``None`` if the key did not exist.
34
+ previous: bytes | None
tikv/py.typed ADDED
File without changes
@@ -0,0 +1,387 @@
1
+ Metadata-Version: 2.5
2
+ Name: tikv-python
3
+ Version: 0.1.0
4
+ Summary: Pure-Python client for TiKV's raw key-value API
5
+ Project-URL: Homepage, https://github.com/slavabobik/tikv-python
6
+ Project-URL: Documentation, https://github.com/slavabobik/tikv-python#readme
7
+ Project-URL: Issues, https://github.com/slavabobik/tikv-python/issues
8
+ Project-URL: Changelog, https://github.com/slavabobik/tikv-python/releases
9
+ Author: Slava Bobik
10
+ License-Expression: Apache-2.0
11
+ License-File: LICENSE
12
+ Keywords: asyncio,database,grpc,key-value,pd,rawkv,tikv
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Framework :: AsyncIO
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3 :: Only
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Programming Language :: Python :: 3.14
24
+ Classifier: Topic :: Database
25
+ Classifier: Typing :: Typed
26
+ Requires-Python: >=3.10
27
+ Requires-Dist: grpcio>=1.60
28
+ Requires-Dist: protobuf>=4.25
29
+ Description-Content-Type: text/markdown
30
+
31
+ <h1 align="center">tikv-python</h1>
32
+
33
+ <p align="center">
34
+ <b>A pure-Python client for <a href="https://tikv.org">TiKV</a>'s raw key-value API.</b><br>
35
+ Blocking and asyncio. No Rust toolchain. Readable in an afternoon.
36
+ </p>
37
+
38
+ <p align="center">
39
+ <a href="https://github.com/slavabobik/tikv-python/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/slavabobik/tikv-python/actions/workflows/ci.yml/badge.svg"></a>
40
+ <a href="https://pypi.org/project/tikv-python/"><img alt="PyPI" src="https://img.shields.io/pypi/v/tikv-python.svg"></a>
41
+ <a href="pyproject.toml"><img alt="Python versions" src="https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fraw.githubusercontent.com%2Fslavabobik%2Ftikv-python%2Fmain%2Fpyproject.toml&query=%24.project.requires-python&label=python&color=3776AB"></a>
42
+ <a href="#compatibility"><img alt="Tested with TiKV 7.5 and 8.5" src="https://img.shields.io/badge/TiKV-7.5%20%7C%208.5-ff5a5f"></a>
43
+ <a href="LICENSE"><img alt="License: Apache 2.0" src="https://img.shields.io/badge/license-Apache%202.0-blue.svg"></a>
44
+ </p>
45
+
46
+ ```python
47
+ import tikv
48
+
49
+ with tikv.RawClient("pd-0:2379") as db:
50
+ db.put(b"user:1", b"Ada")
51
+ db.get(b"user:1") # b"Ada"
52
+
53
+ for key, value in db.scan(prefix=b"user:"): # lazy, crosses regions for you
54
+ print(key, value)
55
+
56
+ db.put(b"session:42", b"...", ttl=3600) # expires in an hour
57
+ ```
58
+
59
+ ## Why
60
+
61
+ TiKV has excellent clients for Go, Rust and Java. For Python, the option so far has
62
+ been [`tikv-client`](https://pypi.org/project/tikv-client/), a set of bindings to the
63
+ Rust client. Bindings are a reasonable way to bootstrap a client, but they leave
64
+ Python users exposed:
65
+
66
+ - **Wheels decide whether you can install it.** The latest release (0.0.4, January
67
+ 2024) ships wheels only for CPython 3.11 on x86-64 and macOS. On Python 3.12+, on
68
+ ARM Linux (Graviton, Axion, Ampere), or in a slim container, `pip install` turns
69
+ into compiling a Rust crate and its gRPC stack from source.
70
+ - **Failures cross a language boundary.** A stuck request or a strange error means
71
+ debugging Rust futures behind an FFI layer, from a Python stack trace that ends
72
+ at the binding.
73
+ - **asyncio is bolted on.** Async calls are bridged into a separate Tokio runtime
74
+ instead of running on your event loop.
75
+ - **Releases are tied to the binding layer.** A fix in TiKV's Rust client reaches
76
+ Python only after the bindings are rebuilt, re-pinned and republished.
77
+
78
+ TiKV speaks plain gRPC, and Python already has a production-grade gRPC stack. So
79
+ this library talks to PD and TiKV directly, the way the Go and Rust clients do:
80
+
81
+ - **Installs anywhere `grpcio` does**, which is practically everywhere. It is a
82
+ `py3-none-any` wheel with two dependencies.
83
+ - **Your event loop, your threads.** `AsyncRawClient` is native asyncio.
84
+ `RawClient` is a plain thread-safe blocking client.
85
+ - **Small enough to own.** About 1,500 lines of typed, commented Python. When
86
+ something goes wrong, you can read exactly what the client did, and every retry
87
+ is logged.
88
+ - **Behaves like the official clients.** Region routing, retries and backoff follow
89
+ client-go and client-rust, verified against a real cluster.
90
+
91
+ ## Features
92
+
93
+ - The full raw API: `get`, `put`, `delete`, batches, range scans (forward and
94
+ reverse), `delete_range`, TTLs, `compare_and_swap` and `checksum`
95
+ - Blocking `RawClient` and asyncio `AsyncRawClient` with identical APIs
96
+ - Automatic routing across regions, following splits, merges, leader changes and
97
+ PD failover
98
+ - A time budget per operation, with exponential backoff and jitter on every
99
+ transient error
100
+ - Lazy, constant-memory scans over any number of regions
101
+ - Mutual TLS
102
+ - Fully typed (`mypy --strict`), `py.typed`, no `Any` in the public API
103
+
104
+ ## Installation
105
+
106
+ ```console
107
+ pip install tikv-python # or: uv add tikv-python
108
+ ```
109
+
110
+ The package imports as `tikv`. It needs Python 3.10+, `grpcio>=1.60` and
111
+ `protobuf>=4.25`.
112
+
113
+ ## Quickstart
114
+
115
+ Point the client at one or more PD servers. It discovers the rest of the cluster on
116
+ its own.
117
+
118
+ ```python
119
+ import tikv
120
+
121
+ with tikv.RawClient(["pd-0:2379", "pd-1:2379"]) as db:
122
+ db.put_many({b"user:1": b"Ada", b"user:2": b"Grace", b"user:3": b"Linus"})
123
+
124
+ db.get(b"user:2") # b"Grace"
125
+ db.get(b"user:9") # None
126
+ db.get_many([b"user:1", b"user:9"]) # {b"user:1": b"Ada"}
127
+
128
+ list(db.scan_keys(prefix=b"user:", reverse=True, limit=2))
129
+ # [b"user:3", b"user:2"]
130
+
131
+ db.delete(b"user:1")
132
+ db.delete_range(prefix=b"user:")
133
+ ```
134
+
135
+ The asyncio client has the same API. Use `await` and `async for`:
136
+
137
+ ```python
138
+ import asyncio
139
+ import tikv
140
+
141
+ async def main() -> None:
142
+ async with tikv.AsyncRawClient("pd-0:2379") as db:
143
+ await asyncio.gather(*(db.put(f"k{i}".encode(), b"v") for i in range(1000)))
144
+ async for key, value in db.scan(b"k", b"l", limit=10):
145
+ print(key, value)
146
+
147
+ asyncio.run(main())
148
+ ```
149
+
150
+ ## Guide
151
+
152
+ ### API at a glance
153
+
154
+ | Method | What it does |
155
+ | --- | --- |
156
+ | `get(key)` | The value, or `None` if the key does not exist |
157
+ | `get_many(keys)` | A `dict` of the keys that exist, in the order requested |
158
+ | `put(key, value, *, ttl=None)` | Set a key, optionally with an expiry |
159
+ | `put_many(items, *, ttl=None)` | Set many keys from a mapping or `(key, value)` pairs |
160
+ | `delete(key)` / `delete_many(keys)` | Delete keys; missing keys are not an error |
161
+ | `delete_range(start, end)` / `delete_range(prefix=…)` | Delete a whole range of keys |
162
+ | `scan(start, end, *, prefix, limit, reverse)` | Lazily iterate over `(key, value)` pairs |
163
+ | `scan_keys(…)` | Like `scan`, without transferring values |
164
+ | `ttl(key)` | Seconds until the key expires: `None` if never, `KeyError` if missing |
165
+ | `checksum(start, end, *, prefix)` | A `Checksum(crc64_xor, keys, size)` computed by the servers |
166
+ | `compare_and_swap(key, expected, new, *, ttl=None)` | Atomic CAS, returning `CasResult(swapped, previous)` |
167
+
168
+ Keys and values are `bytes` (`bytearray` and `memoryview` are fine too). Passing a
169
+ `str` raises `TypeError` instead of silently picking an encoding.
170
+
171
+ ### Ranges and scans
172
+
173
+ Ranges are half-open, `[start, end)`, and `None` means "unbounded":
174
+
175
+ ```python
176
+ db.scan() # every key
177
+ db.scan(b"m") # from b"m" to the end
178
+ db.scan(b"a", b"n") # b"a" up to, but excluding, b"n"
179
+ db.scan(prefix=b"log:") # every key starting with b"log:"
180
+ db.scan(prefix=b"log:", reverse=True, limit=100) # the newest 100 log keys
181
+ ```
182
+
183
+ Scans fetch one page at a time from the right region, so iterating over millions
184
+ of keys uses constant memory. Stop iterating whenever you like.
185
+
186
+ `delete_range` refuses to run without `start` or a non-empty `prefix`, so a missing
187
+ argument cannot wipe your cluster. Deleting everything takes an explicit
188
+ `delete_range(b"")`.
189
+
190
+ ### Atomicity and compare-and-swap
191
+
192
+ Single-key operations are atomic. Batch operations are atomic within a region but
193
+ not across regions; that is how TiKV's raw API works.
194
+
195
+ `compare_and_swap` needs a client in *atomic mode*. In that mode every write goes
196
+ through TiKV's single-row transaction path, which makes CAS linearizable:
197
+
198
+ ```python
199
+ locks = tikv.RawClient("pd-0:2379", atomic=True)
200
+
201
+ swapped, holder = locks.compare_and_swap(b"lock:job-7", None, b"worker-3")
202
+ if not swapped:
203
+ print(f"already held by {holder!r}")
204
+ ```
205
+
206
+ > **Warning:** Every client that writes the same keys must use the same mode.
207
+ > Mixing modes breaks linearizability. `delete_range` is unavailable in atomic
208
+ > mode because TiKV cannot perform it atomically.
209
+
210
+ ### TTL
211
+
212
+ `ttl=` takes seconds, rounded up to a whole second (TiKV's precision), or a
213
+ `datetime.timedelta`. Expiry needs `storage.enable-ttl = true` on the TiKV side.
214
+
215
+ ```python
216
+ db.put(b"otp:5551234", b"839201", ttl=timedelta(minutes=5))
217
+ db.ttl(b"otp:5551234") # 300
218
+ ```
219
+
220
+ ### Errors, timeouts and retries
221
+
222
+ Every operation gets a time budget: `timeout=10.0` seconds by default, set per
223
+ client. Within that budget the client retries every transient failure with
224
+ exponential backoff and jitter:
225
+
226
+ - stale routing after a region split or merge,
227
+ - leader changes and elections,
228
+ - busy or unreachable stores,
229
+ - PD leader failover.
230
+
231
+ Each scan page gets its own budget. When the budget runs out you get
232
+ `tikv.TimeoutError`, which is also a builtin `TimeoutError`, with the last
233
+ underlying error chained as `__cause__`.
234
+
235
+ | Exception | Raised when |
236
+ | --- | --- |
237
+ | `TiKVError` | Base class of everything below |
238
+ | `TimeoutError` | The operation could not complete within `timeout` |
239
+ | `ServerError` | TiKV rejected the request, e.g. a key over `max-key-size` |
240
+ | `RegionError` | A region error that retrying cannot fix, e.g. a write too large for Raft |
241
+ | `PDError` | PD rejected a request, or an endpoint is not PD at all |
242
+ | `UndeterminedError` | A `compare_and_swap` failed in flight and may or may not have applied |
243
+
244
+ `compare_and_swap` is deliberately never replayed after a failure that might have
245
+ applied. If a replayed swap had already succeeded, it would report a spurious
246
+ failure. Read the key to find out what happened.
247
+
248
+ Every retry is logged at `DEBUG` level on the `tikv` logger, with the reason and the
249
+ delay:
250
+
251
+ ```python
252
+ logging.getLogger("tikv").setLevel(logging.DEBUG)
253
+ ```
254
+
255
+ ### Concurrency
256
+
257
+ Create one client and share it across your application. It keeps one HTTP/2
258
+ connection per server, and requests are multiplexed over it.
259
+
260
+ - `RawClient` is thread-safe. Multi-region batches fan out over a small thread
261
+ pool.
262
+ - `AsyncRawClient` is task-safe and fans out with `asyncio`. Like every gRPC
263
+ asyncio channel, it belongs to the event loop it is first used in.
264
+
265
+ ### TLS
266
+
267
+ ```python
268
+ tls = tikv.TLSConfig(ca="ca.pem", cert="client.pem", key="client-key.pem")
269
+ db = tikv.RawClient("pd-0:2379", tls=tls)
270
+ ```
271
+
272
+ ## Performance
273
+
274
+ The numbers below come from a 3-node TiKV 8.5 cluster on Kubernetes, with the
275
+ client running in a pod in the same cluster on Python 3.12. Your hardware will
276
+ differ, but the shape holds: per-request overhead stays well under a millisecond,
277
+ and batches and scans move hundreds of thousands of keys per second.
278
+
279
+ | Operation | Result |
280
+ | --- | --- |
281
+ | `get`, sequential | 0.53 ms median, 0.95 ms p99 |
282
+ | `put`, sequential | 1.75 ms median, 3.82 ms p99 |
283
+ | `put_many`, 50,000 × 100-byte values | ~263,000 keys/s |
284
+ | `get_many`, 50,000 keys | ~344,000 keys/s |
285
+ | `scan`, 50,000 keys | ~376,000 keys/s |
286
+ | `AsyncRawClient.get`, 10,000 concurrent | ~11,500 ops/s from one event loop |
287
+
288
+ ## How it works
289
+
290
+ ```
291
+ your code ──► RawClient / AsyncRawClient
292
+ │
293
+ ├─ region cache ── sorted by start key; maps every key to its region and leader
294
+ │ ▲
295
+ │ └─ PD client ── finds the PD leader, follows failover,
296
+ │ fetches regions (64 at a time when walking a range)
297
+ │
298
+ └─ request executor ── groups keys by region, sends to leaders concurrently,
299
+ handles NotLeader / EpochNotMatch / ServerIsBusy / …,
300
+ re-groups and retries within the time budget
301
+ ```
302
+
303
+ A few design decisions worth knowing:
304
+
305
+ - **One implementation, two flavors.** The asyncio code in `src/tikv/_async` is
306
+ the source of truth. [`scripts/unasync.py`](scripts/unasync.py) mechanically
307
+ generates the blocking client from it, the technique
308
+ [httpcore](https://github.com/encode/httpcore) uses, and CI fails if they drift.
309
+ - **A trimmed protocol.** [`proto/`](proto) is the subset of
310
+ [kvproto](https://github.com/pingcap/kvproto) a raw client needs, with upstream
311
+ names and field numbers. Fields the client never reads are omitted, and protobuf
312
+ preserves them as unknown fields on the wire. The generated modules import in
313
+ about a millisecond and work with protobuf 4.25 through 7.x.
314
+ - **Region errors drive the cache.** Like client-go, the client never polls for
315
+ topology changes. A routing error invalidates or patches exactly the cache
316
+ entries involved, and the affected keys are re-grouped and retried.
317
+ - **Tested against a fake that behaves like the real thing.**
318
+ [`tests/fake_cluster.py`](tests/fake_cluster.py) is an in-process PD + TiKV that
319
+ speaks the real gRPC protocol. It splits and merges regions, moves leaders and
320
+ injects failures, and it validates routing the way TiKV does. CI also runs the
321
+ suite against real TiKV.
322
+
323
+ ## Compatibility
324
+
325
+ | | Supported |
326
+ | --- | --- |
327
+ | Python | 3.10 – 3.14 |
328
+ | TiKV / PD | 7.5 and 8.5 LTS, tested in CI |
329
+ | Storage API | `api-version = 1`, with or without `enable-ttl` |
330
+ | Platforms | Anywhere `grpcio` installs: Linux (x86-64, ARM), macOS, Windows |
331
+
332
+ ## Roadmap
333
+
334
+ These are tracked in [issues](https://github.com/slavabobik/tikv-python/issues):
335
+
336
+ - API v2 keyspaces (`storage.api-version = 2`)
337
+ - Column families other than `default`
338
+ - Chaos tests against a multi-node cluster: leader failover, store loss, PD failover
339
+ - A transactional (TxnKV) client
340
+
341
+ Contributions are very welcome. Open an issue to discuss bigger changes first.
342
+
343
+ ## Development
344
+
345
+ ```console
346
+ git clone https://github.com/slavabobik/tikv-python && cd tikv-python
347
+ uv sync # dev environment in .venv
348
+ uv run pytest # unit tests against the fake cluster
349
+ uv run ruff check && uv run ruff format --check && uv run mypy
350
+ ```
351
+
352
+ Integration tests run against any real cluster. They write only under a unique key
353
+ prefix and delete it afterwards:
354
+
355
+ ```console
356
+ ci/start-cluster.sh v8.5.8 # single-node cluster in Docker
357
+ TIKV_PD_ENDPOINTS=127.0.0.1:2379 uv run pytest -m integration
358
+ ```
359
+
360
+ After changing `src/tikv/_async`, regenerate the blocking client:
361
+
362
+ ```console
363
+ uv run python scripts/unasync.py
364
+ ```
365
+
366
+ After changing `proto/`, regenerate the messages. The pinned generator keeps the
367
+ protobuf floor low:
368
+
369
+ ```console
370
+ uv run --with 'grpcio-tools==1.62.3' python scripts/generate_protos.py
371
+ ```
372
+
373
+ ### Releasing
374
+
375
+ 1. Bump `__version__` in `src/tikv/__init__.py` and merge it to `main`.
376
+ 2. Tag and push the release: `git tag v0.2.0 && git push origin v0.2.0`.
377
+
378
+ The [release workflow](.github/workflows/release.yml) checks that the tag matches
379
+ the version, publishes to PyPI with
380
+ [trusted publishing](https://docs.pypi.org/trusted-publishers/), and creates a
381
+ GitHub release with the distributions attached. You can also run it by hand from
382
+ the Actions tab to publish to TestPyPI or PyPI, optionally under a different
383
+ distribution name; the import name is always `tikv`.
384
+
385
+ ## License
386
+
387
+ [Apache License 2.0](LICENSE), the same as TiKV.