tikv-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tikv/__init__.py +44 -0
- tikv/_async/__init__.py +1 -0
- tikv/_async/_client.py +584 -0
- tikv/_async/_compat.py +53 -0
- tikv/_async/_pd.py +167 -0
- tikv/_async/_transport.py +52 -0
- tikv/_backoff.py +78 -0
- tikv/_errors.py +53 -0
- tikv/_keys.py +109 -0
- tikv/_proto/__init__.py +2 -0
- tikv/_proto/errorpb_pb2.py +68 -0
- tikv/_proto/errorpb_pb2.pyi +200 -0
- tikv/_proto/kvrpcpb_pb2.py +85 -0
- tikv/_proto/kvrpcpb_pb2.pyi +341 -0
- tikv/_proto/metapb_pb2.py +39 -0
- tikv/_proto/metapb_pb2.pyi +92 -0
- tikv/_proto/pdpb_pb2.py +56 -0
- tikv/_proto/pdpb_pb2.pyi +160 -0
- tikv/_region.py +228 -0
- tikv/_rpc.py +109 -0
- tikv/_sync/__init__.py +2 -0
- tikv/_sync/_client.py +585 -0
- tikv/_sync/_compat.py +43 -0
- tikv/_sync/_pd.py +168 -0
- tikv/_sync/_transport.py +53 -0
- tikv/_types.py +34 -0
- tikv/py.typed +0 -0
- tikv_python-0.1.0.dist-info/METADATA +387 -0
- tikv_python-0.1.0.dist-info/RECORD +31 -0
- tikv_python-0.1.0.dist-info/WHEEL +4 -0
- tikv_python-0.1.0.dist-info/licenses/LICENSE +202 -0
tikv/_sync/_compat.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Concurrency primitives for the blocking flavor of the client.
|
|
2
|
+
|
|
3
|
+
Mirrors ``_async/_compat.py``. Unlike the rest of ``_sync``, this module is
|
|
4
|
+
written by hand.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
from collections.abc import Callable, Sequence
|
|
12
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
13
|
+
from typing import TypeVar
|
|
14
|
+
|
|
15
|
+
import grpc
|
|
16
|
+
|
|
17
|
+
T = TypeVar("T")
|
|
18
|
+
R = TypeVar("R")
|
|
19
|
+
|
|
20
|
+
Lock = threading.Lock
|
|
21
|
+
sleep = time.sleep
|
|
22
|
+
Channel = grpc.Channel
|
|
23
|
+
|
|
24
|
+
# Upper bound on requests in flight for one multi-region operation.
|
|
25
|
+
MAX_CONCURRENCY = 16
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def map_concurrently(fn: Callable[[T], R], items: Sequence[T]) -> list[R]:
|
|
29
|
+
"""Run ``fn`` on every item concurrently and return results in order."""
|
|
30
|
+
if len(items) <= 1:
|
|
31
|
+
return [fn(item) for item in items]
|
|
32
|
+
with ThreadPoolExecutor(max_workers=min(len(items), MAX_CONCURRENCY)) as pool:
|
|
33
|
+
return list(pool.map(fn, items))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def open_channel(
|
|
37
|
+
target: str,
|
|
38
|
+
credentials: grpc.ChannelCredentials | None,
|
|
39
|
+
options: Sequence[tuple[str, object]],
|
|
40
|
+
) -> grpc.Channel:
|
|
41
|
+
if credentials is None:
|
|
42
|
+
return grpc.insecure_channel(target, options=options)
|
|
43
|
+
return grpc.secure_channel(target, credentials, options=options)
|
tikv/_sync/_pd.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# Generated from _async/_pd.py by scripts/unasync.py. DO NOT EDIT.
|
|
2
|
+
"""Client for the Placement Driver (PD), TiKV's cluster metadata service."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
|
|
8
|
+
import grpc
|
|
9
|
+
from google.protobuf.message import Message
|
|
10
|
+
|
|
11
|
+
from .._backoff import Backoff, Wait, describe, log
|
|
12
|
+
from .._errors import PDError
|
|
13
|
+
from .._proto import pdpb_pb2 as pdpb
|
|
14
|
+
from .._region import Region, RegionCache, make_region
|
|
15
|
+
from .._rpc import (
|
|
16
|
+
PD_GET_MEMBERS,
|
|
17
|
+
PD_GET_PREV_REGION,
|
|
18
|
+
PD_GET_REGION,
|
|
19
|
+
PD_GET_STORE,
|
|
20
|
+
PD_SCAN_REGIONS,
|
|
21
|
+
Method,
|
|
22
|
+
Response,
|
|
23
|
+
is_retryable,
|
|
24
|
+
strip_scheme,
|
|
25
|
+
)
|
|
26
|
+
from ._compat import Lock, sleep
|
|
27
|
+
from ._transport import Transport
|
|
28
|
+
|
|
29
|
+
CALLER_ID = "tikv-python"
|
|
30
|
+
|
|
31
|
+
# PD answers metadata queries quickly; a slower reply means a sick leader,
|
|
32
|
+
# so give up on it early and rediscover instead of burning the whole budget.
|
|
33
|
+
RPC_TIMEOUT = 3.0
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class PDClient:
|
|
37
|
+
"""Finds the PD leader and asks it where regions and stores are."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, endpoints: Iterable[str], transport: Transport, cache: RegionCache) -> None:
|
|
40
|
+
self._endpoints = list(dict.fromkeys(map(strip_scheme, endpoints)))
|
|
41
|
+
self._transport = transport
|
|
42
|
+
self._cache = cache
|
|
43
|
+
self._lock = Lock()
|
|
44
|
+
self._leader: str | None = None
|
|
45
|
+
self.cluster_id = 0
|
|
46
|
+
|
|
47
|
+
def region(self, key: bytes, backoff: Backoff) -> Region:
|
|
48
|
+
"""The region containing *key*."""
|
|
49
|
+
while True:
|
|
50
|
+
region = self._fetch_region(PD_GET_REGION, key, backoff)
|
|
51
|
+
if region is not None:
|
|
52
|
+
return region
|
|
53
|
+
# PD has no region for this key yet, e.g. right after bootstrap.
|
|
54
|
+
sleep(backoff.delay(Wait.REGION_MISS, PDError(f"no region found for key {key!r}")))
|
|
55
|
+
|
|
56
|
+
def region_by_end(self, key: bytes, backoff: Backoff) -> Region:
|
|
57
|
+
"""The region in which *key* is a valid exclusive upper bound."""
|
|
58
|
+
while True:
|
|
59
|
+
region = self.region(key, backoff)
|
|
60
|
+
if region.start != key or not key:
|
|
61
|
+
return region
|
|
62
|
+
# *key* opens this region, so the keys below it live in the one
|
|
63
|
+
# before. GetPrevRegion returns exactly that (like client-go).
|
|
64
|
+
previous = self._fetch_region(PD_GET_PREV_REGION, key, backoff)
|
|
65
|
+
if previous is not None:
|
|
66
|
+
return previous
|
|
67
|
+
# The regions merged in between: start over.
|
|
68
|
+
sleep(backoff.delay(Wait.REGION_MISS, PDError(f"no region found before key {key!r}")))
|
|
69
|
+
|
|
70
|
+
def _fetch_region(
|
|
71
|
+
self, method: Method[pdpb.GetRegionResponse], key: bytes, backoff: Backoff
|
|
72
|
+
) -> Region | None:
|
|
73
|
+
response = self._call(method, pdpb.GetRegionRequest(region_key=key), backoff)
|
|
74
|
+
if not response.HasField("region"):
|
|
75
|
+
return None
|
|
76
|
+
region = make_region(response.region, response.leader)
|
|
77
|
+
self._cache.put(region)
|
|
78
|
+
return region
|
|
79
|
+
|
|
80
|
+
def scan_regions(self, start: bytes, end: bytes, limit: int, backoff: Backoff) -> list[Region]:
|
|
81
|
+
"""Up to *limit* consecutive regions overlapping ``[start, end)``."""
|
|
82
|
+
request = pdpb.ScanRegionsRequest(start_key=start, end_key=end, limit=limit)
|
|
83
|
+
response = self._call(PD_SCAN_REGIONS, request, backoff)
|
|
84
|
+
if response.regions:
|
|
85
|
+
regions = [make_region(r.region, r.leader) for r in response.regions]
|
|
86
|
+
else: # older PD versions only fill the legacy fields
|
|
87
|
+
regions = [
|
|
88
|
+
make_region(m, leader)
|
|
89
|
+
for m, leader in zip(response.region_metas, response.leaders, strict=False)
|
|
90
|
+
]
|
|
91
|
+
for region in regions:
|
|
92
|
+
self._cache.put(region)
|
|
93
|
+
return regions
|
|
94
|
+
|
|
95
|
+
def store_address(self, store_id: int, backoff: Backoff) -> str:
|
|
96
|
+
"""The client-facing address of TiKV store *store_id*."""
|
|
97
|
+
address = self._cache.store_address(store_id)
|
|
98
|
+
if address is None:
|
|
99
|
+
response = self._call(PD_GET_STORE, pdpb.GetStoreRequest(store_id=store_id), backoff)
|
|
100
|
+
address = strip_scheme(response.store.address)
|
|
101
|
+
self._cache.set_store_address(store_id, address)
|
|
102
|
+
return address
|
|
103
|
+
|
|
104
|
+
def _call(self, method: Method[Response], request: Message, backoff: Backoff) -> Response:
|
|
105
|
+
"""Send *request* to the PD leader, following leader changes."""
|
|
106
|
+
while True:
|
|
107
|
+
leader = self._leader_address(backoff)
|
|
108
|
+
request.header.cluster_id = self.cluster_id # type: ignore[attr-defined]
|
|
109
|
+
request.header.caller_id = CALLER_ID # type: ignore[attr-defined]
|
|
110
|
+
timeout = min(RPC_TIMEOUT, backoff.remaining())
|
|
111
|
+
try:
|
|
112
|
+
response = self._transport.call(leader, method, request, timeout)
|
|
113
|
+
except grpc.RpcError as error:
|
|
114
|
+
if not is_retryable(error):
|
|
115
|
+
raise PDError(f"{method.path} failed: {describe(error)}") from error
|
|
116
|
+
self._leader = None # the leader may have moved; rediscover it
|
|
117
|
+
sleep(backoff.delay(Wait.PD_RPC, error))
|
|
118
|
+
continue
|
|
119
|
+
|
|
120
|
+
header_error = response.header.error # type: ignore[attr-defined]
|
|
121
|
+
if header_error.type == pdpb.OK:
|
|
122
|
+
return response
|
|
123
|
+
failure = PDError(
|
|
124
|
+
f"{method.path} failed: {pdpb.ErrorType.Name(header_error.type)}: {header_error.message}"
|
|
125
|
+
)
|
|
126
|
+
if header_error.type != pdpb.REGION_NOT_FOUND:
|
|
127
|
+
raise failure
|
|
128
|
+
sleep(backoff.delay(Wait.REGION_MISS, failure))
|
|
129
|
+
|
|
130
|
+
def _leader_address(self, backoff: Backoff) -> str:
|
|
131
|
+
leader = self._leader
|
|
132
|
+
if leader is not None:
|
|
133
|
+
return leader
|
|
134
|
+
with self._lock:
|
|
135
|
+
while self._leader is None:
|
|
136
|
+
self._discover(backoff)
|
|
137
|
+
return self._leader
|
|
138
|
+
|
|
139
|
+
def _discover(self, backoff: Backoff) -> None:
|
|
140
|
+
"""Ask the known PD members who the leader is."""
|
|
141
|
+
cause: Exception = PDError("no PD endpoint answered")
|
|
142
|
+
rejected: list[str] = []
|
|
143
|
+
for endpoint in list(self._endpoints):
|
|
144
|
+
timeout = min(RPC_TIMEOUT, backoff.remaining())
|
|
145
|
+
try:
|
|
146
|
+
response = self._transport.call(
|
|
147
|
+
endpoint, PD_GET_MEMBERS, pdpb.GetMembersRequest(), timeout
|
|
148
|
+
)
|
|
149
|
+
except grpc.RpcError as error:
|
|
150
|
+
if not is_retryable(error):
|
|
151
|
+
rejected.append(f"{endpoint} is not a PD server: {describe(error)}")
|
|
152
|
+
cause = error
|
|
153
|
+
continue
|
|
154
|
+
if not response.leader.client_urls:
|
|
155
|
+
cause = PDError(f"PD at {endpoint} reports no leader (election in progress?)")
|
|
156
|
+
continue
|
|
157
|
+
# Learn about every member so we survive losing the seed endpoints.
|
|
158
|
+
for member in response.members:
|
|
159
|
+
for url in member.client_urls:
|
|
160
|
+
if strip_scheme(url) not in self._endpoints:
|
|
161
|
+
self._endpoints.append(strip_scheme(url))
|
|
162
|
+
self.cluster_id = response.header.cluster_id
|
|
163
|
+
self._leader = strip_scheme(response.leader.client_urls[0])
|
|
164
|
+
log.debug("PD leader is %s (cluster %d)", self._leader, self.cluster_id)
|
|
165
|
+
return
|
|
166
|
+
if len(rejected) == len(self._endpoints): # retrying cannot help
|
|
167
|
+
raise PDError("; ".join(rejected)) from cause
|
|
168
|
+
sleep(backoff.delay(Wait.PD_RPC, cause))
|
tikv/_sync/_transport.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Generated from _async/_transport.py by scripts/unasync.py. DO NOT EDIT.
|
|
2
|
+
"""A pool of gRPC channels, one per server address."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import threading
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import grpc
|
|
10
|
+
from google.protobuf.message import Message
|
|
11
|
+
|
|
12
|
+
from .._rpc import CHANNEL_OPTIONS, Method, Response
|
|
13
|
+
from ._compat import Channel, open_channel
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Transport:
|
|
17
|
+
"""Sends unary requests to PD and TiKV servers over shared channels."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, credentials: grpc.ChannelCredentials | None = None) -> None:
|
|
20
|
+
self._credentials = credentials
|
|
21
|
+
self._lock = threading.Lock() # never held across an await
|
|
22
|
+
self._channels: dict[str, Channel] = {}
|
|
23
|
+
self._stubs: dict[tuple[str, str], Any] = {}
|
|
24
|
+
|
|
25
|
+
def call(
|
|
26
|
+
self, address: str, method: Method[Response], request: Message, timeout: float
|
|
27
|
+
) -> Response:
|
|
28
|
+
stub = self._stubs.get((address, method.path))
|
|
29
|
+
if stub is None:
|
|
30
|
+
stub = self._channel(address).unary_unary(
|
|
31
|
+
method.path,
|
|
32
|
+
request_serializer=type(request).SerializeToString,
|
|
33
|
+
response_deserializer=method.response.FromString,
|
|
34
|
+
)
|
|
35
|
+
self._stubs[address, method.path] = stub
|
|
36
|
+
response: Response = stub(request, timeout=timeout)
|
|
37
|
+
return response
|
|
38
|
+
|
|
39
|
+
def _channel(self, address: str) -> Channel:
|
|
40
|
+
with self._lock:
|
|
41
|
+
channel = self._channels.get(address)
|
|
42
|
+
if channel is None:
|
|
43
|
+
channel = open_channel(address, self._credentials, CHANNEL_OPTIONS)
|
|
44
|
+
self._channels[address] = channel
|
|
45
|
+
return channel
|
|
46
|
+
|
|
47
|
+
def close(self) -> None:
|
|
48
|
+
with self._lock:
|
|
49
|
+
channels = list(self._channels.values())
|
|
50
|
+
self._channels.clear()
|
|
51
|
+
self._stubs.clear()
|
|
52
|
+
for channel in channels:
|
|
53
|
+
channel.close()
|
tikv/_types.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Public result types."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import NamedTuple
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Checksum(NamedTuple):
|
|
9
|
+
"""Outcome of :meth:`RawClient.checksum`: a fingerprint of a key range.
|
|
10
|
+
|
|
11
|
+
Equal ranges have equal checksums, which makes this handy for verifying
|
|
12
|
+
copies and migrations without reading the data back.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
#: CRC64 of every key-value pair, XOR-ed together.
|
|
16
|
+
crc64_xor: int
|
|
17
|
+
#: Number of keys in the range.
|
|
18
|
+
keys: int
|
|
19
|
+
#: Total size of the keys and values, in bytes.
|
|
20
|
+
size: int
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class CasResult(NamedTuple):
|
|
24
|
+
"""Outcome of :meth:`RawClient.compare_and_swap`.
|
|
25
|
+
|
|
26
|
+
Unpacks naturally::
|
|
27
|
+
|
|
28
|
+
swapped, previous = client.compare_and_swap(key, b"old", b"new")
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
#: Whether the new value was written.
|
|
32
|
+
swapped: bool
|
|
33
|
+
#: The value before the operation, or ``None`` if the key did not exist.
|
|
34
|
+
previous: bytes | None
|
tikv/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: tikv-python
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Pure-Python client for TiKV's raw key-value API
|
|
5
|
+
Project-URL: Homepage, https://github.com/slavabobik/tikv-python
|
|
6
|
+
Project-URL: Documentation, https://github.com/slavabobik/tikv-python#readme
|
|
7
|
+
Project-URL: Issues, https://github.com/slavabobik/tikv-python/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/slavabobik/tikv-python/releases
|
|
9
|
+
Author: Slava Bobik
|
|
10
|
+
License-Expression: Apache-2.0
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: asyncio,database,grpc,key-value,pd,rawkv,tikv
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Framework :: AsyncIO
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Topic :: Database
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Requires-Dist: grpcio>=1.60
|
|
28
|
+
Requires-Dist: protobuf>=4.25
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
<h1 align="center">tikv-python</h1>
|
|
32
|
+
|
|
33
|
+
<p align="center">
|
|
34
|
+
<b>A pure-Python client for <a href="https://tikv.org">TiKV</a>'s raw key-value API.</b><br>
|
|
35
|
+
Blocking and asyncio. No Rust toolchain. Readable in an afternoon.
|
|
36
|
+
</p>
|
|
37
|
+
|
|
38
|
+
<p align="center">
|
|
39
|
+
<a href="https://github.com/slavabobik/tikv-python/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/slavabobik/tikv-python/actions/workflows/ci.yml/badge.svg"></a>
|
|
40
|
+
<a href="https://pypi.org/project/tikv-python/"><img alt="PyPI" src="https://img.shields.io/pypi/v/tikv-python.svg"></a>
|
|
41
|
+
<a href="pyproject.toml"><img alt="Python versions" src="https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fraw.githubusercontent.com%2Fslavabobik%2Ftikv-python%2Fmain%2Fpyproject.toml&query=%24.project.requires-python&label=python&color=3776AB"></a>
|
|
42
|
+
<a href="#compatibility"><img alt="Tested with TiKV 7.5 and 8.5" src="https://img.shields.io/badge/TiKV-7.5%20%7C%208.5-ff5a5f"></a>
|
|
43
|
+
<a href="LICENSE"><img alt="License: Apache 2.0" src="https://img.shields.io/badge/license-Apache%202.0-blue.svg"></a>
|
|
44
|
+
</p>
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
import tikv
|
|
48
|
+
|
|
49
|
+
with tikv.RawClient("pd-0:2379") as db:
|
|
50
|
+
db.put(b"user:1", b"Ada")
|
|
51
|
+
db.get(b"user:1") # b"Ada"
|
|
52
|
+
|
|
53
|
+
for key, value in db.scan(prefix=b"user:"): # lazy, crosses regions for you
|
|
54
|
+
print(key, value)
|
|
55
|
+
|
|
56
|
+
db.put(b"session:42", b"...", ttl=3600) # expires in an hour
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Why
|
|
60
|
+
|
|
61
|
+
TiKV has excellent clients for Go, Rust and Java. For Python, the option so far has
|
|
62
|
+
been [`tikv-client`](https://pypi.org/project/tikv-client/), a set of bindings to the
|
|
63
|
+
Rust client. Bindings are a reasonable way to bootstrap a client, but they leave
|
|
64
|
+
Python users exposed:
|
|
65
|
+
|
|
66
|
+
- **Wheels decide whether you can install it.** The latest release (0.0.4, January
|
|
67
|
+
2024) ships wheels only for CPython 3.11 on x86-64 and macOS. On Python 3.12+, on
|
|
68
|
+
ARM Linux (Graviton, Axion, Ampere), or in a slim container, `pip install` turns
|
|
69
|
+
into compiling a Rust crate and its gRPC stack from source.
|
|
70
|
+
- **Failures cross a language boundary.** A stuck request or a strange error means
|
|
71
|
+
debugging Rust futures behind an FFI layer, from a Python stack trace that ends
|
|
72
|
+
at the binding.
|
|
73
|
+
- **asyncio is bolted on.** Async calls are bridged into a separate Tokio runtime
|
|
74
|
+
instead of running on your event loop.
|
|
75
|
+
- **Releases are tied to the binding layer.** A fix in TiKV's Rust client reaches
|
|
76
|
+
Python only after the bindings are rebuilt, re-pinned and republished.
|
|
77
|
+
|
|
78
|
+
TiKV speaks plain gRPC, and Python already has a production-grade gRPC stack. So
|
|
79
|
+
this library talks to PD and TiKV directly, the way the Go and Rust clients do:
|
|
80
|
+
|
|
81
|
+
- **Installs anywhere `grpcio` does**, which is practically everywhere. It is a
|
|
82
|
+
`py3-none-any` wheel with two dependencies.
|
|
83
|
+
- **Your event loop, your threads.** `AsyncRawClient` is native asyncio.
|
|
84
|
+
`RawClient` is a plain thread-safe blocking client.
|
|
85
|
+
- **Small enough to own.** About 1,500 lines of typed, commented Python. When
|
|
86
|
+
something goes wrong, you can read exactly what the client did, and every retry
|
|
87
|
+
is logged.
|
|
88
|
+
- **Behaves like the official clients.** Region routing, retries and backoff follow
|
|
89
|
+
client-go and client-rust, verified against a real cluster.
|
|
90
|
+
|
|
91
|
+
## Features
|
|
92
|
+
|
|
93
|
+
- The full raw API: `get`, `put`, `delete`, batches, range scans (forward and
|
|
94
|
+
reverse), `delete_range`, TTLs, `compare_and_swap` and `checksum`
|
|
95
|
+
- Blocking `RawClient` and asyncio `AsyncRawClient` with identical APIs
|
|
96
|
+
- Automatic routing across regions, following splits, merges, leader changes and
|
|
97
|
+
PD failover
|
|
98
|
+
- A time budget per operation, with exponential backoff and jitter on every
|
|
99
|
+
transient error
|
|
100
|
+
- Lazy, constant-memory scans over any number of regions
|
|
101
|
+
- Mutual TLS
|
|
102
|
+
- Fully typed (`mypy --strict`), `py.typed`, no `Any` in the public API
|
|
103
|
+
|
|
104
|
+
## Installation
|
|
105
|
+
|
|
106
|
+
```console
|
|
107
|
+
pip install tikv-python # or: uv add tikv-python
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
The package imports as `tikv`. It needs Python 3.10+, `grpcio>=1.60` and
|
|
111
|
+
`protobuf>=4.25`.
|
|
112
|
+
|
|
113
|
+
## Quickstart
|
|
114
|
+
|
|
115
|
+
Point the client at one or more PD servers. It discovers the rest of the cluster on
|
|
116
|
+
its own.
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
import tikv
|
|
120
|
+
|
|
121
|
+
with tikv.RawClient(["pd-0:2379", "pd-1:2379"]) as db:
|
|
122
|
+
db.put_many({b"user:1": b"Ada", b"user:2": b"Grace", b"user:3": b"Linus"})
|
|
123
|
+
|
|
124
|
+
db.get(b"user:2") # b"Grace"
|
|
125
|
+
db.get(b"user:9") # None
|
|
126
|
+
db.get_many([b"user:1", b"user:9"]) # {b"user:1": b"Ada"}
|
|
127
|
+
|
|
128
|
+
list(db.scan_keys(prefix=b"user:", reverse=True, limit=2))
|
|
129
|
+
# [b"user:3", b"user:2"]
|
|
130
|
+
|
|
131
|
+
db.delete(b"user:1")
|
|
132
|
+
db.delete_range(prefix=b"user:")
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
The asyncio client has the same API. Use `await` and `async for`:
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
import asyncio
|
|
139
|
+
import tikv
|
|
140
|
+
|
|
141
|
+
async def main() -> None:
|
|
142
|
+
async with tikv.AsyncRawClient("pd-0:2379") as db:
|
|
143
|
+
await asyncio.gather(*(db.put(f"k{i}".encode(), b"v") for i in range(1000)))
|
|
144
|
+
async for key, value in db.scan(b"k", b"l", limit=10):
|
|
145
|
+
print(key, value)
|
|
146
|
+
|
|
147
|
+
asyncio.run(main())
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
## Guide
|
|
151
|
+
|
|
152
|
+
### API at a glance
|
|
153
|
+
|
|
154
|
+
| Method | What it does |
|
|
155
|
+
| --- | --- |
|
|
156
|
+
| `get(key)` | The value, or `None` if the key does not exist |
|
|
157
|
+
| `get_many(keys)` | A `dict` of the keys that exist, in the order requested |
|
|
158
|
+
| `put(key, value, *, ttl=None)` | Set a key, optionally with an expiry |
|
|
159
|
+
| `put_many(items, *, ttl=None)` | Set many keys from a mapping or `(key, value)` pairs |
|
|
160
|
+
| `delete(key)` / `delete_many(keys)` | Delete keys; missing keys are not an error |
|
|
161
|
+
| `delete_range(start, end)` / `delete_range(prefix=…)` | Delete a whole range of keys |
|
|
162
|
+
| `scan(start, end, *, prefix, limit, reverse)` | Lazily iterate over `(key, value)` pairs |
|
|
163
|
+
| `scan_keys(…)` | Like `scan`, without transferring values |
|
|
164
|
+
| `ttl(key)` | Seconds until the key expires: `None` if never, `KeyError` if missing |
|
|
165
|
+
| `checksum(start, end, *, prefix)` | A `Checksum(crc64_xor, keys, size)` computed by the servers |
|
|
166
|
+
| `compare_and_swap(key, expected, new, *, ttl=None)` | Atomic CAS, returning `CasResult(swapped, previous)` |
|
|
167
|
+
|
|
168
|
+
Keys and values are `bytes` (`bytearray` and `memoryview` are fine too). Passing a
|
|
169
|
+
`str` raises `TypeError` instead of silently picking an encoding.
|
|
170
|
+
|
|
171
|
+
### Ranges and scans
|
|
172
|
+
|
|
173
|
+
Ranges are half-open, `[start, end)`, and `None` means "unbounded":
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
db.scan() # every key
|
|
177
|
+
db.scan(b"m") # from b"m" to the end
|
|
178
|
+
db.scan(b"a", b"n") # b"a" up to, but excluding, b"n"
|
|
179
|
+
db.scan(prefix=b"log:") # every key starting with b"log:"
|
|
180
|
+
db.scan(prefix=b"log:", reverse=True, limit=100) # the newest 100 log keys
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Scans fetch one page at a time from the right region, so iterating over millions
|
|
184
|
+
of keys uses constant memory. Stop iterating whenever you like.
|
|
185
|
+
|
|
186
|
+
`delete_range` refuses to run without `start` or a non-empty `prefix`, so a missing
|
|
187
|
+
argument cannot wipe your cluster. Deleting everything takes an explicit
|
|
188
|
+
`delete_range(b"")`.
|
|
189
|
+
|
|
190
|
+
### Atomicity and compare-and-swap
|
|
191
|
+
|
|
192
|
+
Single-key operations are atomic. Batch operations are atomic within a region but
|
|
193
|
+
not across regions; that is how TiKV's raw API works.
|
|
194
|
+
|
|
195
|
+
`compare_and_swap` needs a client in *atomic mode*. In that mode every write goes
|
|
196
|
+
through TiKV's single-row transaction path, which makes CAS linearizable:
|
|
197
|
+
|
|
198
|
+
```python
|
|
199
|
+
locks = tikv.RawClient("pd-0:2379", atomic=True)
|
|
200
|
+
|
|
201
|
+
swapped, holder = locks.compare_and_swap(b"lock:job-7", None, b"worker-3")
|
|
202
|
+
if not swapped:
|
|
203
|
+
print(f"already held by {holder!r}")
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
> **Warning:** Every client that writes the same keys must use the same mode.
|
|
207
|
+
> Mixing modes breaks linearizability. `delete_range` is unavailable in atomic
|
|
208
|
+
> mode because TiKV cannot perform it atomically.
|
|
209
|
+
|
|
210
|
+
### TTL
|
|
211
|
+
|
|
212
|
+
`ttl=` takes seconds, rounded up to a whole second (TiKV's precision), or a
|
|
213
|
+
`datetime.timedelta`. Expiry needs `storage.enable-ttl = true` on the TiKV side.
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
db.put(b"otp:5551234", b"839201", ttl=timedelta(minutes=5))
|
|
217
|
+
db.ttl(b"otp:5551234") # 300
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
### Errors, timeouts and retries
|
|
221
|
+
|
|
222
|
+
Every operation gets a time budget: `timeout=10.0` seconds by default, set per
|
|
223
|
+
client. Within that budget the client retries every transient failure with
|
|
224
|
+
exponential backoff and jitter:
|
|
225
|
+
|
|
226
|
+
- stale routing after a region split or merge,
|
|
227
|
+
- leader changes and elections,
|
|
228
|
+
- busy or unreachable stores,
|
|
229
|
+
- PD leader failover.
|
|
230
|
+
|
|
231
|
+
Each scan page gets its own budget. When the budget runs out you get
|
|
232
|
+
`tikv.TimeoutError`, which is also a builtin `TimeoutError`, with the last
|
|
233
|
+
underlying error chained as `__cause__`.
|
|
234
|
+
|
|
235
|
+
| Exception | Raised when |
|
|
236
|
+
| --- | --- |
|
|
237
|
+
| `TiKVError` | Base class of everything below |
|
|
238
|
+
| `TimeoutError` | The operation could not complete within `timeout` |
|
|
239
|
+
| `ServerError` | TiKV rejected the request, e.g. a key over `max-key-size` |
|
|
240
|
+
| `RegionError` | A region error that retrying cannot fix, e.g. a write too large for Raft |
|
|
241
|
+
| `PDError` | PD rejected a request, or an endpoint is not PD at all |
|
|
242
|
+
| `UndeterminedError` | A `compare_and_swap` failed in flight and may or may not have applied |
|
|
243
|
+
|
|
244
|
+
`compare_and_swap` is deliberately never replayed after a failure that might have
|
|
245
|
+
applied. If a replayed swap had already succeeded, it would report a spurious
|
|
246
|
+
failure. Read the key to find out what happened.
|
|
247
|
+
|
|
248
|
+
Every retry is logged at `DEBUG` level on the `tikv` logger, with the reason and the
|
|
249
|
+
delay:
|
|
250
|
+
|
|
251
|
+
```python
|
|
252
|
+
logging.getLogger("tikv").setLevel(logging.DEBUG)
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
### Concurrency
|
|
256
|
+
|
|
257
|
+
Create one client and share it across your application. It keeps one HTTP/2
|
|
258
|
+
connection per server, and requests are multiplexed over it.
|
|
259
|
+
|
|
260
|
+
- `RawClient` is thread-safe. Multi-region batches fan out over a small thread
|
|
261
|
+
pool.
|
|
262
|
+
- `AsyncRawClient` is task-safe and fans out with `asyncio`. Like every gRPC
|
|
263
|
+
asyncio channel, it belongs to the event loop it is first used in.
|
|
264
|
+
|
|
265
|
+
### TLS
|
|
266
|
+
|
|
267
|
+
```python
|
|
268
|
+
tls = tikv.TLSConfig(ca="ca.pem", cert="client.pem", key="client-key.pem")
|
|
269
|
+
db = tikv.RawClient("pd-0:2379", tls=tls)
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
## Performance
|
|
273
|
+
|
|
274
|
+
The numbers below come from a 3-node TiKV 8.5 cluster on Kubernetes, with the
|
|
275
|
+
client running in a pod in the same cluster on Python 3.12. Your hardware will
|
|
276
|
+
differ, but the shape holds: per-request overhead stays well under a millisecond,
|
|
277
|
+
and batches and scans move hundreds of thousands of keys per second.
|
|
278
|
+
|
|
279
|
+
| Operation | Result |
|
|
280
|
+
| --- | --- |
|
|
281
|
+
| `get`, sequential | 0.53 ms median, 0.95 ms p99 |
|
|
282
|
+
| `put`, sequential | 1.75 ms median, 3.82 ms p99 |
|
|
283
|
+
| `put_many`, 50,000 × 100-byte values | ~263,000 keys/s |
|
|
284
|
+
| `get_many`, 50,000 keys | ~344,000 keys/s |
|
|
285
|
+
| `scan`, 50,000 keys | ~376,000 keys/s |
|
|
286
|
+
| `AsyncRawClient.get`, 10,000 concurrent | ~11,500 ops/s from one event loop |
|
|
287
|
+
|
|
288
|
+
## How it works
|
|
289
|
+
|
|
290
|
+
```
|
|
291
|
+
your code ──► RawClient / AsyncRawClient
|
|
292
|
+
│
|
|
293
|
+
├─ region cache ── sorted by start key; maps every key to its region and leader
|
|
294
|
+
│ ▲
|
|
295
|
+
│ └─ PD client ── finds the PD leader, follows failover,
|
|
296
|
+
│ fetches regions (64 at a time when walking a range)
|
|
297
|
+
│
|
|
298
|
+
└─ request executor ── groups keys by region, sends to leaders concurrently,
|
|
299
|
+
handles NotLeader / EpochNotMatch / ServerIsBusy / …,
|
|
300
|
+
re-groups and retries within the time budget
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
A few design decisions worth knowing:
|
|
304
|
+
|
|
305
|
+
- **One implementation, two flavors.** The asyncio code in `src/tikv/_async` is
|
|
306
|
+
the source of truth. [`scripts/unasync.py`](scripts/unasync.py) mechanically
|
|
307
|
+
generates the blocking client from it, the technique
|
|
308
|
+
[httpcore](https://github.com/encode/httpcore) uses, and CI fails if they drift.
|
|
309
|
+
- **A trimmed protocol.** [`proto/`](proto) is the subset of
|
|
310
|
+
[kvproto](https://github.com/pingcap/kvproto) a raw client needs, with upstream
|
|
311
|
+
names and field numbers. Fields the client never reads are omitted, and protobuf
|
|
312
|
+
preserves them as unknown fields on the wire. The generated modules import in
|
|
313
|
+
about a millisecond and work with protobuf 4.25 through 7.x.
|
|
314
|
+
- **Region errors drive the cache.** Like client-go, the client never polls for
|
|
315
|
+
topology changes. A routing error invalidates or patches exactly the cache
|
|
316
|
+
entries involved, and the affected keys are re-grouped and retried.
|
|
317
|
+
- **Tested against a fake that behaves like the real thing.**
|
|
318
|
+
[`tests/fake_cluster.py`](tests/fake_cluster.py) is an in-process PD + TiKV that
|
|
319
|
+
speaks the real gRPC protocol. It splits and merges regions, moves leaders and
|
|
320
|
+
injects failures, and it validates routing the way TiKV does. CI also runs the
|
|
321
|
+
suite against real TiKV.
|
|
322
|
+
|
|
323
|
+
## Compatibility
|
|
324
|
+
|
|
325
|
+
| | Supported |
|
|
326
|
+
| --- | --- |
|
|
327
|
+
| Python | 3.10 – 3.14 |
|
|
328
|
+
| TiKV / PD | 7.5 and 8.5 LTS, tested in CI |
|
|
329
|
+
| Storage API | `api-version = 1`, with or without `enable-ttl` |
|
|
330
|
+
| Platforms | Anywhere `grpcio` installs: Linux (x86-64, ARM), macOS, Windows |
|
|
331
|
+
|
|
332
|
+
## Roadmap
|
|
333
|
+
|
|
334
|
+
These are tracked in [issues](https://github.com/slavabobik/tikv-python/issues):
|
|
335
|
+
|
|
336
|
+
- API v2 keyspaces (`storage.api-version = 2`)
|
|
337
|
+
- Column families other than `default`
|
|
338
|
+
- Chaos tests against a multi-node cluster: leader failover, store loss, PD failover
|
|
339
|
+
- A transactional (TxnKV) client
|
|
340
|
+
|
|
341
|
+
Contributions are very welcome. Open an issue to discuss bigger changes first.
|
|
342
|
+
|
|
343
|
+
## Development
|
|
344
|
+
|
|
345
|
+
```console
|
|
346
|
+
git clone https://github.com/slavabobik/tikv-python && cd tikv-python
|
|
347
|
+
uv sync # dev environment in .venv
|
|
348
|
+
uv run pytest # unit tests against the fake cluster
|
|
349
|
+
uv run ruff check && uv run ruff format --check && uv run mypy
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
Integration tests run against any real cluster. They write only under a unique key
|
|
353
|
+
prefix and delete it afterwards:
|
|
354
|
+
|
|
355
|
+
```console
|
|
356
|
+
ci/start-cluster.sh v8.5.8 # single-node cluster in Docker
|
|
357
|
+
TIKV_PD_ENDPOINTS=127.0.0.1:2379 uv run pytest -m integration
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
After changing `src/tikv/_async`, regenerate the blocking client:
|
|
361
|
+
|
|
362
|
+
```console
|
|
363
|
+
uv run python scripts/unasync.py
|
|
364
|
+
```
|
|
365
|
+
|
|
366
|
+
After changing `proto/`, regenerate the messages. The pinned generator keeps the
|
|
367
|
+
protobuf floor low:
|
|
368
|
+
|
|
369
|
+
```console
|
|
370
|
+
uv run --with 'grpcio-tools==1.62.3' python scripts/generate_protos.py
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
### Releasing
|
|
374
|
+
|
|
375
|
+
1. Bump `__version__` in `src/tikv/__init__.py` and merge it to `main`.
|
|
376
|
+
2. Tag and push the release: `git tag v0.2.0 && git push origin v0.2.0`.
|
|
377
|
+
|
|
378
|
+
The [release workflow](.github/workflows/release.yml) checks that the tag matches
|
|
379
|
+
the version, publishes to PyPI with
|
|
380
|
+
[trusted publishing](https://docs.pypi.org/trusted-publishers/), and creates a
|
|
381
|
+
GitHub release with the distributions attached. You can also run it by hand from
|
|
382
|
+
the Actions tab to publish to TestPyPI or PyPI, optionally under a different
|
|
383
|
+
distribution name; the import name is always `tikv`.
|
|
384
|
+
|
|
385
|
+
## License
|
|
386
|
+
|
|
387
|
+
[Apache License 2.0](LICENSE), the same as TiKV.
|