k3redisutil 0.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- k3redisutil/__init__.py +34 -0
- k3redisutil/redis_proxy_cli.py +389 -0
- k3redisutil/redisutil.py +276 -0
- k3redisutil/synopsis.py +44 -0
- k3redisutil-0.1.5.dist-info/METADATA +128 -0
- k3redisutil-0.1.5.dist-info/RECORD +9 -0
- k3redisutil-0.1.5.dist-info/WHEEL +5 -0
- k3redisutil-0.1.5.dist-info/licenses/LICENSE +21 -0
- k3redisutil-0.1.5.dist-info/top_level.txt +1 -0
k3redisutil/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""
|
|
2
|
+
For using redis more easily.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import version
|
|
6
|
+
|
|
7
|
+
__version__ = version("k3redisutil")
|
|
8
|
+
|
|
9
|
+
from .redisutil import (
|
|
10
|
+
get_client,
|
|
11
|
+
wait_serve,
|
|
12
|
+
normalize_ip_port,
|
|
13
|
+
RedisChannel,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
from .redis_proxy_cli import (
|
|
17
|
+
KeyNotFoundError,
|
|
18
|
+
RedisProxyError,
|
|
19
|
+
SendRequestError,
|
|
20
|
+
ServerResponseError,
|
|
21
|
+
RedisProxyClient,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"get_client",
|
|
26
|
+
"wait_serve",
|
|
27
|
+
"normalize_ip_port",
|
|
28
|
+
"RedisChannel",
|
|
29
|
+
"KeyNotFoundError",
|
|
30
|
+
"RedisProxyError",
|
|
31
|
+
"SendRequestError",
|
|
32
|
+
"ServerResponseError",
|
|
33
|
+
"RedisProxyClient",
|
|
34
|
+
]
|
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# coding: utf-8
|
|
3
|
+
|
|
4
|
+
import logging
|
|
5
|
+
import socket
|
|
6
|
+
|
|
7
|
+
import k3awssign
|
|
8
|
+
from k3confloader import conf
|
|
9
|
+
import k3http
|
|
10
|
+
import k3utfjson
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
DEFAULT_TIMEOUT = 2
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RedisProxyError(Exception):
|
|
17
|
+
"""
|
|
18
|
+
The base class of other exceptions of `RedisProxyClient`.
|
|
19
|
+
It is a subclass of `Exception`.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class SendRequestError(RedisProxyError):
|
|
26
|
+
"""
|
|
27
|
+
It is a subclass of `redisutil.RedisProxyError`.
|
|
28
|
+
Raise if failed to send request to redis proxy server.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class KeyNotFoundError(RedisProxyError):
|
|
35
|
+
"""
|
|
36
|
+
It is a subclass of `redisutil.RedisProxyError`.
|
|
37
|
+
Raise if key not found (redis proxy server return a `404`).
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class ServerResponseError(RedisProxyError):
|
|
44
|
+
"""
|
|
45
|
+
It is a subclass of `redisutil.RedisProxyError`.
|
|
46
|
+
Raise if http-status not in `(200, 404)` return from redis proxy server.
|
|
47
|
+
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
pass
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _proxy(func):
|
|
54
|
+
def _wrapper(self, verb, retry, *args):
|
|
55
|
+
all_hosts = [self.hosts]
|
|
56
|
+
if self.proxy_hosts is not None:
|
|
57
|
+
all_hosts += self.proxy_hosts
|
|
58
|
+
|
|
59
|
+
rst = None
|
|
60
|
+
err_list = []
|
|
61
|
+
if verb == "GET":
|
|
62
|
+
for hosts in all_hosts:
|
|
63
|
+
try:
|
|
64
|
+
return func(self, hosts, verb, retry, *args)
|
|
65
|
+
except RedisProxyError as e:
|
|
66
|
+
err_list.append(e)
|
|
67
|
+
else:
|
|
68
|
+
raise err_list[-1]
|
|
69
|
+
|
|
70
|
+
else:
|
|
71
|
+
for hosts in all_hosts:
|
|
72
|
+
rst = func(self, hosts, verb, retry, *args)
|
|
73
|
+
|
|
74
|
+
return rst
|
|
75
|
+
|
|
76
|
+
return _wrapper
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _retry(func):
|
|
80
|
+
def _wrapper(self, hosts, verb, retry, *args):
|
|
81
|
+
retry = retry or 0
|
|
82
|
+
err_list = []
|
|
83
|
+
for _ in range(retry + 1):
|
|
84
|
+
for ip, port in hosts:
|
|
85
|
+
self.ip = ip
|
|
86
|
+
self.port = port
|
|
87
|
+
|
|
88
|
+
try:
|
|
89
|
+
return func(self, verb, *args)
|
|
90
|
+
|
|
91
|
+
except (k3http.HttpError, socket.error) as e:
|
|
92
|
+
logger.exception(
|
|
93
|
+
"{e} while send request to redis proxy with {ip}:{p}".format(e=repr(e), ip=ip, p=port)
|
|
94
|
+
)
|
|
95
|
+
err_list.append(e)
|
|
96
|
+
|
|
97
|
+
else:
|
|
98
|
+
raise SendRequestError(repr(err_list[-1]))
|
|
99
|
+
|
|
100
|
+
return _wrapper
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class SetAPI(object):
|
|
104
|
+
def __init__(self, cli, redis_op, mtd_info):
|
|
105
|
+
self.cli = cli
|
|
106
|
+
self.redis_op = redis_op.upper()
|
|
107
|
+
self.http_mtd = mtd_info[0]
|
|
108
|
+
self.args_count = mtd_info[1]
|
|
109
|
+
self.opts = list(mtd_info[2]) + ["retry"]
|
|
110
|
+
|
|
111
|
+
def api(self, *args, **argkv):
|
|
112
|
+
mtd_args = []
|
|
113
|
+
|
|
114
|
+
for idx in range(self.args_count):
|
|
115
|
+
if idx < len(args):
|
|
116
|
+
mtd_args.append(args[idx])
|
|
117
|
+
|
|
118
|
+
else:
|
|
119
|
+
left_cnt = self.args_count - len(mtd_args)
|
|
120
|
+
opt_name = self.opts[-left_cnt]
|
|
121
|
+
mtd_args.append(argkv.get(opt_name, None))
|
|
122
|
+
|
|
123
|
+
retry = mtd_args.pop()
|
|
124
|
+
|
|
125
|
+
qs = {}
|
|
126
|
+
# retry in opts, but it is not in qs
|
|
127
|
+
qs_keys = list(self.opts[:-1])
|
|
128
|
+
while len(qs_keys) > 0:
|
|
129
|
+
qs[qs_keys.pop()] = mtd_args.pop()
|
|
130
|
+
|
|
131
|
+
body = None
|
|
132
|
+
if self.http_mtd == "PUT":
|
|
133
|
+
body = k3utfjson.dump(mtd_args.pop())
|
|
134
|
+
|
|
135
|
+
path = [self.redis_op] + mtd_args
|
|
136
|
+
|
|
137
|
+
return self.cli._api(self.http_mtd, retry, path, body, qs)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class RedisProxyClient(object):
|
|
141
|
+
"""
|
|
142
|
+
redis operation, http method, count of args, optional args name
|
|
143
|
+
|
|
144
|
+
### RedisProxyClient.delete
|
|
145
|
+
Delete the specified key.
|
|
146
|
+
Do nothing if the key doesn’t exist.
|
|
147
|
+
|
|
148
|
+
**arguments**:
|
|
149
|
+
`key`:specifies the key to redis.
|
|
150
|
+
`retry`:try to send request for another N times while failed to send request.
|
|
151
|
+
By default, it is `0`.
|
|
152
|
+
|
|
153
|
+
**return**: None
|
|
154
|
+
### RedisProxyClient.get
|
|
155
|
+
Get the value with `key`. Raise a `redisutil.KeyNotFoundError` if the key doesn’t exist.
|
|
156
|
+
|
|
157
|
+
`retry`:try to send request for another N times while failed to send request.
|
|
158
|
+
By default, it is `0`.
|
|
159
|
+
|
|
160
|
+
**return**: the value at `key`.
|
|
161
|
+
|
|
162
|
+
### RedisProxyClient.hdel
|
|
163
|
+
|
|
164
|
+
Delete the specified `hashkey` in the specified `hashname`.
|
|
165
|
+
Raise a `redisutil.KeyNotFoundError` if it doesn’t exist.
|
|
166
|
+
|
|
167
|
+
**arguments**:
|
|
168
|
+
`hashname`:specifies the hash name to redis.
|
|
169
|
+
|
|
170
|
+
`hashkey`:specifies the hash key to redis.
|
|
171
|
+
|
|
172
|
+
`retry`:try to send request for another N times while failed to send request.
|
|
173
|
+
By default, it is `0`.
|
|
174
|
+
|
|
175
|
+
**return**: None
|
|
176
|
+
|
|
177
|
+
### RedisProxyClient.set
|
|
178
|
+
|
|
179
|
+
Set the value at `key` to `val`.
|
|
180
|
+
**arguments**:
|
|
181
|
+
`key`: specifies the key to redis.
|
|
182
|
+
`val`: specifies the value to redis.
|
|
183
|
+
`expire`: the expire time on `key` in msec. Defaults to `None`.
|
|
184
|
+
`retry`: try to send request for another N times while failed to send request.
|
|
185
|
+
By default, it is `0`.
|
|
186
|
+
|
|
187
|
+
**return**: nothing
|
|
188
|
+
|
|
189
|
+
### RedisProxyClient.hget
|
|
190
|
+
|
|
191
|
+
Return the value of `haskkey` within the `haskname`.
|
|
192
|
+
Raise a `redisutil.KeyNotFoundError` if it doesn’t exist.
|
|
193
|
+
|
|
194
|
+
**arguments**:
|
|
195
|
+
`hashname`: specifies the hash name to redis.
|
|
196
|
+
`hashkey`: specifies the hash key to redis.
|
|
197
|
+
`retry`: try to send request for another N times while failed to send request.
|
|
198
|
+
By default, it is `0`.
|
|
199
|
+
|
|
200
|
+
**return**: the value of `hashkey` within the `hashname`.
|
|
201
|
+
|
|
202
|
+
### RedisProxyClient.hset
|
|
203
|
+
|
|
204
|
+
Set `hashkey` to `val` within `hashname`.
|
|
205
|
+
|
|
206
|
+
**arguments**:
|
|
207
|
+
|
|
208
|
+
`hashname`: specifies the hash name to redis.
|
|
209
|
+
|
|
210
|
+
`hashkey`: specifies the hash key to redis.
|
|
211
|
+
|
|
212
|
+
`val`: specifies the value to redis.
|
|
213
|
+
|
|
214
|
+
`expire`: the expire time on `hashkey` within `hashname` in msec. Defaults to `None`.
|
|
215
|
+
|
|
216
|
+
`retry`: try to send request for another N times while failed to send request.
|
|
217
|
+
By default, it is `0`.
|
|
218
|
+
|
|
219
|
+
**return**: nothing
|
|
220
|
+
|
|
221
|
+
### RedisProxyClient.hkeys
|
|
222
|
+
|
|
223
|
+
Return the list of keys within `hashname`.
|
|
224
|
+
Raise a `redisutil.KeyNotFoundError` if the `hashname` doesn’t exist.
|
|
225
|
+
|
|
226
|
+
**arguments**:
|
|
227
|
+
|
|
228
|
+
`hashname`: specifies the hash name to redis.
|
|
229
|
+
|
|
230
|
+
`retry`: try to send request for another N times while failed to send request.
|
|
231
|
+
By default, it is `0`.
|
|
232
|
+
|
|
233
|
+
**return**: a `list` contains all the keys.
|
|
234
|
+
|
|
235
|
+
### RedisProxyClient.hvals
|
|
236
|
+
|
|
237
|
+
Return the list of values within `hashname`.
|
|
238
|
+
Raise a `redisutil.KeyNotFoundError` if the `hashname` doesn’t exist.
|
|
239
|
+
|
|
240
|
+
**arguments**:
|
|
241
|
+
|
|
242
|
+
`hashname`: specifies the hash name to redis.
|
|
243
|
+
|
|
244
|
+
`retry`: try to send request for another N times while failed to send request.
|
|
245
|
+
By default, it is `0`.
|
|
246
|
+
|
|
247
|
+
**return**: a `list` contains all the values.
|
|
248
|
+
|
|
249
|
+
### RedisProxyClient.hgetall
|
|
250
|
+
|
|
251
|
+
Return a dict of the hash’s name/value pairs.
|
|
252
|
+
Raise a `redisutil.KeyNotFoundError` if the `hashname` doesn’t exist.
|
|
253
|
+
|
|
254
|
+
**arguments**:
|
|
255
|
+
|
|
256
|
+
`hashname`:specifies the hash name to redis.
|
|
257
|
+
|
|
258
|
+
`retry`:try to send request for another N times while failed to send request.
|
|
259
|
+
By default, it is `0`.
|
|
260
|
+
|
|
261
|
+
**return**: a `dict` of the hash’s name/value pairs.
|
|
262
|
+
"""
|
|
263
|
+
|
|
264
|
+
methods = {
|
|
265
|
+
# get(key, retry=0)
|
|
266
|
+
"get": ("get", "GET", 2, ()),
|
|
267
|
+
# set(key, val, expire=None, retry=0)
|
|
268
|
+
"set": ("set", "PUT", 4, ("expire",)),
|
|
269
|
+
# hget(hashname, hashkey, retry=0)
|
|
270
|
+
"hget": ("hget", "GET", 3, ()),
|
|
271
|
+
# hset(hashname, hashkey, val, expire=None, retry=0)
|
|
272
|
+
"hset": ("hset", "PUT", 5, ("expire",)),
|
|
273
|
+
# hkeys(hashname, retry=0)
|
|
274
|
+
"hkeys": ("hkeys", "GET", 2, ()),
|
|
275
|
+
# hvals(hashname, retry=0)
|
|
276
|
+
"hvals": ("hvals", "GET", 2, ()),
|
|
277
|
+
# hgetall(hashname, retry=0)
|
|
278
|
+
"hgetall": ("hgetall", "GET", 2, ()),
|
|
279
|
+
# delete(key, retry=0)
|
|
280
|
+
"delete": ("del", "DELETE", 2, ()),
|
|
281
|
+
# hdel(hashname, key, retry=0)
|
|
282
|
+
"hdel": ("hdel", "DELETE", 3, ()),
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
def __init__(self, hosts, proxy_hosts=None, nwr=None, ak_sk=None, timeout=None):
|
|
286
|
+
"""
|
|
287
|
+
A client for redis proxy server.
|
|
288
|
+
:param hosts: a `tuple` each element is a `(ip, port)`, retry with next addr while failed to use prev one.
|
|
289
|
+
:param proxy_hosts: a `tuple` each element is `hosts`, as the backup hosts.
|
|
290
|
+
:param nwr: `tuple` of `(n, w, r)`, count of replicas, count of write, count of read.
|
|
291
|
+
If it is `None`, `config.rp_cli_nwr` is used.
|
|
292
|
+
:param ak_sk: tuple` of `(access_key, secret_key)` to sign the request.
|
|
293
|
+
If it is `None`, `config.rp_cli_ak_sk` is used.
|
|
294
|
+
:param timeout: timeout of connecting redis proxy server in seconds. Defaults to `None`.
|
|
295
|
+
"""
|
|
296
|
+
self.hosts = hosts
|
|
297
|
+
self.proxy_hosts = proxy_hosts
|
|
298
|
+
|
|
299
|
+
if nwr is None:
|
|
300
|
+
nwr = conf.rp_cli_nwr
|
|
301
|
+
|
|
302
|
+
if ak_sk is None:
|
|
303
|
+
ak_sk = conf.rp_cli_ak_sk
|
|
304
|
+
|
|
305
|
+
self.n, self.w, self.r = nwr
|
|
306
|
+
self.access_key, self.secret_key = ak_sk
|
|
307
|
+
|
|
308
|
+
self.timeout = timeout or DEFAULT_TIMEOUT
|
|
309
|
+
self.ver = "/redisproxy/v1"
|
|
310
|
+
|
|
311
|
+
for mtd_name, mtd_info in self.methods.items():
|
|
312
|
+
api_obj = SetAPI(self, mtd_info[0], mtd_info[1:])
|
|
313
|
+
setattr(self, mtd_name, api_obj.api)
|
|
314
|
+
|
|
315
|
+
def _sign_req(self, req):
|
|
316
|
+
sign_payload = True if "body" in req else False
|
|
317
|
+
signer = k3awssign.Signer(self.access_key, self.secret_key)
|
|
318
|
+
sign_ctx = signer.add_auth(req, query_auth=True, sign_payload=sign_payload)
|
|
319
|
+
logger.debug("signing details: {ctx}".format(ctx=sign_ctx))
|
|
320
|
+
|
|
321
|
+
def _make_req_uri(self, params, qs):
|
|
322
|
+
path = [self.ver]
|
|
323
|
+
path.extend(params)
|
|
324
|
+
|
|
325
|
+
qs_list = [
|
|
326
|
+
"n={n}".format(n=self.n),
|
|
327
|
+
"w={w}".format(w=self.w),
|
|
328
|
+
"r={r}".format(r=self.r),
|
|
329
|
+
]
|
|
330
|
+
for k, v in qs.items():
|
|
331
|
+
if v is None:
|
|
332
|
+
continue
|
|
333
|
+
|
|
334
|
+
qs_list.append("{k}={v}".format(k=k, v=v))
|
|
335
|
+
|
|
336
|
+
return "{p}?{qs}".format(p="/".join(path), qs="&".join(qs_list))
|
|
337
|
+
|
|
338
|
+
def _req(self, req):
|
|
339
|
+
if "headers" not in req:
|
|
340
|
+
req["headers"] = {
|
|
341
|
+
"host": "{ip}:{port}".format(ip=self.ip, port=self.port),
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
elif "host" not in req["headers"]:
|
|
345
|
+
req["headers"]["host"] = "{ip}:{port}".format(ip=self.ip, port=self.port)
|
|
346
|
+
|
|
347
|
+
body = req.get("body", None)
|
|
348
|
+
if body is not None:
|
|
349
|
+
req["headers"]["Content-Length"] = len(body)
|
|
350
|
+
|
|
351
|
+
self._sign_req(req)
|
|
352
|
+
|
|
353
|
+
cli = k3http.Client(self.ip, self.port, self.timeout)
|
|
354
|
+
cli.send_request(req["uri"], method=req["verb"], headers=req["headers"])
|
|
355
|
+
if body is not None:
|
|
356
|
+
cli.send_body(req["body"])
|
|
357
|
+
|
|
358
|
+
cli.read_response()
|
|
359
|
+
res = cli.read_body(None)
|
|
360
|
+
|
|
361
|
+
msg = "Status:{s} req:{req} res:{res} server:{ip}:{p}".format(
|
|
362
|
+
s=cli.status, req=repr(req), res=repr(res), ip=self.ip, p=self.port
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
if cli.status == 404:
|
|
366
|
+
raise KeyNotFoundError(msg)
|
|
367
|
+
|
|
368
|
+
elif cli.status != 200:
|
|
369
|
+
raise ServerResponseError(msg)
|
|
370
|
+
|
|
371
|
+
return res
|
|
372
|
+
|
|
373
|
+
@_proxy
|
|
374
|
+
@_retry
|
|
375
|
+
def _api(self, verb, path, body, qs):
|
|
376
|
+
req = {
|
|
377
|
+
"verb": verb,
|
|
378
|
+
"uri": self._make_req_uri(path, qs),
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
if body is not None:
|
|
382
|
+
req["body"] = body
|
|
383
|
+
|
|
384
|
+
if verb == "GET":
|
|
385
|
+
rst = self._req(req)
|
|
386
|
+
return k3utfjson.load(rst)
|
|
387
|
+
|
|
388
|
+
else:
|
|
389
|
+
self._req(req)
|
k3redisutil/redisutil.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
import threading
|
|
4
|
+
import time
|
|
5
|
+
from collections import defaultdict
|
|
6
|
+
|
|
7
|
+
import redis
|
|
8
|
+
|
|
9
|
+
import k3utfjson
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
# redis is thread safe
|
|
14
|
+
|
|
15
|
+
# NOTE: fork may duplicate file descriptor that confuses connection pool.
|
|
16
|
+
_pid_client = defaultdict(dict)
|
|
17
|
+
_lock = threading.RLock()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def get_client(ip_port):
|
|
21
|
+
"""
|
|
22
|
+
Return a process-wise singleton redis client, which is an instance of `redis.StrictRedis`.
|
|
23
|
+
|
|
24
|
+
Redis client returned is shared across the entire process and will not be
|
|
25
|
+
re-created.
|
|
26
|
+
It is also safe to use if a process fork and inherited opened socket file-descriptors.
|
|
27
|
+
:param ip_port: could be a port number in `int` or `long` to get or create a
|
|
28
|
+
client connecting to localhost.
|
|
29
|
+
It can also be tuple of `(ip, port)`.
|
|
30
|
+
:return: an instance of `redis.StrictRedis`.
|
|
31
|
+
"""
|
|
32
|
+
ip_port = normalize_ip_port(ip_port)
|
|
33
|
+
|
|
34
|
+
pid = os.getpid()
|
|
35
|
+
|
|
36
|
+
with _lock:
|
|
37
|
+
o = _pid_client[ip_port]
|
|
38
|
+
|
|
39
|
+
if pid not in o:
|
|
40
|
+
o[pid] = redis.StrictRedis(*ip_port)
|
|
41
|
+
|
|
42
|
+
return _pid_client[ip_port][pid]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def wait_serve(ip_port, timeout=5):
|
|
46
|
+
"""
|
|
47
|
+
Wait for at most `timeout` seconds until redis start serving request.
|
|
48
|
+
It is useful when start a redis server.
|
|
49
|
+
:param ip_port: tuple of `(ip, port)` or a single int/long number as port.
|
|
50
|
+
:param timeout: specifies max waiting time, in seconds.
|
|
51
|
+
:return: Nothing
|
|
52
|
+
"""
|
|
53
|
+
t = time.time() + timeout
|
|
54
|
+
|
|
55
|
+
rcl = get_client(ip_port)
|
|
56
|
+
|
|
57
|
+
while time.time() < t:
|
|
58
|
+
try:
|
|
59
|
+
rcl.hget("foo", "foo")
|
|
60
|
+
logger.info("redis is ready: " + repr(ip_port))
|
|
61
|
+
return
|
|
62
|
+
|
|
63
|
+
except redis.ConnectionError as e:
|
|
64
|
+
logger.info("can not connect to redis: " + repr(ip_port) + " " + repr(e))
|
|
65
|
+
time.sleep(0.1)
|
|
66
|
+
continue
|
|
67
|
+
else:
|
|
68
|
+
logger.error("can not connect to redis: " + repr(ip_port))
|
|
69
|
+
raise
|
|
70
|
+
# if redis does not respond in `timeout` seconds.
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class RedisChannel(object):
|
|
74
|
+
"""
|
|
75
|
+
send message `data` through `channel`.
|
|
76
|
+
`channel` is a list in redis.
|
|
77
|
+
`peer` is "client" or "server".
|
|
78
|
+
|
|
79
|
+
send is a rpush operation that adds an item to the end of a list.
|
|
80
|
+
recv is a lpop operation that pops an item from the start of a list.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
other_peer = {
|
|
84
|
+
"client": "server",
|
|
85
|
+
"server": "client",
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
def __init__(self, ip_port, channel, peer, timeout=None):
|
|
89
|
+
"""
|
|
90
|
+
`redisutil.RedisChannel(ip_port, channel, peer, timeout=None)`
|
|
91
|
+
|
|
92
|
+
Create a redis list based channel for cross process communication.
|
|
93
|
+
|
|
94
|
+
See [redis-list-command](https://redis.io/commands#list).
|
|
95
|
+
|
|
96
|
+
Initializing this class does **NOT** create a socket connecting to redis or
|
|
97
|
+
create any data in redis.
|
|
98
|
+
|
|
99
|
+
A channel support duplex communication.
|
|
100
|
+
Thus in redis it creates two lists for each channel for two way communication:
|
|
101
|
+
`<channel>/client` and `<channel>/server`.
|
|
102
|
+
|
|
103
|
+
Client side uses `<channel>/client` to send message.
|
|
104
|
+
Server side uses `<channel>/server` to send message.
|
|
105
|
+
|
|
106
|
+
A client should initialize `RedisChannel` with argument `peer` set to `client`.
|
|
107
|
+
A server should initialize `RedisChannel` with argument `peer` set to `server`.
|
|
108
|
+
:param ip_port: tuple of `(ip, port)` or a single int/long number as port.
|
|
109
|
+
:param channel: specifies the name of the channel to create.
|
|
110
|
+
A channel should be in URI form, starting with `/`, such as: `/asyncworker/image-process`
|
|
111
|
+
`channel` can also be a tuple of string:
|
|
112
|
+
`('asyncworker', 'image-process')` will be converted to `/asyncworker/image-process`.
|
|
113
|
+
:param peer: specifies this instance is a client or a server.
|
|
114
|
+
It can be `"client"` or `"server"`.
|
|
115
|
+
:param timeout: the expire time of the channel, in second.
|
|
116
|
+
If it is `None`, the channel exists until being cleaned.
|
|
117
|
+
"""
|
|
118
|
+
assert peer in self.other_peer
|
|
119
|
+
|
|
120
|
+
self.ip_port = normalize_ip_port(ip_port)
|
|
121
|
+
self.rcl = get_client(self.ip_port)
|
|
122
|
+
|
|
123
|
+
# convert ['a', 'b'] to '/a/b'
|
|
124
|
+
if isinstance(channel, (list, tuple)):
|
|
125
|
+
channel = "/" + "/".join(channel)
|
|
126
|
+
|
|
127
|
+
self.channel = channel
|
|
128
|
+
self.peer = peer.lower()
|
|
129
|
+
self.send_list_name = "/".join([self.channel, self.peer])
|
|
130
|
+
self.recv_list_name = "/".join([self.channel, self.other_peer[self.peer]])
|
|
131
|
+
self.timeout = timeout
|
|
132
|
+
|
|
133
|
+
def send_msg(self, data):
|
|
134
|
+
"""
|
|
135
|
+
Send data. `data` is json-encoded in redis list.
|
|
136
|
+
:param data: any data type that can be json encoded.
|
|
137
|
+
:return: Nothing
|
|
138
|
+
"""
|
|
139
|
+
j = k3utfjson.dump(data)
|
|
140
|
+
self.rcl.rpush(self.send_list_name, j)
|
|
141
|
+
|
|
142
|
+
if self.timeout is not None:
|
|
143
|
+
self.rcl.expire(self.send_list_name, self.timeout)
|
|
144
|
+
|
|
145
|
+
def recv_msg(self):
|
|
146
|
+
"""
|
|
147
|
+
Receive one message.
|
|
148
|
+
If there is no message in this channel, it returns `None`.
|
|
149
|
+
:return: data that is loaded from json. Or `None` if there is no message in channel.
|
|
150
|
+
"""
|
|
151
|
+
v = self.rcl.lpop(self.recv_list_name)
|
|
152
|
+
if v is None:
|
|
153
|
+
return None
|
|
154
|
+
|
|
155
|
+
return k3utfjson.load(v)
|
|
156
|
+
|
|
157
|
+
def brecv_msg(self, timeout=0):
|
|
158
|
+
"""
|
|
159
|
+
Block to receive one message.
|
|
160
|
+
If there is no message in this channel,
|
|
161
|
+
then block for `timeout` seconds or until
|
|
162
|
+
a value was pushed to the channel.
|
|
163
|
+
|
|
164
|
+
:param timeout: seconds of the block time.
|
|
165
|
+
If it is `0`, then block until a message is available.
|
|
166
|
+
|
|
167
|
+
:return: data that is loaded from json. Or `None` if timeout.
|
|
168
|
+
"""
|
|
169
|
+
v = self.rcl.blpop(self.recv_list_name, timeout=timeout)
|
|
170
|
+
if v is None or len(v) != 2:
|
|
171
|
+
return None
|
|
172
|
+
|
|
173
|
+
# v is a tuple, (key, value)
|
|
174
|
+
_, value = v
|
|
175
|
+
|
|
176
|
+
return k3utfjson.load(value)
|
|
177
|
+
|
|
178
|
+
def recv_last_msg(self):
|
|
179
|
+
"""
|
|
180
|
+
Similar to `RedisChannel.recv_msg` except it returns only the last message it
|
|
181
|
+
sees and removes all previous messages from this channel.
|
|
182
|
+
:return: data that is loaded from json. Or `None` if there is no message in channel.
|
|
183
|
+
"""
|
|
184
|
+
last = None
|
|
185
|
+
while True:
|
|
186
|
+
v = self.rcl.lpop(self.recv_list_name)
|
|
187
|
+
if v is None:
|
|
188
|
+
return k3utfjson.load(last)
|
|
189
|
+
|
|
190
|
+
last = v
|
|
191
|
+
|
|
192
|
+
def brecv_last_msg(self, timeout=0):
|
|
193
|
+
"""
|
|
194
|
+
Similar to `RedisChannel.recv_last_msg` except it blocks for `timeout`
|
|
195
|
+
seconds if the channel is empty.
|
|
196
|
+
:param timeout: seconds of the block time.
|
|
197
|
+
If it is `0`, then block until a message is available.
|
|
198
|
+
:return: data that is loaded from json. Or `None` if timeout.
|
|
199
|
+
"""
|
|
200
|
+
msg = self.brecv_msg(timeout=timeout)
|
|
201
|
+
if msg is None:
|
|
202
|
+
return None
|
|
203
|
+
|
|
204
|
+
last = self.recv_last_msg() or msg
|
|
205
|
+
|
|
206
|
+
return last
|
|
207
|
+
|
|
208
|
+
def peek_msg(self):
|
|
209
|
+
"""
|
|
210
|
+
Similar to `RedisChannel.recv_msg` except it does not remove the message it
|
|
211
|
+
returned.
|
|
212
|
+
:return: data that is loaded from json. Or `None` if there is no message in channel.
|
|
213
|
+
"""
|
|
214
|
+
v = self.rcl.lindex(self.recv_list_name, 0)
|
|
215
|
+
if v is None:
|
|
216
|
+
return None
|
|
217
|
+
|
|
218
|
+
return k3utfjson.load(v)
|
|
219
|
+
|
|
220
|
+
def rpeek_msg(self):
|
|
221
|
+
"""
|
|
222
|
+
Similar to `RedisChannel.peek_msg`
|
|
223
|
+
except it gets message from the tail of the channel.
|
|
224
|
+
:return: JSON decoded message. Or `None` if there is no message in channel.
|
|
225
|
+
"""
|
|
226
|
+
v = self.rcl.lindex(self.recv_list_name, -1)
|
|
227
|
+
if v is None:
|
|
228
|
+
return None
|
|
229
|
+
|
|
230
|
+
return k3utfjson.load(v)
|
|
231
|
+
|
|
232
|
+
def list_channel(self, prefix):
|
|
233
|
+
"""
|
|
234
|
+
List all channel names those start with `prefix`.
|
|
235
|
+
:param prefix: specifies what channel to list. `prefix` must starts with '/', ends with '/'.
|
|
236
|
+
:return: a list of channel names.
|
|
237
|
+
"""
|
|
238
|
+
if isinstance(prefix, (list, tuple)):
|
|
239
|
+
_prefix = "/" + "/".join(prefix) + "/"
|
|
240
|
+
else:
|
|
241
|
+
_prefix = prefix
|
|
242
|
+
|
|
243
|
+
if not _prefix.startswith("/"):
|
|
244
|
+
raise ValueError('prefix must starts with "/", but:' + repr(prefix))
|
|
245
|
+
|
|
246
|
+
if _prefix.endswith("*"):
|
|
247
|
+
raise ValueError('prefix must NOT ends with "*", but:' + repr(prefix))
|
|
248
|
+
|
|
249
|
+
if not _prefix.endswith("/"):
|
|
250
|
+
raise ValueError('prefix must ends with "/", but:' + repr(prefix))
|
|
251
|
+
|
|
252
|
+
_prefix = _prefix + "*"
|
|
253
|
+
channels = self.rcl.keys(_prefix)
|
|
254
|
+
|
|
255
|
+
rst = []
|
|
256
|
+
for c in channels:
|
|
257
|
+
for k in self.other_peer:
|
|
258
|
+
k = "/" + k
|
|
259
|
+
if c.endswith(k.encode()):
|
|
260
|
+
c = c[: -len(k)]
|
|
261
|
+
break
|
|
262
|
+
else:
|
|
263
|
+
logger.info("not a channel: " + repr(c))
|
|
264
|
+
continue
|
|
265
|
+
|
|
266
|
+
if c not in rst:
|
|
267
|
+
rst.append(c)
|
|
268
|
+
|
|
269
|
+
return sorted(rst)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def normalize_ip_port(ip_port):
|
|
273
|
+
if isinstance(ip_port, int):
|
|
274
|
+
ip_port = ("127.0.0.1", ip_port)
|
|
275
|
+
|
|
276
|
+
return ip_port
|
k3redisutil/synopsis.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import k3redisutil
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
# Using redis as a duplex cross process communication channel pool.
|
|
5
|
+
|
|
6
|
+
# client and server with the same channel name "/foo" is a pair
|
|
7
|
+
c = k3redisutil.RedisChannel(6379, "/foo", "client")
|
|
8
|
+
s = k3redisutil.RedisChannel(6379, "/foo", "server")
|
|
9
|
+
|
|
10
|
+
c.send_msg("c2s")
|
|
11
|
+
s.send_msg("s2c")
|
|
12
|
+
|
|
13
|
+
# list channels
|
|
14
|
+
print(c.list_channel("/")) # ["/foo"]
|
|
15
|
+
print(s.recv_msg()) # c2s
|
|
16
|
+
print(c.recv_msg()) # s2c
|
|
17
|
+
|
|
18
|
+
cli = k3redisutil.RedisProxyClient([("127.0.0.1", 2222), ("192.168.0.100", 222)])
|
|
19
|
+
|
|
20
|
+
cli.set("k1", "v1", retry=1)
|
|
21
|
+
cli.set("k2", "v2", expire=1000) # msec
|
|
22
|
+
|
|
23
|
+
print(cli.get("k1", retry=2))
|
|
24
|
+
# out: 'v1'
|
|
25
|
+
|
|
26
|
+
print(cli.get("k2"))
|
|
27
|
+
# out: 'v2'
|
|
28
|
+
|
|
29
|
+
time.sleep(1)
|
|
30
|
+
cli.get("k2")
|
|
31
|
+
# raise a 'redisutil.KeyNotFoundError' because it is timeout
|
|
32
|
+
|
|
33
|
+
cli.hset("hashname1", "k3", "v3")
|
|
34
|
+
cli.hset("hashname2", "k4", "v4", expire=1000)
|
|
35
|
+
|
|
36
|
+
print(cli.hget("hashname1", "k3"))
|
|
37
|
+
# out: 'v3'
|
|
38
|
+
|
|
39
|
+
print(cli.hget("hashname2", "k4"))
|
|
40
|
+
# out: 'v4'
|
|
41
|
+
|
|
42
|
+
time.sleep(1)
|
|
43
|
+
cli.hget("hashname2", "k4")
|
|
44
|
+
# raise a 'redisutil.KeyNotFoundError' because it is timeout
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: k3redisutil
|
|
3
|
+
Version: 0.1.5
|
|
4
|
+
Summary: Redis utilities for easier client management and proxy support
|
|
5
|
+
Author-email: Zhang Yanpo <drdr.xp@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/pykit3/k3redisutil
|
|
8
|
+
Project-URL: Documentation, https://k3redisutil.readthedocs.io
|
|
9
|
+
Keywords: redis,client,proxy,cache
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: redis
|
|
23
|
+
Requires-Dist: k3awssign
|
|
24
|
+
Requires-Dist: k3confloader
|
|
25
|
+
Requires-Dist: k3http
|
|
26
|
+
Requires-Dist: k3utfjson
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
29
|
+
Requires-Dist: ruff; extra == "dev"
|
|
30
|
+
Requires-Dist: coverage; extra == "dev"
|
|
31
|
+
Requires-Dist: k3ut; extra == "dev"
|
|
32
|
+
Requires-Dist: k3thread; extra == "dev"
|
|
33
|
+
Requires-Dist: k3utdocker; extra == "dev"
|
|
34
|
+
Requires-Dist: mock; extra == "dev"
|
|
35
|
+
Provides-Extra: publish
|
|
36
|
+
Requires-Dist: build; extra == "publish"
|
|
37
|
+
Requires-Dist: twine; extra == "publish"
|
|
38
|
+
Requires-Dist: pk3; extra == "publish"
|
|
39
|
+
Provides-Extra: docs
|
|
40
|
+
Requires-Dist: mkdocs>=1.5; extra == "docs"
|
|
41
|
+
Requires-Dist: mkdocs-material>=9.0; extra == "docs"
|
|
42
|
+
Requires-Dist: mkdocstrings[python]>=0.24; extra == "docs"
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
|
|
45
|
+
# k3redisutil
|
|
46
|
+
|
|
47
|
+
[](https://github.com/pykit3/k3redisutil/actions/workflows/python-package.yml)
|
|
48
|
+
[](https://k3redisutil.readthedocs.io/en/stable/?badge=stable)
|
|
49
|
+
[](https://pypi.org/project/k3redisutil)
|
|
50
|
+
|
|
51
|
+
For using redis more easily.
|
|
52
|
+
|
|
53
|
+
k3redisutil is a component of [pykit3] project: a python3 toolkit set.
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
For using redis more easily.
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# Install
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
pip install k3redisutil
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
# Synopsis
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
|
|
70
|
+
import k3redisutil
|
|
71
|
+
import time
|
|
72
|
+
|
|
73
|
+
# Using redis as a duplex cross process communication channel pool.
|
|
74
|
+
|
|
75
|
+
# client and server with the same channel name "/foo" is a pair
|
|
76
|
+
c = k3redisutil.RedisChannel(6379, '/foo', 'client')
|
|
77
|
+
s = k3redisutil.RedisChannel(6379, '/foo', 'server')
|
|
78
|
+
|
|
79
|
+
c.send_msg('c2s')
|
|
80
|
+
s.send_msg('s2c')
|
|
81
|
+
|
|
82
|
+
# list channels
|
|
83
|
+
print(c.list_channel('/')) # ["/foo"]
|
|
84
|
+
print(s.recv_msg()) # c2s
|
|
85
|
+
print(c.recv_msg()) # s2c
|
|
86
|
+
|
|
87
|
+
cli = k3redisutil.RedisProxyClient([('127.0.0.1', 2222), ('192.168.0.100', 222)])
|
|
88
|
+
|
|
89
|
+
cli.set('k1', 'v1', retry=1)
|
|
90
|
+
cli.set('k2', 'v2', expire=1000) # msec
|
|
91
|
+
|
|
92
|
+
print(cli.get('k1', retry=2))
|
|
93
|
+
# out: 'v1'
|
|
94
|
+
|
|
95
|
+
print(cli.get('k2'))
|
|
96
|
+
# out: 'v2'
|
|
97
|
+
|
|
98
|
+
time.sleep(1)
|
|
99
|
+
cli.get('k2')
|
|
100
|
+
# raise a 'redisutil.KeyNotFoundError' because it is timeout
|
|
101
|
+
|
|
102
|
+
cli.hset('hashname1', 'k3', 'v3')
|
|
103
|
+
cli.hset('hashname2', 'k4', 'v4', expire=1000)
|
|
104
|
+
|
|
105
|
+
print(cli.hget('hashname1', 'k3'))
|
|
106
|
+
# out: 'v3'
|
|
107
|
+
|
|
108
|
+
print(cli.hget('hashname2', 'k4'))
|
|
109
|
+
# out: 'v4'
|
|
110
|
+
|
|
111
|
+
time.sleep(1)
|
|
112
|
+
cli.hget('hashname2', 'k4')
|
|
113
|
+
# raise a 'redisutil.KeyNotFoundError' because it is timeout
|
|
114
|
+
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
# Author
|
|
118
|
+
|
|
119
|
+
Zhang Yanpo (张炎泼) <drdr.xp@gmail.com>
|
|
120
|
+
|
|
121
|
+
# Copyright and License
|
|
122
|
+
|
|
123
|
+
The MIT License (MIT)
|
|
124
|
+
|
|
125
|
+
Copyright (c) 2015 Zhang Yanpo (张炎泼) <drdr.xp@gmail.com>
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
[pykit3]: https://github.com/pykit3
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
k3redisutil/__init__.py,sha256=4uepa29qEbbw-w7GtnqFcyzuz8p2dyefGnaun12merU,580
|
|
2
|
+
k3redisutil/redis_proxy_cli.py,sha256=J2q5PKFEPHHwEym4CBh53GCXAdjqVPulfmPN7ppLVdg,11200
|
|
3
|
+
k3redisutil/redisutil.py,sha256=ApGQn4zJO04D0kAmOi7ms9Uyg05NE_DvFuLUtyjznBE,8910
|
|
4
|
+
k3redisutil/synopsis.py,sha256=DZrj01f2Pig9PXDpfnCwsMZgZWM7WNyjJP0Jx3to58s,1065
|
|
5
|
+
k3redisutil-0.1.5.dist-info/licenses/LICENSE,sha256=efDc2MufKQpsx8k2CaRX7tYYsmT6CyOa76by7YATdm0,1066
|
|
6
|
+
k3redisutil-0.1.5.dist-info/METADATA,sha256=78LLg2THhjDbx3FVtV5WIlx0F-_GBoUQwrCSOE1Z4sU,3520
|
|
7
|
+
k3redisutil-0.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
8
|
+
k3redisutil-0.1.5.dist-info/top_level.txt,sha256=xIefklCRQVerLPacYF19qnllG7J0K3Q-jfiSwhfFwyQ,12
|
|
9
|
+
k3redisutil-0.1.5.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020 张炎泼
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
k3redisutil
|