kedis-python 0.1.4__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kedis_python-0.1.4 → kedis_python-0.3.0}/PKG-INFO +26 -14
- {kedis_python-0.1.4 → kedis_python-0.3.0}/README.md +19 -13
- kedis_python-0.3.0/kedis_python/aof.py +138 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/commands.py +133 -60
- kedis_python-0.3.0/kedis_python/help_data.py +711 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/main.py +154 -88
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/network.py +15 -6
- kedis_python-0.3.0/kedis_python/parser.py +621 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/server.py +479 -106
- kedis_python-0.3.0/kedis_python/settings.py +238 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/store.py +118 -69
- kedis_python-0.3.0/kedis_python/textcodec.py +32 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/ui.py +71 -1
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/PKG-INFO +26 -14
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/SOURCES.txt +43 -2
- {kedis_python-0.1.4 → kedis_python-0.3.0}/pyproject.toml +11 -6
- kedis_python-0.3.0/tests/test_aof_persistence.py +280 -0
- kedis_python-0.3.0/tests/test_aof_records.py +140 -0
- kedis_python-0.3.0/tests/test_binary_safety.py +340 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_bitmap_persistence.py +20 -8
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_bitmaps.py +17 -11
- kedis_python-0.3.0/tests/test_chunked_reply.py +409 -0
- kedis_python-0.3.0/tests/test_cli_housekeep.py +121 -0
- kedis_python-0.3.0/tests/test_cli_stream_view.py +118 -0
- kedis_python-0.3.0/tests/test_data_replies.py +123 -0
- kedis_python-0.3.0/tests/test_del_exists.py +107 -0
- kedis_python-0.3.0/tests/test_doc_claims.py +72 -0
- kedis_python-0.3.0/tests/test_docs_data_types.py +152 -0
- kedis_python-0.3.0/tests/test_error_line_cap.py +132 -0
- kedis_python-0.3.0/tests/test_follower_stream.py +495 -0
- kedis_python-0.3.0/tests/test_fuzz.py +286 -0
- kedis_python-0.3.0/tests/test_fuzz_properties.py +63 -0
- kedis_python-0.3.0/tests/test_help_data.py +211 -0
- kedis_python-0.3.0/tests/test_history_setup.py +74 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_hyperloglog.py +9 -2
- kedis_python-0.3.0/tests/test_kesp_decoder.py +254 -0
- kedis_python-0.3.0/tests/test_kesp_encoder.py +208 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_lists.py +1 -1
- kedis_python-0.3.0/tests/test_lrange.py +88 -0
- kedis_python-0.3.0/tests/test_multi_limits.py +301 -0
- kedis_python-0.3.0/tests/test_multi_readonly.py +157 -0
- kedis_python-0.3.0/tests/test_multi_replication.py +475 -0
- kedis_python-0.3.0/tests/test_parser_framing.py +175 -0
- kedis_python-0.3.0/tests/test_parser_limits.py +138 -0
- kedis_python-0.3.0/tests/test_pubsub.py +220 -0
- kedis_python-0.3.0/tests/test_replica_snapshot.py +141 -0
- kedis_python-0.3.0/tests/test_replica_write_deadline.py +279 -0
- kedis_python-0.3.0/tests/test_replication_task.py +217 -0
- kedis_python-0.3.0/tests/test_reply_budgets.py +226 -0
- kedis_python-0.3.0/tests/test_reply_limits.py +123 -0
- kedis_python-0.3.0/tests/test_reply_snapshots.py +198 -0
- kedis_python-0.3.0/tests/test_server_flags.py +427 -0
- kedis_python-0.3.0/tests/test_server_limits.py +345 -0
- kedis_python-0.3.0/tests/test_server_protocol_errors.py +112 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_groups_store.py +13 -5
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_store.py +9 -1
- kedis_python-0.3.0/tests/test_type.py +77 -0
- kedis_python-0.3.0/tests/test_version.py +34 -0
- kedis_python-0.3.0/tests/test_write_deadline.py +242 -0
- kedis_python-0.3.0/tests/test_zadd.py +98 -0
- kedis_python-0.1.4/kedis_python/parser.py +0 -307
- kedis_python-0.1.4/kedis_python/replica.py +0 -454
- {kedis_python-0.1.4 → kedis_python-0.3.0}/LICENSE +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/__init__.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/benchmark.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/benchmark_asyncio.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/hyperloglog.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/kesp_cli.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/simulator.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/skiplist.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/stream.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/dependency_links.txt +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/entry_points.txt +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/requires.txt +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/top_level.txt +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/setup.cfg +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_bench.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_docs_streams.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_housekeeping.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_inline_block.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_kesp.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_kesp2.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_lru.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_parser_casing.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_parser_harden.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_replica.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_commands.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_core.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_groups_commands.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_groups_core.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_groups_server.py +0 -0
- {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_server.py +0 -0
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kedis-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: A Redis-inspired in-memory store in Python.
|
|
5
5
|
Author: V SS Karthik
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Homepage, https://github.com/Hogwarts-coder10/kedis-python
|
|
8
|
+
Keywords: redis,in-memory,key-value,database,streams,pub-sub
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Topic :: Database
|
|
8
14
|
Requires-Python: >=3.9
|
|
9
15
|
Description-Content-Type: text/markdown
|
|
10
16
|
License-File: LICENSE
|
|
@@ -43,16 +49,18 @@ Rather than focusing on Redis compatibility, Kedis focuses on understanding the
|
|
|
43
49
|
## 📚 Engine Documentation
|
|
44
50
|
|
|
45
51
|
Dive into the internal grimoire to understand how Kedis achieves microsecond latency:
|
|
46
|
-
* [Architecture & Pipeline](docs/architecture.md)
|
|
47
|
-
* [KESP Wire Protocol](docs/protocol.md)
|
|
48
|
-
* [Networking & Surge Tank](docs/networking.md)
|
|
49
|
-
* [
|
|
50
|
-
* [
|
|
51
|
-
* [
|
|
52
|
-
* [
|
|
53
|
-
* [
|
|
54
|
-
* [
|
|
55
|
-
* [
|
|
52
|
+
* [Architecture & Pipeline](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/architecture.md)
|
|
53
|
+
* [KESP Wire Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/protocol.md)
|
|
54
|
+
* [Networking & Surge Tank](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/networking.md)
|
|
55
|
+
* [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md)
|
|
56
|
+
* [Fuzzing the Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/fuzzing.md)
|
|
57
|
+
* [Persistence & I/O Drivetrain](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/persistence.md)
|
|
58
|
+
* [Command Spellbook](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/commands.md)
|
|
59
|
+
* [TTL & Lifespan Mechanics](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/ttl.md)
|
|
60
|
+
* [Streams & Consumer Groups](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md)
|
|
61
|
+
* [Pub/Sub Broadcasts](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/pubsub.md)
|
|
62
|
+
* [Benchmarks & Telemetry](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/benchmarks.md)
|
|
63
|
+
* [Future Roadmap](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/roadmap.md)
|
|
56
64
|
|
|
57
65
|
---
|
|
58
66
|
|
|
@@ -120,7 +128,7 @@ XPENDING orders shipping
|
|
|
120
128
|
XACK orders shipping 1718000000000-0
|
|
121
129
|
```
|
|
122
130
|
|
|
123
|
-
Append-only event logs with blocking reads, and consumer groups that share a stream between workers with a pending list and acknowledgements. Streams persist through the AOF, `COMPACT` and snapshots, and replicate with the exact IDs and group state the master chose. See [Streams & Consumer Groups](docs/streams.md).
|
|
131
|
+
Append-only event logs with blocking reads, and consumer groups that share a stream between workers with a pending list and acknowledgements. Streams persist through the AOF, `COMPACT` and snapshots, and replicate with the exact IDs and group state the master chose. See [Streams & Consumer Groups](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md).
|
|
124
132
|
|
|
125
133
|
---
|
|
126
134
|
|
|
@@ -193,7 +201,8 @@ Kedis supports master-replica replication:
|
|
|
193
201
|
|
|
194
202
|
* Replicas connect to a master and perform a `SYNC` handshake to receive the initial dataset
|
|
195
203
|
* The master streams subsequent writes to connected replicas in real time
|
|
196
|
-
* Replicas are read-only — writes issued directly to a replica are rejected
|
|
204
|
+
* Replicas are read-only — writes issued directly to a replica are rejected, and so are writes inside a `MULTI` block: the write gets `-EREADONLY` when it is queued and `EXEC` aborts with `-EXECABORT`
|
|
205
|
+
* A `MULTI`/`EXEC` block runs as one unit on the master and reaches followers as one transaction, which a follower applies as a unit
|
|
197
206
|
|
|
198
207
|
---
|
|
199
208
|
|
|
@@ -204,6 +213,7 @@ Asyncio-based server implementation supporting:
|
|
|
204
213
|
* many concurrent client connections on a single event loop
|
|
205
214
|
* command execution over sockets via the KESP protocol
|
|
206
215
|
* standalone local mode fallback
|
|
216
|
+
* every limit settable at startup, e.g. `kedis-server --port 7001 --max-clients 500` (see [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md))
|
|
207
217
|
|
|
208
218
|
---
|
|
209
219
|
|
|
@@ -306,6 +316,7 @@ The project includes automated tests covering:
|
|
|
306
316
|
* persistence recovery
|
|
307
317
|
* expiration behavior
|
|
308
318
|
* streams and consumer groups, from the data structure up to live TCP sessions
|
|
319
|
+
* protocol fuzzing: a mutation fuzzer, reference readers and property tests (see [Fuzzing the Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/fuzzing.md))
|
|
309
320
|
|
|
310
321
|
Additional coverage is actively being expanded as the project evolves.
|
|
311
322
|
|
|
@@ -377,7 +388,8 @@ Current limitations include:
|
|
|
377
388
|
* limited fault tolerance compared to production databases
|
|
378
389
|
* no memory-based eviction ceiling (LRU tracks usage, but isn't yet tied to a hard memory limit)
|
|
379
390
|
* AOF persistence is not buffered — writes can trigger a sync depending on config
|
|
380
|
-
*
|
|
391
|
+
* `SAVE` and `COMPACT` hold the engine lock while they dump the store, so other commands wait (about 1 s for 3 million items); a write forwarded to followers is encoded on the event loop in one go (about 0.55 s for a command with 1,000,000 arguments). Both are listed in [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md#not-covered-yet)
|
|
392
|
+
* consumer groups have no `XCLAIM`/`XAUTOCLAIM` yet; the full list of stream limits is in [docs/streams.md](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md#-known-limits)
|
|
381
393
|
|
|
382
394
|
These limitations are intentional learning opportunities and areas of active development.
|
|
383
395
|
|
|
@@ -29,16 +29,18 @@ Rather than focusing on Redis compatibility, Kedis focuses on understanding the
|
|
|
29
29
|
## 📚 Engine Documentation
|
|
30
30
|
|
|
31
31
|
Dive into the internal grimoire to understand how Kedis achieves microsecond latency:
|
|
32
|
-
* [Architecture & Pipeline](docs/architecture.md)
|
|
33
|
-
* [KESP Wire Protocol](docs/protocol.md)
|
|
34
|
-
* [Networking & Surge Tank](docs/networking.md)
|
|
35
|
-
* [
|
|
36
|
-
* [
|
|
37
|
-
* [
|
|
38
|
-
* [
|
|
39
|
-
* [
|
|
40
|
-
* [
|
|
41
|
-
* [
|
|
32
|
+
* [Architecture & Pipeline](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/architecture.md)
|
|
33
|
+
* [KESP Wire Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/protocol.md)
|
|
34
|
+
* [Networking & Surge Tank](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/networking.md)
|
|
35
|
+
* [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md)
|
|
36
|
+
* [Fuzzing the Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/fuzzing.md)
|
|
37
|
+
* [Persistence & I/O Drivetrain](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/persistence.md)
|
|
38
|
+
* [Command Spellbook](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/commands.md)
|
|
39
|
+
* [TTL & Lifespan Mechanics](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/ttl.md)
|
|
40
|
+
* [Streams & Consumer Groups](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md)
|
|
41
|
+
* [Pub/Sub Broadcasts](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/pubsub.md)
|
|
42
|
+
* [Benchmarks & Telemetry](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/benchmarks.md)
|
|
43
|
+
* [Future Roadmap](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/roadmap.md)
|
|
42
44
|
|
|
43
45
|
---
|
|
44
46
|
|
|
@@ -106,7 +108,7 @@ XPENDING orders shipping
|
|
|
106
108
|
XACK orders shipping 1718000000000-0
|
|
107
109
|
```
|
|
108
110
|
|
|
109
|
-
Append-only event logs with blocking reads, and consumer groups that share a stream between workers with a pending list and acknowledgements. Streams persist through the AOF, `COMPACT` and snapshots, and replicate with the exact IDs and group state the master chose. See [Streams & Consumer Groups](docs/streams.md).
|
|
111
|
+
Append-only event logs with blocking reads, and consumer groups that share a stream between workers with a pending list and acknowledgements. Streams persist through the AOF, `COMPACT` and snapshots, and replicate with the exact IDs and group state the master chose. See [Streams & Consumer Groups](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md).
|
|
110
112
|
|
|
111
113
|
---
|
|
112
114
|
|
|
@@ -179,7 +181,8 @@ Kedis supports master-replica replication:
|
|
|
179
181
|
|
|
180
182
|
* Replicas connect to a master and perform a `SYNC` handshake to receive the initial dataset
|
|
181
183
|
* The master streams subsequent writes to connected replicas in real time
|
|
182
|
-
* Replicas are read-only — writes issued directly to a replica are rejected
|
|
184
|
+
* Replicas are read-only — writes issued directly to a replica are rejected, and so are writes inside a `MULTI` block: the write gets `-EREADONLY` when it is queued and `EXEC` aborts with `-EXECABORT`
|
|
185
|
+
* A `MULTI`/`EXEC` block runs as one unit on the master and reaches followers as one transaction, which a follower applies as a unit
|
|
183
186
|
|
|
184
187
|
---
|
|
185
188
|
|
|
@@ -190,6 +193,7 @@ Asyncio-based server implementation supporting:
|
|
|
190
193
|
* many concurrent client connections on a single event loop
|
|
191
194
|
* command execution over sockets via the KESP protocol
|
|
192
195
|
* standalone local mode fallback
|
|
196
|
+
* every limit settable at startup, e.g. `kedis-server --port 7001 --max-clients 500` (see [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md))
|
|
193
197
|
|
|
194
198
|
---
|
|
195
199
|
|
|
@@ -292,6 +296,7 @@ The project includes automated tests covering:
|
|
|
292
296
|
* persistence recovery
|
|
293
297
|
* expiration behavior
|
|
294
298
|
* streams and consumer groups, from the data structure up to live TCP sessions
|
|
299
|
+
* protocol fuzzing: a mutation fuzzer, reference readers and property tests (see [Fuzzing the Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/fuzzing.md))
|
|
295
300
|
|
|
296
301
|
Additional coverage is actively being expanded as the project evolves.
|
|
297
302
|
|
|
@@ -363,7 +368,8 @@ Current limitations include:
|
|
|
363
368
|
* limited fault tolerance compared to production databases
|
|
364
369
|
* no memory-based eviction ceiling (LRU tracks usage, but isn't yet tied to a hard memory limit)
|
|
365
370
|
* AOF persistence is not buffered — writes can trigger a sync depending on config
|
|
366
|
-
*
|
|
371
|
+
* `SAVE` and `COMPACT` hold the engine lock while they dump the store, so other commands wait (about 1 s for 3 million items); a write forwarded to followers is encoded on the event loop in one go (about 0.55 s for a command with 1,000,000 arguments). Both are listed in [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md#not-covered-yet)
|
|
372
|
+
* consumer groups have no `XCLAIM`/`XAUTOCLAIM` yet; the full list of stream limits is in [docs/streams.md](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md#-known-limits)
|
|
367
373
|
|
|
368
374
|
These limitations are intentional learning opportunities and areas of active development.
|
|
369
375
|
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""On-disk record format for the append-only file (AOF).
|
|
2
|
+
|
|
3
|
+
Every logged command is one KESP-framed record, the same shape the wire
|
|
4
|
+
protocol uses for commands:
|
|
5
|
+
|
|
6
|
+
A<count>\\n then, for each token: S<length>\\n<payload>\\n
|
|
7
|
+
|
|
8
|
+
Lengths count bytes of the UTF-8 payload, so a token can hold anything: spaces,
|
|
9
|
+
newlines, tabs, an empty string, text that looks like another command. The old
|
|
10
|
+
format joined tokens with spaces on one line, so a value could not survive a
|
|
11
|
+
restart and a value containing a newline could plant commands in the log.
|
|
12
|
+
|
|
13
|
+
Strings are encoded with "surrogatepass", so every Python str round-trips
|
|
14
|
+
exactly, lone surrogates included.
|
|
15
|
+
|
|
16
|
+
Files written by older versions hold one whitespace-separated command per line.
|
|
17
|
+
scan() still reads those, line by line, and a file may mix both: old lines
|
|
18
|
+
first, framed records after.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import re
|
|
22
|
+
|
|
23
|
+
_COUNT = re.compile(rb"(?:0|[1-9][0-9]*)")
|
|
24
|
+
MAX_HEADER_LINE = 32 # "A<n>" / "S<n>" lines are never longer than this
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def encode_record(tokens) -> bytes:
|
|
28
|
+
"""One command as a framed record."""
|
|
29
|
+
parts = [b"A%d\n" % len(tokens)]
|
|
30
|
+
for token in tokens:
|
|
31
|
+
raw = str(token).encode("utf-8", "surrogatepass")
|
|
32
|
+
parts.append(b"S%d\n" % len(raw))
|
|
33
|
+
parts.append(raw)
|
|
34
|
+
parts.append(b"\n")
|
|
35
|
+
return b"".join(parts)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class ScanResult:
|
|
39
|
+
"""How a scan ended.
|
|
40
|
+
|
|
41
|
+
status: "ok" every byte was consumed
|
|
42
|
+
"torn" the file ends in the middle of a record (a crash while
|
|
43
|
+
writing); everything before good_end is intact
|
|
44
|
+
"corrupt" a record is malformed; see reason
|
|
45
|
+
good_end: offset just past the last complete record or line
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
__slots__ = ("good_end", "reason", "status")
|
|
49
|
+
|
|
50
|
+
def __init__(self):
|
|
51
|
+
self.status = "ok"
|
|
52
|
+
self.good_end = 0
|
|
53
|
+
self.reason = ""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _count(buf, start: int, end: int):
|
|
57
|
+
field = bytes(buf[start:end])
|
|
58
|
+
return int(field) if _COUNT.fullmatch(field) else None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def scan(buf, result: ScanResult):
|
|
62
|
+
"""Yields (tokens, end_offset) for each command in buf, a bytes-like object
|
|
63
|
+
that supports find() and slicing (bytes or mmap). Stops at the first
|
|
64
|
+
incomplete or malformed record and says why in result."""
|
|
65
|
+
total = len(buf)
|
|
66
|
+
pos = 0
|
|
67
|
+
|
|
68
|
+
def stop(status: str, reason: str):
|
|
69
|
+
result.status = status
|
|
70
|
+
result.reason = reason
|
|
71
|
+
|
|
72
|
+
while pos < total:
|
|
73
|
+
if buf[pos : pos + 1] == b"A":
|
|
74
|
+
following = buf[pos + 1 : pos + 2]
|
|
75
|
+
if following == b"":
|
|
76
|
+
return stop("torn", "file ends inside a record header")
|
|
77
|
+
# A legacy command word starts "A" + a letter (AUTH, ...). Anything
|
|
78
|
+
# else after the "A" is a record header and must validate as one.
|
|
79
|
+
framed = not following.isalpha()
|
|
80
|
+
else:
|
|
81
|
+
framed = False
|
|
82
|
+
|
|
83
|
+
if not framed:
|
|
84
|
+
# Legacy line: whitespace-separated tokens, as older versions wrote.
|
|
85
|
+
nl = buf.find(b"\n", pos)
|
|
86
|
+
line_end = total if nl == -1 else nl
|
|
87
|
+
next_pos = total if nl == -1 else nl + 1
|
|
88
|
+
tokens = bytes(buf[pos:line_end]).decode("utf-8", "replace").split()
|
|
89
|
+
result.good_end = next_pos
|
|
90
|
+
if tokens:
|
|
91
|
+
yield tokens, next_pos
|
|
92
|
+
pos = next_pos
|
|
93
|
+
continue
|
|
94
|
+
|
|
95
|
+
nl = buf.find(b"\n", pos)
|
|
96
|
+
if nl == -1:
|
|
97
|
+
if total - pos > MAX_HEADER_LINE:
|
|
98
|
+
return stop("corrupt", "array header too long")
|
|
99
|
+
return stop("torn", "file ends inside a record header")
|
|
100
|
+
if nl - pos > MAX_HEADER_LINE:
|
|
101
|
+
return stop("corrupt", "array header too long")
|
|
102
|
+
count = _count(buf, pos + 1, nl)
|
|
103
|
+
if count is None:
|
|
104
|
+
return stop("corrupt", "invalid array header")
|
|
105
|
+
|
|
106
|
+
cursor = nl + 1
|
|
107
|
+
tokens = []
|
|
108
|
+
for _ in range(count):
|
|
109
|
+
if cursor >= total:
|
|
110
|
+
return stop("torn", "file ends between tokens")
|
|
111
|
+
if buf[cursor : cursor + 1] != b"S":
|
|
112
|
+
return stop("corrupt", "expected a string header")
|
|
113
|
+
nl = buf.find(b"\n", cursor)
|
|
114
|
+
if nl == -1:
|
|
115
|
+
if total - cursor > MAX_HEADER_LINE:
|
|
116
|
+
return stop("corrupt", "string header too long")
|
|
117
|
+
return stop("torn", "file ends inside a string header")
|
|
118
|
+
if nl - cursor > MAX_HEADER_LINE:
|
|
119
|
+
return stop("corrupt", "string header too long")
|
|
120
|
+
length = _count(buf, cursor + 1, nl)
|
|
121
|
+
if length is None:
|
|
122
|
+
return stop("corrupt", "invalid string length")
|
|
123
|
+
start = nl + 1
|
|
124
|
+
end = start + length
|
|
125
|
+
if end + 1 > total:
|
|
126
|
+
return stop("torn", "file ends inside a string")
|
|
127
|
+
if buf[end : end + 1] != b"\n":
|
|
128
|
+
return stop("corrupt", "string is not newline-terminated")
|
|
129
|
+
try:
|
|
130
|
+
tokens.append(bytes(buf[start:end]).decode("utf-8", "surrogatepass"))
|
|
131
|
+
except UnicodeDecodeError:
|
|
132
|
+
return stop("corrupt", "string is not valid UTF-8")
|
|
133
|
+
cursor = end + 1
|
|
134
|
+
|
|
135
|
+
result.good_end = cursor
|
|
136
|
+
if tokens:
|
|
137
|
+
yield tokens, cursor
|
|
138
|
+
pos = cursor
|
|
@@ -1,13 +1,17 @@
|
|
|
1
|
+
import math
|
|
1
2
|
import threading
|
|
2
3
|
import time
|
|
3
4
|
from typing import Any
|
|
4
5
|
|
|
5
|
-
from .parser import BulkString
|
|
6
|
+
from .parser import BulkString, KESPEncoder
|
|
6
7
|
from .store import KedisStore
|
|
7
8
|
from .stream import StreamError
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
class CommandHandler:
|
|
12
|
+
# Bytes a pub/sub subscriber may leave unread before the server cuts it off.
|
|
13
|
+
MAX_SUBSCRIBER_BUFFER = 32 * 1024 * 1024
|
|
14
|
+
|
|
11
15
|
def __init__(self, store: KedisStore):
|
|
12
16
|
self.store = store
|
|
13
17
|
|
|
@@ -16,6 +20,9 @@ class CommandHandler:
|
|
|
16
20
|
|
|
17
21
|
# Pub / Sub SwitchBoard
|
|
18
22
|
self._channels = {}
|
|
23
|
+
# The server's event loop, set per connection; pub/sub pushes are queued
|
|
24
|
+
# onto it because commands run in worker threads.
|
|
25
|
+
self.event_loop = None
|
|
19
26
|
|
|
20
27
|
# The O(1) Dispatch Table
|
|
21
28
|
self._commands = {
|
|
@@ -129,57 +136,82 @@ class CommandHandler:
|
|
|
129
136
|
if not tokens:
|
|
130
137
|
return "-ERR empty command or syntax error", []
|
|
131
138
|
|
|
132
|
-
cmd = tokens[0].upper()
|
|
133
|
-
args = tokens[
|
|
134
|
-
1:
|
|
135
|
-
] # (Though your current code passes tokens, let's keep your routing exact)
|
|
136
|
-
|
|
137
139
|
# 🚀 TRACK A: Start the high-precision telemetry stopwatch (in nanoseconds)
|
|
138
140
|
start_time = time.perf_counter_ns()
|
|
141
|
+
try:
|
|
142
|
+
with self._engine_lock:
|
|
143
|
+
return self._run_locked(tokens, client_socket)
|
|
144
|
+
finally:
|
|
145
|
+
self._record_duration(tokens, start_time)
|
|
146
|
+
|
|
147
|
+
def execute_batch(self, batch: list, client_socket=None):
|
|
148
|
+
"""Runs several commands as one unit: the engine lock is taken once, so
|
|
149
|
+
no other client's command can run between them, and the AOF records
|
|
150
|
+
them back to back. Returns (results, forwards): one result per command,
|
|
151
|
+
and the commands a master must send to replicas, in order (see
|
|
152
|
+
replication_payloads). Commands are not skipped or reordered; one that
|
|
153
|
+
fails gives an error result and the rest still run, like the one-at-a-
|
|
154
|
+
time path."""
|
|
155
|
+
results: list = []
|
|
156
|
+
forwards: list = []
|
|
157
|
+
write_commands = self.WRITE_COMMANDS
|
|
158
|
+
starts: list = []
|
|
159
|
+
with self._engine_lock:
|
|
160
|
+
for tokens in batch:
|
|
161
|
+
if not tokens:
|
|
162
|
+
results.append("-ERR empty command or syntax error")
|
|
163
|
+
starts.append(None)
|
|
164
|
+
continue
|
|
165
|
+
starts.append(time.perf_counter_ns())
|
|
166
|
+
result, records = self._run_locked(tokens, client_socket)
|
|
167
|
+
results.append(result)
|
|
168
|
+
if tokens[0].upper() in write_commands:
|
|
169
|
+
forwards.extend(self.replication_payloads(tokens, result, records))
|
|
170
|
+
for tokens, started in zip(batch, starts):
|
|
171
|
+
if started is not None:
|
|
172
|
+
self._record_duration(tokens, started)
|
|
173
|
+
return results, forwards
|
|
174
|
+
|
|
175
|
+
def _run_locked(self, tokens: list[str], client_socket=None):
|
|
176
|
+
"""Runs one command. The caller holds the engine lock."""
|
|
177
|
+
cmd = tokens[0].upper()
|
|
139
178
|
result = None
|
|
140
179
|
records = []
|
|
141
180
|
|
|
142
181
|
try:
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
result = handler_function(tokens, client_socket)
|
|
153
|
-
else:
|
|
154
|
-
handler_function = self._commands[cmd]
|
|
155
|
-
# Note: Depending on how your handlers are registered,
|
|
156
|
-
# they might expect 'tokens' or 'args'. Keeping your existing call:
|
|
157
|
-
result = handler_function(tokens)
|
|
182
|
+
try:
|
|
183
|
+
if cmd == "XGAPPLY" and client_socket is not None:
|
|
184
|
+
# Internal replication command: a network client
|
|
185
|
+
# must not be able to write raw group records.
|
|
186
|
+
result = f"-ERR unknown command '{cmd}'"
|
|
187
|
+
elif cmd in self._commands:
|
|
188
|
+
handler_function = self._commands[cmd]
|
|
189
|
+
if cmd in ["SUBSCRIBE", "PUBLISH"]:
|
|
190
|
+
result = handler_function(tokens, client_socket)
|
|
158
191
|
else:
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
result = f"-ERR wrong number of arguments for '{cmd}' command"
|
|
192
|
+
# Handlers take the whole token list, command name included.
|
|
193
|
+
result = handler_function(tokens)
|
|
194
|
+
else:
|
|
195
|
+
result = f"-ERR unknown command '{cmd}'"
|
|
196
|
+
finally:
|
|
197
|
+
records = self.store.drain_group_records()
|
|
166
198
|
return result, records
|
|
199
|
+
except TypeError:
|
|
200
|
+
return f"-ERR wrong number of arguments for '{cmd}' command", records
|
|
167
201
|
except Exception as e:
|
|
168
|
-
|
|
169
|
-
return result, records
|
|
202
|
+
return f"-ERR {str(e)}", records
|
|
170
203
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
)
|
|
182
|
-
self.store._log_slow_command(tokens, duration_us)
|
|
204
|
+
def _record_duration(self, tokens: list[str], start_time: int) -> None:
|
|
205
|
+
# 🚀 TRACK A: Stop the stopwatch, convert nanoseconds to microseconds
|
|
206
|
+
duration_us = (time.perf_counter_ns() - start_time) // 1000
|
|
207
|
+
|
|
208
|
+
# Feed the metrics to the store's slowlog ring buffer if available
|
|
209
|
+
if (
|
|
210
|
+
hasattr(self, "store")
|
|
211
|
+
and self.store
|
|
212
|
+
and hasattr(self.store, "_log_slow_command")
|
|
213
|
+
):
|
|
214
|
+
self.store._log_slow_command(tokens, duration_us)
|
|
183
215
|
|
|
184
216
|
# ---------------------------------------------------------
|
|
185
217
|
# COMMAND HANDLERS (The isolated engine components)
|
|
@@ -215,23 +247,30 @@ class CommandHandler:
|
|
|
215
247
|
self.store.set(tokens[1], tokens[2])
|
|
216
248
|
return "OK"
|
|
217
249
|
|
|
250
|
+
@staticmethod
|
|
251
|
+
def _data(value):
|
|
252
|
+
"""A stored value as a reply: always data, never read as a status or
|
|
253
|
+
an error, whatever it looks like ("OK", "ERROR x", "-ERR y")."""
|
|
254
|
+
return BulkString(value) if isinstance(value, str) else value
|
|
255
|
+
|
|
218
256
|
def _handle_get(self, tokens: list[str]):
|
|
219
257
|
if len(tokens) != 2:
|
|
220
258
|
return "-ERR wrong number of arguments for 'GET' command"
|
|
221
259
|
try:
|
|
222
|
-
return self.store.get(tokens[1])
|
|
260
|
+
return self._data(self.store.get(tokens[1]))
|
|
223
261
|
except TypeError as e:
|
|
224
262
|
return f"-ERR {str(e)}"
|
|
225
263
|
|
|
226
264
|
def _handle_del(self, tokens: list[str]):
|
|
227
|
-
if len(tokens)
|
|
265
|
+
if len(tokens) < 2:
|
|
228
266
|
return "-ERR wrong number of arguments for 'DEL' command"
|
|
229
|
-
return self.store.delete(tokens[1])
|
|
267
|
+
return sum(self.store.delete(key) for key in tokens[1:])
|
|
230
268
|
|
|
231
269
|
def _handle_exists(self, tokens: list[str]):
|
|
232
|
-
if len(tokens)
|
|
270
|
+
if len(tokens) < 2:
|
|
233
271
|
return "-ERR wrong number of arguments for 'EXISTS' command"
|
|
234
|
-
|
|
272
|
+
# Like Redis, a key named twice counts twice.
|
|
273
|
+
return sum(self.store.exists(key) for key in tokens[1:])
|
|
235
274
|
|
|
236
275
|
def _handle_expire(self, tokens: list[str]):
|
|
237
276
|
if len(tokens) != 3:
|
|
@@ -310,7 +349,7 @@ class CommandHandler:
|
|
|
310
349
|
if len(tokens) != 2:
|
|
311
350
|
return "-ERR wrong number of arguments for 'lpop' command"
|
|
312
351
|
try:
|
|
313
|
-
return self.store.lpop(tokens[1])
|
|
352
|
+
return self._data(self.store.lpop(tokens[1]))
|
|
314
353
|
except TypeError as e:
|
|
315
354
|
return f"-ERR {str(e)}"
|
|
316
355
|
|
|
@@ -318,7 +357,7 @@ class CommandHandler:
|
|
|
318
357
|
if len(tokens) != 2:
|
|
319
358
|
return "-ERR wrong number of arguments for 'rpop' command"
|
|
320
359
|
try:
|
|
321
|
-
return self.store.rpop(tokens[1])
|
|
360
|
+
return self._data(self.store.rpop(tokens[1]))
|
|
322
361
|
except TypeError as e:
|
|
323
362
|
return f"-ERR {str(e)}"
|
|
324
363
|
|
|
@@ -360,7 +399,7 @@ class CommandHandler:
|
|
|
360
399
|
if len(tokens) != 3:
|
|
361
400
|
return "-ERR wrong number of arguments for 'hget' command"
|
|
362
401
|
try:
|
|
363
|
-
return self.store.hget(tokens[1], tokens[2])
|
|
402
|
+
return self._data(self.store.hget(tokens[1], tokens[2]))
|
|
364
403
|
except TypeError as e:
|
|
365
404
|
return f"-ERR {str(e)}"
|
|
366
405
|
|
|
@@ -386,9 +425,15 @@ class CommandHandler:
|
|
|
386
425
|
key = tokens[1]
|
|
387
426
|
added = 0
|
|
388
427
|
try:
|
|
428
|
+
# Validate every pair before applying any: a bad score anywhere
|
|
429
|
+
# means nothing is added.
|
|
430
|
+
pairs = []
|
|
389
431
|
for i in range(2, len(tokens), 2):
|
|
390
432
|
score = float(tokens[i])
|
|
391
|
-
|
|
433
|
+
if math.isnan(score):
|
|
434
|
+
raise ValueError
|
|
435
|
+
pairs.append((score, tokens[i + 1]))
|
|
436
|
+
for score, member in pairs:
|
|
392
437
|
added += self.store.zadd(key, score, member)
|
|
393
438
|
return added
|
|
394
439
|
except ValueError:
|
|
@@ -1053,25 +1098,53 @@ class CommandHandler:
|
|
|
1053
1098
|
receivers = 0
|
|
1054
1099
|
|
|
1055
1100
|
# Construct raw KESP bytes for the broadcast push
|
|
1056
|
-
kesp_payload = (
|
|
1057
|
-
f"A3\n"
|
|
1058
|
-
f"S7\nmessage\n"
|
|
1059
|
-
f"S{len(channel.encode('utf-8'))}\n{channel}\n"
|
|
1060
|
-
f"S{len(message.encode('utf-8'))}\n{message}\n"
|
|
1061
|
-
).encode("utf-8")
|
|
1101
|
+
kesp_payload = KESPEncoder.encode_command(["message", channel, message])
|
|
1062
1102
|
|
|
1063
1103
|
dead_sockets = []
|
|
1064
1104
|
for sock in subscribers:
|
|
1065
1105
|
try:
|
|
1066
|
-
|
|
1106
|
+
self._push(sock, kesp_payload)
|
|
1067
1107
|
receivers += 1
|
|
1068
1108
|
except Exception:
|
|
1069
1109
|
dead_sockets.append(sock)
|
|
1070
1110
|
|
|
1071
1111
|
for dead in dead_sockets:
|
|
1072
1112
|
subscribers.remove(dead)
|
|
1073
|
-
|
|
1074
|
-
|
|
1113
|
+
if not subscribers:
|
|
1114
|
+
del self._channels[channel]
|
|
1115
|
+
|
|
1116
|
+
return receivers
|
|
1117
|
+
|
|
1118
|
+
def _push(self, sub, payload: bytes) -> None:
|
|
1119
|
+
"""Delivers one pub/sub message to a subscriber, raising if it is gone.
|
|
1120
|
+
|
|
1121
|
+
The server hands subscribers over as asyncio StreamWriters, and it runs
|
|
1122
|
+
commands in worker threads, where a StreamWriter must not be touched
|
|
1123
|
+
directly: the write is queued onto the event loop instead. A plain
|
|
1124
|
+
socket (any object with sendall) is written to directly."""
|
|
1125
|
+
if hasattr(sub, "sendall"):
|
|
1126
|
+
sub.sendall(payload)
|
|
1127
|
+
return
|
|
1128
|
+
if sub.is_closing():
|
|
1129
|
+
raise ConnectionError("subscriber is gone")
|
|
1130
|
+
transport = getattr(sub, "transport", None)
|
|
1131
|
+
if (
|
|
1132
|
+
transport is not None
|
|
1133
|
+
and transport.get_write_buffer_size() > self.MAX_SUBSCRIBER_BUFFER
|
|
1134
|
+
):
|
|
1135
|
+
# A subscriber that isn't reading would otherwise make the server
|
|
1136
|
+
# hold every message for it, without limit. Cut it off instead.
|
|
1137
|
+
loop = self.event_loop
|
|
1138
|
+
if loop is not None and loop.is_running():
|
|
1139
|
+
loop.call_soon_threadsafe(sub.close)
|
|
1140
|
+
else:
|
|
1141
|
+
sub.close()
|
|
1142
|
+
raise ConnectionError("subscriber output buffer limit exceeded")
|
|
1143
|
+
loop = self.event_loop
|
|
1144
|
+
if loop is not None and loop.is_running():
|
|
1145
|
+
loop.call_soon_threadsafe(sub.write, payload)
|
|
1146
|
+
else:
|
|
1147
|
+
sub.write(payload)
|
|
1075
1148
|
|
|
1076
1149
|
def cmd_slowlog(self, tokens: list[str]) -> Any:
|
|
1077
1150
|
if len(tokens) < 2:
|