kedis-python 0.1.4__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {kedis_python-0.1.4 → kedis_python-0.3.0}/PKG-INFO +26 -14
  2. {kedis_python-0.1.4 → kedis_python-0.3.0}/README.md +19 -13
  3. kedis_python-0.3.0/kedis_python/aof.py +138 -0
  4. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/commands.py +133 -60
  5. kedis_python-0.3.0/kedis_python/help_data.py +711 -0
  6. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/main.py +154 -88
  7. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/network.py +15 -6
  8. kedis_python-0.3.0/kedis_python/parser.py +621 -0
  9. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/server.py +479 -106
  10. kedis_python-0.3.0/kedis_python/settings.py +238 -0
  11. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/store.py +118 -69
  12. kedis_python-0.3.0/kedis_python/textcodec.py +32 -0
  13. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/ui.py +71 -1
  14. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/PKG-INFO +26 -14
  15. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/SOURCES.txt +43 -2
  16. {kedis_python-0.1.4 → kedis_python-0.3.0}/pyproject.toml +11 -6
  17. kedis_python-0.3.0/tests/test_aof_persistence.py +280 -0
  18. kedis_python-0.3.0/tests/test_aof_records.py +140 -0
  19. kedis_python-0.3.0/tests/test_binary_safety.py +340 -0
  20. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_bitmap_persistence.py +20 -8
  21. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_bitmaps.py +17 -11
  22. kedis_python-0.3.0/tests/test_chunked_reply.py +409 -0
  23. kedis_python-0.3.0/tests/test_cli_housekeep.py +121 -0
  24. kedis_python-0.3.0/tests/test_cli_stream_view.py +118 -0
  25. kedis_python-0.3.0/tests/test_data_replies.py +123 -0
  26. kedis_python-0.3.0/tests/test_del_exists.py +107 -0
  27. kedis_python-0.3.0/tests/test_doc_claims.py +72 -0
  28. kedis_python-0.3.0/tests/test_docs_data_types.py +152 -0
  29. kedis_python-0.3.0/tests/test_error_line_cap.py +132 -0
  30. kedis_python-0.3.0/tests/test_follower_stream.py +495 -0
  31. kedis_python-0.3.0/tests/test_fuzz.py +286 -0
  32. kedis_python-0.3.0/tests/test_fuzz_properties.py +63 -0
  33. kedis_python-0.3.0/tests/test_help_data.py +211 -0
  34. kedis_python-0.3.0/tests/test_history_setup.py +74 -0
  35. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_hyperloglog.py +9 -2
  36. kedis_python-0.3.0/tests/test_kesp_decoder.py +254 -0
  37. kedis_python-0.3.0/tests/test_kesp_encoder.py +208 -0
  38. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_lists.py +1 -1
  39. kedis_python-0.3.0/tests/test_lrange.py +88 -0
  40. kedis_python-0.3.0/tests/test_multi_limits.py +301 -0
  41. kedis_python-0.3.0/tests/test_multi_readonly.py +157 -0
  42. kedis_python-0.3.0/tests/test_multi_replication.py +475 -0
  43. kedis_python-0.3.0/tests/test_parser_framing.py +175 -0
  44. kedis_python-0.3.0/tests/test_parser_limits.py +138 -0
  45. kedis_python-0.3.0/tests/test_pubsub.py +220 -0
  46. kedis_python-0.3.0/tests/test_replica_snapshot.py +141 -0
  47. kedis_python-0.3.0/tests/test_replica_write_deadline.py +279 -0
  48. kedis_python-0.3.0/tests/test_replication_task.py +217 -0
  49. kedis_python-0.3.0/tests/test_reply_budgets.py +226 -0
  50. kedis_python-0.3.0/tests/test_reply_limits.py +123 -0
  51. kedis_python-0.3.0/tests/test_reply_snapshots.py +198 -0
  52. kedis_python-0.3.0/tests/test_server_flags.py +427 -0
  53. kedis_python-0.3.0/tests/test_server_limits.py +345 -0
  54. kedis_python-0.3.0/tests/test_server_protocol_errors.py +112 -0
  55. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_groups_store.py +13 -5
  56. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_store.py +9 -1
  57. kedis_python-0.3.0/tests/test_type.py +77 -0
  58. kedis_python-0.3.0/tests/test_version.py +34 -0
  59. kedis_python-0.3.0/tests/test_write_deadline.py +242 -0
  60. kedis_python-0.3.0/tests/test_zadd.py +98 -0
  61. kedis_python-0.1.4/kedis_python/parser.py +0 -307
  62. kedis_python-0.1.4/kedis_python/replica.py +0 -454
  63. {kedis_python-0.1.4 → kedis_python-0.3.0}/LICENSE +0 -0
  64. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/__init__.py +0 -0
  65. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/benchmark.py +0 -0
  66. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/benchmark_asyncio.py +0 -0
  67. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/hyperloglog.py +0 -0
  68. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/kesp_cli.py +0 -0
  69. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/simulator.py +0 -0
  70. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/skiplist.py +0 -0
  71. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python/stream.py +0 -0
  72. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/dependency_links.txt +0 -0
  73. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/entry_points.txt +0 -0
  74. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/requires.txt +0 -0
  75. {kedis_python-0.1.4 → kedis_python-0.3.0}/kedis_python.egg-info/top_level.txt +0 -0
  76. {kedis_python-0.1.4 → kedis_python-0.3.0}/setup.cfg +0 -0
  77. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_bench.py +0 -0
  78. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_docs_streams.py +0 -0
  79. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_housekeeping.py +0 -0
  80. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_inline_block.py +0 -0
  81. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_kesp.py +0 -0
  82. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_kesp2.py +0 -0
  83. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_lru.py +0 -0
  84. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_parser_casing.py +0 -0
  85. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_parser_harden.py +0 -0
  86. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_replica.py +0 -0
  87. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_commands.py +0 -0
  88. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_core.py +0 -0
  89. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_groups_commands.py +0 -0
  90. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_groups_core.py +0 -0
  91. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_groups_server.py +0 -0
  92. {kedis_python-0.1.4 → kedis_python-0.3.0}/tests/test_stream_server.py +0 -0
@@ -1,10 +1,16 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: kedis-python
3
- Version: 0.1.4
3
+ Version: 0.3.0
4
4
  Summary: A Redis-inspired in-memory store in Python.
5
5
  Author: V SS Karthik
6
6
  License-Expression: MIT
7
7
  Project-URL: Homepage, https://github.com/Hogwarts-coder10/kedis-python
8
+ Keywords: redis,in-memory,key-value,database,streams,pub-sub
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3 :: Only
13
+ Classifier: Topic :: Database
8
14
  Requires-Python: >=3.9
9
15
  Description-Content-Type: text/markdown
10
16
  License-File: LICENSE
@@ -43,16 +49,18 @@ Rather than focusing on Redis compatibility, Kedis focuses on understanding the
43
49
  ## 📚 Engine Documentation
44
50
 
45
51
  Dive into the internal grimoire to understand how Kedis achieves microsecond latency:
46
- * [Architecture & Pipeline](docs/architecture.md)
47
- * [KESP Wire Protocol](docs/protocol.md)
48
- * [Networking & Surge Tank](docs/networking.md)
49
- * [Persistence & I/O Drivetrain](docs/persistence.md)
50
- * [Command Spellbook](docs/commands.md)
51
- * [TTL & Lifespan Mechanics](docs/ttl.md)
52
- * [Streams & Consumer Groups](docs/streams.md)
53
- * [Pub/Sub Broadcasts](docs/pubsub.md)
54
- * [Benchmarks & Telemetry](docs/benchmarks.md)
55
- * [Future Roadmap](docs/roadmap.md)
52
+ * [Architecture & Pipeline](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/architecture.md)
53
+ * [KESP Wire Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/protocol.md)
54
+ * [Networking & Surge Tank](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/networking.md)
55
+ * [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md)
56
+ * [Fuzzing the Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/fuzzing.md)
57
+ * [Persistence & I/O Drivetrain](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/persistence.md)
58
+ * [Command Spellbook](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/commands.md)
59
+ * [TTL & Lifespan Mechanics](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/ttl.md)
60
+ * [Streams & Consumer Groups](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md)
61
+ * [Pub/Sub Broadcasts](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/pubsub.md)
62
+ * [Benchmarks & Telemetry](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/benchmarks.md)
63
+ * [Future Roadmap](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/roadmap.md)
56
64
 
57
65
  ---
58
66
 
@@ -120,7 +128,7 @@ XPENDING orders shipping
120
128
  XACK orders shipping 1718000000000-0
121
129
  ```
122
130
 
123
- Append-only event logs with blocking reads, and consumer groups that share a stream between workers with a pending list and acknowledgements. Streams persist through the AOF, `COMPACT` and snapshots, and replicate with the exact IDs and group state the master chose. See [Streams & Consumer Groups](docs/streams.md).
131
+ Append-only event logs with blocking reads, and consumer groups that share a stream between workers with a pending list and acknowledgements. Streams persist through the AOF, `COMPACT` and snapshots, and replicate with the exact IDs and group state the master chose. See [Streams & Consumer Groups](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md).
124
132
 
125
133
  ---
126
134
 
@@ -193,7 +201,8 @@ Kedis supports master-replica replication:
193
201
 
194
202
  * Replicas connect to a master and perform a `SYNC` handshake to receive the initial dataset
195
203
  * The master streams subsequent writes to connected replicas in real time
196
- * Replicas are read-only — writes issued directly to a replica are rejected
204
+ * Replicas are read-only — writes issued directly to a replica are rejected, and so are writes inside a `MULTI` block: the write gets `-EREADONLY` when it is queued and `EXEC` aborts with `-EXECABORT`
205
+ * A `MULTI`/`EXEC` block runs as one unit on the master and reaches followers as one transaction, which a follower applies as a unit
197
206
 
198
207
  ---
199
208
 
@@ -204,6 +213,7 @@ Asyncio-based server implementation supporting:
204
213
  * many concurrent client connections on a single event loop
205
214
  * command execution over sockets via the KESP protocol
206
215
  * standalone local mode fallback
216
+ * every limit settable at startup, e.g. `kedis-server --port 7001 --max-clients 500` (see [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md))
207
217
 
208
218
  ---
209
219
 
@@ -306,6 +316,7 @@ The project includes automated tests covering:
306
316
  * persistence recovery
307
317
  * expiration behavior
308
318
  * streams and consumer groups, from the data structure up to live TCP sessions
319
+ * protocol fuzzing: a mutation fuzzer, reference readers and property tests (see [Fuzzing the Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/fuzzing.md))
309
320
 
310
321
  Additional coverage is actively being expanded as the project evolves.
311
322
 
@@ -377,7 +388,8 @@ Current limitations include:
377
388
  * limited fault tolerance compared to production databases
378
389
  * no memory-based eviction ceiling (LRU tracks usage, but isn't yet tied to a hard memory limit)
379
390
  * AOF persistence is not buffered — writes can trigger a sync depending on config
380
- * consumer groups have no `XCLAIM`/`XAUTOCLAIM` yet; the full list of stream limits is in [docs/streams.md](docs/streams.md#-known-limits)
391
+ * `SAVE` and `COMPACT` hold the engine lock while they dump the store, so other commands wait (about 1 s for 3 million items); a write forwarded to followers is encoded on the event loop in one go (about 0.55 s for a command with 1,000,000 arguments). Both are listed in [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md#not-covered-yet)
392
+ * consumer groups have no `XCLAIM`/`XAUTOCLAIM` yet; the full list of stream limits is in [docs/streams.md](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md#-known-limits)
381
393
 
382
394
  These limitations are intentional learning opportunities and areas of active development.
383
395
 
@@ -29,16 +29,18 @@ Rather than focusing on Redis compatibility, Kedis focuses on understanding the
29
29
  ## 📚 Engine Documentation
30
30
 
31
31
  Dive into the internal grimoire to understand how Kedis achieves microsecond latency:
32
- * [Architecture & Pipeline](docs/architecture.md)
33
- * [KESP Wire Protocol](docs/protocol.md)
34
- * [Networking & Surge Tank](docs/networking.md)
35
- * [Persistence & I/O Drivetrain](docs/persistence.md)
36
- * [Command Spellbook](docs/commands.md)
37
- * [TTL & Lifespan Mechanics](docs/ttl.md)
38
- * [Streams & Consumer Groups](docs/streams.md)
39
- * [Pub/Sub Broadcasts](docs/pubsub.md)
40
- * [Benchmarks & Telemetry](docs/benchmarks.md)
41
- * [Future Roadmap](docs/roadmap.md)
32
+ * [Architecture & Pipeline](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/architecture.md)
33
+ * [KESP Wire Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/protocol.md)
34
+ * [Networking & Surge Tank](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/networking.md)
35
+ * [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md)
36
+ * [Fuzzing the Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/fuzzing.md)
37
+ * [Persistence & I/O Drivetrain](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/persistence.md)
38
+ * [Command Spellbook](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/commands.md)
39
+ * [TTL & Lifespan Mechanics](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/ttl.md)
40
+ * [Streams & Consumer Groups](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md)
41
+ * [Pub/Sub Broadcasts](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/pubsub.md)
42
+ * [Benchmarks & Telemetry](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/benchmarks.md)
43
+ * [Future Roadmap](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/roadmap.md)
42
44
 
43
45
  ---
44
46
 
@@ -106,7 +108,7 @@ XPENDING orders shipping
106
108
  XACK orders shipping 1718000000000-0
107
109
  ```
108
110
 
109
- Append-only event logs with blocking reads, and consumer groups that share a stream between workers with a pending list and acknowledgements. Streams persist through the AOF, `COMPACT` and snapshots, and replicate with the exact IDs and group state the master chose. See [Streams & Consumer Groups](docs/streams.md).
111
+ Append-only event logs with blocking reads, and consumer groups that share a stream between workers with a pending list and acknowledgements. Streams persist through the AOF, `COMPACT` and snapshots, and replicate with the exact IDs and group state the master chose. See [Streams & Consumer Groups](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md).
110
112
 
111
113
  ---
112
114
 
@@ -179,7 +181,8 @@ Kedis supports master-replica replication:
179
181
 
180
182
  * Replicas connect to a master and perform a `SYNC` handshake to receive the initial dataset
181
183
  * The master streams subsequent writes to connected replicas in real time
182
- * Replicas are read-only — writes issued directly to a replica are rejected
184
+ * Replicas are read-only — writes issued directly to a replica are rejected, and so are writes inside a `MULTI` block: the write gets `-EREADONLY` when it is queued and `EXEC` aborts with `-EXECABORT`
185
+ * A `MULTI`/`EXEC` block runs as one unit on the master and reaches followers as one transaction, which a follower applies as a unit
183
186
 
184
187
  ---
185
188
 
@@ -190,6 +193,7 @@ Asyncio-based server implementation supporting:
190
193
  * many concurrent client connections on a single event loop
191
194
  * command execution over sockets via the KESP protocol
192
195
  * standalone local mode fallback
196
+ * every limit settable at startup, e.g. `kedis-server --port 7001 --max-clients 500` (see [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md))
193
197
 
194
198
  ---
195
199
 
@@ -292,6 +296,7 @@ The project includes automated tests covering:
292
296
  * persistence recovery
293
297
  * expiration behavior
294
298
  * streams and consumer groups, from the data structure up to live TCP sessions
299
+ * protocol fuzzing: a mutation fuzzer, reference readers and property tests (see [Fuzzing the Protocol](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/fuzzing.md))
295
300
 
296
301
  Additional coverage is actively being expanded as the project evolves.
297
302
 
@@ -363,7 +368,8 @@ Current limitations include:
363
368
  * limited fault tolerance compared to production databases
364
369
  * no memory-based eviction ceiling (LRU tracks usage, but isn't yet tied to a hard memory limit)
365
370
  * AOF persistence is not buffered — writes can trigger a sync depending on config
366
- * consumer groups have no `XCLAIM`/`XAUTOCLAIM` yet; the full list of stream limits is in [docs/streams.md](docs/streams.md#-known-limits)
371
+ * `SAVE` and `COMPACT` hold the engine lock while they dump the store, so other commands wait (about 1 s for 3 million items); a write forwarded to followers is encoded on the event loop in one go (about 0.55 s for a command with 1,000,000 arguments). Both are listed in [Limits & Budgets](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/limits.md#not-covered-yet)
372
+ * consumer groups have no `XCLAIM`/`XAUTOCLAIM` yet; the full list of stream limits is in [docs/streams.md](https://github.com/Hogwarts-coder10/kedis-python/blob/main/docs/streams.md#-known-limits)
367
373
 
368
374
  These limitations are intentional learning opportunities and areas of active development.
369
375
 
@@ -0,0 +1,138 @@
1
+ """On-disk record format for the append-only file (AOF).
2
+
3
+ Every logged command is one KESP-framed record, the same shape the wire
4
+ protocol uses for commands:
5
+
6
+ A<count>\\n then, for each token: S<length>\\n<payload>\\n
7
+
8
+ Lengths count bytes of the UTF-8 payload, so a token can hold anything: spaces,
9
+ newlines, tabs, an empty string, text that looks like another command. The old
10
+ format joined tokens with spaces on one line, so a value could not survive a
11
+ restart and a value containing a newline could plant commands in the log.
12
+
13
+ Strings are encoded with "surrogatepass", so every Python str round-trips
14
+ exactly, lone surrogates included.
15
+
16
+ Files written by older versions hold one whitespace-separated command per line.
17
+ scan() still reads those, line by line, and a file may mix both: old lines
18
+ first, framed records after.
19
+ """
20
+
21
+ import re
22
+
23
+ _COUNT = re.compile(rb"(?:0|[1-9][0-9]*)")
24
+ MAX_HEADER_LINE = 32 # "A<n>" / "S<n>" lines are never longer than this
25
+
26
+
27
+ def encode_record(tokens) -> bytes:
28
+ """One command as a framed record."""
29
+ parts = [b"A%d\n" % len(tokens)]
30
+ for token in tokens:
31
+ raw = str(token).encode("utf-8", "surrogatepass")
32
+ parts.append(b"S%d\n" % len(raw))
33
+ parts.append(raw)
34
+ parts.append(b"\n")
35
+ return b"".join(parts)
36
+
37
+
38
+ class ScanResult:
39
+ """How a scan ended.
40
+
41
+ status: "ok" every byte was consumed
42
+ "torn" the file ends in the middle of a record (a crash while
43
+ writing); everything before good_end is intact
44
+ "corrupt" a record is malformed; see reason
45
+ good_end: offset just past the last complete record or line
46
+ """
47
+
48
+ __slots__ = ("good_end", "reason", "status")
49
+
50
+ def __init__(self):
51
+ self.status = "ok"
52
+ self.good_end = 0
53
+ self.reason = ""
54
+
55
+
56
+ def _count(buf, start: int, end: int):
57
+ field = bytes(buf[start:end])
58
+ return int(field) if _COUNT.fullmatch(field) else None
59
+
60
+
61
+ def scan(buf, result: ScanResult):
62
+ """Yields (tokens, end_offset) for each command in buf, a bytes-like object
63
+ that supports find() and slicing (bytes or mmap). Stops at the first
64
+ incomplete or malformed record and says why in result."""
65
+ total = len(buf)
66
+ pos = 0
67
+
68
+ def stop(status: str, reason: str):
69
+ result.status = status
70
+ result.reason = reason
71
+
72
+ while pos < total:
73
+ if buf[pos : pos + 1] == b"A":
74
+ following = buf[pos + 1 : pos + 2]
75
+ if following == b"":
76
+ return stop("torn", "file ends inside a record header")
77
+ # A legacy command word starts "A" + a letter (AUTH, ...). Anything
78
+ # else after the "A" is a record header and must validate as one.
79
+ framed = not following.isalpha()
80
+ else:
81
+ framed = False
82
+
83
+ if not framed:
84
+ # Legacy line: whitespace-separated tokens, as older versions wrote.
85
+ nl = buf.find(b"\n", pos)
86
+ line_end = total if nl == -1 else nl
87
+ next_pos = total if nl == -1 else nl + 1
88
+ tokens = bytes(buf[pos:line_end]).decode("utf-8", "replace").split()
89
+ result.good_end = next_pos
90
+ if tokens:
91
+ yield tokens, next_pos
92
+ pos = next_pos
93
+ continue
94
+
95
+ nl = buf.find(b"\n", pos)
96
+ if nl == -1:
97
+ if total - pos > MAX_HEADER_LINE:
98
+ return stop("corrupt", "array header too long")
99
+ return stop("torn", "file ends inside a record header")
100
+ if nl - pos > MAX_HEADER_LINE:
101
+ return stop("corrupt", "array header too long")
102
+ count = _count(buf, pos + 1, nl)
103
+ if count is None:
104
+ return stop("corrupt", "invalid array header")
105
+
106
+ cursor = nl + 1
107
+ tokens = []
108
+ for _ in range(count):
109
+ if cursor >= total:
110
+ return stop("torn", "file ends between tokens")
111
+ if buf[cursor : cursor + 1] != b"S":
112
+ return stop("corrupt", "expected a string header")
113
+ nl = buf.find(b"\n", cursor)
114
+ if nl == -1:
115
+ if total - cursor > MAX_HEADER_LINE:
116
+ return stop("corrupt", "string header too long")
117
+ return stop("torn", "file ends inside a string header")
118
+ if nl - cursor > MAX_HEADER_LINE:
119
+ return stop("corrupt", "string header too long")
120
+ length = _count(buf, cursor + 1, nl)
121
+ if length is None:
122
+ return stop("corrupt", "invalid string length")
123
+ start = nl + 1
124
+ end = start + length
125
+ if end + 1 > total:
126
+ return stop("torn", "file ends inside a string")
127
+ if buf[end : end + 1] != b"\n":
128
+ return stop("corrupt", "string is not newline-terminated")
129
+ try:
130
+ tokens.append(bytes(buf[start:end]).decode("utf-8", "surrogatepass"))
131
+ except UnicodeDecodeError:
132
+ return stop("corrupt", "string is not valid UTF-8")
133
+ cursor = end + 1
134
+
135
+ result.good_end = cursor
136
+ if tokens:
137
+ yield tokens, cursor
138
+ pos = cursor
@@ -1,13 +1,17 @@
1
+ import math
1
2
  import threading
2
3
  import time
3
4
  from typing import Any
4
5
 
5
- from .parser import BulkString
6
+ from .parser import BulkString, KESPEncoder
6
7
  from .store import KedisStore
7
8
  from .stream import StreamError
8
9
 
9
10
 
10
11
  class CommandHandler:
12
+ # Bytes a pub/sub subscriber may leave unread before the server cuts it off.
13
+ MAX_SUBSCRIBER_BUFFER = 32 * 1024 * 1024
14
+
11
15
  def __init__(self, store: KedisStore):
12
16
  self.store = store
13
17
 
@@ -16,6 +20,9 @@ class CommandHandler:
16
20
 
17
21
  # Pub / Sub SwitchBoard
18
22
  self._channels = {}
23
+ # The server's event loop, set per connection; pub/sub pushes are queued
24
+ # onto it because commands run in worker threads.
25
+ self.event_loop = None
19
26
 
20
27
  # The O(1) Dispatch Table
21
28
  self._commands = {
@@ -129,57 +136,82 @@ class CommandHandler:
129
136
  if not tokens:
130
137
  return "-ERR empty command or syntax error", []
131
138
 
132
- cmd = tokens[0].upper()
133
- args = tokens[
134
- 1:
135
- ] # (Though your current code passes tokens, let's keep your routing exact)
136
-
137
139
  # 🚀 TRACK A: Start the high-precision telemetry stopwatch (in nanoseconds)
138
140
  start_time = time.perf_counter_ns()
141
+ try:
142
+ with self._engine_lock:
143
+ return self._run_locked(tokens, client_socket)
144
+ finally:
145
+ self._record_duration(tokens, start_time)
146
+
147
+ def execute_batch(self, batch: list, client_socket=None):
148
+ """Runs several commands as one unit: the engine lock is taken once, so
149
+ no other client's command can run between them, and the AOF records
150
+ them back to back. Returns (results, forwards): one result per command,
151
+ and the commands a master must send to replicas, in order (see
152
+ replication_payloads). Commands are not skipped or reordered; one that
153
+ fails gives an error result and the rest still run, like the one-at-a-
154
+ time path."""
155
+ results: list = []
156
+ forwards: list = []
157
+ write_commands = self.WRITE_COMMANDS
158
+ starts: list = []
159
+ with self._engine_lock:
160
+ for tokens in batch:
161
+ if not tokens:
162
+ results.append("-ERR empty command or syntax error")
163
+ starts.append(None)
164
+ continue
165
+ starts.append(time.perf_counter_ns())
166
+ result, records = self._run_locked(tokens, client_socket)
167
+ results.append(result)
168
+ if tokens[0].upper() in write_commands:
169
+ forwards.extend(self.replication_payloads(tokens, result, records))
170
+ for tokens, started in zip(batch, starts):
171
+ if started is not None:
172
+ self._record_duration(tokens, started)
173
+ return results, forwards
174
+
175
+ def _run_locked(self, tokens: list[str], client_socket=None):
176
+ """Runs one command. The caller holds the engine lock."""
177
+ cmd = tokens[0].upper()
139
178
  result = None
140
179
  records = []
141
180
 
142
181
  try:
143
- with self._engine_lock:
144
- try:
145
- if cmd == "XGAPPLY" and client_socket is not None:
146
- # Internal replication command: a network client
147
- # must not be able to write raw group records.
148
- result = f"-ERR unknown command '{cmd}'"
149
- elif cmd in self._commands:
150
- if cmd in ["SUBSCRIBE", "PUBLISH"]:
151
- handler_function = self._commands[cmd]
152
- result = handler_function(tokens, client_socket)
153
- else:
154
- handler_function = self._commands[cmd]
155
- # Note: Depending on how your handlers are registered,
156
- # they might expect 'tokens' or 'args'. Keeping your existing call:
157
- result = handler_function(tokens)
182
+ try:
183
+ if cmd == "XGAPPLY" and client_socket is not None:
184
+ # Internal replication command: a network client
185
+ # must not be able to write raw group records.
186
+ result = f"-ERR unknown command '{cmd}'"
187
+ elif cmd in self._commands:
188
+ handler_function = self._commands[cmd]
189
+ if cmd in ["SUBSCRIBE", "PUBLISH"]:
190
+ result = handler_function(tokens, client_socket)
158
191
  else:
159
- result = f"-ERR unknown command '{cmd}'"
160
- finally:
161
- records = self.store.drain_group_records()
162
- return result, records
163
-
164
- except TypeError as e:
165
- result = f"-ERR wrong number of arguments for '{cmd}' command"
192
+ # Handlers take the whole token list, command name included.
193
+ result = handler_function(tokens)
194
+ else:
195
+ result = f"-ERR unknown command '{cmd}'"
196
+ finally:
197
+ records = self.store.drain_group_records()
166
198
  return result, records
199
+ except TypeError:
200
+ return f"-ERR wrong number of arguments for '{cmd}' command", records
167
201
  except Exception as e:
168
- result = f"-ERR {str(e)}"
169
- return result, records
202
+ return f"-ERR {str(e)}", records
170
203
 
171
- finally:
172
- # 🚀 TRACK A: Stop the stopwatch, convert nanoseconds to microseconds
173
- end_time = time.perf_counter_ns()
174
- duration_us = (end_time - start_time) // 1000
175
-
176
- # Feed the metrics to the store's slowlog ring buffer if available
177
- if (
178
- hasattr(self, "store")
179
- and self.store
180
- and hasattr(self.store, "_log_slow_command")
181
- ):
182
- self.store._log_slow_command(tokens, duration_us)
204
+ def _record_duration(self, tokens: list[str], start_time: int) -> None:
205
+ # 🚀 TRACK A: Stop the stopwatch, convert nanoseconds to microseconds
206
+ duration_us = (time.perf_counter_ns() - start_time) // 1000
207
+
208
+ # Feed the metrics to the store's slowlog ring buffer if available
209
+ if (
210
+ hasattr(self, "store")
211
+ and self.store
212
+ and hasattr(self.store, "_log_slow_command")
213
+ ):
214
+ self.store._log_slow_command(tokens, duration_us)
183
215
 
184
216
  # ---------------------------------------------------------
185
217
  # COMMAND HANDLERS (The isolated engine components)
@@ -215,23 +247,30 @@ class CommandHandler:
215
247
  self.store.set(tokens[1], tokens[2])
216
248
  return "OK"
217
249
 
250
+ @staticmethod
251
+ def _data(value):
252
+ """A stored value as a reply: always data, never read as a status or
253
+ an error, whatever it looks like ("OK", "ERROR x", "-ERR y")."""
254
+ return BulkString(value) if isinstance(value, str) else value
255
+
218
256
  def _handle_get(self, tokens: list[str]):
219
257
  if len(tokens) != 2:
220
258
  return "-ERR wrong number of arguments for 'GET' command"
221
259
  try:
222
- return self.store.get(tokens[1])
260
+ return self._data(self.store.get(tokens[1]))
223
261
  except TypeError as e:
224
262
  return f"-ERR {str(e)}"
225
263
 
226
264
  def _handle_del(self, tokens: list[str]):
227
- if len(tokens) != 2:
265
+ if len(tokens) < 2:
228
266
  return "-ERR wrong number of arguments for 'DEL' command"
229
- return self.store.delete(tokens[1])
267
+ return sum(self.store.delete(key) for key in tokens[1:])
230
268
 
231
269
  def _handle_exists(self, tokens: list[str]):
232
- if len(tokens) != 2:
270
+ if len(tokens) < 2:
233
271
  return "-ERR wrong number of arguments for 'EXISTS' command"
234
- return self.store.exists(tokens[1])
272
+ # Like Redis, a key named twice counts twice.
273
+ return sum(self.store.exists(key) for key in tokens[1:])
235
274
 
236
275
  def _handle_expire(self, tokens: list[str]):
237
276
  if len(tokens) != 3:
@@ -310,7 +349,7 @@ class CommandHandler:
310
349
  if len(tokens) != 2:
311
350
  return "-ERR wrong number of arguments for 'lpop' command"
312
351
  try:
313
- return self.store.lpop(tokens[1])
352
+ return self._data(self.store.lpop(tokens[1]))
314
353
  except TypeError as e:
315
354
  return f"-ERR {str(e)}"
316
355
 
@@ -318,7 +357,7 @@ class CommandHandler:
318
357
  if len(tokens) != 2:
319
358
  return "-ERR wrong number of arguments for 'rpop' command"
320
359
  try:
321
- return self.store.rpop(tokens[1])
360
+ return self._data(self.store.rpop(tokens[1]))
322
361
  except TypeError as e:
323
362
  return f"-ERR {str(e)}"
324
363
 
@@ -360,7 +399,7 @@ class CommandHandler:
360
399
  if len(tokens) != 3:
361
400
  return "-ERR wrong number of arguments for 'hget' command"
362
401
  try:
363
- return self.store.hget(tokens[1], tokens[2])
402
+ return self._data(self.store.hget(tokens[1], tokens[2]))
364
403
  except TypeError as e:
365
404
  return f"-ERR {str(e)}"
366
405
 
@@ -386,9 +425,15 @@ class CommandHandler:
386
425
  key = tokens[1]
387
426
  added = 0
388
427
  try:
428
+ # Validate every pair before applying any: a bad score anywhere
429
+ # means nothing is added.
430
+ pairs = []
389
431
  for i in range(2, len(tokens), 2):
390
432
  score = float(tokens[i])
391
- member = tokens[i + 1]
433
+ if math.isnan(score):
434
+ raise ValueError
435
+ pairs.append((score, tokens[i + 1]))
436
+ for score, member in pairs:
392
437
  added += self.store.zadd(key, score, member)
393
438
  return added
394
439
  except ValueError:
@@ -1053,25 +1098,53 @@ class CommandHandler:
1053
1098
  receivers = 0
1054
1099
 
1055
1100
  # Construct raw KESP bytes for the broadcast push
1056
- kesp_payload = (
1057
- f"A3\n"
1058
- f"S7\nmessage\n"
1059
- f"S{len(channel.encode('utf-8'))}\n{channel}\n"
1060
- f"S{len(message.encode('utf-8'))}\n{message}\n"
1061
- ).encode("utf-8")
1101
+ kesp_payload = KESPEncoder.encode_command(["message", channel, message])
1062
1102
 
1063
1103
  dead_sockets = []
1064
1104
  for sock in subscribers:
1065
1105
  try:
1066
- sock.sendall(kesp_payload)
1106
+ self._push(sock, kesp_payload)
1067
1107
  receivers += 1
1068
1108
  except Exception:
1069
1109
  dead_sockets.append(sock)
1070
1110
 
1071
1111
  for dead in dead_sockets:
1072
1112
  subscribers.remove(dead)
1073
-
1074
- return
1113
+ if not subscribers:
1114
+ del self._channels[channel]
1115
+
1116
+ return receivers
1117
+
1118
+ def _push(self, sub, payload: bytes) -> None:
1119
+ """Delivers one pub/sub message to a subscriber, raising if it is gone.
1120
+
1121
+ The server hands subscribers over as asyncio StreamWriters, and it runs
1122
+ commands in worker threads, where a StreamWriter must not be touched
1123
+ directly: the write is queued onto the event loop instead. A plain
1124
+ socket (any object with sendall) is written to directly."""
1125
+ if hasattr(sub, "sendall"):
1126
+ sub.sendall(payload)
1127
+ return
1128
+ if sub.is_closing():
1129
+ raise ConnectionError("subscriber is gone")
1130
+ transport = getattr(sub, "transport", None)
1131
+ if (
1132
+ transport is not None
1133
+ and transport.get_write_buffer_size() > self.MAX_SUBSCRIBER_BUFFER
1134
+ ):
1135
+ # A subscriber that isn't reading would otherwise make the server
1136
+ # hold every message for it, without limit. Cut it off instead.
1137
+ loop = self.event_loop
1138
+ if loop is not None and loop.is_running():
1139
+ loop.call_soon_threadsafe(sub.close)
1140
+ else:
1141
+ sub.close()
1142
+ raise ConnectionError("subscriber output buffer limit exceeded")
1143
+ loop = self.event_loop
1144
+ if loop is not None and loop.is_running():
1145
+ loop.call_soon_threadsafe(sub.write, payload)
1146
+ else:
1147
+ sub.write(payload)
1075
1148
 
1076
1149
  def cmd_slowlog(self, tokens: list[str]) -> Any:
1077
1150
  if len(tokens) < 2: