natsio-kvcodec 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- natsio_kvcodec-0.1.0/PKG-INFO +156 -0
- natsio_kvcodec-0.1.0/README.md +138 -0
- natsio_kvcodec-0.1.0/pyproject.toml +29 -0
- natsio_kvcodec-0.1.0/src/natsio/kvcodec/__init__.py +69 -0
- natsio_kvcodec-0.1.0/src/natsio/kvcodec/errors.py +61 -0
- natsio_kvcodec-0.1.0/src/natsio/kvcodec/key_codecs.py +220 -0
- natsio_kvcodec-0.1.0/src/natsio/kvcodec/py.typed +0 -0
- natsio_kvcodec-0.1.0/src/natsio/kvcodec/value_codecs.py +124 -0
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: natsio-kvcodec
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: natsio extension: transparent KV key/value codecs (base64, path, zlib, chaining) over the core codec seam
|
|
5
|
+
Keywords: nats,nats.io,jetstream,key-value,kv,codec
|
|
6
|
+
Author: Corrupt Mane
|
|
7
|
+
Author-email: Corrupt Mane <corruptmane@gmail.com>
|
|
8
|
+
License-Expression: Apache-2.0
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Framework :: AsyncIO
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Dist: natsio
|
|
16
|
+
Requires-Python: >=3.13
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# natsio-kvcodec
|
|
20
|
+
|
|
21
|
+
Transparent **key/value codecs** for natsio's KV store, over the core's
|
|
22
|
+
`KeyCodec` / `ValueCodec` seam. Encode keys and values on the way in, decode
|
|
23
|
+
them on the way out — `get()`, `keys()`, `watch()`, and `history()` all speak the
|
|
24
|
+
*decoded* form while the bucket stores the *encoded* form.
|
|
25
|
+
|
|
26
|
+
Mirrors [`orbit.go/kvcodec`](https://github.com/synadia-io/orbit.go/tree/main/kvcodec),
|
|
27
|
+
pythonized. Stdlib only; no runtime dependencies beyond `natsio`.
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install natsio-kvcodec
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Usage
|
|
34
|
+
|
|
35
|
+
Pass codecs to any `create_key_value` / `key_value` variant — the natsio core
|
|
36
|
+
already accepts `key_codec=` and `value_codec=`:
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from natsio.kv import KeyValueConfig
|
|
40
|
+
from natsio.kvcodec import PathKeyCodec, ZlibValueCodec
|
|
41
|
+
|
|
42
|
+
kv = await js.create_key_value(
|
|
43
|
+
KeyValueConfig(bucket="cfg"),
|
|
44
|
+
key_codec=PathKeyCodec(), # "/app/db/url" stored as "app.db.url"
|
|
45
|
+
value_codec=ZlibValueCodec(), # values compressed at rest & over the wire
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
await kv.put("/app/db/url", b"postgres://localhost")
|
|
49
|
+
entry = await kv.get("/app/db/url") # entry.key == "/app/db/url"
|
|
50
|
+
await kv.keys() # ["/app/db/url"] — DECODED keys
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Key and value codecs are independent; use either, both, or neither.
|
|
54
|
+
|
|
55
|
+
## Codecs
|
|
56
|
+
|
|
57
|
+
### Key codecs (`str -> str`)
|
|
58
|
+
|
|
59
|
+
| Codec | What it does | orbit.go parity |
|
|
60
|
+
|---|---|---|
|
|
61
|
+
| `Base64KeyCodec` | Per-token raw URL-safe base64 (dots kept as separators). Lets a key carry characters illegal in NATS subjects — modulo the core-friction caveat below. | `Base64Codec` (key side) |
|
|
62
|
+
| `PathKeyCodec` | Filesystem keys `/a/b/c` <-> NATS `a.b.c`. Leading `/` becomes the `_root_` sentinel, trailing `/` trimmed. | `PathCodec` |
|
|
63
|
+
| `NoOpKeyCodec` | Identity (the core's `key_codec=None` already means identity; useful as a chain filler). | `NoOpCodec` |
|
|
64
|
+
| `ChainKeyCodec(*codecs)` | Apply key codecs in sequence (encode first->last, decode last->first). | `KeyChainCodec` |
|
|
65
|
+
|
|
66
|
+
Every key codec guarantees its **encoded output satisfies
|
|
67
|
+
`natsio.kv.validate_key`** (the raw URL-safe base64 alphabet and `_root_`
|
|
68
|
+
sentinel are all NATS-legal), so the core accepts the encoded key as a subject.
|
|
69
|
+
|
|
70
|
+
`Base64KeyCodec`, `PathKeyCodec`, `NoOpKeyCodec`, and an all-filterable
|
|
71
|
+
`ChainKeyCodec` also implement `encode_filter(pattern)` (the core's
|
|
72
|
+
runtime-checkable `natsio.kv.FilterableKeyCodec` protocol), which encodes a
|
|
73
|
+
wildcard pattern while preserving `*`/`>`. **The natsio core calls this** from
|
|
74
|
+
`watch()`, so a wildcard watch under one of these codecs is encoded per token
|
|
75
|
+
(`orders.>` -> `b3JkZXJz.>`) and works end-to-end — see Core Friction.
|
|
76
|
+
|
|
77
|
+
### Value codecs (`bytes -> bytes`)
|
|
78
|
+
|
|
79
|
+
| Codec | What it does | orbit.go parity |
|
|
80
|
+
|---|---|---|
|
|
81
|
+
| `ZlibValueCodec(level=-1)` | Transparent DEFLATE via stdlib `zlib`. | *natsio addition* |
|
|
82
|
+
| `Base64ValueCodec` | Whole-value raw URL-safe base64. | `Base64Codec` (value side) |
|
|
83
|
+
| `NoOpValueCodec` | Identity. | `NoOpCodec` |
|
|
84
|
+
| `ChainValueCodec(*codecs)` | Apply value codecs in sequence. | `ValueChainCodec` |
|
|
85
|
+
|
|
86
|
+
**`ZlibValueCodec` tradeoffs.** It trades CPU for bytes, and *small or
|
|
87
|
+
incompressible values come out larger* (a ~6-byte zlib envelope). It is not
|
|
88
|
+
encryption. If you only want at-rest compression, prefer
|
|
89
|
+
`KeyValueConfig(compression=True)` (server-side S2, zero client CPU); reach for
|
|
90
|
+
this codec when you want the bytes compressed **over the wire** and counted in
|
|
91
|
+
`max_bytes`.
|
|
92
|
+
|
|
93
|
+
## Errors
|
|
94
|
+
|
|
95
|
+
Codecs fail loud (like orbit.go's `error` returns), never silently pass corrupt
|
|
96
|
+
data through:
|
|
97
|
+
|
|
98
|
+
- `NoCodecsError` — empty `ChainKeyCodec()` / `ChainValueCodec()`.
|
|
99
|
+
- `KeyDecodeError` / `ValueDecodeError` — corrupt/non-encoded input (a non-base64
|
|
100
|
+
token, a bad zlib stream).
|
|
101
|
+
- `WildcardNotSupportedError` — `encode_filter` on a chain with a non-filterable
|
|
102
|
+
member.
|
|
103
|
+
|
|
104
|
+
All derive from `KvCodecError`.
|
|
105
|
+
|
|
106
|
+
## orbit.go parity notes
|
|
107
|
+
|
|
108
|
+
- **Two classes per base64, not one.** Go's structural typing lets a single
|
|
109
|
+
`Base64Codec` satisfy both `KeyCodec` and `ValueCodec`. natsio's protocols
|
|
110
|
+
have different `encode` signatures (`str->str` vs `bytes->bytes`), so this
|
|
111
|
+
ships `Base64KeyCodec` and `Base64ValueCodec`.
|
|
112
|
+
- **Naming.** orbit uses `EncodeKey`/`DecodeKey`; the natsio seam uses
|
|
113
|
+
`encode`/`decode`, so codecs follow the natsio spelling.
|
|
114
|
+
- **`ZlibValueCodec` is new.** orbit ships only base64 for values; a value codec
|
|
115
|
+
is the natural home for compression, so natsio adds one.
|
|
116
|
+
- Exact base64/path/filter **test vectors are ported** from orbit's
|
|
117
|
+
`codec_test.go` / `chain_codec_test.go`, so behavior matches byte-for-byte
|
|
118
|
+
(`Base64KeyCodec().encode("test.key...") == "dGVzdA.a2V5..."`, etc.).
|
|
119
|
+
- **`PathKeyCodec` dot caveat (shared with orbit).** It maps `/`<->`.` and cannot
|
|
120
|
+
distinguish a literal `.` in the input from a separator, so feed it
|
|
121
|
+
path-style keys; `a.b` decodes back as `a/b`.
|
|
122
|
+
|
|
123
|
+
## Core friction
|
|
124
|
+
|
|
125
|
+
Building this extension stress-tested the codec seam. Two real gaps surfaced
|
|
126
|
+
(neither is a codec bug — both are in the core, documented here so the behavior
|
|
127
|
+
isn't surprising):
|
|
128
|
+
|
|
129
|
+
1. **The raw key is validated *before* the codec runs, defeating Base64's main
|
|
130
|
+
use case.** `KeyValue._encode_key` calls `validate_key(raw_key)` and *then*
|
|
131
|
+
`validate_key(codec.encode(raw_key))`. So `put("Acme Inc.contact", ...)` with
|
|
132
|
+
`Base64KeyCodec` raises `InvalidKeyError` on the space — even though the codec
|
|
133
|
+
would turn it into the perfectly valid `QWNtZSBJbmM.Y29udGFjdA`. The escape-
|
|
134
|
+
exotic-characters scenario from orbit's own README cannot work end-to-end.
|
|
135
|
+
*Proposal:* when a `key_codec` is set, validate only the **encoded** key; the
|
|
136
|
+
raw key is the user's domain and only the stored subject must be NATS-legal.
|
|
137
|
+
|
|
138
|
+
2. **`keys()` / `iter_keys()` (and `purge_deletes()`) still run the value codec
|
|
139
|
+
on a payload the server deliberately stripped.** Those paths use a
|
|
140
|
+
headers-only (`meta_only`) watch, so every delivery has an empty payload —
|
|
141
|
+
but `_entry_from_msg` unconditionally calls `value_codec.decode(payload)` for
|
|
142
|
+
`PUT` entries. Any value codec that can't decode `b""` (`ZlibValueCodec`,
|
|
143
|
+
encryption, length-prefixed framing) makes `keys()` raise `ValueDecodeError`.
|
|
144
|
+
*Proposal:* skip value decoding for headers-only deliveries (or when the
|
|
145
|
+
payload is empty under `meta_only`). Pinned by a strict `xfail` in
|
|
146
|
+
`test_kvcodec_live.py::test_keys_under_framing_value_codec_is_broken`.
|
|
147
|
+
|
|
148
|
+
A third one — **the filter hook — is now closed.** orbit's `FilterableKeyCodec`
|
|
149
|
+
lets a wildcard watch encode tokens individually (`orders.>` -> `b3JkZXJz.>`).
|
|
150
|
+
The natsio core mirrors this: `watch()` recognises the runtime-checkable
|
|
151
|
+
`natsio.kv.FilterableKeyCodec` protocol (which the codecs here implement) and
|
|
152
|
+
calls `encode_filter()`, so per-token wildcard watches work end-to-end. Under a
|
|
153
|
+
codec the *raw* filter is the caller's domain (it may be in the codec's own
|
|
154
|
+
notation, e.g. `PathKeyCodec`'s `/a/*`) — only the encoded filter must be a
|
|
155
|
+
legal subject filter. A wildcard watch under a *non*-filterable codec is still
|
|
156
|
+
refused (`ConfigError`): encoding the whole key would mangle the `*`/`>`.
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# natsio-kvcodec
|
|
2
|
+
|
|
3
|
+
Transparent **key/value codecs** for natsio's KV store, over the core's
|
|
4
|
+
`KeyCodec` / `ValueCodec` seam. Encode keys and values on the way in, decode
|
|
5
|
+
them on the way out — `get()`, `keys()`, `watch()`, and `history()` all speak the
|
|
6
|
+
*decoded* form while the bucket stores the *encoded* form.
|
|
7
|
+
|
|
8
|
+
Mirrors [`orbit.go/kvcodec`](https://github.com/synadia-io/orbit.go/tree/main/kvcodec),
|
|
9
|
+
pythonized. Stdlib only; no runtime dependencies beyond `natsio`.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install natsio-kvcodec
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Usage
|
|
16
|
+
|
|
17
|
+
Pass codecs to any `create_key_value` / `key_value` variant — the natsio core
|
|
18
|
+
already accepts `key_codec=` and `value_codec=`:
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from natsio.kv import KeyValueConfig
|
|
22
|
+
from natsio.kvcodec import PathKeyCodec, ZlibValueCodec
|
|
23
|
+
|
|
24
|
+
kv = await js.create_key_value(
|
|
25
|
+
KeyValueConfig(bucket="cfg"),
|
|
26
|
+
key_codec=PathKeyCodec(), # "/app/db/url" stored as "app.db.url"
|
|
27
|
+
value_codec=ZlibValueCodec(), # values compressed at rest & over the wire
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
await kv.put("/app/db/url", b"postgres://localhost")
|
|
31
|
+
entry = await kv.get("/app/db/url") # entry.key == "/app/db/url"
|
|
32
|
+
await kv.keys() # ["/app/db/url"] — DECODED keys
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Key and value codecs are independent; use either, both, or neither.
|
|
36
|
+
|
|
37
|
+
## Codecs
|
|
38
|
+
|
|
39
|
+
### Key codecs (`str -> str`)
|
|
40
|
+
|
|
41
|
+
| Codec | What it does | orbit.go parity |
|
|
42
|
+
|---|---|---|
|
|
43
|
+
| `Base64KeyCodec` | Per-token raw URL-safe base64 (dots kept as separators). Lets a key carry characters illegal in NATS subjects — modulo the core-friction caveat below. | `Base64Codec` (key side) |
|
|
44
|
+
| `PathKeyCodec` | Filesystem keys `/a/b/c` <-> NATS `a.b.c`. Leading `/` becomes the `_root_` sentinel, trailing `/` trimmed. | `PathCodec` |
|
|
45
|
+
| `NoOpKeyCodec` | Identity (the core's `key_codec=None` already means identity; useful as a chain filler). | `NoOpCodec` |
|
|
46
|
+
| `ChainKeyCodec(*codecs)` | Apply key codecs in sequence (encode first->last, decode last->first). | `KeyChainCodec` |
|
|
47
|
+
|
|
48
|
+
Every key codec guarantees its **encoded output satisfies
|
|
49
|
+
`natsio.kv.validate_key`** (the raw URL-safe base64 alphabet and `_root_`
|
|
50
|
+
sentinel are all NATS-legal), so the core accepts the encoded key as a subject.
|
|
51
|
+
|
|
52
|
+
`Base64KeyCodec`, `PathKeyCodec`, `NoOpKeyCodec`, and an all-filterable
|
|
53
|
+
`ChainKeyCodec` also implement `encode_filter(pattern)` (the core's
|
|
54
|
+
runtime-checkable `natsio.kv.FilterableKeyCodec` protocol), which encodes a
|
|
55
|
+
wildcard pattern while preserving `*`/`>`. **The natsio core calls this** from
|
|
56
|
+
`watch()`, so a wildcard watch under one of these codecs is encoded per token
|
|
57
|
+
(`orders.>` -> `b3JkZXJz.>`) and works end-to-end — see Core Friction.
|
|
58
|
+
|
|
59
|
+
### Value codecs (`bytes -> bytes`)
|
|
60
|
+
|
|
61
|
+
| Codec | What it does | orbit.go parity |
|
|
62
|
+
|---|---|---|
|
|
63
|
+
| `ZlibValueCodec(level=-1)` | Transparent DEFLATE via stdlib `zlib`. | *natsio addition* |
|
|
64
|
+
| `Base64ValueCodec` | Whole-value raw URL-safe base64. | `Base64Codec` (value side) |
|
|
65
|
+
| `NoOpValueCodec` | Identity. | `NoOpCodec` |
|
|
66
|
+
| `ChainValueCodec(*codecs)` | Apply value codecs in sequence. | `ValueChainCodec` |
|
|
67
|
+
|
|
68
|
+
**`ZlibValueCodec` tradeoffs.** It trades CPU for bytes, and *small or
|
|
69
|
+
incompressible values come out larger* (a ~6-byte zlib envelope). It is not
|
|
70
|
+
encryption. If you only want at-rest compression, prefer
|
|
71
|
+
`KeyValueConfig(compression=True)` (server-side S2, zero client CPU); reach for
|
|
72
|
+
this codec when you want the bytes compressed **over the wire** and counted in
|
|
73
|
+
`max_bytes`.
|
|
74
|
+
|
|
75
|
+
## Errors
|
|
76
|
+
|
|
77
|
+
Codecs fail loud (like orbit.go's `error` returns), never silently pass corrupt
|
|
78
|
+
data through:
|
|
79
|
+
|
|
80
|
+
- `NoCodecsError` — empty `ChainKeyCodec()` / `ChainValueCodec()`.
|
|
81
|
+
- `KeyDecodeError` / `ValueDecodeError` — corrupt/non-encoded input (a non-base64
|
|
82
|
+
token, a bad zlib stream).
|
|
83
|
+
- `WildcardNotSupportedError` — `encode_filter` on a chain with a non-filterable
|
|
84
|
+
member.
|
|
85
|
+
|
|
86
|
+
All derive from `KvCodecError`.
|
|
87
|
+
|
|
88
|
+
## orbit.go parity notes
|
|
89
|
+
|
|
90
|
+
- **Two classes per base64, not one.** Go's structural typing lets a single
|
|
91
|
+
`Base64Codec` satisfy both `KeyCodec` and `ValueCodec`. natsio's protocols
|
|
92
|
+
have different `encode` signatures (`str->str` vs `bytes->bytes`), so this
|
|
93
|
+
ships `Base64KeyCodec` and `Base64ValueCodec`.
|
|
94
|
+
- **Naming.** orbit uses `EncodeKey`/`DecodeKey`; the natsio seam uses
|
|
95
|
+
`encode`/`decode`, so codecs follow the natsio spelling.
|
|
96
|
+
- **`ZlibValueCodec` is new.** orbit ships only base64 for values; a value codec
|
|
97
|
+
is the natural home for compression, so natsio adds one.
|
|
98
|
+
- Exact base64/path/filter **test vectors are ported** from orbit's
|
|
99
|
+
`codec_test.go` / `chain_codec_test.go`, so behavior matches byte-for-byte
|
|
100
|
+
(`Base64KeyCodec().encode("test.key...") == "dGVzdA.a2V5..."`, etc.).
|
|
101
|
+
- **`PathKeyCodec` dot caveat (shared with orbit).** It maps `/`<->`.` and cannot
|
|
102
|
+
distinguish a literal `.` in the input from a separator, so feed it
|
|
103
|
+
path-style keys; `a.b` decodes back as `a/b`.
|
|
104
|
+
|
|
105
|
+
## Core friction
|
|
106
|
+
|
|
107
|
+
Building this extension stress-tested the codec seam. Two real gaps surfaced
|
|
108
|
+
(neither is a codec bug — both are in the core, documented here so the behavior
|
|
109
|
+
isn't surprising):
|
|
110
|
+
|
|
111
|
+
1. **The raw key is validated *before* the codec runs, defeating Base64's main
|
|
112
|
+
use case.** `KeyValue._encode_key` calls `validate_key(raw_key)` and *then*
|
|
113
|
+
`validate_key(codec.encode(raw_key))`. So `put("Acme Inc.contact", ...)` with
|
|
114
|
+
`Base64KeyCodec` raises `InvalidKeyError` on the space — even though the codec
|
|
115
|
+
would turn it into the perfectly valid `QWNtZSBJbmM.Y29udGFjdA`. The escape-
|
|
116
|
+
exotic-characters scenario from orbit's own README cannot work end-to-end.
|
|
117
|
+
*Proposal:* when a `key_codec` is set, validate only the **encoded** key; the
|
|
118
|
+
raw key is the user's domain and only the stored subject must be NATS-legal.
|
|
119
|
+
|
|
120
|
+
2. **`keys()` / `iter_keys()` (and `purge_deletes()`) still run the value codec
|
|
121
|
+
on a payload the server deliberately stripped.** Those paths use a
|
|
122
|
+
headers-only (`meta_only`) watch, so every delivery has an empty payload —
|
|
123
|
+
but `_entry_from_msg` unconditionally calls `value_codec.decode(payload)` for
|
|
124
|
+
`PUT` entries. Any value codec that can't decode `b""` (`ZlibValueCodec`,
|
|
125
|
+
encryption, length-prefixed framing) makes `keys()` raise `ValueDecodeError`.
|
|
126
|
+
*Proposal:* skip value decoding for headers-only deliveries (or when the
|
|
127
|
+
payload is empty under `meta_only`). Pinned by a strict `xfail` in
|
|
128
|
+
`test_kvcodec_live.py::test_keys_under_framing_value_codec_is_broken`.
|
|
129
|
+
|
|
130
|
+
A third one — **the filter hook — is now closed.** orbit's `FilterableKeyCodec`
|
|
131
|
+
lets a wildcard watch encode tokens individually (`orders.>` -> `b3JkZXJz.>`).
|
|
132
|
+
The natsio core mirrors this: `watch()` recognises the runtime-checkable
|
|
133
|
+
`natsio.kv.FilterableKeyCodec` protocol (which the codecs here implement) and
|
|
134
|
+
calls `encode_filter()`, so per-token wildcard watches work end-to-end. Under a
|
|
135
|
+
codec the *raw* filter is the caller's domain (it may be in the codec's own
|
|
136
|
+
notation, e.g. `PathKeyCodec`'s `/a/*`) — only the encoded filter must be a
|
|
137
|
+
legal subject filter. A wildcard watch under a *non*-filterable codec is still
|
|
138
|
+
refused (`ConfigError`): encoding the whole key would mangle the `*`/`>`.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.10,<0.11"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "natsio-kvcodec"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "natsio extension: transparent KV key/value codecs (base64, path, zlib, chaining) over the core codec seam"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "Apache-2.0"
|
|
11
|
+
requires-python = ">=3.13"
|
|
12
|
+
authors = [{ name = "Corrupt Mane", email = "corruptmane@gmail.com" }]
|
|
13
|
+
keywords = ["nats", "nats.io", "jetstream", "key-value", "kv", "codec"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Framework :: AsyncIO",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
19
|
+
"Programming Language :: Python :: 3.13",
|
|
20
|
+
"Typing :: Typed",
|
|
21
|
+
]
|
|
22
|
+
dependencies = ["natsio"]
|
|
23
|
+
|
|
24
|
+
[tool.uv.sources]
|
|
25
|
+
natsio = { workspace = true }
|
|
26
|
+
|
|
27
|
+
[tool.uv.build-backend]
|
|
28
|
+
module-name = "natsio.kvcodec"
|
|
29
|
+
namespace = true
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""natsio.kvcodec — KV key/value codecs over the core's codec seam.
|
|
2
|
+
|
|
3
|
+
Ready-made :class:`natsio.kv.KeyCodec` / :class:`natsio.kv.ValueCodec`
|
|
4
|
+
implementations, mirroring `orbit.go/kvcodec
|
|
5
|
+
<https://github.com/synadia-io/orbit.go/tree/main/kvcodec>`_, for
|
|
6
|
+
:func:`natsio.jetstream.JetStreamContext.create_key_value`'s ``key_codec=`` /
|
|
7
|
+
``value_codec=`` arguments::
|
|
8
|
+
|
|
9
|
+
from natsio.kv import KeyValueConfig
|
|
10
|
+
from natsio.kvcodec import PathKeyCodec, ZlibValueCodec
|
|
11
|
+
|
|
12
|
+
kv = await js.create_key_value(
|
|
13
|
+
KeyValueConfig(bucket="cfg"),
|
|
14
|
+
key_codec=PathKeyCodec(), # "/app/db" stored as "app.db"
|
|
15
|
+
value_codec=ZlibValueCodec(), # values compressed at rest
|
|
16
|
+
)
|
|
17
|
+
await kv.put("/app/db/url", b"postgres://...")
|
|
18
|
+
(await kv.get("/app/db/url")).value # transparently decoded
|
|
19
|
+
|
|
20
|
+
Codecs are transparent: keys/values are encoded on write and decoded on read,
|
|
21
|
+
including the keys returned by ``keys()``/``iter_keys()`` and the entries from
|
|
22
|
+
``watch()``/``history()``.
|
|
23
|
+
|
|
24
|
+
See the package README for the mapping to orbit.go codecs and the "Core
|
|
25
|
+
friction" notes (raw-key pre-validation, and wildcard watches under a key
|
|
26
|
+
codec).
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from .errors import (
|
|
30
|
+
KeyDecodeError,
|
|
31
|
+
KeyEncodeError,
|
|
32
|
+
KvCodecError,
|
|
33
|
+
NoCodecsError,
|
|
34
|
+
ValueDecodeError,
|
|
35
|
+
ValueEncodeError,
|
|
36
|
+
WildcardNotSupportedError,
|
|
37
|
+
)
|
|
38
|
+
from .key_codecs import (
|
|
39
|
+
Base64KeyCodec,
|
|
40
|
+
ChainKeyCodec,
|
|
41
|
+
FilterableKeyCodec,
|
|
42
|
+
NoOpKeyCodec,
|
|
43
|
+
PathKeyCodec,
|
|
44
|
+
)
|
|
45
|
+
from .value_codecs import (
|
|
46
|
+
Base64ValueCodec,
|
|
47
|
+
ChainValueCodec,
|
|
48
|
+
NoOpValueCodec,
|
|
49
|
+
ZlibValueCodec,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
__all__ = [
|
|
53
|
+
"Base64KeyCodec",
|
|
54
|
+
"Base64ValueCodec",
|
|
55
|
+
"ChainKeyCodec",
|
|
56
|
+
"ChainValueCodec",
|
|
57
|
+
"FilterableKeyCodec",
|
|
58
|
+
"KeyDecodeError",
|
|
59
|
+
"KeyEncodeError",
|
|
60
|
+
"KvCodecError",
|
|
61
|
+
"NoCodecsError",
|
|
62
|
+
"NoOpKeyCodec",
|
|
63
|
+
"NoOpValueCodec",
|
|
64
|
+
"PathKeyCodec",
|
|
65
|
+
"ValueDecodeError",
|
|
66
|
+
"ValueEncodeError",
|
|
67
|
+
"WildcardNotSupportedError",
|
|
68
|
+
"ZlibValueCodec",
|
|
69
|
+
]
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Typed errors for kvcodec.
|
|
2
|
+
|
|
3
|
+
Codecs fail *loud*: a corrupt or non-encoded input raises rather than silently
|
|
4
|
+
passing through. This mirrors orbit.go/kvcodec (which returns an ``error`` from
|
|
5
|
+
every ``Decode*``) and keeps a bucket that somehow accumulated foreign/garbage
|
|
6
|
+
data from being misread as if it round-tripped cleanly.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"KeyDecodeError",
|
|
11
|
+
"KeyEncodeError",
|
|
12
|
+
"KvCodecError",
|
|
13
|
+
"NoCodecsError",
|
|
14
|
+
"ValueDecodeError",
|
|
15
|
+
"ValueEncodeError",
|
|
16
|
+
"WildcardNotSupportedError",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class KvCodecError(Exception):
|
|
21
|
+
"""Base class for every kvcodec failure."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class NoCodecsError(KvCodecError, ValueError):
|
|
25
|
+
"""A chain codec was constructed with zero codecs (orbit ``ErrNoCodecs``)."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class KeyEncodeError(KvCodecError):
|
|
29
|
+
"""A key codec could not encode a key (orbit ``ErrKeyEncodeFailed``)."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class KeyDecodeError(KvCodecError):
|
|
33
|
+
"""A key codec could not decode a stored key (orbit ``ErrKeyDecodeFailed``).
|
|
34
|
+
|
|
35
|
+
Raised on corrupt/non-encoded key tokens — e.g. a base64 token that is not
|
|
36
|
+
valid base64.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class ValueEncodeError(KvCodecError):
|
|
41
|
+
"""A value codec could not encode a value (orbit ``ErrValueEncodeFailed``)."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ValueDecodeError(KvCodecError):
|
|
45
|
+
"""A value codec could not decode a stored value (orbit ``ErrValueDecodeFailed``).
|
|
46
|
+
|
|
47
|
+
Raised on corrupt/non-encoded payloads — e.g. a zlib stream with a bad
|
|
48
|
+
checksum, or non-base64 bytes.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class WildcardNotSupportedError(KvCodecError):
|
|
53
|
+
"""A filter with ``*``/``>`` was handed to a codec that cannot preserve them
|
|
54
|
+
(orbit ``ErrWildcardNotSupported``).
|
|
55
|
+
|
|
56
|
+
NOTE: the natsio core does not yet call ``encode_filter`` — it refuses
|
|
57
|
+
wildcard watches under any key codec outright (see the extension README's
|
|
58
|
+
"Core friction" section). This error exists for codecs that *do* implement
|
|
59
|
+
:class:`~natsio.kvcodec.FilterableKeyCodec`, and for the day the seam grows
|
|
60
|
+
a filter hook.
|
|
61
|
+
"""
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""Key codecs — bidirectional ``str -> str`` transforms over the core's
|
|
2
|
+
:class:`natsio.kv.KeyCodec` seam.
|
|
3
|
+
|
|
4
|
+
Every codec here guarantees that the *encoded* form of any NATS-valid key is
|
|
5
|
+
itself a NATS-valid key (it satisfies :func:`natsio.kv.validate_key`), so the
|
|
6
|
+
core accepts it as a subject token. See the extension README for the one place
|
|
7
|
+
this is not enough — the core validates the *raw* key too, which blocks
|
|
8
|
+
Base64's escape-exotic-characters use case.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import base64
|
|
12
|
+
import binascii
|
|
13
|
+
from typing import cast
|
|
14
|
+
|
|
15
|
+
from natsio.kv import FilterableKeyCodec as CoreFilterableKeyCodec
|
|
16
|
+
from natsio.kv import KeyCodec
|
|
17
|
+
|
|
18
|
+
from .errors import (
|
|
19
|
+
KeyDecodeError,
|
|
20
|
+
NoCodecsError,
|
|
21
|
+
WildcardNotSupportedError,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"Base64KeyCodec",
|
|
26
|
+
"ChainKeyCodec",
|
|
27
|
+
"FilterableKeyCodec",
|
|
28
|
+
"NoOpKeyCodec",
|
|
29
|
+
"PathKeyCodec",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
# Subject wildcards. A whole-token ``*`` matches one token; a final ``>``
|
|
33
|
+
# matches the rest. A filter is a key that may contain these; encode_filter
|
|
34
|
+
# must pass them through untouched.
|
|
35
|
+
_WILDCARDS = frozenset({"*", ">"})
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class FilterableKeyCodec(CoreFilterableKeyCodec):
|
|
39
|
+
"""Marker/base for key codecs that can encode a *filter* (a key that may
|
|
40
|
+
contain ``*``/``>`` wildcards) while preserving the wildcards.
|
|
41
|
+
|
|
42
|
+
Explicitly implements the core's runtime-checkable
|
|
43
|
+
:class:`natsio.kv.FilterableKeyCodec` protocol, so the core's ``watch()``
|
|
44
|
+
recognises these codecs (via ``isinstance``) and encodes wildcard filters
|
|
45
|
+
per token instead of refusing them. Mirrors orbit.go's optional
|
|
46
|
+
``FilterableKeyCodec`` interface. A codec is "filterable" iff it provides
|
|
47
|
+
``encode_filter``; :class:`ChainKeyCodec` is filterable only when *every*
|
|
48
|
+
member is.
|
|
49
|
+
|
|
50
|
+
Subclassing is not required — the core protocol is structural — but
|
|
51
|
+
subclassing documents intent and gives you the :meth:`is_filterable`
|
|
52
|
+
helper.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
@staticmethod
|
|
56
|
+
def is_filterable(codec: object) -> bool:
|
|
57
|
+
return isinstance(codec, CoreFilterableKeyCodec)
|
|
58
|
+
|
|
59
|
+
def encode_filter(self, key_filter: str) -> str:
|
|
60
|
+
# Overridden by every concrete filterable codec here; the default keeps
|
|
61
|
+
# the type visible to callers that ``cast`` a checked member.
|
|
62
|
+
raise NotImplementedError
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class NoOpKeyCodec(FilterableKeyCodec):
|
|
66
|
+
"""Identity key codec (orbit ``NoOpCodec``).
|
|
67
|
+
|
|
68
|
+
The core already treats ``key_codec=None`` as identity, so this is only
|
|
69
|
+
useful as an explicit filler inside a :class:`ChainKeyCodec` or when a
|
|
70
|
+
non-optional codec argument is wanted.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
def encode(self, key: str) -> str:
|
|
74
|
+
return key
|
|
75
|
+
|
|
76
|
+
def decode(self, key: str) -> str:
|
|
77
|
+
return key
|
|
78
|
+
|
|
79
|
+
def encode_filter(self, key_filter: str) -> str:
|
|
80
|
+
return key_filter
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _b64_encode_token(token: str) -> str:
|
|
84
|
+
# Raw URL-safe base64 (no ``=`` padding), matching orbit.go's
|
|
85
|
+
# base64.RawURLEncoding. The alphabet ``A-Za-z0-9-_`` is a subset of what a
|
|
86
|
+
# NATS key token allows, so the result is always a valid token.
|
|
87
|
+
return base64.urlsafe_b64encode(token.encode()).rstrip(b"=").decode("ascii")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _b64_decode_token(token: str) -> str:
|
|
91
|
+
padded = token + "=" * (-len(token) % 4)
|
|
92
|
+
try:
|
|
93
|
+
return base64.urlsafe_b64decode(padded).decode("utf-8")
|
|
94
|
+
except (binascii.Error, ValueError) as exc:
|
|
95
|
+
raise KeyDecodeError(f"invalid base64 key token {token!r}: {exc}") from exc
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class Base64KeyCodec(FilterableKeyCodec):
|
|
99
|
+
"""Per-token URL-safe base64 (orbit ``Base64Codec`` key side).
|
|
100
|
+
|
|
101
|
+
The key is split on ``.`` and each token is base64-encoded independently, so
|
|
102
|
+
dots keep their meaning as subject-token separators and wildcard filters
|
|
103
|
+
stay per-token (:meth:`encode_filter`). Because the raw URL-safe alphabet is
|
|
104
|
+
a subset of the NATS key alphabet, encoding a NATS-valid key yields a
|
|
105
|
+
NATS-valid key.
|
|
106
|
+
|
|
107
|
+
Its *purpose* — per orbit — is to let keys carry characters that are not
|
|
108
|
+
valid in NATS subjects (spaces, ``@``, ``:`` ...). The codec does that
|
|
109
|
+
correctly, BUT the natsio core validates the *raw* key before any codec
|
|
110
|
+
runs, so such keys are rejected before this codec sees them. See the
|
|
111
|
+
README's "Core friction" section.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
def encode(self, key: str) -> str:
|
|
115
|
+
return ".".join(_b64_encode_token(t) for t in key.split("."))
|
|
116
|
+
|
|
117
|
+
def decode(self, key: str) -> str:
|
|
118
|
+
return ".".join(_b64_decode_token(t) for t in key.split("."))
|
|
119
|
+
|
|
120
|
+
def encode_filter(self, key_filter: str) -> str:
|
|
121
|
+
return ".".join(t if t in _WILDCARDS else _b64_encode_token(t) for t in key_filter.split("."))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# Path keys that start with ``/`` are stored with this sentinel first token,
|
|
125
|
+
# because a NATS subject cannot start with ``.`` (which is what a bare leading
|
|
126
|
+
# ``/`` would otherwise become). Matches orbit.go's ``_root_``.
|
|
127
|
+
_ROOT_PREFIX = "_root_"
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class PathKeyCodec(FilterableKeyCodec):
|
|
131
|
+
"""Filesystem-style keys ``/a/b/c`` <-> NATS ``a.b.c`` (orbit ``PathCodec``).
|
|
132
|
+
|
|
133
|
+
Rules (mirrored from orbit.go, verified against its test vectors):
|
|
134
|
+
|
|
135
|
+
* a leading ``/`` becomes the ``_root_`` sentinel token (``/`` alone ->
|
|
136
|
+
``_root_``); this keeps the leading-slash distinction round-trippable
|
|
137
|
+
without producing a subject that starts with ``.``;
|
|
138
|
+
* a trailing ``/`` is trimmed (subjects cannot end with ``.``);
|
|
139
|
+
* remaining ``/`` become ``.``.
|
|
140
|
+
|
|
141
|
+
Key-only: values pass through untouched, so pair it with a value codec (or
|
|
142
|
+
none) via :func:`natsio.jetstream.JetStreamContext.create_key_value`'s
|
|
143
|
+
separate ``value_codec=``.
|
|
144
|
+
|
|
145
|
+
Caveat (shared with orbit.go): the codec maps ``/`` <-> ``.`` and cannot
|
|
146
|
+
tell a *literal* ``.`` in the input from a separator, so feed it path-style
|
|
147
|
+
keys (separators are ``/``). A dotted input like ``a.b`` decodes back as
|
|
148
|
+
``a/b`` — it does not round-trip.
|
|
149
|
+
|
|
150
|
+
Edge case worth knowing: an *empty* path segment (``/a//b``) encodes to a
|
|
151
|
+
key with an empty token (``_root_.a..b``), which is NOT a valid NATS key —
|
|
152
|
+
the core's post-encode ``validate_key`` rejects it with ``InvalidKeyError``.
|
|
153
|
+
orbit.go has the same latent output; natsio just catches it at the seam.
|
|
154
|
+
"""
|
|
155
|
+
|
|
156
|
+
def encode(self, key: str) -> str:
|
|
157
|
+
if key.startswith("/"):
|
|
158
|
+
if key == "/":
|
|
159
|
+
return _ROOT_PREFIX
|
|
160
|
+
key = _ROOT_PREFIX + "." + key[1:]
|
|
161
|
+
key = key.removesuffix("/")
|
|
162
|
+
return key.replace("/", ".")
|
|
163
|
+
|
|
164
|
+
def decode(self, key: str) -> str:
|
|
165
|
+
if key == _ROOT_PREFIX:
|
|
166
|
+
return "/"
|
|
167
|
+
prefix = _ROOT_PREFIX + "."
|
|
168
|
+
if key.startswith(prefix):
|
|
169
|
+
return "/" + key[len(prefix) :].replace(".", "/")
|
|
170
|
+
return key.replace(".", "/")
|
|
171
|
+
|
|
172
|
+
def encode_filter(self, key_filter: str) -> str:
|
|
173
|
+
# PathCodec's encode already leaves ``*``/``>`` alone (they contain no
|
|
174
|
+
# ``/``), so filter-encoding is just encode. Matches orbit.go.
|
|
175
|
+
return self.encode(key_filter)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class ChainKeyCodec(FilterableKeyCodec):
|
|
179
|
+
"""Apply several key codecs in sequence (orbit ``KeyChainCodec``).
|
|
180
|
+
|
|
181
|
+
Encoding runs the codecs first-to-last; decoding runs them last-to-first, so
|
|
182
|
+
``decode(encode(k)) == k``. Construct with at least one codec, else
|
|
183
|
+
:class:`~natsio.kvcodec.NoCodecsError`.
|
|
184
|
+
|
|
185
|
+
Example — path notation *then* base64-escape the segments::
|
|
186
|
+
|
|
187
|
+
ChainKeyCodec(PathKeyCodec(), Base64KeyCodec())
|
|
188
|
+
|
|
189
|
+
:meth:`encode_filter` is available only when *every* member is filterable;
|
|
190
|
+
otherwise it raises :class:`~natsio.kvcodec.WildcardNotSupportedError`
|
|
191
|
+
naming the offending position (a chain cannot promise wildcard preservation
|
|
192
|
+
if any link would mangle a ``*``/``>``).
|
|
193
|
+
"""
|
|
194
|
+
|
|
195
|
+
__slots__ = ("_codecs",)
|
|
196
|
+
|
|
197
|
+
def __init__(self, *codecs: KeyCodec) -> None:
|
|
198
|
+
if not codecs:
|
|
199
|
+
raise NoCodecsError("at least one codec must be provided")
|
|
200
|
+
self._codecs: tuple[KeyCodec, ...] = codecs
|
|
201
|
+
|
|
202
|
+
def encode(self, key: str) -> str:
|
|
203
|
+
for codec in self._codecs:
|
|
204
|
+
key = codec.encode(key)
|
|
205
|
+
return key
|
|
206
|
+
|
|
207
|
+
def decode(self, key: str) -> str:
|
|
208
|
+
for codec in reversed(self._codecs):
|
|
209
|
+
key = codec.decode(key)
|
|
210
|
+
return key
|
|
211
|
+
|
|
212
|
+
def encode_filter(self, key_filter: str) -> str:
|
|
213
|
+
for index, codec in enumerate(self._codecs):
|
|
214
|
+
if not FilterableKeyCodec.is_filterable(codec):
|
|
215
|
+
raise WildcardNotSupportedError(
|
|
216
|
+
f"codec {index} ({type(codec).__name__}) does not support wildcard filtering"
|
|
217
|
+
)
|
|
218
|
+
for codec in self._codecs:
|
|
219
|
+
key_filter = cast("FilterableKeyCodec", codec).encode_filter(key_filter)
|
|
220
|
+
return key_filter
|
|
File without changes
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Value codecs — bidirectional ``bytes -> bytes`` transforms over the core's
|
|
2
|
+
:class:`natsio.kv.ValueCodec` seam.
|
|
3
|
+
|
|
4
|
+
Values are never validated by the core (only keys are), so a value codec may
|
|
5
|
+
produce arbitrary bytes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import base64
|
|
9
|
+
import binascii
|
|
10
|
+
import zlib
|
|
11
|
+
|
|
12
|
+
from natsio.kv import ValueCodec
|
|
13
|
+
|
|
14
|
+
from .errors import NoCodecsError, ValueDecodeError
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"Base64ValueCodec",
|
|
18
|
+
"ChainValueCodec",
|
|
19
|
+
"NoOpValueCodec",
|
|
20
|
+
"ZlibValueCodec",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class NoOpValueCodec:
|
|
25
|
+
"""Identity value codec (orbit ``NoOpCodec``, value side).
|
|
26
|
+
|
|
27
|
+
``value_codec=None`` already means identity in the core; use this only as an
|
|
28
|
+
explicit filler in a :class:`ChainValueCodec`.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def encode(self, value: bytes) -> bytes:
|
|
32
|
+
return value
|
|
33
|
+
|
|
34
|
+
def decode(self, value: bytes) -> bytes:
|
|
35
|
+
return value
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Base64ValueCodec:
|
|
39
|
+
"""Whole-value raw URL-safe base64 (orbit ``Base64Codec`` value side).
|
|
40
|
+
|
|
41
|
+
Rarely useful on its own for a KV value (it only inflates size by ~33% and
|
|
42
|
+
adds no safety, since values are opaque bytes to NATS), but it mirrors
|
|
43
|
+
orbit.go and is handy as a chain link or for producing ASCII-safe payloads.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def encode(self, value: bytes) -> bytes:
|
|
47
|
+
return base64.urlsafe_b64encode(value).rstrip(b"=")
|
|
48
|
+
|
|
49
|
+
def decode(self, value: bytes) -> bytes:
|
|
50
|
+
padded = value + b"=" * (-len(value) % 4)
|
|
51
|
+
try:
|
|
52
|
+
return base64.urlsafe_b64decode(padded)
|
|
53
|
+
except (binascii.Error, ValueError) as exc:
|
|
54
|
+
raise ValueDecodeError(f"invalid base64 value: {exc}") from exc
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class ZlibValueCodec:
|
|
58
|
+
"""Transparent DEFLATE compression via stdlib :mod:`zlib`.
|
|
59
|
+
|
|
60
|
+
Not in orbit.go (which ships only base64 for values) — a natsio judgment
|
|
61
|
+
call, since a value codec is the natural home for compression.
|
|
62
|
+
|
|
63
|
+
Tradeoffs, so you use it deliberately:
|
|
64
|
+
|
|
65
|
+
* **CPU for bytes.** Every put compresses and every get decompresses. For
|
|
66
|
+
hot, tiny values that overhead can dwarf the savings.
|
|
67
|
+
* **Small values grow.** zlib adds a ~2-byte header + 4-byte checksum, so
|
|
68
|
+
short/incompressible payloads come out *larger*. Compression is a win for
|
|
69
|
+
larger, repetitive values (JSON, text, logs).
|
|
70
|
+
* **Not encryption.** It hides nothing; it only shrinks.
|
|
71
|
+
* **Server-side compression exists.** ``KeyValueConfig(compression=True)``
|
|
72
|
+
compresses the whole stream at rest (S2), transparently and without client
|
|
73
|
+
CPU. Prefer that unless you specifically want the bytes compressed *over
|
|
74
|
+
the wire* and in ``max_bytes`` accounting — which is what this codec buys
|
|
75
|
+
you.
|
|
76
|
+
|
|
77
|
+
``level`` is the standard zlib 0-9 (default ``-1`` = library default ~6).
|
|
78
|
+
A corrupt/non-zlib payload raises :class:`~natsio.kvcodec.ValueDecodeError`.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
__slots__ = ("_level",)
|
|
82
|
+
|
|
83
|
+
def __init__(self, level: int = -1) -> None:
|
|
84
|
+
if not (-1 <= level <= 9):
|
|
85
|
+
raise ValueError(f"zlib level must be between -1 and 9, got {level}")
|
|
86
|
+
self._level = level
|
|
87
|
+
|
|
88
|
+
def encode(self, value: bytes) -> bytes:
|
|
89
|
+
return zlib.compress(value, self._level)
|
|
90
|
+
|
|
91
|
+
def decode(self, value: bytes) -> bytes:
|
|
92
|
+
try:
|
|
93
|
+
return zlib.decompress(value)
|
|
94
|
+
except zlib.error as exc:
|
|
95
|
+
raise ValueDecodeError(f"invalid zlib value: {exc}") from exc
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class ChainValueCodec:
|
|
99
|
+
"""Apply several value codecs in sequence (orbit ``ValueChainCodec``).
|
|
100
|
+
|
|
101
|
+
Encoding runs first-to-last, decoding last-to-first. Construct with at least
|
|
102
|
+
one codec, else :class:`~natsio.kvcodec.NoCodecsError`.
|
|
103
|
+
|
|
104
|
+
Order matters: ``ChainValueCodec(ZlibValueCodec(), Base64ValueCodec())``
|
|
105
|
+
compresses *then* base64s (ASCII-safe compressed payload); the reverse would
|
|
106
|
+
base64 first and compress incompressible base64 output — usually pointless.
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
__slots__ = ("_codecs",)
|
|
110
|
+
|
|
111
|
+
def __init__(self, *codecs: ValueCodec) -> None:
|
|
112
|
+
if not codecs:
|
|
113
|
+
raise NoCodecsError("at least one codec must be provided")
|
|
114
|
+
self._codecs: tuple[ValueCodec, ...] = codecs
|
|
115
|
+
|
|
116
|
+
def encode(self, value: bytes) -> bytes:
|
|
117
|
+
for codec in self._codecs:
|
|
118
|
+
value = codec.encode(value)
|
|
119
|
+
return value
|
|
120
|
+
|
|
121
|
+
def decode(self, value: bytes) -> bytes:
|
|
122
|
+
for codec in reversed(self._codecs):
|
|
123
|
+
value = codec.decode(value)
|
|
124
|
+
return value
|