natsio-counters 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- natsio_counters-0.1.0/PKG-INFO +110 -0
- natsio_counters-0.1.0/README.md +89 -0
- natsio_counters-0.1.0/pyproject.toml +34 -0
- natsio_counters-0.1.0/src/natsio/counters/__init__.py +61 -0
- natsio_counters-0.1.0/src/natsio/counters/counter.py +183 -0
- natsio_counters-0.1.0/src/natsio/counters/entities.py +143 -0
- natsio_counters-0.1.0/src/natsio/counters/errors.py +39 -0
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: natsio-counters
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: natsio extension: distributed counters (ADR-49) over JetStream counter streams
|
|
5
|
+
Keywords: nats,nats.io,jetstream,counters,distributed-counter,adr-49
|
|
6
|
+
Author: Corrupt Mane
|
|
7
|
+
Author-email: Corrupt Mane <corruptmane@gmail.com>
|
|
8
|
+
License-Expression: Apache-2.0
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Framework :: AsyncIO
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Dist: natsio
|
|
16
|
+
Requires-Python: >=3.13
|
|
17
|
+
Project-URL: Homepage, https://github.com/corruptmane/natsio
|
|
18
|
+
Project-URL: Repository, https://github.com/corruptmane/natsio
|
|
19
|
+
Project-URL: Changelog, https://github.com/corruptmane/natsio/blob/main/extensions/natsio-counters/CHANGELOG.md
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# natsio-counters
|
|
23
|
+
|
|
24
|
+
Distributed counters over JetStream counter streams
|
|
25
|
+
([ADR-49](https://github.com/nats-io/nats-architecture-and-design/blob/main/adr/ADR-49.md)),
|
|
26
|
+
mirroring [`orbit.go/counters`](https://github.com/synadia-io/orbit.go/tree/main/counters).
|
|
27
|
+
Distribution `natsio-counters`, imported as `natsio.counters`. Zero runtime
|
|
28
|
+
dependencies beyond `natsio`. Pre-1.0, no API-stability promises.
|
|
29
|
+
|
|
30
|
+
Requires **nats-server 2.12+** (the `allow_msg_counter` stream feature).
|
|
31
|
+
|
|
32
|
+
## What it is
|
|
33
|
+
|
|
34
|
+
A *counter stream* is a normal JetStream stream created with
|
|
35
|
+
`allow_msg_counter`. Every subject the stream captures becomes an independent,
|
|
36
|
+
**arbitrary-precision** counter. Incrementing is an ordinary publish carrying a
|
|
37
|
+
`Nats-Incr` header; the server folds the delta into a running total, stores it
|
|
38
|
+
as the subject's latest message, and returns the new value in the PubAck — so a
|
|
39
|
+
bare `add` needs no follow-up read.
|
|
40
|
+
|
|
41
|
+
Python `int` is itself unbounded, so it is used directly for both deltas and
|
|
42
|
+
values — no big-integer wrapper.
|
|
43
|
+
|
|
44
|
+
## Usage
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
import natsio
|
|
48
|
+
from natsio.counters import CounterConfig, create_counter, get_counter
|
|
49
|
+
|
|
50
|
+
nc = await natsio.connect("nats://localhost")
|
|
51
|
+
js = nc.jetstream()
|
|
52
|
+
|
|
53
|
+
# Create a counter stream (allow_msg_counter + allow_direct are forced on).
|
|
54
|
+
counter = await create_counter(js, CounterConfig(name="COUNTS", subjects=["events.>"]))
|
|
55
|
+
# ...or bind an existing one: counter = await get_counter(js, "COUNTS")
|
|
56
|
+
|
|
57
|
+
# Increment / decrement — returns the new running total, straight from the PubAck.
|
|
58
|
+
await counter.add("events.orders", 1) # -> 1
|
|
59
|
+
await counter.add("events.orders", 10) # -> 11
|
|
60
|
+
await counter.add("events.orders", -4) # negative deltas are fine -> 7
|
|
61
|
+
await counter.add("events.orders", 0) # zero is a valid no-op increment
|
|
62
|
+
|
|
63
|
+
# Read a single counter's current value.
|
|
64
|
+
value = await counter.load("events.orders") # -> 7
|
|
65
|
+
|
|
66
|
+
# Full entry: value + most-recent increment + source history (aggregation only).
|
|
67
|
+
entry = await counter.get("events.orders")
|
|
68
|
+
print(entry.value, entry.incr, entry.sources)
|
|
69
|
+
|
|
70
|
+
# Enumerate many counters in one batch Direct Get (wildcards allowed).
|
|
71
|
+
async for entry in counter.get_multiple(["events.>"]):
|
|
72
|
+
print(entry.subject, entry.value)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## API
|
|
76
|
+
|
|
77
|
+
| Symbol | Purpose |
|
|
78
|
+
|---|---|
|
|
79
|
+
| `create_counter(js, config)` | Create a counter stream and return its `Counter`. |
|
|
80
|
+
| `get_counter(js, name)` | Bind an existing counter stream by name. |
|
|
81
|
+
| `counter_from_stream(js, stream)` | Wrap an already-fetched counter `Stream` (no round-trip). |
|
|
82
|
+
| `Counter.add(subject, delta) -> int` | Increment/decrement; returns the new total. |
|
|
83
|
+
| `Counter.load(subject) -> int` | Current value of one subject's counter. |
|
|
84
|
+
| `Counter.get(subject) -> CounterEntry` | Value + last increment + sources. |
|
|
85
|
+
| `Counter.get_multiple(subjects) -> AsyncIterator[CounterEntry]` | Batch/wildcard enumeration. |
|
|
86
|
+
| `CounterConfig` | Counter-stream configuration. |
|
|
87
|
+
| `CounterEntry` | `subject`, `value`, `sources`, `incr`. |
|
|
88
|
+
|
|
89
|
+
Errors (all subclass `natsio.jetstream.JetStreamError`): `CounterNotEnabledError`,
|
|
90
|
+
`DirectAccessRequiredError`, `CounterNotFoundError`,
|
|
91
|
+
`CounterSubjectNotInitializedError`, `InvalidCounterValueError`.
|
|
92
|
+
|
|
93
|
+
## ADR-49 wire contract
|
|
94
|
+
|
|
95
|
+
Pinned by the wire-contract tests and exported as constants:
|
|
96
|
+
|
|
97
|
+
- **Increment header** `Nats-Incr` (`COUNTER_INCREMENT_HEADER`) — a base-10
|
|
98
|
+
integer of any size, optional leading sign. A counter stream **rejects any
|
|
99
|
+
publish without it** (`message counter increment is missing`).
|
|
100
|
+
- **Sources header** `Nats-Counter-Sources` (`COUNTER_SOURCES_HEADER`) —
|
|
101
|
+
server-set JSON `{source_stream: {subject: contribution}}`; present only on
|
|
102
|
+
aggregating counters.
|
|
103
|
+
- **Stored payload** `{"val": "<int-as-string>"}` — the value is a *string* so
|
|
104
|
+
it survives languages whose JSON numbers cap at 2^53.
|
|
105
|
+
- **PubAck** carries the new total in its `val` field (fast feedback, no Get).
|
|
106
|
+
|
|
107
|
+
Notes: `allow_msg_counter` can be set **only at stream creation** (it is read-only
|
|
108
|
+
afterwards) and is **incompatible with per-message TTLs and message schedules**,
|
|
109
|
+
so `CounterConfig` intentionally does not expose those knobs. Reads use Direct
|
|
110
|
+
Get, so `allow_direct` is required (and forced on by `create_counter`).
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# natsio-counters
|
|
2
|
+
|
|
3
|
+
Distributed counters over JetStream counter streams
|
|
4
|
+
([ADR-49](https://github.com/nats-io/nats-architecture-and-design/blob/main/adr/ADR-49.md)),
|
|
5
|
+
mirroring [`orbit.go/counters`](https://github.com/synadia-io/orbit.go/tree/main/counters).
|
|
6
|
+
Distribution `natsio-counters`, imported as `natsio.counters`. Zero runtime
|
|
7
|
+
dependencies beyond `natsio`. Pre-1.0, no API-stability promises.
|
|
8
|
+
|
|
9
|
+
Requires **nats-server 2.12+** (the `allow_msg_counter` stream feature).
|
|
10
|
+
|
|
11
|
+
## What it is
|
|
12
|
+
|
|
13
|
+
A *counter stream* is a normal JetStream stream created with
|
|
14
|
+
`allow_msg_counter`. Every subject the stream captures becomes an independent,
|
|
15
|
+
**arbitrary-precision** counter. Incrementing is an ordinary publish carrying a
|
|
16
|
+
`Nats-Incr` header; the server folds the delta into a running total, stores it
|
|
17
|
+
as the subject's latest message, and returns the new value in the PubAck — so a
|
|
18
|
+
bare `add` needs no follow-up read.
|
|
19
|
+
|
|
20
|
+
Python `int` is itself unbounded, so it is used directly for both deltas and
|
|
21
|
+
values — no big-integer wrapper.
|
|
22
|
+
|
|
23
|
+
## Usage
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
import natsio
|
|
27
|
+
from natsio.counters import CounterConfig, create_counter, get_counter
|
|
28
|
+
|
|
29
|
+
nc = await natsio.connect("nats://localhost")
|
|
30
|
+
js = nc.jetstream()
|
|
31
|
+
|
|
32
|
+
# Create a counter stream (allow_msg_counter + allow_direct are forced on).
|
|
33
|
+
counter = await create_counter(js, CounterConfig(name="COUNTS", subjects=["events.>"]))
|
|
34
|
+
# ...or bind an existing one: counter = await get_counter(js, "COUNTS")
|
|
35
|
+
|
|
36
|
+
# Increment / decrement — returns the new running total, straight from the PubAck.
|
|
37
|
+
await counter.add("events.orders", 1) # -> 1
|
|
38
|
+
await counter.add("events.orders", 10) # -> 11
|
|
39
|
+
await counter.add("events.orders", -4) # negative deltas are fine -> 7
|
|
40
|
+
await counter.add("events.orders", 0) # zero is a valid no-op increment
|
|
41
|
+
|
|
42
|
+
# Read a single counter's current value.
|
|
43
|
+
value = await counter.load("events.orders") # -> 7
|
|
44
|
+
|
|
45
|
+
# Full entry: value + most-recent increment + source history (aggregation only).
|
|
46
|
+
entry = await counter.get("events.orders")
|
|
47
|
+
print(entry.value, entry.incr, entry.sources)
|
|
48
|
+
|
|
49
|
+
# Enumerate many counters in one batch Direct Get (wildcards allowed).
|
|
50
|
+
async for entry in counter.get_multiple(["events.>"]):
|
|
51
|
+
print(entry.subject, entry.value)
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## API
|
|
55
|
+
|
|
56
|
+
| Symbol | Purpose |
|
|
57
|
+
|---|---|
|
|
58
|
+
| `create_counter(js, config)` | Create a counter stream and return its `Counter`. |
|
|
59
|
+
| `get_counter(js, name)` | Bind an existing counter stream by name. |
|
|
60
|
+
| `counter_from_stream(js, stream)` | Wrap an already-fetched counter `Stream` (no round-trip). |
|
|
61
|
+
| `Counter.add(subject, delta) -> int` | Increment/decrement; returns the new total. |
|
|
62
|
+
| `Counter.load(subject) -> int` | Current value of one subject's counter. |
|
|
63
|
+
| `Counter.get(subject) -> CounterEntry` | Value + last increment + sources. |
|
|
64
|
+
| `Counter.get_multiple(subjects) -> AsyncIterator[CounterEntry]` | Batch/wildcard enumeration. |
|
|
65
|
+
| `CounterConfig` | Counter-stream configuration. |
|
|
66
|
+
| `CounterEntry` | `subject`, `value`, `sources`, `incr`. |
|
|
67
|
+
|
|
68
|
+
Errors (all subclass `natsio.jetstream.JetStreamError`): `CounterNotEnabledError`,
|
|
69
|
+
`DirectAccessRequiredError`, `CounterNotFoundError`,
|
|
70
|
+
`CounterSubjectNotInitializedError`, `InvalidCounterValueError`.
|
|
71
|
+
|
|
72
|
+
## ADR-49 wire contract
|
|
73
|
+
|
|
74
|
+
Pinned by the wire-contract tests and exported as constants:
|
|
75
|
+
|
|
76
|
+
- **Increment header** `Nats-Incr` (`COUNTER_INCREMENT_HEADER`) — a base-10
|
|
77
|
+
integer of any size, optional leading sign. A counter stream **rejects any
|
|
78
|
+
publish without it** (`message counter increment is missing`).
|
|
79
|
+
- **Sources header** `Nats-Counter-Sources` (`COUNTER_SOURCES_HEADER`) —
|
|
80
|
+
server-set JSON `{source_stream: {subject: contribution}}`; present only on
|
|
81
|
+
aggregating counters.
|
|
82
|
+
- **Stored payload** `{"val": "<int-as-string>"}` — the value is a *string* so
|
|
83
|
+
it survives languages whose JSON numbers cap at 2^53.
|
|
84
|
+
- **PubAck** carries the new total in its `val` field (fast feedback, no Get).
|
|
85
|
+
|
|
86
|
+
Notes: `allow_msg_counter` can be set **only at stream creation** (it is read-only
|
|
87
|
+
afterwards) and is **incompatible with per-message TTLs and message schedules**,
|
|
88
|
+
so `CounterConfig` intentionally does not expose those knobs. Reads use Direct
|
|
89
|
+
Get, so `allow_direct` is required (and forced on by `create_counter`).
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.10,<0.11"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "natsio-counters"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "natsio extension: distributed counters (ADR-49) over JetStream counter streams"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "Apache-2.0"
|
|
11
|
+
requires-python = ">=3.13"
|
|
12
|
+
authors = [{ name = "Corrupt Mane", email = "corruptmane@gmail.com" }]
|
|
13
|
+
keywords = ["nats", "nats.io", "jetstream", "counters", "distributed-counter", "adr-49"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Framework :: AsyncIO",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
19
|
+
"Programming Language :: Python :: 3.13",
|
|
20
|
+
"Typing :: Typed",
|
|
21
|
+
]
|
|
22
|
+
dependencies = ["natsio"]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/corruptmane/natsio"
|
|
26
|
+
Repository = "https://github.com/corruptmane/natsio"
|
|
27
|
+
Changelog = "https://github.com/corruptmane/natsio/blob/main/extensions/natsio-counters/CHANGELOG.md"
|
|
28
|
+
|
|
29
|
+
[tool.uv.sources]
|
|
30
|
+
natsio = { workspace = true }
|
|
31
|
+
|
|
32
|
+
[tool.uv.build-backend]
|
|
33
|
+
module-name = "natsio.counters"
|
|
34
|
+
namespace = true
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Distributed counters over JetStream counter streams (ADR-49).
|
|
2
|
+
|
|
3
|
+
A counter stream (``allow_msg_counter``) turns every subject it captures into
|
|
4
|
+
an independent, arbitrary-precision counter. Increments are ordinary publishes
|
|
5
|
+
carrying a ``Nats-Incr`` header; the server folds them into a running total and
|
|
6
|
+
returns the new value in the PubAck, so a bare ``add`` needs no follow-up read.
|
|
7
|
+
|
|
8
|
+
js = nc.jetstream()
|
|
9
|
+
counter = await create_counter(js, CounterConfig(name="COUNTS", subjects=["events.>"]))
|
|
10
|
+
|
|
11
|
+
total = await counter.add("events.orders", 1) # -> 1
|
|
12
|
+
total = await counter.add("events.orders", 10) # -> 11
|
|
13
|
+
total = await counter.add("inventory.widgets", -3) # negative deltas are fine
|
|
14
|
+
|
|
15
|
+
value = await counter.load("events.orders") # -> 11
|
|
16
|
+
entry = await counter.get("events.orders") # value + last increment + sources
|
|
17
|
+
|
|
18
|
+
async for entry in counter.get_multiple(["events.>"]):
|
|
19
|
+
print(entry.subject, entry.value)
|
|
20
|
+
|
|
21
|
+
Requires server 2.12+ (``allow_msg_counter``). ``allow_msg_counter`` can only
|
|
22
|
+
be set when the stream is created and is incompatible with per-message TTLs and
|
|
23
|
+
message schedules.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from natsio.counters.counter import Counter, counter_from_stream, create_counter, get_counter
|
|
27
|
+
from natsio.counters.entities import (
|
|
28
|
+
COUNTER_INCREMENT_HEADER,
|
|
29
|
+
COUNTER_SOURCES_HEADER,
|
|
30
|
+
CounterConfig,
|
|
31
|
+
CounterEntry,
|
|
32
|
+
CounterSources,
|
|
33
|
+
parse_counter_value,
|
|
34
|
+
parse_sources,
|
|
35
|
+
)
|
|
36
|
+
from natsio.counters.errors import (
|
|
37
|
+
CounterNotEnabledError,
|
|
38
|
+
CounterNotFoundError,
|
|
39
|
+
CounterSubjectNotInitializedError,
|
|
40
|
+
DirectAccessRequiredError,
|
|
41
|
+
InvalidCounterValueError,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
"COUNTER_INCREMENT_HEADER",
|
|
46
|
+
"COUNTER_SOURCES_HEADER",
|
|
47
|
+
"Counter",
|
|
48
|
+
"CounterConfig",
|
|
49
|
+
"CounterEntry",
|
|
50
|
+
"CounterNotEnabledError",
|
|
51
|
+
"CounterNotFoundError",
|
|
52
|
+
"CounterSources",
|
|
53
|
+
"CounterSubjectNotInitializedError",
|
|
54
|
+
"DirectAccessRequiredError",
|
|
55
|
+
"InvalidCounterValueError",
|
|
56
|
+
"counter_from_stream",
|
|
57
|
+
"create_counter",
|
|
58
|
+
"get_counter",
|
|
59
|
+
"parse_counter_value",
|
|
60
|
+
"parse_sources",
|
|
61
|
+
]
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""The Counter handle: increment, load, and enumerate ADR-49 counters."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import AsyncIterator
|
|
4
|
+
from typing import TYPE_CHECKING, Any
|
|
5
|
+
|
|
6
|
+
from natsio.jetstream.entities import DiscardPolicy, StorageCompression, StreamConfig
|
|
7
|
+
from natsio.jetstream.errors import MessageNotFoundError, StreamNotFoundError
|
|
8
|
+
from natsio.jetstream.stream import Stream
|
|
9
|
+
|
|
10
|
+
from .entities import (
|
|
11
|
+
COUNTER_INCREMENT_HEADER,
|
|
12
|
+
CounterConfig,
|
|
13
|
+
CounterEntry,
|
|
14
|
+
parse_counter_value,
|
|
15
|
+
parse_sources,
|
|
16
|
+
)
|
|
17
|
+
from .errors import (
|
|
18
|
+
CounterNotEnabledError,
|
|
19
|
+
CounterNotFoundError,
|
|
20
|
+
CounterSubjectNotInitializedError,
|
|
21
|
+
DirectAccessRequiredError,
|
|
22
|
+
InvalidCounterValueError,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from natsio.jetstream.context import JetStreamContext
|
|
27
|
+
|
|
28
|
+
__all__ = ["Counter", "counter_from_stream", "create_counter", "get_counter"]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Counter:
|
|
32
|
+
"""A handle to a JetStream stream configured for distributed counters.
|
|
33
|
+
|
|
34
|
+
Every subject the stream captures is an independent counter. Obtain via
|
|
35
|
+
`create_counter`, `get_counter`, or `counter_from_stream`.
|
|
36
|
+
|
|
37
|
+
Python ``int`` is arbitrary-precision, so it is used directly for both
|
|
38
|
+
deltas and values — no big-integer wrapper is needed to match the ADR-49
|
|
39
|
+
unbounded-integer semantics.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
__slots__ = ("_ctx", "_stream", "name")
|
|
43
|
+
|
|
44
|
+
def __init__(self, ctx: "JetStreamContext", stream: Stream) -> None:
|
|
45
|
+
config = stream.cached_info.config
|
|
46
|
+
if not config.allow_msg_counter:
|
|
47
|
+
raise CounterNotEnabledError(
|
|
48
|
+
f"stream {config.name!r} is not a counter stream (needs allow_msg_counter=True)"
|
|
49
|
+
)
|
|
50
|
+
if not config.allow_direct:
|
|
51
|
+
raise DirectAccessRequiredError(f"stream {config.name!r} needs allow_direct=True for counter reads")
|
|
52
|
+
self._ctx = ctx
|
|
53
|
+
self._stream = stream
|
|
54
|
+
self.name = config.name
|
|
55
|
+
|
|
56
|
+
def __repr__(self) -> str:
|
|
57
|
+
return f"Counter(name={self.name!r})"
|
|
58
|
+
|
|
59
|
+
# -- writes --------------------------------------------------------------
|
|
60
|
+
|
|
61
|
+
async def add(self, subject: str, delta: int) -> int:
|
|
62
|
+
"""Increment (or, with a negative ``delta``, decrement) ``subject``.
|
|
63
|
+
|
|
64
|
+
Returns the new running total, taken from the PubAck's ``val`` field —
|
|
65
|
+
no follow-up read. ``delta`` of ``0`` is a valid no-op increment that
|
|
66
|
+
still initializes the subject. The ``subject`` must be covered by the
|
|
67
|
+
stream's subjects, else `NoStreamResponseError`.
|
|
68
|
+
"""
|
|
69
|
+
if isinstance(delta, bool) or not isinstance(delta, int):
|
|
70
|
+
raise InvalidCounterValueError(f"counter delta must be an int, got {type(delta).__name__}")
|
|
71
|
+
ack = await self._ctx.publish(subject, b"", headers={COUNTER_INCREMENT_HEADER: str(delta)})
|
|
72
|
+
if not ack.val:
|
|
73
|
+
raise InvalidCounterValueError(f"counter increment for {subject!r} returned no value")
|
|
74
|
+
return int(ack.val, 10)
|
|
75
|
+
|
|
76
|
+
# -- reads ---------------------------------------------------------------
|
|
77
|
+
|
|
78
|
+
async def load(self, subject: str) -> int:
|
|
79
|
+
"""The current value of ``subject``'s counter.
|
|
80
|
+
|
|
81
|
+
Raises `CounterSubjectNotInitializedError` if nothing has been
|
|
82
|
+
added to ``subject`` yet.
|
|
83
|
+
"""
|
|
84
|
+
stored = await self._get_last(subject)
|
|
85
|
+
return parse_counter_value(stored.payload)
|
|
86
|
+
|
|
87
|
+
async def get(self, subject: str) -> CounterEntry:
|
|
88
|
+
"""The full `CounterEntry` for ``subject`` (value, sources, last increment).
|
|
89
|
+
|
|
90
|
+
Raises `CounterSubjectNotInitializedError` if nothing has been
|
|
91
|
+
added to ``subject`` yet.
|
|
92
|
+
"""
|
|
93
|
+
stored = await self._get_last(subject)
|
|
94
|
+
return _entry_from_parts(subject, stored.payload, stored.headers)
|
|
95
|
+
|
|
96
|
+
async def _get_last(self, subject: str) -> Any:
|
|
97
|
+
try:
|
|
98
|
+
return await self._stream.get_msg(subject=subject)
|
|
99
|
+
except MessageNotFoundError:
|
|
100
|
+
raise CounterSubjectNotInitializedError(f"counter not initialized for subject {subject!r}") from None
|
|
101
|
+
|
|
102
|
+
async def get_multiple(self, subjects: list[str]) -> AsyncIterator[CounterEntry]:
|
|
103
|
+
"""Yield a `CounterEntry` for each subject matching ``subjects``.
|
|
104
|
+
|
|
105
|
+
``subjects`` may contain wildcards (``*`` / ``>``); each matched subject
|
|
106
|
+
yields its current entry. Backed by a single batch Direct Get
|
|
107
|
+
(``Stream.get_last_msgs_for``, ``multi_last`` per ADR-31) rather than one
|
|
108
|
+
round-trip per subject.
|
|
109
|
+
"""
|
|
110
|
+
if not subjects:
|
|
111
|
+
return
|
|
112
|
+
async for stored in self._stream.get_last_msgs_for(subjects):
|
|
113
|
+
if not stored.subject:
|
|
114
|
+
continue
|
|
115
|
+
yield _entry_from_parts(stored.subject, stored.payload, stored.headers)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _entry_from_parts(subject: str, payload: bytes, headers: Any) -> CounterEntry:
|
|
119
|
+
value = parse_counter_value(payload)
|
|
120
|
+
sources = parse_sources(headers)
|
|
121
|
+
incr: int | None = None
|
|
122
|
+
if headers is not None:
|
|
123
|
+
raw_incr = headers.get(COUNTER_INCREMENT_HEADER)
|
|
124
|
+
if raw_incr:
|
|
125
|
+
try:
|
|
126
|
+
incr = int(raw_incr, 10)
|
|
127
|
+
except ValueError as exc:
|
|
128
|
+
raise InvalidCounterValueError(f"invalid {COUNTER_INCREMENT_HEADER} header: {raw_incr!r}") from exc
|
|
129
|
+
return CounterEntry(subject=subject, value=value, sources=sources, incr=incr)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _counter_stream_config(config: CounterConfig) -> StreamConfig:
|
|
133
|
+
"""Map a `CounterConfig` onto its ADR-49 backing stream."""
|
|
134
|
+
return StreamConfig(
|
|
135
|
+
name=config.name,
|
|
136
|
+
description=config.description,
|
|
137
|
+
subjects=list(config.subjects),
|
|
138
|
+
storage=config.storage,
|
|
139
|
+
num_replicas=config.replicas,
|
|
140
|
+
max_bytes=config.max_bytes,
|
|
141
|
+
max_msgs_per_subject=config.max_msgs_per_subject,
|
|
142
|
+
max_age=config.max_age,
|
|
143
|
+
placement=config.placement,
|
|
144
|
+
compression=StorageCompression.S2 if config.compression else None,
|
|
145
|
+
metadata=config.metadata,
|
|
146
|
+
discard=DiscardPolicy.OLD,
|
|
147
|
+
allow_msg_counter=True,
|
|
148
|
+
allow_direct=True,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def counter_from_stream(js: "JetStreamContext", stream: Stream) -> Counter:
|
|
153
|
+
"""Wrap an existing counter `Stream` as a `Counter` (no round-trip).
|
|
154
|
+
|
|
155
|
+
Raises `CounterNotEnabledError` /
|
|
156
|
+
`DirectAccessRequiredError` if the stream is not counter-capable.
|
|
157
|
+
"""
|
|
158
|
+
return Counter(js, stream)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
async def create_counter(js: "JetStreamContext", config: CounterConfig) -> Counter:
|
|
162
|
+
"""Create a counter stream from ``config`` and return its `Counter`.
|
|
163
|
+
|
|
164
|
+
Re-creating with an identical configuration is idempotent; an existing
|
|
165
|
+
stream with a different configuration surfaces as the mapped
|
|
166
|
+
`APIError` (``StreamNameInUseError``).
|
|
167
|
+
"""
|
|
168
|
+
stream = await js.create_stream(_counter_stream_config(config))
|
|
169
|
+
return Counter(js, stream)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
async def get_counter(js: "JetStreamContext", name: str) -> Counter:
|
|
173
|
+
"""Bind an existing counter stream by ``name``.
|
|
174
|
+
|
|
175
|
+
Raises `CounterNotFoundError` if no such stream exists, or
|
|
176
|
+
`CounterNotEnabledError` /
|
|
177
|
+
`DirectAccessRequiredError` if it exists but is not a counter.
|
|
178
|
+
"""
|
|
179
|
+
try:
|
|
180
|
+
stream = await js.stream(name)
|
|
181
|
+
except StreamNotFoundError:
|
|
182
|
+
raise CounterNotFoundError(f"no counter stream named {name!r}") from None
|
|
183
|
+
return Counter(js, stream)
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Counter data types and the ADR-49 wire contract (headers, payload).
|
|
2
|
+
|
|
3
|
+
The constants here are the pinned wire contract — kept as plain ``str`` (like
|
|
4
|
+
``natsio.jetstream.headers``) because they go straight onto the wire, and
|
|
5
|
+
asserted byte-for-byte against the oracle in the wire-contract tests.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from datetime import timedelta
|
|
11
|
+
from typing import Any, Final
|
|
12
|
+
|
|
13
|
+
from natsio._internal.protocol import Headers
|
|
14
|
+
from natsio.errors import ConfigError
|
|
15
|
+
from natsio.jetstream.entities import Placement, StorageType
|
|
16
|
+
|
|
17
|
+
from .errors import InvalidCounterValueError
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"COUNTER_INCREMENT_HEADER",
|
|
21
|
+
"COUNTER_SOURCES_HEADER",
|
|
22
|
+
"CounterConfig",
|
|
23
|
+
"CounterEntry",
|
|
24
|
+
"CounterSources",
|
|
25
|
+
"parse_counter_value",
|
|
26
|
+
"parse_sources",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
# -- ADR-49 wire contract ----------------------------------------------------
|
|
30
|
+
# Client-set on a publish to increment (or, with a leading '-', decrement) the
|
|
31
|
+
# per-subject counter. The value is a base-10 integer of arbitrary size.
|
|
32
|
+
COUNTER_INCREMENT_HEADER: Final = "Nats-Incr"
|
|
33
|
+
# Server-set on stored counter messages: a JSON object mapping each contributing
|
|
34
|
+
# source stream to its per-subject contributions (populated on aggregating
|
|
35
|
+
# streams that source from other counters). Absent on plain single-stream
|
|
36
|
+
# counters.
|
|
37
|
+
COUNTER_SOURCES_HEADER: Final = "Nats-Counter-Sources"
|
|
38
|
+
|
|
39
|
+
# The stored counter payload is ``{"val": "<base-10 integer as a string>"}``.
|
|
40
|
+
# The value is a *string* on the wire so it survives languages whose JSON
|
|
41
|
+
# numbers top out at 2**53; Python ``int`` is itself arbitrary-precision, so it
|
|
42
|
+
# is the natural in-memory type on both the read and the write side.
|
|
43
|
+
_PAYLOAD_VALUE_KEY: Final = "val"
|
|
44
|
+
|
|
45
|
+
type CounterSources = dict[str, dict[str, int]]
|
|
46
|
+
"""``{source_stream: {subject: contribution}}`` — see ``COUNTER_SOURCES_HEADER``."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True, slots=True)
|
|
50
|
+
class CounterEntry:
|
|
51
|
+
"""A counter's current state for one subject.
|
|
52
|
+
|
|
53
|
+
``value`` is the running total; ``incr`` is the most recent increment
|
|
54
|
+
(useful for auditing/recounting); ``sources`` is present only when the
|
|
55
|
+
counter aggregates other streams.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
subject: str
|
|
59
|
+
value: int
|
|
60
|
+
sources: CounterSources | None = None
|
|
61
|
+
incr: int | None = None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
65
|
+
class CounterConfig:
|
|
66
|
+
"""Configuration for a counter stream (ADR-49).
|
|
67
|
+
|
|
68
|
+
Maps onto a JetStream stream with ``allow_msg_counter`` (the counter
|
|
69
|
+
behaviour) and ``allow_direct`` (the read path) forced on. Every subject the
|
|
70
|
+
stream captures is an independent counter. ``allow_msg_counter`` is settable
|
|
71
|
+
only at creation and is incompatible with per-message TTLs and message
|
|
72
|
+
schedules, so those knobs are intentionally not exposed here.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
name: str
|
|
76
|
+
subjects: list[str]
|
|
77
|
+
description: str | None = None
|
|
78
|
+
storage: StorageType = StorageType.FILE
|
|
79
|
+
replicas: int = 1
|
|
80
|
+
max_bytes: int = -1
|
|
81
|
+
# Cap the retained increment history per subject (-1 = unlimited). The
|
|
82
|
+
# current value is always the last message per subject regardless.
|
|
83
|
+
max_msgs_per_subject: int = -1
|
|
84
|
+
# Age off old increments (None = never). Does not affect the current value
|
|
85
|
+
# unless a subject goes fully idle past this age.
|
|
86
|
+
max_age: timedelta | None = None
|
|
87
|
+
placement: Placement | None = None
|
|
88
|
+
compression: bool = False
|
|
89
|
+
metadata: dict[str, str] | None = None
|
|
90
|
+
|
|
91
|
+
def __post_init__(self) -> None:
|
|
92
|
+
if not self.subjects:
|
|
93
|
+
raise ConfigError("a counter stream needs at least one subject")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def parse_counter_value(data: bytes) -> int:
|
|
97
|
+
"""Parse a stored counter payload ``{"val": "<int>"}`` into an ``int``.
|
|
98
|
+
|
|
99
|
+
Raises `InvalidCounterValueError` on empty data, malformed JSON,
|
|
100
|
+
a missing ``val`` key, or a non-integer value.
|
|
101
|
+
"""
|
|
102
|
+
if not data:
|
|
103
|
+
raise InvalidCounterValueError("empty counter value")
|
|
104
|
+
try:
|
|
105
|
+
payload: dict[str, Any] = json.loads(data)
|
|
106
|
+
except (ValueError, TypeError) as exc:
|
|
107
|
+
raise InvalidCounterValueError(f"malformed counter payload: {exc}") from exc
|
|
108
|
+
if not isinstance(payload, dict) or _PAYLOAD_VALUE_KEY not in payload:
|
|
109
|
+
raise InvalidCounterValueError(f"counter payload missing {_PAYLOAD_VALUE_KEY!r} field")
|
|
110
|
+
raw = payload[_PAYLOAD_VALUE_KEY]
|
|
111
|
+
try:
|
|
112
|
+
# ``int(str, 10)`` semantics: accept the string form the server writes,
|
|
113
|
+
# reject floats/booleans/None that would slip past a bare ``int()``.
|
|
114
|
+
return int(str(raw), 10)
|
|
115
|
+
except (ValueError, TypeError) as exc:
|
|
116
|
+
raise InvalidCounterValueError(f"invalid counter value: {raw!r}") from exc
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def parse_sources(headers: Headers | None) -> CounterSources | None:
|
|
120
|
+
"""Parse the ``Nats-Counter-Sources`` header, or ``None`` when absent.
|
|
121
|
+
|
|
122
|
+
Raises `InvalidCounterValueError` on malformed JSON or a
|
|
123
|
+
non-integer contribution.
|
|
124
|
+
"""
|
|
125
|
+
if headers is None:
|
|
126
|
+
return None
|
|
127
|
+
raw = headers.get(COUNTER_SOURCES_HEADER)
|
|
128
|
+
if not raw:
|
|
129
|
+
return None
|
|
130
|
+
try:
|
|
131
|
+
decoded: dict[str, dict[str, str]] = json.loads(raw)
|
|
132
|
+
except (ValueError, TypeError) as exc:
|
|
133
|
+
raise InvalidCounterValueError(f"malformed {COUNTER_SOURCES_HEADER} header: {exc}") from exc
|
|
134
|
+
result: CounterSources = {}
|
|
135
|
+
for source_id, contributions in decoded.items():
|
|
136
|
+
parsed: dict[str, int] = {}
|
|
137
|
+
for subject, value in contributions.items():
|
|
138
|
+
try:
|
|
139
|
+
parsed[subject] = int(str(value), 10)
|
|
140
|
+
except (ValueError, TypeError) as exc:
|
|
141
|
+
raise InvalidCounterValueError(f"invalid source contribution for {subject!r}: {value!r}") from exc
|
|
142
|
+
result[source_id] = parsed
|
|
143
|
+
return result
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Counter error types (mirrors orbit.go ``counters/errors.go``)."""
|
|
2
|
+
|
|
3
|
+
from natsio.jetstream.errors import JetStreamError
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"CounterNotEnabledError",
|
|
7
|
+
"CounterNotFoundError",
|
|
8
|
+
"CounterSubjectNotInitializedError",
|
|
9
|
+
"DirectAccessRequiredError",
|
|
10
|
+
"InvalidCounterValueError",
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class CounterNotEnabledError(JetStreamError):
|
|
15
|
+
"""Wrapped a stream that lacks ``allow_msg_counter``.
|
|
16
|
+
|
|
17
|
+
``allow_msg_counter`` can only be set at stream creation (ADR-49); an
|
|
18
|
+
existing plain stream cannot be turned into a counter after the fact.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class DirectAccessRequiredError(JetStreamError):
|
|
23
|
+
"""Wrapped a stream that lacks ``allow_direct``.
|
|
24
|
+
|
|
25
|
+
Reads (``load``/``get``/``get_multiple``) go through Direct Get, so the
|
|
26
|
+
backing stream must have ``allow_direct=True``.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class InvalidCounterValueError(JetStreamError):
|
|
31
|
+
"""A counter delta or a stored counter payload was not a valid integer."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class CounterNotFoundError(JetStreamError):
|
|
35
|
+
"""No counter stream with that name exists."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class CounterSubjectNotInitializedError(JetStreamError):
|
|
39
|
+
"""The subject has no stored increment yet — its counter is uninitialized."""
|