Autarkia-Msgqueue 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- autarkia_msgqueue-0.1.0/.gitignore +11 -0
- autarkia_msgqueue-0.1.0/LICENSE +21 -0
- autarkia_msgqueue-0.1.0/PKG-INFO +242 -0
- autarkia_msgqueue-0.1.0/README.md +215 -0
- autarkia_msgqueue-0.1.0/pyproject.toml +53 -0
- autarkia_msgqueue-0.1.0/src/autarkia_msgqueue/__init__.py +28 -0
- autarkia_msgqueue-0.1.0/src/autarkia_msgqueue/py.typed +0 -0
- autarkia_msgqueue-0.1.0/src/autarkia_msgqueue/queue.py +246 -0
- autarkia_msgqueue-0.1.0/src/autarkia_msgqueue/replay_guard.py +52 -0
- autarkia_msgqueue-0.1.0/tests/test_queue.py +445 -0
- autarkia_msgqueue-0.1.0/tests/test_replay_guard.py +71 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shihao Qi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: Autarkia-Msgqueue
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Reliable offline message queue for WebSocket clients: per-type TTL, message-id dedupe, discard-after-elsewhere-delivery, atomic replay, per-user locks and a receiver-side staleness guard.
|
|
5
|
+
Project-URL: Homepage, https://qishihao.icu/projects/autarkia
|
|
6
|
+
Project-URL: Repository, https://github.com/hakituo/Autarkia-Msgqueue
|
|
7
|
+
Project-URL: Roadmap, https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md
|
|
8
|
+
Project-URL: Autarkia Runtime, https://github.com/hakituo/Autarkia
|
|
9
|
+
Project-URL: Author, https://qishihao.icu
|
|
10
|
+
Author-email: Shihao Qi <qishihao512@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: at-least-once,autarkia,message-replay,offline-queue,reconnect,ttl,websocket
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
<p align="center">
|
|
29
|
+
<picture>
|
|
30
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/hakituo/Autarkia-Msgqueue/main/autarkia-dark.svg">
|
|
31
|
+
<img src="https://raw.githubusercontent.com/hakituo/Autarkia-Msgqueue/main/autarkia.svg" alt="Autarkia" width="300">
|
|
32
|
+
</picture>
|
|
33
|
+
</p>
|
|
34
|
+
|
|
35
|
+
<p align="center">
|
|
36
|
+
<b>Autarkia-Msgqueue</b><br>
|
|
37
|
+
<sub>WebSocket 离线消息可靠性队列 · Reliable offline queues for WebSocket clients</sub>
|
|
38
|
+
</p>
|
|
39
|
+
|
|
40
|
+
<p align="center">
|
|
41
|
+
<a href="https://pypi.org/project/Autarkia-Msgqueue/"><img alt="PyPI" src="https://img.shields.io/pypi/v/autarkia-msgqueue.svg"></a>
|
|
42
|
+
<a href="https://pypi.org/project/Autarkia-Msgqueue/"><img alt="Python" src="https://img.shields.io/pypi/pyversions/autarkia-msgqueue.svg"></a>
|
|
43
|
+
<a href="LICENSE"><img alt="License: MIT" src="https://img.shields.io/badge/license-MIT-blue.svg"></a>
|
|
44
|
+
</p>
|
|
45
|
+
|
|
46
|
+
<p align="center">
|
|
47
|
+
<a href="#中文">简体中文</a> · <a href="#english">English</a>
|
|
48
|
+
</p>
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## 中文
|
|
53
|
+
|
|
54
|
+
**按类型分档的 TTL、`message_id` 去重、已在别端投递后丢弃、原子重放、per-user flush 锁、接收端时效守卫**——把「客户端离线时暂存消息」这件小事做对。
|
|
55
|
+
|
|
56
|
+
来自 [Autarkia](https://qishihao.icu/projects/autarkia) agent 运行时:手机 App 和桌面 bot 共用一个 user id,同一条「晚安」被反复投递。整个系列见[组件库路线图](https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md)。
|
|
57
|
+
|
|
58
|
+
### 为什么
|
|
59
|
+
|
|
60
|
+
存消息很容易,不烦人很难:
|
|
61
|
+
|
|
62
|
+
- **已经投递过还会重放**:主动消息已经送到手机 App,队列里给桌面端留的那份就是重复打扰,必须能按 id 丢掉。
|
|
63
|
+
- **隔夜重放**:「晚安」第二天早上九点再发毫无意义。主动消息要分钟级 TTL,普通消息要一天——一个 TTL 两边都不对。
|
|
64
|
+
- **队列内重复**:同一个 `message_id` 入队两次就会重放两次。
|
|
65
|
+
- **失败即丢**:先出队再发送,socket 一断消息就没了。
|
|
66
|
+
- **并发 flush**:两条重连路径同时掏同一个 deque。
|
|
67
|
+
- **投错人**:两个人设共用一个 user id 时,一条连接只能取走发给自己的消息。
|
|
68
|
+
- **接收端也过期**:客户端 TTL 调大、手工重放、版本不一致,都可能让旧消息晚到,所以接收端独立再查一次年龄。
|
|
69
|
+
|
|
70
|
+
### 安装
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install autarkia-msgqueue
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
零依赖,Python 3.10+。
|
|
77
|
+
|
|
78
|
+
### 快速开始
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
import asyncio
|
|
82
|
+
from autarkia_msgqueue import OfflineMessageQueue
|
|
83
|
+
|
|
84
|
+
queue = OfflineMessageQueue()
|
|
85
|
+
|
|
86
|
+
# 客户端离线:入队(深拷贝)
|
|
87
|
+
queue.store_offline_message(
|
|
88
|
+
"user-1",
|
|
89
|
+
{"type": "proactive_message", "message_id": "evt-9", "text": "晚安"},
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
# 已在别处(手机 App)投递过:不要再重放给桌面端
|
|
93
|
+
queue.discard_offline_message("user-1", "evt-9")
|
|
94
|
+
|
|
95
|
+
async def on_reconnect(user_id):
|
|
96
|
+
async def send(payload) -> bool:
|
|
97
|
+
return await websocket_send(user_id, payload) # 返回假值/抛异常 = 稍后重试
|
|
98
|
+
|
|
99
|
+
result = await queue.flush(user_id, send, accepts=lambda m: m.get("persona") == "a")
|
|
100
|
+
print(result) # FlushResult(sent=0, expired=0, failed=0, skipped=0, remaining=0)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
接收端守卫:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from autarkia_msgqueue import is_stale_offline_replay
|
|
107
|
+
|
|
108
|
+
if is_stale_offline_replay(payload, max_age_seconds=600):
|
|
109
|
+
return # 太旧的重放直接丢
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### 接口一览
|
|
113
|
+
|
|
114
|
+
| API | 保证 |
|
|
115
|
+
| --- | --- |
|
|
116
|
+
| `store_offline_message` | 深拷贝入队;重复或非法载荷返回 `False` |
|
|
117
|
+
| `discard_offline_message` | 删掉该 `message_id` 的全部副本,返回删除条数 |
|
|
118
|
+
| `flush` | FIFO 重放;遇到第一条失败即停止,其余保留 |
|
|
119
|
+
| `QueueConfig` | `default_ttl_seconds`、`proactive_ttl_seconds`、`proactive_types` |
|
|
120
|
+
| `FlushResult` | `sent` / `expired` / `failed` / `skipped` / `remaining`,以及 `.ok` |
|
|
121
|
+
| `offline_replay_age`、`is_stale_offline_replay` | 接收端对重放消息的年龄判断 |
|
|
122
|
+
|
|
123
|
+
重放载荷会带 `is_offline_replay: True` 与 `offline_queued_at`(epoch 秒);队列里的原始消息不会被改写。
|
|
124
|
+
|
|
125
|
+
### 设计说明
|
|
126
|
+
|
|
127
|
+
- **TTL 按消息类型**:`type == "proactive_message"` 或 `is_proactive` 走 `proactive_ttl_seconds`(默认 10 分钟),其余走 `default_ttl_seconds`(默认 24 小时)。过期判据是 `age >= ttl`。
|
|
128
|
+
- **原子重放**:只有 `sender` 返回真值后消息才出队;失败则推回队首并停止本轮,顺序在 socket 抖动后仍然保持。
|
|
129
|
+
- **扫描窗口**:`flush` 只遍历拿到锁时已存在的消息,重放期间新入队的留给下一轮。
|
|
130
|
+
- **路由谓词**:`accepts(payload)` 返回 `False` 时消息留在队列继续扫描,一条连接不会偷走另一个人设的消息。
|
|
131
|
+
- **锁按 user + 事件循环**:队列被另一个事件循环使用时锁会重建,模块级单例不会把锁漏到 `asyncio.run` 之间。
|
|
132
|
+
- **刻意只在内存里**:它是重连缓冲,不是消息存储;进程重启即丢,要持久化请自己做。
|
|
133
|
+
- **测试可确定性**:队列的 `clock=`、`store` / `flush` 的 `now=`、守卫的 `now=` 都能注入,不依赖墙钟。
|
|
134
|
+
|
|
135
|
+
### 系列与链接
|
|
136
|
+
|
|
137
|
+
- 项目页:<https://qishihao.icu/projects/autarkia>
|
|
138
|
+
- 运行时仓库:<https://github.com/hakituo/Autarkia>
|
|
139
|
+
- 本包:<https://github.com/hakituo/Autarkia-Msgqueue>
|
|
140
|
+
- 组件库路线图:<https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md>
|
|
141
|
+
- 作者:Shihao Qi — <https://qishihao.icu>
|
|
142
|
+
|
|
143
|
+
### 许可
|
|
144
|
+
|
|
145
|
+
MIT
|
|
146
|
+
|
|
147
|
+
---
|
|
148
|
+
|
|
149
|
+
## English
|
|
150
|
+
|
|
151
|
+
**Per-type TTL, message-id dedupe, discard-after-elsewhere-delivery, atomic replay, per-user flush locks, and a receiver-side staleness guard** — the small "store messages while the client is offline" problem, done correctly.
|
|
152
|
+
|
|
153
|
+
Extracted from the [Autarkia](https://qishihao.icu/projects/autarkia) agent runtime, where a phone app and a desktop bot shared one user id and kept re-delivering the same goodnight message. See the [component roadmap](https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md).
|
|
154
|
+
|
|
155
|
+
### Why
|
|
156
|
+
|
|
157
|
+
Storing messages for offline clients is easy. Not annoying the user is not:
|
|
158
|
+
|
|
159
|
+
- **Replayed after delivery.** The same proactive message reached the phone app, so the queued copy for the desktop client is a duplicate. It has to be discarded by id.
|
|
160
|
+
- **Overnight replays.** "Goodnight" is worthless at 9 a.m. Proactive messages need minutes-scale TTL, ordinary messages a day; one TTL for both is wrong either way.
|
|
161
|
+
- **Duplicates in queue.** The same `message_id` enqueued twice replays twice.
|
|
162
|
+
- **Lost on failure.** Popping before a successful send loses the message when the socket dies mid-flush.
|
|
163
|
+
- **Concurrent flushes.** Two reconnect paths can empty the same deque under each other's feet.
|
|
164
|
+
- **Wrong recipient.** With two personas sharing a user id, a connection must only take messages addressed to it.
|
|
165
|
+
- **Stale receivers.** A client with an old TTL, a manual flush, or a mismatched version can still receive a replay hours later, so the receiver checks age independently.
|
|
166
|
+
|
|
167
|
+
### Install
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
pip install autarkia-msgqueue
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
No dependencies. Python 3.10+.
|
|
174
|
+
|
|
175
|
+
### Quick start
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
import asyncio
|
|
179
|
+
from autarkia_msgqueue import OfflineMessageQueue
|
|
180
|
+
|
|
181
|
+
queue = OfflineMessageQueue()
|
|
182
|
+
|
|
183
|
+
# The client is offline: park the message (deep-copied).
|
|
184
|
+
queue.store_offline_message(
|
|
185
|
+
"user-1",
|
|
186
|
+
{"type": "proactive_message", "message_id": "evt-9", "text": "晚安"},
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
# Delivered elsewhere (phone app) — never replay it to the desktop client.
|
|
190
|
+
queue.discard_offline_message("user-1", "evt-9")
|
|
191
|
+
|
|
192
|
+
async def on_reconnect(user_id):
|
|
193
|
+
async def send(payload) -> bool:
|
|
194
|
+
return await websocket_send(user_id, payload) # False / raise => retry later
|
|
195
|
+
|
|
196
|
+
result = await queue.flush(user_id, send, accepts=lambda msg: msg.get("persona") == "a")
|
|
197
|
+
print(result) # FlushResult(sent=0, expired=0, failed=0, skipped=0, remaining=0)
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
#### Receiver side
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
from autarkia_msgqueue import is_stale_offline_replay
|
|
204
|
+
|
|
205
|
+
if is_stale_offline_replay(payload, max_age_seconds=600):
|
|
206
|
+
return # drop a replay that is too old to be meaningful
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### Concepts
|
|
210
|
+
|
|
211
|
+
| API | Guarantee |
|
|
212
|
+
| --- | --- |
|
|
213
|
+
| `store_offline_message` | deep copy; returns `False` for duplicates or bad payloads |
|
|
214
|
+
| `discard_offline_message` | removes every copy of that `message_id`, returns the count |
|
|
215
|
+
| `flush` | FIFO replay; stops at the first failure and keeps the rest |
|
|
216
|
+
| `QueueConfig` | `default_ttl_seconds`, `proactive_ttl_seconds`, `proactive_types` |
|
|
217
|
+
| `FlushResult` | `sent` / `expired` / `failed` / `skipped` / `remaining`, plus `.ok` |
|
|
218
|
+
| `offline_replay_age`, `is_stale_offline_replay` | receiver-side age check on replays |
|
|
219
|
+
|
|
220
|
+
Replayed payloads carry `is_offline_replay: True` and `offline_queued_at` (epoch seconds); the stored copy is never mutated.
|
|
221
|
+
|
|
222
|
+
### Design notes
|
|
223
|
+
|
|
224
|
+
- **TTL per message type.** `type == "proactive_message"` or `is_proactive` uses `proactive_ttl_seconds` (default 10 min); everything else uses `default_ttl_seconds` (default 24 h). Expiry is `age >= ttl`.
|
|
225
|
+
- **Atomic replay.** A message leaves the queue only after `sender` returns truthy. On failure it is pushed back to the front and the flush stops, so ordering survives a broken socket.
|
|
226
|
+
- **Scan window.** `flush` only walks the messages present when it takes the lock; anything enqueued mid-flush is left for the next round.
|
|
227
|
+
- **Routing predicate.** `accepts(payload)` returning `False` leaves the message queued and keeps scanning, so one connection never steals another persona's messages.
|
|
228
|
+
- **Locks are per user per event loop.** Locks are recreated when the queue is used from a different event loop, so a module-level singleton does not leak locks across `asyncio.run` calls.
|
|
229
|
+
- **In memory, by design.** The queue is a reconnect buffer, not a message store: a process restart drops it. Persist elsewhere if you need durability.
|
|
230
|
+
- **Deterministic tests.** `clock=` on the queue and `now=` on `store` / `flush` / the guard remove all wall-clock dependence.
|
|
231
|
+
|
|
232
|
+
### The bigger picture
|
|
233
|
+
|
|
234
|
+
- Project page: <https://qishihao.icu/projects/autarkia>
|
|
235
|
+
- Runtime repository: <https://github.com/hakituo/Autarkia>
|
|
236
|
+
- This package: <https://github.com/hakituo/Autarkia-Msgqueue>
|
|
237
|
+
- Roadmap: <https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md>
|
|
238
|
+
- Author: Shihao Qi — <https://qishihao.icu>
|
|
239
|
+
|
|
240
|
+
### License
|
|
241
|
+
|
|
242
|
+
MIT
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<picture>
|
|
3
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/hakituo/Autarkia-Msgqueue/main/autarkia-dark.svg">
|
|
4
|
+
<img src="https://raw.githubusercontent.com/hakituo/Autarkia-Msgqueue/main/autarkia.svg" alt="Autarkia" width="300">
|
|
5
|
+
</picture>
|
|
6
|
+
</p>
|
|
7
|
+
|
|
8
|
+
<p align="center">
|
|
9
|
+
<b>Autarkia-Msgqueue</b><br>
|
|
10
|
+
<sub>WebSocket 离线消息可靠性队列 · Reliable offline queues for WebSocket clients</sub>
|
|
11
|
+
</p>
|
|
12
|
+
|
|
13
|
+
<p align="center">
|
|
14
|
+
<a href="https://pypi.org/project/Autarkia-Msgqueue/"><img alt="PyPI" src="https://img.shields.io/pypi/v/autarkia-msgqueue.svg"></a>
|
|
15
|
+
<a href="https://pypi.org/project/Autarkia-Msgqueue/"><img alt="Python" src="https://img.shields.io/pypi/pyversions/autarkia-msgqueue.svg"></a>
|
|
16
|
+
<a href="LICENSE"><img alt="License: MIT" src="https://img.shields.io/badge/license-MIT-blue.svg"></a>
|
|
17
|
+
</p>
|
|
18
|
+
|
|
19
|
+
<p align="center">
|
|
20
|
+
<a href="#中文">简体中文</a> · <a href="#english">English</a>
|
|
21
|
+
</p>
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## 中文
|
|
26
|
+
|
|
27
|
+
**按类型分档的 TTL、`message_id` 去重、已在别端投递后丢弃、原子重放、per-user flush 锁、接收端时效守卫**——把「客户端离线时暂存消息」这件小事做对。
|
|
28
|
+
|
|
29
|
+
来自 [Autarkia](https://qishihao.icu/projects/autarkia) agent 运行时:手机 App 和桌面 bot 共用一个 user id,同一条「晚安」被反复投递。整个系列见[组件库路线图](https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md)。
|
|
30
|
+
|
|
31
|
+
### 为什么
|
|
32
|
+
|
|
33
|
+
存消息很容易,不烦人很难:
|
|
34
|
+
|
|
35
|
+
- **已经投递过还会重放**:主动消息已经送到手机 App,队列里给桌面端留的那份就是重复打扰,必须能按 id 丢掉。
|
|
36
|
+
- **隔夜重放**:「晚安」第二天早上九点再发毫无意义。主动消息要分钟级 TTL,普通消息要一天——一个 TTL 两边都不对。
|
|
37
|
+
- **队列内重复**:同一个 `message_id` 入队两次就会重放两次。
|
|
38
|
+
- **失败即丢**:先出队再发送,socket 一断消息就没了。
|
|
39
|
+
- **并发 flush**:两条重连路径同时掏同一个 deque。
|
|
40
|
+
- **投错人**:两个人设共用一个 user id 时,一条连接只能取走发给自己的消息。
|
|
41
|
+
- **接收端也过期**:客户端 TTL 调大、手工重放、版本不一致,都可能让旧消息晚到,所以接收端独立再查一次年龄。
|
|
42
|
+
|
|
43
|
+
### 安装
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install autarkia-msgqueue
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
零依赖,Python 3.10+。
|
|
50
|
+
|
|
51
|
+
### 快速开始
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
import asyncio
|
|
55
|
+
from autarkia_msgqueue import OfflineMessageQueue
|
|
56
|
+
|
|
57
|
+
queue = OfflineMessageQueue()
|
|
58
|
+
|
|
59
|
+
# 客户端离线:入队(深拷贝)
|
|
60
|
+
queue.store_offline_message(
|
|
61
|
+
"user-1",
|
|
62
|
+
{"type": "proactive_message", "message_id": "evt-9", "text": "晚安"},
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
# 已在别处(手机 App)投递过:不要再重放给桌面端
|
|
66
|
+
queue.discard_offline_message("user-1", "evt-9")
|
|
67
|
+
|
|
68
|
+
async def on_reconnect(user_id):
|
|
69
|
+
async def send(payload) -> bool:
|
|
70
|
+
return await websocket_send(user_id, payload) # 返回假值/抛异常 = 稍后重试
|
|
71
|
+
|
|
72
|
+
result = await queue.flush(user_id, send, accepts=lambda m: m.get("persona") == "a")
|
|
73
|
+
print(result) # FlushResult(sent=0, expired=0, failed=0, skipped=0, remaining=0)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
接收端守卫:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from autarkia_msgqueue import is_stale_offline_replay
|
|
80
|
+
|
|
81
|
+
if is_stale_offline_replay(payload, max_age_seconds=600):
|
|
82
|
+
return # 太旧的重放直接丢
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### 接口一览
|
|
86
|
+
|
|
87
|
+
| API | 保证 |
|
|
88
|
+
| --- | --- |
|
|
89
|
+
| `store_offline_message` | 深拷贝入队;重复或非法载荷返回 `False` |
|
|
90
|
+
| `discard_offline_message` | 删掉该 `message_id` 的全部副本,返回删除条数 |
|
|
91
|
+
| `flush` | FIFO 重放;遇到第一条失败即停止,其余保留 |
|
|
92
|
+
| `QueueConfig` | `default_ttl_seconds`、`proactive_ttl_seconds`、`proactive_types` |
|
|
93
|
+
| `FlushResult` | `sent` / `expired` / `failed` / `skipped` / `remaining`,以及 `.ok` |
|
|
94
|
+
| `offline_replay_age`、`is_stale_offline_replay` | 接收端对重放消息的年龄判断 |
|
|
95
|
+
|
|
96
|
+
重放载荷会带 `is_offline_replay: True` 与 `offline_queued_at`(epoch 秒);队列里的原始消息不会被改写。
|
|
97
|
+
|
|
98
|
+
### 设计说明
|
|
99
|
+
|
|
100
|
+
- **TTL 按消息类型**:`type == "proactive_message"` 或 `is_proactive` 走 `proactive_ttl_seconds`(默认 10 分钟),其余走 `default_ttl_seconds`(默认 24 小时)。过期判据是 `age >= ttl`。
|
|
101
|
+
- **原子重放**:只有 `sender` 返回真值后消息才出队;失败则推回队首并停止本轮,顺序在 socket 抖动后仍然保持。
|
|
102
|
+
- **扫描窗口**:`flush` 只遍历拿到锁时已存在的消息,重放期间新入队的留给下一轮。
|
|
103
|
+
- **路由谓词**:`accepts(payload)` 返回 `False` 时消息留在队列继续扫描,一条连接不会偷走另一个人设的消息。
|
|
104
|
+
- **锁按 user + 事件循环**:队列被另一个事件循环使用时锁会重建,模块级单例不会把锁漏到 `asyncio.run` 之间。
|
|
105
|
+
- **刻意只在内存里**:它是重连缓冲,不是消息存储;进程重启即丢,要持久化请自己做。
|
|
106
|
+
- **测试可确定性**:队列的 `clock=`、`store` / `flush` 的 `now=`、守卫的 `now=` 都能注入,不依赖墙钟。
|
|
107
|
+
|
|
108
|
+
### 系列与链接
|
|
109
|
+
|
|
110
|
+
- 项目页:<https://qishihao.icu/projects/autarkia>
|
|
111
|
+
- 运行时仓库:<https://github.com/hakituo/Autarkia>
|
|
112
|
+
- 本包:<https://github.com/hakituo/Autarkia-Msgqueue>
|
|
113
|
+
- 组件库路线图:<https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md>
|
|
114
|
+
- 作者:Shihao Qi — <https://qishihao.icu>
|
|
115
|
+
|
|
116
|
+
### 许可
|
|
117
|
+
|
|
118
|
+
MIT
|
|
119
|
+
|
|
120
|
+
---
|
|
121
|
+
|
|
122
|
+
## English
|
|
123
|
+
|
|
124
|
+
**Per-type TTL, message-id dedupe, discard-after-elsewhere-delivery, atomic replay, per-user flush locks, and a receiver-side staleness guard** — the small "store messages while the client is offline" problem, done correctly.
|
|
125
|
+
|
|
126
|
+
Extracted from the [Autarkia](https://qishihao.icu/projects/autarkia) agent runtime, where a phone app and a desktop bot shared one user id and kept re-delivering the same goodnight message. See the [component roadmap](https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md).
|
|
127
|
+
|
|
128
|
+
### Why
|
|
129
|
+
|
|
130
|
+
Storing messages for offline clients is easy. Not annoying the user is not:
|
|
131
|
+
|
|
132
|
+
- **Replayed after delivery.** The same proactive message reached the phone app, so the queued copy for the desktop client is a duplicate. It has to be discarded by id.
|
|
133
|
+
- **Overnight replays.** "Goodnight" is worthless at 9 a.m. Proactive messages need minutes-scale TTL, ordinary messages a day; one TTL for both is wrong either way.
|
|
134
|
+
- **Duplicates in queue.** The same `message_id` enqueued twice replays twice.
|
|
135
|
+
- **Lost on failure.** Popping before a successful send loses the message when the socket dies mid-flush.
|
|
136
|
+
- **Concurrent flushes.** Two reconnect paths can empty the same deque under each other's feet.
|
|
137
|
+
- **Wrong recipient.** With two personas sharing a user id, a connection must only take messages addressed to it.
|
|
138
|
+
- **Stale receivers.** A client with an old TTL, a manual flush, or a mismatched version can still receive a replay hours later, so the receiver checks age independently.
|
|
139
|
+
|
|
140
|
+
### Install
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
pip install autarkia-msgqueue
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
No dependencies. Python 3.10+.
|
|
147
|
+
|
|
148
|
+
### Quick start
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
import asyncio
|
|
152
|
+
from autarkia_msgqueue import OfflineMessageQueue
|
|
153
|
+
|
|
154
|
+
queue = OfflineMessageQueue()
|
|
155
|
+
|
|
156
|
+
# The client is offline: park the message (deep-copied).
|
|
157
|
+
queue.store_offline_message(
|
|
158
|
+
"user-1",
|
|
159
|
+
{"type": "proactive_message", "message_id": "evt-9", "text": "晚安"},
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
# Delivered elsewhere (phone app) — never replay it to the desktop client.
|
|
163
|
+
queue.discard_offline_message("user-1", "evt-9")
|
|
164
|
+
|
|
165
|
+
async def on_reconnect(user_id):
|
|
166
|
+
async def send(payload) -> bool:
|
|
167
|
+
return await websocket_send(user_id, payload) # False / raise => retry later
|
|
168
|
+
|
|
169
|
+
result = await queue.flush(user_id, send, accepts=lambda msg: msg.get("persona") == "a")
|
|
170
|
+
print(result) # FlushResult(sent=0, expired=0, failed=0, skipped=0, remaining=0)
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
#### Receiver side
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
from autarkia_msgqueue import is_stale_offline_replay
|
|
177
|
+
|
|
178
|
+
if is_stale_offline_replay(payload, max_age_seconds=600):
|
|
179
|
+
return # drop a replay that is too old to be meaningful
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### Concepts
|
|
183
|
+
|
|
184
|
+
| API | Guarantee |
|
|
185
|
+
| --- | --- |
|
|
186
|
+
| `store_offline_message` | deep copy; returns `False` for duplicates or bad payloads |
|
|
187
|
+
| `discard_offline_message` | removes every copy of that `message_id`, returns the count |
|
|
188
|
+
| `flush` | FIFO replay; stops at the first failure and keeps the rest |
|
|
189
|
+
| `QueueConfig` | `default_ttl_seconds`, `proactive_ttl_seconds`, `proactive_types` |
|
|
190
|
+
| `FlushResult` | `sent` / `expired` / `failed` / `skipped` / `remaining`, plus `.ok` |
|
|
191
|
+
| `offline_replay_age`, `is_stale_offline_replay` | receiver-side age check on replays |
|
|
192
|
+
|
|
193
|
+
Replayed payloads carry `is_offline_replay: True` and `offline_queued_at` (epoch seconds); the stored copy is never mutated.
|
|
194
|
+
|
|
195
|
+
### Design notes
|
|
196
|
+
|
|
197
|
+
- **TTL per message type.** `type == "proactive_message"` or `is_proactive` uses `proactive_ttl_seconds` (default 10 min); everything else uses `default_ttl_seconds` (default 24 h). Expiry is `age >= ttl`.
|
|
198
|
+
- **Atomic replay.** A message leaves the queue only after `sender` returns truthy. On failure it is pushed back to the front and the flush stops, so ordering survives a broken socket.
|
|
199
|
+
- **Scan window.** `flush` only walks the messages present when it takes the lock; anything enqueued mid-flush is left for the next round.
|
|
200
|
+
- **Routing predicate.** `accepts(payload)` returning `False` leaves the message queued and keeps scanning, so one connection never steals another persona's messages.
|
|
201
|
+
- **Locks are per user per event loop.** Locks are recreated when the queue is used from a different event loop, so a module-level singleton does not leak locks across `asyncio.run` calls.
|
|
202
|
+
- **In memory, by design.** The queue is a reconnect buffer, not a message store: a process restart drops it. Persist elsewhere if you need durability.
|
|
203
|
+
- **Deterministic tests.** `clock=` on the queue and `now=` on `store` / `flush` / the guard remove all wall-clock dependence.
|
|
204
|
+
|
|
205
|
+
### The bigger picture
|
|
206
|
+
|
|
207
|
+
- Project page: <https://qishihao.icu/projects/autarkia>
|
|
208
|
+
- Runtime repository: <https://github.com/hakituo/Autarkia>
|
|
209
|
+
- This package: <https://github.com/hakituo/Autarkia-Msgqueue>
|
|
210
|
+
- Roadmap: <https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md>
|
|
211
|
+
- Author: Shihao Qi — <https://qishihao.icu>
|
|
212
|
+
|
|
213
|
+
### License
|
|
214
|
+
|
|
215
|
+
MIT
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "Autarkia-Msgqueue"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Reliable offline message queue for WebSocket clients: per-type TTL, message-id dedupe, discard-after-elsewhere-delivery, atomic replay, per-user locks and a receiver-side staleness guard."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Shihao Qi", email = "qishihao512@gmail.com" },
|
|
14
|
+
]
|
|
15
|
+
keywords = [
|
|
16
|
+
"websocket",
|
|
17
|
+
"offline-queue",
|
|
18
|
+
"message-replay",
|
|
19
|
+
"at-least-once",
|
|
20
|
+
"reconnect",
|
|
21
|
+
"ttl",
|
|
22
|
+
"autarkia",
|
|
23
|
+
]
|
|
24
|
+
classifiers = [
|
|
25
|
+
"Development Status :: 4 - Beta",
|
|
26
|
+
"Intended Audience :: Developers",
|
|
27
|
+
"License :: OSI Approved :: MIT License",
|
|
28
|
+
"Operating System :: OS Independent",
|
|
29
|
+
"Programming Language :: Python :: 3",
|
|
30
|
+
"Programming Language :: Python :: 3.10",
|
|
31
|
+
"Programming Language :: Python :: 3.11",
|
|
32
|
+
"Programming Language :: Python :: 3.12",
|
|
33
|
+
"Programming Language :: Python :: 3.13",
|
|
34
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
35
|
+
"Typing :: Typed",
|
|
36
|
+
]
|
|
37
|
+
dependencies = []
|
|
38
|
+
|
|
39
|
+
[project.urls]
|
|
40
|
+
Homepage = "https://qishihao.icu/projects/autarkia"
|
|
41
|
+
Repository = "https://github.com/hakituo/Autarkia-Msgqueue"
|
|
42
|
+
Roadmap = "https://github.com/hakituo/Autarkia-tool/blob/main/ROADMAP.md"
|
|
43
|
+
"Autarkia Runtime" = "https://github.com/hakituo/Autarkia"
|
|
44
|
+
Author = "https://qishihao.icu"
|
|
45
|
+
|
|
46
|
+
[tool.pytest.ini_options]
|
|
47
|
+
testpaths = ["tests"]
|
|
48
|
+
|
|
49
|
+
[tool.hatch.build.targets.wheel]
|
|
50
|
+
packages = ["src/autarkia_msgqueue"]
|
|
51
|
+
|
|
52
|
+
[tool.hatch.build.targets.sdist]
|
|
53
|
+
include = ["src", "tests", "README.md", "LICENSE"]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Autarkia-Msgqueue: reliable offline message queue for WebSocket reconnects."""
|
|
2
|
+
|
|
3
|
+
from .queue import (
|
|
4
|
+
DEFAULT_OFFLINE_TTL,
|
|
5
|
+
PROACTIVE_OFFLINE_TTL,
|
|
6
|
+
FlushResult,
|
|
7
|
+
OfflineMessageQueue,
|
|
8
|
+
QueueConfig,
|
|
9
|
+
)
|
|
10
|
+
from .replay_guard import (
|
|
11
|
+
OFFLINE_REPLAY_MAX_AGE_SECONDS,
|
|
12
|
+
is_stale_offline_replay,
|
|
13
|
+
offline_replay_age,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
__version__ = "0.1.0"
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"__version__",
|
|
20
|
+
"OfflineMessageQueue",
|
|
21
|
+
"QueueConfig",
|
|
22
|
+
"FlushResult",
|
|
23
|
+
"DEFAULT_OFFLINE_TTL",
|
|
24
|
+
"PROACTIVE_OFFLINE_TTL",
|
|
25
|
+
"OFFLINE_REPLAY_MAX_AGE_SECONDS",
|
|
26
|
+
"offline_replay_age",
|
|
27
|
+
"is_stale_offline_replay",
|
|
28
|
+
]
|
|
File without changes
|