nobroker 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nobroker-0.1.0/.zero-dep.toml +18 -0
- nobroker-0.1.0/LICENSE +21 -0
- nobroker-0.1.0/MANIFEST.in +18 -0
- nobroker-0.1.0/Makefile +57 -0
- nobroker-0.1.0/PKG-INFO +537 -0
- nobroker-0.1.0/README.md +504 -0
- nobroker-0.1.0/STDLIB.md +391 -0
- nobroker-0.1.0/deps-proof.txt +45 -0
- nobroker-0.1.0/examples/demo.py +167 -0
- nobroker-0.1.0/examples/handlers.py +68 -0
- nobroker-0.1.0/pyproject.toml +61 -0
- nobroker-0.1.0/requirements.txt +9 -0
- nobroker-0.1.0/setup.cfg +4 -0
- nobroker-0.1.0/src/nobroker/__init__.py +56 -0
- nobroker-0.1.0/src/nobroker/backoff.py +58 -0
- nobroker-0.1.0/src/nobroker/bench.py +210 -0
- nobroker-0.1.0/src/nobroker/cli.py +393 -0
- nobroker-0.1.0/src/nobroker/codec.py +170 -0
- nobroker-0.1.0/src/nobroker/errors.py +60 -0
- nobroker-0.1.0/src/nobroker/index.py +358 -0
- nobroker-0.1.0/src/nobroker/job.py +139 -0
- nobroker-0.1.0/src/nobroker/lock.py +170 -0
- nobroker-0.1.0/src/nobroker/logfile.py +463 -0
- nobroker-0.1.0/src/nobroker/py.typed +2 -0
- nobroker-0.1.0/src/nobroker/queue.py +738 -0
- nobroker-0.1.0/src/nobroker/worker.py +268 -0
- nobroker-0.1.0/src/nobroker.egg-info/PKG-INFO +537 -0
- nobroker-0.1.0/src/nobroker.egg-info/SOURCES.txt +41 -0
- nobroker-0.1.0/src/nobroker.egg-info/dependency_links.txt +1 -0
- nobroker-0.1.0/src/nobroker.egg-info/entry_points.txt +2 -0
- nobroker-0.1.0/src/nobroker.egg-info/top_level.txt +1 -0
- nobroker-0.1.0/tests/__init__.py +13 -0
- nobroker-0.1.0/tests/support.py +71 -0
- nobroker-0.1.0/tests/test_backoff.py +60 -0
- nobroker-0.1.0/tests/test_cli.py +210 -0
- nobroker-0.1.0/tests/test_codec.py +148 -0
- nobroker-0.1.0/tests/test_concurrency.py +204 -0
- nobroker-0.1.0/tests/test_crash_recovery.py +274 -0
- nobroker-0.1.0/tests/test_logfile.py +157 -0
- nobroker-0.1.0/tests/test_queue.py +397 -0
- nobroker-0.1.0/tests/test_worker.py +211 -0
- nobroker-0.1.0/tools/build_pyz.py +95 -0
- nobroker-0.1.0/tools/check_deps.py +117 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
track = "D"
|
|
2
|
+
name = "nobroker"
|
|
3
|
+
pitch = "A durable, crash-safe job queue for Python with no broker, no server, and no dependencies -- just a file."
|
|
4
|
+
|
|
5
|
+
language = "python"
|
|
6
|
+
python_requires = ">=3.11"
|
|
7
|
+
entrypoint = "make demo"
|
|
8
|
+
runtime_dependencies = 0
|
|
9
|
+
|
|
10
|
+
bonuses = ["package-killer", "stdlib-log"]
|
|
11
|
+
|
|
12
|
+
# Package Killer: replaces celery + redis (or rq, dramatiq, huey) for the
|
|
13
|
+
# single-machine case -- roughly 40 MB of transitive dependencies and a server
|
|
14
|
+
# process, traded for one file and the standard library.
|
|
15
|
+
replaces = ["celery", "redis", "rq", "dramatiq", "huey", "kombu", "billiard"]
|
|
16
|
+
|
|
17
|
+
# STDLIB Log: see STDLIB.md.
|
|
18
|
+
stdlib_log = "STDLIB.md"
|
nobroker-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ayan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# What goes into the source distribution beyond the package itself.
|
|
2
|
+
include LICENSE
|
|
3
|
+
include README.md
|
|
4
|
+
include STDLIB.md
|
|
5
|
+
include deps-proof.txt
|
|
6
|
+
include requirements.txt
|
|
7
|
+
include .zero-dep.toml
|
|
8
|
+
include Makefile
|
|
9
|
+
recursive-include src/nobroker py.typed
|
|
10
|
+
recursive-include tests *.py
|
|
11
|
+
recursive-include tools *.py
|
|
12
|
+
recursive-include examples *.py
|
|
13
|
+
|
|
14
|
+
# The website and the process notes are not part of the distribution.
|
|
15
|
+
prune web
|
|
16
|
+
prune others
|
|
17
|
+
prune dist
|
|
18
|
+
exclude src/__main__.py
|
nobroker-0.1.0/Makefile
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# nobroker -- one command for everything.
|
|
2
|
+
#
|
|
3
|
+
# make build -> dist/nobroker.pyz, the whole tool as one runnable file
|
|
4
|
+
# make demo -> the end-to-end tour
|
|
5
|
+
# make test -> 122 tests, including crash recovery
|
|
6
|
+
#
|
|
7
|
+
# Every target delegates to make.py, which holds the actual task definitions, so
|
|
8
|
+
# the two can never drift. If you do not have `make` -- which is the default on
|
|
9
|
+
# Windows -- use make.py directly and nothing changes:
|
|
10
|
+
#
|
|
11
|
+
# python make.py build
|
|
12
|
+
#
|
|
13
|
+
# That is deliberate. `make` is a separately installed tool, and this project's
|
|
14
|
+
# claim is that you need Python and nothing else. The Makefile exists because
|
|
15
|
+
# `make test` is what a reviewer's fingers type; make.py exists because it is
|
|
16
|
+
# what actually runs everywhere.
|
|
17
|
+
|
|
18
|
+
PYTHON ?= python3
|
|
19
|
+
|
|
20
|
+
.DEFAULT_GOAL := help
|
|
21
|
+
.PHONY: help build demo video-demo test test-quick bench bench-md check-deps deps-proof cli clean
|
|
22
|
+
|
|
23
|
+
help: ## List every task
|
|
24
|
+
@$(PYTHON) make.py
|
|
25
|
+
|
|
26
|
+
build: ## Bundle the tool into one runnable file (dist/nobroker.pyz)
|
|
27
|
+
@$(PYTHON) make.py build
|
|
28
|
+
|
|
29
|
+
demo: ## Run the end-to-end tour (start here)
|
|
30
|
+
@$(PYTHON) make.py demo
|
|
31
|
+
|
|
32
|
+
video-demo: ## Run the paced demo built for screen recording
|
|
33
|
+
@$(PYTHON) make.py video-demo
|
|
34
|
+
|
|
35
|
+
test: ## Run the full test suite, including crash recovery
|
|
36
|
+
@$(PYTHON) make.py test
|
|
37
|
+
|
|
38
|
+
test-quick: ## Run everything except the slow exhaustive-truncation sweep
|
|
39
|
+
@$(PYTHON) make.py test-quick
|
|
40
|
+
|
|
41
|
+
bench: ## Measure throughput on this machine
|
|
42
|
+
@$(PYTHON) make.py bench
|
|
43
|
+
|
|
44
|
+
bench-md: ## Same, as a Markdown table for the README
|
|
45
|
+
@$(PYTHON) make.py bench-md
|
|
46
|
+
|
|
47
|
+
check-deps: ## Fail if anything outside the standard library is imported
|
|
48
|
+
@$(PYTHON) make.py check-deps
|
|
49
|
+
|
|
50
|
+
deps-proof: ## Regenerate deps-proof.txt
|
|
51
|
+
@$(PYTHON) make.py deps-proof
|
|
52
|
+
|
|
53
|
+
cli: ## Show the CLI help
|
|
54
|
+
@$(PYTHON) make.py cli
|
|
55
|
+
|
|
56
|
+
clean: ## Remove caches, scratch queues and build output
|
|
57
|
+
@$(PYTHON) make.py clean
|
nobroker-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,537 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nobroker
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A durable, broker-less job queue for Python. No server, no dependencies.
|
|
5
|
+
Author-email: Ayan <ayandgp12@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ayanbag/nobroker
|
|
8
|
+
Project-URL: Documentation, https://ayanbag.github.io/nobroker/
|
|
9
|
+
Project-URL: Playground, https://ayanbag.github.io/nobroker/playground.html
|
|
10
|
+
Project-URL: Source, https://github.com/ayanbag/nobroker
|
|
11
|
+
Project-URL: Issues, https://github.com/ayanbag/nobroker/issues
|
|
12
|
+
Project-URL: Changelog, https://github.com/ayanbag/nobroker/releases
|
|
13
|
+
Keywords: queue,job-queue,task-queue,durable,wal,write-ahead-log,zero-dependency,at-least-once,crash-safe,worker,background-jobs
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: POSIX
|
|
18
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
19
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
20
|
+
Classifier: Programming Language :: Python :: 3
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
25
|
+
Classifier: Topic :: Database :: Database Engines/Servers
|
|
26
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
27
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
28
|
+
Classifier: Typing :: Typed
|
|
29
|
+
Requires-Python: >=3.11
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
License-File: LICENSE
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
<p align="center">
|
|
35
|
+
<img src="https://raw.githubusercontent.com/ayanbag/nobroker/master/assets/banner-git.png" alt="nobroker — the job queue with nobody in the middle" width="100%">
|
|
36
|
+
</p>
|
|
37
|
+
|
|
38
|
+
<p align="center">
|
|
39
|
+
<a href="https://github.com/ayanbag/nobroker/actions/workflows/ci.yml"><img alt="tests" src="https://img.shields.io/github/actions/workflow/status/ayanbag/nobroker/ci.yml?branch=main&label=tests&style=flat-square&labelColor=12160F&color=2C6A4E"></a>
|
|
40
|
+
<a href="https://pypi.org/project/nobroker/"><img alt="pypi" src="https://img.shields.io/pypi/v/nobroker?style=flat-square&labelColor=12160F&color=9E3B18"></a>
|
|
41
|
+
<a href="https://pypi.org/project/nobroker/"><img alt="python" src="https://img.shields.io/pypi/pyversions/nobroker?style=flat-square&labelColor=12160F&color=8A6A1F"></a>
|
|
42
|
+
<img alt="dependencies: 0" src="https://img.shields.io/badge/dependencies-0-2C6A4E?style=flat-square&labelColor=12160F">
|
|
43
|
+
<img alt="tests: 124" src="https://img.shields.io/badge/tests-124%20passing-2C6A4E?style=flat-square&labelColor=12160F">
|
|
44
|
+
<img alt="delivery: at-least-once" src="https://img.shields.io/badge/delivery-at--least--once-8A6A1F?style=flat-square&labelColor=12160F">
|
|
45
|
+
<a href="LICENSE"><img alt="license" src="https://img.shields.io/badge/license-MIT-535E55?style=flat-square&labelColor=12160F"></a>
|
|
46
|
+
</p>
|
|
47
|
+
|
|
48
|
+
<p align="center">
|
|
49
|
+
<b>The job queue with nobody in the middle.</b><br>
|
|
50
|
+
<a href="https://ayanbag.github.io/nobroker/">Docs</a> ·
|
|
51
|
+
<a href="https://ayanbag.github.io/nobroker/playground.html">Interactive playground</a> ·
|
|
52
|
+
<a href="STDLIB.md">STDLIB.md</a> ·
|
|
53
|
+
<a href="deps-proof.txt">Dependency proof</a> ·
|
|
54
|
+
<a href="#limits">Limits</a>
|
|
55
|
+
</p>
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
A durable, crash-safe job queue for Python. Enqueue work, lease it, ack it. It
|
|
60
|
+
survives `kill -9`, it is safe across processes, and it needs no Redis, no
|
|
61
|
+
RabbitMQ, no server, and **no dependencies at all** — just the standard library
|
|
62
|
+
and a file.
|
|
63
|
+
|
|
64
|
+
The name is the thesis. Every other job queue makes you run a broker. This one
|
|
65
|
+
doesn't.
|
|
66
|
+
|
|
67
|
+
> Built for the [Zero Dependency Hackathon](https://zerodepshack.com) — **Track D, Data & Storage**.
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## Why this exists
|
|
72
|
+
|
|
73
|
+
You want a background job queue. Your options today:
|
|
74
|
+
|
|
75
|
+
| | What you install | What you operate |
|
|
76
|
+
|---|---|---|
|
|
77
|
+
| Celery | `celery`, `kombu`, `billiard`, `vine`, `amqp`, … | a Redis or RabbitMQ server |
|
|
78
|
+
| RQ | `rq`, `redis` | a Redis server |
|
|
79
|
+
| Dramatiq | `dramatiq`, `pika` or `redis` | a broker |
|
|
80
|
+
| **nobroker** | **nothing** | **nothing** |
|
|
81
|
+
|
|
82
|
+
For a single machine — a cron box, a CLI tool, a desktop app, a small service, a
|
|
83
|
+
CI runner, a Raspberry Pi — the broker is pure operational overhead. The durable
|
|
84
|
+
storage and the mutual exclusion you actually need are both already in your
|
|
85
|
+
kernel. nobroker is what is left when you use them directly.
|
|
86
|
+
|
|
87
|
+
**It is not a Celery replacement for a fleet.** It does not distribute work
|
|
88
|
+
across machines and never will. See [Limits](#limits).
|
|
89
|
+
|
|
90
|
+
---
|
|
91
|
+
|
|
92
|
+
## Install
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
pip install nobroker
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
That installs one package and nothing else — `dependencies = []` is not a
|
|
99
|
+
rounding error, it is the product.
|
|
100
|
+
|
|
101
|
+
Or skip `pip` entirely — from a fresh clone, with nothing installed:
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
git clone https://github.com/ayanbag/nobroker && cd nobroker
|
|
105
|
+
|
|
106
|
+
python make.py demo # end-to-end tour: enqueue, retry, crash, recover, compact
|
|
107
|
+
python make.py build # -> dist/nobroker.pyz, the whole tool as one 40 KB file
|
|
108
|
+
python dist/nobroker.pyz --help
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Requires Python 3.11 or newer. That is the entire dependency list. You can also
|
|
112
|
+
just copy `src/nobroker/` into your project — it is pure standard library, so it
|
|
113
|
+
will run wherever your code does.
|
|
114
|
+
|
|
115
|
+
### The build is one step, and needs nothing either
|
|
116
|
+
|
|
117
|
+
`python make.py build` produces `dist/nobroker.pyz`: a single 40 KB file that
|
|
118
|
+
runs on any Python 3.11+, with no install and nothing unpacked to disk.
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
PYZ=$PWD/dist/nobroker.pyz
|
|
122
|
+
cd /tmp # anywhere at all: no install, no PYTHONPATH
|
|
123
|
+
python "$PYZ" --dir ./q enqueue '{"resize":"photo.jpg"}'
|
|
124
|
+
python "$PYZ" --dir ./q stats
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
No compiler, no bundler, no `pip install` — that is `zipapp` from the standard
|
|
128
|
+
library. `shiv` and `pex` exist to solve the *hard* part of single-file packaging,
|
|
129
|
+
which is vendoring third-party dependencies and their compiled extensions. There
|
|
130
|
+
are none here, so the hard part is absent.
|
|
131
|
+
|
|
132
|
+
**`make` also works** (`make build`, `make test`, …) and is a one-line-per-target
|
|
133
|
+
delegation to `make.py`, so the two cannot drift. `make.py` is the canonical one
|
|
134
|
+
on purpose: `make` is a **separately installed program**, so a build that requires
|
|
135
|
+
it has a dependency that appears in no manifest — and it is absent by default on
|
|
136
|
+
Windows, where `make build` fails before it can print anything. The claim is that
|
|
137
|
+
you need Python and nothing else, and that should be true of the build too.
|
|
138
|
+
|
|
139
|
+
---
|
|
140
|
+
|
|
141
|
+
## Verify the zero-dependency claim in one command
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
python make.py check-deps
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
```
|
|
148
|
+
python: 3.11.9 (win32)
|
|
149
|
+
scanned: src/**/*.py, tests/**/*.py, tools/**/*.py, examples/**/*.py
|
|
150
|
+
method: ast.parse + sys.stdlib_module_names (no imports executed)
|
|
151
|
+
|
|
152
|
+
standard library modules imported (31):
|
|
153
|
+
argparse examples/video_demo.py, src/nobroker/cli.py, tools/check_deps.py
|
|
154
|
+
ast tools/check_deps.py
|
|
155
|
+
…
|
|
156
|
+
first-party modules (1):
|
|
157
|
+
nobroker
|
|
158
|
+
|
|
159
|
+
third-party dependencies: 0
|
|
160
|
+
|
|
161
|
+
VERDICT: nobroker runs on the Python standard library alone.
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
It parses every file in `src/`, `tests/`, `tools/` and `examples/` with `ast` —
|
|
165
|
+
reading the source **without importing it**, since importing a module to inspect
|
|
166
|
+
it runs its top-level code — and checks each imported top-level module against
|
|
167
|
+
`sys.stdlib_module_names`, the frozen set maintained by the people who decide what
|
|
168
|
+
the standard library is. The committed output is [deps-proof.txt](deps-proof.txt),
|
|
169
|
+
and CI fails the build if it ever changes.
|
|
170
|
+
|
|
171
|
+
Proving this with `deptry` would have been self-refuting, so it is 100 lines of
|
|
172
|
+
`ast` instead. The claim is checked at three levels: source imports, the manifest
|
|
173
|
+
(`dependencies = []`, empty [requirements.txt](requirements.txt)), and
|
|
174
|
+
`Requires-Dist` inside the built wheel.
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## Thirty seconds
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
from nobroker import Queue, Worker
|
|
182
|
+
|
|
183
|
+
q = Queue("./jobs")
|
|
184
|
+
q.enqueue({"send_email_to": "ada@example.com"})
|
|
185
|
+
|
|
186
|
+
def handle(job):
|
|
187
|
+
send_email(**job.payload) # raising = retry with backoff
|
|
188
|
+
|
|
189
|
+
Worker(q, handle).run() # Ctrl-C finishes in-flight jobs, then exits
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Or without a worker, if you want the loop yourself:
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
job = q.lease_one() # invisible to everyone else for 30s
|
|
196
|
+
if job:
|
|
197
|
+
try:
|
|
198
|
+
do_the_work(job.payload)
|
|
199
|
+
q.ack(job) # done
|
|
200
|
+
except Exception as exc:
|
|
201
|
+
q.nack(job, error=str(exc)) # retry later, or DLQ after max_attempts
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
Or from a shell:
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
nobroker enqueue '{"resize": "photo.jpg"}' --priority 5
|
|
208
|
+
nobroker stats
|
|
209
|
+
nobroker work myapp.tasks:handle --concurrency 4 # see examples/handlers.py
|
|
210
|
+
nobroker dlq --requeue # after you fix the bug
|
|
211
|
+
nobroker inspect # read the raw log, record by record
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
`examples/handlers.py` has runnable handlers you can point that at from a clone,
|
|
215
|
+
including a slow one built for killing mid-job:
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
python make.py build
|
|
219
|
+
python dist/nobroker.pyz --dir /tmp/q enqueue '{"report":"q3"}'
|
|
220
|
+
python dist/nobroker.pyz --dir /tmp/q work examples.handlers:slow
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
225
|
+
## Delivery semantics: **at-least-once**
|
|
226
|
+
|
|
227
|
+
Read this section before you use nobroker for anything.
|
|
228
|
+
|
|
229
|
+
**A job can be delivered more than once. Your handlers must be idempotent.**
|
|
230
|
+
|
|
231
|
+
This is not a limitation nobroker will fix in a later version — it is the
|
|
232
|
+
strongest honest guarantee any queue of this shape can offer. Here is the exact
|
|
233
|
+
window:
|
|
234
|
+
|
|
235
|
+
1. A worker leases a job and starts running the handler.
|
|
236
|
+
2. The handler completes its side effect (the email is sent, the row is written).
|
|
237
|
+
3. The worker is killed before it can call `ack()`.
|
|
238
|
+
4. The lease expires, the job becomes visible again, another worker runs it.
|
|
239
|
+
|
|
240
|
+
The email is sent twice. No amount of engineering closes that window, because
|
|
241
|
+
step 2 and step 3 are in different systems. Exactly-once would require the
|
|
242
|
+
handler's side effect and the queue's acknowledgement to commit in a *single*
|
|
243
|
+
transaction — which means the queue must live inside your database, and then it
|
|
244
|
+
is not a general-purpose queue any more.
|
|
245
|
+
|
|
246
|
+
**What nobroker does instead:**
|
|
247
|
+
|
|
248
|
+
- **Fencing tokens.** Every lease carries a token. If your lease expires and the
|
|
249
|
+
job is redelivered elsewhere, your `ack()` is *rejected* with `NotLeasedError`
|
|
250
|
+
rather than silently completing someone else's delivery. You find out.
|
|
251
|
+
- **Lease heartbeats.** `Worker` extends the lease of a running handler, so a
|
|
252
|
+
slow-but-healthy worker is not a duplicate-work source. Only real failures
|
|
253
|
+
cause redelivery.
|
|
254
|
+
- **Idempotent enqueue.** `q.enqueue(payload, job_id="order-42")` de-duplicates
|
|
255
|
+
on the key. This is the one place exactly-once *is* honestly available, because
|
|
256
|
+
de-duplication on a key is something a log can actually do. A retried HTTP
|
|
257
|
+
request that enqueues twice schedules one job.
|
|
258
|
+
|
|
259
|
+
---
|
|
260
|
+
|
|
261
|
+
## What it guarantees
|
|
262
|
+
|
|
263
|
+
| Property | How |
|
|
264
|
+
|---|---|
|
|
265
|
+
| If `enqueue()` returned, the job is on disk | `fsync` before return, always (unless you pass `fsync=False`) |
|
|
266
|
+
| A crash at any point leaves every unacked job recoverable | Append-only log; the interrupted write is the only thing lost, and its caller never got a return value |
|
|
267
|
+
| A torn write is detected, never silently applied | CRC32 on every record; recovery truncates back to the last clean one |
|
|
268
|
+
| Log damage is reported, not hidden | `queue.recovery.repaired`, and `nobroker recover` |
|
|
269
|
+
| Two processes never lease the same job | Kernel file lock held across the whole read-modify-append |
|
|
270
|
+
| A dead worker's jobs come back | Visibility timeout; the kernel releases the lock on death |
|
|
271
|
+
| Replaying a log twice gives identical state | No handler reads the clock or calls `random` — see below |
|
|
272
|
+
| Compaction is all-or-nothing | New generation written and fsynced, then one atomic pointer flip |
|
|
273
|
+
|
|
274
|
+
### The property that makes the rest testable
|
|
275
|
+
|
|
276
|
+
**Replay is a pure function of the log.** Nothing in the replay path reads the
|
|
277
|
+
clock, generates a UUID, or samples jitter. Every non-deterministic value — the
|
|
278
|
+
lease deadline, the jittered retry time, the job id — is decided *once* by the
|
|
279
|
+
writer and recorded as an absolute value.
|
|
280
|
+
|
|
281
|
+
That is why the test suite can take a real log, truncate it at **every single
|
|
282
|
+
byte offset**, and assert that recovery lands somewhere consistent at each one.
|
|
283
|
+
A crash can only interrupt a write at a byte boundary, so correctness at all
|
|
284
|
+
~1,300 boundaries of a real log is correctness for any crash that log could have
|
|
285
|
+
suffered.
|
|
286
|
+
|
|
287
|
+
---
|
|
288
|
+
|
|
289
|
+
## How it works
|
|
290
|
+
|
|
291
|
+
```
|
|
292
|
+
jobs/
|
|
293
|
+
emails.000001.log the write-ahead log — the only durable state
|
|
294
|
+
emails.current one line: which generation is authoritative
|
|
295
|
+
emails.lock the cross-process lock
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
A new queue has only two of those: `emails.000000.log` and a zero-byte lock file
|
|
299
|
+
whose only job is to exist, so the kernel has something to arbitrate on. The
|
|
300
|
+
`.current` pointer appears at the first compaction, when there is finally more
|
|
301
|
+
than one generation to choose between.
|
|
302
|
+
|
|
303
|
+
Everything in memory — the priority heap, the lease table, the DLQ — is a *cache*
|
|
304
|
+
of the log. There is no second source of truth to keep consistent with it.
|
|
305
|
+
|
|
306
|
+
**Record framing.** Records are appended, never modified:
|
|
307
|
+
|
|
308
|
+
```
|
|
309
|
+
file header: <8s magic><H version><I generation><H reserved> 16 bytes
|
|
310
|
+
record: <I length><B type><I crc32><json payload> 9 + n bytes
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
The length prefix catches an incomplete record. The CRC catches a full-length
|
|
314
|
+
record with a hole in the middle — the failure that would otherwise be applied
|
|
315
|
+
silently. The payload is JSON on purpose: a durability format you cannot read
|
|
316
|
+
with your eyes is one you cannot debug. `nobroker inspect` prints it.
|
|
317
|
+
|
|
318
|
+
**Every operation is four beats:**
|
|
319
|
+
|
|
320
|
+
1. Take the file lock.
|
|
321
|
+
2. Read forward from our last offset — applying whatever peers appended.
|
|
322
|
+
3. Reclaim leases whose visibility timeout expired.
|
|
323
|
+
4. Append the new records, `fsync`, then apply them in memory.
|
|
324
|
+
|
|
325
|
+
Step 2 is what makes multi-process work with no coordinator: a process that has
|
|
326
|
+
been idle for an hour simply reads forward and finds out what happened. Step 4's
|
|
327
|
+
ordering is not negotiable — the record reaches disk *before* memory changes, so
|
|
328
|
+
a crash between the two replays to the same state.
|
|
329
|
+
|
|
330
|
+
**Compaction** writes a new numbered generation containing one record per live
|
|
331
|
+
job, then flips `emails.current`. That flip is the commit point and it is a
|
|
332
|
+
single atomic `os.replace` of a file nobody holds open. Before it, the old
|
|
333
|
+
generation is authoritative and the new file is ignorable garbage; after it, the
|
|
334
|
+
reverse. Peers notice by comparing an integer.
|
|
335
|
+
|
|
336
|
+
---
|
|
337
|
+
|
|
338
|
+
## Benchmarks
|
|
339
|
+
|
|
340
|
+
Measured on the development machine (Windows 11, NVMe SSD, Python 3.11.9) with
|
|
341
|
+
`python make.py bench`. **Run it yourself — that is the only number that matters
|
|
342
|
+
for your hardware,** and NVMe versus spinning rust changes the fsync rows by
|
|
343
|
+
orders of magnitude.
|
|
344
|
+
|
|
345
|
+
| Operation | ops/sec | µs/op | Notes |
|
|
346
|
+
|---|---:|---:|---|
|
|
347
|
+
| enqueue (fsync per job) | 1,348 | 741.7 | the durability guarantee, paid one job at a time |
|
|
348
|
+
| enqueue_many (one fsync) | 57,676 | 17.3 | batching amortises the fsync; same durability at the batch boundary |
|
|
349
|
+
| enqueue (fsync=False) | 6,014 | 166.3 | **not durable** — a machine crash loses recent jobs. Shown for contrast |
|
|
350
|
+
| lease+ack round trip | 1,433 | 697.9 | leases in batches of 100, acks individually |
|
|
351
|
+
| cold-start replay | 47,732 | 21.0 | full replay on open, including CRC of every record |
|
|
352
|
+
| compact | 537,184 | 1.9 | 2.6 MB → 16 bytes |
|
|
353
|
+
| nack + reschedule | 6,712 | 149.0 | computes backoff, rewrites availability, re-heaps |
|
|
354
|
+
|
|
355
|
+
### Is it faster than Redis?
|
|
356
|
+
|
|
357
|
+
**No.** Redis will do 50,000–100,000 ops/sec on a loopback connection. nobroker
|
|
358
|
+
does ~1,300 durable enqueues/sec, because it calls `fsync` and Redis (by
|
|
359
|
+
default) does not. That is not a fair fight in either direction: nobroker is
|
|
360
|
+
paying for a guarantee Redis is not making.
|
|
361
|
+
|
|
362
|
+
The honest comparisons:
|
|
363
|
+
|
|
364
|
+
- **Against Redis with `appendfsync always`** — the configuration that makes the
|
|
365
|
+
same promise — you are in the same order of magnitude, and nobroker saves you a
|
|
366
|
+
network hop and a server.
|
|
367
|
+
- **Against Redis default (`appendfsync everysec`)** — Redis wins on throughput
|
|
368
|
+
and can lose up to a second of acknowledged writes. `Queue(fsync=False)` is the
|
|
369
|
+
comparable nobroker setting, and it is labelled "not durable" everywhere it
|
|
370
|
+
appears.
|
|
371
|
+
- **On batches**, `enqueue_many` does 57k/sec durably, because fsync costs the
|
|
372
|
+
same for one record as for a thousand.
|
|
373
|
+
|
|
374
|
+
If you need 100k jobs/sec, run a broker. If you need 1,000 jobs/sec that are
|
|
375
|
+
still there after the power cut, this is simpler and there is nothing to operate.
|
|
376
|
+
|
|
377
|
+
---
|
|
378
|
+
|
|
379
|
+
## Limits
|
|
380
|
+
|
|
381
|
+
Stated plainly, because a naive implementation that is honest about its corners
|
|
382
|
+
beats a fast one that hides them.
|
|
383
|
+
|
|
384
|
+
- **Single machine only.** The lock is a kernel file lock; it does not work over
|
|
385
|
+
NFS or SMB and it will not coordinate two hosts. There is no distributed mode
|
|
386
|
+
and there will not be one.
|
|
387
|
+
- **At-least-once, never exactly-once.** See above. Handlers must be idempotent.
|
|
388
|
+
- **Polling, not push.** Workers poll (default every 100 ms). A broker-less queue
|
|
389
|
+
has nobody to send a notification. An idle worker costs one `stat` and one short
|
|
390
|
+
read per poll; it is cheap, but it is not free and it is not zero-latency.
|
|
391
|
+
- **The whole index lives in memory.** One `Job` per job, all of them resident.
|
|
392
|
+
Roughly 500 bytes each, so a million jobs is ~500 MB. Fine for the workloads
|
|
393
|
+
this targets; not a database.
|
|
394
|
+
- **`stats()` is O(n).** It counts by iterating every job. Honest and simple; if
|
|
395
|
+
you have a million jobs and poll stats in a tight loop, you will notice.
|
|
396
|
+
- **Recovery reads the entire log at startup.** ~48k records/sec, so a 1M-record
|
|
397
|
+
log takes ~20 seconds to open. Compact regularly.
|
|
398
|
+
- **No result backend, no chaining, no workflows, no cron DSL, no async
|
|
399
|
+
handlers.** All deliberately out of scope.
|
|
400
|
+
- **Payloads must be JSON-serialisable.** No bytes, no arbitrary objects. That is
|
|
401
|
+
the price of a log you can read.
|
|
402
|
+
- **Windows caveats.** Directory `fsync` does not exist there, so the durability
|
|
403
|
+
of a rename is weaker than on POSIX (the file contents are still fsynced). The
|
|
404
|
+
crash tests use `multiprocessing` with `spawn` instead of `os.fork`, which does
|
|
405
|
+
not exist on Windows.
|
|
406
|
+
- **Clock-dependent.** Visibility timeouts and delays use the wall clock. A large
|
|
407
|
+
backwards clock jump (NTP step, not slew) can delay reclaims by that amount.
|
|
408
|
+
|
|
409
|
+
---
|
|
410
|
+
|
|
411
|
+
## Questions a reviewer will ask
|
|
412
|
+
|
|
413
|
+
### "Why not just use `sqlite3`? It's in the standard library too."
|
|
414
|
+
|
|
415
|
+
SQLite would give me storage and transactions. It would not give me any of the
|
|
416
|
+
things this project actually is: lease semantics, visibility timeouts, backoff
|
|
417
|
+
with jitter, fencing tokens, a dead-letter queue, priority ordering with delayed
|
|
418
|
+
jobs, or crash-recovery semantics I can reason about.
|
|
419
|
+
|
|
420
|
+
A job queue on SQLite is *all of this same code*, written on top of a query layer
|
|
421
|
+
I do not need, plus a schema, plus the `BEGIN IMMEDIATE`/`busy_timeout` dance to
|
|
422
|
+
make polling workers not livelock. It would be more code, not less, and the
|
|
423
|
+
durability story would be "SQLite handles it" — which is true, and also means I
|
|
424
|
+
would have learned nothing about the thing this hackathon is about.
|
|
425
|
+
|
|
426
|
+
The honest counterpoint: SQLite's storage engine is vastly better tested than
|
|
427
|
+
mine. If you need a queue you would bet a company on today, that is a real
|
|
428
|
+
argument for it. If you want to see what the primitives underneath actually cost,
|
|
429
|
+
this is the more interesting build.
|
|
430
|
+
|
|
431
|
+
### "Why is `enqueue` only ~1,300/sec?"
|
|
432
|
+
|
|
433
|
+
Because it calls `fsync` and waits. That is the product. `enqueue_many` gets 57k
|
|
434
|
+
because one fsync covers the batch. `fsync=False` gets 6k and is not durable.
|
|
435
|
+
|
|
436
|
+
### "Isn't a file lock per operation slow?"
|
|
437
|
+
|
|
438
|
+
It was — it was 44% of a non-durable enqueue, spent opening and closing the lock
|
|
439
|
+
file. The descriptor is now held for the queue's lifetime and only the lock is
|
|
440
|
+
taken and released, which was a 2.4× improvement with no semantic change. Both
|
|
441
|
+
`flock` and `msvcrt.locking` associate the lock with the open file description,
|
|
442
|
+
so this is exactly as exclusive as reopening each time.
|
|
443
|
+
|
|
444
|
+
### "What happens if two processes compact at once?"
|
|
445
|
+
|
|
446
|
+
They cannot. Compaction runs inside the same lock as everything else.
|
|
447
|
+
|
|
448
|
+
---
|
|
449
|
+
|
|
450
|
+
## API
|
|
451
|
+
|
|
452
|
+
```python
|
|
453
|
+
Queue(path, name="default", *, fsync=True, visibility_timeout=30.0,
|
|
454
|
+
max_attempts=5, backoff=BackoffPolicy(), lock_timeout=10.0)
|
|
455
|
+
|
|
456
|
+
.enqueue(payload, *, priority=0, delay=0.0, max_attempts=None, job_id=None) -> Job
|
|
457
|
+
.enqueue_many(payloads, *, priority=0, delay=0.0, ...) -> list[Job]
|
|
458
|
+
.lease(count=1, *, visibility_timeout=None) -> list[Job]
|
|
459
|
+
.lease_one(*, visibility_timeout=None) -> Job | None
|
|
460
|
+
.ack(job) # raises NotLeasedError if stale
|
|
461
|
+
.nack(job, *, error=None, delay=None) -> Job # retry, or DLQ if exhausted
|
|
462
|
+
.extend(job, seconds) -> Job # buy a slow handler more time
|
|
463
|
+
.get(job_id) -> Job
|
|
464
|
+
.stats() -> QueueStats
|
|
465
|
+
.list_jobs(state=None, *, limit=None) -> list[Job]
|
|
466
|
+
.dlq(*, limit=None) -> list[Job]
|
|
467
|
+
.requeue_dead(job_id=None) -> int # None revives the whole DLQ
|
|
468
|
+
.purge() -> int
|
|
469
|
+
.compact() -> CompactionResult
|
|
470
|
+
.close()
|
|
471
|
+
|
|
472
|
+
Worker(queue, handler, *, concurrency=1, poll_interval=0.1,
|
|
473
|
+
max_jobs=None, idle_timeout=None, handle_signals=True).run() -> WorkerStats
|
|
474
|
+
|
|
475
|
+
BackoffPolicy(base=1.0, factor=2.0, max_delay=300.0, jitter=0.5)
|
|
476
|
+
```
|
|
477
|
+
|
|
478
|
+
Errors all derive from `NobrokerError`: `NotLeasedError`, `JobNotFoundError`,
|
|
479
|
+
`LockTimeoutError`, `CorruptLogError`, `CompactionError`, `QueueClosedError`,
|
|
480
|
+
`SerializationError`.
|
|
481
|
+
|
|
482
|
+
---
|
|
483
|
+
|
|
484
|
+
## Development
|
|
485
|
+
|
|
486
|
+
`python make.py` on its own lists every task. Each has a `make` alias.
|
|
487
|
+
|
|
488
|
+
```bash
|
|
489
|
+
python make.py build # dist/nobroker.pyz — one runnable file, stdlib zipapp
|
|
490
|
+
python make.py test # 124 tests, ~42s (the truncation sweep dominates)
|
|
491
|
+
python make.py test-quick # ~8s, skips that sweep
|
|
492
|
+
python make.py bench # throughput on your machine
|
|
493
|
+
python make.py bench-md # the same, as a Markdown table for this README
|
|
494
|
+
python make.py check-deps # fails if any non-stdlib import exists anywhere
|
|
495
|
+
python make.py deps-proof # regenerate deps-proof.txt on this machine
|
|
496
|
+
python make.py demo # the tour
|
|
497
|
+
python make.py video-demo # the same material, paced for screen recording
|
|
498
|
+
python make.py clean # caches, scratch queues, build output
|
|
499
|
+
```
|
|
500
|
+
|
|
501
|
+
Arguments pass through: `python make.py bench --jobs 100`, or
|
|
502
|
+
`python make.py video-demo --scenes 6` to run one scene of the demo alone.
|
|
503
|
+
|
|
504
|
+
```
|
|
505
|
+
src/nobroker/ 11 modules, in strict dependency order (errors -> … -> queue)
|
|
506
|
+
src/__main__.py entry point for the zipapp bundle
|
|
507
|
+
tests/ 8 files, 124 tests, unittest only
|
|
508
|
+
tools/ check_deps.py (the dependency proof), build_pyz.py (the bundle)
|
|
509
|
+
examples/ demo.py (the tour), video_demo.py (paced), handlers.py (for `work`)
|
|
510
|
+
web/ docs site and playground — two static files, no build step
|
|
511
|
+
make.py every task; the Makefile delegates here
|
|
512
|
+
```
|
|
513
|
+
|
|
514
|
+
**~3,000 lines of library, ~1,700 lines of tests, 0 dependencies.**
|
|
515
|
+
|
|
516
|
+
The test suite deliberately over-invests in one thing:
|
|
517
|
+
|
|
518
|
+
- `test_crash_recovery.py` — hard kills via `os._exit(9)`, and the truncation
|
|
519
|
+
sweep over every byte offset of a real log
|
|
520
|
+
- `test_concurrency.py` — four separate OS processes racing on one queue, each
|
|
521
|
+
recording which job ids it received; the parent asserts the sets are disjoint
|
|
522
|
+
and complete
|
|
523
|
+
|
|
524
|
+
Both of those caught real bugs during development. The multi-process test found
|
|
525
|
+
a stale cached file offset that made peers overwrite each other's records; the
|
|
526
|
+
compaction test found `os.open` defaulting to text mode on Windows and corrupting
|
|
527
|
+
the log. Neither would have shown up in a single-process happy-path test.
|
|
528
|
+
|
|
529
|
+
See **[STDLIB.md](STDLIB.md)** for all 22 "I would normally have installed X"
|
|
530
|
+
decisions, including the five places the standard library was genuinely the worse
|
|
531
|
+
option.
|
|
532
|
+
|
|
533
|
+
---
|
|
534
|
+
|
|
535
|
+
## License
|
|
536
|
+
|
|
537
|
+
MIT.
|