nobroker 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. nobroker-0.1.0/.zero-dep.toml +18 -0
  2. nobroker-0.1.0/LICENSE +21 -0
  3. nobroker-0.1.0/MANIFEST.in +18 -0
  4. nobroker-0.1.0/Makefile +57 -0
  5. nobroker-0.1.0/PKG-INFO +537 -0
  6. nobroker-0.1.0/README.md +504 -0
  7. nobroker-0.1.0/STDLIB.md +391 -0
  8. nobroker-0.1.0/deps-proof.txt +45 -0
  9. nobroker-0.1.0/examples/demo.py +167 -0
  10. nobroker-0.1.0/examples/handlers.py +68 -0
  11. nobroker-0.1.0/pyproject.toml +61 -0
  12. nobroker-0.1.0/requirements.txt +9 -0
  13. nobroker-0.1.0/setup.cfg +4 -0
  14. nobroker-0.1.0/src/nobroker/__init__.py +56 -0
  15. nobroker-0.1.0/src/nobroker/backoff.py +58 -0
  16. nobroker-0.1.0/src/nobroker/bench.py +210 -0
  17. nobroker-0.1.0/src/nobroker/cli.py +393 -0
  18. nobroker-0.1.0/src/nobroker/codec.py +170 -0
  19. nobroker-0.1.0/src/nobroker/errors.py +60 -0
  20. nobroker-0.1.0/src/nobroker/index.py +358 -0
  21. nobroker-0.1.0/src/nobroker/job.py +139 -0
  22. nobroker-0.1.0/src/nobroker/lock.py +170 -0
  23. nobroker-0.1.0/src/nobroker/logfile.py +463 -0
  24. nobroker-0.1.0/src/nobroker/py.typed +2 -0
  25. nobroker-0.1.0/src/nobroker/queue.py +738 -0
  26. nobroker-0.1.0/src/nobroker/worker.py +268 -0
  27. nobroker-0.1.0/src/nobroker.egg-info/PKG-INFO +537 -0
  28. nobroker-0.1.0/src/nobroker.egg-info/SOURCES.txt +41 -0
  29. nobroker-0.1.0/src/nobroker.egg-info/dependency_links.txt +1 -0
  30. nobroker-0.1.0/src/nobroker.egg-info/entry_points.txt +2 -0
  31. nobroker-0.1.0/src/nobroker.egg-info/top_level.txt +1 -0
  32. nobroker-0.1.0/tests/__init__.py +13 -0
  33. nobroker-0.1.0/tests/support.py +71 -0
  34. nobroker-0.1.0/tests/test_backoff.py +60 -0
  35. nobroker-0.1.0/tests/test_cli.py +210 -0
  36. nobroker-0.1.0/tests/test_codec.py +148 -0
  37. nobroker-0.1.0/tests/test_concurrency.py +204 -0
  38. nobroker-0.1.0/tests/test_crash_recovery.py +274 -0
  39. nobroker-0.1.0/tests/test_logfile.py +157 -0
  40. nobroker-0.1.0/tests/test_queue.py +397 -0
  41. nobroker-0.1.0/tests/test_worker.py +211 -0
  42. nobroker-0.1.0/tools/build_pyz.py +95 -0
  43. nobroker-0.1.0/tools/check_deps.py +117 -0
@@ -0,0 +1,18 @@
1
+ track = "D"
2
+ name = "nobroker"
3
+ pitch = "A durable, crash-safe job queue for Python with no broker, no server, and no dependencies -- just a file."
4
+
5
+ language = "python"
6
+ python_requires = ">=3.11"
7
+ entrypoint = "make demo"
8
+ runtime_dependencies = 0
9
+
10
+ bonuses = ["package-killer", "stdlib-log"]
11
+
12
+ # Package Killer: replaces celery + redis (or rq, dramatiq, huey) for the
13
+ # single-machine case -- roughly 40 MB of transitive dependencies and a server
14
+ # process, traded for one file and the standard library.
15
+ replaces = ["celery", "redis", "rq", "dramatiq", "huey", "kombu", "billiard"]
16
+
17
+ # STDLIB Log: see STDLIB.md.
18
+ stdlib_log = "STDLIB.md"
nobroker-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ayan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,18 @@
1
+ # What goes into the source distribution beyond the package itself.
2
+ include LICENSE
3
+ include README.md
4
+ include STDLIB.md
5
+ include deps-proof.txt
6
+ include requirements.txt
7
+ include .zero-dep.toml
8
+ include Makefile
9
+ recursive-include src/nobroker py.typed
10
+ recursive-include tests *.py
11
+ recursive-include tools *.py
12
+ recursive-include examples *.py
13
+
14
+ # The website and the process notes are not part of the distribution.
15
+ prune web
16
+ prune others
17
+ prune dist
18
+ exclude src/__main__.py
@@ -0,0 +1,57 @@
1
+ # nobroker -- one command for everything.
2
+ #
3
+ # make build -> dist/nobroker.pyz, the whole tool as one runnable file
4
+ # make demo -> the end-to-end tour
5
+ # make test -> 122 tests, including crash recovery
6
+ #
7
+ # Every target delegates to make.py, which holds the actual task definitions, so
8
+ # the two can never drift. If you do not have `make` -- which is the default on
9
+ # Windows -- use make.py directly and nothing changes:
10
+ #
11
+ # python make.py build
12
+ #
13
+ # That is deliberate. `make` is a separately installed tool, and this project's
14
+ # claim is that you need Python and nothing else. The Makefile exists because
15
+ # `make test` is what a reviewer's fingers type; make.py exists because it is
16
+ # what actually runs everywhere.
17
+
18
+ PYTHON ?= python3
19
+
20
+ .DEFAULT_GOAL := help
21
+ .PHONY: help build demo video-demo test test-quick bench bench-md check-deps deps-proof cli clean
22
+
23
+ help: ## List every task
24
+ @$(PYTHON) make.py
25
+
26
+ build: ## Bundle the tool into one runnable file (dist/nobroker.pyz)
27
+ @$(PYTHON) make.py build
28
+
29
+ demo: ## Run the end-to-end tour (start here)
30
+ @$(PYTHON) make.py demo
31
+
32
+ video-demo: ## Run the paced demo built for screen recording
33
+ @$(PYTHON) make.py video-demo
34
+
35
+ test: ## Run the full test suite, including crash recovery
36
+ @$(PYTHON) make.py test
37
+
38
+ test-quick: ## Run everything except the slow exhaustive-truncation sweep
39
+ @$(PYTHON) make.py test-quick
40
+
41
+ bench: ## Measure throughput on this machine
42
+ @$(PYTHON) make.py bench
43
+
44
+ bench-md: ## Same, as a Markdown table for the README
45
+ @$(PYTHON) make.py bench-md
46
+
47
+ check-deps: ## Fail if anything outside the standard library is imported
48
+ @$(PYTHON) make.py check-deps
49
+
50
+ deps-proof: ## Regenerate deps-proof.txt
51
+ @$(PYTHON) make.py deps-proof
52
+
53
+ cli: ## Show the CLI help
54
+ @$(PYTHON) make.py cli
55
+
56
+ clean: ## Remove caches, scratch queues and build output
57
+ @$(PYTHON) make.py clean
@@ -0,0 +1,537 @@
1
+ Metadata-Version: 2.4
2
+ Name: nobroker
3
+ Version: 0.1.0
4
+ Summary: A durable, broker-less job queue for Python. No server, no dependencies.
5
+ Author-email: Ayan <ayandgp12@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/ayanbag/nobroker
8
+ Project-URL: Documentation, https://ayanbag.github.io/nobroker/
9
+ Project-URL: Playground, https://ayanbag.github.io/nobroker/playground.html
10
+ Project-URL: Source, https://github.com/ayanbag/nobroker
11
+ Project-URL: Issues, https://github.com/ayanbag/nobroker/issues
12
+ Project-URL: Changelog, https://github.com/ayanbag/nobroker/releases
13
+ Keywords: queue,job-queue,task-queue,durable,wal,write-ahead-log,zero-dependency,at-least-once,crash-safe,worker,background-jobs
14
+ Classifier: Development Status :: 4 - Beta
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: License :: OSI Approved :: MIT License
17
+ Classifier: Operating System :: POSIX
18
+ Classifier: Operating System :: MacOS :: MacOS X
19
+ Classifier: Operating System :: Microsoft :: Windows
20
+ Classifier: Programming Language :: Python :: 3
21
+ Classifier: Programming Language :: Python :: 3.11
22
+ Classifier: Programming Language :: Python :: 3.12
23
+ Classifier: Programming Language :: Python :: 3.13
24
+ Classifier: Programming Language :: Python :: Implementation :: CPython
25
+ Classifier: Topic :: Database :: Database Engines/Servers
26
+ Classifier: Topic :: System :: Distributed Computing
27
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
28
+ Classifier: Typing :: Typed
29
+ Requires-Python: >=3.11
30
+ Description-Content-Type: text/markdown
31
+ License-File: LICENSE
32
+ Dynamic: license-file
33
+
34
+ <p align="center">
35
+ <img src="https://raw.githubusercontent.com/ayanbag/nobroker/master/assets/banner-git.png" alt="nobroker — the job queue with nobody in the middle" width="100%">
36
+ </p>
37
+
38
+ <p align="center">
39
+ <a href="https://github.com/ayanbag/nobroker/actions/workflows/ci.yml"><img alt="tests" src="https://img.shields.io/github/actions/workflow/status/ayanbag/nobroker/ci.yml?branch=main&label=tests&style=flat-square&labelColor=12160F&color=2C6A4E"></a>
40
+ <a href="https://pypi.org/project/nobroker/"><img alt="pypi" src="https://img.shields.io/pypi/v/nobroker?style=flat-square&labelColor=12160F&color=9E3B18"></a>
41
+ <a href="https://pypi.org/project/nobroker/"><img alt="python" src="https://img.shields.io/pypi/pyversions/nobroker?style=flat-square&labelColor=12160F&color=8A6A1F"></a>
42
+ <img alt="dependencies: 0" src="https://img.shields.io/badge/dependencies-0-2C6A4E?style=flat-square&labelColor=12160F">
43
+ <img alt="tests: 124" src="https://img.shields.io/badge/tests-124%20passing-2C6A4E?style=flat-square&labelColor=12160F">
44
+ <img alt="delivery: at-least-once" src="https://img.shields.io/badge/delivery-at--least--once-8A6A1F?style=flat-square&labelColor=12160F">
45
+ <a href="LICENSE"><img alt="license" src="https://img.shields.io/badge/license-MIT-535E55?style=flat-square&labelColor=12160F"></a>
46
+ </p>
47
+
48
+ <p align="center">
49
+ <b>The job queue with nobody in the middle.</b><br>
50
+ <a href="https://ayanbag.github.io/nobroker/">Docs</a> ·
51
+ <a href="https://ayanbag.github.io/nobroker/playground.html">Interactive playground</a> ·
52
+ <a href="STDLIB.md">STDLIB.md</a> ·
53
+ <a href="deps-proof.txt">Dependency proof</a> ·
54
+ <a href="#limits">Limits</a>
55
+ </p>
56
+
57
+ ---
58
+
59
+ A durable, crash-safe job queue for Python. Enqueue work, lease it, ack it. It
60
+ survives `kill -9`, it is safe across processes, and it needs no Redis, no
61
+ RabbitMQ, no server, and **no dependencies at all** — just the standard library
62
+ and a file.
63
+
64
+ The name is the thesis. Every other job queue makes you run a broker. This one
65
+ doesn't.
66
+
67
+ > Built for the [Zero Dependency Hackathon](https://zerodepshack.com) — **Track D, Data & Storage**.
68
+
69
+ ---
70
+
71
+ ## Why this exists
72
+
73
+ You want a background job queue. Your options today:
74
+
75
+ | | What you install | What you operate |
76
+ |---|---|---|
77
+ | Celery | `celery`, `kombu`, `billiard`, `vine`, `amqp`, … | a Redis or RabbitMQ server |
78
+ | RQ | `rq`, `redis` | a Redis server |
79
+ | Dramatiq | `dramatiq`, `pika` or `redis` | a broker |
80
+ | **nobroker** | **nothing** | **nothing** |
81
+
82
+ For a single machine — a cron box, a CLI tool, a desktop app, a small service, a
83
+ CI runner, a Raspberry Pi — the broker is pure operational overhead. The durable
84
+ storage and the mutual exclusion you actually need are both already in your
85
+ kernel. nobroker is what is left when you use them directly.
86
+
87
+ **It is not a Celery replacement for a fleet.** It does not distribute work
88
+ across machines and never will. See [Limits](#limits).
89
+
90
+ ---
91
+
92
+ ## Install
93
+
94
+ ```bash
95
+ pip install nobroker
96
+ ```
97
+
98
+ That installs one package and nothing else — `dependencies = []` is not a
99
+ rounding error, it is the product.
100
+
101
+ Or skip `pip` entirely — from a fresh clone, with nothing installed:
102
+
103
+ ```bash
104
+ git clone https://github.com/ayanbag/nobroker && cd nobroker
105
+
106
+ python make.py demo # end-to-end tour: enqueue, retry, crash, recover, compact
107
+ python make.py build # -> dist/nobroker.pyz, the whole tool as one 40 KB file
108
+ python dist/nobroker.pyz --help
109
+ ```
110
+
111
+ Requires Python 3.11 or newer. That is the entire dependency list. You can also
112
+ just copy `src/nobroker/` into your project — it is pure standard library, so it
113
+ will run wherever your code does.
114
+
115
+ ### The build is one step, and needs nothing either
116
+
117
+ `python make.py build` produces `dist/nobroker.pyz`: a single 40 KB file that
118
+ runs on any Python 3.11+, with no install and nothing unpacked to disk.
119
+
120
+ ```bash
121
+ PYZ=$PWD/dist/nobroker.pyz
122
+ cd /tmp # anywhere at all: no install, no PYTHONPATH
123
+ python "$PYZ" --dir ./q enqueue '{"resize":"photo.jpg"}'
124
+ python "$PYZ" --dir ./q stats
125
+ ```
126
+
127
+ No compiler, no bundler, no `pip install` — that is `zipapp` from the standard
128
+ library. `shiv` and `pex` exist to solve the *hard* part of single-file packaging,
129
+ which is vendoring third-party dependencies and their compiled extensions. There
130
+ are none here, so the hard part is absent.
131
+
132
+ **`make` also works** (`make build`, `make test`, …) and is a one-line-per-target
133
+ delegation to `make.py`, so the two cannot drift. `make.py` is the canonical one
134
+ on purpose: `make` is a **separately installed program**, so a build that requires
135
+ it has a dependency that appears in no manifest — and it is absent by default on
136
+ Windows, where `make build` fails before it can print anything. The claim is that
137
+ you need Python and nothing else, and that should be true of the build too.
138
+
139
+ ---
140
+
141
+ ## Verify the zero-dependency claim in one command
142
+
143
+ ```bash
144
+ python make.py check-deps
145
+ ```
146
+
147
+ ```
148
+ python: 3.11.9 (win32)
149
+ scanned: src/**/*.py, tests/**/*.py, tools/**/*.py, examples/**/*.py
150
+ method: ast.parse + sys.stdlib_module_names (no imports executed)
151
+
152
+ standard library modules imported (31):
153
+ argparse examples/video_demo.py, src/nobroker/cli.py, tools/check_deps.py
154
+ ast tools/check_deps.py
155
+
156
+ first-party modules (1):
157
+ nobroker
158
+
159
+ third-party dependencies: 0
160
+
161
+ VERDICT: nobroker runs on the Python standard library alone.
162
+ ```
163
+
164
+ It parses every file in `src/`, `tests/`, `tools/` and `examples/` with `ast` —
165
+ reading the source **without importing it**, since importing a module to inspect
166
+ it runs its top-level code — and checks each imported top-level module against
167
+ `sys.stdlib_module_names`, the frozen set maintained by the people who decide what
168
+ the standard library is. The committed output is [deps-proof.txt](deps-proof.txt),
169
+ and CI fails the build if it ever changes.
170
+
171
+ Proving this with `deptry` would have been self-refuting, so it is 100 lines of
172
+ `ast` instead. The claim is checked at three levels: source imports, the manifest
173
+ (`dependencies = []`, empty [requirements.txt](requirements.txt)), and
174
+ `Requires-Dist` inside the built wheel.
175
+
176
+ ---
177
+
178
+ ## Thirty seconds
179
+
180
+ ```python
181
+ from nobroker import Queue, Worker
182
+
183
+ q = Queue("./jobs")
184
+ q.enqueue({"send_email_to": "ada@example.com"})
185
+
186
+ def handle(job):
187
+ send_email(**job.payload) # raising = retry with backoff
188
+
189
+ Worker(q, handle).run() # Ctrl-C finishes in-flight jobs, then exits
190
+ ```
191
+
192
+ Or without a worker, if you want the loop yourself:
193
+
194
+ ```python
195
+ job = q.lease_one() # invisible to everyone else for 30s
196
+ if job:
197
+ try:
198
+ do_the_work(job.payload)
199
+ q.ack(job) # done
200
+ except Exception as exc:
201
+ q.nack(job, error=str(exc)) # retry later, or DLQ after max_attempts
202
+ ```
203
+
204
+ Or from a shell:
205
+
206
+ ```bash
207
+ nobroker enqueue '{"resize": "photo.jpg"}' --priority 5
208
+ nobroker stats
209
+ nobroker work myapp.tasks:handle --concurrency 4 # see examples/handlers.py
210
+ nobroker dlq --requeue # after you fix the bug
211
+ nobroker inspect # read the raw log, record by record
212
+ ```
213
+
214
+ `examples/handlers.py` has runnable handlers you can point that at from a clone,
215
+ including a slow one built for killing mid-job:
216
+
217
+ ```bash
218
+ python make.py build
219
+ python dist/nobroker.pyz --dir /tmp/q enqueue '{"report":"q3"}'
220
+ python dist/nobroker.pyz --dir /tmp/q work examples.handlers:slow
221
+ ```
222
+
223
+ ---
224
+
225
+ ## Delivery semantics: **at-least-once**
226
+
227
+ Read this section before you use nobroker for anything.
228
+
229
+ **A job can be delivered more than once. Your handlers must be idempotent.**
230
+
231
+ This is not a limitation nobroker will fix in a later version — it is the
232
+ strongest honest guarantee any queue of this shape can offer. Here is the exact
233
+ window:
234
+
235
+ 1. A worker leases a job and starts running the handler.
236
+ 2. The handler completes its side effect (the email is sent, the row is written).
237
+ 3. The worker is killed before it can call `ack()`.
238
+ 4. The lease expires, the job becomes visible again, another worker runs it.
239
+
240
+ The email is sent twice. No amount of engineering closes that window, because
241
+ step 2 and step 3 are in different systems. Exactly-once would require the
242
+ handler's side effect and the queue's acknowledgement to commit in a *single*
243
+ transaction — which means the queue must live inside your database, and then it
244
+ is not a general-purpose queue any more.
245
+
246
+ **What nobroker does instead:**
247
+
248
+ - **Fencing tokens.** Every lease carries a token. If your lease expires and the
249
+ job is redelivered elsewhere, your `ack()` is *rejected* with `NotLeasedError`
250
+ rather than silently completing someone else's delivery. You find out.
251
+ - **Lease heartbeats.** `Worker` extends the lease of a running handler, so a
252
+ slow-but-healthy worker is not a duplicate-work source. Only real failures
253
+ cause redelivery.
254
+ - **Idempotent enqueue.** `q.enqueue(payload, job_id="order-42")` de-duplicates
255
+ on the key. This is the one place exactly-once *is* honestly available, because
256
+ de-duplication on a key is something a log can actually do. A retried HTTP
257
+ request that enqueues twice schedules one job.
258
+
259
+ ---
260
+
261
+ ## What it guarantees
262
+
263
+ | Property | How |
264
+ |---|---|
265
+ | If `enqueue()` returned, the job is on disk | `fsync` before return, always (unless you pass `fsync=False`) |
266
+ | A crash at any point leaves every unacked job recoverable | Append-only log; the interrupted write is the only thing lost, and its caller never got a return value |
267
+ | A torn write is detected, never silently applied | CRC32 on every record; recovery truncates back to the last clean one |
268
+ | Log damage is reported, not hidden | `queue.recovery.repaired`, and `nobroker recover` |
269
+ | Two processes never lease the same job | Kernel file lock held across the whole read-modify-append |
270
+ | A dead worker's jobs come back | Visibility timeout; the kernel releases the lock on death |
271
+ | Replaying a log twice gives identical state | No handler reads the clock or calls `random` — see below |
272
+ | Compaction is all-or-nothing | New generation written and fsynced, then one atomic pointer flip |
273
+
274
+ ### The property that makes the rest testable
275
+
276
+ **Replay is a pure function of the log.** Nothing in the replay path reads the
277
+ clock, generates a UUID, or samples jitter. Every non-deterministic value — the
278
+ lease deadline, the jittered retry time, the job id — is decided *once* by the
279
+ writer and recorded as an absolute value.
280
+
281
+ That is why the test suite can take a real log, truncate it at **every single
282
+ byte offset**, and assert that recovery lands somewhere consistent at each one.
283
+ A crash can only interrupt a write at a byte boundary, so correctness at all
284
+ ~1,300 boundaries of a real log is correctness for any crash that log could have
285
+ suffered.
286
+
287
+ ---
288
+
289
+ ## How it works
290
+
291
+ ```
292
+ jobs/
293
+ emails.000001.log the write-ahead log — the only durable state
294
+ emails.current one line: which generation is authoritative
295
+ emails.lock the cross-process lock
296
+ ```
297
+
298
+ A new queue has only two of those: `emails.000000.log` and a zero-byte lock file
299
+ whose only job is to exist, so the kernel has something to arbitrate on. The
300
+ `.current` pointer appears at the first compaction, when there is finally more
301
+ than one generation to choose between.
302
+
303
+ Everything in memory — the priority heap, the lease table, the DLQ — is a *cache*
304
+ of the log. There is no second source of truth to keep consistent with it.
305
+
306
+ **Record framing.** Records are appended, never modified:
307
+
308
+ ```
309
+ file header: <8s magic><H version><I generation><H reserved> 16 bytes
310
+ record: <I length><B type><I crc32><json payload> 9 + n bytes
311
+ ```
312
+
313
+ The length prefix catches an incomplete record. The CRC catches a full-length
314
+ record with a hole in the middle — the failure that would otherwise be applied
315
+ silently. The payload is JSON on purpose: a durability format you cannot read
316
+ with your eyes is one you cannot debug. `nobroker inspect` prints it.
317
+
318
+ **Every operation is four beats:**
319
+
320
+ 1. Take the file lock.
321
+ 2. Read forward from our last offset — applying whatever peers appended.
322
+ 3. Reclaim leases whose visibility timeout expired.
323
+ 4. Append the new records, `fsync`, then apply them in memory.
324
+
325
+ Step 2 is what makes multi-process work with no coordinator: a process that has
326
+ been idle for an hour simply reads forward and finds out what happened. Step 4's
327
+ ordering is not negotiable — the record reaches disk *before* memory changes, so
328
+ a crash between the two replays to the same state.
329
+
330
+ **Compaction** writes a new numbered generation containing one record per live
331
+ job, then flips `emails.current`. That flip is the commit point and it is a
332
+ single atomic `os.replace` of a file nobody holds open. Before it, the old
333
+ generation is authoritative and the new file is ignorable garbage; after it, the
334
+ reverse. Peers notice by comparing an integer.
335
+
336
+ ---
337
+
338
+ ## Benchmarks
339
+
340
+ Measured on the development machine (Windows 11, NVMe SSD, Python 3.11.9) with
341
+ `python make.py bench`. **Run it yourself — that is the only number that matters
342
+ for your hardware,** and NVMe versus spinning rust changes the fsync rows by
343
+ orders of magnitude.
344
+
345
+ | Operation | ops/sec | µs/op | Notes |
346
+ |---|---:|---:|---|
347
+ | enqueue (fsync per job) | 1,348 | 741.7 | the durability guarantee, paid one job at a time |
348
+ | enqueue_many (one fsync) | 57,676 | 17.3 | batching amortises the fsync; same durability at the batch boundary |
349
+ | enqueue (fsync=False) | 6,014 | 166.3 | **not durable** — a machine crash loses recent jobs. Shown for contrast |
350
+ | lease+ack round trip | 1,433 | 697.9 | leases in batches of 100, acks individually |
351
+ | cold-start replay | 47,732 | 21.0 | full replay on open, including CRC of every record |
352
+ | compact | 537,184 | 1.9 | 2.6 MB → 16 bytes |
353
+ | nack + reschedule | 6,712 | 149.0 | computes backoff, rewrites availability, re-heaps |
354
+
355
+ ### Is it faster than Redis?
356
+
357
+ **No.** Redis will do 50,000–100,000 ops/sec on a loopback connection. nobroker
358
+ does ~1,300 durable enqueues/sec, because it calls `fsync` and Redis (by
359
+ default) does not. That is not a fair fight in either direction: nobroker is
360
+ paying for a guarantee Redis is not making.
361
+
362
+ The honest comparisons:
363
+
364
+ - **Against Redis with `appendfsync always`** — the configuration that makes the
365
+ same promise — you are in the same order of magnitude, and nobroker saves you a
366
+ network hop and a server.
367
+ - **Against Redis default (`appendfsync everysec`)** — Redis wins on throughput
368
+ and can lose up to a second of acknowledged writes. `Queue(fsync=False)` is the
369
+ comparable nobroker setting, and it is labelled "not durable" everywhere it
370
+ appears.
371
+ - **On batches**, `enqueue_many` does 57k/sec durably, because fsync costs the
372
+ same for one record as for a thousand.
373
+
374
+ If you need 100k jobs/sec, run a broker. If you need 1,000 jobs/sec that are
375
+ still there after the power cut, this is simpler and there is nothing to operate.
376
+
377
+ ---
378
+
379
+ ## Limits
380
+
381
+ Stated plainly, because a naive implementation that is honest about its corners
382
+ beats a fast one that hides them.
383
+
384
+ - **Single machine only.** The lock is a kernel file lock; it does not work over
385
+ NFS or SMB and it will not coordinate two hosts. There is no distributed mode
386
+ and there will not be one.
387
+ - **At-least-once, never exactly-once.** See above. Handlers must be idempotent.
388
+ - **Polling, not push.** Workers poll (default every 100 ms). A broker-less queue
389
+ has nobody to send a notification. An idle worker costs one `stat` and one short
390
+ read per poll; it is cheap, but it is not free and it is not zero-latency.
391
+ - **The whole index lives in memory.** One `Job` per job, all of them resident.
392
+ Roughly 500 bytes each, so a million jobs is ~500 MB. Fine for the workloads
393
+ this targets; not a database.
394
+ - **`stats()` is O(n).** It counts by iterating every job. Honest and simple; if
395
+ you have a million jobs and poll stats in a tight loop, you will notice.
396
+ - **Recovery reads the entire log at startup.** ~48k records/sec, so a 1M-record
397
+ log takes ~20 seconds to open. Compact regularly.
398
+ - **No result backend, no chaining, no workflows, no cron DSL, no async
399
+ handlers.** All deliberately out of scope.
400
+ - **Payloads must be JSON-serialisable.** No bytes, no arbitrary objects. That is
401
+ the price of a log you can read.
402
+ - **Windows caveats.** Directory `fsync` does not exist there, so the durability
403
+ of a rename is weaker than on POSIX (the file contents are still fsynced). The
404
+ crash tests use `multiprocessing` with `spawn` instead of `os.fork`, which does
405
+ not exist on Windows.
406
+ - **Clock-dependent.** Visibility timeouts and delays use the wall clock. A large
407
+ backwards clock jump (NTP step, not slew) can delay reclaims by that amount.
408
+
409
+ ---
410
+
411
+ ## Questions a reviewer will ask
412
+
413
+ ### "Why not just use `sqlite3`? It's in the standard library too."
414
+
415
+ SQLite would give me storage and transactions. It would not give me any of the
416
+ things this project actually is: lease semantics, visibility timeouts, backoff
417
+ with jitter, fencing tokens, a dead-letter queue, priority ordering with delayed
418
+ jobs, or crash-recovery semantics I can reason about.
419
+
420
+ A job queue on SQLite is *all of this same code*, written on top of a query layer
421
+ I do not need, plus a schema, plus the `BEGIN IMMEDIATE`/`busy_timeout` dance to
422
+ make polling workers not livelock. It would be more code, not less, and the
423
+ durability story would be "SQLite handles it" — which is true, and also means I
424
+ would have learned nothing about the thing this hackathon is about.
425
+
426
+ The honest counterpoint: SQLite's storage engine is vastly better tested than
427
+ mine. If you need a queue you would bet a company on today, that is a real
428
+ argument for it. If you want to see what the primitives underneath actually cost,
429
+ this is the more interesting build.
430
+
431
+ ### "Why is `enqueue` only ~1,300/sec?"
432
+
433
+ Because it calls `fsync` and waits. That is the product. `enqueue_many` gets 57k
434
+ because one fsync covers the batch. `fsync=False` gets 6k and is not durable.
435
+
436
+ ### "Isn't a file lock per operation slow?"
437
+
438
+ It was — it was 44% of a non-durable enqueue, spent opening and closing the lock
439
+ file. The descriptor is now held for the queue's lifetime and only the lock is
440
+ taken and released, which was a 2.4× improvement with no semantic change. Both
441
+ `flock` and `msvcrt.locking` associate the lock with the open file description,
442
+ so this is exactly as exclusive as reopening each time.
443
+
444
+ ### "What happens if two processes compact at once?"
445
+
446
+ They cannot. Compaction runs inside the same lock as everything else.
447
+
448
+ ---
449
+
450
+ ## API
451
+
452
+ ```python
453
+ Queue(path, name="default", *, fsync=True, visibility_timeout=30.0,
454
+ max_attempts=5, backoff=BackoffPolicy(), lock_timeout=10.0)
455
+
456
+ .enqueue(payload, *, priority=0, delay=0.0, max_attempts=None, job_id=None) -> Job
457
+ .enqueue_many(payloads, *, priority=0, delay=0.0, ...) -> list[Job]
458
+ .lease(count=1, *, visibility_timeout=None) -> list[Job]
459
+ .lease_one(*, visibility_timeout=None) -> Job | None
460
+ .ack(job) # raises NotLeasedError if stale
461
+ .nack(job, *, error=None, delay=None) -> Job # retry, or DLQ if exhausted
462
+ .extend(job, seconds) -> Job # buy a slow handler more time
463
+ .get(job_id) -> Job
464
+ .stats() -> QueueStats
465
+ .list_jobs(state=None, *, limit=None) -> list[Job]
466
+ .dlq(*, limit=None) -> list[Job]
467
+ .requeue_dead(job_id=None) -> int # None revives the whole DLQ
468
+ .purge() -> int
469
+ .compact() -> CompactionResult
470
+ .close()
471
+
472
+ Worker(queue, handler, *, concurrency=1, poll_interval=0.1,
473
+ max_jobs=None, idle_timeout=None, handle_signals=True).run() -> WorkerStats
474
+
475
+ BackoffPolicy(base=1.0, factor=2.0, max_delay=300.0, jitter=0.5)
476
+ ```
477
+
478
+ Errors all derive from `NobrokerError`: `NotLeasedError`, `JobNotFoundError`,
479
+ `LockTimeoutError`, `CorruptLogError`, `CompactionError`, `QueueClosedError`,
480
+ `SerializationError`.
481
+
482
+ ---
483
+
484
+ ## Development
485
+
486
+ `python make.py` on its own lists every task. Each has a `make` alias.
487
+
488
+ ```bash
489
+ python make.py build # dist/nobroker.pyz — one runnable file, stdlib zipapp
490
+ python make.py test # 124 tests, ~42s (the truncation sweep dominates)
491
+ python make.py test-quick # ~8s, skips that sweep
492
+ python make.py bench # throughput on your machine
493
+ python make.py bench-md # the same, as a Markdown table for this README
494
+ python make.py check-deps # fails if any non-stdlib import exists anywhere
495
+ python make.py deps-proof # regenerate deps-proof.txt on this machine
496
+ python make.py demo # the tour
497
+ python make.py video-demo # the same material, paced for screen recording
498
+ python make.py clean # caches, scratch queues, build output
499
+ ```
500
+
501
+ Arguments pass through: `python make.py bench --jobs 100`, or
502
+ `python make.py video-demo --scenes 6` to run one scene of the demo alone.
503
+
504
+ ```
505
+ src/nobroker/ 11 modules, in strict dependency order (errors -> … -> queue)
506
+ src/__main__.py entry point for the zipapp bundle
507
+ tests/ 8 files, 124 tests, unittest only
508
+ tools/ check_deps.py (the dependency proof), build_pyz.py (the bundle)
509
+ examples/ demo.py (the tour), video_demo.py (paced), handlers.py (for `work`)
510
+ web/ docs site and playground — two static files, no build step
511
+ make.py every task; the Makefile delegates here
512
+ ```
513
+
514
+ **~3,000 lines of library, ~1,700 lines of tests, 0 dependencies.**
515
+
516
+ The test suite deliberately over-invests in one thing:
517
+
518
+ - `test_crash_recovery.py` — hard kills via `os._exit(9)`, and the truncation
519
+ sweep over every byte offset of a real log
520
+ - `test_concurrency.py` — four separate OS processes racing on one queue, each
521
+ recording which job ids it received; the parent asserts the sets are disjoint
522
+ and complete
523
+
524
+ Both of those caught real bugs during development. The multi-process test found
525
+ a stale cached file offset that made peers overwrite each other's records; the
526
+ compaction test found `os.open` defaulting to text mode on Windows and corrupting
527
+ the log. Neither would have shown up in a single-process happy-path test.
528
+
529
+ See **[STDLIB.md](STDLIB.md)** for all 22 "I would normally have installed X"
530
+ decisions, including the five places the standard library was genuinely the worse
531
+ option.
532
+
533
+ ---
534
+
535
+ ## License
536
+
537
+ MIT.