toro-queue 0.1.0__tar.gz → 0.2.0__tar.gz

toro_queue-0.2.0/.github/workflows/pr-check.yaml

@@ -0,0 +1,88 @@
+name: PR check
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - feature*
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+# Read-only by default — nothing here writes to the repo. SonarCloud PR
+# decoration comes from the SonarCloud GitHub App, not GITHUB_TOKEN write scopes.
+permissions:
+  contents: read
+
+# A new push to the same branch/PR supersedes the previous run.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    name: Lint, types, tests (py${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # Everything the classifiers claim to support.
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    permissions:
+      contents: read
+    services:
+      redis:
+        image: redis:7-alpine
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v6.0.3
+        with:
+          fetch-depth: 0          # full history improves Sonar new-code/blame relevancy
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.2.0
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+
+      - name: Sync dependencies
+        run: uv sync --locked
+
+      - name: Lint (ruff)
+        run: uv run ruff check .
+
+      - name: Format (ruff)
+        run: uv run ruff format --check .
+
+      - name: Type check (ty)
+        run: uv run ty check
+
+      - name: Tests (unit + integration)
+        run: uv run pytest -m "unit or integration" --cov=toro --cov-report=xml
+
+      # Runs only when SONAR_TOKEN is set (skipped on forks / before setup, so the
+      # check stays green), and only once per matrix — one coverage upload.
+      # Config is passed inline; there is no sonar-project.properties.
+      - name: SonarCloud scan
+        if: matrix.python-version == '3.13' && env.SONAR_TOKEN
+        uses: SonarSource/sonarqube-scan-action@v8.2.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        with:
+          args: >
+            -Dsonar.organization=ilovepixelart
+            -Dsonar.projectName=toro
+            -Dsonar.projectKey=ilovepixelart_toro
+            -Dsonar.python.coverage.reportPaths=coverage.xml
+            -Dsonar.sources=toro
+            -Dsonar.tests=tests
+            -Dsonar.test.exclusions=tests/**
+            -Dsonar.coverage.exclusions=tests/**

{toro_queue-0.1.0 → toro_queue-0.2.0}/.github/workflows/release.yml

@@ -12,12 +12,12 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6.0.3
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v8.2.0
       - name: Build sdist + wheel
         run: uv build
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7.0.1
         with:
           name: dist
           path: dist/
@@ -29,7 +29,7 @@ jobs:
     permissions:
       id-token: write
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v8.0.1
         with:
           name: dist
           path: dist/

{toro_queue-0.1.0 → toro_queue-0.2.0}/.gitignore

@@ -8,3 +8,4 @@ __pycache__/
 /build/
 
 .coverage
+coverage.xml

{toro_queue-0.1.0 → toro_queue-0.2.0}/.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 # Install once with:  uvx pre-commit install
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.15
+    rev: v0.15.16
     hooks:
       - id: ruff-check
         args: [--fix]

toro_queue-0.2.0/.vscode/extensions.json

@@ -0,0 +1,8 @@
+{
+    "recommendations": [
+        "ms-python.python",
+        "charliermarsh.ruff",
+        "astral-sh.ty",
+        "streetsidesoftware.code-spell-checker"
+    ]
+}

toro_queue-0.2.0/.vscode/settings.json

@@ -0,0 +1,19 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "charliermarsh.ruff",
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+            "source.fixAll.ruff": "explicit",
+            "source.organizeImports.ruff": "explicit"
+        }
+    },
+    "editor.rulers": [100],
+    "python.testing.pytestEnabled": true,
+    "python.testing.unittestEnabled": false,
+    "cSpell.words": [
+        "BZPOPMIN",
+        "keepalive",
+        "ZPOPMIN",
+        "ZSET"
+    ]
+}

{toro_queue-0.1.0 → toro_queue-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toro-queue
-Version: 0.1.0
+Version: 0.2.0
 Summary: An async-first, Redis-backed job queue for Python.
 Project-URL: Homepage, https://github.com/ilovepixelart/toro
 Project-URL: Repository, https://github.com/ilovepixelart/toro
@@ -30,13 +30,28 @@ Description-Content-Type: text/markdown
 An **async-first**, Redis-backed job queue for Python. Every state transition is
 an atomic Lua script; producing and processing are `asyncio` end to end.
 
+[![Python](https://img.shields.io/pypi/pyversions/toro-queue)](https://pypi.org/project/toro-queue/)
+\
+[![PyPI](https://img.shields.io/pypi/v/toro-queue)](https://pypi.org/project/toro-queue/)
+[![Downloads](https://static.pepy.tech/badge/toro-queue)](https://pepy.tech/project/toro-queue)
+[![License](https://img.shields.io/github/license/ilovepixelart/toro)](https://github.com/ilovepixelart/toro/blob/main/LICENSE)
+\
+[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=ilovepixelart_toro&metric=coverage)](https://sonarcloud.io/summary/new_code?id=ilovepixelart_toro)
+[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=ilovepixelart_toro&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=ilovepixelart_toro)
+\
+[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=ilovepixelart_toro&metric=reliability_rating)](https://sonarcloud.io/summary/new_code?id=ilovepixelart_toro)
+[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=ilovepixelart_toro&metric=sqale_rating)](https://sonarcloud.io/summary/new_code?id=ilovepixelart_toro)
+[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=ilovepixelart_toro&metric=security_rating)](https://sonarcloud.io/summary/new_code?id=ilovepixelart_toro)
+
 ```bash
 pip install toro-queue      # the import name is `toro`
 ```
 
 > Installed as **`toro-queue`** on PyPI (the name `toro` was taken), but you
-> `import toro`. See [DESIGN.md](https://github.com/ilovepixelart/toro/blob/main/DESIGN.md) for the architecture and the
-> at-least-once reliability model.
+> `import toro`. See the [docs](https://github.com/ilovepixelart/toro/tree/main/docs) for the
+> architecture, the reliability model, and the detailed guides.
+
+Pairs with **[matador](https://github.com/ilovepixelart/matador)**, a live web dashboard for your queues.
 
 ## Why toro
 

{toro_queue-0.1.0 → toro_queue-0.2.0}/README.md

@@ -3,13 +3,28 @@
 An **async-first**, Redis-backed job queue for Python. Every state transition is
 an atomic Lua script; producing and processing are `asyncio` end to end.
 
+[![Python](https://img.shields.io/pypi/pyversions/toro-queue)](https://pypi.org/project/toro-queue/)
+\
+[![PyPI](https://img.shields.io/pypi/v/toro-queue)](https://pypi.org/project/toro-queue/)
+[![Downloads](https://static.pepy.tech/badge/toro-queue)](https://pepy.tech/project/toro-queue)
+[![License](https://img.shields.io/github/license/ilovepixelart/toro)](https://github.com/ilovepixelart/toro/blob/main/LICENSE)
+\
+[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=ilovepixelart_toro&metric=coverage)](https://sonarcloud.io/summary/new_code?id=ilovepixelart_toro)
+[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=ilovepixelart_toro&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=ilovepixelart_toro)
+\
+[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=ilovepixelart_toro&metric=reliability_rating)](https://sonarcloud.io/summary/new_code?id=ilovepixelart_toro)
+[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=ilovepixelart_toro&metric=sqale_rating)](https://sonarcloud.io/summary/new_code?id=ilovepixelart_toro)
+[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=ilovepixelart_toro&metric=security_rating)](https://sonarcloud.io/summary/new_code?id=ilovepixelart_toro)
+
 ```bash
 pip install toro-queue      # the import name is `toro`
 ```
 
 > Installed as **`toro-queue`** on PyPI (the name `toro` was taken), but you
-> `import toro`. See [DESIGN.md](https://github.com/ilovepixelart/toro/blob/main/DESIGN.md) for the architecture and the
-> at-least-once reliability model.
+> `import toro`. See the [docs](https://github.com/ilovepixelart/toro/tree/main/docs) for the
+> architecture, the reliability model, and the detailed guides.
+
+Pairs with **[matador](https://github.com/ilovepixelart/matador)**, a live web dashboard for your queues.
 
 ## Why toro
 

toro_queue-0.2.0/docs/architecture.md

@@ -0,0 +1,143 @@
+# Architecture
+
+How toro's core works, and why. Every state transition is an atomic Lua script;
+every job is durable in Redis.
+
+> Prior art: the atomic-Lua and lock/stalled-recovery patterns come from the
+> Node.js Redis-queue ecosystem; the specifics are toro's own.
+
+## Atomic state transitions via Lua
+
+Every state move (`wait→active`, `active→completed/failed/delayed`,
+`delayed→wait`) is a single Redis Lua script, run atomically, so multi-key
+"check-then-act" sequences can't interleave. That removes whole classes of race:
+
+- **pop-then-lock gap** — two workers claiming the same job: the claim pops from
+  the priority set and sets the lock inside one script.
+- **finish-after-steal** — a worker committing a result for a job a stalled sweep
+  already re-queued: guarded by a token check plus `LREM active` returning 0.
+
+Scripts live in `scripts.py`, registered with `redis.asyncio`'s `register_script`.
+The Python side only assembles KEYS/ARGV; the guarantees live in the Lua.
+
+## Claiming a job: the prioritized set + a wakeup marker
+
+All waiting jobs live in one `prioritized` ZSET, scored
+`(PRIORITY_OFFSET - priority) * 2^32 + seq` — a single global order where higher
+priority is more urgent and ties stay FIFO (`seq` is a per-queue counter). This
+*is* the `wait` state; there is no separate fast-lane list, so a low-priority job
+can't starve a high-priority one.
+
+A single ZSET can't be blocking-popped, so wakeup uses a small **base marker**:
+producers `ZADD marker 0 "0"` (idempotent) on enqueue, and idle workers park on
+`BZPOPMIN marker`. The marker only wakes a worker; the real claim is the atomic
+`MOVE_TO_ACTIVE` (`ZPOPMIN prioritized` → push to `active` → set the lock → load
+the job). Because the claim is atomic and idempotent, a missed marker can never
+strand a job.
+
+## Fetch-next inside finish
+
+A busy worker doesn't go back to the blocking wait between jobs. The finish
+scripts (`MOVE_TO_COMPLETED` / `MOVE_TO_FAILED`) commit the current job **and**
+claim the next one in the same round trip; the worker only re-parks on the marker
+when the queue is empty (or it's shutting down, signaled by a fetch flag, so it
+drains cleanly). All claiming funnels through one shared Lua routine
+(`lockAndLoad` / `acquireNext`), used by both the wakeup path and fetch-next.
+
+It's mainly a round-trip win: at concurrency 20, process throughput is roughly
+2.3× a claim-per-job design, because the separate per-job claim and load collapse
+into the finish call.
+
+## At-least-once: locks, tokens, and stalled recovery
+
+The reliability core ([Reliability](reliability.md) is the full guide):
+
+- On claim, the job gets a lock `<id>:lock = <token>` with `PX lockDuration`
+  (default 30s). The token is the claiming worker's; only it can renew or finish.
+- A per-job renewer extends the lock on a timer and clears the job from the
+  `stalled` set while it's alive.
+- A background sweep runs every `stalled_interval` (throttled cluster-wide by a
+  `stalled-check` key): any job in `stalled` whose lock has expired is recovered
+  (`LREM active`, back to the prioritized set, or failed after
+  `max_stalled_count`), then the current `active` list is re-marked as stalled.
+
+The guarantee is **at-least-once**: a job is never lost while Redis persists, but
+its handler can run more than once (bounded by `max_stalled_count`) if a worker
+dies mid-job. Exactly-once *result commit* is enforced by the token-guarded lock
+at finish, not by preventing duplicate handler runs.
+
+## Delayed jobs
+
+Delayed jobs and retries with backoff sit in a `delayed` ZSET scored by their
+process-at timestamp (ms). A one-second promotion loop in the worker moves any
+due jobs into the prioritized set.
+
+## Higher-level features
+
+- **Priorities** — every job is in the one prioritized ZSET above, so priority is
+  a single global order with no starvation, FIFO within a band.
+- **Repeatable / cron** — `add_scheduler(every=ms | cron=...)` stores a template
+  and enqueues the first occurrence as a delayed job; each occurrence mints its
+  successor with a deterministic id when a worker picks it up. `trigger_scheduler`
+  runs one now, `remove_scheduler` stops the chain. See [Scheduling](scheduling.md).
+- **Rate limiting** — a queue-wide token bucket in Redis
+  (`Worker(rate_limit={"max": N, "duration": ms})`), shared by every worker on the
+  queue. An over-limit claim returns a sentinel and the worker waits out the window.
+- **Events** — Redis pub/sub on an `events` channel (`added`, `progress`,
+  `completed`, `failed`); `Queue.result()` awaits the terminal event and
+  `Worker.on(event, fn)` exposes in-process hooks. See [Concepts](concepts.md).
+- **Auto-removal** — `remove_on_complete` / `remove_on_fail` (bool / count /
+  `{count, age}`) enforced inside the finish script, not by a separate sweeper.
+
+## The Lua scripts
+
+Every state change is a Lua script in `scripts.py`, registered once per process
+with `register_script` (run by `EVALSHA`). Python only assembles `KEYS`/`ARGV`.
+
+The scripts share a small library of routines:
+
+| Routine | Does |
+|---|---|
+| `priorityScore` | Packs `(PRIORITY_OFFSET - priority) * 2^32 + seq` for the prioritized ZSET. |
+| `enqueue` | Adds a job to `prioritized` at its score and arms the marker. |
+| `lockAndLoad` | Sets the lock token and loads the hash for a just-claimed id. |
+| `acquireNext` | Pops the top prioritized job into `active` and locks it, honoring the rate limit. |
+| `tryRateLimit` | Token bucket: ms until a token frees, or 0 to proceed. |
+| `recordFinished` | Records a terminal job in `completed`/`failed` and applies auto-removal. |
+
+And the scripts themselves:
+
+| Script | Caller | Does |
+|---|---|---|
+| `ADD_JOB` | producer | Mint/accept an id, write the hash, enqueue or delay, dedup, publish `added`. |
+| `MOVE_TO_ACTIVE` | worker wakeup | Claim the next job: `ZPOPMIN prioritized` → `active` → lock + load. |
+| `MOVE_TO_COMPLETED` | worker finish | Commit the result and fetch-next in one round trip. |
+| `MOVE_TO_FAILED` | worker finish | Retry (to `wait`/`delayed`) or terminally fail, and fetch-next. |
+| `EXTEND_LOCK` | renewer | Token-guarded lock renewal; clears the job from `stalled`. |
+| `MOVE_STALLED` | sweep | Mark-and-sweep recovery of jobs whose lock expired. |
+| `PROMOTE_DELAYED` | promote loop | Move up to `PROMOTE_BATCH` (1000) due delayed jobs to `prioritized`. |
+| `ADD_SCHEDULED` | scheduler | Enqueue a scheduler occurrence under a deterministic id (idempotent). |
+| `PROMOTE_JOB` / `RETRY_JOB` / `REMOVE_JOB` | dashboard | Run a delayed job now / re-enqueue a failed one / delete a job with its lock and logs. |
+
+### Lua → Python return protocol
+
+Scripts signal outcomes with sentinels the worker decodes:
+
+- `RL_SENTINEL` (`"__rl__"`) — a claim hit the rate limiter; the second value is
+  ms until a token frees, so the worker waits instead of busy-spinning.
+- `LOCK_LOST` (`-2`) — a finish ran but the worker no longer held the lock (the
+  job was reclaimed); the result is dropped.
+- `NOT_ACTIVE` (`-3`) — a finish ran but the job was no longer in `active`.
+- `OUTCOME_FAILED` (`1`) vs `0` — `MOVE_TO_FAILED` telling the worker whether the
+  job terminally failed or will retry.
+
+Scores are packed under 2^53 (`PRIORITY_OFFSET = 2^20`, `SEQ_MOD = 2^32`) so ZSET
+double scores stay exact, and the scripts use only plain JSON and integer ARGV —
+no `cmsgpack` / `bit` / `cjson` — so they run on any Redis build.
+
+## Python-specific choices
+
+- **async-first** — `redis.asyncio`, `async def` processors, one event loop;
+  concurrency is N `asyncio` tasks sharing the loop.
+- **Cluster** — a `{braces}` hash-tag in the prefix keeps all of a queue's keys on
+  one slot, which the multi-key Lua scripts require.

toro_queue-0.2.0/docs/concepts.md

@@ -0,0 +1,102 @@
+# Concepts
+
+The mental model behind toro.
+
+## Queue, Worker, Job
+
+toro has a clean producer/consumer split, and both talk to the same Redis.
+
+- A **`Queue`** is the *producer* handle. You use it to enqueue jobs
+  (`queue.add(...)`), schedule repeatable ones, and inspect state (counts,
+  listing, search). Creating a `Queue` opens (or shares) a Redis connection but
+  starts no background work.
+- A **`Worker`** is the *consumer*. You give it a queue name and an `async`
+  processor function; calling `worker.run()` starts claiming jobs, running the
+  processor over each, and recovering jobs from workers that died. A worker also
+  runs small background loops (delayed-job promotion, stalled-job sweep,
+  heartbeat) while it's alive.
+- A **`Job`** is one unit of work. It carries an `id`, a `name` (a label you
+  choose, e.g. `"welcome"`), a JSON-serializable `data` payload, its options, and
+  bookkeeping the system fills in: `state`, `attempts_made`, timestamps
+  (`timestamp`, `processed_on`, `finished_on`), `progress`, `stacktrace`, and
+  either a `returnvalue` or a `failed_reason`. (A job's log lines and its lock
+  live in separate Redis keys, not as fields on the `Job` — see the
+  [data model](data-model.md).)
+
+Producers and consumers never call each other. They coordinate only through
+Redis, which is what lets you run them in different processes or on different
+machines.
+
+## Job states
+
+Every job is in exactly one state at a time. toro exposes them as a `Literal`
+type, `JobState`:
+
+| State | Meaning |
+|---|---|
+| `wait` | Ready to run, waiting for a free worker. (Stored in the priority-ordered set, so "wait" and "prioritized" are the same place.) |
+| `delayed` | Scheduled for the future; not yet runnable. Promoted to `wait` when due. |
+| `active` | Claimed by a worker and currently running. |
+| `completed` | Finished successfully; `returnvalue` holds the result. |
+| `failed` | Exhausted its retry attempts; `failed_reason` holds the error. |
+
+The normal path is `wait → active → completed`. A failure with retries left goes
+`active → wait` (or `active → delayed`, if a backoff delay applies) and tries
+again; only after the last attempt does it land in `failed`. A delayed or
+repeatable job starts in `delayed`. See [Job lifecycle](architecture.md) for the
+exact transitions and [Producing jobs](producing.md) for how delay and retries
+are configured.
+
+## Workers vs. slots
+
+These are easy to conflate but distinct, and the dashboard shows both.
+
+- A **worker** is a running `Worker` instance (one heartbeat, one identity). It
+  lives inside an OS process, but it is *not* the process: you can run several
+  workers in one process, one per process, or spread across machines.
+- A **slot** is one unit of *parallel* work *inside* a worker. A worker created
+  with `concurrency=N` runs N async processing loops, so it can have up to N jobs
+  in flight at once. "Slots" on the dashboard is the sum of every live worker's
+  concurrency: your total throughput capacity.
+
+So `live` counts workers, `slots` counts concurrent capacity. With the default
+`concurrency=1` they happen to match; bump concurrency and slots climb while the
+worker count stays put.
+
+```
+host (machine)
+└── process (pid)
+    └── worker        (a Worker instance, unique id)   ← "live"
+        └── slots     (concurrency async loops)        ← "slots"
+            └── jobs   (one per slot at a time)
+```
+
+Because slots are `asyncio` tasks sharing one event loop (not threads or
+processes), a processor that blocks the loop blocks its sibling slots. Keep
+processors `await`-y.
+
+## Events
+
+toro publishes events to a Redis pub/sub channel: `added` when a job is enqueued
+(published by the add script, atomically with the enqueue), `progress` from a running processor
+(`job.update_progress`), and `completed` / `failed`, which the finish Lua scripts
+publish atomically with the state change. `failed` fires only on terminal failure,
+not on a retry. Two things consume the channel:
+
+- **`await job.result()`** (or `queue.result(job_id)`) on the producer side
+  subscribes and waits for the terminal event, returning the value or raising
+  `JobFailedError`.
+- **A dashboard** (such as [matador](https://github.com/ilovepixelart/matador))
+  subscribes to refresh live as state changes.
+
+`Worker.on(event, fn)` lets a worker react to its own lifecycle with in-process
+callbacks (`completed`, `failed`, `retrying`, `stalled`, `lock-lost`,
+`rate-limited`) — separate from the pub/sub channel above. See
+[Processing jobs](processing.md).
+
+## Reliability in one sentence
+
+toro is **at-least-once**: a job is never lost while Redis persists, but its
+handler can run more than once (bounded) if a worker dies mid-job. Exactly-once
+*result commit* is enforced by a per-job lock token. The full story is in
+[Reliability](reliability.md).

toro_queue-0.2.0/docs/data-model.md

@@ -0,0 +1,63 @@
+# Data model
+
+Everything toro stores lives in Redis under a per-queue prefix. All key names are
+computed in one place (`toro/keys.py`) so the Lua scripts and the Python side can
+never disagree about where something lives.
+
+## Key prefix
+
+For a queue named `<name>` with prefix `<prefix>` (default `toro`), every key
+starts with:
+
+```
+<prefix>:<name>:
+```
+
+So `Queue("emails")` (default prefix) stores everything under `toro:emails:`.
+Using a `{braces}` hash-tag in the prefix forces all of a queue's keys onto one
+Redis Cluster slot, which the multi-key Lua scripts require.
+
+## Queue-wide keys
+
+| Key suffix | Type | Holds |
+|---|---|---|
+| `id` | string (counter) | `INCR`-ed to mint auto job ids. |
+| `prioritized` | ZSET | Waiting jobs in global priority order; score packs (priority, sequence). This *is* the `wait` state. |
+| `marker` | ZSET | A single idempotent base member (`"0"`); idle workers `BZPOPMIN` it to wake. It only signals; the real claim is atomic. |
+| `pc` | string (counter) | Priority sequence counter, so same-priority jobs stay FIFO. |
+| `active` | LIST | Ids currently claimed by a worker and running. |
+| `delayed` | ZSET | Ids scored by their process-at timestamp (ms); promoted to `prioritized` when due. |
+| `completed` | ZSET | Successfully-finished ids, scored by finish time (for auto-removal + listing). |
+| `failed` | ZSET | Terminally-failed ids, scored by finish time. |
+| `meta-paused` | string (flag) | Exists only while the queue is paused; workers stop claiming new jobs. |
+| `events` | pub/sub channel | Carries `added` / `progress` / `completed` / `failed`; drives `result()` and live dashboards. |
+| `limiter` | HASH | The queue-wide rate-limit token bucket (`{tokens, ts}`), shared by every worker. |
+| `stalled` | SET | Candidate ids for the mark-and-sweep recovery pass. |
+| `stalled-check` | string (PX) | Throttle key so the stalled sweep runs about once per interval cluster-wide. |
+| `repeat` | ZSET | Scheduler id -> next-run timestamp. |
+| `workers` | ZSET | Live worker id -> last-heartbeat ms; stale entries pruned lazily on read. |
+| `departed` | LIST (capped) | Recent worker departures: graceful `stopped` or `lost` (crashed). |
+
+## Per-scheduler, per-worker, per-job keys
+
+| Key | Type | Holds |
+|---|---|---|
+| `repeat:<schedulerId>` | HASH | A scheduler's template: `name`, `every`/`cron`, `data`, `opts`. |
+| `worker:<workerId>` | HASH | A worker's presence record: host, pid, concurrency, current jobs, processed/failed counts, state. |
+| `<jobId>` | HASH | The job itself: `name`, `data`, `opts`, `state`, `attemptsMade`, timestamps, `returnvalue`/`failedReason`, `progress`, `stacktrace`, ... |
+| `<jobId>:lock` | string (token, PX) | The per-job lock: the owning worker's token with an expiry. Only the holder may finish or renew it. |
+| `<jobId>:logs` | LIST | Log lines appended by `job.log(...)` from inside a processor. |
+
+Note the job hash key is just `<prefix>:<name>:<jobId>` (no extra segment), so a job
+`5` on `toro:emails:` is the hash `toro:emails:5`, with `toro:emails:5:lock` and
+`toro:emails:5:logs` beside it.
+
+## How the pieces connect
+
+- A job moves between `prioritized` / `active` / `delayed` / `completed` / `failed`
+  as its state changes; the move and the hash update happen in one Lua script. See
+  [Architecture](architecture.md).
+- The `lock` + `stalled` keys are the at-least-once machinery. See
+  [Reliability](reliability.md).
+- `repeat` + `repeat:<id>` drive [scheduling](scheduling.md); `workers` +
+  `worker:<id>` + `departed` drive worker presence in the dashboard.

toro_queue-0.2.0/docs/index.md

@@ -0,0 +1,20 @@
+# toro documentation
+
+Reference docs for how toro works. The [README](../README.md) is the quick start.
+
+## Pages
+
+- **[Concepts](concepts.md)** — the mental model: queues, workers, jobs, the five
+  job states, and the difference between *workers* and *slots*.
+- **[Data model](data-model.md)** — the exact Redis keys a queue uses and what
+  each one stores.
+- **[Reliability](reliability.md)** — the at-least-once guarantee: per-job locks,
+  worker tokens, and stalled-job recovery.
+- **[Producing jobs](producing.md)** — `Queue.add()` and every option (priority,
+  delay, retries/backoff, deduplication, custom ids).
+- **[Processing jobs](processing.md)** — `Worker`: concurrency, lifecycle events,
+  rate limiting, and graceful shutdown.
+- **[Scheduling](scheduling.md)** — repeatable and cron jobs, and how each
+  occurrence schedules the next.
+- **[Architecture](architecture.md)** — the atomic-Lua core and the design
+  decisions behind the queue.