dr-queues 0.1.0__tar.gz

dr_queues-0.1.0/.gitignore

@@ -0,0 +1,4 @@
+.runs/
+.cache/
+__pycache__/
+*.py[cod]

dr_queues-0.1.0/.python-version

@@ -0,0 +1 @@
+3.13

dr_queues-0.1.0/AGENTS.md

@@ -0,0 +1,39 @@
+# Agent Instructions
+
+## Verification
+
+- **Cursor agents:** Do not run `scripts/pre-check.sh` unless the user
+  explicitly asks you to run checks or verification.
+- **Other agents:** Run `scripts/pre-check.sh` after making code changes.
+- Skip `scripts/pre-check.sh` after documentation-only or configuration-only
+  changes (non-Cursor agents).
+- When running checks, treat every issue reported by `scripts/pre-check.sh` as
+  in scope to fix, even when the issue is outside the files you touched.
+- Read failed check output from `.cache/pre-check/`.
+
+## Tests
+
+- Do not add tests unless the user explicitly asks for tests.
+
+## Commits
+
+- After completing a requested sequence of changes, commit the result unless
+  the user specifically asks not to commit at the end.
+- Use a single-line commit message.
+- If your own changes are easy to isolate, commit only your own changes.
+- If your own changes are not cleanly extractable, commit all changes in the
+  files you touched.
+- Before committing code changes, run `scripts/pre-check.sh` (non-Cursor agents,
+  or when the user explicitly asked you to run checks).
+- Before committing documentation-only or configuration-only changes, do not run
+  `scripts/pre-check.sh`.
+
+## Multi-Player Workflow
+
+- Assume other agents and users may be operating in this repo at the same time.
+- Never delete, revert, reset, or otherwise undo unexpected changes in the repo.
+- Assume unexpected changes were made intentionally by the user or another
+  process.
+- If unexpected changes appear suspect or make the requested work difficult,
+  ask how to proceed and describe the concern.
+- Do not undo suspect changes while waiting for guidance.

dr_queues-0.1.0/CHANGELOG.md

@@ -0,0 +1,21 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2026-06-21
+
+### Added
+
+- RabbitMQ multi-stage pipeline runtime (`WorkerPool`, `TerminalTap`, queue chaining)
+- Slim `JobEnvelope` for job state on the wire
+- `PipelineDefinition`, `HandlerRegistry`, and `Pipeline` workflow engine
+- Append-only event sinks: `MongoEventSink` (default), `AmqpEventSink`, `MemoryEventSink`
+- Run manifest for multi-process worker coordination
+- `filter_run_events` analysis helper
+- Console entry points: `dr-queues-demo`, `dr-queues-stage-worker`
+- Reference dummy handlers in `dr_queues.demo_handlers`
+
+[0.1.0]: https://github.com/danielle-rothermel/dr-queues/releases/tag/v0.1.0

dr_queues-0.1.0/CLAUDE.md

@@ -0,0 +1 @@
+@AGENTS.md

dr_queues-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 danielle rothermel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dr_queues-0.1.0/PKG-INFO

@@ -0,0 +1,243 @@
+Metadata-Version: 2.4
+Name: dr-queues
+Version: 0.1.0
+Summary: RabbitMQ multi-stage pipeline runtime with append-only event sinks
+Project-URL: Homepage, https://github.com/danielle-rothermel/dr-queues
+Project-URL: Repository, https://github.com/danielle-rothermel/dr-queues
+Project-URL: Issues, https://github.com/danielle-rothermel/dr-queues/issues
+Author-email: Danielle Rothermel <danielle.rothermel@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: amqp,mongodb,pipeline,queue,rabbitmq,workflow
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.13
+Requires-Dist: pika>=1.4.1
+Requires-Dist: pydantic>=2.13.4
+Requires-Dist: pymongo>=4.16.0
+Requires-Dist: typer>=0.26.7
+Description-Content-Type: text/markdown
+
+# dr-queues
+
+RabbitMQ multi-stage pipeline runtime with append-only event sinks.
+
+dr-queues is a domain-free library for running jobs through chained stage
+queues, scaling worker pools per stage, and recording pipeline lifecycle
+events to durable storage. It is the execution substrate for experiment
+applications such as [dr-bottleneck](https://github.com/danielle-rothermel/dr-bottleneck).
+
+## Install
+
+```bash
+pip install dr-queues
+```
+
+Or with [uv](https://docs.astral.sh/uv/):
+
+```bash
+uv add dr-queues
+```
+
+RabbitMQ and MongoDB are **not** bundled with the package — run them locally
+(Docker Compose) or point `AMQP_URL` / `MONGODB_URL` at your infrastructure.
+
+After install, try the demo CLI:
+
+```bash
+dr-queues-demo --repeats 2 --lanes 1
+```
+
+## What dr-queues is
+
+- AMQP staged pipeline: per-stage pending/completed queue pairs chained together
+- Slim `JobEnvelope` for job state on the wire
+- `WorkerPool` and `TerminalTap` with pluggable step handlers
+- Append-only `EventSink` implementations (MongoDB happy path, AMQP optional)
+- Run manifest for multi-process worker coordination
+- Minimal workflow engine: ordered steps + `HandlerRegistry`
+
+## What dr-queues is not
+
+- LLM calls, prompts, or model profiles
+- Dataset loading, HumanEval, or experiment metrics
+- JSONL report assembly
+- General domain EventBus / webhook dispatch (deferred to a future layer)
+
+Workers append pipeline events **before** acking and forwarding jobs to the
+next stage. That append-before-forward invariant matches an event-sourced
+write-ahead log: durable telemetry precedes propagation.
+
+## Requirements
+
+- Python 3.13+ with [uv](https://docs.astral.sh/uv/)
+- Docker Compose for local RabbitMQ and MongoDB
+
+## Environment
+
+| Variable | Default | Purpose |
+|----------|---------|---------|
+| `AMQP_URL` | `amqp://guest:guest@localhost:5672/` | RabbitMQ connection |
+| `MONGODB_URL` | `mongodb://localhost:27017/dr_queues` | MongoDB event store |
+
+## Local services
+
+```bash
+docker compose up -d
+```
+
+- RabbitMQ management UI: http://localhost:15672 (guest/guest)
+- MongoDB: `mongodb://localhost:27017/dr_queues`
+
+## Quick start
+
+From a git checkout:
+
+```bash
+uv sync
+docker compose up -d
+uv run dr-queues-demo \
+  --repeats 5 \
+  --workers slow=4,transform=4,finalize=2
+```
+
+Or use the repo script wrapper (same CLI):
+
+```bash
+uv run python scripts/run_pipeline_demo.py \
+  --repeats 5 \
+  --workers slow=4,transform=4,finalize=2
+```
+
+The demo runs a 3-stage dummy pipeline (`sleep_ms` → `add_prefix` →
+`record_artifact`) with MongoDB as the default event sink.
+
+Options:
+
+```bash
+dr-queues-demo --sink amqp --repeats 2
+dr-queues-demo --sink both --lanes 1 --repeats 1
+```
+
+Each run writes a manifest to `.runs/{run_id}/manifest.json`. The demo prints
+`run_id=...` at the start — use that value when querying MongoDB (do not use the
+literal placeholder `demo-...`).
+
+On success you should see output like `events=70 terminals=10` for
+`--repeats 5` with the default 2 lanes (10 jobs × 7 events per job).
+
+### Inspect events in MongoDB
+
+Events are stored in the `pipeline_events` collection. Replace
+`YOUR_RUN_ID` with the `run_id` printed by the demo (e.g. `demo-56bd0ce5`).
+
+Count events for one run:
+
+```bash
+mongosh mongodb://localhost:27017/dr_queues \
+  --eval 'db.pipeline_events.countDocuments({run_id: "YOUR_RUN_ID"})'
+```
+
+List all run IDs that have events:
+
+```bash
+mongosh mongodb://localhost:27017/dr_queues \
+  --eval 'db.pipeline_events.distinct("run_id")'
+```
+
+Count all events in the collection (across every run):
+
+```bash
+mongosh mongodb://localhost:27017/dr_queues \
+  --eval 'db.pipeline_events.countDocuments({})'
+```
+
+Preview a few events from a run:
+
+```bash
+mongosh mongodb://localhost:27017/dr_queues \
+  --eval 'db.pipeline_events.find({run_id: "YOUR_RUN_ID"}).limit(3).pretty()'
+```
+
+If counts are zero, check:
+
+- You used the **actual** `run_id` from demo output, not the placeholder text.
+- The demo used the default Mongo sink (`--sink mongo`). AMQP-only runs
+  (`--sink amqp`) do not write to MongoDB.
+- MongoDB is running (`docker compose up -d`) and reachable at `MONGODB_URL`.
+- You re-ran the demo after starting Mongo if the first attempt failed to connect.
+
+## Package layout
+
+| Module | Role |
+|--------|------|
+| `amqp/` | Connection helpers, stage queue pairs |
+| `pipeline/` | `JobEnvelope`, `WorkerPool`, `TerminalTap`, runner |
+| `events/` | `PipelineEvent`, `EventSink`, Mongo/AMQP/memory sinks |
+| `manifest/` | Run manifest read/write, worker CLI helpers |
+| `workflow/` | `PipelineDefinition`, `HandlerRegistry`, `Pipeline` |
+| `analysis/` | `filter_run_events` |
+
+## Public API
+
+Import from `dr_queues`:
+
+- **Setup / run:** `setup_run_queues`, `run_in_process`, `seed_jobs`, `seed_manifest_jobs`
+- **Runtime:** `WorkerPool`, `TerminalTap`, `JobEnvelope`
+- **Workflow:** `PipelineDefinition`, `HandlerRegistry`, `Pipeline`
+- **Events:** `PipelineEvent`, `EventSink`, `MongoEventSink`, `AmqpEventSink`, `MemoryEventSink`
+- **Analysis:** `filter_run_events`
+
+## Detached stage workers
+
+Resize or run a single stage in a separate process:
+
+```bash
+dr-queues-stage-worker \
+  --run-id demo-abc123 \
+  --stage transform \
+  --workers 5 \
+  --replace
+```
+
+Handlers must be registered in the worker process via `--handlers-module`
+(default: `dr_queues.demo_handlers`).
+
+## Future layers
+
+A general EventBus, domain EventAdapter, and webhook/hook dispatch will likely
+live in a separate package or in dr-bottleneck, built on top of dr-queues.
+Pipeline lifecycle events may eventually map to versioned domain event types;
+dr-queues stays focused on queue-based execution and pipeline telemetry.
+
+## Development
+
+```bash
+uv sync
+docker compose up -d
+scripts/pre-check.sh              # ruff, ty, pytest (14 tests)
+uv run pytest -m integration      # integration tests only; needs docker compose
+```
+
+Full manual smoke test:
+
+```bash
+dr-queues-demo \
+  --repeats 5 \
+  --workers slow=4,transform=4,finalize=2
+
+# optional: exercise AMQP event sink instead of Mongo
+dr-queues-demo --sink amqp --repeats 2
+```
+
+Build a wheel locally before publishing:
+
+```bash
+uv build
+tar -tzf dist/dr_queues-*.tar.gz | head
+unzip -l dist/dr_queues-*.whl
+```

dr_queues-0.1.0/README.md

@@ -0,0 +1,219 @@
+# dr-queues
+
+RabbitMQ multi-stage pipeline runtime with append-only event sinks.
+
+dr-queues is a domain-free library for running jobs through chained stage
+queues, scaling worker pools per stage, and recording pipeline lifecycle
+events to durable storage. It is the execution substrate for experiment
+applications such as [dr-bottleneck](https://github.com/danielle-rothermel/dr-bottleneck).
+
+## Install
+
+```bash
+pip install dr-queues
+```
+
+Or with [uv](https://docs.astral.sh/uv/):
+
+```bash
+uv add dr-queues
+```
+
+RabbitMQ and MongoDB are **not** bundled with the package — run them locally
+(Docker Compose) or point `AMQP_URL` / `MONGODB_URL` at your infrastructure.
+
+After install, try the demo CLI:
+
+```bash
+dr-queues-demo --repeats 2 --lanes 1
+```
+
+## What dr-queues is
+
+- AMQP staged pipeline: per-stage pending/completed queue pairs chained together
+- Slim `JobEnvelope` for job state on the wire
+- `WorkerPool` and `TerminalTap` with pluggable step handlers
+- Append-only `EventSink` implementations (MongoDB happy path, AMQP optional)
+- Run manifest for multi-process worker coordination
+- Minimal workflow engine: ordered steps + `HandlerRegistry`
+
+## What dr-queues is not
+
+- LLM calls, prompts, or model profiles
+- Dataset loading, HumanEval, or experiment metrics
+- JSONL report assembly
+- General domain EventBus / webhook dispatch (deferred to a future layer)
+
+Workers append pipeline events **before** acking and forwarding jobs to the
+next stage. That append-before-forward invariant matches an event-sourced
+write-ahead log: durable telemetry precedes propagation.
+
+## Requirements
+
+- Python 3.13+ with [uv](https://docs.astral.sh/uv/)
+- Docker Compose for local RabbitMQ and MongoDB
+
+## Environment
+
+| Variable | Default | Purpose |
+|----------|---------|---------|
+| `AMQP_URL` | `amqp://guest:guest@localhost:5672/` | RabbitMQ connection |
+| `MONGODB_URL` | `mongodb://localhost:27017/dr_queues` | MongoDB event store |
+
+## Local services
+
+```bash
+docker compose up -d
+```
+
+- RabbitMQ management UI: http://localhost:15672 (guest/guest)
+- MongoDB: `mongodb://localhost:27017/dr_queues`
+
+## Quick start
+
+From a git checkout:
+
+```bash
+uv sync
+docker compose up -d
+uv run dr-queues-demo \
+  --repeats 5 \
+  --workers slow=4,transform=4,finalize=2
+```
+
+Or use the repo script wrapper (same CLI):
+
+```bash
+uv run python scripts/run_pipeline_demo.py \
+  --repeats 5 \
+  --workers slow=4,transform=4,finalize=2
+```
+
+The demo runs a 3-stage dummy pipeline (`sleep_ms` → `add_prefix` →
+`record_artifact`) with MongoDB as the default event sink.
+
+Options:
+
+```bash
+dr-queues-demo --sink amqp --repeats 2
+dr-queues-demo --sink both --lanes 1 --repeats 1
+```
+
+Each run writes a manifest to `.runs/{run_id}/manifest.json`. The demo prints
+`run_id=...` at the start — use that value when querying MongoDB (do not use the
+literal placeholder `demo-...`).
+
+On success you should see output like `events=70 terminals=10` for
+`--repeats 5` with the default 2 lanes (10 jobs × 7 events per job).
+
+### Inspect events in MongoDB
+
+Events are stored in the `pipeline_events` collection. Replace
+`YOUR_RUN_ID` with the `run_id` printed by the demo (e.g. `demo-56bd0ce5`).
+
+Count events for one run:
+
+```bash
+mongosh mongodb://localhost:27017/dr_queues \
+  --eval 'db.pipeline_events.countDocuments({run_id: "YOUR_RUN_ID"})'
+```
+
+List all run IDs that have events:
+
+```bash
+mongosh mongodb://localhost:27017/dr_queues \
+  --eval 'db.pipeline_events.distinct("run_id")'
+```
+
+Count all events in the collection (across every run):
+
+```bash
+mongosh mongodb://localhost:27017/dr_queues \
+  --eval 'db.pipeline_events.countDocuments({})'
+```
+
+Preview a few events from a run:
+
+```bash
+mongosh mongodb://localhost:27017/dr_queues \
+  --eval 'db.pipeline_events.find({run_id: "YOUR_RUN_ID"}).limit(3).pretty()'
+```
+
+If counts are zero, check:
+
+- You used the **actual** `run_id` from demo output, not the placeholder text.
+- The demo used the default Mongo sink (`--sink mongo`). AMQP-only runs
+  (`--sink amqp`) do not write to MongoDB.
+- MongoDB is running (`docker compose up -d`) and reachable at `MONGODB_URL`.
+- You re-ran the demo after starting Mongo if the first attempt failed to connect.
+
+## Package layout
+
+| Module | Role |
+|--------|------|
+| `amqp/` | Connection helpers, stage queue pairs |
+| `pipeline/` | `JobEnvelope`, `WorkerPool`, `TerminalTap`, runner |
+| `events/` | `PipelineEvent`, `EventSink`, Mongo/AMQP/memory sinks |
+| `manifest/` | Run manifest read/write, worker CLI helpers |
+| `workflow/` | `PipelineDefinition`, `HandlerRegistry`, `Pipeline` |
+| `analysis/` | `filter_run_events` |
+
+## Public API
+
+Import from `dr_queues`:
+
+- **Setup / run:** `setup_run_queues`, `run_in_process`, `seed_jobs`, `seed_manifest_jobs`
+- **Runtime:** `WorkerPool`, `TerminalTap`, `JobEnvelope`
+- **Workflow:** `PipelineDefinition`, `HandlerRegistry`, `Pipeline`
+- **Events:** `PipelineEvent`, `EventSink`, `MongoEventSink`, `AmqpEventSink`, `MemoryEventSink`
+- **Analysis:** `filter_run_events`
+
+## Detached stage workers
+
+Resize or run a single stage in a separate process:
+
+```bash
+dr-queues-stage-worker \
+  --run-id demo-abc123 \
+  --stage transform \
+  --workers 5 \
+  --replace
+```
+
+Handlers must be registered in the worker process via `--handlers-module`
+(default: `dr_queues.demo_handlers`).
+
+## Future layers
+
+A general EventBus, domain EventAdapter, and webhook/hook dispatch will likely
+live in a separate package or in dr-bottleneck, built on top of dr-queues.
+Pipeline lifecycle events may eventually map to versioned domain event types;
+dr-queues stays focused on queue-based execution and pipeline telemetry.
+
+## Development
+
+```bash
+uv sync
+docker compose up -d
+scripts/pre-check.sh              # ruff, ty, pytest (14 tests)
+uv run pytest -m integration      # integration tests only; needs docker compose
+```
+
+Full manual smoke test:
+
+```bash
+dr-queues-demo \
+  --repeats 5 \
+  --workers slow=4,transform=4,finalize=2
+
+# optional: exercise AMQP event sink instead of Mongo
+dr-queues-demo --sink amqp --repeats 2
+```
+
+Build a wheel locally before publishing:
+
+```bash
+uv build
+tar -tzf dist/dr_queues-*.tar.gz | head
+unzip -l dist/dr_queues-*.whl
+```

dr_queues-0.1.0/docker-compose.yml

@@ -0,0 +1,11 @@
+services:
+  rabbitmq:
+    image: rabbitmq:3-management
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+
+  mongodb:
+    image: mongo:7
+    ports:
+      - "27017:27017"

dr_queues-0.1.0/nbs/hello_world.py

@@ -0,0 +1,35 @@
+import marimo
+
+__generated_with = "0.23.9"
+app = marimo.App(width="columns")
+
+with app.setup:
+    import marimo as mo
+
+
+@app.cell
+def _():
+    mo.md(r"""
+    ## Load Data
+    """)
+    return
+
+
+@app.cell(column=1)
+def _():
+    mo.md(r"""
+    ## Do Exploration
+    """)
+    return
+
+
+@app.cell(column=2, hide_code=True)
+def _():
+    mo.md(r"""
+    (leave space)
+    """)
+    return
+
+
+if __name__ == "__main__":
+    app.run()

dr_queues-0.1.0/pyproject.toml

@@ -0,0 +1,95 @@
+[project]
+name = "dr-queues"
+version = "0.1.0"
+description = "RabbitMQ multi-stage pipeline runtime with append-only event sinks"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [
+    { name = "Danielle Rothermel", email = "danielle.rothermel@gmail.com" }
+]
+requires-python = ">=3.13"
+keywords = ["rabbitmq", "pipeline", "workflow", "queue", "amqp", "mongodb"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: System :: Distributed Computing",
+]
+dependencies = [
+    "pika>=1.4.1",
+    "pydantic>=2.13.4",
+    "pymongo>=4.16.0",
+    "typer>=0.26.7",
+]
+
+[project.urls]
+Homepage = "https://github.com/danielle-rothermel/dr-queues"
+Repository = "https://github.com/danielle-rothermel/dr-queues"
+Issues = "https://github.com/danielle-rothermel/dr-queues/issues"
+
+[project.scripts]
+dr-queues-demo = "dr_queues.cli.demo:run"
+dr-queues-stage-worker = "dr_queues.cli.stage_worker:run"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dr_queues"]
+
+[dependency-groups]
+dev = [
+    "marimo[recommended]>=0.23.10",
+    "pytest>=9.1.1",
+    "ruff>=0.15.18",
+    "ty>=0.0.51",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "integration: tests requiring RabbitMQ and/or MongoDB",
+]
+
+[tool.ruff]
+include = ["scripts/**/*.py", "src/**/*.py", "tests/**/*.py"]
+line-length = 79
+
+[tool.ruff.lint]
+select = [
+    "A",
+    "ARG",
+    "ASYNC",
+    "DTZ",
+    "FA",
+    "FLY",
+    "FURB",
+    "G",
+    "I",
+    "ICN",
+    "ISC",
+    "LOG",
+    "N",
+    "NPY",
+    "PD",
+    "PIE",
+    "PTH",
+    "PT",
+    "RET",
+    "RSE",
+    "RUF",
+    "SLOT",
+    "T10",
+    "TID",
+    "UP",
+    "W",
+    "YTT",
+]
+
+[tool.ty.src]
+include = ["scripts", "src"]
+exclude = ["nbs", "tests"]