queue-max 0.1.0__tar.gz

queue_max-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Robusta Queue Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

queue_max-0.1.0/MANIFEST.in

@@ -0,0 +1,7 @@
+include README.md
+include LICENSE
+include pyproject.toml
+recursive-include src *.py
+recursive-include src *.typed
+recursive-include tests *.py
+recursive-include examples *.py

queue_max-0.1.0/PKG-INFO

@@ -0,0 +1,233 @@
+Metadata-Version: 2.4
+Name: queue-max
+Version: 0.1.0
+Summary: Task queue with SQLite sharding, rate limiting, and circuit breaker
+Author: Alexandre All
+License: MIT
+Project-URL: homepage, https://github.com/all451/queue-max
+Project-URL: repository, https://github.com/all451/queue-max
+Keywords: queue,task-queue,sqlite,background-tasks,worker,sharding
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typing-extensions>=4.5.0
+Provides-Extra: webhook
+Requires-Dist: requests>=2.31.0; extra == "webhook"
+Provides-Extra: django
+Requires-Dist: Django>=3.2; extra == "django"
+Provides-Extra: fastapi
+Requires-Dist: fastapi>=0.100.0; extra == "fastapi"
+Provides-Extra: flask
+Requires-Dist: Flask>=2.0; extra == "flask"
+Provides-Extra: all
+Requires-Dist: requests>=2.31.0; extra == "all"
+Requires-Dist: Django>=3.2; extra == "all"
+Requires-Dist: fastapi>=0.100.0; extra == "all"
+Requires-Dist: Flask>=2.0; extra == "all"
+Dynamic: license-file
+
+# Queue Max
+
+Task queue library with SQLite persistence, sharding, rate limiting, and circuit breaker.
+
+No Redis or RabbitMQ required. Zero external dependencies (except typing-extensions).
+
+## Installation
+
+```bash
+pip install queue-max
+```
+
+With framework integrations:
+
+```bash
+pip install queue-max[django]
+pip install queue-max[fastapi]
+pip install queue-max[flask]
+```
+
+## Quick Start
+
+```python
+from queue_max import Queue, Worker
+
+# Create queue
+queue = Queue()
+
+# Enqueue a job
+queue.enqueue({"task": "send_email", "to": "user@example.com"}, priority=2)
+
+# Define processor
+def process(payload):
+    print(f"Processing: {payload}")
+
+# Start worker
+worker = Worker("worker-1", process, queue)
+worker.start()
+```
+
+## Features
+
+**SQLite Persistence** -- Jobs are stored in SQLite with WAL mode. No external services needed.
+
+**Physical Sharding** -- Multiple .db files allow concurrent read/write across workers.
+
+**Rate Limiting** -- Token bucket algorithm shared across all workers. Configurable per minute.
+
+**Circuit Breaker** -- Stops calling failing services after N consecutive errors. Recovers automatically.
+
+**Retry with Backoff** -- Exponential backoff with jitter for failed jobs. Configurable max retries.
+
+**Heartbeat and Recovery** -- Workers send periodic heartbeats. Orphaned jobs are recovered automatically.
+
+**Priority Queues** -- Three levels: low (0), medium (1), high (2).
+
+**CLI** -- Built-in command line for stats, workers, and queue management.
+
+## Configuration via Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| NUM_SHARDS | 6 | Number of shard databases |
+| RATE_LIMIT_MAX | 160 | Max requests per minute |
+| QUEUE_MAX_RETRIES | 3 | Default max retry attempts |
+| DB_BUSY_TIMEOUT | 30000 | SQLite busy timeout (ms) |
+| HEARTBEAT_INTERVAL | 5000 | Worker heartbeat interval (ms) |
+| STUCK_TIMEOUT | 30000 | Orphan job timeout (ms) |
+| DATA_DIR | ./data | Directory for shard files |
+
+## API Overview
+
+### Queue
+
+```python
+from queue_max import Queue
+
+queue = Queue(shards=6, rate_limit=160, max_retries=3)
+
+# Enqueue jobs
+queue.enqueue(payload, pagina_id=None, priority=0, max_retries=None)
+queue.enqueue_batch([{"payload": {...}}, ...])
+
+# Process jobs
+job = queue.pop_job(worker_id)
+queue.complete_job(job_id, shard_id)
+queue.fail_job(job_id, shard_id, error, permanent=False)
+
+# Management
+queue.retry_failed_jobs()
+queue.cleanup_old_jobs(days=7)
+queue.recover_orphans()
+stats = queue.get_stats()
+```
+
+### Worker
+
+```python
+from queue_max import Worker, WorkerPool
+
+worker = Worker("worker-1", process_function, queue)
+worker.start()
+worker.stop()
+
+pool = WorkerPool([worker1, worker2])
+pool.start_all()
+pool.stop_all()
+```
+
+### Decorator
+
+```python
+from queue_max import task
+
+@task(priority=2, max_retries=3)
+def send_email(to: str, subject: str):
+    return send(to, subject)
+
+send_email.delay("user@example.com", "Hello")
+```
+
+### CLI
+
+```bash
+queue-max stats
+queue-max worker --function mymodule:myfunction --workers 4
+queue-max enqueue --payload '{"task": "test"}' --priority 2
+queue-max list --status failed --limit 20
+queue-max retry
+queue-max purge --days 7
+```
+
+## Framework Integrations
+
+### Django
+
+```python
+# settings.py
+INSTALLED_APPS = ["queue_max.contrib.django", ...]
+QUEUE_MAX = {"SHARDS": 4, "RATE_LIMIT": 160}
+
+# tasks.py
+from queue_max.contrib.django import task
+@task
+def my_task(user_id): ...
+```
+
+Management commands: `python manage.py queue_worker`, `queue_stats`, `queue_purge`.
+
+### FastAPI
+
+```python
+from fastapi import FastAPI
+from queue_max.contrib.fastapi import QueueMiddleware
+
+app = FastAPI()
+app.add_middleware(QueueMiddleware, max_workers=4)
+```
+
+### Flask
+
+```python
+from flask import Flask
+from queue_max.contrib.flask import QueueExtension
+
+app = Flask(__name__)
+queue = QueueExtension(app)
+
+@queue.task
+def my_task(): ...
+```
+
+## Performance
+
+| Cenário | Throughput |
+|---------|-----------|
+| Burst (20 workers, 10 shards) | **~3.300 jobs/sec** |
+| Contenção (10 workers, 1 shard) | **~1.660 jobs/sec** |
+| Com 30% de falhas (8 workers) | **Estável** — circuit breaker não trip |
+
+- Max queue size: 1M+ jobs per shard
+- [Resultados detalhados dos stress tests](docs/stress-test.md)
+
+## Running Tests
+
+```bash
+pip install -e ".[dev]"
+pytest tests/ -v
+```
+
+## License
+
+MIT

queue_max-0.1.0/README.md

@@ -0,0 +1,193 @@
+# Queue Max
+
+Task queue library with SQLite persistence, sharding, rate limiting, and circuit breaker.
+
+No Redis or RabbitMQ required. Zero external dependencies (except typing-extensions).
+
+## Installation
+
+```bash
+pip install queue-max
+```
+
+With framework integrations:
+
+```bash
+pip install queue-max[django]
+pip install queue-max[fastapi]
+pip install queue-max[flask]
+```
+
+## Quick Start
+
+```python
+from queue_max import Queue, Worker
+
+# Create queue
+queue = Queue()
+
+# Enqueue a job
+queue.enqueue({"task": "send_email", "to": "user@example.com"}, priority=2)
+
+# Define processor
+def process(payload):
+    print(f"Processing: {payload}")
+
+# Start worker
+worker = Worker("worker-1", process, queue)
+worker.start()
+```
+
+## Features
+
+**SQLite Persistence** -- Jobs are stored in SQLite with WAL mode. No external services needed.
+
+**Physical Sharding** -- Multiple .db files allow concurrent read/write across workers.
+
+**Rate Limiting** -- Token bucket algorithm shared across all workers. Configurable per minute.
+
+**Circuit Breaker** -- Stops calling failing services after N consecutive errors. Recovers automatically.
+
+**Retry with Backoff** -- Exponential backoff with jitter for failed jobs. Configurable max retries.
+
+**Heartbeat and Recovery** -- Workers send periodic heartbeats. Orphaned jobs are recovered automatically.
+
+**Priority Queues** -- Three levels: low (0), medium (1), high (2).
+
+**CLI** -- Built-in command line for stats, workers, and queue management.
+
+## Configuration via Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| NUM_SHARDS | 6 | Number of shard databases |
+| RATE_LIMIT_MAX | 160 | Max requests per minute |
+| QUEUE_MAX_RETRIES | 3 | Default max retry attempts |
+| DB_BUSY_TIMEOUT | 30000 | SQLite busy timeout (ms) |
+| HEARTBEAT_INTERVAL | 5000 | Worker heartbeat interval (ms) |
+| STUCK_TIMEOUT | 30000 | Orphan job timeout (ms) |
+| DATA_DIR | ./data | Directory for shard files |
+
+## API Overview
+
+### Queue
+
+```python
+from queue_max import Queue
+
+queue = Queue(shards=6, rate_limit=160, max_retries=3)
+
+# Enqueue jobs
+queue.enqueue(payload, pagina_id=None, priority=0, max_retries=None)
+queue.enqueue_batch([{"payload": {...}}, ...])
+
+# Process jobs
+job = queue.pop_job(worker_id)
+queue.complete_job(job_id, shard_id)
+queue.fail_job(job_id, shard_id, error, permanent=False)
+
+# Management
+queue.retry_failed_jobs()
+queue.cleanup_old_jobs(days=7)
+queue.recover_orphans()
+stats = queue.get_stats()
+```
+
+### Worker
+
+```python
+from queue_max import Worker, WorkerPool
+
+worker = Worker("worker-1", process_function, queue)
+worker.start()
+worker.stop()
+
+pool = WorkerPool([worker1, worker2])
+pool.start_all()
+pool.stop_all()
+```
+
+### Decorator
+
+```python
+from queue_max import task
+
+@task(priority=2, max_retries=3)
+def send_email(to: str, subject: str):
+    return send(to, subject)
+
+send_email.delay("user@example.com", "Hello")
+```
+
+### CLI
+
+```bash
+queue-max stats
+queue-max worker --function mymodule:myfunction --workers 4
+queue-max enqueue --payload '{"task": "test"}' --priority 2
+queue-max list --status failed --limit 20
+queue-max retry
+queue-max purge --days 7
+```
+
+## Framework Integrations
+
+### Django
+
+```python
+# settings.py
+INSTALLED_APPS = ["queue_max.contrib.django", ...]
+QUEUE_MAX = {"SHARDS": 4, "RATE_LIMIT": 160}
+
+# tasks.py
+from queue_max.contrib.django import task
+@task
+def my_task(user_id): ...
+```
+
+Management commands: `python manage.py queue_worker`, `queue_stats`, `queue_purge`.
+
+### FastAPI
+
+```python
+from fastapi import FastAPI
+from queue_max.contrib.fastapi import QueueMiddleware
+
+app = FastAPI()
+app.add_middleware(QueueMiddleware, max_workers=4)
+```
+
+### Flask
+
+```python
+from flask import Flask
+from queue_max.contrib.flask import QueueExtension
+
+app = Flask(__name__)
+queue = QueueExtension(app)
+
+@queue.task
+def my_task(): ...
+```
+
+## Performance
+
+| Cenário | Throughput |
+|---------|-----------|
+| Burst (20 workers, 10 shards) | **~3.300 jobs/sec** |
+| Contenção (10 workers, 1 shard) | **~1.660 jobs/sec** |
+| Com 30% de falhas (8 workers) | **Estável** — circuit breaker não trip |
+
+- Max queue size: 1M+ jobs per shard
+- [Resultados detalhados dos stress tests](docs/stress-test.md)
+
+## Running Tests
+
+```bash
+pip install -e ".[dev]"
+pytest tests/ -v
+```
+
+## License
+
+MIT

queue_max-0.1.0/examples/ai_rate_limited.py

@@ -0,0 +1,78 @@
+"""AI/LLM API usage example with rate limiting.
+
+Shows how to use Robusta Queue to handle rate-limited API calls
+for AI/LLM services like OpenAI, Anthropic, etc.
+"""
+
+import time
+
+from queue_max import Queue, Worker
+
+# Configure queue with rate limit matching your API tier
+# e.g., OpenAI free tier: 3 RPM, Tier 1: 500 RPM
+queue = Queue(shards=2, rate_limit=10, max_retries=3)
+
+
+def call_llm_api(payload: dict) -> dict:
+    """Simulate calling an LLM API with rate limiting.
+
+    Replace with actual API call:
+        import openai
+        response = openai.ChatCompletion.create(
+            model="gpt-4",
+            messages=[{"role": "user", "content": payload['prompt']}]
+        )
+    """
+    prompt = payload.get("prompt", "")
+    print(f"  Calling LLM API with prompt: {prompt[:50]}...")
+
+    # Simulate API latency
+    time.sleep(1.0)
+
+    # Simulate rate limit error
+    import random
+
+    if random.random() < 0.1:
+        raise Exception("429 Too Many Requests")
+
+    return {"response": f"Response to: {prompt}", "tokens": len(prompt)}
+
+
+def main():
+    prompts = [
+        "Explain quantum computing in simple terms",
+        "Write a Python decorator for logging",
+        "Summarize the theory of relativity",
+        "Create a recipe for vegan chocolate cake",
+        "Translate 'hello world' to Japanese",
+        "Write a haiku about programming",
+        "Explain how databases work",
+        "Describe the water cycle",
+    ]
+
+    print("  Enqueuing LLM API calls...")
+    for prompt in prompts:
+        result = queue.enqueue(
+            payload={"prompt": prompt, "model": "gpt-4"},
+            priority=1,
+        )
+        print(f"    Enqueued: {prompt[:30]}... (job {result['id']})")
+
+    # Start worker with rate-limited processing
+    worker = Worker("ai-worker", call_llm_api, queue)
+    worker.start()
+
+    print("\n  Processing with rate limiting (10 req/min)...")
+    time.sleep(15)
+
+    worker_stats = worker.get_stats()
+    print(f"\n  Processed: {worker_stats['processed']}")
+    print(f"  Failed: {worker_stats['failed']}")
+    print(f"  Retried: {worker_stats['retried']}")
+
+    worker.stop()
+    print("  Done!")
+
+
+if __name__ == "__main__":
+    main()

queue_max-0.1.0/examples/basic_usage.py

@@ -0,0 +1,43 @@
+"""Basic usage example for Robusta Queue."""
+
+from queue_max import Queue, Worker
+
+
+def process_task(payload: dict) -> str:
+    """Process a task payload."""
+    print(f"  Processing: {payload}")
+    return f"Done: {payload.get('task', 'unknown')}"
+
+
+def main():
+    # 1. Create a queue (3 shards, 100 req/min)
+    queue = Queue(shards=3, rate_limit=100)
+
+    # 2. Enqueue some jobs
+    for i in range(5):
+        result = queue.enqueue(
+            payload={"task": f"job-{i}", "data": f"payload-{i}"},
+            priority=i % 3,  # Mix of priorities
+            pagina_id=i % 3,  # Spread across shards
+        )
+        print(f"  Enqueued job {result['id']} in shard {result['shard_id']}")
+
+    # 3. Process with a worker
+    worker = Worker("example-worker", process_task, queue)
+    worker.start()
+
+    # 4. Let it process for a bit
+    import time
+
+    time.sleep(3)
+
+    # 5. Check stats
+    stats = queue.get_stats()
+    print(f"\n  Queue stats: {stats}")
+
+    worker.stop()
+    print("  Done!")
+
+
+if __name__ == "__main__":
+    main()

queue_max-0.1.0/examples/webhook_processor.py

@@ -0,0 +1,65 @@
+"""Webhook processing example with Robusta Queue."""
+
+import json
+import time
+
+from queue_max import Queue, Worker
+
+# Queue configured for webhook processing
+queue = Queue(shards=4, rate_limit=160, max_retries=3)
+
+
+def process_webhook(payload: dict) -> dict:
+    """Process a webhook payload.
+
+    In real usage, this would call an external API, send emails, etc.
+    """
+    print(f"  Processing webhook: {payload.get('event', 'unknown')}")
+    print(f"  Account: {payload.get('account_id', 'N/A')}")
+
+    # Simulate API call
+    time.sleep(0.5)
+
+    # Simulate occasional failure
+    if payload.get("simulate_error"):
+        raise ConnectionError("External API timeout")
+
+    return {"status": "processed"}
+
+
+def main():
+    # Enqueue sample webhooks
+    webhooks = [
+        {"event": "user.created", "account_id": 101, "user": "alice"},
+        {"event": "payment.received", "account_id": 102, "amount": 99.90},
+        {"event": "order.shipped", "account_id": 101, "order_id": "ORD-123"},
+        {"event": "user.deleted", "account_id": 103, "user": "bob"},
+    ]
+
+    for wh in webhooks:
+        result = queue.enqueue(
+            payload=wh,
+            priority=2,  # Webhooks are high priority
+            pagina_id=wh.get("account_id"),  # Consistent sharding per account
+        )
+        print(f"  Enqueued webhook: {wh['event']} (job {result['id']})")
+
+    # Start worker
+    worker = Worker("webhook-worker", process_webhook, queue)
+    worker.start()
+
+    print("\n  Processing webhooks...")
+    time.sleep(5)
+
+    # Stats
+    stats = queue.get_stats()
+    worker_stats = worker.get_stats()
+    print(f"\n  Queue: {stats['pending']} pending, {stats['failed']} failed")
+    print(f"  Worker: {worker_stats['processed']} processed")
+
+    worker.stop()
+    print("  Done!")
+
+
+if __name__ == "__main__":
+    main()