jqueue 0.1.0__py3-none-any.whl

jqueue/__init__.py

@@ -0,0 +1,95 @@
+"""
+jqueue — object-storage queue with compare-and-set semantics.
+
+Implements the turbopuffer object-storage queue pattern:
+  https://turbopuffer.com/blog/object-storage-queue
+
+The queue state lives in a single JSON file on object storage. Every mutation
+is a compare-and-set (CAS) write — read the file, mutate in memory, write
+back with an If-Match guard. Concurrent writers that lose the CAS race retry
+automatically.
+
+For higher throughput, BrokerQueue batches concurrent operations into a single
+CAS write (group commit), reducing N concurrent writes to O(1) storage
+operations per round-trip.
+
+Quick start
+-----------
+    import asyncio
+    from jqueue import BrokerQueue, HeartbeatManager
+    from jqueue.adapters.storage.memory import InMemoryStorage
+
+    async def main():
+        storage = InMemoryStorage()
+
+        async with BrokerQueue(storage) as q:
+            # Enqueue work
+            await q.enqueue("send_email", b'{"to": "user@example.com"}')
+
+            # Claim and process
+            [job] = await q.dequeue("send_email")
+            async with HeartbeatManager(q, job.id):
+                print(f"Processing job {job.id}")
+            await q.ack(job.id)
+
+    asyncio.run(main())
+
+Storage adapters
+----------------
+Built-in adapters (no extra deps):
+  - InMemoryStorage           — for tests and examples
+  - LocalFileSystemStorage    — POSIX single-machine (fcntl.flock)
+
+Optional adapters (install extras):
+  - S3Storage        (pip install "jqueue[s3]")
+  - GCSStorage       (pip install "jqueue[gcs]")
+
+Custom adapters only need to implement the two-method ObjectStoragePort:
+  async def read() -> tuple[bytes, str | None]
+  async def write(content, if_match=None) -> str
+
+Architecture
+------------
+Follows the Ports & Adapters pattern:
+  domain/   — pure value types (Job, QueueState, JobStatus)
+  ports/    — Protocol interfaces (ObjectStoragePort)
+  core/     — business logic (DirectQueue, BrokerQueue, GroupCommitLoop)
+  adapters/ — concrete storage implementations
+"""
+
+from __future__ import annotations
+
+from jqueue.adapters.storage.filesystem import LocalFileSystemStorage
+from jqueue.adapters.storage.memory import InMemoryStorage
+from jqueue.core.broker import BrokerQueue
+from jqueue.core.direct import DirectQueue
+from jqueue.core.heartbeat import HeartbeatManager
+from jqueue.domain.errors import (
+    CASConflictError,
+    JobNotFoundError,
+    JQueueError,
+    StorageError,
+)
+from jqueue.domain.models import Job, JobStatus, QueueState
+from jqueue.ports.storage import ObjectStoragePort
+
+__all__ = [
+    # Domain models
+    "Job",
+    "JobStatus",
+    "QueueState",
+    # Errors
+    "JQueueError",
+    "CASConflictError",
+    "JobNotFoundError",
+    "StorageError",
+    # Port (for typing custom adapters)
+    "ObjectStoragePort",
+    # High-level queue API
+    "BrokerQueue",
+    "DirectQueue",
+    "HeartbeatManager",
+    # Built-in storage adapters
+    "InMemoryStorage",
+    "LocalFileSystemStorage",
+]

jqueue/adapters/storage/filesystem.py

@@ -0,0 +1,108 @@
+"""
+LocalFileSystemStorage — fcntl.flock-based CAS for POSIX systems.
+
+Suitable for local development, single-machine deployments, or integration
+tests that need a persistent file rather than in-memory state.
+
+NOT suitable for multi-machine deployments — use S3Storage or GCSStorage
+for distributed workloads.
+
+Etag strategy
+-------------
+The etag is a SHA-256 hex digest of the file contents. This is stable,
+deterministic, and always changes when content changes — unlike mtime which
+can be identical across rapid successive writes on fast machines.
+A file that is absent or empty is treated as non-existent; its etag is None.
+The jqueue codec always produces non-empty JSON, so a 0-byte file only occurs
+transiently before the first write completes.
+
+CAS semantics
+-------------
+write(content, if_match) acquires an exclusive flock, re-reads the current
+etag while holding the lock, and raises CASConflictError if it differs from
+if_match. The write is performed atomically within the same lock scope.
+
+POSIX-only (Linux, macOS). Not compatible with NFS or distributed filesystems.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import dataclasses
+import fcntl
+import hashlib
+import os
+from pathlib import Path
+
+from jqueue.domain.errors import CASConflictError
+
+
+@dataclasses.dataclass
+class LocalFileSystemStorage:
+    """
+    Stores the queue state in a local file.
+
+    Parameters
+    ----------
+    path : path to the JSON state file (parent directory created if absent)
+    """
+
+    path: Path
+
+    def __init__(self, path: str | Path) -> None:
+        self.path = Path(path)
+
+    async def read(self) -> tuple[bytes, str | None]:
+        """Return (content, etag). Returns (b"", None) if the file does not exist."""
+        return await asyncio.to_thread(self._sync_read)
+
+    async def write(
+        self,
+        content: bytes,
+        if_match: str | None = None,
+    ) -> str:
+        """CAS write. Raises CASConflictError on etag mismatch."""
+        return await asyncio.to_thread(self._sync_write, content, if_match)
+
+    # ------------------------------------------------------------------ #
+    # Synchronous implementations (executed in a thread-pool worker)      #
+    # ------------------------------------------------------------------ #
+
+    @staticmethod
+    def _etag(data: bytes) -> str:
+        return hashlib.sha256(data).hexdigest()
+
+    def _sync_read(self) -> tuple[bytes, str | None]:
+        if not self.path.exists():
+            return b"", None
+        with open(self.path, "rb") as fh:
+            fcntl.flock(fh, fcntl.LOCK_SH)
+            try:
+                content = fh.read()
+            finally:
+                fcntl.flock(fh, fcntl.LOCK_UN)
+        etag: str | None = self._etag(content) if content else None
+        return content, etag
+
+    def _sync_write(self, content: bytes, if_match: str | None) -> str:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        fd = os.open(str(self.path), os.O_RDWR | os.O_CREAT, 0o644)
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+
+            existing = os.read(fd, os.fstat(fd).st_size)
+            real_etag: str | None = self._etag(existing) if existing else None
+
+            if real_etag != if_match:
+                raise CASConflictError(
+                    f"ETag mismatch: expected {if_match!r}, got {real_etag!r}"
+                )
+
+            os.ftruncate(fd, 0)
+            os.lseek(fd, 0, os.SEEK_SET)
+            os.write(fd, content)
+        finally:
+            fcntl.flock(fd, fcntl.LOCK_UN)
+            os.close(fd)
+
+        return self._etag(content)

jqueue/adapters/storage/gcs.py

@@ -0,0 +1,130 @@
+"""
+GCSStorage — Google Cloud Storage adapter using google-cloud-storage.
+
+Install extras: pip install "jqueue[gcs]"
+
+CAS semantics
+-------------
+GCS supports conditional writes via object generation numbers.
+
+  read()  → returns (content, generation_string) where generation is the
+            integer GCS object generation, stringified to match the etag type
+  write() → uses if_generation_match=int(etag); GCS raises PreconditionFailed
+            on mismatch → CASConflictError
+
+First write (if_match=None):
+  Uses if_generation_match=0 — GCS convention for "blob must not exist yet".
+
+Note: google-cloud-storage is synchronous. All operations are wrapped in
+asyncio.to_thread to avoid blocking the event loop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import dataclasses
+from typing import TYPE_CHECKING
+
+from jqueue.domain.errors import CASConflictError, StorageError
+
+if TYPE_CHECKING:
+    from google.cloud.storage import Client as GCSClient
+
+
+@dataclasses.dataclass
+class GCSStorage:
+    """
+    Google Cloud Storage adapter.
+
+    Parameters
+    ----------
+    bucket_name : GCS bucket name
+    blob_name   : blob path (e.g. "queues/my-queue/state.json")
+    client      : google.cloud.storage.Client — created lazily if omitted
+    """
+
+    bucket_name: str
+    blob_name: str
+    client: GCSClient | None = None
+
+    def _get_client(self) -> GCSClient:
+        if self.client is not None:
+            return self.client
+        try:
+            from google.cloud import storage
+        except ImportError as exc:
+            raise ImportError(
+                "GCSStorage requires google-cloud-storage. "
+                "Install with: pip install 'jqueue[gcs]'"
+            ) from exc
+        return storage.Client()
+
+    async def read(self) -> tuple[bytes, str | None]:
+        """Read the state blob. Returns (b"", None) if the blob does not exist."""
+        try:
+            return await asyncio.to_thread(self._sync_read)
+        except (CASConflictError, StorageError):
+            raise
+        except Exception as exc:
+            raise StorageError("GCS read failed", exc) from exc
+
+    async def write(
+        self,
+        content: bytes,
+        if_match: str | None = None,
+    ) -> str:
+        """CAS write. Raises CASConflictError on generation mismatch."""
+        try:
+            return await asyncio.to_thread(self._sync_write, content, if_match)
+        except (CASConflictError, StorageError):
+            raise
+        except Exception as exc:
+            raise StorageError("GCS write failed", exc) from exc
+
+    # ------------------------------------------------------------------ #
+    # Synchronous implementations (executed in a thread-pool worker)      #
+    # ------------------------------------------------------------------ #
+
+    def _sync_read(self) -> tuple[bytes, str | None]:
+        try:
+            from google.api_core import exceptions as gapi_exc
+        except ImportError as exc:
+            raise ImportError(
+                "GCSStorage requires google-cloud-storage. "
+                "Install with: pip install 'jqueue[gcs]'"
+            ) from exc
+
+        client = self._get_client()
+        blob = client.bucket(self.bucket_name).blob(self.blob_name)
+        try:
+            content: bytes = blob.download_as_bytes()
+            return content, str(blob.generation)
+        except gapi_exc.NotFound:
+            return b"", None
+
+    def _sync_write(self, content: bytes, if_match: str | None) -> str:
+        try:
+            from google.api_core import exceptions as gapi_exc
+        except ImportError as exc:
+            raise ImportError(
+                "GCSStorage requires google-cloud-storage. "
+                "Install with: pip install 'jqueue[gcs]'"
+            ) from exc
+
+        client = self._get_client()
+        blob = client.bucket(self.bucket_name).blob(self.blob_name)
+
+        # if_generation_match=0 → "blob must not exist yet"
+        gen_match: int = 0 if if_match is None else int(if_match)
+
+        try:
+            blob.upload_from_string(
+                content,
+                content_type="application/json",
+                if_generation_match=gen_match,
+            )
+        except gapi_exc.PreconditionFailed as exc:
+            raise CASConflictError("GCS generation mismatch") from exc
+
+        blob.reload()
+        return str(blob.generation)

jqueue/adapters/storage/memory.py

@@ -0,0 +1,62 @@
+"""
+InMemoryStorage — asyncio.Lock-based CAS for testing and development.
+
+Stores the queue state as bytes in memory. Uses an asyncio.Lock to serialize
+reads and writes, faithfully simulating the CAS semantics of real object
+storage backends.
+
+The etag is a simple monotonic integer counter (stringified) that increments
+on every successful write.
+
+Zero external dependencies. Safe for multiple concurrent coroutines in a
+single event loop. NOT safe across processes or threads.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import dataclasses
+
+from jqueue.domain.errors import CASConflictError
+
+
+@dataclasses.dataclass
+class InMemoryStorage:
+    """
+    In-process object storage backed by a bytes buffer.
+
+    Parameters
+    ----------
+    initial_content : optional pre-populated bytes (useful for test setup)
+    """
+
+    initial_content: bytes = b""
+
+    def __post_init__(self) -> None:
+        self._content: bytes = self.initial_content
+        self._etag: str | None = "0" if self.initial_content else None
+        self._counter: int = 0
+        self._lock: asyncio.Lock = asyncio.Lock()
+
+    async def read(self) -> tuple[bytes, str | None]:
+        """Return (content, etag). etag is None until the first write."""
+        async with self._lock:
+            return self._content, self._etag
+
+    async def write(
+        self,
+        content: bytes,
+        if_match: str | None = None,
+    ) -> str:
+        """
+        CAS write. Raises CASConflictError if if_match differs from the current etag.
+        """
+        async with self._lock:
+            if if_match != self._etag:
+                raise CASConflictError(
+                    f"ETag mismatch: expected {if_match!r}, got {self._etag!r}"
+                )
+            self._counter += 1
+            self._etag = str(self._counter)
+            self._content = content
+            return self._etag

jqueue/adapters/storage/s3.py

@@ -0,0 +1,135 @@
+"""
+S3Storage — AWS S3 adapter using aioboto3 and If-Match conditional writes.
+
+Install extras: pip install "jqueue[s3]"
+
+CAS semantics
+-------------
+S3 supports conditional PutObject via the IfMatch parameter (added Aug 2024).
+
+  read()  → returns (content, ETag) where ETag is the S3 object's entity tag
+  write() → passes IfMatch=etag; S3 raises PreconditionFailed on mismatch
+            → CASConflictError
+
+First write (if_match=None):
+  IfMatch is omitted — unconditional put.
+
+Compatible with S3-compatible storage that supports conditional writes:
+  MinIO, Cloudflare R2, Tigris, etc.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import TYPE_CHECKING
+
+from jqueue.domain.errors import CASConflictError, StorageError
+
+if TYPE_CHECKING:
+    from aioboto3 import Session as AioBoto3Session
+
+
+@dataclasses.dataclass
+class S3Storage:
+    """
+    AWS S3 storage adapter.
+
+    Parameters
+    ----------
+    bucket       : S3 bucket name
+    key          : object key (e.g. "queues/my-queue/state.json")
+    session      : aioboto3.Session — created lazily from env vars if omitted
+    region_name  : AWS region passed to the S3 client
+    endpoint_url : custom endpoint for S3-compatible backends (e.g. MinIO)
+    """
+
+    bucket: str
+    key: str
+    session: AioBoto3Session | None = None
+    region_name: str | None = None
+    endpoint_url: str | None = None
+
+    def _get_session(self) -> AioBoto3Session:
+        if self.session is not None:
+            return self.session
+        try:
+            import aioboto3
+        except ImportError as exc:
+            raise ImportError(
+                "S3Storage requires aioboto3. Install with: pip install 'jqueue[s3]'"
+            ) from exc
+        return aioboto3.Session()
+
+    def _client_kwargs(self) -> dict[str, str]:
+        """Build kwargs forwarded to the S3 client constructor."""
+        kwargs: dict[str, str] = {}
+        if self.region_name:
+            kwargs["region_name"] = self.region_name
+        if self.endpoint_url:
+            kwargs["endpoint_url"] = self.endpoint_url
+        return kwargs
+
+    async def read(self) -> tuple[bytes, str | None]:
+        """Read the state object. Returns (b"", None) if the key does not exist."""
+        session = self._get_session()
+        try:
+            async with session.client("s3", **self._client_kwargs()) as s3:
+                try:
+                    response = await s3.get_object(Bucket=self.bucket, Key=self.key)
+                    content: bytes = await response["Body"].read()
+                    etag: str = response["ETag"]
+                    return content, etag
+                except Exception as exc:
+                    if _s3_error_code(exc) in ("NoSuchKey", "404"):
+                        return b"", None
+                    raise
+        except (CASConflictError, StorageError):
+            raise
+        except Exception as exc:
+            raise StorageError("S3 read failed", exc) from exc
+
+    async def write(
+        self,
+        content: bytes,
+        if_match: str | None = None,
+    ) -> str:
+        """CAS write. Raises CASConflictError on ETag mismatch (PreconditionFailed)."""
+        session = self._get_session()
+        try:
+            async with session.client("s3", **self._client_kwargs()) as s3:
+                put_kwargs: dict[str, str | bytes] = {
+                    "Bucket": self.bucket,
+                    "Key": self.key,
+                    "Body": content,
+                    "ContentType": "application/json",
+                }
+                if if_match is not None:
+                    put_kwargs["IfMatch"] = if_match
+
+                try:
+                    response = await s3.put_object(**put_kwargs)
+                    return str(response["ETag"])
+                except Exception as exc:
+                    if _s3_error_code(exc) == "PreconditionFailed":
+                        raise CASConflictError(
+                            "S3 ETag mismatch (PreconditionFailed)"
+                        ) from exc
+                    raise
+        except (CASConflictError, StorageError):
+            raise
+        except Exception as exc:
+            raise StorageError("S3 write failed", exc) from exc
+
+
+def _s3_error_code(exc: Exception) -> str:
+    """Extract the error code from a botocore ClientError, or return ''."""
+    try:
+        response = getattr(exc, "response", None)
+        if isinstance(response, dict):
+            error = response.get("Error", {})
+            if isinstance(error, dict):
+                code = error.get("Code", "")
+                return str(code) if code else ""
+    except Exception:  # noqa: BLE001
+        pass
+    return ""

jqueue/core/broker.py

@@ -0,0 +1,109 @@
+"""
+BrokerQueue — high-throughput queue backed by GroupCommitLoop.
+
+BrokerQueue is an async context manager that starts a GroupCommitLoop on
+__aenter__ and performs a clean shutdown (drain pending ops) on __aexit__.
+
+Usage
+-----
+    from jqueue import BrokerQueue, HeartbeatManager, InMemoryStorage
+
+    async with BrokerQueue(InMemoryStorage()) as q:
+        job = await q.enqueue("send_email", b'{"to": "user@example.com"}')
+
+        [job] = await q.dequeue("send_email")
+        async with HeartbeatManager(q, job.id):
+            process(job.payload)
+        await q.ack(job.id)
+
+Throughput
+----------
+All concurrent callers share a single writer task. While a CAS write is
+in-flight (typically 50–300 ms against real object storage), every enqueue
+and dequeue call that arrives is buffered and committed in the *next* write —
+collapsing N concurrent writes down to O(1) storage round-trips.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from datetime import timedelta
+from types import TracebackType
+
+from jqueue.core.group_commit import GroupCommitLoop
+from jqueue.domain.models import Job, QueueState
+from jqueue.ports.storage import ObjectStoragePort
+
+
+@dataclasses.dataclass
+class BrokerQueue:
+    """
+    Async context manager wrapping a GroupCommitLoop.
+
+    Parameters
+    ----------
+    storage       : any ObjectStoragePort implementation
+    stale_timeout : IN_PROGRESS jobs with a heartbeat older than this are
+                    automatically re-queued on each write cycle (default 5 min)
+    """
+
+    storage: ObjectStoragePort
+    stale_timeout: timedelta = timedelta(minutes=5)
+
+    _loop: GroupCommitLoop = dataclasses.field(init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        self._loop = GroupCommitLoop(
+            storage=self.storage,
+            stale_timeout=self.stale_timeout,
+        )
+
+    async def __aenter__(self) -> BrokerQueue:
+        await self._loop.start()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        await self._loop.stop()
+
+    # ------------------------------------------------------------------ #
+    # Queue operations (delegated to GroupCommitLoop)                     #
+    # ------------------------------------------------------------------ #
+
+    async def enqueue(
+        self,
+        entrypoint: str,
+        payload: bytes,
+        priority: int = 0,
+    ) -> Job:
+        """Add a new job. Returns the committed Job."""
+        return await self._loop.enqueue(entrypoint, payload, priority)
+
+    async def dequeue(
+        self,
+        entrypoint: str | None = None,
+        *,
+        batch_size: int = 1,
+    ) -> list[Job]:
+        """Claim up to batch_size QUEUED jobs and mark them IN_PROGRESS."""
+        return await self._loop.dequeue(entrypoint, batch_size=batch_size)
+
+    async def ack(self, job_id: str) -> None:
+        """Remove a completed job from the queue."""
+        await self._loop.ack(job_id)
+
+    async def nack(self, job_id: str) -> None:
+        """Return a job to QUEUED status."""
+        await self._loop.nack(job_id)
+
+    async def heartbeat(self, job_id: str) -> None:
+        """Refresh the heartbeat timestamp for an IN_PROGRESS job."""
+        await self._loop.heartbeat(job_id)
+
+    async def read_state(self) -> QueueState:
+        """Read-only snapshot of the current queue state."""
+        return await self._loop.read_state()

jqueue/core/codec.py

@@ -0,0 +1,42 @@
+"""
+Codec — serialize and deserialize QueueState to/from bytes using Pydantic v2.
+
+Pydantic v2 handles the full wire format automatically:
+  - bytes fields are encoded as base64 strings in JSON mode
+  - datetime fields are serialized as ISO-8601 strings with UTC offset
+  - Enum values are serialized as their string values
+  - Nested models (Job inside QueueState) are recursively serialized
+
+Wire format (produced by model_dump_json):
+------------------------------------------
+{
+  "version": 3,
+  "jobs": [
+    {
+      "id": "550e8400-...",
+      "entrypoint": "send_email",
+      "payload": "SGVsbG8gV29ybGQ=",   <-- base64-encoded bytes
+      "status": "queued",
+      "priority": 0,
+      "created_at": "2024-01-01T00:00:00+00:00",
+      "heartbeat_at": null
+    }
+  ]
+}
+"""
+
+from __future__ import annotations
+
+from jqueue.domain.models import QueueState
+
+
+def encode(state: QueueState) -> bytes:
+    """Serialize QueueState to UTF-8 JSON bytes."""
+    return state.model_dump_json(indent=2).encode("utf-8")
+
+
+def decode(data: bytes) -> QueueState:
+    """Deserialize UTF-8 JSON bytes to QueueState. Empty bytes → empty state."""
+    if not data:
+        return QueueState()
+    return QueueState.model_validate_json(data)