taraqueue 0.4.0__tar.gz → 0.6.0__tar.gz

{taraqueue-0.4.0 → taraqueue-0.6.0}/CHANGES.rst

@@ -1,3 +1,17 @@
+Version 0.6.0
+-------------
+
+Released 2026-05-22
+
+-   Add Amazon SQS queue.
+
+Version 0.5.0
+-------------
+
+Released 2026-05-13
+
+-   Fix MemoryQueue to behave like RedisQueue.
+
 Version 0.4.0
 -------------
 

{taraqueue-0.4.0 → taraqueue-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taraqueue
-Version: 0.4.0
+Version: 0.6.0
 Summary: Python queue abstraction layer
 Project-URL: Repository, https://github.com/taradix/taraqueue
 Author-email: Marc Tardif <marc@taram.ca>
@@ -15,12 +15,16 @@ Requires-Dist: sphinx<10.0.0,>=9.0.0; extra == 'docs'
 Requires-Dist: sphinxcontrib-log-cabinet<2.0.0,>=1.0.1; extra == 'docs'
 Provides-Extra: redis
 Requires-Dist: redis>=7.4.0; extra == 'redis'
+Provides-Extra: sqs
+Requires-Dist: aiobotocore>=2.21.0; extra == 'sqs'
 Provides-Extra: test
+Requires-Dist: aiobotocore>=2.21.0; extra == 'test'
 Requires-Dist: coverage<8.0.0,>=7.2.3; extra == 'test'
 Requires-Dist: more-itertools<11.1.0,>=11.0.1; extra == 'test'
+Requires-Dist: moto[server,sns,sqs]>=5.1.0; extra == 'test'
 Requires-Dist: pytest-asyncio<2.0.0,>=1.0.0; extra == 'test'
 Requires-Dist: pytest-unique<1.0.0,>=0.1.8; extra == 'test'
-Requires-Dist: pytest-xdocker<1.0.0,>=0.2.8; extra == 'test'
+Requires-Dist: pytest-xdocker<1.0.0,>=0.2.9; extra == 'test'
 Requires-Dist: pytest<10.0.0,>=9.0.0; extra == 'test'
 Requires-Dist: redis>=7.4.0; extra == 'test'
 Description-Content-Type: text/x-rst

{taraqueue-0.4.0 → taraqueue-0.6.0}/docs/taraqueue.rst

@@ -28,6 +28,14 @@ taraqueue.redis module
    :show-inheritance:
    :undoc-members:
 
+taraqueue.sqs module
+--------------------
+
+.. automodule:: taraqueue.sqs
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
 taraqueue.registry module
 -------------------------
 

{taraqueue-0.4.0 → taraqueue-0.6.0}/pyproject.toml

@@ -20,14 +20,20 @@ redis = [
     "redis>=7.4.0",
 ]
 
+sqs = [
+    "aiobotocore>=2.21.0",
+]
+
 test = [
+    "aiobotocore>=2.21.0",
     "coverage>=7.2.3,<8.0.0",
+    "moto[sns,sqs,server]>=5.1.0",
     "more-itertools>=11.0.1,<11.1.0",
     "redis>=7.4.0",
     "pytest>=9.0.0,<10.0.0",
     "pytest-asyncio>=1.0.0,<2.0.0",
     "pytest-unique>=0.1.8,<1.0.0",
-    "pytest-xdocker>=0.2.8,<1.0.0",
+    "pytest-xdocker>=0.2.9,<1.0.0",
 ]
 
 check = [
@@ -47,6 +53,7 @@ taraqueue-queue = "taraqueue.testing.queue"
 [project.entry-points."taraqueue"]
 memory = "taraqueue.memory:MemoryQueue"
 redis = "taraqueue.redis:RedisQueue"
+sqs = "taraqueue.sqs:SQSQueue"
 
 [build-system]
 requires = ["hatchling", "hatch-vcs"]
@@ -110,6 +117,7 @@ ignore = [
 
 [tool.ruff.lint.per-file-ignores]
 "tests/*" = ["S101", "S106"]
+"taraqueue/testing/*" = ["S106", "S603"]
 
 # Pytest options:
 #   https://docs.pytest.org/en/6.2.x/reference.html#ini-options-ref

taraqueue-0.6.0/taraqueue/memory.py

@@ -0,0 +1,58 @@
+"""Memory queue implementation."""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import suppress
+from dataclasses import dataclass, field
+
+from yarl import URL
+
+from taraqueue import Queue, QueueEmpty
+
+_channels: dict[str, list[asyncio.Queue[str]]] = {}
+
+
+@dataclass
+class MemoryQueue(Queue):
+    """In-process queue with fan-out semantics."""
+
+    queue: asyncio.Queue[str] = field(default_factory=asyncio.Queue)
+
+    @classmethod
+    def from_url(cls, url: URL) -> MemoryQueue:
+        return cls()
+
+    async def subscribe(self, topic: str) -> None:
+        """See `Queue.subscribe`."""
+        subscribers = _channels.setdefault(topic, [])
+        if self.queue in subscribers:
+            return
+        subscribers.append(self.queue)
+
+    async def unsubscribe(self, topic: str) -> None:
+        """See `Queue.unsubscribe`."""
+        subscribers = _channels.get(topic)
+        if subscribers is None:
+            return
+        with suppress(ValueError):
+            subscribers.remove(self.queue)
+        if not subscribers:
+            _channels.pop(topic, None)
+
+    async def receive(self, timeout=None) -> str:
+        """See `Queue.receive`."""
+        if not timeout:
+            try:
+                return self.queue.get_nowait()
+            except asyncio.QueueEmpty as e:
+                raise QueueEmpty("Queue is empty") from e
+        try:
+            return await asyncio.wait_for(self.queue.get(), timeout=float(timeout))
+        except TimeoutError as e:
+            raise QueueEmpty("Timed out waiting for message") from e
+
+    async def publish(self, topic: str, message: str) -> None:
+        """See `Queue.publish`."""
+        for q in list(_channels.get(topic, [])):
+            await q.put(message)

{taraqueue-0.4.0 → taraqueue-0.6.0}/taraqueue/redis.py

@@ -1,15 +1,20 @@
 """Redis queue implementation."""
 
+from __future__ import annotations
+
 import os
 from dataclasses import dataclass, field
 from time import time
+from typing import TYPE_CHECKING
 
-from redis.asyncio import Redis
-from redis.asyncio.client import PubSub
 from yarl import URL
 
 from taraqueue import Queue, QueueEmpty
 
+if TYPE_CHECKING:
+    from redis.asyncio import Redis
+    from redis.asyncio.client import PubSub
+
 
 @dataclass
 class RedisQueue(Queue):
@@ -18,14 +23,14 @@ class RedisQueue(Queue):
     pubsub: PubSub = field()
 
     @classmethod
-    def from_env(cls, env=os.environ) -> "RedisQueue":
+    def from_env(cls, env=os.environ) -> RedisQueue:
         host = env.get("REDIS_SLAVEOF_IP", "") or env.get("IPV4_NETWORK", "172.22.1") + ".249"
         port = int(env.get("REDIS_SLAVEOF_PORT", "") or "6379")
         password = env.get("REDIS_PASSWORD")
         return cls.from_host(host, port, password=password)
 
     @classmethod
-    def from_host(cls, host: str, port: int = 6379, password: str | None = None) -> "RedisQueue":
+    def from_host(cls, host: str, port: int = 6379, password: str | None = None) -> RedisQueue:
         from redis.asyncio import StrictRedis
 
         client = StrictRedis(
@@ -39,7 +44,7 @@ class RedisQueue(Queue):
         return cls(client, pubsub)
 
     @classmethod
-    def from_url(cls, url: URL | str) -> "RedisQueue":
+    def from_url(cls, url: URL | str) -> RedisQueue:
         url = URL(url)
         return cls.from_host(url.host, url.port, password=url.password)
 

taraqueue-0.6.0/taraqueue/sqs.py

@@ -0,0 +1,166 @@
+"""Amazon SQS queue implementation using SNS for pub/sub fan-out."""
+
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import dataclass, field
+from time import time
+from typing import TYPE_CHECKING
+
+from yarl import URL
+
+from taraqueue import Queue, QueueEmpty
+
+if TYPE_CHECKING:
+    from types import ModuleType
+
+
+@dataclass
+class SQSQueue(Queue):
+    """Queue backed by Amazon SNS (publish) and SQS (subscribe/receive)."""
+
+    session: object = field(repr=False)
+    region: str = "us-east-1"
+    endpoint_url: str | None = None
+    _subscriptions: dict[str, _Subscription] = field(default_factory=dict, repr=False)
+    _aiobotocore: ModuleType | None = field(default=None, repr=False)
+
+    @classmethod
+    def from_url(cls, url: URL | str) -> SQSQueue:
+        """Create an SQSQueue from a URL."""
+        import aiobotocore.session
+
+        url = URL(url)
+        region = url.path.lstrip("/") or "us-east-1"
+
+        endpoint_url = None
+        if url.host:
+            scheme = "https" if url.port == 443 else "http"
+            port = url.port or 4566
+            endpoint_url = f"{scheme}://{url.host}:{port}"
+
+        session = aiobotocore.session.get_session()
+        if url.user:
+            session.set_credentials(url.user, url.password or "")
+
+        return cls(
+            session=session,
+            region=region,
+            endpoint_url=endpoint_url,
+        )
+
+    def _create_client(self, service: str):
+        return self.session.create_client(
+            service,
+            region_name=self.region,
+            endpoint_url=self.endpoint_url,
+        )
+
+    async def subscribe(self, topic: str) -> None:
+        """See `Queue.subscribe`."""
+        if topic in self._subscriptions:
+            return
+
+        async with self._create_client("sns") as sns:
+            resp = await sns.create_topic(Name=topic)
+            topic_arn = resp["TopicArn"]
+
+        queue_name = f"taraqueue-{topic}-{uuid.uuid4().hex[:8]}"
+        async with self._create_client("sqs") as sqs:
+            resp = await sqs.create_queue(QueueName=queue_name)
+            queue_url = resp["QueueUrl"]
+            attrs = await sqs.get_queue_attributes(QueueUrl=queue_url, AttributeNames=["QueueArn"])
+            queue_arn = attrs["Attributes"]["QueueArn"]
+
+            # Allow SNS to send messages to this SQS queue.
+            policy = json.dumps({
+                "Version": "2012-10-17",
+                "Statement": [{
+                    "Effect": "Allow",
+                    "Principal": "*",
+                    "Action": "sqs:SendMessage",
+                    "Resource": queue_arn,
+                    "Condition": {"ArnEquals": {"aws:SourceArn": topic_arn}},
+                }],
+            })
+            await sqs.set_queue_attributes(QueueUrl=queue_url, Attributes={"Policy": policy})
+
+        async with self._create_client("sns") as sns:
+            resp = await sns.subscribe(
+                TopicArn=topic_arn,
+                Protocol="sqs",
+                Endpoint=queue_arn,
+            )
+            subscription_arn = resp["SubscriptionArn"]
+            # Request raw message delivery so we don't have to unwrap the SNS envelope.
+            await sns.set_subscription_attributes(
+                SubscriptionArn=subscription_arn,
+                AttributeName="RawMessageDelivery",
+                AttributeValue="true",
+            )
+
+        self._subscriptions[topic] = _Subscription(
+            topic_arn=topic_arn,
+            queue_url=queue_url,
+            queue_name=queue_name,
+            subscription_arn=subscription_arn,
+        )
+
+    async def unsubscribe(self, topic: str) -> None:
+        """See `Queue.unsubscribe`."""
+        sub = self._subscriptions.pop(topic, None)
+        if sub is None:
+            return
+
+        async with self._create_client("sns") as sns:
+            await sns.unsubscribe(SubscriptionArn=sub.subscription_arn)
+
+        async with self._create_client("sqs") as sqs:
+            await sqs.delete_queue(QueueUrl=sub.queue_url)
+
+    async def receive(self, timeout=None) -> str:
+        """See `Queue.receive`."""
+        if not self._subscriptions:
+            raise QueueEmpty("Queue is empty")
+
+        wait_time = int(timeout) if timeout and int(timeout) > 0 else 0
+        stop_time = time() + (float(timeout) if timeout else 0)
+
+        while True:
+            for sub in list(self._subscriptions.values()):
+                async with self._create_client("sqs") as sqs:
+                    resp = await sqs.receive_message(
+                        QueueUrl=sub.queue_url,
+                        MaxNumberOfMessages=1,
+                        WaitTimeSeconds=min(wait_time, 20),
+                    )
+                messages = resp.get("Messages", [])
+                if messages:
+                    msg = messages[0]
+                    async with self._create_client("sqs") as sqs:
+                        await sqs.delete_message(
+                            QueueUrl=sub.queue_url,
+                            ReceiptHandle=msg["ReceiptHandle"],
+                        )
+                    return msg["Body"]
+
+            if time() >= stop_time:
+                raise QueueEmpty("Queue is empty")
+
+    async def publish(self, topic: str, message: str) -> None:
+        """See `Queue.publish`."""
+        async with self._create_client("sns") as sns:
+            resp = await sns.create_topic(Name=topic)
+            topic_arn = resp["TopicArn"]
+            await sns.publish(TopicArn=topic_arn, Message=message)
+
+
+@dataclass(frozen=True)
+class _Subscription:
+    """Internal state for a single topic subscription."""
+
+    topic_arn: str
+    queue_url: str
+    queue_name: str
+    subscription_arn: str

{taraqueue-0.4.0 → taraqueue-0.6.0}/taraqueue/testing/compose.py

@@ -66,7 +66,6 @@ class ComposeServer(ProcessServer):
 
     def prepare_func(self, controldir):
         """Prepare the function to run the compose service."""
-        full_name = self.full_name(controldir.basename)
         compose = xdocker.compose().with_project_name(self.project)
         if env_file := self.env_file:
             compose = compose.with_env_file(env_file)
@@ -75,11 +74,8 @@ class ComposeServer(ProcessServer):
 
         command = (
             compose
-            .run(controldir.basename)
-            .with_name(full_name)
-            .with_build()
-            .with_remove()
-            .with_optionals("--use-aliases")
+            .up(controldir.basename)
+            .with_optionals("--no-deps")
         )
 
         return ProcessData(self.pattern, command, timeout=self.timeout)

taraqueue-0.6.0/taraqueue/testing/queue.py

@@ -0,0 +1,88 @@
+"""Queue fixtures."""
+
+import subprocess
+import sys
+import time
+
+import pytest
+from yarl import URL
+
+from taraqueue import Queue
+
+
+@pytest.fixture
+def memory_queue():
+    """Memory queue fixture."""
+    url = URL.build(scheme="memory")
+    return Queue.from_url(url)
+
+
+@pytest.fixture
+def redis_queue(redis_service, taraqueue_env_vars):
+    """Redis queue fixture."""
+    url = URL.build(
+        scheme="redis",
+        host=redis_service.ip,
+        port=6379,
+        password=taraqueue_env_vars["REDIS_PASSWORD"],
+    )
+    return Queue.from_url(url)
+
+
+@pytest.fixture
+def sqs_queue(moto_server):
+    """SQS queue fixture backed by moto server."""
+    url = URL.build(
+        scheme="sqs",
+        host="127.0.0.1",
+        port=moto_server,
+        user="testing",
+        password="testing",
+    )
+    return Queue.from_url(url)
+
+
+@pytest.fixture(scope="session")
+def moto_server():
+    """Start a moto standalone server for AWS service mocking."""
+    import socket
+
+    # Find a free port
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        port = s.getsockname()[1]
+
+    proc = subprocess.Popen(
+        [sys.executable, "-m", "moto.server", "-p", str(port)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    # Wait for the server to be ready
+    for _ in range(50):
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                s.connect(("127.0.0.1", port))
+                break
+        except OSError:
+            time.sleep(0.1)
+    else:
+        proc.terminate()
+        raise RuntimeError("moto server failed to start")
+
+    yield port
+
+    proc.terminate()
+    proc.wait()
+
+
+@pytest.fixture(
+    params=[
+        "memory_queue",
+        "redis_queue",
+        "sqs_queue",
+    ],
+)
+def queue(request):
+    """Queue fixture."""
+    return request.getfixturevalue(request.param)

taraqueue-0.6.0/tests/test_queue.py

@@ -0,0 +1,75 @@
+"""Integration tests for the queue module."""
+
+
+import pytest
+
+from taraqueue import Queue, QueueEmpty
+
+
+async def test_queue_send_receive(queue, unique):
+    """Sending a message to a queue should be the next received message."""
+    topic = unique("text")
+    async with queue.connect(topic) as session:
+        await session.publish(topic, "test")
+        result = await session.receive(5)
+    assert result == "test"
+
+
+async def test_queue_receive_empty(queue, unique):
+    """Receiving from an empty queue should raise a QueueEmpty error."""
+    topic = unique("text")
+    async with queue.connect(topic) as session:
+        with pytest.raises(QueueEmpty):
+            await session.receive()
+
+
+async def test_queue_two_subscribers_both_receive(unique):
+    """Every active subscriber should independently receive each published message.
+
+    This verifies fan-out semantics: two separate Queue instances subscribed to
+    the same channel both receive the message, rather than one consuming it and
+    the other seeing nothing.
+    """
+    topic = unique("text")
+    sub1 = Queue.from_url("memory://")
+    sub2 = Queue.from_url("memory://")
+    publisher = Queue.from_url("memory://")
+
+    async with sub1.connect(topic), sub2.connect(topic):
+        await publisher.publish(topic, "hello")
+        msg1 = await sub1.receive(timeout=5)
+        msg2 = await sub2.receive(timeout=5)
+
+    assert msg1 == "hello"
+    assert msg2 == "hello"
+
+
+async def test_queue_subscribe_two_topics_receives_from_both(unique):
+    """A single subscriber connected to two topics should receive from either."""
+    topic_a = unique("text")
+    topic_b = unique("text")
+    sub = Queue.from_url("memory://")
+    publisher = Queue.from_url("memory://")
+
+    async with sub.connect(topic_a), sub.connect(topic_b):
+        await publisher.publish(topic_a, "from-a")
+        await publisher.publish(topic_b, "from-b")
+        msg1 = await sub.receive(timeout=5)
+        msg2 = await sub.receive(timeout=5)
+
+    assert {msg1, msg2} == {"from-a", "from-b"}
+
+    """A subscriber that connects after a publish should not receive that message.
+
+    This matches Redis pub/sub behaviour: messages are only delivered to
+    subscribers that are active at the moment of publishing.
+    """
+    topic = unique("text")
+    publisher = Queue.from_url("memory://")
+    late_sub = Queue.from_url("memory://")
+
+    await publisher.publish(topic, "early")
+
+    async with late_sub.connect(topic):
+        with pytest.raises(QueueEmpty):
+            await late_sub.receive()

taraqueue-0.6.0/tests/test_sqs.py

@@ -0,0 +1,82 @@
+"""Integration tests for the SQS queue."""
+
+import pytest
+
+from taraqueue import Queue, QueueEmpty
+
+
+async def test_sqs_sns_topic_created_on_subscribe(sqs_queue, unique):
+    """Subscribing should create the SNS topic and SQS queue."""
+    topic = unique("text")
+    await sqs_queue.subscribe(topic)
+    try:
+        assert topic in sqs_queue._subscriptions
+        sub = sqs_queue._subscriptions[topic]
+        assert sub.topic_arn
+        assert sub.queue_url
+        assert sub.subscription_arn
+    finally:
+        await sqs_queue.unsubscribe(topic)
+
+
+async def test_sqs_unsubscribe_cleans_up(sqs_queue, unique):
+    """Unsubscribing should remove the internal subscription state."""
+    topic = unique("text")
+    await sqs_queue.subscribe(topic)
+    await sqs_queue.unsubscribe(topic)
+    assert topic not in sqs_queue._subscriptions
+
+
+async def test_sqs_subscribe_idempotent(sqs_queue, unique):
+    """Subscribing twice to the same topic should not create a duplicate."""
+    topic = unique("text")
+    await sqs_queue.subscribe(topic)
+    try:
+        sub1 = sqs_queue._subscriptions[topic]
+        await sqs_queue.subscribe(topic)
+        sub2 = sqs_queue._subscriptions[topic]
+        assert sub1 is sub2
+    finally:
+        await sqs_queue.unsubscribe(topic)
+
+
+async def test_sqs_receive_empty_no_subscriptions(sqs_queue):
+    """Receiving with no subscriptions should raise QueueEmpty."""
+    with pytest.raises(QueueEmpty):
+        await sqs_queue.receive()
+
+
+async def test_sqs_fan_out(moto_server, unique):
+    """Two SQS subscribers should both receive the same published message."""
+    sub1 = Queue.from_url(f"sqs://test:test@127.0.0.1:{moto_server}")
+    sub2 = Queue.from_url(f"sqs://test:test@127.0.0.1:{moto_server}")
+    publisher = Queue.from_url(f"sqs://test:test@127.0.0.1:{moto_server}")
+
+    topic = unique("text")
+    async with sub1.connect(topic), sub2.connect(topic):
+        await publisher.publish(topic, "fanout-msg")
+        msg1 = await sub1.receive(timeout=5)
+        msg2 = await sub2.receive(timeout=5)
+
+    assert msg1 == "fanout-msg"
+    assert msg2 == "fanout-msg"
+
+
+async def test_sqs_late_subscriber_misses_message(moto_server, unique):
+    """A subscriber that connects after publish should not receive that message."""
+    publisher = Queue.from_url(f"sqs://test:test@127.0.0.1:{moto_server}")
+    late_sub = Queue.from_url(f"sqs://test:test@127.0.0.1:{moto_server}")
+
+    topic = unique("text")
+    await publisher.publish(topic, "early")
+
+    async with late_sub.connect(topic):
+        with pytest.raises(QueueEmpty):
+            await late_sub.receive()
+
+
+async def test_sqs_from_url_with_credentials():
+    """from_url should parse credentials and endpoint from the URL."""
+    q = Queue.from_url("sqs://mykey:mysecret@localhost:4566/eu-west-1")
+    assert q.region == "eu-west-1"
+    assert q.endpoint_url == "http://localhost:4566"