taraqueue 0.0.1.dev0__py3-none-any.whl

taraqueue/queue.py

@@ -0,0 +1,149 @@
+"""Queue abstraction and implementation."""
+
+import os
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from contextlib import asynccontextmanager, suppress
+from time import time
+
+from attrs import define, field
+from yarl import URL
+
+from taraqueue.registry import registry_load
+
+
+class QueueEmpty(Exception):
+    """Raised when the queue is empty."""
+
+
+@define
+class Queue(ABC):
+    """Base queue class."""
+
+    @classmethod
+    def from_url(cls, url: URL | str, registry=None) -> "Queue":
+        if registry is None:
+            registry = registry_load("taraqueue")
+        scheme = URL(url).scheme
+        queue_cls = registry["taraqueue"][scheme]
+        return queue_cls.from_url(url)
+
+    @abstractmethod
+    async def subscribe(self, topic: str) -> None:
+        """Subscribe to a topic before receiving messages."""
+
+    @abstractmethod
+    async def unsubscribe(self, topic: str) -> None:
+        """Unsubscribe from a topic after receiving messages."""
+
+    @abstractmethod
+    async def receive(self, timeout=None) -> str:
+        """Listen for messages on the subscribed topics."""
+
+    @abstractmethod
+    async def publish(self, topic: str, message: str) -> None:
+        """Publish a message to a topic."""
+
+    @asynccontextmanager
+    async def connect(self, topic: str):
+        """Context manager that subscribes on entry and unsubscribes on exit."""
+        await self.subscribe(topic)
+        try:
+            yield self
+        finally:
+            await self.unsubscribe(topic)
+
+
+_global_memory_queues = defaultdict(list)
+
+
+@define
+class MemoryQueue(Queue):
+
+    topics = field(factory=list)
+    queues = field(default=_global_memory_queues)
+
+    @classmethod
+    def from_url(cls, url: URL) -> "MemoryQueue":
+        return cls()
+
+    async def subscribe(self, topic: str) -> None:
+        """See `Queue.subscribe`."""
+        self.topics.append(topic)
+
+    async def unsubscribe(self, topic: str) -> None:
+        """See `Queue.unsubscribe`."""
+        with suppress(ValueError):
+            self.topics.remove(topic)
+
+    async def receive(self, timeout=None) -> str:
+        """See `Queue.receive`."""
+        for topic in self.topics[:]:
+            # Cycle through topics.
+            self.topics.append(self.topics.pop(0))
+            queue = self.queues[topic]
+            with suppress(IndexError):
+                return queue.pop(0)
+
+        raise QueueEmpty("Queue is empty")
+
+    async def publish(self, topic: str, message: str) -> None:
+        """See `Queue.publish`."""
+        queue = self.queues[topic]
+        queue.append(message)
+
+
+@define
+class RedisQueue(Queue):
+
+    client = field()
+    pubsub = field()
+
+    @classmethod
+    def from_env(cls, env=os.environ) -> "RedisQueue":
+        host = env.get("REDIS_SLAVEOF_IP", "") or env.get("IPV4_NETWORK", "172.22.1") + ".249"
+        port = int(env.get("REDIS_SLAVEOF_PORT", "") or "6379")
+        password = env.get("REDISPASS")
+        return cls.from_host(host, port, password=password)
+
+    @classmethod
+    def from_host(cls, host: str, port: int = 6379, password: str | None = None) -> "RedisQueue":
+        from redis.asyncio import StrictRedis
+
+        client = StrictRedis(
+            host=host,
+            port=port,
+            decode_responses=True,
+            db=0,
+            password=password,
+        )
+        pubsub = client.pubsub(ignore_subscribe_messages=True)
+        return cls(client, pubsub)
+
+    @classmethod
+    def from_url(cls, url: URL | str) -> "RedisQueue":
+        url = URL(url)
+        return cls.from_host(url.host, url.port, password=url.password)
+
+    async def subscribe(self, topic: str) -> None:
+        """See `Queue.subscribe`."""
+        await self.pubsub.subscribe(topic)
+
+    async def unsubscribe(self, topic: str) -> None:
+        """See `Queue.unsubscribe`."""
+        await self.pubsub.unsubscribe(topic)
+
+    async def receive(self, timeout=0) -> str:
+        """See `Queue.receive`."""
+        stop_time = time() + timeout
+        while True:
+            remaining_timeout = max(0.0, stop_time - time())
+            message = await self.pubsub.get_message(ignore_subscribe_messages=True, timeout=remaining_timeout)
+            if message:
+                return message["data"]
+            if time() >= stop_time:
+                raise QueueEmpty("Queue is empty")
+
+    async def publish(self, topic: str, message: str) -> None:
+        """See `Queue.publish`."""
+        await self.client.publish(topic, message)

taraqueue/registry.py

@@ -0,0 +1,76 @@
+"""Entry points based registry management."""
+
+import contextlib
+from importlib.metadata import entry_points
+
+
+def get_entry_points(group):
+    """Get the list of pytest_unique entry points."""
+    try:
+        return entry_points().select(group=group)
+    except AttributeError:
+        # Backward compatibility with Python 3.9.
+        return entry_points().get(group, [])
+
+
+def registry_load(group, registry=None):
+    """Find all installed entry points."""
+    if registry is None:
+        registry = {}
+
+    for entry_point in get_entry_points(group):
+        entry = entry_point.load()
+        registry_add(group, entry_point.name, entry, registry)
+
+    return registry
+
+
+def registry_add(group, name, entry, registry=None):
+    """Add an entry to a registry.
+
+    :param group: Group of the entry.
+    :param name: Name of the entry.
+    :param entry: Entry to add.
+    :param registry: Optional registry to update.
+    :return: A registry with the entry.
+    """
+    if registry is None:
+        registry = {
+            group: {},
+        }
+    else:
+        registry.setdefault(group, {})
+
+    registry[group][name] = entry
+    return registry
+
+
+def registry_remove(group, name, registry=None):
+    """Remove an entry from a registry.
+
+    If the entry doesn't exist, return silently.
+
+    :param group: Group of the entry.
+    :param name: Name of the entry.
+    :param registry: Optional registry to update.
+    """
+    if registry is not None:
+        with contextlib.suppress(KeyError):
+            del registry[group][name]
+
+
+def registry_get(group, name, registry=None):
+    """Get an entry from a registry.
+
+    If the registry is not defined or the group is not in the registry,
+    the registry is loaded again.
+
+    :param group: Group of the entry.
+    :param name: Name of the entry.
+    :param registry: Optional registry to get from.
+    :raises KeyError: If not found.
+    """
+    if registry is None or group not in registry:
+        registry = registry_load(group, registry)
+
+    return registry[group][name]

taraqueue/testing/compose.py

@@ -0,0 +1,92 @@
+"""Compose server module."""
+
+from contextlib import contextmanager
+from datetime import datetime
+
+from attrs import define
+from more_itertools import only
+from pytest_xdocker.docker import DockerContainer
+from pytest_xdocker.process import ProcessData, ProcessServer
+from pytest_xdocker.xdocker import xdocker
+
+
+@define(frozen=True)
+class ComposeService:
+    """Compose service.
+
+    :param name: Name of the compose service container.
+    :param network: Network name to use when resolving the IP address.
+    """
+
+    name: str
+    network: str | None = None
+
+    @property
+    def container(self):
+        return DockerContainer(self.name)
+
+    @property
+    def container_id(self):
+        return self.container.inspect["Id"]
+
+    @property
+    def env(self):
+        env = self.container.inspect["Config"]["Env"]
+        return dict(e.split("=", 1) for e in env)
+
+    @property
+    def ip(self):
+        network_settings = self.container.inspect["NetworkSettings"]
+        networks = network_settings["Networks"]
+        if self.network and self.network in networks:
+            return networks[self.network]["IPAddress"]
+        return only(networks.values())["IPAddress"]
+
+    @property
+    def started_at(self):
+        started_at = self.container.inspect["State"]["StartedAt"]
+        return datetime.fromisoformat(started_at)
+
+
+class ComposeServer(ProcessServer):
+    def __init__(self, pattern, project="test", env_file=None, compose_files=None, timeout=180, **kwargs):
+        """Initilize a compose server."""
+        super().__init__(**kwargs)
+        self.pattern = pattern
+        self.project = project
+        self.env_file = env_file
+        self.compose_files = compose_files
+        self.timeout = timeout
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(pattern={self.pattern!r}, project={self.project!r})"
+
+    def full_name(self, name):
+        return f"{self.project}-{name}-1"
+
+    def prepare_func(self, controldir):
+        """Prepare the function to run the compose service."""
+        full_name = self.full_name(controldir.basename)
+        compose = xdocker.compose().with_project_name(self.project)
+        if env_file := self.env_file:
+            compose = compose.with_env_file(env_file)
+        for file in self.compose_files or []:
+            compose = compose.with_file(file)
+
+        command = (
+            compose
+            .run(controldir.basename)
+            .with_name(full_name)
+            .with_build()
+            .with_remove()
+            .with_optionals("--use-aliases")
+        )
+
+        return ProcessData(self.pattern, command, timeout=self.timeout)
+
+    @contextmanager
+    def run(self, name):
+        """Return an `ComposeService` to the running service."""
+        with super().run(name):
+            full_name = self.full_name(name)
+            yield ComposeService(full_name, network=f"{self.project}_default")

taraqueue/testing/queue.py

@@ -0,0 +1,36 @@
+"""Queue fixtures."""
+
+import pytest
+from yarl import URL
+
+from taraqueue.queue import Queue
+
+
+@pytest.fixture
+def memory_queue():
+    """Memory queue fixture."""
+    url = URL.build(scheme="memory")
+    return Queue.from_url(url)
+
+
+@pytest.fixture
+def redis_queue(redis_service, env_vars):
+    """Redis queue fixture."""
+    url = URL.build(
+        scheme="redis",
+        host=redis_service.ip,
+        port=6379,
+        password=env_vars["REDISPASS"],
+    )
+    return Queue.from_url(url)
+
+
+@pytest.fixture(
+    params=[
+        "memory_queue",
+        "redis_queue",
+    ],
+)
+def queue(request):
+    """Queue fixture."""
+    return request.getfixturevalue(request.param)

taraqueue/testing/services.py

@@ -0,0 +1,87 @@
+"""Service fixtures."""
+
+from functools import partial
+from pathlib import Path
+
+import pytest
+
+from taraqueue.testing.compose import ComposeServer
+
+
+@pytest.fixture(scope="session")
+def project():
+    return "test"
+
+
+@pytest.fixture(scope="session")
+def env_vars(project):
+    """Environment variables for the services."""
+    return {
+        "COMPOSE_PROJECT_NAME": project,
+        "REDISPASS": "test",
+    }
+
+
+@pytest.fixture(scope="session")
+def env_file(env_vars, request):
+    """Environment file containing `env_vars`.
+
+    Cached for troubleshooting purposes.
+    """
+    env_file = request.config.cache.makedir("compose") / "env"
+    with env_file.open("w") as f:
+        for k, v in env_vars.items():
+            f.write(f"{k}={v}\n")
+
+    return env_file
+
+
+@pytest.fixture(scope="session")
+def compose_files(request):
+    directory = Path(request.config.rootdir)
+    filenames = ["docker-compose.yml", "compose.yaml", "compose.yml"]
+    while True:
+        for filename in filenames:
+            path = directory / filename
+            if path.exists():
+                all_files = directory.glob(f"{path.stem}.*")
+                ordered_files = sorted(all_files, key=lambda p: len(p.name))
+                return list(ordered_files)
+
+        if directory == directory.parent:
+            raise FileNotFoundError("Docker compose file not found")
+
+        directory = directory.parent
+
+
+@pytest.fixture(scope="session")
+def compose_server(project, env_file, compose_files, process):
+    return partial(
+        ComposeServer,
+        project=project,
+        env_file=env_file,
+        compose_files=compose_files,
+        process=process,
+    )
+
+
+@pytest.fixture(scope="session")
+def redis_service(compose_server):
+    """Redis service fixture."""
+    server = compose_server("Ready to accept connections tcp")
+    with server.run("redis") as service:
+        yield service
+
+
+@pytest.fixture(scope="session")
+def redis_client(redis_service, env_vars):
+    """Redis client to the service fixture."""
+    from redis import StrictRedis
+
+    return StrictRedis(
+        host=redis_service.ip,
+        port=6379,
+        decode_responses=True,
+        db=0,
+        password=env_vars["REDISPASS"],
+    )

taraqueue-0.0.1.dev0.dist-info/METADATA

@@ -0,0 +1,67 @@
+Metadata-Version: 2.4
+Name: taraqueue
+Version: 0.0.1.dev0
+Summary: Python queue abstraction layer
+Project-URL: Repository, https://github.com/taradix/taraqueue
+Author-email: Marc Tardif <marc@taram.ca>
+License-File: LICENSE.rst
+Requires-Python: <4.0,>=3.12
+Requires-Dist: attrs>=26.1.0
+Requires-Dist: yarl>=1.23.0
+Provides-Extra: check
+Requires-Dist: ruff<1.0.0,>=0.15.0; extra == 'check'
+Provides-Extra: docs
+Requires-Dist: sphinx-rtd-theme<3.2.0,>=3.1.0; extra == 'docs'
+Requires-Dist: sphinx<10.0.0,>=9.0.0; extra == 'docs'
+Requires-Dist: sphinxcontrib-log-cabinet<2.0.0,>=1.0.1; extra == 'docs'
+Provides-Extra: redis
+Requires-Dist: redis>=7.4.0; extra == 'redis'
+Provides-Extra: test
+Requires-Dist: coverage<8.0.0,>=7.2.3; extra == 'test'
+Requires-Dist: more-itertools<11.1.0,>=11.0.1; extra == 'test'
+Requires-Dist: pytest-asyncio<2.0.0,>=1.0.0; extra == 'test'
+Requires-Dist: pytest-unique<1.0.0,>=0.1.8; extra == 'test'
+Requires-Dist: pytest-xdocker<1.0.0,>=0.2.8; extra == 'test'
+Requires-Dist: pytest<10.0.0,>=9.0.0; extra == 'test'
+Requires-Dist: redis>=7.4.0; extra == 'test'
+Description-Content-Type: text/x-rst
+
+TaraQueue
+=========
+
+Python queue abstraction layer.
+
+.. image:: https://img.shields.io/badge/license-MIT-blue.svg
+   :target: https://github.com/taradix/taraqueue/blob/master/LICENSE
+   :alt: License
+.. image:: https://img.shields.io/pypi/v/taraqueue.svg
+   :target: https://pypi.python.org/pypi/taraqueue/
+   :alt: PyPI
+.. image:: https://img.shields.io/github/issues-raw/taradix/taraqueue.svg
+   :target: https://github.com/taradix/taraqueue/issues
+   :alt: Issues
+
+Requirements
+------------
+
+You will need the following prerequisites to use taraqueue:
+
+- Python 3.12, 3.13, 3.14
+
+Installation
+------------
+
+To install taraqueue:
+
+.. code-block:: bash
+
+  $ pip install taraqueue
+
+Resources
+---------
+
+- `Documentation <https://taradix.github.io/taraqueue/>`_
+- `Release Notes <http://github.com/taradix/taraqueue/blob/master/CHANGES.rst>`_
+- `Issue Tracker <http://github.com/taradix/taraqueue/issues>`_
+- `Source Code <http://github.com/taradix/taraqueue/>`_
+- `PyPi <https://pypi.org/project/taraqueue/>`_

taraqueue-0.0.1.dev0.dist-info/RECORD

@@ -0,0 +1,12 @@
+taraqueue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+taraqueue/queue.py,sha256=-ya3DVSLTjNQ1nmL1ltpJgPmTkdmQe2W2UHvhUGKfzI,4490
+taraqueue/registry.py,sha256=YcF9FE21HWfzoFbGGtC4GBozj-g5YbQuWdOor9XoPrU,2036
+taraqueue/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+taraqueue/testing/compose.py,sha256=wyI7dkhdXgQra0RcrYqye7RnYSv4WRSOMUVJRloKfDk,2896
+taraqueue/testing/queue.py,sha256=ryIcwE7Txeybairo6HkW9bO9mH7A6Qj9RU3YDiUpG8M,677
+taraqueue/testing/services.py,sha256=YmbsbAiX_kBVziwnNwkuvxn-82zhNE926oxF6V0SfzU,2213
+taraqueue-0.0.1.dev0.dist-info/METADATA,sha256=Bd4BaAxkjIJJ5aJFwOK8gZdAfMjX4JbEw2OUAy-Llqk,2136
+taraqueue-0.0.1.dev0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+taraqueue-0.0.1.dev0.dist-info/entry_points.txt,sha256=IDp7ErMVJHagqZeDuawQyIxa6ZmLF9Kh3Wv3is1up6c,186
+taraqueue-0.0.1.dev0.dist-info/licenses/LICENSE.rst,sha256=aBUUonOC9jBjhQb0VIObBylPnlIVph0P0QnlnZ8z4BM,1068
+taraqueue-0.0.1.dev0.dist-info/RECORD,,

taraqueue-0.0.1.dev0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

taraqueue-0.0.1.dev0.dist-info/entry_points.txt

@@ -0,0 +1,7 @@
+[pytest11]
+taraqueue-queue = taraqueue.testing.queue
+taraqueue-services = taraqueue.testing.services
+
+[taraqueue]
+memory = taraqueue.queue:MemoryQueue
+redis = taraqueue.queue:RedisQueue

taraqueue-0.0.1.dev0.dist-info/licenses/LICENSE.rst

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Marc Tardif
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.