redis-allocator 0.0.1__py3-none-any.whl

redis_allocator/task_queue.py

@@ -0,0 +1,382 @@
+"""Redis-based task queue system for distributed task management.
+
+This module provides a RedisTaskQueue class that enables distributed task
+management and coordination using Redis as the underlying infrastructure.
+"""
+import time
+import pickle
+import base64
+import logging
+from typing import Any, Callable, Optional, List
+from functools import cached_property
+from dataclasses import dataclass
+from enum import Enum
+from threading import Event
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+
+from redis import StrictRedis as Redis
+
+logger = logging.getLogger(__name__)
+
+
+class TaskExecutePolicy(Enum):
+    """Defines different policies for task execution.
+    
+    Attributes:
+        Local: Execute task only locally
+        Remote: Execute task only remotely
+        LocalFirst: Try to execute locally first, then remotely if it fails
+        RemoteFirst: Try to execute remotely first, then locally if it fails
+        Auto: Choose execution mode based on whether a remote listener exists
+    """
+    Local = 0x00
+    Remote = 0x01
+    LocalFirst = 0x02
+    RemoteFirst = 0x03
+    Auto = 0x04
+
+
+@dataclass
+class RedisTask:
+    """Represents a task in the Redis task queue system.
+    
+    This class encapsulates all information related to a task, including
+    its unique ID, name, parameters, and execution status.
+    
+    Attributes:
+        id: Unique identifier for the task
+        name: Name of the task category
+        params: Dictionary of parameters for the task
+        expiry: Unix timestamp when this task expires
+        result: The result of the task, initially None
+        error: Any error that occurred during task execution
+        update_progress_time: Last time the progress was updated
+        current_progress: Current progress value
+        total_progress: Total progress value for completion
+        _save: Internal callable to save task state to Redis
+    """
+    id: str
+    name: str
+    params: dict
+    expiry: float
+    result: Any = None
+    error: Any = None
+    update_progress_time: float = 0.0
+    current_progress: float = 0.0
+    total_progress: float = 100.0
+    _save: Callable[[], None] = None
+
+    def save(self):
+        """Save the current state of the task to Redis."""
+        if self._save is not None:
+            self._save()
+
+    def update(self, current_progress: float, total_progress: float):
+        """Update the progress of the task.
+        
+        Args:
+            current_progress: The current progress value
+            total_progress: The total progress value for completion
+            
+        Raises:
+            TimeoutError: If the task has expired
+        """
+        self.current_progress = current_progress
+        self.total_progress = total_progress
+        self.update_progress_time = time.time()
+        if self.expiry < time.time():
+            raise TimeoutError(f'Task {self.id} in {self.name} has expired')
+        self.save()
+
+
+class RedisTaskQueue:
+    """A class that provides a simple interface for managing a task queue in Redis.
+
+    This class enables distributed task processing through Redis, with support for
+    asynchronous processing, task listening, and result retrieval. Tasks can be 
+    executed locally or remotely based on configurable policies.
+    """
+
+    def __init__(self, redis: Redis, prefix: str, suffix='task-queue', timeout=300, interval=5, 
+                 task_fn: Callable[[RedisTask], Any] = None):
+        """Initialize a RedisTaskQueue instance.
+
+        Args:
+            redis: The Redis client used for interacting with Redis.
+            prefix: The prefix to be added to the task queue key.
+            suffix: The suffix to be added to the task queue key. Default is 'task-queue'.
+            timeout: The query timeout in seconds. Default is 300s.
+            interval: The query interval in seconds. Default is 5s.
+            task_fn: A function to execute tasks locally. Takes a RedisTask and returns Any.
+        """
+        self.redis = redis
+        self.prefix = prefix
+        self.suffix = suffix
+        self.timeout = timeout
+        self.interval = interval
+        self.task_fn = task_fn
+
+    def build_task(self, id: str, name: str, params: dict) -> RedisTask:
+        """Create a new RedisTask instance with the given parameters.
+        
+        Args:
+            id: Unique identifier for the task
+            name: Name of the task category
+            params: Dictionary of parameters for the task
+            
+        Returns:
+            A new RedisTask instance with a save function configured
+        """
+        task = RedisTask(id=id, name=name, params=params, expiry=time.time() + self.timeout)
+        task._save = lambda: self.set_task(task)
+        return task
+
+    def execute_task_remotely(self, task: RedisTask, timeout: Optional[float] = None, once: bool = False) -> Any:
+        """Execute a task remotely by pushing it to the queue.
+        
+        Args:
+            task: The RedisTask to execute
+            timeout: Optional timeout in seconds, defaults to self.timeout
+            once: Whether to delete the result after getting it
+            
+        Returns:
+            The result of the task
+            
+        Raises:
+            TimeoutError: If the task times out
+            Exception: Any exception raised during task execution
+        """
+        self.set_task(task)
+        self.redis.rpush(self._queue_key(task.name), task.id)
+        if timeout is None:
+            timeout = self.timeout
+        while timeout >= 0:
+            time.sleep(self.interval)
+            result = self.get_task(task.id, once)
+            if result is not None:
+                if result.error is not None:
+                    raise result.error
+                elif result.result is not None:
+                    return result.result
+            timeout -= self.interval
+        raise TimeoutError(f'Task {task.id} in {task.name} has expired')
+
+    def execute_task_locally(self, task: RedisTask, timeout: Optional[float] = None) -> Any:
+        """Execute a task locally using the task_fn.
+        
+        Args:
+            task: The RedisTask to execute
+            timeout: Optional timeout in seconds, updates task.expiry if provided
+            
+        Returns:
+            The result of the task
+            
+        Raises:
+            Exception: Any exception raised during task execution
+        """
+        if timeout is not None:
+            task.expiry = time.time() + timeout
+        try:
+            task.result = self.task_fn(task)
+            return task.result
+        except Exception as e:
+            task.error = e
+            raise e
+        finally:
+            task.save()
+
+    @cached_property
+    def _queue_prefix(self) -> str:
+        """Get the prefix for queue keys.
+        
+        Returns:
+            The queue prefix
+        """
+        return f'{self.prefix}|{self.suffix}|task-queue'
+
+    def _queue_key(self, name: str) -> str:
+        """Generate a queue key for the given task name.
+
+        Args:
+            name: The task name.
+            
+        Returns:
+            The formatted queue key.
+        """
+        return f'{self._queue_prefix}|{name}'
+
+    def _queue_name(self, key: str) -> str:
+        """Extract the queue name from a queue key.
+        
+        Args:
+            key: The queue key.
+            
+        Returns:
+            The queue name.
+            
+        Raises:
+            AssertionError: If the key doesn't start with the queue prefix.
+        """
+        assert key.startswith(self._queue_prefix)
+        return key[len(self._queue_prefix) + 1:]
+
+    def _queue_listen_name(self, name: str) -> str:
+        """Generate a listen name for the given task name.
+        
+        Args:
+            name: The task name.
+            
+        Returns:
+            The formatted listen name.
+        """
+        return f'{self.prefix}|{self.suffix}|task-listen|{name}'
+
+    def set_queue_listened(self, name: str) -> None:
+        """Set the queue as being listened to for the given task name.
+
+        Args:
+            name: The task name.
+        """
+        self.redis.setex(self._queue_listen_name(name), self.timeout, '1')
+
+    def _result_key(self, task_id: str) -> str:
+        """Generate a result key for the given task ID.
+        
+        Args:
+            task_id: The task ID.
+            
+        Returns:
+            The formatted result key.
+        """
+        return f'{self.prefix}|{self.suffix}|task-result:{task_id}'
+
+    def set_task(self, task: RedisTask) -> str:
+        """Save a task to Redis.
+        
+        Args:
+            task: The RedisTask to save.
+            
+        Returns:
+            The task ID.
+        """
+        task._save = None
+        t = pickle.dumps(task)
+        result = str(base64.b64encode(t), 'ascii')
+        self.redis.setex(self._result_key(task.id), self.timeout, result)
+        return task.id
+
+    def get_task(self, task_id: str, once: bool = False) -> Optional[RedisTask]:
+        """Get a task from Redis.
+        
+        Args:
+            task_id: The task ID.
+            once: Whether to delete the task after getting it.
+            
+        Returns:
+            The RedisTask, or None if no task is available.
+        """
+        get = self.redis.getdel if once else self.redis.get
+        result = get(self._result_key(task_id))
+        if result is None:
+            return None
+        t = pickle.loads(base64.b64decode(result))
+        t._save = lambda: self.set_task(t)
+        return t
+
+    def has_task(self, task_id: str) -> bool:
+        """Check if a task exists.
+        
+        Args:
+            task_id: The task ID.
+            
+        Returns:
+            True if the task exists, False otherwise.
+        """
+        return self.redis.exists(self._result_key(task_id)) > 0
+
+    @contextmanager
+    def _executor_context(self, max_workers: int = 128):
+        """Create a ThreadPoolExecutor context manager.
+        
+        This is a helper method for testing and internal use.
+        
+        Args:
+            max_workers: The maximum number of worker threads.
+            
+        Yields:
+            The ThreadPoolExecutor instance.
+        """
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            yield executor
+
+    def listen(self, names: List[str], workers: int = 128, event: Optional[Event] = None) -> None:
+        """Listen for tasks on the specified queues.
+        
+        This method continuously polls the specified queues for tasks, 
+        and executes tasks locally when they are received.
+        
+        Args:
+            names: The names of the queues to listen to.
+            workers: The number of worker threads to use. Default is 128.
+            event: An event to signal when to stop listening. Default is None.
+        """
+        with self._executor_context(max_workers=workers) as executor:
+            while event is None or not event.is_set():
+                queue_names = [self._queue_key(name) for name in names]
+                while True:
+                    task = self.redis.blpop(queue_names, timeout=self.interval)
+                    if task is not None:
+                        _queue_key, task_id = task
+                        task = self.get_task(task_id)
+                        executor.submit(self.execute_task_locally, task)
+                    else:
+                        break
+                for name in names:
+                    self.set_queue_listened(name)
+                time.sleep(self.interval)
+
+    def query(self, id: str, name: str, params: dict, timeout: Optional[float] = None,
+              policy: TaskExecutePolicy = TaskExecutePolicy.Auto, once: bool = False) -> Any:
+        """Execute a task according to the specified policy.
+        
+        This method provides a flexible way to execute tasks with different
+        strategies based on the specified policy.
+        
+        Args:
+            id: The task ID.
+            name: The task name.
+            params: The task parameters.
+            timeout: Optional timeout override.
+            policy: The execution policy to use.
+            once: Whether to delete the result after getting it.
+            
+        Returns:
+            The result of the task.
+            
+        Raises:
+            Exception: Any exception raised during task execution.
+        """
+        t = self.build_task(id, name, params)
+        match policy:
+            case TaskExecutePolicy.Local:
+                return self.execute_task_locally(t, timeout)
+            case TaskExecutePolicy.Remote:
+                return self.execute_task_remotely(t)
+            case TaskExecutePolicy.LocalFirst:
+                try:
+                    return self.execute_task_locally(t, timeout)
+                except Exception as e:
+                    logger.exception(f'Failed to execute task {t.id} in {t.name} locally: {e}')
+                    return self.execute_task_remotely(t, timeout)
+            case TaskExecutePolicy.RemoteFirst:
+                try:
+                    return self.execute_task_remotely(t, timeout)
+                except Exception as e:
+                    logger.exception(f'Failed to execute task {t.id} in {t.name} remotely: {e}')
+                    return self.execute_task_locally(t, timeout)
+            case TaskExecutePolicy.Auto:
+                if self.redis.exists(self._queue_listen_name(name)):
+                    return self.execute_task_remotely(t, timeout)
+                else:
+                    return self.execute_task_locally(t, timeout)

redis_allocator-0.0.1.dist-info/METADATA

@@ -0,0 +1,229 @@
+Metadata-Version: 2.4
+Name: redis-allocator
+Version: 0.0.1
+Summary: Redis-based resource allocation system.
+Home-page: https://github.com/invoker-bot/RedisAllocator-python
+Author: Invoker Bot
+Author-email: invoker-bot@outlook.com
+License: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.9
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: redis>=5.0.0
+Provides-Extra: test
+Requires-Dist: pytest>=7.4.3; extra == "test"
+Requires-Dist: pytest-cov>=4.1.0; extra == "test"
+Requires-Dist: pytest-mock>=3.12.0; extra == "test"
+Requires-Dist: fakeredis[lua]>=2.20.1; extra == "test"
+Requires-Dist: flake8>=6.1.0; extra == "test"
+Requires-Dist: freezegun>=1.4.0; extra == "test"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# RedisAllocator
+
+## Project Overview
+
+RedisAllocator is an efficient Redis-based distributed memory allocation system. This system simulates traditional memory allocation mechanisms but implements them in a distributed environment, using Redis as the underlying storage and coordination tool.
+
+> **Note**: Currently, RedisAllocator only supports single Redis instance deployments. For Redis cluster environments, we recommend using RedLock for distributed locking operations.
+
+### Core Features
+
+- **Distributed Locking**: Provides robust distributed locking mechanisms to ensure data consistency in concurrent environments
+- **Resource Allocation**: Implements a distributed resource allocation system with support for:
+  - Priority-based distribution
+  - Soft binding
+  - Garbage collection
+  - Health checking
+- **Task Management**: Implements a distributed task queue system for efficient task processing across multiple workers
+- **Object Allocation**: Supports allocation of resources with priority-based distribution and soft binding
+- **Health Checking**: Monitors the health of distributed instances and automatically handles unhealthy resources
+- **Garbage Collection**: Automatically identifies and reclaims unused resources, optimizing memory usage
+
+
+## Installation
+
+```bash
+pip install redis-allocator
+```
+
+## Quick Start
+
+### Using RedisLock for Distributed Locking
+
+```python
+from redis import Redis
+from redis_allocator import RedisLock
+
+# Initialize Redis client
+redis = Redis(host='localhost', port=6379)
+
+# Create a RedisLock instance
+lock = RedisLock(redis, "myapp", "resource-lock")
+
+# Acquire a lock
+if lock.lock("resource-123", timeout=60):
+    try:
+        # Perform operations with the locked resource
+        print("Resource locked successfully")
+    finally:
+        # Release the lock when done
+        lock.unlock("resource-123")
+```
+
+### Using RedisAllocator for Resource Management
+
+```python
+from redis import Redis
+from redis_allocator import RedisAllocator
+
+# Initialize Redis client
+redis = Redis(host='localhost', port=6379)
+
+# Create a RedisAllocator instance
+allocator = RedisAllocator(
+    redis, 
+    prefix='myapp',
+    suffix='allocator',
+    shared=False  # Whether resources can be shared
+)
+
+# Add resources to the pool
+allocator.extend(['resource-1', 'resource-2', 'resource-3'])
+
+# Allocate a resource key (returns only the key)
+key = allocator.malloc_key(timeout=120)
+if key:
+    try:
+        # Use the allocated resource
+        print(f"Allocated resource: {key}")
+    finally:
+        # Free the resource when done
+        allocator.free_keys(key)
+
+# Allocate a resource with object (returns a RedisAllocatorObject)
+allocated_obj = allocator.malloc(timeout=120)
+if allocated_obj:
+    try:
+        # The key is available as a property
+        print(f"Allocated resource: {allocated_obj.key}")
+        
+        # Update the resource's lock timeout
+        allocated_obj.update(timeout=60)
+    finally:
+        # Free the resource when done
+        allocator.free(allocated_obj)
+
+# Using soft binding (associates a name with a resource)
+allocator.update_soft_bind("worker-1", "resource-1")
+# Later...
+allocator.unbind_soft_bind("worker-1")
+
+# Garbage collection (reclaims unused resources)
+allocator.gc(count=10)  # Check 10 items for cleanup
+```
+
+### Using RedisTaskQueue for Distributed Task Processing
+
+```python
+from redis import Redis
+from redis_allocator import RedisTaskQueue, TaskExecutePolicy
+import json
+
+# Initialize Redis client
+redis = Redis(host='localhost', port=6379)
+
+# Process tasks in a worker
+def process_task(task):
+    # Process the task (task is a RedisTask object)
+    # You can access task.id, task.name, task.params
+    # You can update progress with task.update(current, total)
+    return json.dumps({"result": "processed"})
+
+
+# Create a task queue
+task_queue = RedisTaskQueue(redis, "myapp", task_fn=process_task)
+
+# Submit a task with query method
+result = task_queue.query(
+    id="task-123",
+    name="example-task",
+    params={"input": "data"},
+    timeout=300,  # Optional timeout in seconds
+    policy=TaskExecutePolicy.Auto,  # Execution policy
+    once=False  # Whether to delete the result after getting it
+)
+
+# Start listening for tasks
+task_queue.listen(
+    names=["example-task"],  # List of task names to listen for
+    workers=128,  # Number of worker threads
+    event=None  # Optional event to signal when to stop listening
+)
+```
+
+## Modules
+
+RedisAllocator consists of several modules, each providing specific functionality:
+
+- **lock.py**: Provides `RedisLock` and `RedisLockPool` for distributed locking mechanisms
+- **task_queue.py**: Implements `RedisTaskQueue` for distributed task processing
+- **allocator.py**: Contains `RedisAllocator` and `RedisThreadHealthChecker` for resource allocation
+
+
+## Roadmap
+
+### Phase 1 (Completed)
+- [x] Distributed lock mechanism implementation
+- [x] Task queue processing system
+- [x] Resource allocation and management
+- [x] Basic health checking and monitoring
+- [x] Object allocation with serialization
+- [x] Unit tests for core components
+
+### Phase 2 (In Progress)
+- [ ] Advanced sharding implementation
+- [ ] Performance optimization and benchmarking
+- [ ] Documentation improvement
+- [ ] Enhanced error handling and recovery
+
+### Phase 3 (Planned)
+- [ ] Advanced garbage collection strategies
+- [ ] Redis cluster support
+- [ ] Fault recovery mechanisms
+- [ ] Automated resource scaling
+
+### Phase 4 (Future)
+- [ ] API stability and backward compatibility
+- [ ] Performance monitoring and tuning tools
+- [ ] Advanced features (transaction support, data compression, etc.)
+- [ ] Production environment validation and case studies
+
+## Contributing
+
+Contributions and suggestions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for more information.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Contact
+
+For questions or suggestions, please contact us through GitHub Issues.

redis_allocator-0.0.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+redis_allocator/__init__.py,sha256=x4koK9zx8W1wWnrwt72tPve-LwKdhKbKxTGgacZ8q84,800
+redis_allocator/allocator.py,sha256=qqxKOAPG_QgTp3gxvi8O443dPC9DGr3_iM5nCP3lidM,21831
+redis_allocator/lock.py,sha256=HSCXyBhO3qBMylvByqskekHp6CMgqwbVcwMyP77pBmM,24768
+redis_allocator/task_queue.py,sha256=NalI77hk1u8X-JBWpGoZFv5nJXzfem5fDthiSa9cgFg,13596
+redis_allocator-0.0.1.dist-info/licenses/LICENSE,sha256=NiCCQOo0TQ_djjGpoCphsT59Bgu2sLOk8rVwIRxrqQE,1089
+tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/conftest.py,sha256=gsMBUrUTGxD-ki29rWrtJCZZymb7n7biRV6lWcd7fWo,1124
+tests/test_allocator.py,sha256=F-GHbgWGss5CvbUsQt1AvLib9V0Rxyih1HisHWFi6jU,18608
+tests/test_lock.py,sha256=wA2po5vSgeoyOeH9-bTF3fxVwh-CmR-CSH-ZThVHisY,35294
+tests/test_task_queue.py,sha256=pohhYpw9RIPJma9wQPEF9r4pKkK__PZDd6Y10mEPtjs,33071
+redis_allocator-0.0.1.dist-info/METADATA,sha256=nSxDu1WR1H6NJGdEu3eoVkPivnzEiM04DbVxk5VmLss,7465
+redis_allocator-0.0.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+redis_allocator-0.0.1.dist-info/top_level.txt,sha256=0hXzU7sK5FCeSolTEYxThOt3HOybnwaXv1FLRJvHVgI,22
+redis_allocator-0.0.1.dist-info/RECORD,,

redis_allocator-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (78.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

redis_allocator-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Invoker Bot
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

redis_allocator-0.0.1.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+redis_allocator
+tests