pyarallel 0.1.0__py3-none-any.whl

pyarallel/__init__.py

@@ -0,0 +1,10 @@
+from .core import parallel, RateLimit
+
+__version__ = "0.1.0"
+__all__ = ["parallel", "RateLimit"]
+
+__doc__ = f"""
+pyarallel v{__version__}
+
+Simple parallel processing framework for Python.
+"""

pyarallel/core.py

@@ -0,0 +1,348 @@
+"""
+Pyarallel: A Powerful Parallel Execution Library for Python
+
+This module provides a decorator-based approach to parallel execution, supporting both
+thread and process-based parallelism with advanced features like rate limiting and batch processing.
+
+Key Features:
+- Simple decorator-based API
+- Support for both I/O-bound (threading) and CPU-bound (multiprocessing) tasks
+- Configurable rate limiting with support for per-second, per-minute, and per-hour rates
+- Batch processing for memory efficiency
+- Worker prewarming for latency-critical applications
+- Automatic executor reuse and cleanup
+- Thread-safe implementation
+
+Example Usage:
+    ```python
+    from pyarallel import parallel
+    
+    # Basic I/O-bound task
+    @parallel(max_workers=4)
+    def fetch_url(url: str) -> dict:
+        return requests.get(url).json()
+    
+    # CPU-bound task with rate limiting
+    @parallel(
+        max_workers=4,
+        executor_type="process",
+        rate_limit=(100, "minute")
+    )
+    def process_image(image: bytes) -> bytes:
+        return heavy_processing(image)
+    
+    # Batch processing with prewarming
+    @parallel(
+        max_workers=4,
+        batch_size=10,
+        prewarm=True
+    )
+    def analyze_text(text: str) -> dict:
+        return text_analysis(text)
+    
+    # Use with lists for parallel execution
+    urls = ["http://example1.com", "http://example2.com"]
+    results = fetch_url(urls)  # Processes URLs in parallel
+    
+    # Single items work too
+    result = fetch_url("http://example.com")  # Returns [result]
+    ```
+
+For detailed documentation and examples, see:
+https://github.com/oneryalcin/pyarallel
+"""
+
+from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed
+from functools import wraps
+from typing import Any, Callable, TypeVar, Literal
+from itertools import islice
+import time
+import threading
+from dataclasses import dataclass
+import multiprocessing
+import weakref
+
+T = TypeVar('T')
+
+TimeUnit = Literal["second", "minute", "hour"]
+ExecutorType = Literal["thread", "process"]
+
+# Global executor cache using weak references
+_EXECUTOR_CACHE = weakref.WeakValueDictionary()
+
+@dataclass
+class RateLimit:
+    """
+    Configuration for rate limiting parallel operations.
+    
+    Args:
+        count: Number of operations allowed per interval
+        interval: Time interval for rate limiting ("second", "minute", "hour")
+    
+    Example:
+        ```python
+        # 100 operations per minute
+        rate = RateLimit(100, "minute")
+        
+        @parallel(rate_limit=rate)
+        def my_func(): ...
+        ```
+    """
+    count: float
+    interval: TimeUnit = "second"
+    
+    @property
+    def per_second(self) -> float:
+        """Convert rate to operations per second"""
+        multiplier = {
+            "second": 1,
+            "minute": 60,
+            "hour": 3600
+        }
+        return self.count / multiplier[self.interval]
+
+class TokenBucket:
+    """
+    Thread-safe token bucket algorithm implementation for rate limiting.
+    
+    The token bucket algorithm provides smooth rate limiting with the ability
+    to handle bursts up to the bucket capacity. Tokens are added to the bucket
+    at a fixed rate, and each operation consumes one token.
+    
+    Args:
+        rate_limit: RateLimit configuration
+        capacity: Maximum number of tokens the bucket can hold. Defaults to
+                 the number of operations allowed per interval.
+    """
+    def __init__(self, rate_limit: RateLimit, capacity: int = None):
+        self.rate = rate_limit.per_second
+        self.capacity = capacity or rate_limit.count
+        self.tokens = self.capacity
+        self.last_update = time.time()
+        self.lock = threading.Lock()
+        self.next_allowed = self.last_update  # Track next allowed operation time
+    
+    def get_token(self) -> bool:
+        """
+        Try to get a token from the bucket.
+        
+        Returns:
+            bool: True if a token was acquired, False otherwise
+        """
+        with self.lock:
+            now = time.time()
+            if now < self.next_allowed:
+                return False
+            
+            self.next_allowed = max(self.next_allowed + (1 / self.rate), now + (1 / self.rate))
+            return True
+    
+    def wait_for_token(self):
+        """Block until a token is available"""
+        while True:
+            with self.lock:
+                now = time.time()
+                if now >= self.next_allowed:
+                    self.next_allowed = max(self.next_allowed + (1 / self.rate), now + (1 / self.rate))
+                    return
+                wait_time = self.next_allowed - now
+            
+            time.sleep(max(0.001, wait_time))  # Min sleep 1ms for CPU
+
+def get_executor_class(executor_type: ExecutorType):
+    """Get the appropriate executor class based on type"""
+    return {
+        "thread": ThreadPoolExecutor,
+        "process": ProcessPoolExecutor
+    }[executor_type]
+
+def get_or_create_executor(executor_type: ExecutorType, max_workers: int, prewarm: bool = False):
+    """
+    Get a cached executor or create a new one.
+    
+    This function manages a global cache of executors, allowing them to be
+    reused across multiple calls. The cache uses weak references, so executors
+    are automatically cleaned up when no longer needed.
+    
+    Args:
+        executor_type: Type of executor ("thread" or "process")
+        max_workers: Maximum number of workers
+        prewarm: If True, starts all workers immediately
+    
+    Returns:
+        ThreadPoolExecutor or ProcessPoolExecutor
+    """
+    key = (executor_type, max_workers)
+    executor = _EXECUTOR_CACHE.get(key)
+    
+    if executor is None or executor._shutdown:
+        executor_class = get_executor_class(executor_type)
+        executor = executor_class(max_workers=max_workers)
+        _EXECUTOR_CACHE[key] = executor
+        
+        # Prewarm workers by submitting no-op tasks
+        if prewarm:
+            futures = [executor.submit(lambda: None) for _ in range(max_workers)]
+            for f in futures:
+                f.result()  # Wait for workers to start
+                
+    return executor
+
+def parallel(
+    max_workers: int = None, 
+    batch_size: int = None, 
+    rate_limit: float | tuple[float, TimeUnit] | RateLimit = None,
+    executor_type: ExecutorType = "thread",
+    prewarm: bool = False
+):
+    """
+    Decorator for parallel execution of functions over iterables.
+    
+    This decorator transforms a function that processes a single item into one
+    that can process multiple items in parallel. It supports both thread and
+    process-based parallelism, rate limiting, batch processing, and worker
+    prewarming.
+    
+    The decorated function should take a single item as its first argument.
+    When called with a list/tuple, it will process all items in parallel.
+    When called with a single item, it will process it normally and return
+    a single-item list.
+    
+    Args:
+        max_workers: Maximum number of parallel workers. Defaults to:
+                    - Processes: CPU count
+                    - Threads: CPU count * 5
+        
+        batch_size: Number of items to process in each batch. Useful for
+                   controlling memory usage with large iterables. None means
+                   process all items at once.
+        
+        rate_limit: Rate limiting configuration, specified as:
+                   - float: Operations per second
+                   - tuple[float, TimeUnit]: (count, interval) e.g. (100, "minute")
+                   - RateLimit: RateLimit instance
+        
+        executor_type: Type of parallelism to use:
+                      - "thread": For I/O-bound tasks (default)
+                      - "process": For CPU-bound tasks
+        
+        prewarm: If True, starts all workers immediately. Useful for
+                latency-critical applications where cold start time matters.
+    
+    Returns:
+        Callable: Wrapped function that processes items in parallel
+    
+    Examples:
+        ```python
+        # Basic I/O-bound task
+        @parallel(max_workers=4)
+        def fetch_url(url: str) -> dict:
+            return requests.get(url).json()
+        
+        # CPU-bound task with rate limiting
+        @parallel(
+            max_workers=4,
+            executor_type="process",
+            rate_limit=(100, "minute")
+        )
+        def process_image(image: bytes) -> bytes:
+            return heavy_processing(image)
+        
+        # Batch processing
+        @parallel(max_workers=4, batch_size=10)
+        def analyze_text(text: str) -> dict:
+            return text_analysis(text)
+        
+        # Usage
+        urls = ["http://example1.com", "http://example2.com"]
+        results = fetch_url(urls)  # Parallel processing
+        
+        single_result = fetch_url("http://example.com")  # Returns [result]
+        ```
+    
+    Notes:
+        - The function preserves the original function's docstring and signature
+        - Rate limiting is thread-safe
+        - Executors are reused and automatically cleaned up
+        - Process pools handle rate limiting in the main process
+        - Batch processing helps with memory usage
+        - Prewarming is useful for latency-critical applications
+    """
+    # Calculate workers here so it's consistent
+    workers = max_workers or (
+        multiprocessing.cpu_count() 
+        if executor_type == "process" 
+        else (multiprocessing.cpu_count() * 5)
+    )
+    
+    # Prewarm if requested
+    if prewarm:
+        get_or_create_executor(executor_type, workers, prewarm=True)
+    
+    def decorator(func: Callable[..., T]) -> Callable[..., list[T]]:
+        @wraps(func)
+        def wrapper(*args, **kwargs) -> list[T]:
+            if not args or not isinstance(args[0], (list, tuple)):
+                return [func(*args, **kwargs)]
+            
+            items = args[0]
+            other_args = args[1:]
+            results = []
+            
+            # Convert rate_limit to RateLimit object
+            rate = None
+            if rate_limit is not None:
+                if isinstance(rate_limit, (int, float)):
+                    rate = RateLimit(float(rate_limit))
+                elif isinstance(rate_limit, tuple):
+                    rate = RateLimit(*rate_limit)
+                else:
+                    rate = rate_limit
+            
+            # For process pool, rate limiting happens in the main process
+            bucket = TokenBucket(rate) if rate and executor_type == "thread" else None
+            
+            def rate_limited_func(item):
+                if bucket:
+                    bucket.wait_for_token()
+                return func(item, *other_args, **kwargs)
+            
+            # Get or create cached executor
+            executor = get_or_create_executor(executor_type, workers)
+            
+            try:
+                if batch_size is None:
+                    # Rate limit in main process for ProcessPoolExecutor
+                    if rate and executor_type == "process":
+                        bucket = TokenBucket(rate)
+                        for item in items:
+                            bucket.wait_for_token()
+                    
+                    futures = [
+                        executor.submit(rate_limited_func if executor_type == "thread" else func, item, *other_args, **kwargs)
+                        for item in items
+                    ]
+                    results.extend(f.result() for f in as_completed(futures))
+                else:
+                    # Process in batches
+                    it = iter(items)
+                    while batch := list(islice(it, batch_size)):
+                        # Rate limit in main process for ProcessPoolExecutor
+                        if rate and executor_type == "process":
+                            bucket = TokenBucket(rate)
+                            for _ in batch:
+                                bucket.wait_for_token()
+                                
+                        futures = [
+                            executor.submit(rate_limited_func if executor_type == "thread" else func, item, *other_args, **kwargs)
+                            for item in batch
+                        ]
+                        results.extend(f.result() for f in as_completed(futures))
+            except:
+                # Don't shutdown cached executor on error
+                raise
+                
+            return results
+                
+        return wrapper
+    return decorator

pyarallel-0.1.0.dist-info/METADATA

@@ -0,0 +1,283 @@
+Metadata-Version: 2.4
+Name: pyarallel
+Version: 0.1.0
+Summary: A powerful parallel execution library for Python
+Project-URL: Homepage, https://github.com/oneryalcin/pyarallel
+Project-URL: Repository, https://github.com/oneryalcin/pyarallel.git
+Project-URL: Documentation, https://github.com/oneryalcin/pyarallel
+Author-email: Mehmet Oner Yalcin <oneryalcin@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 Pyarallel Contributors
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE.md
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# Pyarallel
+
+A powerful,feature-rich parallel execution library for Python that makes concurrent programming easy and efficient.
+
+## Features
+
+- **Simple Decorator-Based API**: Just add `@parallel` to your functions
+- **Flexible Parallelism**: Choose between threads (I/O-bound) and processes (CPU-bound)
+- **Smart Rate Limiting**: Control execution rates with per-second, per-minute, or per-hour limits
+- **Batch Processing**: Handle large datasets efficiently with automatic batching
+- **Performance Optimized**: 
+  - Automatic worker pool reuse
+  - Optional worker prewarming for latency-critical applications
+  - Smart defaults based on your system
+- **Production Ready**:
+  - Thread-safe implementation
+  - Memory-efficient with automatic cleanup
+  - Comprehensive error handling
+
+## Installation
+
+```bash
+pip install pyarallel
+```
+
+## Quick Start
+
+```python
+from pyarallel import parallel
+
+# Basic parallel processing
+@parallel(max_workers=4)
+def fetch_url(url: str) -> dict:
+    return requests.get(url).json()
+
+# Process multiple URLs in parallel
+urls = ["http://api1.com", "http://api2.com"]
+results = fetch_url(urls)
+
+# Rate-limited CPU-intensive task
+@parallel(
+    max_workers=4,
+    executor_type="process",
+    rate_limit=(100, "minute")  # 100 ops/minute
+)
+def process_image(image: bytes) -> bytes:
+    return heavy_processing(image)
+
+# Memory-efficient batch processing
+@parallel(max_workers=4, batch_size=10)
+def analyze_text(text: str) -> dict:
+    return text_analysis(text)
+```
+
+## Advanced Usage
+
+### Rate Limiting
+
+Control execution rates using various formats:
+
+```python
+# Operations per second
+@parallel(rate_limit=2.0)
+def func1(): ...
+
+# Operations per minute
+@parallel(rate_limit=(100, "minute"))
+def func2(): ...
+
+# Custom rate limit object
+from pyarallel import RateLimit
+rate = RateLimit(1000, "hour")
+@parallel(rate_limit=rate)
+def func3(): ...
+```
+
+### CPU-Bound Tasks
+
+Use process-based parallelism for CPU-intensive operations:
+
+```python
+@parallel(
+    max_workers=4,
+    executor_type="process",  # Use processes instead of threads
+    batch_size=10            # Process in batches of 10
+)
+def cpu_intensive(data: bytes) -> bytes:
+    return heavy_computation(data)
+```
+
+### Latency-Critical Applications
+
+Prewarm workers to minimize cold start latency:
+
+```python
+@parallel(
+    max_workers=4,
+    prewarm=True  # Start workers immediately
+)
+def latency_critical(item): ...
+```
+
+### Memory-Efficient Processing
+
+Handle large datasets with batch processing:
+
+```python
+@parallel(
+    max_workers=4,
+    batch_size=100  # Process items in batches of 100
+)
+def process_large_dataset(item): ...
+
+# Process millions of items without memory issues
+items = range(1_000_000)
+results = process_large_dataset(items)
+```
+
+## Best Practices
+
+1. **Choose the Right Executor**:
+   - Use `executor_type="thread"` (default) for I/O-bound tasks (network, disk)
+   - Use `executor_type="process"` for CPU-bound tasks (computation)
+
+2. **Optimize Worker Count**:
+   - For I/O-bound: `max_workers = cpu_count * 5` (default)
+   - For CPU-bound: `max_workers = cpu_count` (default)
+
+3. **Control Resource Usage**:
+   - Use `batch_size` for large datasets
+   - Use `rate_limit` to prevent overwhelming resources
+   - Only use `prewarm=True` when cold start latency is critical
+
+4. **Handle Errors Properly**:
+   ```python
+   @parallel()
+   def my_func(item):
+       try:
+           return process(item)
+       except Exception as e:
+           return {"error": str(e), "item": item}
+   ```
+
+## Roadmap
+
+### Observability & Debugging
+- **Advanced Telemetry System**
+  - Task execution metrics (duration, wait times, queue times)
+  - Worker utilization tracking
+  - Error frequency analysis
+  - SQLite persistence for historical data
+  - Interactive visualizations with Plotly
+  - Performance bottleneck identification
+
+- **Rich Logging System**
+  - Configurable log levels per component
+  - Structured logging for machine parsing
+  - Contextual information for debugging
+  - Log rotation and management
+  - Integration with popular logging frameworks
+
+### Advanced Features
+- **Callback System**
+  - Pre/post execution hooks
+  - Error handling callbacks
+  - Progress tracking
+  - Custom metrics collection
+  - State management hooks
+
+- **Smart Scheduling**
+  - Priority queues for tasks
+  - Deadline-aware scheduling
+  - Resource-aware task distribution
+  - Adaptive batch sizing
+  - Dynamic worker scaling
+
+- **Fault Tolerance**
+  - Automatic retries with backoff
+  - Circuit breaker pattern
+  - Fallback strategies
+  - Dead letter queues
+  - Task timeout handling
+
+- **Resource Management**
+  - Memory usage monitoring
+  - CPU utilization tracking
+  - Network bandwidth control
+  - Disk I/O rate limiting
+  - Resource quotas per task
+
+### Developer Experience
+- **CLI Tools**
+  - Task monitoring dashboard
+  - Performance profiling
+  - Configuration management
+  - Log analysis utilities
+  - Telemetry visualization
+
+
+### Enterprise Features
+- **Integration**
+  - Distributed tracing (OpenTelemetry)
+  - Metrics export (Prometheus)
+  - Log aggregation (ELK Stack)
+
+Want to contribute? Check out our [CONTRIBUTING.md](CONTRIBUTING.md) guide!
+
+## API Reference
+
+### @parallel Decorator
+
+```python
+@parallel(
+    max_workers: int = None,          # Maximum workers (default: based on CPU)
+    batch_size: int = None,           # Items per batch (default: all at once)
+    rate_limit: Union[                # Rate limiting configuration
+        float,                        # - Operations per second
+        Tuple[float, str],           # - (count, interval)
+        RateLimit                     # - RateLimit object
+    ] = None,
+    executor_type: str = "thread",    # "thread" or "process"
+    prewarm: bool = False            # Prewarm workers
+)
+```
+
+### RateLimit Class
+
+```python
+class RateLimit:
+    def __init__(self, count: float, interval: str = "second"):
+        """
+        Args:
+            count: Operations allowed per interval
+            interval: "second", "minute", or "hour"
+        """
+```
+
+## Contributing
+
+Contributions are welcome! Please check out our [Contributing Guide](CONTRIBUTING.md).
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

pyarallel-0.1.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+pyarallel/__init__.py,sha256=XQA0uDv38wz3OWRRKj0JE1M-Teee9y_1TsWU5ajOvSY,191
+pyarallel/core.py,sha256=5kTmCXyRH4agzUkAlHgAJ2c_Pmbrhe_wVC-JaG_z7i4,12522
+pyarallel-0.1.0.dist-info/METADATA,sha256=TOLwHbW2zohxJOrVEE2xpRBlCXxbuiFNSjqp2jj__ZQ,8252
+pyarallel-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pyarallel-0.1.0.dist-info/licenses/LICENSE.md,sha256=6jlQa2lR3rODieDf4bPeHXaOFHLOLbw2Q8jD2RAkz68,1079
+pyarallel-0.1.0.dist-info/RECORD,,

pyarallel-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pyarallel-0.1.0.dist-info/licenses/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Pyarallel Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.