pyarallel 0.1.0__tar.gz → 0.1.2__tar.gz

{pyarallel-0.1.0 → pyarallel-0.1.2}/.gitignore

@@ -33,6 +33,7 @@ ENV/
 *.swo
 
 # Test
+.coverage.*
 .coverage
 htmlcov/
 .pytest_cache/
@@ -41,3 +42,7 @@ htmlcov/
 # Misc
 .DS_Store
 *.log
+
+
+# docs 
+site

pyarallel-0.1.2/Makefile

@@ -0,0 +1,36 @@
+.PHONY: help test docs-serve docs-deploy format lint clean
+
+help:
+	@echo "Available commands:"
+	@echo "  make test         Run pytest suite"
+	@echo "  make docs-serve   Start mkdocs development server"
+	@echo "  make docs-deploy  Deploy documentation to GitHub Pages"
+	@echo "  make format      Format code with black and isort"
+	@echo "  make lint        Run mypy for type checking"
+	@echo "  make clean       Remove build artifacts"
+
+test:
+	pytest tests/ -v
+
+docs-serve:
+	mkdocs serve
+
+docs-deploy:
+	python -c "import pyarallel; from pathlib import Path; yml = Path('mkdocs.yml').read_text(); Path('mkdocs.yml').write_text(yml.replace('version: .*', f'version: {pyarallel.__version__}'))"
+	mkdocs gh-deploy
+
+format:
+	black .
+	isort .
+
+lint:
+	mypy pyarallel/
+
+clean:
+	rm -rf build/
+	rm -rf dist/
+	rm -rf *.egg-info/
+	find . -type d -name __pycache__ -exec rm -rf {} +
+	find . -type f -name '*.pyc' -delete
+	find . -type f -name '*.pyo' -delete
+	find . -type f -name '*.pyd' -delete

{pyarallel-0.1.0 → pyarallel-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyarallel
-Version: 0.1.0
+Version: 0.1.2
 Summary: A powerful parallel execution library for Python
 Project-URL: Homepage, https://github.com/oneryalcin/pyarallel
 Project-URL: Repository, https://github.com/oneryalcin/pyarallel.git
@@ -39,6 +39,9 @@ Description-Content-Type: text/markdown
 
 # Pyarallel
 
+[![Docs](https://img.shields.io/badge/docs-live-brightgreen)](https://oneryalcin.github.io/pyarallel/) [![PyPI version](https://img.shields.io/pypi/v/pyarallel)](https://pypi.org/project/pyarallel/) [![PyPI Downloads](https://static.pepy.tech/badge/pyarallel/month)](https://pepy.tech/project/pyarallel)
+
+
 A powerful,feature-rich parallel execution library for Python that makes concurrent programming easy and efficient.
 
 ## Features
@@ -56,6 +59,10 @@ A powerful,feature-rich parallel execution library for Python that makes concurr
   - Memory-efficient with automatic cleanup
   - Comprehensive error handling
 
+## Documentation
+
+Check out the [documentation](https://oneryalcin.github.io/pyarallel/) for detailed usage instructions and examples.
+
 ## Installation
 
 ```bash
@@ -180,6 +187,157 @@ results = process_large_dataset(items)
            return {"error": str(e), "item": item}
    ```
 
+## Configuration
+
+Pyarallel features a robust configuration system built on Pydantic, offering type validation, environment variable support, and thread-safe configuration management.
+
+### Basic Configuration
+
+```python
+from pyarallel import ConfigManager
+
+# Get the thread-safe singleton configuration manager
+config = ConfigManager.get_instance()
+
+# Update configuration with type validation
+config.update_config({
+    "execution": {
+        "default_max_workers": 8,
+        "default_executor_type": "thread",
+        "default_batch_size": 100,
+        "prewarm_pools": True
+    },
+    "rate_limiting": {
+        "default_rate": 1000,
+        "default_interval": "minute",
+        "burst_tolerance": 1.5
+    }
+})
+
+# Access configuration using dot notation
+workers = config.execution.default_max_workers
+rate = config.rate_limiting.default_rate
+
+# Category-specific updates
+config.update_execution(max_workers=16)
+config.update_rate_limiting(rate=2000)
+```
+
+### Environment Variables
+
+Configure Pyarallel using environment variables with the `PYARALLEL_` prefix. The system automatically handles type coercion and validation:
+
+```bash
+# Execution settings
+export PYARALLEL_MAX_WORKERS=4
+export PYARALLEL_EXECUTOR_TYPE=thread
+export PYARALLEL_BATCH_SIZE=100
+
+# Rate limiting
+export PYARALLEL_RATE_LIMIT=100/minute
+export PYARALLEL_FAIL_FAST=true
+
+# Complex values (using JSON)
+export PYARALLEL_RETRY_CONFIG='{"max_attempts": 3, "backoff": 1.5}'
+```
+
+### Configuration Schema
+
+The configuration system uses a structured schema with the following categories:
+
+```python
+{
+    "execution": {
+        "default_max_workers": int,        # Default worker count
+        "default_executor_type": str,      # "thread" or "process"
+        "default_batch_size": Optional[int], # Default batch size
+        "prewarm_pools": bool             # Enable worker prewarming
+    },
+    "rate_limiting": {
+        "default_rate": Optional[float],   # Default operations per interval
+        "default_interval": str,          # "second", "minute", "hour"
+        "burst_tolerance": float          # Burst allowance factor
+    },
+    "error_handling": {
+        "max_retries": int,               # Maximum retry attempts
+        "retry_backoff": float,           # Backoff multiplier
+        "fail_fast": bool                 # Stop on first error
+    },
+    "monitoring": {
+        "enable_logging": bool,           # Enable detailed logging
+        "log_level": str,                # Logging level
+        "sentry_dsn": Optional[str],     # Sentry integration
+        "metrics_enabled": bool          # Enable metrics collection
+    }
+}
+```
+
+### Best Practices
+
+1. **Use Environment Variables for Deployment**:
+   - Keep configuration in environment variables for different environments
+   - Use the `PYARALLEL_` prefix to avoid conflicts
+   - Complex values can be passed as JSON strings
+
+2. **Validate Configuration Early**:
+   - Set up configuration at application startup
+   - Use type validation to catch issues early
+   - Test configuration with sample data
+
+3. **Thread-Safe Updates**:
+   - Always use `ConfigManager.get_instance()` for thread-safe access
+   - Make configuration changes before starting parallel operations
+   - Use category-specific update methods for better type safety
+
+4. **Configuration Inheritance**:
+   - Global settings serve as defaults
+   - Decorator arguments override global configuration
+   - Environment variables take precedence over code-based configuration
+
+### Runtime Configuration Warnings
+
+Pyarallel includes built-in warnings to help identify potential performance issues:
+
+```python
+# Warning for high worker count
+@parallel(max_workers=150)  # Triggers warning about system impact
+def high_worker_task(): ...
+
+# Warning for inefficient process pool configuration
+@parallel(
+    executor_type="process",
+    batch_size=1  # Triggers warning about inefficient batch size
+)
+def inefficient_task(): ...
+```
+
+### Configuration Inheritance
+
+Pyarallel uses a hierarchical configuration system:
+
+1. **Default Values**: Built-in defaults (4 workers, thread executor, batch size 10)
+2. **Global Configuration**: Set via ConfigManager
+3. **Environment Variables**: Override global config
+4. **Decorator Arguments**: Highest precedence, override all other settings
+
+```python
+# Global configuration (lowest precedence)
+config = ConfigManager.get_instance()
+config.update_config({
+    "execution": {
+        "default_max_workers": 8,
+        "default_executor_type": "thread"
+    }
+})
+
+# Environment variables (middle precedence)
+# export PYARALLEL_MAX_WORKERS=16
+
+# Decorator arguments (highest precedence)
+@parallel(max_workers=4)  # This value wins
+def my_func(): ...
+```
+
 ## Roadmap
 
 ### Observability & Debugging
@@ -235,7 +393,6 @@ results = process_large_dataset(items)
   - Log analysis utilities
   - Telemetry visualization
 
-
 ### Enterprise Features
 - **Integration**
   - Distributed tracing (OpenTelemetry)

{pyarallel-0.1.0 → pyarallel-0.1.2}/README.md

@@ -1,5 +1,8 @@
 # Pyarallel
 
+[![Docs](https://img.shields.io/badge/docs-live-brightgreen)](https://oneryalcin.github.io/pyarallel/) [![PyPI version](https://img.shields.io/pypi/v/pyarallel)](https://pypi.org/project/pyarallel/) [![PyPI Downloads](https://static.pepy.tech/badge/pyarallel/month)](https://pepy.tech/project/pyarallel)
+
+
 A powerful,feature-rich parallel execution library for Python that makes concurrent programming easy and efficient.
 
 ## Features
@@ -17,6 +20,10 @@ A powerful,feature-rich parallel execution library for Python that makes concurr
   - Memory-efficient with automatic cleanup
   - Comprehensive error handling
 
+## Documentation
+
+Check out the [documentation](https://oneryalcin.github.io/pyarallel/) for detailed usage instructions and examples.
+
 ## Installation
 
 ```bash
@@ -141,6 +148,157 @@ results = process_large_dataset(items)
            return {"error": str(e), "item": item}
    ```
 
+## Configuration
+
+Pyarallel features a robust configuration system built on Pydantic, offering type validation, environment variable support, and thread-safe configuration management.
+
+### Basic Configuration
+
+```python
+from pyarallel import ConfigManager
+
+# Get the thread-safe singleton configuration manager
+config = ConfigManager.get_instance()
+
+# Update configuration with type validation
+config.update_config({
+    "execution": {
+        "default_max_workers": 8,
+        "default_executor_type": "thread",
+        "default_batch_size": 100,
+        "prewarm_pools": True
+    },
+    "rate_limiting": {
+        "default_rate": 1000,
+        "default_interval": "minute",
+        "burst_tolerance": 1.5
+    }
+})
+
+# Access configuration using dot notation
+workers = config.execution.default_max_workers
+rate = config.rate_limiting.default_rate
+
+# Category-specific updates
+config.update_execution(max_workers=16)
+config.update_rate_limiting(rate=2000)
+```
+
+### Environment Variables
+
+Configure Pyarallel using environment variables with the `PYARALLEL_` prefix. The system automatically handles type coercion and validation:
+
+```bash
+# Execution settings
+export PYARALLEL_MAX_WORKERS=4
+export PYARALLEL_EXECUTOR_TYPE=thread
+export PYARALLEL_BATCH_SIZE=100
+
+# Rate limiting
+export PYARALLEL_RATE_LIMIT=100/minute
+export PYARALLEL_FAIL_FAST=true
+
+# Complex values (using JSON)
+export PYARALLEL_RETRY_CONFIG='{"max_attempts": 3, "backoff": 1.5}'
+```
+
+### Configuration Schema
+
+The configuration system uses a structured schema with the following categories:
+
+```python
+{
+    "execution": {
+        "default_max_workers": int,        # Default worker count
+        "default_executor_type": str,      # "thread" or "process"
+        "default_batch_size": Optional[int], # Default batch size
+        "prewarm_pools": bool             # Enable worker prewarming
+    },
+    "rate_limiting": {
+        "default_rate": Optional[float],   # Default operations per interval
+        "default_interval": str,          # "second", "minute", "hour"
+        "burst_tolerance": float          # Burst allowance factor
+    },
+    "error_handling": {
+        "max_retries": int,               # Maximum retry attempts
+        "retry_backoff": float,           # Backoff multiplier
+        "fail_fast": bool                 # Stop on first error
+    },
+    "monitoring": {
+        "enable_logging": bool,           # Enable detailed logging
+        "log_level": str,                # Logging level
+        "sentry_dsn": Optional[str],     # Sentry integration
+        "metrics_enabled": bool          # Enable metrics collection
+    }
+}
+```
+
+### Best Practices
+
+1. **Use Environment Variables for Deployment**:
+   - Keep configuration in environment variables for different environments
+   - Use the `PYARALLEL_` prefix to avoid conflicts
+   - Complex values can be passed as JSON strings
+
+2. **Validate Configuration Early**:
+   - Set up configuration at application startup
+   - Use type validation to catch issues early
+   - Test configuration with sample data
+
+3. **Thread-Safe Updates**:
+   - Always use `ConfigManager.get_instance()` for thread-safe access
+   - Make configuration changes before starting parallel operations
+   - Use category-specific update methods for better type safety
+
+4. **Configuration Inheritance**:
+   - Global settings serve as defaults
+   - Decorator arguments override global configuration
+   - Environment variables take precedence over code-based configuration
+
+### Runtime Configuration Warnings
+
+Pyarallel includes built-in warnings to help identify potential performance issues:
+
+```python
+# Warning for high worker count
+@parallel(max_workers=150)  # Triggers warning about system impact
+def high_worker_task(): ...
+
+# Warning for inefficient process pool configuration
+@parallel(
+    executor_type="process",
+    batch_size=1  # Triggers warning about inefficient batch size
+)
+def inefficient_task(): ...
+```
+
+### Configuration Inheritance
+
+Pyarallel uses a hierarchical configuration system:
+
+1. **Default Values**: Built-in defaults (4 workers, thread executor, batch size 10)
+2. **Global Configuration**: Set via ConfigManager
+3. **Environment Variables**: Override global config
+4. **Decorator Arguments**: Highest precedence, override all other settings
+
+```python
+# Global configuration (lowest precedence)
+config = ConfigManager.get_instance()
+config.update_config({
+    "execution": {
+        "default_max_workers": 8,
+        "default_executor_type": "thread"
+    }
+})
+
+# Environment variables (middle precedence)
+# export PYARALLEL_MAX_WORKERS=16
+
+# Decorator arguments (highest precedence)
+@parallel(max_workers=4)  # This value wins
+def my_func(): ...
+```
+
 ## Roadmap
 
 ### Observability & Debugging
@@ -196,7 +354,6 @@ results = process_large_dataset(items)
   - Log analysis utilities
   - Telemetry visualization
 
-
 ### Enterprise Features
 - **Integration**
   - Distributed tracing (OpenTelemetry)

pyarallel-0.1.2/TASKS.md

@@ -0,0 +1,123 @@
+# Configuration System Implementation Tasks
+
+## 1. Core Configuration System [✅]
+- [✅] Create `config.py` with dataclass-based configuration
+- [✅] Implement singleton config manager
+- [✅] Add type validation for config values
+- [✅] Add merge strategy for partial updates
+
+**Test Cases:**
+```python
+def test_config_defaults():
+    """Default values are set correctly"""
+
+def test_config_validation():
+    """Invalid values raise proper exceptions"""
+
+def test_partial_update():
+    """Partial config updates don't affect other values"""
+```
+
+## 2. Environment Variables Support [✅]
+- [✅] Add env var parsing in config manager
+- [✅] Implement type coercion (str -> proper type)
+- [✅] Add prefix support (PYARALLEL_*)
+- [✅] Support complex values (lists, dicts via JSON)
+
+**Test Cases:**
+```python
+def test_env_var_loading():
+    """Config loads from environment variables"""
+
+def test_env_var_types():
+    """Environment variables are properly typed"""
+
+def test_env_var_prefix():
+    """Only PYARALLEL_* vars are loaded"""
+```
+
+## 3. Runtime Configuration API [✅]
+- [✅] Add global `set()` method
+- [✅] Add category-specific setters
+- [✅] Add value getters with dot notation
+- [✅] Implement config validation hooks
+
+**Test Cases:**
+```python
+def test_global_set():
+    """Global config can be set"""
+
+def test_category_set():
+    """Category-specific settings work"""
+
+def test_dot_notation():
+    """Dot notation access works for nested config"""
+```
+
+## 4. Decorator Integration [✅]
+- [✅] Update parallel decorator to use config
+- [✅] Add config override in decorator
+- [✅] Implement inheritance rules
+- [✅] Add runtime config warnings
+
+**Test Cases:**
+```python
+def test_decorator_defaults():
+    """Decorator uses global defaults"""
+
+def test_decorator_override():
+    """Decorator args override global config"""
+
+def test_runtime_warnings():
+    """Warnings for problematic configs"""
+```
+
+## 5. Documentation [ ]
+- [ ] Add configuration section to README
+- [ ] Document all environment variables
+- [ ] Add configuration examples
+- [ ] Document best practices
+
+## Configuration Schema
+```python
+{
+    "execution": {
+        "default_max_workers": int,
+        "default_executor_type": str,
+        "default_batch_size": Optional[int],
+        "prewarm_pools": bool
+    },
+    "rate_limiting": {
+        "default_rate": Optional[float],
+        "default_interval": str,
+        "burst_tolerance": float
+    },
+    "error_handling": {
+        "max_retries": int,
+        "retry_backoff": float,
+        "fail_fast": bool
+    },
+    "monitoring": {
+        "enable_logging": bool,
+        "log_level": str,
+        "sentry_dsn": Optional[str],
+        "metrics_enabled": bool
+    }
+}
+```
+
+## Environment Variables
+```
+PYARALLEL_MAX_WORKERS=4
+PYARALLEL_EXECUTOR_TYPE=thread
+PYARALLEL_BATCH_SIZE=10
+PYARALLEL_RATE_LIMIT=100/minute
+PYARALLEL_FAIL_FAST=true
+PYARALLEL_SENTRY_DSN=https://...
+```
+
+## Progress Tracking
+- ✅ Task completed
+- 🚧 In progress
+- ❌ Blocked
+- [ ] Not started

pyarallel-0.1.2/docs/api-reference/configuration-api.md

@@ -0,0 +1,49 @@
+# API Reference
+
+## Configuration API
+
+### ConfigManager
+
+Singleton class for managing global configuration.
+
+```python
+from pyarallel import ConfigManager
+
+config_manager = ConfigManager.get_instance()
+```
+
+#### Methods
+
+- `get_config()`: Get current configuration
+- `update_config(config: dict)`: Update configuration with new values
+- `reset_config()`: Reset to default configuration
+
+#### Configuration Options
+
+```python
+{
+    "execution": {
+        "default_max_workers": 4,
+        "default_executor_type": "thread",
+        "default_batch_size": 10
+    }
+}
+```
+
+#### Examples
+
+```python
+# Get config manager instance
+config_manager = ConfigManager.get_instance()
+
+# Update global configuration
+config_manager.update_config({
+    "execution": {
+        "default_max_workers": 8,
+        "default_executor_type": "process"
+    }
+})
+
+# Get current configuration
+config = config_manager.get_config()
+```

pyarallel-0.1.2/docs/api-reference/decorators.md

@@ -0,0 +1,64 @@
+# API Reference
+
+## Decorator API
+
+### @parallel
+
+The main decorator for enabling parallel execution of functions.
+
+```python
+from pyarallel import parallel
+
+@parallel(
+    max_workers: int = None,
+    batch_size: int = None,
+    rate_limit: float | tuple[float, TimeUnit] | RateLimit = None,
+    executor_type: Literal["thread", "process"] = None,
+    prewarm: bool = False
+)
+def function(item, *args, **kwargs) -> Any: ...
+```
+
+#### Parameters
+
+- `max_workers` (int, optional): Maximum number of parallel workers. Defaults to global configuration.
+- `batch_size` (int, optional): Number of items to process in each batch. Defaults to global configuration.
+- `rate_limit`: Rate limiting configuration. Can be:
+  - A float (operations per second)
+  - A tuple of (count, unit) where unit is "second", "minute", or "hour"
+  - A RateLimit instance
+- `executor_type` (str, optional): Type of parallelism to use ("thread" or "process"). Defaults to global configuration.
+- `prewarm` (bool): If True, starts all workers immediately.
+
+#### Returns
+
+A wrapped function that accepts either:
+- A single item (returns a single-item list)
+- A list/tuple of items (processes in parallel and returns a list of results)
+
+#### Examples
+
+```python
+# Basic usage with threads
+@parallel(max_workers=4)
+def fetch_url(url: str) -> dict:
+    return requests.get(url).json()
+
+# Process-based execution with rate limiting
+@parallel(
+    max_workers=4,
+    executor_type="process",
+    rate_limit=(100, "minute")
+)
+def process_image(image: bytes) -> bytes:
+    return heavy_processing(image)
+
+# Batch processing with prewarming
+@parallel(
+    max_workers=4,
+    batch_size=10,
+    prewarm=True
+)
+def analyze_text(text: str) -> dict:
+    return text_analysis(text)
+```