robotframework-parallel-requests 0.1.0__py3-none-any.whl

robot_parallel_requests/rate_limiter.py

@@ -0,0 +1,76 @@
+"""Rate limiting utilities using token bucket algorithm."""
+import time
+import threading
+from typing import Optional
+
+
+class TokenBucket:
+    """Token bucket rate limiter for controlling request rates."""
+    
+    def __init__(self, rate: float, burst_size: int):
+        """
+        Initialize token bucket.
+        
+        Args:
+            rate: Tokens per second (e.g., 1.75 for 105 requests/minute)
+            burst_size: Maximum tokens that can accumulate
+        """
+        if rate <= 0:
+            raise ValueError(f"Rate must be positive, got: {rate}")
+        if burst_size <= 0:
+            raise ValueError(f"Burst size must be positive, got: {burst_size}")
+
+        self.rate = rate
+        self.burst_size = burst_size
+        self.tokens = float(self.burst_size)
+        self.last_update = time.time()
+        self.lock = threading.Lock()
+    
+    def acquire(self, tokens: int = 1, blocking: bool = True, timeout: Optional[float] = None) -> bool:
+        """
+        Acquire tokens from the bucket.
+        
+        Args:
+            tokens: Number of tokens to acquire
+            blocking: If True, wait until tokens available
+            timeout: Maximum time to wait if blocking
+            
+        Returns:
+            True if tokens acquired, False otherwise
+        """
+        if self.rate <= 0:
+            raise ValueError("Cannot acquire tokens when rate is zero or negative")
+
+        deadline = None if timeout is None else time.time() + timeout
+        
+        while True:
+            with self.lock:
+                now = time.time()
+                elapsed = now - self.last_update
+                
+                # Add tokens based on elapsed time
+                self.tokens = min(self.burst_size, self.tokens + elapsed * self.rate)
+                self.last_update = now
+                
+                if self.tokens >= tokens:
+                    self.tokens -= tokens
+                    return True
+                
+                if not blocking:
+                    return False
+                
+                if deadline and now >= deadline:
+                    return False
+                
+                # Calculate wait time
+                tokens_needed = tokens - self.tokens
+                wait_time = tokens_needed / self.rate
+                
+            # Release lock while sleeping
+            if deadline:
+                wait_time = min(wait_time, deadline - time.time())
+            
+            if wait_time > 0:
+                time.sleep(wait_time)
+            else:
+                return False

robot_parallel_requests/response_store.py

@@ -0,0 +1,33 @@
+from threading import Lock
+from typing import Dict, Any, Optional
+
+
+class ResponseStore:
+    """Simple in-memory store for responses and exceptions keyed by id."""
+
+    def __init__(self):
+        self._responses: Dict[str, Any] = {}
+        self._lock = Lock()
+
+    def set_response(self, id: str, value: Any) -> None:
+        with self._lock:
+            self._responses[id] = value
+
+    def get(self, id: str) -> Optional[Any]:
+        with self._lock:
+            return self._responses.get(id)
+
+    def has(self, id: str) -> bool:
+        with self._lock:
+            return id in self._responses
+
+    def __contains__(self, id: str) -> bool:
+        return self.has(id)
+
+    def all_ids(self):
+        with self._lock:
+            return list(self._responses.keys())
+
+    def clear(self):
+        with self._lock:
+            self._responses.clear()

robot_parallel_requests/retry.py

@@ -0,0 +1,106 @@
+"""Retry logic with exponential backoff and jitter."""
+import random
+import time
+from typing import List, Optional, Callable, Any, Tuple
+from dataclasses import dataclass, field
+
+import httpx
+
+
+# Transport-level failures that are usually safe to retry by default.
+DEFAULT_RETRY_EXCEPTIONS: List[type] = [
+    httpx.TimeoutException,
+    httpx.NetworkError,
+    httpx.RemoteProtocolError,
+]
+
+
+@dataclass
+class RetryPolicy:
+    """Configuration for retry behavior with exponential backoff."""
+    max_retries: int = 3
+    backoff_factor: float = 2.0
+    retry_statuses: Optional[List[int]] = None
+    retry_exceptions: Optional[List[type]] = None
+    jitter: float = 0.1  # Fraction of wait time added as random jitter (0 disables)
+
+    def __post_init__(self):
+        if self.retry_statuses is None:
+            self.retry_statuses = [429, 500, 502, 503, 504]
+        if self.retry_exceptions is None:
+            self.retry_exceptions = list(DEFAULT_RETRY_EXCEPTIONS)
+        if self.jitter < 0:
+            raise ValueError(f"jitter must be >= 0, got: {self.jitter}")
+
+    def should_retry(
+        self,
+        attempt: int,
+        status_code: Optional[int] = None,
+        exception: Optional[Exception] = None,
+    ) -> bool:
+        """Determine if a request should be retried."""
+        if attempt >= self.max_retries:
+            return False
+
+        if status_code and status_code in self.retry_statuses:
+            return True
+
+        if exception and any(isinstance(exception, exc_type) for exc_type in self.retry_exceptions):
+            return True
+
+        return False
+
+    def get_wait_time(self, attempt: int) -> float:
+        """Calculate wait time before retry using exponential backoff + jitter."""
+        base = self.backoff_factor ** attempt
+        if self.jitter <= 0:
+            return base
+        return base + (random.random() * self.jitter * base)
+
+
+def retry_with_backoff(func: Callable, policy: RetryPolicy, *args, **kwargs) -> Tuple[Any, int]:
+    """
+    Execute function with retry and exponential backoff.
+
+    Args:
+        func: Function to execute
+        policy: Retry policy configuration
+        *args: Positional arguments for func
+        **kwargs: Keyword arguments for func
+
+    Returns:
+        Tuple of (result, retry_count) where retry_count is the number of re-attempts
+
+    Raises:
+        Last exception if all retries exhausted
+    """
+    attempt = 0
+    last_exception = None
+
+    while attempt <= policy.max_retries:
+        try:
+            result = func(*args, **kwargs)
+
+            # Check if result has status_code (httpx.Response)
+            if hasattr(result, 'status_code'):
+                if policy.should_retry(attempt, status_code=result.status_code):
+                    wait_time = policy.get_wait_time(attempt)
+                    time.sleep(wait_time)
+                    attempt += 1
+                    continue
+
+            return result, attempt
+
+        except Exception as exc:
+            last_exception = exc
+            if policy.should_retry(attempt, exception=exc):
+                wait_time = policy.get_wait_time(attempt)
+                time.sleep(wait_time)
+                attempt += 1
+                continue
+            raise
+
+    # If we exhausted retries, raise last exception or return last result
+    if last_exception:
+        raise last_exception
+    raise RuntimeError("Retry loop exhausted without a result or exception")

robot_parallel_requests/session.py

@@ -0,0 +1,27 @@
+"""Session management for parallel requests."""
+from typing import Optional, Dict
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Session:
+    """Represents a named session with base URL and default headers."""
+    alias: str
+    base_url: Optional[str] = None
+    headers: Dict[str, str] = field(default_factory=dict)
+    
+    def resolve_url(self, url: str) -> str:
+        """Resolve URL against base_url if relative."""
+        if self.base_url and not url.startswith(('http://', 'https://')):
+            # Remove leading slash from url if base_url ends with slash
+            base = self.base_url.rstrip('/')
+            url_part = url.lstrip('/')
+            return f"{base}/{url_part}"
+        return url
+    
+    def merge_headers(self, request_headers: Optional[Dict[str, str]] = None) -> Dict[str, str]:
+        """Merge session headers with request-specific headers."""
+        merged = self.headers.copy()
+        if request_headers:
+            merged.update(request_headers)
+        return merged

robot_parallel_requests/tasks.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional, TYPE_CHECKING
+import uuid
+
+if TYPE_CHECKING:
+    from .rate_limiter import TokenBucket
+    from .retry import RetryPolicy
+
+
+@dataclass
+class RequestTask:
+    method: str
+    url: str
+    kwargs: Dict[str, Any] = field(default_factory=dict)
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    session_name: Optional[str] = None
+    rate_limiter: Optional['TokenBucket'] = None
+    retry_policy: Optional['RetryPolicy'] = None

robot_parallel_requests/transport/__init__.py

@@ -0,0 +1,3 @@
+from .base import TransportBase
+
+__all__ = ["TransportBase"]

robot_parallel_requests/transport/base.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+from typing import Any
+from ..tasks import RequestTask
+
+
+class TransportBase(ABC):
+    @abstractmethod
+    def send(self, task: RequestTask):
+        """Send a RequestTask and return a response-like object or raise."""

robot_parallel_requests/transport/httpx_async_future.py

@@ -0,0 +1,82 @@
+"""Skeleton for future async httpx transport.
+
+This module shows how to add high-concurrency async support later.
+Not implemented in MVP, but provides a roadmap for v1.2+.
+"""
+
+# from typing import Any
+# import httpx
+# import asyncio
+# from .tasks import RequestTask
+# from .transport.base import TransportBase
+
+
+# class HttpxAsyncTransport(TransportBase):
+#     """Async httpx transport for high-concurrency scenarios (10k+).
+#
+#     Usage:
+#         transport = HttpxAsyncTransport()
+#         worker = AsyncWorkerPool(transport, max_workers=1000)
+#         # or provide an event loop runner
+#     """
+#
+#     def __init__(self, timeout: float = 30.0):
+#         self._client = httpx.AsyncClient(timeout=timeout)
+#
+#     async def send(self, task: RequestTask) -> Any:
+#         """Send async request and return httpx.Response."""
+#         resp = await self._client.request(task.method, task.url, **task.kwargs)
+#         return resp
+#
+#     async def close(self):
+#         await self._client.aclose()
+
+
+# class AsyncWorkerPool:
+#     """Asyncio-based worker pool for high-concurrency.
+#
+#     Manages an event loop and dispatches coroutines to handle requests.
+#     """
+#
+#     def __init__(self, transport: HttpxAsyncTransport, max_workers: int = 1000):
+#         self.transport = transport
+#         self.max_workers = max_workers
+#         self._event_loop = None  # Managed separately or in background thread
+#         self._tasks = []
+#
+#     def submit(self, task: RequestTask) -> str:
+#         """Queue a task for async execution."""
+#         coro = self.transport.send(task)
+#         # Schedule on event loop (implementation TBD)
+#         return task.id
+#
+#     async def wait_all_async(self, timeout: float = None):
+#         """Await all pending coroutines."""
+#         pass
+#
+#     def shutdown(self):
+#         """Cleanup event loop and transport."""
+#         pass
+
+
+# STRATEGY FOR BRIDGING TO ROBOT:
+# 
+# Since Robot Framework keywords are synchronous, we'd need to:
+# 1. Run the async event loop in a background thread
+# 2. Submit tasks to it from keywords (thread-safe queue)
+# 3. Use asyncio.run_coroutine_threadsafe() for thread-safe calls
+# 4. or use trio/curio for better integration
+#
+# Example:
+#     def __init__(self):
+#         self.loop = asyncio.new_event_loop()
+#         self.thread = threading.Thread(target=self.loop.run_forever, daemon=True)
+#         self.thread.start()
+#
+#     def submit(self, task):
+#         future = asyncio.run_coroutine_threadsafe(
+#             self._send_async(task),
+#             self.loop
+#         )
+#         self._futures[task.id] = future
+#         return task.id

robot_parallel_requests/transport/httpx_sync.py

@@ -0,0 +1,53 @@
+import httpx
+from typing import Any, Optional
+from ..tasks import RequestTask
+from .base import TransportBase
+
+
+def limits_for_workers(max_workers: int) -> httpx.Limits:
+    """Size the connection pool so workers are not starved waiting for sockets."""
+    workers = max(1, int(max_workers))
+    # Keepalive slightly above workers so a rolling set of warm connections is available.
+    keepalive = max(workers, min(workers + 5, workers * 2))
+    max_connections = max(workers * 2, keepalive)
+    return httpx.Limits(
+        max_connections=max_connections,
+        max_keepalive_connections=keepalive,
+    )
+
+
+class HttpxSyncTransport(TransportBase):
+    def __init__(
+        self,
+        timeout: float = 30.0,
+        max_workers: int = 5,
+        limits: Optional[httpx.Limits] = None,
+        http2: bool = False,
+    ):
+        self.max_workers = max(1, int(max_workers))
+        self.http2 = bool(http2)
+        self._limits = limits if limits is not None else limits_for_workers(self.max_workers)
+        try:
+            self._client = httpx.Client(
+                timeout=timeout,
+                limits=self._limits,
+                http2=self.http2,
+            )
+        except ImportError as exc:
+            if self.http2:
+                raise ImportError(
+                    "HTTP/2 support requires the optional 'h2' package. "
+                    "Install with: pip install 'robotframework-parallel-requests[http2]'"
+                ) from exc
+            raise
+
+    def send(self, task: RequestTask) -> Any:
+        # task.kwargs expected to be compatible with httpx.Client.request
+        resp = self._client.request(task.method, task.url, **task.kwargs)
+        return resp
+
+    def close(self):
+        try:
+            self._client.close()
+        except Exception:
+            pass

robot_parallel_requests/worker.py

@@ -0,0 +1,136 @@
+from concurrent.futures import ThreadPoolExecutor, Future, wait, ALL_COMPLETED
+from typing import Optional, List, Tuple
+import time
+
+from .tasks import RequestTask
+from .response_store import ResponseStore
+from .metrics import MetricsCollector, RequestMetric
+from .retry import retry_with_backoff
+
+
+class WorkerPool:
+    def __init__(self, transport, max_workers: int = 5, metrics_collector: Optional[MetricsCollector] = None):
+        self.transport = transport
+        self.max_workers = max_workers
+        self._executor = ThreadPoolExecutor(max_workers=max_workers)
+        self._store = ResponseStore()
+        self._futures: dict[str, Future] = {}
+        self._pending_ids: List[str] = []
+        self.metrics_collector = metrics_collector
+
+    def submit(self, task: RequestTask) -> str:
+        if task.id in self._futures or self._store.has(task.id):
+            raise ValueError(f"Duplicate request id: {task.id}")
+
+        future = self._executor.submit(self._run_task, task)
+        self._futures[task.id] = future
+        self._pending_ids.append(task.id)
+        return task.id
+
+    def _send_once(self, task: RequestTask):
+        if task.rate_limiter:
+            task.rate_limiter.acquire()
+        return self.transport.send(task)
+
+    def _run_task(self, task: RequestTask):
+        start_time = time.perf_counter()
+        wall_start = time.time()
+        metric = RequestMetric(
+            request_id=task.id,
+            method=task.method,
+            url=task.url,
+            timestamp=wall_start,
+        )
+        retry_count = 0
+
+        try:
+            if task.retry_policy:
+                resp, retry_count = retry_with_backoff(
+                    self._send_once, task.retry_policy, task
+                )
+            else:
+                resp = self._send_once(task)
+
+            metric.duration = time.perf_counter() - start_time
+            metric.retries = retry_count
+
+            if hasattr(resp, 'status_code'):
+                metric.status_code = resp.status_code
+
+            self._store.set_response(task.id, resp)
+
+        except Exception as exc:
+            metric.duration = time.perf_counter() - start_time
+            metric.retries = retry_count
+            metric.error = str(exc)
+            self._store.set_response(task.id, exc)
+
+        finally:
+            metric.completed_at = time.time()
+            if self.metrics_collector:
+                self.metrics_collector.record_request(metric)
+
+    def get_response(self, id: str):
+        if id not in self._futures and not self._store.has(id):
+            raise KeyError(
+                f"Unknown response id: {id!r}. Queue a request first or check the id."
+            )
+        if not self._store.has(id):
+            future = self._futures.get(id)
+            if future is not None and not future.done():
+                raise LookupError(
+                    f"Response for id {id!r} is not ready yet. "
+                    f"Call Parallel Wait For All Requests first."
+                )
+            raise LookupError(f"No response stored for id: {id!r}")
+        return self._store.get(id)
+
+    def get_responses_in_order(self, ids: List[str]) -> list:
+        # Soft lookup: incomplete timed-out ids may still be absent from the store.
+        return [self._store.get(request_id) for request_id in ids]
+
+    def wait_all(
+        self,
+        timeout: Optional[float] = None,
+        cancel_pending: bool = True,
+    ) -> Tuple[int, int, List[str]]:
+        """Wait for pending requests. Returns (completed_count, incomplete_count, batch_ids)."""
+        if timeout is not None and not isinstance(timeout, float):
+            try:
+                timeout = float(timeout)
+            except Exception:
+                raise ValueError(f"Timeout must be a float or convertible to float, got: {timeout!r}")
+
+        batch_ids = list(self._pending_ids)
+        if not batch_ids:
+            return 0, 0, batch_ids
+
+        pending_futures = [self._futures[request_id] for request_id in batch_ids]
+        done, not_done = wait(pending_futures, timeout=timeout, return_when=ALL_COMPLETED)
+
+        if cancel_pending and not_done:
+            for future in not_done:
+                future.cancel()
+
+        completed_count = len(done)
+        incomplete_count = len(not_done)
+        self._pending_ids.clear()
+
+        return completed_count, incomplete_count, batch_ids
+
+    def shutdown(self):
+        try:
+            self._executor.shutdown(wait=True, cancel_futures=True)
+        except TypeError:
+            # cancel_futures added in Python 3.9
+            try:
+                self._executor.shutdown(wait=True)
+            except Exception:
+                pass
+        except Exception:
+            pass
+        try:
+            if hasattr(self.transport, "close"):
+                self.transport.close()
+        except Exception:
+            pass