overload-cli 0.1.0__py3-none-any.whl

overload/engine/models.py

@@ -0,0 +1,254 @@
+from __future__ import annotations
+
+import statistics
+from collections import defaultdict
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class TestType(str, Enum):
+    LOAD = "load"
+    STRESS = "stress"
+    SPIKE = "spike"
+    SOAK = "soak"
+    RAMP = "ramp"
+    BURST = "burst"
+    BREAKPOINT = "breakpoint"
+    CUSTOM = "custom"
+    RATE_LIMIT = "ratelimit"
+    SEQUENTIAL = "sequential"
+
+
+class RequestDistribution(str, Enum):
+    ROUND_ROBIN = "round-robin"
+    RANDOM = "random"
+
+
+@dataclass
+class RequestResult:
+    request_name: str
+    method: str
+    url: str
+    status_code: int
+    latency_ms: float
+    timestamp: float
+    error: str | None = None
+    headers_sent: dict[str, str] = field(default_factory=dict)
+    headers_received: dict[str, str] = field(default_factory=dict)
+    body_size_bytes: int = 0
+    response_body: str | None = None
+
+
+@dataclass
+class RunProgress:
+    run_id: str
+    total_requests: int
+    completed_requests: int
+    current_rps: float
+    phase: str
+    elapsed_seconds: float
+    error_count: int = 0
+    status_codes: dict[int, int] = field(default_factory=dict)
+    avg_latency_ms: float = 0.0
+    recent_results: list[dict] = field(default_factory=list)
+
+
+@dataclass
+class PatternConfig:
+    concurrency: int = 20
+    timeout_seconds: float = 30.0
+    verify_ssl: bool = True
+    follow_redirects: bool = True
+    save_responses: bool = False
+    distribution: RequestDistribution = RequestDistribution.ROUND_ROBIN
+    think_time_ms: int = 0
+
+    # Load test
+    target_rps: int = 50
+    ramp_up_seconds: int = 30
+    hold_duration_seconds: int = 300
+    ramp_down_seconds: int = 10
+
+    # Stress test
+    start_rps: int = 10
+    step_rps: int = 20
+    step_duration_seconds: int = 30
+    failure_threshold_pct: float = 80.0
+    max_rps: int = 500
+
+    # Spike test
+    baseline_rps: int = 20
+    spike_rps: int = 200
+    baseline_duration_seconds: int = 60
+    spike_duration_seconds: int = 30
+    recovery_duration_seconds: int = 60
+
+    # Soak test
+    soak_rps: int = 30
+    soak_duration_seconds: int = 1800
+
+    # Ramp test
+    ramp_start_rps: int = 10
+    ramp_end_rps: int = 200
+
+    # Burst test
+    total_requests: int = 200
+
+    # Breakpoint test
+    precision_rps: int = 5
+    latency_threshold_ms: float = 2000.0
+    error_threshold_pct: float = 10.0
+
+    # Custom test
+    stages: list[dict] = field(default_factory=list)
+
+    # Rate limit test
+    rate_limit_cap: int = 60
+    rate_limit_requests: int = 120
+
+    # Sequential runner
+    iterations: int = 1
+    delay_ms: int = 0
+
+
+@dataclass
+class Threshold:
+    metric: str
+    operator: str
+    value: float
+
+
+@dataclass
+class AssertionResult:
+    metric: str
+    operator: str
+    expected: float
+    actual: float
+    passed: bool
+
+
+@dataclass
+class Verdict:
+    passed: bool
+    results: list[AssertionResult]
+
+
+class Stats:
+    def __init__(self) -> None:
+        self.results: list[RequestResult] = []
+
+    def add(self, result: RequestResult) -> None:
+        self.results.append(result)
+
+    def add_all(self, results: list[RequestResult]) -> None:
+        self.results.extend(results)
+
+    @property
+    def total(self) -> int:
+        return len(self.results)
+
+    @property
+    def success_count(self) -> int:
+        return sum(1 for r in self.results if 200 <= r.status_code < 400)
+
+    @property
+    def error_count(self) -> int:
+        return sum(
+            1
+            for r in self.results
+            if r.status_code < 200 or r.status_code >= 400
+        )
+
+    @property
+    def rate_limited_count(self) -> int:
+        return sum(1 for r in self.results if r.status_code == 429)
+
+    def compute(self) -> dict | None:
+        if not self.results:
+            return None
+
+        status_counts: dict[int, int] = defaultdict(int)
+        for r in self.results:
+            status_counts[r.status_code] += 1
+
+        latencies = [r.latency_ms for r in self.results]
+        sorted_latencies = sorted(latencies)
+
+        ok = sum(v for k, v in status_counts.items() if 200 <= k < 400)
+        rl = status_counts.get(429, 0)
+        total = len(self.results)
+
+        t0 = self.results[0].timestamp
+        buckets: dict[int, dict[int, int]] = defaultdict(lambda: defaultdict(int))
+        for r in self.results:
+            buckets[int(r.timestamp - t0)][r.status_code] += 1
+
+        per_second = []
+        for sec in sorted(buckets):
+            b = buckets[sec]
+            sec_total = sum(b.values())
+            sec_ok = sum(v for k, v in b.items() if 200 <= k < 300)
+            sec_redirect = sum(v for k, v in b.items() if 300 <= k < 400)
+            sec_rl = b.get(429, 0)
+            sec_client = sum(v for k, v in b.items() if 400 <= k < 500 and k != 429)
+            sec_server = sum(v for k, v in b.items() if k >= 500)
+            sec_conn = sum(v for k, v in b.items() if k <= 0)
+            per_second.append({
+                "second": sec,
+                "total": sec_total,
+                "ok": sec_ok + sec_redirect,
+                "rate_limited": sec_rl,
+                "client_errors": sec_client,
+                "server_errors": sec_server,
+                "conn_errors": sec_conn,
+                "errors": sec_total - sec_ok - sec_redirect - sec_rl,
+            })
+
+        timeline = [
+            {
+                "timestamp": round(r.timestamp - t0, 3),
+                "status": r.status_code,
+                "latency_ms": round(r.latency_ms, 1),
+                "request_name": r.request_name,
+            }
+            for r in self.results
+        ]
+
+        request_log = []
+        for r in self.results:
+            entry = {
+                "timestamp": round(r.timestamp - t0, 3),
+                "status": r.status_code,
+                "latency_ms": round(r.latency_ms, 1),
+                "method": r.method,
+                "url": r.url,
+                "request_name": r.request_name,
+                "error": r.error,
+            }
+            if r.response_body is not None:
+                entry["response_body"] = r.response_body
+            request_log.append(entry)
+
+        p95_idx = int(len(sorted_latencies) * 0.95)
+        p99_idx = int(len(sorted_latencies) * 0.99)
+
+        return {
+            "total": total,
+            "ok": ok,
+            "rate_limited": rl,
+            "errors": total - ok - rl,
+            "status_codes": dict(status_counts),
+            "latency": {
+                "min": round(min(latencies), 1),
+                "median": round(statistics.median(latencies), 1),
+                "mean": round(statistics.mean(latencies), 1),
+                "p95": round(sorted_latencies[min(p95_idx, total - 1)], 1),
+                "p99": round(sorted_latencies[min(p99_idx, total - 1)], 1),
+                "max": round(max(latencies), 1),
+            },
+            "per_second": per_second,
+            "timeline": timeline,
+            "request_log": request_log,
+            "duration_seconds": round(max(self.results[-1].timestamp - t0, 0.1), 1) if total > 1 else 0.1,
+            "avg_rps": round(total / max(self.results[-1].timestamp - t0, 0.1), 1),
+        }

overload/engine/rate_limiter.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from collections.abc import Callable, Coroutine
+from typing import Any
+
+from overload.collection.models import ParsedRequest
+from overload.collection.variables import VariableContext
+from overload.engine.http_client import HttpClient
+from overload.engine.load_patterns import _fire_one, _pick_request
+from overload.engine.models import (
+    PatternConfig,
+    RequestDistribution,
+    RequestResult,
+    RunProgress,
+)
+
+logger = logging.getLogger(__name__)
+
+ProgressCallback = Callable[[RunProgress], Coroutine[Any, Any, None]]
+
+
+async def run_rate_limit_test(
+    client: HttpClient,
+    requests: list[ParsedRequest],
+    variables: VariableContext,
+    config: PatternConfig,
+    run_id: str,
+    cancel_event: asyncio.Event,
+    on_progress: ProgressCallback | None = None,
+) -> tuple[list[RequestResult], list[dict]]:
+    cap = config.rate_limit_cap
+    total_requests = config.rate_limit_requests
+    concurrency = config.concurrency
+    sem = asyncio.Semaphore(concurrency)
+    all_results: list[RequestResult] = []
+    ramp_rows: list[dict] = []
+    start_time = time.monotonic()
+    request_idx = 0
+
+    logger.info("Rate Limit Test: cap=%d, requests=%d, concurrency=%d", cap, total_requests, concurrency)
+
+    # Phase 1: Burst test
+    if on_progress:
+        await on_progress(RunProgress(
+            run_id=run_id, total_requests=total_requests,
+            completed_requests=0, current_rps=0,
+            phase=f"Burst: {total_requests} requests", elapsed_seconds=0,
+        ))
+
+    burst_tasks = [
+        asyncio.create_task(
+            _fire_one(
+                client,
+                _pick_request(requests, i, RequestDistribution.ROUND_ROBIN),
+                variables, sem,
+            )
+        )
+        for i in range(total_requests)
+    ]
+
+    for coro in asyncio.as_completed(burst_tasks):
+        if cancel_event.is_set():
+            for t in burst_tasks:
+                t.cancel()
+            break
+        result = await coro
+        all_results.append(result)
+        request_idx += 1
+
+    # Phase 2: Ramp to find threshold
+    if not cancel_event.is_set():
+        logger.info("Rate limit ramp: 10 -> %d req/s", cap * 2)
+
+        for rps in range(10, cap * 2 + 1, 10):
+            if cancel_event.is_set():
+                break
+
+            interval = 1.0 / rps
+            batch_tasks = []
+            batch_start = time.monotonic()
+
+            for i in range(rps):
+                if cancel_event.is_set():
+                    break
+                delay = batch_start + i * interval - time.monotonic()
+                if delay > 0:
+                    await asyncio.sleep(delay)
+                req = _pick_request(requests, request_idx, RequestDistribution.ROUND_ROBIN)
+                request_idx += 1
+                batch_tasks.append(
+                    asyncio.create_task(_fire_one(client, req, variables, sem))
+                )
+
+            batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True)
+            valid = [r for r in batch_results if isinstance(r, RequestResult)]
+            all_results.extend(valid)
+
+            total = len(valid)
+            rl = sum(1 for r in valid if r.status_code == 429)
+            ok = sum(1 for r in valid if 200 <= r.status_code < 400)
+            pct = rl * 100 // total if total else 0
+
+            ramp_rows.append({"rps": rps, "ok": ok, "rate_limited": rl, "pct": pct})
+
+            logger.info("%d/s: ok=%d, 429=%d (%d%%)", rps, ok, rl, pct)
+
+            if on_progress:
+                elapsed = time.monotonic() - start_time
+                await on_progress(RunProgress(
+                    run_id=run_id, total_requests=0,
+                    completed_requests=len(all_results),
+                    current_rps=rps,
+                    phase=f"Ramp: {rps} req/s ({pct}% throttled)",
+                    elapsed_seconds=round(elapsed, 1),
+                ))
+
+            if pct >= 50:
+                logger.info("Rate limiting confirmed at %d req/s", rps)
+                break
+
+    return all_results, ramp_rows

overload/engine/runner.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from collections.abc import Callable, Coroutine
+from typing import Any
+
+from overload.collection.models import ParsedRequest
+from overload.collection.variables import VariableContext
+from overload.engine.http_client import HttpClient
+from overload.engine.models import PatternConfig, RequestResult, RunProgress
+
+logger = logging.getLogger(__name__)
+
+ProgressCallback = Callable[[RunProgress], Coroutine[Any, Any, None]]
+
+
+async def run_sequential(
+    client: HttpClient,
+    requests: list[ParsedRequest],
+    variables: VariableContext,
+    config: PatternConfig,
+    run_id: str,
+    cancel_event: asyncio.Event,
+    on_progress: ProgressCallback | None = None,
+) -> list[RequestResult]:
+    iterations = config.iterations
+    delay_ms = config.delay_ms
+    total = len(requests) * iterations
+    all_results: list[RequestResult] = []
+    start_time = time.monotonic()
+
+    logger.info(
+        "Sequential: %d requests x %d iterations, delay=%dms",
+        len(requests), iterations, delay_ms,
+    )
+
+    if on_progress:
+        try:
+            await on_progress(RunProgress(
+                run_id=run_id,
+                total_requests=total,
+                completed_requests=0,
+                current_rps=0,
+                phase="Preparing sequential run...",
+                elapsed_seconds=0,
+            ))
+        except Exception:
+            logger.exception("Error in progress callback")
+
+    for iteration in range(1, iterations + 1):
+        if cancel_event.is_set():
+            break
+
+        for idx, request in enumerate(requests):
+            if cancel_event.is_set():
+                break
+
+            result = await client.execute(request, variables)
+            all_results.append(result)
+
+            logger.debug(
+                "Iter %d/%d, Req %d/%d: %s %s -> %d (%.1fms)",
+                iteration, iterations, idx + 1, len(requests),
+                result.method, result.url, result.status_code, result.latency_ms,
+            )
+
+            if on_progress:
+                elapsed = time.monotonic() - start_time
+                try:
+                    await on_progress(RunProgress(
+                        run_id=run_id,
+                        total_requests=total,
+                        completed_requests=len(all_results),
+                        current_rps=len(all_results) / max(elapsed, 0.001),
+                        phase=f"Iteration {iteration}/{iterations}: {request.name}",
+                        elapsed_seconds=round(elapsed, 1),
+                    ))
+                except Exception:
+                    logger.exception("Error in progress callback")
+
+            if delay_ms > 0 and not (iteration == iterations and idx == len(requests) - 1):
+                await asyncio.sleep(delay_ms / 1000.0)
+
+    return all_results

overload/report/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

overload/report/exporters.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import csv
+import json
+import logging
+import os
+
+from overload.engine.models import Stats
+from overload.utils.naming import stamped_filename
+
+logger = logging.getLogger(__name__)
+
+
+def export_json(
+    stats: Stats,
+    test_type: str,
+    run_id: str,
+    output_dir: str = ".",
+    ramp_rows: list[dict] | None = None,
+) -> str:
+    computed = stats.compute()
+    if not computed:
+        logger.warning("No results to export")
+        return ""
+
+    payload = {
+        "meta": {"run_id": run_id, "test_type": test_type},
+        "stats": computed,
+        "ramp_rows": ramp_rows or [],
+    }
+
+    filename = stamped_filename("overload_results", run_id, ".json")
+    filepath = os.path.join(output_dir, filename)
+    os.makedirs(output_dir, exist_ok=True)
+
+    with open(filepath, "w", encoding="utf-8") as f:
+        json.dump(payload, f, indent=2)
+
+    logger.info("JSON export: %s", os.path.abspath(filepath))
+    return filepath
+
+
+def export_csv(
+    stats: Stats,
+    run_id: str,
+    output_dir: str = ".",
+) -> str:
+    if not stats.results:
+        logger.warning("No results to export")
+        return ""
+
+    filename = stamped_filename("overload_results", run_id, ".csv")
+    filepath = os.path.join(output_dir, filename)
+    os.makedirs(output_dir, exist_ok=True)
+
+    fieldnames = [
+        "timestamp", "request_name", "method", "url",
+        "status_code", "latency_ms", "error", "body_size_bytes",
+    ]
+
+    with open(filepath, "w", encoding="utf-8", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for r in stats.results:
+            writer.writerow({
+                "timestamp": r.timestamp,
+                "request_name": r.request_name,
+                "method": r.method,
+                "url": r.url,
+                "status_code": r.status_code,
+                "latency_ms": round(r.latency_ms, 1),
+                "error": r.error or "",
+                "body_size_bytes": r.body_size_bytes,
+            })
+
+    logger.info("CSV export: %s", os.path.abspath(filepath))
+    return filepath

overload/report/generator.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader
+
+from overload.engine.models import Stats
+from overload.utils.naming import generate_run_id, stamped_filename
+
+logger = logging.getLogger(__name__)
+
+TEMPLATES_DIR = Path(__file__).parent / "templates"
+
+
+def _create_jinja_env() -> Environment:
+    return Environment(
+        loader=FileSystemLoader(str(TEMPLATES_DIR)),
+        autoescape=True,
+    )
+
+
+def generate_report(
+    stats: Stats,
+    test_type: str,
+    config: dict,
+    run_id: str | None = None,
+    ramp_rows: list[dict] | None = None,
+    output_dir: str = "reports",
+    verdict: dict | None = None,
+) -> str:
+    run_id = run_id or generate_run_id()
+    computed = stats.compute()
+    if not computed:
+        logger.warning("No results to generate report from")
+        return ""
+
+    payload: dict = {
+        "meta": {
+            "run_id": run_id,
+            "test_type": test_type,
+            "config": config,
+        },
+        "stats": computed,
+        "ramp_rows": ramp_rows or [],
+    }
+    if verdict is not None:
+        payload["verdict"] = verdict
+
+    data_json = json.dumps(payload, separators=(",", ":"))
+    data_json = data_json.replace("</", "<\\/")
+
+    env = _create_jinja_env()
+    template = env.get_template("report.html")
+    html = template.render(
+        run_id=run_id,
+        test_type=test_type,
+        data_json=data_json,
+    )
+
+    filename = stamped_filename("overload_report", run_id, ".html")
+    filepath = os.path.join(output_dir, filename)
+    os.makedirs(output_dir, exist_ok=True)
+
+    with open(filepath, "w", encoding="utf-8") as f:
+        f.write(html)
+
+    logger.info("Report generated: %s", os.path.abspath(filepath))
+    return filepath