jac-loadtest 0.2.2__tar.gz → 0.2.3__tar.gz

{jac_loadtest-0.2.2 → jac_loadtest-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jac-loadtest
-Version: 0.2.2
+Version: 0.2.3
 Summary: HAR-based load testing CLI for jac-scale applications
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown

{jac_loadtest-0.2.2 → jac_loadtest-0.2.3}/jac_loadtest/cli.py

@@ -15,7 +15,7 @@ def run(args: object) -> None:
     from jac_loadtest.core.har_parser import parse_har
     from jac_loadtest.core.engine import run_all_vus
     from jac_loadtest.core.metrics import MetricsCollector
-    from jac_loadtest.output.reporter import render_console
+    from jac_loadtest.output.reporter import render_console, render_json, render_html
 
     config = from_args(args)
 
@@ -70,17 +70,72 @@ def run(args: object) -> None:
         print(f"Error: {exc}", file=sys.stderr)
         sys.exit(2)
 
-    metrics = MetricsCollector(max_samples=config.max_samples)
+    from rich.console import Console as _Console
+    _console = _Console(stderr=True)
+
     t_start = time.time()
 
     try:
-        asyncio.run(
-            run_all_vus(entries, config, metrics, topology=topology, auth_provider=auth_provider)
-        )
+        with _console.status(f"Running load test — {config.vus} VUs..."):
+            if config.workers > 1:
+                from jac_loadtest.core.process_runner import run_multiprocess
+                metrics = run_multiprocess(entries, config, topology, auth_provider)
+            else:
+                metrics = MetricsCollector(max_samples=config.max_samples)
+                asyncio.run(
+                    run_all_vus(entries, config, metrics, topology=topology, auth_provider=auth_provider)
+                )
     except AuthenticationError as exc:
         print(f"Error: {exc}", file=sys.stderr)
         sys.exit(2)
 
     duration_s = time.time() - t_start
-    stats = metrics.compute_endpoint_stats(duration_s)
-    render_console(stats, config, actual_duration_s=duration_s)
+    stats = metrics.compute_endpoint_stats()
+    snapshots = metrics.generate_timeseries(t_start)
+    completion_p50, completion_p95, completion_p99 = metrics.completion_percentiles(t_start)
+
+    fmt = config.report_format
+
+    if fmt == "json":
+        output = render_json(
+            stats, config,
+            actual_duration_s=duration_s,
+            total_rps=metrics.global_rps(duration_s),
+            snapshots=snapshots,
+            completion_p50_s=completion_p50,
+            completion_p95_s=completion_p95,
+            completion_p99_s=completion_p99,
+        )
+        if config.report_out:
+            with open(config.report_out, "w", encoding="utf-8") as fh:
+                fh.write(output)
+            print(f"JSON report written to {config.report_out}")
+        else:
+            print(output)
+
+    elif fmt == "html":
+        if not config.report_out:
+            print("Error: --report-out <path> is required for --report-format html", file=sys.stderr)
+            sys.exit(2)
+        output = render_html(
+            stats, config,
+            actual_duration_s=duration_s,
+            total_rps=metrics.global_rps(duration_s),
+            snapshots=snapshots,
+            completion_p50_s=completion_p50,
+            completion_p95_s=completion_p95,
+            completion_p99_s=completion_p99,
+        )
+        with open(config.report_out, "w", encoding="utf-8") as fh:
+            fh.write(output)
+        print(f"HTML report written to {config.report_out}", file=sys.stderr)
+
+    else:
+        render_console(
+            stats, config,
+            actual_duration_s=duration_s,
+            total_rps=metrics.global_rps(duration_s),
+            completion_p50_s=completion_p50,
+            completion_p95_s=completion_p95,
+            completion_p99_s=completion_p99,
+        )

{jac_loadtest-0.2.2 → jac_loadtest-0.2.3}/jac_loadtest/config.py

@@ -4,6 +4,7 @@ Phase 0: dataclass with built-in defaults only.
 Phase 2 will add jac.toml reading via jac_scale.config_loader.
 """
 from __future__ import annotations
+import os
 from dataclasses import dataclass, field
 from typing import Any
 
@@ -14,6 +15,7 @@ BUILT_IN_DEFAULTS: dict = {
     "iterations": 1,
     "ramp_up": "0s",
     "timeout": "30s",
+    "workers": os.cpu_count() or 1,
     "mode": "monolith",
     "think_time": "none",
     "think_time_scale": 1.0,
@@ -40,6 +42,7 @@ class LoadTestConfig:
     iterations: int  = 1
     ramp_up: str = "0s"
     timeout: str = "30s"
+    workers: int = 1
 
     # Traffic
     mode: str = "monolith"
@@ -129,6 +132,7 @@ def from_args(args: object) -> LoadTestConfig:
         iterations=resolve("iterations"),
         mode=resolve("mode"),
         vus=resolve("vus"),
+        workers=resolve("workers"),
         duration=resolve("duration"),
         ramp_up=resolve("ramp_up"),
         timeout=resolve("timeout"),

{jac_loadtest-0.2.2 → jac_loadtest-0.2.3}/jac_loadtest/core/engine.py

@@ -6,10 +6,12 @@ from __future__ import annotations
 
 import asyncio
 import signal
+import socket
 import sys
 from typing import TYPE_CHECKING
 
 import aiohttp
+from aiohttp.abc import AbstractResolver as _AbstractResolver
 
 from jac_loadtest.config import parse_duration
 from jac_loadtest.core.metrics import RequestResult, normalize_path
@@ -22,12 +24,37 @@ if TYPE_CHECKING:
     from jac_loadtest.bridge.auth import AuthProvider
 
 
+class _PreResolvedResolver(_AbstractResolver):
+    """Resolver that returns a pre-resolved IP for known hostnames, falling back to
+    system DNS for anything else. Injected into each worker's TCPConnector so workers
+    never issue their own DNS lookups for the target host."""
+
+    def __init__(self, host_map: dict[str, str]) -> None:
+        self._host_map = host_map
+        self._fallback = aiohttp.ThreadedResolver()
+
+    async def resolve(  # type: ignore[override]
+        self, hostname: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET
+    ) -> list[dict]:
+        if hostname in self._host_map:
+            ip = self._host_map[hostname]
+            addr_family = socket.AF_INET6 if ":" in ip else socket.AF_INET
+            return [{"hostname": hostname, "host": ip, "port": port, "family": addr_family, "proto": 0, "flags": 0}]
+        return await self._fallback.resolve(hostname, port, family)  # type: ignore[return-value]
+
+    async def close(self) -> None:
+        await self._fallback.close()
+
+
 async def run_all_vus(
     entries: list[HarEntry],
     config: LoadTestConfig,
     metrics: MetricsCollector,
     topology: TopologyRouter | None = None,
     auth_provider: AuthProvider | None = None,
+    vu_id_offset: int = 0,
+    pre_authed_tokens: dict[int, str] | None = None,
+    pre_resolved_hosts: dict[str, str] | None = None,
 ) -> None:
     """Spawn N virtual user coroutines and run until duration/iterations/stop signal."""
     stop_requested = asyncio.Event()
@@ -44,24 +71,51 @@ async def run_all_vus(
 
     signal.signal(signal.SIGINT, _on_first_sigint)
 
+    timeout = aiohttp.ClientTimeout(total=parse_duration(config.timeout))
     ramp_up_seconds = parse_duration(config.ramp_up)
 
-    # Limit concurrent logins to avoid exhausting OS sockets when VU count is large.
-    auth_semaphore = asyncio.Semaphore(50)
+    # Use pre-computed tokens when provided (multi-process path: auth ran in main process).
+    # Otherwise authenticate here (single-process path).
+    token_by_vu: dict[int, str] = {}
+    if pre_authed_tokens is not None:
+        token_by_vu = pre_authed_tokens
+    elif auth_provider is not None:
+        if not config.url:
+            raise ValueError("auth_provider requires --url to be set")
+        n_creds = len(auth_provider._credentials)
+        # Unique credential indices needed by this worker's VU slice.
+        cred_indices = {(vu_id_offset + i) % n_creds for i in range(config.vus)}
+        sem = asyncio.Semaphore(min(50, len(cred_indices)))
+
+        async def _do_auth(cred_idx: int) -> tuple[int, str]:
+            async with sem:
+                async with aiohttp.ClientSession(timeout=timeout) as auth_session:
+                    if config.url is not None:
+                        tok = await auth_provider.authenticate(cred_idx, auth_session, config.url)
+            return cred_idx, tok
+
+        auth_results: list[tuple[int, str]] = list(
+            await asyncio.gather(*[_do_auth(idx) for idx in cred_indices])
+        )
+        token_by_cred = dict(auth_results)
+        token_by_vu = {
+            vu_id_offset + i: token_by_cred[(vu_id_offset + i) % n_creds]
+            for i in range(config.vus)
+        }
 
     tasks = [
         asyncio.create_task(
             _run_vu(
-                vu_id=i,
+                vu_id=vu_id_offset + i,
                 delay=(i / config.vus) * ramp_up_seconds if config.vus > 1 else 0.0,
                 entries=entries,
                 config=config,
                 metrics=metrics,
                 stop_requested=stop_requested,
                 loop=loop,
-                auth_provider=auth_provider,
+                token=token_by_vu.get(vu_id_offset + i),
                 topology=topology,
-                auth_semaphore=auth_semaphore,
+                pre_resolved_hosts=pre_resolved_hosts or {},
             )
         )
         for i in range(config.vus)
@@ -81,40 +135,34 @@ async def _run_vu(
     metrics: MetricsCollector,
     stop_requested: asyncio.Event,
     loop: asyncio.AbstractEventLoop,
-    auth_provider: AuthProvider | None = None,
+    token: str | None = None,
     topology: TopologyRouter | None = None,
-    auth_semaphore: asyncio.Semaphore | None = None,
+    pre_resolved_hosts: dict[str, str] | None = None,
 ) -> None:
-    """Single virtual user: wait ramp delay, authenticate, then replay HAR entries."""
+    """Single virtual user: wait ramp delay, then replay HAR entries with a pre-fetched token."""
     if delay > 0:
         await asyncio.sleep(delay)
 
     timeout = aiohttp.ClientTimeout(total=parse_duration(config.timeout))
-    duration_seconds = parse_duration(config.duration)
-    t_start = loop.time()
+   
     iteration = 0
     # Warn once per unrouted path within this VU to avoid spam across iterations.
     _warned_unrouted: set[str] = set()
 
-    async with aiohttp.ClientSession(timeout=timeout) as session:
-        # Authenticate once before entering the request loop.
-        token: str | None = None
-        if auth_provider is not None:
-            if not config.url:
-                raise ValueError("auth_provider requires --url to be set")
-            async with (auth_semaphore or asyncio.Semaphore()):
-                token = await auth_provider.authenticate(vu_id, session, config.url)
-
+    connector = (
+        aiohttp.TCPConnector(resolver=_PreResolvedResolver(pre_resolved_hosts))
+        if pre_resolved_hosts
+        else aiohttp.TCPConnector()
+    )
+    async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
         while not stop_requested.is_set():
-            if loop.time() - t_start >= duration_seconds:
-                break
             if config.iterations is not None and iteration >= config.iterations:
                 break
 
             for entry in entries:
                 if stop_requested.is_set():
                     break
-                # Skip the HAR-recorded login; auth is handled by the pre-loop step.
+                # Skip the HAR-recorded login; auth is handled by pre-fetched token.
                 if entry.is_login and token is not None:
                     continue
                 result = await _send_request(
@@ -127,8 +175,6 @@ async def _run_vu(
                     topology=topology,
                 )
                 if result is None:
-                    # topology.resolve() found no route (e.g. /static/* fetch entries
-                    # that look like API calls but have no matching service prefix).
                     from urllib.parse import urlparse as _up
                     path = _up(entry.url).path
                     if path not in _warned_unrouted:
@@ -162,8 +208,6 @@ async def _send_request(
     if token:
         headers["Authorization"] = f"Bearer {token}"
 
-    # Resolve the final request URL and service label via topology.
-    # topology=None means monolith mode: use entry.url as-is, label "monolith".
     if topology is not None:
         try:
             request_url, service_name = topology.resolve(entry.url)
@@ -172,9 +216,10 @@ async def _send_request(
     else:
         request_url, service_name = entry.url, "monolith"
 
-    # Endpoint identifier for metrics uses the original entry URL (path only, normalized).
+    import time as _time
     endpoint = normalize_path(entry.url)
-    t0 = loop.time()
+    t0 = loop.time()   # high-res clock for latency measurement only
+    ts = _time.time()  # wall-clock for cross-process timestamp comparison
 
     try:
         async with session.request(
@@ -192,7 +237,7 @@ async def _send_request(
                 status=resp.status,
                 latency_ms=latency_ms,
                 bytes_received=len(body),
-                timestamp=t0,
+                timestamp=ts,
                 vu_id=vu_id,
                 error_type=None,
                 occurrence=entry.occurrence,
@@ -206,13 +251,41 @@ async def _send_request(
             status=0,
             latency_ms=parse_duration(config.timeout) * 1000,
             bytes_received=0,
-            timestamp=t0,
+            timestamp=ts,
             vu_id=vu_id,
             error_type="TIMEOUT",
             occurrence=entry.occurrence,
             total_occurrences=entry.total_occurrences,
         )
 
+    except aiohttp.ClientConnectorDNSError:
+        return RequestResult(
+            endpoint=endpoint,
+            service=service_name,
+            status=0,
+            latency_ms=0.0,
+            bytes_received=0,
+            timestamp=ts,
+            vu_id=vu_id,
+            error_type="DNS_ERROR",
+            occurrence=entry.occurrence,
+            total_occurrences=entry.total_occurrences,
+        )
+
+    except aiohttp.ClientSSLError:
+        return RequestResult(
+            endpoint=endpoint,
+            service=service_name,
+            status=0,
+            latency_ms=0.0,
+            bytes_received=0,
+            timestamp=ts,
+            vu_id=vu_id,
+            error_type="SSL_ERROR",
+            occurrence=entry.occurrence,
+            total_occurrences=entry.total_occurrences,
+        )
+
     except aiohttp.ClientConnectorError:
         return RequestResult(
             endpoint=endpoint,
@@ -220,9 +293,51 @@ async def _send_request(
             status=0,
             latency_ms=0.0,
             bytes_received=0,
-            timestamp=t0,
+            timestamp=ts,
             vu_id=vu_id,
             error_type="CONNECTION_REFUSED",
             occurrence=entry.occurrence,
             total_occurrences=entry.total_occurrences,
         )
+
+    except aiohttp.ServerDisconnectedError:
+        return RequestResult(
+            endpoint=endpoint,
+            service=service_name,
+            status=0,
+            latency_ms=(loop.time() - t0) * 1000,
+            bytes_received=0,
+            timestamp=ts,
+            vu_id=vu_id,
+            error_type="SERVER_DISCONNECTED",
+            occurrence=entry.occurrence,
+            total_occurrences=entry.total_occurrences,
+        )
+
+    except aiohttp.ClientOSError:
+        return RequestResult(
+            endpoint=endpoint,
+            service=service_name,
+            status=0,
+            latency_ms=(loop.time() - t0) * 1000,
+            bytes_received=0,
+            timestamp=ts,
+            vu_id=vu_id,
+            error_type="CONNECTION_RESET",
+            occurrence=entry.occurrence,
+            total_occurrences=entry.total_occurrences,
+        )
+
+    except Exception as e:
+        return RequestResult(
+            endpoint=endpoint,
+            service=service_name,
+            status=0,
+            latency_ms=(loop.time() - t0) * 1000,
+            bytes_received=0,
+            timestamp=ts,
+            vu_id=vu_id,
+            error_type=str(e).upper() or type(e).__name__.upper(),
+            occurrence=entry.occurrence,
+            total_occurrences=entry.total_occurrences,
+        )

{jac_loadtest-0.2.2 → jac_loadtest-0.2.3}/jac_loadtest/core/metrics.py

@@ -3,7 +3,7 @@
 Three-layer storage (Phase 4):
   Layer 1 — total_count (always accurate RPS)
   Layer 2 — deque(maxlen=max_samples) of RequestResult (percentiles)
-  Layer 3 — list[StatsSnapshot] every 5s (time-series charts)
+  Layer 3 — list[StatsSnapshot] generated post-run by binning samples into 10s intervals
 
 Phase 0: dataclasses only.
 """
@@ -22,7 +22,7 @@ class RequestResult:
     bytes_received: int
     timestamp: float
     vu_id: int
-    error_type: str | None  # None | "TIMEOUT" | "CONNECTION_REFUSED" | "DNS_ERROR" | "SSL_ERROR"
+    error_type: str | None  # None | "TIMEOUT" | "CONNECTION_REFUSED" | "DNS_ERROR" | "SSL_ERROR" | "SERVER_DISCONNECTED" | "CONNECTION_RESET"
     occurrence: int = 1
     total_occurrences: int = 1
 
@@ -41,7 +41,6 @@ class EndpointStats:
     p50_ms: float
     p95_ms: float
     p99_ms: float
-    rps: float
     error_breakdown: dict[str, int] = field(default_factory=dict)
 
 
@@ -53,6 +52,7 @@ class StatsSnapshot:
     p99_ms: float
     rps: float
     error_rate_pct: float
+    total_requests: int = 0
 
 
 def percentile(latencies: list[float], p: float) -> float:
@@ -88,20 +88,22 @@ class MetricsCollector:
     def __init__(self, max_samples: int = 1_000_000) -> None:
         self.total_count: int = 0
         self._samples: deque[RequestResult] = deque(maxlen=max_samples)
-        self._snapshots: list[StatsSnapshot] = []
 
     def record(self, result: RequestResult) -> None:
         self.total_count += 1
         self._samples.append(result)
 
-    def compute_endpoint_stats(self, duration_seconds: float) -> list[EndpointStats]:
+    def global_rps(self, duration_seconds: float) -> float:
+        """Return total requests per second across all endpoints."""
+        return self.total_count / max(duration_seconds, 0.001)
+
+    def compute_endpoint_stats(self) -> list[EndpointStats]:
         """Aggregate per-endpoint stats from collected samples."""
         groups: dict[str, list[RequestResult]] = {}
         for result in self._samples:
             groups.setdefault(result.endpoint, []).append(result)
 
         stats: list[EndpointStats] = []
-        safe_duration = max(duration_seconds, 0.001)
 
         for endpoint, results in groups.items():
             latencies = [r.latency_ms for r in results]
@@ -142,29 +144,57 @@ class MetricsCollector:
                     p50_ms=percentile(latencies, 50),
                     p95_ms=percentile(latencies, 95),
                     p99_ms=percentile(latencies, 99),
-                    rps=self.total_count / safe_duration,
                     error_breakdown=error_breakdown,
                 )
             )
 
         return stats
 
-    def flush_snapshot(self, timestamp: float, duration_seconds: float) -> None:
-        """Record a 5-second interval snapshot (for time-series charts)."""
-        latencies = [r.latency_ms for r in self._samples]
-        safe_duration = max(duration_seconds, 0.001)
-        total = len(self._samples)
-        error_count = sum(
-            1 for r in self._samples
-            if r.error_type is not None or not (200 <= r.status < 300)
-        )
-        self._snapshots.append(
-            StatsSnapshot(
-                timestamp=timestamp,
+    def completion_percentiles(self, t_start: float) -> tuple[float, float, float]:
+        """Return (p50, p95, p99) elapsed seconds from t_start at which requests completed.
+
+        Each value answers: "by this elapsed time, that fraction of all requests had finished."
+        Completion time per request = (timestamp - t_start) + latency_ms / 1000.
+        """
+        times = [
+            (r.timestamp - t_start) + r.latency_ms / 1000.0
+            for r in self._samples
+        ]
+        return percentile(times, 50), percentile(times, 95), percentile(times, 99)
+
+    def generate_timeseries(self, t_start: float, interval: float = 10.0) -> list[StatsSnapshot]:
+        """Bin all samples into `interval`-second buckets and return one StatsSnapshot per bucket.
+
+        Uses wall-clock timestamps stored on each RequestResult so samples from all
+        worker processes are comparable on the same timeline.
+        """
+        if not self._samples:
+            return []
+
+        buckets: dict[int, list[RequestResult]] = {}
+        for r in self._samples:
+            bucket = int((r.timestamp - t_start) / interval)
+            if bucket >= 0:
+                buckets.setdefault(bucket, []).append(r)
+
+        snapshots: list[StatsSnapshot] = []
+        running_total = 0
+        for bucket_idx in sorted(buckets.keys()):
+            results = buckets[bucket_idx]
+            running_total += len(results)
+            latencies = [r.latency_ms for r in results]
+            errors = sum(
+                1 for r in results
+                if r.error_type is not None or not (200 <= r.status < 300)
+            )
+            snapshots.append(StatsSnapshot(
+                timestamp=(bucket_idx + 1) * interval,
                 p50_ms=percentile(latencies, 50),
                 p95_ms=percentile(latencies, 95),
                 p99_ms=percentile(latencies, 99),
-                rps=self.total_count / safe_duration,
-                error_rate_pct=(error_count / total * 100.0) if total else 0.0,
-            )
-        )
+                rps=running_total / ((bucket_idx + 1) * interval),
+                error_rate_pct=(errors / len(results) * 100.0) if results else 0.0,
+                total_requests=running_total,
+            ))
+
+        return snapshots