ml-loadtest 1.0.0__py3-none-any.whl

ml_loadtest/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.0.0"

ml_loadtest/analyze.py

@@ -0,0 +1,518 @@
+import argparse
+import io
+import json
+from dataclasses import asdict, dataclass
+from datetime import datetime
+from enum import Enum
+from pathlib import Path
+
+from dacite import Config as DaciteConfig
+from dacite import from_dict
+
+from ml_loadtest import __version__ as package_version
+from ml_loadtest.locustfile import EndpointStats, InstanceInfo, Mode, Report
+
+
+@dataclass
+class PerformanceMetrics:
+    """Baseline performance metrics."""
+
+    version: str
+    timestamp: str
+    p99_ms: float
+    rps: float
+    distribution_weights: dict[str, float]
+    endpoint_stats: dict[str, EndpointStats]
+    individual_capacities: dict[str, EndpointStats]
+    instance_info: InstanceInfo | None = None
+
+    @classmethod
+    def from_reports(cls, reports: dict[str, Report]) -> "PerformanceMetrics":
+        """Create PerformanceMetrics from a dictionary of reports.
+
+        :param reports: Dictionary mapping report names to Report objects.
+        :returns: A new PerformanceMetrics instance with extracted metrics.
+        """
+        p99_ms, rps = 0.0, 0.0
+        distribution_weights = {}
+        endpoint_stats = {}
+
+        # Parse production report
+        for r in reports.values():
+            if r.mode == Mode.PRODUCTION:
+                p99_ms = r.p99_ms
+                rps = r.rps
+                distribution_weights = r.weights
+                for ep, stats in r.endpoint_stats.items():
+                    endpoint_stats[ep] = EndpointStats(
+                        p99_ms=stats.p99_ms,
+                        rps=stats.rps,
+                    )
+                break
+
+        # Parse individual reports
+        individual = {}
+        for ep, r in reports.items():
+            if r.mode == Mode.INDIVIDUAL:
+                individual[ep] = EndpointStats(
+                    p99_ms=r.p99_ms,
+                    rps=r.rps,
+                )
+
+        # Create baseline structure
+        return PerformanceMetrics(
+            version=package_version,
+            timestamp=datetime.now().isoformat(),
+            p99_ms=p99_ms,
+            rps=rps,
+            distribution_weights=distribution_weights,
+            endpoint_stats=endpoint_stats,
+            individual_capacities=individual,
+        )
+
+
+@dataclass
+class MetricComparison:
+    """Comparison between current and baseline."""
+
+    metric: str
+    baseline_value: float
+    current_value: float
+    change_pct: float
+    regression: bool
+
+    def __str__(self) -> str:
+        """Return a string representation of the comparison.
+
+        :returns: Formatted string showing the comparison with symbols and arrows.
+        """
+        symbol = "❌" if self.regression else "✅"
+        direction = "↑" if self.change_pct > 0 else "↓" if self.change_pct < 0 else "→"
+        return f"{symbol} {self.metric}: {self.baseline_value:.1f} → {self.current_value:.1f} ({direction} {abs(self.change_pct * 100):.1f}%)"  # noqa: E501
+
+
+@dataclass
+class InstanceComparison:
+    """Comparison of instance information."""
+
+    field: str
+    baseline_value: str | int | None
+    current_value: str | int | None
+
+    def __str__(self) -> str:
+        """Return a string representation of the instance comparison.
+
+        :returns: Formatted string showing the instance comparison.
+        """
+        return f"⚠️  Different {self.field}: {self.baseline_value} → {self.current_value}"
+
+
+def compute_rate_limits(
+    individual_capacities: dict[str, EndpointStats],
+    endpoint_stats: dict[str, EndpointStats],
+    safety_factor: float,
+) -> tuple[dict[str, dict[str, int | None]], int | None, int | None]:
+    """Compute rate limits from capacity data.
+
+    :param individual_capacities: Per-endpoint stats at 100% traffic.
+    :param endpoint_stats: Per-endpoint stats at production weights.
+    :param safety_factor: Safety factor multiplier (0-1).
+    :returns: Tuple of (per_endpoint_limits, total_standard, total_burst).
+        per_endpoint_limits maps endpoint to {"per_second": int|None, "burst": int|None}.
+        total_standard/total_burst are None if any endpoint has missing stats.
+    """
+    rate_limits: dict[str, dict[str, int | None]] = {}
+    has_missed_stats = False
+    for ep, stats in individual_capacities.items():
+        per_second: int | None = None
+        if ep in endpoint_stats:
+            per_second = max(1, int(endpoint_stats[ep].rps * safety_factor))
+        else:
+            has_missed_stats = True
+        burst = max(1, int(stats.rps * safety_factor))
+        rate_limits[ep] = {"per_second": per_second, "burst": burst}
+
+    total_standard: int | None = None
+    total_burst: int | None = None
+    if not has_missed_stats:
+        total_standard = sum(v["per_second"] for v in rate_limits.values() if v["per_second"] is not None)
+        total_burst = sum(v["burst"] for v in rate_limits.values() if v["burst"] is not None)
+
+    return rate_limits, total_standard, total_burst
+
+
+class LoadTestAnalyzer:
+    """Analyze load test results and manage baselines."""
+
+    def __init__(
+        self,
+        loadtest_report_file: str,
+        baseline_file: str,
+        safety_factor: float = 0.8,
+        regression_threshold: float = 0.1,
+    ) -> None:
+        """Initialize the LoadTestAnalyzer.
+
+        :param loadtest_report_file: Path to the load test report JSON file.
+        :param baseline_file: Path to the baseline JSON file.
+        :param safety_factor: Safety factor for rate limit calculations (0-1).
+        :param regression_threshold: Threshold for detecting performance regression (0-1).
+        """
+        self.loadtest_report_file = Path(loadtest_report_file)
+        self.baseline_file = Path(baseline_file)
+        self.safety_factor = safety_factor
+        self.regression_threshold = regression_threshold
+
+    def load_from_report(self) -> PerformanceMetrics:
+        """Load and parse report results.
+
+        :returns: PerformanceMetrics object containing the parsed results.
+        :raises FileNotFoundError: If the results file is not found.
+        """
+        if not self.loadtest_report_file.exists():
+            raise FileNotFoundError(f"Results file not found: {self.loadtest_report_file}")
+
+        with self.loadtest_report_file.open() as f:
+            data = json.load(f)
+
+        reports = {}
+        for name, metrics in data["metrics"].items():
+            reports[name] = from_dict(
+                data_class=Report,
+                data=metrics,
+                config=DaciteConfig(cast=[Enum]),
+            )
+        performance_metrics = PerformanceMetrics.from_reports(reports)
+
+        instance_info = from_dict(
+            data_class=InstanceInfo,
+            data=data.get("instance_info", {}),
+            config=DaciteConfig(cast=[Enum]),
+        )
+        performance_metrics.instance_info = instance_info
+
+        return performance_metrics
+
+    def save_baseline(self, metrics: PerformanceMetrics) -> None:
+        """Save baseline to file.
+
+        :param metrics: PerformanceMetrics object to save as baseline.
+        """
+        with self.baseline_file.open("w") as f:
+            json.dump(asdict(metrics), f, indent=4)
+
+    def load_baseline(self) -> PerformanceMetrics | None:
+        """Load baseline from file.
+
+        :returns: PerformanceMetrics object if baseline exists, None otherwise.
+        """
+        if not self.baseline_file.exists():
+            return None
+
+        with self.baseline_file.open() as f:
+            data = json.load(f)
+
+        return from_dict(data_class=PerformanceMetrics, data=data)
+
+    def compare_metrics(self, current: PerformanceMetrics, baseline: PerformanceMetrics) -> list[MetricComparison]:
+        """Compare current results with baseline.
+
+        :param current: Current performance metrics to compare.
+        :param baseline: Baseline performance metrics to compare against.
+        :returns: List of MetricComparison objects detailing the differences.
+        """
+        comparisons = []
+
+        # Compare global RPS
+        current_rps = current.rps
+        baseline_rps = baseline.rps
+        rps_change = ((current_rps - baseline_rps) / baseline_rps) if baseline_rps > 0 else 0
+        rps_regression = rps_change < -self.regression_threshold
+        comparisons.append(
+            MetricComparison(
+                metric="Global RPS",
+                baseline_value=baseline_rps,
+                current_value=current_rps,
+                change_pct=rps_change,
+                regression=rps_regression,
+            )
+        )
+
+        # Compare per-endpoint RPS
+        for ep, current_stats in current.endpoint_stats.items():
+            if ep == "global":
+                continue
+            if ep in baseline.endpoint_stats:
+                current_ep_rps = current_stats.rps
+                baseline_ep_rps = baseline.endpoint_stats[ep].rps
+                ep_rps_change = ((current_ep_rps - baseline_ep_rps) / baseline_ep_rps) if baseline_ep_rps > 0 else 0
+                ep_rps_regression = ep_rps_change < -self.regression_threshold
+                comparisons.append(
+                    MetricComparison(
+                        metric=f"{ep} RPS",
+                        baseline_value=baseline_ep_rps,
+                        current_value=current_ep_rps,
+                        change_pct=ep_rps_change,
+                        regression=ep_rps_regression,
+                    )
+                )
+
+        # Compare per-endpoint max RPS
+        for ep, current_stats in current.individual_capacities.items():
+            if ep in baseline.individual_capacities:
+                baseline_ep_rps = baseline.individual_capacities[ep].rps
+                current_ep_rps = current_stats.rps
+                ep_rps_change = ((current_ep_rps - baseline_ep_rps) / baseline_ep_rps) if baseline_ep_rps > 0 else 0
+                ep_rps_regression = ep_rps_change < -self.regression_threshold
+                comparisons.append(
+                    MetricComparison(
+                        metric=f"{ep} Max RPS",
+                        baseline_value=baseline_ep_rps,
+                        current_value=current_ep_rps,
+                        change_pct=ep_rps_change,
+                        regression=ep_rps_regression,
+                    )
+                )
+
+        return comparisons
+
+    @staticmethod
+    def compare_instance_info(current: PerformanceMetrics, baseline: PerformanceMetrics) -> list[InstanceComparison]:
+        """Compare instance information between current and baseline.
+
+        :param current: Current performance metrics containing instance info.
+        :param baseline: Baseline performance metrics containing instance info.
+        :returns: List of InstanceComparison objects detailing the differences.
+        """
+        comparisons = []
+
+        if current.instance_info and baseline.instance_info:
+            if current.instance_info.instance_type != baseline.instance_info.instance_type:
+                comparisons.append(
+                    InstanceComparison(
+                        field="instance type",
+                        baseline_value=baseline.instance_info.instance_type,
+                        current_value=current.instance_info.instance_type,
+                    )
+                )
+            if current.instance_info.cpu_count_physical_cores != baseline.instance_info.cpu_count_physical_cores:
+                comparisons.append(
+                    InstanceComparison(
+                        field="CPU physical cores",
+                        baseline_value=baseline.instance_info.cpu_count_physical_cores,
+                        current_value=current.instance_info.cpu_count_physical_cores,
+                    )
+                )
+            if current.instance_info.cpu_count_logical_cores != baseline.instance_info.cpu_count_logical_cores:
+                comparisons.append(
+                    InstanceComparison(
+                        field="CPU logical cores",
+                        baseline_value=baseline.instance_info.cpu_count_logical_cores,
+                        current_value=current.instance_info.cpu_count_logical_cores,
+                    )
+                )
+            if current.instance_info.cpu_model != baseline.instance_info.cpu_model:
+                comparisons.append(
+                    InstanceComparison(
+                        field="CPU model",
+                        baseline_value=baseline.instance_info.cpu_model,
+                        current_value=current.instance_info.cpu_model,
+                    )
+                )
+
+        return comparisons
+
+    def print_and_save_final_report(
+        self,
+        current: PerformanceMetrics,
+        metric_comparisons: list[MetricComparison] | None = None,
+        instance_comparisons: list[InstanceComparison] | None = None,
+        output_file: str | None = None,
+    ) -> None:
+        """Print comprehensive analysis report.
+
+        :param current: Current performance metrics to report.
+        :param metric_comparisons: Optional list of metric comparisons with baseline.
+        :param instance_comparisons: Optional list of instance comparisons with baseline.
+        :param output_file: Optional path to save the report (without extension).
+        """
+        report_buffer = io.StringIO()
+
+        def write_line(line: str) -> None:
+            """Write a line to both console and report buffer.
+
+            :param line: The line to write.
+            """
+            print(line)
+            report_buffer.write(line + "\n")
+
+        write_line("=" * 80)
+        write_line("LOAD TEST ANALYSIS REPORT")
+        write_line("=" * 80)
+
+        # Instance information
+        if current.instance_info:
+            write_line("\n🖥️  Instance Information:")
+            write_line(f"  OS: {current.instance_info.os_name} {current.instance_info.os_version}")
+            write_line(f"  Instance Type: {current.instance_info.instance_type}")
+            write_line(f"  CPU Physical Cores: {current.instance_info.cpu_count_physical_cores}")
+            write_line(f"  CPU Logical Cores: {current.instance_info.cpu_count_logical_cores}")
+            write_line(f"  CPU Model: {current.instance_info.cpu_model}")
+            write_line(f"  Memory (GB): {current.instance_info.memory_gb}")
+
+        # Current test summary
+        write_line("\n📊 Production Configuration Results:")
+        write_line(f"  P99 Latency: {current.p99_ms:.1f}ms")
+        write_line(f"  Total RPS: {current.rps:.2f}")
+
+        write_line("\n📈 Per-Endpoint Performance:")
+        for endpoint, stats in current.endpoint_stats.items():
+            write_line(f"  {endpoint}:")
+            write_line(f"    P99: {stats.p99_ms:.1f}ms")
+            write_line(f"    RPS: {stats.rps:.2f}")
+
+        # Individual endpoint capacities
+        write_line("\n💪 Individual Endpoint Capacities (100% traffic):")
+        for ep, stats in current.individual_capacities.items():
+            write_line(f"  {ep}:")
+            write_line(f"    P99: {stats.p99_ms:.1f}ms")
+            write_line(f"    Max RPS: {stats.rps:.2f}")
+
+        # Comparison with baseline
+        if metric_comparisons:
+            write_line("\n" + "=" * 80)
+            write_line("COMPARISON WITH BASELINE")
+            write_line("=" * 80)
+
+            has_regression = any(c.regression for c in metric_comparisons)
+
+            for comp in metric_comparisons:
+                write_line(f"  {comp}")
+
+            if has_regression:
+                write_line("\n⚠️  PERFORMANCE REGRESSION DETECTED")
+            else:
+                write_line("\n✅ PERFORMANCE ACCEPTABLE")
+
+        # Warnings if there are instance differences
+        if instance_comparisons:
+            write_line("\n⚠️  Instance Differences Detected:")
+            for warning in instance_comparisons:
+                write_line(f"  {warning}")
+            write_line("\n  Note: Performance comparisons may not be valid across different instance types")
+
+        # Rate limit recommendations
+        write_line("\n" + "=" * 80)
+        write_line("RECOMMENDED RATE LIMITS")
+        write_line("=" * 80)
+
+        write_line(f"\n🔒 Rate Limits (Safety Factor: {self.safety_factor * 100:.0f}%):")
+
+        rate_limits, total_standard, total_burst = compute_rate_limits(
+            current.individual_capacities, current.endpoint_stats, self.safety_factor
+        )
+
+        for ep, limits in rate_limits.items():
+            if limits["per_second"] is None:
+                write_line(f"⚠️Warning: No valid stats for endpoint {ep}, rate limits may be incomplete")
+            per_second_display: int | str = limits["per_second"] if limits["per_second"] is not None else "N/A"
+            burst_display: int | str = limits["burst"] if limits["burst"] is not None else "N/A"
+            write_line(f"  {ep}:")
+            if per_second_display != "N/A":
+                write_line(f"    Standard Limit: {per_second_display} req/s")
+            if burst_display != "N/A":
+                write_line(f"    Burst Limit:    {burst_display} req/s")
+
+        if total_standard is not None and total_burst is not None:
+            write_line("\n  GLOBAL TOTALS:")
+            write_line(f"    Standard Limit: {total_standard} req/s")
+            write_line(f"    Burst Limit:    {total_burst} req/s")
+
+        write_line("\n" + "=" * 80)
+
+        # Save test report to file if specified
+        if output_file:
+            with open(f"{output_file}.txt", "w") as f:
+                f.write(report_buffer.getvalue())
+
+
+def main() -> None:
+    """Main entry point for the load test analyzer."""
+    parser = argparse.ArgumentParser(description="Analyze load test results and manage performance baselines")
+    parser.add_argument(
+        "--input-file",
+        default="report_loadtest_results.json",
+        help="Path to experiment results JSON file",
+    )
+    parser.add_argument(
+        "--output-file",
+        default="report_analysis_results",
+        help="Path to save analysis results without extension (default: report_analysis_results)",
+    )
+    parser.add_argument(
+        "--baseline",
+        default="baseline.json",
+        help="Path to baseline JSON file",
+    )
+    parser.add_argument(
+        "--update-baseline",
+        action="store_true",
+        help="Update baseline with current results",
+    )
+    parser.add_argument(
+        "--safety-factor",
+        type=float,
+        default=0.7,
+        help="Safety factor for rate limits (default: 0.7 = 70%)",
+    )
+    parser.add_argument(
+        "--regression-threshold",
+        type=float,
+        default=0.1,
+        help="Percentage threshold for regression detection (default: 0.1 = 10%)",
+    )
+
+    args = parser.parse_args()
+
+    analyzer = LoadTestAnalyzer(
+        loadtest_report_file=args.input_file,
+        baseline_file=args.baseline,
+        safety_factor=args.safety_factor,
+        regression_threshold=args.regression_threshold,
+    )
+
+    current_metrics = analyzer.load_from_report()
+    baseline_metrics = analyzer.load_baseline()
+    metric_comparisons, instance_comparisons = None, None
+
+    if baseline_metrics is None:
+        # No baseline exists, create one
+        print("📝 No baseline found, creating from current results...")
+        analyzer.save_baseline(current_metrics)
+        print("✅ Baseline created successfully")
+
+    elif args.update_baseline:
+        # Force update baseline
+        print("🔄 Updating baseline with current results...")
+        analyzer.save_baseline(current_metrics)
+        print("✅ Baseline updated")
+
+    else:
+        # Compare with existing baseline
+        print("📊 Comparing with baseline...")
+        metric_comparisons = analyzer.compare_metrics(current_metrics, baseline_metrics)
+        instance_comparisons = analyzer.compare_instance_info(current_metrics, baseline_metrics)
+
+    # Print comprehensive report
+    analyzer.print_and_save_final_report(
+        current=current_metrics,
+        metric_comparisons=metric_comparisons,
+        instance_comparisons=instance_comparisons,
+        output_file=args.output_file,
+    )
+
+    print("\n✅ Analysis complete")
+
+
+if __name__ == "__main__":
+    main()

ml_loadtest/cli.py

@@ -0,0 +1,29 @@
+"""CLI helper to provide path to installed locustfile for direct locust usage."""
+
+import sys
+from pathlib import Path
+
+
+def get_locustfile_path() -> str:
+    """Return absolute path to installed locustfile.py.
+
+    Returns:
+        Absolute path string to the locustfile.py in the installed package.
+    """
+    package_dir = Path(__file__).parent
+    locustfile = package_dir / "locustfile.py"
+    return str(locustfile.absolute())
+
+
+def main() -> None:
+    """CLI entry point that prints locustfile path.
+
+    Usage:
+        locust -f $(ml-loadtest-file) --host http://api:8000 [any locust params]
+    """
+    print(get_locustfile_path())
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

ml_loadtest/examples/__init__.py

@@ -0,0 +1,17 @@
+"""Example TaskSet implementations for ml-loadtest demonstration and testing."""
+
+from ml_loadtest.examples.demo_tasks import (
+    EchoTaskSet,
+    HealthCheckTaskSet,
+    ImageProcessingTaskSet,
+    PredictionTaskSet,
+    StatusTaskSet,
+)
+
+__all__ = [
+    "EchoTaskSet",
+    "HealthCheckTaskSet",
+    "ImageProcessingTaskSet",
+    "PredictionTaskSet",
+    "StatusTaskSet",
+]