haoline 0.3.0__py3-none-any.whl

haoline/tests/test_compare_visualizations.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python
+# Copyright (c) 2025 HaoLine Contributors
+# SPDX-License-Identifier: MIT
+
+"""Unit tests for compare_visualizations module."""
+
+import tempfile
+from pathlib import Path
+from unittest import TestCase
+
+from ..compare_visualizations import (
+    CalibrationRecommendation,
+    TradeoffPoint,
+    analyze_tradeoffs,
+    build_enhanced_markdown,
+    compute_tradeoff_points,
+    generate_calibration_recommendations,
+    generate_memory_savings_chart,
+    generate_tradeoff_chart,
+    is_available,
+)
+
+
+def create_sample_compare_json() -> dict:
+    """Create a sample comparison JSON for testing."""
+    return {
+        "model_family_id": "resnet50_test",
+        "baseline_precision": "fp32",
+        "architecture_compatible": True,
+        "compatibility_warnings": [],
+        "variants": [
+            {
+                "precision": "fp32",
+                "quantization_scheme": "none",
+                "model_path": "resnet_fp32.onnx",
+                "size_bytes": 102400000,
+                "total_params": 25000000,
+                "total_flops": 4100000000,
+                "memory_bytes": 102400000,
+                "metrics": {
+                    "f1_macro": 0.931,
+                    "latency_ms_p50": 14.5,
+                    "throughput_qps": 680,
+                },
+                "hardware_estimates": None,
+                "deltas_vs_baseline": None,
+            },
+            {
+                "precision": "fp16",
+                "quantization_scheme": "fp16",
+                "model_path": "resnet_fp16.onnx",
+                "size_bytes": 51200000,
+                "total_params": 25000000,
+                "total_flops": 4100000000,
+                "memory_bytes": 51200000,
+                "metrics": {
+                    "f1_macro": 0.929,
+                    "latency_ms_p50": 9.1,
+                    "throughput_qps": 1080,
+                },
+                "hardware_estimates": None,
+                "deltas_vs_baseline": {
+                    "size_bytes": -51200000,
+                    "f1_macro": -0.002,
+                    "latency_ms_p50": -5.4,
+                },
+            },
+            {
+                "precision": "int8",
+                "quantization_scheme": "int8",
+                "model_path": "resnet_int8.onnx",
+                "size_bytes": 25600000,
+                "total_params": 25000000,
+                "total_flops": 4100000000,
+                "memory_bytes": 25600000,
+                "metrics": {
+                    "f1_macro": 0.915,
+                    "latency_ms_p50": 5.2,
+                    "throughput_qps": 1850,
+                },
+                "hardware_estimates": None,
+                "deltas_vs_baseline": {
+                    "size_bytes": -76800000,
+                    "f1_macro": -0.016,
+                    "latency_ms_p50": -9.3,
+                },
+            },
+        ],
+    }
+
+
+class TestIsAvailable(TestCase):
+    """Test visualization availability check."""
+
+    def test_is_available_returns_bool(self):
+        """is_available should return a boolean."""
+        result = is_available()
+        self.assertIsInstance(result, bool)
+
+
+class TestComputeTradeoffPoints(TestCase):
+    """Tests for compute_tradeoff_points function."""
+
+    def test_basic_computation(self):
+        """Should compute tradeoff points for all variants."""
+        compare_json = create_sample_compare_json()
+        points = compute_tradeoff_points(compare_json)
+
+        self.assertEqual(len(points), 3)
+        self.assertIsInstance(points[0], TradeoffPoint)
+
+    def test_baseline_speedup_is_one(self):
+        """Baseline variant should have speedup of 1.0."""
+        compare_json = create_sample_compare_json()
+        points = compute_tradeoff_points(compare_json)
+
+        # Find fp32 (baseline)
+        fp32_point = next(p for p in points if p.precision == "fp32")
+        self.assertAlmostEqual(fp32_point.speedup, 1.0, places=2)
+        self.assertAlmostEqual(fp32_point.accuracy_delta, 0.0, places=5)
+
+    def test_faster_variant_has_higher_speedup(self):
+        """Faster variant should have speedup > 1.0."""
+        compare_json = create_sample_compare_json()
+        points = compute_tradeoff_points(compare_json)
+
+        int8_point = next(p for p in points if p.precision == "int8")
+        # int8 is faster: 14.5 / 5.2 ≈ 2.79x speedup
+        self.assertGreater(int8_point.speedup, 2.0)
+
+    def test_accuracy_delta_computed(self):
+        """Should compute accuracy delta relative to baseline."""
+        compare_json = create_sample_compare_json()
+        points = compute_tradeoff_points(compare_json)
+
+        fp16_point = next(p for p in points if p.precision == "fp16")
+        # fp16: 0.929 - 0.931 = -0.002
+        self.assertAlmostEqual(fp16_point.accuracy_delta, -0.002, places=5)
+
+    def test_empty_variants(self):
+        """Should handle empty variants list."""
+        compare_json = {"variants": []}
+        points = compute_tradeoff_points(compare_json)
+        self.assertEqual(len(points), 0)
+
+
+class TestAnalyzeTradeoffs(TestCase):
+    """Tests for analyze_tradeoffs function."""
+
+    def test_returns_recommendations(self):
+        """Should return analysis with recommendations."""
+        compare_json = create_sample_compare_json()
+        analysis = analyze_tradeoffs(compare_json)
+
+        self.assertIn("recommendations", analysis)
+        self.assertIsInstance(analysis["recommendations"], list)
+
+    def test_identifies_best_variants(self):
+        """Should identify best variants for different criteria."""
+        compare_json = create_sample_compare_json()
+        analysis = analyze_tradeoffs(compare_json)
+
+        self.assertIn("best_speed", analysis)
+        self.assertIn("smallest", analysis)
+
+    def test_tradeoff_points_included(self):
+        """Should include tradeoff points in analysis."""
+        compare_json = create_sample_compare_json()
+        analysis = analyze_tradeoffs(compare_json)
+
+        self.assertIn("tradeoff_points", analysis)
+        self.assertEqual(len(analysis["tradeoff_points"]), 3)
+
+
+class TestCalibrationRecommendations(TestCase):
+    """Tests for generate_calibration_recommendations function."""
+
+    def test_returns_recommendations(self):
+        """Should return list of CalibrationRecommendation objects."""
+        compare_json = create_sample_compare_json()
+        recs = generate_calibration_recommendations(compare_json)
+
+        self.assertIsInstance(recs, list)
+        for rec in recs:
+            self.assertIsInstance(rec, CalibrationRecommendation)
+
+    def test_int8_accuracy_warning(self):
+        """Should warn about INT8 accuracy drop > 2%."""
+        # Modify to have significant INT8 accuracy drop
+        compare_json = create_sample_compare_json()
+        # INT8 has 1.6% drop, which is below threshold, so let's make it worse
+        compare_json["variants"][2]["metrics"]["f1_macro"] = 0.90  # 3.1% drop
+
+        recs = generate_calibration_recommendations(compare_json)
+
+        # Should have warning about calibration
+        warnings = [r for r in recs if r.severity == "warning"]
+        self.assertGreater(len(warnings), 0)
+
+
+class TestBuildEnhancedMarkdown(TestCase):
+    """Tests for build_enhanced_markdown function."""
+
+    def test_basic_markdown_generation(self):
+        """Should generate valid Markdown."""
+        compare_json = create_sample_compare_json()
+        md = build_enhanced_markdown(compare_json, include_charts=False)
+
+        self.assertIn("# Quantization Impact Report", md)
+        self.assertIn("resnet50_test", md)
+        self.assertIn("FP32", md)
+        self.assertIn("FP16", md)
+        self.assertIn("INT8", md)
+
+    def test_includes_tradeoff_analysis(self):
+        """Should include trade-off analysis section."""
+        compare_json = create_sample_compare_json()
+        md = build_enhanced_markdown(compare_json, include_charts=False)
+
+        self.assertIn("## Trade-off Analysis", md)
+        self.assertIn("### Recommendations", md)
+
+    def test_includes_variant_table(self):
+        """Should include variant comparison table."""
+        compare_json = create_sample_compare_json()
+        md = build_enhanced_markdown(compare_json, include_charts=False)
+
+        self.assertIn("## Variant Comparison", md)
+        self.assertIn("| Precision |", md)
+
+    def test_compatibility_warnings_shown(self):
+        """Should show compatibility warnings if present."""
+        compare_json = create_sample_compare_json()
+        compare_json["architecture_compatible"] = False
+        compare_json["compatibility_warnings"] = ["Test warning"]
+
+        md = build_enhanced_markdown(compare_json, include_charts=False)
+        self.assertIn("⚠️ Compatibility Warnings", md)
+        self.assertIn("Test warning", md)
+
+
+class TestChartGeneration(TestCase):
+    """Tests for chart generation functions."""
+
+    def test_tradeoff_chart_no_matplotlib(self):
+        """Should return None if matplotlib not available."""
+        # This test will pass if matplotlib IS available, as it will
+        # generate the chart. We just verify it doesn't crash.
+        compare_json = create_sample_compare_json()
+        points = compute_tradeoff_points(compare_json)
+
+        result = generate_tradeoff_chart(points)
+        # Result is either bytes (matplotlib available) or None
+        if is_available():
+            self.assertIsInstance(result, bytes)
+            self.assertGreater(len(result), 0)
+        else:
+            self.assertIsNone(result)
+
+    def test_tradeoff_chart_to_file(self):
+        """Should save chart to file if path provided."""
+        if not is_available():
+            self.skipTest("matplotlib not available")
+
+        compare_json = create_sample_compare_json()
+        points = compute_tradeoff_points(compare_json)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output_path = Path(tmpdir) / "tradeoff.png"
+            result = generate_tradeoff_chart(points, output_path)
+
+            self.assertIsNotNone(result)
+            self.assertTrue(output_path.exists())
+            self.assertGreater(output_path.stat().st_size, 0)
+
+    def test_memory_savings_chart(self):
+        """Should generate memory savings chart."""
+        if not is_available():
+            self.skipTest("matplotlib not available")
+
+        compare_json = create_sample_compare_json()
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output_path = Path(tmpdir) / "memory.png"
+            result = generate_memory_savings_chart(compare_json, output_path)
+
+            self.assertIsNotNone(result)
+            self.assertTrue(output_path.exists())
+
+    def test_empty_points_returns_none(self):
+        """Should return None for empty points list."""
+        result = generate_tradeoff_chart([])
+        self.assertIsNone(result)

haoline/tests/test_edge_analysis.py

@@ -0,0 +1,243 @@
+# Copyright (c) 2025 HaoLine Contributors
+# SPDX-License-Identifier: MIT
+
+"""Tests for edge-centric analysis."""
+
+from __future__ import annotations
+
+import tempfile
+from pathlib import Path
+
+import numpy as np
+import onnx
+import pytest
+from onnx import TensorProto, helper
+
+from ..analyzer import ONNXGraphLoader
+from ..edge_analysis import (
+    PRECISION_EDGE_COLORS,
+    EdgeAnalyzer,
+    EdgeInfo,
+    compute_edge_thickness,
+    format_tensor_shape,
+    format_tensor_size,
+    generate_edge_tooltip,
+    get_edge_color,
+)
+
+
+def create_simple_model() -> onnx.ModelProto:
+    """Create a simple model for edge testing."""
+    X = helper.make_tensor_value_info("X", TensorProto.FLOAT, [1, 3, 224, 224])
+    Y = helper.make_tensor_value_info("Y", TensorProto.FLOAT, [1, 64, 112, 112])
+
+    weight = helper.make_tensor(
+        "conv_weight",
+        TensorProto.FLOAT,
+        [64, 3, 7, 7],
+        np.random.randn(64, 3, 7, 7).astype(np.float32).flatten().tolist(),
+    )
+
+    conv = helper.make_node(
+        "Conv",
+        ["X", "conv_weight"],
+        ["conv_out"],
+        kernel_shape=[7, 7],
+        strides=[2, 2],
+        pads=[3, 3, 3, 3],
+    )
+    relu = helper.make_node("Relu", ["conv_out"], ["Y"])
+
+    graph = helper.make_graph([conv, relu], "simple", [X], [Y], [weight])
+    model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)])
+    return model
+
+
+def create_residual_model() -> onnx.ModelProto:
+    """Create a model with skip connection."""
+    X = helper.make_tensor_value_info("X", TensorProto.FLOAT, [1, 64, 56, 56])
+    Y = helper.make_tensor_value_info("Y", TensorProto.FLOAT, [1, 64, 56, 56])
+
+    weight1 = helper.make_tensor(
+        "w1",
+        TensorProto.FLOAT,
+        [64, 64, 3, 3],
+        np.random.randn(64, 64, 3, 3).astype(np.float32).flatten().tolist(),
+    )
+    weight2 = helper.make_tensor(
+        "w2",
+        TensorProto.FLOAT,
+        [64, 64, 3, 3],
+        np.random.randn(64, 64, 3, 3).astype(np.float32).flatten().tolist(),
+    )
+
+    # Conv -> BN -> ReLU -> Conv -> BN -> Add(skip) -> ReLU
+    nodes = [
+        helper.make_node("Conv", ["X", "w1"], ["c1"], kernel_shape=[3, 3], pads=[1, 1, 1, 1]),
+        helper.make_node("Relu", ["c1"], ["r1"]),
+        helper.make_node("Conv", ["r1", "w2"], ["c2"], kernel_shape=[3, 3], pads=[1, 1, 1, 1]),
+        helper.make_node("Add", ["X", "c2"], ["add_out"]),  # Skip connection
+        helper.make_node("Relu", ["add_out"], ["Y"]),
+    ]
+
+    graph = helper.make_graph(nodes, "residual", [X], [Y], [weight1, weight2])
+    model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)])
+    return model
+
+
+class TestEdgeAnalysis:
+    """Tests for EdgeAnalyzer."""
+
+    def test_extract_edges(self):
+        """Test edge extraction from model."""
+        model = create_simple_model()
+
+        with tempfile.NamedTemporaryFile(suffix=".onnx", delete=False) as f:
+            onnx.save(model, f.name)
+            model_path = Path(f.name)
+
+        try:
+            loader = ONNXGraphLoader()
+            _, graph_info = loader.load(model_path)
+
+            analyzer = EdgeAnalyzer()
+            result = analyzer.analyze(graph_info)
+
+            # Should have edges for: X, conv_weight, conv_out, Y
+            assert len(result.edges) >= 3
+
+            # Find the weight edge
+            weight_edges = [e for e in result.edges if e.is_weight]
+            assert len(weight_edges) >= 1
+        finally:
+            model_path.unlink()
+
+    def test_skip_connection_detection(self):
+        """Test skip connection detection."""
+        model = create_residual_model()
+
+        with tempfile.NamedTemporaryFile(suffix=".onnx", delete=False) as f:
+            onnx.save(model, f.name)
+            model_path = Path(f.name)
+
+        try:
+            loader = ONNXGraphLoader()
+            _, graph_info = loader.load(model_path)
+
+            analyzer = EdgeAnalyzer()
+            result = analyzer.analyze(graph_info)
+
+            # Should detect skip connection for X -> Add
+            [e for e in result.edges if e.is_skip_connection]
+            # May or may not detect depending on topological distance
+            # At minimum the analysis should run without error
+            assert result is not None
+        finally:
+            model_path.unlink()
+
+    def test_memory_profile(self):
+        """Test memory profile calculation."""
+        model = create_simple_model()
+
+        with tempfile.NamedTemporaryFile(suffix=".onnx", delete=False) as f:
+            onnx.save(model, f.name)
+            model_path = Path(f.name)
+
+        try:
+            loader = ONNXGraphLoader()
+            _, graph_info = loader.load(model_path)
+
+            analyzer = EdgeAnalyzer()
+            result = analyzer.analyze(graph_info)
+
+            # Memory profile should have entries
+            assert len(result.memory_profile) > 0
+
+            # Peak should be non-zero
+            assert result.peak_activation_bytes >= 0
+        finally:
+            model_path.unlink()
+
+
+class TestEdgeVisualization:
+    """Tests for edge visualization helpers."""
+
+    def test_edge_thickness_scaling(self):
+        """Test edge thickness computation."""
+        # Very small tensor
+        assert compute_edge_thickness(100) == 1  # Below min threshold
+
+        # 1KB
+        kb = compute_edge_thickness(1024)
+        assert 1 <= kb <= 10
+
+        # 1MB - should be thicker
+        mb = compute_edge_thickness(1024 * 1024)
+        assert mb > kb
+
+        # 1GB - should be even thicker
+        gb = compute_edge_thickness(1024 * 1024 * 1024)
+        assert gb > mb
+        assert gb <= 10
+
+    def test_format_tensor_shape(self):
+        """Test tensor shape formatting."""
+        assert format_tensor_shape([]) == "[]"
+        assert format_tensor_shape([1, 3, 224, 224]) == "[1, 3, 224, 224]"
+        assert format_tensor_shape([1, "batch", 768]) == "[1, batch, 768]"
+
+    def test_format_tensor_size(self):
+        """Test tensor size formatting."""
+        assert format_tensor_size(100) == "100 B"
+        assert format_tensor_size(1024) == "1.0 KB"
+        assert format_tensor_size(1024 * 1024) == "1.0 MB"
+        assert format_tensor_size(1024 * 1024 * 1024) == "1.00 GB"
+
+    def test_edge_colors(self):
+        """Test edge color assignment."""
+        # Normal edge
+        edge = EdgeInfo(
+            tensor_name="test",
+            source_node="node1",
+            target_nodes=["node2"],
+            shape=[1, 64, 56, 56],
+            dtype="float32",
+            size_bytes=1024 * 1024,
+            is_weight=False,
+            precision="fp32",
+        )
+        assert get_edge_color(edge) == PRECISION_EDGE_COLORS["fp32"]
+
+        # Bottleneck edge
+        edge.is_bottleneck = True
+        assert get_edge_color(edge) == "#E74C3C"  # Red
+
+        # Attention edge (without bottleneck)
+        edge.is_bottleneck = False
+        edge.is_attention_qk = True
+        assert get_edge_color(edge) == "#E67E22"  # Orange
+
+    def test_edge_tooltip(self):
+        """Test tooltip generation."""
+        edge = EdgeInfo(
+            tensor_name="attention_scores",
+            source_node="matmul_qk",
+            target_nodes=["softmax"],
+            shape=[1, 12, 512, 512],
+            dtype="float32",
+            size_bytes=12 * 512 * 512 * 4,
+            is_weight=False,
+            precision="fp32",
+            is_attention_qk=True,
+        )
+
+        tooltip = generate_edge_tooltip(edge)
+
+        assert "attention_scores" in tooltip
+        assert "[1, 12, 512, 512]" in tooltip
+        assert "fp32" in tooltip
+        assert "O(seq" in tooltip  # O(seq^2) warning
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])