wisent 0.5.13__py3-none-any.whl → 0.5.15__py3-none-any.whl

wisent/tests/examples/cli/multi_steering/test_multi_steer_with_different_parameters.py

@@ -0,0 +1,205 @@
+"""
+Test for multi_steer_with_different_parameters.sh example.
+
+This test validates different parameter combinations for multi-steering:
+normalized vs unnormalized weights, target norm scaling, and different methods.
+"""
+
+import subprocess
+import pytest
+import tempfile
+import os
+
+
+def setup_test_vectors(tmpdir):
+    """Helper function to create test vectors for multi-steering tests."""
+    vector1_path = os.path.join(tmpdir, "vector1.pt")
+    vector2_path = os.path.join(tmpdir, "vector2.pt")
+    
+    # Train first vector
+    result1 = subprocess.run(
+        [
+            "python", "-m", "wisent.core.main", "generate-vector-from-task",
+            "--task", "boolq",
+            "--trait-label", "test1",
+            "--output", vector1_path,
+            "--model", "meta-llama/Llama-3.2-1B-Instruct",
+            "--num-pairs", "10",
+            "--layers", "3",
+            "--token-aggregation", "average",
+            "--method", "caa",
+            "--normalize",
+            "--device", "cpu"
+        ],
+        capture_output=True,
+        text=True,
+        timeout=300
+    )
+    
+    assert result1.returncode == 0, f"Failed to create vector1: {result1.stderr}"
+    
+    # Train second vector
+    result2 = subprocess.run(
+        [
+            "python", "-m", "wisent.core.main", "generate-vector-from-task",
+            "--task", "boolq",
+            "--trait-label", "test2",
+            "--output", vector2_path,
+            "--model", "meta-llama/Llama-3.2-1B-Instruct",
+            "--num-pairs", "10",
+            "--layers", "3",
+            "--token-aggregation", "average",
+            "--method", "caa",
+            "--normalize",
+            "--device", "cpu"
+        ],
+        capture_output=True,
+        text=True,
+        timeout=300
+    )
+    
+    assert result2.returncode == 0, f"Failed to create vector2: {result2.stderr}"
+    
+    return vector1_path, vector2_path
+
+
+def test_normalized_weights():
+    """Test multi-steer with normalized weights (sum to 1.0)."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        vector1_path, vector2_path = setup_test_vectors(tmpdir)
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{vector1_path}:0.6",
+                "--vector", f"{vector2_path}:0.4",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", "Explain AI.",
+                "--max-new-tokens", "50",
+                "--normalize-weights",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+
+
+def test_unnormalized_weights():
+    """Test multi-steer with unnormalized weights for stronger effect."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        vector1_path, vector2_path = setup_test_vectors(tmpdir)
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{vector1_path}:2.0",
+                "--vector", f"{vector2_path}:1.5",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", "Explain AI.",
+                "--max-new-tokens", "50",
+                "--allow-unnormalized",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+
+
+def test_target_norm_scaling():
+    """Test multi-steer with target norm scaling."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        vector1_path, vector2_path = setup_test_vectors(tmpdir)
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{vector1_path}:0.5",
+                "--vector", f"{vector2_path}:0.5",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", "Explain AI.",
+                "--max-new-tokens", "50",
+                "--target-norm", "10.0",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+
+
+def test_subtle_steering_with_small_norm():
+    """Test subtle steering with small target norm."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        vector1_path, vector2_path = setup_test_vectors(tmpdir)
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{vector1_path}:0.5",
+                "--vector", f"{vector2_path}:0.5",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", "Explain AI.",
+                "--max-new-tokens", "50",
+                "--target-norm", "2.0",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+
+
+def test_save_combined_vector():
+    """Test saving combined vector for reuse."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        vector1_path, vector2_path = setup_test_vectors(tmpdir)
+        combined_path = os.path.join(tmpdir, "combined.pt")
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{vector1_path}:0.5",
+                "--vector", f"{vector2_path}:0.5",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", "Explain AI.",
+                "--max-new-tokens", "50",
+                "--normalize-weights",
+                "--save-combined", combined_path,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        assert os.path.exists(combined_path), "Combined vector was not saved"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

wisent/tests/examples/cli/multi_steering/test_train_and_multi_steer.py

@@ -0,0 +1,174 @@
+"""
+Test for train_and_multi_steer.sh example.
+
+This test validates training multiple steering vectors and combining them
+with different weights during inference using the multi-steer command.
+"""
+
+import subprocess
+import pytest
+import tempfile
+import os
+
+
+def test_train_multiple_vectors_and_combine():
+    """Test training multiple vectors and combining them with multi-steer."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        vector1_path = os.path.join(tmpdir, "vector1.pt")
+        vector2_path = os.path.join(tmpdir, "vector2.pt")
+        combined_path = os.path.join(tmpdir, "combined.pt")
+        
+        # Train first vector
+        train1_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "generate-vector-from-task",
+                "--task", "boolq",
+                "--trait-label", "truthfulness",
+                "--output", vector1_path,
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--num-pairs", "10",
+                "--layers", "3",
+                "--token-aggregation", "average",
+                "--method", "caa",
+                "--normalize",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert train1_result.returncode == 0, f"Training vector1 failed: {train1_result.stderr}"
+        assert os.path.exists(vector1_path), "Vector1 was not saved"
+        
+        # Train second vector
+        train2_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "generate-vector-from-task",
+                "--task", "boolq",
+                "--trait-label", "helpfulness",
+                "--output", vector2_path,
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--num-pairs", "10",
+                "--layers", "3",
+                "--token-aggregation", "average",
+                "--method", "caa",
+                "--normalize",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert train2_result.returncode == 0, f"Training vector2 failed: {train2_result.stderr}"
+        assert os.path.exists(vector2_path), "Vector2 was not saved"
+        
+        # Combine vectors with equal weights
+        combine_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{vector1_path}:0.5",
+                "--vector", f"{vector2_path}:0.5",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", "What is AI?",
+                "--max-new-tokens", "50",
+                "--normalize-weights",
+                "--save-combined", combined_path,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        # Should complete without error
+        assert combine_result.returncode == 0, f"Combining failed: {combine_result.stderr}"
+        
+        # Combined vector should be saved
+        assert os.path.exists(combined_path), "Combined vector was not saved"
+        
+        # Output should contain generated text
+        assert len(combine_result.stdout) > 0, "No output generated"
+
+
+def test_multi_steer_with_different_emphasis():
+    """Test combining vectors with different weight emphasis."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        vector1_path = os.path.join(tmpdir, "vector1.pt")
+        vector2_path = os.path.join(tmpdir, "vector2.pt")
+        
+        # Train vectors (abbreviated for test)
+        train1_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "generate-vector-from-task",
+                "--task", "boolq",
+                "--trait-label", "trait1",
+                "--output", vector1_path,
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--num-pairs", "10",
+                "--layers", "3",
+                "--token-aggregation", "average",
+                "--method", "caa",
+                "--normalize",
+                "--device", "cpu"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert train1_result.returncode == 0
+        
+        train2_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "generate-vector-from-task",
+                "--task", "boolq",
+                "--trait-label", "trait2",
+                "--output", vector2_path,
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--num-pairs", "10",
+                "--layers", "3",
+                "--token-aggregation", "average",
+                "--method", "caa",
+                "--normalize",
+                "--device", "cpu"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert train2_result.returncode == 0
+        
+        # Emphasize first vector (70% vs 30%)
+        combine_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{vector1_path}:0.7",
+                "--vector", f"{vector2_path}:0.3",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", "Explain quantum computing.",
+                "--max-new-tokens", "50",
+                "--normalize-weights",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        # Should complete without error
+        assert combine_result.returncode == 0, f"Combining failed: {combine_result.stderr}"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

wisent/tests/examples/cli/optimizer/test_optimize_sample_size.py

@@ -0,0 +1,102 @@
+"""
+Test for optimize_sample_size.sh example.
+
+This test validates that the optimize-sample-size command works correctly
+with minimal parameters to verify the example script would work.
+"""
+
+import subprocess
+import pytest
+import tempfile
+import os
+
+
+def test_optimize_sample_size_basic():
+    """Test basic sample size optimization command."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "optimize-sample-size",
+                "meta-llama/Llama-3.2-1B-Instruct",  # Small model for testing
+                "--task", "boolq",
+                "--layer", "3",
+                "--token-aggregation", "average",
+                "--sample-sizes", "5", "10",
+                "--test-size", "20",
+                "--limit", "30",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        # Should complete without error
+        assert result.returncode == 0, f"Command failed with: {result.stderr}"
+        
+        # Should mention sample sizes in output
+        assert "5" in result.stdout or "5" in result.stderr
+        assert "10" in result.stdout or "10" in result.stderr
+
+
+def test_optimize_sample_size_with_steering():
+    """Test sample size optimization in steering mode."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "optimize-sample-size",
+                "meta-llama/Llama-3.2-1B-Instruct",
+                "--task", "boolq",
+                "--layer", "3",
+                "--token-aggregation", "final",
+                "--steering-mode",
+                "--steering-method", "CAA",
+                "--steering-strength", "1.0",
+                "--sample-sizes", "5", "10",
+                "--test-size", "15",
+                "--limit", "25",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        # Should complete without error
+        assert result.returncode == 0, f"Command failed with: {result.stderr}"
+        
+        # Should mention steering in output
+        assert "steering" in result.stdout.lower() or "steering" in result.stderr.lower()
+
+
+def test_optimize_sample_size_custom_threshold():
+    """Test sample size optimization with custom threshold."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "optimize-sample-size",
+                "meta-llama/Llama-3.2-1B-Instruct",
+                "--task", "boolq",
+                "--layer", "3",
+                "--token-aggregation", "max",
+                "--threshold", "0.6",
+                "--sample-sizes", "5", "10",
+                "--test-size", "15",
+                "--limit", "25",
+                "--device", "cpu",
+                "--seed", "123",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        # Should complete without error
+        assert result.returncode == 0, f"Command failed with: {result.stderr}"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

wisent/tests/examples/cli/optimizer/test_optimizer_examples.py

@@ -0,0 +1,59 @@
+"""
+Tests for optimizer examples (classification and steering).
+
+Validates parameter optimization commands.
+"""
+
+import subprocess
+import pytest
+import tempfile
+import os
+
+
+def test_optimize_classification_parameters():
+    """Test optimizing classification parameters."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "optimize-classification",
+                "meta-llama/Llama-3.2-1B-Instruct",
+                "--limit", "10",
+                "--optimization-metric", "f1",
+                "--max-time-per-task", "5.0",
+                "--layer-range", "2-4",
+                "--aggregation-methods", "average", "final",
+                "--threshold-range", "0.5", "0.6",
+                "--device", "cpu"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+
+
+def test_optimize_steering_parameters():
+    """Test optimizing steering parameters."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "optimize-steering", "comprehensive",
+                "meta-llama/Llama-3.2-1B-Instruct",
+                "--tasks", "boolq",
+                "--methods", "CAA",
+                "--limit", "10",
+                "--max-time-per-task", "5.0",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

wisent/tests/examples/cli/steering/test_create_steering_vectors.py

@@ -0,0 +1,135 @@
+"""
+Tests for steering vector creation examples.
+
+Validates creating steering vectors from tasks, synthetic pairs, and activations.
+"""
+
+import subprocess
+import pytest
+import tempfile
+import os
+import json
+
+
+def test_create_vector_from_task():
+    """Test creating steering vector from lm-eval task."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        output_file = os.path.join(tmpdir, "vector.pt")
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "generate-vector-from-task",
+                "--task", "boolq",
+                "--trait-label", "correctness",
+                "--output", output_file,
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--num-pairs", "10",
+                "--layers", "3",
+                "--token-aggregation", "average",
+                "--method", "caa",
+                "--normalize",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        assert os.path.exists(output_file), "Vector file not created"
+
+
+def test_create_vector_from_synthetic():
+    """Test creating steering vector from synthetic pairs."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        output_file = os.path.join(tmpdir, "synthetic_vector.pt")
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "generate-vector-from-synthetic",
+                "--trait", "helpfulness",
+                "--output", output_file,
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--num-pairs", "5",
+                "--layers", "3",
+                "--token-aggregation", "average",
+                "--method", "caa",
+                "--normalize",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        assert os.path.exists(output_file), "Vector file not created"
+
+
+def test_create_vector_from_activations():
+    """Test creating steering vector from enriched pairs."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # First create pairs with activations
+        pairs_file = os.path.join(tmpdir, "pairs.json")
+        enriched_file = os.path.join(tmpdir, "enriched_pairs.json")
+        vector_file = os.path.join(tmpdir, "vector.pt")
+        
+        # Create test pairs
+        pairs = [
+            {
+                "prompt": "How should I respond?",
+                "positive_response": {"model_response": "Good answer."},
+                "negative_response": {"model_response": "Bad answer."},
+                "label": "quality"
+            },
+            {
+                "prompt": "What information is best?",
+                "positive_response": {"model_response": "Helpful info."},
+                "negative_response": {"model_response": "Unhelpful info."},
+                "label": "quality"
+            }
+        ]
+        with open(pairs_file, 'w') as f:
+            json.dump(pairs, f)
+        
+        # Enrich with activations
+        enrich_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "get-activations",
+                pairs_file,
+                "--output", enriched_file,
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layers", "3",
+                "--token-aggregation", "average",
+                "--device", "cpu"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert enrich_result.returncode == 0
+        
+        # Create steering vector from enriched pairs
+        vector_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "create-steering-vector",
+                enriched_file,
+                "--output", vector_file,
+                "--method", "caa",
+                "--normalize",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert vector_result.returncode == 0, f"Vector creation failed: {vector_result.stderr}"
+        assert os.path.exists(vector_file), "Vector file not created"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])