wisent 0.5.13__py3-none-any.whl → 0.5.15__py3-none-any.whl

wisent/tests/examples/cli/generate/test_generate_with_classifier.py

@@ -0,0 +1,146 @@
+"""
+Test for generate_with_classifier.sh example.
+
+This test validates classifier training and inference-only mode
+for real-time monitoring during generation.
+"""
+
+import subprocess
+import pytest
+import tempfile
+import os
+
+
+def test_classifier_train_only():
+    """Test training a classifier and saving it (train-only mode)."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        classifier_path = os.path.join(tmpdir, "classifier.pt")
+        output_dir = os.path.join(tmpdir, "training_logs")
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "tasks", "boolq",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--classifier-type", "logistic",
+                "--limit", "20",
+                "--train-only",
+                "--save-classifier", classifier_path,
+                "--output", output_dir,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        # Should complete without error
+        assert result.returncode == 0, f"Command failed with: {result.stderr}"
+        
+        # Classifier file should be saved
+        assert os.path.exists(classifier_path), "Classifier was not saved"
+
+
+def test_classifier_inference_only():
+    """Test using a trained classifier during generation (inference-only mode)."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        classifier_path = os.path.join(tmpdir, "classifier.pt")
+        training_output = os.path.join(tmpdir, "training")
+        inference_output = os.path.join(tmpdir, "inference")
+        
+        # First train the classifier
+        train_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "tasks", "boolq",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--classifier-type", "logistic",
+                "--limit", "20",
+                "--train-only",
+                "--save-classifier", classifier_path,
+                "--output", training_output,
+                "--device", "cpu"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert train_result.returncode == 0, f"Training failed: {train_result.stderr}"
+        assert os.path.exists(classifier_path), "Classifier was not saved"
+        
+        # Now use it for inference
+        inference_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "tasks", "boolq",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--limit", "10",
+                "--inference-only",
+                "--load-classifier", classifier_path,
+                "--output", inference_output,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        # Should complete without error
+        assert inference_result.returncode == 0, f"Inference failed: {inference_result.stderr}"
+
+
+def test_classifier_with_threshold():
+    """Test classifier with custom detection threshold."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        classifier_path = os.path.join(tmpdir, "classifier.pt")
+        training_output = os.path.join(tmpdir, "training")
+        inference_output = os.path.join(tmpdir, "inference")
+        
+        # Train classifier
+        train_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "tasks", "boolq",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--classifier-type", "logistic",
+                "--limit", "20",
+                "--train-only",
+                "--save-classifier", classifier_path,
+                "--output", training_output,
+                "--device", "cpu"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert train_result.returncode == 0
+        
+        # Use with custom threshold
+        inference_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "tasks", "boolq",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--limit", "10",
+                "--inference-only",
+                "--load-classifier", classifier_path,
+                "--detection-threshold", "0.7",
+                "--output", inference_output,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        # Should complete without error
+        assert inference_result.returncode == 0, f"Inference failed: {inference_result.stderr}"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

wisent/tests/examples/cli/generate/test_generate_with_steering.py

@@ -0,0 +1,149 @@
+"""
+Test for generate_with_steering.sh example.
+
+This test validates steering vector training and inference-only mode
+for controlling model behavior during generation.
+"""
+
+import subprocess
+import pytest
+import tempfile
+import os
+
+
+def test_steering_vector_train_only():
+    """Test training a steering vector and saving it (train-only mode)."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        vector_path = os.path.join(tmpdir, "vector.pt")
+        output_dir = os.path.join(tmpdir, "training_logs")
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "tasks", "boolq",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--steering-method", "CAA",
+                "--limit", "20",
+                "--train-only",
+                "--save-steering-vector", vector_path,
+                "--output", output_dir,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        # Should complete without error
+        assert result.returncode == 0, f"Command failed with: {result.stderr}"
+        
+        # Vector file should be saved
+        assert os.path.exists(vector_path), "Steering vector was not saved"
+
+
+def test_steering_vector_inference_only():
+    """Test using a trained steering vector during generation (inference-only mode)."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        vector_path = os.path.join(tmpdir, "vector.pt")
+        training_output = os.path.join(tmpdir, "training")
+        inference_output = os.path.join(tmpdir, "inference")
+        
+        # First train the vector
+        train_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "tasks", "boolq",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--steering-method", "CAA",
+                "--limit", "20",
+                "--train-only",
+                "--save-steering-vector", vector_path,
+                "--output", training_output,
+                "--device", "cpu"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        assert train_result.returncode == 0, f"Training failed: {train_result.stderr}"
+        assert os.path.exists(vector_path), "Vector was not saved"
+        
+        # Now use it for inference
+        inference_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "tasks", "boolq",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--steering-method", "CAA",
+                "--steering-strength", "1.5",
+                "--limit", "10",
+                "--inference-only",
+                "--load-steering-vector", vector_path,
+                "--output", inference_output,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+        
+        # Should complete without error
+        assert inference_result.returncode == 0, f"Inference failed: {inference_result.stderr}"
+
+
+def test_steering_with_caa_l2():
+    """Test steering with CAA method using strong steering strength."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        vector_path = os.path.join(tmpdir, "vector.pt")
+        training_output = os.path.join(tmpdir, "training")
+        inference_output = os.path.join(tmpdir, "inference")
+
+        # Train with CAA
+        train_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "tasks", "boolq",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--steering-method", "CAA",
+                "--limit", "20",
+                "--train-only",
+                "--save-steering-vector", vector_path,
+                "--output", training_output,
+                "--device", "cpu"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+
+        assert train_result.returncode == 0
+
+        # Use with stronger steering
+        inference_result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "tasks", "boolq",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--steering-method", "CAA",
+                "--steering-strength", "2.0",
+                "--limit", "10",
+                "--inference-only",
+                "--load-steering-vector", vector_path,
+                "--output", inference_output,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+
+        # Should complete without error
+        assert inference_result.returncode == 0, f"Inference failed: {inference_result.stderr}"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

wisent/tests/examples/cli/generate/test_only_generate.py

@@ -0,0 +1,110 @@
+"""
+Test for only_generate.sh example.
+
+This test validates that the generate-responses command works correctly
+for basic response generation without steering or classification.
+"""
+
+import subprocess
+import pytest
+import tempfile
+import os
+import json
+
+
+def test_generate_basic():
+    """Test basic response generation from a task."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        output_file = os.path.join(tmpdir, "responses.json")
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "generate-responses",
+                "meta-llama/Llama-3.2-1B-Instruct",
+                "--task", "boolq",
+                "--num-questions", "3",
+                "--max-new-tokens", "50",
+                "--temperature", "0.7",
+                "--top-p", "0.95",
+                "--output", output_file,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        # Should complete without error
+        assert result.returncode == 0, f"Command failed with: {result.stderr}"
+        
+        # Output file should exist
+        assert os.path.exists(output_file), "Output file was not created"
+        
+        # Output file should contain valid JSON
+        with open(output_file, 'r') as f:
+            data = json.load(f)
+            assert isinstance(data, (list, dict)), "Output should be JSON"
+
+
+def test_generate_deterministic():
+    """Test deterministic generation with temperature=0."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        output_file = os.path.join(tmpdir, "deterministic.json")
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "generate-responses",
+                "meta-llama/Llama-3.2-1B-Instruct",
+                "--task", "boolq",
+                "--num-questions", "2",
+                "--max-new-tokens", "30",
+                "--temperature", "0.0",
+                "--output", output_file,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        # Should complete without error
+        assert result.returncode == 0, f"Command failed with: {result.stderr}"
+        
+        # Output file should exist
+        assert os.path.exists(output_file), "Output file was not created"
+
+
+def test_generate_creative():
+    """Test creative generation with higher temperature."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        output_file = os.path.join(tmpdir, "creative.json")
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "generate-responses",
+                "meta-llama/Llama-3.2-1B-Instruct",
+                "--task", "boolq",
+                "--num-questions", "2",
+                "--max-new-tokens", "40",
+                "--temperature", "1.0",
+                "--top-p", "0.9",
+                "--output", output_file,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        # Should complete without error
+        assert result.returncode == 0, f"Command failed with: {result.stderr}"
+        
+        # Output file should exist
+        assert os.path.exists(output_file), "Output file was not created"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

wisent/tests/examples/cli/multi_steering/test_multi_steer_from_trained_vectors.py

@@ -0,0 +1,210 @@
+"""
+Test for multi_steer_from_trained_vectors.sh example.
+
+This test validates practical use cases of combining pre-trained vectors
+for different personas and scenarios (tech doc writer, teacher, etc).
+"""
+
+import subprocess
+import pytest
+import tempfile
+import os
+
+
+def create_named_vector(tmpdir, name, trait_label):
+    """Helper to create a named test vector."""
+    vector_path = os.path.join(tmpdir, f"{name}.pt")
+    
+    result = subprocess.run(
+        [
+            "python", "-m", "wisent.core.main", "generate-vector-from-task",
+            "--task", "boolq",
+            "--trait-label", trait_label,
+            "--output", vector_path,
+            "--model", "meta-llama/Llama-3.2-1B-Instruct",
+            "--num-pairs", "10",
+            "--layers", "3",
+            "--token-aggregation", "average",
+            "--method", "caa",
+            "--normalize",
+            "--device", "cpu"
+        ],
+        capture_output=True,
+        text=True,
+        timeout=300
+    )
+    
+    assert result.returncode == 0, f"Failed to create {name} vector: {result.stderr}"
+    return vector_path
+
+
+def test_technical_documentation_persona():
+    """Test combining vectors for technical documentation writer persona."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        formal_vector = create_named_vector(tmpdir, "formal", "formal_tone")
+        technical_vector = create_named_vector(tmpdir, "technical", "technical")
+        combined_path = os.path.join(tmpdir, "tech_doc.pt")
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{formal_vector}:0.5",
+                "--vector", f"{technical_vector}:0.5",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", "Explain REST APIs.",
+                "--max-new-tokens", "50",
+                "--normalize-weights",
+                "--save-combined", combined_path,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        assert os.path.exists(combined_path), "Combined vector not saved"
+
+
+def test_friendly_teacher_persona():
+    """Test combining vectors for friendly teacher persona."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        friendly_vector = create_named_vector(tmpdir, "friendly", "friendly")
+        detailed_vector = create_named_vector(tmpdir, "detailed", "detailed")
+        combined_path = os.path.join(tmpdir, "teacher.pt")
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{friendly_vector}:0.6",
+                "--vector", f"{detailed_vector}:0.4",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", "How does photosynthesis work?",
+                "--max-new-tokens", "50",
+                "--normalize-weights",
+                "--save-combined", combined_path,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        assert os.path.exists(combined_path), "Combined vector not saved"
+
+
+def test_executive_summary_persona():
+    """Test combining vectors for executive summary writer."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        concise_vector = create_named_vector(tmpdir, "concise", "concise")
+        formal_vector = create_named_vector(tmpdir, "formal", "formal")
+        combined_path = os.path.join(tmpdir, "executive.pt")
+        
+        result = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{concise_vector}:0.6",
+                "--vector", f"{formal_vector}:0.4",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", "Benefits of cloud computing.",
+                "--max-new-tokens", "50",
+                "--normalize-weights",
+                "--save-combined", combined_path,
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        assert os.path.exists(combined_path), "Combined vector not saved"
+
+
+def test_comparing_weight_ratios():
+    """Test comparing different weight ratios for the same prompt."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        vector1 = create_named_vector(tmpdir, "v1", "trait1")
+        vector2 = create_named_vector(tmpdir, "v2", "trait2")
+        
+        prompt = "Explain machine learning."
+        
+        # Configuration A: More weight on vector1
+        result_a = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{vector1}:0.7",
+                "--vector", f"{vector2}:0.3",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", prompt,
+                "--max-new-tokens", "50",
+                "--normalize-weights",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        assert result_a.returncode == 0, f"Config A failed: {result_a.stderr}"
+        
+        # Configuration B: More weight on vector2
+        result_b = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{vector1}:0.3",
+                "--vector", f"{vector2}:0.7",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", prompt,
+                "--max-new-tokens", "50",
+                "--normalize-weights",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        assert result_b.returncode == 0, f"Config B failed: {result_b.stderr}"
+        
+        # Configuration C: Balanced
+        result_c = subprocess.run(
+            [
+                "python", "-m", "wisent.core.main", "multi-steer",
+                "--vector", f"{vector1}:0.5",
+                "--vector", f"{vector2}:0.5",
+                "--model", "meta-llama/Llama-3.2-1B-Instruct",
+                "--layer", "3",
+                "--method", "CAA",
+                "--prompt", prompt,
+                "--max-new-tokens", "50",
+                "--normalize-weights",
+                "--device", "cpu",
+                "--verbose"
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180
+        )
+        
+        assert result_c.returncode == 0, f"Config C failed: {result_c.stderr}"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])