wisent 0.5.12__py3-none-any.whl → 0.5.13__py3-none-any.whl

wisent/core/parser_arguments/agent_parser.py

@@ -0,0 +1,110 @@
+"""Parser setup for the 'agent' command."""
+
+
+def setup_agent_parser(parser):
+    """Set up the agent subcommand parser."""
+    parser.add_argument("prompt", type=str, help="Prompt to send to the autonomous agent")
+    parser.add_argument("--model", type=str, default="meta-llama/Llama-3.1-8B-Instruct", help="Model to use")
+    parser.add_argument("--layer", type=int, help="Layer to use (overrides parameter file)")
+    parser.add_argument(
+        "--quality-threshold", type=float, default=0.3, help="Quality threshold for classifiers (default: 0.3)"
+    )
+    parser.add_argument(
+        "--time-budget",
+        type=float,
+        default=10.0,
+        help="Time budget in minutes for creating classifiers (default: 10.0)",
+    )
+    parser.add_argument("--max-attempts", type=int, default=3, help="Maximum improvement attempts (default: 3)")
+    parser.add_argument(
+        "--max-classifiers", type=int, default=None, help="Maximum classifiers to use (default: no limit)"
+    )
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose output")
+
+    # Steering method arguments
+    parser.add_argument(
+        "--steering-method",
+        type=str,
+        default="CAA",
+        choices=["CAA", "HPR", "DAC", "BiPO", "KSteering"],
+        help="Steering method to use (default: CAA)",
+    )
+    parser.add_argument(
+        "--steering-strength", type=float, default=1.0, help="Strength of steering vector application (default: 1.0)"
+    )
+    parser.add_argument("--steering-mode", action="store_true", help="Enable steering mode")
+
+    # Normalization parameters
+    parser.add_argument("--normalize-mode", action="store_true", help="Enable normalization of steering vectors")
+    parser.add_argument(
+        "--normalization-method",
+        type=str,
+        default="none",
+        choices=["none", "l2_unit", "l2_norm", "max_norm"],
+        help="Normalization method for steering vectors (default: none)",
+    )
+    parser.add_argument("--target-norm", type=float, default=None, help="Target norm for steering vectors")
+
+    # HPR (Householder Pseudo-Rotation) parameters
+    parser.add_argument("--hpr-beta", type=float, default=1.0, help="Beta parameter for HPR steering (default: 1.0)")
+
+    # DAC (Dynamic Activation Composition) parameters
+    parser.add_argument("--dac-dynamic-control", action="store_true", help="Enable dynamic control for DAC steering")
+    parser.add_argument(
+        "--dac-entropy-threshold", type=float, default=1.0, help="Entropy threshold for DAC steering (default: 1.0)"
+    )
+
+    # BiPO (Bi-directional Preference Optimization) parameters
+    parser.add_argument("--bipo-beta", type=float, default=0.1, help="Beta parameter for BiPO steering (default: 0.1)")
+    parser.add_argument(
+        "--bipo-learning-rate", type=float, default=5e-4, help="Learning rate for BiPO steering (default: 5e-4)"
+    )
+    parser.add_argument(
+        "--bipo-epochs", type=int, default=100, help="Number of epochs for BiPO steering (default: 100)"
+    )
+
+    # KSteering parameters
+    parser.add_argument(
+        "--ksteering-num-labels", type=int, default=6, help="Number of labels for K-steering (default: 6)"
+    )
+    parser.add_argument(
+        "--ksteering-hidden-dim", type=int, default=512, help="Hidden dimension for K-steering (default: 512)"
+    )
+    parser.add_argument(
+        "--ksteering-learning-rate", type=float, default=1e-3, help="Learning rate for K-steering (default: 1e-3)"
+    )
+    parser.add_argument(
+        "--ksteering-classifier-epochs", type=int, default=100, help="Classifier epochs for K-steering (default: 100)"
+    )
+    parser.add_argument(
+        "--ksteering-target-labels",
+        type=str,
+        default="0",
+        help="Target labels for K-steering (comma-separated, default: '0')",
+    )
+    parser.add_argument(
+        "--ksteering-avoid-labels",
+        type=str,
+        default="",
+        help="Avoid labels for K-steering (comma-separated, default: '')",
+    )
+    parser.add_argument(
+        "--ksteering-alpha", type=float, default=50.0, help="Alpha parameter for K-steering (default: 50.0)"
+    )
+
+    # Quality Control System parameters
+    parser.add_argument(
+        "--enable-quality-control",
+        action="store_true",
+        default=True,
+        help="Enable new quality control system (default: True)",
+    )
+    parser.add_argument(
+        "--max-quality-attempts",
+        type=int,
+        default=5,
+        help="Maximum attempts to achieve acceptable quality (default: 5)",
+    )
+    parser.add_argument(
+        "--show-parameter-reasoning", action="store_true", help="Display model's reasoning for parameter choices"
+    )

wisent/core/parser_arguments/configure_model_parser.py

@@ -0,0 +1,7 @@
+"""Parser setup for the 'configure-model' command."""
+
+
+def setup_configure_model_parser(parser):
+    """Set up the configure-model subcommand parser."""
+    parser.add_argument("model", type=str, help="Model name to configure")
+    parser.add_argument("--force", action="store_true", help="Force reconfiguration even if model already has a config")

wisent/core/parser_arguments/create_steering_vector_parser.py

@@ -0,0 +1,59 @@
+"""Parser for the create-steering-vector command."""
+
+import argparse
+
+
+def setup_create_steering_vector_parser(parser: argparse.ArgumentParser) -> None:
+    """
+    Set up the create-steering-vector command parser.
+
+    This command loads enriched pairs (with activations) from JSON and creates
+    steering vectors using a specified method (e.g., CAA).
+    """
+    # Input/Output
+    parser.add_argument(
+        "enriched_pairs_file",
+        type=str,
+        help="Path to JSON file containing contrastive pairs with activations"
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        required=True,
+        help="Output file path for steering vector (JSON)"
+    )
+
+    # Steering method
+    parser.add_argument(
+        "--method",
+        type=str,
+        choices=["caa"],
+        default="caa",
+        help="Steering method to use (default: caa)"
+    )
+
+    # Method parameters
+    parser.add_argument(
+        "--normalize",
+        action="store_true",
+        default=True,
+        help="L2-normalize steering vectors (default: True)"
+    )
+    parser.add_argument(
+        "--no-normalize",
+        action="store_false",
+        dest="normalize",
+        help="Do not L2-normalize steering vectors"
+    )
+
+    # Display options
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable verbose output"
+    )
+    parser.add_argument(
+        "--timing",
+        action="store_true",
+        help="Show timing information"
+    )

wisent/core/parser_arguments/evaluate_parser.py

@@ -0,0 +1,40 @@
+"""Parser setup for the 'evaluate' command."""
+
+
+def setup_evaluate_parser(parser):
+    """Set up the evaluate subcommand parser for single-prompt evaluation."""
+
+    # Required arguments
+    parser.add_argument("--vector", type=str, required=True, help="Path to steering vector file (.pt)")
+    parser.add_argument("--prompt", type=str, required=True, help="Prompt to evaluate")
+    parser.add_argument(
+        "--model", type=str, required=True, help="Model name or path (used for both generation and evaluation)"
+    )
+    parser.add_argument("--trait", type=str, required=True, help="Trait name (e.g., 'catholic', 'cynical')")
+
+    # Optional model configuration
+    parser.add_argument("--device", type=str, default=None, help="Device to run on (default: auto-detect)")
+
+    # Optional steering parameters
+    parser.add_argument(
+        "--steering-strength", type=float, default=2.0, help="Steering strength to apply (default: 2.0)"
+    )
+    parser.add_argument("--max-new-tokens", type=int, default=100, help="Maximum new tokens to generate (default: 100)")
+    parser.add_argument(
+        "--trait-description",
+        type=str,
+        default=None,
+        help="Optional description of the trait (default: use trait name)",
+    )
+
+    # Optional threshold parameters
+    parser.add_argument(
+        "--trait-threshold", type=float, default=None, help="Minimum trait quality threshold (-1 to 1 scale)"
+    )
+    parser.add_argument(
+        "--answer-threshold", type=float, default=None, help="Minimum answer quality threshold (0 to 1 scale)"
+    )
+
+    # Output options
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose output")
+    parser.add_argument("--json", action="store_true", help="Output results as JSON")

wisent/core/parser_arguments/evaluate_responses_parser.py

@@ -0,0 +1,10 @@
+"""Parser setup for the 'evaluate-responses' command."""
+
+
+def setup_evaluate_responses_parser(parser):
+    """Set up the evaluate-responses command parser."""
+    parser.add_argument("--input", type=str, required=True, help="Input JSON file with generated responses")
+    parser.add_argument("--output", type=str, required=True, help="Output JSON file for evaluation results")
+    parser.add_argument("--task", type=str, help="Task name (optional, overrides task from input JSON)")
+    parser.add_argument("--trait", type=str, help="Personality trait to evaluate (optional, for personalization tasks)")
+    parser.add_argument("--verbose", action="store_true", help="Verbose output")

wisent/core/parser_arguments/full_optimize_parser.py

@@ -0,0 +1,115 @@
+"""Parser setup for the 'full-optimize' command."""
+
+
+def setup_full_optimizer_parser(parser):
+    """Set up the full-optimize subcommand parser."""
+    parser.add_argument("model", type=str, help="Model name or path to optimize")
+
+    # Task selection - mutually exclusive options
+    task_group = parser.add_mutually_exclusive_group()
+    task_group.add_argument("--tasks", type=str, nargs="+", help="Specific tasks to optimize")
+    task_group.add_argument(
+        "--skills", type=str, nargs="+", help="Select tasks by skill categories (e.g., coding, mathematics, reasoning)"
+    )
+    task_group.add_argument(
+        "--risks",
+        type=str,
+        nargs="+",
+        help="Select tasks by risk categories (e.g., harmfulness, toxicity, hallucination)",
+    )
+
+    # General limit that applies to all optimizations unless overridden
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=100,
+        help="Sample limit for all optimizations (default: 100). Can be overridden by specific limits below",
+    )
+
+    # Specific limits (override general limit if provided)
+    parser.add_argument(
+        "--classification-limit",
+        type=int,
+        default=None,
+        help="Sample limit for classification optimization (overrides --limit)",
+    )
+    parser.add_argument(
+        "--sample-size-limit",
+        type=int,
+        default=None,
+        help="Sample limit for sample size optimization (overrides --limit)",
+    )
+    parser.add_argument(
+        "--steering-limit", type=int, default=None, help="Sample limit for steering optimization (overrides --limit)"
+    )
+
+    parser.add_argument(
+        "--sample-sizes",
+        type=int,
+        nargs="+",
+        default=[5, 10, 20, 50, 100, 200, 500],
+        help="Sample sizes to test (default: 5 10 20 50 100 200 500)",
+    )
+    parser.add_argument(
+        "--skip-classification", action="store_true", help="Skip classification optimization and use existing config"
+    )
+    parser.add_argument("--skip-sample-size", action="store_true", help="Skip sample size optimization")
+    parser.add_argument("--skip-classifier-training", action="store_true", help="Skip final classifier training step")
+    parser.add_argument("--skip-control-vectors", action="store_true", help="Skip control vector training step")
+
+    # Steering optimization options
+    parser.add_argument("--skip-steering", action="store_true", help="Skip steering optimization")
+    parser.add_argument(
+        "--steering-methods",
+        type=str,
+        nargs="+",
+        choices=["CAA", "HPR", "DAC", "BiPO", "KSteering"],
+        default=["CAA", "HPR", "DAC", "BiPO", "KSteering"],
+        help="Steering methods to test (default: all methods with parameter variations)",
+    )
+    parser.add_argument(
+        "--steering-layer-range", type=str, default=None, help="Layer range for steering optimization (e.g., '0-5')"
+    )
+    parser.add_argument(
+        "--steering-strength-range",
+        type=float,
+        nargs="+",
+        default=[0.5, 1.0, 1.5, 2.0],
+        help="Steering strengths to test (default: 0.5 1.0 1.5 2.0)",
+    )
+    # Task selection options
+    parser.add_argument(
+        "--num-tasks",
+        type=int,
+        default=None,
+        help="Number of tasks to randomly select from matched tasks (default: all)",
+    )
+    parser.add_argument(
+        "--min-quality-score",
+        type=int,
+        default=2,
+        choices=[1, 2, 3, 4, 5],
+        help="Minimum quality score for tasks (default: 2)",
+    )
+    parser.add_argument(
+        "--task-seed", type=int, default=None, help="Random seed for task selection (for reproducibility)"
+    )
+
+    parser.add_argument(
+        "--max-time-per-task", type=float, default=20.0, help="Maximum time per task in minutes (default: 20.0)"
+    )
+
+    parser.add_argument("--device", type=str, default=None, help="Device to run on")
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose output")
+    parser.add_argument("--save-plots", action="store_true", help="Save plots for both optimizations")
+
+    # Timing calibration options
+    parser.add_argument(
+        "--skip-timing-estimation", action="store_true", help="Skip timing estimation and proceed without time warnings"
+    )
+    parser.add_argument("--calibration-file", type=str, default=None, help="File to save/load calibration data")
+    parser.add_argument(
+        "--calibrate-only",
+        action="store_true",
+        help="Only run calibration and exit (saves to --calibration-file if provided)",
+    )

wisent/core/parser_arguments/generate_pairs_from_task_parser.py

@@ -0,0 +1,33 @@
+"""Parser setup for the 'generate-pairs-from-task' command."""
+
+
+def setup_generate_pairs_from_task_parser(parser):
+    """Set up the generate-pairs-from-task subcommand parser."""
+    parser.add_argument(
+        "task_name",
+        type=str,
+        help="Name of the lm-eval task (e.g., 'truthfulqa_mc1', 'hellaswag')"
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        required=True,
+        help="Output file path for the generated pairs (JSON format)"
+    )
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=None,
+        help="Maximum number of pairs to generate (default: all available)"
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Random seed for reproducibility (default: 42)"
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging"
+    )

wisent/core/parser_arguments/generate_pairs_parser.py

@@ -0,0 +1,29 @@
+"""Parser setup for the 'generate-pairs' command."""
+
+
+def setup_generate_pairs_parser(parser):
+    """Set up the generate-pairs subcommand parser."""
+    parser.add_argument(
+        "--trait", type=str, required=True, help="Natural language description of the desired trait or behavior"
+    )
+    parser.add_argument(
+        "--num-pairs", type=int, default=30, help="Number of contrastive pairs to generate (default: 30)"
+    )
+    parser.add_argument(
+        "--output", type=str, required=True, help="Output file path for the generated pairs (JSON format)"
+    )
+    parser.add_argument(
+        "--model", type=str, default="meta-llama/Llama-3.1-8B-Instruct", help="Model name or path to use for generation"
+    )
+    parser.add_argument("--device", type=str, default=None, help="Device to run on")
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose output")
+    parser.add_argument(
+        "--similarity-threshold",
+        type=float,
+        default=0.8,
+        help="Similarity threshold for deduplication (0-1, higher = more strict)",
+    )
+    parser.add_argument("--timing", action="store_true", help="Show detailed timing for each generation step")
+    parser.add_argument(
+        "--max-workers", type=int, default=4, help="Number of parallel workers for generation (default: 4)"
+    )

wisent/core/parser_arguments/generate_responses_parser.py

@@ -0,0 +1,15 @@
+"""Parser setup for the 'generate-responses' command."""
+
+
+def setup_generate_responses_parser(parser):
+    """Set up the generate-responses command parser."""
+    parser.add_argument("model", type=str, help="Model name or path")
+    parser.add_argument("--task", type=str, required=True, help="Task name (e.g., arc_easy, truthfulqa_mc1)")
+    parser.add_argument("--num-questions", type=int, default=10, help="Number of questions to generate responses for (default: 10)")
+    parser.add_argument("--max-new-tokens", type=int, default=128, help="Maximum tokens to generate (default: 128)")
+    parser.add_argument("--temperature", type=float, default=0.7, help="Temperature for generation (default: 0.7)")
+    parser.add_argument("--top-p", type=float, default=0.95, help="Top-p for nucleus sampling (default: 0.95)")
+    parser.add_argument("--device", type=str, default=None, help="Device to use (cpu, cuda, mps)")
+    parser.add_argument("--output", type=str, required=True, help="Output file path for results")
+    parser.add_argument("--use-steering", action="store_true", help="Use steering during generation")
+    parser.add_argument("--verbose", action="store_true", help="Verbose output")

wisent/core/parser_arguments/generate_vector_from_synthetic_parser.py

@@ -0,0 +1,127 @@
+"""Parser for the generate-vector-from-synthetic command."""
+
+import argparse
+
+
+def setup_generate_vector_from_synthetic_parser(parser: argparse.ArgumentParser) -> None:
+    """
+    Set up the generate-vector-from-synthetic command parser.
+    
+    This command runs the complete pipeline:
+    1. Generate synthetic contrastive pairs for a trait
+    2. Collect activations from those pairs
+    3. Create steering vectors from the activations
+    
+    All in one command.
+    """
+    # Trait to generate pairs for
+    parser.add_argument(
+        "--trait",
+        type=str,
+        required=True,
+        help="Trait to generate contrastive pairs for (e.g., 'helpfulness', 'toxicity')"
+    )
+    
+    # Output
+    parser.add_argument(
+        "--output",
+        type=str,
+        required=True,
+        help="Output file path for the final steering vector (JSON)"
+    )
+    
+    # Model configuration
+    parser.add_argument(
+        "--model",
+        type=str,
+        default="meta-llama/Llama-3.2-1B-Instruct",
+        help="HuggingFace model name or path (default: meta-llama/Llama-3.2-1B-Instruct)"
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="cpu",
+        help="Device to use (e.g., 'cpu', 'cuda', 'cuda:0')"
+    )
+    
+    # Pair generation
+    parser.add_argument(
+        "--num-pairs",
+        type=int,
+        default=20,
+        help="Number of contrastive pairs to generate (default: 20)"
+    )
+    parser.add_argument(
+        "--similarity-threshold",
+        type=float,
+        default=0.8,
+        help="Cosine similarity threshold for filtering pairs (default: 0.8)"
+    )
+    
+    # Activation collection
+    parser.add_argument(
+        "--layers",
+        type=str,
+        default=None,
+        help="Comma-separated layer indices (e.g., '8,12,16') or 'all' (default: all layers)"
+    )
+    parser.add_argument(
+        "--token-aggregation",
+        type=str,
+        choices=["average", "final", "first", "max", "continuation"],
+        default="average",
+        help="How to aggregate token activations (default: average)"
+    )
+    parser.add_argument(
+        "--prompt-strategy",
+        type=str,
+        choices=["chat_template", "direct_completion", "instruction_following", "multiple_choice", "role_playing"],
+        default="chat_template",
+        help="Prompt construction strategy (default: chat_template)"
+    )
+    
+    # Steering vector creation
+    parser.add_argument(
+        "--method",
+        type=str,
+        choices=["caa"],
+        default="caa",
+        help="Steering method to use (default: caa)"
+    )
+    parser.add_argument(
+        "--normalize",
+        action="store_true",
+        default=True,
+        help="L2-normalize steering vectors (default: True)"
+    )
+    parser.add_argument(
+        "--no-normalize",
+        action="store_false",
+        dest="normalize",
+        help="Do not L2-normalize steering vectors"
+    )
+    
+    # Intermediate file handling
+    parser.add_argument(
+        "--keep-intermediate",
+        action="store_true",
+        help="Keep intermediate files (pairs and enriched pairs)"
+    )
+    parser.add_argument(
+        "--intermediate-dir",
+        type=str,
+        default=None,
+        help="Directory for intermediate files (default: same as output)"
+    )
+    
+    # Display options
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable verbose output"
+    )
+    parser.add_argument(
+        "--timing",
+        action="store_true",
+        help="Show timing information"
+    )

wisent/core/parser_arguments/generate_vector_from_task_parser.py

@@ -0,0 +1,127 @@
+"""Parser for the generate-vector-from-task command."""
+
+import argparse
+
+
+def setup_generate_vector_from_task_parser(parser: argparse.ArgumentParser) -> None:
+    """
+    Set up the generate-vector-from-task command parser.
+    
+    This command runs the complete pipeline:
+    1. Generate contrastive pairs from an lm-eval task
+    2. Collect activations from those pairs
+    3. Create steering vectors from the activations
+    
+    All in one command.
+    """
+    # Task source
+    parser.add_argument(
+        "--task",
+        type=str,
+        required=True,
+        help="Name of the lm-eval task to use (e.g., 'mmlu', 'hellaswag')"
+    )
+    parser.add_argument(
+        "--trait-label",
+        type=str,
+        required=True,
+        help="Label for the trait being steered (e.g., 'accuracy', 'correctness')"
+    )
+    
+    # Output
+    parser.add_argument(
+        "--output",
+        type=str,
+        required=True,
+        help="Output file path for the final steering vector (JSON)"
+    )
+    
+    # Model configuration
+    parser.add_argument(
+        "--model",
+        type=str,
+        default="meta-llama/Llama-3.2-1B-Instruct",
+        help="HuggingFace model name or path (default: meta-llama/Llama-3.2-1B-Instruct)"
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="cpu",
+        help="Device to use (e.g., 'cpu', 'cuda', 'cuda:0')"
+    )
+    
+    # Pair generation
+    parser.add_argument(
+        "--num-pairs",
+        type=int,
+        default=50,
+        help="Number of contrastive pairs to generate (default: 50)"
+    )
+    
+    # Activation collection
+    parser.add_argument(
+        "--layers",
+        type=str,
+        default=None,
+        help="Comma-separated layer indices (e.g., '8,12,16') or 'all' (default: all layers)"
+    )
+    parser.add_argument(
+        "--token-aggregation",
+        type=str,
+        choices=["average", "final", "first", "max", "continuation"],
+        default="average",
+        help="How to aggregate token activations (default: average)"
+    )
+    parser.add_argument(
+        "--prompt-strategy",
+        type=str,
+        choices=["chat_template", "direct_completion", "instruction_following", "multiple_choice", "role_playing"],
+        default="chat_template",
+        help="Prompt construction strategy (default: chat_template)"
+    )
+    
+    # Steering vector creation
+    parser.add_argument(
+        "--method",
+        type=str,
+        choices=["caa"],
+        default="caa",
+        help="Steering method to use (default: caa)"
+    )
+    parser.add_argument(
+        "--normalize",
+        action="store_true",
+        default=True,
+        help="L2-normalize steering vectors (default: True)"
+    )
+    parser.add_argument(
+        "--no-normalize",
+        action="store_false",
+        dest="normalize",
+        help="Do not L2-normalize steering vectors"
+    )
+    
+    # Intermediate file handling
+    parser.add_argument(
+        "--keep-intermediate",
+        action="store_true",
+        help="Keep intermediate files (pairs and enriched pairs)"
+    )
+    parser.add_argument(
+        "--intermediate-dir",
+        type=str,
+        default=None,
+        help="Directory for intermediate files (default: same as output)"
+    )
+    
+    # Display options
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable verbose output"
+    )
+    parser.add_argument(
+        "--timing",
+        action="store_true",
+        help="Show timing information"
+    )