wisent 0.5.11__py3-none-any.whl → 0.5.13__py3-none-any.whl

wisent/core/benchmark_extractors.py

@@ -0,0 +1,293 @@
+"""
+Answer extractors for TaskInterface benchmarks.
+
+These extractors parse model outputs to extract answers for validation.
+Different from LMEvalBenchmarkExtractor which creates contrastive pairs.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Optional
+import re
+
+
+class BenchmarkExtractor(ABC):
+    """Base class for benchmark answer extraction."""
+
+    @abstractmethod
+    def extract_answer(self, text: str) -> Optional[str]:
+        """Extract answer from model's generated text."""
+        pass
+
+    def normalize_answer(self, answer: str) -> str:
+        """Normalize answer for comparison."""
+        if answer is None:
+            return ""
+        return answer.lower().strip()
+
+    def check_answer(self, predicted: str, expected: str) -> bool:
+        """Check if predicted answer matches expected."""
+        return self.normalize_answer(predicted) == self.normalize_answer(expected)
+
+
+class GSM8KExtractor(BenchmarkExtractor):
+    """Extractor for GSM8K and math tasks."""
+
+    def extract_answer(self, text: str) -> Optional[str]:
+        """Extract numerical answer from text."""
+        if not text:
+            return None
+
+        # Strategy 1: JSON format {"final_answer": "123"}
+        try:
+            import json
+            if "{" in text and "}" in text:
+                # Find JSON-like structures
+                json_match = re.search(r'\{[^}]*"final_answer"[^}]*\}', text)
+                if json_match:
+                    data = json.loads(json_match.group(0))
+                    answer = data.get("final_answer")
+                    if answer:
+                        # Remove commas and non-numeric characters except decimal and minus
+                        answer = re.sub(r'[^\d.\-]', '', str(answer))
+                        if answer and answer.replace('.', '').replace('-', '').isdigit():
+                            return answer
+        except Exception:
+            pass
+
+        # Strategy 2: GSM8K format "#### 123"
+        hash_match = re.search(r'####\s*([\d,.\-]+)', text)
+        if hash_match:
+            answer = hash_match.group(1).replace(',', '')
+            return answer
+
+        # Strategy 3: "The answer is 123" or "The final answer is 123"
+        answer_patterns = [
+            r'(?:the\s+)?(?:final\s+)?answer\s+is\s*:?\s*([\d,.\-]+)',
+            r'(?:therefore|thus|so),?\s+(?:the\s+)?(?:final\s+)?answer\s+is\s*:?\s*([\d,.\-]+)',
+            r'=\s*([\d,.\-]+)\s*$',
+        ]
+
+        for pattern in answer_patterns:
+            match = re.search(pattern, text, re.IGNORECASE)
+            if match:
+                answer = match.group(1).replace(',', '')
+                return answer
+
+        # Strategy 4: Last number in text (fallback)
+        numbers = re.findall(r'-?\d+(?:\.\d+)?', text)
+        if numbers:
+            return numbers[-1]
+
+        return None
+
+    def check_answer(self, predicted: str, expected: str) -> bool:
+        """Compare numerical answers with tolerance."""
+        if predicted is None:
+            return False
+        try:
+            pred_float = float(predicted)
+            expected_float = float(expected)
+            return abs(pred_float - expected_float) < 1e-6
+        except (ValueError, TypeError):
+            return self.normalize_answer(predicted) == self.normalize_answer(expected)
+
+
+class LiveCodeBenchExtractor(BenchmarkExtractor):
+    """Extractor for coding tasks."""
+
+    def extract_answer(self, text: str) -> Optional[str]:
+        """Extract code from model response."""
+        if not text:
+            return None
+
+        # Strategy 1: Extract from markdown code blocks
+        code_block_patterns = [
+            r'```python\s*(.*?)```',
+            r'```\s*(.*?)```',
+            r'`([^`]+)`',
+        ]
+
+        for pattern in code_block_patterns:
+            matches = re.findall(pattern, text, re.DOTALL | re.IGNORECASE)
+            if matches:
+                # Return the longest code block found
+                code = max(matches, key=len)
+                return code.strip()
+
+        # Strategy 2: Look for function definitions
+        func_match = re.search(r'(def\s+\w+.*?)(?:\n\n|\Z)', text, re.DOTALL)
+        if func_match:
+            return func_match.group(1).strip()
+
+        # Strategy 3: Look for class definitions
+        class_match = re.search(r'(class\s+\w+.*?)(?:\n\n|\Z)', text, re.DOTALL)
+        if class_match:
+            return class_match.group(1).strip()
+
+        # Strategy 4: Return entire text if it looks like code
+        if 'def ' in text or 'class ' in text or 'import ' in text:
+            return text.strip()
+
+        return None
+
+    def check_answer(self, predicted: str, expected: str) -> bool:
+        """For coding tasks, basic check: code is not empty."""
+        if predicted is None:
+            return False
+        # Basic check: code is not empty
+        return len(predicted.strip()) > 0
+
+
+class HLEExtractor(BenchmarkExtractor):
+    """Extractor for HLE tasks."""
+
+    def extract_answer(self, text: str) -> Optional[str]:
+        """Extract answer from HLE response."""
+        if not text:
+            return None
+
+        # Strategy 1: JSON format {"answer": "X"}
+        try:
+            import json
+            if "{" in text and "}" in text:
+                json_match = re.search(r'\{[^}]*"answer"[^}]*\}', text)
+                if json_match:
+                    data = json.loads(json_match.group(0))
+                    return str(data.get("answer", ""))
+        except Exception:
+            pass
+
+        # Strategy 2: "Answer: X" format
+        answer_match = re.search(r'answer\s*:\s*(.+?)(?:\n|$)', text, re.IGNORECASE)
+        if answer_match:
+            return answer_match.group(1).strip()
+
+        # Strategy 3: Multiple choice (A, B, C, D)
+        mc_match = re.search(r'\b([A-D])\b', text)
+        if mc_match:
+            return mc_match.group(1)
+
+        # Strategy 4: First line (fallback)
+        first_line = text.split('\n')[0].strip()
+        if first_line:
+            return first_line
+
+        return None
+
+    def check_answer(self, predicted: str, expected: str) -> bool:
+        """Compare answers with case-insensitive substring matching."""
+        if predicted is None:
+            return False
+        pred_norm = self.normalize_answer(predicted)
+        exp_norm = self.normalize_answer(expected)
+        return exp_norm in pred_norm or pred_norm in exp_norm
+
+
+class SuperGPQAExtractor(BenchmarkExtractor):
+    """Extractor for SuperGPQA science tasks."""
+
+    def extract_answer(self, text: str) -> Optional[str]:
+        """Extract answer from SuperGPQA response."""
+        if not text:
+            return None
+
+        # Strategy 1: "Answer: A" format
+        answer_match = re.search(r'answer\s*:\s*([A-D])', text, re.IGNORECASE)
+        if answer_match:
+            return answer_match.group(1).upper()
+
+        # Strategy 2: (A) or [A] format
+        bracket_match = re.search(r'[\(\[]\s*([A-D])\s*[\)\]]', text, re.IGNORECASE)
+        if bracket_match:
+            return bracket_match.group(1).upper()
+
+        # Strategy 3: Standalone letter
+        letter_match = re.search(r'\b([A-D])\b', text)
+        if letter_match:
+            return letter_match.group(1).upper()
+
+        # Strategy 4: First character if it's A-D
+        first_char = text.strip()[0].upper() if text.strip() else None
+        if first_char in ['A', 'B', 'C', 'D']:
+            return first_char
+
+        return None
+
+    def check_answer(self, predicted: str, expected: str) -> bool:
+        """Compare answers (case-insensitive letter comparison)."""
+        if predicted is None:
+            return False
+        return self.normalize_answer(predicted) == self.normalize_answer(expected)
+
+
+# Registry mapping task names to extractors
+_EXTRACTOR_REGISTRY = {}
+
+
+def _populate_registry():
+    """Populate the extractor registry."""
+    global _EXTRACTOR_REGISTRY
+
+    # Math tasks use GSM8KExtractor
+    math_tasks = [
+        "gsm8k", "math", "math500", "hendrycks_math",
+        "aime", "aime2024", "aime2025",
+        "hmmt", "hmmt_feb_2025",
+        "polymath", "polymath_en_medium", "polymath_zh_medium",
+        "polymath_en_high", "polymath_zh_high",
+        "livemathbench", "livemathbench_cnmo_en", "livemathbench_cnmo_zh",
+    ]
+    for task in math_tasks:
+        _EXTRACTOR_REGISTRY[task] = GSM8KExtractor()
+
+    # Coding tasks use LiveCodeBenchExtractor
+    coding_tasks = [
+        "livecodebench", "humaneval", "mbpp", "humaneval_plus", "mbpp_plus",
+        "instructhumaneval", "apps", "ds1000",
+        "multiple_py", "multiple_js", "multiple_java", "multiple_cpp", "multiple_rs", "multiple_go",
+        "conala", "concode", "mercury", "recode",
+        "codexglue_code_to_text_python", "codexglue_code_to_text_go",
+        "codexglue_code_to_text_ruby", "codexglue_code_to_text_java",
+        "codexglue_code_to_text_javascript", "codexglue_code_to_text_php",
+    ]
+    for task in coding_tasks:
+        _EXTRACTOR_REGISTRY[task] = LiveCodeBenchExtractor()
+
+    # HLE tasks use HLEExtractor
+    hle_tasks = ["hle", "hle_exact_match", "hle_multiple_choice"]
+    for task in hle_tasks:
+        _EXTRACTOR_REGISTRY[task] = HLEExtractor()
+
+    # Science tasks use SuperGPQAExtractor
+    science_tasks = ["supergpqa", "supergpqa_physics", "supergpqa_chemistry", "supergpqa_biology"]
+    for task in science_tasks:
+        _EXTRACTOR_REGISTRY[task] = SuperGPQAExtractor()
+
+    # QA tasks use generic extractor (HLEExtractor works well for these)
+    qa_tasks = ["truthfulqa_mc1", "mmlu", "squad2"]
+    for task in qa_tasks:
+        _EXTRACTOR_REGISTRY[task] = HLEExtractor()
+
+
+# Populate on module load
+_populate_registry()
+
+
+def get_extractor(task_name: str) -> BenchmarkExtractor:
+    """Get the appropriate extractor for a task."""
+    if task_name in _EXTRACTOR_REGISTRY:
+        return _EXTRACTOR_REGISTRY[task_name]
+
+    # Fallback logic based on task name patterns
+    task_lower = task_name.lower()
+    if any(keyword in task_lower for keyword in ["math", "aime", "hmmt", "gsm", "arithmetic"]):
+        return GSM8KExtractor()
+    elif any(keyword in task_lower for keyword in ["code", "human", "mbpp", "programming"]):
+        return LiveCodeBenchExtractor()
+    elif "hle" in task_lower:
+        return HLEExtractor()
+    elif any(keyword in task_lower for keyword in ["gpqa", "science", "physics", "chemistry", "biology"]):
+        return SuperGPQAExtractor()
+    else:
+        # Default fallback
+        return HLEExtractor()

wisent/core/bigcode_integration.py

@@ -325,9 +325,12 @@ class BigCodeEvaluator:
 
     def _execute_in_docker(self, sample: Dict, generation: str, task_name: str) -> Dict:
         """Execute code in Docker container."""
-        # TODO: Implement Docker execution
-        logger.warning("Docker execution not yet implemented, using subprocess")
-        return self._execute_in_subprocess(sample, generation, task_name)
+        raise NotImplementedError(
+            "Docker execution is not yet implemented. "
+            "To execute code safely, please implement Docker containerization for code execution, "
+            "or use the subprocess executor by setting docker_executor=None (note: subprocess execution "
+            "is less secure and should only be used in trusted environments)."
+        )
 
     def _execute_in_subprocess(self, sample: Dict, generation: str, task_name: str) -> Dict:
         """Execute code in subprocess (less secure)."""
@@ -540,10 +543,20 @@ def {expected_name}(*args, **kwargs):
         return total_passed / total_samples if total_samples > 0 else 0.0
 
     def _evaluate_text_generation(self, task: BigCodeTask, generations: List[str]) -> List[float]:
-        """Evaluate text generation tasks (e.g., code-to-text)."""
-        # TODO: Implement BLEU scoring
-        logger.warning("Text generation evaluation not yet implemented")
-        return [0.0] * len(generations)
+        """Evaluate text generation tasks (e.g., code-to-text).
+
+        Raises:
+            NotImplementedError: BLEU scoring is not yet implemented
+        """
+        raise NotImplementedError(
+            "BLEU scoring for text generation tasks is not yet implemented. "
+            "This requires:\n"
+            "  1. Reference text extraction from task samples\n"
+            "  2. BLEU score computation (using sacrebleu or similar library)\n"
+            "  3. Proper tokenization and n-gram matching\n"
+            "Please implement BLEU scoring or use alternative evaluation metrics "
+            "(exact match, F1 score) for code generation tasks."
+        )
 
 
 # Main interface for BigCode integration

wisent/core/branding.py

@@ -0,0 +1,108 @@
+"""
+Wisent project branding assets (ASCII art logo, project description).
+
+This module provides reusable branding components that can be used by:
+- CLI interfaces
+- Documentation
+- Web interfaces
+- Any other presentation layer
+
+The ASCII art represents a Wisent (European bison), the project's namesake.
+"""
+
+from __future__ import annotations
+
+__all__ = ["WISENT_ASCII_LOGO", "PROJECT_TAGLINE", "render_banner", "get_logo"]
+
+
+WISENT_ASCII_LOGO = """
+.................  .:--++*##%%%%##**+=-:.  .................
+..             .:=*%@@@@@@@%%%%%%%@@@@@@%*=:.             ..
+.           .-*%@@@%#+=-::.........:-=+#%@@@%*=.           .
+.         -*%@@@#=:.                    .:=*%@@@*-.        .
+.      .-#@@@*=.                            .-*@@@#-.      .
+.     :#@@@*:                                  :+%@@#-     .
+.   .+@@@*:                                      :+@@@+.   .
+.  .*@@@@%*=:.                                     -%@@#:  .
+. .#@@#=*%@@@%*-:.                                  .#@@%: .
+..*@@%.  .-+#@@@@#+-:.                               .*@@%..
+.=@@@-       :-+#@@@@%*=:.                            .%@@*.
+:#@@+           .:-+#@@@@%#+=:.                        -@@@-
+=@@@:                .-=*%@@@@%#+=:..                  .#@@+
++@@@*=:.                 .:-+*%@@@@%#*=-:..             *@@+
++@@@@@@#+-..                  .:-=*#@@@@@%#*+--..       +@@+
++@@#-+%@@@%:                        .:-=*#%@@@@@%#*+=-:.*@@+
+=@@%. .=@@@:                             ..:-=+#%%@@@@@%@@@+
+:%@@=  :@@@-                                    ..::-=+#@@@=
+.+@@%. .#@@*                                           +@@#:
+..%@@*. =@@@:                                         =@@@-.
+. :%@@*..#@@#.                         .:..          =@@@= .
+.  :%@@*.:%@@*.                       :#@@%#*+=-::..+@@@=  .
+.   :#@@%-:%@@#:                    .+@@@#%%@@@@@@%%@@%-   .
+.    .+@@@*=#@@%-                 .=%@@%=...::-=#@@@@*.    .
+.      :*@@@#%@@@*:             .=%@@@+.     .:*%@@#-      .
+.        :+%@@@@@@@*-.       :=*@@@%+.    .-+%@@@*-.       .
+.          .=*%@@@@@@#+:.:-+#@@@%*-. .:-+#%@@@#+:          .
+.             .-+#%@@@@@@@@@@@@#*+**#@@@@@%*=:.            .
+..............   ..-=+*#%%%@@@@@@@@%%#*=-:.   ..............
+ ...................  ....:::::::::.... ...................
+""".strip()
+
+PROJECT_TAGLINE = "Steering vectors & activation tooling"
+
+
+def get_logo(width: int = 48) -> str:
+    """
+    Get the Wisent ASCII logo, optionally centered to a specific width.
+
+    Args:
+        width: Total width to center the logo within (default: 48)
+
+    Returns:
+        ASCII art logo as a string
+    """
+    return "\n".join(line.center(width) for line in WISENT_ASCII_LOGO.splitlines())
+
+
+def render_banner(title: str, width: int = 48, use_color: bool = True) -> str:
+    """
+    Render a banner with the Wisent logo, title, and tagline.
+
+    Args:
+        title: Title text to display (e.g., "Wisent-Guard CLI")
+        width: Width for centering (default: 48)
+        use_color: Whether to use ANSI color codes (default: True)
+
+    Returns:
+        Formatted banner as a string
+    """
+    logo = get_logo(width)
+
+    if use_color:
+        GREEN = "\x1b[32m"
+        BOLD = "\x1b[1m"
+        OFF = "\x1b[0m"
+        banner = f"{GREEN}{logo}{OFF}\n"
+        banner += f"{BOLD}{GREEN}{title}{OFF} – {PROJECT_TAGLINE}\n"
+    else:
+        banner = f"{logo}\n"
+        banner += f"{title} – {PROJECT_TAGLINE}\n"
+
+    return banner
+
+
+def print_banner(title: str, width: int = 48, use_color: bool = True) -> None:
+    """
+    Print a banner with the Wisent logo, title, and tagline.
+
+    Args:
+        title: Title text to display (e.g., "Wisent-Guard CLI")
+        width: Width for centering (default: 48)
+        use_color: Whether to use ANSI color codes (default: True)
+    """
+    print(render_banner(title, width, use_color))
+
+
+if __name__ == "__main__":
+    # Demo the branding
+    print_banner("Wisent-Guard")

wisent/core/cli/__init__.py

@@ -0,0 +1,15 @@
+"""CLI execution logic for Wisent commands."""
+
+from .tasks import execute_tasks
+from .generate_pairs_from_task import execute_generate_pairs_from_task
+from .generate_pairs import execute_generate_pairs
+from .get_activations import execute_get_activations
+from .create_steering_vector import execute_create_steering_vector
+from .generate_vector_from_task import execute_generate_vector_from_task
+from .generate_vector_from_synthetic import execute_generate_vector_from_synthetic
+from .optimize_classification import execute_optimize_classification
+from .optimize_steering import execute_optimize_steering
+from .generate_responses import execute_generate_responses
+from .evaluate_responses import execute_evaluate_responses
+
+__all__ = ['execute_tasks', 'execute_generate_pairs_from_task', 'execute_generate_pairs', 'execute_get_activations', 'execute_create_steering_vector', 'execute_generate_vector_from_task', 'execute_generate_vector_from_synthetic', 'execute_optimize_classification', 'execute_optimize_steering', 'execute_generate_responses', 'execute_evaluate_responses']

wisent/core/cli/create_steering_vector.py

@@ -0,0 +1,138 @@
+"""Create steering vector command execution logic."""
+
+import sys
+import json
+import os
+import time
+import torch
+from collections import defaultdict
+
+
+def execute_create_steering_vector(args):
+    """Execute the create-steering-vector command - load enriched pairs and create steering vectors."""
+    from wisent.core.steering_methods.methods.caa import CAAMethod
+
+    print(f"\n🎯 Creating steering vectors from enriched pairs")
+    print(f"   Input file: {args.enriched_pairs_file}")
+    print(f"   Method: {args.method}")
+
+    start_time = time.time() if args.timing else None
+
+    try:
+        # 1. Load enriched pairs from JSON
+        print(f"\n📂 Loading enriched pairs...")
+        if not os.path.exists(args.enriched_pairs_file):
+            raise FileNotFoundError(f"Enriched pairs file not found: {args.enriched_pairs_file}")
+
+        with open(args.enriched_pairs_file, 'r') as f:
+            data = json.load(f)
+
+        # Extract metadata
+        trait_label = data.get('trait_label', 'unknown')
+        model = data.get('model', 'unknown')
+        layers = data.get('layers', [])
+        token_aggregation = data.get('token_aggregation', 'unknown')
+        pairs_list = data.get('pairs', [])
+
+        print(f"   ✓ Loaded {len(pairs_list)} pairs")
+        print(f"   ✓ Model: {model}")
+        print(f"   ✓ Layers: {layers}")
+        print(f"   ✓ Token aggregation: {token_aggregation}")
+
+        # 2. Organize activations by layer
+        print(f"\n📊 Organizing activations by layer...")
+
+        # Structure: {layer_str: {"positive": [tensors], "negative": [tensors]}}
+        layer_activations = defaultdict(lambda: {"positive": [], "negative": []})
+
+        for pair in pairs_list:
+            # Extract positive activations
+            pos_layers = pair['positive_response'].get('layers_activations', {})
+            for layer_str, activation_list in pos_layers.items():
+                if activation_list is not None:
+                    tensor = torch.tensor(activation_list, dtype=torch.float32)
+                    layer_activations[layer_str]["positive"].append(tensor)
+
+            # Extract negative activations
+            neg_layers = pair['negative_response'].get('layers_activations', {})
+            for layer_str, activation_list in neg_layers.items():
+                if activation_list is not None:
+                    tensor = torch.tensor(activation_list, dtype=torch.float32)
+                    layer_activations[layer_str]["negative"].append(tensor)
+
+        available_layers = sorted(layer_activations.keys(), key=int)
+        print(f"   ✓ Found activations for {len(available_layers)} layers: {available_layers}")
+
+        # 3. Create steering method instance
+        print(f"\n🧠 Initializing {args.method.upper()} steering method...")
+
+        if args.method == "caa":
+            method = CAAMethod(kwargs={"normalize": args.normalize})
+        else:
+            raise ValueError(f"Unknown method: {args.method}")
+
+        print(f"   ✓ Method initialized (normalize={args.normalize})")
+
+        # 4. Generate steering vectors for each layer
+        print(f"\n⚡ Generating steering vectors...")
+        steering_vectors = {}
+
+        for layer_str in available_layers:
+            pos_list = layer_activations[layer_str]["positive"]
+            neg_list = layer_activations[layer_str]["negative"]
+
+            if args.verbose:
+                print(f"   Processing layer {layer_str}: {len(pos_list)} positive, {len(neg_list)} negative")
+
+            if not pos_list or not neg_list:
+                print(f"   ⚠️  Skipping layer {layer_str}: missing activations")
+                continue
+
+            # Generate steering vector for this layer
+            vector = method.train_for_layer(pos_list, neg_list)
+            steering_vectors[layer_str] = vector.tolist()  # Convert to list for JSON
+
+        print(f"   ✓ Generated {len(steering_vectors)} steering vectors")
+
+        # 5. Save steering vectors to JSON
+        print(f"\n💾 Saving steering vectors to '{args.output}'...")
+        output_data = {
+            'trait_label': trait_label,
+            'model': model,
+            'method': args.method,
+            'normalize': args.normalize,
+            'token_aggregation': token_aggregation,
+            'num_pairs': len(pairs_list),
+            'layers': list(steering_vectors.keys()),
+            'steering_vectors': steering_vectors,
+            'metadata': {
+                'source_file': args.enriched_pairs_file,
+                'creation_time': time.strftime('%Y-%m-%d %H:%M:%S'),
+            }
+        }
+
+        os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
+        with open(args.output, 'w') as f:
+            json.dump(output_data, f, indent=2)
+
+        print(f"   ✓ Saved steering vectors to: {args.output}")
+
+        # 6. Display statistics
+        print(f"\n📈 Steering Vector Statistics:")
+        for layer_str in sorted(steering_vectors.keys(), key=int):
+            vector = torch.tensor(steering_vectors[layer_str])
+            norm = torch.linalg.norm(vector).item()
+            print(f"   Layer {layer_str}: dim={len(vector)}, norm={norm:.4f}")
+
+        if args.timing:
+            elapsed = time.time() - start_time
+            print(f"   ⏱️  Total time: {elapsed:.2f}s")
+
+        print(f"\n✅ Steering vector creation completed successfully!\\n")
+
+    except Exception as e:
+        print(f"\n❌ Error: {str(e)}", file=sys.stderr)
+        if args.verbose:
+            import traceback
+            traceback.print_exc()
+        sys.exit(1)