wisent 0.1.1__py3-none-any.whl → 0.5.1__py3-none-any.whl

wisent/core/tasks/supergpqa_task.py

@@ -0,0 +1,220 @@
+"""
+SuperGPQA task implementation for task-agnostic architecture.
+"""
+
+from typing import Dict, Any, List, Optional
+from datasets import load_dataset
+from ..task_interface import TaskInterface
+from ..benchmark_extractors import SuperGPQAExtractor
+
+
+class SuperGPQATask(TaskInterface):
+    """SuperGPQA scientific reasoning task implementation."""
+    
+    def __init__(self, discipline_filter: Optional[str] = None, difficulty_filter: Optional[str] = None, 
+                 calculation_only: Optional[bool] = None, limit: Optional[int] = None):
+        """Initialize SuperGPQA task.
+        
+        Args:
+            discipline_filter: Filter by discipline (Science, Engineering, etc.)
+            difficulty_filter: Filter by difficulty level
+            calculation_only: If True, only include calculation problems; if False, exclude them
+            limit: Maximum number of examples to load
+        """
+        self.dataset_name = "m-a-p/SuperGPQA"
+        self.discipline_filter = discipline_filter
+        self.difficulty_filter = difficulty_filter
+        self.calculation_only = calculation_only
+        self.limit = limit
+        self.field_filter = None  # Can be set by subclasses
+        self._extractor = SuperGPQAExtractor()
+        self._data = None  # Cache for loaded data
+    
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load SuperGPQA data from HuggingFace datasets."""
+        dataset = load_dataset(self.dataset_name, split="train")
+        
+        # Apply filters
+        filtered_data = self._filter_and_process(dataset)
+        
+        # Apply limit
+        effective_limit = limit or self.limit
+        if effective_limit:
+            filtered_data = filtered_data[:effective_limit]
+        
+        return filtered_data
+    
+    def _filter_and_process(self, dataset) -> List[Dict[str, Any]]:
+        """Filter data by discipline, field, difficulty, and calculation type, then convert to internal format."""
+        filtered_data = []
+        
+        for item in dataset:
+            # Apply discipline filter
+            if self.discipline_filter and item.get('discipline') != self.discipline_filter:
+                continue
+            
+            # Apply field filter (for subject-specific tasks)
+            if self.field_filter and item.get('field') != self.field_filter:
+                continue
+            
+            # Apply difficulty filter
+            if self.difficulty_filter and item.get('difficulty') != self.difficulty_filter:
+                continue
+            
+            # Apply calculation filter
+            if self.calculation_only is not None:
+                if self.calculation_only and not item.get('is_calculation', False):
+                    continue
+                elif not self.calculation_only and item.get('is_calculation', False):
+                    continue
+            
+            # Convert to internal format
+            processed_item = {
+                'uuid': item.get('uuid', ''),
+                'question': item.get('question', ''),
+                'options': item.get('options', []),
+                'answer': item.get('answer', ''),
+                'answer_letter': item.get('answer_letter', ''),
+                'discipline': item.get('discipline', ''),
+                'field': item.get('field', ''),
+                'subfield': item.get('subfield', ''),
+                'difficulty': item.get('difficulty', ''),
+                'is_calculation': item.get('is_calculation', False),
+                'metadata': {
+                    'dataset': self.dataset_name
+                }
+            }
+            
+            filtered_data.append(processed_item)
+        
+        return filtered_data
+    
+    def get_extractor(self) -> SuperGPQAExtractor:
+        """Get the SuperGPQA benchmark extractor."""
+        return self._extractor
+    
+    def get_name(self) -> str:
+        """Get the task name."""
+        name = "supergpqa"
+        if self.discipline_filter:
+            name += f"_{self.discipline_filter.lower()}"
+        if self.difficulty_filter:
+            name += f"_{self.difficulty_filter.lower()}"
+        if self.calculation_only is not None:
+            name += "_calc" if self.calculation_only else "_nocalc"
+        return name
+    
+    def get_description(self) -> str:
+        """Get the task description."""
+        desc = "SuperGPQA: Large-scale dataset of scientific multiple-choice questions across disciplines"
+        filters = []
+        if self.discipline_filter:
+            filters.append(f"discipline: {self.discipline_filter}")
+        if self.difficulty_filter:
+            filters.append(f"difficulty: {self.difficulty_filter}")
+        if self.calculation_only is not None:
+            filters.append("calculation problems only" if self.calculation_only else "non-calculation problems only")
+        
+        if filters:
+            desc += f" (filtered: {', '.join(filters)})"
+        return desc
+    
+    def get_categories(self) -> List[str]:
+        """Get the task categories."""
+        return ["science", "reasoning", "multiple_choice", "knowledge"]
+    
+    def get_task_info(self) -> Dict[str, Any]:
+        """Get information about the SuperGPQA task."""
+        return {
+            "task_name": self.get_name(),
+            "description": self.get_description(),
+            "source": self.dataset_name,
+            "task_type": "multiple_choice",
+            "evaluation_method": "exact_match",
+            "filters": {
+                "discipline": self.discipline_filter,
+                "difficulty": self.difficulty_filter,
+                "calculation_only": self.calculation_only
+            }
+        }
+    
+    def validate_sample(self, sample: Dict[str, Any]) -> bool:
+        """Validate that a sample has required SuperGPQA fields."""
+        required_fields = ["question", "options", "answer", "answer_letter"]
+        return all(field in sample for field in required_fields)
+    
+    # Methods to match lm-eval interface
+    def has_validation_docs(self) -> bool:
+        """Check if task has validation documents."""
+        return False  # SuperGPQA doesn't have separate validation sets
+    
+    def has_test_docs(self) -> bool:
+        """Check if task has test documents."""
+        return True  # All samples are considered test docs
+    
+    def test_docs(self) -> List[Dict[str, Any]]:
+        """Get test documents."""
+        if self._data is None:
+            self._data = self.load_data()
+        return self._data
+    
+    def validation_docs(self) -> List[Dict[str, Any]]:
+        """Get validation documents."""
+        return []  # No separate validation set
+    
+    def doc_to_text(self, doc: Dict[str, Any]) -> str:
+        """Convert document to text prompt."""
+        question = doc.get('question', '')
+        options = doc.get('options', [])
+        
+        # Format as multiple choice question
+        if options:
+            choices = []
+            for i, option in enumerate(options):
+                letter = chr(ord('A') + i)
+                choices.append(f"{letter}. {option}")
+            return f"{question}\n\n" + "\n".join(choices)
+        else:
+            return question
+
+
+class SuperGPQAPhysicsTask(SuperGPQATask):
+    """SuperGPQA task filtered to Physics questions only."""
+    
+    def __init__(self, difficulty_filter: Optional[str] = None, calculation_only: Optional[bool] = None, 
+                 limit: Optional[int] = None):
+        # Filter by discipline=Science and field=Physics
+        super().__init__(discipline_filter="Science", difficulty_filter=difficulty_filter, 
+                        calculation_only=calculation_only, limit=limit)
+        self.field_filter = "Physics"  # Add field filtering
+    
+    def get_name(self) -> str:
+        return "supergpqa_physics"
+
+
+class SuperGPQAChemistryTask(SuperGPQATask):
+    """SuperGPQA task filtered to Chemistry questions only."""
+    
+    def __init__(self, difficulty_filter: Optional[str] = None, calculation_only: Optional[bool] = None, 
+                 limit: Optional[int] = None):
+        # Filter by discipline=Science and field=Chemistry  
+        super().__init__(discipline_filter="Science", difficulty_filter=difficulty_filter, 
+                        calculation_only=calculation_only, limit=limit)
+        self.field_filter = "Chemistry"  # Add field filtering
+    
+    def get_name(self) -> str:
+        return "supergpqa_chemistry"
+
+
+class SuperGPQABiologyTask(SuperGPQATask):
+    """SuperGPQA task filtered to Biology questions only."""
+    
+    def __init__(self, difficulty_filter: Optional[str] = None, calculation_only: Optional[bool] = None, 
+                 limit: Optional[int] = None):
+        # Filter by discipline=Science and field=Biology
+        super().__init__(discipline_filter="Science", difficulty_filter=difficulty_filter, 
+                        calculation_only=calculation_only, limit=limit)
+        self.field_filter = "Biology"  # Add field filtering
+    
+    def get_name(self) -> str:
+        return "supergpqa_biology"

wisent/core/time_estimator.py

@@ -0,0 +1,149 @@
+"""Time estimation for optimization operations using runtime calibration"""
+import time
+from typing import Dict, Tuple, Optional
+from pathlib import Path
+
+from .timing_calibration import TimingCalibrator
+
+
+class OptimizationTimeEstimator:
+    """Estimates time required for optimization operations using calibration"""
+    
+    def __init__(
+        self, 
+        model_name: str, 
+        verbose: bool = True,
+        skip_calibration: bool = False,
+        calibration_file: Optional[Path] = None,
+        calibrate_only: bool = False
+    ):
+        self.model_name = model_name
+        self.verbose = verbose
+        self.calibrator = TimingCalibrator(verbose=verbose)
+        
+        # Get number of layers in the model
+        from . import Model
+        model = Model(name=model_name)
+        if hasattr(model, 'model') and hasattr(model.model, 'config'):
+            if hasattr(model.model.config, 'num_hidden_layers'):
+                self.total_layers = model.model.config.num_hidden_layers
+            elif hasattr(model.model.config, 'n_layer'):
+                self.total_layers = model.model.config.n_layer
+            else:
+                raise RuntimeError(f"Cannot determine number of layers for model {model_name}")
+        else:
+            raise RuntimeError(f"Cannot access model configuration for {model_name}")
+        
+        # Handle calibration
+        if skip_calibration:
+            raise RuntimeError("Calibration cannot be skipped. Accurate timing requires calibration.")
+        
+        if calibration_file and calibration_file.exists():
+            # Load from file
+            if not self.calibrator.load_from_file(calibration_file):
+                raise RuntimeError(f"Failed to load calibration from {calibration_file}")
+            self.timing = self.calibrator.timings
+            if self.timing["training_time"] is None or self.timing["steering_time"] is None:
+                raise RuntimeError(f"Calibration file {calibration_file} contains invalid data")
+        else:
+            # Run calibration
+            if verbose:
+                print(f"\n🔧 Running timing calibration for {model_name}...")
+            
+            self.timing = self.calibrator.run_calibration(model_name)
+            
+            # Save calibration if file path provided
+            if calibration_file:
+                self.calibrator.save_to_file(calibration_file)
+        
+        self.calibrate_only = calibrate_only
+    
+    def estimate_classification_time(
+        self,
+        num_tasks: int,
+        sample_limit: int = 200,
+        layers: Optional[list] = None
+    ) -> Tuple[float, Dict[str, float]]:
+        """
+        Estimate time for classification optimization.
+        
+        Returns:
+            Tuple of (total_seconds, breakdown)
+        """
+        num_layers = len(layers) if layers else min(5, self.total_layers)
+        
+        total_time, breakdown = self.calibrator.estimate_optimization_time(
+            num_tasks=num_tasks,
+            num_layers=num_layers,
+            samples_per_task=sample_limit,
+            include_sample_size_opt=False,
+            include_classifier_training=False,
+            include_control_vectors=False
+        )
+        
+        return total_time, {"classification": total_time}
+    
+    def estimate_full_optimization_time(
+        self,
+        num_tasks: int,
+        classification_limit: int = 200,
+        sample_sizes: list = None,
+        sample_size_limit: int = 1000,
+        include_sample_size_opt: bool = True,
+        include_classifier_training: bool = True,
+        include_control_vectors: bool = True
+    ) -> Tuple[float, Dict[str, float]]:
+        """
+        Estimate time for full optimization pipeline.
+        
+        Returns:
+            Tuple of (total_seconds, breakdown)
+        """
+        # Typical number of layers tested in classification
+        num_layers = min(5, self.total_layers)
+        
+        # Control vectors typically test more layers
+        cv_layers = min(10, self.total_layers)
+        
+        # Sample sizes must be provided
+        if sample_sizes is None:
+            raise RuntimeError("sample_sizes must be provided for full optimization time estimation")
+        
+        return self.calibrator.estimate_optimization_time(
+            num_tasks=num_tasks,
+            num_layers=num_layers,
+            samples_per_task=classification_limit,
+            sample_sizes=sample_sizes,
+            sample_size_limit=sample_size_limit,
+            include_sample_size_opt=include_sample_size_opt,
+            include_classifier_training=include_classifier_training,
+            include_control_vectors=include_control_vectors,
+            num_cv_layers=cv_layers
+        )
+    
+    @staticmethod
+    def format_time(seconds: float) -> str:
+        """Format time in human-readable format"""
+        if seconds < 60:
+            return f"{seconds:.0f} seconds"
+        elif seconds < 3600:
+            minutes = seconds / 60
+            return f"{minutes:.0f} minutes"
+        else:
+            hours = seconds / 3600
+            minutes = (seconds % 3600) / 60
+            if minutes > 0:
+                return f"{hours:.0f} hours {minutes:.0f} minutes"
+            else:
+                return f"{hours:.0f} hours"
+    
+    def print_time_breakdown(self, total_time: float, breakdown: Dict[str, float]):
+        """Print a formatted time breakdown"""
+        print(f"\n⏱️  ESTIMATED OPTIMIZATION TIME:")
+        print(f"   Total: {self.format_time(total_time)}")
+        
+        if len(breakdown) > 1:
+            print("\n   Breakdown:")
+            for phase, time_sec in breakdown.items():
+                if time_sec > 0:
+                    print(f"   - {phase.replace('_', ' ').title()}: {self.format_time(time_sec)}")

wisent/core/timing_calibration.py

@@ -0,0 +1,174 @@
+"""Runtime timing calibration for optimization time estimation"""
+import time
+import json
+import subprocess
+import sys
+from typing import Dict, Optional, Tuple
+from pathlib import Path
+
+
+class TimingCalibrator:
+    """Measures actual optimization timing on the current system"""
+    
+    def __init__(self, verbose: bool = True):
+        self.verbose = verbose
+        self.timings = {
+            "training_time": None,  # Time for training command
+            "steering_time": None,  # Time for steering command
+        }
+    
+    def run_calibration(self, model_name: str) -> Dict[str, float]:
+        """
+        Run calibration by measuring training and steering times.
+        Uses exactly one task, one layer, and 10 samples.
+        
+        Args:
+            model_name: Model to calibrate timing for
+            
+        Returns:
+            Dictionary with training_time and steering_time
+        """
+        # Get model layer count
+        from . import Model
+        model = Model(name=model_name)
+        if hasattr(model, 'model') and hasattr(model.model, 'config'):
+            if hasattr(model.model.config, 'num_hidden_layers'):
+                total_layers = model.model.config.num_hidden_layers
+            elif hasattr(model.model.config, 'n_layer'):
+                total_layers = model.model.config.n_layer
+            else:
+                raise RuntimeError(f"Cannot determine number of layers for model {model_name}")
+        else:
+            raise RuntimeError(f"Cannot access model config for {model_name}")
+        
+        # Use middle layer for calibration
+        calibration_layer = total_layers // 2
+        
+        if self.verbose:
+            print(f"\n🔧 Running timing calibration for {model_name}...")
+            print(f"   Task: arc_easy")
+            print(f"   Layer: {calibration_layer}")
+            print(f"   Samples: 10")
+        
+        # 1. Measure training time
+        if self.verbose:
+            print(f"\n📊 Measuring training time...")
+        
+        start_time = time.time()
+        
+        cmd = [
+            sys.executable, "-m", "wisent_guard.cli", 
+            "optimize-classification",
+            model_name,
+            "--tasks", "arc_easy",
+            "--limit", "10",
+            "--layer-range", f"{calibration_layer},{calibration_layer}",
+            "--no-save",
+            "--skip-timing-estimation"
+        ]
+        
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        
+        if result.returncode != 0:
+            raise RuntimeError(f"Training calibration failed:\n{result.stderr}")
+            
+        self.timings["training_time"] = time.time() - start_time
+        
+        # 2. Skip steering calibration - it's too complex for quick calibration
+        # Users can run steering separately if needed
+        if self.verbose:
+            print(f"\n📊 Skipping steering calibration (too complex for quick estimate)")
+        
+        self.timings["steering_time"] = None
+        
+        if self.verbose:
+            print(f"\n✅ Calibration complete!")
+            print(f"   Training time: {self.timings['training_time']:.3f}s")
+            print(f"   Steering time: {self.timings['steering_time']:.3f}s")
+        
+        return self.timings
+    
+    def save_to_file(self, filepath: Path):
+        """Save calibration results to a file"""
+        with open(filepath, 'w') as f:
+            json.dump(self.timings, f, indent=2)
+        if self.verbose:
+            print(f"💾 Saved calibration to {filepath}")
+    
+    def load_from_file(self, filepath: Path) -> bool:
+        """Load calibration results from a file"""
+        if not filepath.exists():
+            return False
+        
+        try:
+            with open(filepath, 'r') as f:
+                self.timings = json.load(f)
+            if self.verbose:
+                print(f"📂 Loaded calibration from {filepath}")
+            return True
+        except Exception as e:
+            if self.verbose:
+                print(f"⚠️  Failed to load calibration: {e}")
+            return False
+    
+    def estimate_optimization_time(
+        self,
+        num_tasks: int,
+        num_layers: int,
+        samples_per_task: int = 1000,
+        sample_sizes: list = None,
+        sample_size_limit: int = 1000,
+        include_sample_size_opt: bool = True,
+        include_classifier_training: bool = True,
+        include_control_vectors: bool = True,
+        num_cv_layers: int = None
+    ) -> Tuple[float, Dict[str, float]]:
+        """
+        Estimate total optimization time based on calibration.
+        
+        Linear scaling from base measurements: 1 task, 1 layer, 10 samples.
+        
+        Returns:
+            Tuple of (total_seconds, breakdown_dict)
+        """
+        if self.timings["training_time"] is None:
+            raise RuntimeError("No calibration data available. Run calibration first.")
+        
+        # Base measurements from calibration
+        base_training = self.timings["training_time"]  # Time for 1 task, 1 layer, 10 samples
+        base_steering = self.timings["steering_time"]  # Time for 1 task, 1 layer, 10 samples
+        
+        breakdown = {}
+        
+        # Classification optimization: scales linearly with tasks, layers, and samples
+        classification_time = base_training * num_tasks * num_layers * (samples_per_task / 10)
+        breakdown["classification"] = classification_time
+        
+        # Sample size optimization: tests multiple sample sizes on ONE layer per task
+        if include_sample_size_opt and sample_sizes:
+            # Calculate average sample size from the provided list
+            avg_sample_size = sum(sample_sizes) / len(sample_sizes)
+            # Each test uses sample_size_limit samples from the dataset
+            sample_size_time = base_training * num_tasks * len(sample_sizes) * (min(avg_sample_size, sample_size_limit) / 10)
+            breakdown["sample_size"] = sample_size_time
+        else:
+            breakdown["sample_size"] = 0
+        
+        # Classifier training: one run per task with full samples
+        if include_classifier_training:
+            classifier_time = base_training * num_tasks * (samples_per_task / 10)
+            breakdown["classifier_training"] = classifier_time
+        else:
+            breakdown["classifier_training"] = 0
+        
+        # Control vector generation: skip if no steering calibration
+        if include_control_vectors and base_steering is not None:
+            cv_layers = num_cv_layers or num_layers
+            control_vectors_time = base_steering * num_tasks * cv_layers * (samples_per_task / 10)
+            breakdown["control_vectors"] = control_vectors_time
+        else:
+            breakdown["control_vectors"] = 0
+        
+        total_time = sum(breakdown.values())
+        
+        return total_time, breakdown

wisent/core/tracking/__init__.py

@@ -0,0 +1,54 @@
+"""
+Performance tracking module for wisent-guard.
+
+This module provides comprehensive monitoring capabilities including:
+- Memory usage tracking (CPU and GPU)
+- Latency/timing analysis
+- Performance profiling and optimization insights
+"""
+
+from .memory import (
+    MemoryTracker,
+    MemorySnapshot,
+    MemoryStats,
+    get_global_tracker as get_global_memory_tracker,
+    track_memory,
+    get_memory_info,
+    format_memory_usage
+)
+
+from .latency import (
+    LatencyTracker,
+    TimingEvent,
+    LatencyStats,
+    get_global_tracker as get_global_latency_tracker,
+    time_function,
+    time_operation,
+    get_timing_summary,
+    format_timing_summary,
+    reset_timing,
+    Operations
+)
+
+__all__ = [
+    # Memory tracking
+    "MemoryTracker",
+    "MemorySnapshot", 
+    "MemoryStats",
+    "get_global_memory_tracker",
+    "track_memory",
+    "get_memory_info",
+    "format_memory_usage",
+    
+    # Latency tracking
+    "LatencyTracker",
+    "TimingEvent",
+    "LatencyStats", 
+    "get_global_latency_tracker",
+    "time_function",
+    "time_operation",
+    "get_timing_summary",
+    "format_timing_summary",
+    "reset_timing",
+    "Operations"
+]