wisent 0.1.1__py3-none-any.whl → 0.5.1__py3-none-any.whl

wisent/core/task_interface.py

@@ -0,0 +1,132 @@
+"""
+Task-agnostic interface for benchmark integration.
+
+This module provides a unified interface for integrating different benchmarks
+without depending on lm-evaluation-harness.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Type
+
+from .benchmark_extractors import BenchmarkExtractor
+
+
+class TaskInterface(ABC):
+    """Abstract interface for benchmark tasks."""
+
+    @abstractmethod
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load task data."""
+
+    @abstractmethod
+    def get_extractor(self) -> BenchmarkExtractor:
+        """Get the benchmark extractor for this task."""
+
+    @abstractmethod
+    def get_name(self) -> str:
+        """Get the task name."""
+
+    @abstractmethod
+    def get_description(self) -> str:
+        """Get the task description."""
+
+    @abstractmethod
+    def get_categories(self) -> List[str]:
+        """Get the task categories (e.g., ['coding', 'reasoning'])."""
+
+
+class TaskRegistry:
+    """Registry for managing available tasks."""
+
+    def __init__(self):
+        self._tasks: Dict[str, Type[TaskInterface]] = {}
+
+    def register_task(self, name: str, task_class: Type[TaskInterface]):
+        """Register a new task."""
+        self._tasks[name] = task_class
+
+    def get_task(self, name: str, limit: Optional[int] = None) -> TaskInterface:
+        """Get a task instance by name."""
+        if name not in self._tasks:
+            raise ValueError(f"Task '{name}' not found. Available tasks: {list(self._tasks.keys())}")
+
+        task_factory = self._tasks[name]
+
+        # Handle different task factory types
+        if callable(task_factory):
+            # Try calling with limit parameter
+            try:
+                return task_factory(limit=limit)
+            except TypeError:
+                # Fallback for factories that don't accept limit
+                return task_factory()
+        else:
+            # Direct class instantiation
+            return task_factory()
+
+    def list_tasks(self) -> List[str]:
+        """List all available task names."""
+        return list(self._tasks.keys())
+
+    def get_task_info(self, name: str) -> Dict[str, Any]:
+        """Get information about a specific task."""
+        task = self.get_task(name)
+        return {"name": task.get_name(), "description": task.get_description(), "categories": task.get_categories()}
+
+    def list_task_info(self) -> List[Dict[str, Any]]:
+        """List information about all available tasks."""
+        return [self.get_task_info(name) for name in self.list_tasks()]
+
+
+# Global task registry instance
+_task_registry = TaskRegistry()
+
+
+def register_task(name: str, task_class: Type[TaskInterface]):
+    """Register a new task globally."""
+    _task_registry.register_task(name, task_class)
+
+
+def get_task(name: str, limit: Optional[int] = None) -> TaskInterface:
+    """Get a task instance by name."""
+    # Ensure tasks are registered before attempting to get a task
+    _ensure_tasks_registered()
+
+    # Check if this is a file path (contains / or \\ or ends with .json)
+    if "/" in name or "\\" in name or name.endswith(".json"):
+        # Treat as file path and load directly
+        from .tasks.file_task import FileTask
+
+        return FileTask(name, limit=limit)
+
+    # Otherwise, try to get from registry
+    try:
+        return _task_registry.get_task(name, limit=limit)
+    except ValueError:
+        raise ValueError(
+            f"Task '{name}' not found in registry. Available tasks: {list(_task_registry._tasks.keys())}. To load a custom dataset, provide a file path ending with .json"
+        )
+
+
+def list_tasks() -> List[str]:
+    """List all available task names."""
+    _ensure_tasks_registered()
+    return _task_registry.list_tasks()
+
+
+def get_task_info(name: str) -> Dict[str, Any]:
+    """Get information about a specific task."""
+    return _task_registry.get_task_info(name)
+
+
+def list_task_info() -> List[Dict[str, Any]]:
+    """List information about all available tasks."""
+    return _task_registry.list_task_info()
+
+
+def _ensure_tasks_registered():
+    """Ensure all tasks are registered in the global registry."""
+    if len(_task_registry._tasks) == 0:  # Only register if not already done
+        # Import tasks module to trigger registration
+        # This is crucial for CLI usage where tasks module isn't imported elsewhere
+        from . import tasks  # noqa: F401 # This imports __init__.py which calls register_all_tasks()

wisent/core/task_selector.py

@@ -0,0 +1,189 @@
+"""
+Task selector for choosing tasks based on skills and risks tags.
+"""
+
+import json
+import os
+import random
+import logging
+from typing import List, Dict, Any, Optional, Set
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class TaskSelector:
+    """Select tasks based on skills and risks criteria."""
+    
+    def __init__(self):
+        """Initialize the task selector by loading metadata."""
+        self.base_path = Path(__file__).parent.parent / "parameters" / "tasks"
+        self.skills = self._load_json("skills.json")
+        self.risks = self._load_json("risks.json")
+        self.tasks_data = self._load_json("tasks.json")
+        self.tasks = self.tasks_data.get("tasks", {})
+        
+    def _load_json(self, filename: str) -> Any:
+        """Load JSON file from parameters/tasks directory."""
+        filepath = self.base_path / filename
+        try:
+            with open(filepath, 'r') as f:
+                return json.load(f)
+        except Exception as e:
+            logger.error(f"Failed to load {filename}: {e}")
+            return {} if filename == "tasks.json" else []
+    
+    def get_available_skills(self) -> List[str]:
+        """Get list of available skills."""
+        return self.skills
+    
+    def get_available_risks(self) -> List[str]:
+        """Get list of available risks."""
+        return self.risks
+    
+    def find_tasks_by_tags(
+        self, 
+        skills: Optional[List[str]] = None, 
+        risks: Optional[List[str]] = None,
+        min_quality_score: int = 2
+    ) -> List[str]:
+        """
+        Find tasks that match the given skills and/or risks.
+        
+        Args:
+            skills: List of skill tags to match
+            risks: List of risk tags to match
+            min_quality_score: Minimum quality score for tasks (default: 2)
+            
+        Returns:
+            List of task names that match the criteria
+        """
+        if not skills and not risks:
+            # Return all tasks if no criteria specified
+            return [
+                task_name for task_name, task_data in self.tasks.items()
+                if task_data.get("quality_score", 0) >= min_quality_score
+            ]
+        
+        # Convert to sets for efficient lookup
+        required_tags = set()
+        if skills:
+            required_tags.update(skills)
+        if risks:
+            required_tags.update(risks)
+        
+        matched_tasks = []
+        for task_name, task_data in self.tasks.items():
+            # Check quality score
+            if task_data.get("quality_score", 0) < min_quality_score:
+                continue
+                
+            # Check if task has any of the required tags
+            task_tags = set(task_data.get("tags", []))
+            if task_tags.intersection(required_tags):
+                matched_tasks.append(task_name)
+        
+        return matched_tasks
+    
+    def select_random_tasks(
+        self,
+        skills: Optional[List[str]] = None,
+        risks: Optional[List[str]] = None,
+        num_tasks: Optional[int] = None,
+        min_quality_score: int = 2,
+        seed: Optional[int] = None
+    ) -> List[str]:
+        """
+        Select random tasks based on skills/risks criteria.
+        
+        Args:
+            skills: List of skill tags to match
+            risks: List of risk tags to match
+            num_tasks: Number of tasks to select (None = all matching tasks)
+            min_quality_score: Minimum quality score for tasks
+            seed: Random seed for reproducibility
+            
+        Returns:
+            List of randomly selected task names
+        """
+        # Find matching tasks
+        matched_tasks = self.find_tasks_by_tags(skills, risks, min_quality_score)
+        
+        if not matched_tasks:
+            logger.warning(f"No tasks found matching skills={skills}, risks={risks}")
+            return []
+        
+        # Set random seed if provided
+        if seed is not None:
+            random.seed(seed)
+        
+        # Select tasks
+        if num_tasks is None or num_tasks >= len(matched_tasks):
+            selected = matched_tasks
+        else:
+            selected = random.sample(matched_tasks, num_tasks)
+        
+        logger.info(f"Selected {len(selected)} tasks from {len(matched_tasks)} matching tasks")
+        return selected
+    
+    def validate_skills_and_risks(
+        self, 
+        skills: Optional[List[str]] = None, 
+        risks: Optional[List[str]] = None
+    ) -> Dict[str, List[str]]:
+        """
+        Validate provided skills and risks against available options.
+        
+        Returns:
+            Dictionary with 'invalid_skills' and 'invalid_risks' lists
+        """
+        invalid = {"invalid_skills": [], "invalid_risks": []}
+        
+        if skills:
+            valid_skills = set(self.skills)
+            invalid["invalid_skills"] = [s for s in skills if s not in valid_skills]
+        
+        if risks:
+            valid_risks = set(self.risks)
+            invalid["invalid_risks"] = [r for r in risks if r not in valid_risks]
+        
+        return invalid
+
+
+def get_tasks_for_skills_and_risks(
+    skills: Optional[List[str]] = None,
+    risks: Optional[List[str]] = None,
+    num_tasks: Optional[int] = None,
+    min_quality_score: int = 2,
+    seed: Optional[int] = None
+) -> List[str]:
+    """
+    Convenience function to get tasks matching skills/risks criteria.
+    
+    Args:
+        skills: List of skill tags to match
+        risks: List of risk tags to match
+        num_tasks: Number of tasks to select (None = all)
+        min_quality_score: Minimum quality score
+        seed: Random seed
+        
+    Returns:
+        List of task names
+    """
+    selector = TaskSelector()
+    
+    # Validate inputs
+    invalid = selector.validate_skills_and_risks(skills, risks)
+    if invalid["invalid_skills"]:
+        logger.warning(f"Invalid skills: {invalid['invalid_skills']}")
+    if invalid["invalid_risks"]:
+        logger.warning(f"Invalid risks: {invalid['invalid_risks']}")
+    
+    # Select tasks
+    return selector.select_random_tasks(
+        skills=skills,
+        risks=risks,
+        num_tasks=num_tasks,
+        min_quality_score=min_quality_score,
+        seed=seed
+    )

wisent/core/tasks/__init__.py

@@ -0,0 +1,175 @@
+"""
+Task implementations for wisent-guard.
+
+This package contains task-agnostic implementations for various benchmarks.
+"""
+
+from ..task_interface import register_task
+from .aime_task import AIMETask
+from .hle_task import HLEExactMatchTask, HLEMultipleChoiceTask, HLETask
+from .hmmt_task import HMMTTask
+from .livecodebench_task import LiveCodeBenchTask
+from .livemathbench_task import LiveMathBenchTask
+from .lm_eval_task import (
+    AppsTask,
+    CodexglueCodeToTextGoTask,
+    CodexglueCodeToTextJavascriptTask,
+    CodexglueCodeToTextJavaTask,
+    CodexglueCodeToTextPhpTask,
+    CodexglueCodeToTextPythonTask,
+    CodexglueCodeToTextRubyTask,
+    ConalaTask,
+    ConcodeTask,
+    DS1000Task,
+    GSM8KTask,
+    HumanEvalPlusTask,
+    HumanEvalTask,
+    InstructHumanEvalTask,
+    MBPPPlusTask,
+    MBPPTask,
+    MercuryTask,
+    MMLUTask,
+    MultipleCppTask,
+    MultipleGoTask,
+    MultipleJavaTask,
+    MultipleJsTask,
+    MultiplePyTask,
+    MultipleRsTask,
+    RecodeTask,
+    Squad2Task,
+    TruthfulQATask,
+)
+from .math500_task import Math500Task
+from .polymath_task import PolyMathTask
+from .supergpqa_task import SuperGPQABiologyTask, SuperGPQAChemistryTask, SuperGPQAPhysicsTask, SuperGPQATask
+
+
+def register_all_tasks():
+    """Register all available tasks."""
+    # Register LiveCodeBench task
+    register_task("livecodebench", lambda limit=None: LiveCodeBenchTask(release_version="release_v1", limit=limit))
+
+    # Register common lm-eval tasks
+    register_task("gsm8k", GSM8KTask)
+    register_task("truthfulqa_mc1", TruthfulQATask)
+    register_task("mmlu", MMLUTask)
+
+    # Register all coding tasks
+    register_task("mbpp", MBPPTask)
+    register_task("humaneval", HumanEvalTask)
+    register_task("mbpp_plus", MBPPPlusTask)
+    register_task("instructhumaneval", InstructHumanEvalTask)
+    register_task("humaneval_plus", HumanEvalPlusTask)
+    register_task("conala", ConalaTask)
+    register_task("concode", ConcodeTask)
+    register_task("mercury", MercuryTask)
+    register_task("apps", AppsTask)
+    register_task("ds1000", DS1000Task)
+    register_task("multiple_py", MultiplePyTask)
+    register_task("multiple_js", MultipleJsTask)
+    register_task("multiple_java", MultipleJavaTask)
+    register_task("multiple_cpp", MultipleCppTask)
+    register_task("multiple_rs", MultipleRsTask)
+    register_task("multiple_go", MultipleGoTask)
+    register_task("codexglue_code_to_text_python", CodexglueCodeToTextPythonTask)
+    register_task("codexglue_code_to_text_go", CodexglueCodeToTextGoTask)
+    register_task("codexglue_code_to_text_ruby", CodexglueCodeToTextRubyTask)
+    register_task("codexglue_code_to_text_java", CodexglueCodeToTextJavaTask)
+    register_task("codexglue_code_to_text_javascript", CodexglueCodeToTextJavascriptTask)
+    register_task("codexglue_code_to_text_php", CodexglueCodeToTextPhpTask)
+    register_task("recode", RecodeTask)
+    register_task("squad2", Squad2Task)
+
+    # Register HLE tasks
+    register_task("hle", lambda limit=None: HLETask(limit=limit))
+    register_task("hle_exact_match", lambda limit=None: HLEExactMatchTask(limit=limit))
+    register_task("hle_multiple_choice", lambda limit=None: HLEMultipleChoiceTask(limit=limit))
+
+    # Register MATH-500 tasks
+    register_task("math500", lambda limit=None: Math500Task(limit=limit))
+    register_task("math", lambda limit=None: Math500Task(limit=limit))
+    register_task("hendrycks_math", lambda limit=None: Math500Task(limit=limit))
+
+    # Register AIME tasks (general + year-specific)
+    register_task("aime", lambda limit=None: AIMETask(year="2025", limit=limit))  # Default: latest year (2025)
+    register_task("aime2025", lambda limit=None: AIMETask(year="2025", limit=limit))
+    register_task("aime2024", lambda limit=None: AIMETask(year="2024", limit=limit))
+
+    # Register HMMT tasks (general + competition-specific)
+    register_task(
+        "hmmt", lambda limit=None: HMMTTask(competition="feb_2025", limit=limit)
+    )  # Default: latest competition
+    register_task("hmmt_feb_2025", lambda limit=None: HMMTTask(competition="feb_2025", limit=limit))
+
+    # Register PolyMath tasks (Chinese and English, medium difficulty)
+    register_task(
+        "polymath", lambda limit=None: PolyMathTask(language="en", difficulty="medium", limit=limit)
+    )  # Default: English medium
+    register_task(
+        "polymath_en_medium", lambda limit=None: PolyMathTask(language="en", difficulty="medium", limit=limit)
+    )
+    register_task(
+        "polymath_zh_medium", lambda limit=None: PolyMathTask(language="zh", difficulty="medium", limit=limit)
+    )
+    register_task("polymath_en_high", lambda limit=None: PolyMathTask(language="en", difficulty="high", limit=limit))
+    register_task("polymath_zh_high", lambda limit=None: PolyMathTask(language="zh", difficulty="high", limit=limit))
+
+    # Register LiveMathBench tasks (CNMO 2024 Chinese and English)
+    register_task("livemathbench", lambda limit=None: LiveMathBenchTask(language="en", limit=limit))  # Default: English
+    register_task("livemathbench_cnmo_en", lambda limit=None: LiveMathBenchTask(language="en", limit=limit))
+    register_task("livemathbench_cnmo_zh", lambda limit=None: LiveMathBenchTask(language="zh", limit=limit))
+
+    # Register SuperGPQA tasks (scientific reasoning)
+    register_task("supergpqa", lambda limit=None: SuperGPQATask(limit=limit))  # Default: all subjects
+    register_task("supergpqa_physics", lambda limit=None: SuperGPQAPhysicsTask(limit=limit))
+    register_task("supergpqa_chemistry", lambda limit=None: SuperGPQAChemistryTask(limit=limit))
+    register_task("supergpqa_biology", lambda limit=None: SuperGPQABiologyTask(limit=limit))
+
+
+# Auto-register tasks when the module is imported
+register_all_tasks()
+
+
+__all__ = [
+    "AIMETask",
+    "AppsTask",
+    "CodexglueCodeToTextGoTask",
+    "CodexglueCodeToTextJavaTask",
+    "CodexglueCodeToTextJavascriptTask",
+    "CodexglueCodeToTextPhpTask",
+    "CodexglueCodeToTextPythonTask",
+    "CodexglueCodeToTextRubyTask",
+    "ConalaTask",
+    "ConcodeTask",
+    "DS1000Task",
+    "GSM8KTask",
+    "HLEExactMatchTask",
+    "HLEMultipleChoiceTask",
+    "HLETask",
+    "HMMTTask",
+    "HumanEvalPlusTask",
+    "HumanEvalTask",
+    "InstructHumanEvalTask",
+    "LiveCodeBenchTask",
+    "LiveMathBenchTask",
+    "MBPPPlusTask",
+    "MBPPTask",
+    "MMLUTask",
+    "Math500Task",
+    "MercuryTask",
+    "MultipleCppTask",
+    "MultipleGoTask",
+    "MultipleJavaTask",
+    "MultipleJsTask",
+    "MultiplePyTask",
+    "MultipleRsTask",
+    "PolyMathTask",
+    "RecodeTask",
+    "Squad2Task",
+    "SuperGPQABiologyTask",
+    "SuperGPQAChemistryTask",
+    "SuperGPQAPhysicsTask",
+    "SuperGPQATask",
+    "TruthfulQATask",
+    "register_all_tasks",
+]

wisent/core/tasks/aime_task.py

@@ -0,0 +1,141 @@
+"""
+AIME task implementation for task-agnostic architecture.
+"""
+
+from typing import Dict, Any, List, Optional
+from ..task_interface import TaskInterface
+from ..benchmark_extractors import GSM8KExtractor
+import datasets
+
+
+class AIMETask(TaskInterface):
+    """General AIME mathematical contest task implementation."""
+    
+    # Dataset configurations for different years
+    DATASET_CONFIGS = {
+        "2024": {
+            "source": "Maxwell-Jia/AIME_2024",
+            "split": "train",
+            "fields": {"problem": "Problem", "answer": "Answer"},
+            "description": "30 high-difficulty AIME contest problems from 2024"
+        },
+        "2025": {
+            "source": "MathArena/aime_2025", 
+            "split": "train",
+            "fields": {"problem": "problem", "answer": "answer"},
+            "description": "30 high-difficulty AIME contest problems from 2025 (MathArena)"
+        }
+    }
+    
+    def __init__(self, year: str = "2025", limit: Optional[int] = None):
+        """
+        Initialize AIME task for specified year.
+        
+        Args:
+            year: AIME year to load ("2024", "2025"). Default: "2025" (latest)
+            limit: Maximum number of samples to load
+        """
+        if year not in self.DATASET_CONFIGS:
+            available = list(self.DATASET_CONFIGS.keys())
+            raise ValueError(f"AIME year '{year}' not supported. Available: {available}")
+            
+        self.year = year
+        self.config = self.DATASET_CONFIGS[year]
+        self._limit = limit
+        self._data = None  # Cache for loaded data
+        self._extractor = GSM8KExtractor()  # Reuse enhanced GSM8K extractor
+    
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load AIME data from HuggingFace for specified year."""
+        # Load dataset based on year configuration
+        dataset = datasets.load_dataset(
+            self.config["source"], 
+            split=self.config["split"]
+        )
+        
+        # Apply limit
+        effective_limit = limit or self._limit
+        if effective_limit:
+            dataset = dataset.select(range(min(effective_limit, len(dataset))))
+        
+        # Convert to list and normalize field names
+        data = [dict(item) for item in dataset]
+        
+        # Normalize field names for consistent processing
+        normalized_data = []
+        problem_field = self.config["fields"]["problem"]
+        answer_field = self.config["fields"]["answer"]
+        
+        for item in data:
+            normalized_item = dict(item)  # Keep all original fields
+            
+            # Ensure consistent field names for extractor
+            if problem_field in item:
+                normalized_item["Problem"] = item[problem_field]
+                normalized_item["question"] = item[problem_field]  # For question/answer format
+            
+            if answer_field in item:
+                normalized_item["Answer"] = item[answer_field]
+                normalized_item["answer"] = item[answer_field]  # For question/answer format
+            
+            normalized_data.append(normalized_item)
+        
+        return normalized_data
+            
+    
+    def get_task_info(self) -> Dict[str, Any]:
+        """Get information about the AIME task."""
+        return {
+            "task_name": f"aime{self.year}" if self.year != "2025" else "aime",
+            "year": self.year,
+            "description": self.config["description"],
+            "source": self.config["source"],
+            "task_type": "text_generation",
+            "evaluation_method": "mathematical_equivalence"
+        }
+    
+    def validate_sample(self, sample: Dict[str, Any]) -> bool:
+        """Validate that a sample has required AIME fields."""
+        problem_field = self.config["fields"]["problem"]
+        answer_field = self.config["fields"]["answer"]
+        
+        return all(field in sample for field in [problem_field, answer_field])
+    
+    def get_extractor(self) -> GSM8KExtractor:
+        """Get the benchmark extractor for this task."""
+        return self._extractor
+    
+    def get_name(self) -> str:
+        """Get the task name."""
+        return f"aime{self.year}" if self.year != "2025" else "aime"
+    
+    def get_description(self) -> str:
+        """Get the task description."""
+        return f"AIME {self.year} contest problems requiring advanced mathematical reasoning"
+    
+    def get_categories(self) -> List[str]:
+        """Get the task categories."""
+        return ["mathematics", "reasoning", "contest", "text_generation"]
+    
+    # Methods to match lm-eval interface
+    def has_validation_docs(self) -> bool:
+        """Check if task has validation documents."""
+        return False  # AIME doesn't have separate validation sets
+    
+    def has_test_docs(self) -> bool:
+        """Check if task has test documents."""
+        return True  # All samples are considered test docs
+    
+    def test_docs(self) -> List[Dict[str, Any]]:
+        """Get test documents."""
+        if self._data is None:
+            self._data = self.load_data()
+        return self._data
+    
+    def validation_docs(self) -> List[Dict[str, Any]]:
+        """Get validation documents."""
+        return []  # No separate validation set
+    
+    def doc_to_text(self, doc: Dict[str, Any]) -> str:
+        """Convert document to text prompt."""
+        return doc.get('Problem', '')