wisent 0.1.1__py3-none-any.whl → 0.5.2__py3-none-any.whl

wisent/core/tasks/file_task.py

@@ -0,0 +1,211 @@
+"""
+File-based task implementation for loading custom datasets from JSON files.
+
+This allows users to easily test the optimization pipeline with their own datasets
+without needing to implement complex task classes or modify the core system.
+"""
+
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from ..benchmark_extractors import GSM8KExtractor
+from ..task_interface import TaskInterface
+
+
+class FileTask(TaskInterface):
+    """Task that loads data from a JSON file."""
+
+    def __init__(self, file_path: str, task_name: Optional[str] = None, limit: Optional[int] = None):
+        """
+        Initialize a file-based task.
+
+        Args:
+            file_path: Path to JSON file containing the dataset
+            task_name: Optional custom name for the task (defaults to filename)
+            limit: Optional limit on number of samples to load
+        """
+        self.file_path = Path(file_path)
+        self._limit = limit
+        self._data = None  # Cache for loaded data
+        self._extractor = GSM8KExtractor()  # Reuse GSM8K extractor for QA format
+
+        # Set task name
+        if task_name:
+            self._task_name = task_name
+        else:
+            self._task_name = self.file_path.stem.lower()
+
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load data from the JSON file."""
+        if not self.file_path.exists():
+            raise FileNotFoundError(f"Dataset file not found: {self.file_path}")
+
+        try:
+            with open(self.file_path, encoding="utf-8") as f:
+                data = json.load(f)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON in file {self.file_path}: {e}")
+        except Exception as e:
+            raise RuntimeError(f"Failed to load file {self.file_path}: {e}")
+
+        # Ensure data is a list
+        if not isinstance(data, list):
+            raise ValueError(f"JSON file must contain a list of objects, got {type(data).__name__}")
+
+        # Validate samples
+        for i, sample in enumerate(data):
+            if not self.validate_sample(sample):
+                raise ValueError(f"Invalid sample at index {i}: {sample}")
+
+        # Apply limit
+        effective_limit = limit or self._limit
+        if effective_limit:
+            data = data[: min(effective_limit, len(data))]
+
+        return data
+
+    def get_extractor(self) -> GSM8KExtractor:
+        """Get the benchmark extractor for this task."""
+        return self._extractor
+
+    def get_name(self) -> str:
+        """Get the task name."""
+        return self._task_name
+
+    def get_description(self) -> str:
+        """Get the task description."""
+        return f"Custom dataset loaded from {self.file_path.name}"
+
+    def get_categories(self) -> List[str]:
+        """Get the task categories."""
+        return ["custom", "file_based", "text_generation"]
+
+    def validate_sample(self, sample: Dict[str, Any]) -> bool:
+        """
+        Validate that a sample has the required format.
+
+        Expected format:
+        {
+            "question": "Question text",
+            "answer": "Expected answer"
+        }
+
+        Optional fields:
+        - "problem": Alternative to "question"
+        - Any other fields will be preserved but ignored
+        """
+        if not isinstance(sample, dict):
+            return False
+
+        # Check for question field (or alternative names)
+        question = sample.get("question") or sample.get("problem")
+        if not question or not isinstance(question, str):
+            return False
+
+        # Check for answer field
+        answer = sample.get("answer")
+        if answer is None:
+            return False
+
+        return True
+
+    # Methods to match lm-eval interface
+    def has_validation_docs(self) -> bool:
+        """Check if task has validation documents."""
+        return False  # File tasks don't have separate validation sets
+
+    def has_test_docs(self) -> bool:
+        """Check if task has test documents."""
+        return True  # All samples are considered test docs
+
+    def test_docs(self) -> List[Dict[str, Any]]:
+        """Get test documents."""
+        if self._data is None:
+            self._data = self.load_data()
+        return self._data
+
+    def validation_docs(self) -> List[Dict[str, Any]]:
+        """Get validation documents."""
+        return []  # No separate validation set
+
+    def doc_to_text(self, doc: Dict[str, Any]) -> str:
+        """Convert document to text prompt."""
+        question = doc.get("question") or doc.get("problem", "")
+        return f"Question: {question}\nAnswer:"
+
+    def get_task_info(self) -> Dict[str, Any]:
+        """Get information about the file task."""
+        return {
+            "task_name": self._task_name,
+            "description": self.get_description(),
+            "source": str(self.file_path),
+            "task_type": "text_generation",
+            "evaluation_method": "exact_match",
+            "num_samples": len(self.test_docs()) if self._data else "unknown",
+        }
+
+
+def create_file_task(file_path: str, task_name: Optional[str] = None) -> callable:
+    """
+    Create a task factory function for a file-based task.
+
+    This is the recommended way to create file tasks for registration.
+
+    Args:
+        file_path: Path to the JSON dataset file
+        task_name: Optional custom name for the task
+
+    Returns:
+        A factory function that creates FileTask instances
+    """
+
+    def task_factory(limit: Optional[int] = None) -> FileTask:
+        return FileTask(file_path=file_path, task_name=task_name, limit=limit)
+
+    return task_factory
+
+
+def register_file_task(task_name: str, file_path: str, registry=None):
+    """
+    Register a file-based task with the global task registry.
+
+    Args:
+        task_name: Name to register the task under
+        file_path: Path to the JSON dataset file
+        registry: Optional registry to use (defaults to global registry)
+    """
+    from ..task_interface import register_task
+
+    task_factory = create_file_task(file_path, task_name)
+    register_task(task_name, task_factory)
+
+
+def load_tasks_from_directory(directory: str, pattern: str = "*.json", prefix: str = ""):
+    """
+    Load all JSON files in a directory as tasks.
+
+    Args:
+        directory: Directory to search for JSON files
+        pattern: File pattern to match (default: "*.json")
+        prefix: Optional prefix to add to task names
+    """
+    directory_path = Path(directory)
+
+    if not directory_path.exists():
+        raise FileNotFoundError(f"Directory not found: {directory}")
+
+    if not directory_path.is_dir():
+        raise ValueError(f"Path is not a directory: {directory}")
+
+    loaded_tasks = []
+
+    for json_file in directory_path.glob(pattern):
+        try:
+            task_name = f"{prefix}{json_file.stem}".lower()
+            register_file_task(task_name, str(json_file))
+            loaded_tasks.append(task_name)
+        except Exception as e:
+            print(f"Warning: Failed to load task from {json_file}: {e}")
+
+    return loaded_tasks

wisent/core/tasks/hle_task.py

@@ -0,0 +1,180 @@
+"""
+HLE (Human-Level Evaluation) task implementation for task-agnostic architecture.
+"""
+
+from typing import Dict, Any, List, Optional
+from datasets import load_dataset
+from ..task_interface import TaskInterface
+from ..benchmark_extractors import HLEExtractor
+
+
+class HLETask(TaskInterface):
+    """HLE (Human-Level Evaluation) task implementation."""
+    
+    def __init__(self, category_filter: Optional[str] = None, answer_type_filter: Optional[str] = None, 
+                 limit: Optional[int] = None):
+        """Initialize HLE task.
+        
+        Args:
+            category_filter: Filter by category (Math, Physics, CS, etc.)
+            answer_type_filter: Filter by answer type ('exactMatch' or 'multipleChoice')
+            limit: Maximum number of examples to load
+        """
+        self.dataset_name = "cais/hle"
+        self.category_filter = category_filter
+        self.answer_type_filter = answer_type_filter
+        self.limit = limit
+        self._extractor = HLEExtractor()
+        self._data = None  # Cache for loaded data
+    
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load HLE data from HuggingFace datasets."""
+        dataset = load_dataset(self.dataset_name, split="test")
+        
+        # Filter out multimodal examples for initial implementation
+        text_only_data = [
+            item for item in dataset 
+            if not item.get('image') and not item.get('image_1') and not item.get('image_2')
+        ]
+        
+        # Apply additional filters
+        filtered_data = self._filter_and_process(text_only_data)
+        
+        # Apply limit
+        effective_limit = limit or self.limit
+        if effective_limit:
+            filtered_data = filtered_data[:effective_limit]
+        
+        return filtered_data
+    
+    def _filter_and_process(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Filter data by category and answer type, and convert to internal format."""
+        filtered_data = []
+        
+        for item in data:
+            # Apply category filter
+            if self.category_filter and item.get('category') != self.category_filter:
+                continue
+            
+            # Apply answer type filter
+            if self.answer_type_filter and item.get('answer_type') != self.answer_type_filter:
+                continue
+            
+            # Convert to internal format
+            processed_item = {
+                'question': item.get('question', ''),
+                'answer': item.get('answer', ''),
+                'answer_type': item.get('answer_type', ''),
+                'category': item.get('category', ''),
+                'raw_subject': item.get('raw_subject', ''),
+                'rationale': item.get('rationale', ''),
+                'author_name': item.get('author_name', ''),
+                'id': item.get('id', ''),
+                'metadata': {
+                    'canary': item.get('canary', ''),
+                    'dataset': self.dataset_name
+                }
+            }
+            
+            # For multiple choice, parse choices from question text if needed
+            if item.get('answer_type') == 'multipleChoice':
+                processed_item['parsed_choices'] = self._parse_choices_from_question(item.get('question', ''))
+            
+            filtered_data.append(processed_item)
+        
+        return filtered_data
+    
+    def _parse_choices_from_question(self, question: str) -> List[str]:
+        """Parse multiple choice options from question text."""
+        # Look for patterns like "A. ", "B. ", etc.
+        import re
+        choices = []
+        patterns = [
+            r'([A-E])\.\s+(.+?)(?=\n[A-E]\.|$)',  # "A. option" format
+            r'([A-E])\)\s+(.+?)(?=\n[A-E]\)|$)',  # "A) option" format
+        ]
+        
+        for pattern in patterns:
+            matches = re.findall(pattern, question, re.MULTILINE | re.DOTALL)
+            if matches:
+                choices = [f"{letter}. {text.strip()}" for letter, text in matches]
+                break
+        
+        return choices
+    
+    def get_extractor(self) -> HLEExtractor:
+        """Get the HLE benchmark extractor."""
+        return self._extractor
+    
+    def get_name(self) -> str:
+        """Get the task name."""
+        return "hle"
+    
+    def get_description(self) -> str:
+        """Get the task description."""
+        desc = "HLE (Human-Level Evaluation): Multimodal benchmark for human-level reasoning across multiple domains"
+        if self.category_filter:
+            desc += f" (filtered to {self.category_filter})"
+        if self.answer_type_filter:
+            desc += f" ({self.answer_type_filter} questions only)"
+        return desc
+    
+    def get_categories(self) -> List[str]:
+        """Get the task categories."""
+        return ["reasoning", "knowledge", "multimodal", "evaluation"]
+    
+    # Methods to match lm-eval interface
+    def has_validation_docs(self) -> bool:
+        """Check if task has validation documents."""
+        return False  # HLE doesn't have separate validation sets
+    
+    def has_test_docs(self) -> bool:
+        """Check if task has test documents."""
+        return True  # All samples are considered test docs
+    
+    def test_docs(self) -> List[Dict[str, Any]]:
+        """Get test documents."""
+        if self._data is None:
+            self._data = self.load_data()
+        return self._data
+    
+    def validation_docs(self) -> List[Dict[str, Any]]:
+        """Get validation documents."""
+        return []  # No separate validation set
+    
+    def doc_to_text(self, doc: Dict[str, Any]) -> str:
+        """Convert document to text prompt."""
+        # For HLE, the question already contains the choices for multiple choice
+        return doc.get('question', '')
+
+
+class HLEExactMatchTask(HLETask):
+    """HLE task filtered to exact match questions only."""
+    
+    def __init__(self, category_filter: Optional[str] = None, limit: Optional[int] = None):
+        super().__init__(category_filter=category_filter, answer_type_filter='exactMatch', limit=limit)
+    
+    def get_name(self) -> str:
+        return "hle_exact_match"
+    
+    def get_description(self) -> str:
+        desc = "HLE Exact Match: Text-based questions requiring exact string matching"
+        if self.category_filter:
+            desc += f" (filtered to {self.category_filter})"
+        return desc
+
+
+class HLEMultipleChoiceTask(HLETask):
+    """HLE task filtered to multiple choice questions only."""
+    
+    def __init__(self, category_filter: Optional[str] = None, limit: Optional[int] = None):
+        super().__init__(category_filter=category_filter, answer_type_filter='multipleChoice', limit=limit)
+    
+    def get_name(self) -> str:
+        return "hle_multiple_choice"
+    
+    def get_description(self) -> str:
+        desc = "HLE Multiple Choice: Questions with A/B/C/D/E answer options"
+        if self.category_filter:
+            desc += f" (filtered to {self.category_filter})"
+        return desc

wisent/core/tasks/hmmt_task.py

@@ -0,0 +1,119 @@
+"""
+HMMT (Harvard-MIT Math Tournament) task implementation for task-agnostic architecture.
+"""
+
+from typing import Dict, Any, List, Optional
+from ..task_interface import TaskInterface
+from ..benchmark_extractors import GSM8KExtractor
+import datasets
+
+
+class HMMTTask(TaskInterface):
+    """HMMT (Harvard-MIT Math Tournament) mathematical contest task implementation."""
+    
+    # Dataset configurations for different HMMT competitions
+    DATASET_CONFIGS = {
+        "feb_2025": {
+            "source": "MathArena/hmmt_feb_2025",
+            "split": "train",
+            "fields": {"problem": "problem", "answer": "answer"},
+            "description": "30 high-difficulty HMMT February 2025 contest problems"
+        }
+    }
+    
+    def __init__(self, competition: str = "feb_2025", limit: Optional[int] = None):
+        """
+        Initialize HMMT task for specified competition.
+        
+        Args:
+            competition: HMMT competition to load ("feb_2025"). Default: "feb_2025" (latest)
+            limit: Maximum number of samples to load
+        """
+        if competition not in self.DATASET_CONFIGS:
+            available = list(self.DATASET_CONFIGS.keys())
+            raise ValueError(f"HMMT competition '{competition}' not supported. Available: {available}")
+            
+        self.competition = competition
+        self.config = self.DATASET_CONFIGS[competition]
+        self._limit = limit
+        self._data = None  # Cache for loaded data
+        self._extractor = GSM8KExtractor()  # Reuse enhanced GSM8K extractor
+    
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load HMMT data from HuggingFace for specified competition."""
+        # Load dataset based on competition configuration
+        dataset = datasets.load_dataset(
+            self.config["source"], 
+            split=self.config["split"]
+        )
+        
+        # Apply limit
+        effective_limit = limit or self._limit
+        if effective_limit:
+            dataset = dataset.select(range(min(effective_limit, len(dataset))))
+        
+        # Convert to list and normalize field names
+        data = [dict(item) for item in dataset]
+        
+        # Normalize field names for consistent processing
+        normalized_data = []
+        problem_field = self.config["fields"]["problem"]
+        answer_field = self.config["fields"]["answer"]
+        
+        for item in data:
+            normalized_item = dict(item)  # Keep all original fields
+            
+            # Ensure consistent field names for extractor
+            if problem_field in item:
+                normalized_item["Problem"] = item[problem_field]
+                normalized_item["question"] = item[problem_field]  # For question/answer format
+            
+            if answer_field in item:
+                normalized_item["Answer"] = item[answer_field]
+                normalized_item["answer"] = item[answer_field]  # For question/answer format
+            
+            normalized_data.append(normalized_item)
+        
+        return normalized_data
+            
+    
+    def get_task_info(self) -> Dict[str, Any]:
+        """Get information about the HMMT task."""
+        return {
+            "task_name": f"hmmt_{self.competition}" if self.competition != "feb_2025" else "hmmt",
+            "competition": self.competition,
+            "description": self.config["description"],
+            "source": self.config["source"],
+            "task_type": "text_generation",
+            "evaluation_method": "mathematical_equivalence"
+        }
+    
+    def validate_sample(self, sample: Dict[str, Any]) -> bool:
+        """Validate that a sample has required HMMT fields."""
+        problem_field = self.config["fields"]["problem"]
+        answer_field = self.config["fields"]["answer"]
+        
+        return all(field in sample for field in [problem_field, answer_field])
+    
+    def get_extractor(self) -> GSM8KExtractor:
+        """Get the benchmark extractor for this task."""
+        return self._extractor
+    
+    def get_name(self) -> str:
+        """Get the task name."""
+        return f"hmmt_{self.competition}" if self.competition != "feb_2025" else "hmmt"
+    
+    def get_description(self) -> str:
+        """Get the task description."""
+        return f"HMMT {self.competition.replace('_', ' ').title()} contest problems requiring advanced mathematical reasoning"
+    
+    def get_categories(self) -> List[str]:
+        """Get the task categories."""
+        return ["mathematics", "reasoning", "contest", "text_generation"]
+    
+    @classmethod
+    def get_supported_competitions(cls) -> List[str]:
+        """Get list of supported HMMT competitions."""
+        return list(cls.DATASET_CONFIGS.keys())
+
+