wisent 0.1.1__py3-none-any.whl → 0.5.1__py3-none-any.whl

wisent/core/tasks/lm_eval_task.py

@@ -0,0 +1,455 @@
+"""
+LM-Evaluation-Harness task wrapper for task-agnostic architecture.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from ..benchmark_extractors import BenchmarkExtractor, get_extractor
+from ..task_interface import TaskInterface
+
+
+class LMEvalTask(TaskInterface):
+    """Wrapper for lm-evaluation-harness tasks."""
+
+    def __init__(self, task_name: str, description: str, categories: List[str]):
+        self.task_name = task_name
+        self._description = description
+        self._categories = categories
+        self._extractor = get_extractor(task_name)
+
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load data directly from lm-eval without Model dependency."""
+        try:
+            # Load data directly from lm-eval without creating a Model instance
+            from lm_eval.tasks import get_task_dict
+
+            # Get task directly from lm-eval
+            task_dict = get_task_dict([self.task_name])
+            if self.task_name not in task_dict:
+                print(f"Warning: Task '{self.task_name}' not found in lm-eval")
+                return []
+
+            task = task_dict[self.task_name]
+
+            # Get the task's test documents
+            docs = []
+            if hasattr(task, "test_docs"):
+                # For lm-eval versions with test_docs method
+                docs = list(task.test_docs())
+            elif hasattr(task, "dataset"):
+                # For newer lm-eval versions
+                dataset = task.dataset
+                if hasattr(dataset, "test"):
+                    docs = list(dataset.test)
+                elif hasattr(dataset, "validation"):
+                    docs = list(dataset.validation)
+                else:
+                    # Fallback to the main dataset
+                    docs = list(dataset)
+
+            # Ensure docs are in dictionary format
+            processed_docs = []
+            for doc in docs:
+                if isinstance(doc, dict):
+                    processed_docs.append(doc)
+                elif isinstance(doc, str):
+                    # Handle string documents by wrapping them
+                    processed_docs.append({"text": doc})
+                else:
+                    # Try to convert to dict if possible
+                    try:
+                        processed_docs.append(dict(doc))
+                    except:
+                        processed_docs.append({"data": str(doc)})
+
+            docs = processed_docs
+
+            # Apply limit if specified
+            if limit and len(docs) > limit:
+                docs = docs[:limit]
+
+            return docs
+
+        except Exception as e:
+            print(f"Warning: Could not load lm-eval task '{self.task_name}': {e}")
+            return []
+
+    def get_extractor(self) -> BenchmarkExtractor:
+        """Get the benchmark extractor for this task."""
+        return self._extractor
+
+    def get_name(self) -> str:
+        """Get the task name."""
+        return self.task_name
+
+    def get_description(self) -> str:
+        """Get the task description."""
+        return self._description
+
+    def get_categories(self) -> List[str]:
+        """Get the task categories."""
+        return self._categories
+
+
+class MBPPTask(LMEvalTask):
+    """MBPP task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="mbpp",
+            description="MBPP: Mostly Basic Python Problems coding benchmark",
+            categories=["coding", "reasoning", "python"],
+        )
+
+
+class HumanEvalTask(LMEvalTask):
+    """HumanEval task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="humaneval",
+            description="HumanEval: Human Evaluation of Python coding problems",
+            categories=["coding", "reasoning", "python"],
+        )
+
+
+class MBPPPlusTask(LMEvalTask):
+    """MBPP Plus task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="mbpp_plus",
+            description="MBPP Plus: Extended version of MBPP with additional test cases",
+            categories=["coding", "reasoning", "python"],
+        )
+
+
+class GSM8KTask(LMEvalTask):
+    """GSM8K task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="gsm8k",
+            description="GSM8K: Grade School Math 8K problems",
+            categories=["mathematics", "reasoning", "arithmetic"],
+        )
+
+
+class TruthfulQATask(LMEvalTask):
+    """TruthfulQA task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="truthfulqa_mc1",
+            description="TruthfulQA: Truthfulness evaluation benchmark",
+            categories=["hallucination", "general-knowledge", "reasoning"],
+        )
+
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load TruthfulQA data, which only has validation split."""
+        try:
+            from lm_eval.tasks import get_task_dict
+
+            # Get task directly from lm-eval
+            task_dict = get_task_dict([self.task_name])
+            if self.task_name not in task_dict:
+                print(f"Warning: Task '{self.task_name}' not found in lm-eval")
+                return []
+
+            task = task_dict[self.task_name]
+
+            # TruthfulQA only has validation split, access it directly
+            docs = []
+            if hasattr(task, "dataset") and "validation" in task.dataset:
+                validation_data = task.dataset["validation"]
+                docs = list(validation_data)
+
+            # Apply limit if specified
+            if limit and len(docs) > limit:
+                docs = docs[:limit]
+
+            return docs
+
+        except Exception as e:
+            print(f"Warning: Could not load TruthfulQA task '{self.task_name}': {e}")
+            import traceback
+
+            traceback.print_exc()
+            return []
+
+
+class MMLUTask(LMEvalTask):
+    """MMLU task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="mmlu",
+            description="MMLU: Massive Multitask Language Understanding",
+            categories=["general-knowledge", "science", "reasoning"],
+        )
+
+
+# === CODING TASKS ===
+
+
+class InstructHumanEvalTask(LMEvalTask):
+    """InstructHumanEval task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="instructhumaneval",
+            description="InstructHumanEval: Instruction-following HumanEval benchmark",
+            categories=["coding", "reasoning", "python", "instruction-following"],
+        )
+
+
+class HumanEvalPlusTask(LMEvalTask):
+    """HumanEval Plus task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="humaneval_plus",
+            description="HumanEval Plus: Extended HumanEval with more tests",
+            categories=["coding", "reasoning", "python"],
+        )
+
+
+class ConalaTask(LMEvalTask):
+    """Conala task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="conala",
+            description="Conala: Code generation from natural language",
+            categories=["coding", "reasoning", "python", "nl2code"],
+        )
+
+
+class ConcodeTask(LMEvalTask):
+    """Concode task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="concode",
+            description="Concode: Code completion benchmark",
+            categories=["coding", "reasoning", "completion"],
+        )
+
+
+class MercuryTask(LMEvalTask):
+    """Mercury task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="mercury",
+            description="Mercury: Code generation benchmark",
+            categories=["coding", "reasoning"],
+        )
+
+
+class AppsTask(LMEvalTask):
+    """APPS task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="apps",
+            description="APPS: Automated Programming Problems Synthesis",
+            categories=["coding", "reasoning", "python", "competitive"],
+        )
+
+
+class DS1000Task(LMEvalTask):
+    """DS1000 task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="ds1000",
+            description="DS1000: Data Science coding tasks",
+            categories=["coding", "reasoning", "python", "data-science"],
+        )
+
+
+class MultiplePyTask(LMEvalTask):
+    """Multiple-Py task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="multiple_py",
+            description="Multiple-Py: Multi-language Python tasks",
+            categories=["coding", "reasoning", "python", "multi-language"],
+        )
+
+
+class MultipleJsTask(LMEvalTask):
+    """Multiple-JS task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="multiple_js",
+            description="Multiple-JS: Multi-language JavaScript tasks",
+            categories=["coding", "reasoning", "javascript", "multi-language"],
+        )
+
+
+class MultipleJavaTask(LMEvalTask):
+    """Multiple-Java task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="multiple_java",
+            description="Multiple-Java: Multi-language Java tasks",
+            categories=["coding", "reasoning", "java", "multi-language"],
+        )
+
+
+class MultipleCppTask(LMEvalTask):
+    """Multiple-Cpp task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="multiple_cpp",
+            description="Multiple-Cpp: Multi-language C++ tasks",
+            categories=["coding", "reasoning", "cpp", "multi-language"],
+        )
+
+
+class MultipleRsTask(LMEvalTask):
+    """Multiple-Rs task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="multiple_rs",
+            description="Multiple-Rs: Multi-language Rust tasks",
+            categories=["coding", "reasoning", "rust", "multi-language"],
+        )
+
+
+class MultipleGoTask(LMEvalTask):
+    """Multiple-Go task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="multiple_go",
+            description="Multiple-Go: Multi-language Go tasks",
+            categories=["coding", "reasoning", "go", "multi-language"],
+        )
+
+
+class CodexglueCodeToTextPythonTask(LMEvalTask):
+    """CodexGlue Code-to-Text Python task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="codexglue_code_to_text_python",
+            description="CodexGlue Code-to-Text Python: Python code summarization",
+            categories=["coding", "reasoning", "python", "code-to-text"],
+        )
+
+
+class CodexglueCodeToTextGoTask(LMEvalTask):
+    """CodexGlue Code-to-Text Go task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="codexglue_code_to_text_go",
+            description="CodexGlue Code-to-Text Go: Go code summarization",
+            categories=["coding", "reasoning", "go", "code-to-text"],
+        )
+
+
+class CodexglueCodeToTextRubyTask(LMEvalTask):
+    """CodexGlue Code-to-Text Ruby task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="codexglue_code_to_text_ruby",
+            description="CodexGlue Code-to-Text Ruby: Ruby code summarization",
+            categories=["coding", "reasoning", "ruby", "code-to-text"],
+        )
+
+
+class CodexglueCodeToTextJavaTask(LMEvalTask):
+    """CodexGlue Code-to-Text Java task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="codexglue_code_to_text_java",
+            description="CodexGlue Code-to-Text Java: Java code summarization",
+            categories=["coding", "reasoning", "java", "code-to-text"],
+        )
+
+
+class CodexglueCodeToTextJavascriptTask(LMEvalTask):
+    """CodexGlue Code-to-Text JavaScript task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="codexglue_code_to_text_javascript",
+            description="CodexGlue Code-to-Text JavaScript: JavaScript code summarization",
+            categories=["coding", "reasoning", "javascript", "code-to-text"],
+        )
+
+
+class CodexglueCodeToTextPhpTask(LMEvalTask):
+    """CodexGlue Code-to-Text PHP task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="codexglue_code_to_text_php",
+            description="CodexGlue Code-to-Text PHP: PHP code summarization",
+            categories=["coding", "reasoning", "php", "code-to-text"],
+        )
+
+
+class RecodeTask(LMEvalTask):
+    """Recode task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="recode",
+            description="Recode: Perturbed HumanEval natural generation",
+            categories=["coding", "reasoning", "python", "perturbation"],
+        )
+
+
+class Squad2Task(LMEvalTask):
+    """SQuAD2 task implementation."""
+
+    def __init__(self):
+        super().__init__(
+            task_name="squadv2",
+            description="SQuAD2: Stanford Question Answering Dataset 2.0",
+            categories=["reading-comprehension", "qa", "natural-language"],
+        )
+
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load SQuAD2 data, which only has validation split."""
+        try:
+            from lm_eval.tasks import get_task_dict
+
+            # Get task directly from lm-eval
+            task_dict = get_task_dict([self.task_name])
+            if self.task_name not in task_dict:
+                print(f"Warning: Task '{self.task_name}' not found in lm-eval")
+                return []
+
+            task = task_dict[self.task_name]
+
+            # SQuAD2 only has validation split, access it directly
+            docs = []
+            if hasattr(task, "dataset") and "validation" in task.dataset:
+                validation_data = task.dataset["validation"]
+                docs = list(validation_data)
+
+            # Apply limit if specified
+            if limit and len(docs) > limit:
+                docs = docs[:limit]
+
+            return docs
+
+        except Exception as e:
+            print(f"Warning: Could not load SQuAD2 task '{self.task_name}': {e}")
+            import traceback
+
+            traceback.print_exc()
+            return []

wisent/core/tasks/math500_task.py

@@ -0,0 +1,84 @@
+"""
+MATH-500 task implementation for task-agnostic architecture.
+"""
+
+from typing import Dict, Any, List, Optional
+from ..task_interface import TaskInterface
+from ..benchmark_extractors import GSM8KExtractor
+import datasets
+
+
+class Math500Task(TaskInterface):
+    """MATH-500 mathematical reasoning task implementation."""
+    
+    def __init__(self, limit: Optional[int] = None):
+        self._limit = limit
+        self._data = None  # Cache for loaded data
+        self._extractor = GSM8KExtractor()  # Reuse GSM8K extractor
+    
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load MATH-500 data from HuggingFace."""
+        dataset = datasets.load_dataset("HuggingFaceH4/MATH-500", split="test")
+        
+        # Apply limit
+        effective_limit = limit or self._limit
+        if effective_limit:
+            dataset = dataset.select(range(min(effective_limit, len(dataset))))
+        
+        # Convert to list of dictionaries
+        return [dict(item) for item in dataset]
+            
+    
+    def get_task_info(self) -> Dict[str, Any]:
+        """Get information about the MATH-500 task."""
+        return {
+            "task_name": "math500",
+            "description": "500 mathematical reasoning problems from OpenAI's MATH dataset",
+            "source": "HuggingFaceH4/MATH-500",
+            "task_type": "text_generation",
+            "evaluation_method": "mathematical_equivalence"
+        }
+    
+    def validate_sample(self, sample: Dict[str, Any]) -> bool:
+        """Validate that a sample has required MATH-500 fields."""
+        required_fields = ["problem", "answer"]
+        return all(field in sample for field in required_fields)
+    
+    def get_extractor(self) -> GSM8KExtractor:
+        """Get the benchmark extractor for this task."""
+        return self._extractor
+    
+    def get_name(self) -> str:
+        """Get the task name."""
+        return "math500"
+    
+    def get_description(self) -> str:
+        """Get the task description."""
+        return "500 mathematical reasoning problems from OpenAI's MATH dataset requiring multi-step solutions"
+    
+    def get_categories(self) -> List[str]:
+        """Get the task categories."""
+        return ["mathematics", "reasoning", "text_generation"]
+    
+    # Methods to match lm-eval interface
+    def has_validation_docs(self) -> bool:
+        """Check if task has validation documents."""
+        return False  # MATH-500 doesn't have separate validation sets
+    
+    def has_test_docs(self) -> bool:
+        """Check if task has test documents."""
+        return True  # All samples are considered test docs
+    
+    def test_docs(self) -> List[Dict[str, Any]]:
+        """Get test documents."""
+        if self._data is None:
+            self._data = self.load_data()
+        return self._data
+    
+    def validation_docs(self) -> List[Dict[str, Any]]:
+        """Get validation documents."""
+        return []  # No separate validation set
+    
+    def doc_to_text(self, doc: Dict[str, Any]) -> str:
+        """Convert document to text prompt."""
+        return doc.get('problem', '')

wisent/core/tasks/polymath_task.py

@@ -0,0 +1,146 @@
+"""
+PolyMath multilingual mathematical reasoning task implementation for task-agnostic architecture.
+"""
+
+from typing import Dict, Any, List, Optional
+from ..task_interface import TaskInterface
+from ..benchmark_extractors import GSM8KExtractor
+import datasets
+
+
+class PolyMathTask(TaskInterface):
+    """PolyMath multilingual mathematical reasoning task implementation."""
+    
+    # Dataset configurations for different language-difficulty combinations
+    DATASET_CONFIGS = {
+        "zh_medium": {
+            "source": "Qwen/PolyMath",
+            "language": "zh",
+            "split": "medium",
+            "fields": {"problem": "question", "answer": "answer"},
+            "description": "125 medium-difficulty mathematical problems in Chinese"
+        },
+        "en_medium": {
+            "source": "Qwen/PolyMath",
+            "language": "en", 
+            "split": "medium",
+            "fields": {"problem": "question", "answer": "answer"},
+            "description": "125 medium-difficulty mathematical problems in English"
+        },
+        "zh_high": {
+            "source": "Qwen/PolyMath",
+            "language": "zh",
+            "split": "high",
+            "fields": {"problem": "question", "answer": "answer"},
+            "description": "125 high-difficulty mathematical problems in Chinese"
+        },
+        "en_high": {
+            "source": "Qwen/PolyMath",
+            "language": "en",
+            "split": "high", 
+            "fields": {"problem": "question", "answer": "answer"},
+            "description": "125 high-difficulty mathematical problems in English"
+        }
+    }
+    
+    def __init__(self, language: str = "en", difficulty: str = "medium", limit: Optional[int] = None):
+        """
+        Initialize PolyMath task for specified language and difficulty.
+        
+        Args:
+            language: Language code ("en" for English, "zh" for Chinese). Default: "en"
+            difficulty: Difficulty level ("medium", "high"). Default: "medium" 
+            limit: Maximum number of samples to load
+        """
+        config_key = f"{language}_{difficulty}"
+        if config_key not in self.DATASET_CONFIGS:
+            available = list(self.DATASET_CONFIGS.keys())
+            raise ValueError(f"PolyMath config '{config_key}' not supported. Available: {available}")
+            
+        self.language = language
+        self.difficulty = difficulty
+        self.config_key = config_key
+        self.config = self.DATASET_CONFIGS[config_key]
+        self._limit = limit
+        self._data = None  # Cache for loaded data
+        self._extractor = GSM8KExtractor()  # Reuse enhanced GSM8K extractor
+    
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load PolyMath data from HuggingFace for specified language and difficulty."""
+        # Load dataset based on language and difficulty configuration
+        dataset = datasets.load_dataset(
+            self.config["source"],
+            self.config["language"], 
+            split=self.config["split"]
+        )
+        
+        # Apply limit
+        effective_limit = limit or self._limit
+        if effective_limit:
+            dataset = dataset.select(range(min(effective_limit, len(dataset))))
+        
+        # Convert to list and normalize field names
+        data = [dict(item) for item in dataset]
+        
+        # Normalize field names for consistent processing
+        normalized_data = []
+        problem_field = self.config["fields"]["problem"]
+        answer_field = self.config["fields"]["answer"]
+        
+        for item in data:
+            normalized_item = dict(item)  # Keep all original fields
+            
+            # Ensure consistent field names for extractor
+            if problem_field in item:
+                normalized_item["Problem"] = item[problem_field]
+                normalized_item["question"] = item[problem_field]  # For question/answer format
+            
+            if answer_field in item:
+                normalized_item["Answer"] = item[answer_field]
+                normalized_item["answer"] = item[answer_field]  # For question/answer format
+            
+            normalized_data.append(normalized_item)
+        
+        return normalized_data
+            
+    
+    def get_task_info(self) -> Dict[str, Any]:
+        """Get information about the PolyMath task."""
+        return {
+            "task_name": f"polymath_{self.config_key}",
+            "language": self.language,
+            "difficulty": self.difficulty,
+            "description": self.config["description"],
+            "source": self.config["source"],
+            "task_type": "text_generation",
+            "evaluation_method": "mathematical_equivalence"
+        }
+    
+    def validate_sample(self, sample: Dict[str, Any]) -> bool:
+        """Validate that a sample has required PolyMath fields."""
+        problem_field = self.config["fields"]["problem"]
+        answer_field = self.config["fields"]["answer"]
+        
+        return all(field in sample for field in [problem_field, answer_field])
+    
+    def get_extractor(self) -> GSM8KExtractor:
+        """Get the benchmark extractor for this task."""
+        return self._extractor
+    
+    def get_name(self) -> str:
+        """Get the task name."""
+        return f"polymath_{self.config_key}"
+    
+    def get_description(self) -> str:
+        """Get the task description."""
+        lang_name = "Chinese" if self.language == "zh" else "English"
+        return f"PolyMath {self.difficulty}-difficulty mathematical problems in {lang_name}"
+    
+    def get_categories(self) -> List[str]:
+        """Get the task categories."""
+        return ["mathematics", "reasoning", "multilingual", "text_generation"]
+    
+    @classmethod
+    def get_supported_configs(cls) -> List[str]:
+        """Get list of supported PolyMath language-difficulty configurations."""
+        return list(cls.DATASET_CONFIGS.keys())