wisent 0.5.11__py3-none-any.whl → 0.5.13__py3-none-any.whl

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_pairs_generation.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from wisent.core.contrastive_pairs.lm_eval_pairs.lm_extractor_registry import get_extractor
-from wisent.cli.cli_logger import setup_logger, bind
+from wisent.core.cli_logger import setup_logger, bind
 
 if TYPE_CHECKING:
     from lm_eval.api.task import ConfigurableTask

wisent/core/data_loaders/__init__.py

@@ -0,0 +1,235 @@
+"""
+Data loaders for various benchmarks.
+
+This module provides data loaders for tasks that need special handling.
+"""
+
+from typing import Any, Dict, List, Optional
+from dataclasses import dataclass
+
+
+__all__ = ["LiveCodeBenchLoader", "LiveCodeBenchProblem"]
+
+
+@dataclass
+class LiveCodeBenchProblem:
+    """
+    A LiveCodeBench coding problem.
+
+    Attributes:
+        question_title: Title of the problem
+        question_content: Full problem description
+        platform: Source platform (codeforces, leetcode, atcoder)
+        question_id: Unique problem identifier
+        contest_id: Contest identifier
+        contest_date: Date of the contest
+        starter_code: Optional starter code template
+        difficulty: Problem difficulty (easy, medium, hard)
+        public_test_cases: Public test cases
+        private_test_cases: Private test cases
+        metadata: Additional metadata
+        answer: Correct answer/solution (for TaskInterface compatibility)
+        good_code: Code that passes tests (from wisent-core)
+        bad_code: Code that fails tests (from wisent-core)
+    """
+    question_title: str
+    question_content: str
+    platform: str
+    question_id: str
+    contest_id: str
+    contest_date: str
+    starter_code: str
+    difficulty: str
+    public_test_cases: List[Any]
+    private_test_cases: List[Any]
+    metadata: Dict[str, Any]
+    answer: Optional[str] = None
+    good_code: Optional[str] = None
+    bad_code: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary format."""
+        return {
+            "task_id": self.question_id,
+            "question_id": self.question_id,  # Also include question_id for compatibility
+            "question_title": self.question_title,
+            "question_content": self.question_content,
+            "platform": self.platform.upper(),
+            "contest_id": self.contest_id,
+            "contest_date": self.contest_date,
+            "starter_code": self.starter_code,
+            "difficulty": self.difficulty.upper(),
+            "public_test_cases": [
+                {
+                    "input": tc if isinstance(tc, str) else str(tc),
+                    "output": "",
+                    "testtype": "FUNCTIONAL"
+                }
+                for tc in self.public_test_cases[:3]  # Limit to first 3 for brevity
+            ] if self.public_test_cases else [],
+            "metadata": self.metadata,
+            "answer": self.good_code,  # Use good_code as the correct answer
+            "good_code": self.good_code,
+            "bad_code": self.bad_code,
+        }
+
+
+class LiveCodeBenchLoader:
+    """
+    LiveCodeBench data loader.
+
+    Loads real coding problems from the LiveCodeBench dataset on HuggingFace.
+    Dataset: livecodebench/code_generation_lite
+
+    Also loads pre-generated good/bad code solutions from local cache.
+    """
+
+    def __init__(self, solution_cache_dir: Optional[str] = None):
+        self._dataset_name = "livecodebench/code_generation_lite"
+        self._cached_dataset = None
+        self._solution_data = None
+        self.solution_cache_dir = solution_cache_dir or "./livecodebench_solutions"
+
+    def list_available_versions(self) -> List[str]:
+        """List available LiveCodeBench versions."""
+        # The dataset doesn't have explicit versions, but we can filter by date ranges
+        return ["release_v1", "release_v2", "all"]
+
+    def get_version_info(self, version: str) -> Dict[str, Any]:
+        """Get information about a specific version."""
+        version_info = {
+            "release_v1": {
+                "version": "release_v1",
+                "description": "LiveCodeBench Release V1 (May 2023 - Oct 2023)",
+                "problems": "~500",
+                "date_range": "2023-05-01 to 2023-10-31",
+            },
+            "release_v2": {
+                "version": "release_v2",
+                "description": "LiveCodeBench Release V2 (Nov 2023 - Apr 2024)",
+                "problems": "~500",
+                "date_range": "2023-11-01 to 2024-04-30",
+            },
+            "all": {
+                "version": "all",
+                "description": "All LiveCodeBench problems",
+                "problems": "1055",
+                "date_range": "2023-05-01 to 2024-12-31",
+            }
+        }
+        return version_info.get(version, version_info["all"])
+
+    def _load_solution_data(self) -> Dict[str, Any]:
+        """
+        Load pre-generated AI model solutions from local cache.
+
+        Returns:
+            Dictionary with question_id -> {good_code, bad_code, difficulty} mapping.
+        """
+        import json
+        from pathlib import Path
+
+        if self._solution_data is not None:
+            return self._solution_data
+
+        cache_file = Path(self.solution_cache_dir) / "solutions.json"
+
+        if not cache_file.exists():
+            import logging
+            logging.warning(
+                f"Solutions cache not found at {cache_file}. "
+                f"Run solution generation first using LiveCodeBenchSolutionGenerator. "
+                f"Problems will have no answer field."
+            )
+            self._solution_data = {}
+            return {}
+
+        with open(cache_file, 'r') as f:
+            data = json.load(f)
+
+        # Create mapping from question_id to solutions
+        solution_map = {}
+        for problem in data.get("problems", []):
+            question_id = problem.get("question_id")
+            if question_id and problem.get("good_example") and problem.get("bad_example"):
+                solution_map[question_id] = {
+                    "good_code": problem["good_example"].get("code", ""),
+                    "bad_code": problem["bad_example"].get("code", ""),
+                    "difficulty": problem.get("difficulty", "unknown"),
+                }
+
+        self._solution_data = solution_map
+        return solution_map
+
+    def load_problems(
+        self,
+        release_version: str = "all",
+        limit: Optional[int] = None
+    ) -> List[LiveCodeBenchProblem]:
+        """
+        Load LiveCodeBench problems from HuggingFace.
+
+        Arguments:
+            release_version: Version to load (release_v1, release_v2, or all)
+            limit: Maximum number of problems to load
+
+        Returns:
+            List of LiveCodeBenchProblem objects
+        """
+        from datasets import load_dataset
+
+        # Load dataset (cached after first load)
+        if self._cached_dataset is None:
+            self._cached_dataset = load_dataset(self._dataset_name, split="test")
+
+        dataset = self._cached_dataset
+
+        # Filter by version if needed
+        if release_version == "release_v1":
+            # Filter problems from May 2023 - Oct 2023
+            dataset = dataset.filter(
+                lambda x: x["contest_date"] >= "2023-05-01" and x["contest_date"] <= "2023-10-31"
+            )
+        elif release_version == "release_v2":
+            # Filter problems from Nov 2023 - Apr 2024
+            dataset = dataset.filter(
+                lambda x: x["contest_date"] >= "2023-11-01" and x["contest_date"] <= "2024-04-30"
+            )
+        # "all" or any other value: use all problems
+
+        # Apply limit
+        if limit:
+            dataset = dataset.select(range(min(limit, len(dataset))))
+
+        # Load solution data from wisent-core
+        solution_map = self._load_solution_data()
+
+        # Convert to LiveCodeBenchProblem objects
+        problems = []
+        for item in dataset:
+            question_id = item.get("question_id", "")
+
+            # Get solutions if available
+            solutions = solution_map.get(question_id, {})
+            good_code = solutions.get("good_code")
+            bad_code = solutions.get("bad_code")
+
+            problem = LiveCodeBenchProblem(
+                question_title=item.get("question_title", ""),
+                question_content=item.get("question_content", ""),
+                platform=item.get("platform", ""),
+                question_id=question_id,
+                contest_id=str(item.get("contest_id", "")),
+                contest_date=item.get("contest_date", ""),
+                starter_code=item.get("starter_code", ""),
+                difficulty=item.get("difficulty", ""),
+                public_test_cases=item.get("public_test_cases", []),
+                private_test_cases=item.get("private_test_cases", []),
+                metadata=item.get("metadata", {}),
+                answer=good_code,  # Set answer field for TaskInterface
+                good_code=good_code,
+                bad_code=bad_code,
+            )
+            problems.append(problem)
+
+        return problems

wisent/core/data_loaders/loaders/lm_loader.py

@@ -91,8 +91,8 @@ class LMEvalDataLoader(BaseDataLoader):
         train_set = ContrastivePairSet("lm_eval_train", train_pairs, task_type=task_name)
         test_set = ContrastivePairSet("lm_eval_test", test_pairs, task_type=task_name)
 
-        train_set.validate()
-        test_set.validate()
+        train_set.validate(raise_on_critical=False)
+        test_set.validate(raise_on_critical=False)
 
         return LoadDataResult(
             train_qa_pairs=train_set,

wisent/core/data_loaders/loaders/task_interface_loader.py

@@ -0,0 +1,300 @@
+from __future__ import annotations
+from typing import Any, Optional
+import logging
+import random
+
+from wisent.core.data_loaders.core.atoms import BaseDataLoader, DataLoaderError, LoadDataResult
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import PositiveResponse, NegativeResponse
+from wisent.core.contrastive_pairs.core.set import ContrastivePairSet
+from wisent.core.task_interface import get_task, list_tasks
+from wisent.core.task_interface import TaskInterface
+
+__all__ = [
+    "TaskInterfaceDataLoader",
+]
+
+log = logging.getLogger(__name__)
+
+
+class TaskInterfaceDataLoader(BaseDataLoader):
+    """
+    Load contrastive pairs from TaskInterface tasks (AIME, HMMT, LiveCodeBench, etc.).
+
+    This loader bridges TaskInterface tasks with the CLI training pipeline by:
+    1. Loading problem data from TaskInterface tasks
+    2. Converting problems into contrastive pairs (correct/incorrect answers)
+    3. Splitting into train/test sets
+
+    Usage:
+        wisent train model meta-llama/Llama-3.2-1B-Instruct loader task_interface task gsm8k training_limit 100
+        wisent train model meta-llama/Llama-3.2-1B-Instruct loader task_interface task aime training_limit 50
+        wisent train model meta-llama/Llama-3.2-1B-Instruct loader task_interface task livecodebench training_limit 200
+    """
+
+    name = "task_interface"
+    description = "Load from TaskInterface tasks (AIME, HMMT, LiveCodeBench, GSM8K, etc.)"
+
+    def load(
+        self,
+        task: Optional[str] = None,
+        split_ratio: Optional[float] = None,
+        seed: int = 42,
+        limit: Optional[int] = None,
+        training_limit: Optional[int] = None,
+        testing_limit: Optional[int] = None,
+        **kwargs: Any,
+    ) -> LoadDataResult:
+        """
+        Load contrastive pairs from a TaskInterface task.
+
+        Arguments:
+            task: Name of the TaskInterface task (e.g., 'gsm8k', 'aime', 'livecodebench')
+            split_ratio: Fraction of data for training (default: 0.8)
+            seed: Random seed for splitting
+            limit: Total number of problems to load
+            training_limit: Maximum training examples
+            testing_limit: Maximum testing examples
+            **kwargs: Additional arguments passed to the task
+
+        Returns:
+            LoadDataResult with train/test contrastive pairs
+
+        Raises:
+            DataLoaderError: If task is not specified or not found
+        """
+        if not task:
+            available = list_tasks()
+            raise DataLoaderError(
+                f"TaskInterface loader requires a 'task' parameter. "
+                f"Available tasks: {', '.join(available[:10])}..."
+            )
+
+        # Ensure split ratio is valid
+        split_ratio = self._effective_split(split_ratio)
+
+        # Load the task
+        try:
+            task_obj: TaskInterface = get_task(task, limit=limit)
+        except ValueError as e:
+            available = list_tasks()
+            raise DataLoaderError(
+                f"TaskInterface task '{task}' not found. "
+                f"Available tasks: {', '.join(available[:20])}..."
+            ) from e
+
+        # Load problem data
+        log.info(f"Loading data from TaskInterface task: {task}")
+        problems = task_obj.load_data(limit=limit)
+
+        if not problems:
+            raise DataLoaderError(f"TaskInterface task '{task}' returned no data")
+
+        log.info(f"Loaded {len(problems)} problems from {task}")
+
+        # Convert problems to contrastive pairs
+        pairs = self._convert_to_contrastive_pairs(task_obj, problems)
+
+        if not pairs:
+            raise DataLoaderError(
+                f"Could not generate any contrastive pairs from {task}. "
+                f"Problems may be missing required fields."
+            )
+
+        log.info(f"Generated {len(pairs)} contrastive pairs")
+
+        # Shuffle and split
+        random.seed(seed)
+        random.shuffle(pairs)
+
+        split_idx = int(len(pairs) * split_ratio)
+        train_pairs = pairs[:split_idx]
+        test_pairs = pairs[split_idx:]
+
+        # Apply limits
+        if training_limit:
+            train_pairs = train_pairs[:training_limit]
+        if testing_limit:
+            test_pairs = test_pairs[:testing_limit]
+
+        log.info(f"Split: {len(train_pairs)} train, {len(test_pairs)} test")
+
+        # Create ContrastivePairSets
+        train_set = ContrastivePairSet(name=f"{task}_train", pairs=train_pairs, task_type="classification")
+        test_set = ContrastivePairSet(name=f"{task}_test", pairs=test_pairs, task_type="classification")
+
+        return LoadDataResult(
+            train_qa_pairs=train_set,
+            test_qa_pairs=test_set,
+            task_type="classification",
+            lm_task_data=None,  # TaskInterface tasks don't use lm-eval format
+        )
+
+    def _convert_to_contrastive_pairs(
+        self,
+        task_obj: TaskInterface,
+        problems: list[dict[str, Any]],
+    ) -> list[ContrastivePair]:
+        """
+        Convert task problems into contrastive pairs.
+
+        For each problem, we create a contrastive pair with:
+        - Positive response: The correct answer
+        - Negative response: An incorrect answer (generated or from problem data)
+
+        Arguments:
+            task_obj: The TaskInterface task object
+            problems: List of problem dictionaries
+
+        Returns:
+            List of ContrastivePair objects
+        """
+        pairs = []
+        extractor = task_obj.get_extractor()
+        task_name = task_obj.get_name()
+
+        for idx, problem in enumerate(problems):
+            try:
+                # Get the prompt/question
+                if hasattr(task_obj, 'doc_to_text'):
+                    prompt = task_obj.doc_to_text(problem)
+                else:
+                    # Fallback: extract prompt from problem dict
+                    prompt = self._extract_prompt_from_problem(problem, task_name)
+
+                if not prompt:
+                    log.warning(f"Problem {idx} has no prompt, skipping")
+                    continue
+
+                # Get correct answer
+                correct_answer = self._get_correct_answer(problem, task_name)
+                if not correct_answer:
+                    log.warning(f"Problem {idx} has no correct answer, skipping")
+                    continue
+
+                # Generate incorrect answer
+                incorrect_answer = self._generate_incorrect_answer(
+                    problem, correct_answer, task_name, extractor
+                )
+                if not incorrect_answer:
+                    log.warning(f"Problem {idx}: could not generate incorrect answer, skipping")
+                    continue
+
+                # Create contrastive pair
+                pair = ContrastivePair(
+                    prompt=prompt,
+                    positive_response=PositiveResponse(model_response=correct_answer),
+                    negative_response=NegativeResponse(model_response=incorrect_answer),
+                    label="correct",
+                )
+                pairs.append(pair)
+
+            except Exception as e:
+                log.warning(f"Problem {idx}: failed to create pair: {e}")
+                continue
+
+        return pairs
+
+    def _extract_prompt_from_problem(self, problem: dict[str, Any], task_name: str) -> Optional[str]:
+        """Extract prompt/question from a problem dict."""
+        # Try common field names for prompts
+        prompt_fields = ["question", "prompt", "problem", "text", "input", "query", "doc"]
+
+        for field in prompt_fields:
+            if field in problem:
+                value = problem[field]
+                if isinstance(value, str) and value.strip():
+                    return value.strip()
+
+        # If no prompt found, try to construct from problem data
+        if task_name == "gsm8k" and "question" in problem:
+            return problem["question"]
+
+        return None
+
+    def _get_correct_answer(self, problem: dict[str, Any], task_name: str) -> Optional[str]:
+        """Extract the correct answer from a problem."""
+        # Try common field names
+        answer_fields = ["answer", "target", "label", "solution", "expected_output"]
+
+        for field in answer_fields:
+            if field in problem:
+                answer = problem[field]
+                if isinstance(answer, (str, int, float)):
+                    return str(answer)
+                elif isinstance(answer, dict):
+                    # Try to extract from nested dict
+                    if "answer" in answer:
+                        return str(answer["answer"])
+                    if "text" in answer:
+                        return str(answer["text"])
+
+        return None
+
+    def _generate_incorrect_answer(
+        self,
+        problem: dict[str, Any],
+        correct_answer: str,
+        task_name: str,
+        extractor: Any,
+    ) -> Optional[str]:
+        """
+        Generate an incorrect answer for a problem.
+
+        Strategy:
+        1. Check if problem has bad_code/incorrect answer field (for LiveCodeBench)
+        2. If problem has multiple choices, use an incorrect choice
+        3. For numerical answers, perturb the number
+        4. For text answers, use a generic incorrect response
+        """
+        # Strategy 0: Check for explicit bad_code field (LiveCodeBench)
+        if "bad_code" in problem and problem["bad_code"]:
+            return problem["bad_code"]
+
+        # Strategy 1: Check for multiple choice options
+        choices_fields = ["choices", "options", "mc1_targets", "mc2_targets"]
+        for field in choices_fields:
+            if field in problem:
+                choices = problem[field]
+                if isinstance(choices, dict):
+                    # Handle mc1_targets/mc2_targets format
+                    if "choices" in choices and "labels" in choices:
+                        incorrect_indices = [
+                            i for i, label in enumerate(choices["labels"]) if label == 0
+                        ]
+                        if incorrect_indices:
+                            return choices["choices"][random.choice(incorrect_indices)]
+                elif isinstance(choices, list) and choices:
+                    # Filter out correct answer
+                    incorrect_choices = [
+                        c for c in choices
+                        if self._normalize_for_comparison(str(c)) != self._normalize_for_comparison(correct_answer)
+                    ]
+                    if incorrect_choices:
+                        return str(random.choice(incorrect_choices))
+
+        # Strategy 2: Numerical answer perturbation
+        try:
+            correct_num = float(correct_answer.strip())
+            # Perturb by 10-50%
+            perturbation = random.uniform(1.1, 1.5) if random.random() > 0.5 else random.uniform(0.5, 0.9)
+            incorrect_num = correct_num * perturbation
+            return str(int(incorrect_num) if correct_num == int(correct_num) else round(incorrect_num, 2))
+        except (ValueError, AttributeError):
+            pass
+
+        # Strategy 3: Generic incorrect responses by task type
+        if task_name in ["gsm8k", "math500", "aime", "hmmt", "polymath", "livemathbench"]:
+            # Math tasks: slightly wrong number
+            return str(random.randint(0, 1000))
+        elif task_name in ["livecodebench", "humaneval", "mbpp"]:
+            # Coding tasks: empty or syntax error
+            return "# Incomplete solution\npass"
+        else:
+            # Generic incorrect response
+            return "I don't know"
+
+    @staticmethod
+    def _normalize_for_comparison(text: str) -> str:
+        """Normalize text for comparison."""
+        return text.lower().strip()

wisent/{cli/data_loaders/data_loader_rotator.py → core/data_loaders/rotator.py}

@@ -14,7 +14,7 @@ class DataLoaderRotator:
     def __init__(
         self,
         loader: Union[str, BaseDataLoader, Type[BaseDataLoader], None] = None,
-        loaders_location: Union[str, Path] = "wisent_guard.core.data_loaders.loaders",
+        loaders_location: Union[str, Path] = "wisent.core.data_loaders.loaders",
         autoload: bool = True,
         **default_loader_kwargs: Any,
     ) -> None:

wisent/core/download_full_benchmarks.py

@@ -1276,6 +1276,80 @@ class FullBenchmarkDownloader:
 
         return pairs
 
+    def _perturb_code_to_break(self, code: str) -> str:
+        """
+        Perturb correct code to make it broken/unable to execute at runtime.
+
+        Introduces various types of bugs:
+        - Syntax errors (missing colons, parentheses)
+        - Runtime errors (undefined variables)
+        - Logic errors (wrong operators)
+        - Type errors (wrong return values)
+
+        Args:
+            code: Correct Python code
+
+        Returns:
+            Broken version of the code
+        """
+        lines = code.split('\n')
+        if not lines:
+            return "pass  # Broken code"
+
+        # Choose a random perturbation strategy
+        perturbation_type = random.choice([
+            'remove_colon',
+            'remove_return',
+            'wrong_variable',
+            'syntax_error',
+            'wrong_operator',
+            'incomplete_code'
+        ])
+
+        if perturbation_type == 'remove_colon':
+            # Remove colons from function/if/for statements
+            for i, line in enumerate(lines):
+                if any(keyword in line for keyword in ['def ', 'if ', 'for ', 'while ', 'elif ', 'else:']):
+                    lines[i] = line.replace(':', '')
+                    break
+
+        elif perturbation_type == 'remove_return':
+            # Remove return statement to break function
+            for i, line in enumerate(lines):
+                if 'return ' in line:
+                    lines[i] = line.replace('return ', '# return ')
+                    break
+
+        elif perturbation_type == 'wrong_variable':
+            # Use undefined variable name
+            for i, line in enumerate(lines):
+                if '=' in line and 'def ' not in line:
+                    lines[i] = line.replace('=', '= undefined_variable +')
+                    break
+
+        elif perturbation_type == 'syntax_error':
+            # Add syntax error by removing closing parenthesis
+            for i, line in enumerate(lines):
+                if '(' in line and ')' in line:
+                    lines[i] = line.replace(')', '', 1)
+                    break
+
+        elif perturbation_type == 'wrong_operator':
+            # Change operators to break logic
+            for i, line in enumerate(lines):
+                if any(op in line for op in ['+', '-', '*', '/', '<', '>', '==']):
+                    line = line.replace('+', '-', 1) if '+' in line else line
+                    line = line.replace('<', '>', 1) if '<' in line else line
+                    lines[i] = line
+                    break
+
+        elif perturbation_type == 'incomplete_code':
+            # Return only first half of code to make it incomplete
+            lines = lines[:max(1, len(lines) // 2)]
+            lines.append("    # Incomplete implementation")
+
+        return '\n'.join(lines)
+
     def _convert_mbpp_format(self, sample: Dict[str, Any]) -> List[Dict[str, Any]]:
         """Convert MBPP/HumanEval code generation format (task_id, code, prompt, test)."""
         task_id = sample.get("task_id", "")
@@ -1285,16 +1359,19 @@ class FullBenchmarkDownloader:
 
         # For code generation tasks, we create contrastive pairs based on:
         # Correct: The reference code solution
-        # Incorrect: A placeholder for incorrect/buggy code (since we don't have real incorrect solutions)
+        # Incorrect: Perturbed version with bugs that prevent runtime execution
 
         pairs = []
 
+        # Generate incorrect code by perturbing the correct solution
+        incorrect_code = self._perturb_code_to_break(code)
+
         # Create a contrastive pair with the coding prompt
         pairs.append(
             {
                 "question": f"Write Python code to solve this problem:\n\n{prompt}",
                 "correct_answer": code,
-                "incorrect_answer": "# This is a placeholder for incorrect code\n# In practice, this would be buggy or incomplete code\npass",  # TODO
+                "incorrect_answer": incorrect_code,
                 "metadata": {
                     "task_id": task_id,
                     "test_cases": test,