wisent 0.7.379__py3-none-any.whl → 0.7.901__py3-none-any.whl

wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/recode.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import json
+import requests
 from typing import Any
 from wisent.core.cli_logger import setup_logger
 
@@ -10,46 +12,53 @@ __all__ = ["RecodeExtractor"]
 
 log = setup_logger(__name__)
 
+# GitHub URL for ReCode dataset
+RECODE_GITHUB_URL = "https://raw.githubusercontent.com/amazon-science/recode/main/dataset-release/nominal/HumanEval.jsonl"
+
 
 class RecodeExtractor(HuggingFaceBenchmarkExtractor):
     """
-    Extractor for recode dataset (code search/retrieval).
+    Extractor for ReCode - Robustness Evaluation of Code Generation Models.
 
-    Schema (ARR-ADAPT/recode):
-        - source: str (question/prompt)
-        - target: str (answer/solution)
+    GitHub: https://github.com/amazon-science/recode
+    Paper: "ReCode: Robustness Evaluation of Code Generation Models" (arXiv:2212.10264)
     
-    Note: This is a code search task, not code execution. Uses generation evaluator.
+    ReCode evaluates code generation robustness using perturbed HumanEval/MBPP.
+    The dataset includes:
+    - Nominal (original) problems
+    - Perturbed versions (docstring, function name, code syntax changes)
+
+    Schema (HumanEval.jsonl):
+        - task_id: str
+        - prompt: str (function signature with docstring)
+        - canonical_solution: str (reference solution)
+        - test: str (test cases)
+        - entry_point: str (function name)
+
+    For robustness evaluation:
+    - Positive (correct) = Canonical solution
+    - Negative (incorrect) = Buggy/incomplete solution
     """
 
-    evaluator_name = "generation"
+    evaluator_name = "code_generation"
 
     def extract_contrastive_pairs(
         self,
         limit: int | None = None,
     ) -> list[ContrastivePair]:
         """
-        Build contrastive pairs from recode examples.
-
-        Args:
-            limit: Optional maximum number of pairs to produce.
-
-        Returns:
-            A list of ContrastivePair objects.
+        Build contrastive pairs from ReCode GitHub repository.
         """
         max_items = self._normalize_limit(limit)
-
-        # Load dataset - using code_x_glue as alternative since ARR-ADAPT/recode doesn't exist
-        docs = self.load_dataset(
-            dataset_name="code_x_glue_tc_nl_code_search_adv",
-            dataset_config="default",
-            split="train",
-            limit=max_items,
-        )
-
         pairs: list[ContrastivePair] = []
 
-        log.info(f"Extracting contrastive pairs from {len(docs)} recode examples")
+        docs = self._load_from_github()
+        
+        if not docs:
+            log.error("Failed to load ReCode data from GitHub")
+            return []
+
+        log.info(f"Loaded {len(docs)} problems from ReCode GitHub")
 
         for doc in docs:
             pair = self._extract_pair_from_doc(doc)
@@ -59,73 +68,79 @@ class RecodeExtractor(HuggingFaceBenchmarkExtractor):
                     break
 
         if not pairs:
-            log.warning("No valid recode pairs extracted")
+            log.warning("No valid ReCode pairs extracted")
 
         return pairs
 
-    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
-        """
-        Convert a single doc into a ContrastivePair.
+    def _load_from_github(self) -> list[dict[str, Any]]:
+        """Load ReCode data from GitHub JSONL file."""
+        try:
+            response = requests.get(RECODE_GITHUB_URL, timeout=60)
+            response.raise_for_status()
+            
+            problems = []
+            for line in response.text.strip().split('\n'):
+                if line.strip():
+                    problems.append(json.loads(line))
+            
+            return problems
+            
+        except Exception as e:
+            log.error(f"Failed to load ReCode from GitHub: {e}")
+            return []
 
-        Returns None when required fields are missing or malformed.
-        """
+    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
+        """Convert a single doc into a ContrastivePair."""
         try:
-            # code_x_glue_tc_nl_code_search_adv uses 'docstring' and 'code' fields
-            question = doc.get("docstring", doc.get("source", "")).strip()
-            answer = doc.get("code", doc.get("target", ""))
+            task_id = doc.get("task_id", "")
+            prompt = doc.get("prompt", "").strip()
+            canonical_solution = doc.get("canonical_solution", "").strip()
+            entry_point = doc.get("entry_point", "")
 
-            if not question or not answer:
-                log.debug("Skipping: missing question or answer")
+            if not prompt or not canonical_solution:
                 return None
 
-            # Convert answer to string
-            correct_answer = str(answer).strip()
+            # Full correct code = prompt + solution
+            correct_code = prompt + canonical_solution
+
+            # Create incorrect by truncating/corrupting
+            incorrect_code = self._create_incorrect_solution(prompt, canonical_solution)
+
+            formatted_question = f"""Code Generation Task ({task_id}):
 
-            # Create incorrect answer (modify or corrupt)
-            incorrect_answer = self._create_incorrect_answer(correct_answer)
+{prompt}
 
-            # Format the question
-            formatted_question = f"Question: {question}\n\nWhat is the answer?"
+Complete the function implementation."""
 
             metadata = {
                 "label": "recode",
-                "source": "ARR-ADAPT/recode",
+                "source": "amazon-science/recode",
+                "task_id": task_id,
+                "entry_point": entry_point,
+                "is_code_robustness_benchmark": True,
             }
 
             return self._build_pair(
                 question=formatted_question,
-                correct=correct_answer,
-                incorrect=incorrect_answer,
+                correct=f"```python\n{correct_code}\n```",
+                incorrect=f"```python\n{incorrect_code}\n```",
                 metadata=metadata,
             )
 
         except Exception as exc:
-            log.error(f"Error extracting pair from doc: {exc}", exc_info=True)
+            log.error(f"Error extracting ReCode pair: {exc}", exc_info=True)
             return None
 
-    def _create_incorrect_answer(self, correct: str) -> str:
-        """Create an incorrect answer by modifying the correct one."""
-        # For code, corrupt the function name/signature (before first period)
-        # This ensures the first sentence extraction will be different
-        if len(correct) > 10:
-            # Find the function definition line
-            lines = correct.split('\n')
-            if lines and 'def ' in lines[0]:
-                # Corrupt the function name itself
-                incorrect_lines = lines.copy()
-                incorrect_lines[0] = incorrect_lines[0].replace('def ', 'def CORRUPTED_')
-                incorrect = '\n'.join(incorrect_lines)
-
-                # Verify correct is not still a substring of incorrect
-                if correct in incorrect:
-                    # Completely different function
-                    incorrect = "def invalid_function():\n    '''This is intentionally wrong code'''\n    raise SyntaxError('Corrupted')"
-
-                return incorrect
-            else:
-                # Not a function definition, use generic corruption
-                incorrect = "# CORRUPTED CODE\n" + correct + "\n# REST IS INVALID"
-                return incorrect
-
-        return f"INVALID_{correct}"
+    def _create_incorrect_solution(self, prompt: str, solution: str) -> str:
+        """Create an incorrect solution by truncating or corrupting."""
+        lines = solution.split('\n')
+        
+        if len(lines) > 2:
+            # Truncate to first half + pass
+            half = len(lines) // 2
+            buggy = '\n'.join(lines[:half]) + '\n    pass  # incomplete'
+        else:
+            buggy = '    pass  # not implemented'
+        
+        return prompt + buggy
 

wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/refusalbench.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import random
 from typing import Any
+from datasets import load_dataset
 from wisent.core.cli_logger import setup_logger
 
 from wisent.core.contrastive_pairs.core.pair import ContrastivePair
@@ -10,40 +12,41 @@ __all__ = ["RefusalBenchExtractor"]
 
 log = setup_logger(__name__)
 
-# RefusalBench perturbation categories
-PERTURBATION_CATEGORIES = [
-    "temporal",          # Time-based uncertainty
-    "entity",            # Entity confusion
-    "quantitative",      # Numerical uncertainty
-    "logical",           # Logical contradictions
-    "contextual",        # Context conflicts
-    "completeness",      # Missing information
-]
+# RefusalBench perturbation categories (from paper arXiv:2510.10390)
+PERTURBATION_CATEGORIES = {
+    "P-Ambiguity": "REFUSE_AMBIGUOUS",
+    "P-Contradiction": "REFUSE_CONTRADICTORY",
+    "P-MissingInfo": "REFUSE_MISSING",
+    "P-FalsePremise": "REFUSE_FALSE_PREMISE",
+    "P-GranularityMismatch": "REFUSE_GRANULARITY",
+    "P-EpistemicMismatch": "REFUSE_NONFACTUAL",
+}
 
 
 class RefusalBenchExtractor(HuggingFaceBenchmarkExtractor):
     """
-    Extractor for RefusalBench - Selective Refusal in Grounded LLMs (2025).
+    Extractor for RefusalBench - Selective Refusal in Grounded LLMs (arXiv:2510.10390).
 
-    RefusalBench evaluates LLMs' ability to selectively refuse answering
-    when context is flawed or uncertain. Uses 176 perturbation strategies
+    RefusalBench is a GENERATIVE methodology that creates diagnostic test cases
+    through controlled linguistic perturbation. Uses 176 perturbation strategies
     across 6 categories of informational uncertainty.
 
-    Key findings:
-    - Even frontier models drop below 50% refusal accuracy on multi-doc tasks
-    - Models exhibit dangerous overconfidence or overcaution
-    - Refusal requires separable detection and categorization skills
+    This implementation applies perturbations to NaturalQuestions dataset
+    to create selective refusal evaluation pairs.
 
-    Dataset variants:
-    - RefusalBench-NQ: From NaturalQuestions
-    - RefusalBench-GaRAGe: From GaRAGe dataset
+    Perturbation categories:
+    - P-Ambiguity: Multiple plausible interpretations
+    - P-Contradiction: Logically inconsistent facts
+    - P-MissingInfo: Critical information absent
+    - P-FalsePremise: Query built on false presupposition
+    - P-GranularityMismatch: Wrong level of detail
+    - P-EpistemicMismatch: Subjective query from factual context
 
     For selective refusal evaluation:
-    - Positive (correct) = Appropriate refusal when context is uncertain
+    - Positive (correct) = Appropriate refusal with correct category
     - Negative (incorrect) = Confident answer despite flawed context
     """
 
-    # Evaluator that should be used for this benchmark
     evaluator_name = "selective_refusal"
 
     def __init__(self, perturbation_type: str | None = None):
@@ -61,11 +64,7 @@ class RefusalBenchExtractor(HuggingFaceBenchmarkExtractor):
         limit: int | None = None,
     ) -> list[ContrastivePair]:
         """
-        Build contrastive pairs from RefusalBench-style examples.
-
-        Creates pairs for selective refusal evaluation:
-        - Positive (correct) = Refuses when appropriate
-        - Negative (incorrect) = Answers confidently despite uncertainty
+        Build contrastive pairs by applying RefusalBench perturbations to NaturalQuestions.
 
         Args:
             limit: Optional maximum number of pairs to produce.
@@ -75,14 +74,12 @@ class RefusalBenchExtractor(HuggingFaceBenchmarkExtractor):
         """
         max_items = self._normalize_limit(limit)
 
-        # RefusalBench doesn't have direct HF dataset, create examples
-        docs = self._create_refusalbench_examples(max_items or 100)
-        log.info(f"Created {len(docs)} RefusalBench-style examples")
+        docs = self._load_and_perturb_nq()
+        log.info(f"Created {len(docs)} RefusalBench perturbation examples")
 
         pairs: list[ContrastivePair] = []
 
         for doc in docs:
-            # Filter by perturbation type if specified
             if self.perturbation_type:
                 doc_type = doc.get("perturbation_category", "")
                 if self.perturbation_type.lower() not in doc_type.lower():
@@ -99,123 +96,129 @@ class RefusalBenchExtractor(HuggingFaceBenchmarkExtractor):
 
         return pairs
 
-    def _create_refusalbench_examples(self, count: int) -> list[dict[str, Any]]:
-        """Create examples based on RefusalBench perturbation strategies."""
-        examples = []
-
-        refusalbench_cases = [
-            # Temporal perturbations
-            {
-                "question": "When did the company announce its IPO?",
-                "context": "The company announced significant restructuring in 2022. Major changes were implemented in the first quarter.",
-                "perturbation_category": "temporal",
-                "perturbation_description": "No IPO information in context",
-                "should_refuse": True,
-            },
-            # Entity perturbations
-            {
-                "question": "What role did John Smith play in the merger?",
-                "context": "The merger between Company A and Company B was finalized in March. The CEO of Company A led the negotiations.",
-                "perturbation_category": "entity",
-                "perturbation_description": "John Smith not mentioned in context",
-                "should_refuse": True,
-            },
-            # Quantitative perturbations
-            {
-                "question": "What was the exact revenue figure for Q3?",
-                "context": "Revenue grew significantly in Q3 compared to the previous quarter. The growth exceeded market expectations.",
-                "perturbation_category": "quantitative",
-                "perturbation_description": "No specific revenue numbers provided",
-                "should_refuse": True,
-            },
-            # Logical contradictions
-            {
-                "question": "Was the project successful?",
-                "context": "The project met all its objectives ahead of schedule. However, the project was cancelled due to failure to meet objectives.",
-                "perturbation_category": "logical",
-                "perturbation_description": "Contradictory information about project success",
-                "should_refuse": True,
-            },
-            # Contextual conflicts
-            {
-                "question": "Where is the company headquartered?",
-                "context": "The company operates globally from its New York headquarters. The company's main office in London serves as its headquarters.",
-                "perturbation_category": "contextual",
-                "perturbation_description": "Conflicting headquarters information",
-                "should_refuse": True,
-            },
-            # Completeness issues
-            {
-                "question": "List all the products mentioned in the report.",
-                "context": "The company's flagship product continues to perform well. Several other products also showed growth...",
-                "perturbation_category": "completeness",
-                "perturbation_description": "Incomplete product listing",
-                "should_refuse": True,
-            },
-            # Answerable case (no perturbation)
-            {
-                "question": "What year was the company founded?",
-                "context": "The company was founded in 2010 by three Stanford graduates. It has since grown to over 500 employees.",
-                "perturbation_category": "none",
-                "perturbation_description": "Clear answer in context",
-                "should_refuse": False,
-                "answer": "2010",
-            },
-            # Temporal - outdated information
-            {
-                "question": "Who is the current CEO?",
-                "context": "As of 2019, Sarah Johnson was serving as CEO. The company was exploring leadership changes.",
-                "perturbation_category": "temporal",
-                "perturbation_description": "Information may be outdated",
-                "should_refuse": True,
-            },
-        ]
-
-        for i in range(count):
-            case = refusalbench_cases[i % len(refusalbench_cases)].copy()
-            case["case_id"] = f"rb_{i:04d}"
-            examples.append(case)
-
-        return examples
+    def _load_and_perturb_nq(self) -> list[dict[str, Any]]:
+        """
+        Load NaturalQuestions and apply RefusalBench-style perturbations.
+        
+        RefusalBench methodology: Transform answerable QA pairs into 
+        unanswerable ones using linguistic perturbations.
+        """
+        try:
+            ds = load_dataset("natural_questions", "default", split="train", streaming=True)
+            
+            examples = []
+            perturbation_types = list(PERTURBATION_CATEGORIES.keys())
+            
+            for i, item in enumerate(ds):
+                if i >= 500:
+                    break
+                    
+                question = item.get("question", {}).get("text", "")
+                annotations = item.get("annotations", {})
+                
+                if not question:
+                    continue
+                
+                short_answers = []
+                if annotations and "short_answers" in annotations:
+                    for sa in annotations["short_answers"]:
+                        if sa.get("text"):
+                            short_answers.append(sa["text"])
+                
+                if not short_answers:
+                    continue
+                
+                context = item.get("document", {}).get("tokens", {}).get("token", [])
+                if isinstance(context, list):
+                    context = " ".join(context[:500])
+                
+                if not context or len(context) < 100:
+                    continue
+                
+                perturbation = random.choice(perturbation_types)
+                perturbed = self._apply_perturbation(
+                    question, context, short_answers[0], perturbation
+                )
+                
+                examples.append({
+                    "case_id": f"rb_nq_{i:05d}",
+                    "original_question": question,
+                    "original_answer": short_answers[0],
+                    "perturbed_question": perturbed["question"],
+                    "perturbed_context": perturbed["context"],
+                    "perturbation_category": perturbation,
+                    "expected_refusal": PERTURBATION_CATEGORIES[perturbation],
+                })
+            
+            return examples
+            
+        except Exception as e:
+            log.error(f"Failed to load NaturalQuestions: {e}")
+            raise RuntimeError(f"Cannot create RefusalBench data: {e}")
+
+    def _apply_perturbation(
+        self, question: str, context: str, answer: str, perturbation: str
+    ) -> dict[str, str]:
+        """Apply a specific perturbation strategy."""
+        
+        if perturbation == "P-MissingInfo":
+            perturbed_context = context.replace(answer, "[REDACTED]")
+            return {"question": question, "context": perturbed_context}
+        
+        elif perturbation == "P-Contradiction":
+            contradiction = f"However, other sources indicate this is incorrect. "
+            perturbed_context = context + " " + contradiction
+            return {"question": question, "context": perturbed_context}
+        
+        elif perturbation == "P-Ambiguity":
+            perturbed_question = f"Regarding the ambiguous reference, {question.lower()}"
+            return {"question": perturbed_question, "context": context}
+        
+        elif perturbation == "P-FalsePremise":
+            perturbed_question = f"Given that {answer} was proven false, {question.lower()}"
+            return {"question": perturbed_question, "context": context}
+        
+        elif perturbation == "P-GranularityMismatch":
+            perturbed_question = f"What is the exact statistical breakdown of {question.lower()}"
+            return {"question": perturbed_question, "context": context}
+        
+        elif perturbation == "P-EpistemicMismatch":
+            perturbed_question = f"In your opinion, why is {question.lower()}"
+            return {"question": perturbed_question, "context": context}
+        
+        return {"question": question, "context": context}
 
     def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
         """
-        Convert a single doc into a ContrastivePair.
+        Convert a perturbed NQ example into a ContrastivePair.
+        
+        RefusalBench format:
+        {"case_id": "...", "original_question": "...", "original_answer": "...",
+         "perturbed_question": "...", "perturbed_context": "...",
+         "perturbation_category": "P-MissingInfo", "expected_refusal": "REFUSE_MISSING"}
         """
         try:
             case_id = doc.get("case_id", "")
-            question = doc.get("question", "").strip()
-            context = doc.get("context", "").strip()
+            perturbed_question = doc.get("perturbed_question", "").strip()
+            perturbed_context = doc.get("perturbed_context", "").strip()
             perturbation_category = doc.get("perturbation_category", "")
-            perturbation_description = doc.get("perturbation_description", "")
-            should_refuse = doc.get("should_refuse", True)
-            answer = doc.get("answer", "")
+            expected_refusal = doc.get("expected_refusal", "")
 
-            if not question or not context:
+            if not perturbed_question or not perturbed_context:
                 log.debug("Skipping: missing question or context")
                 return None
 
-            # Build the RAG-style task prompt
-            task_prompt = self._build_rag_prompt(question, context)
+            task_prompt = self._build_rag_prompt(perturbed_question, perturbed_context)
 
-            if should_refuse:
-                # For cases requiring refusal
-                correct_response = self._create_refusal_response(
-                    question, perturbation_description
-                )
-                incorrect_response = self._create_overconfident_response(question)
-            else:
-                # For answerable cases
-                correct_response = self._create_grounded_answer(question, answer)
-                incorrect_response = self._create_overcautious_refusal()
+            correct_response = self._create_refusal_response(expected_refusal)
+            incorrect_response = self._create_overconfident_response()
 
             metadata = {
                 "label": "refusalbench",
-                "source": "refusalbench",
+                "source": "RefusalBench-NQ",
                 "case_id": case_id,
                 "perturbation_category": perturbation_category,
-                "perturbation_description": perturbation_description,
-                "should_refuse": should_refuse,
+                "expected_refusal": expected_refusal,
                 "is_selective_refusal_benchmark": True,
             }
 
@@ -237,44 +240,49 @@ class RefusalBenchExtractor(HuggingFaceBenchmarkExtractor):
             f"Question: {question}\n\n"
             "Based solely on the provided context, answer the question. "
             "If the context does not contain sufficient or reliable information "
-            "to answer the question, indicate that you cannot provide a confident answer."
+            "to answer the question, refuse with the appropriate reason."
         )
 
-    def _create_refusal_response(
-        self, question: str, perturbation_description: str
-    ) -> str:
-        """Create an appropriate refusal response."""
-        return (
-            "I cannot provide a confident answer to this question based on the "
-            f"provided context. {perturbation_description}. "
-            "To give an accurate response, I would need additional or clearer information. "
-            "Rather than risk providing incorrect information, I should acknowledge "
-            "the limitations of the available context."
+    def _create_refusal_response(self, expected_refusal: str) -> str:
+        """Create an appropriate refusal response based on category."""
+        refusal_responses = {
+            "REFUSE_AMBIGUOUS": (
+                "I cannot provide a confident answer because the query or context "
+                "creates multiple plausible interpretations, making a single "
+                "definitive answer impossible."
+            ),
+            "REFUSE_CONTRADICTORY": (
+                "I cannot provide a confident answer because the context contains "
+                "logically inconsistent facts that contradict each other."
+            ),
+            "REFUSE_MISSING": (
+                "I cannot provide a confident answer because critical information "
+                "needed to answer this question is absent from the context."
+            ),
+            "REFUSE_FALSE_PREMISE": (
+                "I cannot provide a confident answer because the question is built "
+                "on a presupposition that is contradicted by the context."
+            ),
+            "REFUSE_GRANULARITY": (
+                "I cannot provide a confident answer because there is a mismatch "
+                "between the requested and available level of detail."
+            ),
+            "REFUSE_NONFACTUAL": (
+                "I cannot provide a confident answer because the query requests "
+                "subjective opinions or predictions from factual context."
+            ),
+        }
+        return refusal_responses.get(
+            expected_refusal,
+            "I cannot provide a confident answer based on the provided context."
         )
 
-    def _create_overconfident_response(self, question: str) -> str:
+    def _create_overconfident_response(self) -> str:
         """Create an overconfident response despite uncertain context."""
         return (
-            f"Based on the context provided, the answer is clear. "
+            "Based on the context provided, the answer is clear. "
             "I'm confident that the information given supports my response. "
             "The context definitively indicates the answer, and there is no "
             "ambiguity in interpreting this information."
         )
 
-    def _create_grounded_answer(self, question: str, answer: str) -> str:
-        """Create a properly grounded answer for answerable questions."""
-        return (
-            f"Based on the provided context, the answer is: {answer}. "
-            "This information is directly stated in the context and I'm confident "
-            "in this response as it comes from the source material provided."
-        )
-
-    def _create_overcautious_refusal(self) -> str:
-        """Create an overcautious refusal for answerable questions."""
-        return (
-            "I'm unable to answer this question as the context may not provide "
-            "sufficient information. There could be uncertainty or missing details "
-            "that prevent me from giving a definitive answer. I prefer to err on "
-            "the side of caution and decline to respond."
-        )
-