wisent 0.5.12__py3-none-any.whl → 0.5.13__py3-none-any.whl

wisent/core/contrastive_pairs/lm_eval_pairs/lm_extractor_manifest.py

@@ -1,8 +1,56 @@
 __all__ = [
     "EXTRACTORS",
 ]
-base_import: str = "wisent_guard.core.contrastive_pairs.lm_eval_pairs.lm_task_extractors."
+base_import: str = "wisent.core.contrastive_pairs.lm_eval_pairs.lm_task_extractors."
 EXTRACTORS: dict[str, str] = {
     # key → "module_path:ClassName" (supports dotted attr path after ':')
+    "arc_challenge": f"{base_import}arc_challenge:Arc_ChallengeExtractor",
+    "arc_easy": f"{base_import}arc_easy:Arc_EasyExtractor",
+    "arithmetic": f"{base_import}arithmetic:ArithmeticExtractor",
+    "asdiv": f"{base_import}asdiv:ASDivExtractor",
+    "boolq": f"{base_import}boolq:BoolQExtractor",
+    "cb": f"{base_import}cb:CBExtractor",
+    "copa": f"{base_import}copa:COPAExtractor",
+    "coqa": f"{base_import}coqa:CoQAExtractor",
+    "drop": f"{base_import}drop:DROPExtractor",
+    "gsm8k": f"{base_import}gsm8k:GSM8KExtractor",
+    "headqa": f"{base_import}headqa:HeadQAExtractor",
+    "hellaswag": f"{base_import}hellaswag:HellaSwagExtractor",
+    "logiqa": f"{base_import}logiqa:LogiQAExtractor",
+    "logiqa2": f"{base_import}logiqa2:LogiQA2Extractor",
+    "mc_taco": f"{base_import}mc-taco:MCTACOExtractor",
+    "medqa": f"{base_import}medqa:MedQAExtractor",
+    "mrpc": f"{base_import}mrpc:MRPCExtractor",
+    "multirc": f"{base_import}multirc:MultiRCExtractor",
+    "mutual": f"{base_import}mutual:MutualExtractor",
+    "openbookqa": f"{base_import}openbookqa:OpenBookQAExtractor",
+    "pawsx": f"{base_import}pawsx:PAWSXExtractor",
+    "piqa": f"{base_import}piqa:PIQAExtractor",
+    "prost": f"{base_import}prost:PROSTExtractor",
+    "pubmedqa": f"{base_import}pubmedqa:PubMedQAExtractor",
+    "qa4mre": f"{base_import}qa4mre:QA4MREExtractor",
+    "qasper": f"{base_import}qasper:QASPERExtractor",
+    "qnli": f"{base_import}qnli:QNLIExtractor",
+    "qqp": f"{base_import}qqp:QQPExtractor",
+    "race": f"{base_import}race:RaceExtractor",
+    "record": f"{base_import}record:ReCoRDExtractor",
+    "rte": f"{base_import}rte:RTEExtractor",
+    "sciq": f"{base_import}sciq:SciQExtractor",
+    "social_iqa": f"{base_import}social_iqa:Social_IQAExtractor",
+    "squad2": f"{base_import}squad2:SQuAD2Extractor",
+    "sst2": f"{base_import}sst2:SST2Extractor",
+    "swag": f"{base_import}swag:SWAGExtractor",
+    "triviaqa": f"{base_import}triviaqa:TriviaQAExtractor",
+    "truthfulqa_gen": f"{base_import}truthfulqa_gen:TruthfulQA_GenExtractor",
+    "truthfulqa_mc1": f"{base_import}truthfulqa_mc1:TruthfulQA_MC1Extractor",
+    "truthfulqa_mc2": f"{base_import}truthfulqa_mc2:TruthfulQA_MC2Extractor",
+    "webqs": f"{base_import}webqs:WebQuestionsExtractor",
+    "wic": f"{base_import}wic:WiCExtractor",
     "winogrande": f"{base_import}winogrande:WinograndeExtractor",
+    "wnli": f"{base_import}wnli:WNLIExtractor",
+    "wsc": f"{base_import}wsc:WSCExtractor",
+    "xnli": f"{base_import}xnli:XNLIExtractor",
+    "xstorycloze": f"{base_import}xstorycloze:XStoryClozeExtractor",
+    "xwinograd": f"{base_import}xwinograd:XWinogradExtractor",
+    "livecodebench": f"{base_import}livecodebench:LiveCodeBenchExtractor",
 }

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_challenge.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+from typing import Any, TYPE_CHECKING
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import NegativeResponse, PositiveResponse
+from wisent.core.contrastive_pairs.lm_eval_pairs.atoms import LMEvalBenchmarkExtractor
+from wisent.core.cli_logger import setup_logger, bind
+
+if TYPE_CHECKING:
+    from lm_eval.api.task import ConfigurableTask
+
+
+__all__ = ["Arc_ChallengeExtractor"]
+_LOG = setup_logger(__name__)
+
+
+class Arc_ChallengeExtractor(LMEvalBenchmarkExtractor):
+    """Extractor for the Arc_Challenge benchmark."""
+
+    def extract_contrastive_pairs(
+        self,
+        lm_eval_task_data: ConfigurableTask,
+        limit: int | None = None,
+    ) -> list[ContrastivePair]:
+        """
+        Build contrastive pairs from Arc_Challenge docs.
+
+        Arc_Challenge schema:
+            - question
+            - choices: dict,
+            - choices["text"]: list with possible choices strings
+            - answerKey: str
+
+        Args:
+            lm_eval_task_data: lm-eval task instance for Arc_Challenge.
+            limit: Optional maximum number of pairs to produce.
+
+        Returns:
+            A list of ContrastivePair objects.
+        """
+        log = bind(_LOG, task=getattr(lm_eval_task_data, "NAME", "unknown"))
+
+        max_items = self._normalize_limit(limit)
+        docs = self.load_docs(lm_eval_task_data, max_items)
+
+        pairs: list[ContrastivePair] = []
+
+        log.info("Extracting contrastive pairs", extra={"doc_count": len(docs)})
+
+        for doc in docs:
+            pair = self._extract_pair_from_doc(doc)
+            if pair is not None:
+                pairs.append(pair)
+                if max_items is not None and len(pairs) >= max_items:
+                    break
+
+        if not pairs:
+            task_name = getattr(lm_eval_task_data, "NAME", type(lm_eval_task_data).__name__)
+            log.warning("No valid Arc_Challenge pairs extracted", extra={"task": task_name})
+
+        return pairs
+    
+    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
+        """
+        Convert a single Arc_Challenge doc into a ContrastivePair, if possible.
+        Returns None when required fields are missing or malformed.
+        """
+        log = bind(_LOG, doc_id=doc.get("id", "unknown"))
+
+        try:
+            question = str(doc.get("question", "")).strip()
+            choices_dict = doc.get("choices", {})
+            choices = choices_dict["text"]
+            answer = str(doc.get("answerKey", "")).strip()
+            answer_idx = int(ord(answer) - ord('A'))
+
+            if not question or not choices or not (0 <= answer_idx < len(choices)):
+                log.debug(
+                    "Skipping doc due to missing/invalid fields",
+                    extra={"doc": doc},
+                )
+                return None
+            
+            correct = choices[answer_idx]
+            incorrect = choices[(answer_idx+1)%len(choices)]
+
+            question = f"{question}"
+            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
+
+            metadata = {
+                "label": "arc_easy",
+            }
+
+            return self._build_pair(
+                question=formatted_question,
+                correct=correct,
+                incorrect=incorrect,
+                metadata=metadata,
+            )
+
+        except Exception as exc:  
+            log.error("Error extracting pair from doc", exc_info=exc, extra={"doc": doc})
+            return None
+
+    @staticmethod
+    def _build_pair(
+        question: str,
+        correct: str,
+        incorrect: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> ContrastivePair:
+        positive_response = PositiveResponse(model_response=correct)
+        negative_response = NegativeResponse(model_response=incorrect)
+        return ContrastivePair(prompt=question, positive_response=positive_response, negative_response=negative_response, label=metadata.get("label"))

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_easy.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+from typing import Any, TYPE_CHECKING
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import NegativeResponse, PositiveResponse
+from wisent.core.contrastive_pairs.lm_eval_pairs.atoms import LMEvalBenchmarkExtractor
+from wisent.core.cli_logger import setup_logger, bind
+
+if TYPE_CHECKING:
+    from lm_eval.api.task import ConfigurableTask
+
+
+__all__ = ["Arc_EasyExtractor"]
+_LOG = setup_logger(__name__)
+
+
+class Arc_EasyExtractor(LMEvalBenchmarkExtractor):
+    """Extractor for the Arc_Easy benchmark."""
+
+    def extract_contrastive_pairs(
+        self,
+        lm_eval_task_data: ConfigurableTask,
+        limit: int | None = None,
+    ) -> list[ContrastivePair]:
+        """
+        Build contrastive pairs from Arc_Easy docs.
+
+        Arc_Easy schema:
+            - question
+            - choices: dict,
+            - choices["text"]: list with possible choices strings
+            - answerKey: str
+
+        Args:
+            lm_eval_task_data: lm-eval task instance for Arc_Easy.
+            limit: Optional maximum number of pairs to produce.
+
+        Returns:
+            A list of ContrastivePair objects.
+        """
+        log = bind(_LOG, task=getattr(lm_eval_task_data, "NAME", "unknown"))
+
+        max_items = self._normalize_limit(limit)
+        docs = self.load_docs(lm_eval_task_data, max_items)
+
+        pairs: list[ContrastivePair] = []
+
+        log.info("Extracting contrastive pairs", extra={"doc_count": len(docs)})
+
+        for doc in docs:
+            pair = self._extract_pair_from_doc(doc)
+            if pair is not None:
+                pairs.append(pair)
+                if max_items is not None and len(pairs) >= max_items:
+                    break
+
+        if not pairs:
+            task_name = getattr(lm_eval_task_data, "NAME", type(lm_eval_task_data).__name__)
+            log.warning("No valid Arc_Easy pairs extracted", extra={"task": task_name})
+
+        return pairs
+    
+    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
+        """
+        Convert a single Arc_Easy doc into a ContrastivePair, if possible.
+        Returns None when required fields are missing or malformed.
+        """
+        log = bind(_LOG, doc_id=doc.get("id", "unknown"))
+
+        try:
+            question = str(doc.get("question", "")).strip()
+            choices_dict = doc.get("choices", {})
+            choices = choices_dict["text"]
+            answer = str(doc.get("answerKey", "")).strip()
+            answer_idx = int(ord(answer) - ord('A'))
+
+            if not question or not choices or not (0 <= answer_idx < len(choices)):
+                log.debug(
+                    "Skipping doc due to missing/invalid fields",
+                    extra={"doc": doc},
+                )
+                return None
+            
+            correct = choices[answer_idx]
+            incorrect = choices[(answer_idx+1)%len(choices)]
+
+            question = f"{question}"
+            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
+
+            metadata = {
+                "label": "arc_easy",
+            }
+
+            return self._build_pair(
+                question=formatted_question,
+                correct=correct,
+                incorrect=incorrect,
+                metadata=metadata,
+            )
+
+        except Exception as exc:  
+            log.error("Error extracting pair from doc", exc_info=exc, extra={"doc": doc})
+            return None
+
+    @staticmethod
+    def _build_pair(
+        question: str,
+        correct: str,
+        incorrect: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> ContrastivePair:
+        positive_response = PositiveResponse(model_response=correct)
+        negative_response = NegativeResponse(model_response=incorrect)
+        return ContrastivePair(prompt=question, positive_response=positive_response, negative_response=negative_response, label=metadata.get("label"))

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arithmetic.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import random
+from typing import Any, TYPE_CHECKING
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import NegativeResponse, PositiveResponse
+from wisent.core.contrastive_pairs.lm_eval_pairs.atoms import LMEvalBenchmarkExtractor
+from wisent.core.cli_logger import setup_logger, bind
+
+if TYPE_CHECKING:
+    from lm_eval.api.task import ConfigurableTask
+
+
+__all__ = ["ArithmeticExtractor"]
+_LOG = setup_logger(__name__)
+
+
+class ArithmeticExtractor(LMEvalBenchmarkExtractor):
+    """Extractor for the Arithmetic benchmark."""
+
+    def extract_contrastive_pairs(
+        self,
+        lm_eval_task_data: ConfigurableTask,
+        limit: int | None = None,
+    ) -> list[ContrastivePair]:
+        """
+        Build contrastive pairs from Arithmetic docs.
+
+        Arithmetic schema:
+            - context: str
+            - completion: str
+            
+        Args:
+            lm_eval_task_data: lm-eval task instance for Arithmetic.
+            limit: Optional maximum number of pairs to produce.
+
+        Returns:
+            A list of ContrastivePair objects.
+        """
+        log = bind(_LOG, task=getattr(lm_eval_task_data, "NAME", "unknown"))
+
+        max_items = self._normalize_limit(limit)
+        docs = self.load_docs(lm_eval_task_data, max_items)
+
+        pairs: list[ContrastivePair] = []
+
+        log.info("Extracting contrastive pairs", extra={"doc_count": len(docs)})
+
+        for doc in docs:
+            pair = self._extract_pair_from_doc(doc)
+            if pair is not None:
+                pairs.append(pair)
+                if max_items is not None and len(pairs) >= max_items:
+                    break
+
+        if not pairs:
+            task_name = getattr(lm_eval_task_data, "NAME", type(lm_eval_task_data).__name__)
+            log.warning("No valid Arithmetic pairs extracted", extra={"task": task_name})
+
+        return pairs
+    
+    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
+        """
+        Convert a single Arithmetic doc into a ContrastivePair, if possible.
+        Returns None when required fields are missing or malformed.
+        """
+        log = bind(_LOG, doc_id=doc.get("id", "unknown"))
+
+        try:
+            context = str(doc.get("context", "")).strip()
+            completion = str(doc.get("completion", "")).strip()
+
+            if not context or not completion:
+                log.debug(
+                    "Skipping doc due to missing/invalid fields",
+                    extra={"doc": doc},
+                )
+                return None
+
+            correct = completion
+            incorrect_val = float(completion) + 1
+            incorrect = str(int(incorrect_val)) if incorrect_val == int(incorrect_val) else str(incorrect_val)
+
+            formatted_question = f"{context}\nA. {incorrect}\nB. {correct}"
+
+            metadata = {
+                "label": "arithmetic",
+            }
+
+            return self._build_pair(
+                question=formatted_question,
+                correct=correct,
+                incorrect=incorrect,
+                metadata=metadata,
+            )
+
+        except Exception as exc:  
+            log.error("Error extracting pair from doc", exc_info=exc, extra={"doc": doc})
+            return None
+
+    @staticmethod
+    def _build_pair(
+        question: str,
+        correct: str,
+        incorrect: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> ContrastivePair:
+        positive_response = PositiveResponse(model_response=correct)
+        negative_response = NegativeResponse(model_response=incorrect)
+        return ContrastivePair(prompt=question, positive_response=positive_response, negative_response=negative_response, label=metadata.get("label"))

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/asdiv.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import random
+import re
+from typing import Any, TYPE_CHECKING
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import NegativeResponse, PositiveResponse
+from wisent.core.contrastive_pairs.lm_eval_pairs.atoms import LMEvalBenchmarkExtractor
+from wisent.core.cli_logger import setup_logger, bind
+
+if TYPE_CHECKING:
+    from lm_eval.api.task import ConfigurableTask
+
+
+__all__ = ["ASDivExtractor"]
+_LOG = setup_logger(__name__)
+
+
+class ASDivExtractor(LMEvalBenchmarkExtractor):
+    """Extractor for the ASDiv benchmark."""
+
+    def extract_contrastive_pairs(
+        self,
+        lm_eval_task_data: ConfigurableTask,
+        limit: int | None = None,
+    ) -> list[ContrastivePair]:
+        """
+        Build contrastive pairs from ASDiv docs.
+
+        ASDiv schema:
+            - body: str
+            - question: str
+            - answer: str
+            
+        Args:
+            lm_eval_task_data: lm-eval task instance for ASDiv.
+            limit: Optional maximum number of pairs to produce.
+
+        Returns:
+            A list of ContrastivePair objects.
+        """
+        log = bind(_LOG, task=getattr(lm_eval_task_data, "NAME", "unknown"))
+
+        max_items = self._normalize_limit(limit)
+        docs = self.load_docs(lm_eval_task_data, max_items)
+
+        pairs: list[ContrastivePair] = []
+
+        log.info("Extracting contrastive pairs", extra={"doc_count": len(docs)})
+
+        for doc in docs:
+            pair = self._extract_pair_from_doc(doc)
+            if pair is not None:
+                pairs.append(pair)
+                if max_items is not None and len(pairs) >= max_items:
+                    break
+
+        if not pairs:
+            task_name = getattr(lm_eval_task_data, "NAME", type(lm_eval_task_data).__name__)
+            log.warning("No valid ASDiv pairs extracted", extra={"task": task_name})
+
+        return pairs
+    
+    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
+        """
+        Convert a single ASDiv doc into a ContrastivePair, if possible.
+        Returns None when required fields are missing or malformed.
+        """
+        log = bind(_LOG, doc_id=doc.get("id", "unknown"))
+
+        try:
+            body = str(doc.get("body", "")).strip()
+            question = str(doc.get("question", "")).strip()
+            answer = str(doc.get("answer", "")).strip()
+
+            if not question or not answer:
+                log.debug(
+                    "Skipping doc due to missing/invalid fields",
+                    extra={"doc": doc},
+                )
+                return None
+            
+            match = re.search(r'-?\d+(?:\.\d+)?', answer.replace(',', '')) 
+            if not match:
+                log.debug("Skipping doc due to missing numerical answer", extra={"doc": doc})
+                return None
+            numerical_answer = match.group()
+            correct = numerical_answer
+            incorrect_val = float(numerical_answer) + 1
+            incorrect = str(int(incorrect_val)) if incorrect_val == int(incorrect_val) else str(incorrect_val)
+
+            formatted_question = f"{body}\nQuestion:{question}\nA. {incorrect}\nB. {correct}"
+
+            metadata = {
+                "label": "asdiv",
+            }
+
+            return self._build_pair(
+                question=formatted_question,
+                correct=correct,
+                incorrect=incorrect,
+                metadata=metadata,
+            )
+
+        except Exception as exc:  
+            log.error("Error extracting pair from doc", exc_info=exc, extra={"doc": doc})
+            return None
+
+    @staticmethod
+    def _build_pair(
+        question: str,
+        correct: str,
+        incorrect: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> ContrastivePair:
+        positive_response = PositiveResponse(model_response=correct)
+        negative_response = NegativeResponse(model_response=incorrect)
+        return ContrastivePair(prompt=question, positive_response=positive_response, negative_response=negative_response, label=metadata.get("label"))

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/boolq.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import random
+from typing import Any, TYPE_CHECKING
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import NegativeResponse, PositiveResponse
+from wisent.core.contrastive_pairs.lm_eval_pairs.atoms import LMEvalBenchmarkExtractor
+from wisent.core.cli_logger import setup_logger, bind
+
+if TYPE_CHECKING:
+    from lm_eval.api.task import ConfigurableTask
+
+
+__all__ = ["BoolQExtractor"]
+_LOG = setup_logger(__name__)
+
+
+class BoolQExtractor(LMEvalBenchmarkExtractor):
+    """Extractor for the BoolQ benchmark."""
+
+    def extract_contrastive_pairs(
+        self,
+        lm_eval_task_data: ConfigurableTask,
+        limit: int | None = None,
+        preferred_doc: str | None = None,
+    ) -> list[ContrastivePair]:
+        """
+        Build contrastive pairs from BoolQ docs.
+
+        BoolQ schema:
+            - passage: str
+            - question: str
+            - label: 0 or 1
+
+        Args:
+            lm_eval_task_data: lm-eval task instance for BoolQ.
+            limit: Optional maximum number of pairs to produce.
+            preferred_doc: Preferred document source ("validation", "test", "training", "fewshot")
+
+        Returns:
+            A list of ContrastivePair objects.
+        """
+        log = bind(_LOG, task=getattr(lm_eval_task_data, "NAME", "unknown"))
+
+        max_items = self._normalize_limit(limit)
+        docs = self.load_docs(lm_eval_task_data, max_items, preferred_doc=preferred_doc)
+
+        pairs: list[ContrastivePair] = []
+
+        log.info("Extracting contrastive pairs", extra={"doc_count": len(docs)})
+
+        for doc in docs:
+            pair = self._extract_pair_from_doc(doc)
+            if pair is not None:
+                pairs.append(pair)
+                if max_items is not None and len(pairs) >= max_items:
+                    break
+
+        if not pairs:
+            task_name = getattr(lm_eval_task_data, "NAME", type(lm_eval_task_data).__name__)
+            log.warning("No valid BoolQ pairs extracted", extra={"task": task_name})
+
+        return pairs
+    
+    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
+        """
+        Convert a single BoolQ doc into a ContrastivePair, if possible.
+        Returns None when required fields are missing or malformed.
+        """
+        log = bind(_LOG, doc_id=doc.get("id", "unknown"))
+
+        try:
+            passage = str(doc.get("passage", "")).strip()
+            question = str(doc.get("question", "")).strip()
+            label = doc.get("label")
+
+            if not passage or not question or label not in {0, 1}:
+                log.debug(
+                    "Skipping doc due to missing/invalid fields",
+                    extra={"doc": doc},
+                )
+                return None
+
+            #formatted_question = f"{passage}\nQuestion: {question}?\nAnswer:\nA. Yes\nB. No"
+            formatted_question = f"{passage}\nQuestion: {question}?"
+
+            correct = "Yes" if label == 1 else "No"
+            incorrect = "No" if label == 1 else "Yes"
+
+            metadata = {
+                "label": "boolq",
+            }
+
+            return self._build_pair(
+                question=formatted_question,
+                correct=correct,
+                incorrect=incorrect,
+                metadata=metadata,
+            )
+
+        except Exception as exc:  
+            log.error("Error extracting pair from doc", exc_info=exc, extra={"doc": doc})
+            return None
+
+    @staticmethod
+    def _build_pair(
+        question: str,
+        correct: str,
+        incorrect: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> ContrastivePair:
+        positive_response = PositiveResponse(model_response=correct)
+        negative_response = NegativeResponse(model_response=incorrect)
+        return ContrastivePair(prompt=question, positive_response=positive_response, negative_response=negative_response, label=metadata.get("label"))