wisent 0.5.12__py3-none-any.whl → 0.5.14__py3-none-any.whl

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/triviaqa.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import random
+from typing import Any, TYPE_CHECKING
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import NegativeResponse, PositiveResponse
+from wisent.core.contrastive_pairs.lm_eval_pairs.atoms import LMEvalBenchmarkExtractor
+from wisent.core.cli_logger import setup_logger, bind
+
+if TYPE_CHECKING:
+    from lm_eval.api.task import ConfigurableTask
+
+
+__all__ = ["TriviaQAExtractor"]
+_LOG = setup_logger(__name__)
+
+
+class TriviaQAExtractor(LMEvalBenchmarkExtractor):
+    """Extractor for the TriviaQA benchmark."""
+
+    def extract_contrastive_pairs(
+        self,
+        lm_eval_task_data: ConfigurableTask,
+        limit: int | None = None,
+    ) -> list[ContrastivePair]:
+        """
+        Build contrastive pairs from TriviaQA docs.
+
+        TriviaQA schema:
+            - question: str 
+            - answer: dict
+            
+        Args:
+            lm_eval_task_data: lm-eval task instance for TriviaQA.
+            limit: Optional maximum number of pairs to produce.
+
+        Returns:
+            A list of ContrastivePair objects.
+        """
+        log = bind(_LOG, task=getattr(lm_eval_task_data, "NAME", "unknown"))
+
+        max_items = self._normalize_limit(limit)
+        docs = self.load_docs(lm_eval_task_data, max_items)
+
+        pairs: list[ContrastivePair] = []
+
+        log.info("Extracting contrastive pairs", extra={"doc_count": len(docs)})
+
+        for doc in docs:
+            pair = self._extract_pair_from_doc(doc)
+            if pair is not None:
+                pairs.append(pair)
+                if max_items is not None and len(pairs) >= max_items:
+                    break
+
+        if not pairs:
+            task_name = getattr(lm_eval_task_data, "NAME", type(lm_eval_task_data).__name__)
+            log.warning("No valid TriviaQA pairs extracted", extra={"task": task_name})
+
+        return pairs
+    
+    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
+        """
+        Convert a single TriviaQA doc into a ContrastivePair, if possible.
+        Returns None when required fields are missing or malformed.
+        """
+        log = bind(_LOG, doc_id=doc.get("id", "unknown"))
+
+        try:
+            question = str(doc.get("question", "")).strip()
+            answer = doc.get("answer", {})
+
+            if not question or not answer:
+                log.debug(
+                    "Skipping doc due to missing/invalid fields",
+                    extra={"doc": doc},
+                )
+                return None
+            
+            correct = answer["value"]
+            incorrect = None
+            # Generate incorrect answer
+            try:
+                # Try to convert to number
+                num = float(correct)
+                # Check if it's an integer
+                if num.is_integer():
+                    incorrect = str(int(num) + 1)
+                else:
+                    incorrect = str(num + 1)
+            except ValueError:
+                # It's a string, shuffle the letters until different
+                letters = list(correct)
+                incorrect = correct
+                random.shuffle(letters)
+                incorrect = ''.join(letters)
+                if incorrect == correct:
+                    incorrect += "k"
+
+            formatted_question = f"Question: {question}\nAnswer:\nA. {incorrect}\nB. {correct}"
+
+            metadata = {
+                "label": "triviaqa",
+            }
+
+            return self._build_pair(
+                question=formatted_question,
+                correct=correct,
+                incorrect=incorrect,
+                metadata=metadata,
+            )
+
+        except Exception as exc:  
+            log.error("Error extracting pair from doc", exc_info=exc, extra={"doc": doc})
+            return None
+
+    @staticmethod
+    def _build_pair(
+        question: str,
+        correct: str,
+        incorrect: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> ContrastivePair:
+        positive_response = PositiveResponse(model_response=correct)
+        negative_response = NegativeResponse(model_response=incorrect)
+        return ContrastivePair(prompt=question, positive_response=positive_response, negative_response=negative_response, label=metadata.get("label"))

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_gen.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import random
+from typing import Any, TYPE_CHECKING
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import NegativeResponse, PositiveResponse
+from wisent.core.contrastive_pairs.lm_eval_pairs.atoms import LMEvalBenchmarkExtractor
+from wisent.core.cli_logger import setup_logger, bind
+
+if TYPE_CHECKING:
+    from lm_eval.api.task import ConfigurableTask
+
+
+__all__ = ["TruthfulQA_GenExtractor"]
+_LOG = setup_logger(__name__)
+
+
+class TruthfulQA_GenExtractor(LMEvalBenchmarkExtractor):
+    """Extractor for the TruthfulQA_Gen benchmark."""
+
+    def extract_contrastive_pairs(
+        self,
+        lm_eval_task_data: ConfigurableTask,
+        limit: int | None = None,
+    ) -> list[ContrastivePair]:
+        """
+        Build contrastive pairs from TruthfulQA_Gen docs.
+
+        TruthfulQA_Gen schema:
+            - question: str
+            - best_answer: str
+            - incorrect_answers: list
+            
+        Args:
+            lm_eval_task_data: lm-eval task instance for TruthfulQA_Gen.
+            limit: Optional maximum number of pairs to produce.
+
+        Returns:
+            A list of ContrastivePair objects.
+        """
+        log = bind(_LOG, task=getattr(lm_eval_task_data, "NAME", "unknown"))
+
+        max_items = self._normalize_limit(limit)
+        docs = self.load_docs(lm_eval_task_data, max_items)
+
+        pairs: list[ContrastivePair] = []
+
+        log.info("Extracting contrastive pairs", extra={"doc_count": len(docs)})
+
+        for doc in docs:
+            pair = self._extract_pair_from_doc(doc)
+            if pair is not None:
+                pairs.append(pair)
+                if max_items is not None and len(pairs) >= max_items:
+                    break
+
+        if not pairs:
+            task_name = getattr(lm_eval_task_data, "NAME", type(lm_eval_task_data).__name__)
+            log.warning("No valid TruthfulQA_MC2 pairs extracted", extra={"task": task_name})
+
+        return pairs
+    
+    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
+        """
+        Convert a single TruthfulQA_Gen doc into a ContrastivePair, if possible.
+        Returns None when required fields are missing or malformed.
+        """
+        log = bind(_LOG, doc_id=doc.get("id", "unknown"))
+
+        try:
+            question = str(doc.get("question", "")).strip()
+            best_answer = str(doc.get("best_answer", "")).strip()
+            incorrect_answers = doc.get("incorrect_answers", [])
+
+            if not question or not best_answer or not incorrect_answers:
+                log.debug(
+                    "Skipping doc due to missing/invalid fields",
+                    extra={"doc": doc},
+                )
+                return None
+            
+            correct = best_answer
+            incorrect = incorrect_answers[0]
+
+            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
+
+            metadata = {
+                "label": "truthfulqa_gen",
+            }
+
+            return self._build_pair(
+                question=formatted_question,
+                correct=correct,
+                incorrect=incorrect,
+                metadata=metadata,
+            )
+
+        except Exception as exc:  
+            log.error("Error extracting pair from doc", exc_info=exc, extra={"doc": doc})
+            return None
+
+    @staticmethod
+    def _build_pair(
+        question: str,
+        correct: str,
+        incorrect: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> ContrastivePair:
+        positive_response = PositiveResponse(model_response=correct)
+        negative_response = NegativeResponse(model_response=incorrect)
+        return ContrastivePair(prompt=question, positive_response=positive_response, negative_response=negative_response, label=metadata.get("label"))

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc1.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import random
+from typing import Any, TYPE_CHECKING
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import NegativeResponse, PositiveResponse
+from wisent.core.contrastive_pairs.lm_eval_pairs.atoms import LMEvalBenchmarkExtractor
+from wisent.core.cli_logger import setup_logger, bind
+
+if TYPE_CHECKING:
+    from lm_eval.api.task import ConfigurableTask
+
+
+__all__ = ["TruthfulQA_MC1Extractor"]
+_LOG = setup_logger(__name__)
+
+
+class TruthfulQA_MC1Extractor(LMEvalBenchmarkExtractor):
+    """Extractor for the TruthfulQA_MC1 benchmark."""
+
+    def extract_contrastive_pairs(
+        self,
+        lm_eval_task_data: ConfigurableTask,
+        limit: int | None = None,
+    ) -> list[ContrastivePair]:
+        """
+        Build contrastive pairs from TruthfulQA_MC1 docs.
+
+        TruthfulQA_MC1 schema:
+            - question: str
+            - mc1_targets: dict
+            
+        Args:
+            lm_eval_task_data: lm-eval task instance for TruthfulQA_MC1.
+            limit: Optional maximum number of pairs to produce.
+
+        Returns:
+            A list of ContrastivePair objects.
+        """
+        log = bind(_LOG, task=getattr(lm_eval_task_data, "NAME", "unknown"))
+
+        max_items = self._normalize_limit(limit)
+        docs = self.load_docs(lm_eval_task_data, max_items)
+
+        pairs: list[ContrastivePair] = []
+
+        log.info("Extracting contrastive pairs", extra={"doc_count": len(docs)})
+
+        for doc in docs:
+            pair = self._extract_pair_from_doc(doc)
+            if pair is not None:
+                pairs.append(pair)
+                if max_items is not None and len(pairs) >= max_items:
+                    break
+
+        if not pairs:
+            task_name = getattr(lm_eval_task_data, "NAME", type(lm_eval_task_data).__name__)
+            log.warning("No valid TruthfulQA_MC1 pairs extracted", extra={"task": task_name})
+
+        return pairs
+    
+    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
+        """
+        Convert a single TruthfulQA_MC1 doc into a ContrastivePair, if possible.
+        Returns None when required fields are missing or malformed.
+        """
+        log = bind(_LOG, doc_id=doc.get("id", "unknown"))
+
+        try:
+            question = str(doc.get("question", "")).strip()
+            mc1_targets = doc.get("mc1_targets")
+            options = mc1_targets["choices"]
+            labels = mc1_targets["labels"]
+
+            if not question or not options or not labels:
+                log.debug(
+                    "Skipping doc due to missing/invalid fields",
+                    extra={"doc": doc},
+                )
+                return None
+            
+            # Find correct answer
+            for i in range(len(labels)):
+                if labels[i] == 1:
+                    answer_idx = i
+            
+            correct = options[answer_idx]
+            incorrect = options[(answer_idx+1)%len(options)]
+
+            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
+
+            metadata = {
+                "label": "truthfulqa_mc1",
+            }
+
+            return self._build_pair(
+                question=formatted_question,
+                correct=correct,
+                incorrect=incorrect,
+                metadata=metadata,
+            )
+
+        except Exception as exc:  
+            log.error("Error extracting pair from doc", exc_info=exc, extra={"doc": doc})
+            return None
+
+    @staticmethod
+    def _build_pair(
+        question: str,
+        correct: str,
+        incorrect: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> ContrastivePair:
+        positive_response = PositiveResponse(model_response=correct)
+        negative_response = NegativeResponse(model_response=incorrect)
+        return ContrastivePair(prompt=question, positive_response=positive_response, negative_response=negative_response, label=metadata.get("label"))

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc2.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import random
+from typing import Any, TYPE_CHECKING
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import NegativeResponse, PositiveResponse
+from wisent.core.contrastive_pairs.lm_eval_pairs.atoms import LMEvalBenchmarkExtractor
+from wisent.core.cli_logger import setup_logger, bind
+
+if TYPE_CHECKING:
+    from lm_eval.api.task import ConfigurableTask
+
+
+__all__ = ["TruthfulQA_MC2Extractor"]
+_LOG = setup_logger(__name__)
+
+
+class TruthfulQA_MC2Extractor(LMEvalBenchmarkExtractor):
+    """Extractor for the TruthfulQA_MC2 benchmark."""
+
+    def extract_contrastive_pairs(
+        self,
+        lm_eval_task_data: ConfigurableTask,
+        limit: int | None = None,
+    ) -> list[ContrastivePair]:
+        """
+        Build contrastive pairs from TruthfulQA_MC2 docs.
+
+        TruthfulQA_MC1 schema:
+            - question: str
+            - mc1_targets: dict
+            
+        Args:
+            lm_eval_task_data: lm-eval task instance for TruthfulQA_MC2.
+            limit: Optional maximum number of pairs to produce.
+
+        Returns:
+            A list of ContrastivePair objects.
+        """
+        log = bind(_LOG, task=getattr(lm_eval_task_data, "NAME", "unknown"))
+
+        max_items = self._normalize_limit(limit)
+        docs = self.load_docs(lm_eval_task_data, max_items)
+
+        pairs: list[ContrastivePair] = []
+
+        log.info("Extracting contrastive pairs", extra={"doc_count": len(docs)})
+
+        for doc in docs:
+            pair = self._extract_pair_from_doc(doc)
+            if pair is not None:
+                pairs.append(pair)
+                if max_items is not None and len(pairs) >= max_items:
+                    break
+
+        if not pairs:
+            task_name = getattr(lm_eval_task_data, "NAME", type(lm_eval_task_data).__name__)
+            log.warning("No valid TruthfulQA_MC2 pairs extracted", extra={"task": task_name})
+
+        return pairs
+    
+    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
+        """
+        Convert a single TruthfulQA_MC2 doc into a ContrastivePair, if possible.
+        Returns None when required fields are missing or malformed.
+        """
+        log = bind(_LOG, doc_id=doc.get("id", "unknown"))
+
+        try:
+            question = str(doc.get("question", "")).strip()
+            mc1_targets = doc.get("mc1_targets")
+            options = mc1_targets["choices"]
+            labels = mc1_targets["labels"]
+
+            if not question or not options or not labels:
+                log.debug(
+                    "Skipping doc due to missing/invalid fields",
+                    extra={"doc": doc},
+                )
+                return None
+            
+            # Find correct answer
+            for i in range(len(labels)):
+                if labels[i] == 1:
+                    answer_idx = i
+            
+            correct = options[answer_idx]
+            incorrect = options[(answer_idx+1)%len(options)]
+
+            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
+
+            metadata = {
+                "label": "truthfulqa_mc2",
+            }
+
+            return self._build_pair(
+                question=formatted_question,
+                correct=correct,
+                incorrect=incorrect,
+                metadata=metadata,
+            )
+
+        except Exception as exc:  
+            log.error("Error extracting pair from doc", exc_info=exc, extra={"doc": doc})
+            return None
+
+    @staticmethod
+    def _build_pair(
+        question: str,
+        correct: str,
+        incorrect: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> ContrastivePair:
+        positive_response = PositiveResponse(model_response=correct)
+        negative_response = NegativeResponse(model_response=incorrect)
+        return ContrastivePair(prompt=question, positive_response=positive_response, negative_response=negative_response, label=metadata.get("label"))

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/webqs.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import random
+from typing import Any, TYPE_CHECKING
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import NegativeResponse, PositiveResponse
+from wisent.core.contrastive_pairs.lm_eval_pairs.atoms import LMEvalBenchmarkExtractor
+from wisent.core.cli_logger import setup_logger, bind
+
+if TYPE_CHECKING:
+    from lm_eval.api.task import ConfigurableTask
+
+
+__all__ = ["WebQuestionsExtractor"]
+_LOG = setup_logger(__name__)
+
+
+class WebQuestionsExtractor(LMEvalBenchmarkExtractor):
+    """Extractor for the WebQuestions benchmark."""
+
+    def extract_contrastive_pairs(
+        self,
+        lm_eval_task_data: ConfigurableTask,
+        limit: int | None = None,
+    ) -> list[ContrastivePair]:
+        """
+        Build contrastive pairs from WebQuestions docs.
+
+        WebQuestions schema:
+            - question: str 
+            - answers: list 
+            
+        Args:
+            lm_eval_task_data: lm-eval task instance for WebQuestions.
+            limit: Optional maximum number of pairs to produce.
+
+        Returns:
+            A list of ContrastivePair objects.
+        """
+        log = bind(_LOG, task=getattr(lm_eval_task_data, "NAME", "unknown"))
+
+        max_items = self._normalize_limit(limit)
+        docs = self.load_docs(lm_eval_task_data, max_items)
+
+        pairs: list[ContrastivePair] = []
+
+        log.info("Extracting contrastive pairs", extra={"doc_count": len(docs)})
+
+        for doc in docs:
+            pair = self._extract_pair_from_doc(doc)
+            if pair is not None:
+                pairs.append(pair)
+                if max_items is not None and len(pairs) >= max_items:
+                    break
+
+        if not pairs:
+            task_name = getattr(lm_eval_task_data, "NAME", type(lm_eval_task_data).__name__)
+            log.warning("No valid WebQuestions pairs extracted", extra={"task": task_name})
+
+        return pairs
+    
+    def _extract_pair_from_doc(self, doc: dict[str, Any]) -> ContrastivePair | None:
+        """
+        Convert a single WebQuestions doc into a ContrastivePair, if possible.
+        Returns None when required fields are missing or malformed.
+        """
+        log = bind(_LOG, doc_id=doc.get("id", "unknown"))
+
+        try:
+            question = str(doc.get("question", "")).strip()
+            answers = doc.get("answers", [])
+
+            if not question or not answers:
+                log.debug(
+                    "Skipping doc due to missing/invalid fields",
+                    extra={"doc": doc},
+                )
+                return None
+            
+            correct = answers[0]
+            incorrect = None
+            # Generate incorrect answer
+            try:
+                # Try to convert to number
+                num = float(correct)
+                # Check if it's an integer
+                if num.is_integer():
+                    incorrect = str(int(num) + 1)
+                else:
+                    incorrect = str(num + 1)
+            except ValueError:
+                # It's a string, shuffle the letters until different
+                letters = list(correct)
+                incorrect = correct
+                random.shuffle(letters)
+                incorrect = ''.join(letters)
+                if incorrect == correct:
+                    incorrect += "k"
+
+            formatted_question = f"Question: {question}\nAnswer:\nA. {incorrect}\nB. {correct}"
+
+            metadata = {
+                "label": "webquestions",
+            }
+
+            return self._build_pair(
+                question=formatted_question,
+                correct=correct,
+                incorrect=incorrect,
+                metadata=metadata,
+            )
+
+        except Exception as exc:  
+            log.error("Error extracting pair from doc", exc_info=exc, extra={"doc": doc})
+            return None
+
+    @staticmethod
+    def _build_pair(
+        question: str,
+        correct: str,
+        incorrect: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> ContrastivePair:
+        positive_response = PositiveResponse(model_response=correct)
+        negative_response = NegativeResponse(model_response=incorrect)
+        return ContrastivePair(prompt=question, positive_response=positive_response, negative_response=negative_response, label=metadata.get("label"))