PyPI - wisent - Versions diffs - 0.7.379__py3-none-any.whl → 0.7.901__py3-none-any.whl - Mend

wisent 0.7.379py3-none-any.whl → 0.7.901py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (1020) hide show

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/metabench.py CHANGED Viewed

@@ -125,14 +125,12 @@ class MetabenchExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "metabench",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/minerva_math.py CHANGED Viewed

@@ -143,14 +143,12 @@ class MinervaMathExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "minerva_math",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu.py CHANGED Viewed

@@ -115,14 +115,12 @@ class MMLUExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "mmlu",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlusr.py CHANGED Viewed

@@ -154,20 +154,19 @@ class MmlusrExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
-            # Build prompt matching lm-eval format
+            # Build prompt - raw question without MC formatting
             correct = choices[answer_idx]
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            # Format exactly as lm-eval does it
-            formatted_question = f"{question}\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}\nAnswer:"
+            prompt = question
             metadata = {
                 "label": "mmlusr",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mrpc.py CHANGED Viewed

@@ -79,7 +79,7 @@ class MRPCExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
-            formatted_question = f"Sentence 1: {sentence1}\nSentence 2: {sentence2}. Do both sequences mean the same thing?\nAnswer:\nA. Yes\nB. No"
+            prompt = f"Sentence 1: {sentence1}\nSentence 2: {sentence2}. Do both sequences mean the same thing?"
             correct = "Yes" if label == 1 else "No"
             incorrect = "No" if label == 1 else "Yes"
@@ -89,7 +89,7 @@ class MRPCExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multiblimp.py CHANGED Viewed

@@ -100,11 +100,8 @@ class MultiblimpExtractor(LMEvalBenchmarkExtractor):
                     log.debug("Skipping doc with missing sen/wrong_sen", extra={"doc": doc})
                     return None
-                # Prompt: present both sentences as choices (matching lm-eval format)
-                # Since doc_to_text is empty, we format as multiple choice
-                prompt = "Which sentence is grammatically correct?\nA. {}\nB. {}".format(
-                    correct_sentence, incorrect_sentence
-                )
+                # Raw prompt without A./B. formatting
+                prompt = "Which sentence is grammatically correct?"
                 metadata = {"label": "multiblimp"}

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multirc.py CHANGED Viewed

@@ -82,7 +82,7 @@ class MultiRCExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
-            formatted_question = f"{paragraph}\nQuestion: {question}\nAnswer: {answer}\nIs this answer correct?\nA. Yes\nB. No"
+            prompt = f"{paragraph}\nQuestion: {question}\nAnswer: {answer}\nIs this answer correct?"
             correct = "Yes" if label == 1 else "No"
             incorrect = "No" if label == 1 else "Yes"
@@ -92,7 +92,7 @@ class MultiRCExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mutual.py CHANGED Viewed

@@ -84,14 +84,14 @@ class MutualExtractor(LMEvalBenchmarkExtractor):
             correct = options[answer_idx]
             incorrect = options[(answer_idx+1)%len(options)]
-            formatted_question = (f"{article}\nA. {incorrect}\nB. {correct}")
+            prompt = article
             metadata = {
                 "label": "mutual",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/non.py CHANGED Viewed

@@ -80,12 +80,10 @@ class NonExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "non"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval.py CHANGED Viewed

@@ -144,14 +144,12 @@ class NorevalExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "noreval",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_exact.py CHANGED Viewed

@@ -128,14 +128,12 @@ class NorevalExactMatchExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "noreval_exact",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_gen_exact.py CHANGED Viewed

@@ -136,14 +136,12 @@ class NorevalGenerationExactMatchExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "noreval_gen_exact",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc.py CHANGED Viewed

@@ -107,12 +107,12 @@ class NorevalMultipleChoiceExtractor(LMEvalBenchmarkExtractor):
                     log.debug("Skipping doc due to empty correct/wrong fields", extra={"doc": doc})
                     return None
-                formatted_question = f"Which sentence is grammatically correct?\nA. {incorrect}\nB. {correct}"
+                prompt = f"Which sentence is grammatically correct?"
                 metadata = {"label": "noreval_ncb"}
                 return self._build_pair(
-                    question=formatted_question,
+                    question=question,
                     correct=correct,
                     incorrect=incorrect,
                     metadata=metadata,
@@ -151,12 +151,10 @@ class NorevalMultipleChoiceExtractor(LMEvalBenchmarkExtractor):
                 correct = str(choices[correct_idx]).strip()
                 incorrect = str(choices[incorrect_idx]).strip()
-                formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
                 metadata = {"label": "noreval_truthfulqa"}
                 return self._build_pair(
-                    question=formatted_question,
+                    question=question,
                     correct=correct,
                     incorrect=incorrect,
                     metadata=metadata,
@@ -192,12 +190,10 @@ class NorevalMultipleChoiceExtractor(LMEvalBenchmarkExtractor):
                 incorrect_idx = (answer_idx + 1) % len(choice_texts)
                 incorrect = str(choice_texts[incorrect_idx]).strip()
-                formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
                 metadata = {"label": "noreval_nrk_quiz"}
                 return self._build_pair(
-                    question=formatted_question,
+                    question=question,
                     correct=correct,
                     incorrect=incorrect,
                     metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc_log_likelihoods.py CHANGED Viewed

@@ -102,12 +102,12 @@ class NorevalMultipleChoiceExtractor(LMEvalBenchmarkExtractor):
                     log.debug("Skipping doc due to empty correct/wrong fields", extra={"doc": doc})
                     return None
-                formatted_question = f"Which sentence is grammatically correct?\nA. {incorrect}\nB. {correct}"
+                prompt = f"Which sentence is grammatically correct?"
                 metadata = {"label": "noreval_ncb"}
                 return self._build_pair(
-                    question=formatted_question,
+                    question=question,
                     correct=correct,
                     incorrect=incorrect,
                     metadata=metadata,
@@ -146,12 +146,10 @@ class NorevalMultipleChoiceExtractor(LMEvalBenchmarkExtractor):
                 correct = str(choices[correct_idx]).strip()
                 incorrect = str(choices[incorrect_idx]).strip()
-                formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
                 metadata = {"label": "noreval_truthfulqa"}
                 return self._build_pair(
-                    question=formatted_question,
+                    question=question,
                     correct=correct,
                     incorrect=incorrect,
                     metadata=metadata,
@@ -187,12 +185,10 @@ class NorevalMultipleChoiceExtractor(LMEvalBenchmarkExtractor):
                 incorrect_idx = (answer_idx + 1) % len(choice_texts)
                 incorrect = str(choice_texts[incorrect_idx]).strip()
-                formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
                 metadata = {"label": "noreval_nrk_quiz"}
                 return self._build_pair(
-                    question=formatted_question,
+                    question=question,
                     correct=correct,
                     incorrect=incorrect,
                     metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/nq_open.py CHANGED Viewed

@@ -106,14 +106,14 @@ class NQOpenExtractor(LMEvalBenchmarkExtractor):
                 if incorrect == correct:
                     incorrect += "k"
-            formatted_question = f"Question: {question}\nAnswer:\nA. {incorrect}\nB. {correct}"
+            prompt = f"Question: {question}\nAnswer:"
             metadata = {
                 "label": "nq_open",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_arc_multilingual.py CHANGED Viewed

@@ -138,14 +138,12 @@ class OkapiArcMultilingualExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "okapi/arc_multilingual",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_hellaswag_multilingual.py CHANGED Viewed

@@ -145,14 +145,12 @@ class OkapiHellaswagMultilingualExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "okapi/hellaswag_multilingual",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_mmlu_multilingual.py CHANGED Viewed

@@ -133,14 +133,12 @@ class OkapiMmluMultilingualExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "okapi/mmlu_multilingual",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_truthfulqa_multilingual.py CHANGED Viewed

@@ -117,10 +117,9 @@ class OkapiTruthfulqaMultilingualExtractor(LMEvalBenchmarkExtractor):
                     incorrect = choices[incorrect_idx].strip() if isinstance(choices[incorrect_idx], str) else str(choices[incorrect_idx])
                     if correct and incorrect:
-                        formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
                         metadata = {"label": "okapi/truthfulqa_multilingual"}
                         return self._build_pair(
-                            question=formatted_question,
+                            question=question,
                             correct=correct,
                             incorrect=incorrect,
                             metadata=metadata,
@@ -180,14 +179,12 @@ class OkapiTruthfulqaMultilingualExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "okapi/truthfulqa_multilingual",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph.py CHANGED Viewed

@@ -157,14 +157,12 @@ class OlaphExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "olaph",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/openbookqa.py CHANGED Viewed

@@ -89,14 +89,14 @@ class OpenBookQAExtractor(LMEvalBenchmarkExtractor):
             incorrect = endings[(answer_idx+1)%len(endings)]
             question = f"{question_stem}"
-            formatted_question = f"{question}\nA. {incorrect}\nB. {correct}"
+            prompt = f"{question}"
             metadata = {
                 "label": "openbookqa",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/option.py CHANGED Viewed

@@ -80,12 +80,10 @@ class OptionExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "option"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafraseja.py CHANGED Viewed

@@ -83,12 +83,10 @@ class ParafrasejaExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "parafraseja"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafrases.py CHANGED Viewed

@@ -83,12 +83,10 @@ class ParafrasesExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "parafrases"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws.py CHANGED Viewed

@@ -80,12 +80,10 @@ class PawsExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "paws"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws_x.py CHANGED Viewed

@@ -125,14 +125,12 @@ class PawsXExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "paws-x",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pawsx.py CHANGED Viewed

@@ -82,7 +82,7 @@ class PawsXExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
-            formatted_question = f"Is sentence '{sentence1}' paraphrase of sentence '{sentence2}'?\nA. Yes\nB. No"
+            prompt = f"Is sentence '{sentence1}' paraphrase of sentence '{sentence2}'?"
             # label == 1 means paraphrase (positive), label == 0 means not paraphrase (negative)
             correct = "Yes" if label == 1 else "No"
@@ -93,7 +93,7 @@ class PawsXExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/persona.py CHANGED Viewed

@@ -219,12 +219,10 @@ class PersonaExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "persona"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases.py CHANGED Viewed

@@ -117,12 +117,10 @@ class PhrasesExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "phrases"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile.py CHANGED Viewed

@@ -132,14 +132,12 @@ class PileExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "pile",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/piqa.py CHANGED Viewed

@@ -84,7 +84,7 @@ class PIQAExtractor(LMEvalBenchmarkExtractor):
                 return None
             question = f"Question: {goal}\nAnswer:"
-            formatted_question = f"{question}\nA. {sol1}\nB. {sol2}"
+            prompt = f"{question}"
             correct = sol1 if label == 0 else sol2
             incorrect = sol2 if label == 0 else sol1
@@ -94,7 +94,7 @@ class PIQAExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench.py CHANGED Viewed

@@ -126,14 +126,12 @@ class PortugueseBenchExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "portuguese_bench",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prompt.py CHANGED Viewed

@@ -80,12 +80,10 @@ class PromptExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "prompt"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prost.py CHANGED Viewed

@@ -86,14 +86,14 @@ class ProstExtractor(LMEvalBenchmarkExtractor):
             correct = answers[label]
             incorrect = answers[(label+1)%len(answers)]
-            formatted_question = f"{context}\nQuestion: {question}\nAnswer:\nA. {incorrect}\nB. {correct}"
+            prompt = f"{context}\nQuestion: {question}\nAnswer:"
             metadata = {
                 "label": "prost",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent 0.7.379__py3-none-any.whl → 0.7.901__py3-none-any.whl

wisent 0.7.379py3-none-any.whl → 0.7.901py3-none-any.whl