PyPI - wisent - Versions diffs - 0.7.701__py3-none-any.whl → 0.7.1045__py3-none-any.whl - Mend

wisent 0.7.701py3-none-any.whl → 0.7.1045py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (391) hide show

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_mmlu_multilingual.py CHANGED Viewed

@@ -133,14 +133,12 @@ class OkapiMmluMultilingualExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "okapi/mmlu_multilingual",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_truthfulqa_multilingual.py CHANGED Viewed

@@ -117,10 +117,9 @@ class OkapiTruthfulqaMultilingualExtractor(LMEvalBenchmarkExtractor):
                     incorrect = choices[incorrect_idx].strip() if isinstance(choices[incorrect_idx], str) else str(choices[incorrect_idx])
                     if correct and incorrect:
-                        formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
                         metadata = {"label": "okapi/truthfulqa_multilingual"}
                         return self._build_pair(
-                            question=formatted_question,
+                            question=question,
                             correct=correct,
                             incorrect=incorrect,
                             metadata=metadata,
@@ -180,14 +179,12 @@ class OkapiTruthfulqaMultilingualExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "okapi/truthfulqa_multilingual",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph.py CHANGED Viewed

@@ -157,14 +157,12 @@ class OlaphExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "olaph",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/openbookqa.py CHANGED Viewed

@@ -89,14 +89,14 @@ class OpenBookQAExtractor(LMEvalBenchmarkExtractor):
             incorrect = endings[(answer_idx+1)%len(endings)]
             question = f"{question_stem}"
-            formatted_question = f"{question}\nA. {incorrect}\nB. {correct}"
+            prompt = f"{question}"
             metadata = {
                 "label": "openbookqa",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/option.py CHANGED Viewed

@@ -80,12 +80,10 @@ class OptionExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "option"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafraseja.py CHANGED Viewed

@@ -83,12 +83,10 @@ class ParafrasejaExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "parafraseja"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafrases.py CHANGED Viewed

@@ -83,12 +83,10 @@ class ParafrasesExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "parafrases"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws.py CHANGED Viewed

@@ -80,12 +80,10 @@ class PawsExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "paws"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws_x.py CHANGED Viewed

@@ -125,14 +125,12 @@ class PawsXExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "paws-x",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pawsx.py CHANGED Viewed

@@ -82,7 +82,7 @@ class PawsXExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
-            formatted_question = f"Is sentence '{sentence1}' paraphrase of sentence '{sentence2}'?\nA. Yes\nB. No"
+            prompt = f"Is sentence '{sentence1}' paraphrase of sentence '{sentence2}'?"
             # label == 1 means paraphrase (positive), label == 0 means not paraphrase (negative)
             correct = "Yes" if label == 1 else "No"
@@ -93,7 +93,7 @@ class PawsXExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/persona.py CHANGED Viewed

@@ -219,12 +219,10 @@ class PersonaExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "persona"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases.py CHANGED Viewed

@@ -117,12 +117,10 @@ class PhrasesExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "phrases"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile.py CHANGED Viewed

@@ -132,14 +132,12 @@ class PileExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "pile",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/piqa.py CHANGED Viewed

@@ -84,7 +84,7 @@ class PIQAExtractor(LMEvalBenchmarkExtractor):
                 return None
             question = f"Question: {goal}\nAnswer:"
-            formatted_question = f"{question}\nA. {sol1}\nB. {sol2}"
+            prompt = f"{question}"
             correct = sol1 if label == 0 else sol2
             incorrect = sol2 if label == 0 else sol1
@@ -94,7 +94,7 @@ class PIQAExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench.py CHANGED Viewed

@@ -126,14 +126,12 @@ class PortugueseBenchExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "portuguese_bench",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prompt.py CHANGED Viewed

@@ -80,12 +80,10 @@ class PromptExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "prompt"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prost.py CHANGED Viewed

@@ -86,14 +86,14 @@ class ProstExtractor(LMEvalBenchmarkExtractor):
             correct = answers[label]
             incorrect = answers[(label+1)%len(answers)]
-            formatted_question = f"{context}\nQuestion: {question}\nAnswer:\nA. {incorrect}\nB. {correct}"
+            prompt = f"{context}\nQuestion: {question}\nAnswer:"
             metadata = {
                 "label": "prost",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pubmedqa.py CHANGED Viewed

@@ -80,7 +80,7 @@ class PubMedQAExtractor(LMEvalBenchmarkExtractor):
                 return None
             formatted_context = " ".join(s.strip() for s in contexts if isinstance(s, str) and s.strip())
-            formatted_question = f"Abstract: {formatted_context}\nQuestion: {question}\nAnswer:\nA. yes\nB. no"
+            prompt = f"Abstract: {formatted_context}\nQuestion: {question}"
             correct = final_decision
             incorrect = "yes" if correct == "no" else "no"
@@ -90,7 +90,7 @@ class PubMedQAExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qa4mre.py CHANGED Viewed

@@ -90,14 +90,14 @@ class QA4MREExtractor(LMEvalBenchmarkExtractor):
             correct = answers[answer]
             incorrect = answers[(answer+1)%len(answers)]
-            formatted_question = f"{document_str}\nQuestion: {question_str}?\nAnswer:\nA. {incorrect}\nB. {correct}"
+            prompt = f"{document_str}\nQuestion: {question_str}?\nAnswer:"
             metadata = {
                 "label": "qa4mre",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper.py CHANGED Viewed

@@ -86,7 +86,7 @@ class QasperExtractor(LMEvalBenchmarkExtractor):
                 return None
-            formatted_question = f"TITLE: {title}\nABSTRACT: {abstract}\nQ: {question}\nA. yes\nB. no"
+            prompt = f"TITLE: {title}\nABSTRACT: {abstract}\nQ: {question}"
             correct = answer
             incorrect = "yes" if answer == "no" else "no"
@@ -96,7 +96,7 @@ class QasperExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper_bool.py CHANGED Viewed

@@ -80,7 +80,7 @@ class QasperBoolExtractor(LMEvalBenchmarkExtractor):
                 return None
-            formatted_question = f"TITLE: {title}\nABSTRACT: {abstract}\nQ: {question}\nA. yes\nB. no"
+            prompt = f"TITLE: {title}\nABSTRACT: {abstract}\nQ: {question}"
             correct = answer
             incorrect = "yes" if answer == "no" else "no"
@@ -90,7 +90,7 @@ class QasperBoolExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnli.py CHANGED Viewed

@@ -79,7 +79,7 @@ class QNLIExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
-            formatted_question = f"{question}\n{sentence}\nQuestion: Does this response answer the question?\nAnswer:\nA. Yes\nB. No"
+            prompt = f"{question}\n{sentence}\nQuestion: Does this response answer the question?"
             correct = "Yes" if label == 0 else "No"
             incorrect = "No" if label == 0 else "Yes"
@@ -89,7 +89,7 @@ class QNLIExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnlieu.py CHANGED Viewed

@@ -80,12 +80,10 @@ class QnlieuExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "qnlieu"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qqp.py CHANGED Viewed

@@ -79,7 +79,7 @@ class QQPExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
-            formatted_question = f"{question1}\n{question2}\nQuestion: Do both qiestions ask the same thing?\nAnswer:\nA. Yes\nB. No"
+            prompt = f"{question1}\n{question2}\nQuestion: Do both questions ask the same thing?"
             correct = "Yes" if label == 1 else "No"
             incorrect = "No" if label == 1 else "Yes"
@@ -89,7 +89,7 @@ class QQPExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/race.py CHANGED Viewed

@@ -95,14 +95,14 @@ class RACEExtractor(LMEvalBenchmarkExtractor):
             correct = options[answer_idx]
             incorrect = options[(answer_idx+1)%len(options)]
-            formatted_question = f"{article}\nQuestion: {question}?\nAnswer:\nA. {incorrect}\nB. {correct}"
+            prompt = f"{article}\nQuestion: {question}?\nAnswer:"
             metadata = {
                 "label": "race",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/random.py CHANGED Viewed

@@ -80,12 +80,10 @@ class RandomExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "random"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/record.py CHANGED Viewed

@@ -96,14 +96,14 @@ class RecordExtractor(LMEvalBenchmarkExtractor):
             # Remove @highlight prefix
             passage = passage.replace('@highlight', '')
-            formatted_question = f"Passage: {passage}\n\nQuery: {query}\nWhich option correctly completes the sentence at @placeholder?\nA. {incorrect}\nB. {correct}"
+            prompt = f"Passage: {passage}\n\nQuery: {query}\nWhich option correctly completes the sentence at @placeholder?"
             metadata = {
                 "label": "record",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/reversed.py CHANGED Viewed

@@ -83,12 +83,10 @@ class ReversedExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "reversed"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/rte.py CHANGED Viewed

@@ -79,7 +79,7 @@ class RTEExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
-            formatted_question = f"{sentence1}\nQuestion: {sentence2} True or False?\nAnswer:\nA. True\nB. False"
+            prompt = f"{sentence1}\nQuestion: {sentence2} True or False?"
             correct = "True" if label == 0 else "False"
             incorrect = "False" if label == 0 else "True"
@@ -89,7 +89,7 @@ class RTEExtractor(LMEvalBenchmarkExtractor):
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ruler.py CHANGED Viewed

@@ -141,14 +141,12 @@ class RulerExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "ruler",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sciq.py CHANGED Viewed

@@ -84,14 +84,14 @@ class SciQExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
-            formatted_question = f"{support}\nQuestion: {question}\nAnswer:\nA. {incorrect}\nB. {correct}"
+            prompt = f"{support}\nQuestion: {question}\nAnswer:"
             metadata = {
                 "label": "sciq",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/score.py CHANGED Viewed

@@ -150,12 +150,10 @@ class ScoreExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(cleaned_choices)
             incorrect = cleaned_choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "score_robustness"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls.py CHANGED Viewed

@@ -132,14 +132,12 @@ class ScrollsExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "scrolls",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls_mc.py CHANGED Viewed

@@ -128,14 +128,12 @@ class ScrollsMultipleChoiceExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {
                 "label": "scrolls_mc",
             }
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/self.py CHANGED Viewed

@@ -83,12 +83,10 @@ class SelfExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "self"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue.py CHANGED Viewed

@@ -104,12 +104,10 @@ class SglueExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "sglue"}
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue_rte.py CHANGED Viewed

@@ -92,7 +92,8 @@ class SglueRteExtractor(LMEvalBenchmarkExtractor):
                 correct = "False"
                 incorrect = "True"
-            prompt = f"Premise: {premise}\nHypothesis: {hypothesis} True or False?\nAnswer:\nA. {incorrect}\nB. {correct}"
+            # Raw prompt without A./B. formatting
+            prompt = f"Premise: {premise}\nHypothesis: {hypothesis} True or False?"
             metadata = {"label": "sglue_rte"}

wisent 0.7.701__py3-none-any.whl → 0.7.1045__py3-none-any.whl

wisent 0.7.701py3-none-any.whl → 0.7.1045py3-none-any.whl