wisent 0.7.701__py3-none-any.whl → 0.7.901__py3-none-any.whl

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_truthfulqa_multilingual.py

@@ -117,10 +117,9 @@ class OkapiTruthfulqaMultilingualExtractor(LMEvalBenchmarkExtractor):
                     incorrect = choices[incorrect_idx].strip() if isinstance(choices[incorrect_idx], str) else str(choices[incorrect_idx])
 
                     if correct and incorrect:
-                        formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
                         metadata = {"label": "okapi/truthfulqa_multilingual"}
                         return self._build_pair(
-                            question=formatted_question,
+                            question=question,
                             correct=correct,
                             incorrect=incorrect,
                             metadata=metadata,
@@ -180,14 +179,12 @@ class OkapiTruthfulqaMultilingualExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
 
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
-
             metadata = {
                 "label": "okapi/truthfulqa_multilingual",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph.py

@@ -157,14 +157,12 @@ class OlaphExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
 
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
-
             metadata = {
                 "label": "olaph",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/openbookqa.py

@@ -89,14 +89,14 @@ class OpenBookQAExtractor(LMEvalBenchmarkExtractor):
             incorrect = endings[(answer_idx+1)%len(endings)]
 
             question = f"{question_stem}"
-            formatted_question = f"{question}\nA. {incorrect}\nB. {correct}"
+            prompt = f"{question}"
 
             metadata = {
                 "label": "openbookqa",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/option.py

@@ -80,12 +80,10 @@ class OptionExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "option"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafraseja.py

@@ -83,12 +83,10 @@ class ParafrasejaExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "parafraseja"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafrases.py

@@ -83,12 +83,10 @@ class ParafrasesExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "parafrases"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws.py

@@ -80,12 +80,10 @@ class PawsExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "paws"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws_x.py

@@ -125,14 +125,12 @@ class PawsXExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
 
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
-
             metadata = {
                 "label": "paws-x",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pawsx.py

@@ -82,7 +82,7 @@ class PawsXExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
             
-            formatted_question = f"Is sentence '{sentence1}' paraphrase of sentence '{sentence2}'?\nA. Yes\nB. No"
+            prompt = f"Is sentence '{sentence1}' paraphrase of sentence '{sentence2}'?"
 
             # label == 1 means paraphrase (positive), label == 0 means not paraphrase (negative)
             correct = "Yes" if label == 1 else "No"
@@ -93,7 +93,7 @@ class PawsXExtractor(LMEvalBenchmarkExtractor):
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/persona.py

@@ -219,12 +219,10 @@ class PersonaExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "persona"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases.py

@@ -117,12 +117,10 @@ class PhrasesExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "phrases"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile.py

@@ -132,14 +132,12 @@ class PileExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
 
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
-
             metadata = {
                 "label": "pile",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/piqa.py

@@ -84,7 +84,7 @@ class PIQAExtractor(LMEvalBenchmarkExtractor):
                 return None
 
             question = f"Question: {goal}\nAnswer:"
-            formatted_question = f"{question}\nA. {sol1}\nB. {sol2}"
+            prompt = f"{question}"
 
             correct = sol1 if label == 0 else sol2
             incorrect = sol2 if label == 0 else sol1
@@ -94,7 +94,7 @@ class PIQAExtractor(LMEvalBenchmarkExtractor):
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench.py

@@ -126,14 +126,12 @@ class PortugueseBenchExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
 
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
-
             metadata = {
                 "label": "portuguese_bench",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prompt.py

@@ -80,12 +80,10 @@ class PromptExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "prompt"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prost.py

@@ -86,14 +86,14 @@ class ProstExtractor(LMEvalBenchmarkExtractor):
             correct = answers[label]
             incorrect = answers[(label+1)%len(answers)]
 
-            formatted_question = f"{context}\nQuestion: {question}\nAnswer:\nA. {incorrect}\nB. {correct}"
+            prompt = f"{context}\nQuestion: {question}\nAnswer:"
 
             metadata = {
                 "label": "prost",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pubmedqa.py

@@ -80,7 +80,7 @@ class PubMedQAExtractor(LMEvalBenchmarkExtractor):
                 return None
         
             formatted_context = " ".join(s.strip() for s in contexts if isinstance(s, str) and s.strip())
-            formatted_question = f"Abstract: {formatted_context}\nQuestion: {question}\nAnswer:\nA. yes\nB. no"
+            prompt = f"Abstract: {formatted_context}\nQuestion: {question}"
 
             correct = final_decision
             incorrect = "yes" if correct == "no" else "no"
@@ -90,7 +90,7 @@ class PubMedQAExtractor(LMEvalBenchmarkExtractor):
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qa4mre.py

@@ -90,14 +90,14 @@ class QA4MREExtractor(LMEvalBenchmarkExtractor):
             correct = answers[answer]
             incorrect = answers[(answer+1)%len(answers)]
 
-            formatted_question = f"{document_str}\nQuestion: {question_str}?\nAnswer:\nA. {incorrect}\nB. {correct}"
+            prompt = f"{document_str}\nQuestion: {question_str}?\nAnswer:"
 
             metadata = {
                 "label": "qa4mre",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper.py

@@ -86,7 +86,7 @@ class QasperExtractor(LMEvalBenchmarkExtractor):
                 return None
 
 
-            formatted_question = f"TITLE: {title}\nABSTRACT: {abstract}\nQ: {question}\nA. yes\nB. no"
+            prompt = f"TITLE: {title}\nABSTRACT: {abstract}\nQ: {question}"
 
             correct = answer
             incorrect = "yes" if answer == "no" else "no"
@@ -96,7 +96,7 @@ class QasperExtractor(LMEvalBenchmarkExtractor):
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper_bool.py

@@ -80,7 +80,7 @@ class QasperBoolExtractor(LMEvalBenchmarkExtractor):
                 return None
 
 
-            formatted_question = f"TITLE: {title}\nABSTRACT: {abstract}\nQ: {question}\nA. yes\nB. no"
+            prompt = f"TITLE: {title}\nABSTRACT: {abstract}\nQ: {question}"
 
             correct = answer
             incorrect = "yes" if answer == "no" else "no"
@@ -90,7 +90,7 @@ class QasperBoolExtractor(LMEvalBenchmarkExtractor):
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnli.py

@@ -79,7 +79,7 @@ class QNLIExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
         
-            formatted_question = f"{question}\n{sentence}\nQuestion: Does this response answer the question?\nAnswer:\nA. Yes\nB. No"
+            prompt = f"{question}\n{sentence}\nQuestion: Does this response answer the question?"
 
             correct = "Yes" if label == 0 else "No"
             incorrect = "No" if label == 0 else "Yes"
@@ -89,7 +89,7 @@ class QNLIExtractor(LMEvalBenchmarkExtractor):
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnlieu.py

@@ -80,12 +80,10 @@ class QnlieuExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "qnlieu"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qqp.py

@@ -79,7 +79,7 @@ class QQPExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
         
-            formatted_question = f"{question1}\n{question2}\nQuestion: Do both qiestions ask the same thing?\nAnswer:\nA. Yes\nB. No"
+            prompt = f"{question1}\n{question2}\nQuestion: Do both questions ask the same thing?"
 
             correct = "Yes" if label == 1 else "No"
             incorrect = "No" if label == 1 else "Yes"
@@ -89,7 +89,7 @@ class QQPExtractor(LMEvalBenchmarkExtractor):
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/race.py

@@ -95,14 +95,14 @@ class RACEExtractor(LMEvalBenchmarkExtractor):
             correct = options[answer_idx]
             incorrect = options[(answer_idx+1)%len(options)]
 
-            formatted_question = f"{article}\nQuestion: {question}?\nAnswer:\nA. {incorrect}\nB. {correct}"
+            prompt = f"{article}\nQuestion: {question}?\nAnswer:"
 
             metadata = {
                 "label": "race",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/random.py

@@ -80,12 +80,10 @@ class RandomExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "random"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/record.py

@@ -96,14 +96,14 @@ class RecordExtractor(LMEvalBenchmarkExtractor):
             # Remove @highlight prefix
             passage = passage.replace('@highlight', '')
 
-            formatted_question = f"Passage: {passage}\n\nQuery: {query}\nWhich option correctly completes the sentence at @placeholder?\nA. {incorrect}\nB. {correct}"
+            prompt = f"Passage: {passage}\n\nQuery: {query}\nWhich option correctly completes the sentence at @placeholder?"
 
             metadata = {
                 "label": "record",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/reversed.py

@@ -83,12 +83,10 @@ class ReversedExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "reversed"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/rte.py

@@ -79,7 +79,7 @@ class RTEExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
         
-            formatted_question = f"{sentence1}\nQuestion: {sentence2} True or False?\nAnswer:\nA. True\nB. False"
+            prompt = f"{sentence1}\nQuestion: {sentence2} True or False?"
 
             correct = "True" if label == 0 else "False"
             incorrect = "False" if label == 0 else "True"
@@ -89,7 +89,7 @@ class RTEExtractor(LMEvalBenchmarkExtractor):
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ruler.py

@@ -141,14 +141,12 @@ class RulerExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
 
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
-
             metadata = {
                 "label": "ruler",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sciq.py

@@ -84,14 +84,14 @@ class SciQExtractor(LMEvalBenchmarkExtractor):
                 )
                 return None
         
-            formatted_question = f"{support}\nQuestion: {question}\nAnswer:\nA. {incorrect}\nB. {correct}"
+            prompt = f"{support}\nQuestion: {question}\nAnswer:"
 
             metadata = {
                 "label": "sciq",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=prompt,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/score.py

@@ -150,12 +150,10 @@ class ScoreExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(cleaned_choices)
             incorrect = cleaned_choices[incorrect_idx]
 
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
-
             metadata = {"label": "score_robustness"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls.py

@@ -132,14 +132,12 @@ class ScrollsExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
 
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
-
             metadata = {
                 "label": "scrolls",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls_mc.py

@@ -128,14 +128,12 @@ class ScrollsMultipleChoiceExtractor(LMEvalBenchmarkExtractor):
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = choices[incorrect_idx]
 
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
-
             metadata = {
                 "label": "scrolls_mc",
             }
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/self.py

@@ -83,12 +83,10 @@ class SelfExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "self"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue.py

@@ -104,12 +104,10 @@ class SglueExtractor(LMEvalBenchmarkExtractor):
             correct = str(choices[answer_idx]).strip()
             incorrect_idx = (answer_idx + 1) % len(choices)
             incorrect = str(choices[incorrect_idx]).strip()
-
-            formatted_question = f"Question: {question}\nA. {incorrect}\nB. {correct}"
             metadata = {"label": "sglue"}
 
             return self._build_pair(
-                question=formatted_question,
+                question=question,
                 correct=correct,
                 incorrect=incorrect,
                 metadata=metadata,

wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue_rte.py

@@ -92,7 +92,8 @@ class SglueRteExtractor(LMEvalBenchmarkExtractor):
                 correct = "False"
                 incorrect = "True"
 
-            prompt = f"Premise: {premise}\nHypothesis: {hypothesis} True or False?\nAnswer:\nA. {incorrect}\nB. {correct}"
+            # Raw prompt without A./B. formatting
+            prompt = f"Premise: {premise}\nHypothesis: {hypothesis} True or False?"
 
             metadata = {"label": "sglue_rte"}