wisent 0.7.901__py3-none-any.whl → 0.7.1116__py3-none-any.whl

wisent/examples/scripts/results/test_afrimmlu_evaluation.json

@@ -1,324 +0,0 @@
-{
-  "task_name": "afrimmlu",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 15,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Ked\u1ee5 \u1ee5d\u1ecb emume Steeti a ch\u1ecdr\u1ecd?...",
-      "positive_response": "[",
-      "negative_response": "'",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '[' (log_prob=-0.500), Expected: '['"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '[' (log_prob=-0.500), Expected: '''"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 1,
-      "prompt": "\u12a8\u121a\u12a8\u1270\u1209\u1275 \u12cd\u1235\u1325 \u12e8\u12cb\u130b (P) \u1260\u134d\u1341\u121d \u1349\u12ad\u12ad\u122d \u12cd\u1235\u1325 \u1208\u121d\u1295 \u12a8\u1205\u12f3\u130d \u1308\u1262 (MR) \u130b\u122d \u12a5\u1295\u12f0\u121a\u1270\u12ab\u12a8\u120d\u1363 \u12a5\u1293 \u12cb\u130b (P) \u1260\u1265\u1278\u129d\u1290\u1275 \u12c8\u12ed\u121d \u134d\u133d\u121d\u1293 \u1260\u130e\u12f0\u1208\u12cd \u1349\u12ad\u12ad\u122d \u12a8\u12a0\u1290\u1235\u1270\u129b \u1308\u1262 \u1208\u121d\u1295 \u12a5\u1295\u12f0\u121a\u1260\u120d\u1325 \u1260\u1275\u12ad\u12ad\u120d \u12e8\u121a\u12eb\u1235\u1228\u12f3\u12cd \u12e8\u1275\u129b\u12cd \u1290\u12cd? I. MR = P = \u1208\u134d\u1339\u121d \u12cd\u12f5\u12f5\u122d \u1260\u12a0\u130d\u12f5\u121d \u1270\u130d\u1263\u122d \u120b\u12ed \u134d\u120b\u130e\u1275. II. P > MR \u134d\u133d\u121d\u1293 \u12e8\u130e\u12f0\u1208\u12cd \u1349\u12ad\u12ad\u122d \u12a5\u1295\u12f0 \u1241\u120d\u1241\u120d \u1270\u1295\u1238\u122b\u1273\u127d \u1270\u130d\u1263\u122b\u1275\u1362 III. \u134d\u120b\u130e\u1275 \u12a5\u1293 \u12cb\u130b \u134d\u133d\u121d\u1293 \u12e8\u130e\u12f0\u1208\u12cd \u12cd\u12f5\u12f5\u122d \u12a5\u1295\u12f0 \u124b\u121a \u1270\u130d\u1263\u122d \u1290\u12cd \u12e8\u121a\u12c8\u12a8\u1209\u1275\u1362...",
-      "positive_response": "'",
-      "negative_response": "I",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: '''"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: 'I'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 2,
-      "prompt": "Na 1997, \u1ee4l\u1ecd ak\u1ee5 \u1ee5wa ch\u1ecdp\u1ee5tara na fraksh\u1ecdn nke nd\u1ecb b\u1ecb n'\u1ee5wa niile bi na mba nd\u1ecb na-akpata obere ego--ya b\u1ee5 mba kwa onye \u1ecdb\u1ee5la n'eri \u1ee5gw\u1ecd \u1ecdnwa nke $1230 ma\u1ecdb\u1ee5 n'erughi --d\u1ecb ihe d\u1ecb ka...",
-      "positive_response": "5",
-      "negative_response": "%",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '5' (log_prob=-0.500), Expected: '5'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '5' (log_prob=-0.500), Expected: '%'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 3,
-      "prompt": "Mu gihe cyacyo cy'agahebuzo, ni irihe tandukaniro riri hagati y'abantu bo mu Majyepfo n'abo mu bindi bice bya Leta Zunze Ubumwe z'Amerika bemera ko amashuri atandukana n'ayandi?...",
-      "positive_response": "'",
-      "negative_response": "8",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: '''"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: '8'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 4,
-      "prompt": "Ni iyihe mibare yabaturage itubwira igipimo cyuburumbuke aho umubare wabana bazavuka uzaba uhagije kugira ngo usimbure ababyeyi no kwishyura impfu zabana bapfa bakiri bato?...",
-      "positive_response": "'",
-      "negative_response": "U",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: '''"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: 'U'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 5,
-      "prompt": "A 2016, kusan mutane nawa a Amurka ba su da matsuguni?...",
-      "positive_response": "'",
-      "negative_response": "5",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: '''"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: '5'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 6,
-      "prompt": "Pour quel nombre l'\u00e9quation ci-apr\u00e8s est-elle vraie? : 26,1 mm = _ dm....",
-      "positive_response": "[",
-      "negative_response": "'",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '[' (log_prob=-0.500), Expected: '['"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '[' (log_prob=-0.500), Expected: '''"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 7,
-      "prompt": "Je, kuna vikwazo vyovyote vya kuingizwa kwa uhalifu wa kimila chini ya sheria ya Kiingereza?...",
-      "positive_response": "a",
-      "negative_response": "k",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'a' (log_prob=-0.500), Expected: 'a'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'a' (log_prob=-0.500), Expected: 'k'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 8,
-      "prompt": "Akwai wasu iyakoki dan shigar laifukan al'ada a \u0199ar\u0199ashin dokar Ingilishi?...",
-      "positive_response": "a",
-      "negative_response": "b",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'a' (log_prob=-0.500), Expected: 'a'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'a' (log_prob=-0.500), Expected: 'b'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 9,
-      "prompt": "Kufikia 2019, ni takriban asilimia ngapi ya watu duniani wanaishi katika umaskini uliokithiri (chini ya $1.90 kwa siku)?...",
-      "positive_response": "'",
-      "negative_response": "5",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: '''"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: '5'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 10,
-      "prompt": "Which treaties are considered as 'source of international law' under article 38 ICJ Statute?...",
-      "positive_response": "'",
-      "negative_response": "A",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: '''"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: 'A'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 11,
-      "prompt": "Umupaka w'ibishoboka byo gukora uzagaragara nk'umurongo ugororotse igihe...",
-      "positive_response": "b",
-      "negative_response": "u",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'b' (log_prob=-0.500), Expected: 'b'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'b' (log_prob=-0.500), Expected: 'u'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 12,
-      "prompt": "T\u00ed \u00ecd\u00e1s\u00ed l\u00e1ti \u00ecta t\u00ed \u00f2 d\u00e1ra b\u00e1 darap\u1ecd\u0300 m\u1ecd\u0301 \u00ecp\u00e8s\u00e8 ohun \u00e8l\u00f2 al\u00e1d\u00e0ni, \u00e8wo n\u00edn\u00fa \u00e0w\u1ecdn w\u1ecd\u0300ny\u00ec\u00ed ni \u00ecg\u00e9s\u1eb9\u0300 l\u00e1ti \u1ecd\u0300d\u1ecd\u0300 \u00ecj\u1ecdba t\u00ed \u00f3 \u1e63e\u00e9\u1e63e k\u00f3 gb\u00e9 \u1ecdj\u00e0 n\u00e1\u00e0 l\u1ecd s\u00edbi \u00e0b\u00e1j\u00e1de t\u00f3 d\u00e1ra?...",
-      "positive_response": "T",
-      "negative_response": "i",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'T' (log_prob=-0.500), Expected: 'T'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'T' (log_prob=-0.500), Expected: 'i'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 13,
-      "prompt": "What was GDP per capita in the United States in 1850 when adjusting for inflation and PPP in 2011 prices?...",
-      "positive_response": "'",
-      "negative_response": "A",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: '''"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: ''' (log_prob=-0.500), Expected: 'A'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 14,
-      "prompt": "Ni uwuhe mubare w'abana bafite hagati y'imyaka 13 na 15 muri Misiri bavuze ko bahohotewe nibura rimwe mu mezi abiri ashize, guhera mu mwaka wa 2015?...",
-      "positive_response": "0",
-      "negative_response": "%",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '0' (log_prob=-0.500), Expected: '0'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '0' (log_prob=-0.500), Expected: '%'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_afrimmlu_pairs.json

@@ -1,92 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Ked\u1ee5 \u1ee5d\u1ecb emume Steeti a ch\u1ecdr\u1ecd?",
-    "positive_response": "[",
-    "negative_response": "'"
-  },
-  {
-    "pair_id": 1,
-    "prompt": "\u12a8\u121a\u12a8\u1270\u1209\u1275 \u12cd\u1235\u1325 \u12e8\u12cb\u130b (P) \u1260\u134d\u1341\u121d \u1349\u12ad\u12ad\u122d \u12cd\u1235\u1325 \u1208\u121d\u1295 \u12a8\u1205\u12f3\u130d \u1308\u1262 (MR) \u130b\u122d \u12a5\u1295\u12f0\u121a\u1270\u12ab\u12a8\u120d\u1363 \u12a5\u1293 \u12cb\u130b (P) \u1260\u1265\u1278\u129d\u1290\u1275 \u12c8\u12ed\u121d \u134d\u133d\u121d\u1293 \u1260\u130e\u12f0\u1208\u12cd \u1349\u12ad\u12ad\u122d \u12a8\u12a0\u1290\u1235\u1270\u129b \u1308\u1262 \u1208\u121d\u1295 \u12a5\u1295\u12f0\u121a\u1260\u120d\u1325 \u1260\u1275\u12ad\u12ad\u120d \u12e8\u121a\u12eb\u1235\u1228\u12f3\u12cd \u12e8\u1275\u129b\u12cd \u1290\u12cd? I. MR = P = \u1208\u134d\u1339\u121d \u12cd\u12f5\u12f5\u122d \u1260\u12a0\u130d\u12f5\u121d \u1270\u130d\u1263\u122d \u120b\u12ed \u134d\u120b\u130e\u1275. II. P > MR \u134d\u133d\u121d\u1293 \u12e8\u130e\u12f0\u1208\u12cd \u1349\u12ad\u12ad\u122d \u12a5\u1295\u12f0 \u1241\u120d\u1241\u120d \u1270\u1295\u1238\u122b\u1273\u127d \u1270\u130d\u1263\u122b\u1275\u1362 III. \u134d\u120b\u130e\u1275 \u12a5\u1293 \u12cb\u130b \u134d\u133d\u121d\u1293 \u12e8\u130e\u12f0\u1208\u12cd \u12cd\u12f5\u12f5\u122d \u12a5\u1295\u12f0 \u124b\u121a \u1270\u130d\u1263\u122d \u1290\u12cd \u12e8\u121a\u12c8\u12a8\u1209\u1275\u1362",
-    "positive_response": "'",
-    "negative_response": "I"
-  },
-  {
-    "pair_id": 2,
-    "prompt": "Na 1997, \u1ee4l\u1ecd ak\u1ee5 \u1ee5wa ch\u1ecdp\u1ee5tara na fraksh\u1ecdn nke nd\u1ecb b\u1ecb n'\u1ee5wa niile bi na mba nd\u1ecb na-akpata obere ego--ya b\u1ee5 mba kwa onye \u1ecdb\u1ee5la n'eri \u1ee5gw\u1ecd \u1ecdnwa nke $1230 ma\u1ecdb\u1ee5 n'erughi --d\u1ecb ihe d\u1ecb ka",
-    "positive_response": "5",
-    "negative_response": "%"
-  },
-  {
-    "pair_id": 3,
-    "prompt": "Mu gihe cyacyo cy'agahebuzo, ni irihe tandukaniro riri hagati y'abantu bo mu Majyepfo n'abo mu bindi bice bya Leta Zunze Ubumwe z'Amerika bemera ko amashuri atandukana n'ayandi?",
-    "positive_response": "'",
-    "negative_response": "8"
-  },
-  {
-    "pair_id": 4,
-    "prompt": "Ni iyihe mibare yabaturage itubwira igipimo cyuburumbuke aho umubare wabana bazavuka uzaba uhagije kugira ngo usimbure ababyeyi no kwishyura impfu zabana bapfa bakiri bato?",
-    "positive_response": "'",
-    "negative_response": "U"
-  },
-  {
-    "pair_id": 5,
-    "prompt": "A 2016, kusan mutane nawa a Amurka ba su da matsuguni?",
-    "positive_response": "'",
-    "negative_response": "5"
-  },
-  {
-    "pair_id": 6,
-    "prompt": "Pour quel nombre l'\u00e9quation ci-apr\u00e8s est-elle vraie? : 26,1 mm = _ dm.",
-    "positive_response": "[",
-    "negative_response": "'"
-  },
-  {
-    "pair_id": 7,
-    "prompt": "Je, kuna vikwazo vyovyote vya kuingizwa kwa uhalifu wa kimila chini ya sheria ya Kiingereza?",
-    "positive_response": "a",
-    "negative_response": "k"
-  },
-  {
-    "pair_id": 8,
-    "prompt": "Akwai wasu iyakoki dan shigar laifukan al'ada a \u0199ar\u0199ashin dokar Ingilishi?",
-    "positive_response": "a",
-    "negative_response": "b"
-  },
-  {
-    "pair_id": 9,
-    "prompt": "Kufikia 2019, ni takriban asilimia ngapi ya watu duniani wanaishi katika umaskini uliokithiri (chini ya $1.90 kwa siku)?",
-    "positive_response": "'",
-    "negative_response": "5"
-  },
-  {
-    "pair_id": 10,
-    "prompt": "Which treaties are considered as 'source of international law' under article 38 ICJ Statute?",
-    "positive_response": "'",
-    "negative_response": "A"
-  },
-  {
-    "pair_id": 11,
-    "prompt": "Umupaka w'ibishoboka byo gukora uzagaragara nk'umurongo ugororotse igihe",
-    "positive_response": "b",
-    "negative_response": "u"
-  },
-  {
-    "pair_id": 12,
-    "prompt": "T\u00ed \u00ecd\u00e1s\u00ed l\u00e1ti \u00ecta t\u00ed \u00f2 d\u00e1ra b\u00e1 darap\u1ecd\u0300 m\u1ecd\u0301 \u00ecp\u00e8s\u00e8 ohun \u00e8l\u00f2 al\u00e1d\u00e0ni, \u00e8wo n\u00edn\u00fa \u00e0w\u1ecdn w\u1ecd\u0300ny\u00ec\u00ed ni \u00ecg\u00e9s\u1eb9\u0300 l\u00e1ti \u1ecd\u0300d\u1ecd\u0300 \u00ecj\u1ecdba t\u00ed \u00f3 \u1e63e\u00e9\u1e63e k\u00f3 gb\u00e9 \u1ecdj\u00e0 n\u00e1\u00e0 l\u1ecd s\u00edbi \u00e0b\u00e1j\u00e1de t\u00f3 d\u00e1ra?",
-    "positive_response": "T",
-    "negative_response": "i"
-  },
-  {
-    "pair_id": 13,
-    "prompt": "What was GDP per capita in the United States in 1850 when adjusting for inflation and PPP in 2011 prices?",
-    "positive_response": "'",
-    "negative_response": "A"
-  },
-  {
-    "pair_id": 14,
-    "prompt": "Ni uwuhe mubare w'abana bafite hagati y'imyaka 13 na 15 muri Misiri bavuze ko bahohotewe nibura rimwe mu mezi abiri ashize, guhera mu mwaka wa 2015?",
-    "positive_response": "0",
-    "negative_response": "%"
-  }
-]