wisent 0.7.379__py3-none-any.whl → 0.7.701__py3-none-any.whl

wisent/examples/scripts/results/test_lingoly_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "lingoly",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Below is a problem sheet from a lingusitics exam. You will first see the entire sheet, then be asked...",
-      "positive_response": "You (pl) will sit down because your (pl) snake is dirty",
-      "negative_response": "incorrect translation",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'You (pl) will sit down because your (pl) snake is dirty' (log_prob=-0.500), Expected: 'You (pl) will sit down because your (pl) snake is dirty'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'You (pl) will sit down because your (pl) snake is dirty' (log_prob=-0.500), Expected: 'incorrect translation'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_lingoly_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Below is a problem sheet from a lingusitics exam. You will first see the entire sheet, then be asked to respond to specific questions from the sheet. Your answers to the questions should rely only on reasoning about the information provided in the sheet.\n                Problem 5. Tawala (30 marks)\nTawala is an Oceanic language spoken in Papua New Guinea by 10,000 people who live in hamlets and small villages on the East Cape peninsula.\n\n                 Here are some Tawala sentences and their translations. (Plural (pl) and singular (sg) are shown in the translation.)\n\n1\tApo onadewadewa babana uhilage\tYou (pl) will be good because you (sg) are dying.\n2\tApo onanae babana odaladala\tYou (pl) will go because you (pl) are crawling.\n3\tBaha anona ma lawa memehi\tThe speech has substance but the people are settled\n4\tEga Limi natuna babana Limi ega natuna\tThat is not Limi\u2019s child because Limi has no children.\t\n5\tGala gobugobuhi babana tahaya bigabigana\tThe clothes are stained because the path is swampy.\t\n6\tHiduhuduhuna babana baha ega sigasigana\tThey are sitting down because the speech has no end\t\n7\tHinam ginouli idanedanenehi ma ega dewadewana\tYour (sg) mother is stealing things and is never good\n8\tInapa ega unanenae nu tahaya apo hinapanim\tIf you (sg) aren\u2019t going to the path, they will\timprison you (sg)\n9\tInapa unapeu apo unagobu\tIf you (sg) fall, you (sg) will be dirty\n10\tLawa pipeuna imae\tThe falling person is staying\n11\tMotam daodaohi ega hilhilagehi\tYour (sg) long snakes are not dead\n12\tNatuhi ega lawa memena natuna\tTheir child is not the settled person\u2019s child\n13\tOgaleya ma ega igalemi babana itowotowolo\tYou (pl) see him but he does not see you (pl) because he is standing\n14\tTahaya ibigabiga ma apo mawa inadao\tThe path is muddy (because it has rained) and the distance will be long\n15\tTewela daladalana ogaleya ma gala higobu\tYou (pl) see a crawling child and the clothes are dirty.\n16\tTewela towotowolohi hipeu\tThe standing children fall.\n\n\n                 \nTranslate between English and Tawala. Include markings for plural (pl) and singular (sg) where necessary:\nA Apo onaduhuna babana motami igobu\nB Inapa unatowolo apo unamae\nC Tewela hidewadewa ma natum dewadewana\nD Unenae nu tahaya daodaona\nE You (pl) steal a swampy thing.\nF If the path is long, you (sg) will be dirty.\nG Limi\u2019s settled child is dying but is not dead.\nH The crawling children have substance.\n\n                 \n\n                Now respond to the following questions:\n                \nTranslate between English and Tawala. Include markings for plural (pl) and singular (sg) where necessary:\nA Apo onaduhuna babana motami igobu\nB Inapa unatowolo apo unamae\nC Tewela hidewadewa ma natum dewadewana\nD Unenae nu tahaya daodaona\nE You (pl) steal a swampy thing.\nF If the path is long, you (sg) will be dirty.\nG Limi\u2019s settled child is dying but is not dead.\nH The crawling children have substance.\n\n\n                Format your response as a json file with the keys as provided below:\n                {\"A\": \"\", \"B\": \"\", \"C\": \"\", \"D\": \"\", \"E\": \"\", \"F\": \"\", \"G\": \"\", \"H\": \"\"}",
-    "positive_response": "You (pl) will sit down because your (pl) snake is dirty",
-    "negative_response": "incorrect translation"
-  }
-]

wisent/examples/scripts/results/test_livecodebench_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "livecodebench",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Question: There are three cards with letters $\\texttt{a}$, $\\texttt{b}$, $\\texttt{c}$ placed in a ro...",
-      "positive_response": "\nt = int(input())\nfor _ in range(t):\n    s = input().strip()\n    if s == 'abc':\n        print(\"YES\")\n        continue\n    chars = list(s)\n    found = False\n    # Check all possible single swaps\n    for i, j in [(0, 1), (0, 2), (1, 2)]:\n        temp = chars.copy()\n        temp[i], temp[j] = temp[j], temp[i]\n        if ''.join(temp) == 'abc':\n            found = True\n            break\n    print(\"YES\" if found else \"NO\")\n",
-      "negative_response": "# Read the number of test cases\nt = int(input())\n\nfor _ in range(t):\n    # Read the input string\n    s = input()\n    \n    # Check if the string is already \"abc\"\n    if s == \"abc\":\n        print(\"YES\")\n        continue\n    \n    # Check if the string can be converted to \"abc\" by swapping two characters\n    if \"\".join(sorted(s)) == \"abc\":\n        print(\"YES\")\n    else:\n        print(\"NO\")",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\nt = int(input())\nfor _ in range(t):\n    s = input().strip()\n    if s == 'abc':\n        print(\"YES\")\n        continue\n    chars = list(s)\n    found = False\n    # Check all possible single swaps\n    for i, j in [(0, 1), (0, 2), (1, 2)]:\n        temp = chars.copy()\n        temp[i], temp[j] = temp[j], temp[i]\n        if ''.join(temp) == 'abc':\n            found = True\n            break\n    print(\"YES\" if found else \"NO\")\n' (log_prob=-0.500), Expected: '\nt = int(input())\nfor _ in range(t):\n    s = input().strip()\n    if s == 'abc':\n        print(\"YES\")\n        continue\n    chars = list(s)\n    found = False\n    # Check all possible single swaps\n    for i, j in [(0, 1), (0, 2), (1, 2)]:\n        temp = chars.copy()\n        temp[i], temp[j] = temp[j], temp[i]\n        if ''.join(temp) == 'abc':\n            found = True\n            break\n    print(\"YES\" if found else \"NO\")\n'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\nt = int(input())\nfor _ in range(t):\n    s = input().strip()\n    if s == 'abc':\n        print(\"YES\")\n        continue\n    chars = list(s)\n    found = False\n    # Check all possible single swaps\n    for i, j in [(0, 1), (0, 2), (1, 2)]:\n        temp = chars.copy()\n        temp[i], temp[j] = temp[j], temp[i]\n        if ''.join(temp) == 'abc':\n            found = True\n            break\n    print(\"YES\" if found else \"NO\")\n' (log_prob=-0.500), Expected: '# Read the number of test cases\nt = int(input())\n\nfor _ in range(t):\n    # Read the input string\n    s = input()\n    \n    # Check if the string is already \"abc\"\n    if s == \"abc\":\n        print(\"YES\")\n        continue\n    \n    # Check if the string can be converted to \"abc\" by swapping two characters\n    if \"\".join(sorted(s)) == \"abc\":\n        print(\"YES\")\n    else:\n        print(\"NO\")'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_livecodebench_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Question: There are three cards with letters $\\texttt{a}$, $\\texttt{b}$, $\\texttt{c}$ placed in a row in some order. You can do the following operation at most once: \n\n \n-  Pick two cards, and swap them.  Is it possible that the row becomes $\\texttt{abc}$ after the operation? Output \"YES\" if it is possible, and \"NO\" otherwise.\n\nInput\n\nThe first line contains a single integer $t$ ($1 \\leq t \\leq 6$)\u00a0\u2014 the number of test cases.\n\nThe only line of each test case contains a single string consisting of each of the three characters $\\texttt{a}$, $\\texttt{b}$, and $\\texttt{c}$ exactly once, representing the cards.\n\nOutput\n\nFor each test case, output \"YES\" if you can make the row $\\texttt{abc}$ with at most one operation, or \"NO\" otherwise.\n\nYou can output the answer in any case (for example, the strings \"yEs\", \"yes\", \"Yes\" and \"YES\" will be recognized as a positive answer).Sample Input 1:\n6\n\nabc\n\nacb\n\nbac\n\nbca\n\ncab\n\ncba\n\n\n\nSample Output 1:\n\nYES\nYES\nYES\nNO\nNO\nYES\n\n\nNote\n\nIn the first test case, we don't need to do any operations, since the row is already $\\texttt{abc}$.\n\nIn the second test case, we can swap $\\texttt{c}$ and $\\texttt{b}$: $\\texttt{acb} \\to \\texttt{abc}$.\n\nIn the third test case, we can swap $\\texttt{b}$ and $\\texttt{a}$: $\\texttt{bac} \\to \\texttt{abc}$.\n\nIn the fourth test case, it is impossible to make $\\texttt{abc}$ using at most one operation.\n\nWrite a solution:",
-    "positive_response": "\nt = int(input())\nfor _ in range(t):\n    s = input().strip()\n    if s == 'abc':\n        print(\"YES\")\n        continue\n    chars = list(s)\n    found = False\n    # Check all possible single swaps\n    for i, j in [(0, 1), (0, 2), (1, 2)]:\n        temp = chars.copy()\n        temp[i], temp[j] = temp[j], temp[i]\n        if ''.join(temp) == 'abc':\n            found = True\n            break\n    print(\"YES\" if found else \"NO\")\n",
-    "negative_response": "# Read the number of test cases\nt = int(input())\n\nfor _ in range(t):\n    # Read the input string\n    s = input()\n    \n    # Check if the string is already \"abc\"\n    if s == \"abc\":\n        print(\"YES\")\n        continue\n    \n    # Check if the string can be converted to \"abc\" by swapping two characters\n    if \"\".join(sorted(s)) == \"abc\":\n        print(\"YES\")\n    else:\n        print(\"NO\")"
-  }
-]

wisent/examples/scripts/results/test_livemathbench_cnmo_en_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "livemathbench_cnmo_en",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Question: If the real number $m > 1$ satisfies $\\log_9(\\log_8 m) = 2024$, then what is the value of ...",
-      "positive_response": "$4049$",
-      "negative_response": "4050",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '$4049$' (log_prob=-0.500), Expected: '$4049$'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '$4049$' (log_prob=-0.500), Expected: '4050'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_livemathbench_cnmo_en_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Question: If the real number $m > 1$ satisfies $\\log_9(\\log_8 m) = 2024$, then what is the value of $\\log_3(\\log_2 m)$?\n\nWhat is the answer?",
-    "positive_response": "$4049$",
-    "negative_response": "4050"
-  }
-]

wisent/examples/scripts/results/test_livemathbench_cnmo_zh_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "livemathbench_cnmo_zh",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Question: \u82e5\u5b9e\u6570 $m > 1$ \u6ee1\u8db3 $\\log_9(\\log_8 m) = 2024$\uff0c\u5219 $\\log_3(\\log_2 m)$ \u7684\u503c\u4e3a\uff1f\n\nWhat is the answer?...",
-      "positive_response": "$4049$",
-      "negative_response": "4050",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '$4049$' (log_prob=-0.500), Expected: '$4049$'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '$4049$' (log_prob=-0.500), Expected: '4050'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_livemathbench_cnmo_zh_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Question: \u82e5\u5b9e\u6570 $m > 1$ \u6ee1\u8db3 $\\log_9(\\log_8 m) = 2024$\uff0c\u5219 $\\log_3(\\log_2 m)$ \u7684\u503c\u4e3a\uff1f\n\nWhat is the answer?",
-    "positive_response": "$4049$",
-    "negative_response": "4050"
-  }
-]

wisent/examples/scripts/results/test_llama_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "llama",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Question: Juan and LaKeisha roll a few objects down a ramp. They want to see which object rolls the ...",
-      "positive_response": "Record the details of the investigation.",
-      "negative_response": "Put the objects in groups.",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'Record the details of the investigation.' (log_prob=-0.500), Expected: 'Record the details of the investigation.'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'Record the details of the investigation.' (log_prob=-0.500), Expected: 'Put the objects in groups.'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_llama_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Question: Juan and LaKeisha roll a few objects down a ramp. They want to see which object rolls the farthest. What should they do so they can repeat their investigation?\nA. Put the objects in groups.\nB. Record the details of the investigation.",
-    "positive_response": "Record the details of the investigation.",
-    "negative_response": "Put the objects in groups."
-  }
-]

wisent/examples/scripts/results/test_logiqa2_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "logiqa2",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Passage: Jupiter is a gas giant planet and the largest planet in the solar system. Its mass is 2.5 t...",
-      "positive_response": "The satellite and the planets around it were formed from the same gas and dust at the same time.",
-      "negative_response": "After hundreds of millions of years, the satellite may slowly fall onto the planet.",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'The satellite and the planets around it were formed from the same gas and dust at the same time.' (log_prob=-0.500), Expected: 'The satellite and the planets around it were formed from the same gas and dust at the same time.'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'The satellite and the planets around it were formed from the same gas and dust at the same time.' (log_prob=-0.500), Expected: 'After hundreds of millions of years, the satellite may slowly fall onto the planet.'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_logiqa2_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Passage: Jupiter is a gas giant planet and the largest planet in the solar system. Its mass is 2.5 times the total mass of the other seven planets in the solar system. Observations have found that most of the more than 70 moons surrounding Jupiter are composed of water ice. Therefore, Jupiter's atmosphere should contain a considerable amount of water.\nQuestion: Which of the followings, if true, can best support the above statement?\nA. After hundreds of millions of years, the satellite may slowly fall onto the planet.\nB. The satellite and the planets around it were formed from the same gas and dust at the same time.",
-    "positive_response": "The satellite and the planets around it were formed from the same gas and dust at the same time.",
-    "negative_response": "After hundreds of millions of years, the satellite may slowly fall onto the planet."
-  }
-]

wisent/examples/scripts/results/test_logiqa_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "logiqa",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Passage: Black Americans are twice as likely to suffer from hypertension as white Americans. The sam...",
-      "positive_response": "The blood pressure of the descendants of Senegalese and Gambians is usually not high, and the history of Senegal and Gambia has not been short of salt.",
-      "negative_response": "The unusually high salt intake in certain parts of Africa is a serious problem that threatens the health of residents.",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'The blood pressure of the descendants of Senegalese and Gambians is usually not high, and the history of Senegal and Gambia has not been short of salt.' (log_prob=-0.500), Expected: 'The blood pressure of the descendants of Senegalese and Gambians is usually not high, and the history of Senegal and Gambia has not been short of salt.'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'The blood pressure of the descendants of Senegalese and Gambians is usually not high, and the history of Senegal and Gambia has not been short of salt.' (log_prob=-0.500), Expected: 'The unusually high salt intake in certain parts of Africa is a serious problem that threatens the health of residents.'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_logiqa_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Passage: Black Americans are twice as likely to suffer from hypertension as white Americans. The same is true when comparing Westernized black Africans to white Africans. The researchers hypothesized that the reason why westernized black people suffer from hypertension is the result of the interaction of two reasons? one is the high salt content of western foods, and the other is the adaptation mechanism of black genetic genes to the salt-deficient environment .\nQuestion: The following conclusions about contemporary westernized African blacks, if the item is true, can it best support the researchers' hypothesis?\nA. The unusually high salt intake in certain parts of Africa is a serious problem that threatens the health of residents.\nB. The blood pressure of the descendants of Senegalese and Gambians is usually not high, and the history of Senegal and Gambia has not been short of salt.",
-    "positive_response": "The blood pressure of the descendants of Senegalese and Gambians is usually not high, and the history of Senegal and Gambia has not been short of salt.",
-    "negative_response": "The unusually high salt intake in certain parts of Africa is a serious problem that threatens the health of residents."
-  }
-]

wisent/examples/scripts/results/test_m_mmlu_evaluation.json

@@ -1,51 +0,0 @@
-{
-  "task_name": "m_mmlu",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 2,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Question: \u092f\u094b \u092a\u094d\u0930\u0936\u094d\u0928 \u0928\u093f\u092e\u094d\u0928\u0932\u093f\u0916\u093f\u0924 \u0938\u0942\u091a\u0928\u093e\u0915\u094b \u092c\u093e\u0930\u0947\u092e\u093e \u0939\u094b \u0964\n\u090f\u0921\u093f\u0938\u0928\u0915\u094b \u0915\u093e\u0930\u094d\u092f\u0915\u094d\u0930\u092e \u092f\u0941\u0926\u094d\u0927\u092a\u091b\u093f \u091a\u093e\u0939\u093f\u090f\u0915\u094b \u0915\u093e\u092e\u0926\u093e\u0930 \u0906\u0936\u093e\u0939\u0930\u0942 ...",
-      "positive_response": "\u0935\u0943\u0939\u0926\u093e\u0915\u093e\u0930 \u0924\u0925\u093e \u0938\u0928\u0915\u0940\u0915\u0930\u0923\u0915\u094b \u0935\u0943\u0926\u094d\u0927\u093f",
-      "negative_response": "\u0935\u093f\u0936\u094d\u0935 \u0936\u0915\u094d\u0924\u093f \u0915\u094b \u0930\u0941\u092a\u092e\u093e \u0909\u092d\u093f\u0928\u0947 \u0938\u092e\u0941\u0926\u093e\u092f\u0915\u094b \u0909\u0926\u092f",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\u0935\u0943\u0939\u0926\u093e\u0915\u093e\u0930 \u0924\u0925\u093e \u0938\u0928\u0915\u0940\u0915\u0930\u0923\u0915\u094b \u0935\u0943\u0926\u094d\u0927\u093f' (log_prob=-0.500), Expected: '\u0935\u0943\u0939\u0926\u093e\u0915\u093e\u0930 \u0924\u0925\u093e \u0938\u0928\u0915\u0940\u0915\u0930\u0923\u0915\u094b \u0935\u0943\u0926\u094d\u0927\u093f'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\u0935\u0943\u0939\u0926\u093e\u0915\u093e\u0930 \u0924\u0925\u093e \u0938\u0928\u0915\u0940\u0915\u0930\u0923\u0915\u094b \u0935\u0943\u0926\u094d\u0927\u093f' (log_prob=-0.500), Expected: '\u0935\u093f\u0936\u094d\u0935 \u0936\u0915\u094d\u0924\u093f \u0915\u094b \u0930\u0941\u092a\u092e\u093e \u0909\u092d\u093f\u0928\u0947 \u0938\u092e\u0941\u0926\u093e\u092f\u0915\u094b \u0909\u0926\u092f'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 1,
-      "prompt": "Question: Un escriptor tenia una propietat a la ciutat. Aquesta propietat era l'\u00fanica que tenia el p...",
-      "positive_response": "no va transmetre al comerciant un termini de temps legal.",
-      "negative_response": "era nul, invalid, i no tenia efecte legal.",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'no va transmetre al comerciant un termini de temps legal.' (log_prob=-0.500), Expected: 'no va transmetre al comerciant un termini de temps legal.'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'no va transmetre al comerciant un termini de temps legal.' (log_prob=-0.500), Expected: 'era nul, invalid, i no tenia efecte legal.'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_m_mmlu_pairs.json

@@ -1,14 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Question: \u092f\u094b \u092a\u094d\u0930\u0936\u094d\u0928 \u0928\u093f\u092e\u094d\u0928\u0932\u093f\u0916\u093f\u0924 \u0938\u0942\u091a\u0928\u093e\u0915\u094b \u092c\u093e\u0930\u0947\u092e\u093e \u0939\u094b \u0964\n\u090f\u0921\u093f\u0938\u0928\u0915\u094b \u0915\u093e\u0930\u094d\u092f\u0915\u094d\u0930\u092e \u092f\u0941\u0926\u094d\u0927\u092a\u091b\u093f \u091a\u093e\u0939\u093f\u090f\u0915\u094b \u0915\u093e\u092e\u0926\u093e\u0930 \u0906\u0936\u093e\u0939\u0930\u0942 \u092a\u0942\u0930\u093e \u0928\u092d\u090f\u092e\u093e, \u092c\u093f\u091f\u0947\u0928\u092e\u093e \u0930\u0942\u0938\u092e\u093e \u092d\u090f\u0915\u094b \u091c\u0938\u094d\u0924\u094b \u090f\u0915 \u0938\u093e\u0901\u091a\u094b \u092e\u0939\u093e\u092e\u093e\u0930\u0940 \u0906\u0907\u092a\u0941\u0917\u094d\u0926\u0948 \u091c\u0938\u094d\u0924\u094b \u0932\u093e\u0917\u094d\u0928\u0947 \u092c\u093f\u0936\u094d\u0935\u093e\u0938\u092e\u093e \u092c\u0928\u093e\u0907\u090f\u0915\u094b \u0939\u094b \u0964 \u0968\u0966\u0968\u0966 \u0938\u093e\u0932\u092e\u093e \u0924\u094d\u092f\u094b \u092d\u092f \u0938\u0941\u0938\u094d\u0924\u093f\u0928\u0947 \u0925\u093e\u0932\u094d\u092f\u094b \u0930 \u090f\u0921\u093f\u0938\u0928\u0915\u094b \u0928\u0940\u0924\u093f\u0932\u093e\u0908 \u0905\u0924\u094d\u092f\u0927\u093f\u0915 \u092e\u093e\u0928\u093f\u0928\u0947 \u0932\u093e\u0917\u0947\u0915\u094b \u0925\u093f\u092f\u094b \u0964 \u090f\u0909\u091f\u093e \u0938\u094d\u0925\u093e\u0928\u0940\u092f \u0915\u093e\u0909\u0928\u094d\u0938\u093f\u0932\u0932\u0947 \u0938\u093f\u0924\u092e\u094d\u092c\u0930 \u0967\u096f\u0968\u0966 \u092e\u093e \u0938\u093e\u092e\u094d\u092f\u0915\u093e \u0930\u0941\u092a\u092e\u093e \u0905\u0928\u0941\u092e\u094b\u0926\u0928 \u0917\u0930\u093f\u090f\u0915\u094b \u0907\u091c\u0932\u094d\u0932\u093e \u0938\u094d\u091f\u094d\u0930\u0940\u091f \u092f\u094b\u091c\u0928\u093e \u0928\u092f\u093e\u0901 \u0926\u0943\u0937\u094d\u091f\u093f\u092d\u0919\u094d\u0917\u093f\u0915\u094b \u0936\u093f\u0915\u093e\u0930 \u092d\u092f\u094b \u0964 \u0915\u093e\u0909\u0928\u094d\u0938\u093f\u0932\u0915\u094b \u0906\u0936\u094d\u091a\u0930\u094d\u092f \u092d\u090f\u0915\u094b \u0915\u0941\u0930\u093e, \u0938\u0930\u0915\u093e\u0930\u0915\u094b \u0939\u093e\u0909\u0938\u093f\u0919 \u092c\u094b\u0930\u094d\u0921\u0932\u0947 \u092f\u094b\u091c\u0928\u093e\u0932\u093e\u0908 \u0930\u094b\u0915\u0947\u0915\u094b \u0925\u093f\u092f\u094b, \"\u0938\u092e\u092f\u0938\u092e\u0947\u0924 \u0915\u093e\u0909\u0928\u094d\u0938\u093f\u0932\u0915\u094b \u0935\u0930\u094d\u0924\u092e\u093e\u0928 \u0915\u0930\u094d\u0924\u0935\u094d\u092f\u0939\u0930\u0942 \u0930 \u0909\u092a\u0932\u092c\u094d\u0927 \u092a\u0948\u0938\u093e \u0938\u092e\u0947\u0924 \u0927\u094d\u092f\u093e\u0928 \u0926\u093f\u090f\u0930\" \u0930 \u092e\u0947 \u0967\u096f\u0968\u0967 \u092e\u093e \u0938\u0930\u0915\u093e\u0930\u0932\u0947 \u0928\u093f\u0935\u093e\u0938\u0940 \u092a\u094d\u0930\u094b\u0917\u094d\u0930\u093e\u092e\u0915\u094b \u092c\u0939\u0941\u0924\u0948 \u0915\u092e\u0940 \u0917\u0930\u094d\u0926\u0948, \u0906\u0927\u093e\u0930 \u0918\u091f\u0915\u094b \u0918\u094b\u0937\u0923\u093e \u0917\u0930\u094d\u092f\u094b\u0964\n\u0964 \u092f\u093e\u0928\u0940 \u0939\u0947\u092e\u094b\u0928\u0940 \u0939\u094b\u092c\u0939\u093e\u0909\u0938 \u0926\u094d\u0935\u093e\u0930\u093e \u090f\u0915 \u0915\u093e\u092e\u0926\u093e\u0930 \u0907\u0924\u093f\u0939\u093e\u0938\u0915\u093e\u0930 \u0926\u094d\u0935\u093e\u0930\u093e \u090f\u0915 \u0917\u094d\u0930\u0928\u094d\u0925\u092c\u093e\u091f \u0938\u092e\u094d\u092a\u093e\u0926\u093f\u0924 \u0907\u0917\u094d\u0932\u093f\u0938 \u0917\u0943\u0939\u0915\u094b \u0932\u094b\u0915\u0924\u093e\u0928\u094d\u0924\u094d\u0930\u093f\u0915 \u0906\u0935\u093e\u0938, \u0967\u096f\u096f\u096a \u0935\u0930\u094d\u0937\u0926\u0947\u0916\u093f\n\u092a\u0939\u093f\u0932\u094b \u0935\u093f\u0936\u094d\u0935 \u092f\u0941\u0926\u094d\u0927\u092a\u091b\u093f \u0928\u093f\u092e\u094d\u0928\u0932\u093f\u0916\u093f\u0924 \u0915\u0941\u0928 \u092c\u093e\u0930\u094d\u0924\u093e\u092e\u093e \u092a\u094d\u0930\u092c\u0932 \u0905\u0927\u093f\u0935\u0947\u0936\u0928 \u0917\u0930\u093f\u0930\u0939\u0947\u0915\u094b \u091b?\nA. \u0935\u093f\u0936\u094d\u0935 \u0936\u0915\u094d\u0924\u093f \u0915\u094b \u0930\u0941\u092a\u092e\u093e \u0909\u092d\u093f\u0928\u0947 \u0938\u092e\u0941\u0926\u093e\u092f\u0915\u094b \u0909\u0926\u092f\nB. \u0935\u0943\u0939\u0926\u093e\u0915\u093e\u0930 \u0924\u0925\u093e \u0938\u0928\u0915\u0940\u0915\u0930\u0923\u0915\u094b \u0935\u0943\u0926\u094d\u0927\u093f",
-    "positive_response": "\u0935\u0943\u0939\u0926\u093e\u0915\u093e\u0930 \u0924\u0925\u093e \u0938\u0928\u0915\u0940\u0915\u0930\u0923\u0915\u094b \u0935\u0943\u0926\u094d\u0927\u093f",
-    "negative_response": "\u0935\u093f\u0936\u094d\u0935 \u0936\u0915\u094d\u0924\u093f \u0915\u094b \u0930\u0941\u092a\u092e\u093e \u0909\u092d\u093f\u0928\u0947 \u0938\u092e\u0941\u0926\u093e\u092f\u0915\u094b \u0909\u0926\u092f"
-  },
-  {
-    "pair_id": 1,
-    "prompt": "Question: Un escriptor tenia una propietat a la ciutat. Aquesta propietat era l'\u00fanica que tenia el propietari. El edifici tenia tres plantes, amb una botiga a la planta baixa i apartaments a les altres dues plantes. El propietari va signar un contracte de lloguer amb un comerciant, qui llogaria la primera planta per obrir una botiga d'articles esportius. Despr\u00e9s d'identificar les parts, les paraules operatives del contracte de lloguer eren les seg\u00fcents: \"El propietari accepta de llogar per tres anys la primera planta del seu edifici a la ciutat al llogater, reservant-se el dret d'un cost anual de lloguer de $12,000 pagable per endavant en quotes mensuals de $1,000\". En el moment de la signatura del contracte de lloguer entre l'escriptor i el comerciant\nA. era nul, invalid, i no tenia efecte legal.\nB. no va transmetre al comerciant un termini de temps legal.",
-    "positive_response": "no va transmetre al comerciant un termini de temps legal.",
-    "negative_response": "era nul, invalid, i no tenia efecte legal."
-  }
-]

wisent/examples/scripts/results/test_mastermind/test_mastermind_evaluation.json

@@ -1,51 +0,0 @@
-{
-  "task_name": "mastermind",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 2,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Your goal is to find the secret four-color code. The following colors are possible: green, black, or...",
-      "positive_response": "green, blue, orange, black",
-      "negative_response": "green, yellow, orange, black",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'green, blue, orange, black' (log_prob=-0.500), Expected: 'green, blue, orange, black'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'green, blue, orange, black' (log_prob=-0.500), Expected: 'green, yellow, orange, black'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 1,
-      "prompt": "Your goal is to find the secret three-color code. The following colors are possible: orange, green, ...",
-      "positive_response": "orange, purple, brown",
-      "negative_response": "orange, orange, purple",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'orange, purple, brown' (log_prob=-0.500), Expected: 'orange, purple, brown'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'orange, purple, brown' (log_prob=-0.500), Expected: 'orange, orange, purple'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_mastermind/test_mastermind_pairs.json

@@ -1,14 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Your goal is to find the secret four-color code. The following colors are possible: green, black, orange, blue, purple, yellow.\nSome guesses have already been made. I will provide feedback for each guess made with which it is possible to unambigiously determine the secret code.\n\nPrevious Guesses:\nGuess: ['black', 'green', 'purple', 'yellow']. Hint: two colors are in the secret code but in the wrong positions.\nGuess: ['green', 'orange', 'orange', 'purple']. Hint: two colors are in the correct positions.\nGuess: ['green', 'orange', 'blue', 'black']. Hint: two colors are in the correct positions and two colors are in the secret code but in the wrong positions.\nGuess: ['green', 'orange', 'black', 'blue']. Hint: one color is in the correct position and three colors are in the secret code but in the wrong position.\n\nThe secret code is:",
-    "positive_response": "green, blue, orange, black",
-    "negative_response": "green, yellow, orange, black"
-  },
-  {
-    "pair_id": 1,
-    "prompt": "Your goal is to find the secret three-color code. The following colors are possible: orange, green, brown, pink, purple.\nSome guesses have already been made. I will provide feedback for each guess made with which it is possible to unambigiously determine the secret code.\n\nPrevious Guesses:\nGuess: ['orange', 'pink', 'green']. Hint: one color is in the correct position.\nGuess: ['green', 'orange', 'brown']. Hint: one color is in the correct position and one color is in the secret code but in the wrong position.\nGuess: ['orange', 'brown', 'brown']. Hint: two colors are in the correct positions.\n\nThe secret code is:",
-    "positive_response": "orange, purple, brown",
-    "negative_response": "orange, orange, purple"
-  }
-]

wisent/examples/scripts/results/test_mastermind_24_easy/test_mastermind_24_easy_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "mastermind_24_easy",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Your goal is to find the secret two-color code. The following colors are possible: yellow, blue, ora...",
-      "positive_response": "orange, blue",
-      "negative_response": "green, green",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'orange, blue' (log_prob=-0.500), Expected: 'orange, blue'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'orange, blue' (log_prob=-0.500), Expected: 'green, green'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_mastermind_24_easy/test_mastermind_24_easy_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Your goal is to find the secret two-color code. The following colors are possible: yellow, blue, orange, green.\nSome guesses have already been made. I will provide feedback for each guess made with which it is possible to unambigiously determine the secret code.\n\nPrevious Guesses:\nGuess: ['blue', 'orange']. Hint: two colors are in the secret code but in the wrong positions.\n\nThe secret code is:",
-    "positive_response": "orange, blue",
-    "negative_response": "green, green"
-  }
-]

wisent/examples/scripts/results/test_mastermind_evaluation.json

@@ -1,51 +0,0 @@
-{
-  "task_name": "mastermind",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 2,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Your goal is to find the secret four-color code. The following colors are possible: green, black, or...",
-      "positive_response": "green, blue, orange, black",
-      "negative_response": "green, yellow, orange, black",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'green, blue, orange, black' (log_prob=-0.500), Expected: 'green, blue, orange, black'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'green, blue, orange, black' (log_prob=-0.500), Expected: 'green, yellow, orange, black'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 1,
-      "prompt": "Your goal is to find the secret three-color code. The following colors are possible: orange, green, ...",
-      "positive_response": "orange, purple, brown",
-      "negative_response": "orange, orange, purple",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'orange, purple, brown' (log_prob=-0.500), Expected: 'orange, purple, brown'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'orange, purple, brown' (log_prob=-0.500), Expected: 'orange, orange, purple'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_mastermind_pairs.json

@@ -1,14 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Your goal is to find the secret four-color code. The following colors are possible: green, black, orange, blue, purple, yellow.\nSome guesses have already been made. I will provide feedback for each guess made with which it is possible to unambigiously determine the secret code.\n\nPrevious Guesses:\nGuess: ['black', 'green', 'purple', 'yellow']. Hint: two colors are in the secret code but in the wrong positions.\nGuess: ['green', 'orange', 'orange', 'purple']. Hint: two colors are in the correct positions.\nGuess: ['green', 'orange', 'blue', 'black']. Hint: two colors are in the correct positions and two colors are in the secret code but in the wrong positions.\nGuess: ['green', 'orange', 'black', 'blue']. Hint: one color is in the correct position and three colors are in the secret code but in the wrong position.\n\nThe secret code is:",
-    "positive_response": "green, blue, orange, black",
-    "negative_response": "green, yellow, orange, black"
-  },
-  {
-    "pair_id": 1,
-    "prompt": "Your goal is to find the secret three-color code. The following colors are possible: orange, green, brown, pink, purple.\nSome guesses have already been made. I will provide feedback for each guess made with which it is possible to unambigiously determine the secret code.\n\nPrevious Guesses:\nGuess: ['orange', 'pink', 'green']. Hint: one color is in the correct position.\nGuess: ['green', 'orange', 'brown']. Hint: one color is in the correct position and one color is in the secret code but in the wrong position.\nGuess: ['orange', 'brown', 'brown']. Hint: two colors are in the correct positions.\n\nThe secret code is:",
-    "positive_response": "orange, purple, brown",
-    "negative_response": "orange, orange, purple"
-  }
-]

wisent/examples/scripts/results/test_math500_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "math500",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Question: Convert the point $(0,3)$ in rectangular coordinates to polar coordinates.  Enter your ans...",
-      "positive_response": "\\left( 3, \\frac{\\pi}{2} \\right)",
-      "negative_response": "\\left( 3, \\frac{\\pi}{2} \\right) + 1",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\\left( 3, \\frac{\\pi}{2} \\right)' (log_prob=-0.500), Expected: '\\left( 3, \\frac{\\pi}{2} \\right)'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\\left( 3, \\frac{\\pi}{2} \\right)' (log_prob=-0.500), Expected: '\\left( 3, \\frac{\\pi}{2} \\right) + 1'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_math500_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Question: Convert the point $(0,3)$ in rectangular coordinates to polar coordinates.  Enter your answer in the form $(r,\\theta),$ where $r > 0$ and $0 \\le \\theta < 2 \\pi.$\n\nWhat is the answer?",
-    "positive_response": "\\left( 3, \\frac{\\pi}{2} \\right)",
-    "negative_response": "\\left( 3, \\frac{\\pi}{2} \\right) + 1"
-  }
-]