wisent 0.7.379__py3-none-any.whl → 0.7.901__py3-none-any.whl

wisent/examples/scripts/results/test_iwslt2017_group/test_iwslt2017_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "iwslt2017",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Translate the following from ar to en:\n\u0627\u0644\u0633\u0646\u0629 \u0627\u0644\u0645\u0627\u0636\u064a\u0629 \u0639\u0631\u0636\u062a \u0647\u0627\u062a\u064a\u0646 \u0627\u0644\u0634\u0631\u064a\u062d\u062a\u064a\u0646 \u0644\u0643\u064a \u0623\u0648\u0636\u062d \u0623\u0646 \u0627\u0644\u063a\u0637\u0627\u0621 \u0627\u0644\u062c\u0644\u064a\u062f\u064a...",
-      "positive_response": "Last year I showed these two slides so that  demonstrate that the arctic ice cap,  which for most of the last three million years  has been the size of the lower 48 states,  has shrunk by 40 percent.",
-      "negative_response": "year lower the two I been shrunk most states, that of slides has three percent. for showed these ice by million last the size the 48 has of 40 so cap, years Last arctic the which that demonstrate",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'Last year I showed these two slides so that  demonstrate that the arctic ice cap,  which for most of the last three million years  has been the size of the lower 48 states,  has shrunk by 40 percent.' (log_prob=-0.500), Expected: 'Last year I showed these two slides so that  demonstrate that the arctic ice cap,  which for most of the last three million years  has been the size of the lower 48 states,  has shrunk by 40 percent.'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'Last year I showed these two slides so that  demonstrate that the arctic ice cap,  which for most of the last three million years  has been the size of the lower 48 states,  has shrunk by 40 percent.' (log_prob=-0.500), Expected: 'year lower the two I been shrunk most states, that of slides has three percent. for showed these ice by million last the size the 48 has of 40 so cap, years Last arctic the which that demonstrate'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_iwslt2017_group/test_iwslt2017_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Translate the following from ar to en:\n\u0627\u0644\u0633\u0646\u0629 \u0627\u0644\u0645\u0627\u0636\u064a\u0629 \u0639\u0631\u0636\u062a \u0647\u0627\u062a\u064a\u0646 \u0627\u0644\u0634\u0631\u064a\u062d\u062a\u064a\u0646 \u0644\u0643\u064a \u0623\u0648\u0636\u062d \u0623\u0646 \u0627\u0644\u063a\u0637\u0627\u0621 \u0627\u0644\u062c\u0644\u064a\u062f\u064a \u0627\u0644\u0642\u0637\u0628\u064a\u060c \u0627\u0644\u0630\u064a \u0643\u0627\u0646 \u062e\u0644\u0627\u0644 \u0627\u0644\u062b\u0644\u0627\u062b\u0629 \u0645\u0644\u0627\u064a\u064a\u0646 \u0633\u0646\u0629 \u0627\u0644\u0645\u0627\u0636\u064a\u0629 \u0641\u064a \u062d\u062c\u0645 \u0623\u0642\u0644\u0647 \u062b\u0645\u0627\u0646\u064a\u0629 \u0648\u0623\u0631\u0628\u0639\u064a\u0646 \u060c \u0642\u062f \u062a\u0642\u0644\u0635 \u0628\u0646\u0633\u0628\u0629 \u0623\u0631\u0628\u0639\u064a\u0646 \u0641\u064a \u0627\u0644\u0645\u0627\u0626\u0629.",
-    "positive_response": "Last year I showed these two slides so that  demonstrate that the arctic ice cap,  which for most of the last three million years  has been the size of the lower 48 states,  has shrunk by 40 percent.",
-    "negative_response": "year lower the two I been shrunk most states, that of slides has three percent. for showed these ice by million last the size the 48 has of 40 so cap, years Last arctic the which that demonstrate"
-  }
-]

wisent/examples/scripts/results/test_japanese_leaderboard_evaluation.json

@@ -1,51 +0,0 @@
-{
-  "task_name": "japanese_leaderboard",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 2,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Question: \u751f\u7269\u306e\u4e2d\u3067\u3001\u552f\u4e00\u4eba\u9593\u304c\u4f7f\u3048\u308b\u3001\u76f8\u624b\u306b\u4f1d\u3048\u308b\u624b\u6bb5\u3068\u306f?\nA. \u3054\u98ef\nB. \u5a18...",
-      "positive_response": "\u5a18",
-      "negative_response": "\u3054\u98ef",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\u5a18' (log_prob=-0.500), Expected: '\u5a18'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\u5a18' (log_prob=-0.500), Expected: '\u3054\u98ef'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 1,
-      "prompt": "Question: \u30c8\u30a5\u30ea\u30fc\u306e\u3053\u3068\u3092\u306a\u3093\u3068\u8a00\u3046\uff1f\nA. \u30ac\u30fc\u30c7\u30f3\nB. \u30ad\u30e5\u30a6\u30ea...",
-      "positive_response": "\u30ad\u30e5\u30a6\u30ea",
-      "negative_response": "\u30ac\u30fc\u30c7\u30f3",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\u30ad\u30e5\u30a6\u30ea' (log_prob=-0.500), Expected: '\u30ad\u30e5\u30a6\u30ea'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\u30ad\u30e5\u30a6\u30ea' (log_prob=-0.500), Expected: '\u30ac\u30fc\u30c7\u30f3'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_japanese_leaderboard_pairs.json

@@ -1,14 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Question: \u751f\u7269\u306e\u4e2d\u3067\u3001\u552f\u4e00\u4eba\u9593\u304c\u4f7f\u3048\u308b\u3001\u76f8\u624b\u306b\u4f1d\u3048\u308b\u624b\u6bb5\u3068\u306f?\nA. \u3054\u98ef\nB. \u5a18",
-    "positive_response": "\u5a18",
-    "negative_response": "\u3054\u98ef"
-  },
-  {
-    "pair_id": 1,
-    "prompt": "Question: \u30c8\u30a5\u30ea\u30fc\u306e\u3053\u3068\u3092\u306a\u3093\u3068\u8a00\u3046\uff1f\nA. \u30ac\u30fc\u30c7\u30f3\nB. \u30ad\u30e5\u30a6\u30ea",
-    "positive_response": "\u30ad\u30e5\u30a6\u30ea",
-    "negative_response": "\u30ac\u30fc\u30c7\u30f3"
-  }
-]

wisent/examples/scripts/results/test_jsonschema_bench/test_jsonschema_bench_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "jsonschema_bench",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "JSON schema: {\n    \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n    \"additionalProperties\":...",
-      "positive_response": "{}",
-      "negative_response": "{invalid",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '{}' (log_prob=-0.500), Expected: '{}'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '{}' (log_prob=-0.500), Expected: '{invalid'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_jsonschema_bench/test_jsonschema_bench_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "JSON schema: {\n    \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n    \"additionalProperties\": false,\n    \"definitions\": {\n        \"absolute_path\": {\n            \"description\": \"A path only. Query string and/or fragment are not allowed.\",\n            \"pattern\": \"^/(([a-zA-Z0-9._~!$&'()*+,;=:@-]|%[0-9a-fA-F]{2})+(/([a-zA-Z0-9._~!$&'()*+,;=:@-]|%[0-9a-fA-F]{2})*)*)?$\",\n            \"type\": \"string\"\n        },\n        \"access_limited\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"auth_bypass_ids\": {\n                    \"$ref\": \"#/definitions/guid_list\",\n                    \"description\": \"Deprecated: auth_bypass_ids should be sent as a separate field\"\n                },\n                \"organisations\": {\n                    \"$ref\": \"#/definitions/guid_list\",\n                    \"description\": \"A list of organisation content ids permitted access to this item\"\n                },\n                \"users\": {\n                    \"items\": {\n                        \"type\": \"string\"\n                    },\n                    \"type\": \"array\"\n                }\n            },\n            \"type\": \"object\"\n        },\n        \"analytics_identifier\": {\n            \"anyOf\": [\n                {\n                    \"type\": \"string\"\n                },\n                {\n                    \"type\": \"null\"\n                }\n            ],\n            \"description\": \"A short identifier we send to Google Analytics for multi-valued fields. This means we avoid the truncated values we would get if we sent the path or slug of eg organisations.\"\n        },\n        \"body_html_and_govspeak\": {\n            \"anyOf\": [\n                {\n                    \"$ref\": \"#/definitions/multiple_content_types\"\n                }\n            ],\n            \"description\": \"The main content provided as HTML with the govspeak markdown it's rendered from\"\n        },\n        \"country\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"name\": {\n                    \"type\": \"string\"\n                },\n                \"slug\": {\n                    \"type\": \"string\"\n                },\n                \"synonyms\": {\n                    \"items\": {\n                        \"type\": \"string\"\n                    },\n                    \"type\": \"array\"\n                }\n            },\n            \"required\": [\n                \"slug\",\n                \"name\"\n            ],\n            \"type\": \"object\"\n        },\n        \"description\": {\n            \"type\": \"string\"\n        },\n        \"description_optional\": {\n            \"anyOf\": [\n                {\n                    \"$ref\": \"#/definitions/description\"\n                },\n                {\n                    \"type\": \"null\"\n                }\n            ]\n        },\n        \"details\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"alert_status\": {\n                    \"items\": {\n                        \"type\": \"string\"\n                    },\n                    \"type\": \"array\"\n                },\n                \"change_description\": {\n                    \"type\": \"string\"\n                },\n                \"country\": {\n                    \"$ref\": \"#/definitions/country\"\n                },\n                \"document\": {\n                    \"$ref\": \"#/definitions/file_attachment_asset\"\n                },\n                \"email_signup_link\": {\n                    \"$ref\": \"#/definitions/email_signup_link\"\n                },\n                \"image\": {\n                    \"$ref\": \"#/definitions/image_asset\"\n                },\n                \"max_cache_time\": {\n                    \"$ref\": \"#/definitions/max_cache_time\"\n                },\n                \"parts\": {\n                    \"$ref\": \"#/definitions/parts\"\n                },\n                \"publishing_request_id\": {\n                    \"$ref\": \"#/definitions/publishing_request_id\"\n                },\n                \"reviewed_at\": {\n                    \"_format\": \"date-time\",\n                    \"type\": \"string\"\n                },\n                \"summary\": {\n                    \"$ref\": \"#/definitions/multiple_content_types\"\n                },\n                \"updated_at\": {\n                    \"_format\": \"date-time\",\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n                \"summary\",\n                \"country\",\n                \"updated_at\",\n                \"reviewed_at\",\n                \"change_description\",\n                \"alert_status\",\n                \"email_signup_link\",\n                \"parts\"\n            ],\n            \"type\": \"object\"\n        },\n        \"email_signup_link\": {\n            \"_format\": \"uri\",\n            \"description\": \"Path to email signup form. TODO: Check if can be switched to use links instead\",\n            \"type\": \"string\"\n        },\n        \"file_attachment_asset\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"accessible\": {\n                    \"type\": \"boolean\"\n                },\n                \"alternative_format_contact_email\": {\n                    \"type\": \"string\"\n                },\n                \"attachment_type\": {\n                    \"enum\": [\n                        \"file\"\n                    ],\n                    \"type\": \"string\"\n                },\n                \"content_type\": {\n                    \"type\": \"string\"\n                },\n                \"file_size\": {\n                    \"type\": \"integer\"\n                },\n                \"filename\": {\n                    \"type\": \"string\"\n                },\n                \"id\": {\n                    \"type\": \"string\"\n                },\n                \"locale\": {\n                    \"$ref\": \"#/definitions/locale\"\n                },\n                \"number_of_pages\": {\n                    \"type\": \"integer\"\n                },\n                \"preview_url\": {\n                    \"_format\": \"uri\",\n                    \"type\": \"string\"\n                },\n                \"title\": {\n                    \"type\": \"string\"\n                },\n                \"url\": {\n                    \"_format\": \"uri\",\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n                \"attachment_type\",\n                \"content_type\",\n                \"id\",\n                \"url\"\n            ],\n            \"type\": \"object\"\n        },\n        \"first_published_at\": {\n            \"_format\": \"date-time\",\n            \"description\": \"The date the content was first published.  Automatically determined by the publishing-api, unless overridden by the publishing application.\",\n            \"type\": \"string\"\n        },\n        \"guid\": {\n            \"pattern\": \"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$\",\n            \"type\": \"string\"\n        },\n        \"guid_list\": {\n            \"items\": {\n                \"$ref\": \"#/definitions/guid\"\n            },\n            \"type\": \"array\",\n            \"_uniqueItems\": true\n        },\n        \"image_asset\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"alt_text\": {\n                    \"type\": \"string\"\n                },\n                \"caption\": {\n                    \"type\": \"string\"\n                },\n                \"content_type\": {\n                    \"type\": \"string\"\n                },\n                \"credit\": {\n                    \"type\": \"string\"\n                },\n                \"url\": {\n                    \"_format\": \"uri\",\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n                \"content_type\",\n                \"url\"\n            ],\n            \"type\": \"object\"\n        },\n        \"locale\": {\n            \"enum\": [\n                \"ar\",\n                \"az\",\n                \"be\",\n                \"bg\",\n                \"bn\",\n                \"cs\",\n                \"cy\",\n                \"da\",\n                \"de\",\n                \"dr\",\n                \"el\",\n                \"en\",\n                \"es\",\n                \"es-419\",\n                \"et\",\n                \"fa\",\n                \"fi\",\n                \"fr\",\n                \"gd\",\n                \"he\",\n                \"hi\",\n                \"hr\",\n                \"hu\",\n                \"hy\",\n                \"id\",\n                \"is\",\n                \"it\",\n                \"ja\",\n                \"ka\",\n                \"kk\",\n                \"ko\",\n                \"lt\",\n                \"lv\",\n                \"ms\",\n                \"mt\",\n                \"nl\",\n                \"no\",\n                \"pl\",\n                \"ps\",\n                \"pt\",\n                \"ro\",\n                \"ru\",\n                \"si\",\n                \"sk\",\n                \"sl\",\n                \"so\",\n                \"sq\",\n                \"sr\",\n                \"sv\",\n                \"sw\",\n                \"ta\",\n                \"th\",\n                \"tk\",\n                \"tr\",\n                \"uk\",\n                \"ur\",\n                \"uz\",\n                \"vi\",\n                \"zh\",\n                \"zh-hk\",\n                \"zh-tw\"\n            ],\n            \"type\": \"string\"\n        },\n        \"max_cache_time\": {\n            \"description\": \"The maximum length of time the content should be cached, in seconds\",\n            \"type\": \"integer\"\n        },\n        \"multiple_content_types\": {\n            \"items\": {\n                \"additionalProperties\": false,\n                \"properties\": {\n                    \"content\": {\n                        \"type\": \"string\"\n                    },\n                    \"content_type\": {\n                        \"type\": \"string\"\n                    }\n                },\n                \"required\": [\n                    \"content_type\",\n                    \"content\"\n                ],\n                \"type\": \"object\"\n            },\n            \"type\": \"array\"\n        },\n        \"parts\": {\n            \"items\": {\n                \"additionalProperties\": false,\n                \"properties\": {\n                    \"body\": {\n                        \"$ref\": \"#/definitions/body_html_and_govspeak\"\n                    },\n                    \"slug\": {\n                        \"_format\": \"uri\",\n                        \"type\": \"string\"\n                    },\n                    \"title\": {\n                        \"type\": \"string\"\n                    }\n                },\n                \"required\": [\n                    \"title\",\n                    \"slug\",\n                    \"body\"\n                ],\n                \"type\": \"object\"\n            },\n            \"type\": \"array\"\n        },\n        \"public_updated_at\": {\n            \"_format\": \"date-time\",\n            \"description\": \"When the content was last significantly changed (a major update). Shown to users.  Automatically determined by the publishing-api, unless overridden by the publishing application.\",\n            \"type\": \"string\"\n        },\n        \"publishing_app_name\": {\n            \"description\": \"The application that published this item.\",\n            \"enum\": [\n                \"calculators\",\n                \"calendars\",\n                \"collections-publisher\",\n                \"contacts\",\n                \"content-publisher\",\n                \"content-tagger\",\n                \"email-alert-frontend\",\n                \"external-link-tracker\",\n                \"feedback\",\n                \"frontend\",\n                \"government-frontend\",\n                \"hmrc-manuals-api\",\n                \"info-frontend\",\n                \"licencefinder\",\n                \"local-links-manager\",\n                \"manuals-frontend\",\n                \"manuals-publisher\",\n                \"maslow\",\n                \"performanceplatform-big-screen-view\",\n                \"publisher\",\n                \"rummager\",\n                \"search-admin\",\n                \"search-api\",\n                \"service-manual-publisher\",\n                \"share-sale-publisher\",\n                \"short-url-manager\",\n                \"smartanswers\",\n                \"special-route-publisher\",\n                \"specialist-publisher\",\n                \"static\",\n                \"tariff\",\n                \"travel-advice-publisher\",\n                \"whitehall\"\n            ],\n            \"type\": \"string\"\n        },\n        \"publishing_request_id\": {\n            \"description\": \"A unique identifier used to track publishing requests to rendered content\",\n            \"oneOf\": [\n                {\n                    \"type\": \"string\"\n                },\n                {\n                    \"type\": \"null\"\n                }\n            ]\n        },\n        \"rendering_app\": {\n            \"description\": \"The application that renders this item.\",\n            \"enum\": [\n                \"calculators\",\n                \"calendars\",\n                \"collections\",\n                \"content-store\",\n                \"email-alert-frontend\",\n                \"email-campaign-frontend\",\n                \"feedback\",\n                \"finder-frontend\",\n                \"frontend\",\n                \"government-frontend\",\n                \"info-frontend\",\n                \"licencefinder\",\n                \"manuals-frontend\",\n                \"performanceplatform-big-screen-view\",\n                \"rummager\",\n                \"search-api\",\n                \"service-manual-frontend\",\n                \"smartanswers\",\n                \"spotlight\",\n                \"static\",\n                \"tariff\",\n                \"whitehall-admin\",\n                \"whitehall-frontend\"\n            ],\n            \"type\": \"string\"\n        },\n        \"route\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"path\": {\n                    \"type\": \"string\"\n                },\n                \"type\": {\n                    \"enum\": [\n                        \"prefix\",\n                        \"exact\"\n                    ]\n                }\n            },\n            \"required\": [\n                \"path\",\n                \"type\"\n            ],\n            \"type\": \"object\"\n        },\n        \"routes\": {\n            \"items\": {\n                \"$ref\": \"#/definitions/route\"\n            },\n            \"minItems\": 1,\n            \"type\": \"array\"\n        },\n        \"title\": {\n            \"type\": \"string\"\n        },\n        \"update_type\": {\n            \"enum\": [\n                \"major\",\n                \"minor\",\n                \"republish\"\n            ]\n        }\n    },\n    \"properties\": {\n        \"access_limited\": {\n            \"$ref\": \"#/definitions/access_limited\"\n        },\n        \"analytics_identifier\": {\n            \"$ref\": \"#/definitions/analytics_identifier\"\n        },\n        \"auth_bypass_ids\": {\n            \"$ref\": \"#/definitions/guid_list\",\n            \"description\": \"A list of ids that will allow access to this item for non-authenticated users\"\n        },\n        \"base_path\": {\n            \"$ref\": \"#/definitions/absolute_path\"\n        },\n        \"bulk_publishing\": {\n            \"type\": \"boolean\"\n        },\n        \"change_note\": {\n            \"type\": [\n                \"null\",\n                \"string\"\n            ]\n        },\n        \"description\": {\n            \"$ref\": \"#/definitions/description_optional\"\n        },\n        \"details\": {\n            \"$ref\": \"#/definitions/details\"\n        },\n        \"document_type\": {\n            \"enum\": [\n                \"travel_advice\"\n            ],\n            \"type\": \"string\"\n        },\n        \"first_published_at\": {\n            \"$ref\": \"#/definitions/first_published_at\"\n        },\n        \"last_edited_at\": {\n            \"_format\": \"date-time\",\n            \"description\": \"Last time when the content received a major or minor update.\",\n            \"type\": \"string\"\n        },\n        \"links\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"policy_areas\": {\n                    \"$ref\": \"#/definitions/guid_list\",\n                    \"description\": \"A largely deprecated tag currently only used to power email alerts.\"\n                }\n            },\n            \"type\": \"object\"\n        },\n        \"locale\": {\n            \"$ref\": \"#/definitions/locale\"\n        },\n        \"need_ids\": {\n            \"items\": {\n                \"type\": \"string\"\n            },\n            \"type\": \"array\"\n        },\n        \"phase\": {\n            \"description\": \"The service design phase of this content item - https://www.gov.uk/service-manual/phases\",\n            \"enum\": [\n                \"alpha\",\n                \"beta\",\n                \"live\"\n            ],\n            \"type\": \"string\"\n        },\n        \"previous_version\": {\n            \"type\": \"string\"\n        },\n        \"public_updated_at\": {\n            \"$ref\": \"#/definitions/public_updated_at\"\n        },\n        \"publishing_app\": {\n            \"$ref\": \"#/definitions/publishing_app_name\"\n        },\n        \"redirects\": {\n            \"additionalItems\": false,\n            \"items\": {},\n            \"type\": \"array\"\n        },\n        \"rendering_app\": {\n            \"$ref\": \"#/definitions/rendering_app\"\n        },\n        \"routes\": {\n            \"$ref\": \"#/definitions/routes\"\n        },\n        \"schema_name\": {\n            \"enum\": [\n                \"travel_advice\"\n            ],\n            \"type\": \"string\"\n        },\n        \"title\": {\n            \"$ref\": \"#/definitions/title\"\n        },\n        \"update_type\": {\n            \"$ref\": \"#/definitions/update_type\"\n        }\n    },\n    \"required\": [\n        \"base_path\",\n        \"details\",\n        \"document_type\",\n        \"publishing_app\",\n        \"rendering_app\",\n        \"routes\",\n        \"schema_name\",\n        \"title\"\n    ],\n    \"type\": \"object\"\n}\\n\\nJSON object: ",
-    "positive_response": "{}",
-    "negative_response": "{invalid"
-  }
-]

wisent/examples/scripts/results/test_jsonschema_bench_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "jsonschema_bench",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "JSON schema: {\n    \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n    \"additionalProperties\":...",
-      "positive_response": "{}",
-      "negative_response": "{invalid",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '{}' (log_prob=-0.500), Expected: '{}'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '{}' (log_prob=-0.500), Expected: '{invalid'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_jsonschema_bench_final/test_jsonschema_bench_evaluation.json

@@ -1,30 +0,0 @@
-{
-  "task_name": "jsonschema_bench",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 1,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "JSON schema: {\n    \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n    \"additionalProperties\":...",
-      "positive_response": "{}",
-      "negative_response": "{invalid",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '{}' (log_prob=-0.500), Expected: '{}'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '{}' (log_prob=-0.500), Expected: '{invalid'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_jsonschema_bench_final/test_jsonschema_bench_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "JSON schema: {\n    \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n    \"additionalProperties\": false,\n    \"definitions\": {\n        \"absolute_path\": {\n            \"description\": \"A path only. Query string and/or fragment are not allowed.\",\n            \"pattern\": \"^/(([a-zA-Z0-9._~!$&'()*+,;=:@-]|%[0-9a-fA-F]{2})+(/([a-zA-Z0-9._~!$&'()*+,;=:@-]|%[0-9a-fA-F]{2})*)*)?$\",\n            \"type\": \"string\"\n        },\n        \"access_limited\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"auth_bypass_ids\": {\n                    \"$ref\": \"#/definitions/guid_list\",\n                    \"description\": \"Deprecated: auth_bypass_ids should be sent as a separate field\"\n                },\n                \"organisations\": {\n                    \"$ref\": \"#/definitions/guid_list\",\n                    \"description\": \"A list of organisation content ids permitted access to this item\"\n                },\n                \"users\": {\n                    \"items\": {\n                        \"type\": \"string\"\n                    },\n                    \"type\": \"array\"\n                }\n            },\n            \"type\": \"object\"\n        },\n        \"analytics_identifier\": {\n            \"anyOf\": [\n                {\n                    \"type\": \"string\"\n                },\n                {\n                    \"type\": \"null\"\n                }\n            ],\n            \"description\": \"A short identifier we send to Google Analytics for multi-valued fields. This means we avoid the truncated values we would get if we sent the path or slug of eg organisations.\"\n        },\n        \"body_html_and_govspeak\": {\n            \"anyOf\": [\n                {\n                    \"$ref\": \"#/definitions/multiple_content_types\"\n                }\n            ],\n            \"description\": \"The main content provided as HTML with the govspeak markdown it's rendered from\"\n        },\n        \"country\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"name\": {\n                    \"type\": \"string\"\n                },\n                \"slug\": {\n                    \"type\": \"string\"\n                },\n                \"synonyms\": {\n                    \"items\": {\n                        \"type\": \"string\"\n                    },\n                    \"type\": \"array\"\n                }\n            },\n            \"required\": [\n                \"slug\",\n                \"name\"\n            ],\n            \"type\": \"object\"\n        },\n        \"description\": {\n            \"type\": \"string\"\n        },\n        \"description_optional\": {\n            \"anyOf\": [\n                {\n                    \"$ref\": \"#/definitions/description\"\n                },\n                {\n                    \"type\": \"null\"\n                }\n            ]\n        },\n        \"details\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"alert_status\": {\n                    \"items\": {\n                        \"type\": \"string\"\n                    },\n                    \"type\": \"array\"\n                },\n                \"change_description\": {\n                    \"type\": \"string\"\n                },\n                \"country\": {\n                    \"$ref\": \"#/definitions/country\"\n                },\n                \"document\": {\n                    \"$ref\": \"#/definitions/file_attachment_asset\"\n                },\n                \"email_signup_link\": {\n                    \"$ref\": \"#/definitions/email_signup_link\"\n                },\n                \"image\": {\n                    \"$ref\": \"#/definitions/image_asset\"\n                },\n                \"max_cache_time\": {\n                    \"$ref\": \"#/definitions/max_cache_time\"\n                },\n                \"parts\": {\n                    \"$ref\": \"#/definitions/parts\"\n                },\n                \"publishing_request_id\": {\n                    \"$ref\": \"#/definitions/publishing_request_id\"\n                },\n                \"reviewed_at\": {\n                    \"_format\": \"date-time\",\n                    \"type\": \"string\"\n                },\n                \"summary\": {\n                    \"$ref\": \"#/definitions/multiple_content_types\"\n                },\n                \"updated_at\": {\n                    \"_format\": \"date-time\",\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n                \"summary\",\n                \"country\",\n                \"updated_at\",\n                \"reviewed_at\",\n                \"change_description\",\n                \"alert_status\",\n                \"email_signup_link\",\n                \"parts\"\n            ],\n            \"type\": \"object\"\n        },\n        \"email_signup_link\": {\n            \"_format\": \"uri\",\n            \"description\": \"Path to email signup form. TODO: Check if can be switched to use links instead\",\n            \"type\": \"string\"\n        },\n        \"file_attachment_asset\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"accessible\": {\n                    \"type\": \"boolean\"\n                },\n                \"alternative_format_contact_email\": {\n                    \"type\": \"string\"\n                },\n                \"attachment_type\": {\n                    \"enum\": [\n                        \"file\"\n                    ],\n                    \"type\": \"string\"\n                },\n                \"content_type\": {\n                    \"type\": \"string\"\n                },\n                \"file_size\": {\n                    \"type\": \"integer\"\n                },\n                \"filename\": {\n                    \"type\": \"string\"\n                },\n                \"id\": {\n                    \"type\": \"string\"\n                },\n                \"locale\": {\n                    \"$ref\": \"#/definitions/locale\"\n                },\n                \"number_of_pages\": {\n                    \"type\": \"integer\"\n                },\n                \"preview_url\": {\n                    \"_format\": \"uri\",\n                    \"type\": \"string\"\n                },\n                \"title\": {\n                    \"type\": \"string\"\n                },\n                \"url\": {\n                    \"_format\": \"uri\",\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n                \"attachment_type\",\n                \"content_type\",\n                \"id\",\n                \"url\"\n            ],\n            \"type\": \"object\"\n        },\n        \"first_published_at\": {\n            \"_format\": \"date-time\",\n            \"description\": \"The date the content was first published.  Automatically determined by the publishing-api, unless overridden by the publishing application.\",\n            \"type\": \"string\"\n        },\n        \"guid\": {\n            \"pattern\": \"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$\",\n            \"type\": \"string\"\n        },\n        \"guid_list\": {\n            \"items\": {\n                \"$ref\": \"#/definitions/guid\"\n            },\n            \"type\": \"array\",\n            \"_uniqueItems\": true\n        },\n        \"image_asset\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"alt_text\": {\n                    \"type\": \"string\"\n                },\n                \"caption\": {\n                    \"type\": \"string\"\n                },\n                \"content_type\": {\n                    \"type\": \"string\"\n                },\n                \"credit\": {\n                    \"type\": \"string\"\n                },\n                \"url\": {\n                    \"_format\": \"uri\",\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n                \"content_type\",\n                \"url\"\n            ],\n            \"type\": \"object\"\n        },\n        \"locale\": {\n            \"enum\": [\n                \"ar\",\n                \"az\",\n                \"be\",\n                \"bg\",\n                \"bn\",\n                \"cs\",\n                \"cy\",\n                \"da\",\n                \"de\",\n                \"dr\",\n                \"el\",\n                \"en\",\n                \"es\",\n                \"es-419\",\n                \"et\",\n                \"fa\",\n                \"fi\",\n                \"fr\",\n                \"gd\",\n                \"he\",\n                \"hi\",\n                \"hr\",\n                \"hu\",\n                \"hy\",\n                \"id\",\n                \"is\",\n                \"it\",\n                \"ja\",\n                \"ka\",\n                \"kk\",\n                \"ko\",\n                \"lt\",\n                \"lv\",\n                \"ms\",\n                \"mt\",\n                \"nl\",\n                \"no\",\n                \"pl\",\n                \"ps\",\n                \"pt\",\n                \"ro\",\n                \"ru\",\n                \"si\",\n                \"sk\",\n                \"sl\",\n                \"so\",\n                \"sq\",\n                \"sr\",\n                \"sv\",\n                \"sw\",\n                \"ta\",\n                \"th\",\n                \"tk\",\n                \"tr\",\n                \"uk\",\n                \"ur\",\n                \"uz\",\n                \"vi\",\n                \"zh\",\n                \"zh-hk\",\n                \"zh-tw\"\n            ],\n            \"type\": \"string\"\n        },\n        \"max_cache_time\": {\n            \"description\": \"The maximum length of time the content should be cached, in seconds\",\n            \"type\": \"integer\"\n        },\n        \"multiple_content_types\": {\n            \"items\": {\n                \"additionalProperties\": false,\n                \"properties\": {\n                    \"content\": {\n                        \"type\": \"string\"\n                    },\n                    \"content_type\": {\n                        \"type\": \"string\"\n                    }\n                },\n                \"required\": [\n                    \"content_type\",\n                    \"content\"\n                ],\n                \"type\": \"object\"\n            },\n            \"type\": \"array\"\n        },\n        \"parts\": {\n            \"items\": {\n                \"additionalProperties\": false,\n                \"properties\": {\n                    \"body\": {\n                        \"$ref\": \"#/definitions/body_html_and_govspeak\"\n                    },\n                    \"slug\": {\n                        \"_format\": \"uri\",\n                        \"type\": \"string\"\n                    },\n                    \"title\": {\n                        \"type\": \"string\"\n                    }\n                },\n                \"required\": [\n                    \"title\",\n                    \"slug\",\n                    \"body\"\n                ],\n                \"type\": \"object\"\n            },\n            \"type\": \"array\"\n        },\n        \"public_updated_at\": {\n            \"_format\": \"date-time\",\n            \"description\": \"When the content was last significantly changed (a major update). Shown to users.  Automatically determined by the publishing-api, unless overridden by the publishing application.\",\n            \"type\": \"string\"\n        },\n        \"publishing_app_name\": {\n            \"description\": \"The application that published this item.\",\n            \"enum\": [\n                \"calculators\",\n                \"calendars\",\n                \"collections-publisher\",\n                \"contacts\",\n                \"content-publisher\",\n                \"content-tagger\",\n                \"email-alert-frontend\",\n                \"external-link-tracker\",\n                \"feedback\",\n                \"frontend\",\n                \"government-frontend\",\n                \"hmrc-manuals-api\",\n                \"info-frontend\",\n                \"licencefinder\",\n                \"local-links-manager\",\n                \"manuals-frontend\",\n                \"manuals-publisher\",\n                \"maslow\",\n                \"performanceplatform-big-screen-view\",\n                \"publisher\",\n                \"rummager\",\n                \"search-admin\",\n                \"search-api\",\n                \"service-manual-publisher\",\n                \"share-sale-publisher\",\n                \"short-url-manager\",\n                \"smartanswers\",\n                \"special-route-publisher\",\n                \"specialist-publisher\",\n                \"static\",\n                \"tariff\",\n                \"travel-advice-publisher\",\n                \"whitehall\"\n            ],\n            \"type\": \"string\"\n        },\n        \"publishing_request_id\": {\n            \"description\": \"A unique identifier used to track publishing requests to rendered content\",\n            \"oneOf\": [\n                {\n                    \"type\": \"string\"\n                },\n                {\n                    \"type\": \"null\"\n                }\n            ]\n        },\n        \"rendering_app\": {\n            \"description\": \"The application that renders this item.\",\n            \"enum\": [\n                \"calculators\",\n                \"calendars\",\n                \"collections\",\n                \"content-store\",\n                \"email-alert-frontend\",\n                \"email-campaign-frontend\",\n                \"feedback\",\n                \"finder-frontend\",\n                \"frontend\",\n                \"government-frontend\",\n                \"info-frontend\",\n                \"licencefinder\",\n                \"manuals-frontend\",\n                \"performanceplatform-big-screen-view\",\n                \"rummager\",\n                \"search-api\",\n                \"service-manual-frontend\",\n                \"smartanswers\",\n                \"spotlight\",\n                \"static\",\n                \"tariff\",\n                \"whitehall-admin\",\n                \"whitehall-frontend\"\n            ],\n            \"type\": \"string\"\n        },\n        \"route\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"path\": {\n                    \"type\": \"string\"\n                },\n                \"type\": {\n                    \"enum\": [\n                        \"prefix\",\n                        \"exact\"\n                    ]\n                }\n            },\n            \"required\": [\n                \"path\",\n                \"type\"\n            ],\n            \"type\": \"object\"\n        },\n        \"routes\": {\n            \"items\": {\n                \"$ref\": \"#/definitions/route\"\n            },\n            \"minItems\": 1,\n            \"type\": \"array\"\n        },\n        \"title\": {\n            \"type\": \"string\"\n        },\n        \"update_type\": {\n            \"enum\": [\n                \"major\",\n                \"minor\",\n                \"republish\"\n            ]\n        }\n    },\n    \"properties\": {\n        \"access_limited\": {\n            \"$ref\": \"#/definitions/access_limited\"\n        },\n        \"analytics_identifier\": {\n            \"$ref\": \"#/definitions/analytics_identifier\"\n        },\n        \"auth_bypass_ids\": {\n            \"$ref\": \"#/definitions/guid_list\",\n            \"description\": \"A list of ids that will allow access to this item for non-authenticated users\"\n        },\n        \"base_path\": {\n            \"$ref\": \"#/definitions/absolute_path\"\n        },\n        \"bulk_publishing\": {\n            \"type\": \"boolean\"\n        },\n        \"change_note\": {\n            \"type\": [\n                \"null\",\n                \"string\"\n            ]\n        },\n        \"description\": {\n            \"$ref\": \"#/definitions/description_optional\"\n        },\n        \"details\": {\n            \"$ref\": \"#/definitions/details\"\n        },\n        \"document_type\": {\n            \"enum\": [\n                \"travel_advice\"\n            ],\n            \"type\": \"string\"\n        },\n        \"first_published_at\": {\n            \"$ref\": \"#/definitions/first_published_at\"\n        },\n        \"last_edited_at\": {\n            \"_format\": \"date-time\",\n            \"description\": \"Last time when the content received a major or minor update.\",\n            \"type\": \"string\"\n        },\n        \"links\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"policy_areas\": {\n                    \"$ref\": \"#/definitions/guid_list\",\n                    \"description\": \"A largely deprecated tag currently only used to power email alerts.\"\n                }\n            },\n            \"type\": \"object\"\n        },\n        \"locale\": {\n            \"$ref\": \"#/definitions/locale\"\n        },\n        \"need_ids\": {\n            \"items\": {\n                \"type\": \"string\"\n            },\n            \"type\": \"array\"\n        },\n        \"phase\": {\n            \"description\": \"The service design phase of this content item - https://www.gov.uk/service-manual/phases\",\n            \"enum\": [\n                \"alpha\",\n                \"beta\",\n                \"live\"\n            ],\n            \"type\": \"string\"\n        },\n        \"previous_version\": {\n            \"type\": \"string\"\n        },\n        \"public_updated_at\": {\n            \"$ref\": \"#/definitions/public_updated_at\"\n        },\n        \"publishing_app\": {\n            \"$ref\": \"#/definitions/publishing_app_name\"\n        },\n        \"redirects\": {\n            \"additionalItems\": false,\n            \"items\": {},\n            \"type\": \"array\"\n        },\n        \"rendering_app\": {\n            \"$ref\": \"#/definitions/rendering_app\"\n        },\n        \"routes\": {\n            \"$ref\": \"#/definitions/routes\"\n        },\n        \"schema_name\": {\n            \"enum\": [\n                \"travel_advice\"\n            ],\n            \"type\": \"string\"\n        },\n        \"title\": {\n            \"$ref\": \"#/definitions/title\"\n        },\n        \"update_type\": {\n            \"$ref\": \"#/definitions/update_type\"\n        }\n    },\n    \"required\": [\n        \"base_path\",\n        \"details\",\n        \"document_type\",\n        \"publishing_app\",\n        \"rendering_app\",\n        \"routes\",\n        \"schema_name\",\n        \"title\"\n    ],\n    \"type\": \"object\"\n}\\n\\nJSON object: ",
-    "positive_response": "{}",
-    "negative_response": "{invalid"
-  }
-]

wisent/examples/scripts/results/test_jsonschema_bench_pairs.json

@@ -1,8 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "JSON schema: {\n    \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n    \"additionalProperties\": false,\n    \"definitions\": {\n        \"absolute_path\": {\n            \"description\": \"A path only. Query string and/or fragment are not allowed.\",\n            \"pattern\": \"^/(([a-zA-Z0-9._~!$&'()*+,;=:@-]|%[0-9a-fA-F]{2})+(/([a-zA-Z0-9._~!$&'()*+,;=:@-]|%[0-9a-fA-F]{2})*)*)?$\",\n            \"type\": \"string\"\n        },\n        \"access_limited\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"auth_bypass_ids\": {\n                    \"$ref\": \"#/definitions/guid_list\",\n                    \"description\": \"Deprecated: auth_bypass_ids should be sent as a separate field\"\n                },\n                \"organisations\": {\n                    \"$ref\": \"#/definitions/guid_list\",\n                    \"description\": \"A list of organisation content ids permitted access to this item\"\n                },\n                \"users\": {\n                    \"items\": {\n                        \"type\": \"string\"\n                    },\n                    \"type\": \"array\"\n                }\n            },\n            \"type\": \"object\"\n        },\n        \"analytics_identifier\": {\n            \"anyOf\": [\n                {\n                    \"type\": \"string\"\n                },\n                {\n                    \"type\": \"null\"\n                }\n            ],\n            \"description\": \"A short identifier we send to Google Analytics for multi-valued fields. This means we avoid the truncated values we would get if we sent the path or slug of eg organisations.\"\n        },\n        \"body_html_and_govspeak\": {\n            \"anyOf\": [\n                {\n                    \"$ref\": \"#/definitions/multiple_content_types\"\n                }\n            ],\n            \"description\": \"The main content provided as HTML with the govspeak markdown it's rendered from\"\n        },\n        \"country\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"name\": {\n                    \"type\": \"string\"\n                },\n                \"slug\": {\n                    \"type\": \"string\"\n                },\n                \"synonyms\": {\n                    \"items\": {\n                        \"type\": \"string\"\n                    },\n                    \"type\": \"array\"\n                }\n            },\n            \"required\": [\n                \"slug\",\n                \"name\"\n            ],\n            \"type\": \"object\"\n        },\n        \"description\": {\n            \"type\": \"string\"\n        },\n        \"description_optional\": {\n            \"anyOf\": [\n                {\n                    \"$ref\": \"#/definitions/description\"\n                },\n                {\n                    \"type\": \"null\"\n                }\n            ]\n        },\n        \"details\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"alert_status\": {\n                    \"items\": {\n                        \"type\": \"string\"\n                    },\n                    \"type\": \"array\"\n                },\n                \"change_description\": {\n                    \"type\": \"string\"\n                },\n                \"country\": {\n                    \"$ref\": \"#/definitions/country\"\n                },\n                \"document\": {\n                    \"$ref\": \"#/definitions/file_attachment_asset\"\n                },\n                \"email_signup_link\": {\n                    \"$ref\": \"#/definitions/email_signup_link\"\n                },\n                \"image\": {\n                    \"$ref\": \"#/definitions/image_asset\"\n                },\n                \"max_cache_time\": {\n                    \"$ref\": \"#/definitions/max_cache_time\"\n                },\n                \"parts\": {\n                    \"$ref\": \"#/definitions/parts\"\n                },\n                \"publishing_request_id\": {\n                    \"$ref\": \"#/definitions/publishing_request_id\"\n                },\n                \"reviewed_at\": {\n                    \"_format\": \"date-time\",\n                    \"type\": \"string\"\n                },\n                \"summary\": {\n                    \"$ref\": \"#/definitions/multiple_content_types\"\n                },\n                \"updated_at\": {\n                    \"_format\": \"date-time\",\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n                \"summary\",\n                \"country\",\n                \"updated_at\",\n                \"reviewed_at\",\n                \"change_description\",\n                \"alert_status\",\n                \"email_signup_link\",\n                \"parts\"\n            ],\n            \"type\": \"object\"\n        },\n        \"email_signup_link\": {\n            \"_format\": \"uri\",\n            \"description\": \"Path to email signup form. TODO: Check if can be switched to use links instead\",\n            \"type\": \"string\"\n        },\n        \"file_attachment_asset\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"accessible\": {\n                    \"type\": \"boolean\"\n                },\n                \"alternative_format_contact_email\": {\n                    \"type\": \"string\"\n                },\n                \"attachment_type\": {\n                    \"enum\": [\n                        \"file\"\n                    ],\n                    \"type\": \"string\"\n                },\n                \"content_type\": {\n                    \"type\": \"string\"\n                },\n                \"file_size\": {\n                    \"type\": \"integer\"\n                },\n                \"filename\": {\n                    \"type\": \"string\"\n                },\n                \"id\": {\n                    \"type\": \"string\"\n                },\n                \"locale\": {\n                    \"$ref\": \"#/definitions/locale\"\n                },\n                \"number_of_pages\": {\n                    \"type\": \"integer\"\n                },\n                \"preview_url\": {\n                    \"_format\": \"uri\",\n                    \"type\": \"string\"\n                },\n                \"title\": {\n                    \"type\": \"string\"\n                },\n                \"url\": {\n                    \"_format\": \"uri\",\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n                \"attachment_type\",\n                \"content_type\",\n                \"id\",\n                \"url\"\n            ],\n            \"type\": \"object\"\n        },\n        \"first_published_at\": {\n            \"_format\": \"date-time\",\n            \"description\": \"The date the content was first published.  Automatically determined by the publishing-api, unless overridden by the publishing application.\",\n            \"type\": \"string\"\n        },\n        \"guid\": {\n            \"pattern\": \"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$\",\n            \"type\": \"string\"\n        },\n        \"guid_list\": {\n            \"items\": {\n                \"$ref\": \"#/definitions/guid\"\n            },\n            \"type\": \"array\",\n            \"_uniqueItems\": true\n        },\n        \"image_asset\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"alt_text\": {\n                    \"type\": \"string\"\n                },\n                \"caption\": {\n                    \"type\": \"string\"\n                },\n                \"content_type\": {\n                    \"type\": \"string\"\n                },\n                \"credit\": {\n                    \"type\": \"string\"\n                },\n                \"url\": {\n                    \"_format\": \"uri\",\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n                \"content_type\",\n                \"url\"\n            ],\n            \"type\": \"object\"\n        },\n        \"locale\": {\n            \"enum\": [\n                \"ar\",\n                \"az\",\n                \"be\",\n                \"bg\",\n                \"bn\",\n                \"cs\",\n                \"cy\",\n                \"da\",\n                \"de\",\n                \"dr\",\n                \"el\",\n                \"en\",\n                \"es\",\n                \"es-419\",\n                \"et\",\n                \"fa\",\n                \"fi\",\n                \"fr\",\n                \"gd\",\n                \"he\",\n                \"hi\",\n                \"hr\",\n                \"hu\",\n                \"hy\",\n                \"id\",\n                \"is\",\n                \"it\",\n                \"ja\",\n                \"ka\",\n                \"kk\",\n                \"ko\",\n                \"lt\",\n                \"lv\",\n                \"ms\",\n                \"mt\",\n                \"nl\",\n                \"no\",\n                \"pl\",\n                \"ps\",\n                \"pt\",\n                \"ro\",\n                \"ru\",\n                \"si\",\n                \"sk\",\n                \"sl\",\n                \"so\",\n                \"sq\",\n                \"sr\",\n                \"sv\",\n                \"sw\",\n                \"ta\",\n                \"th\",\n                \"tk\",\n                \"tr\",\n                \"uk\",\n                \"ur\",\n                \"uz\",\n                \"vi\",\n                \"zh\",\n                \"zh-hk\",\n                \"zh-tw\"\n            ],\n            \"type\": \"string\"\n        },\n        \"max_cache_time\": {\n            \"description\": \"The maximum length of time the content should be cached, in seconds\",\n            \"type\": \"integer\"\n        },\n        \"multiple_content_types\": {\n            \"items\": {\n                \"additionalProperties\": false,\n                \"properties\": {\n                    \"content\": {\n                        \"type\": \"string\"\n                    },\n                    \"content_type\": {\n                        \"type\": \"string\"\n                    }\n                },\n                \"required\": [\n                    \"content_type\",\n                    \"content\"\n                ],\n                \"type\": \"object\"\n            },\n            \"type\": \"array\"\n        },\n        \"parts\": {\n            \"items\": {\n                \"additionalProperties\": false,\n                \"properties\": {\n                    \"body\": {\n                        \"$ref\": \"#/definitions/body_html_and_govspeak\"\n                    },\n                    \"slug\": {\n                        \"_format\": \"uri\",\n                        \"type\": \"string\"\n                    },\n                    \"title\": {\n                        \"type\": \"string\"\n                    }\n                },\n                \"required\": [\n                    \"title\",\n                    \"slug\",\n                    \"body\"\n                ],\n                \"type\": \"object\"\n            },\n            \"type\": \"array\"\n        },\n        \"public_updated_at\": {\n            \"_format\": \"date-time\",\n            \"description\": \"When the content was last significantly changed (a major update). Shown to users.  Automatically determined by the publishing-api, unless overridden by the publishing application.\",\n            \"type\": \"string\"\n        },\n        \"publishing_app_name\": {\n            \"description\": \"The application that published this item.\",\n            \"enum\": [\n                \"calculators\",\n                \"calendars\",\n                \"collections-publisher\",\n                \"contacts\",\n                \"content-publisher\",\n                \"content-tagger\",\n                \"email-alert-frontend\",\n                \"external-link-tracker\",\n                \"feedback\",\n                \"frontend\",\n                \"government-frontend\",\n                \"hmrc-manuals-api\",\n                \"info-frontend\",\n                \"licencefinder\",\n                \"local-links-manager\",\n                \"manuals-frontend\",\n                \"manuals-publisher\",\n                \"maslow\",\n                \"performanceplatform-big-screen-view\",\n                \"publisher\",\n                \"rummager\",\n                \"search-admin\",\n                \"search-api\",\n                \"service-manual-publisher\",\n                \"share-sale-publisher\",\n                \"short-url-manager\",\n                \"smartanswers\",\n                \"special-route-publisher\",\n                \"specialist-publisher\",\n                \"static\",\n                \"tariff\",\n                \"travel-advice-publisher\",\n                \"whitehall\"\n            ],\n            \"type\": \"string\"\n        },\n        \"publishing_request_id\": {\n            \"description\": \"A unique identifier used to track publishing requests to rendered content\",\n            \"oneOf\": [\n                {\n                    \"type\": \"string\"\n                },\n                {\n                    \"type\": \"null\"\n                }\n            ]\n        },\n        \"rendering_app\": {\n            \"description\": \"The application that renders this item.\",\n            \"enum\": [\n                \"calculators\",\n                \"calendars\",\n                \"collections\",\n                \"content-store\",\n                \"email-alert-frontend\",\n                \"email-campaign-frontend\",\n                \"feedback\",\n                \"finder-frontend\",\n                \"frontend\",\n                \"government-frontend\",\n                \"info-frontend\",\n                \"licencefinder\",\n                \"manuals-frontend\",\n                \"performanceplatform-big-screen-view\",\n                \"rummager\",\n                \"search-api\",\n                \"service-manual-frontend\",\n                \"smartanswers\",\n                \"spotlight\",\n                \"static\",\n                \"tariff\",\n                \"whitehall-admin\",\n                \"whitehall-frontend\"\n            ],\n            \"type\": \"string\"\n        },\n        \"route\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"path\": {\n                    \"type\": \"string\"\n                },\n                \"type\": {\n                    \"enum\": [\n                        \"prefix\",\n                        \"exact\"\n                    ]\n                }\n            },\n            \"required\": [\n                \"path\",\n                \"type\"\n            ],\n            \"type\": \"object\"\n        },\n        \"routes\": {\n            \"items\": {\n                \"$ref\": \"#/definitions/route\"\n            },\n            \"minItems\": 1,\n            \"type\": \"array\"\n        },\n        \"title\": {\n            \"type\": \"string\"\n        },\n        \"update_type\": {\n            \"enum\": [\n                \"major\",\n                \"minor\",\n                \"republish\"\n            ]\n        }\n    },\n    \"properties\": {\n        \"access_limited\": {\n            \"$ref\": \"#/definitions/access_limited\"\n        },\n        \"analytics_identifier\": {\n            \"$ref\": \"#/definitions/analytics_identifier\"\n        },\n        \"auth_bypass_ids\": {\n            \"$ref\": \"#/definitions/guid_list\",\n            \"description\": \"A list of ids that will allow access to this item for non-authenticated users\"\n        },\n        \"base_path\": {\n            \"$ref\": \"#/definitions/absolute_path\"\n        },\n        \"bulk_publishing\": {\n            \"type\": \"boolean\"\n        },\n        \"change_note\": {\n            \"type\": [\n                \"null\",\n                \"string\"\n            ]\n        },\n        \"description\": {\n            \"$ref\": \"#/definitions/description_optional\"\n        },\n        \"details\": {\n            \"$ref\": \"#/definitions/details\"\n        },\n        \"document_type\": {\n            \"enum\": [\n                \"travel_advice\"\n            ],\n            \"type\": \"string\"\n        },\n        \"first_published_at\": {\n            \"$ref\": \"#/definitions/first_published_at\"\n        },\n        \"last_edited_at\": {\n            \"_format\": \"date-time\",\n            \"description\": \"Last time when the content received a major or minor update.\",\n            \"type\": \"string\"\n        },\n        \"links\": {\n            \"additionalProperties\": false,\n            \"properties\": {\n                \"policy_areas\": {\n                    \"$ref\": \"#/definitions/guid_list\",\n                    \"description\": \"A largely deprecated tag currently only used to power email alerts.\"\n                }\n            },\n            \"type\": \"object\"\n        },\n        \"locale\": {\n            \"$ref\": \"#/definitions/locale\"\n        },\n        \"need_ids\": {\n            \"items\": {\n                \"type\": \"string\"\n            },\n            \"type\": \"array\"\n        },\n        \"phase\": {\n            \"description\": \"The service design phase of this content item - https://www.gov.uk/service-manual/phases\",\n            \"enum\": [\n                \"alpha\",\n                \"beta\",\n                \"live\"\n            ],\n            \"type\": \"string\"\n        },\n        \"previous_version\": {\n            \"type\": \"string\"\n        },\n        \"public_updated_at\": {\n            \"$ref\": \"#/definitions/public_updated_at\"\n        },\n        \"publishing_app\": {\n            \"$ref\": \"#/definitions/publishing_app_name\"\n        },\n        \"redirects\": {\n            \"additionalItems\": false,\n            \"items\": {},\n            \"type\": \"array\"\n        },\n        \"rendering_app\": {\n            \"$ref\": \"#/definitions/rendering_app\"\n        },\n        \"routes\": {\n            \"$ref\": \"#/definitions/routes\"\n        },\n        \"schema_name\": {\n            \"enum\": [\n                \"travel_advice\"\n            ],\n            \"type\": \"string\"\n        },\n        \"title\": {\n            \"$ref\": \"#/definitions/title\"\n        },\n        \"update_type\": {\n            \"$ref\": \"#/definitions/update_type\"\n        }\n    },\n    \"required\": [\n        \"base_path\",\n        \"details\",\n        \"document_type\",\n        \"publishing_app\",\n        \"rendering_app\",\n        \"routes\",\n        \"schema_name\",\n        \"title\"\n    ],\n    \"type\": \"object\"\n}\\n\\nJSON object: ",
-    "positive_response": "{}",
-    "negative_response": "{invalid"
-  }
-]

wisent/examples/scripts/results/test_kbl_evaluation.json

@@ -1,51 +0,0 @@
-{
-  "task_name": "kbl",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 2,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "\uc778\uacfc\uad00\uacc4\uc5d0 \uad00\ud55c \uc124\uba85 \uc911 \uc633\uc9c0 \uc54a\uc740 \uac83\uc740? (\ub2e4\ud23c\uc774 \uc788\ub294 \uacbd\uc6b0 \ud310\ub840\uc5d0 \uc758\ud568)...",
-      "positive_response": "C",
-      "negative_response": "Z",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'C' (log_prob=-0.500), Expected: 'C'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'C' (log_prob=-0.500), Expected: 'Z'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 1,
-      "prompt": "\ub2e4\uc74c \uc0ac\ub840\uc5d0 \uad00\ud55c \uc124\uba85 \uc911 \uc633\uc740 \uac83\uc740? (\ub2e8, \ub2f4\ubc30\uc18c\ub9e4\uc778\uc9c0\uc815\ucc98\ubd84\uc758 \ubc95\uc801 \uc131\uaca9\uc740 \uac15\ud559\uc0c1 \u2018 \ud2b9\ud5c8 \u2019 \uc784\uc744 \uc804\uc81c\ub85c \ud558\uba70, \ub2e4\ud23c\uc774 \uc788\ub294 \uacbd\uc6b0 \ud310\ub840\uc5d0 \uc758\ud568) \u25cb \u7532\uc740 \uac74\ubb3c 1\uce35\uc5d0\uc11c \ub2f4\ubc30\uc18c...",
-      "positive_response": "A",
-      "negative_response": "Z",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'A' (log_prob=-0.500), Expected: 'A'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'A' (log_prob=-0.500), Expected: 'Z'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_kbl_fixed/test_kbl_evaluation.json

@@ -1,51 +0,0 @@
-{
-  "task_name": "kbl",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 2,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "\uc778\uacfc\uad00\uacc4\uc5d0 \uad00\ud55c \uc124\uba85 \uc911 \uc633\uc9c0 \uc54a\uc740 \uac83\uc740? (\ub2e4\ud23c\uc774 \uc788\ub294 \uacbd\uc6b0 \ud310\ub840\uc5d0 \uc758\ud568)...",
-      "positive_response": "C",
-      "negative_response": "Z",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'C' (log_prob=-0.500), Expected: 'C'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'C' (log_prob=-0.500), Expected: 'Z'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 1,
-      "prompt": "\ub2e4\uc74c \uc0ac\ub840\uc5d0 \uad00\ud55c \uc124\uba85 \uc911 \uc633\uc740 \uac83\uc740? (\ub2e8, \ub2f4\ubc30\uc18c\ub9e4\uc778\uc9c0\uc815\ucc98\ubd84\uc758 \ubc95\uc801 \uc131\uaca9\uc740 \uac15\ud559\uc0c1 \u2018 \ud2b9\ud5c8 \u2019 \uc784\uc744 \uc804\uc81c\ub85c \ud558\uba70, \ub2e4\ud23c\uc774 \uc788\ub294 \uacbd\uc6b0 \ud310\ub840\uc5d0 \uc758\ud568) \u25cb \u7532\uc740 \uac74\ubb3c 1\uce35\uc5d0\uc11c \ub2f4\ubc30\uc18c...",
-      "positive_response": "A",
-      "negative_response": "Z",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'A' (log_prob=-0.500), Expected: 'A'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: 'A' (log_prob=-0.500), Expected: 'Z'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_kbl_fixed/test_kbl_pairs.json

@@ -1,14 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "\uc778\uacfc\uad00\uacc4\uc5d0 \uad00\ud55c \uc124\uba85 \uc911 \uc633\uc9c0 \uc54a\uc740 \uac83\uc740? (\ub2e4\ud23c\uc774 \uc788\ub294 \uacbd\uc6b0 \ud310\ub840\uc5d0 \uc758\ud568)",
-    "positive_response": "C",
-    "negative_response": "Z"
-  },
-  {
-    "pair_id": 1,
-    "prompt": "\ub2e4\uc74c \uc0ac\ub840\uc5d0 \uad00\ud55c \uc124\uba85 \uc911 \uc633\uc740 \uac83\uc740? (\ub2e8, \ub2f4\ubc30\uc18c\ub9e4\uc778\uc9c0\uc815\ucc98\ubd84\uc758 \ubc95\uc801 \uc131\uaca9\uc740 \uac15\ud559\uc0c1 \u2018 \ud2b9\ud5c8 \u2019 \uc784\uc744 \uc804\uc81c\ub85c \ud558\uba70, \ub2e4\ud23c\uc774 \uc788\ub294 \uacbd\uc6b0 \ud310\ub840\uc5d0 \uc758\ud568) \u25cb \u7532\uc740 \uac74\ubb3c 1\uce35\uc5d0\uc11c \ub2f4\ubc30\uc18c\ub9e4\uc778 \uc9c0\uc815\uc744 \ubc1b\uc544 \ub2f4\ubc30\uc18c\ub9e4\uc5c5\uc744 \ud558\uace0 \uc788\uc5c8\ub294\ub370, \uad00\ud560 \uad6c\uccad\uc7a5 A\ub294 \ubc95\ub839\uc0c1\uc758 \uac70\ub9ac\uc81c\ud55c \uaddc\uc815\uc744 \uc704\ubc18\ud558\uc5ec \uadf8 \uc601\uc5c5\uc18c\uc5d0\uc11c 30\ubbf8\ud130 \ub5a8\uc5b4\uc9c4 \uc778\uc811 \uc544\ud30c\ud2b8 \uc0c1\uac00\uc5d0\uc11c \u4e59\uc774 \ub2f4\ubc30\uc18c\ub9e4\uc5c5\uc744 \ud560 \uc218 \uc788\ub3c4\ub85d \ub2f4\ubc30\uc18c\ub9e4\uc778 \uc2e0\uaddc\uc9c0\uc815\ucc98\ubd84\uc744 \ud558\uc600\ub2e4. \u25cb \u4e19\uacfc \u4e01\uc740 \uac19\uc740 \uc0c1\uac00\uc758 1\uce35\uacfc 2\uce35\uc5d0\uc11c \uac01\uac01 \ub2f4\ubc30\uc18c\ub9e4\uc5c5\uc744 \ud558\uace0\uc790 \uad00\ud560 \uad6c\uccad\uc7a5 B\uc5d0\uac8c \ub2f4\ubc30\uc18c\ub9e4\uc778 \uc9c0\uc815\uc2e0\uccad\uc744 \ud558\uc600\uc73c\ub098, B\ub294 \u4e01\uc5d0\uac8c\ub9cc \ub2f4\ubc30\uc18c\ub9e4\uc778\uc9c0\uc815\ucc98\ubd84\uc744 \ud558\uc600\ub2e4.",
-    "positive_response": "A",
-    "negative_response": "Z"
-  }
-]

wisent/examples/scripts/results/test_kbl_pairs.json

@@ -1,14 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "\uc778\uacfc\uad00\uacc4\uc5d0 \uad00\ud55c \uc124\uba85 \uc911 \uc633\uc9c0 \uc54a\uc740 \uac83\uc740? (\ub2e4\ud23c\uc774 \uc788\ub294 \uacbd\uc6b0 \ud310\ub840\uc5d0 \uc758\ud568)",
-    "positive_response": "C",
-    "negative_response": "Z"
-  },
-  {
-    "pair_id": 1,
-    "prompt": "\ub2e4\uc74c \uc0ac\ub840\uc5d0 \uad00\ud55c \uc124\uba85 \uc911 \uc633\uc740 \uac83\uc740? (\ub2e8, \ub2f4\ubc30\uc18c\ub9e4\uc778\uc9c0\uc815\ucc98\ubd84\uc758 \ubc95\uc801 \uc131\uaca9\uc740 \uac15\ud559\uc0c1 \u2018 \ud2b9\ud5c8 \u2019 \uc784\uc744 \uc804\uc81c\ub85c \ud558\uba70, \ub2e4\ud23c\uc774 \uc788\ub294 \uacbd\uc6b0 \ud310\ub840\uc5d0 \uc758\ud568) \u25cb \u7532\uc740 \uac74\ubb3c 1\uce35\uc5d0\uc11c \ub2f4\ubc30\uc18c\ub9e4\uc778 \uc9c0\uc815\uc744 \ubc1b\uc544 \ub2f4\ubc30\uc18c\ub9e4\uc5c5\uc744 \ud558\uace0 \uc788\uc5c8\ub294\ub370, \uad00\ud560 \uad6c\uccad\uc7a5 A\ub294 \ubc95\ub839\uc0c1\uc758 \uac70\ub9ac\uc81c\ud55c \uaddc\uc815\uc744 \uc704\ubc18\ud558\uc5ec \uadf8 \uc601\uc5c5\uc18c\uc5d0\uc11c 30\ubbf8\ud130 \ub5a8\uc5b4\uc9c4 \uc778\uc811 \uc544\ud30c\ud2b8 \uc0c1\uac00\uc5d0\uc11c \u4e59\uc774 \ub2f4\ubc30\uc18c\ub9e4\uc5c5\uc744 \ud560 \uc218 \uc788\ub3c4\ub85d \ub2f4\ubc30\uc18c\ub9e4\uc778 \uc2e0\uaddc\uc9c0\uc815\ucc98\ubd84\uc744 \ud558\uc600\ub2e4. \u25cb \u4e19\uacfc \u4e01\uc740 \uac19\uc740 \uc0c1\uac00\uc758 1\uce35\uacfc 2\uce35\uc5d0\uc11c \uac01\uac01 \ub2f4\ubc30\uc18c\ub9e4\uc5c5\uc744 \ud558\uace0\uc790 \uad00\ud560 \uad6c\uccad\uc7a5 B\uc5d0\uac8c \ub2f4\ubc30\uc18c\ub9e4\uc778 \uc9c0\uc815\uc2e0\uccad\uc744 \ud558\uc600\uc73c\ub098, B\ub294 \u4e01\uc5d0\uac8c\ub9cc \ub2f4\ubc30\uc18c\ub9e4\uc778\uc9c0\uc815\ucc98\ubd84\uc744 \ud558\uc600\ub2e4.",
-    "positive_response": "A",
-    "negative_response": "Z"
-  }
-]

wisent/examples/scripts/results/test_kmmlu_evaluation.json

@@ -1,51 +0,0 @@
-{
-  "task_name": "kmmlu",
-  "model_name": "mock",
-  "evaluator_name": "log_likelihoods",
-  "num_pairs": 2,
-  "all_correct": true,
-  "pairs": [
-    {
-      "pair_id": 0,
-      "prompt": "Question: \uc774\uc0c1\uae30\uccb4\ub97c T1\uc5d0\uc11c T2\uae4c\uc9c0 \uc77c\uc815\uc555\ub825\uacfc \uc77c\uc815\uc6a9\uc801\uc5d0\uc11c \uac00\uc5f4\ud560 \ub54c \uc5f4\uc6a9\ub7c9\uc5d0 \uad00\ud55c \uc2dd \uc911 \uc633\uc740 \uac83\uc740? (\ub2e8, CP\ub294 \uc815\uc555\uc5f4\uc6a9\ub7c9\uc774\uace0, CV\ub294 \uc815\uc801\uc5f4\uc6a9\ub7c9\uc774\ub2e4.)\nA. \u25b3U...",
-      "positive_response": "\u25b3U=CV\u318d\u25b3T-W",
-      "negative_response": "\u25b3U=R\u318d\u25b3T\u318dCP",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\u25b3U=CV\u318d\u25b3T-W' (log_prob=-0.500), Expected: '\u25b3U=CV\u318d\u25b3T-W'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\u25b3U=CV\u318d\u25b3T-W' (log_prob=-0.500), Expected: '\u25b3U=R\u318d\u25b3T\u318dCP'"
-      },
-      "both_correct": true
-    },
-    {
-      "pair_id": 1,
-      "prompt": "Question: \uac01\uad6d \ubbfc\uac04\uacbd\ube44\uc6d0\uc758 \ubc95\uc801\uc9c0\uc704\uc640 \uad8c\ud55c\uc5d0 \uad00\ud55c \uc124\uba85\uc73c\ub85c \uc633\uc9c0 \uc54a\uc740 \uac83\uc740?\nA. \uc77c\ubcf8\uc5d0\uc11c \ubbfc\uac04\uacbd\ube44\uc6d0\uc740 \uc5c5\ubb34\uc758 \ud2b9\uc218\uc131\uc73c\ub85c \uc778\ud574 \ud5cc\ubc95\uc5d0 \uaddc\uc815\ub41c \uad6d\ubbfc\uc758 \uad8c\ub9ac\ub97c \uce68\ud574\ud560 \uc6b0\ub824\uac00 ...",
-      "positive_response": "\ubbf8\uad6d\uc5d0\uc11c \ubbfc\uac04\uacbd\ube44\uc6d0\uc774 \uacbd\ucc30\uacfc \ud611\ub825 \ub610\ub294 \uae30\uc18c\ub97c \ubaa9\uc801\uc73c\ub85c \uc99d\uac70\ub97c \uc218\uc9d1\ud558\uc5ec \uacbd\ucc30\uc5d0 \uc81c\uacf5\ud558\ub294 \ub300\ub9ac\uc778\uc73c\ub85c \ud65c\ub3d9\ud55c \uacbd\uc6b0 \ud5cc\ubc95\uc801 \uc81c\ud55c\uc774 \ub530\ub978\ub2e4.",
-      "negative_response": "\uc77c\ubcf8\uc5d0\uc11c \ubbfc\uac04\uacbd\ube44\uc6d0\uc740 \uc5c5\ubb34\uc758 \ud2b9\uc218\uc131\uc73c\ub85c \uc778\ud574 \ud5cc\ubc95\uc5d0 \uaddc\uc815\ub41c \uad6d\ubbfc\uc758 \uad8c\ub9ac\ub97c \uce68\ud574\ud560 \uc6b0\ub824\uac00 \uc788\uc73c\ubbc0\ub85c \uc8fc\uc758\uac00 \ud544\uc694\ud558\ub2e4.",
-      "positive_evaluation": {
-        "ground_truth": "TRUTHFUL",
-        "confidence": 1.0,
-        "expected": "TRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\ubbf8\uad6d\uc5d0\uc11c \ubbfc\uac04\uacbd\ube44\uc6d0\uc774 \uacbd\ucc30\uacfc \ud611\ub825 \ub610\ub294 \uae30\uc18c\ub97c \ubaa9\uc801\uc73c\ub85c \uc99d\uac70\ub97c \uc218\uc9d1\ud558\uc5ec \uacbd\ucc30\uc5d0 \uc81c\uacf5\ud558\ub294 \ub300\ub9ac\uc778\uc73c\ub85c \ud65c\ub3d9\ud55c \uacbd\uc6b0 \ud5cc\ubc95\uc801 \uc81c\ud55c\uc774 \ub530\ub978\ub2e4.' (log_prob=-0.500), Expected: '\ubbf8\uad6d\uc5d0\uc11c \ubbfc\uac04\uacbd\ube44\uc6d0\uc774 \uacbd\ucc30\uacfc \ud611\ub825 \ub610\ub294 \uae30\uc18c\ub97c \ubaa9\uc801\uc73c\ub85c \uc99d\uac70\ub97c \uc218\uc9d1\ud558\uc5ec \uacbd\ucc30\uc5d0 \uc81c\uacf5\ud558\ub294 \ub300\ub9ac\uc778\uc73c\ub85c \ud65c\ub3d9\ud55c \uacbd\uc6b0 \ud5cc\ubc95\uc801 \uc81c\ud55c\uc774 \ub530\ub978\ub2e4.'"
-      },
-      "negative_evaluation": {
-        "ground_truth": "UNTRUTHFUL",
-        "confidence": 0.0,
-        "expected": "UNTRUTHFUL",
-        "correct": true,
-        "details": "Predicted: '\ubbf8\uad6d\uc5d0\uc11c \ubbfc\uac04\uacbd\ube44\uc6d0\uc774 \uacbd\ucc30\uacfc \ud611\ub825 \ub610\ub294 \uae30\uc18c\ub97c \ubaa9\uc801\uc73c\ub85c \uc99d\uac70\ub97c \uc218\uc9d1\ud558\uc5ec \uacbd\ucc30\uc5d0 \uc81c\uacf5\ud558\ub294 \ub300\ub9ac\uc778\uc73c\ub85c \ud65c\ub3d9\ud55c \uacbd\uc6b0 \ud5cc\ubc95\uc801 \uc81c\ud55c\uc774 \ub530\ub978\ub2e4.' (log_prob=-0.500), Expected: '\uc77c\ubcf8\uc5d0\uc11c \ubbfc\uac04\uacbd\ube44\uc6d0\uc740 \uc5c5\ubb34\uc758 \ud2b9\uc218\uc131\uc73c\ub85c \uc778\ud574 \ud5cc\ubc95\uc5d0 \uaddc\uc815\ub41c \uad6d\ubbfc\uc758 \uad8c\ub9ac\ub97c \uce68\ud574\ud560 \uc6b0\ub824\uac00 \uc788\uc73c\ubbc0\ub85c \uc8fc\uc758\uac00 \ud544\uc694\ud558\ub2e4.'"
-      },
-      "both_correct": true
-    }
-  ]
-}

wisent/examples/scripts/results/test_kmmlu_pairs.json

@@ -1,14 +0,0 @@
-[
-  {
-    "pair_id": 0,
-    "prompt": "Question: \uc774\uc0c1\uae30\uccb4\ub97c T1\uc5d0\uc11c T2\uae4c\uc9c0 \uc77c\uc815\uc555\ub825\uacfc \uc77c\uc815\uc6a9\uc801\uc5d0\uc11c \uac00\uc5f4\ud560 \ub54c \uc5f4\uc6a9\ub7c9\uc5d0 \uad00\ud55c \uc2dd \uc911 \uc633\uc740 \uac83\uc740? (\ub2e8, CP\ub294 \uc815\uc555\uc5f4\uc6a9\ub7c9\uc774\uace0, CV\ub294 \uc815\uc801\uc5f4\uc6a9\ub7c9\uc774\ub2e4.)\nA. \u25b3U=R\u318d\u25b3T\u318dCP\nB. \u25b3U=CV\u318d\u25b3T-W",
-    "positive_response": "\u25b3U=CV\u318d\u25b3T-W",
-    "negative_response": "\u25b3U=R\u318d\u25b3T\u318dCP"
-  },
-  {
-    "pair_id": 1,
-    "prompt": "Question: \uac01\uad6d \ubbfc\uac04\uacbd\ube44\uc6d0\uc758 \ubc95\uc801\uc9c0\uc704\uc640 \uad8c\ud55c\uc5d0 \uad00\ud55c \uc124\uba85\uc73c\ub85c \uc633\uc9c0 \uc54a\uc740 \uac83\uc740?\nA. \uc77c\ubcf8\uc5d0\uc11c \ubbfc\uac04\uacbd\ube44\uc6d0\uc740 \uc5c5\ubb34\uc758 \ud2b9\uc218\uc131\uc73c\ub85c \uc778\ud574 \ud5cc\ubc95\uc5d0 \uaddc\uc815\ub41c \uad6d\ubbfc\uc758 \uad8c\ub9ac\ub97c \uce68\ud574\ud560 \uc6b0\ub824\uac00 \uc788\uc73c\ubbc0\ub85c \uc8fc\uc758\uac00 \ud544\uc694\ud558\ub2e4.\nB. \ubbf8\uad6d\uc5d0\uc11c \ubbfc\uac04\uacbd\ube44\uc6d0\uc774 \uacbd\ucc30\uacfc \ud611\ub825 \ub610\ub294 \uae30\uc18c\ub97c \ubaa9\uc801\uc73c\ub85c \uc99d\uac70\ub97c \uc218\uc9d1\ud558\uc5ec \uacbd\ucc30\uc5d0 \uc81c\uacf5\ud558\ub294 \ub300\ub9ac\uc778\uc73c\ub85c \ud65c\ub3d9\ud55c \uacbd\uc6b0 \ud5cc\ubc95\uc801 \uc81c\ud55c\uc774 \ub530\ub978\ub2e4.",
-    "positive_response": "\ubbf8\uad6d\uc5d0\uc11c \ubbfc\uac04\uacbd\ube44\uc6d0\uc774 \uacbd\ucc30\uacfc \ud611\ub825 \ub610\ub294 \uae30\uc18c\ub97c \ubaa9\uc801\uc73c\ub85c \uc99d\uac70\ub97c \uc218\uc9d1\ud558\uc5ec \uacbd\ucc30\uc5d0 \uc81c\uacf5\ud558\ub294 \ub300\ub9ac\uc778\uc73c\ub85c \ud65c\ub3d9\ud55c \uacbd\uc6b0 \ud5cc\ubc95\uc801 \uc81c\ud55c\uc774 \ub530\ub978\ub2e4.",
-    "negative_response": "\uc77c\ubcf8\uc5d0\uc11c \ubbfc\uac04\uacbd\ube44\uc6d0\uc740 \uc5c5\ubb34\uc758 \ud2b9\uc218\uc131\uc73c\ub85c \uc778\ud574 \ud5cc\ubc95\uc5d0 \uaddc\uc815\ub41c \uad6d\ubbfc\uc758 \uad8c\ub9ac\ub97c \uce68\ud574\ud560 \uc6b0\ub824\uac00 \uc788\uc73c\ubbc0\ub85c \uc8fc\uc758\uac00 \ud544\uc694\ud558\ub2e4."
-  }
-]