wisent 0.7.925__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1065) hide show
  1. wisent-0.7.925/LICENSE +21 -0
  2. wisent-0.7.925/PKG-INFO +68 -0
  3. wisent-0.7.925/README.md +18 -0
  4. wisent-0.7.925/pyproject.toml +85 -0
  5. wisent-0.7.925/setup.cfg +4 -0
  6. wisent-0.7.925/setup.py +78 -0
  7. wisent-0.7.925/tests/test_multimodal_adapters.py +666 -0
  8. wisent-0.7.925/tests/test_multimodal_pipeline.py +661 -0
  9. wisent-0.7.925/wisent/__init__.py +64 -0
  10. wisent-0.7.925/wisent/cli.py +114 -0
  11. wisent-0.7.925/wisent/comparison/__init__.py +1 -0
  12. wisent-0.7.925/wisent/comparison/detect_bos_features.py +275 -0
  13. wisent-0.7.925/wisent/comparison/fgaa.py +465 -0
  14. wisent-0.7.925/wisent/comparison/lora.py +500 -0
  15. wisent-0.7.925/wisent/comparison/main.py +444 -0
  16. wisent-0.7.925/wisent/comparison/ours.py +76 -0
  17. wisent-0.7.925/wisent/comparison/sae.py +304 -0
  18. wisent-0.7.925/wisent/comparison/utils.py +381 -0
  19. wisent-0.7.925/wisent/core/__init__.py +40 -0
  20. wisent-0.7.925/wisent/core/activations/__init__.py +42 -0
  21. wisent-0.7.925/wisent/core/activations/activation_cache.py +393 -0
  22. wisent-0.7.925/wisent/core/activations/activations.py +79 -0
  23. wisent-0.7.925/wisent/core/activations/activations_collector.py +279 -0
  24. wisent-0.7.925/wisent/core/activations/classifier_inference_strategy.py +195 -0
  25. wisent-0.7.925/wisent/core/activations/core/__init__.py +0 -0
  26. wisent-0.7.925/wisent/core/activations/core/atoms.py +135 -0
  27. wisent-0.7.925/wisent/core/activations/extraction_strategy.py +484 -0
  28. wisent-0.7.925/wisent/core/adapters/__init__.py +22 -0
  29. wisent-0.7.925/wisent/core/adapters/audio.py +616 -0
  30. wisent-0.7.925/wisent/core/adapters/base.py +420 -0
  31. wisent-0.7.925/wisent/core/adapters/multimodal.py +738 -0
  32. wisent-0.7.925/wisent/core/adapters/robotics.py +643 -0
  33. wisent-0.7.925/wisent/core/adapters/text.py +441 -0
  34. wisent-0.7.925/wisent/core/adapters/video.py +555 -0
  35. wisent-0.7.925/wisent/core/agent/__init__.py +1 -0
  36. wisent-0.7.925/wisent/core/agent/budget.py +644 -0
  37. wisent-0.7.925/wisent/core/agent/device_benchmarks.py +691 -0
  38. wisent-0.7.925/wisent/core/agent/diagnose/__init__.py +1 -0
  39. wisent-0.7.925/wisent/core/agent/diagnose/agent_classifier_decision.py +641 -0
  40. wisent-0.7.925/wisent/core/agent/diagnose/classifier_marketplace.py +554 -0
  41. wisent-0.7.925/wisent/core/agent/diagnose/create_classifier.py +1155 -0
  42. wisent-0.7.925/wisent/core/agent/diagnose/response_diagnostics.py +273 -0
  43. wisent-0.7.925/wisent/core/agent/diagnose/select_classifiers.py +507 -0
  44. wisent-0.7.925/wisent/core/agent/diagnose/synthetic_classifier_option.py +755 -0
  45. wisent-0.7.925/wisent/core/agent/diagnose/tasks/__init__.py +33 -0
  46. wisent-0.7.925/wisent/core/agent/diagnose/tasks/task_manager.py +1453 -0
  47. wisent-0.7.925/wisent/core/agent/diagnose/tasks/task_relevance.py +94 -0
  48. wisent-0.7.925/wisent/core/agent/diagnose/tasks/task_selector.py +151 -0
  49. wisent-0.7.925/wisent/core/agent/diagnose.py +249 -0
  50. wisent-0.7.925/wisent/core/agent/steer.py +215 -0
  51. wisent-0.7.925/wisent/core/agent/timeout.py +134 -0
  52. wisent-0.7.925/wisent/core/autonomous_agent.py +1158 -0
  53. wisent-0.7.925/wisent/core/benchmark_extractors.py +372 -0
  54. wisent-0.7.925/wisent/core/benchmark_registry.py +151 -0
  55. wisent-0.7.925/wisent/core/bigcode_extractors.py +26 -0
  56. wisent-0.7.925/wisent/core/bigcode_integration.py +886 -0
  57. wisent-0.7.925/wisent/core/branding.py +108 -0
  58. wisent-0.7.925/wisent/core/classifier/__init__.py +1 -0
  59. wisent-0.7.925/wisent/core/classifier/models/__init__.py +1 -0
  60. wisent-0.7.925/wisent/core/classifiers/__init__.py +1 -0
  61. wisent-0.7.925/wisent/core/classifiers/classifiers/__init__.py +0 -0
  62. wisent-0.7.925/wisent/core/classifiers/classifiers/core/__init__.py +0 -0
  63. wisent-0.7.925/wisent/core/classifiers/classifiers/core/atoms.py +749 -0
  64. wisent-0.7.925/wisent/core/classifiers/classifiers/models/__init__.py +0 -0
  65. wisent-0.7.925/wisent/core/classifiers/classifiers/models/logistic.py +29 -0
  66. wisent-0.7.925/wisent/core/classifiers/classifiers/models/mlp.py +47 -0
  67. wisent-0.7.925/wisent/core/classifiers/classifiers/rotator.py +137 -0
  68. wisent-0.7.925/wisent/core/classifiers/core/__init__.py +1 -0
  69. wisent-0.7.925/wisent/core/classifiers/models/__init__.py +1 -0
  70. wisent-0.7.925/wisent/core/classifiers/pipeline_steps/__init__.py +1 -0
  71. wisent-0.7.925/wisent/core/cli/__init__.py +27 -0
  72. wisent-0.7.925/wisent/core/cli/agent/__init__.py +15 -0
  73. wisent-0.7.925/wisent/core/cli/agent/apply_steering.py +188 -0
  74. wisent-0.7.925/wisent/core/cli/agent/evaluate_response.py +126 -0
  75. wisent-0.7.925/wisent/core/cli/agent/generate_synthetic_pairs.py +123 -0
  76. wisent-0.7.925/wisent/core/cli/agent/main.py +139 -0
  77. wisent-0.7.925/wisent/core/cli/agent/train_classifier.py +184 -0
  78. wisent-0.7.925/wisent/core/cli/check_linearity.py +158 -0
  79. wisent-0.7.925/wisent/core/cli/cluster_benchmarks.py +470 -0
  80. wisent-0.7.925/wisent/core/cli/create_steering_vector.py +314 -0
  81. wisent-0.7.925/wisent/core/cli/diagnose_pairs.py +153 -0
  82. wisent-0.7.925/wisent/core/cli/diagnose_vectors.py +407 -0
  83. wisent-0.7.925/wisent/core/cli/estimate_unified_goodness_time.py +430 -0
  84. wisent-0.7.925/wisent/core/cli/evaluate_refusal.py +241 -0
  85. wisent-0.7.925/wisent/core/cli/evaluate_responses.py +926 -0
  86. wisent-0.7.925/wisent/core/cli/generate_humanization_pairs.py +128 -0
  87. wisent-0.7.925/wisent/core/cli/generate_pairs.py +175 -0
  88. wisent-0.7.925/wisent/core/cli/generate_pairs_from_task.py +61 -0
  89. wisent-0.7.925/wisent/core/cli/generate_responses.py +160 -0
  90. wisent-0.7.925/wisent/core/cli/generate_vector_from_synthetic.py +217 -0
  91. wisent-0.7.925/wisent/core/cli/generate_vector_from_task.py +243 -0
  92. wisent-0.7.925/wisent/core/cli/geometry_search.py +137 -0
  93. wisent-0.7.925/wisent/core/cli/get_activations.py +168 -0
  94. wisent-0.7.925/wisent/core/cli/inference_config.py +84 -0
  95. wisent-0.7.925/wisent/core/cli/inference_config_cli.py +54 -0
  96. wisent-0.7.925/wisent/core/cli/method_optimizer.py +860 -0
  97. wisent-0.7.925/wisent/core/cli/modify_weights.py +661 -0
  98. wisent-0.7.925/wisent/core/cli/multi_steer.py +112 -0
  99. wisent-0.7.925/wisent/core/cli/optimization_cache.py +298 -0
  100. wisent-0.7.925/wisent/core/cli/optimize.py +660 -0
  101. wisent-0.7.925/wisent/core/cli/optimize_classification.py +472 -0
  102. wisent-0.7.925/wisent/core/cli/optimize_sample_size.py +376 -0
  103. wisent-0.7.925/wisent/core/cli/optimize_steering.py +3695 -0
  104. wisent-0.7.925/wisent/core/cli/optimize_weights.py +1347 -0
  105. wisent-0.7.925/wisent/core/cli/preview_pairs.py +203 -0
  106. wisent-0.7.925/wisent/core/cli/steering_method_trainer.py +642 -0
  107. wisent-0.7.925/wisent/core/cli/steering_search_space.py +513 -0
  108. wisent-0.7.925/wisent/core/cli/tasks.py +854 -0
  109. wisent-0.7.925/wisent/core/cli/train_unified_goodness.py +680 -0
  110. wisent-0.7.925/wisent/core/cli_logger.py +22 -0
  111. wisent-0.7.925/wisent/core/config_manager.py +1731 -0
  112. wisent-0.7.925/wisent/core/contrastive_pairs/__init__.py +15 -0
  113. wisent-0.7.925/wisent/core/contrastive_pairs/core/__init__.py +0 -0
  114. wisent-0.7.925/wisent/core/contrastive_pairs/core/atoms.py +45 -0
  115. wisent-0.7.925/wisent/core/contrastive_pairs/core/buliders.py +59 -0
  116. wisent-0.7.925/wisent/core/contrastive_pairs/core/pair.py +183 -0
  117. wisent-0.7.925/wisent/core/contrastive_pairs/core/response.py +153 -0
  118. wisent-0.7.925/wisent/core/contrastive_pairs/core/serialization.py +306 -0
  119. wisent-0.7.925/wisent/core/contrastive_pairs/core/set.py +192 -0
  120. wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/__init__.py +79 -0
  121. wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/activations.py +53 -0
  122. wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/base.py +73 -0
  123. wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/control_vectors.py +3060 -0
  124. wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/coverage.py +79 -0
  125. wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/divergence.py +98 -0
  126. wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/duplicates.py +118 -0
  127. wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/linearity.py +315 -0
  128. wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/vector_quality.py +621 -0
  129. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/__init__.py +1 -0
  130. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/atoms.py +255 -0
  131. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_extractor_manifest.py +456 -0
  132. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_extractor_registry.py +136 -0
  133. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/__init__.py +50 -0
  134. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentbench.py +225 -0
  135. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentharm.py +267 -0
  136. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentic_search.py +134 -0
  137. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aider_polyglot.py +202 -0
  138. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime.py +118 -0
  139. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime2024.py +74 -0
  140. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime2025.py +73 -0
  141. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/alpaca_eval.py +153 -0
  142. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/apps.py +296 -0
  143. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/arena_hard.py +179 -0
  144. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/atis.py +89 -0
  145. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/babilong.py +96 -0
  146. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bangla_mmlu.py +108 -0
  147. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/basqueglue.py +217 -0
  148. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bec2016eu.py +99 -0
  149. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bfcl.py +283 -0
  150. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bhtc_v2.py +87 -0
  151. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/chain_of_thought.py +89 -0
  152. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/chinese_simpleqa.py +209 -0
  153. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/cluewsc.py +177 -0
  154. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/cnn_dailymail.py +92 -0
  155. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codeforces.py +368 -0
  156. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue.py +109 -0
  157. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/coding_benchmarks.py +464 -0
  158. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/coedit_gec.py +79 -0
  159. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/conala.py +133 -0
  160. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/concode.py +111 -0
  161. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/dbpedia_14.py +91 -0
  162. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/doc_vqa.py +102 -0
  163. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/donotanswer.py +236 -0
  164. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ds1000.py +129 -0
  165. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ds_1000.py +155 -0
  166. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/epec_koref_bin.py +85 -0
  167. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ethos_binary.py +82 -0
  168. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/evalita_mp.py +165 -0
  169. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/evalita_sp_sum_task_fp_small_p1.py +89 -0
  170. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/facts_grounding.py +181 -0
  171. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/faithbench.py +272 -0
  172. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/financial_tweets.py +100 -0
  173. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flames.py +227 -0
  174. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flan_held_in.py +98 -0
  175. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flores.py +583 -0
  176. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/frames.py +159 -0
  177. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/get_negative_example_livecodebench.py +146 -0
  178. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/get_positive_example_livecodebench.py +140 -0
  179. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/gpt3_translation_benchmarks.py +98 -0
  180. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hallucinations_leaderboard.py +347 -0
  181. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/halueval.py +246 -0
  182. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/harmbench.py +250 -0
  183. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/healthbench.py +181 -0
  184. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hle.py +106 -0
  185. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hmmt.py +117 -0
  186. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/humaneval.py +160 -0
  187. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/humanevalpack.py +102 -0
  188. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/iwslt2017_ar_en.py +98 -0
  189. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/iwslt2017_en_ar.py +98 -0
  190. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/jailbreakbench.py +258 -0
  191. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/law_stack_exchange.py +101 -0
  192. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ledgar.py +118 -0
  193. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench.py +61 -0
  194. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench_contrastive_pair_generator.py +491 -0
  195. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench_v6.py +263 -0
  196. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livemathbench.py +268 -0
  197. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/llama.py +96 -0
  198. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/longform_writing.py +175 -0
  199. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/m_mmlu.py +96 -0
  200. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/math.py +186 -0
  201. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/math500.py +181 -0
  202. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/meddialog.py +79 -0
  203. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/medical_abstracts.py +101 -0
  204. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/medium_priority_benchmarks.py +737 -0
  205. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mercury.py +134 -0
  206. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mmlu_redux.py +194 -0
  207. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mmlusr.py +108 -0
  208. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multimedqa.py +99 -0
  209. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multipl_e.py +109 -0
  210. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple.py +96 -0
  211. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_choice.py +87 -0
  212. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_cpp.py +128 -0
  213. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_go.py +128 -0
  214. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_java.py +128 -0
  215. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_js.py +128 -0
  216. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_py.py +15 -0
  217. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_rs.py +128 -0
  218. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/non_greedy_robustness_agieval_aqua_rat.py +92 -0
  219. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/olympiadbench.py +277 -0
  220. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/openllm.py +99 -0
  221. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/option_order_robustness_agieval_aqua_rat.py +92 -0
  222. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/or_bench.py +300 -0
  223. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/penn_treebank.py +80 -0
  224. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/planbench.py +176 -0
  225. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/polymath.py +500 -0
  226. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/prompt_robustness_agieval_aqua_rat.py +92 -0
  227. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/pythia.py +99 -0
  228. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/recode.py +146 -0
  229. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/refusalbench.py +288 -0
  230. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/scicode.py +275 -0
  231. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/self_consistency.py +90 -0
  232. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/simpleqa.py +164 -0
  233. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/sorry_bench.py +211 -0
  234. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/stsb.py +79 -0
  235. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_lm_eval_v1.py +99 -0
  236. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_lm_eval_v1_seq2seq.py +98 -0
  237. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_t5_prompt.py +123 -0
  238. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_gpqa.py +106 -0
  239. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/swe_bench.py +428 -0
  240. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/swe_bench_verified.py +158 -0
  241. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/sycophancy_eval.py +205 -0
  242. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/t0_eval.py +79 -0
  243. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tag.py +98 -0
  244. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tau_bench.py +205 -0
  245. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tmlu.py +109 -0
  246. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/toolbench.py +266 -0
  247. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/toolemu.py +252 -0
  248. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/travelplanner.py +286 -0
  249. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/truthfulqa_generation.py +128 -0
  250. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/unfair_tos.py +83 -0
  251. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/vaxx_stance.py +86 -0
  252. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wiceu.py +85 -0
  253. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wikitext103.py +97 -0
  254. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wildguard.py +280 -0
  255. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt14_en_fr.py +97 -0
  256. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt14_fr_en.py +97 -0
  257. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_de_en.py +90 -0
  258. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_en_de.py +90 -0
  259. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_en_ro.py +90 -0
  260. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_ro_en.py +90 -0
  261. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt_ro_en_t5_prompt.py +90 -0
  262. wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/xsum.py +81 -0
  263. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/__init__.py +0 -0
  264. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/atoms.py +265 -0
  265. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/__init__.py +472 -0
  266. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aclue.py +24 -0
  267. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/acp.py +33 -0
  268. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/acpbench.py +39 -0
  269. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/advanced_ai_risk.py +59 -0
  270. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aexams.py +14 -0
  271. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrimgsm.py +10 -0
  272. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrimmlu.py +10 -0
  273. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrixnli.py +9 -0
  274. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench.py +14 -0
  275. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_adr.py +9 -0
  276. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_afriqa.py +9 -0
  277. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_afrisenti.py +9 -0
  278. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_belebele.py +9 -0
  279. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_flores.py +9 -0
  280. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_injongointent.py +9 -0
  281. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_mafand.py +9 -0
  282. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhaner.py +9 -0
  283. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhanews.py +9 -0
  284. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhapos.py +9 -0
  285. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_naijarc.py +9 -0
  286. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_nollysenti.py +9 -0
  287. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_ntrex.py +9 -0
  288. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_openai_mmlu.py +9 -0
  289. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_salt.py +9 -0
  290. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_sib.py +9 -0
  291. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_uhura_arc_easy.py +9 -0
  292. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_xlsum.py +9 -0
  293. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/agieval.py +33 -0
  294. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/anli.py +9 -0
  295. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arab_culture.py +24 -0
  296. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_acva.py +67 -0
  297. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_acva_light.py +67 -0
  298. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_complete.py +24 -0
  299. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_light.py +81 -0
  300. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabicmmlu.py +59 -0
  301. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aradice.py +36 -0
  302. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arc.py +61 -0
  303. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arithmetic.py +19 -0
  304. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/basque_bench.py +37 -0
  305. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bbh.py +121 -0
  306. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bbq.py +9 -0
  307. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/belebele.py +293 -0
  308. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bertaqa.py +25 -0
  309. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bigbench.py +300 -0
  310. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/blimp.py +76 -0
  311. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/careqa.py +9 -0
  312. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/catalan_bench.py +43 -0
  313. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ceval_valid.py +61 -0
  314. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/cmmlu.py +76 -0
  315. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/code_x_glue.py +16 -0
  316. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/copal_id.py +11 -0
  317. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/crows_pairs.py +31 -0
  318. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/csatqa.py +15 -0
  319. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/darija.py +29 -0
  320. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/darijammlu.py +57 -0
  321. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/egymmlu.py +62 -0
  322. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/eus.py +76 -0
  323. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/evalita_mp.py +93 -0
  324. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/fld.py +9 -0
  325. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/flores.py +466 -0
  326. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/freebase.py +9 -0
  327. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/french_bench.py +23 -0
  328. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/galician_bench.py +41 -0
  329. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/glianorex.py +11 -0
  330. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/global_mmlu.py +115 -0
  331. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gpqa.py +27 -0
  332. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gsm8k.py +9 -0
  333. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gsm8k_platinum.py +9 -0
  334. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/haerae.py +14 -0
  335. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/headqa.py +11 -0
  336. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hellaswag.py +39 -0
  337. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hendrycks_ethics.py +14 -0
  338. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hendrycks_math.py +9 -0
  339. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hrm8k.py +20 -0
  340. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/inverse.py +22 -0
  341. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/japanese_leaderboard.py +20 -0
  342. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/jsonschema_bench.py +9 -0
  343. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kbl.py +85 -0
  344. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kmmlu.py +281 -0
  345. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kobest.py +14 -0
  346. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kormedmcqa.py +9 -0
  347. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/lambada.py +28 -0
  348. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/leaderboard.py +52 -0
  349. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/libra.py +9 -0
  350. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/lingoly.py +11 -0
  351. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/longbench.py +9 -0
  352. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/m.py +43 -0
  353. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mastermind.py +9 -0
  354. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mathqa.py +9 -0
  355. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/med.py +24 -0
  356. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/meddialog.py +12 -0
  357. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/medqa.py +9 -0
  358. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mela.py +18 -0
  359. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/metabench.py +36 -0
  360. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mgsm.py +44 -0
  361. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/minerva_math.py +16 -0
  362. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mlqa.py +58 -0
  363. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu.py +70 -0
  364. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_pro.py +23 -0
  365. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_pro_plus.py +23 -0
  366. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_prox.py +191 -0
  367. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlusr.py +9 -0
  368. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmmu.py +46 -0
  369. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/model_written_evals.py +9 -0
  370. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/multiblimp.py +111 -0
  371. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/non.py +23 -0
  372. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/noreval.py +143 -0
  373. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/noridiom.py +20 -0
  374. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/nortruthfulqa.py +32 -0
  375. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/nrk.py +20 -0
  376. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi.py +9 -0
  377. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_arc_multilingual.py +10 -0
  378. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_hellaswag_multilingual.py +24 -0
  379. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_mmlu_multilingual.py +24 -0
  380. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_truthfulqa_multilingual.py +34 -0
  381. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/paloma.py +25 -0
  382. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/pawsx.py +9 -0
  383. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/persona.py +144 -0
  384. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/pile.py +31 -0
  385. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/polemo2.py +9 -0
  386. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/portuguese_bench.py +31 -0
  387. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/prompt.py +23 -0
  388. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/qa4mre.py +12 -0
  389. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/qasper.py +11 -0
  390. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ru.py +19 -0
  391. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ruler.py +9 -0
  392. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/score.py +20 -0
  393. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/scrolls.py +9 -0
  394. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/self_consistency.py +11 -0
  395. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/spanish_bench.py +38 -0
  396. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/storycloze.py +9 -0
  397. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/super_glue_t5_prompt.py +17 -0
  398. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tinyBenchmarks.py +9 -0
  399. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tmlu.py +9 -0
  400. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tmmluplus.py +80 -0
  401. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/translation.py +9 -0
  402. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/truthfulqa.py +76 -0
  403. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/truthfulqa_multi.py +24 -0
  404. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/turkishmmlu.py +30 -0
  405. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/unitxt.py +23 -0
  406. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/unscramble.py +9 -0
  407. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/winogender.py +16 -0
  408. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmdp.py +12 -0
  409. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmt14.py +16 -0
  410. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmt16.py +22 -0
  411. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wsc273.py +9 -0
  412. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xcopa.py +21 -0
  413. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xnli.py +28 -0
  414. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xnli_eu.py +12 -0
  415. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xquad.py +22 -0
  416. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xstorycloze.py +22 -0
  417. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xwinograd.py +15 -0
  418. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_extractor_manifest.py +484 -0
  419. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_extractor_registry.py +158 -0
  420. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/__init__.py +125 -0
  421. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aclue.py +169 -0
  422. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/acp_bench.py +205 -0
  423. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/acp_bench_hard.py +183 -0
  424. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/advanced.py +128 -0
  425. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aexams.py +182 -0
  426. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrimgsm.py +98 -0
  427. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrimmlu.py +111 -0
  428. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrixnli.py +129 -0
  429. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrobench_cot.py +88 -0
  430. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrobench_mc.py +107 -0
  431. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ag.py +134 -0
  432. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/agieval_aqua_rat.py +129 -0
  433. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ai2_arc.py +114 -0
  434. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anagrams1.py +81 -0
  435. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anagrams2.py +81 -0
  436. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anli.py +140 -0
  437. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabculture.py +178 -0
  438. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic.py +96 -0
  439. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_exams.py +102 -0
  440. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_leaderboard_complete.py +166 -0
  441. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_leaderboard_light.py +166 -0
  442. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabicmmlu.py +165 -0
  443. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aradice.py +266 -0
  444. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc.py +131 -0
  445. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_challenge.py +117 -0
  446. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_easy.py +117 -0
  447. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_gen.py +101 -0
  448. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_mc.py +106 -0
  449. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/argument.py +134 -0
  450. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arithmetic.py +114 -0
  451. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/asdiv.py +122 -0
  452. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/assin.py +103 -0
  453. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/babi.py +147 -0
  454. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench.py +153 -0
  455. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench_gen.py +168 -0
  456. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench_mc.py +139 -0
  457. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bbh.py +133 -0
  458. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bbq.py +167 -0
  459. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/belebele.py +179 -0
  460. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/benchmarks.py +153 -0
  461. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bertaqa.py +163 -0
  462. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bhs.py +153 -0
  463. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bhtc.py +141 -0
  464. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bigbench.py +170 -0
  465. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/blimp.py +169 -0
  466. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/blimp_nl.py +150 -0
  467. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/boolq.py +134 -0
  468. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/boolq_seq2seq.py +117 -0
  469. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/c4.py +148 -0
  470. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cabbq.py +150 -0
  471. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cabreu.py +127 -0
  472. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/careqa.py +167 -0
  473. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench.py +153 -0
  474. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench_gen.py +119 -0
  475. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench_mc.py +113 -0
  476. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalanqa.py +169 -0
  477. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catcola.py +137 -0
  478. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cb.py +124 -0
  479. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ceval.py +221 -0
  480. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ceval_valid.py +161 -0
  481. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/chain.py +108 -0
  482. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/chartqa.py +236 -0
  483. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/claim.py +149 -0
  484. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/click.py +150 -0
  485. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cmmlu.py +164 -0
  486. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cnn.py +142 -0
  487. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cocoteros.py +146 -0
  488. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/code_x_glue.py +119 -0
  489. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coedit.py +147 -0
  490. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cola.py +83 -0
  491. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/commonsense.py +105 -0
  492. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/commonsense_qa.py +125 -0
  493. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/copa.py +124 -0
  494. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/copal_id.py +167 -0
  495. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coqa.py +161 -0
  496. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coqcat.py +114 -0
  497. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/crows_pairs.py +158 -0
  498. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/csatqa.py +150 -0
  499. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cycle.py +105 -0
  500. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cycle_letters.py +81 -0
  501. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darija_bench.py +219 -0
  502. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darijahellaswag.py +170 -0
  503. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darijammlu.py +150 -0
  504. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/dbpedia.py +155 -0
  505. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/discrim_eval.py +150 -0
  506. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/doc.py +105 -0
  507. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/drop.py +129 -0
  508. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/egyhellaswag.py +125 -0
  509. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/egymmlu.py +180 -0
  510. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/epec.py +140 -0
  511. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq.py +105 -0
  512. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench.py +192 -0
  513. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench_ca.py +150 -0
  514. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench_es.py +150 -0
  515. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/esbbq.py +150 -0
  516. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/escola.py +85 -0
  517. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ethics.py +133 -0
  518. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ethos.py +99 -0
  519. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus.py +105 -0
  520. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_exams.py +223 -0
  521. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_proficiency.py +157 -0
  522. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_reading.py +157 -0
  523. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_trivia.py +157 -0
  524. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/evalita_llm.py +164 -0
  525. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/evalita_sp.py +109 -0
  526. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/fda.py +105 -0
  527. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/financial.py +105 -0
  528. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/flan.py +112 -0
  529. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/fld.py +143 -0
  530. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench.py +200 -0
  531. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench_mc.py +98 -0
  532. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench_perplexity.py +86 -0
  533. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galcola.py +109 -0
  534. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench.py +153 -0
  535. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench_gen.py +118 -0
  536. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench_mc.py +112 -0
  537. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gaokao.py +141 -0
  538. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/glianorex.py +116 -0
  539. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/global_mmlu.py +169 -0
  540. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/global_piqa.py +150 -0
  541. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/glue.py +109 -0
  542. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gpqa.py +161 -0
  543. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gpt3.py +108 -0
  544. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/groundcocoa.py +182 -0
  545. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gsm.py +108 -0
  546. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gsm8k.py +134 -0
  547. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/haerae.py +150 -0
  548. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/headqa.py +112 -0
  549. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hellaswag.py +125 -0
  550. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hendrycks_ethics.py +221 -0
  551. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hendrycks_math.py +238 -0
  552. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/histoires_morales.py +177 -0
  553. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hle.py +111 -0
  554. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hrm8k.py +201 -0
  555. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/humaneval.py +124 -0
  556. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/humaneval_infilling.py +150 -0
  557. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/icelandic_winogrande.py +150 -0
  558. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ifeval.py +118 -0
  559. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/inverse.py +105 -0
  560. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/inverse_scaling.py +190 -0
  561. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/iwslt2017.py +117 -0
  562. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ja.py +105 -0
  563. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard.py +153 -0
  564. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard_gen.py +224 -0
  565. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard_mc.py +120 -0
  566. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/jsonschema_bench.py +123 -0
  567. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kbl.py +140 -0
  568. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu.py +166 -0
  569. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu_cot.py +88 -0
  570. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu_mc.py +107 -0
  571. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kobest.py +163 -0
  572. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kormedmcqa.py +148 -0
  573. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada.py +147 -0
  574. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_cloze.py +183 -0
  575. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_multilingual.py +183 -0
  576. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_multilingual_stablelm.py +141 -0
  577. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/law.py +105 -0
  578. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/leaderboard.py +192 -0
  579. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/libra.py +165 -0
  580. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lingoly.py +201 -0
  581. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/llama3.py +150 -0
  582. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lm_syneval.py +150 -0
  583. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logieval.py +82 -0
  584. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logiqa.py +115 -0
  585. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logiqa2.py +114 -0
  586. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/longbench.py +150 -0
  587. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/longbenchv2.py +150 -0
  588. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mastermind.py +201 -0
  589. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mathqa.py +137 -0
  590. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mbpp.py +164 -0
  591. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mc-taco.py +115 -0
  592. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/med_concepts_qa.py +222 -0
  593. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/meddialog.py +178 -0
  594. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medical.py +105 -0
  595. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mediqa_qa2019.py +123 -0
  596. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medmcqa.py +167 -0
  597. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medqa.py +118 -0
  598. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medtext.py +108 -0
  599. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mela.py +96 -0
  600. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/meqsum.py +115 -0
  601. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/metabench.py +152 -0
  602. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mgsm.py +122 -0
  603. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mimic_repsum.py +140 -0
  604. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/minerva_math.py +170 -0
  605. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mlqa.py +143 -0
  606. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu.py +142 -0
  607. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_cot.py +88 -0
  608. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_mc.py +107 -0
  609. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_pro.py +145 -0
  610. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlusr.py +188 -0
  611. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmmu.py +150 -0
  612. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mnli.py +113 -0
  613. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/model_written_evals.py +115 -0
  614. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/moral_stories.py +151 -0
  615. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mrpc.py +111 -0
  616. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mts_dialog.py +118 -0
  617. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mts_dialog_perplexity.py +97 -0
  618. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multiblimp.py +131 -0
  619. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multilingual.py +106 -0
  620. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multirc.py +114 -0
  621. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mutual.py +113 -0
  622. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/non.py +105 -0
  623. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval.py +171 -0
  624. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_exact.py +155 -0
  625. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_gen.py +277 -0
  626. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_gen_exact.py +163 -0
  627. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc.py +224 -0
  628. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc_log_likelihoods.py +219 -0
  629. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noticia.py +105 -0
  630. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/nq_open.py +135 -0
  631. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi.py +27 -0
  632. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_arc_multilingual.py +165 -0
  633. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_hellaswag_multilingual.py +172 -0
  634. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_mmlu_multilingual.py +160 -0
  635. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_truthfulqa_multilingual.py +206 -0
  636. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph.py +184 -0
  637. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph_perplexity.py +97 -0
  638. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/openbookqa.py +118 -0
  639. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/option.py +105 -0
  640. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paloma.py +205 -0
  641. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafraseja.py +108 -0
  642. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafrases.py +108 -0
  643. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws.py +105 -0
  644. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws_x.py +152 -0
  645. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pawsx.py +115 -0
  646. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/persona.py +244 -0
  647. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases.py +142 -0
  648. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases_ca_va.py +82 -0
  649. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile.py +159 -0
  650. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile_10k.py +140 -0
  651. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/piqa.py +116 -0
  652. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/polemo2.py +135 -0
  653. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench.py +153 -0
  654. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench_gen.py +121 -0
  655. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench_mc.py +103 -0
  656. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prompt.py +105 -0
  657. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prost.py +115 -0
  658. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pubmedqa.py +112 -0
  659. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qa4mre.py +119 -0
  660. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper.py +118 -0
  661. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper_bool.py +112 -0
  662. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnli.py +111 -0
  663. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnlieu.py +105 -0
  664. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qqp.py +111 -0
  665. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/quac.py +111 -0
  666. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/race.py +124 -0
  667. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/random.py +105 -0
  668. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/realtoxicityprompts.py +124 -0
  669. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/record.py +125 -0
  670. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/reversed.py +108 -0
  671. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/rte.py +111 -0
  672. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ruler.py +168 -0
  673. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sciq.py +113 -0
  674. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/score.py +175 -0
  675. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls.py +159 -0
  676. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls_mc.py +155 -0
  677. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/self.py +108 -0
  678. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue.py +129 -0
  679. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue_rte.py +120 -0
  680. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/simple_cooccurrence_bias.py +121 -0
  681. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/siqa.py +206 -0
  682. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/social_iqa.py +114 -0
  683. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench.py +153 -0
  684. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench_gen.py +117 -0
  685. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench_mc.py +110 -0
  686. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/squad2.py +129 -0
  687. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/squad_completion.py +121 -0
  688. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sst2.py +111 -0
  689. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/storycloze.py +246 -0
  690. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/summarization.py +105 -0
  691. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/super.py +105 -0
  692. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/super_glue.py +152 -0
  693. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/superglue.py +111 -0
  694. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/supergpqa.py +111 -0
  695. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/swag.py +115 -0
  696. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/swde.py +177 -0
  697. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sycophancy.py +115 -0
  698. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/t0.py +108 -0
  699. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/teca.py +108 -0
  700. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinyarc.py +108 -0
  701. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinybenchmarks.py +153 -0
  702. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinygsm8k.py +108 -0
  703. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinyhellaswag.py +108 -0
  704. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinymmlu.py +108 -0
  705. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinytruthfulqa.py +111 -0
  706. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinywinogrande.py +108 -0
  707. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tmmluplus.py +179 -0
  708. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/toxigen.py +91 -0
  709. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/translation.py +149 -0
  710. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/triviaqa.py +130 -0
  711. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa.py +110 -0
  712. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc1.py +125 -0
  713. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc2.py +138 -0
  714. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_multi.py +142 -0
  715. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turblimp_core.py +150 -0
  716. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu.py +159 -0
  717. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu_cot.py +104 -0
  718. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu_mc.py +100 -0
  719. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/twenty_newsgroups.py +111 -0
  720. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/unitxt.py +131 -0
  721. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/unscramble.py +153 -0
  722. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/vaxx.py +95 -0
  723. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/webqs.py +130 -0
  724. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wic.py +121 -0
  725. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wikitext.py +146 -0
  726. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/winogender.py +139 -0
  727. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/winogrande.py +118 -0
  728. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmdp.py +153 -0
  729. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmt14.py +110 -0
  730. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmt16.py +118 -0
  731. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wnli.py +114 -0
  732. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wsc.py +117 -0
  733. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wsc273.py +178 -0
  734. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xcopa.py +195 -0
  735. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xlsum.py +145 -0
  736. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xnli.py +131 -0
  737. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xquad.py +201 -0
  738. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xstorycloze.py +128 -0
  739. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xwinograd.py +124 -0
  740. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/yahoo.py +108 -0
  741. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/zhoblimp.py +153 -0
  742. wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_pairs_generation.py +223 -0
  743. wisent-0.7.925/wisent/core/data_loaders/__init__.py +235 -0
  744. wisent-0.7.925/wisent/core/data_loaders/core/__init__.py +0 -0
  745. wisent-0.7.925/wisent/core/data_loaders/core/atoms.py +99 -0
  746. wisent-0.7.925/wisent/core/data_loaders/loaders/__init__.py +0 -0
  747. wisent-0.7.925/wisent/core/data_loaders/loaders/custom.py +120 -0
  748. wisent-0.7.925/wisent/core/data_loaders/loaders/huggingface_loader.py +153 -0
  749. wisent-0.7.925/wisent/core/data_loaders/loaders/lm_loader.py +505 -0
  750. wisent-0.7.925/wisent/core/data_loaders/loaders/lm_loader_special_cases.py +496 -0
  751. wisent-0.7.925/wisent/core/data_loaders/loaders/task_interface_loader.py +300 -0
  752. wisent-0.7.925/wisent/core/data_loaders/rotator.py +118 -0
  753. wisent-0.7.925/wisent/core/detection_handling.py +259 -0
  754. wisent-0.7.925/wisent/core/diversity_processors.py +193 -0
  755. wisent-0.7.925/wisent/core/download_full_benchmarks.py +1512 -0
  756. wisent-0.7.925/wisent/core/errors/__init__.py +203 -0
  757. wisent-0.7.925/wisent/core/errors/error_codes.py +763 -0
  758. wisent-0.7.925/wisent/core/errors/error_handler.py +134 -0
  759. wisent-0.7.925/wisent/core/evaluators/__init__.py +0 -0
  760. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/__init__.py +42 -0
  761. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/aime_evaluator.py +90 -0
  762. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/apps_evaluator.py +133 -0
  763. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/__init__.py +0 -0
  764. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/metrics/__init__.py +0 -0
  765. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/metrics/core/__init__.py +0 -0
  766. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/metrics/core/atoms.py +36 -0
  767. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/metrics/evaluator.py +368 -0
  768. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/metrics/passk.py +67 -0
  769. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/__init__.py +0 -0
  770. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/core/__init__.py +0 -0
  771. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/core/atoms.py +27 -0
  772. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/cpp_sanitizer.py +62 -0
  773. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/java_sanitizer.py +78 -0
  774. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/python_sanitizer.py +94 -0
  775. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/utils.py +126 -0
  776. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/providers/__init__.py +18 -0
  777. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/providers/core/__init__.py +0 -0
  778. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/providers/core/atoms.py +31 -0
  779. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/providers/livecodebench/__init__.py +3 -0
  780. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/providers/livecodebench/provider.py +305 -0
  781. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/Dockerfile +31 -0
  782. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/__init__.py +0 -0
  783. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/__init__.py +0 -0
  784. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/atoms.py +105 -0
  785. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/runtime.py +143 -0
  786. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/entrypoint.py +121 -0
  787. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/recipes.py +60 -0
  788. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/solution_generator.py +258 -0
  789. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/conala_evaluator.py +195 -0
  790. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/exact_match_evaluator.py +81 -0
  791. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/f1_evaluator.py +173 -0
  792. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/generation_evaluator.py +488 -0
  793. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/livemathbench_evaluator.py +393 -0
  794. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/log_likelihoods_evaluator.py +202 -0
  795. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_evaluator.py +119 -0
  796. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_parsing/__init__.py +1 -0
  797. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_parsing/core.py +1640 -0
  798. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_parsing/extract_boxed.py +48 -0
  799. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_parsing/is_equiv.py +159 -0
  800. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_parsing/scripts.py +919 -0
  801. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/perplexity_evaluator.py +175 -0
  802. wisent-0.7.925/wisent/core/evaluators/benchmark_specific/polymath_evaluator.py +114 -0
  803. wisent-0.7.925/wisent/core/evaluators/core/__init__.py +5 -0
  804. wisent-0.7.925/wisent/core/evaluators/core/atoms.py +166 -0
  805. wisent-0.7.925/wisent/core/evaluators/custom/__init__.py +20 -0
  806. wisent-0.7.925/wisent/core/evaluators/custom/custom_evaluator.py +382 -0
  807. wisent-0.7.925/wisent/core/evaluators/custom/examples/__init__.py +37 -0
  808. wisent-0.7.925/wisent/core/evaluators/custom/examples/desklib_detector.py +166 -0
  809. wisent-0.7.925/wisent/core/evaluators/custom/examples/gptzero.py +185 -0
  810. wisent-0.7.925/wisent/core/evaluators/custom/examples/humanization.py +79 -0
  811. wisent-0.7.925/wisent/core/evaluators/custom/examples/humanization_coherent.py +181 -0
  812. wisent-0.7.925/wisent/core/evaluators/custom/examples/roberta_detector.py +173 -0
  813. wisent-0.7.925/wisent/core/evaluators/oracles/__init__.py +0 -0
  814. wisent-0.7.925/wisent/core/evaluators/oracles/interactive.py +73 -0
  815. wisent-0.7.925/wisent/core/evaluators/oracles/nlp_evaluator.py +440 -0
  816. wisent-0.7.925/wisent/core/evaluators/oracles/truthfulqa_gen_evaluator.py +150 -0
  817. wisent-0.7.925/wisent/core/evaluators/oracles/user_specified.py +67 -0
  818. wisent-0.7.925/wisent/core/evaluators/personalization/__init__.py +12 -0
  819. wisent-0.7.925/wisent/core/evaluators/personalization/alignment.py +166 -0
  820. wisent-0.7.925/wisent/core/evaluators/personalization/coherence.py +371 -0
  821. wisent-0.7.925/wisent/core/evaluators/personalization/difference.py +73 -0
  822. wisent-0.7.925/wisent/core/evaluators/rotator.py +217 -0
  823. wisent-0.7.925/wisent/core/evaluators/steering_evaluators.py +386 -0
  824. wisent-0.7.925/wisent/core/evaluators/synthetic_evaluator.py +377 -0
  825. wisent-0.7.925/wisent/core/geometry_runner.py +995 -0
  826. wisent-0.7.925/wisent/core/geometry_search_space.py +237 -0
  827. wisent-0.7.925/wisent/core/hyperparameter_optimizer.py +547 -0
  828. wisent-0.7.925/wisent/core/layer.py +17 -0
  829. wisent-0.7.925/wisent/core/lm_eval_harness_ground_truth.py +1427 -0
  830. wisent-0.7.925/wisent/core/main.py +107 -0
  831. wisent-0.7.925/wisent/core/managed_cached_benchmarks.py +609 -0
  832. wisent-0.7.925/wisent/core/mixed_benchmark_sampler.py +366 -0
  833. wisent-0.7.925/wisent/core/modalities/__init__.py +545 -0
  834. wisent-0.7.925/wisent/core/model_persistence.py +302 -0
  835. wisent-0.7.925/wisent/core/models/__init__.py +23 -0
  836. wisent-0.7.925/wisent/core/models/core/__init__.py +0 -0
  837. wisent-0.7.925/wisent/core/models/core/atoms.py +467 -0
  838. wisent-0.7.925/wisent/core/models/inference_config.py +127 -0
  839. wisent-0.7.925/wisent/core/models/wisent_model.py +894 -0
  840. wisent-0.7.925/wisent/core/multi_steering.py +397 -0
  841. wisent-0.7.925/wisent/core/opti/__init__.py +0 -0
  842. wisent-0.7.925/wisent/core/opti/core/__init__.py +0 -0
  843. wisent-0.7.925/wisent/core/opti/core/atoms.py +177 -0
  844. wisent-0.7.925/wisent/core/opti/methods/__init__.py +10 -0
  845. wisent-0.7.925/wisent/core/opti/methods/opti_classificator.py +172 -0
  846. wisent-0.7.925/wisent/core/opti/methods/opti_steering.py +139 -0
  847. wisent-0.7.925/wisent/core/opti/methods/opti_weights.py +550 -0
  848. wisent-0.7.925/wisent/core/optuna/__init__.py +54 -0
  849. wisent-0.7.925/wisent/core/optuna/classifier/__init__.py +25 -0
  850. wisent-0.7.925/wisent/core/optuna/classifier/activation_generator.py +353 -0
  851. wisent-0.7.925/wisent/core/optuna/classifier/classifier_cache.py +509 -0
  852. wisent-0.7.925/wisent/core/optuna/classifier/optuna_classifier_optimizer.py +685 -0
  853. wisent-0.7.925/wisent/core/optuna/steering/__init__.py +20 -0
  854. wisent-0.7.925/wisent/core/optuna/steering/bigcode_evaluator_wrapper.py +200 -0
  855. wisent-0.7.925/wisent/core/optuna/steering/data_utils.py +342 -0
  856. wisent-0.7.925/wisent/core/optuna/steering/metrics.py +412 -0
  857. wisent-0.7.925/wisent/core/optuna/steering/steering_optimization.py +1101 -0
  858. wisent-0.7.925/wisent/core/parser.py +1662 -0
  859. wisent-0.7.925/wisent/core/parser_arguments/__init__.py +10 -0
  860. wisent-0.7.925/wisent/core/parser_arguments/agent_parser.py +122 -0
  861. wisent-0.7.925/wisent/core/parser_arguments/check_linearity_parser.py +92 -0
  862. wisent-0.7.925/wisent/core/parser_arguments/cluster_benchmarks_parser.py +31 -0
  863. wisent-0.7.925/wisent/core/parser_arguments/configure_model_parser.py +7 -0
  864. wisent-0.7.925/wisent/core/parser_arguments/create_steering_vector_parser.py +67 -0
  865. wisent-0.7.925/wisent/core/parser_arguments/diagnose_pairs_parser.py +25 -0
  866. wisent-0.7.925/wisent/core/parser_arguments/diagnose_vectors_parser.py +72 -0
  867. wisent-0.7.925/wisent/core/parser_arguments/evaluate_parser.py +40 -0
  868. wisent-0.7.925/wisent/core/parser_arguments/evaluate_refusal_parser.py +32 -0
  869. wisent-0.7.925/wisent/core/parser_arguments/evaluate_responses_parser.py +12 -0
  870. wisent-0.7.925/wisent/core/parser_arguments/full_optimize_parser.py +194 -0
  871. wisent-0.7.925/wisent/core/parser_arguments/generate_pairs_from_task_parser.py +33 -0
  872. wisent-0.7.925/wisent/core/parser_arguments/generate_pairs_parser.py +43 -0
  873. wisent-0.7.925/wisent/core/parser_arguments/generate_responses_parser.py +16 -0
  874. wisent-0.7.925/wisent/core/parser_arguments/generate_vector_from_synthetic_parser.py +148 -0
  875. wisent-0.7.925/wisent/core/parser_arguments/generate_vector_from_task_parser.py +162 -0
  876. wisent-0.7.925/wisent/core/parser_arguments/generate_vector_parser.py +89 -0
  877. wisent-0.7.925/wisent/core/parser_arguments/geometry_search_parser.py +61 -0
  878. wisent-0.7.925/wisent/core/parser_arguments/get_activations_parser.py +81 -0
  879. wisent-0.7.925/wisent/core/parser_arguments/inference_config_parser.py +65 -0
  880. wisent-0.7.925/wisent/core/parser_arguments/main_parser.py +236 -0
  881. wisent-0.7.925/wisent/core/parser_arguments/model_config_parser.py +59 -0
  882. wisent-0.7.925/wisent/core/parser_arguments/modify_weights_parser.py +309 -0
  883. wisent-0.7.925/wisent/core/parser_arguments/monitor_parser.py +17 -0
  884. wisent-0.7.925/wisent/core/parser_arguments/multi_steer_parser.py +48 -0
  885. wisent-0.7.925/wisent/core/parser_arguments/nonsense_parser.py +26 -0
  886. wisent-0.7.925/wisent/core/parser_arguments/optimization_cache_parser.py +64 -0
  887. wisent-0.7.925/wisent/core/parser_arguments/optimize_classification_parser.py +108 -0
  888. wisent-0.7.925/wisent/core/parser_arguments/optimize_parser.py +142 -0
  889. wisent-0.7.925/wisent/core/parser_arguments/optimize_sample_size_parser.py +58 -0
  890. wisent-0.7.925/wisent/core/parser_arguments/optimize_steering_parser.py +724 -0
  891. wisent-0.7.925/wisent/core/parser_arguments/optimize_weights_parser.py +409 -0
  892. wisent-0.7.925/wisent/core/parser_arguments/synthetic_parser.py +117 -0
  893. wisent-0.7.925/wisent/core/parser_arguments/tasks_parser.py +579 -0
  894. wisent-0.7.925/wisent/core/parser_arguments/train_unified_goodness_parser.py +172 -0
  895. wisent-0.7.925/wisent/core/parser_arguments/utils.py +107 -0
  896. wisent-0.7.925/wisent/core/prompts/__init__.py +0 -0
  897. wisent-0.7.925/wisent/core/prompts/core/__init__.py +0 -0
  898. wisent-0.7.925/wisent/core/prompts/core/atom.py +57 -0
  899. wisent-0.7.925/wisent/core/prompts/core/prompt_formater.py +148 -0
  900. wisent-0.7.925/wisent/core/prompts/prompt_stratiegies/__init__.py +0 -0
  901. wisent-0.7.925/wisent/core/prompts/prompt_stratiegies/direct_completion.py +26 -0
  902. wisent-0.7.925/wisent/core/prompts/prompt_stratiegies/instruction_following.py +26 -0
  903. wisent-0.7.925/wisent/core/prompts/prompt_stratiegies/multiple_choice.py +31 -0
  904. wisent-0.7.925/wisent/core/prompts/prompt_stratiegies/role_playing.py +33 -0
  905. wisent-0.7.925/wisent/core/representation.py +5 -0
  906. wisent-0.7.925/wisent/core/save_results.py +277 -0
  907. wisent-0.7.925/wisent/core/steering.py +662 -0
  908. wisent-0.7.925/wisent/core/steering_method.py +20 -0
  909. wisent-0.7.925/wisent/core/steering_methods/__init__.py +54 -0
  910. wisent-0.7.925/wisent/core/steering_methods/core/__init__.py +0 -0
  911. wisent-0.7.925/wisent/core/steering_methods/core/atoms.py +153 -0
  912. wisent-0.7.925/wisent/core/steering_methods/methods/__init__.py +0 -0
  913. wisent-0.7.925/wisent/core/steering_methods/methods/caa.py +45 -0
  914. wisent-0.7.925/wisent/core/steering_methods/methods/hyperplane.py +75 -0
  915. wisent-0.7.925/wisent/core/steering_methods/methods/prism.py +587 -0
  916. wisent-0.7.925/wisent/core/steering_methods/methods/pulse.py +672 -0
  917. wisent-0.7.925/wisent/core/steering_methods/methods/titan.py +1050 -0
  918. wisent-0.7.925/wisent/core/steering_methods/preflight.py +322 -0
  919. wisent-0.7.925/wisent/core/steering_methods/registry.py +689 -0
  920. wisent-0.7.925/wisent/core/steering_methods/rotator.py +121 -0
  921. wisent-0.7.925/wisent/core/steering_optimizer.py +1503 -0
  922. wisent-0.7.925/wisent/core/synthetic/__init__.py +0 -0
  923. wisent-0.7.925/wisent/core/synthetic/cleaners/__init__.py +0 -0
  924. wisent-0.7.925/wisent/core/synthetic/cleaners/core/__init__.py +0 -0
  925. wisent-0.7.925/wisent/core/synthetic/cleaners/core/atoms.py +58 -0
  926. wisent-0.7.925/wisent/core/synthetic/cleaners/deduper_cleaner.py +53 -0
  927. wisent-0.7.925/wisent/core/synthetic/cleaners/methods/__init__.py +0 -0
  928. wisent-0.7.925/wisent/core/synthetic/cleaners/methods/base_dedupers.py +321 -0
  929. wisent-0.7.925/wisent/core/synthetic/cleaners/methods/base_refusalers.py +286 -0
  930. wisent-0.7.925/wisent/core/synthetic/cleaners/methods/core/__init__.py +0 -0
  931. wisent-0.7.925/wisent/core/synthetic/cleaners/methods/core/atoms.py +47 -0
  932. wisent-0.7.925/wisent/core/synthetic/cleaners/pairs_cleaner.py +90 -0
  933. wisent-0.7.925/wisent/core/synthetic/cleaners/refusaler_cleaner.py +133 -0
  934. wisent-0.7.925/wisent/core/synthetic/db_instructions/__init__.py +0 -0
  935. wisent-0.7.925/wisent/core/synthetic/db_instructions/core/__init__.py +0 -0
  936. wisent-0.7.925/wisent/core/synthetic/db_instructions/core/atoms.py +25 -0
  937. wisent-0.7.925/wisent/core/synthetic/db_instructions/mini_dp.py +115 -0
  938. wisent-0.7.925/wisent/core/synthetic/generators/__init__.py +0 -0
  939. wisent-0.7.925/wisent/core/synthetic/generators/core/__init__.py +0 -0
  940. wisent-0.7.925/wisent/core/synthetic/generators/core/atoms.py +73 -0
  941. wisent-0.7.925/wisent/core/synthetic/generators/diversities/__init__.py +0 -0
  942. wisent-0.7.925/wisent/core/synthetic/generators/diversities/core/__init__.py +0 -0
  943. wisent-0.7.925/wisent/core/synthetic/generators/diversities/core/core.py +68 -0
  944. wisent-0.7.925/wisent/core/synthetic/generators/diversities/methods/__init__.py +0 -0
  945. wisent-0.7.925/wisent/core/synthetic/generators/diversities/methods/fast_diversity.py +249 -0
  946. wisent-0.7.925/wisent/core/synthetic/generators/nonsense_generator.py +162 -0
  947. wisent-0.7.925/wisent/core/synthetic/generators/pairs_generator.py +313 -0
  948. wisent-0.7.925/wisent/core/task_interface.py +143 -0
  949. wisent-0.7.925/wisent/core/task_selector.py +232 -0
  950. wisent-0.7.925/wisent/core/tasks/__init__.py +218 -0
  951. wisent-0.7.925/wisent/core/tasks/aime_task.py +142 -0
  952. wisent-0.7.925/wisent/core/tasks/file_task.py +212 -0
  953. wisent-0.7.925/wisent/core/tasks/hle_task.py +180 -0
  954. wisent-0.7.925/wisent/core/tasks/hmmt_task.py +120 -0
  955. wisent-0.7.925/wisent/core/tasks/livecodebench_task.py +94 -0
  956. wisent-0.7.925/wisent/core/tasks/livemathbench_task.py +159 -0
  957. wisent-0.7.925/wisent/core/tasks/lm_eval_task.py +611 -0
  958. wisent-0.7.925/wisent/core/tasks/math500_task.py +84 -0
  959. wisent-0.7.925/wisent/core/tasks/polymath_task.py +147 -0
  960. wisent-0.7.925/wisent/core/tasks/supergpqa_task.py +220 -0
  961. wisent-0.7.925/wisent/core/time_estimator.py +155 -0
  962. wisent-0.7.925/wisent/core/timing_calibration.py +176 -0
  963. wisent-0.7.925/wisent/core/tracking/__init__.py +54 -0
  964. wisent-0.7.925/wisent/core/tracking/latency.py +620 -0
  965. wisent-0.7.925/wisent/core/tracking/memory.py +360 -0
  966. wisent-0.7.925/wisent/core/trainers/__init__.py +0 -0
  967. wisent-0.7.925/wisent/core/trainers/core/__init__.py +11 -0
  968. wisent-0.7.925/wisent/core/trainers/core/atoms.py +45 -0
  969. wisent-0.7.925/wisent/core/trainers/steering_trainer.py +356 -0
  970. wisent-0.7.925/wisent/core/universal_subspace.py +918 -0
  971. wisent-0.7.925/wisent/core/user_model_config.py +158 -0
  972. wisent-0.7.925/wisent/core/utils/__init__.py +64 -0
  973. wisent-0.7.925/wisent/core/utils/base_rotator.py +292 -0
  974. wisent-0.7.925/wisent/core/utils/dataset_splits.py +197 -0
  975. wisent-0.7.925/wisent/core/utils/device.py +279 -0
  976. wisent-0.7.925/wisent/core/utils/layer_combinations.py +70 -0
  977. wisent-0.7.925/wisent/core/weight_modification/__init__.py +134 -0
  978. wisent-0.7.925/wisent/core/weight_modification/additive.py +340 -0
  979. wisent-0.7.925/wisent/core/weight_modification/directional.py +1357 -0
  980. wisent-0.7.925/wisent/core/weight_modification/export.py +359 -0
  981. wisent-0.7.925/wisent/core/weight_modification/multi_direction.py +410 -0
  982. wisent-0.7.925/wisent/core/weight_modification/utils.py +236 -0
  983. wisent-0.7.925/wisent/core/wisent.py +660 -0
  984. wisent-0.7.925/wisent/examples/__init__.py +1 -0
  985. wisent-0.7.925/wisent/examples/scripts/__init__.py +1 -0
  986. wisent-0.7.925/wisent/examples/scripts/benchmark_tags.json +2140 -0
  987. wisent-0.7.925/wisent/examples/scripts/count_all_benchmarks.py +121 -0
  988. wisent-0.7.925/wisent/examples/scripts/discover_directions.py +469 -0
  989. wisent-0.7.925/wisent/examples/scripts/extract_benchmark_info.py +71 -0
  990. wisent-0.7.925/wisent/examples/scripts/lm_eval_readme.json +4 -0
  991. wisent-0.7.925/wisent/examples/scripts/search_all_short_names.py +31 -0
  992. wisent-0.7.925/wisent/examples/scripts/test_all_benchmarks.py +138 -0
  993. wisent-0.7.925/wisent/examples/scripts/test_all_benchmarks_new.py +28 -0
  994. wisent-0.7.925/wisent/examples/scripts/test_contrastive_pairs_all_supported.py +230 -0
  995. wisent-0.7.925/wisent/examples/scripts/test_nonsense_baseline.py +261 -0
  996. wisent-0.7.925/wisent/examples/scripts/test_one_benchmark.py +324 -0
  997. wisent-0.7.925/wisent/examples/scripts/test_one_coding_benchmark.py +293 -0
  998. wisent-0.7.925/wisent/parameters/__init__.py +1 -0
  999. wisent-0.7.925/wisent/parameters/lm_eval/all_lm_eval_task_families.json +169 -0
  1000. wisent-0.7.925/wisent/parameters/lm_eval/broken_in_lm_eval.json +187 -0
  1001. wisent-0.7.925/wisent/parameters/lm_eval/category_directions.json +137 -0
  1002. wisent-0.7.925/wisent/parameters/lm_eval/evaluations_not_lm_eval_tasks.json +0 -0
  1003. wisent-0.7.925/wisent/parameters/lm_eval/evaluator_check.json +3476 -0
  1004. wisent-0.7.925/wisent/parameters/lm_eval/final_verification.json +24782 -0
  1005. wisent-0.7.925/wisent/parameters/lm_eval/group_task_evaluators.json +1833 -0
  1006. wisent-0.7.925/wisent/parameters/lm_eval/group_tasks.json +150 -0
  1007. wisent-0.7.925/wisent/parameters/lm_eval/individual_tasks.json +402 -0
  1008. wisent-0.7.925/wisent/parameters/lm_eval/no_readmes.json +1 -0
  1009. wisent-0.7.925/wisent/parameters/lm_eval/not_lm_eval_tasks.json +110 -0
  1010. wisent-0.7.925/wisent/parameters/lm_eval/read_tasks.json +208 -0
  1011. wisent-0.7.925/wisent/parameters/lm_eval/readme_files.json +208 -0
  1012. wisent-0.7.925/wisent/parameters/lm_eval/repair_plan.json +282 -0
  1013. wisent-0.7.925/wisent/parameters/lm_eval/track_progress_not_lm_eval_tasks.json +77 -0
  1014. wisent-0.7.925/wisent/parameters/lm_eval/weak_contrastive_pairs.json +38 -0
  1015. wisent-0.7.925/wisent/parameters/lm_eval/working_benchmarks.json +206 -0
  1016. wisent-0.7.925/wisent/parameters/lm_eval/working_benchmarks_categorized.json +236 -0
  1017. wisent-0.7.925/wisent/parameters/tasks/missing_task_families.json +2963 -0
  1018. wisent-0.7.925/wisent/parameters/tasks/remaining_tasks_to_implement.json +199 -0
  1019. wisent-0.7.925/wisent/parameters/tasks/risks.json +10 -0
  1020. wisent-0.7.925/wisent/parameters/tasks/skills.json +14 -0
  1021. wisent-0.7.925/wisent/parameters/tasks/tasks.json +56031 -0
  1022. wisent-0.7.925/wisent/scripts/run_quality_metrics_sweep.sh +310 -0
  1023. wisent-0.7.925/wisent/tests/__init__.py +0 -0
  1024. wisent-0.7.925/wisent/tests/examples/__init__.py +0 -0
  1025. wisent-0.7.925/wisent/tests/examples/cli/__init__.py +0 -0
  1026. wisent-0.7.925/wisent/tests/examples/cli/activations/__init__.py +0 -0
  1027. wisent-0.7.925/wisent/tests/examples/cli/activations/test_get_activations.py +127 -0
  1028. wisent-0.7.925/wisent/tests/examples/cli/classifier/__init__.py +0 -0
  1029. wisent-0.7.925/wisent/tests/examples/cli/classifier/test_classifier_examples.py +141 -0
  1030. wisent-0.7.925/wisent/tests/examples/cli/contrastive_pairs/__init__.py +0 -0
  1031. wisent-0.7.925/wisent/tests/examples/cli/contrastive_pairs/test_generate_pairs.py +89 -0
  1032. wisent-0.7.925/wisent/tests/examples/cli/evaluation/__init__.py +0 -0
  1033. wisent-0.7.925/wisent/tests/examples/cli/evaluation/test_evaluation_examples.py +117 -0
  1034. wisent-0.7.925/wisent/tests/examples/cli/generate/__init__.py +0 -0
  1035. wisent-0.7.925/wisent/tests/examples/cli/generate/test_generate_with_classifier.py +146 -0
  1036. wisent-0.7.925/wisent/tests/examples/cli/generate/test_generate_with_steering.py +149 -0
  1037. wisent-0.7.925/wisent/tests/examples/cli/generate/test_only_generate.py +110 -0
  1038. wisent-0.7.925/wisent/tests/examples/cli/multi_steering/__init__.py +0 -0
  1039. wisent-0.7.925/wisent/tests/examples/cli/multi_steering/test_multi_steer_from_trained_vectors.py +210 -0
  1040. wisent-0.7.925/wisent/tests/examples/cli/multi_steering/test_multi_steer_with_different_parameters.py +205 -0
  1041. wisent-0.7.925/wisent/tests/examples/cli/multi_steering/test_train_and_multi_steer.py +174 -0
  1042. wisent-0.7.925/wisent/tests/examples/cli/optimizer/__init__.py +0 -0
  1043. wisent-0.7.925/wisent/tests/examples/cli/optimizer/test_optimize_sample_size.py +102 -0
  1044. wisent-0.7.925/wisent/tests/examples/cli/optimizer/test_optimizer_examples.py +59 -0
  1045. wisent-0.7.925/wisent/tests/examples/cli/steering/__init__.py +0 -0
  1046. wisent-0.7.925/wisent/tests/examples/cli/steering/test_create_steering_vectors.py +135 -0
  1047. wisent-0.7.925/wisent/tests/examples/cli/synthetic/__init__.py +0 -0
  1048. wisent-0.7.925/wisent/tests/examples/cli/synthetic/test_synthetic_pairs.py +45 -0
  1049. wisent-0.7.925/wisent/tests/nosense/__init__.py +6 -0
  1050. wisent-0.7.925/wisent/tests/nosense/base_nosense.py +81 -0
  1051. wisent-0.7.925/wisent/tests/nosense/math500_nosense.py +72 -0
  1052. wisent-0.7.925/wisent/tests/nosense/test_robustness.py +336 -0
  1053. wisent-0.7.925/wisent/tests/test_aggregation_geometry.py +236 -0
  1054. wisent-0.7.925/wisent/tests/test_all_cli_commands.py +674 -0
  1055. wisent-0.7.925/wisent/tests/test_detector_accuracy.py +163 -0
  1056. wisent-0.7.925/wisent/tests/test_geometry_comprehensive.py +327 -0
  1057. wisent-0.7.925/wisent/tests/test_geometry_exhaustive.py +1202 -0
  1058. wisent-0.7.925/wisent/tests/test_titan_geometry.py +257 -0
  1059. wisent-0.7.925/wisent/tests/visualize_geometry.py +342 -0
  1060. wisent-0.7.925/wisent.egg-info/PKG-INFO +68 -0
  1061. wisent-0.7.925/wisent.egg-info/SOURCES.txt +1063 -0
  1062. wisent-0.7.925/wisent.egg-info/dependency_links.txt +1 -0
  1063. wisent-0.7.925/wisent.egg-info/entry_points.txt +2 -0
  1064. wisent-0.7.925/wisent.egg-info/requires.txt +23 -0
  1065. wisent-0.7.925/wisent.egg-info/top_level.txt +1 -0
wisent-0.7.925/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2023-2024 Wisent Team
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,68 @@
1
+ Metadata-Version: 2.4
2
+ Name: wisent
3
+ Version: 0.7.925
4
+ Summary: Monitor and influence AI Brains
5
+ Home-page: https://github.com/wisent-ai/wisent
6
+ Author: Lukasz Bartoszcze and the Wisent Team
7
+ Author-email: lukasz.bartoszcze@wisent.ai
8
+ Keywords: nlp,machine learning,language models,safety,guardrails,lm-evaluation-harness
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
15
+ Requires-Python: >=3.8
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+ Requires-Dist: torch>=1.9.0
19
+ Requires-Dist: transformers>=4.20.0
20
+ Requires-Dist: tqdm>=4.50.0
21
+ Requires-Dist: scikit-learn>=0.24.0
22
+ Requires-Dist: pandas>=1.2.0
23
+ Requires-Dist: numpy>=1.21.0
24
+ Requires-Dist: datasets>=2.0.0
25
+ Requires-Dist: sentence-transformers>=2.0.0
26
+ Requires-Dist: faiss-cpu>=1.7.0
27
+ Requires-Dist: uncensorbench>=0.2.0
28
+ Requires-Dist: pebble>=5.0.0
29
+ Requires-Dist: latex2sympy2_extended>=1.0.0
30
+ Requires-Dist: sae_lens>=0.1.0
31
+ Requires-Dist: trl>=0.7.0
32
+ Provides-Extra: harness
33
+ Requires-Dist: lm-eval==0.4.8; extra == "harness"
34
+ Provides-Extra: cuda
35
+ Requires-Dist: flash-attn>=2.5.0; extra == "cuda"
36
+ Provides-Extra: sparsify
37
+ Requires-Dist: sparsify>=0.1.0; extra == "sparsify"
38
+ Dynamic: author
39
+ Dynamic: author-email
40
+ Dynamic: classifier
41
+ Dynamic: description
42
+ Dynamic: description-content-type
43
+ Dynamic: home-page
44
+ Dynamic: keywords
45
+ Dynamic: license-file
46
+ Dynamic: provides-extra
47
+ Dynamic: requires-dist
48
+ Dynamic: requires-python
49
+ Dynamic: summary
50
+
51
+ <p align="center">
52
+ <img src="banner.png" alt="Wisent Banner" width="100%">
53
+ </p>
54
+
55
+ <p align="center">
56
+ <code>pip install wisent</code>
57
+ </p>
58
+
59
+
60
+
61
+ ## Overview
62
+
63
+ Wisent allows you to control your AI by identifying brain patterns corresponding to responses you don't like, like hallucinations or harmful outputs. We use contrastive pairs of representations to detect when a model might be generating harmful content or hallucinating. Learn more at [wisent.ai/documentation](https://www.wisent.ai/documentation).
64
+
65
+
66
+ ## License
67
+
68
+ This project is licensed under the MIT License - see the LICENSE file for details.
@@ -0,0 +1,18 @@
1
+ <p align="center">
2
+ <img src="banner.png" alt="Wisent Banner" width="100%">
3
+ </p>
4
+
5
+ <p align="center">
6
+ <code>pip install wisent</code>
7
+ </p>
8
+
9
+
10
+
11
+ ## Overview
12
+
13
+ Wisent allows you to control your AI by identifying brain patterns corresponding to responses you don't like, like hallucinations or harmful outputs. We use contrastive pairs of representations to detect when a model might be generating harmful content or hallucinating. Learn more at [wisent.ai/documentation](https://www.wisent.ai/documentation).
14
+
15
+
16
+ ## License
17
+
18
+ This project is licensed under the MIT License - see the LICENSE file for details.
@@ -0,0 +1,85 @@
1
+ [tool.pytest.ini_options]
2
+ testpaths = ["wisent/tests"]
3
+ python_files = ["test_*.py"]
4
+ python_classes = ["Test*"]
5
+ python_functions = ["test_*"]
6
+ addopts = ["-v", "--tb=short"]
7
+ markers = [
8
+ "slow: marks tests as slow (deselect with '-m \"not slow\"')",
9
+ "heavy: marks tests as very resource-intensive with large model downloads (deselect with '-m \"not heavy\"')",
10
+ "model_download: marks tests that download models from HuggingFace (deselect with '-m \"not model_download\"')",
11
+ "integration: marks tests as integration tests",
12
+ "unit: marks tests as unit tests",
13
+ "docker: marks tests for Docker-based execution",
14
+ "performance: marks tests for performance evaluation",
15
+ "cli: marks tests for CLI integration",
16
+ "sandbox_required: marks tests that require sandbox environment (includes coding tasks and trust_remote_code)",
17
+ "bigcode_required: marks tests that require bigcode-evaluation-harness installation",
18
+ ]
19
+ filterwarnings = [
20
+ "ignore::DeprecationWarning",
21
+ "ignore::PendingDeprecationWarning",
22
+ ]
23
+
24
+ [tool.ruff]
25
+ line-length = 120
26
+ target-version = "py39"
27
+ exclude = [
28
+ ".bzr",
29
+ ".direnv",
30
+ ".eggs",
31
+ ".git",
32
+ ".git-rewrite",
33
+ ".hg",
34
+ ".mypy_cache",
35
+ ".nox",
36
+ ".pants.d",
37
+ ".pytype",
38
+ ".ruff_cache",
39
+ ".svn",
40
+ ".tox",
41
+ ".venv",
42
+ "__pypackages__",
43
+ "_build",
44
+ "buck-out",
45
+ "build",
46
+ "dist",
47
+ "node_modules",
48
+ "venv",
49
+ "evaluation",
50
+ "bigcode-evaluation-harness",
51
+ ]
52
+
53
+ [tool.ruff.lint]
54
+ # Enable common rule sets
55
+ select = ["E4", "E7", "E9", "F", "I", "N", "UP", "B", "A", "C4", "ISC", "ICN", "PIE", "T20", "PT", "Q", "RSE", "RET", "SLF", "SIM", "TCH", "ARG", "ERA", "PGH", "RUF"]
56
+ ignore = [
57
+ "PLR0913", # Too many arguments in function definition
58
+ "PLR0912", # Too many branches
59
+ "PLR0915", # Too many statements
60
+ "C901", # Function is too complex
61
+ "PLR2004", # Magic value used in comparison
62
+ "TRY003", # Avoid specifying long messages outside exception class
63
+ "PLW2901", # Redefined loop variable
64
+ "B008", # Do not perform function calls in argument defaults
65
+ "ARG002", # Unused method argument
66
+ "PIE800", # Unnecessary spread operator,
67
+ "T201" # print
68
+ ]
69
+ fixable = ["ALL"]
70
+ unfixable = []
71
+
72
+ [tool.ruff.format]
73
+ # Like Black, use double quotes for strings.
74
+ quote-style = "double"
75
+ # Like Black, indent with spaces, rather than tabs.
76
+ indent-style = "space"
77
+ # Like Black, respect magic trailing commas.
78
+ skip-magic-trailing-comma = false
79
+ # Like Black, automatically detect the appropriate line ending.
80
+ line-ending = "auto"
81
+
82
+ [tool.ruff.lint.isort]
83
+ known-first-party = ["wisent"]
84
+ force-single-line = false
85
+ combine-as-imports = true
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,78 @@
1
+ import os
2
+ from setuptools import setup, find_packages
3
+
4
+ # Read the contents of the README file
5
+ with open("README.md", encoding="utf-8") as f:
6
+ long_description = f.read()
7
+
8
+ # Read version from __init__.py
9
+ with open(os.path.join("wisent", "__init__.py"), encoding="utf-8") as f:
10
+ for line in f:
11
+ if line.startswith("__version__"):
12
+ version = line.split("=")[1].strip().strip('"').strip("'")
13
+ break
14
+
15
+ setup(
16
+ name="wisent",
17
+ version=version,
18
+ author="Lukasz Bartoszcze and the Wisent Team",
19
+ author_email="lukasz.bartoszcze@wisent.ai", # Replace with your email
20
+ description="Monitor and influence AI Brains",
21
+ long_description=long_description,
22
+ long_description_content_type="text/markdown",
23
+ url="https://github.com/wisent-ai/wisent", # Replace with your GitHub repo
24
+ packages=find_packages(exclude=["patches", "patches.*"]), # Exclude patches directory
25
+ include_package_data=True,
26
+ package_data={
27
+ "wisent": [
28
+ "core/evaluators/benchmark_specific/coding/safe_docker/Dockerfile",
29
+ "core/evaluators/benchmark_specific/coding/safe_docker/entrypoint.py",
30
+ "parameters/lm_eval/*.json",
31
+ "parameters/tasks/*.json",
32
+ "examples/scripts/*.json",
33
+ "scripts/*.sh",
34
+ ],
35
+ },
36
+ classifiers=[
37
+ "Programming Language :: Python :: 3",
38
+ "License :: OSI Approved :: MIT License",
39
+ "Operating System :: OS Independent",
40
+ "Development Status :: 4 - Beta",
41
+ "Intended Audience :: Science/Research",
42
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
43
+ ],
44
+ python_requires=">=3.8",
45
+ install_requires=[
46
+ "torch>=1.9.0",
47
+ "transformers>=4.20.0",
48
+ "tqdm>=4.50.0",
49
+ "scikit-learn>=0.24.0",
50
+ "pandas>=1.2.0",
51
+ "numpy>=1.21.0",
52
+ "datasets>=2.0.0",
53
+ "sentence-transformers>=2.0.0",
54
+ "faiss-cpu>=1.7.0",
55
+ "uncensorbench>=0.2.0",
56
+ "pebble>=5.0.0",
57
+ "latex2sympy2_extended>=1.0.0",
58
+ "sae_lens>=0.1.0",
59
+ "trl>=0.7.0",
60
+ ],
61
+ extras_require={
62
+ "harness": [
63
+ "lm-eval==0.4.8",
64
+ ],
65
+ "cuda": [
66
+ "flash-attn>=2.5.0",
67
+ ],
68
+ "sparsify": [
69
+ "sparsify>=0.1.0",
70
+ ],
71
+ },
72
+ entry_points={
73
+ "console_scripts": [
74
+ "wisent=wisent.core.main:main",
75
+ ],
76
+ },
77
+ keywords="nlp, machine learning, language models, safety, guardrails, lm-evaluation-harness",
78
+ )