wisent 0.7.925__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wisent-0.7.925/LICENSE +21 -0
- wisent-0.7.925/PKG-INFO +68 -0
- wisent-0.7.925/README.md +18 -0
- wisent-0.7.925/pyproject.toml +85 -0
- wisent-0.7.925/setup.cfg +4 -0
- wisent-0.7.925/setup.py +78 -0
- wisent-0.7.925/tests/test_multimodal_adapters.py +666 -0
- wisent-0.7.925/tests/test_multimodal_pipeline.py +661 -0
- wisent-0.7.925/wisent/__init__.py +64 -0
- wisent-0.7.925/wisent/cli.py +114 -0
- wisent-0.7.925/wisent/comparison/__init__.py +1 -0
- wisent-0.7.925/wisent/comparison/detect_bos_features.py +275 -0
- wisent-0.7.925/wisent/comparison/fgaa.py +465 -0
- wisent-0.7.925/wisent/comparison/lora.py +500 -0
- wisent-0.7.925/wisent/comparison/main.py +444 -0
- wisent-0.7.925/wisent/comparison/ours.py +76 -0
- wisent-0.7.925/wisent/comparison/sae.py +304 -0
- wisent-0.7.925/wisent/comparison/utils.py +381 -0
- wisent-0.7.925/wisent/core/__init__.py +40 -0
- wisent-0.7.925/wisent/core/activations/__init__.py +42 -0
- wisent-0.7.925/wisent/core/activations/activation_cache.py +393 -0
- wisent-0.7.925/wisent/core/activations/activations.py +79 -0
- wisent-0.7.925/wisent/core/activations/activations_collector.py +279 -0
- wisent-0.7.925/wisent/core/activations/classifier_inference_strategy.py +195 -0
- wisent-0.7.925/wisent/core/activations/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/activations/core/atoms.py +135 -0
- wisent-0.7.925/wisent/core/activations/extraction_strategy.py +484 -0
- wisent-0.7.925/wisent/core/adapters/__init__.py +22 -0
- wisent-0.7.925/wisent/core/adapters/audio.py +616 -0
- wisent-0.7.925/wisent/core/adapters/base.py +420 -0
- wisent-0.7.925/wisent/core/adapters/multimodal.py +738 -0
- wisent-0.7.925/wisent/core/adapters/robotics.py +643 -0
- wisent-0.7.925/wisent/core/adapters/text.py +441 -0
- wisent-0.7.925/wisent/core/adapters/video.py +555 -0
- wisent-0.7.925/wisent/core/agent/__init__.py +1 -0
- wisent-0.7.925/wisent/core/agent/budget.py +644 -0
- wisent-0.7.925/wisent/core/agent/device_benchmarks.py +691 -0
- wisent-0.7.925/wisent/core/agent/diagnose/__init__.py +1 -0
- wisent-0.7.925/wisent/core/agent/diagnose/agent_classifier_decision.py +641 -0
- wisent-0.7.925/wisent/core/agent/diagnose/classifier_marketplace.py +554 -0
- wisent-0.7.925/wisent/core/agent/diagnose/create_classifier.py +1155 -0
- wisent-0.7.925/wisent/core/agent/diagnose/response_diagnostics.py +273 -0
- wisent-0.7.925/wisent/core/agent/diagnose/select_classifiers.py +507 -0
- wisent-0.7.925/wisent/core/agent/diagnose/synthetic_classifier_option.py +755 -0
- wisent-0.7.925/wisent/core/agent/diagnose/tasks/__init__.py +33 -0
- wisent-0.7.925/wisent/core/agent/diagnose/tasks/task_manager.py +1453 -0
- wisent-0.7.925/wisent/core/agent/diagnose/tasks/task_relevance.py +94 -0
- wisent-0.7.925/wisent/core/agent/diagnose/tasks/task_selector.py +151 -0
- wisent-0.7.925/wisent/core/agent/diagnose.py +249 -0
- wisent-0.7.925/wisent/core/agent/steer.py +215 -0
- wisent-0.7.925/wisent/core/agent/timeout.py +134 -0
- wisent-0.7.925/wisent/core/autonomous_agent.py +1158 -0
- wisent-0.7.925/wisent/core/benchmark_extractors.py +372 -0
- wisent-0.7.925/wisent/core/benchmark_registry.py +151 -0
- wisent-0.7.925/wisent/core/bigcode_extractors.py +26 -0
- wisent-0.7.925/wisent/core/bigcode_integration.py +886 -0
- wisent-0.7.925/wisent/core/branding.py +108 -0
- wisent-0.7.925/wisent/core/classifier/__init__.py +1 -0
- wisent-0.7.925/wisent/core/classifier/models/__init__.py +1 -0
- wisent-0.7.925/wisent/core/classifiers/__init__.py +1 -0
- wisent-0.7.925/wisent/core/classifiers/classifiers/__init__.py +0 -0
- wisent-0.7.925/wisent/core/classifiers/classifiers/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/classifiers/classifiers/core/atoms.py +749 -0
- wisent-0.7.925/wisent/core/classifiers/classifiers/models/__init__.py +0 -0
- wisent-0.7.925/wisent/core/classifiers/classifiers/models/logistic.py +29 -0
- wisent-0.7.925/wisent/core/classifiers/classifiers/models/mlp.py +47 -0
- wisent-0.7.925/wisent/core/classifiers/classifiers/rotator.py +137 -0
- wisent-0.7.925/wisent/core/classifiers/core/__init__.py +1 -0
- wisent-0.7.925/wisent/core/classifiers/models/__init__.py +1 -0
- wisent-0.7.925/wisent/core/classifiers/pipeline_steps/__init__.py +1 -0
- wisent-0.7.925/wisent/core/cli/__init__.py +27 -0
- wisent-0.7.925/wisent/core/cli/agent/__init__.py +15 -0
- wisent-0.7.925/wisent/core/cli/agent/apply_steering.py +188 -0
- wisent-0.7.925/wisent/core/cli/agent/evaluate_response.py +126 -0
- wisent-0.7.925/wisent/core/cli/agent/generate_synthetic_pairs.py +123 -0
- wisent-0.7.925/wisent/core/cli/agent/main.py +139 -0
- wisent-0.7.925/wisent/core/cli/agent/train_classifier.py +184 -0
- wisent-0.7.925/wisent/core/cli/check_linearity.py +158 -0
- wisent-0.7.925/wisent/core/cli/cluster_benchmarks.py +470 -0
- wisent-0.7.925/wisent/core/cli/create_steering_vector.py +314 -0
- wisent-0.7.925/wisent/core/cli/diagnose_pairs.py +153 -0
- wisent-0.7.925/wisent/core/cli/diagnose_vectors.py +407 -0
- wisent-0.7.925/wisent/core/cli/estimate_unified_goodness_time.py +430 -0
- wisent-0.7.925/wisent/core/cli/evaluate_refusal.py +241 -0
- wisent-0.7.925/wisent/core/cli/evaluate_responses.py +926 -0
- wisent-0.7.925/wisent/core/cli/generate_humanization_pairs.py +128 -0
- wisent-0.7.925/wisent/core/cli/generate_pairs.py +175 -0
- wisent-0.7.925/wisent/core/cli/generate_pairs_from_task.py +61 -0
- wisent-0.7.925/wisent/core/cli/generate_responses.py +160 -0
- wisent-0.7.925/wisent/core/cli/generate_vector_from_synthetic.py +217 -0
- wisent-0.7.925/wisent/core/cli/generate_vector_from_task.py +243 -0
- wisent-0.7.925/wisent/core/cli/geometry_search.py +137 -0
- wisent-0.7.925/wisent/core/cli/get_activations.py +168 -0
- wisent-0.7.925/wisent/core/cli/inference_config.py +84 -0
- wisent-0.7.925/wisent/core/cli/inference_config_cli.py +54 -0
- wisent-0.7.925/wisent/core/cli/method_optimizer.py +860 -0
- wisent-0.7.925/wisent/core/cli/modify_weights.py +661 -0
- wisent-0.7.925/wisent/core/cli/multi_steer.py +112 -0
- wisent-0.7.925/wisent/core/cli/optimization_cache.py +298 -0
- wisent-0.7.925/wisent/core/cli/optimize.py +660 -0
- wisent-0.7.925/wisent/core/cli/optimize_classification.py +472 -0
- wisent-0.7.925/wisent/core/cli/optimize_sample_size.py +376 -0
- wisent-0.7.925/wisent/core/cli/optimize_steering.py +3695 -0
- wisent-0.7.925/wisent/core/cli/optimize_weights.py +1347 -0
- wisent-0.7.925/wisent/core/cli/preview_pairs.py +203 -0
- wisent-0.7.925/wisent/core/cli/steering_method_trainer.py +642 -0
- wisent-0.7.925/wisent/core/cli/steering_search_space.py +513 -0
- wisent-0.7.925/wisent/core/cli/tasks.py +854 -0
- wisent-0.7.925/wisent/core/cli/train_unified_goodness.py +680 -0
- wisent-0.7.925/wisent/core/cli_logger.py +22 -0
- wisent-0.7.925/wisent/core/config_manager.py +1731 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/__init__.py +15 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/core/atoms.py +45 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/core/buliders.py +59 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/core/pair.py +183 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/core/response.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/core/serialization.py +306 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/core/set.py +192 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/__init__.py +79 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/activations.py +53 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/base.py +73 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/control_vectors.py +3060 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/coverage.py +79 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/divergence.py +98 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/duplicates.py +118 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/linearity.py +315 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/diagnostics/vector_quality.py +621 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/__init__.py +1 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/atoms.py +255 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_extractor_manifest.py +456 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_extractor_registry.py +136 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/__init__.py +50 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentbench.py +225 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentharm.py +267 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentic_search.py +134 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aider_polyglot.py +202 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime.py +118 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime2024.py +74 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime2025.py +73 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/alpaca_eval.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/apps.py +296 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/arena_hard.py +179 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/atis.py +89 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/babilong.py +96 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bangla_mmlu.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/basqueglue.py +217 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bec2016eu.py +99 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bfcl.py +283 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bhtc_v2.py +87 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/chain_of_thought.py +89 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/chinese_simpleqa.py +209 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/cluewsc.py +177 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/cnn_dailymail.py +92 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codeforces.py +368 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue.py +109 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/coding_benchmarks.py +464 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/coedit_gec.py +79 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/conala.py +133 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/concode.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/dbpedia_14.py +91 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/doc_vqa.py +102 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/donotanswer.py +236 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ds1000.py +129 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ds_1000.py +155 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/epec_koref_bin.py +85 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ethos_binary.py +82 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/evalita_mp.py +165 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/evalita_sp_sum_task_fp_small_p1.py +89 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/facts_grounding.py +181 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/faithbench.py +272 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/financial_tweets.py +100 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flames.py +227 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flan_held_in.py +98 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flores.py +583 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/frames.py +159 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/get_negative_example_livecodebench.py +146 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/get_positive_example_livecodebench.py +140 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/gpt3_translation_benchmarks.py +98 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hallucinations_leaderboard.py +347 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/halueval.py +246 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/harmbench.py +250 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/healthbench.py +181 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hle.py +106 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hmmt.py +117 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/humaneval.py +160 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/humanevalpack.py +102 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/iwslt2017_ar_en.py +98 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/iwslt2017_en_ar.py +98 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/jailbreakbench.py +258 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/law_stack_exchange.py +101 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ledgar.py +118 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench.py +61 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench_contrastive_pair_generator.py +491 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench_v6.py +263 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livemathbench.py +268 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/llama.py +96 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/longform_writing.py +175 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/m_mmlu.py +96 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/math.py +186 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/math500.py +181 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/meddialog.py +79 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/medical_abstracts.py +101 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/medium_priority_benchmarks.py +737 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mercury.py +134 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mmlu_redux.py +194 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mmlusr.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multimedqa.py +99 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multipl_e.py +109 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple.py +96 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_choice.py +87 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_cpp.py +128 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_go.py +128 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_java.py +128 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_js.py +128 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_py.py +15 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_rs.py +128 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/non_greedy_robustness_agieval_aqua_rat.py +92 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/olympiadbench.py +277 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/openllm.py +99 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/option_order_robustness_agieval_aqua_rat.py +92 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/or_bench.py +300 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/penn_treebank.py +80 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/planbench.py +176 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/polymath.py +500 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/prompt_robustness_agieval_aqua_rat.py +92 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/pythia.py +99 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/recode.py +146 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/refusalbench.py +288 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/scicode.py +275 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/self_consistency.py +90 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/simpleqa.py +164 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/sorry_bench.py +211 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/stsb.py +79 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_lm_eval_v1.py +99 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_lm_eval_v1_seq2seq.py +98 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_t5_prompt.py +123 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_gpqa.py +106 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/swe_bench.py +428 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/swe_bench_verified.py +158 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/sycophancy_eval.py +205 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/t0_eval.py +79 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tag.py +98 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tau_bench.py +205 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tmlu.py +109 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/toolbench.py +266 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/toolemu.py +252 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/travelplanner.py +286 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/truthfulqa_generation.py +128 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/unfair_tos.py +83 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/vaxx_stance.py +86 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wiceu.py +85 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wikitext103.py +97 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wildguard.py +280 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt14_en_fr.py +97 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt14_fr_en.py +97 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_de_en.py +90 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_en_de.py +90 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_en_ro.py +90 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_ro_en.py +90 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt_ro_en_t5_prompt.py +90 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/xsum.py +81 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/__init__.py +0 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/atoms.py +265 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/__init__.py +472 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aclue.py +24 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/acp.py +33 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/acpbench.py +39 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/advanced_ai_risk.py +59 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aexams.py +14 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrimgsm.py +10 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrimmlu.py +10 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrixnli.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench.py +14 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_adr.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_afriqa.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_afrisenti.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_belebele.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_flores.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_injongointent.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_mafand.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhaner.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhanews.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhapos.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_naijarc.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_nollysenti.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_ntrex.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_openai_mmlu.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_salt.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_sib.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_uhura_arc_easy.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_xlsum.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/agieval.py +33 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/anli.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arab_culture.py +24 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_acva.py +67 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_acva_light.py +67 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_complete.py +24 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_light.py +81 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabicmmlu.py +59 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aradice.py +36 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arc.py +61 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arithmetic.py +19 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/basque_bench.py +37 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bbh.py +121 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bbq.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/belebele.py +293 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bertaqa.py +25 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bigbench.py +300 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/blimp.py +76 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/careqa.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/catalan_bench.py +43 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ceval_valid.py +61 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/cmmlu.py +76 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/code_x_glue.py +16 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/copal_id.py +11 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/crows_pairs.py +31 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/csatqa.py +15 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/darija.py +29 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/darijammlu.py +57 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/egymmlu.py +62 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/eus.py +76 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/evalita_mp.py +93 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/fld.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/flores.py +466 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/freebase.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/french_bench.py +23 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/galician_bench.py +41 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/glianorex.py +11 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/global_mmlu.py +115 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gpqa.py +27 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gsm8k.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gsm8k_platinum.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/haerae.py +14 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/headqa.py +11 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hellaswag.py +39 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hendrycks_ethics.py +14 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hendrycks_math.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hrm8k.py +20 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/inverse.py +22 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/japanese_leaderboard.py +20 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/jsonschema_bench.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kbl.py +85 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kmmlu.py +281 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kobest.py +14 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kormedmcqa.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/lambada.py +28 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/leaderboard.py +52 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/libra.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/lingoly.py +11 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/longbench.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/m.py +43 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mastermind.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mathqa.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/med.py +24 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/meddialog.py +12 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/medqa.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mela.py +18 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/metabench.py +36 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mgsm.py +44 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/minerva_math.py +16 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mlqa.py +58 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu.py +70 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_pro.py +23 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_pro_plus.py +23 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_prox.py +191 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlusr.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmmu.py +46 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/model_written_evals.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/multiblimp.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/non.py +23 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/noreval.py +143 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/noridiom.py +20 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/nortruthfulqa.py +32 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/nrk.py +20 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_arc_multilingual.py +10 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_hellaswag_multilingual.py +24 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_mmlu_multilingual.py +24 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_truthfulqa_multilingual.py +34 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/paloma.py +25 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/pawsx.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/persona.py +144 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/pile.py +31 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/polemo2.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/portuguese_bench.py +31 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/prompt.py +23 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/qa4mre.py +12 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/qasper.py +11 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ru.py +19 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ruler.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/score.py +20 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/scrolls.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/self_consistency.py +11 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/spanish_bench.py +38 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/storycloze.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/super_glue_t5_prompt.py +17 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tinyBenchmarks.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tmlu.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tmmluplus.py +80 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/translation.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/truthfulqa.py +76 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/truthfulqa_multi.py +24 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/turkishmmlu.py +30 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/unitxt.py +23 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/unscramble.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/winogender.py +16 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmdp.py +12 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmt14.py +16 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmt16.py +22 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wsc273.py +9 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xcopa.py +21 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xnli.py +28 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xnli_eu.py +12 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xquad.py +22 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xstorycloze.py +22 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xwinograd.py +15 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_extractor_manifest.py +484 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_extractor_registry.py +158 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/__init__.py +125 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aclue.py +169 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/acp_bench.py +205 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/acp_bench_hard.py +183 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/advanced.py +128 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aexams.py +182 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrimgsm.py +98 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrimmlu.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrixnli.py +129 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrobench_cot.py +88 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrobench_mc.py +107 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ag.py +134 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/agieval_aqua_rat.py +129 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ai2_arc.py +114 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anagrams1.py +81 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anagrams2.py +81 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anli.py +140 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabculture.py +178 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic.py +96 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_exams.py +102 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_leaderboard_complete.py +166 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_leaderboard_light.py +166 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabicmmlu.py +165 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aradice.py +266 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc.py +131 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_challenge.py +117 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_easy.py +117 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_gen.py +101 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_mc.py +106 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/argument.py +134 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arithmetic.py +114 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/asdiv.py +122 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/assin.py +103 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/babi.py +147 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench_gen.py +168 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench_mc.py +139 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bbh.py +133 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bbq.py +167 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/belebele.py +179 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/benchmarks.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bertaqa.py +163 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bhs.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bhtc.py +141 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bigbench.py +170 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/blimp.py +169 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/blimp_nl.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/boolq.py +134 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/boolq_seq2seq.py +117 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/c4.py +148 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cabbq.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cabreu.py +127 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/careqa.py +167 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench_gen.py +119 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench_mc.py +113 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalanqa.py +169 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catcola.py +137 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cb.py +124 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ceval.py +221 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ceval_valid.py +161 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/chain.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/chartqa.py +236 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/claim.py +149 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/click.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cmmlu.py +164 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cnn.py +142 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cocoteros.py +146 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/code_x_glue.py +119 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coedit.py +147 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cola.py +83 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/commonsense.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/commonsense_qa.py +125 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/copa.py +124 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/copal_id.py +167 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coqa.py +161 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coqcat.py +114 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/crows_pairs.py +158 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/csatqa.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cycle.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cycle_letters.py +81 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darija_bench.py +219 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darijahellaswag.py +170 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darijammlu.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/dbpedia.py +155 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/discrim_eval.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/doc.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/drop.py +129 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/egyhellaswag.py +125 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/egymmlu.py +180 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/epec.py +140 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench.py +192 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench_ca.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench_es.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/esbbq.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/escola.py +85 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ethics.py +133 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ethos.py +99 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_exams.py +223 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_proficiency.py +157 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_reading.py +157 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_trivia.py +157 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/evalita_llm.py +164 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/evalita_sp.py +109 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/fda.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/financial.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/flan.py +112 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/fld.py +143 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench.py +200 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench_mc.py +98 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench_perplexity.py +86 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galcola.py +109 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench_gen.py +118 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench_mc.py +112 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gaokao.py +141 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/glianorex.py +116 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/global_mmlu.py +169 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/global_piqa.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/glue.py +109 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gpqa.py +161 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gpt3.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/groundcocoa.py +182 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gsm.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gsm8k.py +134 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/haerae.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/headqa.py +112 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hellaswag.py +125 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hendrycks_ethics.py +221 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hendrycks_math.py +238 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/histoires_morales.py +177 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hle.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hrm8k.py +201 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/humaneval.py +124 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/humaneval_infilling.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/icelandic_winogrande.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ifeval.py +118 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/inverse.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/inverse_scaling.py +190 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/iwslt2017.py +117 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ja.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard_gen.py +224 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard_mc.py +120 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/jsonschema_bench.py +123 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kbl.py +140 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu.py +166 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu_cot.py +88 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu_mc.py +107 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kobest.py +163 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kormedmcqa.py +148 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada.py +147 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_cloze.py +183 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_multilingual.py +183 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_multilingual_stablelm.py +141 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/law.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/leaderboard.py +192 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/libra.py +165 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lingoly.py +201 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/llama3.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lm_syneval.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logieval.py +82 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logiqa.py +115 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logiqa2.py +114 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/longbench.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/longbenchv2.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mastermind.py +201 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mathqa.py +137 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mbpp.py +164 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mc-taco.py +115 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/med_concepts_qa.py +222 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/meddialog.py +178 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medical.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mediqa_qa2019.py +123 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medmcqa.py +167 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medqa.py +118 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medtext.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mela.py +96 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/meqsum.py +115 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/metabench.py +152 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mgsm.py +122 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mimic_repsum.py +140 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/minerva_math.py +170 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mlqa.py +143 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu.py +142 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_cot.py +88 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_mc.py +107 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_pro.py +145 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlusr.py +188 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmmu.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mnli.py +113 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/model_written_evals.py +115 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/moral_stories.py +151 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mrpc.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mts_dialog.py +118 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mts_dialog_perplexity.py +97 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multiblimp.py +131 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multilingual.py +106 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multirc.py +114 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mutual.py +113 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/non.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval.py +171 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_exact.py +155 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_gen.py +277 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_gen_exact.py +163 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc.py +224 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc_log_likelihoods.py +219 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noticia.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/nq_open.py +135 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi.py +27 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_arc_multilingual.py +165 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_hellaswag_multilingual.py +172 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_mmlu_multilingual.py +160 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_truthfulqa_multilingual.py +206 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph.py +184 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph_perplexity.py +97 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/openbookqa.py +118 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/option.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paloma.py +205 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafraseja.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafrases.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws_x.py +152 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pawsx.py +115 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/persona.py +244 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases.py +142 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases_ca_va.py +82 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile.py +159 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile_10k.py +140 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/piqa.py +116 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/polemo2.py +135 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench_gen.py +121 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench_mc.py +103 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prompt.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prost.py +115 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pubmedqa.py +112 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qa4mre.py +119 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper.py +118 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper_bool.py +112 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnli.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnlieu.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qqp.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/quac.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/race.py +124 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/random.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/realtoxicityprompts.py +124 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/record.py +125 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/reversed.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/rte.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ruler.py +168 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sciq.py +113 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/score.py +175 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls.py +159 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls_mc.py +155 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/self.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue.py +129 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue_rte.py +120 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/simple_cooccurrence_bias.py +121 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/siqa.py +206 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/social_iqa.py +114 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench_gen.py +117 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench_mc.py +110 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/squad2.py +129 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/squad_completion.py +121 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sst2.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/storycloze.py +246 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/summarization.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/super.py +105 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/super_glue.py +152 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/superglue.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/supergpqa.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/swag.py +115 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/swde.py +177 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sycophancy.py +115 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/t0.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/teca.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinyarc.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinybenchmarks.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinygsm8k.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinyhellaswag.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinymmlu.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinytruthfulqa.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinywinogrande.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tmmluplus.py +179 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/toxigen.py +91 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/translation.py +149 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/triviaqa.py +130 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa.py +110 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc1.py +125 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc2.py +138 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_multi.py +142 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turblimp_core.py +150 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu.py +159 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu_cot.py +104 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu_mc.py +100 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/twenty_newsgroups.py +111 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/unitxt.py +131 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/unscramble.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/vaxx.py +95 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/webqs.py +130 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wic.py +121 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wikitext.py +146 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/winogender.py +139 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/winogrande.py +118 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmdp.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmt14.py +110 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmt16.py +118 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wnli.py +114 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wsc.py +117 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wsc273.py +178 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xcopa.py +195 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xlsum.py +145 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xnli.py +131 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xquad.py +201 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xstorycloze.py +128 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xwinograd.py +124 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/yahoo.py +108 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/zhoblimp.py +153 -0
- wisent-0.7.925/wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_pairs_generation.py +223 -0
- wisent-0.7.925/wisent/core/data_loaders/__init__.py +235 -0
- wisent-0.7.925/wisent/core/data_loaders/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/data_loaders/core/atoms.py +99 -0
- wisent-0.7.925/wisent/core/data_loaders/loaders/__init__.py +0 -0
- wisent-0.7.925/wisent/core/data_loaders/loaders/custom.py +120 -0
- wisent-0.7.925/wisent/core/data_loaders/loaders/huggingface_loader.py +153 -0
- wisent-0.7.925/wisent/core/data_loaders/loaders/lm_loader.py +505 -0
- wisent-0.7.925/wisent/core/data_loaders/loaders/lm_loader_special_cases.py +496 -0
- wisent-0.7.925/wisent/core/data_loaders/loaders/task_interface_loader.py +300 -0
- wisent-0.7.925/wisent/core/data_loaders/rotator.py +118 -0
- wisent-0.7.925/wisent/core/detection_handling.py +259 -0
- wisent-0.7.925/wisent/core/diversity_processors.py +193 -0
- wisent-0.7.925/wisent/core/download_full_benchmarks.py +1512 -0
- wisent-0.7.925/wisent/core/errors/__init__.py +203 -0
- wisent-0.7.925/wisent/core/errors/error_codes.py +763 -0
- wisent-0.7.925/wisent/core/errors/error_handler.py +134 -0
- wisent-0.7.925/wisent/core/evaluators/__init__.py +0 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/__init__.py +42 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/aime_evaluator.py +90 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/apps_evaluator.py +133 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/__init__.py +0 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/metrics/__init__.py +0 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/metrics/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/metrics/core/atoms.py +36 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/metrics/evaluator.py +368 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/metrics/passk.py +67 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/__init__.py +0 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/core/atoms.py +27 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/cpp_sanitizer.py +62 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/java_sanitizer.py +78 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/python_sanitizer.py +94 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/utils.py +126 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/providers/__init__.py +18 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/providers/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/providers/core/atoms.py +31 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/providers/livecodebench/__init__.py +3 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/providers/livecodebench/provider.py +305 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/Dockerfile +31 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/__init__.py +0 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/atoms.py +105 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/runtime.py +143 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/entrypoint.py +121 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/safe_docker/recipes.py +60 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/coding/solution_generator.py +258 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/conala_evaluator.py +195 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/exact_match_evaluator.py +81 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/f1_evaluator.py +173 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/generation_evaluator.py +488 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/livemathbench_evaluator.py +393 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/log_likelihoods_evaluator.py +202 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_evaluator.py +119 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_parsing/__init__.py +1 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_parsing/core.py +1640 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_parsing/extract_boxed.py +48 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_parsing/is_equiv.py +159 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/math_parsing/scripts.py +919 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/perplexity_evaluator.py +175 -0
- wisent-0.7.925/wisent/core/evaluators/benchmark_specific/polymath_evaluator.py +114 -0
- wisent-0.7.925/wisent/core/evaluators/core/__init__.py +5 -0
- wisent-0.7.925/wisent/core/evaluators/core/atoms.py +166 -0
- wisent-0.7.925/wisent/core/evaluators/custom/__init__.py +20 -0
- wisent-0.7.925/wisent/core/evaluators/custom/custom_evaluator.py +382 -0
- wisent-0.7.925/wisent/core/evaluators/custom/examples/__init__.py +37 -0
- wisent-0.7.925/wisent/core/evaluators/custom/examples/desklib_detector.py +166 -0
- wisent-0.7.925/wisent/core/evaluators/custom/examples/gptzero.py +185 -0
- wisent-0.7.925/wisent/core/evaluators/custom/examples/humanization.py +79 -0
- wisent-0.7.925/wisent/core/evaluators/custom/examples/humanization_coherent.py +181 -0
- wisent-0.7.925/wisent/core/evaluators/custom/examples/roberta_detector.py +173 -0
- wisent-0.7.925/wisent/core/evaluators/oracles/__init__.py +0 -0
- wisent-0.7.925/wisent/core/evaluators/oracles/interactive.py +73 -0
- wisent-0.7.925/wisent/core/evaluators/oracles/nlp_evaluator.py +440 -0
- wisent-0.7.925/wisent/core/evaluators/oracles/truthfulqa_gen_evaluator.py +150 -0
- wisent-0.7.925/wisent/core/evaluators/oracles/user_specified.py +67 -0
- wisent-0.7.925/wisent/core/evaluators/personalization/__init__.py +12 -0
- wisent-0.7.925/wisent/core/evaluators/personalization/alignment.py +166 -0
- wisent-0.7.925/wisent/core/evaluators/personalization/coherence.py +371 -0
- wisent-0.7.925/wisent/core/evaluators/personalization/difference.py +73 -0
- wisent-0.7.925/wisent/core/evaluators/rotator.py +217 -0
- wisent-0.7.925/wisent/core/evaluators/steering_evaluators.py +386 -0
- wisent-0.7.925/wisent/core/evaluators/synthetic_evaluator.py +377 -0
- wisent-0.7.925/wisent/core/geometry_runner.py +995 -0
- wisent-0.7.925/wisent/core/geometry_search_space.py +237 -0
- wisent-0.7.925/wisent/core/hyperparameter_optimizer.py +547 -0
- wisent-0.7.925/wisent/core/layer.py +17 -0
- wisent-0.7.925/wisent/core/lm_eval_harness_ground_truth.py +1427 -0
- wisent-0.7.925/wisent/core/main.py +107 -0
- wisent-0.7.925/wisent/core/managed_cached_benchmarks.py +609 -0
- wisent-0.7.925/wisent/core/mixed_benchmark_sampler.py +366 -0
- wisent-0.7.925/wisent/core/modalities/__init__.py +545 -0
- wisent-0.7.925/wisent/core/model_persistence.py +302 -0
- wisent-0.7.925/wisent/core/models/__init__.py +23 -0
- wisent-0.7.925/wisent/core/models/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/models/core/atoms.py +467 -0
- wisent-0.7.925/wisent/core/models/inference_config.py +127 -0
- wisent-0.7.925/wisent/core/models/wisent_model.py +894 -0
- wisent-0.7.925/wisent/core/multi_steering.py +397 -0
- wisent-0.7.925/wisent/core/opti/__init__.py +0 -0
- wisent-0.7.925/wisent/core/opti/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/opti/core/atoms.py +177 -0
- wisent-0.7.925/wisent/core/opti/methods/__init__.py +10 -0
- wisent-0.7.925/wisent/core/opti/methods/opti_classificator.py +172 -0
- wisent-0.7.925/wisent/core/opti/methods/opti_steering.py +139 -0
- wisent-0.7.925/wisent/core/opti/methods/opti_weights.py +550 -0
- wisent-0.7.925/wisent/core/optuna/__init__.py +54 -0
- wisent-0.7.925/wisent/core/optuna/classifier/__init__.py +25 -0
- wisent-0.7.925/wisent/core/optuna/classifier/activation_generator.py +353 -0
- wisent-0.7.925/wisent/core/optuna/classifier/classifier_cache.py +509 -0
- wisent-0.7.925/wisent/core/optuna/classifier/optuna_classifier_optimizer.py +685 -0
- wisent-0.7.925/wisent/core/optuna/steering/__init__.py +20 -0
- wisent-0.7.925/wisent/core/optuna/steering/bigcode_evaluator_wrapper.py +200 -0
- wisent-0.7.925/wisent/core/optuna/steering/data_utils.py +342 -0
- wisent-0.7.925/wisent/core/optuna/steering/metrics.py +412 -0
- wisent-0.7.925/wisent/core/optuna/steering/steering_optimization.py +1101 -0
- wisent-0.7.925/wisent/core/parser.py +1662 -0
- wisent-0.7.925/wisent/core/parser_arguments/__init__.py +10 -0
- wisent-0.7.925/wisent/core/parser_arguments/agent_parser.py +122 -0
- wisent-0.7.925/wisent/core/parser_arguments/check_linearity_parser.py +92 -0
- wisent-0.7.925/wisent/core/parser_arguments/cluster_benchmarks_parser.py +31 -0
- wisent-0.7.925/wisent/core/parser_arguments/configure_model_parser.py +7 -0
- wisent-0.7.925/wisent/core/parser_arguments/create_steering_vector_parser.py +67 -0
- wisent-0.7.925/wisent/core/parser_arguments/diagnose_pairs_parser.py +25 -0
- wisent-0.7.925/wisent/core/parser_arguments/diagnose_vectors_parser.py +72 -0
- wisent-0.7.925/wisent/core/parser_arguments/evaluate_parser.py +40 -0
- wisent-0.7.925/wisent/core/parser_arguments/evaluate_refusal_parser.py +32 -0
- wisent-0.7.925/wisent/core/parser_arguments/evaluate_responses_parser.py +12 -0
- wisent-0.7.925/wisent/core/parser_arguments/full_optimize_parser.py +194 -0
- wisent-0.7.925/wisent/core/parser_arguments/generate_pairs_from_task_parser.py +33 -0
- wisent-0.7.925/wisent/core/parser_arguments/generate_pairs_parser.py +43 -0
- wisent-0.7.925/wisent/core/parser_arguments/generate_responses_parser.py +16 -0
- wisent-0.7.925/wisent/core/parser_arguments/generate_vector_from_synthetic_parser.py +148 -0
- wisent-0.7.925/wisent/core/parser_arguments/generate_vector_from_task_parser.py +162 -0
- wisent-0.7.925/wisent/core/parser_arguments/generate_vector_parser.py +89 -0
- wisent-0.7.925/wisent/core/parser_arguments/geometry_search_parser.py +61 -0
- wisent-0.7.925/wisent/core/parser_arguments/get_activations_parser.py +81 -0
- wisent-0.7.925/wisent/core/parser_arguments/inference_config_parser.py +65 -0
- wisent-0.7.925/wisent/core/parser_arguments/main_parser.py +236 -0
- wisent-0.7.925/wisent/core/parser_arguments/model_config_parser.py +59 -0
- wisent-0.7.925/wisent/core/parser_arguments/modify_weights_parser.py +309 -0
- wisent-0.7.925/wisent/core/parser_arguments/monitor_parser.py +17 -0
- wisent-0.7.925/wisent/core/parser_arguments/multi_steer_parser.py +48 -0
- wisent-0.7.925/wisent/core/parser_arguments/nonsense_parser.py +26 -0
- wisent-0.7.925/wisent/core/parser_arguments/optimization_cache_parser.py +64 -0
- wisent-0.7.925/wisent/core/parser_arguments/optimize_classification_parser.py +108 -0
- wisent-0.7.925/wisent/core/parser_arguments/optimize_parser.py +142 -0
- wisent-0.7.925/wisent/core/parser_arguments/optimize_sample_size_parser.py +58 -0
- wisent-0.7.925/wisent/core/parser_arguments/optimize_steering_parser.py +724 -0
- wisent-0.7.925/wisent/core/parser_arguments/optimize_weights_parser.py +409 -0
- wisent-0.7.925/wisent/core/parser_arguments/synthetic_parser.py +117 -0
- wisent-0.7.925/wisent/core/parser_arguments/tasks_parser.py +579 -0
- wisent-0.7.925/wisent/core/parser_arguments/train_unified_goodness_parser.py +172 -0
- wisent-0.7.925/wisent/core/parser_arguments/utils.py +107 -0
- wisent-0.7.925/wisent/core/prompts/__init__.py +0 -0
- wisent-0.7.925/wisent/core/prompts/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/prompts/core/atom.py +57 -0
- wisent-0.7.925/wisent/core/prompts/core/prompt_formater.py +148 -0
- wisent-0.7.925/wisent/core/prompts/prompt_stratiegies/__init__.py +0 -0
- wisent-0.7.925/wisent/core/prompts/prompt_stratiegies/direct_completion.py +26 -0
- wisent-0.7.925/wisent/core/prompts/prompt_stratiegies/instruction_following.py +26 -0
- wisent-0.7.925/wisent/core/prompts/prompt_stratiegies/multiple_choice.py +31 -0
- wisent-0.7.925/wisent/core/prompts/prompt_stratiegies/role_playing.py +33 -0
- wisent-0.7.925/wisent/core/representation.py +5 -0
- wisent-0.7.925/wisent/core/save_results.py +277 -0
- wisent-0.7.925/wisent/core/steering.py +662 -0
- wisent-0.7.925/wisent/core/steering_method.py +20 -0
- wisent-0.7.925/wisent/core/steering_methods/__init__.py +54 -0
- wisent-0.7.925/wisent/core/steering_methods/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/steering_methods/core/atoms.py +153 -0
- wisent-0.7.925/wisent/core/steering_methods/methods/__init__.py +0 -0
- wisent-0.7.925/wisent/core/steering_methods/methods/caa.py +45 -0
- wisent-0.7.925/wisent/core/steering_methods/methods/hyperplane.py +75 -0
- wisent-0.7.925/wisent/core/steering_methods/methods/prism.py +587 -0
- wisent-0.7.925/wisent/core/steering_methods/methods/pulse.py +672 -0
- wisent-0.7.925/wisent/core/steering_methods/methods/titan.py +1050 -0
- wisent-0.7.925/wisent/core/steering_methods/preflight.py +322 -0
- wisent-0.7.925/wisent/core/steering_methods/registry.py +689 -0
- wisent-0.7.925/wisent/core/steering_methods/rotator.py +121 -0
- wisent-0.7.925/wisent/core/steering_optimizer.py +1503 -0
- wisent-0.7.925/wisent/core/synthetic/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/cleaners/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/cleaners/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/cleaners/core/atoms.py +58 -0
- wisent-0.7.925/wisent/core/synthetic/cleaners/deduper_cleaner.py +53 -0
- wisent-0.7.925/wisent/core/synthetic/cleaners/methods/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/cleaners/methods/base_dedupers.py +321 -0
- wisent-0.7.925/wisent/core/synthetic/cleaners/methods/base_refusalers.py +286 -0
- wisent-0.7.925/wisent/core/synthetic/cleaners/methods/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/cleaners/methods/core/atoms.py +47 -0
- wisent-0.7.925/wisent/core/synthetic/cleaners/pairs_cleaner.py +90 -0
- wisent-0.7.925/wisent/core/synthetic/cleaners/refusaler_cleaner.py +133 -0
- wisent-0.7.925/wisent/core/synthetic/db_instructions/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/db_instructions/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/db_instructions/core/atoms.py +25 -0
- wisent-0.7.925/wisent/core/synthetic/db_instructions/mini_dp.py +115 -0
- wisent-0.7.925/wisent/core/synthetic/generators/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/generators/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/generators/core/atoms.py +73 -0
- wisent-0.7.925/wisent/core/synthetic/generators/diversities/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/generators/diversities/core/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/generators/diversities/core/core.py +68 -0
- wisent-0.7.925/wisent/core/synthetic/generators/diversities/methods/__init__.py +0 -0
- wisent-0.7.925/wisent/core/synthetic/generators/diversities/methods/fast_diversity.py +249 -0
- wisent-0.7.925/wisent/core/synthetic/generators/nonsense_generator.py +162 -0
- wisent-0.7.925/wisent/core/synthetic/generators/pairs_generator.py +313 -0
- wisent-0.7.925/wisent/core/task_interface.py +143 -0
- wisent-0.7.925/wisent/core/task_selector.py +232 -0
- wisent-0.7.925/wisent/core/tasks/__init__.py +218 -0
- wisent-0.7.925/wisent/core/tasks/aime_task.py +142 -0
- wisent-0.7.925/wisent/core/tasks/file_task.py +212 -0
- wisent-0.7.925/wisent/core/tasks/hle_task.py +180 -0
- wisent-0.7.925/wisent/core/tasks/hmmt_task.py +120 -0
- wisent-0.7.925/wisent/core/tasks/livecodebench_task.py +94 -0
- wisent-0.7.925/wisent/core/tasks/livemathbench_task.py +159 -0
- wisent-0.7.925/wisent/core/tasks/lm_eval_task.py +611 -0
- wisent-0.7.925/wisent/core/tasks/math500_task.py +84 -0
- wisent-0.7.925/wisent/core/tasks/polymath_task.py +147 -0
- wisent-0.7.925/wisent/core/tasks/supergpqa_task.py +220 -0
- wisent-0.7.925/wisent/core/time_estimator.py +155 -0
- wisent-0.7.925/wisent/core/timing_calibration.py +176 -0
- wisent-0.7.925/wisent/core/tracking/__init__.py +54 -0
- wisent-0.7.925/wisent/core/tracking/latency.py +620 -0
- wisent-0.7.925/wisent/core/tracking/memory.py +360 -0
- wisent-0.7.925/wisent/core/trainers/__init__.py +0 -0
- wisent-0.7.925/wisent/core/trainers/core/__init__.py +11 -0
- wisent-0.7.925/wisent/core/trainers/core/atoms.py +45 -0
- wisent-0.7.925/wisent/core/trainers/steering_trainer.py +356 -0
- wisent-0.7.925/wisent/core/universal_subspace.py +918 -0
- wisent-0.7.925/wisent/core/user_model_config.py +158 -0
- wisent-0.7.925/wisent/core/utils/__init__.py +64 -0
- wisent-0.7.925/wisent/core/utils/base_rotator.py +292 -0
- wisent-0.7.925/wisent/core/utils/dataset_splits.py +197 -0
- wisent-0.7.925/wisent/core/utils/device.py +279 -0
- wisent-0.7.925/wisent/core/utils/layer_combinations.py +70 -0
- wisent-0.7.925/wisent/core/weight_modification/__init__.py +134 -0
- wisent-0.7.925/wisent/core/weight_modification/additive.py +340 -0
- wisent-0.7.925/wisent/core/weight_modification/directional.py +1357 -0
- wisent-0.7.925/wisent/core/weight_modification/export.py +359 -0
- wisent-0.7.925/wisent/core/weight_modification/multi_direction.py +410 -0
- wisent-0.7.925/wisent/core/weight_modification/utils.py +236 -0
- wisent-0.7.925/wisent/core/wisent.py +660 -0
- wisent-0.7.925/wisent/examples/__init__.py +1 -0
- wisent-0.7.925/wisent/examples/scripts/__init__.py +1 -0
- wisent-0.7.925/wisent/examples/scripts/benchmark_tags.json +2140 -0
- wisent-0.7.925/wisent/examples/scripts/count_all_benchmarks.py +121 -0
- wisent-0.7.925/wisent/examples/scripts/discover_directions.py +469 -0
- wisent-0.7.925/wisent/examples/scripts/extract_benchmark_info.py +71 -0
- wisent-0.7.925/wisent/examples/scripts/lm_eval_readme.json +4 -0
- wisent-0.7.925/wisent/examples/scripts/search_all_short_names.py +31 -0
- wisent-0.7.925/wisent/examples/scripts/test_all_benchmarks.py +138 -0
- wisent-0.7.925/wisent/examples/scripts/test_all_benchmarks_new.py +28 -0
- wisent-0.7.925/wisent/examples/scripts/test_contrastive_pairs_all_supported.py +230 -0
- wisent-0.7.925/wisent/examples/scripts/test_nonsense_baseline.py +261 -0
- wisent-0.7.925/wisent/examples/scripts/test_one_benchmark.py +324 -0
- wisent-0.7.925/wisent/examples/scripts/test_one_coding_benchmark.py +293 -0
- wisent-0.7.925/wisent/parameters/__init__.py +1 -0
- wisent-0.7.925/wisent/parameters/lm_eval/all_lm_eval_task_families.json +169 -0
- wisent-0.7.925/wisent/parameters/lm_eval/broken_in_lm_eval.json +187 -0
- wisent-0.7.925/wisent/parameters/lm_eval/category_directions.json +137 -0
- wisent-0.7.925/wisent/parameters/lm_eval/evaluations_not_lm_eval_tasks.json +0 -0
- wisent-0.7.925/wisent/parameters/lm_eval/evaluator_check.json +3476 -0
- wisent-0.7.925/wisent/parameters/lm_eval/final_verification.json +24782 -0
- wisent-0.7.925/wisent/parameters/lm_eval/group_task_evaluators.json +1833 -0
- wisent-0.7.925/wisent/parameters/lm_eval/group_tasks.json +150 -0
- wisent-0.7.925/wisent/parameters/lm_eval/individual_tasks.json +402 -0
- wisent-0.7.925/wisent/parameters/lm_eval/no_readmes.json +1 -0
- wisent-0.7.925/wisent/parameters/lm_eval/not_lm_eval_tasks.json +110 -0
- wisent-0.7.925/wisent/parameters/lm_eval/read_tasks.json +208 -0
- wisent-0.7.925/wisent/parameters/lm_eval/readme_files.json +208 -0
- wisent-0.7.925/wisent/parameters/lm_eval/repair_plan.json +282 -0
- wisent-0.7.925/wisent/parameters/lm_eval/track_progress_not_lm_eval_tasks.json +77 -0
- wisent-0.7.925/wisent/parameters/lm_eval/weak_contrastive_pairs.json +38 -0
- wisent-0.7.925/wisent/parameters/lm_eval/working_benchmarks.json +206 -0
- wisent-0.7.925/wisent/parameters/lm_eval/working_benchmarks_categorized.json +236 -0
- wisent-0.7.925/wisent/parameters/tasks/missing_task_families.json +2963 -0
- wisent-0.7.925/wisent/parameters/tasks/remaining_tasks_to_implement.json +199 -0
- wisent-0.7.925/wisent/parameters/tasks/risks.json +10 -0
- wisent-0.7.925/wisent/parameters/tasks/skills.json +14 -0
- wisent-0.7.925/wisent/parameters/tasks/tasks.json +56031 -0
- wisent-0.7.925/wisent/scripts/run_quality_metrics_sweep.sh +310 -0
- wisent-0.7.925/wisent/tests/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/cli/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/cli/activations/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/cli/activations/test_get_activations.py +127 -0
- wisent-0.7.925/wisent/tests/examples/cli/classifier/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/cli/classifier/test_classifier_examples.py +141 -0
- wisent-0.7.925/wisent/tests/examples/cli/contrastive_pairs/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/cli/contrastive_pairs/test_generate_pairs.py +89 -0
- wisent-0.7.925/wisent/tests/examples/cli/evaluation/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/cli/evaluation/test_evaluation_examples.py +117 -0
- wisent-0.7.925/wisent/tests/examples/cli/generate/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/cli/generate/test_generate_with_classifier.py +146 -0
- wisent-0.7.925/wisent/tests/examples/cli/generate/test_generate_with_steering.py +149 -0
- wisent-0.7.925/wisent/tests/examples/cli/generate/test_only_generate.py +110 -0
- wisent-0.7.925/wisent/tests/examples/cli/multi_steering/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/cli/multi_steering/test_multi_steer_from_trained_vectors.py +210 -0
- wisent-0.7.925/wisent/tests/examples/cli/multi_steering/test_multi_steer_with_different_parameters.py +205 -0
- wisent-0.7.925/wisent/tests/examples/cli/multi_steering/test_train_and_multi_steer.py +174 -0
- wisent-0.7.925/wisent/tests/examples/cli/optimizer/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/cli/optimizer/test_optimize_sample_size.py +102 -0
- wisent-0.7.925/wisent/tests/examples/cli/optimizer/test_optimizer_examples.py +59 -0
- wisent-0.7.925/wisent/tests/examples/cli/steering/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/cli/steering/test_create_steering_vectors.py +135 -0
- wisent-0.7.925/wisent/tests/examples/cli/synthetic/__init__.py +0 -0
- wisent-0.7.925/wisent/tests/examples/cli/synthetic/test_synthetic_pairs.py +45 -0
- wisent-0.7.925/wisent/tests/nosense/__init__.py +6 -0
- wisent-0.7.925/wisent/tests/nosense/base_nosense.py +81 -0
- wisent-0.7.925/wisent/tests/nosense/math500_nosense.py +72 -0
- wisent-0.7.925/wisent/tests/nosense/test_robustness.py +336 -0
- wisent-0.7.925/wisent/tests/test_aggregation_geometry.py +236 -0
- wisent-0.7.925/wisent/tests/test_all_cli_commands.py +674 -0
- wisent-0.7.925/wisent/tests/test_detector_accuracy.py +163 -0
- wisent-0.7.925/wisent/tests/test_geometry_comprehensive.py +327 -0
- wisent-0.7.925/wisent/tests/test_geometry_exhaustive.py +1202 -0
- wisent-0.7.925/wisent/tests/test_titan_geometry.py +257 -0
- wisent-0.7.925/wisent/tests/visualize_geometry.py +342 -0
- wisent-0.7.925/wisent.egg-info/PKG-INFO +68 -0
- wisent-0.7.925/wisent.egg-info/SOURCES.txt +1063 -0
- wisent-0.7.925/wisent.egg-info/dependency_links.txt +1 -0
- wisent-0.7.925/wisent.egg-info/entry_points.txt +2 -0
- wisent-0.7.925/wisent.egg-info/requires.txt +23 -0
- wisent-0.7.925/wisent.egg-info/top_level.txt +1 -0
wisent-0.7.925/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023-2024 Wisent Team
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
wisent-0.7.925/PKG-INFO
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: wisent
|
|
3
|
+
Version: 0.7.925
|
|
4
|
+
Summary: Monitor and influence AI Brains
|
|
5
|
+
Home-page: https://github.com/wisent-ai/wisent
|
|
6
|
+
Author: Lukasz Bartoszcze and the Wisent Team
|
|
7
|
+
Author-email: lukasz.bartoszcze@wisent.ai
|
|
8
|
+
Keywords: nlp,machine learning,language models,safety,guardrails,lm-evaluation-harness
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
15
|
+
Requires-Python: >=3.8
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: torch>=1.9.0
|
|
19
|
+
Requires-Dist: transformers>=4.20.0
|
|
20
|
+
Requires-Dist: tqdm>=4.50.0
|
|
21
|
+
Requires-Dist: scikit-learn>=0.24.0
|
|
22
|
+
Requires-Dist: pandas>=1.2.0
|
|
23
|
+
Requires-Dist: numpy>=1.21.0
|
|
24
|
+
Requires-Dist: datasets>=2.0.0
|
|
25
|
+
Requires-Dist: sentence-transformers>=2.0.0
|
|
26
|
+
Requires-Dist: faiss-cpu>=1.7.0
|
|
27
|
+
Requires-Dist: uncensorbench>=0.2.0
|
|
28
|
+
Requires-Dist: pebble>=5.0.0
|
|
29
|
+
Requires-Dist: latex2sympy2_extended>=1.0.0
|
|
30
|
+
Requires-Dist: sae_lens>=0.1.0
|
|
31
|
+
Requires-Dist: trl>=0.7.0
|
|
32
|
+
Provides-Extra: harness
|
|
33
|
+
Requires-Dist: lm-eval==0.4.8; extra == "harness"
|
|
34
|
+
Provides-Extra: cuda
|
|
35
|
+
Requires-Dist: flash-attn>=2.5.0; extra == "cuda"
|
|
36
|
+
Provides-Extra: sparsify
|
|
37
|
+
Requires-Dist: sparsify>=0.1.0; extra == "sparsify"
|
|
38
|
+
Dynamic: author
|
|
39
|
+
Dynamic: author-email
|
|
40
|
+
Dynamic: classifier
|
|
41
|
+
Dynamic: description
|
|
42
|
+
Dynamic: description-content-type
|
|
43
|
+
Dynamic: home-page
|
|
44
|
+
Dynamic: keywords
|
|
45
|
+
Dynamic: license-file
|
|
46
|
+
Dynamic: provides-extra
|
|
47
|
+
Dynamic: requires-dist
|
|
48
|
+
Dynamic: requires-python
|
|
49
|
+
Dynamic: summary
|
|
50
|
+
|
|
51
|
+
<p align="center">
|
|
52
|
+
<img src="banner.png" alt="Wisent Banner" width="100%">
|
|
53
|
+
</p>
|
|
54
|
+
|
|
55
|
+
<p align="center">
|
|
56
|
+
<code>pip install wisent</code>
|
|
57
|
+
</p>
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
## Overview
|
|
62
|
+
|
|
63
|
+
Wisent allows you to control your AI by identifying brain patterns corresponding to responses you don't like, like hallucinations or harmful outputs. We use contrastive pairs of representations to detect when a model might be generating harmful content or hallucinating. Learn more at [wisent.ai/documentation](https://www.wisent.ai/documentation).
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
## License
|
|
67
|
+
|
|
68
|
+
This project is licensed under the MIT License - see the LICENSE file for details.
|
wisent-0.7.925/README.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="banner.png" alt="Wisent Banner" width="100%">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
<code>pip install wisent</code>
|
|
7
|
+
</p>
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
## Overview
|
|
12
|
+
|
|
13
|
+
Wisent allows you to control your AI by identifying brain patterns corresponding to responses you don't like, like hallucinations or harmful outputs. We use contrastive pairs of representations to detect when a model might be generating harmful content or hallucinating. Learn more at [wisent.ai/documentation](https://www.wisent.ai/documentation).
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
## License
|
|
17
|
+
|
|
18
|
+
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
[tool.pytest.ini_options]
|
|
2
|
+
testpaths = ["wisent/tests"]
|
|
3
|
+
python_files = ["test_*.py"]
|
|
4
|
+
python_classes = ["Test*"]
|
|
5
|
+
python_functions = ["test_*"]
|
|
6
|
+
addopts = ["-v", "--tb=short"]
|
|
7
|
+
markers = [
|
|
8
|
+
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
|
|
9
|
+
"heavy: marks tests as very resource-intensive with large model downloads (deselect with '-m \"not heavy\"')",
|
|
10
|
+
"model_download: marks tests that download models from HuggingFace (deselect with '-m \"not model_download\"')",
|
|
11
|
+
"integration: marks tests as integration tests",
|
|
12
|
+
"unit: marks tests as unit tests",
|
|
13
|
+
"docker: marks tests for Docker-based execution",
|
|
14
|
+
"performance: marks tests for performance evaluation",
|
|
15
|
+
"cli: marks tests for CLI integration",
|
|
16
|
+
"sandbox_required: marks tests that require sandbox environment (includes coding tasks and trust_remote_code)",
|
|
17
|
+
"bigcode_required: marks tests that require bigcode-evaluation-harness installation",
|
|
18
|
+
]
|
|
19
|
+
filterwarnings = [
|
|
20
|
+
"ignore::DeprecationWarning",
|
|
21
|
+
"ignore::PendingDeprecationWarning",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[tool.ruff]
|
|
25
|
+
line-length = 120
|
|
26
|
+
target-version = "py39"
|
|
27
|
+
exclude = [
|
|
28
|
+
".bzr",
|
|
29
|
+
".direnv",
|
|
30
|
+
".eggs",
|
|
31
|
+
".git",
|
|
32
|
+
".git-rewrite",
|
|
33
|
+
".hg",
|
|
34
|
+
".mypy_cache",
|
|
35
|
+
".nox",
|
|
36
|
+
".pants.d",
|
|
37
|
+
".pytype",
|
|
38
|
+
".ruff_cache",
|
|
39
|
+
".svn",
|
|
40
|
+
".tox",
|
|
41
|
+
".venv",
|
|
42
|
+
"__pypackages__",
|
|
43
|
+
"_build",
|
|
44
|
+
"buck-out",
|
|
45
|
+
"build",
|
|
46
|
+
"dist",
|
|
47
|
+
"node_modules",
|
|
48
|
+
"venv",
|
|
49
|
+
"evaluation",
|
|
50
|
+
"bigcode-evaluation-harness",
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
[tool.ruff.lint]
|
|
54
|
+
# Enable common rule sets
|
|
55
|
+
select = ["E4", "E7", "E9", "F", "I", "N", "UP", "B", "A", "C4", "ISC", "ICN", "PIE", "T20", "PT", "Q", "RSE", "RET", "SLF", "SIM", "TCH", "ARG", "ERA", "PGH", "RUF"]
|
|
56
|
+
ignore = [
|
|
57
|
+
"PLR0913", # Too many arguments in function definition
|
|
58
|
+
"PLR0912", # Too many branches
|
|
59
|
+
"PLR0915", # Too many statements
|
|
60
|
+
"C901", # Function is too complex
|
|
61
|
+
"PLR2004", # Magic value used in comparison
|
|
62
|
+
"TRY003", # Avoid specifying long messages outside exception class
|
|
63
|
+
"PLW2901", # Redefined loop variable
|
|
64
|
+
"B008", # Do not perform function calls in argument defaults
|
|
65
|
+
"ARG002", # Unused method argument
|
|
66
|
+
"PIE800", # Unnecessary spread operator,
|
|
67
|
+
"T201" # print
|
|
68
|
+
]
|
|
69
|
+
fixable = ["ALL"]
|
|
70
|
+
unfixable = []
|
|
71
|
+
|
|
72
|
+
[tool.ruff.format]
|
|
73
|
+
# Like Black, use double quotes for strings.
|
|
74
|
+
quote-style = "double"
|
|
75
|
+
# Like Black, indent with spaces, rather than tabs.
|
|
76
|
+
indent-style = "space"
|
|
77
|
+
# Like Black, respect magic trailing commas.
|
|
78
|
+
skip-magic-trailing-comma = false
|
|
79
|
+
# Like Black, automatically detect the appropriate line ending.
|
|
80
|
+
line-ending = "auto"
|
|
81
|
+
|
|
82
|
+
[tool.ruff.lint.isort]
|
|
83
|
+
known-first-party = ["wisent"]
|
|
84
|
+
force-single-line = false
|
|
85
|
+
combine-as-imports = true
|
wisent-0.7.925/setup.cfg
ADDED
wisent-0.7.925/setup.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from setuptools import setup, find_packages
|
|
3
|
+
|
|
4
|
+
# Read the contents of the README file
|
|
5
|
+
with open("README.md", encoding="utf-8") as f:
|
|
6
|
+
long_description = f.read()
|
|
7
|
+
|
|
8
|
+
# Read version from __init__.py
|
|
9
|
+
with open(os.path.join("wisent", "__init__.py"), encoding="utf-8") as f:
|
|
10
|
+
for line in f:
|
|
11
|
+
if line.startswith("__version__"):
|
|
12
|
+
version = line.split("=")[1].strip().strip('"').strip("'")
|
|
13
|
+
break
|
|
14
|
+
|
|
15
|
+
setup(
|
|
16
|
+
name="wisent",
|
|
17
|
+
version=version,
|
|
18
|
+
author="Lukasz Bartoszcze and the Wisent Team",
|
|
19
|
+
author_email="lukasz.bartoszcze@wisent.ai", # Replace with your email
|
|
20
|
+
description="Monitor and influence AI Brains",
|
|
21
|
+
long_description=long_description,
|
|
22
|
+
long_description_content_type="text/markdown",
|
|
23
|
+
url="https://github.com/wisent-ai/wisent", # Replace with your GitHub repo
|
|
24
|
+
packages=find_packages(exclude=["patches", "patches.*"]), # Exclude patches directory
|
|
25
|
+
include_package_data=True,
|
|
26
|
+
package_data={
|
|
27
|
+
"wisent": [
|
|
28
|
+
"core/evaluators/benchmark_specific/coding/safe_docker/Dockerfile",
|
|
29
|
+
"core/evaluators/benchmark_specific/coding/safe_docker/entrypoint.py",
|
|
30
|
+
"parameters/lm_eval/*.json",
|
|
31
|
+
"parameters/tasks/*.json",
|
|
32
|
+
"examples/scripts/*.json",
|
|
33
|
+
"scripts/*.sh",
|
|
34
|
+
],
|
|
35
|
+
},
|
|
36
|
+
classifiers=[
|
|
37
|
+
"Programming Language :: Python :: 3",
|
|
38
|
+
"License :: OSI Approved :: MIT License",
|
|
39
|
+
"Operating System :: OS Independent",
|
|
40
|
+
"Development Status :: 4 - Beta",
|
|
41
|
+
"Intended Audience :: Science/Research",
|
|
42
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
43
|
+
],
|
|
44
|
+
python_requires=">=3.8",
|
|
45
|
+
install_requires=[
|
|
46
|
+
"torch>=1.9.0",
|
|
47
|
+
"transformers>=4.20.0",
|
|
48
|
+
"tqdm>=4.50.0",
|
|
49
|
+
"scikit-learn>=0.24.0",
|
|
50
|
+
"pandas>=1.2.0",
|
|
51
|
+
"numpy>=1.21.0",
|
|
52
|
+
"datasets>=2.0.0",
|
|
53
|
+
"sentence-transformers>=2.0.0",
|
|
54
|
+
"faiss-cpu>=1.7.0",
|
|
55
|
+
"uncensorbench>=0.2.0",
|
|
56
|
+
"pebble>=5.0.0",
|
|
57
|
+
"latex2sympy2_extended>=1.0.0",
|
|
58
|
+
"sae_lens>=0.1.0",
|
|
59
|
+
"trl>=0.7.0",
|
|
60
|
+
],
|
|
61
|
+
extras_require={
|
|
62
|
+
"harness": [
|
|
63
|
+
"lm-eval==0.4.8",
|
|
64
|
+
],
|
|
65
|
+
"cuda": [
|
|
66
|
+
"flash-attn>=2.5.0",
|
|
67
|
+
],
|
|
68
|
+
"sparsify": [
|
|
69
|
+
"sparsify>=0.1.0",
|
|
70
|
+
],
|
|
71
|
+
},
|
|
72
|
+
entry_points={
|
|
73
|
+
"console_scripts": [
|
|
74
|
+
"wisent=wisent.core.main:main",
|
|
75
|
+
],
|
|
76
|
+
},
|
|
77
|
+
keywords="nlp, machine learning, language models, safety, guardrails, lm-evaluation-harness",
|
|
78
|
+
)
|