wisent 0.7.379__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wisent/__init__.py +64 -0
- wisent/cli.py +114 -0
- wisent/core/__init__.py +40 -0
- wisent/core/activations/__init__.py +26 -0
- wisent/core/activations/activations.py +97 -0
- wisent/core/activations/activations_collector.py +506 -0
- wisent/core/activations/core/__init__.py +0 -0
- wisent/core/activations/core/atoms.py +219 -0
- wisent/core/activations/prompt_construction_strategy.py +47 -0
- wisent/core/adapters/__init__.py +22 -0
- wisent/core/adapters/audio.py +616 -0
- wisent/core/adapters/base.py +420 -0
- wisent/core/adapters/multimodal.py +738 -0
- wisent/core/adapters/robotics.py +643 -0
- wisent/core/adapters/text.py +441 -0
- wisent/core/adapters/video.py +555 -0
- wisent/core/agent/__init__.py +1 -0
- wisent/core/agent/budget.py +644 -0
- wisent/core/agent/device_benchmarks.py +691 -0
- wisent/core/agent/diagnose/__init__.py +1 -0
- wisent/core/agent/diagnose/agent_classifier_decision.py +641 -0
- wisent/core/agent/diagnose/classifier_marketplace.py +554 -0
- wisent/core/agent/diagnose/create_classifier.py +1155 -0
- wisent/core/agent/diagnose/response_diagnostics.py +273 -0
- wisent/core/agent/diagnose/select_classifiers.py +507 -0
- wisent/core/agent/diagnose/synthetic_classifier_option.py +755 -0
- wisent/core/agent/diagnose/tasks/__init__.py +33 -0
- wisent/core/agent/diagnose/tasks/task_manager.py +1453 -0
- wisent/core/agent/diagnose/tasks/task_relevance.py +94 -0
- wisent/core/agent/diagnose/tasks/task_selector.py +151 -0
- wisent/core/agent/diagnose.py +249 -0
- wisent/core/agent/steer.py +215 -0
- wisent/core/agent/timeout.py +134 -0
- wisent/core/autonomous_agent.py +1158 -0
- wisent/core/benchmark_extractors.py +372 -0
- wisent/core/benchmark_registry.py +151 -0
- wisent/core/bigcode_extractors.py +26 -0
- wisent/core/bigcode_integration.py +886 -0
- wisent/core/branding.py +108 -0
- wisent/core/classifier/__init__.py +1 -0
- wisent/core/classifier/models/__init__.py +1 -0
- wisent/core/classifiers/__init__.py +1 -0
- wisent/core/classifiers/classifiers/__init__.py +0 -0
- wisent/core/classifiers/classifiers/core/__init__.py +0 -0
- wisent/core/classifiers/classifiers/core/atoms.py +748 -0
- wisent/core/classifiers/classifiers/models/__init__.py +0 -0
- wisent/core/classifiers/classifiers/models/logistic.py +29 -0
- wisent/core/classifiers/classifiers/models/mlp.py +47 -0
- wisent/core/classifiers/classifiers/rotator.py +137 -0
- wisent/core/classifiers/core/__init__.py +1 -0
- wisent/core/classifiers/models/__init__.py +1 -0
- wisent/core/classifiers/pipeline_steps/__init__.py +1 -0
- wisent/core/cli/__init__.py +26 -0
- wisent/core/cli/agent/__init__.py +15 -0
- wisent/core/cli/agent/apply_steering.py +192 -0
- wisent/core/cli/agent/evaluate_response.py +128 -0
- wisent/core/cli/agent/generate_synthetic_pairs.py +123 -0
- wisent/core/cli/agent/main.py +139 -0
- wisent/core/cli/agent/train_classifier.py +173 -0
- wisent/core/cli/check_linearity.py +126 -0
- wisent/core/cli/create_steering_vector.py +304 -0
- wisent/core/cli/diagnose_pairs.py +153 -0
- wisent/core/cli/diagnose_vectors.py +404 -0
- wisent/core/cli/estimate_unified_goodness_time.py +428 -0
- wisent/core/cli/evaluate_refusal.py +241 -0
- wisent/core/cli/evaluate_responses.py +926 -0
- wisent/core/cli/generate_humanization_pairs.py +128 -0
- wisent/core/cli/generate_pairs.py +175 -0
- wisent/core/cli/generate_pairs_from_task.py +108 -0
- wisent/core/cli/generate_responses.py +160 -0
- wisent/core/cli/generate_vector_from_synthetic.py +217 -0
- wisent/core/cli/generate_vector_from_task.py +248 -0
- wisent/core/cli/get_activations.py +192 -0
- wisent/core/cli/inference_config.py +84 -0
- wisent/core/cli/inference_config_cli.py +54 -0
- wisent/core/cli/modify_weights.py +660 -0
- wisent/core/cli/multi_steer.py +112 -0
- wisent/core/cli/optimization_cache.py +298 -0
- wisent/core/cli/optimize.py +621 -0
- wisent/core/cli/optimize_classification.py +473 -0
- wisent/core/cli/optimize_sample_size.py +390 -0
- wisent/core/cli/optimize_steering.py +3421 -0
- wisent/core/cli/optimize_weights.py +1287 -0
- wisent/core/cli/steering_method_trainer.py +641 -0
- wisent/core/cli/steering_search_space.py +508 -0
- wisent/core/cli/tasks.py +940 -0
- wisent/core/cli/train_unified_goodness.py +681 -0
- wisent/core/cli_logger.py +22 -0
- wisent/core/config_manager.py +1731 -0
- wisent/core/contrastive_pairs/__init__.py +15 -0
- wisent/core/contrastive_pairs/core/__init__.py +0 -0
- wisent/core/contrastive_pairs/core/atoms.py +45 -0
- wisent/core/contrastive_pairs/core/buliders.py +59 -0
- wisent/core/contrastive_pairs/core/pair.py +183 -0
- wisent/core/contrastive_pairs/core/response.py +153 -0
- wisent/core/contrastive_pairs/core/serialization.py +306 -0
- wisent/core/contrastive_pairs/core/set.py +192 -0
- wisent/core/contrastive_pairs/diagnostics/__init__.py +79 -0
- wisent/core/contrastive_pairs/diagnostics/activations.py +53 -0
- wisent/core/contrastive_pairs/diagnostics/base.py +73 -0
- wisent/core/contrastive_pairs/diagnostics/control_vectors.py +1655 -0
- wisent/core/contrastive_pairs/diagnostics/coverage.py +79 -0
- wisent/core/contrastive_pairs/diagnostics/divergence.py +98 -0
- wisent/core/contrastive_pairs/diagnostics/duplicates.py +118 -0
- wisent/core/contrastive_pairs/diagnostics/linearity.py +325 -0
- wisent/core/contrastive_pairs/diagnostics/vector_quality.py +620 -0
- wisent/core/contrastive_pairs/huggingface_pairs/__init__.py +1 -0
- wisent/core/contrastive_pairs/huggingface_pairs/atoms.py +255 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_extractor_manifest.py +470 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_extractor_registry.py +136 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/__init__.py +44 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentbench.py +225 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentharm.py +267 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentic_search.py +444 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aider_polyglot.py +225 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime.py +118 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime2024.py +74 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime2025.py +73 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/alpaca_eval.py +153 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/apps.py +182 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/arena_hard.py +179 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/atis.py +89 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/babilong.py +96 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bangla_mmlu.py +108 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/basqueglue.py +217 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bec2016eu.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bfcl.py +283 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bhtc_v2.py +87 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/browsecomp.py +245 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/chain_of_thought.py +89 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/chinese_simpleqa.py +209 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/cluewsc.py +177 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/cnn_dailymail.py +92 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codeforces.py +378 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue.py +109 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text.py +15 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_go.py +64 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_java.py +65 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_javascript.py +65 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_php.py +65 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_python.py +65 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_ruby.py +65 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/coding_benchmarks.py +844 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/coedit_gec.py +79 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/conala.py +133 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/concode.py +111 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/dbpedia_14.py +91 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/doc_vqa.py +102 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/donotanswer.py +236 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ds1000.py +129 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ds_1000.py +155 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/epec_koref_bin.py +85 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ethos_binary.py +82 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/evalita_mp.py +165 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/evalita_sp_sum_task_fp_small_p1.py +89 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/facts_grounding.py +181 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/faithbench.py +295 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/financial_tweets.py +100 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flames.py +270 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flan_held_in.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flores.py +572 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/frames.py +143 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/freebase.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/get_negative_example_livecodebench.py +146 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/get_positive_example_livecodebench.py +140 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/gpt3_translation_benchmarks.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hallucinations_leaderboard.py +389 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/halueval.py +246 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/harmbench.py +250 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/healthbench.py +181 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hle.py +106 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hmmt.py +117 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/humaneval.py +119 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/humanevalpack.py +102 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/instruct_humaneval.py +180 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/instructhumaneval.py +129 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/iwslt2017_ar_en.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/iwslt2017_en_ar.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/jailbreakbench.py +258 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/law_stack_exchange.py +101 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ledgar.py +118 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench.py +61 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench_contrastive_pair_generator.py +491 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench_v6.py +263 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livemathbench.py +230 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/llama.py +96 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/longform_writing.py +285 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/m_mmlu.py +96 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/math.py +186 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/math500.py +146 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mbpp.py +142 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/meddialog.py +79 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/medical_abstracts.py +101 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/medium_priority_benchmarks.py +787 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mercury.py +111 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mmlu_redux.py +194 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mmlusr.py +108 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multimedqa.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multipl_e.py +109 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple.py +96 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_choice.py +87 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_cpp.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_go.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_java.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_js.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_py.py +15 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_rs.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/non_greedy_robustness_agieval_aqua_rat.py +92 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/olympiadbench.py +287 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/openllm.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/option_order_robustness_agieval_aqua_rat.py +92 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/or_bench.py +300 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/penn_treebank.py +80 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/planbench.py +317 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/polymath.py +467 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/prompt_robustness_agieval_aqua_rat.py +92 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/pythia.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/recode.py +131 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/refusalbench.py +280 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/scicode.py +275 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/self_consistency.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/simpleqa.py +145 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/sorry_bench.py +211 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/stsb.py +79 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_lm_eval_v1.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_lm_eval_v1_seq2seq.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_t5_prompt.py +123 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_gpqa.py +106 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/swe_bench.py +428 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/swe_bench_verified.py +158 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/sycophancy_eval.py +205 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/t0_eval.py +79 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tag.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tau_bench.py +305 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tmlu.py +109 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/toolbench.py +360 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/toolemu.py +386 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/travelplanner.py +286 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/truthfulqa_generation.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/unfair_tos.py +83 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/vaxx_stance.py +86 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wiceu.py +85 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wikitext103.py +97 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wildguard.py +280 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt14_en_fr.py +97 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt14_fr_en.py +97 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_de_en.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_en_de.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_en_ro.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_ro_en.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt_ro_en_t5_prompt.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/xsum.py +81 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/__init__.py +0 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/atoms.py +265 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/__init__.py +472 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aclue.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/acp.py +33 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/acpbench.py +39 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/advanced_ai_risk.py +59 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aexams.py +14 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrimgsm.py +10 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrimmlu.py +10 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrixnli.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench.py +14 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_adr.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_afriqa.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_afrisenti.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_belebele.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_flores.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_injongointent.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_mafand.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhaner.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhanews.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhapos.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_naijarc.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_nollysenti.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_ntrex.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_openai_mmlu.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_salt.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_sib.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_uhura_arc_easy.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_xlsum.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/agieval.py +33 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/anli.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arab_culture.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_acva.py +67 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_acva_light.py +67 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_complete.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_light.py +81 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabicmmlu.py +59 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aradice.py +36 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arc.py +61 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arithmetic.py +19 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/basque_bench.py +37 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bbh.py +121 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bbq.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/belebele.py +293 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bertaqa.py +25 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bigbench.py +300 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/blimp.py +76 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/careqa.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/catalan_bench.py +43 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ceval_valid.py +61 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/cmmlu.py +76 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/code_x_glue.py +16 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/copal_id.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/crows_pairs.py +31 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/csatqa.py +15 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/darija.py +29 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/darijammlu.py +57 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/egymmlu.py +62 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/eus.py +76 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/evalita_mp.py +93 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/fld.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/flores.py +466 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/freebase.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/french_bench.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/galician_bench.py +41 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/glianorex.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/global_mmlu.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gpqa.py +27 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gsm8k.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gsm8k_platinum.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/haerae.py +14 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/headqa.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hellaswag.py +39 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hendrycks_ethics.py +14 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hendrycks_math.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hrm8k.py +20 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/inverse.py +22 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/japanese_leaderboard.py +20 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/jsonschema_bench.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kbl.py +85 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kmmlu.py +281 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kobest.py +14 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kormedmcqa.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/lambada.py +28 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/leaderboard.py +52 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/libra.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/lingoly.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/longbench.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/m.py +43 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mastermind.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mathqa.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/med.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/meddialog.py +12 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/medqa.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mela.py +18 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/metabench.py +36 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mgsm.py +44 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/minerva_math.py +16 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mlqa.py +58 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu.py +70 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_pro.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_pro_plus.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_prox.py +191 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlusr.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmmu.py +46 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/model_written_evals.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/multiblimp.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/non.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/noreval.py +143 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/noridiom.py +20 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/nortruthfulqa.py +32 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/nrk.py +20 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_arc_multilingual.py +10 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_hellaswag_multilingual.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_mmlu_multilingual.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_truthfulqa_multilingual.py +34 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/paloma.py +25 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/pawsx.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/persona.py +144 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/pile.py +31 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/polemo2.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/portuguese_bench.py +31 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/prompt.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/qa4mre.py +12 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/qasper.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ru.py +19 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ruler.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/score.py +20 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/scrolls.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/self_consistency.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/spanish_bench.py +38 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/storycloze.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/super_glue_t5_prompt.py +17 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tinyBenchmarks.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tmlu.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tmmluplus.py +80 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/translation.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/truthfulqa.py +76 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/truthfulqa_multi.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/turkishmmlu.py +30 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/unitxt.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/unscramble.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/winogender.py +16 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmdp.py +12 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmt14.py +16 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmt16.py +22 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wsc273.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xcopa.py +21 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xnli.py +28 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xnli_eu.py +12 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xquad.py +22 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xstorycloze.py +22 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xwinograd.py +15 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_extractor_manifest.py +478 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_extractor_registry.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/__init__.py +125 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aclue.py +171 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/acp_bench.py +207 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/acp_bench_hard.py +185 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/advanced.py +130 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aexams.py +184 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrimgsm.py +98 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrimmlu.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrixnli.py +129 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrobench_cot.py +88 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrobench_mc.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ag.py +134 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/agieval.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ai2_arc.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anagrams1.py +81 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anagrams2.py +81 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anli.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabculture.py +180 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic.py +98 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_exams.py +104 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_leaderboard_complete.py +168 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_leaderboard_light.py +168 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabicmmlu.py +167 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aradice.py +268 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc.py +133 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_challenge.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_easy.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_gen.py +101 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_mc.py +106 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/argument.py +134 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arithmetic.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/asdiv.py +122 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/assin.py +103 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/babi.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench_gen.py +168 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench_mc.py +139 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bbh.py +133 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bbq.py +169 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/belebele.py +181 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/benchmarks.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bertaqa.py +165 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bhs.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bhtc.py +143 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bigbench.py +170 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/blimp.py +171 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/blimp_nl.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/boolq.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/boolq_seq2seq.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/c4.py +150 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cabbq.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cabreu.py +127 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/careqa.py +169 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench_gen.py +119 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench_mc.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalanqa.py +171 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catcola.py +139 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cb.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ceval.py +223 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ceval_valid.py +163 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/chain.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/chartqa.py +238 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/claim.py +151 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/click.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cmmlu.py +166 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cnn.py +144 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cocoteros.py +148 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/code2text.py +161 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/code_x_glue.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/codexglue.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coedit.py +149 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cola.py +83 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/commonsense.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/commonsense_qa.py +127 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/copa.py +124 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/copal_id.py +169 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coqa.py +162 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coqcat.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/crows_pairs.py +158 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/csatqa.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cycle.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cycle_letters.py +81 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darija_bench.py +221 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darijahellaswag.py +174 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darijammlu.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/dbpedia.py +157 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/discrim_eval.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/doc.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/drop.py +129 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/egyhellaswag.py +125 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/egymmlu.py +180 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/epec.py +142 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench.py +194 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench_ca.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench_es.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/esbbq.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/escola.py +85 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ethics.py +135 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ethos.py +99 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_exams.py +225 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_proficiency.py +159 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_reading.py +159 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_trivia.py +159 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/evalita_llm.py +166 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/evalita_sp.py +109 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/fda.py +105 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/financial.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/flan.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/fld.py +143 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench.py +202 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench_mc.py +98 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench_perplexity.py +86 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galcola.py +109 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench_gen.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench_mc.py +112 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gaokao.py +141 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/glianorex.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/global_mmlu.py +171 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/global_piqa.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/glue.py +109 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gpqa.py +161 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gpt3.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/groundcocoa.py +184 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gsm.py +108 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gsm8k.py +134 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/haerae.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/headqa.py +112 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hellaswag.py +125 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hendrycks_ethics.py +225 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hendrycks_math.py +191 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/histoires_morales.py +179 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hle.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hrm8k.py +203 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/humaneval.py +124 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/humaneval_infilling.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/icelandic_winogrande.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ifeval.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/inverse.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/inverse_scaling.py +192 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/iwslt2017.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ja.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard_gen.py +224 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard_mc.py +120 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/jsonschema_bench.py +123 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kbl.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu.py +168 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu_cot.py +88 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu_mc.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kobest.py +165 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kormedmcqa.py +160 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada.py +147 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_cloze.py +185 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_multilingual.py +185 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_multilingual_stablelm.py +141 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/law.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/leaderboard.py +194 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/libra.py +165 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lingoly.py +203 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/livemathbench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/llama3.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lm_syneval.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logieval.py +82 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logiqa.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logiqa2.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/longbench.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/longbenchv2.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mastermind.py +203 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mathqa.py +137 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mbpp.py +123 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mc-taco.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/med_concepts_qa.py +224 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/meddialog.py +180 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medical.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mediqa_qa2019.py +123 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medmcqa.py +169 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medqa.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medtext.py +108 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mela.py +96 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/meqsum.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/metabench.py +154 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mgsm.py +122 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mimic_repsum.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/minerva_math.py +172 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mlqa.py +143 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu.py +144 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_cot.py +88 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_mc.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_pro.py +145 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlusr.py +189 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmmu.py +150 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mnli.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/model_written_evals.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/moral_stories.py +151 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mrpc.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mts_dialog.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mts_dialog_perplexity.py +97 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multiblimp.py +134 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multilingual.py +106 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multirc.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mutual.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/non.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval.py +173 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_exact.py +157 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_gen.py +277 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_gen_exact.py +165 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc.py +228 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc_log_likelihoods.py +223 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noticia.py +105 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/nq_open.py +135 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi.py +27 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_arc_multilingual.py +167 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_hellaswag_multilingual.py +174 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_mmlu_multilingual.py +162 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_truthfulqa_multilingual.py +209 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph.py +186 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph_perplexity.py +97 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/openbookqa.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/option.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paloma.py +205 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafraseja.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafrases.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws_x.py +154 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pawsx.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/persona.py +246 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases.py +144 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases_ca_va.py +82 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile.py +161 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile_10k.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/piqa.py +116 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/polemo2.py +135 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/polymath.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench_gen.py +121 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench_mc.py +103 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prompt.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prost.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pubmedqa.py +112 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qa4mre.py +119 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper_bool.py +112 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnli.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnlieu.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qqp.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/quac.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/race.py +124 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/random.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/realtoxicityprompts.py +124 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/record.py +125 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/reversed.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/rte.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ruler.py +170 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sciq.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/score.py +177 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls.py +161 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls_mc.py +157 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/self.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue.py +131 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue_rte.py +119 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/simple_cooccurrence_bias.py +121 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/siqa.py +209 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/social_iqa.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench_gen.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench_mc.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/squad2.py +129 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/squad_completion.py +121 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sst2.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/storycloze.py +250 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/summarization.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/super.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/super_glue.py +154 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/superglue.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/supergpqa.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/swag.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/swde.py +179 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sycophancy.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/t0.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/teca.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinyarc.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinybenchmarks.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinygsm8k.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinyhellaswag.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinymmlu.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinytruthfulqa.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinywinogrande.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tmmluplus.py +181 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/toxigen.py +91 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/translation.py +149 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/triviaqa.py +130 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa.py +112 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc1.py +120 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc2.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_multi.py +142 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turblimp_core.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu.py +161 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu_cot.py +104 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu_mc.py +102 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/twenty_newsgroups.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/unitxt.py +131 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/unscramble.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/vaxx.py +95 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/webqs.py +130 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wic.py +122 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wikitext.py +146 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/winogender.py +139 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/winogrande.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmdp.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmt14.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmt16.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wnli.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wsc.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wsc273.py +180 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xcopa.py +197 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xlsum.py +147 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xnli.py +131 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xquad.py +203 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xstorycloze.py +129 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xwinograd.py +124 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/yahoo.py +108 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/zhoblimp.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_pairs_generation.py +56 -0
- wisent/core/data_loaders/__init__.py +235 -0
- wisent/core/data_loaders/core/__init__.py +0 -0
- wisent/core/data_loaders/core/atoms.py +99 -0
- wisent/core/data_loaders/loaders/__init__.py +0 -0
- wisent/core/data_loaders/loaders/custom.py +120 -0
- wisent/core/data_loaders/loaders/huggingface_loader.py +153 -0
- wisent/core/data_loaders/loaders/lm_loader.py +494 -0
- wisent/core/data_loaders/loaders/lm_loader_special_cases.py +496 -0
- wisent/core/data_loaders/loaders/task_interface_loader.py +300 -0
- wisent/core/data_loaders/rotator.py +118 -0
- wisent/core/detection_handling.py +259 -0
- wisent/core/diversity_processors.py +193 -0
- wisent/core/download_full_benchmarks.py +1512 -0
- wisent/core/errors/__init__.py +203 -0
- wisent/core/errors/error_codes.py +763 -0
- wisent/core/errors/error_handler.py +134 -0
- wisent/core/evaluators/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/__init__.py +42 -0
- wisent/core/evaluators/benchmark_specific/aime_evaluator.py +90 -0
- wisent/core/evaluators/benchmark_specific/coding/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/metrics/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/metrics/core/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/metrics/core/atoms.py +36 -0
- wisent/core/evaluators/benchmark_specific/coding/metrics/evaluator.py +363 -0
- wisent/core/evaluators/benchmark_specific/coding/metrics/passk.py +67 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/core/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/core/atoms.py +27 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/cpp_sanitizer.py +62 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/java_sanitizer.py +78 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/python_sanitizer.py +94 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/utils.py +126 -0
- wisent/core/evaluators/benchmark_specific/coding/providers/__init__.py +18 -0
- wisent/core/evaluators/benchmark_specific/coding/providers/core/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/providers/core/atoms.py +31 -0
- wisent/core/evaluators/benchmark_specific/coding/providers/livecodebench/__init__.py +3 -0
- wisent/core/evaluators/benchmark_specific/coding/providers/livecodebench/provider.py +305 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/Dockerfile +31 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/atoms.py +105 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/runtime.py +143 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/entrypoint.py +121 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/recipes.py +60 -0
- wisent/core/evaluators/benchmark_specific/coding/solution_generator.py +258 -0
- wisent/core/evaluators/benchmark_specific/conala_evaluator.py +332 -0
- wisent/core/evaluators/benchmark_specific/exact_match_evaluator.py +81 -0
- wisent/core/evaluators/benchmark_specific/f1_evaluator.py +173 -0
- wisent/core/evaluators/benchmark_specific/generation_evaluator.py +488 -0
- wisent/core/evaluators/benchmark_specific/livemathbench_evaluator.py +393 -0
- wisent/core/evaluators/benchmark_specific/log_likelihoods_evaluator.py +202 -0
- wisent/core/evaluators/benchmark_specific/math_evaluator.py +119 -0
- wisent/core/evaluators/benchmark_specific/math_parsing/__init__.py +1 -0
- wisent/core/evaluators/benchmark_specific/math_parsing/core.py +1640 -0
- wisent/core/evaluators/benchmark_specific/math_parsing/extract_boxed.py +48 -0
- wisent/core/evaluators/benchmark_specific/math_parsing/is_equiv.py +159 -0
- wisent/core/evaluators/benchmark_specific/math_parsing/scripts.py +919 -0
- wisent/core/evaluators/benchmark_specific/perplexity_evaluator.py +175 -0
- wisent/core/evaluators/benchmark_specific/polymath_evaluator.py +114 -0
- wisent/core/evaluators/core/__init__.py +5 -0
- wisent/core/evaluators/core/atoms.py +166 -0
- wisent/core/evaluators/custom/__init__.py +20 -0
- wisent/core/evaluators/custom/custom_evaluator.py +382 -0
- wisent/core/evaluators/custom/examples/__init__.py +37 -0
- wisent/core/evaluators/custom/examples/desklib_detector.py +166 -0
- wisent/core/evaluators/custom/examples/gptzero.py +185 -0
- wisent/core/evaluators/custom/examples/humanization.py +79 -0
- wisent/core/evaluators/custom/examples/humanization_coherent.py +127 -0
- wisent/core/evaluators/custom/examples/roberta_detector.py +173 -0
- wisent/core/evaluators/oracles/__init__.py +0 -0
- wisent/core/evaluators/oracles/interactive.py +73 -0
- wisent/core/evaluators/oracles/nlp_evaluator.py +440 -0
- wisent/core/evaluators/oracles/truthfulqa_gen_evaluator.py +168 -0
- wisent/core/evaluators/oracles/user_specified.py +67 -0
- wisent/core/evaluators/personalization/__init__.py +12 -0
- wisent/core/evaluators/personalization/alignment.py +166 -0
- wisent/core/evaluators/personalization/coherence.py +325 -0
- wisent/core/evaluators/personalization/difference.py +73 -0
- wisent/core/evaluators/rotator.py +217 -0
- wisent/core/evaluators/steering_evaluators.py +386 -0
- wisent/core/evaluators/synthetic_evaluator.py +377 -0
- wisent/core/hyperparameter_optimizer.py +547 -0
- wisent/core/layer.py +17 -0
- wisent/core/lm_eval_harness_ground_truth.py +1431 -0
- wisent/core/main.py +101 -0
- wisent/core/managed_cached_benchmarks.py +609 -0
- wisent/core/mixed_benchmark_sampler.py +366 -0
- wisent/core/modalities/__init__.py +545 -0
- wisent/core/model_persistence.py +302 -0
- wisent/core/models/__init__.py +23 -0
- wisent/core/models/core/__init__.py +0 -0
- wisent/core/models/core/atoms.py +465 -0
- wisent/core/models/inference_config.py +127 -0
- wisent/core/models/wisent_model.py +893 -0
- wisent/core/multi_steering.py +397 -0
- wisent/core/opti/__init__.py +0 -0
- wisent/core/opti/core/__init__.py +0 -0
- wisent/core/opti/core/atoms.py +177 -0
- wisent/core/opti/methods/__init__.py +10 -0
- wisent/core/opti/methods/opti_classificator.py +172 -0
- wisent/core/opti/methods/opti_steering.py +139 -0
- wisent/core/opti/methods/opti_weights.py +523 -0
- wisent/core/optuna/__init__.py +54 -0
- wisent/core/optuna/classifier/__init__.py +25 -0
- wisent/core/optuna/classifier/activation_generator.py +351 -0
- wisent/core/optuna/classifier/classifier_cache.py +509 -0
- wisent/core/optuna/classifier/optuna_classifier_optimizer.py +685 -0
- wisent/core/optuna/steering/__init__.py +20 -0
- wisent/core/optuna/steering/bigcode_evaluator_wrapper.py +200 -0
- wisent/core/optuna/steering/data_utils.py +342 -0
- wisent/core/optuna/steering/metrics.py +412 -0
- wisent/core/optuna/steering/steering_optimization.py +1096 -0
- wisent/core/parser.py +1662 -0
- wisent/core/parser_arguments/__init__.py +10 -0
- wisent/core/parser_arguments/agent_parser.py +122 -0
- wisent/core/parser_arguments/check_linearity_parser.py +82 -0
- wisent/core/parser_arguments/configure_model_parser.py +7 -0
- wisent/core/parser_arguments/create_steering_vector_parser.py +67 -0
- wisent/core/parser_arguments/diagnose_pairs_parser.py +25 -0
- wisent/core/parser_arguments/diagnose_vectors_parser.py +72 -0
- wisent/core/parser_arguments/evaluate_parser.py +40 -0
- wisent/core/parser_arguments/evaluate_refusal_parser.py +32 -0
- wisent/core/parser_arguments/evaluate_responses_parser.py +12 -0
- wisent/core/parser_arguments/full_optimize_parser.py +194 -0
- wisent/core/parser_arguments/generate_pairs_from_task_parser.py +33 -0
- wisent/core/parser_arguments/generate_pairs_parser.py +43 -0
- wisent/core/parser_arguments/generate_responses_parser.py +16 -0
- wisent/core/parser_arguments/generate_vector_from_synthetic_parser.py +148 -0
- wisent/core/parser_arguments/generate_vector_from_task_parser.py +149 -0
- wisent/core/parser_arguments/generate_vector_parser.py +89 -0
- wisent/core/parser_arguments/get_activations_parser.py +90 -0
- wisent/core/parser_arguments/inference_config_parser.py +65 -0
- wisent/core/parser_arguments/main_parser.py +220 -0
- wisent/core/parser_arguments/model_config_parser.py +59 -0
- wisent/core/parser_arguments/modify_weights_parser.py +309 -0
- wisent/core/parser_arguments/monitor_parser.py +17 -0
- wisent/core/parser_arguments/multi_steer_parser.py +48 -0
- wisent/core/parser_arguments/nonsense_parser.py +26 -0
- wisent/core/parser_arguments/optimization_cache_parser.py +64 -0
- wisent/core/parser_arguments/optimize_classification_parser.py +108 -0
- wisent/core/parser_arguments/optimize_parser.py +142 -0
- wisent/core/parser_arguments/optimize_sample_size_parser.py +58 -0
- wisent/core/parser_arguments/optimize_steering_parser.py +617 -0
- wisent/core/parser_arguments/optimize_weights_parser.py +403 -0
- wisent/core/parser_arguments/synthetic_parser.py +117 -0
- wisent/core/parser_arguments/tasks_parser.py +591 -0
- wisent/core/parser_arguments/train_unified_goodness_parser.py +172 -0
- wisent/core/parser_arguments/utils.py +107 -0
- wisent/core/prompts/__init__.py +0 -0
- wisent/core/prompts/core/__init__.py +0 -0
- wisent/core/prompts/core/atom.py +57 -0
- wisent/core/prompts/core/prompt_formater.py +148 -0
- wisent/core/prompts/prompt_stratiegies/__init__.py +0 -0
- wisent/core/prompts/prompt_stratiegies/direct_completion.py +26 -0
- wisent/core/prompts/prompt_stratiegies/instruction_following.py +26 -0
- wisent/core/prompts/prompt_stratiegies/multiple_choice.py +31 -0
- wisent/core/prompts/prompt_stratiegies/role_playing.py +33 -0
- wisent/core/representation.py +5 -0
- wisent/core/save_results.py +277 -0
- wisent/core/steering.py +660 -0
- wisent/core/steering_method.py +20 -0
- wisent/core/steering_methods/__init__.py +54 -0
- wisent/core/steering_methods/core/__init__.py +0 -0
- wisent/core/steering_methods/core/atoms.py +154 -0
- wisent/core/steering_methods/methods/__init__.py +0 -0
- wisent/core/steering_methods/methods/caa.py +45 -0
- wisent/core/steering_methods/methods/prism.py +588 -0
- wisent/core/steering_methods/methods/pulse.py +641 -0
- wisent/core/steering_methods/methods/titan.py +1005 -0
- wisent/core/steering_methods/preflight.py +322 -0
- wisent/core/steering_methods/registry.py +649 -0
- wisent/core/steering_methods/rotator.py +121 -0
- wisent/core/steering_optimizer.py +1503 -0
- wisent/core/synthetic/__init__.py +0 -0
- wisent/core/synthetic/cleaners/__init__.py +0 -0
- wisent/core/synthetic/cleaners/core/__init__.py +0 -0
- wisent/core/synthetic/cleaners/core/atoms.py +58 -0
- wisent/core/synthetic/cleaners/deduper_cleaner.py +53 -0
- wisent/core/synthetic/cleaners/methods/__init__.py +0 -0
- wisent/core/synthetic/cleaners/methods/base_dedupers.py +321 -0
- wisent/core/synthetic/cleaners/methods/base_refusalers.py +286 -0
- wisent/core/synthetic/cleaners/methods/core/__init__.py +0 -0
- wisent/core/synthetic/cleaners/methods/core/atoms.py +47 -0
- wisent/core/synthetic/cleaners/pairs_cleaner.py +90 -0
- wisent/core/synthetic/cleaners/refusaler_cleaner.py +133 -0
- wisent/core/synthetic/db_instructions/__init__.py +0 -0
- wisent/core/synthetic/db_instructions/core/__init__.py +0 -0
- wisent/core/synthetic/db_instructions/core/atoms.py +25 -0
- wisent/core/synthetic/db_instructions/mini_dp.py +115 -0
- wisent/core/synthetic/generators/__init__.py +0 -0
- wisent/core/synthetic/generators/core/__init__.py +0 -0
- wisent/core/synthetic/generators/core/atoms.py +73 -0
- wisent/core/synthetic/generators/diversities/__init__.py +0 -0
- wisent/core/synthetic/generators/diversities/core/__init__.py +0 -0
- wisent/core/synthetic/generators/diversities/core/core.py +68 -0
- wisent/core/synthetic/generators/diversities/methods/__init__.py +0 -0
- wisent/core/synthetic/generators/diversities/methods/fast_diversity.py +249 -0
- wisent/core/synthetic/generators/nonsense_generator.py +150 -0
- wisent/core/synthetic/generators/pairs_generator.py +313 -0
- wisent/core/task_interface.py +143 -0
- wisent/core/task_selector.py +232 -0
- wisent/core/tasks/__init__.py +218 -0
- wisent/core/tasks/aime_task.py +142 -0
- wisent/core/tasks/file_task.py +212 -0
- wisent/core/tasks/hle_task.py +180 -0
- wisent/core/tasks/hmmt_task.py +120 -0
- wisent/core/tasks/livecodebench_task.py +94 -0
- wisent/core/tasks/livemathbench_task.py +159 -0
- wisent/core/tasks/lm_eval_task.py +611 -0
- wisent/core/tasks/math500_task.py +84 -0
- wisent/core/tasks/polymath_task.py +147 -0
- wisent/core/tasks/supergpqa_task.py +220 -0
- wisent/core/time_estimator.py +155 -0
- wisent/core/timing_calibration.py +176 -0
- wisent/core/tracking/__init__.py +54 -0
- wisent/core/tracking/latency.py +620 -0
- wisent/core/tracking/memory.py +360 -0
- wisent/core/trainers/__init__.py +0 -0
- wisent/core/trainers/core/__init__.py +11 -0
- wisent/core/trainers/core/atoms.py +45 -0
- wisent/core/trainers/steering_trainer.py +365 -0
- wisent/core/universal_subspace.py +918 -0
- wisent/core/user_model_config.py +158 -0
- wisent/core/utils/__init__.py +64 -0
- wisent/core/utils/base_rotator.py +292 -0
- wisent/core/utils/dataset_splits.py +197 -0
- wisent/core/utils/device.py +279 -0
- wisent/core/weight_modification/__init__.py +134 -0
- wisent/core/weight_modification/additive.py +340 -0
- wisent/core/weight_modification/directional.py +1357 -0
- wisent/core/weight_modification/export.py +359 -0
- wisent/core/weight_modification/multi_direction.py +410 -0
- wisent/core/weight_modification/utils.py +236 -0
- wisent/core/wisent.py +660 -0
- wisent/examples/contrastive_pairs/humanization_human_vs_ai.json +2112 -0
- wisent/examples/scripts/1/test_basqueglue_evaluation.json +51 -0
- wisent/examples/scripts/1/test_basqueglue_pairs.json +14 -0
- wisent/examples/scripts/1/test_bec2016eu_evaluation.json +51 -0
- wisent/examples/scripts/1/test_bec2016eu_pairs.json +14 -0
- wisent/examples/scripts/1/test_belebele_evaluation.json +51 -0
- wisent/examples/scripts/1/test_belebele_pairs.json +14 -0
- wisent/examples/scripts/1/test_benchmarks_evaluation.json +51 -0
- wisent/examples/scripts/1/test_benchmarks_pairs.json +14 -0
- wisent/examples/scripts/1/test_bertaqa_evaluation.json +51 -0
- wisent/examples/scripts/1/test_bertaqa_pairs.json +14 -0
- wisent/examples/scripts/1/test_bhtc_v2_evaluation.json +30 -0
- wisent/examples/scripts/1/test_bhtc_v2_pairs.json +8 -0
- wisent/examples/scripts/1/test_boolq-seq2seq_evaluation.json +30 -0
- wisent/examples/scripts/1/test_boolq-seq2seq_pairs.json +8 -0
- wisent/examples/scripts/1/test_cabreu_evaluation.json +30 -0
- wisent/examples/scripts/1/test_cabreu_pairs.json +8 -0
- wisent/examples/scripts/1/test_careqa_en_evaluation.json +30 -0
- wisent/examples/scripts/1/test_careqa_en_pairs.json +8 -0
- wisent/examples/scripts/1/test_careqa_evaluation.json +30 -0
- wisent/examples/scripts/1/test_careqa_pairs.json +8 -0
- wisent/examples/scripts/1/test_catalanqa_evaluation.json +30 -0
- wisent/examples/scripts/1/test_catalanqa_pairs.json +8 -0
- wisent/examples/scripts/1/test_catcola_evaluation.json +30 -0
- wisent/examples/scripts/1/test_catcola_pairs.json +8 -0
- wisent/examples/scripts/1/test_chartqa_evaluation.json +30 -0
- wisent/examples/scripts/1/test_chartqa_pairs.json +8 -0
- wisent/examples/scripts/1/test_claim_stance_topic_evaluation.json +30 -0
- wisent/examples/scripts/1/test_claim_stance_topic_pairs.json +8 -0
- wisent/examples/scripts/1/test_cnn_dailymail_evaluation.json +30 -0
- wisent/examples/scripts/1/test_cnn_dailymail_pairs.json +8 -0
- wisent/examples/scripts/1/test_cocoteros_es_evaluation.json +30 -0
- wisent/examples/scripts/1/test_cocoteros_es_pairs.json +8 -0
- wisent/examples/scripts/1/test_coedit_gec_evaluation.json +30 -0
- wisent/examples/scripts/1/test_coedit_gec_pairs.json +8 -0
- wisent/examples/scripts/1/test_cola_evaluation.json +30 -0
- wisent/examples/scripts/1/test_cola_pairs.json +8 -0
- wisent/examples/scripts/1/test_coqcat_evaluation.json +30 -0
- wisent/examples/scripts/1/test_coqcat_pairs.json +8 -0
- wisent/examples/scripts/1/test_dbpedia_14_evaluation.json +30 -0
- wisent/examples/scripts/1/test_dbpedia_14_pairs.json +8 -0
- wisent/examples/scripts/1/test_epec_koref_bin_evaluation.json +30 -0
- wisent/examples/scripts/1/test_epec_koref_bin_pairs.json +8 -0
- wisent/examples/scripts/1/test_ethos_binary_evaluation.json +30 -0
- wisent/examples/scripts/1/test_ethos_binary_pairs.json +8 -0
- wisent/examples/scripts/2/test_afrimgsm_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/2/test_afrimgsm_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/2/test_afrimmlu_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/2/test_afrimmlu_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/2/test_afrixnli_en_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/2/test_afrixnli_en_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/2/test_arc_ar_evaluation.json +30 -0
- wisent/examples/scripts/2/test_arc_ar_pairs.json +8 -0
- wisent/examples/scripts/2/test_atis_evaluation.json +30 -0
- wisent/examples/scripts/2/test_atis_pairs.json +8 -0
- wisent/examples/scripts/2/test_babi_evaluation.json +30 -0
- wisent/examples/scripts/2/test_babi_pairs.json +8 -0
- wisent/examples/scripts/2/test_babilong_evaluation.json +30 -0
- wisent/examples/scripts/2/test_babilong_pairs.json +8 -0
- wisent/examples/scripts/2/test_bangla_mmlu_evaluation.json +30 -0
- wisent/examples/scripts/2/test_bangla_mmlu_pairs.json +8 -0
- wisent/examples/scripts/2/test_basque-glue_pairs.json +14 -0
- wisent/examples/scripts/benchmark_tags.json +2140 -0
- wisent/examples/scripts/lm_eval_readme.json +4 -0
- wisent/examples/scripts/results/benchmark_descriptions.json +1244 -0
- wisent/examples/scripts/results/benchmark_evaluation_methods.json +66 -0
- wisent/examples/scripts/results/benchmark_evaluator_mapping.json +2781 -0
- wisent/examples/scripts/results/benchmark_evaluator_mapping_updated.json +30536 -0
- wisent/examples/scripts/results/benchmark_evaluators_clean.json +469 -0
- wisent/examples/scripts/results/benchmark_methods_summary.json +260 -0
- wisent/examples/scripts/results/benchmark_pair_creation_methods.json +66 -0
- wisent/examples/scripts/results/benchmark_pair_totals.json +269 -0
- wisent/examples/scripts/results/benchmark_tags.json +917 -0
- wisent/examples/scripts/results/benchmark_test_summary_nov4.json +71 -0
- wisent/examples/scripts/results/coding_benchmarks_test_code_status.json +150 -0
- wisent/examples/scripts/results/failing_benchmarks.json +946 -0
- wisent/examples/scripts/results/failing_benchmarks_list.json +41 -0
- wisent/examples/scripts/results/failing_benchmarks_test_results.json +945 -0
- wisent/examples/scripts/results/missing_benchmark_tags.json +341 -0
- wisent/examples/scripts/results/test_20_newsgroups_evaluation.json +30 -0
- wisent/examples/scripts/results/test_20_newsgroups_pairs.json +8 -0
- wisent/examples/scripts/results/test_AraDICE_evaluation.json +51 -0
- wisent/examples/scripts/results/test_AraDICE_pairs.json +14 -0
- wisent/examples/scripts/results/test_AraDiCE_boolq_egy/test_AraDiCE_boolq_egy_evaluation.json +30 -0
- wisent/examples/scripts/results/test_AraDiCE_boolq_egy/test_AraDiCE_boolq_egy_pairs.json +8 -0
- wisent/examples/scripts/results/test_ArabCulture_evaluation.json +51 -0
- wisent/examples/scripts/results/test_ArabCulture_pairs.json +14 -0
- wisent/examples/scripts/results/test_Tag_evaluation.json +30 -0
- wisent/examples/scripts/results/test_Tag_pairs.json +8 -0
- wisent/examples/scripts/results/test_aclue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_aclue_pairs.json +14 -0
- wisent/examples/scripts/results/test_acp_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_acp_bench_hard_evaluation.json +51 -0
- wisent/examples/scripts/results/test_acp_bench_hard_pairs.json +14 -0
- wisent/examples/scripts/results/test_acp_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_advanced_ai_risk_evaluation.json +51 -0
- wisent/examples/scripts/results/test_advanced_ai_risk_pairs.json +14 -0
- wisent/examples/scripts/results/test_aexams_evaluation.json +51 -0
- wisent/examples/scripts/results/test_aexams_pairs.json +14 -0
- wisent/examples/scripts/results/test_afrimgsm_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/results/test_afrimgsm_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/results/test_afrimmlu_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/results/test_afrimmlu_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/results/test_afrixnli_en_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/results/test_afrixnli_en_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/results/test_ag_news_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ag_news_pairs.json +8 -0
- wisent/examples/scripts/results/test_agieval_evaluation.json +51 -0
- wisent/examples/scripts/results/test_agieval_pairs.json +14 -0
- wisent/examples/scripts/results/test_aime2024_evaluation.json +30 -0
- wisent/examples/scripts/results/test_aime2024_pairs.json +8 -0
- wisent/examples/scripts/results/test_aime2025_evaluation.json +30 -0
- wisent/examples/scripts/results/test_aime2025_pairs.json +8 -0
- wisent/examples/scripts/results/test_aime_evaluation.json +30 -0
- wisent/examples/scripts/results/test_aime_pairs.json +8 -0
- wisent/examples/scripts/results/test_anagrams1_evaluation.json +30 -0
- wisent/examples/scripts/results/test_anagrams1_pairs.json +8 -0
- wisent/examples/scripts/results/test_anagrams2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_anagrams2_pairs.json +8 -0
- wisent/examples/scripts/results/test_anli_evaluation.json +30 -0
- wisent/examples/scripts/results/test_anli_pairs.json +8 -0
- wisent/examples/scripts/results/test_apps_evaluation.json +30 -0
- wisent/examples/scripts/results/test_apps_pairs.json +8 -0
- wisent/examples/scripts/results/test_arabic_exams_evaluation.json +30 -0
- wisent/examples/scripts/results/test_arabic_exams_pairs.json +8 -0
- wisent/examples/scripts/results/test_arabic_leaderboard_complete_evaluation.json +51 -0
- wisent/examples/scripts/results/test_arabic_leaderboard_complete_pairs.json +14 -0
- wisent/examples/scripts/results/test_arabic_leaderboard_light_evaluation.json +51 -0
- wisent/examples/scripts/results/test_arabic_leaderboard_light_pairs.json +14 -0
- wisent/examples/scripts/results/test_arabicmmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_arabicmmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_aradice/test_aradice_evaluation.json +51 -0
- wisent/examples/scripts/results/test_aradice/test_aradice_pairs.json +14 -0
- wisent/examples/scripts/results/test_aradice3/test_aradice_evaluation.json +51 -0
- wisent/examples/scripts/results/test_aradice3/test_aradice_pairs.json +14 -0
- wisent/examples/scripts/results/test_arc_ar_evaluation.json +30 -0
- wisent/examples/scripts/results/test_arc_ar_pairs.json +8 -0
- wisent/examples/scripts/results/test_arc_challenge_evaluation.json +30 -0
- wisent/examples/scripts/results/test_arc_challenge_pairs.json +8 -0
- wisent/examples/scripts/results/test_arc_easy_evaluation.json +30 -0
- wisent/examples/scripts/results/test_arc_easy_pairs.json +8 -0
- wisent/examples/scripts/results/test_argument_topic_evaluation.json +30 -0
- wisent/examples/scripts/results/test_argument_topic_pairs.json +8 -0
- wisent/examples/scripts/results/test_arithmetic_evaluation.json +51 -0
- wisent/examples/scripts/results/test_arithmetic_pairs.json +14 -0
- wisent/examples/scripts/results/test_asdiv_evaluation.json +30 -0
- wisent/examples/scripts/results/test_asdiv_pairs.json +8 -0
- wisent/examples/scripts/results/test_assin_entailment_evaluation.json +30 -0
- wisent/examples/scripts/results/test_assin_entailment_pairs.json +8 -0
- wisent/examples/scripts/results/test_atis_evaluation.json +30 -0
- wisent/examples/scripts/results/test_atis_pairs.json +8 -0
- wisent/examples/scripts/results/test_babi_evaluation.json +30 -0
- wisent/examples/scripts/results/test_babi_pairs.json +8 -0
- wisent/examples/scripts/results/test_babilong_evaluation.json +30 -0
- wisent/examples/scripts/results/test_babilong_pairs.json +8 -0
- wisent/examples/scripts/results/test_bangla_mmlu_evaluation.json +30 -0
- wisent/examples/scripts/results/test_bangla_mmlu_pairs.json +8 -0
- wisent/examples/scripts/results/test_banking77_evaluation.json +30 -0
- wisent/examples/scripts/results/test_banking77_pairs.json +8 -0
- wisent/examples/scripts/results/test_basque/test_basque-glue_pairs.json +14 -0
- wisent/examples/scripts/results/test_basque-glue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_basque-glue_pairs.json +14 -0
- wisent/examples/scripts/results/test_basque2/test_basque-glue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_basque2/test_basque-glue_pairs.json +14 -0
- wisent/examples/scripts/results/test_basque_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_basque_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_basque_glue/test_basque-glue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_basque_glue/test_basque-glue_pairs.json +14 -0
- wisent/examples/scripts/results/test_basqueglue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_basqueglue_pairs.json +14 -0
- wisent/examples/scripts/results/test_bbh_evaluation.json +51 -0
- wisent/examples/scripts/results/test_bbh_pairs.json +14 -0
- wisent/examples/scripts/results/test_bbq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_bbq_pairs.json +8 -0
- wisent/examples/scripts/results/test_bec2016eu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_bec2016eu_pairs.json +14 -0
- wisent/examples/scripts/results/test_belebele_evaluation.json +51 -0
- wisent/examples/scripts/results/test_belebele_pairs.json +14 -0
- wisent/examples/scripts/results/test_benchmarks_evaluation.json +51 -0
- wisent/examples/scripts/results/test_benchmarks_pairs.json +14 -0
- wisent/examples/scripts/results/test_bertaqa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_bertaqa_pairs.json +14 -0
- wisent/examples/scripts/results/test_bhtc_v2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_bhtc_v2_pairs.json +8 -0
- wisent/examples/scripts/results/test_bigbench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_bigbench_pairs.json +14 -0
- wisent/examples/scripts/results/test_blimp_evaluation.json +51 -0
- wisent/examples/scripts/results/test_blimp_pairs.json +14 -0
- wisent/examples/scripts/results/test_boolq/test_boolq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_boolq/test_boolq_pairs.json +8 -0
- wisent/examples/scripts/results/test_boolq-seq2seq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_boolq-seq2seq_pairs.json +8 -0
- wisent/examples/scripts/results/test_boolq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_boolq_pairs.json +8 -0
- wisent/examples/scripts/results/test_c4_evaluation.json +30 -0
- wisent/examples/scripts/results/test_c4_pairs.json +8 -0
- wisent/examples/scripts/results/test_cabreu_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cabreu_pairs.json +8 -0
- wisent/examples/scripts/results/test_careqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_careqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_catalan_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_catalan_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_catalanqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_catalanqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_catcola_evaluation.json +30 -0
- wisent/examples/scripts/results/test_catcola_pairs.json +8 -0
- wisent/examples/scripts/results/test_cb_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cb_pairs.json +8 -0
- wisent/examples/scripts/results/test_ceval/test_ceval_evaluation.json +51 -0
- wisent/examples/scripts/results/test_ceval/test_ceval_pairs.json +14 -0
- wisent/examples/scripts/results/test_ceval_accountant/test_ceval-valid_accountant_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ceval_accountant/test_ceval-valid_accountant_pairs.json +8 -0
- wisent/examples/scripts/results/test_ceval_evaluation.json +51 -0
- wisent/examples/scripts/results/test_ceval_pairs.json +14 -0
- wisent/examples/scripts/results/test_ceval_valid/test_ceval_valid_evaluation.json +51 -0
- wisent/examples/scripts/results/test_ceval_valid/test_ceval_valid_pairs.json +14 -0
- wisent/examples/scripts/results/test_chain_of_thought_evaluation.json +51 -0
- wisent/examples/scripts/results/test_chain_of_thought_pairs.json +14 -0
- wisent/examples/scripts/results/test_chartqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_chartqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_claim_stance_topic_evaluation.json +30 -0
- wisent/examples/scripts/results/test_claim_stance_topic_pairs.json +8 -0
- wisent/examples/scripts/results/test_cmmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_cmmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_cnn_dailymail_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cnn_dailymail_pairs.json +8 -0
- wisent/examples/scripts/results/test_cocoteros_es_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cocoteros_es_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_go_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_go_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_java_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_java_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_javascript_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_javascript_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_php_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_php_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_python_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_python_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_ruby_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_ruby_pairs.json +8 -0
- wisent/examples/scripts/results/test_coedit_gec_evaluation.json +30 -0
- wisent/examples/scripts/results/test_coedit_gec_pairs.json +8 -0
- wisent/examples/scripts/results/test_cola_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cola_pairs.json +8 -0
- wisent/examples/scripts/results/test_commonsense_qa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_commonsense_qa_pairs.json +8 -0
- wisent/examples/scripts/results/test_conala_evaluation.json +30 -0
- wisent/examples/scripts/results/test_conala_pairs.json +8 -0
- wisent/examples/scripts/results/test_concode_evaluation.json +30 -0
- wisent/examples/scripts/results/test_concode_pairs.json +8 -0
- wisent/examples/scripts/results/test_copa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_copa_pairs.json +8 -0
- wisent/examples/scripts/results/test_copal_id_evaluation.json +30 -0
- wisent/examples/scripts/results/test_copal_id_pairs.json +8 -0
- wisent/examples/scripts/results/test_coqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_coqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_coqcat_evaluation.json +30 -0
- wisent/examples/scripts/results/test_coqcat_pairs.json +8 -0
- wisent/examples/scripts/results/test_crows_pairs_evaluation.json +51 -0
- wisent/examples/scripts/results/test_crows_pairs_pairs.json +14 -0
- wisent/examples/scripts/results/test_csatqa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_csatqa_pairs.json +14 -0
- wisent/examples/scripts/results/test_cycle_letters_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cycle_letters_pairs.json +8 -0
- wisent/examples/scripts/results/test_darija_bench/test_darija_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_darija_bench/test_darija_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_darija_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_darija_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_darijahellaswag_evaluation.json +30 -0
- wisent/examples/scripts/results/test_darijahellaswag_pairs.json +8 -0
- wisent/examples/scripts/results/test_darijammlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_darijammlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_dbpedia_14_evaluation.json +30 -0
- wisent/examples/scripts/results/test_dbpedia_14_pairs.json +8 -0
- wisent/examples/scripts/results/test_drop_evaluation.json +30 -0
- wisent/examples/scripts/results/test_drop_pairs.json +8 -0
- wisent/examples/scripts/results/test_ds1000_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ds1000_pairs.json +8 -0
- wisent/examples/scripts/results/test_egyhellaswag_evaluation.json +30 -0
- wisent/examples/scripts/results/test_egyhellaswag_pairs.json +8 -0
- wisent/examples/scripts/results/test_egymmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_egymmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_epec_koref_bin_evaluation.json +30 -0
- wisent/examples/scripts/results/test_epec_koref_bin_pairs.json +8 -0
- wisent/examples/scripts/results/test_eq_bench_evaluation.json +30 -0
- wisent/examples/scripts/results/test_eq_bench_pairs.json +8 -0
- wisent/examples/scripts/results/test_escola_evaluation.json +30 -0
- wisent/examples/scripts/results/test_escola_pairs.json +8 -0
- wisent/examples/scripts/results/test_ethics_cm_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ethics_cm_pairs.json +8 -0
- wisent/examples/scripts/results/test_ethos_binary_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ethos_binary_pairs.json +8 -0
- wisent/examples/scripts/results/test_eus_exams/test_eus_exams_evaluation.json +51 -0
- wisent/examples/scripts/results/test_eus_exams/test_eus_exams_pairs.json +14 -0
- wisent/examples/scripts/results/test_eus_exams_es_evaluation.json +51 -0
- wisent/examples/scripts/results/test_eus_exams_es_pairs.json +14 -0
- wisent/examples/scripts/results/test_eus_exams_evaluation.json +51 -0
- wisent/examples/scripts/results/test_eus_exams_pairs.json +14 -0
- wisent/examples/scripts/results/test_eus_proficiency_evaluation.json +30 -0
- wisent/examples/scripts/results/test_eus_proficiency_pairs.json +8 -0
- wisent/examples/scripts/results/test_eus_reading_evaluation.json +30 -0
- wisent/examples/scripts/results/test_eus_reading_pairs.json +8 -0
- wisent/examples/scripts/results/test_eus_trivia_evaluation.json +30 -0
- wisent/examples/scripts/results/test_eus_trivia_pairs.json +8 -0
- wisent/examples/scripts/results/test_evalita-mp_evaluation.json +51 -0
- wisent/examples/scripts/results/test_evalita-mp_pairs.json +14 -0
- wisent/examples/scripts/results/test_evalita-sp_sum_task_fp-small_p1_evaluation.json +30 -0
- wisent/examples/scripts/results/test_evalita-sp_sum_task_fp-small_p1_pairs.json +8 -0
- wisent/examples/scripts/results/test_evalita_LLM_evaluation.json +51 -0
- wisent/examples/scripts/results/test_evalita_LLM_pairs.json +14 -0
- wisent/examples/scripts/results/test_evalita_llm/test_evalita_llm_evaluation.json +51 -0
- wisent/examples/scripts/results/test_evalita_llm/test_evalita_llm_pairs.json +14 -0
- wisent/examples/scripts/results/test_evalita_mp/test_evalita-mp_te_prompt-1_evaluation.json +30 -0
- wisent/examples/scripts/results/test_evalita_mp/test_evalita-mp_te_prompt-1_pairs.json +8 -0
- wisent/examples/scripts/results/test_evalita_mp2/test_evalita_mp_evaluation.json +51 -0
- wisent/examples/scripts/results/test_evalita_mp2/test_evalita_mp_pairs.json +14 -0
- wisent/examples/scripts/results/test_evalita_sp2/test_evalita-sp_sum_task_fp-small_p1_evaluation.json +30 -0
- wisent/examples/scripts/results/test_evalita_sp2/test_evalita-sp_sum_task_fp-small_p1_pairs.json +8 -0
- wisent/examples/scripts/results/test_fda_evaluation.json +30 -0
- wisent/examples/scripts/results/test_fda_pairs.json +8 -0
- wisent/examples/scripts/results/test_financial_tweets_evaluation.json +30 -0
- wisent/examples/scripts/results/test_financial_tweets_pairs.json +8 -0
- wisent/examples/scripts/results/test_fld/test_fld_evaluation.json +30 -0
- wisent/examples/scripts/results/test_fld/test_fld_pairs.json +8 -0
- wisent/examples/scripts/results/test_fld_evaluation.json +30 -0
- wisent/examples/scripts/results/test_fld_fixed/test_fld_evaluation.json +30 -0
- wisent/examples/scripts/results/test_fld_fixed/test_fld_pairs.json +8 -0
- wisent/examples/scripts/results/test_fld_pairs.json +8 -0
- wisent/examples/scripts/results/test_flores_evaluation.json +51 -0
- wisent/examples/scripts/results/test_flores_pairs.json +14 -0
- wisent/examples/scripts/results/test_freebase_evaluation.json +30 -0
- wisent/examples/scripts/results/test_freebase_pairs.json +8 -0
- wisent/examples/scripts/results/test_french_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_french_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_galcola_evaluation.json +30 -0
- wisent/examples/scripts/results/test_galcola_pairs.json +8 -0
- wisent/examples/scripts/results/test_galician_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_galician_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_glianorex_evaluation.json +30 -0
- wisent/examples/scripts/results/test_glianorex_pairs.json +8 -0
- wisent/examples/scripts/results/test_global_mmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_global_mmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_glue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_glue_pairs.json +14 -0
- wisent/examples/scripts/results/test_gpqa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_gpqa_pairs.json +14 -0
- wisent/examples/scripts/results/test_gpt3_translation_benchmarks_evaluation.json +51 -0
- wisent/examples/scripts/results/test_gpt3_translation_benchmarks_pairs.json +14 -0
- wisent/examples/scripts/results/test_groundcocoa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_groundcocoa_pairs.json +8 -0
- wisent/examples/scripts/results/test_gsm8k_evaluation.json +30 -0
- wisent/examples/scripts/results/test_gsm8k_pairs.json +8 -0
- wisent/examples/scripts/results/test_haerae_evaluation.json +51 -0
- wisent/examples/scripts/results/test_haerae_pairs.json +14 -0
- wisent/examples/scripts/results/test_headqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_headqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_hellaswag_evaluation.json +30 -0
- wisent/examples/scripts/results/test_hellaswag_pairs.json +8 -0
- wisent/examples/scripts/results/test_hendrycks_ethics_evaluation.json +51 -0
- wisent/examples/scripts/results/test_hendrycks_ethics_pairs.json +14 -0
- wisent/examples/scripts/results/test_hendrycks_math_evaluation.json +51 -0
- wisent/examples/scripts/results/test_hendrycks_math_pairs.json +14 -0
- wisent/examples/scripts/results/test_histoires_morales_evaluation.json +30 -0
- wisent/examples/scripts/results/test_histoires_morales_pairs.json +8 -0
- wisent/examples/scripts/results/test_hmmt_evaluation.json +30 -0
- wisent/examples/scripts/results/test_hmmt_feb_2025_evaluation.json +30 -0
- wisent/examples/scripts/results/test_hmmt_feb_2025_pairs.json +8 -0
- wisent/examples/scripts/results/test_hmmt_pairs.json +8 -0
- wisent/examples/scripts/results/test_hrm8k_evaluation.json +51 -0
- wisent/examples/scripts/results/test_hrm8k_pairs.json +14 -0
- wisent/examples/scripts/results/test_humaneval_evaluation.json +30 -0
- wisent/examples/scripts/results/test_humaneval_pairs.json +8 -0
- wisent/examples/scripts/results/test_humaneval_plus_evaluation.json +30 -0
- wisent/examples/scripts/results/test_humaneval_plus_pairs.json +8 -0
- wisent/examples/scripts/results/test_ifeval_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ifeval_pairs.json +8 -0
- wisent/examples/scripts/results/test_instruct_humaneval/test_instruct_humaneval_evaluation.json +30 -0
- wisent/examples/scripts/results/test_instruct_humaneval/test_instruct_humaneval_pairs.json +8 -0
- wisent/examples/scripts/results/test_instruct_humaneval_evaluation.json +30 -0
- wisent/examples/scripts/results/test_instruct_humaneval_pairs.json +8 -0
- wisent/examples/scripts/results/test_inverse_scaling_evaluation.json +51 -0
- wisent/examples/scripts/results/test_inverse_scaling_hindsight_neglect_10shot_evaluation.json +30 -0
- wisent/examples/scripts/results/test_inverse_scaling_hindsight_neglect_10shot_pairs.json +8 -0
- wisent/examples/scripts/results/test_inverse_scaling_mc/test_inverse_scaling_mc_evaluation.json +51 -0
- wisent/examples/scripts/results/test_inverse_scaling_mc/test_inverse_scaling_mc_pairs.json +14 -0
- wisent/examples/scripts/results/test_inverse_scaling_pairs.json +14 -0
- wisent/examples/scripts/results/test_iwslt2017-ar-en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_iwslt2017-ar-en_pairs.json +8 -0
- wisent/examples/scripts/results/test_iwslt2017-en-ar_evaluation.json +30 -0
- wisent/examples/scripts/results/test_iwslt2017-en-ar_pairs.json +8 -0
- wisent/examples/scripts/results/test_iwslt2017_ar_en/test_iwslt2017-ar-en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_iwslt2017_ar_en/test_iwslt2017-ar-en_pairs.json +8 -0
- wisent/examples/scripts/results/test_iwslt2017_en_ar/test_iwslt2017-en-ar_evaluation.json +30 -0
- wisent/examples/scripts/results/test_iwslt2017_en_ar/test_iwslt2017-en-ar_pairs.json +8 -0
- wisent/examples/scripts/results/test_iwslt2017_group/test_iwslt2017_evaluation.json +30 -0
- wisent/examples/scripts/results/test_iwslt2017_group/test_iwslt2017_pairs.json +8 -0
- wisent/examples/scripts/results/test_japanese_leaderboard_evaluation.json +51 -0
- wisent/examples/scripts/results/test_japanese_leaderboard_pairs.json +14 -0
- wisent/examples/scripts/results/test_jsonschema_bench/test_jsonschema_bench_evaluation.json +30 -0
- wisent/examples/scripts/results/test_jsonschema_bench/test_jsonschema_bench_pairs.json +8 -0
- wisent/examples/scripts/results/test_jsonschema_bench_evaluation.json +30 -0
- wisent/examples/scripts/results/test_jsonschema_bench_final/test_jsonschema_bench_evaluation.json +30 -0
- wisent/examples/scripts/results/test_jsonschema_bench_final/test_jsonschema_bench_pairs.json +8 -0
- wisent/examples/scripts/results/test_jsonschema_bench_pairs.json +8 -0
- wisent/examples/scripts/results/test_kbl_evaluation.json +51 -0
- wisent/examples/scripts/results/test_kbl_fixed/test_kbl_evaluation.json +51 -0
- wisent/examples/scripts/results/test_kbl_fixed/test_kbl_pairs.json +14 -0
- wisent/examples/scripts/results/test_kbl_pairs.json +14 -0
- wisent/examples/scripts/results/test_kmmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_kmmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_kobest_evaluation.json +51 -0
- wisent/examples/scripts/results/test_kobest_pairs.json +14 -0
- wisent/examples/scripts/results/test_kormedmcqa/test_kormedmcqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_kormedmcqa/test_kormedmcqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_kormedmcqa_dentist/test_kormedmcqa_dentist_evaluation.json +30 -0
- wisent/examples/scripts/results/test_kormedmcqa_dentist/test_kormedmcqa_dentist_pairs.json +8 -0
- wisent/examples/scripts/results/test_kormedmcqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_kormedmcqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_cloze_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_cloze_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_final/test_lambada_openai_mt_stablelm_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_final/test_lambada_openai_mt_stablelm_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_multilingual/test_lambada_multilingual_evaluation.json +51 -0
- wisent/examples/scripts/results/test_lambada_multilingual/test_lambada_multilingual_pairs.json +14 -0
- wisent/examples/scripts/results/test_lambada_multilingual_evaluation.json +51 -0
- wisent/examples/scripts/results/test_lambada_multilingual_pairs.json +14 -0
- wisent/examples/scripts/results/test_lambada_multilingual_stablelm_evaluation.json +51 -0
- wisent/examples/scripts/results/test_lambada_multilingual_stablelm_pairs.json +14 -0
- wisent/examples/scripts/results/test_lambada_openai_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_openai_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_stablelm_en_fixed/test_lambada_openai_mt_stablelm_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_stablelm_en_fixed/test_lambada_openai_mt_stablelm_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_stablelm_fixed/test_lambada_openai_mt_stablelm_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_stablelm_fixed/test_lambada_openai_mt_stablelm_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_standard_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_standard_pairs.json +8 -0
- wisent/examples/scripts/results/test_leaderboard_evaluation.json +51 -0
- wisent/examples/scripts/results/test_leaderboard_pairs.json +14 -0
- wisent/examples/scripts/results/test_libra/test_libra_evaluation.json +51 -0
- wisent/examples/scripts/results/test_libra/test_libra_pairs.json +14 -0
- wisent/examples/scripts/results/test_libra_evaluation.json +51 -0
- wisent/examples/scripts/results/test_libra_pairs.json +14 -0
- wisent/examples/scripts/results/test_lingoly_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lingoly_pairs.json +8 -0
- wisent/examples/scripts/results/test_livecodebench_evaluation.json +30 -0
- wisent/examples/scripts/results/test_livecodebench_pairs.json +8 -0
- wisent/examples/scripts/results/test_livemathbench_cnmo_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_livemathbench_cnmo_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_livemathbench_cnmo_zh_evaluation.json +30 -0
- wisent/examples/scripts/results/test_livemathbench_cnmo_zh_pairs.json +8 -0
- wisent/examples/scripts/results/test_llama_evaluation.json +30 -0
- wisent/examples/scripts/results/test_llama_pairs.json +8 -0
- wisent/examples/scripts/results/test_logiqa2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_logiqa2_pairs.json +8 -0
- wisent/examples/scripts/results/test_logiqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_logiqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_m_mmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_m_mmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_mastermind/test_mastermind_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mastermind/test_mastermind_pairs.json +14 -0
- wisent/examples/scripts/results/test_mastermind_24_easy/test_mastermind_24_easy_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mastermind_24_easy/test_mastermind_24_easy_pairs.json +8 -0
- wisent/examples/scripts/results/test_mastermind_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mastermind_pairs.json +14 -0
- wisent/examples/scripts/results/test_math500_evaluation.json +30 -0
- wisent/examples/scripts/results/test_math500_pairs.json +8 -0
- wisent/examples/scripts/results/test_math_evaluation.json +30 -0
- wisent/examples/scripts/results/test_math_pairs.json +8 -0
- wisent/examples/scripts/results/test_mathqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mathqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_mbpp_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mbpp_pairs.json +8 -0
- wisent/examples/scripts/results/test_mbpp_plus_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mbpp_plus_pairs.json +8 -0
- wisent/examples/scripts/results/test_mc_taco_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mc_taco_pairs.json +8 -0
- wisent/examples/scripts/results/test_med_concepts_qa/test_med_concepts_qa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_med_concepts_qa/test_med_concepts_qa_pairs.json +14 -0
- wisent/examples/scripts/results/test_med_concepts_qa_atc_easy/test_med_concepts_qa_atc_easy_evaluation.json +30 -0
- wisent/examples/scripts/results/test_med_concepts_qa_atc_easy/test_med_concepts_qa_atc_easy_pairs.json +8 -0
- wisent/examples/scripts/results/test_med_concepts_qa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_med_concepts_qa_pairs.json +14 -0
- wisent/examples/scripts/results/test_meddialog_evaluation.json +30 -0
- wisent/examples/scripts/results/test_meddialog_pairs.json +8 -0
- wisent/examples/scripts/results/test_meddialog_raw_perplexity/test_meddialog_raw_perplexity_evaluation.json +30 -0
- wisent/examples/scripts/results/test_meddialog_raw_perplexity/test_meddialog_raw_perplexity_pairs.json +8 -0
- wisent/examples/scripts/results/test_mediqa_qa2019_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mediqa_qa2019_pairs.json +8 -0
- wisent/examples/scripts/results/test_medmcqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_medmcqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_medqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_medqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_medtext_evaluation.json +30 -0
- wisent/examples/scripts/results/test_medtext_pairs.json +8 -0
- wisent/examples/scripts/results/test_mela_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mela_pairs.json +14 -0
- wisent/examples/scripts/results/test_meqsum_evaluation.json +30 -0
- wisent/examples/scripts/results/test_meqsum_pairs.json +8 -0
- wisent/examples/scripts/results/test_mercury_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mercury_pairs.json +8 -0
- wisent/examples/scripts/results/test_metabench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_metabench_pairs.json +14 -0
- wisent/examples/scripts/results/test_mgsm_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mgsm_pairs.json +14 -0
- wisent/examples/scripts/results/test_mimic_repsum_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mimic_repsum_pairs.json +8 -0
- wisent/examples/scripts/results/test_minerva_math_evaluation.json +51 -0
- wisent/examples/scripts/results/test_minerva_math_pairs.json +14 -0
- wisent/examples/scripts/results/test_mlqa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mlqa_pairs.json +14 -0
- wisent/examples/scripts/results/test_mmlu-pro-plus_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mmlu-pro-plus_pairs.json +14 -0
- wisent/examples/scripts/results/test_mmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_mmlu_pro_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mmlu_pro_pairs.json +14 -0
- wisent/examples/scripts/results/test_mmlu_prox_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mmlu_prox_pairs.json +14 -0
- wisent/examples/scripts/results/test_mmlusr_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mmlusr_pairs.json +8 -0
- wisent/examples/scripts/results/test_mmmu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mmmu_pairs.json +14 -0
- wisent/examples/scripts/results/test_mnli_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mnli_pairs.json +8 -0
- wisent/examples/scripts/results/test_model_written_evals_evaluation.json +51 -0
- wisent/examples/scripts/results/test_model_written_evals_pairs.json +14 -0
- wisent/examples/scripts/results/test_moral_stories_evaluation.json +30 -0
- wisent/examples/scripts/results/test_moral_stories_pairs.json +8 -0
- wisent/examples/scripts/results/test_mts_dialog_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mts_dialog_pairs.json +8 -0
- wisent/examples/scripts/results/test_multiblimp_evaluation.json +51 -0
- wisent/examples/scripts/results/test_multiblimp_pairs.json +14 -0
- wisent/examples/scripts/results/test_multimedqa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_multimedqa_pairs.json +14 -0
- wisent/examples/scripts/results/test_multipl_e_evaluation.json +30 -0
- wisent/examples/scripts/results/test_multipl_e_pairs.json +8 -0
- wisent/examples/scripts/results/test_mutual_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mutual_pairs.json +8 -0
- wisent/examples/scripts/results/test_non_greedy_robustness_agieval_aqua_rat_evaluation.json +30 -0
- wisent/examples/scripts/results/test_non_greedy_robustness_agieval_aqua_rat_pairs.json +8 -0
- wisent/examples/scripts/results/test_noreval_evaluation.json +51 -0
- wisent/examples/scripts/results/test_noreval_pairs.json +14 -0
- wisent/examples/scripts/results/test_noticia_evaluation.json +30 -0
- wisent/examples/scripts/results/test_noticia_pairs.json +8 -0
- wisent/examples/scripts/results/test_nq_open_evaluation.json +30 -0
- wisent/examples/scripts/results/test_nq_open_pairs.json +8 -0
- wisent/examples/scripts/results/test_olaph_evaluation.json +30 -0
- wisent/examples/scripts/results/test_olaph_pairs.json +8 -0
- wisent/examples/scripts/results/test_openbookqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_openbookqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_openllm_evaluation.json +51 -0
- wisent/examples/scripts/results/test_openllm_pairs.json +14 -0
- wisent/examples/scripts/results/test_option_order_robustness_agieval_aqua_rat_evaluation.json +30 -0
- wisent/examples/scripts/results/test_option_order_robustness_agieval_aqua_rat_pairs.json +8 -0
- wisent/examples/scripts/results/test_paloma_evaluation.json +51 -0
- wisent/examples/scripts/results/test_paloma_pairs.json +14 -0
- wisent/examples/scripts/results/test_passkey/test_passkey_evaluation.json +30 -0
- wisent/examples/scripts/results/test_passkey/test_passkey_pairs.json +8 -0
- wisent/examples/scripts/results/test_paws-x_evaluation.json +51 -0
- wisent/examples/scripts/results/test_paws-x_pairs.json +14 -0
- wisent/examples/scripts/results/test_paws_en/test_paws_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_paws_en/test_paws_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_penn_treebank_evaluation.json +30 -0
- wisent/examples/scripts/results/test_penn_treebank_pairs.json +8 -0
- wisent/examples/scripts/results/test_pile_10k/test_pile_10k_evaluation.json +30 -0
- wisent/examples/scripts/results/test_pile_10k/test_pile_10k_pairs.json +8 -0
- wisent/examples/scripts/results/test_piqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_piqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_polemo2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_polemo2_pairs.json +8 -0
- wisent/examples/scripts/results/test_polymath_en_high_evaluation.json +30 -0
- wisent/examples/scripts/results/test_polymath_en_high_pairs.json +8 -0
- wisent/examples/scripts/results/test_polymath_en_medium_evaluation.json +30 -0
- wisent/examples/scripts/results/test_polymath_en_medium_pairs.json +8 -0
- wisent/examples/scripts/results/test_polymath_zh_high_evaluation.json +30 -0
- wisent/examples/scripts/results/test_polymath_zh_high_pairs.json +8 -0
- wisent/examples/scripts/results/test_polymath_zh_medium_evaluation.json +30 -0
- wisent/examples/scripts/results/test_polymath_zh_medium_pairs.json +8 -0
- wisent/examples/scripts/results/test_portuguese_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_portuguese_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_prompt_robustness_agieval_aqua_rat/test_prompt_robustness_agieval_aqua_rat_evaluation.json +30 -0
- wisent/examples/scripts/results/test_prompt_robustness_agieval_aqua_rat/test_prompt_robustness_agieval_aqua_rat_pairs.json +8 -0
- wisent/examples/scripts/results/test_prompt_robustness_agieval_aqua_rat_evaluation.json +30 -0
- wisent/examples/scripts/results/test_prompt_robustness_agieval_aqua_rat_pairs.json +8 -0
- wisent/examples/scripts/results/test_prost_evaluation.json +30 -0
- wisent/examples/scripts/results/test_prost_pairs.json +8 -0
- wisent/examples/scripts/results/test_ptb_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ptb_pairs.json +8 -0
- wisent/examples/scripts/results/test_pubmedqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_pubmedqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_pythia_evaluation.json +51 -0
- wisent/examples/scripts/results/test_pythia_pairs.json +14 -0
- wisent/examples/scripts/results/test_qa4mre_evaluation.json +30 -0
- wisent/examples/scripts/results/test_qa4mre_pairs.json +8 -0
- wisent/examples/scripts/results/test_qasper_evaluation.json +30 -0
- wisent/examples/scripts/results/test_qasper_pairs.json +8 -0
- wisent/examples/scripts/results/test_race_evaluation.json +30 -0
- wisent/examples/scripts/results/test_race_pairs.json +8 -0
- wisent/examples/scripts/results/test_realtoxicityprompts_evaluation.json +30 -0
- wisent/examples/scripts/results/test_realtoxicityprompts_pairs.json +8 -0
- wisent/examples/scripts/results/test_recode_evaluation.json +30 -0
- wisent/examples/scripts/results/test_recode_pairs.json +8 -0
- wisent/examples/scripts/results/test_record_evaluation.json +30 -0
- wisent/examples/scripts/results/test_record_pairs.json +8 -0
- wisent/examples/scripts/results/test_ruler_evaluation.json +51 -0
- wisent/examples/scripts/results/test_ruler_pairs.json +14 -0
- wisent/examples/scripts/results/test_sciq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_sciq_pairs.json +8 -0
- wisent/examples/scripts/results/test_score_evaluation.json +51 -0
- wisent/examples/scripts/results/test_score_pairs.json +14 -0
- wisent/examples/scripts/results/test_self_consistency_evaluation.json +30 -0
- wisent/examples/scripts/results/test_self_consistency_pairs.json +8 -0
- wisent/examples/scripts/results/test_siqa/test_siqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_siqa/test_siqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_siqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_siqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_spanish_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_spanish_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_squad2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_squad2_pairs.json +8 -0
- wisent/examples/scripts/results/test_squadv2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_squadv2_pairs.json +8 -0
- wisent/examples/scripts/results/test_super-glue-lm-eval-v1-seq2seq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_super-glue-lm-eval-v1-seq2seq_pairs.json +8 -0
- wisent/examples/scripts/results/test_super-glue-lm-eval-v1_evaluation.json +51 -0
- wisent/examples/scripts/results/test_super-glue-lm-eval-v1_pairs.json +14 -0
- wisent/examples/scripts/results/test_swag_evaluation.json +30 -0
- wisent/examples/scripts/results/test_swag_pairs.json +8 -0
- wisent/examples/scripts/results/test_tinyBenchmarks_evaluation.json +51 -0
- wisent/examples/scripts/results/test_tinyBenchmarks_pairs.json +14 -0
- wisent/examples/scripts/results/test_tmmluplus_evaluation.json +51 -0
- wisent/examples/scripts/results/test_tmmluplus_pairs.json +14 -0
- wisent/examples/scripts/results/test_translation_evaluation.json +51 -0
- wisent/examples/scripts/results/test_translation_pairs.json +14 -0
- wisent/examples/scripts/results/test_triviaqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_triviaqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_truthfulqa-multi_evaluation.json +51 -0
- wisent/examples/scripts/results/test_truthfulqa-multi_pairs.json +14 -0
- wisent/examples/scripts/results/test_truthfulqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_truthfulqa_mc1_evaluation.json +30 -0
- wisent/examples/scripts/results/test_truthfulqa_mc1_pairs.json +8 -0
- wisent/examples/scripts/results/test_truthfulqa_mc2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_truthfulqa_mc2_pairs.json +8 -0
- wisent/examples/scripts/results/test_truthfulqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_turkishmmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_turkishmmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_unfair_tos_evaluation.json +30 -0
- wisent/examples/scripts/results/test_unfair_tos_pairs.json +8 -0
- wisent/examples/scripts/results/test_unscramble_evaluation.json +51 -0
- wisent/examples/scripts/results/test_unscramble_pairs.json +14 -0
- wisent/examples/scripts/results/test_webqs_evaluation.json +30 -0
- wisent/examples/scripts/results/test_webqs_pairs.json +8 -0
- wisent/examples/scripts/results/test_wikitext103_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wikitext103_pairs.json +8 -0
- wisent/examples/scripts/results/test_wikitext_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wikitext_pairs.json +8 -0
- wisent/examples/scripts/results/test_winogender_evaluation.json +51 -0
- wisent/examples/scripts/results/test_winogender_pairs.json +14 -0
- wisent/examples/scripts/results/test_winogrande_evaluation.json +30 -0
- wisent/examples/scripts/results/test_winogrande_pairs.json +8 -0
- wisent/examples/scripts/results/test_wmdp_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wmdp_pairs.json +8 -0
- wisent/examples/scripts/results/test_wmt-ro-en-t5-prompt_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wmt-ro-en-t5-prompt_pairs.json +8 -0
- wisent/examples/scripts/results/test_wmt14_en_fr_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wmt14_en_fr_pairs.json +8 -0
- wisent/examples/scripts/results/test_wmt16_en_de_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wmt16_en_de_pairs.json +8 -0
- wisent/examples/scripts/results/test_wmt16_ro_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wmt16_ro_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_wsc273_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wsc273_pairs.json +8 -0
- wisent/examples/scripts/results/test_xcopa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_xcopa_pairs.json +14 -0
- wisent/examples/scripts/results/test_xnli_eu_evaluation.json +30 -0
- wisent/examples/scripts/results/test_xnli_eu_pairs.json +8 -0
- wisent/examples/scripts/results/test_xnli_evaluation.json +51 -0
- wisent/examples/scripts/results/test_xnli_pairs.json +14 -0
- wisent/examples/scripts/results/test_xquad_evaluation.json +51 -0
- wisent/examples/scripts/results/test_xquad_pairs.json +14 -0
- wisent/examples/scripts/results/test_xstorycloze_evaluation.json +51 -0
- wisent/examples/scripts/results/test_xstorycloze_pairs.json +14 -0
- wisent/examples/scripts/results/test_xsum_evaluation.json +30 -0
- wisent/examples/scripts/results/test_xsum_pairs.json +8 -0
- wisent/examples/scripts/results/test_xwinograd_evaluation.json +51 -0
- wisent/examples/scripts/results/test_xwinograd_pairs.json +14 -0
- wisent/examples/scripts/results/test_yahoo_answers_topics_evaluation.json +30 -0
- wisent/examples/scripts/results/test_yahoo_answers_topics_pairs.json +8 -0
- wisent/parameters/__init__.py +1 -0
- wisent/parameters/lm_eval/all_lm_eval_task_families.json +169 -0
- wisent/parameters/lm_eval/broken_in_lm_eval.json +10 -0
- wisent/parameters/lm_eval/evaluations_not_lm_eval_tasks.json +0 -0
- wisent/parameters/lm_eval/evaluator_check.json +3476 -0
- wisent/parameters/lm_eval/final_verification.json +24782 -0
- wisent/parameters/lm_eval/group_task_evaluators.json +1833 -0
- wisent/parameters/lm_eval/group_tasks.json +150 -0
- wisent/parameters/lm_eval/individual_tasks.json +402 -0
- wisent/parameters/lm_eval/no_readmes.json +1 -0
- wisent/parameters/lm_eval/not_lm_eval_tasks.json +110 -0
- wisent/parameters/lm_eval/read_tasks.json +208 -0
- wisent/parameters/lm_eval/readme_files.json +208 -0
- wisent/parameters/lm_eval/track_progress_not_lm_eval_tasks.json +128 -0
- wisent/parameters/tasks/missing_task_families.json +2963 -0
- wisent/parameters/tasks/remaining_tasks_to_implement.json +199 -0
- wisent/parameters/tasks/risks.json +10 -0
- wisent/parameters/tasks/skills.json +14 -0
- wisent/parameters/tasks/tasks.json +56031 -0
- wisent/scripts/run_quality_metrics_sweep.sh +315 -0
- wisent/tests/__init__.py +0 -0
- wisent/tests/examples/__init__.py +0 -0
- wisent/tests/examples/cli/__init__.py +0 -0
- wisent/tests/examples/cli/activations/__init__.py +0 -0
- wisent/tests/examples/cli/activations/test_get_activations.py +127 -0
- wisent/tests/examples/cli/classifier/__init__.py +0 -0
- wisent/tests/examples/cli/classifier/test_classifier_examples.py +141 -0
- wisent/tests/examples/cli/contrastive_pairs/__init__.py +0 -0
- wisent/tests/examples/cli/contrastive_pairs/test_generate_pairs.py +89 -0
- wisent/tests/examples/cli/evaluation/__init__.py +0 -0
- wisent/tests/examples/cli/evaluation/test_evaluation_examples.py +117 -0
- wisent/tests/examples/cli/generate/__init__.py +0 -0
- wisent/tests/examples/cli/generate/test_generate_with_classifier.py +146 -0
- wisent/tests/examples/cli/generate/test_generate_with_steering.py +149 -0
- wisent/tests/examples/cli/generate/test_only_generate.py +110 -0
- wisent/tests/examples/cli/multi_steering/__init__.py +0 -0
- wisent/tests/examples/cli/multi_steering/test_multi_steer_from_trained_vectors.py +210 -0
- wisent/tests/examples/cli/multi_steering/test_multi_steer_with_different_parameters.py +205 -0
- wisent/tests/examples/cli/multi_steering/test_train_and_multi_steer.py +174 -0
- wisent/tests/examples/cli/optimizer/__init__.py +0 -0
- wisent/tests/examples/cli/optimizer/test_optimize_sample_size.py +102 -0
- wisent/tests/examples/cli/optimizer/test_optimizer_examples.py +59 -0
- wisent/tests/examples/cli/steering/__init__.py +0 -0
- wisent/tests/examples/cli/steering/test_create_steering_vectors.py +135 -0
- wisent/tests/examples/cli/synthetic/__init__.py +0 -0
- wisent/tests/examples/cli/synthetic/test_synthetic_pairs.py +45 -0
- wisent/tests/nosense/__init__.py +6 -0
- wisent/tests/nosense/base_nosense.py +81 -0
- wisent/tests/nosense/math500_nosense.py +72 -0
- wisent/tests/nosense/test_robustness.py +336 -0
- wisent/tests/test_all_cli_commands.py +674 -0
- wisent/tests/test_geometry_comprehensive.py +327 -0
- wisent/tests/test_titan_geometry.py +257 -0
- wisent/tests/visualize_geometry.py +148 -0
- wisent-0.7.379.dist-info/METADATA +64 -0
- wisent-0.7.379.dist-info/RECORD +1720 -0
- wisent-0.7.379.dist-info/WHEEL +5 -0
- wisent-0.7.379.dist-info/entry_points.txt +2 -0
- wisent-0.7.379.dist-info/licenses/LICENSE +21 -0
- wisent-0.7.379.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"task_name": "arc_challenge",
|
|
3
|
+
"model_name": "mock",
|
|
4
|
+
"evaluator_name": "log_likelihoods",
|
|
5
|
+
"num_pairs": 1,
|
|
6
|
+
"all_correct": true,
|
|
7
|
+
"pairs": [
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 0,
|
|
10
|
+
"prompt": "Question: Juan and LaKeisha roll a few objects down a ramp. They want to see which object rolls the ...",
|
|
11
|
+
"positive_response": "Record the details of the investigation.",
|
|
12
|
+
"negative_response": "Put the objects in groups.",
|
|
13
|
+
"positive_evaluation": {
|
|
14
|
+
"ground_truth": "TRUTHFUL",
|
|
15
|
+
"confidence": 1.0,
|
|
16
|
+
"expected": "TRUTHFUL",
|
|
17
|
+
"correct": true,
|
|
18
|
+
"details": "Predicted: 'Record the details of the investigation.' (log_prob=-0.500), Expected: 'Record the details of the investigation.'"
|
|
19
|
+
},
|
|
20
|
+
"negative_evaluation": {
|
|
21
|
+
"ground_truth": "UNTRUTHFUL",
|
|
22
|
+
"confidence": 0.0,
|
|
23
|
+
"expected": "UNTRUTHFUL",
|
|
24
|
+
"correct": true,
|
|
25
|
+
"details": "Predicted: 'Record the details of the investigation.' (log_prob=-0.500), Expected: 'Put the objects in groups.'"
|
|
26
|
+
},
|
|
27
|
+
"both_correct": true
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"pair_id": 0,
|
|
4
|
+
"prompt": "Question: Juan and LaKeisha roll a few objects down a ramp. They want to see which object rolls the farthest. What should they do so they can repeat their investigation?\nA. Put the objects in groups.\nB. Record the details of the investigation.",
|
|
5
|
+
"positive_response": "Record the details of the investigation.",
|
|
6
|
+
"negative_response": "Put the objects in groups."
|
|
7
|
+
}
|
|
8
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"task_name": "arc_easy",
|
|
3
|
+
"model_name": "mock",
|
|
4
|
+
"evaluator_name": "log_likelihoods",
|
|
5
|
+
"num_pairs": 1,
|
|
6
|
+
"all_correct": true,
|
|
7
|
+
"pairs": [
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 0,
|
|
10
|
+
"prompt": "Question: Which technology was developed most recently?\nA. television\nB. cellular telephone...",
|
|
11
|
+
"positive_response": "cellular telephone",
|
|
12
|
+
"negative_response": "television",
|
|
13
|
+
"positive_evaluation": {
|
|
14
|
+
"ground_truth": "TRUTHFUL",
|
|
15
|
+
"confidence": 1.0,
|
|
16
|
+
"expected": "TRUTHFUL",
|
|
17
|
+
"correct": true,
|
|
18
|
+
"details": "Predicted: 'cellular telephone' (log_prob=-0.500), Expected: 'cellular telephone'"
|
|
19
|
+
},
|
|
20
|
+
"negative_evaluation": {
|
|
21
|
+
"ground_truth": "UNTRUTHFUL",
|
|
22
|
+
"confidence": 0.0,
|
|
23
|
+
"expected": "UNTRUTHFUL",
|
|
24
|
+
"correct": true,
|
|
25
|
+
"details": "Predicted: 'cellular telephone' (log_prob=-0.500), Expected: 'television'"
|
|
26
|
+
},
|
|
27
|
+
"both_correct": true
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"task_name": "argument_topic",
|
|
3
|
+
"model_name": "mock",
|
|
4
|
+
"evaluator_name": "log_likelihoods",
|
|
5
|
+
"num_pairs": 1,
|
|
6
|
+
"all_correct": true,
|
|
7
|
+
"pairs": [
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 0,
|
|
10
|
+
"prompt": "Classify the Topic of the following Argument to one of these options: affirmative action, algorithmi...",
|
|
11
|
+
"positive_response": "holocaust denial",
|
|
12
|
+
"negative_response": "affirmative action",
|
|
13
|
+
"positive_evaluation": {
|
|
14
|
+
"ground_truth": "TRUTHFUL",
|
|
15
|
+
"confidence": 1.0,
|
|
16
|
+
"expected": "TRUTHFUL",
|
|
17
|
+
"correct": true,
|
|
18
|
+
"details": "Predicted: 'holocaust denial' (log_prob=-0.500), Expected: 'holocaust denial'"
|
|
19
|
+
},
|
|
20
|
+
"negative_evaluation": {
|
|
21
|
+
"ground_truth": "UNTRUTHFUL",
|
|
22
|
+
"confidence": 0.0,
|
|
23
|
+
"expected": "UNTRUTHFUL",
|
|
24
|
+
"correct": true,
|
|
25
|
+
"details": "Predicted: 'holocaust denial' (log_prob=-0.500), Expected: 'affirmative action'"
|
|
26
|
+
},
|
|
27
|
+
"both_correct": true
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"pair_id": 0,
|
|
4
|
+
"prompt": "Classify the Topic of the following Argument to one of these options: affirmative action, algorithmic trading, assisted suicide, atheism, austerity regime, blockade of the gaza strip, cancel pride parades, cannabis, capital punishment, collectivism, compulsory voting, cosmetic surgery, cosmetic surgery for minors, embryonic stem cell research, entrapment, executive compensation, factory farming, fast food, fight urbanization, flag burning, foster care, gender-neutral language, guantanamo bay detention camp, holocaust denial, homeopathy, homeschooling, human cloning, intellectual property rights, intelligence tests, journalism, judicial activism, libertarianism, marriage, missionary work, multi-party system, naturopathy, organ trade, payday loans, polygamy, private military companies, prostitution, racial profiling, retirement, safe spaces, school prayer, sex selection, social media, space exploration, stay-at-home dads, student loans, surrogacy, targeted killing, telemarketing, television, the abolition of nuclear weapons, the church of scientology, the development of autonomous cars, the olympic games, the right to keep and bear arms, the three-strikes laws, the use of child actors, the use of economic sanctions, the use of public defenders, the use of school uniform, the vow of celibacy, vocational education, whaling, wikipedia, women in combat, zero-tolerance policy in schools, zoos.\nArgument:\ndenying the holocaust is a valid exercise of freedom of speech\nTopic:",
|
|
5
|
+
"positive_response": "holocaust denial",
|
|
6
|
+
"negative_response": "affirmative action"
|
|
7
|
+
}
|
|
8
|
+
]
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
{
|
|
2
|
+
"task_name": "arithmetic",
|
|
3
|
+
"model_name": "mock",
|
|
4
|
+
"evaluator_name": "log_likelihoods",
|
|
5
|
+
"num_pairs": 2,
|
|
6
|
+
"all_correct": true,
|
|
7
|
+
"pairs": [
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 0,
|
|
10
|
+
"prompt": "Question: What is 78 plus 74?\nAnswer:\nA. 153\nB. 152...",
|
|
11
|
+
"positive_response": "152",
|
|
12
|
+
"negative_response": "153",
|
|
13
|
+
"positive_evaluation": {
|
|
14
|
+
"ground_truth": "TRUTHFUL",
|
|
15
|
+
"confidence": 1.0,
|
|
16
|
+
"expected": "TRUTHFUL",
|
|
17
|
+
"correct": true,
|
|
18
|
+
"details": "Predicted: '152' (log_prob=-0.500), Expected: '152'"
|
|
19
|
+
},
|
|
20
|
+
"negative_evaluation": {
|
|
21
|
+
"ground_truth": "UNTRUTHFUL",
|
|
22
|
+
"confidence": 0.0,
|
|
23
|
+
"expected": "UNTRUTHFUL",
|
|
24
|
+
"correct": true,
|
|
25
|
+
"details": "Predicted: '152' (log_prob=-0.500), Expected: '153'"
|
|
26
|
+
},
|
|
27
|
+
"both_correct": true
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"pair_id": 1,
|
|
31
|
+
"prompt": "Question: What is 6204 plus 3166?\nAnswer:\nA. 9371\nB. 9370...",
|
|
32
|
+
"positive_response": "9370",
|
|
33
|
+
"negative_response": "9371",
|
|
34
|
+
"positive_evaluation": {
|
|
35
|
+
"ground_truth": "TRUTHFUL",
|
|
36
|
+
"confidence": 1.0,
|
|
37
|
+
"expected": "TRUTHFUL",
|
|
38
|
+
"correct": true,
|
|
39
|
+
"details": "Predicted: '9370' (log_prob=-0.500), Expected: '9370'"
|
|
40
|
+
},
|
|
41
|
+
"negative_evaluation": {
|
|
42
|
+
"ground_truth": "UNTRUTHFUL",
|
|
43
|
+
"confidence": 0.0,
|
|
44
|
+
"expected": "UNTRUTHFUL",
|
|
45
|
+
"correct": true,
|
|
46
|
+
"details": "Predicted: '9370' (log_prob=-0.500), Expected: '9371'"
|
|
47
|
+
},
|
|
48
|
+
"both_correct": true
|
|
49
|
+
}
|
|
50
|
+
]
|
|
51
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"pair_id": 0,
|
|
4
|
+
"prompt": "Question: What is 78 plus 74?\nAnswer:\nA. 153\nB. 152",
|
|
5
|
+
"positive_response": "152",
|
|
6
|
+
"negative_response": "153"
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 1,
|
|
10
|
+
"prompt": "Question: What is 6204 plus 3166?\nAnswer:\nA. 9371\nB. 9370",
|
|
11
|
+
"positive_response": "9370",
|
|
12
|
+
"negative_response": "9371"
|
|
13
|
+
}
|
|
14
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"task_name": "asdiv",
|
|
3
|
+
"model_name": "mock",
|
|
4
|
+
"evaluator_name": "log_likelihoods",
|
|
5
|
+
"num_pairs": 1,
|
|
6
|
+
"all_correct": true,
|
|
7
|
+
"pairs": [
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 0,
|
|
10
|
+
"prompt": "Seven red apples and two green apples are in the basket.\nQuestion:How many apples are in the basket?...",
|
|
11
|
+
"positive_response": "9",
|
|
12
|
+
"negative_response": "10",
|
|
13
|
+
"positive_evaluation": {
|
|
14
|
+
"ground_truth": "TRUTHFUL",
|
|
15
|
+
"confidence": 1.0,
|
|
16
|
+
"expected": "TRUTHFUL",
|
|
17
|
+
"correct": true,
|
|
18
|
+
"details": "Predicted: '9' (log_prob=-0.500), Expected: '9'"
|
|
19
|
+
},
|
|
20
|
+
"negative_evaluation": {
|
|
21
|
+
"ground_truth": "UNTRUTHFUL",
|
|
22
|
+
"confidence": 0.0,
|
|
23
|
+
"expected": "UNTRUTHFUL",
|
|
24
|
+
"correct": true,
|
|
25
|
+
"details": "Predicted: '9' (log_prob=-0.500), Expected: '10'"
|
|
26
|
+
},
|
|
27
|
+
"both_correct": true
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"task_name": "assin_entailment",
|
|
3
|
+
"model_name": "mock",
|
|
4
|
+
"evaluator_name": "log_likelihoods",
|
|
5
|
+
"num_pairs": 1,
|
|
6
|
+
"all_correct": true,
|
|
7
|
+
"pairs": [
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 0,
|
|
10
|
+
"prompt": "De acordo com o relat\u00f3rio, foram notificados 6.052 casos suspeitos de dengue, sendo 641 descartados....",
|
|
11
|
+
"positive_response": "De acordo com o relat\u00f3rio, foram notificados 6.052 casos suspeitos de dengue, sendo 641 descartados., certo? Tamb\u00e9m, Do total de casos notificados, 10.768 foram confirmados como dengue e 15.202 descartados.",
|
|
12
|
+
"negative_response": "De acordo com o relat\u00f3rio, foram notificados 6.052 casos suspeitos de dengue, sendo 641 descartados., certo? Sim, Do total de casos notificados, 10.768 foram confirmados como dengue e 15.202 descartados.",
|
|
13
|
+
"positive_evaluation": {
|
|
14
|
+
"ground_truth": "TRUTHFUL",
|
|
15
|
+
"confidence": 1.0,
|
|
16
|
+
"expected": "TRUTHFUL",
|
|
17
|
+
"correct": true,
|
|
18
|
+
"details": "Predicted: 'De acordo com o relat\u00f3rio, foram notificados 6.052 casos suspeitos de dengue, sendo 641 descartados., certo? Tamb\u00e9m, Do total de casos notificados, 10.768 foram confirmados como dengue e 15.202 descartados.' (log_prob=-0.500), Expected: 'De acordo com o relat\u00f3rio, foram notificados 6.052 casos suspeitos de dengue, sendo 641 descartados., certo? Tamb\u00e9m, Do total de casos notificados, 10.768 foram confirmados como dengue e 15.202 descartados.'"
|
|
19
|
+
},
|
|
20
|
+
"negative_evaluation": {
|
|
21
|
+
"ground_truth": "UNTRUTHFUL",
|
|
22
|
+
"confidence": 0.0,
|
|
23
|
+
"expected": "UNTRUTHFUL",
|
|
24
|
+
"correct": true,
|
|
25
|
+
"details": "Predicted: 'De acordo com o relat\u00f3rio, foram notificados 6.052 casos suspeitos de dengue, sendo 641 descartados., certo? Tamb\u00e9m, Do total de casos notificados, 10.768 foram confirmados como dengue e 15.202 descartados.' (log_prob=-0.500), Expected: 'De acordo com o relat\u00f3rio, foram notificados 6.052 casos suspeitos de dengue, sendo 641 descartados., certo? Sim, Do total de casos notificados, 10.768 foram confirmados como dengue e 15.202 descartados.'"
|
|
26
|
+
},
|
|
27
|
+
"both_correct": true
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"pair_id": 0,
|
|
4
|
+
"prompt": "De acordo com o relat\u00f3rio, foram notificados 6.052 casos suspeitos de dengue, sendo 641 descartados.",
|
|
5
|
+
"positive_response": "De acordo com o relat\u00f3rio, foram notificados 6.052 casos suspeitos de dengue, sendo 641 descartados., certo? Tamb\u00e9m, Do total de casos notificados, 10.768 foram confirmados como dengue e 15.202 descartados.",
|
|
6
|
+
"negative_response": "De acordo com o relat\u00f3rio, foram notificados 6.052 casos suspeitos de dengue, sendo 641 descartados., certo? Sim, Do total de casos notificados, 10.768 foram confirmados como dengue e 15.202 descartados."
|
|
7
|
+
}
|
|
8
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"task_name": "atis",
|
|
3
|
+
"model_name": "mock",
|
|
4
|
+
"evaluator_name": "log_likelihoods",
|
|
5
|
+
"num_pairs": 1,
|
|
6
|
+
"all_correct": true,
|
|
7
|
+
"pairs": [
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 0,
|
|
10
|
+
"prompt": "From the following Text, extract the objects for which the entity type expressed is one of aircraft_...",
|
|
11
|
+
"positive_response": "charlotte: fromloc.city_name, las vegas: toloc.city_name, st. louis: stoploc.city_name",
|
|
12
|
+
"negative_response": "st. louis: stoploc.city_name, las vegas: toloc.city_name, charlotte: fromloc.city_name",
|
|
13
|
+
"positive_evaluation": {
|
|
14
|
+
"ground_truth": "TRUTHFUL",
|
|
15
|
+
"confidence": 1.0,
|
|
16
|
+
"expected": "TRUTHFUL",
|
|
17
|
+
"correct": true,
|
|
18
|
+
"details": "Predicted: 'charlotte: fromloc.city_name, las vegas: toloc.city_name, st. louis: stoploc.city_name' (log_prob=-0.500), Expected: 'charlotte: fromloc.city_name, las vegas: toloc.city_name, st. louis: stoploc.city_name'"
|
|
19
|
+
},
|
|
20
|
+
"negative_evaluation": {
|
|
21
|
+
"ground_truth": "UNTRUTHFUL",
|
|
22
|
+
"confidence": 0.0,
|
|
23
|
+
"expected": "UNTRUTHFUL",
|
|
24
|
+
"correct": true,
|
|
25
|
+
"details": "Predicted: 'charlotte: fromloc.city_name, las vegas: toloc.city_name, st. louis: stoploc.city_name' (log_prob=-0.500), Expected: 'st. louis: stoploc.city_name, las vegas: toloc.city_name, charlotte: fromloc.city_name'"
|
|
26
|
+
},
|
|
27
|
+
"both_correct": true
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"pair_id": 0,
|
|
4
|
+
"prompt": "From the following Text, extract the objects for which the entity type expressed is one of aircraft_code, airline_code, airline_name, airport_code, airport_name, arrive_date.date_relative, arrive_date.day_name, arrive_date.day_number, arrive_date.month_name, arrive_date.today_relative, arrive_time.end_time, arrive_time.period_mod, arrive_time.period_of_day, arrive_time.start_time, arrive_time.time, arrive_time.time_relative, city_name, class_type, connect, cost_relative, day_name, day_number, days_code, depart_date.date_relative, depart_date.day_name, depart_date.day_number, depart_date.month_name, depart_date.today_relative, depart_date.year, depart_time.end_time, depart_time.period_mod, depart_time.period_of_day, depart_time.start_time, depart_time.time, depart_time.time_relative, economy, fare_amount, fare_basis_code, flight_days, flight_mod, flight_number, flight_stop, flight_time, fromloc.airport_code, fromloc.airport_name, fromloc.city_name, fromloc.state_code, fromloc.state_name, meal, meal_code, meal_description, mod, month_name, or, period_of_day, restriction_code, return_date.date_relative, return_date.day_name, return_date.day_number, return_date.month_name, return_date.today_relative, return_time.period_mod, return_time.period_of_day, round_trip, state_code, state_name, stoploc.airport_name, stoploc.city_name, stoploc.state_code, time, time_relative, today_relative, toloc.airport_code, toloc.airport_name, toloc.city_name, toloc.country_name, toloc.state_code, toloc.state_name, transport_type.\nText:\ni would like to find a flight from charlotte to las vegas that makes a stop in st. louis\nentity type:",
|
|
5
|
+
"positive_response": "charlotte: fromloc.city_name, las vegas: toloc.city_name, st. louis: stoploc.city_name",
|
|
6
|
+
"negative_response": "st. louis: stoploc.city_name, las vegas: toloc.city_name, charlotte: fromloc.city_name"
|
|
7
|
+
}
|
|
8
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"task_name": "babi",
|
|
3
|
+
"model_name": "mock",
|
|
4
|
+
"evaluator_name": "log_likelihoods",
|
|
5
|
+
"num_pairs": 1,
|
|
6
|
+
"all_correct": true,
|
|
7
|
+
"pairs": [
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 0,
|
|
10
|
+
"prompt": "Passage: John travelled to the hallway.\nMary journeyed to the bathroom.\n\nQuestion: Where is John?...",
|
|
11
|
+
"positive_response": "hallway",
|
|
12
|
+
"negative_response": "not hallway",
|
|
13
|
+
"positive_evaluation": {
|
|
14
|
+
"ground_truth": "TRUTHFUL",
|
|
15
|
+
"confidence": 1.0,
|
|
16
|
+
"expected": "TRUTHFUL",
|
|
17
|
+
"correct": true,
|
|
18
|
+
"details": "Predicted: 'hallway' (log_prob=-0.500), Expected: 'hallway'"
|
|
19
|
+
},
|
|
20
|
+
"negative_evaluation": {
|
|
21
|
+
"ground_truth": "UNTRUTHFUL",
|
|
22
|
+
"confidence": 0.0,
|
|
23
|
+
"expected": "UNTRUTHFUL",
|
|
24
|
+
"correct": true,
|
|
25
|
+
"details": "Predicted: 'hallway' (log_prob=-0.500), Expected: 'not hallway'"
|
|
26
|
+
},
|
|
27
|
+
"both_correct": true
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"task_name": "babilong",
|
|
3
|
+
"model_name": "mock",
|
|
4
|
+
"evaluator_name": "log_likelihoods",
|
|
5
|
+
"num_pairs": 1,
|
|
6
|
+
"all_correct": true,
|
|
7
|
+
"pairs": [
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 0,
|
|
10
|
+
"prompt": "Context: \u2013 \u0421\u043c\u043e\u0442\u0440\u0438, \u0414\u0436\u043e, \u2013 \u0441\u043a\u0430\u0437\u0430\u043b\u0430 \u0411\u043e\u043d\u0438\u0442\u0430, \u043f\u043e\u043a\u0430\u0437\u0430\u0432 \u043f\u0430\u043b\u044c\u0446\u0435\u043c. \u041a \u043a\u043e\u043d\u0432\u043e\u044e \u043f\u043e\u0434\u044a\u0435\u0445\u0430\u043b \u0433\u0440\u044f\u0437\u043d\u043e-\u0437\u0435\u043b\u0435\u043d\u044b\u0439 \u043f\u0438\u043a\u0430\u043f \"\u0448...",
|
|
11
|
+
"positive_response": "\u0441\u043f\u0430\u043b\u044c\u043d\u044f",
|
|
12
|
+
"negative_response": "\u0432\u0430\u043d\u043d\u0430\u044f",
|
|
13
|
+
"positive_evaluation": {
|
|
14
|
+
"ground_truth": "TRUTHFUL",
|
|
15
|
+
"confidence": 1.0,
|
|
16
|
+
"expected": "TRUTHFUL",
|
|
17
|
+
"correct": true,
|
|
18
|
+
"details": "Predicted: '\u0441\u043f\u0430\u043b\u044c\u043d\u044f' (log_prob=-0.500), Expected: '\u0441\u043f\u0430\u043b\u044c\u043d\u044f'"
|
|
19
|
+
},
|
|
20
|
+
"negative_evaluation": {
|
|
21
|
+
"ground_truth": "UNTRUTHFUL",
|
|
22
|
+
"confidence": 0.0,
|
|
23
|
+
"expected": "UNTRUTHFUL",
|
|
24
|
+
"correct": true,
|
|
25
|
+
"details": "Predicted: '\u0441\u043f\u0430\u043b\u044c\u043d\u044f' (log_prob=-0.500), Expected: '\u0432\u0430\u043d\u043d\u0430\u044f'"
|
|
26
|
+
},
|
|
27
|
+
"both_correct": true
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"pair_id": 0,
|
|
4
|
+
"prompt": "Context: \u2013 \u0421\u043c\u043e\u0442\u0440\u0438, \u0414\u0436\u043e, \u2013 \u0441\u043a\u0430\u0437\u0430\u043b\u0430 \u0411\u043e\u043d\u0438\u0442\u0430, \u043f\u043e\u043a\u0430\u0437\u0430\u0432 \u043f\u0430\u043b\u044c\u0446\u0435\u043c. \u041a \u043a\u043e\u043d\u0432\u043e\u044e \u043f\u043e\u0434\u044a\u0435\u0445\u0430\u043b \u0433\u0440\u044f\u0437\u043d\u043e-\u0437\u0435\u043b\u0435\u043d\u044b\u0439 \u043f\u0438\u043a\u0430\u043f \"\u0448\u0435\u0432\u0440\u043e\u043b\u0435\" \u043d\u0430 \u043c\u043e\u0449\u043d\u044b\u0445 \u043a\u043e\u043b\u0435\u0441\u0430\u0445, \u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u0441\u043a\u0430\u044f \u0434\u0432\u0435\u0440\u044c \u0440\u0430\u0441\u043f\u0430\u0445\u043d\u0443\u043b\u0430\u0441\u044c \u0438 \u043e\u0442\u0442\u0443\u0434\u0430 \u0432\u044b\u0431\u0440\u0430\u043b\u0441\u044f \u0414\u0436\u043e \u2013 \u043d\u0430\u0448 \u0434\u0440\u0443\u0433, \u043f\u0430\u0440\u0442\u043d\u0435\u0440 \u043f\u043e \u0431\u0438\u0437\u043d\u0435\u0441\u0443, \u0434\u0430 \u0438 \u043f\u0440\u043e\u0441\u0442\u043e \u043a\u043e\u043b\u043b\u0435\u0433\u0430. \u0420\u043e\u0441\u043b\u044b\u0439, \u043a\u043e\u0440\u043e\u0442\u043a\u043e \u0441\u0442\u0440\u0438\u0436\u0435\u043d\u044b\u0439, \u0432 \u0441\u0432\u0435\u0442\u043b\u044b\u0445 \u0431\u0440\u044e\u043a\u0430\u0445 \u0445\u0430\u043a\u0438 \u0438 \u0447\u0435\u0440\u043d\u043e\u0439 \u0444\u0443\u0442\u0431\u043e\u043b\u043a\u0435. \u041a \u043d\u0435\u043c\u0443 \u043f\u043e\u0434\u043e\u0448\u0435\u043b \u043a\u0442\u043e-\u0442\u043e \u0438\u0437 \u043f\u0440\u0438\u0435\u0445\u0430\u0432\u0448\u0438\u0445 \u0441 \u043a\u043e\u043d\u0432\u043e\u0435\u043c, \u043e\u043d\u0438 \u0437\u0430\u0433\u043e\u0432\u043e\u0440\u0438\u043b\u0438, \u043d\u0435 \u0437\u0430\u0431\u044b\u0432 \u043f\u043e\u0445\u043b\u043e\u043f\u0430\u0442\u044c \u0434\u0440\u0443\u0433 \u0434\u0440\u0443\u0433\u0430 \u043f\u043e \u0441\u043f\u0438\u043d\u0430\u043c. \u2013 \u041e\u043d \u043a\u0430\u043a\u0438\u0435-\u0442\u043e \u0437\u0430\u043f\u0447\u0430\u0441\u0442\u0438 \u0436\u0434\u0430\u043b \u0442\u0430\u043a, \u0447\u0442\u043e \u0441\u043f\u0430\u0442\u044c \u043d\u0435 \u043c\u043e\u0433, \u043a\u0430\u0436\u0435\u0442\u0441\u044f, \u2013 \u043e\u0442\u043e\u0437\u0432\u0430\u043b\u0441\u044f \u044f. \u0421\u043e\u0431\u0435\u0441\u0435\u0434\u043d\u0438\u043a \u0414\u0436\u043e \u043e\u0442\u043a\u0438\u043d\u0443\u043b \u043f\u043e\u043b\u043e\u0433 \u0442\u0435\u043d\u0442\u0430 \u043d\u0430 \u0433\u0440\u0443\u0437\u043e\u0432\u0438\u043a\u0435, \u0437\u0430\u043b\u0435\u0437 \u0432\u043d\u0443\u0442\u0440\u044c \u0438 \u043d\u0430\u0447\u0430\u043b \u043f\u043e\u0434\u0430\u0432\u0430\u0442\u044c \u043e\u0442\u0442\u0443\u0434\u0430 \u043a\u0430\u0440\u0442\u043e\u043d\u043d\u044b\u0435 \u043a\u043e\u0440\u043e\u0431\u043a\u0438, \u0443\u0432\u0435\u0441\u0438\u0441\u0442\u044b\u0435 \u0441 \u0432\u0438\u0434\u0443, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u0414\u0436\u043e \u0443\u043a\u043b\u0430\u0434\u044b\u0432\u0430\u043b \u0432 \u043f\u0438\u043a\u0430\u043f. \u2013 \u0422\u0435\u043f\u0435\u0440\u044c \u0443\u0441\u043d\u0435\u0442 \u0441\u043f\u043e\u043a\u043e\u0439\u043d\u043e? \u2013 \u041d\u0430\u0432\u0435\u0440\u043d\u043e\u0435. \u042f \u043e\u0431\u043d\u044f\u043b \u0435\u0435 \u0437\u0430 \u0442\u0430\u043b\u0438\u044e \u0438 \u043f\u0440\u0438\u0442\u044f\u043d\u0443\u043b \u043a \u0441\u0435\u0431\u0435 \u0431\u043b\u0438\u0436\u0435. \u0417\u0430 \u0441\u043f\u0438\u043d\u043e\u0439 \u043e\u0442\u043a\u0440\u044b\u043b\u0430\u0441\u044c \u0434\u0432\u0435\u0440\u044c \u0438 \u0433\u043e\u043b\u043e\u0441 \u043c\u0435\u0434\u0441\u0435\u0441\u0442\u0440\u044b \u0434\u043e\u043a\u0442\u043e\u0440\u0430 \u041c\u0430\u0440\u0442\u0438\u043d\u0435\u0441\u0430, \u0432\u044b\u0441\u043e\u043a\u043e\u0439 \u0438 \u0442\u043e\u0449\u0435\u0439 \u0431\u043b\u043e\u043d\u0434\u0438\u043d\u043a\u0438 \u0425\u044d\u0439\u0437\u0435\u043b, \u043f\u0440\u0438\u0433\u043b\u0430\u0441\u0438\u043b \u0411\u043e\u043d\u0438\u0442\u0443 \u0432\u043e\u0439\u0442\u0438. \u0410\u043b\u0435\u043a\u0441\u0430\u043d\u0434\u0440\u0430 \u043f\u0435\u0440\u0435\u043c\u0435\u0449\u0430\u0435\u0442\u0441\u044f \u0432 \u0441\u043f\u0430\u043b\u044c\u043d\u044e. \u0410 \u044f \u043e\u0441\u0442\u0430\u043b\u0441\u044f \u0436\u0434\u0430\u0442\u044c \u0432 \u043f\u0440\u0438\u0435\u043c\u043d\u043e\u0439, \u0443\u0441\u0435\u0432\u0448\u0438\u0441\u044c \u043d\u0430 \u0434\u0438\u0432\u0430\u043d \u0438 \u0434\u043e\u0441\u0442\u0430\u0432 \u0438\u0437 \u0441\u0443\u043c\u043a\u0438 \u043a\u043d\u0438\u0433\u0443. \u0414\u0443\u043c\u0430\u044e, \u0447\u0442\u043e \u043e\u0436\u0438\u0434\u0430\u043d\u0438\u0435 \u043d\u0435 \u043f\u044f\u0442\u044c \u043c\u0438\u043d\u0443\u0442 \u0437\u0430\u0439\u043c\u0435\u0442. \u041c\u043e\u0436\u043d\u043e \u0431\u044b \u043f\u043e\u0439\u0442\u0438 \u043f\u0440\u043e\u0433\u0443\u043b\u044f\u0442\u044c\u0441\u044f, \u0441 \u0442\u0435\u043c \u0436\u0435 \u0414\u0436\u043e \u043f\u043e\u043e\u0431\u0449\u0430\u0442\u044c\u0441\u044f, \u043d\u043e \u0432\u0434\u0440\u0443\u0433 \u043f\u043e\u043d\u0430\u0434\u043e\u0431\u043b\u044e\u0441\u044c \u0437\u0430\u0447\u0435\u043c-\u043d\u0438\u0431\u0443\u0434\u044c? \u0414\u0432\u0435\u0440\u044c \u0431\u044b\u043b\u0430 \u0442\u043e\u043b\u0441\u0442\u0430\u044f, \u0438\u0437-\u0437\u0430 \u043d\u0435\u0435 \u043d\u0438 \u0437\u0432\u0443\u043a\u0430 \u043d\u0435 \u0434\u043e\u043d\u043e\u0441\u0438\u043b\u043e\u0441\u044c. \u0427\u0438\u0442\u0430\u043b\u043e\u0441\u044c \u0438\u0437-\u0437\u0430 \u0432\u043e\u043b\u043d\u0435\u043d\u0438\u044f \u043f\u043b\u043e\u0445\u043e, \u0432\u0441\u0435 \u0432\u0440\u0435\u043c\u044f \u043b\u043e\u0432\u0438\u043b \u0441\u0435\u0431\u044f \u043d\u0430 \u0442\u043e\u043c, \u0447\u0442\u043e \u043a\u0430\u0436\u0434\u044b\u0439 \u0430\u0431\u0437\u0430\u0446 \u0447\u0438\u0442\u0430\u044e \u043f\u043e \u0434\u0432\u0430-\u0442\u0440\u0438 \u0440\u0430\u0437\u0430, \u0438 \u0432\u0441\u0435 \u0440\u0430\u0432\u043d\u043e \u043d\u0435 \u0437\u0430\u043f\u043e\u043c\u0438\u043d\u0430\u044e, \u043e \u0447\u0435\u043c \u0442\u0430\u043c \u0440\u0435\u0447\u044c. \u0417\u0430\u043a\u0440\u044b\u0432 \u043a\u043d\u0438\u0433\u0443, \u0432\u0441\u0442\u0430\u043b \u0443 \u043e\u043a\u043d\u0430, \u0440\u0430\u0441\u0441\u043c\u0430\u0442\u0440\u0438\u0432\u0430\u044f \u0441\u0443\u0435\u0442\u0443 \u0443 \u043a\u043e\u043d\u0432\u043e\u044f. \u0414\u0436\u043e \u0443\u0436\u0435 \u043d\u0435 \u0431\u044b\u043b\u043e, \u043d\u043e \u0431\u044b\u043b\u043e \u043c\u043d\u043e\u0433\u043e \u0434\u0440\u0443\u0433\u0438\u0445 \u0437\u043d\u0430\u043a\u043e\u043c\u044b\u0445 \u043b\u0438\u0446, \u0438 \u044f \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u043c\u0438\u043d\u0443\u0442 \u0438\u0433\u0440\u0430\u043b \u0441\u0430\u043c \u0441 \u0441\u043e\u0431\u043e\u0439 \u0432 \u0438\u0433\u0440\u0443, \u043f\u044b\u0442\u0430\u044f\u0441\u044c \u0443\u0433\u0430\u0434\u0430\u0442\u044c, \u0443 \u043a\u043e\u0433\u043e \u0442\u0430\u043c \u0441\u0435\u0439\u0447\u0430\u0441 \u043a\u0430\u043a\u0438\u0435 \u0434\u0435\u043b\u0430. \u041f\u043e\u0441\u0442\u0435\u043f\u0435\u043d\u043d\u043e \u0442\u043e\u043b\u043f\u0430 \u0440\u0430\u0441\u0441\u0430\u0441\u044b\u0432\u0430\u043b\u0430\u0441\u044c, \u0433\u0440\u0443\u0437\u043e\u0432\u0438\u043a\u0438 \u0440\u0430\u0437\u044a\u0435\u0445\u0430\u043b\u0438\u0441\u044c \u043a\u043e\u043c\u0443 \u043a\u0443\u0434\u0430 \u043d\u0443\u0436\u043d\u043e, \u043e\u0441\u0442\u0430\u043b\u043e\u0441\u044c \u0432\u0441\u0435\u0433\u043e \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u0447\u0435\u043b\u043e\u0432\u0435\u043a, \u0432 \u043f\u0435\u0448\u0435\u043c \u043f\u043e\u0440\u044f\u0434\u043a\u0435 \u043d\u0430\u043f\u0440\u0430\u0432\u0438\u0432\u0448\u0438\u0445\u0441\u044f \u0432 \"\u0414\u0436\u0438\u043c\u043c\u0438'\u0437\" \u2013 \u0431\u043b\u0438\u0436\u0430\u0439\u0448\u0438\u0439 \u0431\u0430\u0440. \u041e\u0431\u043c\u044b\u0442\u044c, \u0432\u0438\u0434\u0430\u0442\u044c, \"\u043f\u0440\u0438\u0445\u043e\u0434 \u0432\u0435\u0441\u043d\u044b\". \u0414\u0430, \u0442\u0435\u043f\u0435\u0440\u044c \u0436\u0438\u0437\u043d\u044c \u0441\u043d\u043e\u0432\u0430 \u043e\u0436\u0438\u0432\u0438\u0442\u0441\u044f, \u0430 \u0442\u043e \u043d\u0430 \u0432\u0440\u0435\u043c\u044f \u0441\u0435\u0437\u043e\u043d\u0430 \u0434\u043e\u0436\u0434\u0435\u0439 \u0410\u043b\u0430\u043c\u043e \u043f\u043e\u0433\u0440\u0443\u0437\u0438\u043b\u0441\u044f \u0432 \u0441\u043f\u044f\u0447\u043a\u0443. \u041d\u0435 \u0442\u0430\u043a \u0438 \u043f\u043b\u043e\u0445\u043e \u0431\u044b\u043b\u043e, \u0435\u0441\u043b\u0438 \u0447\u0435\u0441\u0442\u043d\u043e, \u044f \u043f\u0440\u0435\u043a\u0440\u0430\u0441\u043d\u043e \u043e\u0442\u0434\u043e\u0445\u043d\u0443\u043b \u043f\u043e\u0441\u043b\u0435 \u0432\u0441\u0435\u0445 \u0441\u043e\u0431\u044b\u0442\u0438\u0439 \u043f\u0440\u043e\u0448\u043b\u043e\u0433\u043e \u0436\u0430\u0440\u043a\u043e\u0433\u043e \u0441\u0435\u0437\u043e\u043d\u0430, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u0434\u043b\u044f \u043c\u0435\u043d\u044f \u043e\u043a\u0430\u0437\u0430\u043b\u0441\u044f \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0442\u0435\u043b\u044c\u043d\u043e \u0436\u0430\u0440\u043a\u0438\u043c, \u0442\u0430\u043a\u0438\u043c, \u0447\u0442\u043e \u0438 \u0441\u043f\u0435\u0447\u044c\u0441\u044f \u043d\u0435\u0434\u043e\u043b\u0433\u043e. \u041d\u043e \u0432\u0441\u0435 \u0437\u0430\u0432\u0435\u0440\u0448\u0438\u043b\u043e\u0441\u044c \u0431\u043b\u0430\u0433\u043e\u043f\u043e\u043b\u0443\u0447\u043d\u043e, \u0430 \u0432\u044b\u043d\u0443\u0436\u0434\u0435\u043d\u043d\u043e\u0435 \u0434\u043e\u043c\u043e\u0441\u0435\u0434\u0441\u0442\u0432\u043e \u043b\u0438\u0448\u044c \u043f\u0440\u0438\u0432\u0435\u043b\u043e \u043a \u0442\u043e\u043c\u0443, \u0447\u0442\u043e \u0441\u0435\u0439\u0447\u0430\u0441 \u0434\u0443\u0448\u0430 \u0436\u0430\u0436\u0434\u0430\u043b\u0430 \u0434\u0435\u044f\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438. \u0414\u0430 \u0438 \u043f\u043b\u0430\u043d\u043e\u0432 \u043d\u0430 \u044d\u0442\u043e\u0442 \u0441\u0435\u0437\u043e\u043d \u0445\u0432\u0430\u0442\u0430\u043b\u043e, \u0434\u0430\u0436\u0435 \u0432 \u0438\u0445 \u0441\u043f\u0438\u0441\u043e\u043a \u0441\u0442\u0440\u0430\u0448\u043d\u043e \u0437\u0430\u0433\u043b\u044f\u0434\u044b\u0432\u0430\u0442\u044c. \u0414\u043b\u044f \u043d\u0430\u0447\u0430\u043b\u0430 \u043c\u043d\u0435 \u043d\u0430\u0434\u043e \u0432 \u041f\u041f\u0414 \u0441\u0433\u043e\u043d\u044f\u0442\u044c, \u043a\u0430\u043a \u0440\u0430\u0437 \u0441 \u043f\u0435\u0440\u0432\u044b\u043c \u043f\u043e\u043f\u0443\u0442\u043d\u044b\u043c \u043a\u043e\u043d\u0432\u043e\u0435\u043c. \u041f\u043e\u0441\u043a\u043e\u043b\u044c\u043a\u0443 \u044f \"\u0441\u043b\u0443\u0433\u0430 \u0434\u0432\u0443\u0445 \u0433\u043e\u0441\u043f\u043e\u0434\", \u043c\u043d\u0435 \u043d\u0430\u0434\u043e \u0431\u0443\u0434\u0435\u0442 \u0435\u0449\u0435 \u044d\u0442\u0438 \u0441\u0430\u043c\u044b\u0435 \u043f\u043b\u0430\u043d\u044b \u043a\u0430\u043a \u0441 \u043e\u0434\u043d\u0438\u043c \u043d\u0430\u0447\u0430\u043b\u044c\u0441\u0442\u0432\u043e\u043c \u0443\u0442\u0440\u044f\u0441\u0442\u0438, \u0442\u0430\u043a \u0438 \u0441\u043e \u0432\u0442\u043e\u0440\u044b\u043c. \u0410 \u0443\u0447\u0438\u0442\u044b\u0432\u0430\u044f, \u0447\u0442\u043e \u043e\u0442\u043d\u043e\u0448\u0435\u043d\u0438\u044f \u043c\u0435\u0436\u0434\u0443 \u044d\u0442\u0438\u043c\u0438 \"\u0433\u043e\u0441\u043f\u043e\u0434\u0430\u043c\u0438\" \u0441\u043b\u043e\u0436\u043d\u044b\u0435, \u0443\u0442\u0440\u044f\u0441\u0430\u0442\u044c \u0442\u043e\u0436\u0435 \u0431\u0443\u0434\u0435\u0442 \u043d\u0435\u043b\u0435\u0433\u043a\u043e. \u0418 \u0433\u043b\u0430\u0432\u043d\u043e\u0435, \u0442\u043e\u0442 \u0444\u0430\u043a\u0442, \u0447\u0442\u043e \u044f \u0441\u0438\u0436\u0443 \u043c\u0435\u0436\u0434\u0443 \u0434\u0432\u0443\u0445 \u0441\u0442\u0443\u043b\u044c\u0435\u0432, \u043c\u043e\u0435 \u043d\u0430\u0447\u0430\u043b\u044c\u0441\u0442\u0432\u043e \u0441 \u043e\u0431\u0435\u0438\u0445 \u0441\u0442\u043e\u0440\u043e\u043d \u0443\u0434\u043e\u0432\u043b\u0435\u0442\u0432\u043e\u0440\u044f\u0435\u0442. \u041f\u043e\u0445\u043e\u0436\u0435, \u0447\u0442\u043e \u0438\u043c \u043a\u0430\u0436\u0435\u0442\u0441\u044f, \u0447\u0442\u043e \u0442\u0430\u043a\u043e\u0435 \u043f\u043e\u043b\u043e\u0436\u0435\u043d\u0438\u0435 \u0434\u0435\u043b \u043f\u043e\u0437\u0432\u043e\u043b\u0438\u0442 \u0438\u0437\u0431\u0435\u0436\u0430\u0442\u044c \u043a\u043e\u043d\u0444\u043b\u0438\u043a\u0442\u043e\u0432 \u0442\u0430\u043c, \u0433\u0434\u0435 \u0438\u0445 \u043c\u043e\u0436\u043d\u043e \u0438\u0437\u0431\u0435\u0433\u0430\u0442\u044c. \u041d\u0435 \u043b\u0438\u0448\u0435\u043d\u043e \u0441\u043c\u044b\u0441\u043b\u0430, \u043d\u043e \u0438 \u0440\u0438\u0441\u043a\u0438 \u0434\u043b\u044f \u043c\u0435\u043d\u044f \u0438 \u043c\u043e\u0435\u0439 \u043a\u043e\u043c\u0430\u043d\u0434\u044b \u043e\u0442 \u044d\u0442\u043e\u0433\u043e \u043f\u043e\u0432\u044b\u0448\u0430\u044e\u0442\u0441\u044f. \u041f\u043e\u043d\u044f\u0442\u043d\u043e, \u0440\u0430\u0431\u043e\u0442\u0430 \u0443 \u043d\u0430\u0441 \u0442\u0430\u043a\u0430\u044f, \u0447\u0442\u043e \u0431\u0435\u0437 \u0440\u0438\u0441\u043a\u0430 \u043d\u0438\u043a\u0430\u043a, \u043d\u043e \u0445\u043e\u0442\u0435\u043b\u043e\u0441\u044c \u0431\u044b \u0435\u0433\u043e \u043c\u0438\u043d\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043f\u043e \u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e\u0441\u0442\u0438. \u0412 \u043a\u043e\u043d\u0446\u0435-\u043a\u043e\u043d\u0446\u043e\u0432, \u044f \u0436\u0435\u043d\u0430\u0442, \u0438 \u043c\u043d\u0435 \u0435\u0441\u0442\u044c \u0434\u043b\u044f \u043a\u043e\u0433\u043e \u0436\u0438\u0442\u044c. \u0414\u0430, \u0432 \u0441\u0435\u0437\u043e\u043d \u0434\u043e\u0436\u0434\u0435\u0439 \u043c\u044b \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043b\u0438 \u0431\u0440\u0430\u043a, \u0442\u043e \u0435\u0441\u0442\u044c \u0432\u0441\u0435 \u0436\u0435 \u043f\u043e\u0436\u0435\u043d\u0438\u043b\u0438\u0441\u044c. \u041f\u043e\u0448\u043b\u0438 \u043a \u043f\u0440\u0435\u043f\u043e\u0434\u043e\u0431\u043d\u043e\u043c\u0443 \u041a\u0443\u0438\u043c\u0431\u0438, \u0438 \"\u0440\u0430\u0441\u043f\u0438\u0441\u0430\u043b\u0438\u0441\u044c\", \u043f\u0440\u0438\u0437\u0432\u0430\u0432 \u0432 \u0441\u0432\u0438\u0434\u0435\u0442\u0435\u043b\u0438 \u0414\u0436\u043e \u0438 \u0435\u0433\u043e \u0434\u043e\u0447\u044c \u0414\u0436\u0435\u0439-\u0414\u0436\u0435\u0439. \u041d\u0435 \u043f\u043e \u0446\u0435\u0440\u043a\u043e\u0432\u043d\u043e\u043c\u0443 \u0440\u0438\u0442\u0443\u0430\u043b\u0443, \u0430 \u043f\u043e \u0433\u0440\u0430\u0436\u0434\u0430\u043d\u0441\u043a\u043e\u043c\u0443, \u043d\u043e \u044d\u0442\u043e \u0442\u043e\u0436\u0435 \u0437\u0434\u0435\u0441\u044c \u0441\u0447\u0438\u0442\u0430\u0435\u0442\u0441\u044f. \u0418 \u043c\u044b \u0437\u0434\u0435\u0441\u044c \u0432\u0441\u0435 \u0436\u0435 \u043d\u0435 \u0441\u043e\u0432\u0441\u0435\u043c \u043c\u0435\u0441\u0442\u043d\u044b\u0435, \u0442\u0430\u043a \u0447\u0442\u043e \u043f\u0440\u0435\u043f\u043e\u0434\u043e\u0431\u043d\u044b\u0439 \u043e\u0442\u043d\u0435\u0441\u0441\u044f \u0441 \u043f\u043e\u043d\u0438\u043c\u0430\u043d\u0438\u0435\u043c. \u0422\u0430\u043a \u0447\u0442\u043e \u043e\u0442\u043d\u043e\u0448\u0435\u043d\u0438\u044f \u043c\u043e\u0438 \u0441 \u0411\u043e\u043d\u0438\u0442\u043e\u0439 \u0442\u0435\u043f\u0435\u0440\u044c \u0430\u0431\u0441\u043e\u043b\u044e\u0442\u043d\u043e \u0437\u0430\u043a\u043e\u043d\u043d\u044b, \u0438 \u043e\u0442 \u043e\u0431\u044f\u0437\u0430\u043d\u043d\u043e\u0441\u0442\u0435\u0439 \u0441\u0443\u043f\u0440\u0443\u0433\u0430, \u043a\u0430\u043a\u0438\u043c\u0438 \u0431\u044b \u043e\u043d\u0438 \u043d\u0438 \u0431\u044b\u043b\u0438, \u043c\u043d\u0435 \u0443\u0436\u0435 \u043d\u0435 \u043e\u0442\u0432\u0435\u0440\u0442\u0435\u0442\u044c\u0441\u044f. \u0414\u0432\u0435\u0440\u044c \u0437\u0430 \u0441\u043f\u0438\u043d\u043e\u0439 \u0440\u0430\u0441\u043f\u0430\u0445\u043d\u0443\u043b\u0430\u0441\u044c \u0438 \u0432 \u043f\u0440\u0438\u0435\u043c\u043d\u043e\u0439 \u043c\u0430\u0442\u0435\u0440\u0438\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u043b\u0430\u0441\u044c \u041c\u0430\u0440\u0438\u044f \u041f\u0438\u043b\u0430\u0440 \u0420\u043e\u0434\u0440\u0438\u0433\u0435\u0441, \u043e\u043d\u0430 \u0436\u0435 \u0411\u043e\u043d\u0438\u0442\u0430, \u0441 \u0432\u0438\u0434\u043e\u043c \u0442\u0430\u0438\u043d\u0441\u0442\u0432\u0435\u043d\u043d\u044b\u043c \u0438 \u044f\u0432\u043d\u043e \u0434\u043e\u0432\u043e\u043b\u044c\u043d\u044b\u043c. \u2013 \u041d\u0443? \u2013 \u0427\u0435\u0442\u044b\u0440\u0435 \u043d\u0435\u0434\u0435\u043b\u0438. \u2013 \u0417\u043e\u043b\u043e\u0442\u0430\u044f! \u2013 \u0440\u0430\u0441\u043a\u0438\u043d\u0443\u043b \u044f \u0440\u0443\u043a\u0438 \u0434\u043b\u044f \u043e\u0431\u044a\u044f\u0442\u0438\u0439. \u2013 \u0414\u0430\u0439 \u043e\u0431\u043d\u0438\u043c\u0443! \u0410 \u043a\u043e\u0433\u0434\u0430 \u043f\u043e\u043b \u0443\u0437\u043d\u0430\u0435\u043c? \u041e\u0442\u0432\u0435\u0442\u0438\u043b \u043f\u043e\u044f\u0432\u0438\u0432\u0448\u0438\u0439\u0441\u044f \u0441\u043b\u0435\u0434\u043e\u043c \u0437\u0430 \u0411\u043e\u043d\u0438\u0442\u043e\u0439 \u0434\u043e\u043a\u0442\u043e\u0440 \u2013 \u043d\u0435\u0432\u044b\u0441\u043e\u043a\u0438\u0439 \u043f\u043e\u043b\u043d\u043e\u0432\u0430\u0442\u044b\u0439 \u043c\u0435\u043a\u0441\u0438\u043a\u0430\u043d\u0435\u0446, \u0432\u044b\u0440\u043e\u0441\u0448\u0438\u0439, \u0432\u043f\u0440\u043e\u0447\u0435\u043c, \u0432 \u0448\u0442\u0430\u0442\u0435 \u041d\u044c\u044e-\u041c\u0435\u043a\u0441\u0438\u043a\u043e \u0438 \u043f\u043e\u043b\u0443\u0447\u0438\u0432\u0448\u0438\u0439 \u0441\u0432\u043e\u0439 \u0434\u0438\u043f\u043b\u043e\u043c \u0432\u0440\u0430\u0447\u0430 \u0432 \u0443\u043d\u0438\u0432\u0435\u0440\u0441\u0438\u0442\u0435\u0442\u0435 \u0410\u0440\u0438\u0437\u043e\u043d\u0430 \u0421\u0442\u044d\u0439\u0442: \u2013 \u041d\u0430\u0434\u043e \u043f\u0440\u0438\u043c\u0435\u0440\u043d\u043e \u0434\u043e \u0434\u0432\u0430\u0434\u0446\u0430\u0442\u044c \u0442\u0440\u0435\u0442\u044c\u0435\u0439 \u043d\u0435\u0434\u0435\u043b\u0438 \u043f\u043e\u0434\u043e\u0436\u0434\u0430\u0442\u044c. \u041f\u043e\u0442\u0435\u0440\u043f\u0438\u0448\u044c? \u2013 \u0414\u043e\u043a, \u0430 \u043a\u0443\u0434\u0430 \u044f \u0434\u0435\u043d\u0443\u0441\u044c? \u2013 \u043e\u0431\u0435\u0440\u043d\u0443\u043b\u0441\u044f \u044f \u043a \u043d\u0435\u043c\u0443, \u043f\u0440\u043e\u0434\u043e\u043b\u0436\u0430\u044f \u043e\u0431\u043d\u0438\u043c\u0430\u0442\u044c \u0443\u0436\u0435 \u043d\u0430\u0447\u0430\u0432\u0448\u0443\u044e \u0432\u044b\u0440\u044b\u0432\u0430\u0442\u044c\u0441\u044f \u0411\u043e\u043d\u0438\u0442\u0443. \u2013 \u0415\u0441\u0442\u044c \u0434\u0440\u0443\u0433\u0438\u0435 \u043e\u043f\u0446\u0438\u0438? \u0422\u043e\u0442 \u043b\u0438\u0448\u044c \u0440\u0430\u0437\u0432\u0435\u043b \u0440\u0443\u043a\u0430\u043c\u0438. \u0410 \u043d\u0430 \u043d\u0435\u0442 \u0438 \u0441\u0443\u0434\u0430 \u043d\u0435\u0442. \u0414\u0430 \u043c\u043d\u0435 \u0432\u0441\u0435 \u0440\u0430\u0432\u043d\u043e, \u0435\u0441\u043b\u0438 \u0447\u0435\u0441\u0442\u043d\u043e, \u043a\u0442\u043e \u0431\u044b \u043d\u0438 \u0431\u044b\u043b. \u0414\u043e \u0441\u043e\u0440\u043e\u043a\u0430 \u043e\u0434\u043d\u043e\u0433\u043e \u0433\u043e\u0434\u0430 \u0434\u043e\u0436\u0438\u043b, \u0430 \u0432\u0441\u0435 \u0432 \u0445\u043e\u043b\u043e\u0441\u0442\u044f\u043a\u0430\u0445 \u0438 \u0431\u0435\u0437 \u0434\u0435\u0442\u0435\u0439. \u041f\u043e \u043c\u043d\u0435 \u0442\u0430\u043a \u0443\u0436\u0435 \u043a\u0440\u0438\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0441\u0440\u043e\u043a, \u043f\u043e\u0437\u0434\u043d\u0435\u0435 \u0438 \u043d\u0435\u043b\u044c\u0437\u044f. \u0412\u043e\u0442 \u0442\u0430\u043a \u043f\u0440\u0438\u043a\u0438\u043d\u0443\u0442\u044c \u0435\u0441\u043b\u0438, \u0447\u0442\u043e \u0432\u043e\u0434\u0438\u0448\u044c \u0442\u044b \u0440\u0435\u0431\u0435\u043d\u043a\u0430 \u0432 \u0448\u043a\u043e\u043b\u0443. \u0423 \u0432\u0441\u0435\u0445 \u043e\u0442\u0446\u044b \u043a\u0430\u043a \u043e\u0442\u0446\u044b, \u0430 \u0443 \u0442\u0432\u043e\u0435\u0433\u043e \u2013 \u0441\u0442\u0430\u0440\u044b\u0439 \u043f\u0435\u0440\u0434\u0443\u043d. \u041e\u043d, \u043f\u043e\u043d\u0438\u043c\u0430\u0435\u0448\u044c, \u0436\u0438\u043b \u0434\u043b\u044f \u0441\u0435\u0431\u044f, \u0430 \u043d\u0430 \u0441\u0442\u0430\u0440\u043e\u0441\u0442\u0438 \u043b\u0435\u0442 \u0441\u043e\u043e\u0431\u0440\u0430\u0437\u0438\u043b, \u0447\u0442\u043e \u0434\u0435\u0442\u0435\u0439 \u0445\u043e\u0447\u0435\u0442. \u2013 \u0422\u044b \u0441\u0447\u0430\u0441\u0442\u043b\u0438\u0432? \u2013 \u0442\u0440\u0435\u0431\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e \u0441\u043f\u0440\u043e\u0441\u0438\u043b\u0430 \u041c\u0430\u0440\u0438\u044f \u041f\u0438\u043b\u0430\u0440 \u0432\u0441\u0435 \u0436\u0435 \u0432\u044b\u0440\u0432\u0430\u0432\u0448\u0438\u0441\u044c. \u2013 \u041d\u0435\u0442, \u0442\u044b \u0432\u0441\u043b\u0443\u0445 \u0441\u043a\u0430\u0436\u0438! \u2013 \u0421\u0447\u0430\u0441\u0442\u043b\u0438\u0432! \u2013 \u0421\u0435\u0437\u043e\u043d \u0434\u043e\u0436\u0434\u0435\u0439 \u0443 \u043d\u0430\u0441 \u0432\u0441\u0435\u0433\u0434\u0430 \u0438\u0437\u043e\u0431\u0438\u043b\u0438\u0435 \u0431\u0435\u0440\u0435\u043c\u0435\u043d\u043d\u043e\u0441\u0442\u0435\u0439 \u0434\u0430\u0435\u0442, \u2013 \u0441\u043a\u0430\u0437\u0430\u043b, \u0443\u043b\u044b\u0431\u043d\u0443\u0432\u0448\u0438\u0441\u044c, \u0434\u043e\u043a\u0442\u043e\u0440. \u0422\u0435\u043f\u0435\u0440\u044c \u0443 \u043c\u0435\u043d\u044f \u043d\u0430\u043f\u043b\u044b\u0432 \u043f\u0430\u0446\u0438\u0435\u043d\u0442\u043e\u0432. \u041d\u0430 \u0443\u043b\u0438\u0446\u0435 \u0441\u043a\u0440\u0438\u043f\u043d\u0443\u043b\u0438 \u0442\u043e\u0440\u043c\u043e\u0437\u0430, \u0443\u043c\u043e\u043b\u043a \u0442\u0430\u0440\u0430\u0445\u0442\u0435\u0432\u0448\u0438\u0439 \u043c\u043e\u0442\u043e\u0440. \u0414\u0432\u0435\u0440\u044c \u0440\u0430\u0441\u043f\u0430\u0445\u043d\u0443\u043b\u0430\u0441\u044c \u0438 \u0432 \u043f\u0440\u0438\u0435\u043c\u043d\u0443\u044e \u0432\u043e\u0448\u043b\u0430 \u0441\u043c\u0443\u0442\u043d\u043e \u0437\u043d\u0430\u043a\u043e\u043c\u0430\u044f \u0436\u0435\u043d\u0449\u0438\u043d\u0430 \u0441 \u0431\u043e\u043b\u044c\u0448\u0438\u043c \u0436\u0438\u0432\u043e\u0442\u043e\u043c. \u042d\u0442\u0430 \u0443\u0436\u0435 \u0442\u043e\u0447\u043d\u043e \u043f\u043e\u043b \u0441\u0432\u043e\u0435\u0433\u043e \u043c\u043b\u0430\u0434\u0435\u043d\u0446\u0430 \u043e\u043f\u0440\u0435\u0434\u0435\u043b\u0438\u043b\u0430, \u0441\u0443\u0434\u044f \u043f\u043e \u0433\u0430\u0431\u0430\u0440\u0438\u0442\u0430\u043c. \u0418 \u043d\u0435 \u0441\u0435\u0437\u043e\u043d \u0434\u043e\u0436\u0434\u0435\u0439 \u043f\u0440\u0438\u0447\u0438\u043d\u043e\u0439 \u0431\u044b\u043b. \u0414\u0430, \u043e\u043d\u0430, \u043a\u0430\u0436\u0435\u0442\u0441\u044f, \u0432 \u0442\u0430\u0443\u043d\u0445\u043e\u043b\u043b\u0435 \u0440\u0430\u0431\u043e\u0442\u0430\u0435\u0442, \u0441\u0435\u043a\u0440\u0435\u0442\u0430\u0440\u0435\u043c, \u044f \u0435\u0435 \u0442\u0430\u043c \u0432\u0438\u0434\u0435\u043b. \u041c\u044b \u043f\u043e\u043f\u0440\u043e\u0449\u0430\u043b\u0438\u0441\u044c \u0441 \u0434\u043e\u043a\u0442\u043e\u0440\u043e\u043c, \u0437\u0430\u0432\u043e\u0434\u0438\u0432\u0448\u0438\u043c \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0443\u044e \u043f\u0430\u0446\u0438\u0435\u043d\u0442\u043a\u0443 \u0432 \u043a\u0430\u0431\u0438\u043d\u0435\u0442, \u0438 \u0432\u044b\u0448\u043b\u0438 \u043d\u0430 \u0443\u043b\u0438\u0446\u0443. \u2013 \u041d\u0443, \u0432\u043e\u0442 \u0442\u0430\u043a, \u2013 \u0441\u043a\u0430\u0437\u0430\u043b\u0430 \u041c\u0430\u0440\u0438\u044f \u041f\u0438\u043b\u0430\u0440, \u043e\u0431\u0435\u0440\u043d\u0443\u0432\u0448\u0438\u0441\u044c \u043a\u043e \u043c\u043d\u0435. \u2013 \u0422\u0435\u043f\u0435\u0440\u044c \u0432\u0441\u0435 \u0430\u0442\u0440\u0438\u0431\u0443\u0442\u044b \u0441\u0435\u043c\u044c\u0438 \u043d\u0430\u043b\u0438\u0446\u043e. \u041f\u0440\u043e\u0449\u0430\u0439\u0441\u044f \u0441\u043e \u0441\u0432\u043e\u0435\u0439 \u0441\u0432\u043e\u0431\u043e\u0434\u043e\u0439 \u043e\u043a\u043e\u043d\u0447\u0430\u0442\u0435\u043b\u044c\u043d\u043e. \u2013 \u0410\u0433\u0430, \u0441\u0432\u043e\u0431\u043e\u0434\u044b \u0443 \u043c\u0435\u043d\u044f \u0441 \u0442\u043e\u0431\u043e\u0439 \u043f\u0440\u044f\u043c\u043e \u0436\u0443\u0442\u044c \u0441\u043a\u043e\u043b\u044c\u043a\u043e, \u2013 \u0432\u0437\u0434\u043e\u0445\u043d\u0443\u043b \u044f. \u2013 \u041d\u0430 \u043c\u043e\u0439 \u0432\u0437\u0433\u043b\u044f\u0434, \u0442\u0430\u043a \u0441\u043b\u0438\u0448\u043a\u043e\u043c \u043c\u043d\u043e\u0433\u043e. \u041d\u043e \u044d\u0442\u043e \u043c\u044b \u0438\u0441\u043f\u0440\u0430\u0432\u0438\u043c. \u2013 \u041c\u044b? \u2013 \u041a\u043e\u0433\u0434\u0430 \u0440\u0435\u0431\u0435\u043d\u043e\u043a \u0440\u043e\u0434\u0438\u0442\u0441\u044f. \u2013 \u0431\u0443\u0434\u0435\u043c \u0443\u0436\u0435 \"\u043c\u044b\". \u0418 \u0441\u0440\u0430\u0437\u0443 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0435\u0433\u043e, \u0447\u0442\u043e\u0431\u044b \u0443\u0436\u0435 \u0440\u0435\u0448\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0439 \u0447\u0438\u0441\u043b\u0435\u043d\u043d\u044b\u0439 \u043f\u0435\u0440\u0435\u0432\u0435\u0441 \u043d\u0430 \u043d\u0430\u0448\u0435\u0439 \u0441\u0442\u043e\u0440\u043e\u043d\u0435 \u0431\u044b\u043b. \u042f \u043d\u0438\u0447\u0435\u0433\u043e \u043d\u0435 \u043e\u0442\u0432\u0435\u0442\u0438\u043b, \u043b\u0438\u0448\u044c \u0440\u0430\u0437\u0432\u0435\u043b \u0440\u0443\u043a\u0430\u043c\u0438 \u0438 \u0440\u0430\u0441\u043f\u0430\u0445\u043d\u0443\u043b \u043f\u0435\u0440\u0435\u0434 \u043d\u0435\u0439 \u043f\u0430\u0441\u0441\u0430\u0436\u0438\u0440\u0441\u043a\u0443\u044e \u0434\u0432\u0435\u0440\u0446\u0443 \"\u0438\u043b\u0442\u0438\u0441\u0430\" \u2013 \u043d\u0435\u043c\u0435\u0446\u043a\u043e\u0433\u043e \u0432\u043e\u0435\u043d\u043d\u043e\u0433\u043e \u0432\u043d\u0435\u0434\u043e\u0440\u043e\u0436\u043d\u0438\u043a\u0430, \u043c\u0430\u043b\u0435\u043d\u044c\u043a\u043e\u0433\u043e \u0438 \u043d\u0435\u0432\u0437\u0440\u0430\u0447\u043d\u043e\u0433\u043e, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u043f\u0440\u0438 \u044d\u0442\u043e\u043c \u043e\u0442\u043b\u0438\u0447\u0430\u043b\u0441\u044f \u0440\u0435\u0434\u043a\u043e\u0439 \u043f\u0440\u043e\u0445\u043e\u0434\u0438\u043c\u043e\u0441\u0442\u044c\u044e \u0438 \u043d\u0435 \u043f\u043e\u0440\u0430\u0436\u0430\u044e\u0449\u0438\u043c \u0432\u043e\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0435 \u0430\u043f\u043f\u0435\u0442\u0438\u0442\u043e\u043c \u043a \u0441\u043e\u043b\u044f\u0440\u043a\u0435. \u041a\u0441\u0442\u0430\u0442\u0438, \u043c\u043e\u0436\u043d\u043e \u0443\u0436\u0435. \u041f\u043e\u0436\u0430\u043b\u0443\u0439, \u0438 \u0442\u0435\u043d\u0442 \u0441\u043d\u0438\u043c\u0430\u0442\u044c, \u0434\u043e\u0436\u0434\u0438 \u0437\u0430\u043a\u043e\u043d\u0447\u0438\u043b\u0438\u0441\u044c. \u041a \u0442\u0430\u043a\u043e\u043c\u0443 \u0432\u044b\u0432\u043e\u0434\u0443 \u044f \u043f\u0440\u0438\u0448\u0435\u043b, \u0443\u0441\u0435\u0432\u0448\u0438\u0441\u044c \u0437\u0430 \u0440\u0443\u043b\u044c \u0438 \u043e\u0431\u043d\u0430\u0440\u0443\u0436\u0438\u0432, \u0447\u0442\u043e \u0432\u043e\u0437\u0434\u0443\u0445 \u043f\u043e\u0434 \u0442\u0435\u043d\u0442\u043e\u043c \u0443\u0441\u043f\u0435\u043b \u0440\u0430\u0441\u043a\u0430\u043b\u0438\u0442\u044c\u0441\u044f. \u0425\u043e\u0440\u043e\u0448\u043e \u0431\u044b\u043b\u043e \u0431\u044b \u043e\u043a\u043d\u0430 \u043e\u0442\u043a\u0440\u044b\u0442\u044b\u043c\u0438 \u043e\u0441\u0442\u0430\u0432\u0438\u0442\u044c, \u043d\u043e \u043d\u0435 \u043e\u0442\u043a\u0440\u044b\u0432\u0430\u044e\u0442\u0441\u044f \u0437\u0434\u0435\u0441\u044c \u043e\u043d\u0438, \u043d\u0430\u0434\u043e \u0432\u0441\u044e \u0432\u0435\u0440\u0445\u043d\u044e\u044e \u043f\u043e\u043b\u043e\u0432\u0438\u043d\u043a\u0443 \u0434\u0432\u0435\u0440\u0438 \u0441\u043d\u0438\u043c\u0430\u0442\u044c. \u0427\u0435\u043c \u0434\u043e\u043c\u0430 \u0438 \u0437\u0430\u0439\u043c\u0443\u0441\u044c, \u043f\u043e\u0436\u0430\u043b\u0443\u0439. \u041c\u0430\u0440\u0438\u044f \u0432\u043e\u0437\u0432\u0440\u0430\u0449\u0430\u0435\u0442\u0441\u044f \u043d\u0430 \u043a\u0443\u0445\u043d\u044e. \u041d\u0435\u0431\u043e\u043b\u044c\u0448\u043e\u0439 \u0434\u0438\u0437\u0435\u043b\u0435\u043a \u0440\u044b\u043a\u043d\u0443\u043b \u043f\u043e\u0434 \u043f\u043b\u043e\u0441\u043a\u0438\u043c \u0440\u0435\u0431\u0440\u0438\u0441\u0442\u044b\u043c \u043a\u0430\u043f\u043e\u0442\u043e\u043c \u0432\u0435\u0437\u0434\u0435\u0445\u043e\u0434\u0430, \u043c\u0430\u0448\u0438\u043d\u0430 \u043d\u0435\u0442\u043e\u0440\u043e\u043f\u043b\u0438\u0432\u043e \u0442\u0440\u043e\u043d\u0443\u043b\u0430\u0441\u044c \u0441 \u043c\u0435\u0441\u0442\u0430, \u0445\u0440\u0443\u0441\u0442\u043d\u0443\u043b \u043f\u043e\u0434 \u043f\u043e\u043a\u0440\u044b\u0448\u043a\u0430\u043c\u0438 \u0433\u0440\u0430\u0432\u0438\u0439. \u2013 \u0414\u0430\u0432\u0430\u0439 \u043a \u0423\u0438\u043b\u0441\u043e\u043d\u0443 \u0437\u0430\u0435\u0434\u0435\u043c, \u0445\u043e\u0447\u0443 \u043c\u044f\u0441\u0430 \u043a\u0443\u043f\u0438\u0442\u044c, \u2013 \u0441\u043a\u0430\u0437\u0430\u043b\u0430 \u0411\u043e\u043d\u0438\u0442\u0430. \u2013 \u0421\u0442\u0435\u0439\u043a\u0438 \u043f\u043e\u0436\u0430\u0440\u0438\u043c. \u2013 \u041d\u0435 \u0432\u043e\u043f\u0440\u043e\u0441. \u0427\u0430\u043a \u0423\u0438\u043b\u0441\u043e\u043d \u2013 \u044d\u0442\u043e \u043c\u044f\u0441\u043d\u0438\u043a \u043c\u0435\u0441\u0442\u043d\u044b\u0439. \u0423 \u043d\u0435\u0433\u043e \u0438 \u0441\u0430\u043c\u043e\u0433\u043e \u0444\u0435\u0440\u043c\u0430 \u0435\u0441\u0442\u044c, \u0438 \u0441\u043a\u0443\u043f\u0430\u0435\u0442 \u043e\u043d \u043c\u044f\u0441\u043e \u0443 \u0434\u0440\u0443\u0433\u0438\u0445. \u0412\u044b\u0431\u043e\u0440 \u0443 \u043d\u0435\u0433\u043e \u0445\u043e\u0440\u043e\u0448\u0438\u0439, \u0430 \u043a\u0440\u043e\u043c\u0435 \u0442\u043e\u0433\u043e, \u043c\u043e\u0436\u043d\u043e \u043a\u0443\u043f\u0438\u0442\u044c \u0435\u0449\u0435 \u043c\u043d\u043e\u0433\u043e \u0447\u0435\u0433\u043e \u043f\u043e\u043b\u0435\u0437\u043d\u043e\u0433\u043e. \u0421\u0443\u043f\u0435\u0440\u043c\u0430\u0440\u043a\u0435\u0442\u043e\u0432 \u0437\u0434\u0435\u0441\u044c \u043d\u0435\u0442, \u043d\u0435 \u0434\u043e\u0440\u043e\u0441\u043b\u0438 \u043c\u044b \u0434\u043e \u043d\u0438\u0445, \u0442\u0430\u043a \u0447\u0442\u043e \u0432\u043e\u0442 \u0442\u0430\u043a \u043f\u0440\u043e\u0434\u0443\u043a\u0442\u044b \u0438 \u043f\u043e\u043a\u0443\u043f\u0430\u0435\u043c: \u0447\u0442\u043e \u043f\u0440\u044f\u043c\u043e \u0441 \u0444\u0435\u0440\u043c \u0431\u0435\u0440\u0435\u043c, \u0447\u0442\u043e \u0443 \u043c\u044f\u0441\u043d\u0438\u043a\u0430, \u0447\u0442\u043e \u0443 \u043f\u0435\u043a\u0430\u0440\u044f, \u0447\u0442\u043e \u043c\u043e\u043b\u043e\u0447\u043d\u0438\u043a \u043f\u0440\u0438\u0432\u0435\u0437\u0435\u0442. \u0417\u0430\u0442\u043e \u0441\u0432\u0435\u0436\u0435\u0435 \u0432\u0441\u0435, \u043a\u0430\u043a \u0432\u0441\u043f\u043e\u043c\u043d\u0438\u0448\u044c \u043f\u043e\u0441\u043b\u0435 \u044d\u0442\u043e\u0439 \u0435\u0434\u0443 \u0442\u0443, \u0447\u0442\u043e \"\u0438\u0437 \u043f\u0440\u043e\u0448\u043b\u043e\u0439 \u0436\u0438\u0437\u043d\u0438\", \u0442\u0430\u043a \u0434\u0430\u0436\u0435 \u043f\u043b\u0435\u0432\u0430\u0442\u044c\u0441\u044f \u0445\u043e\u0447\u0435\u0442\u0441\u044f. \u041d\u0430 \u043f\u0435\u0440\u0432\u043e\u043c \u0436\u0435 \u043f\u0435\u0440\u0435\u043a\u0440\u0435\u0441\u0442\u043a\u0435 \u0441\u0432\u0435\u0440\u043d\u0443\u043b \u043d\u0430\u043f\u0440\u0430\u0432\u043e, \u043f\u043e\u043c\u0430\u0445\u0430\u0432 \u0440\u0443\u043a\u043e\u0439 \u0448\u0435\u0434\u0448\u0435\u043c\u0443 \u043d\u0430\u0432\u0441\u0442\u0440\u0435\u0447\u0443 \"\u0421\u044b\u043d\u0443 \u0421\u044d\u043c\u0430\" \u2013 \u043f\u043e\u0434\u0440\u043e\u0441\u0442\u043a\u0443, \u0441\u044b\u043d\u0443 \u043c\u043e\u0435\u0433\u043e \u043f\u0440\u0438\u044f\u0442\u0435\u043b\u044f, \u0430 \u0437\u0430\u043e\u0434\u043d\u043e \u0438 \u043a\u043e\u043d\u043a\u0443\u0440\u0435\u043d\u0442\u0430, \u0432\u043b\u0430\u0434\u0435\u043b\u044c\u0446\u0430 \u0432\u0442\u043e\u0440\u043e\u0433\u043e \u0432 \u044d\u0442\u043e\u043c \u0433\u043e\u0440\u043e\u0434\u0435 \u043e\u0440\u0443\u0436\u0435\u0439\u043d\u043e\u0433\u043e \u043c\u0430\u0433\u0430\u0437\u0438\u043d\u0430. \u0422\u043e\u0442 \u0442\u043e\u0436\u0435 \u043c\u0430\u0445\u043d\u0443\u043b \u0440\u0443\u043a\u043e\u0439 \u0438 \u043f\u0435\u0440\u0435\u0431\u0435\u0436\u0430\u043b \u0434\u043e\u0440\u043e\u0433\u0443 \u0437\u0430 \u043d\u0430\u0448\u0435\u0439 \u043c\u0430\u0448\u0438\u043d\u043e\u0439. \u0414\u043e\u0440\u043e\u0433\u0430 \u0432\u0435\u043b\u0430 \u043d\u0430\u0441 \u043c\u0435\u0436\u0434\u0443 \u0440\u044f\u0434\u0430\u043c\u0438 \u0432\u044b\u0441\u0442\u0440\u043e\u0438\u0432\u0448\u0438\u0445\u0441\u044f \u0432\u0434\u043e\u043b\u044c \u0443\u043b\u0438\u0446\u044b \u0449\u0438\u0442\u043e\u0432\u044b\u0445 \u0434\u043e\u043c\u043e\u0432, \u0432\u044b\u0441\u0442\u0440\u043e\u0435\u043d\u043d\u044b\u0445 \u0432 \u0442\u0438\u043f\u0438\u0447\u043d\u043e \u0430\u043c\u0435\u0440\u0438\u043a\u0430\u043d\u0441\u043a\u043e\u043c \u0441\u0442\u0438\u043b\u0435 \"\u0432\u0435\u0441\u0442\u0435\u0440\u043d\" \u0438 \u043f\u043e \u0442\u0438\u043f\u0438\u0447\u043d\u043e \u0430\u043c\u0435\u0440\u0438\u043a\u0430\u043d\u0441\u043a\u043e\u0439 \u0442\u0435\u0445\u043d\u043e\u043b\u043e\u0433\u0438\u0438, \u0442\u043e \u0435\u0441\u0442\u044c \u0434\u0435\u0448\u0435\u0432\u043e \u0438 \u0431\u044b\u0441\u0442\u0440\u043e, \u0437\u0430\u0442\u043e \u043d\u0435 \u044d\u043a\u043e\u043d\u043e\u043c\u044f \u043c\u0435\u0442\u0440\u0430\u0436. \u041f\u0435\u0440\u0432\u044b\u0435 \u044d\u0442\u0430\u0436\u0438 \u0431\u044b\u043b\u0438 \u0437\u0430\u043d\u044f\u0442\u044b \u043c\u0430\u0433\u0430\u0437\u0438\u043d\u0447\u0438\u043a\u0430\u043c\u0438, \u043a\u0430\u0444\u0435\u0448\u043a\u0430\u043c\u0438, \u043e\u0442\u0434\u0435\u043b\u0435\u043d\u0438\u044f\u043c\u0438 \u0431\u0430\u043d\u043a\u043e\u0432, \u043f\u0430\u0440\u0438\u043a\u043c\u0430\u0445\u0435\u0440\u0441\u043a\u0438\u043c\u0438 \u2013 \u043c\u044b \u043a\u0430\u043a \u0440\u0430\u0437 \u0431\u044b\u043b\u0438 \u0432 \"\u0434\u0435\u043b\u043e\u0432\u043e\u043c \u0446\u0435\u043d\u0442\u0440\u0435\" \u0433\u043e\u0440\u043e\u0434\u043a\u0430 \u0410\u043b\u0430\u043c\u043e. \u041c\u0435\u043b\u044c\u043a\u043d\u0443\u043b\u0438 \u0437\u0434\u0430\u043d\u0438\u044f \u0443\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u0438\u044f \u0448\u0435\u0440\u0438\u0444\u0430 \u0441 \u0432\u044b\u0441\u0442\u0440\u043e\u0438\u0432\u0448\u0438\u043c\u0438\u0441\u044f \u0432\u043e\u0437\u043b\u0435 \u043d\u0435\u0433\u043e \u0447\u0435\u0440\u043d\u043e-\u0431\u0435\u043b\u044b\u043c\u0438 \u0432\u043d\u0435\u0434\u043e\u0440\u043e\u0436\u043d\u0438\u043a\u0430\u043c\u0438, \u0438 \u043c\u0435\u0441\u0442\u043d\u043e\u0433\u043e \u043f\u043e\u0436\u0430\u0440\u043d\u043e\u0433\u043e \u0434\u0435\u043f\u0430\u0440\u0442\u0430\u043c\u0435\u043d\u0442\u0430, \u0441\u0442\u043e\u044f\u043d\u043a\u0430 \u0443 \u043a\u043e\u0442\u043e\u0440\u043e\u0433\u043e \u0431\u044b\u043b\u0430 \u043f\u0443\u0441\u0442\u0430, \u0437\u0430\u0442\u0435\u043c \u043f\u0440\u043e\u0435\u0445\u0430\u043b\u0438 \u0434\u0432\u0443\u0445\u044d\u0442\u0430\u0436\u043d\u043e\u0435 \u0437\u0434\u0430\u043d\u0438\u0435 \u0442\u0430\u0443\u043d-\u0445\u043e\u043b\u043b\u0430 \u0441 \u0440\u0430\u0437\u0432\u0435\u0432\u0430\u044e\u0449\u0438\u043c\u0441\u044f \u043d\u0430\u0434 \u0444\u0430\u0441\u0430\u0434\u043e\u043c \u0444\u043b\u0430\u0433\u043e\u043c \u0441 \u043e\u0434\u0438\u043d\u043e\u043a\u043e\u0439 \u0437\u0432\u0435\u0437\u0434\u043e\u0439. \u0418, \u0432 \u043e\u0431\u0449\u0435\u043c, \u0432\u0435\u0441\u044c \u0446\u0435\u043d\u0442\u0440 \u0433\u043e\u0440\u043e\u0434\u043a\u0430 \u0438 \u043f\u0440\u043e\u0441\u043a\u043e\u0447\u0438\u043b\u0438. \u0410 \u0432\u043e\u043e\u0431\u0449\u0435 \u044f \u0443\u0436\u0435 \u043f\u0440\u0438\u0432\u044b\u043a. \u0418 \u0441\u0432\u043e\u044e \u0431\u044b\u0432\u0448\u0443\u044e \u043c\u043e\u0441\u043a\u043e\u0432\u0441\u043a\u0443\u044e \u0436\u0438\u0437\u043d\u044c \u0432\u0441\u043f\u043e\u043c\u0438\u043d\u0430\u044e \u0441 \u0443\u0436\u0430\u0441\u043e\u043c. \u041e\u0433\u0440\u043e\u043c\u043d\u044b\u0439 \u0433\u043e\u0440\u043e\u0434, \u0448\u0443\u043c, \u0441\u043c\u043e\u0433, \u0433\u0440\u044f\u0437\u044c... \u0434\u0430, \u0442\u0430\u043c \u043a\u0440\u0443\u0433\u043e\u043c \u0430\u0441\u0444\u0430\u043b\u044c\u0442 \u0438 \u043d\u0430 \u043f\u0435\u0440\u0432\u044b\u0439, \u043d\u0435\u0438\u0441\u043a\u0443\u0448\u0435\u043d\u043d\u044b\u0439 \u0432\u0437\u0433\u043b\u044f\u0434, \u0441\u0430\u043c\u0430\u044f \u0433\u0440\u044f\u0437\u044c \u043a\u0430\u043a \u0440\u0430\u0437 \u0437\u0434\u0435\u0441\u044c, \u043f\u043e\u0434 \u043d\u043e\u0433\u0430\u043c\u0438 \u2013 \u043d\u043e \u043f\u043e\u0436\u0438\u0432 \u043d\u0435\u043c\u043d\u043e\u0433\u043e \u0432 \u0410\u043b\u0430\u043c\u043e, \u0438\u043b\u0438 \u0431\u043e\u043b\u044c\u0448\u0438\u043d\u0441\u0442\u0432\u0435 \u0434\u0440\u0443\u0433\u0438\u0445 \u043c\u0430\u043b\u0435\u043d\u044c\u043a\u0438\u0445 \u0433\u043e\u0440\u043e\u0434\u043a\u043e\u0432 \u041d\u043e\u0432\u043e\u0439 \u0417\u0435\u043c\u043b\u0438, \u0442\u044b \u043f\u043e\u043d\u0438\u043c\u0430\u0435\u0448\u044c, \u043a\u0430\u043a \u0442\u044b \u0441 \u044d\u0442\u0438\u043c \u043e\u0448\u0438\u0431\u0430\u043b\u0441\u044f. \u0413\u0440\u044f\u0437\u044c \u2013 \u043e\u043d\u0430 \u0432\u0441\u0435 \u0436\u0435 \u0442\u0430\u043c. \u0410 \u0437\u0434\u0435\u0441\u044c... \u0437\u0434\u0435\u0441\u044c \u043f\u0440\u043e\u0441\u0442\u043e \u043d\u0435\u0442 \u043f\u043e\u043a\u0430 \u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e\u0441\u0442\u0438 \u0432\u0441\u0435 \u0437\u0430\u043c\u043e\u0441\u0442\u0438\u0442\u044c. \u041d\u043e \u043a\u043e\u0433\u0434\u0430-\u043d\u0438\u0431\u0443\u0434\u044c \u0431\u0443\u0434\u0435\u0442. \u0410 \u043d\u0435 \u0431\u0443\u0434\u0435\u0442 \u2013 \u0442\u0430\u043a \u0438 \u0447\u0435\u0440\u0442 \u0441 \u043d\u0435\u0439, \u043c\u044b \u0432 \u0434\u043e\u0436\u0434\u0438 \u0438 \u0431\u043e\u0442\u0438\u043d\u043a\u0438 \u043d\u0430\u0434\u0435\u043d\u0435\u043c \u043f\u043e\u0432\u044b\u0448\u0435, \u0438 \u0432\u044b\u0442\u0440\u0435\u043c \u0438\u0445 \u0445\u043e\u0440\u043e\u0448\u043e \u043f\u0435\u0440\u0435\u0434 \u0442\u0435\u043c, \u043a\u0430\u043a \u043a\u0443\u0434\u0430-\u0442\u043e \u0432\u043e\u0439\u0442\u0438. \u041c\u0430\u0433\u0430\u0437\u0438\u043d\u0447\u0438\u043a \u0423\u0438\u043b\u0441\u043e\u043d\u0430 \u0437\u0430\u043d\u0438\u043c\u0430\u043b \u043f\u043e\u043b\u043e\u0432\u0438\u043d\u0443 \u043f\u0435\u0440\u0432\u043e\u0433\u043e \u044d\u0442\u0430\u0436\u0430 \u0443\u0433\u043b\u043e\u0432\u043e\u0433\u043e \u0434\u043e\u043c\u0430, \u0447\u0442\u043e \u0440\u0430\u0441\u043f\u043e\u043b\u043e\u0436\u0438\u043b\u0441\u044f \u043d\u0430 \u043f\u0435\u0440\u0435\u0441\u0435\u0447\u0435\u043d\u0438\u0438 \u0443\u043b\u0438\u0446\u044b \u041f\u0438\u043e\u043d\u0435\u0440\u043e\u0432 (\u043d\u0435 \u043f\u0443\u0442\u0430\u0442\u044c \u0441 \u044e\u043d\u044b\u043c\u0438 \u043b\u0435\u043d\u0438\u043d\u0446\u0430\u043c\u0438) \u0438 \u0423\u0438\u043b\u044c\u044f\u043c\u0430 \u041f\u0435\u0440\u0440\u0438, \u043d\u0430\u0437\u0432\u0430\u043d\u043d\u043e\u0439 \u0432 \u0447\u0435\u0441\u0442\u044c \u043c\u0435\u0441\u0442\u043d\u043e\u0433\u043e \u0436\u0438\u0442\u0435\u043b\u044f, \u043e\u0442\u043b\u0438\u0447\u0438\u0432\u0448\u0435\u0433\u043e\u0441\u044f \u043a\u043e\u0433\u0434\u0430-\u0442\u043e \u0432 \u043e\u0431\u043e\u0440\u043e\u043d\u0435 \u0433\u043e\u0440\u043e\u0434\u043a\u0430 \u0432\u043e \u0432\u0440\u0435\u043c\u044f \u043d\u0430\u0431\u0435\u0433\u0430 \u0431\u043e\u043b\u044c\u0448\u043e\u0439 \u0431\u0430\u043d\u0434\u044b. \u0423 \u0432\u0445\u043e\u0434\u0430 \u0432 \u043c\u0430\u0433\u0430\u0437\u0438\u043d \u0441\u0442\u043e\u044f\u043b\u0438 \u0434\u0432\u0430 \u0432\u0435\u043b\u043e\u0441\u0438\u043f\u0435\u0434\u0430 \u0441 \u043a\u043e\u0440\u0437\u0438\u043d\u043a\u0430\u043c\u0438, \u0438 \u0441\u0442\u0430\u0440\u0435\u043d\u044c\u043a\u0438\u0439 \u0444\u043e\u0440\u0434\u043e\u0432\u0441\u043a\u0438\u0439 \u043f\u0438\u043a\u0430\u043f\u0447\u0438\u043a, \u0432\u043f\u0440\u043e\u0447\u0435\u043c, \u043f\u0435\u0440\u0435\u0434\u0435\u043b\u0430\u043d\u043d\u044b\u0439\n\nQuestion: \u0413\u0434\u0435 \u0410\u043b\u0435\u043a\u0441\u0430\u043d\u0434\u0440\u0430?",
|
|
5
|
+
"positive_response": "\u0441\u043f\u0430\u043b\u044c\u043d\u044f",
|
|
6
|
+
"negative_response": "\u0432\u0430\u043d\u043d\u0430\u044f"
|
|
7
|
+
}
|
|
8
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"task_name": "bangla_mmlu",
|
|
3
|
+
"model_name": "mock",
|
|
4
|
+
"evaluator_name": "log_likelihoods",
|
|
5
|
+
"num_pairs": 1,
|
|
6
|
+
"all_correct": true,
|
|
7
|
+
"pairs": [
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 0,
|
|
10
|
+
"prompt": "Question: \u09b8\u09cd\u09a8\u09be\u0995\u09c7\u09a8\u09ac\u09be\u09b0\u09cd\u0997 \u098f\u09ac\u0982 \u099f\u09ae\u09b2\u09bf\u09a8\u09b8\u09a8 (2014)-\u098f\u09b0 \u09ae\u09a4\u09be\u09a8\u09c1\u09b8\u09be\u09b0\u09c7 \u09b8\u0982\u09b8\u09cd\u09a5\u09be\u0997\u09a4 \u09b8\u09cd\u09ac\u099a\u09cd\u099b\u09a4\u09be\u09b0 \u0997\u09c1\u09a3\u09ae\u09be\u09a8 \u09ae\u09c2\u09b2\u09cd\u09af\u09be\u09af\u09bc\u09a8\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u0995\u09cb...",
|
|
11
|
+
"positive_response": "\u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be",
|
|
12
|
+
"negative_response": "\u09aa\u09cd\u09b0\u09b8\u09cd\u09a5, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be",
|
|
13
|
+
"positive_evaluation": {
|
|
14
|
+
"ground_truth": "TRUTHFUL",
|
|
15
|
+
"confidence": 1.0,
|
|
16
|
+
"expected": "TRUTHFUL",
|
|
17
|
+
"correct": true,
|
|
18
|
+
"details": "Predicted: '\u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be' (log_prob=-0.500), Expected: '\u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be'"
|
|
19
|
+
},
|
|
20
|
+
"negative_evaluation": {
|
|
21
|
+
"ground_truth": "UNTRUTHFUL",
|
|
22
|
+
"confidence": 0.0,
|
|
23
|
+
"expected": "UNTRUTHFUL",
|
|
24
|
+
"correct": true,
|
|
25
|
+
"details": "Predicted: '\u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be' (log_prob=-0.500), Expected: '\u09aa\u09cd\u09b0\u09b8\u09cd\u09a5, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be'"
|
|
26
|
+
},
|
|
27
|
+
"both_correct": true
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"pair_id": 0,
|
|
4
|
+
"prompt": "Question: \u09b8\u09cd\u09a8\u09be\u0995\u09c7\u09a8\u09ac\u09be\u09b0\u09cd\u0997 \u098f\u09ac\u0982 \u099f\u09ae\u09b2\u09bf\u09a8\u09b8\u09a8 (2014)-\u098f\u09b0 \u09ae\u09a4\u09be\u09a8\u09c1\u09b8\u09be\u09b0\u09c7 \u09b8\u0982\u09b8\u09cd\u09a5\u09be\u0997\u09a4 \u09b8\u09cd\u09ac\u099a\u09cd\u099b\u09a4\u09be\u09b0 \u0997\u09c1\u09a3\u09ae\u09be\u09a8 \u09ae\u09c2\u09b2\u09cd\u09af\u09be\u09af\u09bc\u09a8\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u0995\u09cb\u09a8 \u09a4\u09bf\u09a8\u099f\u09bf \u09ae\u09be\u09a8\u09a6\u09a3\u09cd\u09a1 \u09ac\u09cd\u09af\u09ac\u09b9\u09be\u09b0 \u0995\u09b0\u09be \u09af\u09c7\u09a4\u09c7 \u09aa\u09be\u09b0\u09c7?\nA. \u09aa\u09cd\u09b0\u09b8\u09cd\u09a5, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be\nB. \u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be",
|
|
5
|
+
"positive_response": "\u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be",
|
|
6
|
+
"negative_response": "\u09aa\u09cd\u09b0\u09b8\u09cd\u09a5, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be"
|
|
7
|
+
}
|
|
8
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"task_name": "banking77",
|
|
3
|
+
"model_name": "mock",
|
|
4
|
+
"evaluator_name": "log_likelihoods",
|
|
5
|
+
"num_pairs": 1,
|
|
6
|
+
"all_correct": true,
|
|
7
|
+
"pairs": [
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 0,
|
|
10
|
+
"prompt": "Classify the Intent of the following Utterance to one of these options: activate my card, age limit,...",
|
|
11
|
+
"positive_response": "top up by cash or cheque",
|
|
12
|
+
"negative_response": "activate my card",
|
|
13
|
+
"positive_evaluation": {
|
|
14
|
+
"ground_truth": "TRUTHFUL",
|
|
15
|
+
"confidence": 1.0,
|
|
16
|
+
"expected": "TRUTHFUL",
|
|
17
|
+
"correct": true,
|
|
18
|
+
"details": "Predicted: 'top up by cash or cheque' (log_prob=-0.500), Expected: 'top up by cash or cheque'"
|
|
19
|
+
},
|
|
20
|
+
"negative_evaluation": {
|
|
21
|
+
"ground_truth": "UNTRUTHFUL",
|
|
22
|
+
"confidence": 0.0,
|
|
23
|
+
"expected": "UNTRUTHFUL",
|
|
24
|
+
"correct": true,
|
|
25
|
+
"details": "Predicted: 'top up by cash or cheque' (log_prob=-0.500), Expected: 'activate my card'"
|
|
26
|
+
},
|
|
27
|
+
"both_correct": true
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"pair_id": 0,
|
|
4
|
+
"prompt": "Classify the Intent of the following Utterance to one of these options: activate my card, age limit, apple pay or google pay, atm support, automatic top up, balance not updated after bank transfer, balance not updated after cheque or cash deposit, beneficiary not allowed, cancel transfer, card about to expire, card acceptance, card arrival, card delivery estimate, card linking, card not working, card payment fee charged, card payment not recognised, card payment wrong exchange rate, card swallowed, cash withdrawal charge, cash withdrawal not recognised, change pin, compromised card, contactless not working, country support, declined card payment, declined cash withdrawal, declined transfer, direct debit payment not recognised, disposable card limits, edit personal details, exchange charge, exchange rate, exchange via app, extra charge on statement, failed transfer, fiat currency support, get disposable virtual card, get physical card, getting spare card, getting virtual card, lost or stolen card, lost or stolen phone, order physical card, passcode forgotten, pending card payment, pending cash withdrawal, pending top up, pending transfer, pin blocked, receiving money, Refund not showing up, request refund, reverted card payment?, supported cards and currencies, terminate account, top up by bank transfer charge, top up by card charge, top up by cash or cheque, top up failed, top up limits, top up reverted, topping up by card, transaction charged twice, transfer fee charged, transfer into account, transfer not received by recipient, transfer timing, unable to verify identity, verify my identity, verify source of funds, verify top up, virtual card not working, visa or mastercard, why verify identity, wrong amount of cash received, wrong exchange rate for cash withdrawal.\nUtterance:\ncan i deposit a check here?\nIntent:",
|
|
5
|
+
"positive_response": "top up by cash or cheque",
|
|
6
|
+
"negative_response": "activate my card"
|
|
7
|
+
}
|
|
8
|
+
]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"pair_id": 0,
|
|
4
|
+
"prompt": "Question: Hegazkinak kontrola galdu du. Ai, ene, .....\nA. gurea egin digu!\nB. gureak egin du!",
|
|
5
|
+
"positive_response": "gureak egin du!",
|
|
6
|
+
"negative_response": "gurea egin digu!"
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"pair_id": 1,
|
|
10
|
+
"prompt": "Question: Larrialdietako Harrerak ingresuen harreraren bulego delegatu gisafuntzionatzen du:\nA. Ezingo du jardun bulego delegatu gisa, jaiegunetan izan ezik.\nB. Egunez.",
|
|
11
|
+
"positive_response": "Egunez.",
|
|
12
|
+
"negative_response": "Ezingo du jardun bulego delegatu gisa, jaiegunetan izan ezik."
|
|
13
|
+
}
|
|
14
|
+
]
|