wisent 0.7.701__py3-none-any.whl → 0.7.1045__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wisent/__init__.py +1 -1
- wisent/comparison/__init__.py +1 -0
- wisent/comparison/detect_bos_features.py +275 -0
- wisent/comparison/fgaa.py +465 -0
- wisent/comparison/lora.py +669 -0
- wisent/comparison/lora_dpo.py +592 -0
- wisent/comparison/main.py +444 -0
- wisent/comparison/ours.py +76 -0
- wisent/comparison/sae.py +304 -0
- wisent/comparison/utils.py +381 -0
- wisent/core/activations/activation_cache.py +393 -0
- wisent/core/activations/activations.py +3 -3
- wisent/core/activations/activations_collector.py +12 -7
- wisent/core/activations/classifier_inference_strategy.py +12 -11
- wisent/core/activations/extraction_strategy.py +260 -84
- wisent/core/classifiers/classifiers/core/atoms.py +3 -2
- wisent/core/cli/__init__.py +2 -1
- wisent/core/cli/agent/train_classifier.py +16 -3
- wisent/core/cli/check_linearity.py +35 -3
- wisent/core/cli/cluster_benchmarks.py +4 -6
- wisent/core/cli/create_steering_vector.py +6 -4
- wisent/core/cli/diagnose_vectors.py +7 -4
- wisent/core/cli/estimate_unified_goodness_time.py +6 -4
- wisent/core/cli/generate_pairs_from_task.py +9 -56
- wisent/core/cli/generate_vector_from_task.py +11 -20
- wisent/core/cli/geometry_search.py +137 -0
- wisent/core/cli/get_activations.py +2 -2
- wisent/core/cli/method_optimizer.py +4 -3
- wisent/core/cli/modify_weights.py +3 -2
- wisent/core/cli/optimize_sample_size.py +1 -1
- wisent/core/cli/optimize_steering.py +14 -16
- wisent/core/cli/optimize_weights.py +2 -1
- wisent/core/cli/preview_pairs.py +203 -0
- wisent/core/cli/steering_method_trainer.py +3 -3
- wisent/core/cli/tasks.py +19 -76
- wisent/core/cli/train_unified_goodness.py +3 -3
- wisent/core/contrastive_pairs/diagnostics/control_vectors.py +4 -4
- wisent/core/contrastive_pairs/diagnostics/linearity.py +7 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentic_search.py +37 -347
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aider_polyglot.py +113 -136
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codeforces.py +2 -12
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/coding_benchmarks.py +124 -504
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/faithbench.py +40 -63
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flames.py +46 -89
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flores.py +15 -4
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/frames.py +36 -20
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hallucinations_leaderboard.py +3 -45
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livemathbench.py +42 -4
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/longform_writing.py +2 -112
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/math500.py +39 -4
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/medium_priority_benchmarks.py +475 -525
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mercury.py +65 -42
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/olympiadbench.py +2 -12
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/planbench.py +78 -219
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/polymath.py +37 -4
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/recode.py +84 -69
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/refusalbench.py +168 -160
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/simpleqa.py +44 -25
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tau_bench.py +3 -103
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/toolbench.py +3 -97
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/toolemu.py +48 -182
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_extractor_manifest.py +3 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_extractor_registry.py +19 -1
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aclue.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/acp_bench.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/acp_bench_hard.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/advanced.py +2 -4
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aexams.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrimmlu.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrixnli.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabculture.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_exams.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_leaderboard_complete.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_leaderboard_light.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabicmmlu.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aradice.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_challenge.py +1 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_easy.py +1 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arithmetic.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/asdiv.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/babi.py +36 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bbq.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/belebele.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/benchmarks.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bertaqa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bhs.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bhtc.py +3 -5
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/blimp.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/blimp_nl.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/boolq.py +22 -5
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/c4.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cabbq.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/careqa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalanqa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catcola.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cb.py +10 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ceval.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ceval_valid.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/chain.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/chartqa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/claim.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/click.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cmmlu.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cnn.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cocoteros.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coedit.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/commonsense.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/commonsense_qa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/copa.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/copal_id.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coqa.py +3 -4
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/csatqa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cycle.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darija_bench.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darijahellaswag.py +2 -6
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darijammlu.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/dbpedia.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/discrim_eval.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/doc.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/drop.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/epec.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench_ca.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench_es.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/esbbq.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ethics.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_exams.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_proficiency.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_reading.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_trivia.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/evalita_llm.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/financial.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/flan.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gaokao.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/glianorex.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/global_mmlu.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/global_piqa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gpt3.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/groundcocoa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/haerae.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/headqa.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hellaswag.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hendrycks_ethics.py +5 -9
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hendrycks_math.py +63 -16
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/histoires_morales.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hrm8k.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/humaneval_infilling.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/icelandic_winogrande.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/inverse.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/inverse_scaling.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ja.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard_mc.py +1 -1
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kobest.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kormedmcqa.py +5 -17
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_cloze.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_multilingual.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/law.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/leaderboard.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lingoly.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/llama3.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lm_syneval.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logiqa.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logiqa2.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/longbench.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/longbenchv2.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mastermind.py +2 -4
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mc-taco.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/med_concepts_qa.py +2 -4
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/meddialog.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medical.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medmcqa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medqa.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mela.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/metabench.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/minerva_math.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlusr.py +3 -4
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mrpc.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multiblimp.py +2 -5
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multirc.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mutual.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/non.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_exact.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_gen_exact.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc.py +4 -8
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc_log_likelihoods.py +4 -8
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/nq_open.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_arc_multilingual.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_hellaswag_multilingual.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_mmlu_multilingual.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_truthfulqa_multilingual.py +2 -5
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/openbookqa.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/option.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafraseja.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafrases.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws_x.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pawsx.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/persona.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/piqa.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prompt.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prost.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pubmedqa.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qa4mre.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper_bool.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnli.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnlieu.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qqp.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/race.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/random.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/record.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/reversed.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/rte.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ruler.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sciq.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/score.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls_mc.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/self.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue_rte.py +2 -1
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/siqa.py +4 -7
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/social_iqa.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/storycloze.py +2 -6
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/summarization.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/super.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/super_glue.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/swag.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/swde.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sycophancy.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/t0.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/teca.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinyarc.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinybenchmarks.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinygsm8k.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinyhellaswag.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinymmlu.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinytruthfulqa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinywinogrande.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tmmluplus.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/triviaqa.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc1.py +9 -4
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc2.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turblimp_core.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu_mc.py +0 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/unscramble.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/vaxx.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/webqs.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wic.py +3 -4
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/winogrande.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmdp.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wnli.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wsc.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wsc273.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xcopa.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xlsum.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xnli.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xquad.py +2 -4
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xstorycloze.py +2 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xwinograd.py +2 -2
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/zhoblimp.py +1 -3
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_pairs_generation.py +173 -6
- wisent/core/data_loaders/loaders/lm_loader.py +12 -1
- wisent/core/geometry_runner.py +995 -0
- wisent/core/geometry_search_space.py +237 -0
- wisent/core/hyperparameter_optimizer.py +1 -1
- wisent/core/main.py +3 -0
- wisent/core/models/core/atoms.py +5 -3
- wisent/core/models/wisent_model.py +1 -1
- wisent/core/optuna/classifier/optuna_classifier_optimizer.py +2 -2
- wisent/core/parser_arguments/check_linearity_parser.py +12 -2
- wisent/core/parser_arguments/generate_vector_from_synthetic_parser.py +2 -2
- wisent/core/parser_arguments/generate_vector_from_task_parser.py +6 -13
- wisent/core/parser_arguments/geometry_search_parser.py +61 -0
- wisent/core/parser_arguments/get_activations_parser.py +5 -14
- wisent/core/parser_arguments/main_parser.py +8 -0
- wisent/core/parser_arguments/train_unified_goodness_parser.py +2 -2
- wisent/core/steering.py +5 -3
- wisent/core/steering_methods/methods/hyperplane.py +2 -1
- wisent/core/synthetic/generators/nonsense_generator.py +30 -18
- wisent/core/trainers/steering_trainer.py +2 -2
- wisent/core/utils/device.py +27 -27
- wisent/core/utils/layer_combinations.py +70 -0
- wisent/examples/__init__.py +1 -0
- wisent/examples/scripts/__init__.py +1 -0
- wisent/examples/scripts/count_all_benchmarks.py +121 -0
- wisent/examples/scripts/discover_directions.py +469 -0
- wisent/examples/scripts/extract_benchmark_info.py +71 -0
- wisent/examples/scripts/search_all_short_names.py +31 -0
- wisent/examples/scripts/test_all_benchmarks.py +138 -0
- wisent/examples/scripts/test_all_benchmarks_new.py +28 -0
- wisent/examples/scripts/test_contrastive_pairs_all_supported.py +230 -0
- wisent/examples/scripts/test_nonsense_baseline.py +261 -0
- wisent/examples/scripts/test_one_benchmark.py +324 -0
- wisent/examples/scripts/test_one_coding_benchmark.py +293 -0
- wisent/parameters/lm_eval/broken_in_lm_eval.json +179 -2
- wisent/parameters/lm_eval/category_directions.json +137 -0
- wisent/parameters/lm_eval/repair_plan.json +282 -0
- wisent/parameters/lm_eval/weak_contrastive_pairs.json +38 -0
- wisent/parameters/lm_eval/working_benchmarks.json +206 -0
- wisent/parameters/lm_eval/working_benchmarks_categorized.json +236 -0
- wisent/tests/test_detector_accuracy.py +1 -1
- wisent/tests/visualize_geometry.py +1 -1
- {wisent-0.7.701.dist-info → wisent-0.7.1045.dist-info}/METADATA +5 -1
- {wisent-0.7.701.dist-info → wisent-0.7.1045.dist-info}/RECORD +328 -358
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/browsecomp.py +0 -245
- wisent/examples/contrastive_pairs/humanization_human_vs_ai.json +0 -2112
- wisent/examples/scripts/1/test_basqueglue_evaluation.json +0 -51
- wisent/examples/scripts/1/test_basqueglue_pairs.json +0 -14
- wisent/examples/scripts/1/test_bec2016eu_evaluation.json +0 -51
- wisent/examples/scripts/1/test_bec2016eu_pairs.json +0 -14
- wisent/examples/scripts/1/test_belebele_evaluation.json +0 -51
- wisent/examples/scripts/1/test_belebele_pairs.json +0 -14
- wisent/examples/scripts/1/test_benchmarks_evaluation.json +0 -51
- wisent/examples/scripts/1/test_benchmarks_pairs.json +0 -14
- wisent/examples/scripts/1/test_bertaqa_evaluation.json +0 -51
- wisent/examples/scripts/1/test_bertaqa_pairs.json +0 -14
- wisent/examples/scripts/1/test_bhtc_v2_evaluation.json +0 -30
- wisent/examples/scripts/1/test_bhtc_v2_pairs.json +0 -8
- wisent/examples/scripts/1/test_boolq-seq2seq_evaluation.json +0 -30
- wisent/examples/scripts/1/test_boolq-seq2seq_pairs.json +0 -8
- wisent/examples/scripts/1/test_cabreu_evaluation.json +0 -30
- wisent/examples/scripts/1/test_cabreu_pairs.json +0 -8
- wisent/examples/scripts/1/test_careqa_en_evaluation.json +0 -30
- wisent/examples/scripts/1/test_careqa_en_pairs.json +0 -8
- wisent/examples/scripts/1/test_careqa_evaluation.json +0 -30
- wisent/examples/scripts/1/test_careqa_pairs.json +0 -8
- wisent/examples/scripts/1/test_catalanqa_evaluation.json +0 -30
- wisent/examples/scripts/1/test_catalanqa_pairs.json +0 -8
- wisent/examples/scripts/1/test_catcola_evaluation.json +0 -30
- wisent/examples/scripts/1/test_catcola_pairs.json +0 -8
- wisent/examples/scripts/1/test_chartqa_evaluation.json +0 -30
- wisent/examples/scripts/1/test_chartqa_pairs.json +0 -8
- wisent/examples/scripts/1/test_claim_stance_topic_evaluation.json +0 -30
- wisent/examples/scripts/1/test_claim_stance_topic_pairs.json +0 -8
- wisent/examples/scripts/1/test_cnn_dailymail_evaluation.json +0 -30
- wisent/examples/scripts/1/test_cnn_dailymail_pairs.json +0 -8
- wisent/examples/scripts/1/test_cocoteros_es_evaluation.json +0 -30
- wisent/examples/scripts/1/test_cocoteros_es_pairs.json +0 -8
- wisent/examples/scripts/1/test_coedit_gec_evaluation.json +0 -30
- wisent/examples/scripts/1/test_coedit_gec_pairs.json +0 -8
- wisent/examples/scripts/1/test_cola_evaluation.json +0 -30
- wisent/examples/scripts/1/test_cola_pairs.json +0 -8
- wisent/examples/scripts/1/test_coqcat_evaluation.json +0 -30
- wisent/examples/scripts/1/test_coqcat_pairs.json +0 -8
- wisent/examples/scripts/1/test_dbpedia_14_evaluation.json +0 -30
- wisent/examples/scripts/1/test_dbpedia_14_pairs.json +0 -8
- wisent/examples/scripts/1/test_epec_koref_bin_evaluation.json +0 -30
- wisent/examples/scripts/1/test_epec_koref_bin_pairs.json +0 -8
- wisent/examples/scripts/1/test_ethos_binary_evaluation.json +0 -30
- wisent/examples/scripts/1/test_ethos_binary_pairs.json +0 -8
- wisent/examples/scripts/2/test_afrimgsm_direct_amh_evaluation.json +0 -30
- wisent/examples/scripts/2/test_afrimgsm_direct_amh_pairs.json +0 -8
- wisent/examples/scripts/2/test_afrimmlu_direct_amh_evaluation.json +0 -30
- wisent/examples/scripts/2/test_afrimmlu_direct_amh_pairs.json +0 -8
- wisent/examples/scripts/2/test_afrixnli_en_direct_amh_evaluation.json +0 -30
- wisent/examples/scripts/2/test_afrixnli_en_direct_amh_pairs.json +0 -8
- wisent/examples/scripts/2/test_arc_ar_evaluation.json +0 -30
- wisent/examples/scripts/2/test_arc_ar_pairs.json +0 -8
- wisent/examples/scripts/2/test_atis_evaluation.json +0 -30
- wisent/examples/scripts/2/test_atis_pairs.json +0 -8
- wisent/examples/scripts/2/test_babi_evaluation.json +0 -30
- wisent/examples/scripts/2/test_babi_pairs.json +0 -8
- wisent/examples/scripts/2/test_babilong_evaluation.json +0 -30
- wisent/examples/scripts/2/test_babilong_pairs.json +0 -8
- wisent/examples/scripts/2/test_bangla_mmlu_evaluation.json +0 -30
- wisent/examples/scripts/2/test_bangla_mmlu_pairs.json +0 -8
- wisent/examples/scripts/2/test_basque-glue_pairs.json +0 -14
- {wisent-0.7.701.dist-info → wisent-0.7.1045.dist-info}/WHEEL +0 -0
- {wisent-0.7.701.dist-info → wisent-0.7.1045.dist-info}/entry_points.txt +0 -0
- {wisent-0.7.701.dist-info → wisent-0.7.1045.dist-info}/licenses/LICENSE +0 -0
- {wisent-0.7.701.dist-info → wisent-0.7.1045.dist-info}/top_level.txt +0 -0
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"task_name": "atis",
|
|
3
|
-
"model_name": "mock",
|
|
4
|
-
"evaluator_name": "log_likelihoods",
|
|
5
|
-
"num_pairs": 1,
|
|
6
|
-
"all_correct": true,
|
|
7
|
-
"pairs": [
|
|
8
|
-
{
|
|
9
|
-
"pair_id": 0,
|
|
10
|
-
"prompt": "From the following Text, extract the objects for which the entity type expressed is one of aircraft_...",
|
|
11
|
-
"positive_response": "charlotte: fromloc.city_name, las vegas: toloc.city_name, st. louis: stoploc.city_name",
|
|
12
|
-
"negative_response": "st. louis: stoploc.city_name, las vegas: toloc.city_name, charlotte: fromloc.city_name",
|
|
13
|
-
"positive_evaluation": {
|
|
14
|
-
"ground_truth": "TRUTHFUL",
|
|
15
|
-
"confidence": 1.0,
|
|
16
|
-
"expected": "TRUTHFUL",
|
|
17
|
-
"correct": true,
|
|
18
|
-
"details": "Predicted: 'charlotte: fromloc.city_name, las vegas: toloc.city_name, st. louis: stoploc.city_name' (log_prob=-0.500), Expected: 'charlotte: fromloc.city_name, las vegas: toloc.city_name, st. louis: stoploc.city_name'"
|
|
19
|
-
},
|
|
20
|
-
"negative_evaluation": {
|
|
21
|
-
"ground_truth": "UNTRUTHFUL",
|
|
22
|
-
"confidence": 0.0,
|
|
23
|
-
"expected": "UNTRUTHFUL",
|
|
24
|
-
"correct": true,
|
|
25
|
-
"details": "Predicted: 'charlotte: fromloc.city_name, las vegas: toloc.city_name, st. louis: stoploc.city_name' (log_prob=-0.500), Expected: 'st. louis: stoploc.city_name, las vegas: toloc.city_name, charlotte: fromloc.city_name'"
|
|
26
|
-
},
|
|
27
|
-
"both_correct": true
|
|
28
|
-
}
|
|
29
|
-
]
|
|
30
|
-
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
[
|
|
2
|
-
{
|
|
3
|
-
"pair_id": 0,
|
|
4
|
-
"prompt": "From the following Text, extract the objects for which the entity type expressed is one of aircraft_code, airline_code, airline_name, airport_code, airport_name, arrive_date.date_relative, arrive_date.day_name, arrive_date.day_number, arrive_date.month_name, arrive_date.today_relative, arrive_time.end_time, arrive_time.period_mod, arrive_time.period_of_day, arrive_time.start_time, arrive_time.time, arrive_time.time_relative, city_name, class_type, connect, cost_relative, day_name, day_number, days_code, depart_date.date_relative, depart_date.day_name, depart_date.day_number, depart_date.month_name, depart_date.today_relative, depart_date.year, depart_time.end_time, depart_time.period_mod, depart_time.period_of_day, depart_time.start_time, depart_time.time, depart_time.time_relative, economy, fare_amount, fare_basis_code, flight_days, flight_mod, flight_number, flight_stop, flight_time, fromloc.airport_code, fromloc.airport_name, fromloc.city_name, fromloc.state_code, fromloc.state_name, meal, meal_code, meal_description, mod, month_name, or, period_of_day, restriction_code, return_date.date_relative, return_date.day_name, return_date.day_number, return_date.month_name, return_date.today_relative, return_time.period_mod, return_time.period_of_day, round_trip, state_code, state_name, stoploc.airport_name, stoploc.city_name, stoploc.state_code, time, time_relative, today_relative, toloc.airport_code, toloc.airport_name, toloc.city_name, toloc.country_name, toloc.state_code, toloc.state_name, transport_type.\nText:\ni would like to find a flight from charlotte to las vegas that makes a stop in st. louis\nentity type:",
|
|
5
|
-
"positive_response": "charlotte: fromloc.city_name, las vegas: toloc.city_name, st. louis: stoploc.city_name",
|
|
6
|
-
"negative_response": "st. louis: stoploc.city_name, las vegas: toloc.city_name, charlotte: fromloc.city_name"
|
|
7
|
-
}
|
|
8
|
-
]
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"task_name": "babi",
|
|
3
|
-
"model_name": "mock",
|
|
4
|
-
"evaluator_name": "log_likelihoods",
|
|
5
|
-
"num_pairs": 1,
|
|
6
|
-
"all_correct": true,
|
|
7
|
-
"pairs": [
|
|
8
|
-
{
|
|
9
|
-
"pair_id": 0,
|
|
10
|
-
"prompt": "Passage: John travelled to the hallway.\nMary journeyed to the bathroom.\n\nQuestion: Where is John?...",
|
|
11
|
-
"positive_response": "hallway",
|
|
12
|
-
"negative_response": "not hallway",
|
|
13
|
-
"positive_evaluation": {
|
|
14
|
-
"ground_truth": "TRUTHFUL",
|
|
15
|
-
"confidence": 1.0,
|
|
16
|
-
"expected": "TRUTHFUL",
|
|
17
|
-
"correct": true,
|
|
18
|
-
"details": "Predicted: 'hallway' (log_prob=-0.500), Expected: 'hallway'"
|
|
19
|
-
},
|
|
20
|
-
"negative_evaluation": {
|
|
21
|
-
"ground_truth": "UNTRUTHFUL",
|
|
22
|
-
"confidence": 0.0,
|
|
23
|
-
"expected": "UNTRUTHFUL",
|
|
24
|
-
"correct": true,
|
|
25
|
-
"details": "Predicted: 'hallway' (log_prob=-0.500), Expected: 'not hallway'"
|
|
26
|
-
},
|
|
27
|
-
"both_correct": true
|
|
28
|
-
}
|
|
29
|
-
]
|
|
30
|
-
}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"task_name": "babilong",
|
|
3
|
-
"model_name": "mock",
|
|
4
|
-
"evaluator_name": "log_likelihoods",
|
|
5
|
-
"num_pairs": 1,
|
|
6
|
-
"all_correct": true,
|
|
7
|
-
"pairs": [
|
|
8
|
-
{
|
|
9
|
-
"pair_id": 0,
|
|
10
|
-
"prompt": "Context: \u2013 \u0421\u043c\u043e\u0442\u0440\u0438, \u0414\u0436\u043e, \u2013 \u0441\u043a\u0430\u0437\u0430\u043b\u0430 \u0411\u043e\u043d\u0438\u0442\u0430, \u043f\u043e\u043a\u0430\u0437\u0430\u0432 \u043f\u0430\u043b\u044c\u0446\u0435\u043c. \u041a \u043a\u043e\u043d\u0432\u043e\u044e \u043f\u043e\u0434\u044a\u0435\u0445\u0430\u043b \u0433\u0440\u044f\u0437\u043d\u043e-\u0437\u0435\u043b\u0435\u043d\u044b\u0439 \u043f\u0438\u043a\u0430\u043f \"\u0448...",
|
|
11
|
-
"positive_response": "\u0441\u043f\u0430\u043b\u044c\u043d\u044f",
|
|
12
|
-
"negative_response": "\u0432\u0430\u043d\u043d\u0430\u044f",
|
|
13
|
-
"positive_evaluation": {
|
|
14
|
-
"ground_truth": "TRUTHFUL",
|
|
15
|
-
"confidence": 1.0,
|
|
16
|
-
"expected": "TRUTHFUL",
|
|
17
|
-
"correct": true,
|
|
18
|
-
"details": "Predicted: '\u0441\u043f\u0430\u043b\u044c\u043d\u044f' (log_prob=-0.500), Expected: '\u0441\u043f\u0430\u043b\u044c\u043d\u044f'"
|
|
19
|
-
},
|
|
20
|
-
"negative_evaluation": {
|
|
21
|
-
"ground_truth": "UNTRUTHFUL",
|
|
22
|
-
"confidence": 0.0,
|
|
23
|
-
"expected": "UNTRUTHFUL",
|
|
24
|
-
"correct": true,
|
|
25
|
-
"details": "Predicted: '\u0441\u043f\u0430\u043b\u044c\u043d\u044f' (log_prob=-0.500), Expected: '\u0432\u0430\u043d\u043d\u0430\u044f'"
|
|
26
|
-
},
|
|
27
|
-
"both_correct": true
|
|
28
|
-
}
|
|
29
|
-
]
|
|
30
|
-
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
[
|
|
2
|
-
{
|
|
3
|
-
"pair_id": 0,
|
|
4
|
-
"prompt": "Context: \u2013 \u0421\u043c\u043e\u0442\u0440\u0438, \u0414\u0436\u043e, \u2013 \u0441\u043a\u0430\u0437\u0430\u043b\u0430 \u0411\u043e\u043d\u0438\u0442\u0430, \u043f\u043e\u043a\u0430\u0437\u0430\u0432 \u043f\u0430\u043b\u044c\u0446\u0435\u043c. \u041a \u043a\u043e\u043d\u0432\u043e\u044e \u043f\u043e\u0434\u044a\u0435\u0445\u0430\u043b \u0433\u0440\u044f\u0437\u043d\u043e-\u0437\u0435\u043b\u0435\u043d\u044b\u0439 \u043f\u0438\u043a\u0430\u043f \"\u0448\u0435\u0432\u0440\u043e\u043b\u0435\" \u043d\u0430 \u043c\u043e\u0449\u043d\u044b\u0445 \u043a\u043e\u043b\u0435\u0441\u0430\u0445, \u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u0441\u043a\u0430\u044f \u0434\u0432\u0435\u0440\u044c \u0440\u0430\u0441\u043f\u0430\u0445\u043d\u0443\u043b\u0430\u0441\u044c \u0438 \u043e\u0442\u0442\u0443\u0434\u0430 \u0432\u044b\u0431\u0440\u0430\u043b\u0441\u044f \u0414\u0436\u043e \u2013 \u043d\u0430\u0448 \u0434\u0440\u0443\u0433, \u043f\u0430\u0440\u0442\u043d\u0435\u0440 \u043f\u043e \u0431\u0438\u0437\u043d\u0435\u0441\u0443, \u0434\u0430 \u0438 \u043f\u0440\u043e\u0441\u0442\u043e \u043a\u043e\u043b\u043b\u0435\u0433\u0430. \u0420\u043e\u0441\u043b\u044b\u0439, \u043a\u043e\u0440\u043e\u0442\u043a\u043e \u0441\u0442\u0440\u0438\u0436\u0435\u043d\u044b\u0439, \u0432 \u0441\u0432\u0435\u0442\u043b\u044b\u0445 \u0431\u0440\u044e\u043a\u0430\u0445 \u0445\u0430\u043a\u0438 \u0438 \u0447\u0435\u0440\u043d\u043e\u0439 \u0444\u0443\u0442\u0431\u043e\u043b\u043a\u0435. \u041a \u043d\u0435\u043c\u0443 \u043f\u043e\u0434\u043e\u0448\u0435\u043b \u043a\u0442\u043e-\u0442\u043e \u0438\u0437 \u043f\u0440\u0438\u0435\u0445\u0430\u0432\u0448\u0438\u0445 \u0441 \u043a\u043e\u043d\u0432\u043e\u0435\u043c, \u043e\u043d\u0438 \u0437\u0430\u0433\u043e\u0432\u043e\u0440\u0438\u043b\u0438, \u043d\u0435 \u0437\u0430\u0431\u044b\u0432 \u043f\u043e\u0445\u043b\u043e\u043f\u0430\u0442\u044c \u0434\u0440\u0443\u0433 \u0434\u0440\u0443\u0433\u0430 \u043f\u043e \u0441\u043f\u0438\u043d\u0430\u043c. \u2013 \u041e\u043d \u043a\u0430\u043a\u0438\u0435-\u0442\u043e \u0437\u0430\u043f\u0447\u0430\u0441\u0442\u0438 \u0436\u0434\u0430\u043b \u0442\u0430\u043a, \u0447\u0442\u043e \u0441\u043f\u0430\u0442\u044c \u043d\u0435 \u043c\u043e\u0433, \u043a\u0430\u0436\u0435\u0442\u0441\u044f, \u2013 \u043e\u0442\u043e\u0437\u0432\u0430\u043b\u0441\u044f \u044f. \u0421\u043e\u0431\u0435\u0441\u0435\u0434\u043d\u0438\u043a \u0414\u0436\u043e \u043e\u0442\u043a\u0438\u043d\u0443\u043b \u043f\u043e\u043b\u043e\u0433 \u0442\u0435\u043d\u0442\u0430 \u043d\u0430 \u0433\u0440\u0443\u0437\u043e\u0432\u0438\u043a\u0435, \u0437\u0430\u043b\u0435\u0437 \u0432\u043d\u0443\u0442\u0440\u044c \u0438 \u043d\u0430\u0447\u0430\u043b \u043f\u043e\u0434\u0430\u0432\u0430\u0442\u044c \u043e\u0442\u0442\u0443\u0434\u0430 \u043a\u0430\u0440\u0442\u043e\u043d\u043d\u044b\u0435 \u043a\u043e\u0440\u043e\u0431\u043a\u0438, \u0443\u0432\u0435\u0441\u0438\u0441\u0442\u044b\u0435 \u0441 \u0432\u0438\u0434\u0443, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u0414\u0436\u043e \u0443\u043a\u043b\u0430\u0434\u044b\u0432\u0430\u043b \u0432 \u043f\u0438\u043a\u0430\u043f. \u2013 \u0422\u0435\u043f\u0435\u0440\u044c \u0443\u0441\u043d\u0435\u0442 \u0441\u043f\u043e\u043a\u043e\u0439\u043d\u043e? \u2013 \u041d\u0430\u0432\u0435\u0440\u043d\u043e\u0435. \u042f \u043e\u0431\u043d\u044f\u043b \u0435\u0435 \u0437\u0430 \u0442\u0430\u043b\u0438\u044e \u0438 \u043f\u0440\u0438\u0442\u044f\u043d\u0443\u043b \u043a \u0441\u0435\u0431\u0435 \u0431\u043b\u0438\u0436\u0435. \u0417\u0430 \u0441\u043f\u0438\u043d\u043e\u0439 \u043e\u0442\u043a\u0440\u044b\u043b\u0430\u0441\u044c \u0434\u0432\u0435\u0440\u044c \u0438 \u0433\u043e\u043b\u043e\u0441 \u043c\u0435\u0434\u0441\u0435\u0441\u0442\u0440\u044b \u0434\u043e\u043a\u0442\u043e\u0440\u0430 \u041c\u0430\u0440\u0442\u0438\u043d\u0435\u0441\u0430, \u0432\u044b\u0441\u043e\u043a\u043e\u0439 \u0438 \u0442\u043e\u0449\u0435\u0439 \u0431\u043b\u043e\u043d\u0434\u0438\u043d\u043a\u0438 \u0425\u044d\u0439\u0437\u0435\u043b, \u043f\u0440\u0438\u0433\u043b\u0430\u0441\u0438\u043b \u0411\u043e\u043d\u0438\u0442\u0443 \u0432\u043e\u0439\u0442\u0438. \u0410\u043b\u0435\u043a\u0441\u0430\u043d\u0434\u0440\u0430 \u043f\u0435\u0440\u0435\u043c\u0435\u0449\u0430\u0435\u0442\u0441\u044f \u0432 \u0441\u043f\u0430\u043b\u044c\u043d\u044e. \u0410 \u044f \u043e\u0441\u0442\u0430\u043b\u0441\u044f \u0436\u0434\u0430\u0442\u044c \u0432 \u043f\u0440\u0438\u0435\u043c\u043d\u043e\u0439, \u0443\u0441\u0435\u0432\u0448\u0438\u0441\u044c \u043d\u0430 \u0434\u0438\u0432\u0430\u043d \u0438 \u0434\u043e\u0441\u0442\u0430\u0432 \u0438\u0437 \u0441\u0443\u043c\u043a\u0438 \u043a\u043d\u0438\u0433\u0443. \u0414\u0443\u043c\u0430\u044e, \u0447\u0442\u043e \u043e\u0436\u0438\u0434\u0430\u043d\u0438\u0435 \u043d\u0435 \u043f\u044f\u0442\u044c \u043c\u0438\u043d\u0443\u0442 \u0437\u0430\u0439\u043c\u0435\u0442. \u041c\u043e\u0436\u043d\u043e \u0431\u044b \u043f\u043e\u0439\u0442\u0438 \u043f\u0440\u043e\u0433\u0443\u043b\u044f\u0442\u044c\u0441\u044f, \u0441 \u0442\u0435\u043c \u0436\u0435 \u0414\u0436\u043e \u043f\u043e\u043e\u0431\u0449\u0430\u0442\u044c\u0441\u044f, \u043d\u043e \u0432\u0434\u0440\u0443\u0433 \u043f\u043e\u043d\u0430\u0434\u043e\u0431\u043b\u044e\u0441\u044c \u0437\u0430\u0447\u0435\u043c-\u043d\u0438\u0431\u0443\u0434\u044c? \u0414\u0432\u0435\u0440\u044c \u0431\u044b\u043b\u0430 \u0442\u043e\u043b\u0441\u0442\u0430\u044f, \u0438\u0437-\u0437\u0430 \u043d\u0435\u0435 \u043d\u0438 \u0437\u0432\u0443\u043a\u0430 \u043d\u0435 \u0434\u043e\u043d\u043e\u0441\u0438\u043b\u043e\u0441\u044c. \u0427\u0438\u0442\u0430\u043b\u043e\u0441\u044c \u0438\u0437-\u0437\u0430 \u0432\u043e\u043b\u043d\u0435\u043d\u0438\u044f \u043f\u043b\u043e\u0445\u043e, \u0432\u0441\u0435 \u0432\u0440\u0435\u043c\u044f \u043b\u043e\u0432\u0438\u043b \u0441\u0435\u0431\u044f \u043d\u0430 \u0442\u043e\u043c, \u0447\u0442\u043e \u043a\u0430\u0436\u0434\u044b\u0439 \u0430\u0431\u0437\u0430\u0446 \u0447\u0438\u0442\u0430\u044e \u043f\u043e \u0434\u0432\u0430-\u0442\u0440\u0438 \u0440\u0430\u0437\u0430, \u0438 \u0432\u0441\u0435 \u0440\u0430\u0432\u043d\u043e \u043d\u0435 \u0437\u0430\u043f\u043e\u043c\u0438\u043d\u0430\u044e, \u043e \u0447\u0435\u043c \u0442\u0430\u043c \u0440\u0435\u0447\u044c. \u0417\u0430\u043a\u0440\u044b\u0432 \u043a\u043d\u0438\u0433\u0443, \u0432\u0441\u0442\u0430\u043b \u0443 \u043e\u043a\u043d\u0430, \u0440\u0430\u0441\u0441\u043c\u0430\u0442\u0440\u0438\u0432\u0430\u044f \u0441\u0443\u0435\u0442\u0443 \u0443 \u043a\u043e\u043d\u0432\u043e\u044f. \u0414\u0436\u043e \u0443\u0436\u0435 \u043d\u0435 \u0431\u044b\u043b\u043e, \u043d\u043e \u0431\u044b\u043b\u043e \u043c\u043d\u043e\u0433\u043e \u0434\u0440\u0443\u0433\u0438\u0445 \u0437\u043d\u0430\u043a\u043e\u043c\u044b\u0445 \u043b\u0438\u0446, \u0438 \u044f \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u043c\u0438\u043d\u0443\u0442 \u0438\u0433\u0440\u0430\u043b \u0441\u0430\u043c \u0441 \u0441\u043e\u0431\u043e\u0439 \u0432 \u0438\u0433\u0440\u0443, \u043f\u044b\u0442\u0430\u044f\u0441\u044c \u0443\u0433\u0430\u0434\u0430\u0442\u044c, \u0443 \u043a\u043e\u0433\u043e \u0442\u0430\u043c \u0441\u0435\u0439\u0447\u0430\u0441 \u043a\u0430\u043a\u0438\u0435 \u0434\u0435\u043b\u0430. \u041f\u043e\u0441\u0442\u0435\u043f\u0435\u043d\u043d\u043e \u0442\u043e\u043b\u043f\u0430 \u0440\u0430\u0441\u0441\u0430\u0441\u044b\u0432\u0430\u043b\u0430\u0441\u044c, \u0433\u0440\u0443\u0437\u043e\u0432\u0438\u043a\u0438 \u0440\u0430\u0437\u044a\u0435\u0445\u0430\u043b\u0438\u0441\u044c \u043a\u043e\u043c\u0443 \u043a\u0443\u0434\u0430 \u043d\u0443\u0436\u043d\u043e, \u043e\u0441\u0442\u0430\u043b\u043e\u0441\u044c \u0432\u0441\u0435\u0433\u043e \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u0447\u0435\u043b\u043e\u0432\u0435\u043a, \u0432 \u043f\u0435\u0448\u0435\u043c \u043f\u043e\u0440\u044f\u0434\u043a\u0435 \u043d\u0430\u043f\u0440\u0430\u0432\u0438\u0432\u0448\u0438\u0445\u0441\u044f \u0432 \"\u0414\u0436\u0438\u043c\u043c\u0438'\u0437\" \u2013 \u0431\u043b\u0438\u0436\u0430\u0439\u0448\u0438\u0439 \u0431\u0430\u0440. \u041e\u0431\u043c\u044b\u0442\u044c, \u0432\u0438\u0434\u0430\u0442\u044c, \"\u043f\u0440\u0438\u0445\u043e\u0434 \u0432\u0435\u0441\u043d\u044b\". \u0414\u0430, \u0442\u0435\u043f\u0435\u0440\u044c \u0436\u0438\u0437\u043d\u044c \u0441\u043d\u043e\u0432\u0430 \u043e\u0436\u0438\u0432\u0438\u0442\u0441\u044f, \u0430 \u0442\u043e \u043d\u0430 \u0432\u0440\u0435\u043c\u044f \u0441\u0435\u0437\u043e\u043d\u0430 \u0434\u043e\u0436\u0434\u0435\u0439 \u0410\u043b\u0430\u043c\u043e \u043f\u043e\u0433\u0440\u0443\u0437\u0438\u043b\u0441\u044f \u0432 \u0441\u043f\u044f\u0447\u043a\u0443. \u041d\u0435 \u0442\u0430\u043a \u0438 \u043f\u043b\u043e\u0445\u043e \u0431\u044b\u043b\u043e, \u0435\u0441\u043b\u0438 \u0447\u0435\u0441\u0442\u043d\u043e, \u044f \u043f\u0440\u0435\u043a\u0440\u0430\u0441\u043d\u043e \u043e\u0442\u0434\u043e\u0445\u043d\u0443\u043b \u043f\u043e\u0441\u043b\u0435 \u0432\u0441\u0435\u0445 \u0441\u043e\u0431\u044b\u0442\u0438\u0439 \u043f\u0440\u043e\u0448\u043b\u043e\u0433\u043e \u0436\u0430\u0440\u043a\u043e\u0433\u043e \u0441\u0435\u0437\u043e\u043d\u0430, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u0434\u043b\u044f \u043c\u0435\u043d\u044f \u043e\u043a\u0430\u0437\u0430\u043b\u0441\u044f \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0442\u0435\u043b\u044c\u043d\u043e \u0436\u0430\u0440\u043a\u0438\u043c, \u0442\u0430\u043a\u0438\u043c, \u0447\u0442\u043e \u0438 \u0441\u043f\u0435\u0447\u044c\u0441\u044f \u043d\u0435\u0434\u043e\u043b\u0433\u043e. \u041d\u043e \u0432\u0441\u0435 \u0437\u0430\u0432\u0435\u0440\u0448\u0438\u043b\u043e\u0441\u044c \u0431\u043b\u0430\u0433\u043e\u043f\u043e\u043b\u0443\u0447\u043d\u043e, \u0430 \u0432\u044b\u043d\u0443\u0436\u0434\u0435\u043d\u043d\u043e\u0435 \u0434\u043e\u043c\u043e\u0441\u0435\u0434\u0441\u0442\u0432\u043e \u043b\u0438\u0448\u044c \u043f\u0440\u0438\u0432\u0435\u043b\u043e \u043a \u0442\u043e\u043c\u0443, \u0447\u0442\u043e \u0441\u0435\u0439\u0447\u0430\u0441 \u0434\u0443\u0448\u0430 \u0436\u0430\u0436\u0434\u0430\u043b\u0430 \u0434\u0435\u044f\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438. \u0414\u0430 \u0438 \u043f\u043b\u0430\u043d\u043e\u0432 \u043d\u0430 \u044d\u0442\u043e\u0442 \u0441\u0435\u0437\u043e\u043d \u0445\u0432\u0430\u0442\u0430\u043b\u043e, \u0434\u0430\u0436\u0435 \u0432 \u0438\u0445 \u0441\u043f\u0438\u0441\u043e\u043a \u0441\u0442\u0440\u0430\u0448\u043d\u043e \u0437\u0430\u0433\u043b\u044f\u0434\u044b\u0432\u0430\u0442\u044c. \u0414\u043b\u044f \u043d\u0430\u0447\u0430\u043b\u0430 \u043c\u043d\u0435 \u043d\u0430\u0434\u043e \u0432 \u041f\u041f\u0414 \u0441\u0433\u043e\u043d\u044f\u0442\u044c, \u043a\u0430\u043a \u0440\u0430\u0437 \u0441 \u043f\u0435\u0440\u0432\u044b\u043c \u043f\u043e\u043f\u0443\u0442\u043d\u044b\u043c \u043a\u043e\u043d\u0432\u043e\u0435\u043c. \u041f\u043e\u0441\u043a\u043e\u043b\u044c\u043a\u0443 \u044f \"\u0441\u043b\u0443\u0433\u0430 \u0434\u0432\u0443\u0445 \u0433\u043e\u0441\u043f\u043e\u0434\", \u043c\u043d\u0435 \u043d\u0430\u0434\u043e \u0431\u0443\u0434\u0435\u0442 \u0435\u0449\u0435 \u044d\u0442\u0438 \u0441\u0430\u043c\u044b\u0435 \u043f\u043b\u0430\u043d\u044b \u043a\u0430\u043a \u0441 \u043e\u0434\u043d\u0438\u043c \u043d\u0430\u0447\u0430\u043b\u044c\u0441\u0442\u0432\u043e\u043c \u0443\u0442\u0440\u044f\u0441\u0442\u0438, \u0442\u0430\u043a \u0438 \u0441\u043e \u0432\u0442\u043e\u0440\u044b\u043c. \u0410 \u0443\u0447\u0438\u0442\u044b\u0432\u0430\u044f, \u0447\u0442\u043e \u043e\u0442\u043d\u043e\u0448\u0435\u043d\u0438\u044f \u043c\u0435\u0436\u0434\u0443 \u044d\u0442\u0438\u043c\u0438 \"\u0433\u043e\u0441\u043f\u043e\u0434\u0430\u043c\u0438\" \u0441\u043b\u043e\u0436\u043d\u044b\u0435, \u0443\u0442\u0440\u044f\u0441\u0430\u0442\u044c \u0442\u043e\u0436\u0435 \u0431\u0443\u0434\u0435\u0442 \u043d\u0435\u043b\u0435\u0433\u043a\u043e. \u0418 \u0433\u043b\u0430\u0432\u043d\u043e\u0435, \u0442\u043e\u0442 \u0444\u0430\u043a\u0442, \u0447\u0442\u043e \u044f \u0441\u0438\u0436\u0443 \u043c\u0435\u0436\u0434\u0443 \u0434\u0432\u0443\u0445 \u0441\u0442\u0443\u043b\u044c\u0435\u0432, \u043c\u043e\u0435 \u043d\u0430\u0447\u0430\u043b\u044c\u0441\u0442\u0432\u043e \u0441 \u043e\u0431\u0435\u0438\u0445 \u0441\u0442\u043e\u0440\u043e\u043d \u0443\u0434\u043e\u0432\u043b\u0435\u0442\u0432\u043e\u0440\u044f\u0435\u0442. \u041f\u043e\u0445\u043e\u0436\u0435, \u0447\u0442\u043e \u0438\u043c \u043a\u0430\u0436\u0435\u0442\u0441\u044f, \u0447\u0442\u043e \u0442\u0430\u043a\u043e\u0435 \u043f\u043e\u043b\u043e\u0436\u0435\u043d\u0438\u0435 \u0434\u0435\u043b \u043f\u043e\u0437\u0432\u043e\u043b\u0438\u0442 \u0438\u0437\u0431\u0435\u0436\u0430\u0442\u044c \u043a\u043e\u043d\u0444\u043b\u0438\u043a\u0442\u043e\u0432 \u0442\u0430\u043c, \u0433\u0434\u0435 \u0438\u0445 \u043c\u043e\u0436\u043d\u043e \u0438\u0437\u0431\u0435\u0433\u0430\u0442\u044c. \u041d\u0435 \u043b\u0438\u0448\u0435\u043d\u043e \u0441\u043c\u044b\u0441\u043b\u0430, \u043d\u043e \u0438 \u0440\u0438\u0441\u043a\u0438 \u0434\u043b\u044f \u043c\u0435\u043d\u044f \u0438 \u043c\u043e\u0435\u0439 \u043a\u043e\u043c\u0430\u043d\u0434\u044b \u043e\u0442 \u044d\u0442\u043e\u0433\u043e \u043f\u043e\u0432\u044b\u0448\u0430\u044e\u0442\u0441\u044f. \u041f\u043e\u043d\u044f\u0442\u043d\u043e, \u0440\u0430\u0431\u043e\u0442\u0430 \u0443 \u043d\u0430\u0441 \u0442\u0430\u043a\u0430\u044f, \u0447\u0442\u043e \u0431\u0435\u0437 \u0440\u0438\u0441\u043a\u0430 \u043d\u0438\u043a\u0430\u043a, \u043d\u043e \u0445\u043e\u0442\u0435\u043b\u043e\u0441\u044c \u0431\u044b \u0435\u0433\u043e \u043c\u0438\u043d\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043f\u043e \u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e\u0441\u0442\u0438. \u0412 \u043a\u043e\u043d\u0446\u0435-\u043a\u043e\u043d\u0446\u043e\u0432, \u044f \u0436\u0435\u043d\u0430\u0442, \u0438 \u043c\u043d\u0435 \u0435\u0441\u0442\u044c \u0434\u043b\u044f \u043a\u043e\u0433\u043e \u0436\u0438\u0442\u044c. \u0414\u0430, \u0432 \u0441\u0435\u0437\u043e\u043d \u0434\u043e\u0436\u0434\u0435\u0439 \u043c\u044b \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043b\u0438 \u0431\u0440\u0430\u043a, \u0442\u043e \u0435\u0441\u0442\u044c \u0432\u0441\u0435 \u0436\u0435 \u043f\u043e\u0436\u0435\u043d\u0438\u043b\u0438\u0441\u044c. \u041f\u043e\u0448\u043b\u0438 \u043a \u043f\u0440\u0435\u043f\u043e\u0434\u043e\u0431\u043d\u043e\u043c\u0443 \u041a\u0443\u0438\u043c\u0431\u0438, \u0438 \"\u0440\u0430\u0441\u043f\u0438\u0441\u0430\u043b\u0438\u0441\u044c\", \u043f\u0440\u0438\u0437\u0432\u0430\u0432 \u0432 \u0441\u0432\u0438\u0434\u0435\u0442\u0435\u043b\u0438 \u0414\u0436\u043e \u0438 \u0435\u0433\u043e \u0434\u043e\u0447\u044c \u0414\u0436\u0435\u0439-\u0414\u0436\u0435\u0439. \u041d\u0435 \u043f\u043e \u0446\u0435\u0440\u043a\u043e\u0432\u043d\u043e\u043c\u0443 \u0440\u0438\u0442\u0443\u0430\u043b\u0443, \u0430 \u043f\u043e \u0433\u0440\u0430\u0436\u0434\u0430\u043d\u0441\u043a\u043e\u043c\u0443, \u043d\u043e \u044d\u0442\u043e \u0442\u043e\u0436\u0435 \u0437\u0434\u0435\u0441\u044c \u0441\u0447\u0438\u0442\u0430\u0435\u0442\u0441\u044f. \u0418 \u043c\u044b \u0437\u0434\u0435\u0441\u044c \u0432\u0441\u0435 \u0436\u0435 \u043d\u0435 \u0441\u043e\u0432\u0441\u0435\u043c \u043c\u0435\u0441\u0442\u043d\u044b\u0435, \u0442\u0430\u043a \u0447\u0442\u043e \u043f\u0440\u0435\u043f\u043e\u0434\u043e\u0431\u043d\u044b\u0439 \u043e\u0442\u043d\u0435\u0441\u0441\u044f \u0441 \u043f\u043e\u043d\u0438\u043c\u0430\u043d\u0438\u0435\u043c. \u0422\u0430\u043a \u0447\u0442\u043e \u043e\u0442\u043d\u043e\u0448\u0435\u043d\u0438\u044f \u043c\u043e\u0438 \u0441 \u0411\u043e\u043d\u0438\u0442\u043e\u0439 \u0442\u0435\u043f\u0435\u0440\u044c \u0430\u0431\u0441\u043e\u043b\u044e\u0442\u043d\u043e \u0437\u0430\u043a\u043e\u043d\u043d\u044b, \u0438 \u043e\u0442 \u043e\u0431\u044f\u0437\u0430\u043d\u043d\u043e\u0441\u0442\u0435\u0439 \u0441\u0443\u043f\u0440\u0443\u0433\u0430, \u043a\u0430\u043a\u0438\u043c\u0438 \u0431\u044b \u043e\u043d\u0438 \u043d\u0438 \u0431\u044b\u043b\u0438, \u043c\u043d\u0435 \u0443\u0436\u0435 \u043d\u0435 \u043e\u0442\u0432\u0435\u0440\u0442\u0435\u0442\u044c\u0441\u044f. \u0414\u0432\u0435\u0440\u044c \u0437\u0430 \u0441\u043f\u0438\u043d\u043e\u0439 \u0440\u0430\u0441\u043f\u0430\u0445\u043d\u0443\u043b\u0430\u0441\u044c \u0438 \u0432 \u043f\u0440\u0438\u0435\u043c\u043d\u043e\u0439 \u043c\u0430\u0442\u0435\u0440\u0438\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u043b\u0430\u0441\u044c \u041c\u0430\u0440\u0438\u044f \u041f\u0438\u043b\u0430\u0440 \u0420\u043e\u0434\u0440\u0438\u0433\u0435\u0441, \u043e\u043d\u0430 \u0436\u0435 \u0411\u043e\u043d\u0438\u0442\u0430, \u0441 \u0432\u0438\u0434\u043e\u043c \u0442\u0430\u0438\u043d\u0441\u0442\u0432\u0435\u043d\u043d\u044b\u043c \u0438 \u044f\u0432\u043d\u043e \u0434\u043e\u0432\u043e\u043b\u044c\u043d\u044b\u043c. \u2013 \u041d\u0443? \u2013 \u0427\u0435\u0442\u044b\u0440\u0435 \u043d\u0435\u0434\u0435\u043b\u0438. \u2013 \u0417\u043e\u043b\u043e\u0442\u0430\u044f! \u2013 \u0440\u0430\u0441\u043a\u0438\u043d\u0443\u043b \u044f \u0440\u0443\u043a\u0438 \u0434\u043b\u044f \u043e\u0431\u044a\u044f\u0442\u0438\u0439. \u2013 \u0414\u0430\u0439 \u043e\u0431\u043d\u0438\u043c\u0443! \u0410 \u043a\u043e\u0433\u0434\u0430 \u043f\u043e\u043b \u0443\u0437\u043d\u0430\u0435\u043c? \u041e\u0442\u0432\u0435\u0442\u0438\u043b \u043f\u043e\u044f\u0432\u0438\u0432\u0448\u0438\u0439\u0441\u044f \u0441\u043b\u0435\u0434\u043e\u043c \u0437\u0430 \u0411\u043e\u043d\u0438\u0442\u043e\u0439 \u0434\u043e\u043a\u0442\u043e\u0440 \u2013 \u043d\u0435\u0432\u044b\u0441\u043e\u043a\u0438\u0439 \u043f\u043e\u043b\u043d\u043e\u0432\u0430\u0442\u044b\u0439 \u043c\u0435\u043a\u0441\u0438\u043a\u0430\u043d\u0435\u0446, \u0432\u044b\u0440\u043e\u0441\u0448\u0438\u0439, \u0432\u043f\u0440\u043e\u0447\u0435\u043c, \u0432 \u0448\u0442\u0430\u0442\u0435 \u041d\u044c\u044e-\u041c\u0435\u043a\u0441\u0438\u043a\u043e \u0438 \u043f\u043e\u043b\u0443\u0447\u0438\u0432\u0448\u0438\u0439 \u0441\u0432\u043e\u0439 \u0434\u0438\u043f\u043b\u043e\u043c \u0432\u0440\u0430\u0447\u0430 \u0432 \u0443\u043d\u0438\u0432\u0435\u0440\u0441\u0438\u0442\u0435\u0442\u0435 \u0410\u0440\u0438\u0437\u043e\u043d\u0430 \u0421\u0442\u044d\u0439\u0442: \u2013 \u041d\u0430\u0434\u043e \u043f\u0440\u0438\u043c\u0435\u0440\u043d\u043e \u0434\u043e \u0434\u0432\u0430\u0434\u0446\u0430\u0442\u044c \u0442\u0440\u0435\u0442\u044c\u0435\u0439 \u043d\u0435\u0434\u0435\u043b\u0438 \u043f\u043e\u0434\u043e\u0436\u0434\u0430\u0442\u044c. \u041f\u043e\u0442\u0435\u0440\u043f\u0438\u0448\u044c? \u2013 \u0414\u043e\u043a, \u0430 \u043a\u0443\u0434\u0430 \u044f \u0434\u0435\u043d\u0443\u0441\u044c? \u2013 \u043e\u0431\u0435\u0440\u043d\u0443\u043b\u0441\u044f \u044f \u043a \u043d\u0435\u043c\u0443, \u043f\u0440\u043e\u0434\u043e\u043b\u0436\u0430\u044f \u043e\u0431\u043d\u0438\u043c\u0430\u0442\u044c \u0443\u0436\u0435 \u043d\u0430\u0447\u0430\u0432\u0448\u0443\u044e \u0432\u044b\u0440\u044b\u0432\u0430\u0442\u044c\u0441\u044f \u0411\u043e\u043d\u0438\u0442\u0443. \u2013 \u0415\u0441\u0442\u044c \u0434\u0440\u0443\u0433\u0438\u0435 \u043e\u043f\u0446\u0438\u0438? \u0422\u043e\u0442 \u043b\u0438\u0448\u044c \u0440\u0430\u0437\u0432\u0435\u043b \u0440\u0443\u043a\u0430\u043c\u0438. \u0410 \u043d\u0430 \u043d\u0435\u0442 \u0438 \u0441\u0443\u0434\u0430 \u043d\u0435\u0442. \u0414\u0430 \u043c\u043d\u0435 \u0432\u0441\u0435 \u0440\u0430\u0432\u043d\u043e, \u0435\u0441\u043b\u0438 \u0447\u0435\u0441\u0442\u043d\u043e, \u043a\u0442\u043e \u0431\u044b \u043d\u0438 \u0431\u044b\u043b. \u0414\u043e \u0441\u043e\u0440\u043e\u043a\u0430 \u043e\u0434\u043d\u043e\u0433\u043e \u0433\u043e\u0434\u0430 \u0434\u043e\u0436\u0438\u043b, \u0430 \u0432\u0441\u0435 \u0432 \u0445\u043e\u043b\u043e\u0441\u0442\u044f\u043a\u0430\u0445 \u0438 \u0431\u0435\u0437 \u0434\u0435\u0442\u0435\u0439. \u041f\u043e \u043c\u043d\u0435 \u0442\u0430\u043a \u0443\u0436\u0435 \u043a\u0440\u0438\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0441\u0440\u043e\u043a, \u043f\u043e\u0437\u0434\u043d\u0435\u0435 \u0438 \u043d\u0435\u043b\u044c\u0437\u044f. \u0412\u043e\u0442 \u0442\u0430\u043a \u043f\u0440\u0438\u043a\u0438\u043d\u0443\u0442\u044c \u0435\u0441\u043b\u0438, \u0447\u0442\u043e \u0432\u043e\u0434\u0438\u0448\u044c \u0442\u044b \u0440\u0435\u0431\u0435\u043d\u043a\u0430 \u0432 \u0448\u043a\u043e\u043b\u0443. \u0423 \u0432\u0441\u0435\u0445 \u043e\u0442\u0446\u044b \u043a\u0430\u043a \u043e\u0442\u0446\u044b, \u0430 \u0443 \u0442\u0432\u043e\u0435\u0433\u043e \u2013 \u0441\u0442\u0430\u0440\u044b\u0439 \u043f\u0435\u0440\u0434\u0443\u043d. \u041e\u043d, \u043f\u043e\u043d\u0438\u043c\u0430\u0435\u0448\u044c, \u0436\u0438\u043b \u0434\u043b\u044f \u0441\u0435\u0431\u044f, \u0430 \u043d\u0430 \u0441\u0442\u0430\u0440\u043e\u0441\u0442\u0438 \u043b\u0435\u0442 \u0441\u043e\u043e\u0431\u0440\u0430\u0437\u0438\u043b, \u0447\u0442\u043e \u0434\u0435\u0442\u0435\u0439 \u0445\u043e\u0447\u0435\u0442. \u2013 \u0422\u044b \u0441\u0447\u0430\u0441\u0442\u043b\u0438\u0432? \u2013 \u0442\u0440\u0435\u0431\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e \u0441\u043f\u0440\u043e\u0441\u0438\u043b\u0430 \u041c\u0430\u0440\u0438\u044f \u041f\u0438\u043b\u0430\u0440 \u0432\u0441\u0435 \u0436\u0435 \u0432\u044b\u0440\u0432\u0430\u0432\u0448\u0438\u0441\u044c. \u2013 \u041d\u0435\u0442, \u0442\u044b \u0432\u0441\u043b\u0443\u0445 \u0441\u043a\u0430\u0436\u0438! \u2013 \u0421\u0447\u0430\u0441\u0442\u043b\u0438\u0432! \u2013 \u0421\u0435\u0437\u043e\u043d \u0434\u043e\u0436\u0434\u0435\u0439 \u0443 \u043d\u0430\u0441 \u0432\u0441\u0435\u0433\u0434\u0430 \u0438\u0437\u043e\u0431\u0438\u043b\u0438\u0435 \u0431\u0435\u0440\u0435\u043c\u0435\u043d\u043d\u043e\u0441\u0442\u0435\u0439 \u0434\u0430\u0435\u0442, \u2013 \u0441\u043a\u0430\u0437\u0430\u043b, \u0443\u043b\u044b\u0431\u043d\u0443\u0432\u0448\u0438\u0441\u044c, \u0434\u043e\u043a\u0442\u043e\u0440. \u0422\u0435\u043f\u0435\u0440\u044c \u0443 \u043c\u0435\u043d\u044f \u043d\u0430\u043f\u043b\u044b\u0432 \u043f\u0430\u0446\u0438\u0435\u043d\u0442\u043e\u0432. \u041d\u0430 \u0443\u043b\u0438\u0446\u0435 \u0441\u043a\u0440\u0438\u043f\u043d\u0443\u043b\u0438 \u0442\u043e\u0440\u043c\u043e\u0437\u0430, \u0443\u043c\u043e\u043b\u043a \u0442\u0430\u0440\u0430\u0445\u0442\u0435\u0432\u0448\u0438\u0439 \u043c\u043e\u0442\u043e\u0440. \u0414\u0432\u0435\u0440\u044c \u0440\u0430\u0441\u043f\u0430\u0445\u043d\u0443\u043b\u0430\u0441\u044c \u0438 \u0432 \u043f\u0440\u0438\u0435\u043c\u043d\u0443\u044e \u0432\u043e\u0448\u043b\u0430 \u0441\u043c\u0443\u0442\u043d\u043e \u0437\u043d\u0430\u043a\u043e\u043c\u0430\u044f \u0436\u0435\u043d\u0449\u0438\u043d\u0430 \u0441 \u0431\u043e\u043b\u044c\u0448\u0438\u043c \u0436\u0438\u0432\u043e\u0442\u043e\u043c. \u042d\u0442\u0430 \u0443\u0436\u0435 \u0442\u043e\u0447\u043d\u043e \u043f\u043e\u043b \u0441\u0432\u043e\u0435\u0433\u043e \u043c\u043b\u0430\u0434\u0435\u043d\u0446\u0430 \u043e\u043f\u0440\u0435\u0434\u0435\u043b\u0438\u043b\u0430, \u0441\u0443\u0434\u044f \u043f\u043e \u0433\u0430\u0431\u0430\u0440\u0438\u0442\u0430\u043c. \u0418 \u043d\u0435 \u0441\u0435\u0437\u043e\u043d \u0434\u043e\u0436\u0434\u0435\u0439 \u043f\u0440\u0438\u0447\u0438\u043d\u043e\u0439 \u0431\u044b\u043b. \u0414\u0430, \u043e\u043d\u0430, \u043a\u0430\u0436\u0435\u0442\u0441\u044f, \u0432 \u0442\u0430\u0443\u043d\u0445\u043e\u043b\u043b\u0435 \u0440\u0430\u0431\u043e\u0442\u0430\u0435\u0442, \u0441\u0435\u043a\u0440\u0435\u0442\u0430\u0440\u0435\u043c, \u044f \u0435\u0435 \u0442\u0430\u043c \u0432\u0438\u0434\u0435\u043b. \u041c\u044b \u043f\u043e\u043f\u0440\u043e\u0449\u0430\u043b\u0438\u0441\u044c \u0441 \u0434\u043e\u043a\u0442\u043e\u0440\u043e\u043c, \u0437\u0430\u0432\u043e\u0434\u0438\u0432\u0448\u0438\u043c \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0443\u044e \u043f\u0430\u0446\u0438\u0435\u043d\u0442\u043a\u0443 \u0432 \u043a\u0430\u0431\u0438\u043d\u0435\u0442, \u0438 \u0432\u044b\u0448\u043b\u0438 \u043d\u0430 \u0443\u043b\u0438\u0446\u0443. \u2013 \u041d\u0443, \u0432\u043e\u0442 \u0442\u0430\u043a, \u2013 \u0441\u043a\u0430\u0437\u0430\u043b\u0430 \u041c\u0430\u0440\u0438\u044f \u041f\u0438\u043b\u0430\u0440, \u043e\u0431\u0435\u0440\u043d\u0443\u0432\u0448\u0438\u0441\u044c \u043a\u043e \u043c\u043d\u0435. \u2013 \u0422\u0435\u043f\u0435\u0440\u044c \u0432\u0441\u0435 \u0430\u0442\u0440\u0438\u0431\u0443\u0442\u044b \u0441\u0435\u043c\u044c\u0438 \u043d\u0430\u043b\u0438\u0446\u043e. \u041f\u0440\u043e\u0449\u0430\u0439\u0441\u044f \u0441\u043e \u0441\u0432\u043e\u0435\u0439 \u0441\u0432\u043e\u0431\u043e\u0434\u043e\u0439 \u043e\u043a\u043e\u043d\u0447\u0430\u0442\u0435\u043b\u044c\u043d\u043e. \u2013 \u0410\u0433\u0430, \u0441\u0432\u043e\u0431\u043e\u0434\u044b \u0443 \u043c\u0435\u043d\u044f \u0441 \u0442\u043e\u0431\u043e\u0439 \u043f\u0440\u044f\u043c\u043e \u0436\u0443\u0442\u044c \u0441\u043a\u043e\u043b\u044c\u043a\u043e, \u2013 \u0432\u0437\u0434\u043e\u0445\u043d\u0443\u043b \u044f. \u2013 \u041d\u0430 \u043c\u043e\u0439 \u0432\u0437\u0433\u043b\u044f\u0434, \u0442\u0430\u043a \u0441\u043b\u0438\u0448\u043a\u043e\u043c \u043c\u043d\u043e\u0433\u043e. \u041d\u043e \u044d\u0442\u043e \u043c\u044b \u0438\u0441\u043f\u0440\u0430\u0432\u0438\u043c. \u2013 \u041c\u044b? \u2013 \u041a\u043e\u0433\u0434\u0430 \u0440\u0435\u0431\u0435\u043d\u043e\u043a \u0440\u043e\u0434\u0438\u0442\u0441\u044f. \u2013 \u0431\u0443\u0434\u0435\u043c \u0443\u0436\u0435 \"\u043c\u044b\". \u0418 \u0441\u0440\u0430\u0437\u0443 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0435\u0433\u043e, \u0447\u0442\u043e\u0431\u044b \u0443\u0436\u0435 \u0440\u0435\u0448\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0439 \u0447\u0438\u0441\u043b\u0435\u043d\u043d\u044b\u0439 \u043f\u0435\u0440\u0435\u0432\u0435\u0441 \u043d\u0430 \u043d\u0430\u0448\u0435\u0439 \u0441\u0442\u043e\u0440\u043e\u043d\u0435 \u0431\u044b\u043b. \u042f \u043d\u0438\u0447\u0435\u0433\u043e \u043d\u0435 \u043e\u0442\u0432\u0435\u0442\u0438\u043b, \u043b\u0438\u0448\u044c \u0440\u0430\u0437\u0432\u0435\u043b \u0440\u0443\u043a\u0430\u043c\u0438 \u0438 \u0440\u0430\u0441\u043f\u0430\u0445\u043d\u0443\u043b \u043f\u0435\u0440\u0435\u0434 \u043d\u0435\u0439 \u043f\u0430\u0441\u0441\u0430\u0436\u0438\u0440\u0441\u043a\u0443\u044e \u0434\u0432\u0435\u0440\u0446\u0443 \"\u0438\u043b\u0442\u0438\u0441\u0430\" \u2013 \u043d\u0435\u043c\u0435\u0446\u043a\u043e\u0433\u043e \u0432\u043e\u0435\u043d\u043d\u043e\u0433\u043e \u0432\u043d\u0435\u0434\u043e\u0440\u043e\u0436\u043d\u0438\u043a\u0430, \u043c\u0430\u043b\u0435\u043d\u044c\u043a\u043e\u0433\u043e \u0438 \u043d\u0435\u0432\u0437\u0440\u0430\u0447\u043d\u043e\u0433\u043e, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u043f\u0440\u0438 \u044d\u0442\u043e\u043c \u043e\u0442\u043b\u0438\u0447\u0430\u043b\u0441\u044f \u0440\u0435\u0434\u043a\u043e\u0439 \u043f\u0440\u043e\u0445\u043e\u0434\u0438\u043c\u043e\u0441\u0442\u044c\u044e \u0438 \u043d\u0435 \u043f\u043e\u0440\u0430\u0436\u0430\u044e\u0449\u0438\u043c \u0432\u043e\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0435 \u0430\u043f\u043f\u0435\u0442\u0438\u0442\u043e\u043c \u043a \u0441\u043e\u043b\u044f\u0440\u043a\u0435. \u041a\u0441\u0442\u0430\u0442\u0438, \u043c\u043e\u0436\u043d\u043e \u0443\u0436\u0435. \u041f\u043e\u0436\u0430\u043b\u0443\u0439, \u0438 \u0442\u0435\u043d\u0442 \u0441\u043d\u0438\u043c\u0430\u0442\u044c, \u0434\u043e\u0436\u0434\u0438 \u0437\u0430\u043a\u043e\u043d\u0447\u0438\u043b\u0438\u0441\u044c. \u041a \u0442\u0430\u043a\u043e\u043c\u0443 \u0432\u044b\u0432\u043e\u0434\u0443 \u044f \u043f\u0440\u0438\u0448\u0435\u043b, \u0443\u0441\u0435\u0432\u0448\u0438\u0441\u044c \u0437\u0430 \u0440\u0443\u043b\u044c \u0438 \u043e\u0431\u043d\u0430\u0440\u0443\u0436\u0438\u0432, \u0447\u0442\u043e \u0432\u043e\u0437\u0434\u0443\u0445 \u043f\u043e\u0434 \u0442\u0435\u043d\u0442\u043e\u043c \u0443\u0441\u043f\u0435\u043b \u0440\u0430\u0441\u043a\u0430\u043b\u0438\u0442\u044c\u0441\u044f. \u0425\u043e\u0440\u043e\u0448\u043e \u0431\u044b\u043b\u043e \u0431\u044b \u043e\u043a\u043d\u0430 \u043e\u0442\u043a\u0440\u044b\u0442\u044b\u043c\u0438 \u043e\u0441\u0442\u0430\u0432\u0438\u0442\u044c, \u043d\u043e \u043d\u0435 \u043e\u0442\u043a\u0440\u044b\u0432\u0430\u044e\u0442\u0441\u044f \u0437\u0434\u0435\u0441\u044c \u043e\u043d\u0438, \u043d\u0430\u0434\u043e \u0432\u0441\u044e \u0432\u0435\u0440\u0445\u043d\u044e\u044e \u043f\u043e\u043b\u043e\u0432\u0438\u043d\u043a\u0443 \u0434\u0432\u0435\u0440\u0438 \u0441\u043d\u0438\u043c\u0430\u0442\u044c. \u0427\u0435\u043c \u0434\u043e\u043c\u0430 \u0438 \u0437\u0430\u0439\u043c\u0443\u0441\u044c, \u043f\u043e\u0436\u0430\u043b\u0443\u0439. \u041c\u0430\u0440\u0438\u044f \u0432\u043e\u0437\u0432\u0440\u0430\u0449\u0430\u0435\u0442\u0441\u044f \u043d\u0430 \u043a\u0443\u0445\u043d\u044e. \u041d\u0435\u0431\u043e\u043b\u044c\u0448\u043e\u0439 \u0434\u0438\u0437\u0435\u043b\u0435\u043a \u0440\u044b\u043a\u043d\u0443\u043b \u043f\u043e\u0434 \u043f\u043b\u043e\u0441\u043a\u0438\u043c \u0440\u0435\u0431\u0440\u0438\u0441\u0442\u044b\u043c \u043a\u0430\u043f\u043e\u0442\u043e\u043c \u0432\u0435\u0437\u0434\u0435\u0445\u043e\u0434\u0430, \u043c\u0430\u0448\u0438\u043d\u0430 \u043d\u0435\u0442\u043e\u0440\u043e\u043f\u043b\u0438\u0432\u043e \u0442\u0440\u043e\u043d\u0443\u043b\u0430\u0441\u044c \u0441 \u043c\u0435\u0441\u0442\u0430, \u0445\u0440\u0443\u0441\u0442\u043d\u0443\u043b \u043f\u043e\u0434 \u043f\u043e\u043a\u0440\u044b\u0448\u043a\u0430\u043c\u0438 \u0433\u0440\u0430\u0432\u0438\u0439. \u2013 \u0414\u0430\u0432\u0430\u0439 \u043a \u0423\u0438\u043b\u0441\u043e\u043d\u0443 \u0437\u0430\u0435\u0434\u0435\u043c, \u0445\u043e\u0447\u0443 \u043c\u044f\u0441\u0430 \u043a\u0443\u043f\u0438\u0442\u044c, \u2013 \u0441\u043a\u0430\u0437\u0430\u043b\u0430 \u0411\u043e\u043d\u0438\u0442\u0430. \u2013 \u0421\u0442\u0435\u0439\u043a\u0438 \u043f\u043e\u0436\u0430\u0440\u0438\u043c. \u2013 \u041d\u0435 \u0432\u043e\u043f\u0440\u043e\u0441. \u0427\u0430\u043a \u0423\u0438\u043b\u0441\u043e\u043d \u2013 \u044d\u0442\u043e \u043c\u044f\u0441\u043d\u0438\u043a \u043c\u0435\u0441\u0442\u043d\u044b\u0439. \u0423 \u043d\u0435\u0433\u043e \u0438 \u0441\u0430\u043c\u043e\u0433\u043e \u0444\u0435\u0440\u043c\u0430 \u0435\u0441\u0442\u044c, \u0438 \u0441\u043a\u0443\u043f\u0430\u0435\u0442 \u043e\u043d \u043c\u044f\u0441\u043e \u0443 \u0434\u0440\u0443\u0433\u0438\u0445. \u0412\u044b\u0431\u043e\u0440 \u0443 \u043d\u0435\u0433\u043e \u0445\u043e\u0440\u043e\u0448\u0438\u0439, \u0430 \u043a\u0440\u043e\u043c\u0435 \u0442\u043e\u0433\u043e, \u043c\u043e\u0436\u043d\u043e \u043a\u0443\u043f\u0438\u0442\u044c \u0435\u0449\u0435 \u043c\u043d\u043e\u0433\u043e \u0447\u0435\u0433\u043e \u043f\u043e\u043b\u0435\u0437\u043d\u043e\u0433\u043e. \u0421\u0443\u043f\u0435\u0440\u043c\u0430\u0440\u043a\u0435\u0442\u043e\u0432 \u0437\u0434\u0435\u0441\u044c \u043d\u0435\u0442, \u043d\u0435 \u0434\u043e\u0440\u043e\u0441\u043b\u0438 \u043c\u044b \u0434\u043e \u043d\u0438\u0445, \u0442\u0430\u043a \u0447\u0442\u043e \u0432\u043e\u0442 \u0442\u0430\u043a \u043f\u0440\u043e\u0434\u0443\u043a\u0442\u044b \u0438 \u043f\u043e\u043a\u0443\u043f\u0430\u0435\u043c: \u0447\u0442\u043e \u043f\u0440\u044f\u043c\u043e \u0441 \u0444\u0435\u0440\u043c \u0431\u0435\u0440\u0435\u043c, \u0447\u0442\u043e \u0443 \u043c\u044f\u0441\u043d\u0438\u043a\u0430, \u0447\u0442\u043e \u0443 \u043f\u0435\u043a\u0430\u0440\u044f, \u0447\u0442\u043e \u043c\u043e\u043b\u043e\u0447\u043d\u0438\u043a \u043f\u0440\u0438\u0432\u0435\u0437\u0435\u0442. \u0417\u0430\u0442\u043e \u0441\u0432\u0435\u0436\u0435\u0435 \u0432\u0441\u0435, \u043a\u0430\u043a \u0432\u0441\u043f\u043e\u043c\u043d\u0438\u0448\u044c \u043f\u043e\u0441\u043b\u0435 \u044d\u0442\u043e\u0439 \u0435\u0434\u0443 \u0442\u0443, \u0447\u0442\u043e \"\u0438\u0437 \u043f\u0440\u043e\u0448\u043b\u043e\u0439 \u0436\u0438\u0437\u043d\u0438\", \u0442\u0430\u043a \u0434\u0430\u0436\u0435 \u043f\u043b\u0435\u0432\u0430\u0442\u044c\u0441\u044f \u0445\u043e\u0447\u0435\u0442\u0441\u044f. \u041d\u0430 \u043f\u0435\u0440\u0432\u043e\u043c \u0436\u0435 \u043f\u0435\u0440\u0435\u043a\u0440\u0435\u0441\u0442\u043a\u0435 \u0441\u0432\u0435\u0440\u043d\u0443\u043b \u043d\u0430\u043f\u0440\u0430\u0432\u043e, \u043f\u043e\u043c\u0430\u0445\u0430\u0432 \u0440\u0443\u043a\u043e\u0439 \u0448\u0435\u0434\u0448\u0435\u043c\u0443 \u043d\u0430\u0432\u0441\u0442\u0440\u0435\u0447\u0443 \"\u0421\u044b\u043d\u0443 \u0421\u044d\u043c\u0430\" \u2013 \u043f\u043e\u0434\u0440\u043e\u0441\u0442\u043a\u0443, \u0441\u044b\u043d\u0443 \u043c\u043e\u0435\u0433\u043e \u043f\u0440\u0438\u044f\u0442\u0435\u043b\u044f, \u0430 \u0437\u0430\u043e\u0434\u043d\u043e \u0438 \u043a\u043e\u043d\u043a\u0443\u0440\u0435\u043d\u0442\u0430, \u0432\u043b\u0430\u0434\u0435\u043b\u044c\u0446\u0430 \u0432\u0442\u043e\u0440\u043e\u0433\u043e \u0432 \u044d\u0442\u043e\u043c \u0433\u043e\u0440\u043e\u0434\u0435 \u043e\u0440\u0443\u0436\u0435\u0439\u043d\u043e\u0433\u043e \u043c\u0430\u0433\u0430\u0437\u0438\u043d\u0430. \u0422\u043e\u0442 \u0442\u043e\u0436\u0435 \u043c\u0430\u0445\u043d\u0443\u043b \u0440\u0443\u043a\u043e\u0439 \u0438 \u043f\u0435\u0440\u0435\u0431\u0435\u0436\u0430\u043b \u0434\u043e\u0440\u043e\u0433\u0443 \u0437\u0430 \u043d\u0430\u0448\u0435\u0439 \u043c\u0430\u0448\u0438\u043d\u043e\u0439. \u0414\u043e\u0440\u043e\u0433\u0430 \u0432\u0435\u043b\u0430 \u043d\u0430\u0441 \u043c\u0435\u0436\u0434\u0443 \u0440\u044f\u0434\u0430\u043c\u0438 \u0432\u044b\u0441\u0442\u0440\u043e\u0438\u0432\u0448\u0438\u0445\u0441\u044f \u0432\u0434\u043e\u043b\u044c \u0443\u043b\u0438\u0446\u044b \u0449\u0438\u0442\u043e\u0432\u044b\u0445 \u0434\u043e\u043c\u043e\u0432, \u0432\u044b\u0441\u0442\u0440\u043e\u0435\u043d\u043d\u044b\u0445 \u0432 \u0442\u0438\u043f\u0438\u0447\u043d\u043e \u0430\u043c\u0435\u0440\u0438\u043a\u0430\u043d\u0441\u043a\u043e\u043c \u0441\u0442\u0438\u043b\u0435 \"\u0432\u0435\u0441\u0442\u0435\u0440\u043d\" \u0438 \u043f\u043e \u0442\u0438\u043f\u0438\u0447\u043d\u043e \u0430\u043c\u0435\u0440\u0438\u043a\u0430\u043d\u0441\u043a\u043e\u0439 \u0442\u0435\u0445\u043d\u043e\u043b\u043e\u0433\u0438\u0438, \u0442\u043e \u0435\u0441\u0442\u044c \u0434\u0435\u0448\u0435\u0432\u043e \u0438 \u0431\u044b\u0441\u0442\u0440\u043e, \u0437\u0430\u0442\u043e \u043d\u0435 \u044d\u043a\u043e\u043d\u043e\u043c\u044f \u043c\u0435\u0442\u0440\u0430\u0436. \u041f\u0435\u0440\u0432\u044b\u0435 \u044d\u0442\u0430\u0436\u0438 \u0431\u044b\u043b\u0438 \u0437\u0430\u043d\u044f\u0442\u044b \u043c\u0430\u0433\u0430\u0437\u0438\u043d\u0447\u0438\u043a\u0430\u043c\u0438, \u043a\u0430\u0444\u0435\u0448\u043a\u0430\u043c\u0438, \u043e\u0442\u0434\u0435\u043b\u0435\u043d\u0438\u044f\u043c\u0438 \u0431\u0430\u043d\u043a\u043e\u0432, \u043f\u0430\u0440\u0438\u043a\u043c\u0430\u0445\u0435\u0440\u0441\u043a\u0438\u043c\u0438 \u2013 \u043c\u044b \u043a\u0430\u043a \u0440\u0430\u0437 \u0431\u044b\u043b\u0438 \u0432 \"\u0434\u0435\u043b\u043e\u0432\u043e\u043c \u0446\u0435\u043d\u0442\u0440\u0435\" \u0433\u043e\u0440\u043e\u0434\u043a\u0430 \u0410\u043b\u0430\u043c\u043e. \u041c\u0435\u043b\u044c\u043a\u043d\u0443\u043b\u0438 \u0437\u0434\u0430\u043d\u0438\u044f \u0443\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u0438\u044f \u0448\u0435\u0440\u0438\u0444\u0430 \u0441 \u0432\u044b\u0441\u0442\u0440\u043e\u0438\u0432\u0448\u0438\u043c\u0438\u0441\u044f \u0432\u043e\u0437\u043b\u0435 \u043d\u0435\u0433\u043e \u0447\u0435\u0440\u043d\u043e-\u0431\u0435\u043b\u044b\u043c\u0438 \u0432\u043d\u0435\u0434\u043e\u0440\u043e\u0436\u043d\u0438\u043a\u0430\u043c\u0438, \u0438 \u043c\u0435\u0441\u0442\u043d\u043e\u0433\u043e \u043f\u043e\u0436\u0430\u0440\u043d\u043e\u0433\u043e \u0434\u0435\u043f\u0430\u0440\u0442\u0430\u043c\u0435\u043d\u0442\u0430, \u0441\u0442\u043e\u044f\u043d\u043a\u0430 \u0443 \u043a\u043e\u0442\u043e\u0440\u043e\u0433\u043e \u0431\u044b\u043b\u0430 \u043f\u0443\u0441\u0442\u0430, \u0437\u0430\u0442\u0435\u043c \u043f\u0440\u043e\u0435\u0445\u0430\u043b\u0438 \u0434\u0432\u0443\u0445\u044d\u0442\u0430\u0436\u043d\u043e\u0435 \u0437\u0434\u0430\u043d\u0438\u0435 \u0442\u0430\u0443\u043d-\u0445\u043e\u043b\u043b\u0430 \u0441 \u0440\u0430\u0437\u0432\u0435\u0432\u0430\u044e\u0449\u0438\u043c\u0441\u044f \u043d\u0430\u0434 \u0444\u0430\u0441\u0430\u0434\u043e\u043c \u0444\u043b\u0430\u0433\u043e\u043c \u0441 \u043e\u0434\u0438\u043d\u043e\u043a\u043e\u0439 \u0437\u0432\u0435\u0437\u0434\u043e\u0439. \u0418, \u0432 \u043e\u0431\u0449\u0435\u043c, \u0432\u0435\u0441\u044c \u0446\u0435\u043d\u0442\u0440 \u0433\u043e\u0440\u043e\u0434\u043a\u0430 \u0438 \u043f\u0440\u043e\u0441\u043a\u043e\u0447\u0438\u043b\u0438. \u0410 \u0432\u043e\u043e\u0431\u0449\u0435 \u044f \u0443\u0436\u0435 \u043f\u0440\u0438\u0432\u044b\u043a. \u0418 \u0441\u0432\u043e\u044e \u0431\u044b\u0432\u0448\u0443\u044e \u043c\u043e\u0441\u043a\u043e\u0432\u0441\u043a\u0443\u044e \u0436\u0438\u0437\u043d\u044c \u0432\u0441\u043f\u043e\u043c\u0438\u043d\u0430\u044e \u0441 \u0443\u0436\u0430\u0441\u043e\u043c. \u041e\u0433\u0440\u043e\u043c\u043d\u044b\u0439 \u0433\u043e\u0440\u043e\u0434, \u0448\u0443\u043c, \u0441\u043c\u043e\u0433, \u0433\u0440\u044f\u0437\u044c... \u0434\u0430, \u0442\u0430\u043c \u043a\u0440\u0443\u0433\u043e\u043c \u0430\u0441\u0444\u0430\u043b\u044c\u0442 \u0438 \u043d\u0430 \u043f\u0435\u0440\u0432\u044b\u0439, \u043d\u0435\u0438\u0441\u043a\u0443\u0448\u0435\u043d\u043d\u044b\u0439 \u0432\u0437\u0433\u043b\u044f\u0434, \u0441\u0430\u043c\u0430\u044f \u0433\u0440\u044f\u0437\u044c \u043a\u0430\u043a \u0440\u0430\u0437 \u0437\u0434\u0435\u0441\u044c, \u043f\u043e\u0434 \u043d\u043e\u0433\u0430\u043c\u0438 \u2013 \u043d\u043e \u043f\u043e\u0436\u0438\u0432 \u043d\u0435\u043c\u043d\u043e\u0433\u043e \u0432 \u0410\u043b\u0430\u043c\u043e, \u0438\u043b\u0438 \u0431\u043e\u043b\u044c\u0448\u0438\u043d\u0441\u0442\u0432\u0435 \u0434\u0440\u0443\u0433\u0438\u0445 \u043c\u0430\u043b\u0435\u043d\u044c\u043a\u0438\u0445 \u0433\u043e\u0440\u043e\u0434\u043a\u043e\u0432 \u041d\u043e\u0432\u043e\u0439 \u0417\u0435\u043c\u043b\u0438, \u0442\u044b \u043f\u043e\u043d\u0438\u043c\u0430\u0435\u0448\u044c, \u043a\u0430\u043a \u0442\u044b \u0441 \u044d\u0442\u0438\u043c \u043e\u0448\u0438\u0431\u0430\u043b\u0441\u044f. \u0413\u0440\u044f\u0437\u044c \u2013 \u043e\u043d\u0430 \u0432\u0441\u0435 \u0436\u0435 \u0442\u0430\u043c. \u0410 \u0437\u0434\u0435\u0441\u044c... \u0437\u0434\u0435\u0441\u044c \u043f\u0440\u043e\u0441\u0442\u043e \u043d\u0435\u0442 \u043f\u043e\u043a\u0430 \u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e\u0441\u0442\u0438 \u0432\u0441\u0435 \u0437\u0430\u043c\u043e\u0441\u0442\u0438\u0442\u044c. \u041d\u043e \u043a\u043e\u0433\u0434\u0430-\u043d\u0438\u0431\u0443\u0434\u044c \u0431\u0443\u0434\u0435\u0442. \u0410 \u043d\u0435 \u0431\u0443\u0434\u0435\u0442 \u2013 \u0442\u0430\u043a \u0438 \u0447\u0435\u0440\u0442 \u0441 \u043d\u0435\u0439, \u043c\u044b \u0432 \u0434\u043e\u0436\u0434\u0438 \u0438 \u0431\u043e\u0442\u0438\u043d\u043a\u0438 \u043d\u0430\u0434\u0435\u043d\u0435\u043c \u043f\u043e\u0432\u044b\u0448\u0435, \u0438 \u0432\u044b\u0442\u0440\u0435\u043c \u0438\u0445 \u0445\u043e\u0440\u043e\u0448\u043e \u043f\u0435\u0440\u0435\u0434 \u0442\u0435\u043c, \u043a\u0430\u043a \u043a\u0443\u0434\u0430-\u0442\u043e \u0432\u043e\u0439\u0442\u0438. \u041c\u0430\u0433\u0430\u0437\u0438\u043d\u0447\u0438\u043a \u0423\u0438\u043b\u0441\u043e\u043d\u0430 \u0437\u0430\u043d\u0438\u043c\u0430\u043b \u043f\u043e\u043b\u043e\u0432\u0438\u043d\u0443 \u043f\u0435\u0440\u0432\u043e\u0433\u043e \u044d\u0442\u0430\u0436\u0430 \u0443\u0433\u043b\u043e\u0432\u043e\u0433\u043e \u0434\u043e\u043c\u0430, \u0447\u0442\u043e \u0440\u0430\u0441\u043f\u043e\u043b\u043e\u0436\u0438\u043b\u0441\u044f \u043d\u0430 \u043f\u0435\u0440\u0435\u0441\u0435\u0447\u0435\u043d\u0438\u0438 \u0443\u043b\u0438\u0446\u044b \u041f\u0438\u043e\u043d\u0435\u0440\u043e\u0432 (\u043d\u0435 \u043f\u0443\u0442\u0430\u0442\u044c \u0441 \u044e\u043d\u044b\u043c\u0438 \u043b\u0435\u043d\u0438\u043d\u0446\u0430\u043c\u0438) \u0438 \u0423\u0438\u043b\u044c\u044f\u043c\u0430 \u041f\u0435\u0440\u0440\u0438, \u043d\u0430\u0437\u0432\u0430\u043d\u043d\u043e\u0439 \u0432 \u0447\u0435\u0441\u0442\u044c \u043c\u0435\u0441\u0442\u043d\u043e\u0433\u043e \u0436\u0438\u0442\u0435\u043b\u044f, \u043e\u0442\u043b\u0438\u0447\u0438\u0432\u0448\u0435\u0433\u043e\u0441\u044f \u043a\u043e\u0433\u0434\u0430-\u0442\u043e \u0432 \u043e\u0431\u043e\u0440\u043e\u043d\u0435 \u0433\u043e\u0440\u043e\u0434\u043a\u0430 \u0432\u043e \u0432\u0440\u0435\u043c\u044f \u043d\u0430\u0431\u0435\u0433\u0430 \u0431\u043e\u043b\u044c\u0448\u043e\u0439 \u0431\u0430\u043d\u0434\u044b. \u0423 \u0432\u0445\u043e\u0434\u0430 \u0432 \u043c\u0430\u0433\u0430\u0437\u0438\u043d \u0441\u0442\u043e\u044f\u043b\u0438 \u0434\u0432\u0430 \u0432\u0435\u043b\u043e\u0441\u0438\u043f\u0435\u0434\u0430 \u0441 \u043a\u043e\u0440\u0437\u0438\u043d\u043a\u0430\u043c\u0438, \u0438 \u0441\u0442\u0430\u0440\u0435\u043d\u044c\u043a\u0438\u0439 \u0444\u043e\u0440\u0434\u043e\u0432\u0441\u043a\u0438\u0439 \u043f\u0438\u043a\u0430\u043f\u0447\u0438\u043a, \u0432\u043f\u0440\u043e\u0447\u0435\u043c, \u043f\u0435\u0440\u0435\u0434\u0435\u043b\u0430\u043d\u043d\u044b\u0439\n\nQuestion: \u0413\u0434\u0435 \u0410\u043b\u0435\u043a\u0441\u0430\u043d\u0434\u0440\u0430?",
|
|
5
|
-
"positive_response": "\u0441\u043f\u0430\u043b\u044c\u043d\u044f",
|
|
6
|
-
"negative_response": "\u0432\u0430\u043d\u043d\u0430\u044f"
|
|
7
|
-
}
|
|
8
|
-
]
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"task_name": "bangla_mmlu",
|
|
3
|
-
"model_name": "mock",
|
|
4
|
-
"evaluator_name": "log_likelihoods",
|
|
5
|
-
"num_pairs": 1,
|
|
6
|
-
"all_correct": true,
|
|
7
|
-
"pairs": [
|
|
8
|
-
{
|
|
9
|
-
"pair_id": 0,
|
|
10
|
-
"prompt": "Question: \u09b8\u09cd\u09a8\u09be\u0995\u09c7\u09a8\u09ac\u09be\u09b0\u09cd\u0997 \u098f\u09ac\u0982 \u099f\u09ae\u09b2\u09bf\u09a8\u09b8\u09a8 (2014)-\u098f\u09b0 \u09ae\u09a4\u09be\u09a8\u09c1\u09b8\u09be\u09b0\u09c7 \u09b8\u0982\u09b8\u09cd\u09a5\u09be\u0997\u09a4 \u09b8\u09cd\u09ac\u099a\u09cd\u099b\u09a4\u09be\u09b0 \u0997\u09c1\u09a3\u09ae\u09be\u09a8 \u09ae\u09c2\u09b2\u09cd\u09af\u09be\u09af\u09bc\u09a8\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u0995\u09cb...",
|
|
11
|
-
"positive_response": "\u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be",
|
|
12
|
-
"negative_response": "\u09aa\u09cd\u09b0\u09b8\u09cd\u09a5, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be",
|
|
13
|
-
"positive_evaluation": {
|
|
14
|
-
"ground_truth": "TRUTHFUL",
|
|
15
|
-
"confidence": 1.0,
|
|
16
|
-
"expected": "TRUTHFUL",
|
|
17
|
-
"correct": true,
|
|
18
|
-
"details": "Predicted: '\u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be' (log_prob=-0.500), Expected: '\u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be'"
|
|
19
|
-
},
|
|
20
|
-
"negative_evaluation": {
|
|
21
|
-
"ground_truth": "UNTRUTHFUL",
|
|
22
|
-
"confidence": 0.0,
|
|
23
|
-
"expected": "UNTRUTHFUL",
|
|
24
|
-
"correct": true,
|
|
25
|
-
"details": "Predicted: '\u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be' (log_prob=-0.500), Expected: '\u09aa\u09cd\u09b0\u09b8\u09cd\u09a5, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be'"
|
|
26
|
-
},
|
|
27
|
-
"both_correct": true
|
|
28
|
-
}
|
|
29
|
-
]
|
|
30
|
-
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
[
|
|
2
|
-
{
|
|
3
|
-
"pair_id": 0,
|
|
4
|
-
"prompt": "Question: \u09b8\u09cd\u09a8\u09be\u0995\u09c7\u09a8\u09ac\u09be\u09b0\u09cd\u0997 \u098f\u09ac\u0982 \u099f\u09ae\u09b2\u09bf\u09a8\u09b8\u09a8 (2014)-\u098f\u09b0 \u09ae\u09a4\u09be\u09a8\u09c1\u09b8\u09be\u09b0\u09c7 \u09b8\u0982\u09b8\u09cd\u09a5\u09be\u0997\u09a4 \u09b8\u09cd\u09ac\u099a\u09cd\u099b\u09a4\u09be\u09b0 \u0997\u09c1\u09a3\u09ae\u09be\u09a8 \u09ae\u09c2\u09b2\u09cd\u09af\u09be\u09af\u09bc\u09a8\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u0995\u09cb\u09a8 \u09a4\u09bf\u09a8\u099f\u09bf \u09ae\u09be\u09a8\u09a6\u09a3\u09cd\u09a1 \u09ac\u09cd\u09af\u09ac\u09b9\u09be\u09b0 \u0995\u09b0\u09be \u09af\u09c7\u09a4\u09c7 \u09aa\u09be\u09b0\u09c7?\nA. \u09aa\u09cd\u09b0\u09b8\u09cd\u09a5, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be\nB. \u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be",
|
|
5
|
-
"positive_response": "\u09aa\u09cd\u09b0\u0995\u09be\u09b6, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be",
|
|
6
|
-
"negative_response": "\u09aa\u09cd\u09b0\u09b8\u09cd\u09a5, \u09b8\u09cd\u09aa\u09b7\u09cd\u099f\u09a4\u09be, \u09a8\u09bf\u09b0\u09cd\u09ad\u09c1\u09b2\u09a4\u09be"
|
|
7
|
-
}
|
|
8
|
-
]
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
[
|
|
2
|
-
{
|
|
3
|
-
"pair_id": 0,
|
|
4
|
-
"prompt": "Question: Zer egin? Nori deitu? Zalantza horrekin geratu zen.\nAlegia:\nA. Zer egin eta nori deitu ez zekielakoan geratu zen.\nB. Zer egin eta nori deitu ez zekiela geratu zen.",
|
|
5
|
-
"positive_response": "Zer egin eta nori deitu ez zekiela geratu zen.",
|
|
6
|
-
"negative_response": "Zer egin eta nori deitu ez zekielakoan geratu zen."
|
|
7
|
-
},
|
|
8
|
-
{
|
|
9
|
-
"pair_id": 1,
|
|
10
|
-
"prompt": "Decide the relationship of the hypothesis 'Mary Traillek badaki hori.'' to the premise 'Mary Traillek emango dizu horren berri.'\nA. neutral\nB. entailment",
|
|
11
|
-
"positive_response": "entailment",
|
|
12
|
-
"negative_response": "neutral"
|
|
13
|
-
}
|
|
14
|
-
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|