wisent 0.7.379__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wisent/__init__.py +64 -0
- wisent/cli.py +114 -0
- wisent/core/__init__.py +40 -0
- wisent/core/activations/__init__.py +26 -0
- wisent/core/activations/activations.py +97 -0
- wisent/core/activations/activations_collector.py +506 -0
- wisent/core/activations/core/__init__.py +0 -0
- wisent/core/activations/core/atoms.py +219 -0
- wisent/core/activations/prompt_construction_strategy.py +47 -0
- wisent/core/adapters/__init__.py +22 -0
- wisent/core/adapters/audio.py +616 -0
- wisent/core/adapters/base.py +420 -0
- wisent/core/adapters/multimodal.py +738 -0
- wisent/core/adapters/robotics.py +643 -0
- wisent/core/adapters/text.py +441 -0
- wisent/core/adapters/video.py +555 -0
- wisent/core/agent/__init__.py +1 -0
- wisent/core/agent/budget.py +644 -0
- wisent/core/agent/device_benchmarks.py +691 -0
- wisent/core/agent/diagnose/__init__.py +1 -0
- wisent/core/agent/diagnose/agent_classifier_decision.py +641 -0
- wisent/core/agent/diagnose/classifier_marketplace.py +554 -0
- wisent/core/agent/diagnose/create_classifier.py +1155 -0
- wisent/core/agent/diagnose/response_diagnostics.py +273 -0
- wisent/core/agent/diagnose/select_classifiers.py +507 -0
- wisent/core/agent/diagnose/synthetic_classifier_option.py +755 -0
- wisent/core/agent/diagnose/tasks/__init__.py +33 -0
- wisent/core/agent/diagnose/tasks/task_manager.py +1453 -0
- wisent/core/agent/diagnose/tasks/task_relevance.py +94 -0
- wisent/core/agent/diagnose/tasks/task_selector.py +151 -0
- wisent/core/agent/diagnose.py +249 -0
- wisent/core/agent/steer.py +215 -0
- wisent/core/agent/timeout.py +134 -0
- wisent/core/autonomous_agent.py +1158 -0
- wisent/core/benchmark_extractors.py +372 -0
- wisent/core/benchmark_registry.py +151 -0
- wisent/core/bigcode_extractors.py +26 -0
- wisent/core/bigcode_integration.py +886 -0
- wisent/core/branding.py +108 -0
- wisent/core/classifier/__init__.py +1 -0
- wisent/core/classifier/models/__init__.py +1 -0
- wisent/core/classifiers/__init__.py +1 -0
- wisent/core/classifiers/classifiers/__init__.py +0 -0
- wisent/core/classifiers/classifiers/core/__init__.py +0 -0
- wisent/core/classifiers/classifiers/core/atoms.py +748 -0
- wisent/core/classifiers/classifiers/models/__init__.py +0 -0
- wisent/core/classifiers/classifiers/models/logistic.py +29 -0
- wisent/core/classifiers/classifiers/models/mlp.py +47 -0
- wisent/core/classifiers/classifiers/rotator.py +137 -0
- wisent/core/classifiers/core/__init__.py +1 -0
- wisent/core/classifiers/models/__init__.py +1 -0
- wisent/core/classifiers/pipeline_steps/__init__.py +1 -0
- wisent/core/cli/__init__.py +26 -0
- wisent/core/cli/agent/__init__.py +15 -0
- wisent/core/cli/agent/apply_steering.py +192 -0
- wisent/core/cli/agent/evaluate_response.py +128 -0
- wisent/core/cli/agent/generate_synthetic_pairs.py +123 -0
- wisent/core/cli/agent/main.py +139 -0
- wisent/core/cli/agent/train_classifier.py +173 -0
- wisent/core/cli/check_linearity.py +126 -0
- wisent/core/cli/create_steering_vector.py +304 -0
- wisent/core/cli/diagnose_pairs.py +153 -0
- wisent/core/cli/diagnose_vectors.py +404 -0
- wisent/core/cli/estimate_unified_goodness_time.py +428 -0
- wisent/core/cli/evaluate_refusal.py +241 -0
- wisent/core/cli/evaluate_responses.py +926 -0
- wisent/core/cli/generate_humanization_pairs.py +128 -0
- wisent/core/cli/generate_pairs.py +175 -0
- wisent/core/cli/generate_pairs_from_task.py +108 -0
- wisent/core/cli/generate_responses.py +160 -0
- wisent/core/cli/generate_vector_from_synthetic.py +217 -0
- wisent/core/cli/generate_vector_from_task.py +248 -0
- wisent/core/cli/get_activations.py +192 -0
- wisent/core/cli/inference_config.py +84 -0
- wisent/core/cli/inference_config_cli.py +54 -0
- wisent/core/cli/modify_weights.py +660 -0
- wisent/core/cli/multi_steer.py +112 -0
- wisent/core/cli/optimization_cache.py +298 -0
- wisent/core/cli/optimize.py +621 -0
- wisent/core/cli/optimize_classification.py +473 -0
- wisent/core/cli/optimize_sample_size.py +390 -0
- wisent/core/cli/optimize_steering.py +3421 -0
- wisent/core/cli/optimize_weights.py +1287 -0
- wisent/core/cli/steering_method_trainer.py +641 -0
- wisent/core/cli/steering_search_space.py +508 -0
- wisent/core/cli/tasks.py +940 -0
- wisent/core/cli/train_unified_goodness.py +681 -0
- wisent/core/cli_logger.py +22 -0
- wisent/core/config_manager.py +1731 -0
- wisent/core/contrastive_pairs/__init__.py +15 -0
- wisent/core/contrastive_pairs/core/__init__.py +0 -0
- wisent/core/contrastive_pairs/core/atoms.py +45 -0
- wisent/core/contrastive_pairs/core/buliders.py +59 -0
- wisent/core/contrastive_pairs/core/pair.py +183 -0
- wisent/core/contrastive_pairs/core/response.py +153 -0
- wisent/core/contrastive_pairs/core/serialization.py +306 -0
- wisent/core/contrastive_pairs/core/set.py +192 -0
- wisent/core/contrastive_pairs/diagnostics/__init__.py +79 -0
- wisent/core/contrastive_pairs/diagnostics/activations.py +53 -0
- wisent/core/contrastive_pairs/diagnostics/base.py +73 -0
- wisent/core/contrastive_pairs/diagnostics/control_vectors.py +1655 -0
- wisent/core/contrastive_pairs/diagnostics/coverage.py +79 -0
- wisent/core/contrastive_pairs/diagnostics/divergence.py +98 -0
- wisent/core/contrastive_pairs/diagnostics/duplicates.py +118 -0
- wisent/core/contrastive_pairs/diagnostics/linearity.py +325 -0
- wisent/core/contrastive_pairs/diagnostics/vector_quality.py +620 -0
- wisent/core/contrastive_pairs/huggingface_pairs/__init__.py +1 -0
- wisent/core/contrastive_pairs/huggingface_pairs/atoms.py +255 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_extractor_manifest.py +470 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_extractor_registry.py +136 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/__init__.py +44 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentbench.py +225 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentharm.py +267 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/agentic_search.py +444 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aider_polyglot.py +225 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime.py +118 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime2024.py +74 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/aime2025.py +73 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/alpaca_eval.py +153 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/apps.py +182 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/arena_hard.py +179 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/atis.py +89 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/babilong.py +96 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bangla_mmlu.py +108 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/basqueglue.py +217 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bec2016eu.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bfcl.py +283 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/bhtc_v2.py +87 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/browsecomp.py +245 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/chain_of_thought.py +89 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/chinese_simpleqa.py +209 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/cluewsc.py +177 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/cnn_dailymail.py +92 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codeforces.py +378 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue.py +109 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text.py +15 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_go.py +64 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_java.py +65 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_javascript.py +65 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_php.py +65 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_python.py +65 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/codexglue_code_to_text_ruby.py +65 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/coding_benchmarks.py +844 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/coedit_gec.py +79 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/conala.py +133 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/concode.py +111 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/dbpedia_14.py +91 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/doc_vqa.py +102 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/donotanswer.py +236 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ds1000.py +129 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ds_1000.py +155 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/epec_koref_bin.py +85 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ethos_binary.py +82 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/evalita_mp.py +165 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/evalita_sp_sum_task_fp_small_p1.py +89 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/facts_grounding.py +181 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/faithbench.py +295 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/financial_tweets.py +100 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flames.py +270 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flan_held_in.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/flores.py +572 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/frames.py +143 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/freebase.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/get_negative_example_livecodebench.py +146 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/get_positive_example_livecodebench.py +140 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/gpt3_translation_benchmarks.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hallucinations_leaderboard.py +389 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/halueval.py +246 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/harmbench.py +250 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/healthbench.py +181 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hle.py +106 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/hmmt.py +117 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/humaneval.py +119 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/humanevalpack.py +102 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/instruct_humaneval.py +180 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/instructhumaneval.py +129 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/iwslt2017_ar_en.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/iwslt2017_en_ar.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/jailbreakbench.py +258 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/law_stack_exchange.py +101 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/ledgar.py +118 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench.py +61 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench_contrastive_pair_generator.py +491 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livecodebench_v6.py +263 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/livemathbench.py +230 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/llama.py +96 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/longform_writing.py +285 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/m_mmlu.py +96 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/math.py +186 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/math500.py +146 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mbpp.py +142 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/meddialog.py +79 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/medical_abstracts.py +101 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/medium_priority_benchmarks.py +787 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mercury.py +111 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mmlu_redux.py +194 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/mmlusr.py +108 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multimedqa.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multipl_e.py +109 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple.py +96 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_choice.py +87 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_cpp.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_go.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_java.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_js.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_py.py +15 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/multiple_rs.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/non_greedy_robustness_agieval_aqua_rat.py +92 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/olympiadbench.py +287 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/openllm.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/option_order_robustness_agieval_aqua_rat.py +92 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/or_bench.py +300 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/penn_treebank.py +80 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/planbench.py +317 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/polymath.py +467 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/prompt_robustness_agieval_aqua_rat.py +92 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/pythia.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/recode.py +131 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/refusalbench.py +280 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/scicode.py +275 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/self_consistency.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/simpleqa.py +145 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/sorry_bench.py +211 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/stsb.py +79 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_lm_eval_v1.py +99 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_lm_eval_v1_seq2seq.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_glue_t5_prompt.py +123 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/super_gpqa.py +106 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/swe_bench.py +428 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/swe_bench_verified.py +158 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/sycophancy_eval.py +205 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/t0_eval.py +79 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tag.py +98 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tau_bench.py +305 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/tmlu.py +109 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/toolbench.py +360 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/toolemu.py +386 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/travelplanner.py +286 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/truthfulqa_generation.py +128 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/unfair_tos.py +83 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/vaxx_stance.py +86 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wiceu.py +85 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wikitext103.py +97 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wildguard.py +280 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt14_en_fr.py +97 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt14_fr_en.py +97 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_de_en.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_en_de.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_en_ro.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt16_ro_en.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/wmt_ro_en_t5_prompt.py +90 -0
- wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/xsum.py +81 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/__init__.py +0 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/atoms.py +265 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/__init__.py +472 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aclue.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/acp.py +33 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/acpbench.py +39 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/advanced_ai_risk.py +59 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aexams.py +14 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrimgsm.py +10 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrimmlu.py +10 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrixnli.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench.py +14 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_adr.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_afriqa.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_afrisenti.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_belebele.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_flores.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_injongointent.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_mafand.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhaner.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhanews.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_masakhapos.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_naijarc.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_nollysenti.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_ntrex.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_openai_mmlu.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_salt.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_sib.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_uhura_arc_easy.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/afrobench_xlsum.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/agieval.py +33 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/anli.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arab_culture.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_acva.py +67 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_acva_light.py +67 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_complete.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabic_leaderboard_light.py +81 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arabicmmlu.py +59 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/aradice.py +36 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arc.py +61 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/arithmetic.py +19 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/basque_bench.py +37 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bbh.py +121 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bbq.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/belebele.py +293 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bertaqa.py +25 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/bigbench.py +300 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/blimp.py +76 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/careqa.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/catalan_bench.py +43 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ceval_valid.py +61 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/cmmlu.py +76 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/code_x_glue.py +16 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/copal_id.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/crows_pairs.py +31 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/csatqa.py +15 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/darija.py +29 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/darijammlu.py +57 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/egymmlu.py +62 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/eus.py +76 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/evalita_mp.py +93 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/fld.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/flores.py +466 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/freebase.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/french_bench.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/galician_bench.py +41 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/glianorex.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/global_mmlu.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gpqa.py +27 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gsm8k.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/gsm8k_platinum.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/haerae.py +14 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/headqa.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hellaswag.py +39 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hendrycks_ethics.py +14 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hendrycks_math.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/hrm8k.py +20 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/inverse.py +22 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/japanese_leaderboard.py +20 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/jsonschema_bench.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kbl.py +85 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kmmlu.py +281 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kobest.py +14 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/kormedmcqa.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/lambada.py +28 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/leaderboard.py +52 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/libra.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/lingoly.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/longbench.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/m.py +43 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mastermind.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mathqa.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/med.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/meddialog.py +12 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/medqa.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mela.py +18 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/metabench.py +36 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mgsm.py +44 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/minerva_math.py +16 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mlqa.py +58 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu.py +70 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_pro.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_pro_plus.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlu_prox.py +191 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmlusr.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/mmmu.py +46 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/model_written_evals.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/multiblimp.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/non.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/noreval.py +143 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/noridiom.py +20 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/nortruthfulqa.py +32 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/nrk.py +20 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_arc_multilingual.py +10 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_hellaswag_multilingual.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_mmlu_multilingual.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/okapi_truthfulqa_multilingual.py +34 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/paloma.py +25 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/pawsx.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/persona.py +144 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/pile.py +31 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/polemo2.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/portuguese_bench.py +31 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/prompt.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/qa4mre.py +12 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/qasper.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ru.py +19 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/ruler.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/score.py +20 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/scrolls.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/self_consistency.py +11 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/spanish_bench.py +38 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/storycloze.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/super_glue_t5_prompt.py +17 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tinyBenchmarks.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tmlu.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/tmmluplus.py +80 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/translation.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/truthfulqa.py +76 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/truthfulqa_multi.py +24 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/turkishmmlu.py +30 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/unitxt.py +23 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/unscramble.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/winogender.py +16 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmdp.py +12 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmt14.py +16 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wmt16.py +22 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/wsc273.py +9 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xcopa.py +21 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xnli.py +28 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xnli_eu.py +12 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xquad.py +22 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xstorycloze.py +22 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/group_task_manifests/xwinograd.py +15 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_extractor_manifest.py +478 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_extractor_registry.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/__init__.py +125 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aclue.py +171 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/acp_bench.py +207 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/acp_bench_hard.py +185 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/advanced.py +130 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aexams.py +184 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrimgsm.py +98 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrimmlu.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrixnli.py +129 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrobench_cot.py +88 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/afrobench_mc.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ag.py +134 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/agieval.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ai2_arc.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anagrams1.py +81 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anagrams2.py +81 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/anli.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabculture.py +180 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic.py +98 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_exams.py +104 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_leaderboard_complete.py +168 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabic_leaderboard_light.py +168 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arabicmmlu.py +167 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/aradice.py +268 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc.py +133 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_challenge.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_easy.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_gen.py +101 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arc_mc.py +106 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/argument.py +134 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/arithmetic.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/asdiv.py +122 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/assin.py +103 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/babi.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench_gen.py +168 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/basque_bench_mc.py +139 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bbh.py +133 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bbq.py +169 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/belebele.py +181 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/benchmarks.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bertaqa.py +165 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bhs.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bhtc.py +143 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/bigbench.py +170 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/blimp.py +171 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/blimp_nl.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/boolq.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/boolq_seq2seq.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/c4.py +150 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cabbq.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cabreu.py +127 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/careqa.py +169 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench_gen.py +119 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalan_bench_mc.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catalanqa.py +171 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/catcola.py +139 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cb.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ceval.py +223 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ceval_valid.py +163 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/chain.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/chartqa.py +238 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/claim.py +151 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/click.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cmmlu.py +166 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cnn.py +144 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cocoteros.py +148 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/code2text.py +161 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/code_x_glue.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/codexglue.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coedit.py +149 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cola.py +83 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/commonsense.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/commonsense_qa.py +127 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/copa.py +124 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/copal_id.py +169 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coqa.py +162 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/coqcat.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/crows_pairs.py +158 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/csatqa.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cycle.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/cycle_letters.py +81 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darija_bench.py +221 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darijahellaswag.py +174 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/darijammlu.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/dbpedia.py +157 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/discrim_eval.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/doc.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/drop.py +129 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/egyhellaswag.py +125 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/egymmlu.py +180 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/epec.py +142 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench.py +194 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench_ca.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eq_bench_es.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/esbbq.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/escola.py +85 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ethics.py +135 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ethos.py +99 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_exams.py +225 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_proficiency.py +159 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_reading.py +159 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/eus_trivia.py +159 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/evalita_llm.py +166 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/evalita_sp.py +109 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/fda.py +105 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/financial.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/flan.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/fld.py +143 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench.py +202 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench_mc.py +98 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/french_bench_perplexity.py +86 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galcola.py +109 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench_gen.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/galician_bench_mc.py +112 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gaokao.py +141 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/glianorex.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/global_mmlu.py +171 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/global_piqa.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/glue.py +109 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gpqa.py +161 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gpt3.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/groundcocoa.py +184 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gsm.py +108 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/gsm8k.py +134 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/haerae.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/headqa.py +112 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hellaswag.py +125 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hendrycks_ethics.py +225 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hendrycks_math.py +191 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/histoires_morales.py +179 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hle.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/hrm8k.py +203 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/humaneval.py +124 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/humaneval_infilling.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/icelandic_winogrande.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ifeval.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/inverse.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/inverse_scaling.py +192 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/iwslt2017.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ja.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard_gen.py +224 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/japanese_leaderboard_mc.py +120 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/jsonschema_bench.py +123 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kbl.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu.py +168 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu_cot.py +88 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kmmlu_mc.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kobest.py +165 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/kormedmcqa.py +160 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada.py +147 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_cloze.py +185 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_multilingual.py +185 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lambada_multilingual_stablelm.py +141 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/law.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/leaderboard.py +194 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/libra.py +165 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lingoly.py +203 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/livemathbench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/llama3.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/lm_syneval.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logieval.py +82 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logiqa.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/logiqa2.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/longbench.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/longbenchv2.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mastermind.py +203 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mathqa.py +137 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mbpp.py +123 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mc-taco.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/med_concepts_qa.py +224 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/meddialog.py +180 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medical.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mediqa_qa2019.py +123 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medmcqa.py +169 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medqa.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/medtext.py +108 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mela.py +96 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/meqsum.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/metabench.py +154 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mgsm.py +122 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mimic_repsum.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/minerva_math.py +172 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mlqa.py +143 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu.py +144 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_cot.py +88 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_mc.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlu_pro.py +145 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmlusr.py +189 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mmmu.py +150 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mnli.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/model_written_evals.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/moral_stories.py +151 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mrpc.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mts_dialog.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mts_dialog_perplexity.py +97 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multiblimp.py +134 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multilingual.py +106 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/multirc.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/mutual.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/non.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval.py +173 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_exact.py +157 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_gen.py +277 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_gen_exact.py +165 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc.py +228 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noreval_mc_log_likelihoods.py +223 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/noticia.py +105 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/nq_open.py +135 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi.py +27 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_arc_multilingual.py +167 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_hellaswag_multilingual.py +174 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_mmlu_multilingual.py +162 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/okapi_truthfulqa_multilingual.py +209 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph.py +186 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/olaph_perplexity.py +97 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/openbookqa.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/option.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paloma.py +205 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafraseja.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/parafrases.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/paws_x.py +154 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pawsx.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/persona.py +246 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases.py +144 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/phrases_ca_va.py +82 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile.py +161 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pile_10k.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/piqa.py +116 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/polemo2.py +135 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/polymath.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench_gen.py +121 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/portuguese_bench_mc.py +103 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prompt.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/prost.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/pubmedqa.py +112 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qa4mre.py +119 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qasper_bool.py +112 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnli.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qnlieu.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/qqp.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/quac.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/race.py +124 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/random.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/realtoxicityprompts.py +124 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/record.py +125 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/reversed.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/rte.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/ruler.py +170 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sciq.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/score.py +177 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls.py +161 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/scrolls_mc.py +157 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/self.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue.py +131 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sglue_rte.py +119 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/simple_cooccurrence_bias.py +121 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/siqa.py +209 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/social_iqa.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench_gen.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/spanish_bench_mc.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/squad2.py +129 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/squad_completion.py +121 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sst2.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/storycloze.py +250 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/summarization.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/super.py +107 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/super_glue.py +154 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/superglue.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/supergpqa.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/swag.py +115 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/swde.py +179 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/sycophancy.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/t0.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/teca.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinyarc.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinybenchmarks.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinygsm8k.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinyhellaswag.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinymmlu.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinytruthfulqa.py +113 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tinywinogrande.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/tmmluplus.py +181 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/toxigen.py +91 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/translation.py +149 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/triviaqa.py +130 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa.py +112 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc1.py +120 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_mc2.py +140 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/truthfulqa_multi.py +142 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turblimp_core.py +152 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu.py +161 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu_cot.py +104 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/turkishmmlu_mc.py +102 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/twenty_newsgroups.py +111 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/unitxt.py +131 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/unscramble.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/vaxx.py +95 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/webqs.py +130 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wic.py +122 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wikitext.py +146 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/winogender.py +139 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/winogrande.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmdp.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmt14.py +110 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wmt16.py +118 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wnli.py +114 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wsc.py +117 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/wsc273.py +180 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xcopa.py +197 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xlsum.py +147 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xnli.py +131 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xquad.py +203 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xstorycloze.py +129 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/xwinograd.py +124 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/yahoo.py +108 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_extractors/zhoblimp.py +155 -0
- wisent/core/contrastive_pairs/lm_eval_pairs/lm_task_pairs_generation.py +56 -0
- wisent/core/data_loaders/__init__.py +235 -0
- wisent/core/data_loaders/core/__init__.py +0 -0
- wisent/core/data_loaders/core/atoms.py +99 -0
- wisent/core/data_loaders/loaders/__init__.py +0 -0
- wisent/core/data_loaders/loaders/custom.py +120 -0
- wisent/core/data_loaders/loaders/huggingface_loader.py +153 -0
- wisent/core/data_loaders/loaders/lm_loader.py +494 -0
- wisent/core/data_loaders/loaders/lm_loader_special_cases.py +496 -0
- wisent/core/data_loaders/loaders/task_interface_loader.py +300 -0
- wisent/core/data_loaders/rotator.py +118 -0
- wisent/core/detection_handling.py +259 -0
- wisent/core/diversity_processors.py +193 -0
- wisent/core/download_full_benchmarks.py +1512 -0
- wisent/core/errors/__init__.py +203 -0
- wisent/core/errors/error_codes.py +763 -0
- wisent/core/errors/error_handler.py +134 -0
- wisent/core/evaluators/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/__init__.py +42 -0
- wisent/core/evaluators/benchmark_specific/aime_evaluator.py +90 -0
- wisent/core/evaluators/benchmark_specific/coding/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/metrics/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/metrics/core/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/metrics/core/atoms.py +36 -0
- wisent/core/evaluators/benchmark_specific/coding/metrics/evaluator.py +363 -0
- wisent/core/evaluators/benchmark_specific/coding/metrics/passk.py +67 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/core/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/core/atoms.py +27 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/cpp_sanitizer.py +62 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/java_sanitizer.py +78 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/python_sanitizer.py +94 -0
- wisent/core/evaluators/benchmark_specific/coding/output_sanitizer/utils.py +126 -0
- wisent/core/evaluators/benchmark_specific/coding/providers/__init__.py +18 -0
- wisent/core/evaluators/benchmark_specific/coding/providers/core/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/providers/core/atoms.py +31 -0
- wisent/core/evaluators/benchmark_specific/coding/providers/livecodebench/__init__.py +3 -0
- wisent/core/evaluators/benchmark_specific/coding/providers/livecodebench/provider.py +305 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/Dockerfile +31 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/__init__.py +0 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/atoms.py +105 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/core/runtime.py +143 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/entrypoint.py +121 -0
- wisent/core/evaluators/benchmark_specific/coding/safe_docker/recipes.py +60 -0
- wisent/core/evaluators/benchmark_specific/coding/solution_generator.py +258 -0
- wisent/core/evaluators/benchmark_specific/conala_evaluator.py +332 -0
- wisent/core/evaluators/benchmark_specific/exact_match_evaluator.py +81 -0
- wisent/core/evaluators/benchmark_specific/f1_evaluator.py +173 -0
- wisent/core/evaluators/benchmark_specific/generation_evaluator.py +488 -0
- wisent/core/evaluators/benchmark_specific/livemathbench_evaluator.py +393 -0
- wisent/core/evaluators/benchmark_specific/log_likelihoods_evaluator.py +202 -0
- wisent/core/evaluators/benchmark_specific/math_evaluator.py +119 -0
- wisent/core/evaluators/benchmark_specific/math_parsing/__init__.py +1 -0
- wisent/core/evaluators/benchmark_specific/math_parsing/core.py +1640 -0
- wisent/core/evaluators/benchmark_specific/math_parsing/extract_boxed.py +48 -0
- wisent/core/evaluators/benchmark_specific/math_parsing/is_equiv.py +159 -0
- wisent/core/evaluators/benchmark_specific/math_parsing/scripts.py +919 -0
- wisent/core/evaluators/benchmark_specific/perplexity_evaluator.py +175 -0
- wisent/core/evaluators/benchmark_specific/polymath_evaluator.py +114 -0
- wisent/core/evaluators/core/__init__.py +5 -0
- wisent/core/evaluators/core/atoms.py +166 -0
- wisent/core/evaluators/custom/__init__.py +20 -0
- wisent/core/evaluators/custom/custom_evaluator.py +382 -0
- wisent/core/evaluators/custom/examples/__init__.py +37 -0
- wisent/core/evaluators/custom/examples/desklib_detector.py +166 -0
- wisent/core/evaluators/custom/examples/gptzero.py +185 -0
- wisent/core/evaluators/custom/examples/humanization.py +79 -0
- wisent/core/evaluators/custom/examples/humanization_coherent.py +127 -0
- wisent/core/evaluators/custom/examples/roberta_detector.py +173 -0
- wisent/core/evaluators/oracles/__init__.py +0 -0
- wisent/core/evaluators/oracles/interactive.py +73 -0
- wisent/core/evaluators/oracles/nlp_evaluator.py +440 -0
- wisent/core/evaluators/oracles/truthfulqa_gen_evaluator.py +168 -0
- wisent/core/evaluators/oracles/user_specified.py +67 -0
- wisent/core/evaluators/personalization/__init__.py +12 -0
- wisent/core/evaluators/personalization/alignment.py +166 -0
- wisent/core/evaluators/personalization/coherence.py +325 -0
- wisent/core/evaluators/personalization/difference.py +73 -0
- wisent/core/evaluators/rotator.py +217 -0
- wisent/core/evaluators/steering_evaluators.py +386 -0
- wisent/core/evaluators/synthetic_evaluator.py +377 -0
- wisent/core/hyperparameter_optimizer.py +547 -0
- wisent/core/layer.py +17 -0
- wisent/core/lm_eval_harness_ground_truth.py +1431 -0
- wisent/core/main.py +101 -0
- wisent/core/managed_cached_benchmarks.py +609 -0
- wisent/core/mixed_benchmark_sampler.py +366 -0
- wisent/core/modalities/__init__.py +545 -0
- wisent/core/model_persistence.py +302 -0
- wisent/core/models/__init__.py +23 -0
- wisent/core/models/core/__init__.py +0 -0
- wisent/core/models/core/atoms.py +465 -0
- wisent/core/models/inference_config.py +127 -0
- wisent/core/models/wisent_model.py +893 -0
- wisent/core/multi_steering.py +397 -0
- wisent/core/opti/__init__.py +0 -0
- wisent/core/opti/core/__init__.py +0 -0
- wisent/core/opti/core/atoms.py +177 -0
- wisent/core/opti/methods/__init__.py +10 -0
- wisent/core/opti/methods/opti_classificator.py +172 -0
- wisent/core/opti/methods/opti_steering.py +139 -0
- wisent/core/opti/methods/opti_weights.py +523 -0
- wisent/core/optuna/__init__.py +54 -0
- wisent/core/optuna/classifier/__init__.py +25 -0
- wisent/core/optuna/classifier/activation_generator.py +351 -0
- wisent/core/optuna/classifier/classifier_cache.py +509 -0
- wisent/core/optuna/classifier/optuna_classifier_optimizer.py +685 -0
- wisent/core/optuna/steering/__init__.py +20 -0
- wisent/core/optuna/steering/bigcode_evaluator_wrapper.py +200 -0
- wisent/core/optuna/steering/data_utils.py +342 -0
- wisent/core/optuna/steering/metrics.py +412 -0
- wisent/core/optuna/steering/steering_optimization.py +1096 -0
- wisent/core/parser.py +1662 -0
- wisent/core/parser_arguments/__init__.py +10 -0
- wisent/core/parser_arguments/agent_parser.py +122 -0
- wisent/core/parser_arguments/check_linearity_parser.py +82 -0
- wisent/core/parser_arguments/configure_model_parser.py +7 -0
- wisent/core/parser_arguments/create_steering_vector_parser.py +67 -0
- wisent/core/parser_arguments/diagnose_pairs_parser.py +25 -0
- wisent/core/parser_arguments/diagnose_vectors_parser.py +72 -0
- wisent/core/parser_arguments/evaluate_parser.py +40 -0
- wisent/core/parser_arguments/evaluate_refusal_parser.py +32 -0
- wisent/core/parser_arguments/evaluate_responses_parser.py +12 -0
- wisent/core/parser_arguments/full_optimize_parser.py +194 -0
- wisent/core/parser_arguments/generate_pairs_from_task_parser.py +33 -0
- wisent/core/parser_arguments/generate_pairs_parser.py +43 -0
- wisent/core/parser_arguments/generate_responses_parser.py +16 -0
- wisent/core/parser_arguments/generate_vector_from_synthetic_parser.py +148 -0
- wisent/core/parser_arguments/generate_vector_from_task_parser.py +149 -0
- wisent/core/parser_arguments/generate_vector_parser.py +89 -0
- wisent/core/parser_arguments/get_activations_parser.py +90 -0
- wisent/core/parser_arguments/inference_config_parser.py +65 -0
- wisent/core/parser_arguments/main_parser.py +220 -0
- wisent/core/parser_arguments/model_config_parser.py +59 -0
- wisent/core/parser_arguments/modify_weights_parser.py +309 -0
- wisent/core/parser_arguments/monitor_parser.py +17 -0
- wisent/core/parser_arguments/multi_steer_parser.py +48 -0
- wisent/core/parser_arguments/nonsense_parser.py +26 -0
- wisent/core/parser_arguments/optimization_cache_parser.py +64 -0
- wisent/core/parser_arguments/optimize_classification_parser.py +108 -0
- wisent/core/parser_arguments/optimize_parser.py +142 -0
- wisent/core/parser_arguments/optimize_sample_size_parser.py +58 -0
- wisent/core/parser_arguments/optimize_steering_parser.py +617 -0
- wisent/core/parser_arguments/optimize_weights_parser.py +403 -0
- wisent/core/parser_arguments/synthetic_parser.py +117 -0
- wisent/core/parser_arguments/tasks_parser.py +591 -0
- wisent/core/parser_arguments/train_unified_goodness_parser.py +172 -0
- wisent/core/parser_arguments/utils.py +107 -0
- wisent/core/prompts/__init__.py +0 -0
- wisent/core/prompts/core/__init__.py +0 -0
- wisent/core/prompts/core/atom.py +57 -0
- wisent/core/prompts/core/prompt_formater.py +148 -0
- wisent/core/prompts/prompt_stratiegies/__init__.py +0 -0
- wisent/core/prompts/prompt_stratiegies/direct_completion.py +26 -0
- wisent/core/prompts/prompt_stratiegies/instruction_following.py +26 -0
- wisent/core/prompts/prompt_stratiegies/multiple_choice.py +31 -0
- wisent/core/prompts/prompt_stratiegies/role_playing.py +33 -0
- wisent/core/representation.py +5 -0
- wisent/core/save_results.py +277 -0
- wisent/core/steering.py +660 -0
- wisent/core/steering_method.py +20 -0
- wisent/core/steering_methods/__init__.py +54 -0
- wisent/core/steering_methods/core/__init__.py +0 -0
- wisent/core/steering_methods/core/atoms.py +154 -0
- wisent/core/steering_methods/methods/__init__.py +0 -0
- wisent/core/steering_methods/methods/caa.py +45 -0
- wisent/core/steering_methods/methods/prism.py +588 -0
- wisent/core/steering_methods/methods/pulse.py +641 -0
- wisent/core/steering_methods/methods/titan.py +1005 -0
- wisent/core/steering_methods/preflight.py +322 -0
- wisent/core/steering_methods/registry.py +649 -0
- wisent/core/steering_methods/rotator.py +121 -0
- wisent/core/steering_optimizer.py +1503 -0
- wisent/core/synthetic/__init__.py +0 -0
- wisent/core/synthetic/cleaners/__init__.py +0 -0
- wisent/core/synthetic/cleaners/core/__init__.py +0 -0
- wisent/core/synthetic/cleaners/core/atoms.py +58 -0
- wisent/core/synthetic/cleaners/deduper_cleaner.py +53 -0
- wisent/core/synthetic/cleaners/methods/__init__.py +0 -0
- wisent/core/synthetic/cleaners/methods/base_dedupers.py +321 -0
- wisent/core/synthetic/cleaners/methods/base_refusalers.py +286 -0
- wisent/core/synthetic/cleaners/methods/core/__init__.py +0 -0
- wisent/core/synthetic/cleaners/methods/core/atoms.py +47 -0
- wisent/core/synthetic/cleaners/pairs_cleaner.py +90 -0
- wisent/core/synthetic/cleaners/refusaler_cleaner.py +133 -0
- wisent/core/synthetic/db_instructions/__init__.py +0 -0
- wisent/core/synthetic/db_instructions/core/__init__.py +0 -0
- wisent/core/synthetic/db_instructions/core/atoms.py +25 -0
- wisent/core/synthetic/db_instructions/mini_dp.py +115 -0
- wisent/core/synthetic/generators/__init__.py +0 -0
- wisent/core/synthetic/generators/core/__init__.py +0 -0
- wisent/core/synthetic/generators/core/atoms.py +73 -0
- wisent/core/synthetic/generators/diversities/__init__.py +0 -0
- wisent/core/synthetic/generators/diversities/core/__init__.py +0 -0
- wisent/core/synthetic/generators/diversities/core/core.py +68 -0
- wisent/core/synthetic/generators/diversities/methods/__init__.py +0 -0
- wisent/core/synthetic/generators/diversities/methods/fast_diversity.py +249 -0
- wisent/core/synthetic/generators/nonsense_generator.py +150 -0
- wisent/core/synthetic/generators/pairs_generator.py +313 -0
- wisent/core/task_interface.py +143 -0
- wisent/core/task_selector.py +232 -0
- wisent/core/tasks/__init__.py +218 -0
- wisent/core/tasks/aime_task.py +142 -0
- wisent/core/tasks/file_task.py +212 -0
- wisent/core/tasks/hle_task.py +180 -0
- wisent/core/tasks/hmmt_task.py +120 -0
- wisent/core/tasks/livecodebench_task.py +94 -0
- wisent/core/tasks/livemathbench_task.py +159 -0
- wisent/core/tasks/lm_eval_task.py +611 -0
- wisent/core/tasks/math500_task.py +84 -0
- wisent/core/tasks/polymath_task.py +147 -0
- wisent/core/tasks/supergpqa_task.py +220 -0
- wisent/core/time_estimator.py +155 -0
- wisent/core/timing_calibration.py +176 -0
- wisent/core/tracking/__init__.py +54 -0
- wisent/core/tracking/latency.py +620 -0
- wisent/core/tracking/memory.py +360 -0
- wisent/core/trainers/__init__.py +0 -0
- wisent/core/trainers/core/__init__.py +11 -0
- wisent/core/trainers/core/atoms.py +45 -0
- wisent/core/trainers/steering_trainer.py +365 -0
- wisent/core/universal_subspace.py +918 -0
- wisent/core/user_model_config.py +158 -0
- wisent/core/utils/__init__.py +64 -0
- wisent/core/utils/base_rotator.py +292 -0
- wisent/core/utils/dataset_splits.py +197 -0
- wisent/core/utils/device.py +279 -0
- wisent/core/weight_modification/__init__.py +134 -0
- wisent/core/weight_modification/additive.py +340 -0
- wisent/core/weight_modification/directional.py +1357 -0
- wisent/core/weight_modification/export.py +359 -0
- wisent/core/weight_modification/multi_direction.py +410 -0
- wisent/core/weight_modification/utils.py +236 -0
- wisent/core/wisent.py +660 -0
- wisent/examples/contrastive_pairs/humanization_human_vs_ai.json +2112 -0
- wisent/examples/scripts/1/test_basqueglue_evaluation.json +51 -0
- wisent/examples/scripts/1/test_basqueglue_pairs.json +14 -0
- wisent/examples/scripts/1/test_bec2016eu_evaluation.json +51 -0
- wisent/examples/scripts/1/test_bec2016eu_pairs.json +14 -0
- wisent/examples/scripts/1/test_belebele_evaluation.json +51 -0
- wisent/examples/scripts/1/test_belebele_pairs.json +14 -0
- wisent/examples/scripts/1/test_benchmarks_evaluation.json +51 -0
- wisent/examples/scripts/1/test_benchmarks_pairs.json +14 -0
- wisent/examples/scripts/1/test_bertaqa_evaluation.json +51 -0
- wisent/examples/scripts/1/test_bertaqa_pairs.json +14 -0
- wisent/examples/scripts/1/test_bhtc_v2_evaluation.json +30 -0
- wisent/examples/scripts/1/test_bhtc_v2_pairs.json +8 -0
- wisent/examples/scripts/1/test_boolq-seq2seq_evaluation.json +30 -0
- wisent/examples/scripts/1/test_boolq-seq2seq_pairs.json +8 -0
- wisent/examples/scripts/1/test_cabreu_evaluation.json +30 -0
- wisent/examples/scripts/1/test_cabreu_pairs.json +8 -0
- wisent/examples/scripts/1/test_careqa_en_evaluation.json +30 -0
- wisent/examples/scripts/1/test_careqa_en_pairs.json +8 -0
- wisent/examples/scripts/1/test_careqa_evaluation.json +30 -0
- wisent/examples/scripts/1/test_careqa_pairs.json +8 -0
- wisent/examples/scripts/1/test_catalanqa_evaluation.json +30 -0
- wisent/examples/scripts/1/test_catalanqa_pairs.json +8 -0
- wisent/examples/scripts/1/test_catcola_evaluation.json +30 -0
- wisent/examples/scripts/1/test_catcola_pairs.json +8 -0
- wisent/examples/scripts/1/test_chartqa_evaluation.json +30 -0
- wisent/examples/scripts/1/test_chartqa_pairs.json +8 -0
- wisent/examples/scripts/1/test_claim_stance_topic_evaluation.json +30 -0
- wisent/examples/scripts/1/test_claim_stance_topic_pairs.json +8 -0
- wisent/examples/scripts/1/test_cnn_dailymail_evaluation.json +30 -0
- wisent/examples/scripts/1/test_cnn_dailymail_pairs.json +8 -0
- wisent/examples/scripts/1/test_cocoteros_es_evaluation.json +30 -0
- wisent/examples/scripts/1/test_cocoteros_es_pairs.json +8 -0
- wisent/examples/scripts/1/test_coedit_gec_evaluation.json +30 -0
- wisent/examples/scripts/1/test_coedit_gec_pairs.json +8 -0
- wisent/examples/scripts/1/test_cola_evaluation.json +30 -0
- wisent/examples/scripts/1/test_cola_pairs.json +8 -0
- wisent/examples/scripts/1/test_coqcat_evaluation.json +30 -0
- wisent/examples/scripts/1/test_coqcat_pairs.json +8 -0
- wisent/examples/scripts/1/test_dbpedia_14_evaluation.json +30 -0
- wisent/examples/scripts/1/test_dbpedia_14_pairs.json +8 -0
- wisent/examples/scripts/1/test_epec_koref_bin_evaluation.json +30 -0
- wisent/examples/scripts/1/test_epec_koref_bin_pairs.json +8 -0
- wisent/examples/scripts/1/test_ethos_binary_evaluation.json +30 -0
- wisent/examples/scripts/1/test_ethos_binary_pairs.json +8 -0
- wisent/examples/scripts/2/test_afrimgsm_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/2/test_afrimgsm_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/2/test_afrimmlu_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/2/test_afrimmlu_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/2/test_afrixnli_en_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/2/test_afrixnli_en_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/2/test_arc_ar_evaluation.json +30 -0
- wisent/examples/scripts/2/test_arc_ar_pairs.json +8 -0
- wisent/examples/scripts/2/test_atis_evaluation.json +30 -0
- wisent/examples/scripts/2/test_atis_pairs.json +8 -0
- wisent/examples/scripts/2/test_babi_evaluation.json +30 -0
- wisent/examples/scripts/2/test_babi_pairs.json +8 -0
- wisent/examples/scripts/2/test_babilong_evaluation.json +30 -0
- wisent/examples/scripts/2/test_babilong_pairs.json +8 -0
- wisent/examples/scripts/2/test_bangla_mmlu_evaluation.json +30 -0
- wisent/examples/scripts/2/test_bangla_mmlu_pairs.json +8 -0
- wisent/examples/scripts/2/test_basque-glue_pairs.json +14 -0
- wisent/examples/scripts/benchmark_tags.json +2140 -0
- wisent/examples/scripts/lm_eval_readme.json +4 -0
- wisent/examples/scripts/results/benchmark_descriptions.json +1244 -0
- wisent/examples/scripts/results/benchmark_evaluation_methods.json +66 -0
- wisent/examples/scripts/results/benchmark_evaluator_mapping.json +2781 -0
- wisent/examples/scripts/results/benchmark_evaluator_mapping_updated.json +30536 -0
- wisent/examples/scripts/results/benchmark_evaluators_clean.json +469 -0
- wisent/examples/scripts/results/benchmark_methods_summary.json +260 -0
- wisent/examples/scripts/results/benchmark_pair_creation_methods.json +66 -0
- wisent/examples/scripts/results/benchmark_pair_totals.json +269 -0
- wisent/examples/scripts/results/benchmark_tags.json +917 -0
- wisent/examples/scripts/results/benchmark_test_summary_nov4.json +71 -0
- wisent/examples/scripts/results/coding_benchmarks_test_code_status.json +150 -0
- wisent/examples/scripts/results/failing_benchmarks.json +946 -0
- wisent/examples/scripts/results/failing_benchmarks_list.json +41 -0
- wisent/examples/scripts/results/failing_benchmarks_test_results.json +945 -0
- wisent/examples/scripts/results/missing_benchmark_tags.json +341 -0
- wisent/examples/scripts/results/test_20_newsgroups_evaluation.json +30 -0
- wisent/examples/scripts/results/test_20_newsgroups_pairs.json +8 -0
- wisent/examples/scripts/results/test_AraDICE_evaluation.json +51 -0
- wisent/examples/scripts/results/test_AraDICE_pairs.json +14 -0
- wisent/examples/scripts/results/test_AraDiCE_boolq_egy/test_AraDiCE_boolq_egy_evaluation.json +30 -0
- wisent/examples/scripts/results/test_AraDiCE_boolq_egy/test_AraDiCE_boolq_egy_pairs.json +8 -0
- wisent/examples/scripts/results/test_ArabCulture_evaluation.json +51 -0
- wisent/examples/scripts/results/test_ArabCulture_pairs.json +14 -0
- wisent/examples/scripts/results/test_Tag_evaluation.json +30 -0
- wisent/examples/scripts/results/test_Tag_pairs.json +8 -0
- wisent/examples/scripts/results/test_aclue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_aclue_pairs.json +14 -0
- wisent/examples/scripts/results/test_acp_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_acp_bench_hard_evaluation.json +51 -0
- wisent/examples/scripts/results/test_acp_bench_hard_pairs.json +14 -0
- wisent/examples/scripts/results/test_acp_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_advanced_ai_risk_evaluation.json +51 -0
- wisent/examples/scripts/results/test_advanced_ai_risk_pairs.json +14 -0
- wisent/examples/scripts/results/test_aexams_evaluation.json +51 -0
- wisent/examples/scripts/results/test_aexams_pairs.json +14 -0
- wisent/examples/scripts/results/test_afrimgsm_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/results/test_afrimgsm_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/results/test_afrimmlu_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/results/test_afrimmlu_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/results/test_afrixnli_en_direct_amh_evaluation.json +30 -0
- wisent/examples/scripts/results/test_afrixnli_en_direct_amh_pairs.json +8 -0
- wisent/examples/scripts/results/test_ag_news_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ag_news_pairs.json +8 -0
- wisent/examples/scripts/results/test_agieval_evaluation.json +51 -0
- wisent/examples/scripts/results/test_agieval_pairs.json +14 -0
- wisent/examples/scripts/results/test_aime2024_evaluation.json +30 -0
- wisent/examples/scripts/results/test_aime2024_pairs.json +8 -0
- wisent/examples/scripts/results/test_aime2025_evaluation.json +30 -0
- wisent/examples/scripts/results/test_aime2025_pairs.json +8 -0
- wisent/examples/scripts/results/test_aime_evaluation.json +30 -0
- wisent/examples/scripts/results/test_aime_pairs.json +8 -0
- wisent/examples/scripts/results/test_anagrams1_evaluation.json +30 -0
- wisent/examples/scripts/results/test_anagrams1_pairs.json +8 -0
- wisent/examples/scripts/results/test_anagrams2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_anagrams2_pairs.json +8 -0
- wisent/examples/scripts/results/test_anli_evaluation.json +30 -0
- wisent/examples/scripts/results/test_anli_pairs.json +8 -0
- wisent/examples/scripts/results/test_apps_evaluation.json +30 -0
- wisent/examples/scripts/results/test_apps_pairs.json +8 -0
- wisent/examples/scripts/results/test_arabic_exams_evaluation.json +30 -0
- wisent/examples/scripts/results/test_arabic_exams_pairs.json +8 -0
- wisent/examples/scripts/results/test_arabic_leaderboard_complete_evaluation.json +51 -0
- wisent/examples/scripts/results/test_arabic_leaderboard_complete_pairs.json +14 -0
- wisent/examples/scripts/results/test_arabic_leaderboard_light_evaluation.json +51 -0
- wisent/examples/scripts/results/test_arabic_leaderboard_light_pairs.json +14 -0
- wisent/examples/scripts/results/test_arabicmmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_arabicmmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_aradice/test_aradice_evaluation.json +51 -0
- wisent/examples/scripts/results/test_aradice/test_aradice_pairs.json +14 -0
- wisent/examples/scripts/results/test_aradice3/test_aradice_evaluation.json +51 -0
- wisent/examples/scripts/results/test_aradice3/test_aradice_pairs.json +14 -0
- wisent/examples/scripts/results/test_arc_ar_evaluation.json +30 -0
- wisent/examples/scripts/results/test_arc_ar_pairs.json +8 -0
- wisent/examples/scripts/results/test_arc_challenge_evaluation.json +30 -0
- wisent/examples/scripts/results/test_arc_challenge_pairs.json +8 -0
- wisent/examples/scripts/results/test_arc_easy_evaluation.json +30 -0
- wisent/examples/scripts/results/test_arc_easy_pairs.json +8 -0
- wisent/examples/scripts/results/test_argument_topic_evaluation.json +30 -0
- wisent/examples/scripts/results/test_argument_topic_pairs.json +8 -0
- wisent/examples/scripts/results/test_arithmetic_evaluation.json +51 -0
- wisent/examples/scripts/results/test_arithmetic_pairs.json +14 -0
- wisent/examples/scripts/results/test_asdiv_evaluation.json +30 -0
- wisent/examples/scripts/results/test_asdiv_pairs.json +8 -0
- wisent/examples/scripts/results/test_assin_entailment_evaluation.json +30 -0
- wisent/examples/scripts/results/test_assin_entailment_pairs.json +8 -0
- wisent/examples/scripts/results/test_atis_evaluation.json +30 -0
- wisent/examples/scripts/results/test_atis_pairs.json +8 -0
- wisent/examples/scripts/results/test_babi_evaluation.json +30 -0
- wisent/examples/scripts/results/test_babi_pairs.json +8 -0
- wisent/examples/scripts/results/test_babilong_evaluation.json +30 -0
- wisent/examples/scripts/results/test_babilong_pairs.json +8 -0
- wisent/examples/scripts/results/test_bangla_mmlu_evaluation.json +30 -0
- wisent/examples/scripts/results/test_bangla_mmlu_pairs.json +8 -0
- wisent/examples/scripts/results/test_banking77_evaluation.json +30 -0
- wisent/examples/scripts/results/test_banking77_pairs.json +8 -0
- wisent/examples/scripts/results/test_basque/test_basque-glue_pairs.json +14 -0
- wisent/examples/scripts/results/test_basque-glue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_basque-glue_pairs.json +14 -0
- wisent/examples/scripts/results/test_basque2/test_basque-glue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_basque2/test_basque-glue_pairs.json +14 -0
- wisent/examples/scripts/results/test_basque_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_basque_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_basque_glue/test_basque-glue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_basque_glue/test_basque-glue_pairs.json +14 -0
- wisent/examples/scripts/results/test_basqueglue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_basqueglue_pairs.json +14 -0
- wisent/examples/scripts/results/test_bbh_evaluation.json +51 -0
- wisent/examples/scripts/results/test_bbh_pairs.json +14 -0
- wisent/examples/scripts/results/test_bbq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_bbq_pairs.json +8 -0
- wisent/examples/scripts/results/test_bec2016eu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_bec2016eu_pairs.json +14 -0
- wisent/examples/scripts/results/test_belebele_evaluation.json +51 -0
- wisent/examples/scripts/results/test_belebele_pairs.json +14 -0
- wisent/examples/scripts/results/test_benchmarks_evaluation.json +51 -0
- wisent/examples/scripts/results/test_benchmarks_pairs.json +14 -0
- wisent/examples/scripts/results/test_bertaqa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_bertaqa_pairs.json +14 -0
- wisent/examples/scripts/results/test_bhtc_v2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_bhtc_v2_pairs.json +8 -0
- wisent/examples/scripts/results/test_bigbench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_bigbench_pairs.json +14 -0
- wisent/examples/scripts/results/test_blimp_evaluation.json +51 -0
- wisent/examples/scripts/results/test_blimp_pairs.json +14 -0
- wisent/examples/scripts/results/test_boolq/test_boolq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_boolq/test_boolq_pairs.json +8 -0
- wisent/examples/scripts/results/test_boolq-seq2seq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_boolq-seq2seq_pairs.json +8 -0
- wisent/examples/scripts/results/test_boolq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_boolq_pairs.json +8 -0
- wisent/examples/scripts/results/test_c4_evaluation.json +30 -0
- wisent/examples/scripts/results/test_c4_pairs.json +8 -0
- wisent/examples/scripts/results/test_cabreu_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cabreu_pairs.json +8 -0
- wisent/examples/scripts/results/test_careqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_careqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_catalan_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_catalan_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_catalanqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_catalanqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_catcola_evaluation.json +30 -0
- wisent/examples/scripts/results/test_catcola_pairs.json +8 -0
- wisent/examples/scripts/results/test_cb_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cb_pairs.json +8 -0
- wisent/examples/scripts/results/test_ceval/test_ceval_evaluation.json +51 -0
- wisent/examples/scripts/results/test_ceval/test_ceval_pairs.json +14 -0
- wisent/examples/scripts/results/test_ceval_accountant/test_ceval-valid_accountant_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ceval_accountant/test_ceval-valid_accountant_pairs.json +8 -0
- wisent/examples/scripts/results/test_ceval_evaluation.json +51 -0
- wisent/examples/scripts/results/test_ceval_pairs.json +14 -0
- wisent/examples/scripts/results/test_ceval_valid/test_ceval_valid_evaluation.json +51 -0
- wisent/examples/scripts/results/test_ceval_valid/test_ceval_valid_pairs.json +14 -0
- wisent/examples/scripts/results/test_chain_of_thought_evaluation.json +51 -0
- wisent/examples/scripts/results/test_chain_of_thought_pairs.json +14 -0
- wisent/examples/scripts/results/test_chartqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_chartqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_claim_stance_topic_evaluation.json +30 -0
- wisent/examples/scripts/results/test_claim_stance_topic_pairs.json +8 -0
- wisent/examples/scripts/results/test_cmmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_cmmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_cnn_dailymail_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cnn_dailymail_pairs.json +8 -0
- wisent/examples/scripts/results/test_cocoteros_es_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cocoteros_es_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_go_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_go_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_java_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_java_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_javascript_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_javascript_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_php_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_php_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_python_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_python_pairs.json +8 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_ruby_evaluation.json +30 -0
- wisent/examples/scripts/results/test_codexglue_code_to_text_ruby_pairs.json +8 -0
- wisent/examples/scripts/results/test_coedit_gec_evaluation.json +30 -0
- wisent/examples/scripts/results/test_coedit_gec_pairs.json +8 -0
- wisent/examples/scripts/results/test_cola_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cola_pairs.json +8 -0
- wisent/examples/scripts/results/test_commonsense_qa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_commonsense_qa_pairs.json +8 -0
- wisent/examples/scripts/results/test_conala_evaluation.json +30 -0
- wisent/examples/scripts/results/test_conala_pairs.json +8 -0
- wisent/examples/scripts/results/test_concode_evaluation.json +30 -0
- wisent/examples/scripts/results/test_concode_pairs.json +8 -0
- wisent/examples/scripts/results/test_copa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_copa_pairs.json +8 -0
- wisent/examples/scripts/results/test_copal_id_evaluation.json +30 -0
- wisent/examples/scripts/results/test_copal_id_pairs.json +8 -0
- wisent/examples/scripts/results/test_coqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_coqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_coqcat_evaluation.json +30 -0
- wisent/examples/scripts/results/test_coqcat_pairs.json +8 -0
- wisent/examples/scripts/results/test_crows_pairs_evaluation.json +51 -0
- wisent/examples/scripts/results/test_crows_pairs_pairs.json +14 -0
- wisent/examples/scripts/results/test_csatqa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_csatqa_pairs.json +14 -0
- wisent/examples/scripts/results/test_cycle_letters_evaluation.json +30 -0
- wisent/examples/scripts/results/test_cycle_letters_pairs.json +8 -0
- wisent/examples/scripts/results/test_darija_bench/test_darija_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_darija_bench/test_darija_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_darija_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_darija_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_darijahellaswag_evaluation.json +30 -0
- wisent/examples/scripts/results/test_darijahellaswag_pairs.json +8 -0
- wisent/examples/scripts/results/test_darijammlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_darijammlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_dbpedia_14_evaluation.json +30 -0
- wisent/examples/scripts/results/test_dbpedia_14_pairs.json +8 -0
- wisent/examples/scripts/results/test_drop_evaluation.json +30 -0
- wisent/examples/scripts/results/test_drop_pairs.json +8 -0
- wisent/examples/scripts/results/test_ds1000_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ds1000_pairs.json +8 -0
- wisent/examples/scripts/results/test_egyhellaswag_evaluation.json +30 -0
- wisent/examples/scripts/results/test_egyhellaswag_pairs.json +8 -0
- wisent/examples/scripts/results/test_egymmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_egymmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_epec_koref_bin_evaluation.json +30 -0
- wisent/examples/scripts/results/test_epec_koref_bin_pairs.json +8 -0
- wisent/examples/scripts/results/test_eq_bench_evaluation.json +30 -0
- wisent/examples/scripts/results/test_eq_bench_pairs.json +8 -0
- wisent/examples/scripts/results/test_escola_evaluation.json +30 -0
- wisent/examples/scripts/results/test_escola_pairs.json +8 -0
- wisent/examples/scripts/results/test_ethics_cm_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ethics_cm_pairs.json +8 -0
- wisent/examples/scripts/results/test_ethos_binary_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ethos_binary_pairs.json +8 -0
- wisent/examples/scripts/results/test_eus_exams/test_eus_exams_evaluation.json +51 -0
- wisent/examples/scripts/results/test_eus_exams/test_eus_exams_pairs.json +14 -0
- wisent/examples/scripts/results/test_eus_exams_es_evaluation.json +51 -0
- wisent/examples/scripts/results/test_eus_exams_es_pairs.json +14 -0
- wisent/examples/scripts/results/test_eus_exams_evaluation.json +51 -0
- wisent/examples/scripts/results/test_eus_exams_pairs.json +14 -0
- wisent/examples/scripts/results/test_eus_proficiency_evaluation.json +30 -0
- wisent/examples/scripts/results/test_eus_proficiency_pairs.json +8 -0
- wisent/examples/scripts/results/test_eus_reading_evaluation.json +30 -0
- wisent/examples/scripts/results/test_eus_reading_pairs.json +8 -0
- wisent/examples/scripts/results/test_eus_trivia_evaluation.json +30 -0
- wisent/examples/scripts/results/test_eus_trivia_pairs.json +8 -0
- wisent/examples/scripts/results/test_evalita-mp_evaluation.json +51 -0
- wisent/examples/scripts/results/test_evalita-mp_pairs.json +14 -0
- wisent/examples/scripts/results/test_evalita-sp_sum_task_fp-small_p1_evaluation.json +30 -0
- wisent/examples/scripts/results/test_evalita-sp_sum_task_fp-small_p1_pairs.json +8 -0
- wisent/examples/scripts/results/test_evalita_LLM_evaluation.json +51 -0
- wisent/examples/scripts/results/test_evalita_LLM_pairs.json +14 -0
- wisent/examples/scripts/results/test_evalita_llm/test_evalita_llm_evaluation.json +51 -0
- wisent/examples/scripts/results/test_evalita_llm/test_evalita_llm_pairs.json +14 -0
- wisent/examples/scripts/results/test_evalita_mp/test_evalita-mp_te_prompt-1_evaluation.json +30 -0
- wisent/examples/scripts/results/test_evalita_mp/test_evalita-mp_te_prompt-1_pairs.json +8 -0
- wisent/examples/scripts/results/test_evalita_mp2/test_evalita_mp_evaluation.json +51 -0
- wisent/examples/scripts/results/test_evalita_mp2/test_evalita_mp_pairs.json +14 -0
- wisent/examples/scripts/results/test_evalita_sp2/test_evalita-sp_sum_task_fp-small_p1_evaluation.json +30 -0
- wisent/examples/scripts/results/test_evalita_sp2/test_evalita-sp_sum_task_fp-small_p1_pairs.json +8 -0
- wisent/examples/scripts/results/test_fda_evaluation.json +30 -0
- wisent/examples/scripts/results/test_fda_pairs.json +8 -0
- wisent/examples/scripts/results/test_financial_tweets_evaluation.json +30 -0
- wisent/examples/scripts/results/test_financial_tweets_pairs.json +8 -0
- wisent/examples/scripts/results/test_fld/test_fld_evaluation.json +30 -0
- wisent/examples/scripts/results/test_fld/test_fld_pairs.json +8 -0
- wisent/examples/scripts/results/test_fld_evaluation.json +30 -0
- wisent/examples/scripts/results/test_fld_fixed/test_fld_evaluation.json +30 -0
- wisent/examples/scripts/results/test_fld_fixed/test_fld_pairs.json +8 -0
- wisent/examples/scripts/results/test_fld_pairs.json +8 -0
- wisent/examples/scripts/results/test_flores_evaluation.json +51 -0
- wisent/examples/scripts/results/test_flores_pairs.json +14 -0
- wisent/examples/scripts/results/test_freebase_evaluation.json +30 -0
- wisent/examples/scripts/results/test_freebase_pairs.json +8 -0
- wisent/examples/scripts/results/test_french_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_french_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_galcola_evaluation.json +30 -0
- wisent/examples/scripts/results/test_galcola_pairs.json +8 -0
- wisent/examples/scripts/results/test_galician_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_galician_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_glianorex_evaluation.json +30 -0
- wisent/examples/scripts/results/test_glianorex_pairs.json +8 -0
- wisent/examples/scripts/results/test_global_mmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_global_mmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_glue_evaluation.json +51 -0
- wisent/examples/scripts/results/test_glue_pairs.json +14 -0
- wisent/examples/scripts/results/test_gpqa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_gpqa_pairs.json +14 -0
- wisent/examples/scripts/results/test_gpt3_translation_benchmarks_evaluation.json +51 -0
- wisent/examples/scripts/results/test_gpt3_translation_benchmarks_pairs.json +14 -0
- wisent/examples/scripts/results/test_groundcocoa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_groundcocoa_pairs.json +8 -0
- wisent/examples/scripts/results/test_gsm8k_evaluation.json +30 -0
- wisent/examples/scripts/results/test_gsm8k_pairs.json +8 -0
- wisent/examples/scripts/results/test_haerae_evaluation.json +51 -0
- wisent/examples/scripts/results/test_haerae_pairs.json +14 -0
- wisent/examples/scripts/results/test_headqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_headqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_hellaswag_evaluation.json +30 -0
- wisent/examples/scripts/results/test_hellaswag_pairs.json +8 -0
- wisent/examples/scripts/results/test_hendrycks_ethics_evaluation.json +51 -0
- wisent/examples/scripts/results/test_hendrycks_ethics_pairs.json +14 -0
- wisent/examples/scripts/results/test_hendrycks_math_evaluation.json +51 -0
- wisent/examples/scripts/results/test_hendrycks_math_pairs.json +14 -0
- wisent/examples/scripts/results/test_histoires_morales_evaluation.json +30 -0
- wisent/examples/scripts/results/test_histoires_morales_pairs.json +8 -0
- wisent/examples/scripts/results/test_hmmt_evaluation.json +30 -0
- wisent/examples/scripts/results/test_hmmt_feb_2025_evaluation.json +30 -0
- wisent/examples/scripts/results/test_hmmt_feb_2025_pairs.json +8 -0
- wisent/examples/scripts/results/test_hmmt_pairs.json +8 -0
- wisent/examples/scripts/results/test_hrm8k_evaluation.json +51 -0
- wisent/examples/scripts/results/test_hrm8k_pairs.json +14 -0
- wisent/examples/scripts/results/test_humaneval_evaluation.json +30 -0
- wisent/examples/scripts/results/test_humaneval_pairs.json +8 -0
- wisent/examples/scripts/results/test_humaneval_plus_evaluation.json +30 -0
- wisent/examples/scripts/results/test_humaneval_plus_pairs.json +8 -0
- wisent/examples/scripts/results/test_ifeval_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ifeval_pairs.json +8 -0
- wisent/examples/scripts/results/test_instruct_humaneval/test_instruct_humaneval_evaluation.json +30 -0
- wisent/examples/scripts/results/test_instruct_humaneval/test_instruct_humaneval_pairs.json +8 -0
- wisent/examples/scripts/results/test_instruct_humaneval_evaluation.json +30 -0
- wisent/examples/scripts/results/test_instruct_humaneval_pairs.json +8 -0
- wisent/examples/scripts/results/test_inverse_scaling_evaluation.json +51 -0
- wisent/examples/scripts/results/test_inverse_scaling_hindsight_neglect_10shot_evaluation.json +30 -0
- wisent/examples/scripts/results/test_inverse_scaling_hindsight_neglect_10shot_pairs.json +8 -0
- wisent/examples/scripts/results/test_inverse_scaling_mc/test_inverse_scaling_mc_evaluation.json +51 -0
- wisent/examples/scripts/results/test_inverse_scaling_mc/test_inverse_scaling_mc_pairs.json +14 -0
- wisent/examples/scripts/results/test_inverse_scaling_pairs.json +14 -0
- wisent/examples/scripts/results/test_iwslt2017-ar-en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_iwslt2017-ar-en_pairs.json +8 -0
- wisent/examples/scripts/results/test_iwslt2017-en-ar_evaluation.json +30 -0
- wisent/examples/scripts/results/test_iwslt2017-en-ar_pairs.json +8 -0
- wisent/examples/scripts/results/test_iwslt2017_ar_en/test_iwslt2017-ar-en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_iwslt2017_ar_en/test_iwslt2017-ar-en_pairs.json +8 -0
- wisent/examples/scripts/results/test_iwslt2017_en_ar/test_iwslt2017-en-ar_evaluation.json +30 -0
- wisent/examples/scripts/results/test_iwslt2017_en_ar/test_iwslt2017-en-ar_pairs.json +8 -0
- wisent/examples/scripts/results/test_iwslt2017_group/test_iwslt2017_evaluation.json +30 -0
- wisent/examples/scripts/results/test_iwslt2017_group/test_iwslt2017_pairs.json +8 -0
- wisent/examples/scripts/results/test_japanese_leaderboard_evaluation.json +51 -0
- wisent/examples/scripts/results/test_japanese_leaderboard_pairs.json +14 -0
- wisent/examples/scripts/results/test_jsonschema_bench/test_jsonschema_bench_evaluation.json +30 -0
- wisent/examples/scripts/results/test_jsonschema_bench/test_jsonschema_bench_pairs.json +8 -0
- wisent/examples/scripts/results/test_jsonschema_bench_evaluation.json +30 -0
- wisent/examples/scripts/results/test_jsonschema_bench_final/test_jsonschema_bench_evaluation.json +30 -0
- wisent/examples/scripts/results/test_jsonschema_bench_final/test_jsonschema_bench_pairs.json +8 -0
- wisent/examples/scripts/results/test_jsonschema_bench_pairs.json +8 -0
- wisent/examples/scripts/results/test_kbl_evaluation.json +51 -0
- wisent/examples/scripts/results/test_kbl_fixed/test_kbl_evaluation.json +51 -0
- wisent/examples/scripts/results/test_kbl_fixed/test_kbl_pairs.json +14 -0
- wisent/examples/scripts/results/test_kbl_pairs.json +14 -0
- wisent/examples/scripts/results/test_kmmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_kmmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_kobest_evaluation.json +51 -0
- wisent/examples/scripts/results/test_kobest_pairs.json +14 -0
- wisent/examples/scripts/results/test_kormedmcqa/test_kormedmcqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_kormedmcqa/test_kormedmcqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_kormedmcqa_dentist/test_kormedmcqa_dentist_evaluation.json +30 -0
- wisent/examples/scripts/results/test_kormedmcqa_dentist/test_kormedmcqa_dentist_pairs.json +8 -0
- wisent/examples/scripts/results/test_kormedmcqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_kormedmcqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_cloze_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_cloze_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_final/test_lambada_openai_mt_stablelm_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_final/test_lambada_openai_mt_stablelm_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_multilingual/test_lambada_multilingual_evaluation.json +51 -0
- wisent/examples/scripts/results/test_lambada_multilingual/test_lambada_multilingual_pairs.json +14 -0
- wisent/examples/scripts/results/test_lambada_multilingual_evaluation.json +51 -0
- wisent/examples/scripts/results/test_lambada_multilingual_pairs.json +14 -0
- wisent/examples/scripts/results/test_lambada_multilingual_stablelm_evaluation.json +51 -0
- wisent/examples/scripts/results/test_lambada_multilingual_stablelm_pairs.json +14 -0
- wisent/examples/scripts/results/test_lambada_openai_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_openai_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_stablelm_en_fixed/test_lambada_openai_mt_stablelm_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_stablelm_en_fixed/test_lambada_openai_mt_stablelm_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_stablelm_fixed/test_lambada_openai_mt_stablelm_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_stablelm_fixed/test_lambada_openai_mt_stablelm_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_lambada_standard_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lambada_standard_pairs.json +8 -0
- wisent/examples/scripts/results/test_leaderboard_evaluation.json +51 -0
- wisent/examples/scripts/results/test_leaderboard_pairs.json +14 -0
- wisent/examples/scripts/results/test_libra/test_libra_evaluation.json +51 -0
- wisent/examples/scripts/results/test_libra/test_libra_pairs.json +14 -0
- wisent/examples/scripts/results/test_libra_evaluation.json +51 -0
- wisent/examples/scripts/results/test_libra_pairs.json +14 -0
- wisent/examples/scripts/results/test_lingoly_evaluation.json +30 -0
- wisent/examples/scripts/results/test_lingoly_pairs.json +8 -0
- wisent/examples/scripts/results/test_livecodebench_evaluation.json +30 -0
- wisent/examples/scripts/results/test_livecodebench_pairs.json +8 -0
- wisent/examples/scripts/results/test_livemathbench_cnmo_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_livemathbench_cnmo_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_livemathbench_cnmo_zh_evaluation.json +30 -0
- wisent/examples/scripts/results/test_livemathbench_cnmo_zh_pairs.json +8 -0
- wisent/examples/scripts/results/test_llama_evaluation.json +30 -0
- wisent/examples/scripts/results/test_llama_pairs.json +8 -0
- wisent/examples/scripts/results/test_logiqa2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_logiqa2_pairs.json +8 -0
- wisent/examples/scripts/results/test_logiqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_logiqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_m_mmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_m_mmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_mastermind/test_mastermind_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mastermind/test_mastermind_pairs.json +14 -0
- wisent/examples/scripts/results/test_mastermind_24_easy/test_mastermind_24_easy_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mastermind_24_easy/test_mastermind_24_easy_pairs.json +8 -0
- wisent/examples/scripts/results/test_mastermind_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mastermind_pairs.json +14 -0
- wisent/examples/scripts/results/test_math500_evaluation.json +30 -0
- wisent/examples/scripts/results/test_math500_pairs.json +8 -0
- wisent/examples/scripts/results/test_math_evaluation.json +30 -0
- wisent/examples/scripts/results/test_math_pairs.json +8 -0
- wisent/examples/scripts/results/test_mathqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mathqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_mbpp_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mbpp_pairs.json +8 -0
- wisent/examples/scripts/results/test_mbpp_plus_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mbpp_plus_pairs.json +8 -0
- wisent/examples/scripts/results/test_mc_taco_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mc_taco_pairs.json +8 -0
- wisent/examples/scripts/results/test_med_concepts_qa/test_med_concepts_qa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_med_concepts_qa/test_med_concepts_qa_pairs.json +14 -0
- wisent/examples/scripts/results/test_med_concepts_qa_atc_easy/test_med_concepts_qa_atc_easy_evaluation.json +30 -0
- wisent/examples/scripts/results/test_med_concepts_qa_atc_easy/test_med_concepts_qa_atc_easy_pairs.json +8 -0
- wisent/examples/scripts/results/test_med_concepts_qa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_med_concepts_qa_pairs.json +14 -0
- wisent/examples/scripts/results/test_meddialog_evaluation.json +30 -0
- wisent/examples/scripts/results/test_meddialog_pairs.json +8 -0
- wisent/examples/scripts/results/test_meddialog_raw_perplexity/test_meddialog_raw_perplexity_evaluation.json +30 -0
- wisent/examples/scripts/results/test_meddialog_raw_perplexity/test_meddialog_raw_perplexity_pairs.json +8 -0
- wisent/examples/scripts/results/test_mediqa_qa2019_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mediqa_qa2019_pairs.json +8 -0
- wisent/examples/scripts/results/test_medmcqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_medmcqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_medqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_medqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_medtext_evaluation.json +30 -0
- wisent/examples/scripts/results/test_medtext_pairs.json +8 -0
- wisent/examples/scripts/results/test_mela_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mela_pairs.json +14 -0
- wisent/examples/scripts/results/test_meqsum_evaluation.json +30 -0
- wisent/examples/scripts/results/test_meqsum_pairs.json +8 -0
- wisent/examples/scripts/results/test_mercury_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mercury_pairs.json +8 -0
- wisent/examples/scripts/results/test_metabench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_metabench_pairs.json +14 -0
- wisent/examples/scripts/results/test_mgsm_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mgsm_pairs.json +14 -0
- wisent/examples/scripts/results/test_mimic_repsum_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mimic_repsum_pairs.json +8 -0
- wisent/examples/scripts/results/test_minerva_math_evaluation.json +51 -0
- wisent/examples/scripts/results/test_minerva_math_pairs.json +14 -0
- wisent/examples/scripts/results/test_mlqa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mlqa_pairs.json +14 -0
- wisent/examples/scripts/results/test_mmlu-pro-plus_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mmlu-pro-plus_pairs.json +14 -0
- wisent/examples/scripts/results/test_mmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_mmlu_pro_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mmlu_pro_pairs.json +14 -0
- wisent/examples/scripts/results/test_mmlu_prox_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mmlu_prox_pairs.json +14 -0
- wisent/examples/scripts/results/test_mmlusr_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mmlusr_pairs.json +8 -0
- wisent/examples/scripts/results/test_mmmu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_mmmu_pairs.json +14 -0
- wisent/examples/scripts/results/test_mnli_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mnli_pairs.json +8 -0
- wisent/examples/scripts/results/test_model_written_evals_evaluation.json +51 -0
- wisent/examples/scripts/results/test_model_written_evals_pairs.json +14 -0
- wisent/examples/scripts/results/test_moral_stories_evaluation.json +30 -0
- wisent/examples/scripts/results/test_moral_stories_pairs.json +8 -0
- wisent/examples/scripts/results/test_mts_dialog_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mts_dialog_pairs.json +8 -0
- wisent/examples/scripts/results/test_multiblimp_evaluation.json +51 -0
- wisent/examples/scripts/results/test_multiblimp_pairs.json +14 -0
- wisent/examples/scripts/results/test_multimedqa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_multimedqa_pairs.json +14 -0
- wisent/examples/scripts/results/test_multipl_e_evaluation.json +30 -0
- wisent/examples/scripts/results/test_multipl_e_pairs.json +8 -0
- wisent/examples/scripts/results/test_mutual_evaluation.json +30 -0
- wisent/examples/scripts/results/test_mutual_pairs.json +8 -0
- wisent/examples/scripts/results/test_non_greedy_robustness_agieval_aqua_rat_evaluation.json +30 -0
- wisent/examples/scripts/results/test_non_greedy_robustness_agieval_aqua_rat_pairs.json +8 -0
- wisent/examples/scripts/results/test_noreval_evaluation.json +51 -0
- wisent/examples/scripts/results/test_noreval_pairs.json +14 -0
- wisent/examples/scripts/results/test_noticia_evaluation.json +30 -0
- wisent/examples/scripts/results/test_noticia_pairs.json +8 -0
- wisent/examples/scripts/results/test_nq_open_evaluation.json +30 -0
- wisent/examples/scripts/results/test_nq_open_pairs.json +8 -0
- wisent/examples/scripts/results/test_olaph_evaluation.json +30 -0
- wisent/examples/scripts/results/test_olaph_pairs.json +8 -0
- wisent/examples/scripts/results/test_openbookqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_openbookqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_openllm_evaluation.json +51 -0
- wisent/examples/scripts/results/test_openllm_pairs.json +14 -0
- wisent/examples/scripts/results/test_option_order_robustness_agieval_aqua_rat_evaluation.json +30 -0
- wisent/examples/scripts/results/test_option_order_robustness_agieval_aqua_rat_pairs.json +8 -0
- wisent/examples/scripts/results/test_paloma_evaluation.json +51 -0
- wisent/examples/scripts/results/test_paloma_pairs.json +14 -0
- wisent/examples/scripts/results/test_passkey/test_passkey_evaluation.json +30 -0
- wisent/examples/scripts/results/test_passkey/test_passkey_pairs.json +8 -0
- wisent/examples/scripts/results/test_paws-x_evaluation.json +51 -0
- wisent/examples/scripts/results/test_paws-x_pairs.json +14 -0
- wisent/examples/scripts/results/test_paws_en/test_paws_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_paws_en/test_paws_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_penn_treebank_evaluation.json +30 -0
- wisent/examples/scripts/results/test_penn_treebank_pairs.json +8 -0
- wisent/examples/scripts/results/test_pile_10k/test_pile_10k_evaluation.json +30 -0
- wisent/examples/scripts/results/test_pile_10k/test_pile_10k_pairs.json +8 -0
- wisent/examples/scripts/results/test_piqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_piqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_polemo2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_polemo2_pairs.json +8 -0
- wisent/examples/scripts/results/test_polymath_en_high_evaluation.json +30 -0
- wisent/examples/scripts/results/test_polymath_en_high_pairs.json +8 -0
- wisent/examples/scripts/results/test_polymath_en_medium_evaluation.json +30 -0
- wisent/examples/scripts/results/test_polymath_en_medium_pairs.json +8 -0
- wisent/examples/scripts/results/test_polymath_zh_high_evaluation.json +30 -0
- wisent/examples/scripts/results/test_polymath_zh_high_pairs.json +8 -0
- wisent/examples/scripts/results/test_polymath_zh_medium_evaluation.json +30 -0
- wisent/examples/scripts/results/test_polymath_zh_medium_pairs.json +8 -0
- wisent/examples/scripts/results/test_portuguese_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_portuguese_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_prompt_robustness_agieval_aqua_rat/test_prompt_robustness_agieval_aqua_rat_evaluation.json +30 -0
- wisent/examples/scripts/results/test_prompt_robustness_agieval_aqua_rat/test_prompt_robustness_agieval_aqua_rat_pairs.json +8 -0
- wisent/examples/scripts/results/test_prompt_robustness_agieval_aqua_rat_evaluation.json +30 -0
- wisent/examples/scripts/results/test_prompt_robustness_agieval_aqua_rat_pairs.json +8 -0
- wisent/examples/scripts/results/test_prost_evaluation.json +30 -0
- wisent/examples/scripts/results/test_prost_pairs.json +8 -0
- wisent/examples/scripts/results/test_ptb_evaluation.json +30 -0
- wisent/examples/scripts/results/test_ptb_pairs.json +8 -0
- wisent/examples/scripts/results/test_pubmedqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_pubmedqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_pythia_evaluation.json +51 -0
- wisent/examples/scripts/results/test_pythia_pairs.json +14 -0
- wisent/examples/scripts/results/test_qa4mre_evaluation.json +30 -0
- wisent/examples/scripts/results/test_qa4mre_pairs.json +8 -0
- wisent/examples/scripts/results/test_qasper_evaluation.json +30 -0
- wisent/examples/scripts/results/test_qasper_pairs.json +8 -0
- wisent/examples/scripts/results/test_race_evaluation.json +30 -0
- wisent/examples/scripts/results/test_race_pairs.json +8 -0
- wisent/examples/scripts/results/test_realtoxicityprompts_evaluation.json +30 -0
- wisent/examples/scripts/results/test_realtoxicityprompts_pairs.json +8 -0
- wisent/examples/scripts/results/test_recode_evaluation.json +30 -0
- wisent/examples/scripts/results/test_recode_pairs.json +8 -0
- wisent/examples/scripts/results/test_record_evaluation.json +30 -0
- wisent/examples/scripts/results/test_record_pairs.json +8 -0
- wisent/examples/scripts/results/test_ruler_evaluation.json +51 -0
- wisent/examples/scripts/results/test_ruler_pairs.json +14 -0
- wisent/examples/scripts/results/test_sciq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_sciq_pairs.json +8 -0
- wisent/examples/scripts/results/test_score_evaluation.json +51 -0
- wisent/examples/scripts/results/test_score_pairs.json +14 -0
- wisent/examples/scripts/results/test_self_consistency_evaluation.json +30 -0
- wisent/examples/scripts/results/test_self_consistency_pairs.json +8 -0
- wisent/examples/scripts/results/test_siqa/test_siqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_siqa/test_siqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_siqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_siqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_spanish_bench_evaluation.json +51 -0
- wisent/examples/scripts/results/test_spanish_bench_pairs.json +14 -0
- wisent/examples/scripts/results/test_squad2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_squad2_pairs.json +8 -0
- wisent/examples/scripts/results/test_squadv2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_squadv2_pairs.json +8 -0
- wisent/examples/scripts/results/test_super-glue-lm-eval-v1-seq2seq_evaluation.json +30 -0
- wisent/examples/scripts/results/test_super-glue-lm-eval-v1-seq2seq_pairs.json +8 -0
- wisent/examples/scripts/results/test_super-glue-lm-eval-v1_evaluation.json +51 -0
- wisent/examples/scripts/results/test_super-glue-lm-eval-v1_pairs.json +14 -0
- wisent/examples/scripts/results/test_swag_evaluation.json +30 -0
- wisent/examples/scripts/results/test_swag_pairs.json +8 -0
- wisent/examples/scripts/results/test_tinyBenchmarks_evaluation.json +51 -0
- wisent/examples/scripts/results/test_tinyBenchmarks_pairs.json +14 -0
- wisent/examples/scripts/results/test_tmmluplus_evaluation.json +51 -0
- wisent/examples/scripts/results/test_tmmluplus_pairs.json +14 -0
- wisent/examples/scripts/results/test_translation_evaluation.json +51 -0
- wisent/examples/scripts/results/test_translation_pairs.json +14 -0
- wisent/examples/scripts/results/test_triviaqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_triviaqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_truthfulqa-multi_evaluation.json +51 -0
- wisent/examples/scripts/results/test_truthfulqa-multi_pairs.json +14 -0
- wisent/examples/scripts/results/test_truthfulqa_evaluation.json +30 -0
- wisent/examples/scripts/results/test_truthfulqa_mc1_evaluation.json +30 -0
- wisent/examples/scripts/results/test_truthfulqa_mc1_pairs.json +8 -0
- wisent/examples/scripts/results/test_truthfulqa_mc2_evaluation.json +30 -0
- wisent/examples/scripts/results/test_truthfulqa_mc2_pairs.json +8 -0
- wisent/examples/scripts/results/test_truthfulqa_pairs.json +8 -0
- wisent/examples/scripts/results/test_turkishmmlu_evaluation.json +51 -0
- wisent/examples/scripts/results/test_turkishmmlu_pairs.json +14 -0
- wisent/examples/scripts/results/test_unfair_tos_evaluation.json +30 -0
- wisent/examples/scripts/results/test_unfair_tos_pairs.json +8 -0
- wisent/examples/scripts/results/test_unscramble_evaluation.json +51 -0
- wisent/examples/scripts/results/test_unscramble_pairs.json +14 -0
- wisent/examples/scripts/results/test_webqs_evaluation.json +30 -0
- wisent/examples/scripts/results/test_webqs_pairs.json +8 -0
- wisent/examples/scripts/results/test_wikitext103_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wikitext103_pairs.json +8 -0
- wisent/examples/scripts/results/test_wikitext_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wikitext_pairs.json +8 -0
- wisent/examples/scripts/results/test_winogender_evaluation.json +51 -0
- wisent/examples/scripts/results/test_winogender_pairs.json +14 -0
- wisent/examples/scripts/results/test_winogrande_evaluation.json +30 -0
- wisent/examples/scripts/results/test_winogrande_pairs.json +8 -0
- wisent/examples/scripts/results/test_wmdp_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wmdp_pairs.json +8 -0
- wisent/examples/scripts/results/test_wmt-ro-en-t5-prompt_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wmt-ro-en-t5-prompt_pairs.json +8 -0
- wisent/examples/scripts/results/test_wmt14_en_fr_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wmt14_en_fr_pairs.json +8 -0
- wisent/examples/scripts/results/test_wmt16_en_de_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wmt16_en_de_pairs.json +8 -0
- wisent/examples/scripts/results/test_wmt16_ro_en_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wmt16_ro_en_pairs.json +8 -0
- wisent/examples/scripts/results/test_wsc273_evaluation.json +30 -0
- wisent/examples/scripts/results/test_wsc273_pairs.json +8 -0
- wisent/examples/scripts/results/test_xcopa_evaluation.json +51 -0
- wisent/examples/scripts/results/test_xcopa_pairs.json +14 -0
- wisent/examples/scripts/results/test_xnli_eu_evaluation.json +30 -0
- wisent/examples/scripts/results/test_xnli_eu_pairs.json +8 -0
- wisent/examples/scripts/results/test_xnli_evaluation.json +51 -0
- wisent/examples/scripts/results/test_xnli_pairs.json +14 -0
- wisent/examples/scripts/results/test_xquad_evaluation.json +51 -0
- wisent/examples/scripts/results/test_xquad_pairs.json +14 -0
- wisent/examples/scripts/results/test_xstorycloze_evaluation.json +51 -0
- wisent/examples/scripts/results/test_xstorycloze_pairs.json +14 -0
- wisent/examples/scripts/results/test_xsum_evaluation.json +30 -0
- wisent/examples/scripts/results/test_xsum_pairs.json +8 -0
- wisent/examples/scripts/results/test_xwinograd_evaluation.json +51 -0
- wisent/examples/scripts/results/test_xwinograd_pairs.json +14 -0
- wisent/examples/scripts/results/test_yahoo_answers_topics_evaluation.json +30 -0
- wisent/examples/scripts/results/test_yahoo_answers_topics_pairs.json +8 -0
- wisent/parameters/__init__.py +1 -0
- wisent/parameters/lm_eval/all_lm_eval_task_families.json +169 -0
- wisent/parameters/lm_eval/broken_in_lm_eval.json +10 -0
- wisent/parameters/lm_eval/evaluations_not_lm_eval_tasks.json +0 -0
- wisent/parameters/lm_eval/evaluator_check.json +3476 -0
- wisent/parameters/lm_eval/final_verification.json +24782 -0
- wisent/parameters/lm_eval/group_task_evaluators.json +1833 -0
- wisent/parameters/lm_eval/group_tasks.json +150 -0
- wisent/parameters/lm_eval/individual_tasks.json +402 -0
- wisent/parameters/lm_eval/no_readmes.json +1 -0
- wisent/parameters/lm_eval/not_lm_eval_tasks.json +110 -0
- wisent/parameters/lm_eval/read_tasks.json +208 -0
- wisent/parameters/lm_eval/readme_files.json +208 -0
- wisent/parameters/lm_eval/track_progress_not_lm_eval_tasks.json +128 -0
- wisent/parameters/tasks/missing_task_families.json +2963 -0
- wisent/parameters/tasks/remaining_tasks_to_implement.json +199 -0
- wisent/parameters/tasks/risks.json +10 -0
- wisent/parameters/tasks/skills.json +14 -0
- wisent/parameters/tasks/tasks.json +56031 -0
- wisent/scripts/run_quality_metrics_sweep.sh +315 -0
- wisent/tests/__init__.py +0 -0
- wisent/tests/examples/__init__.py +0 -0
- wisent/tests/examples/cli/__init__.py +0 -0
- wisent/tests/examples/cli/activations/__init__.py +0 -0
- wisent/tests/examples/cli/activations/test_get_activations.py +127 -0
- wisent/tests/examples/cli/classifier/__init__.py +0 -0
- wisent/tests/examples/cli/classifier/test_classifier_examples.py +141 -0
- wisent/tests/examples/cli/contrastive_pairs/__init__.py +0 -0
- wisent/tests/examples/cli/contrastive_pairs/test_generate_pairs.py +89 -0
- wisent/tests/examples/cli/evaluation/__init__.py +0 -0
- wisent/tests/examples/cli/evaluation/test_evaluation_examples.py +117 -0
- wisent/tests/examples/cli/generate/__init__.py +0 -0
- wisent/tests/examples/cli/generate/test_generate_with_classifier.py +146 -0
- wisent/tests/examples/cli/generate/test_generate_with_steering.py +149 -0
- wisent/tests/examples/cli/generate/test_only_generate.py +110 -0
- wisent/tests/examples/cli/multi_steering/__init__.py +0 -0
- wisent/tests/examples/cli/multi_steering/test_multi_steer_from_trained_vectors.py +210 -0
- wisent/tests/examples/cli/multi_steering/test_multi_steer_with_different_parameters.py +205 -0
- wisent/tests/examples/cli/multi_steering/test_train_and_multi_steer.py +174 -0
- wisent/tests/examples/cli/optimizer/__init__.py +0 -0
- wisent/tests/examples/cli/optimizer/test_optimize_sample_size.py +102 -0
- wisent/tests/examples/cli/optimizer/test_optimizer_examples.py +59 -0
- wisent/tests/examples/cli/steering/__init__.py +0 -0
- wisent/tests/examples/cli/steering/test_create_steering_vectors.py +135 -0
- wisent/tests/examples/cli/synthetic/__init__.py +0 -0
- wisent/tests/examples/cli/synthetic/test_synthetic_pairs.py +45 -0
- wisent/tests/nosense/__init__.py +6 -0
- wisent/tests/nosense/base_nosense.py +81 -0
- wisent/tests/nosense/math500_nosense.py +72 -0
- wisent/tests/nosense/test_robustness.py +336 -0
- wisent/tests/test_all_cli_commands.py +674 -0
- wisent/tests/test_geometry_comprehensive.py +327 -0
- wisent/tests/test_titan_geometry.py +257 -0
- wisent/tests/visualize_geometry.py +148 -0
- wisent-0.7.379.dist-info/METADATA +64 -0
- wisent-0.7.379.dist-info/RECORD +1720 -0
- wisent-0.7.379.dist-info/WHEEL +5 -0
- wisent-0.7.379.dist-info/entry_points.txt +2 -0
- wisent-0.7.379.dist-info/licenses/LICENSE +21 -0
- wisent-0.7.379.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1731 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Unified Configuration Manager for Wisent.
|
|
3
|
+
|
|
4
|
+
Stores all optimized parameters in a single location with a consistent structure:
|
|
5
|
+
- Classification parameters (layer, threshold, aggregation, etc.)
|
|
6
|
+
- Steering parameters (layer, strength, method, etc.)
|
|
7
|
+
- Weight modification parameters (directional projection/additive settings)
|
|
8
|
+
|
|
9
|
+
All configs are stored per model, with task-specific overrides where applicable.
|
|
10
|
+
|
|
11
|
+
Location: ~/.wisent/configs/{model_name}.json
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
from dataclasses import dataclass, field, asdict
|
|
17
|
+
from datetime import datetime
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Optional, Dict, Any, List, TypeVar, Type
|
|
20
|
+
import numpy as np
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# Default config location
|
|
24
|
+
DEFAULT_CONFIG_DIR = os.path.expanduser("~/.wisent/configs")
|
|
25
|
+
|
|
26
|
+
# Type variable for SerializableConfig
|
|
27
|
+
T = TypeVar("T", bound="SerializableConfig")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class SerializableConfig:
|
|
31
|
+
"""Mixin providing to_dict() and from_dict() for dataclasses.
|
|
32
|
+
|
|
33
|
+
All simple config dataclasses should inherit from this to avoid
|
|
34
|
+
duplicating serialization logic.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
38
|
+
"""Convert config to dictionary."""
|
|
39
|
+
return asdict(self)
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def from_dict(cls: Type[T], data: Dict[str, Any]) -> T:
|
|
43
|
+
"""Create config from dictionary, filtering to valid fields only."""
|
|
44
|
+
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class NumpyEncoder(json.JSONEncoder):
|
|
48
|
+
"""Custom JSON encoder to handle numpy types."""
|
|
49
|
+
def default(self, obj):
|
|
50
|
+
if isinstance(obj, (np.integer, np.int64)):
|
|
51
|
+
return int(obj)
|
|
52
|
+
if isinstance(obj, (np.floating, np.float64)):
|
|
53
|
+
return float(obj)
|
|
54
|
+
if isinstance(obj, np.ndarray):
|
|
55
|
+
return obj.tolist()
|
|
56
|
+
return super().default(obj)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class ClassificationConfig(SerializableConfig):
|
|
61
|
+
"""Classification optimization parameters."""
|
|
62
|
+
layer: int = 12
|
|
63
|
+
token_aggregation: str = "average"
|
|
64
|
+
detection_threshold: float = 0.6
|
|
65
|
+
classifier_type: str = "logistic"
|
|
66
|
+
prompt_construction_strategy: str = "multiple_choice"
|
|
67
|
+
token_targeting_strategy: str = "last_token"
|
|
68
|
+
|
|
69
|
+
# Metrics from optimization
|
|
70
|
+
accuracy: float = 0.0
|
|
71
|
+
f1_score: float = 0.0
|
|
72
|
+
precision: float = 0.0
|
|
73
|
+
recall: float = 0.0
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass
|
|
77
|
+
class SteeringConfig(SerializableConfig):
|
|
78
|
+
"""Steering optimization parameters."""
|
|
79
|
+
layer: int = 12
|
|
80
|
+
strength: float = 1.0
|
|
81
|
+
method: str = "CAA"
|
|
82
|
+
token_aggregation: str = "average"
|
|
83
|
+
prompt_strategy: str = "question_only"
|
|
84
|
+
normalize_mode: str = "none"
|
|
85
|
+
strategy: str = "constant" # Steering strategy: constant, initial_only, diminishing
|
|
86
|
+
|
|
87
|
+
# Metrics from optimization
|
|
88
|
+
score: float = 0.0
|
|
89
|
+
metric: str = "accuracy"
|
|
90
|
+
|
|
91
|
+
# ==========================================================================
|
|
92
|
+
# METHOD-SPECIFIC PARAMETERS
|
|
93
|
+
# ==========================================================================
|
|
94
|
+
|
|
95
|
+
# PRISM parameters
|
|
96
|
+
num_directions: int = 1 # Number of steering directions
|
|
97
|
+
direction_weighting: str = "primary_only" # primary_only, equal, learned, decay
|
|
98
|
+
retain_weight: float = 0.0 # Weight for retaining original behavior
|
|
99
|
+
independence_weight: float = 0.05 # Weight for direction independence loss
|
|
100
|
+
prism_optimization_steps: int = 100 # Optimization steps for PRISM
|
|
101
|
+
use_caa_init: bool = True # Initialize from CAA direction
|
|
102
|
+
cone_constraint: bool = True # Use cone constraint
|
|
103
|
+
min_cosine_similarity: float = 0.3 # Min cosine sim for cone
|
|
104
|
+
max_cosine_similarity: float = 0.95 # Max cosine sim for cone
|
|
105
|
+
|
|
106
|
+
# PULSE parameters
|
|
107
|
+
sensor_layer: int = -1 # Layer to sense activation patterns (-1 = auto)
|
|
108
|
+
steering_layers: str = "" # Comma-separated steering layer indices
|
|
109
|
+
condition_threshold: float = 0.5 # Threshold for conditional steering
|
|
110
|
+
gate_temperature: float = 0.5 # Temperature for gating
|
|
111
|
+
per_layer_scaling: bool = True # Use per-layer scaling
|
|
112
|
+
use_entropy_scaling: bool = False # Scale by entropy
|
|
113
|
+
max_alpha: float = 2.0 # Maximum steering intensity
|
|
114
|
+
learn_threshold: bool = True # Learn threshold during training
|
|
115
|
+
pulse_optimization_steps: int = 100 # Optimization steps for PULSE
|
|
116
|
+
|
|
117
|
+
# TITAN parameters
|
|
118
|
+
gate_hidden_dim: int = 64 # Hidden dimension for gate network
|
|
119
|
+
intensity_hidden_dim: int = 32 # Hidden dimension for intensity network
|
|
120
|
+
behavior_weight: float = 1.0 # Weight for behavior loss
|
|
121
|
+
sparse_weight: float = 0.05 # Weight for sparsity loss
|
|
122
|
+
titan_optimization_steps: int = 200 # Optimization steps for TITAN
|
|
123
|
+
titan_learning_rate: float = 0.005 # Learning rate for TITAN
|
|
124
|
+
|
|
125
|
+
# Generic method parameters storage (for future methods)
|
|
126
|
+
method_params: Dict[str, Any] = field(default_factory=dict)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@dataclass
|
|
130
|
+
class WeightModificationConfig(SerializableConfig):
|
|
131
|
+
"""Weight modification (directional projection/additive) parameters."""
|
|
132
|
+
method: str = "directional" # directional or additive
|
|
133
|
+
|
|
134
|
+
# Directional projection parameters
|
|
135
|
+
max_weight: float = 1.0
|
|
136
|
+
min_weight: float = 0.0
|
|
137
|
+
max_weight_position: float = 0.5 # As ratio of total layers
|
|
138
|
+
min_weight_distance: float = 0.5 # As ratio of total layers
|
|
139
|
+
strength: float = 1.0
|
|
140
|
+
num_pairs: int = 100
|
|
141
|
+
|
|
142
|
+
# Additive parameters
|
|
143
|
+
alpha: float = 1.0
|
|
144
|
+
additive_method: str = "bias" # bias, weight, or both
|
|
145
|
+
|
|
146
|
+
# Common parameters
|
|
147
|
+
components: List[str] = field(default_factory=lambda: ["self_attn.o_proj", "mlp.down_proj"])
|
|
148
|
+
normalize_vectors: bool = True
|
|
149
|
+
norm_preserve: bool = True
|
|
150
|
+
use_biprojection: bool = True
|
|
151
|
+
use_kernel: bool = True
|
|
152
|
+
|
|
153
|
+
# Metrics
|
|
154
|
+
score: float = 0.0
|
|
155
|
+
baseline_score: float = 0.0
|
|
156
|
+
output_dir: str = ""
|
|
157
|
+
|
|
158
|
+
@classmethod
|
|
159
|
+
def from_dict(cls, data: Dict[str, Any]) -> "WeightModificationConfig":
|
|
160
|
+
# Handle components field specially since it's a list
|
|
161
|
+
if "components" in data and isinstance(data["components"], str):
|
|
162
|
+
data["components"] = [data["components"]]
|
|
163
|
+
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class NestedConfigMixin:
|
|
167
|
+
"""Mixin for configs with nested classification/steering/weight_modification configs.
|
|
168
|
+
|
|
169
|
+
Subclasses must define a `_name_field` class attribute specifying the primary name field
|
|
170
|
+
(e.g., "task_name" or "trait_name").
|
|
171
|
+
"""
|
|
172
|
+
_name_field: str = "name" # Override in subclass
|
|
173
|
+
|
|
174
|
+
classification: Optional[ClassificationConfig] = None
|
|
175
|
+
steering: Optional[SteeringConfig] = None
|
|
176
|
+
weight_modification: Optional[WeightModificationConfig] = None
|
|
177
|
+
updated_at: str = ""
|
|
178
|
+
optimization_method: str = "manual"
|
|
179
|
+
|
|
180
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
181
|
+
result = {
|
|
182
|
+
self._name_field: getattr(self, self._name_field),
|
|
183
|
+
"updated_at": self.updated_at,
|
|
184
|
+
"optimization_method": self.optimization_method,
|
|
185
|
+
}
|
|
186
|
+
if self.classification:
|
|
187
|
+
result["classification"] = self.classification.to_dict()
|
|
188
|
+
if self.steering:
|
|
189
|
+
result["steering"] = self.steering.to_dict()
|
|
190
|
+
if self.weight_modification:
|
|
191
|
+
result["weight_modification"] = self.weight_modification.to_dict()
|
|
192
|
+
return result
|
|
193
|
+
|
|
194
|
+
@classmethod
|
|
195
|
+
def _from_dict_common(cls, config, data: Dict[str, Any]):
|
|
196
|
+
"""Populate nested configs from dict data."""
|
|
197
|
+
if "classification" in data:
|
|
198
|
+
config.classification = ClassificationConfig.from_dict(data["classification"])
|
|
199
|
+
if "steering" in data:
|
|
200
|
+
config.steering = SteeringConfig.from_dict(data["steering"])
|
|
201
|
+
if "weight_modification" in data:
|
|
202
|
+
config.weight_modification = WeightModificationConfig.from_dict(data["weight_modification"])
|
|
203
|
+
return config
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@dataclass
|
|
207
|
+
class TaskConfig(NestedConfigMixin):
|
|
208
|
+
"""Configuration for a specific benchmark task (e.g., hellaswag, gsm8k, mmlu)."""
|
|
209
|
+
_name_field: str = field(default="task_name", init=False, repr=False)
|
|
210
|
+
|
|
211
|
+
task_name: str = ""
|
|
212
|
+
classification: Optional[ClassificationConfig] = None
|
|
213
|
+
steering: Optional[SteeringConfig] = None
|
|
214
|
+
weight_modification: Optional[WeightModificationConfig] = None
|
|
215
|
+
updated_at: str = field(default_factory=lambda: datetime.now().isoformat())
|
|
216
|
+
optimization_method: str = "manual"
|
|
217
|
+
|
|
218
|
+
@classmethod
|
|
219
|
+
def from_dict(cls, data: Dict[str, Any]) -> "TaskConfig":
|
|
220
|
+
config = cls(
|
|
221
|
+
task_name=data.get("task_name", ""),
|
|
222
|
+
updated_at=data.get("updated_at", datetime.now().isoformat()),
|
|
223
|
+
optimization_method=data.get("optimization_method", "manual"),
|
|
224
|
+
)
|
|
225
|
+
return cls._from_dict_common(config, data)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@dataclass
|
|
229
|
+
class TraitConfig(NestedConfigMixin):
|
|
230
|
+
"""Configuration for a specific behavioral trait (e.g., coding, honesty, helpfulness)."""
|
|
231
|
+
_name_field: str = field(default="trait_name", init=False, repr=False)
|
|
232
|
+
|
|
233
|
+
trait_name: str = ""
|
|
234
|
+
classification: Optional[ClassificationConfig] = None
|
|
235
|
+
steering: Optional[SteeringConfig] = None
|
|
236
|
+
weight_modification: Optional[WeightModificationConfig] = None
|
|
237
|
+
updated_at: str = field(default_factory=lambda: datetime.now().isoformat())
|
|
238
|
+
optimization_method: str = "manual"
|
|
239
|
+
|
|
240
|
+
@classmethod
|
|
241
|
+
def from_dict(cls, data: Dict[str, Any]) -> "TraitConfig":
|
|
242
|
+
config = cls(
|
|
243
|
+
trait_name=data.get("trait_name", ""),
|
|
244
|
+
updated_at=data.get("updated_at", datetime.now().isoformat()),
|
|
245
|
+
optimization_method=data.get("optimization_method", "manual"),
|
|
246
|
+
)
|
|
247
|
+
return cls._from_dict_common(config, data)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
@dataclass
|
|
251
|
+
class ModelConfig:
|
|
252
|
+
"""Complete configuration for a model."""
|
|
253
|
+
model_name: str
|
|
254
|
+
num_layers: int = 0
|
|
255
|
+
|
|
256
|
+
# Default configs (used when no task/trait-specific config exists)
|
|
257
|
+
default_classification: Optional[ClassificationConfig] = None
|
|
258
|
+
default_steering: Optional[SteeringConfig] = None
|
|
259
|
+
default_weight_modification: Optional[WeightModificationConfig] = None
|
|
260
|
+
|
|
261
|
+
# Task-specific configs (benchmark tasks: hellaswag, gsm8k, mmlu, etc.)
|
|
262
|
+
tasks: Dict[str, TaskConfig] = field(default_factory=dict)
|
|
263
|
+
|
|
264
|
+
# Trait-specific configs (behavioral traits: coding, honesty, helpfulness, etc.)
|
|
265
|
+
traits: Dict[str, TraitConfig] = field(default_factory=dict)
|
|
266
|
+
|
|
267
|
+
# Metadata
|
|
268
|
+
created_at: str = field(default_factory=lambda: datetime.now().isoformat())
|
|
269
|
+
updated_at: str = field(default_factory=lambda: datetime.now().isoformat())
|
|
270
|
+
config_version: str = "2.1" # Bumped version for traits support
|
|
271
|
+
|
|
272
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
273
|
+
result = {
|
|
274
|
+
"model_name": self.model_name,
|
|
275
|
+
"num_layers": self.num_layers,
|
|
276
|
+
"created_at": self.created_at,
|
|
277
|
+
"updated_at": self.updated_at,
|
|
278
|
+
"config_version": self.config_version,
|
|
279
|
+
"tasks": {k: v.to_dict() for k, v in self.tasks.items()},
|
|
280
|
+
"traits": {k: v.to_dict() for k, v in self.traits.items()},
|
|
281
|
+
}
|
|
282
|
+
if self.default_classification:
|
|
283
|
+
result["default_classification"] = self.default_classification.to_dict()
|
|
284
|
+
if self.default_steering:
|
|
285
|
+
result["default_steering"] = self.default_steering.to_dict()
|
|
286
|
+
if self.default_weight_modification:
|
|
287
|
+
result["default_weight_modification"] = self.default_weight_modification.to_dict()
|
|
288
|
+
return result
|
|
289
|
+
|
|
290
|
+
@classmethod
|
|
291
|
+
def from_dict(cls, data: Dict[str, Any]) -> "ModelConfig":
|
|
292
|
+
config = cls(
|
|
293
|
+
model_name=data.get("model_name", ""),
|
|
294
|
+
num_layers=data.get("num_layers", 0),
|
|
295
|
+
created_at=data.get("created_at", datetime.now().isoformat()),
|
|
296
|
+
updated_at=data.get("updated_at", datetime.now().isoformat()),
|
|
297
|
+
config_version=data.get("config_version", "2.1"),
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
if "default_classification" in data:
|
|
301
|
+
config.default_classification = ClassificationConfig.from_dict(data["default_classification"])
|
|
302
|
+
if "default_steering" in data:
|
|
303
|
+
config.default_steering = SteeringConfig.from_dict(data["default_steering"])
|
|
304
|
+
if "default_weight_modification" in data:
|
|
305
|
+
config.default_weight_modification = WeightModificationConfig.from_dict(data["default_weight_modification"])
|
|
306
|
+
|
|
307
|
+
if "tasks" in data:
|
|
308
|
+
config.tasks = {
|
|
309
|
+
k: TaskConfig.from_dict(v) for k, v in data["tasks"].items()
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
if "traits" in data:
|
|
313
|
+
config.traits = {
|
|
314
|
+
k: TraitConfig.from_dict(v) for k, v in data["traits"].items()
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
return config
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
class WisentConfigManager:
|
|
321
|
+
"""
|
|
322
|
+
Unified configuration manager for all Wisent optimization parameters.
|
|
323
|
+
|
|
324
|
+
Stores one JSON file per model at ~/.wisent/configs/{model_name}.json
|
|
325
|
+
|
|
326
|
+
Structure:
|
|
327
|
+
{
|
|
328
|
+
"model_name": "meta-llama/Llama-3.2-1B-Instruct",
|
|
329
|
+
"num_layers": 16,
|
|
330
|
+
"config_version": "2.1",
|
|
331
|
+
|
|
332
|
+
// Default configs (fallback when no specific config exists)
|
|
333
|
+
"default_classification": { ... },
|
|
334
|
+
"default_steering": { ... },
|
|
335
|
+
"default_weight_modification": { ... },
|
|
336
|
+
|
|
337
|
+
// Task-specific configs (benchmark tasks)
|
|
338
|
+
"tasks": {
|
|
339
|
+
"hellaswag": {
|
|
340
|
+
"task_name": "hellaswag",
|
|
341
|
+
"classification": { ... },
|
|
342
|
+
"steering": { ... },
|
|
343
|
+
"weight_modification": { ... }
|
|
344
|
+
},
|
|
345
|
+
"gsm8k": { ... }
|
|
346
|
+
},
|
|
347
|
+
|
|
348
|
+
// Trait-specific configs (behavioral traits)
|
|
349
|
+
"traits": {
|
|
350
|
+
"coding": {
|
|
351
|
+
"trait_name": "coding",
|
|
352
|
+
"classification": { ... },
|
|
353
|
+
"steering": { ... },
|
|
354
|
+
"weight_modification": { ... }
|
|
355
|
+
},
|
|
356
|
+
"honesty": { ... },
|
|
357
|
+
"helpfulness": { ... }
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
"""
|
|
361
|
+
|
|
362
|
+
def __init__(self, config_dir: Optional[str] = None):
|
|
363
|
+
"""
|
|
364
|
+
Initialize the config manager.
|
|
365
|
+
|
|
366
|
+
Args:
|
|
367
|
+
config_dir: Directory to store config files. Defaults to ~/.wisent/configs/
|
|
368
|
+
"""
|
|
369
|
+
self.config_dir = Path(config_dir or DEFAULT_CONFIG_DIR)
|
|
370
|
+
self.config_dir.mkdir(parents=True, exist_ok=True)
|
|
371
|
+
self._cache: Dict[str, ModelConfig] = {}
|
|
372
|
+
|
|
373
|
+
def _sanitize_model_name(self, model_name: str) -> str:
|
|
374
|
+
"""Convert model name to a safe filename."""
|
|
375
|
+
sanitized = model_name.replace("/", "_").replace("\\", "_").replace(":", "_")
|
|
376
|
+
sanitized = "".join(c for c in sanitized if c.isalnum() or c in "._-")
|
|
377
|
+
return sanitized
|
|
378
|
+
|
|
379
|
+
def _get_config_path(self, model_name: str) -> Path:
|
|
380
|
+
"""Get the full path to the config file for a model."""
|
|
381
|
+
sanitized_name = self._sanitize_model_name(model_name)
|
|
382
|
+
return self.config_dir / f"{sanitized_name}.json"
|
|
383
|
+
|
|
384
|
+
def _load_model_config(self, model_name: str) -> ModelConfig:
|
|
385
|
+
"""Load or create a model config."""
|
|
386
|
+
if model_name in self._cache:
|
|
387
|
+
return self._cache[model_name]
|
|
388
|
+
|
|
389
|
+
config_path = self._get_config_path(model_name)
|
|
390
|
+
|
|
391
|
+
if config_path.exists():
|
|
392
|
+
try:
|
|
393
|
+
with open(config_path) as f:
|
|
394
|
+
data = json.load(f)
|
|
395
|
+
config = ModelConfig.from_dict(data)
|
|
396
|
+
except (json.JSONDecodeError, KeyError) as e:
|
|
397
|
+
print(f"Warning: Failed to load config for {model_name}: {e}")
|
|
398
|
+
config = ModelConfig(model_name=model_name)
|
|
399
|
+
else:
|
|
400
|
+
config = ModelConfig(model_name=model_name)
|
|
401
|
+
|
|
402
|
+
self._cache[model_name] = config
|
|
403
|
+
return config
|
|
404
|
+
|
|
405
|
+
def _save_model_config(self, config: ModelConfig) -> Path:
|
|
406
|
+
"""Save a model config to disk."""
|
|
407
|
+
config.updated_at = datetime.now().isoformat()
|
|
408
|
+
config_path = self._get_config_path(config.model_name)
|
|
409
|
+
|
|
410
|
+
with open(config_path, "w") as f:
|
|
411
|
+
json.dump(config.to_dict(), f, indent=2, cls=NumpyEncoder)
|
|
412
|
+
|
|
413
|
+
self._cache[config.model_name] = config
|
|
414
|
+
return config_path
|
|
415
|
+
|
|
416
|
+
# ========== Classification Methods ==========
|
|
417
|
+
|
|
418
|
+
def save_classification_config(
|
|
419
|
+
self,
|
|
420
|
+
model_name: str,
|
|
421
|
+
task_name: Optional[str] = None,
|
|
422
|
+
layer: int = 12,
|
|
423
|
+
token_aggregation: str = "average",
|
|
424
|
+
detection_threshold: float = 0.6,
|
|
425
|
+
classifier_type: str = "logistic",
|
|
426
|
+
prompt_construction_strategy: str = "multiple_choice",
|
|
427
|
+
token_targeting_strategy: str = "last_token",
|
|
428
|
+
accuracy: float = 0.0,
|
|
429
|
+
f1_score: float = 0.0,
|
|
430
|
+
precision: float = 0.0,
|
|
431
|
+
recall: float = 0.0,
|
|
432
|
+
optimization_method: str = "manual",
|
|
433
|
+
set_as_default: bool = False,
|
|
434
|
+
) -> Path:
|
|
435
|
+
"""
|
|
436
|
+
Save classification config for a model/task.
|
|
437
|
+
|
|
438
|
+
Args:
|
|
439
|
+
model_name: Model name/path
|
|
440
|
+
task_name: Task name (None for default config)
|
|
441
|
+
set_as_default: If True, also set as the default config
|
|
442
|
+
... other classification parameters
|
|
443
|
+
|
|
444
|
+
Returns:
|
|
445
|
+
Path to the saved config file
|
|
446
|
+
"""
|
|
447
|
+
config = self._load_model_config(model_name)
|
|
448
|
+
|
|
449
|
+
classification = ClassificationConfig(
|
|
450
|
+
layer=layer,
|
|
451
|
+
token_aggregation=token_aggregation,
|
|
452
|
+
detection_threshold=detection_threshold,
|
|
453
|
+
classifier_type=classifier_type,
|
|
454
|
+
prompt_construction_strategy=prompt_construction_strategy,
|
|
455
|
+
token_targeting_strategy=token_targeting_strategy,
|
|
456
|
+
accuracy=accuracy,
|
|
457
|
+
f1_score=f1_score,
|
|
458
|
+
precision=precision,
|
|
459
|
+
recall=recall,
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
if task_name:
|
|
463
|
+
# Save to task-specific config
|
|
464
|
+
if task_name not in config.tasks:
|
|
465
|
+
config.tasks[task_name] = TaskConfig(task_name=task_name)
|
|
466
|
+
config.tasks[task_name].classification = classification
|
|
467
|
+
config.tasks[task_name].optimization_method = optimization_method
|
|
468
|
+
config.tasks[task_name].updated_at = datetime.now().isoformat()
|
|
469
|
+
|
|
470
|
+
if set_as_default or not task_name:
|
|
471
|
+
config.default_classification = classification
|
|
472
|
+
|
|
473
|
+
return self._save_model_config(config)
|
|
474
|
+
|
|
475
|
+
def get_classification_config(
|
|
476
|
+
self,
|
|
477
|
+
model_name: str,
|
|
478
|
+
task_name: Optional[str] = None,
|
|
479
|
+
) -> Optional[ClassificationConfig]:
|
|
480
|
+
"""
|
|
481
|
+
Get classification config for a model/task.
|
|
482
|
+
|
|
483
|
+
Args:
|
|
484
|
+
model_name: Model name/path
|
|
485
|
+
task_name: Task name (returns task-specific if exists, else default)
|
|
486
|
+
|
|
487
|
+
Returns:
|
|
488
|
+
ClassificationConfig or None
|
|
489
|
+
"""
|
|
490
|
+
config = self._load_model_config(model_name)
|
|
491
|
+
|
|
492
|
+
# Try task-specific first
|
|
493
|
+
if task_name and task_name in config.tasks:
|
|
494
|
+
task_config = config.tasks[task_name]
|
|
495
|
+
if task_config.classification:
|
|
496
|
+
return task_config.classification
|
|
497
|
+
|
|
498
|
+
# Fall back to default
|
|
499
|
+
return config.default_classification
|
|
500
|
+
|
|
501
|
+
# ========== Steering Methods ==========
|
|
502
|
+
|
|
503
|
+
def save_steering_config(
|
|
504
|
+
self,
|
|
505
|
+
model_name: str,
|
|
506
|
+
task_name: Optional[str] = None,
|
|
507
|
+
layer: int = 12,
|
|
508
|
+
strength: float = 1.0,
|
|
509
|
+
method: str = "CAA",
|
|
510
|
+
token_aggregation: str = "average",
|
|
511
|
+
prompt_strategy: str = "question_only",
|
|
512
|
+
normalize_mode: str = "none",
|
|
513
|
+
strategy: str = "constant",
|
|
514
|
+
score: float = 0.0,
|
|
515
|
+
metric: str = "accuracy",
|
|
516
|
+
optimization_method: str = "manual",
|
|
517
|
+
set_as_default: bool = False,
|
|
518
|
+
# PRISM parameters
|
|
519
|
+
num_directions: int = 1,
|
|
520
|
+
direction_weighting: str = "primary_only",
|
|
521
|
+
retain_weight: float = 0.0,
|
|
522
|
+
independence_weight: float = 0.05,
|
|
523
|
+
prism_optimization_steps: int = 100,
|
|
524
|
+
use_caa_init: bool = True,
|
|
525
|
+
cone_constraint: bool = True,
|
|
526
|
+
min_cosine_similarity: float = 0.3,
|
|
527
|
+
max_cosine_similarity: float = 0.95,
|
|
528
|
+
# PULSE parameters
|
|
529
|
+
sensor_layer: int = -1,
|
|
530
|
+
steering_layers: str = "",
|
|
531
|
+
condition_threshold: float = 0.5,
|
|
532
|
+
gate_temperature: float = 0.5,
|
|
533
|
+
per_layer_scaling: bool = True,
|
|
534
|
+
use_entropy_scaling: bool = False,
|
|
535
|
+
max_alpha: float = 2.0,
|
|
536
|
+
learn_threshold: bool = True,
|
|
537
|
+
pulse_optimization_steps: int = 100,
|
|
538
|
+
# TITAN parameters
|
|
539
|
+
gate_hidden_dim: int = 64,
|
|
540
|
+
intensity_hidden_dim: int = 32,
|
|
541
|
+
behavior_weight: float = 1.0,
|
|
542
|
+
sparse_weight: float = 0.05,
|
|
543
|
+
titan_optimization_steps: int = 200,
|
|
544
|
+
titan_learning_rate: float = 0.005,
|
|
545
|
+
# Generic method params
|
|
546
|
+
method_params: Optional[Dict[str, Any]] = None,
|
|
547
|
+
) -> Path:
|
|
548
|
+
"""Save steering config for a model/task."""
|
|
549
|
+
config = self._load_model_config(model_name)
|
|
550
|
+
|
|
551
|
+
steering = SteeringConfig(
|
|
552
|
+
layer=layer,
|
|
553
|
+
strength=strength,
|
|
554
|
+
method=method,
|
|
555
|
+
token_aggregation=token_aggregation,
|
|
556
|
+
prompt_strategy=prompt_strategy,
|
|
557
|
+
normalize_mode=normalize_mode,
|
|
558
|
+
strategy=strategy,
|
|
559
|
+
score=score,
|
|
560
|
+
metric=metric,
|
|
561
|
+
# PRISM
|
|
562
|
+
num_directions=num_directions,
|
|
563
|
+
direction_weighting=direction_weighting,
|
|
564
|
+
retain_weight=retain_weight,
|
|
565
|
+
independence_weight=independence_weight,
|
|
566
|
+
prism_optimization_steps=prism_optimization_steps,
|
|
567
|
+
use_caa_init=use_caa_init,
|
|
568
|
+
cone_constraint=cone_constraint,
|
|
569
|
+
min_cosine_similarity=min_cosine_similarity,
|
|
570
|
+
max_cosine_similarity=max_cosine_similarity,
|
|
571
|
+
# PULSE
|
|
572
|
+
sensor_layer=sensor_layer,
|
|
573
|
+
steering_layers=steering_layers,
|
|
574
|
+
condition_threshold=condition_threshold,
|
|
575
|
+
gate_temperature=gate_temperature,
|
|
576
|
+
per_layer_scaling=per_layer_scaling,
|
|
577
|
+
use_entropy_scaling=use_entropy_scaling,
|
|
578
|
+
max_alpha=max_alpha,
|
|
579
|
+
learn_threshold=learn_threshold,
|
|
580
|
+
pulse_optimization_steps=pulse_optimization_steps,
|
|
581
|
+
# TITAN
|
|
582
|
+
gate_hidden_dim=gate_hidden_dim,
|
|
583
|
+
intensity_hidden_dim=intensity_hidden_dim,
|
|
584
|
+
behavior_weight=behavior_weight,
|
|
585
|
+
sparse_weight=sparse_weight,
|
|
586
|
+
titan_optimization_steps=titan_optimization_steps,
|
|
587
|
+
titan_learning_rate=titan_learning_rate,
|
|
588
|
+
# Generic
|
|
589
|
+
method_params=method_params or {},
|
|
590
|
+
)
|
|
591
|
+
|
|
592
|
+
if task_name:
|
|
593
|
+
if task_name not in config.tasks:
|
|
594
|
+
config.tasks[task_name] = TaskConfig(task_name=task_name)
|
|
595
|
+
config.tasks[task_name].steering = steering
|
|
596
|
+
config.tasks[task_name].optimization_method = optimization_method
|
|
597
|
+
config.tasks[task_name].updated_at = datetime.now().isoformat()
|
|
598
|
+
|
|
599
|
+
if set_as_default or not task_name:
|
|
600
|
+
config.default_steering = steering
|
|
601
|
+
|
|
602
|
+
return self._save_model_config(config)
|
|
603
|
+
|
|
604
|
+
def get_steering_config(
|
|
605
|
+
self,
|
|
606
|
+
model_name: str,
|
|
607
|
+
task_name: Optional[str] = None,
|
|
608
|
+
) -> Optional[SteeringConfig]:
|
|
609
|
+
"""Get steering config for a model/task."""
|
|
610
|
+
config = self._load_model_config(model_name)
|
|
611
|
+
|
|
612
|
+
if task_name and task_name in config.tasks:
|
|
613
|
+
task_config = config.tasks[task_name]
|
|
614
|
+
if task_config.steering:
|
|
615
|
+
return task_config.steering
|
|
616
|
+
|
|
617
|
+
return config.default_steering
|
|
618
|
+
|
|
619
|
+
# ========== Weight Modification Methods ==========
|
|
620
|
+
|
|
621
|
+
def save_weight_modification_config(
|
|
622
|
+
self,
|
|
623
|
+
model_name: str,
|
|
624
|
+
task_name: Optional[str] = None,
|
|
625
|
+
trait_label: str = "",
|
|
626
|
+
method: str = "directional",
|
|
627
|
+
max_weight: float = 1.0,
|
|
628
|
+
min_weight: float = 0.0,
|
|
629
|
+
max_weight_position: float = 0.5,
|
|
630
|
+
min_weight_distance: float = 0.5,
|
|
631
|
+
strength: float = 1.0,
|
|
632
|
+
num_pairs: int = 100,
|
|
633
|
+
alpha: float = 1.0,
|
|
634
|
+
additive_method: str = "bias",
|
|
635
|
+
components: Optional[List[str]] = None,
|
|
636
|
+
normalize_vectors: bool = True,
|
|
637
|
+
norm_preserve: bool = True,
|
|
638
|
+
use_biprojection: bool = True,
|
|
639
|
+
use_kernel: bool = True,
|
|
640
|
+
score: float = 0.0,
|
|
641
|
+
baseline_score: float = 0.0,
|
|
642
|
+
output_dir: str = "",
|
|
643
|
+
optimization_method: str = "manual",
|
|
644
|
+
set_as_default: bool = False,
|
|
645
|
+
) -> Path:
|
|
646
|
+
"""Save weight modification config for a model/task."""
|
|
647
|
+
config = self._load_model_config(model_name)
|
|
648
|
+
|
|
649
|
+
weight_mod = WeightModificationConfig(
|
|
650
|
+
method=method,
|
|
651
|
+
max_weight=max_weight,
|
|
652
|
+
min_weight=min_weight,
|
|
653
|
+
max_weight_position=max_weight_position,
|
|
654
|
+
min_weight_distance=min_weight_distance,
|
|
655
|
+
strength=strength,
|
|
656
|
+
num_pairs=num_pairs,
|
|
657
|
+
alpha=alpha,
|
|
658
|
+
additive_method=additive_method,
|
|
659
|
+
components=components or ["self_attn.o_proj", "mlp.down_proj"],
|
|
660
|
+
normalize_vectors=normalize_vectors,
|
|
661
|
+
norm_preserve=norm_preserve,
|
|
662
|
+
use_biprojection=use_biprojection,
|
|
663
|
+
use_kernel=use_kernel,
|
|
664
|
+
score=score,
|
|
665
|
+
baseline_score=baseline_score,
|
|
666
|
+
output_dir=output_dir,
|
|
667
|
+
)
|
|
668
|
+
|
|
669
|
+
if task_name:
|
|
670
|
+
if task_name not in config.tasks:
|
|
671
|
+
config.tasks[task_name] = TaskConfig(task_name=task_name)
|
|
672
|
+
config.tasks[task_name].weight_modification = weight_mod
|
|
673
|
+
config.tasks[task_name].optimization_method = optimization_method
|
|
674
|
+
config.tasks[task_name].updated_at = datetime.now().isoformat()
|
|
675
|
+
|
|
676
|
+
if set_as_default or not task_name:
|
|
677
|
+
config.default_weight_modification = weight_mod
|
|
678
|
+
|
|
679
|
+
return self._save_model_config(config)
|
|
680
|
+
|
|
681
|
+
def get_weight_modification_config(
|
|
682
|
+
self,
|
|
683
|
+
model_name: str,
|
|
684
|
+
task_name: Optional[str] = None,
|
|
685
|
+
) -> Optional[WeightModificationConfig]:
|
|
686
|
+
"""Get weight modification config for a model/task."""
|
|
687
|
+
config = self._load_model_config(model_name)
|
|
688
|
+
|
|
689
|
+
if task_name and task_name in config.tasks:
|
|
690
|
+
task_config = config.tasks[task_name]
|
|
691
|
+
if task_config.weight_modification:
|
|
692
|
+
return task_config.weight_modification
|
|
693
|
+
|
|
694
|
+
return config.default_weight_modification
|
|
695
|
+
|
|
696
|
+
# ========== Trait-specific Methods ==========
|
|
697
|
+
|
|
698
|
+
def save_trait_classification_config(
|
|
699
|
+
self,
|
|
700
|
+
model_name: str,
|
|
701
|
+
trait_name: str,
|
|
702
|
+
layer: int = 12,
|
|
703
|
+
token_aggregation: str = "average",
|
|
704
|
+
detection_threshold: float = 0.6,
|
|
705
|
+
classifier_type: str = "logistic",
|
|
706
|
+
prompt_construction_strategy: str = "multiple_choice",
|
|
707
|
+
token_targeting_strategy: str = "last_token",
|
|
708
|
+
accuracy: float = 0.0,
|
|
709
|
+
f1_score: float = 0.0,
|
|
710
|
+
precision: float = 0.0,
|
|
711
|
+
recall: float = 0.0,
|
|
712
|
+
optimization_method: str = "manual",
|
|
713
|
+
set_as_default: bool = False,
|
|
714
|
+
) -> Path:
|
|
715
|
+
"""Save classification config for a trait."""
|
|
716
|
+
config = self._load_model_config(model_name)
|
|
717
|
+
|
|
718
|
+
classification = ClassificationConfig(
|
|
719
|
+
layer=layer,
|
|
720
|
+
token_aggregation=token_aggregation,
|
|
721
|
+
detection_threshold=detection_threshold,
|
|
722
|
+
classifier_type=classifier_type,
|
|
723
|
+
prompt_construction_strategy=prompt_construction_strategy,
|
|
724
|
+
token_targeting_strategy=token_targeting_strategy,
|
|
725
|
+
accuracy=accuracy,
|
|
726
|
+
f1_score=f1_score,
|
|
727
|
+
precision=precision,
|
|
728
|
+
recall=recall,
|
|
729
|
+
)
|
|
730
|
+
|
|
731
|
+
if trait_name not in config.traits:
|
|
732
|
+
config.traits[trait_name] = TraitConfig(trait_name=trait_name)
|
|
733
|
+
config.traits[trait_name].classification = classification
|
|
734
|
+
config.traits[trait_name].optimization_method = optimization_method
|
|
735
|
+
config.traits[trait_name].updated_at = datetime.now().isoformat()
|
|
736
|
+
|
|
737
|
+
if set_as_default:
|
|
738
|
+
config.default_classification = classification
|
|
739
|
+
|
|
740
|
+
return self._save_model_config(config)
|
|
741
|
+
|
|
742
|
+
def get_trait_classification_config(
|
|
743
|
+
self,
|
|
744
|
+
model_name: str,
|
|
745
|
+
trait_name: str,
|
|
746
|
+
) -> Optional[ClassificationConfig]:
|
|
747
|
+
"""Get classification config for a trait."""
|
|
748
|
+
config = self._load_model_config(model_name)
|
|
749
|
+
|
|
750
|
+
if trait_name in config.traits:
|
|
751
|
+
trait_config = config.traits[trait_name]
|
|
752
|
+
if trait_config.classification:
|
|
753
|
+
return trait_config.classification
|
|
754
|
+
|
|
755
|
+
return config.default_classification
|
|
756
|
+
|
|
757
|
+
def save_trait_steering_config(
|
|
758
|
+
self,
|
|
759
|
+
model_name: str,
|
|
760
|
+
trait_name: str,
|
|
761
|
+
layer: int = 12,
|
|
762
|
+
strength: float = 1.0,
|
|
763
|
+
method: str = "CAA",
|
|
764
|
+
token_aggregation: str = "average",
|
|
765
|
+
prompt_strategy: str = "question_only",
|
|
766
|
+
normalize_mode: str = "none",
|
|
767
|
+
score: float = 0.0,
|
|
768
|
+
metric: str = "accuracy",
|
|
769
|
+
optimization_method: str = "manual",
|
|
770
|
+
set_as_default: bool = False,
|
|
771
|
+
) -> Path:
|
|
772
|
+
"""Save steering config for a trait."""
|
|
773
|
+
config = self._load_model_config(model_name)
|
|
774
|
+
|
|
775
|
+
steering = SteeringConfig(
|
|
776
|
+
layer=layer,
|
|
777
|
+
strength=strength,
|
|
778
|
+
method=method,
|
|
779
|
+
token_aggregation=token_aggregation,
|
|
780
|
+
prompt_strategy=prompt_strategy,
|
|
781
|
+
normalize_mode=normalize_mode,
|
|
782
|
+
score=score,
|
|
783
|
+
metric=metric,
|
|
784
|
+
)
|
|
785
|
+
|
|
786
|
+
if trait_name not in config.traits:
|
|
787
|
+
config.traits[trait_name] = TraitConfig(trait_name=trait_name)
|
|
788
|
+
config.traits[trait_name].steering = steering
|
|
789
|
+
config.traits[trait_name].optimization_method = optimization_method
|
|
790
|
+
config.traits[trait_name].updated_at = datetime.now().isoformat()
|
|
791
|
+
|
|
792
|
+
if set_as_default:
|
|
793
|
+
config.default_steering = steering
|
|
794
|
+
|
|
795
|
+
return self._save_model_config(config)
|
|
796
|
+
|
|
797
|
+
def get_trait_steering_config(
|
|
798
|
+
self,
|
|
799
|
+
model_name: str,
|
|
800
|
+
trait_name: str,
|
|
801
|
+
) -> Optional[SteeringConfig]:
|
|
802
|
+
"""Get steering config for a trait."""
|
|
803
|
+
config = self._load_model_config(model_name)
|
|
804
|
+
|
|
805
|
+
if trait_name in config.traits:
|
|
806
|
+
trait_config = config.traits[trait_name]
|
|
807
|
+
if trait_config.steering:
|
|
808
|
+
return trait_config.steering
|
|
809
|
+
|
|
810
|
+
return config.default_steering
|
|
811
|
+
|
|
812
|
+
def save_trait_weight_modification_config(
|
|
813
|
+
self,
|
|
814
|
+
model_name: str,
|
|
815
|
+
trait_name: str,
|
|
816
|
+
method: str = "directional",
|
|
817
|
+
max_weight: float = 1.0,
|
|
818
|
+
min_weight: float = 0.0,
|
|
819
|
+
max_weight_position: float = 0.5,
|
|
820
|
+
min_weight_distance: float = 0.5,
|
|
821
|
+
strength: float = 1.0,
|
|
822
|
+
num_pairs: int = 100,
|
|
823
|
+
alpha: float = 1.0,
|
|
824
|
+
additive_method: str = "bias",
|
|
825
|
+
components: Optional[List[str]] = None,
|
|
826
|
+
normalize_vectors: bool = True,
|
|
827
|
+
norm_preserve: bool = True,
|
|
828
|
+
use_biprojection: bool = True,
|
|
829
|
+
use_kernel: bool = True,
|
|
830
|
+
score: float = 0.0,
|
|
831
|
+
baseline_score: float = 0.0,
|
|
832
|
+
output_dir: str = "",
|
|
833
|
+
optimization_method: str = "manual",
|
|
834
|
+
set_as_default: bool = False,
|
|
835
|
+
) -> Path:
|
|
836
|
+
"""Save weight modification config for a trait."""
|
|
837
|
+
config = self._load_model_config(model_name)
|
|
838
|
+
|
|
839
|
+
weight_mod = WeightModificationConfig(
|
|
840
|
+
method=method,
|
|
841
|
+
max_weight=max_weight,
|
|
842
|
+
min_weight=min_weight,
|
|
843
|
+
max_weight_position=max_weight_position,
|
|
844
|
+
min_weight_distance=min_weight_distance,
|
|
845
|
+
strength=strength,
|
|
846
|
+
num_pairs=num_pairs,
|
|
847
|
+
alpha=alpha,
|
|
848
|
+
additive_method=additive_method,
|
|
849
|
+
components=components or ["self_attn.o_proj", "mlp.down_proj"],
|
|
850
|
+
normalize_vectors=normalize_vectors,
|
|
851
|
+
norm_preserve=norm_preserve,
|
|
852
|
+
use_biprojection=use_biprojection,
|
|
853
|
+
use_kernel=use_kernel,
|
|
854
|
+
score=score,
|
|
855
|
+
baseline_score=baseline_score,
|
|
856
|
+
output_dir=output_dir,
|
|
857
|
+
)
|
|
858
|
+
|
|
859
|
+
if trait_name not in config.traits:
|
|
860
|
+
config.traits[trait_name] = TraitConfig(trait_name=trait_name)
|
|
861
|
+
config.traits[trait_name].weight_modification = weight_mod
|
|
862
|
+
config.traits[trait_name].optimization_method = optimization_method
|
|
863
|
+
config.traits[trait_name].updated_at = datetime.now().isoformat()
|
|
864
|
+
|
|
865
|
+
if set_as_default:
|
|
866
|
+
config.default_weight_modification = weight_mod
|
|
867
|
+
|
|
868
|
+
return self._save_model_config(config)
|
|
869
|
+
|
|
870
|
+
def get_trait_weight_modification_config(
|
|
871
|
+
self,
|
|
872
|
+
model_name: str,
|
|
873
|
+
trait_name: str,
|
|
874
|
+
) -> Optional[WeightModificationConfig]:
|
|
875
|
+
"""Get weight modification config for a trait."""
|
|
876
|
+
config = self._load_model_config(model_name)
|
|
877
|
+
|
|
878
|
+
if trait_name in config.traits:
|
|
879
|
+
trait_config = config.traits[trait_name]
|
|
880
|
+
if trait_config.weight_modification:
|
|
881
|
+
return trait_config.weight_modification
|
|
882
|
+
|
|
883
|
+
return config.default_weight_modification
|
|
884
|
+
|
|
885
|
+
# ========== General Methods ==========
|
|
886
|
+
|
|
887
|
+
def get_model_config(self, model_name: str) -> ModelConfig:
|
|
888
|
+
"""Get the full model config."""
|
|
889
|
+
return self._load_model_config(model_name)
|
|
890
|
+
|
|
891
|
+
def has_config(self, model_name: str) -> bool:
|
|
892
|
+
"""Check if a model has any saved configuration."""
|
|
893
|
+
config_path = self._get_config_path(model_name)
|
|
894
|
+
return config_path.exists()
|
|
895
|
+
|
|
896
|
+
def list_models(self) -> List[str]:
|
|
897
|
+
"""List all models with saved configurations."""
|
|
898
|
+
models = []
|
|
899
|
+
for path in self.config_dir.glob("*.json"):
|
|
900
|
+
try:
|
|
901
|
+
with open(path) as f:
|
|
902
|
+
data = json.load(f)
|
|
903
|
+
models.append(data.get("model_name", path.stem))
|
|
904
|
+
except (json.JSONDecodeError, KeyError):
|
|
905
|
+
continue
|
|
906
|
+
return models
|
|
907
|
+
|
|
908
|
+
def list_tasks(self, model_name: str) -> List[str]:
|
|
909
|
+
"""List all tasks with saved configurations for a model."""
|
|
910
|
+
config = self._load_model_config(model_name)
|
|
911
|
+
return list(config.tasks.keys())
|
|
912
|
+
|
|
913
|
+
def list_traits(self, model_name: str) -> List[str]:
|
|
914
|
+
"""List all traits with saved configurations for a model."""
|
|
915
|
+
config = self._load_model_config(model_name)
|
|
916
|
+
return list(config.traits.keys())
|
|
917
|
+
|
|
918
|
+
def delete_config(self, model_name: str) -> bool:
|
|
919
|
+
"""Delete all configuration for a model."""
|
|
920
|
+
config_path = self._get_config_path(model_name)
|
|
921
|
+
if config_path.exists():
|
|
922
|
+
config_path.unlink()
|
|
923
|
+
if model_name in self._cache:
|
|
924
|
+
del self._cache[model_name]
|
|
925
|
+
return True
|
|
926
|
+
return False
|
|
927
|
+
|
|
928
|
+
def delete_task_config(self, model_name: str, task_name: str) -> bool:
|
|
929
|
+
"""Delete configuration for a specific task."""
|
|
930
|
+
config = self._load_model_config(model_name)
|
|
931
|
+
if task_name in config.tasks:
|
|
932
|
+
del config.tasks[task_name]
|
|
933
|
+
self._save_model_config(config)
|
|
934
|
+
return True
|
|
935
|
+
return False
|
|
936
|
+
|
|
937
|
+
def delete_trait_config(self, model_name: str, trait_name: str) -> bool:
|
|
938
|
+
"""Delete configuration for a specific trait."""
|
|
939
|
+
config = self._load_model_config(model_name)
|
|
940
|
+
if trait_name in config.traits:
|
|
941
|
+
del config.traits[trait_name]
|
|
942
|
+
self._save_model_config(config)
|
|
943
|
+
return True
|
|
944
|
+
return False
|
|
945
|
+
|
|
946
|
+
|
|
947
|
+
# Global instance
|
|
948
|
+
_config_manager: Optional[WisentConfigManager] = None
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def get_config_manager() -> WisentConfigManager:
|
|
952
|
+
"""Get the global config manager instance."""
|
|
953
|
+
global _config_manager
|
|
954
|
+
if _config_manager is None:
|
|
955
|
+
_config_manager = WisentConfigManager()
|
|
956
|
+
return _config_manager
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
# Convenience functions
|
|
960
|
+
def save_classification_config(model_name: str, **kwargs) -> Path:
|
|
961
|
+
"""Save classification config using global manager."""
|
|
962
|
+
return get_config_manager().save_classification_config(model_name, **kwargs)
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
def get_classification_config(model_name: str, task_name: Optional[str] = None) -> Optional[ClassificationConfig]:
|
|
966
|
+
"""Get classification config using global manager."""
|
|
967
|
+
return get_config_manager().get_classification_config(model_name, task_name)
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def save_steering_config(model_name: str, **kwargs) -> Path:
|
|
971
|
+
"""Save steering config using global manager."""
|
|
972
|
+
return get_config_manager().save_steering_config(model_name, **kwargs)
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
def get_steering_config(model_name: str, task_name: Optional[str] = None) -> Optional[SteeringConfig]:
|
|
976
|
+
"""Get steering config using global manager."""
|
|
977
|
+
return get_config_manager().get_steering_config(model_name, task_name)
|
|
978
|
+
|
|
979
|
+
|
|
980
|
+
def save_weight_modification_config(model_name: str, **kwargs) -> Path:
|
|
981
|
+
"""Save weight modification config using global manager."""
|
|
982
|
+
return get_config_manager().save_weight_modification_config(model_name, **kwargs)
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
def get_weight_modification_config(
|
|
986
|
+
model_name: str,
|
|
987
|
+
task_name: Optional[str] = None,
|
|
988
|
+
) -> Optional[WeightModificationConfig]:
|
|
989
|
+
"""Get weight modification config using global manager."""
|
|
990
|
+
return get_config_manager().get_weight_modification_config(model_name, task_name)
|
|
991
|
+
|
|
992
|
+
|
|
993
|
+
# Trait convenience functions
|
|
994
|
+
def save_trait_classification_config(model_name: str, trait_name: str, **kwargs) -> Path:
|
|
995
|
+
"""Save classification config for a trait using global manager."""
|
|
996
|
+
return get_config_manager().save_trait_classification_config(model_name, trait_name, **kwargs)
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def get_trait_classification_config(model_name: str, trait_name: str) -> Optional[ClassificationConfig]:
|
|
1000
|
+
"""Get classification config for a trait using global manager."""
|
|
1001
|
+
return get_config_manager().get_trait_classification_config(model_name, trait_name)
|
|
1002
|
+
|
|
1003
|
+
|
|
1004
|
+
def save_trait_steering_config(model_name: str, trait_name: str, **kwargs) -> Path:
|
|
1005
|
+
"""Save steering config for a trait using global manager."""
|
|
1006
|
+
return get_config_manager().save_trait_steering_config(model_name, trait_name, **kwargs)
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
def get_trait_steering_config(model_name: str, trait_name: str) -> Optional[SteeringConfig]:
|
|
1010
|
+
"""Get steering config for a trait using global manager."""
|
|
1011
|
+
return get_config_manager().get_trait_steering_config(model_name, trait_name)
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
def save_trait_weight_modification_config(model_name: str, trait_name: str, **kwargs) -> Path:
|
|
1015
|
+
"""Save weight modification config for a trait using global manager."""
|
|
1016
|
+
return get_config_manager().save_trait_weight_modification_config(model_name, trait_name, **kwargs)
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def get_trait_weight_modification_config(model_name: str, trait_name: str) -> Optional[WeightModificationConfig]:
|
|
1020
|
+
"""Get weight modification config for a trait using global manager."""
|
|
1021
|
+
return get_config_manager().get_trait_weight_modification_config(model_name, trait_name)
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
# ========== Backward Compatibility Layer ==========
|
|
1025
|
+
# These functions provide compatibility with the old OptimizationCache API
|
|
1026
|
+
|
|
1027
|
+
@dataclass
|
|
1028
|
+
class OptimizationResult:
|
|
1029
|
+
"""Backward-compatible result class for steering optimization cache."""
|
|
1030
|
+
model: str
|
|
1031
|
+
task: str
|
|
1032
|
+
layer: int
|
|
1033
|
+
strength: float
|
|
1034
|
+
method: str = "CAA"
|
|
1035
|
+
token_aggregation: str = "average"
|
|
1036
|
+
prompt_strategy: str = "question_only"
|
|
1037
|
+
strategy: str = "constant"
|
|
1038
|
+
score: float = 0.0
|
|
1039
|
+
metric: str = "accuracy"
|
|
1040
|
+
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
|
1041
|
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
|
1042
|
+
|
|
1043
|
+
# Method-specific parameters
|
|
1044
|
+
# PRISM
|
|
1045
|
+
num_directions: int = 1
|
|
1046
|
+
direction_weighting: str = "primary_only"
|
|
1047
|
+
retain_weight: float = 0.0
|
|
1048
|
+
independence_weight: float = 0.05
|
|
1049
|
+
prism_optimization_steps: int = 100
|
|
1050
|
+
use_caa_init: bool = True
|
|
1051
|
+
cone_constraint: bool = True
|
|
1052
|
+
min_cosine_similarity: float = 0.3
|
|
1053
|
+
max_cosine_similarity: float = 0.95
|
|
1054
|
+
# PULSE
|
|
1055
|
+
sensor_layer: int = -1
|
|
1056
|
+
steering_layers: str = ""
|
|
1057
|
+
condition_threshold: float = 0.5
|
|
1058
|
+
gate_temperature: float = 0.5
|
|
1059
|
+
per_layer_scaling: bool = True
|
|
1060
|
+
use_entropy_scaling: bool = False
|
|
1061
|
+
max_alpha: float = 2.0
|
|
1062
|
+
learn_threshold: bool = True
|
|
1063
|
+
pulse_optimization_steps: int = 100
|
|
1064
|
+
# TITAN
|
|
1065
|
+
gate_hidden_dim: int = 64
|
|
1066
|
+
intensity_hidden_dim: int = 32
|
|
1067
|
+
behavior_weight: float = 1.0
|
|
1068
|
+
sparse_weight: float = 0.05
|
|
1069
|
+
titan_optimization_steps: int = 200
|
|
1070
|
+
titan_learning_rate: float = 0.005
|
|
1071
|
+
# Generic
|
|
1072
|
+
method_params: Dict[str, Any] = field(default_factory=dict)
|
|
1073
|
+
|
|
1074
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
1075
|
+
return asdict(self)
|
|
1076
|
+
|
|
1077
|
+
@classmethod
|
|
1078
|
+
def from_dict(cls, data: Dict[str, Any]) -> "OptimizationResult":
|
|
1079
|
+
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
|
|
1080
|
+
|
|
1081
|
+
|
|
1082
|
+
def store_optimization(
|
|
1083
|
+
model: str,
|
|
1084
|
+
task: str,
|
|
1085
|
+
layer: int,
|
|
1086
|
+
strength: float,
|
|
1087
|
+
method: str = "CAA",
|
|
1088
|
+
token_aggregation: str = "average",
|
|
1089
|
+
prompt_strategy: str = "question_only",
|
|
1090
|
+
strategy: str = "constant",
|
|
1091
|
+
score: float = 0.0,
|
|
1092
|
+
metric: str = "accuracy",
|
|
1093
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
1094
|
+
set_as_default: bool = False,
|
|
1095
|
+
# PRISM parameters
|
|
1096
|
+
num_directions: int = 1,
|
|
1097
|
+
direction_weighting: str = "primary_only",
|
|
1098
|
+
retain_weight: float = 0.0,
|
|
1099
|
+
independence_weight: float = 0.05,
|
|
1100
|
+
prism_optimization_steps: int = 100,
|
|
1101
|
+
use_caa_init: bool = True,
|
|
1102
|
+
cone_constraint: bool = True,
|
|
1103
|
+
min_cosine_similarity: float = 0.3,
|
|
1104
|
+
max_cosine_similarity: float = 0.95,
|
|
1105
|
+
# PULSE parameters
|
|
1106
|
+
sensor_layer: int = -1,
|
|
1107
|
+
steering_layers: str = "",
|
|
1108
|
+
condition_threshold: float = 0.5,
|
|
1109
|
+
gate_temperature: float = 0.5,
|
|
1110
|
+
per_layer_scaling: bool = True,
|
|
1111
|
+
use_entropy_scaling: bool = False,
|
|
1112
|
+
max_alpha: float = 2.0,
|
|
1113
|
+
learn_threshold: bool = True,
|
|
1114
|
+
pulse_optimization_steps: int = 100,
|
|
1115
|
+
# TITAN parameters
|
|
1116
|
+
gate_hidden_dim: int = 64,
|
|
1117
|
+
intensity_hidden_dim: int = 32,
|
|
1118
|
+
behavior_weight: float = 1.0,
|
|
1119
|
+
sparse_weight: float = 0.05,
|
|
1120
|
+
titan_optimization_steps: int = 200,
|
|
1121
|
+
titan_learning_rate: float = 0.005,
|
|
1122
|
+
# Generic
|
|
1123
|
+
method_params: Optional[Dict[str, Any]] = None,
|
|
1124
|
+
) -> str:
|
|
1125
|
+
"""
|
|
1126
|
+
Backward-compatible function to store steering optimization result.
|
|
1127
|
+
Maps to the new unified config manager.
|
|
1128
|
+
"""
|
|
1129
|
+
config_path = save_steering_config(
|
|
1130
|
+
model_name=model,
|
|
1131
|
+
task_name=task,
|
|
1132
|
+
layer=layer,
|
|
1133
|
+
strength=strength,
|
|
1134
|
+
method=method,
|
|
1135
|
+
token_aggregation=token_aggregation,
|
|
1136
|
+
prompt_strategy=prompt_strategy,
|
|
1137
|
+
strategy=strategy,
|
|
1138
|
+
score=score,
|
|
1139
|
+
metric=metric,
|
|
1140
|
+
optimization_method="optuna" if metadata else "manual",
|
|
1141
|
+
set_as_default=set_as_default,
|
|
1142
|
+
# PRISM
|
|
1143
|
+
num_directions=num_directions,
|
|
1144
|
+
direction_weighting=direction_weighting,
|
|
1145
|
+
retain_weight=retain_weight,
|
|
1146
|
+
independence_weight=independence_weight,
|
|
1147
|
+
prism_optimization_steps=prism_optimization_steps,
|
|
1148
|
+
use_caa_init=use_caa_init,
|
|
1149
|
+
cone_constraint=cone_constraint,
|
|
1150
|
+
min_cosine_similarity=min_cosine_similarity,
|
|
1151
|
+
max_cosine_similarity=max_cosine_similarity,
|
|
1152
|
+
# PULSE
|
|
1153
|
+
sensor_layer=sensor_layer,
|
|
1154
|
+
steering_layers=steering_layers,
|
|
1155
|
+
condition_threshold=condition_threshold,
|
|
1156
|
+
gate_temperature=gate_temperature,
|
|
1157
|
+
per_layer_scaling=per_layer_scaling,
|
|
1158
|
+
use_entropy_scaling=use_entropy_scaling,
|
|
1159
|
+
max_alpha=max_alpha,
|
|
1160
|
+
learn_threshold=learn_threshold,
|
|
1161
|
+
pulse_optimization_steps=pulse_optimization_steps,
|
|
1162
|
+
# TITAN
|
|
1163
|
+
gate_hidden_dim=gate_hidden_dim,
|
|
1164
|
+
intensity_hidden_dim=intensity_hidden_dim,
|
|
1165
|
+
behavior_weight=behavior_weight,
|
|
1166
|
+
sparse_weight=sparse_weight,
|
|
1167
|
+
titan_optimization_steps=titan_optimization_steps,
|
|
1168
|
+
titan_learning_rate=titan_learning_rate,
|
|
1169
|
+
# Generic
|
|
1170
|
+
method_params=method_params,
|
|
1171
|
+
)
|
|
1172
|
+
# Return a cache key for backward compatibility
|
|
1173
|
+
model_normalized = model.replace("/", "_").replace("\\", "_")
|
|
1174
|
+
return f"{model_normalized}::{task}::{method}"
|
|
1175
|
+
|
|
1176
|
+
|
|
1177
|
+
def get_cached_optimization(
|
|
1178
|
+
model: str,
|
|
1179
|
+
task: str,
|
|
1180
|
+
method: str = "CAA",
|
|
1181
|
+
use_default: bool = True
|
|
1182
|
+
) -> Optional[OptimizationResult]:
|
|
1183
|
+
"""
|
|
1184
|
+
Backward-compatible function to get cached steering optimization result.
|
|
1185
|
+
Maps to the new unified config manager.
|
|
1186
|
+
"""
|
|
1187
|
+
steering = get_steering_config(model, task)
|
|
1188
|
+
|
|
1189
|
+
if steering is None:
|
|
1190
|
+
return None
|
|
1191
|
+
|
|
1192
|
+
# Only return if method matches (or method is wildcard)
|
|
1193
|
+
if method != "*" and steering.method != method:
|
|
1194
|
+
return None
|
|
1195
|
+
|
|
1196
|
+
return OptimizationResult(
|
|
1197
|
+
model=model,
|
|
1198
|
+
task=task,
|
|
1199
|
+
layer=steering.layer,
|
|
1200
|
+
strength=steering.strength,
|
|
1201
|
+
method=steering.method,
|
|
1202
|
+
token_aggregation=steering.token_aggregation,
|
|
1203
|
+
prompt_strategy=steering.prompt_strategy,
|
|
1204
|
+
strategy=steering.strategy,
|
|
1205
|
+
score=steering.score,
|
|
1206
|
+
metric=steering.metric,
|
|
1207
|
+
# PRISM
|
|
1208
|
+
num_directions=steering.num_directions,
|
|
1209
|
+
direction_weighting=steering.direction_weighting,
|
|
1210
|
+
retain_weight=steering.retain_weight,
|
|
1211
|
+
independence_weight=steering.independence_weight,
|
|
1212
|
+
prism_optimization_steps=steering.prism_optimization_steps,
|
|
1213
|
+
use_caa_init=steering.use_caa_init,
|
|
1214
|
+
cone_constraint=steering.cone_constraint,
|
|
1215
|
+
min_cosine_similarity=steering.min_cosine_similarity,
|
|
1216
|
+
max_cosine_similarity=steering.max_cosine_similarity,
|
|
1217
|
+
# PULSE
|
|
1218
|
+
sensor_layer=steering.sensor_layer,
|
|
1219
|
+
steering_layers=steering.steering_layers,
|
|
1220
|
+
condition_threshold=steering.condition_threshold,
|
|
1221
|
+
gate_temperature=steering.gate_temperature,
|
|
1222
|
+
per_layer_scaling=steering.per_layer_scaling,
|
|
1223
|
+
use_entropy_scaling=steering.use_entropy_scaling,
|
|
1224
|
+
max_alpha=steering.max_alpha,
|
|
1225
|
+
learn_threshold=steering.learn_threshold,
|
|
1226
|
+
pulse_optimization_steps=steering.pulse_optimization_steps,
|
|
1227
|
+
# TITAN
|
|
1228
|
+
gate_hidden_dim=steering.gate_hidden_dim,
|
|
1229
|
+
intensity_hidden_dim=steering.intensity_hidden_dim,
|
|
1230
|
+
behavior_weight=steering.behavior_weight,
|
|
1231
|
+
sparse_weight=steering.sparse_weight,
|
|
1232
|
+
titan_optimization_steps=steering.titan_optimization_steps,
|
|
1233
|
+
titan_learning_rate=steering.titan_learning_rate,
|
|
1234
|
+
# Generic
|
|
1235
|
+
method_params=steering.method_params,
|
|
1236
|
+
)
|
|
1237
|
+
|
|
1238
|
+
|
|
1239
|
+
# ========== Weight Modification Backward Compatibility ==========
|
|
1240
|
+
# These functions provide compatibility with the old WeightModificationCache API
|
|
1241
|
+
|
|
1242
|
+
@dataclass
|
|
1243
|
+
class WeightModificationResult:
|
|
1244
|
+
"""Backward-compatible result class for weight modification cache."""
|
|
1245
|
+
model: str
|
|
1246
|
+
task: str
|
|
1247
|
+
trait_label: str
|
|
1248
|
+
method: str = "directional"
|
|
1249
|
+
max_weight: float = 1.0
|
|
1250
|
+
min_weight: float = 0.0
|
|
1251
|
+
max_weight_position: float = 0.5
|
|
1252
|
+
min_weight_distance: float = 0.5
|
|
1253
|
+
strength: float = 1.0
|
|
1254
|
+
num_pairs: int = 100
|
|
1255
|
+
alpha: float = 1.0
|
|
1256
|
+
additive_method: str = "bias"
|
|
1257
|
+
components: List[str] = field(default_factory=lambda: ["self_attn.o_proj", "mlp.down_proj"])
|
|
1258
|
+
normalize_vectors: bool = True
|
|
1259
|
+
norm_preserve: bool = True
|
|
1260
|
+
use_biprojection: bool = True
|
|
1261
|
+
use_kernel: bool = True
|
|
1262
|
+
score: float = 0.0
|
|
1263
|
+
metric: str = "accuracy"
|
|
1264
|
+
baseline_score: float = 0.0
|
|
1265
|
+
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
|
1266
|
+
output_dir: str = ""
|
|
1267
|
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
|
1268
|
+
|
|
1269
|
+
|
|
1270
|
+
def store_weight_modification(
|
|
1271
|
+
model: str,
|
|
1272
|
+
task: str,
|
|
1273
|
+
trait_label: str,
|
|
1274
|
+
method: str = "directional",
|
|
1275
|
+
max_weight: float = 1.0,
|
|
1276
|
+
min_weight: float = 0.0,
|
|
1277
|
+
max_weight_position: float = 0.5,
|
|
1278
|
+
min_weight_distance: float = 0.5,
|
|
1279
|
+
strength: float = 1.0,
|
|
1280
|
+
num_pairs: int = 100,
|
|
1281
|
+
alpha: float = 1.0,
|
|
1282
|
+
additive_method: str = "bias",
|
|
1283
|
+
components: Optional[List[str]] = None,
|
|
1284
|
+
normalize_vectors: bool = True,
|
|
1285
|
+
norm_preserve: bool = True,
|
|
1286
|
+
use_biprojection: bool = True,
|
|
1287
|
+
use_kernel: bool = True,
|
|
1288
|
+
score: float = 0.0,
|
|
1289
|
+
metric: str = "accuracy",
|
|
1290
|
+
baseline_score: float = 0.0,
|
|
1291
|
+
output_dir: str = "",
|
|
1292
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
1293
|
+
set_as_default: bool = False,
|
|
1294
|
+
) -> str:
|
|
1295
|
+
"""
|
|
1296
|
+
Backward-compatible function to store weight modification result.
|
|
1297
|
+
Maps to the new unified config manager.
|
|
1298
|
+
|
|
1299
|
+
If trait_label is provided, stores as a trait config.
|
|
1300
|
+
If task is provided without trait_label, stores as a task config.
|
|
1301
|
+
"""
|
|
1302
|
+
# If trait_label is provided, store as trait config
|
|
1303
|
+
if trait_label:
|
|
1304
|
+
save_trait_weight_modification_config(
|
|
1305
|
+
model_name=model,
|
|
1306
|
+
trait_name=trait_label,
|
|
1307
|
+
method=method,
|
|
1308
|
+
max_weight=max_weight,
|
|
1309
|
+
min_weight=min_weight,
|
|
1310
|
+
max_weight_position=max_weight_position,
|
|
1311
|
+
min_weight_distance=min_weight_distance,
|
|
1312
|
+
strength=strength,
|
|
1313
|
+
num_pairs=num_pairs,
|
|
1314
|
+
alpha=alpha,
|
|
1315
|
+
additive_method=additive_method,
|
|
1316
|
+
components=components,
|
|
1317
|
+
normalize_vectors=normalize_vectors,
|
|
1318
|
+
norm_preserve=norm_preserve,
|
|
1319
|
+
use_biprojection=use_biprojection,
|
|
1320
|
+
use_kernel=use_kernel,
|
|
1321
|
+
score=score,
|
|
1322
|
+
baseline_score=baseline_score,
|
|
1323
|
+
output_dir=output_dir,
|
|
1324
|
+
optimization_method="optuna" if metadata else "manual",
|
|
1325
|
+
set_as_default=set_as_default,
|
|
1326
|
+
)
|
|
1327
|
+
else:
|
|
1328
|
+
# Store as task config
|
|
1329
|
+
save_weight_modification_config(
|
|
1330
|
+
model_name=model,
|
|
1331
|
+
task_name=task,
|
|
1332
|
+
method=method,
|
|
1333
|
+
max_weight=max_weight,
|
|
1334
|
+
min_weight=min_weight,
|
|
1335
|
+
max_weight_position=max_weight_position,
|
|
1336
|
+
min_weight_distance=min_weight_distance,
|
|
1337
|
+
strength=strength,
|
|
1338
|
+
num_pairs=num_pairs,
|
|
1339
|
+
alpha=alpha,
|
|
1340
|
+
additive_method=additive_method,
|
|
1341
|
+
components=components,
|
|
1342
|
+
normalize_vectors=normalize_vectors,
|
|
1343
|
+
norm_preserve=norm_preserve,
|
|
1344
|
+
use_biprojection=use_biprojection,
|
|
1345
|
+
use_kernel=use_kernel,
|
|
1346
|
+
score=score,
|
|
1347
|
+
baseline_score=baseline_score,
|
|
1348
|
+
output_dir=output_dir,
|
|
1349
|
+
optimization_method="optuna" if metadata else "manual",
|
|
1350
|
+
set_as_default=set_as_default,
|
|
1351
|
+
)
|
|
1352
|
+
|
|
1353
|
+
# Return a cache key for backward compatibility
|
|
1354
|
+
model_normalized = model.replace("/", "_").replace("\\", "_")
|
|
1355
|
+
return f"{model_normalized}::{task}::{trait_label}::{method}"
|
|
1356
|
+
|
|
1357
|
+
|
|
1358
|
+
def get_cached_weight_modification(
|
|
1359
|
+
model: str,
|
|
1360
|
+
task: str,
|
|
1361
|
+
trait_label: str,
|
|
1362
|
+
method: str = "directional",
|
|
1363
|
+
use_default: bool = True,
|
|
1364
|
+
) -> Optional[WeightModificationResult]:
|
|
1365
|
+
"""
|
|
1366
|
+
Backward-compatible function to get cached weight modification result.
|
|
1367
|
+
Maps to the new unified config manager.
|
|
1368
|
+
|
|
1369
|
+
If trait_label is provided, gets from trait config.
|
|
1370
|
+
If task is provided without trait_label, gets from task config.
|
|
1371
|
+
"""
|
|
1372
|
+
# If trait_label is provided, get from trait config
|
|
1373
|
+
if trait_label:
|
|
1374
|
+
weight_mod = get_trait_weight_modification_config(model, trait_label)
|
|
1375
|
+
else:
|
|
1376
|
+
weight_mod = get_weight_modification_config(model, task)
|
|
1377
|
+
|
|
1378
|
+
if weight_mod is None:
|
|
1379
|
+
return None
|
|
1380
|
+
|
|
1381
|
+
# Only return if method matches
|
|
1382
|
+
if method != "*" and weight_mod.method != method:
|
|
1383
|
+
return None
|
|
1384
|
+
|
|
1385
|
+
return WeightModificationResult(
|
|
1386
|
+
model=model,
|
|
1387
|
+
task=task,
|
|
1388
|
+
trait_label=trait_label,
|
|
1389
|
+
method=weight_mod.method,
|
|
1390
|
+
max_weight=weight_mod.max_weight,
|
|
1391
|
+
min_weight=weight_mod.min_weight,
|
|
1392
|
+
max_weight_position=weight_mod.max_weight_position,
|
|
1393
|
+
min_weight_distance=weight_mod.min_weight_distance,
|
|
1394
|
+
strength=weight_mod.strength,
|
|
1395
|
+
num_pairs=weight_mod.num_pairs,
|
|
1396
|
+
alpha=weight_mod.alpha,
|
|
1397
|
+
additive_method=weight_mod.additive_method,
|
|
1398
|
+
components=weight_mod.components,
|
|
1399
|
+
normalize_vectors=weight_mod.normalize_vectors,
|
|
1400
|
+
norm_preserve=weight_mod.norm_preserve,
|
|
1401
|
+
use_biprojection=weight_mod.use_biprojection,
|
|
1402
|
+
use_kernel=weight_mod.use_kernel,
|
|
1403
|
+
score=weight_mod.score,
|
|
1404
|
+
baseline_score=weight_mod.baseline_score,
|
|
1405
|
+
output_dir=weight_mod.output_dir,
|
|
1406
|
+
)
|
|
1407
|
+
|
|
1408
|
+
|
|
1409
|
+
def get_weight_modification_cache():
|
|
1410
|
+
"""
|
|
1411
|
+
Backward-compatible function that returns the global config manager.
|
|
1412
|
+
This allows existing code expecting a cache object to work.
|
|
1413
|
+
"""
|
|
1414
|
+
return get_config_manager()
|
|
1415
|
+
|
|
1416
|
+
|
|
1417
|
+
# ========== Steering Cache Backward Compatibility ==========
|
|
1418
|
+
# Provides compatibility with the old OptimizationCache class API
|
|
1419
|
+
|
|
1420
|
+
class OptimizationCache:
|
|
1421
|
+
"""
|
|
1422
|
+
Backward-compatible wrapper class for the unified config manager.
|
|
1423
|
+
Provides the same interface as the old OptimizationCache class.
|
|
1424
|
+
"""
|
|
1425
|
+
|
|
1426
|
+
def __init__(self):
|
|
1427
|
+
self._manager = get_config_manager()
|
|
1428
|
+
self._defaults: Dict[str, str] = {}
|
|
1429
|
+
|
|
1430
|
+
def _make_key(self, model: str, task: str, method: str = "CAA") -> str:
|
|
1431
|
+
model_normalized = model.replace("/", "_").replace("\\", "_")
|
|
1432
|
+
return f"{model_normalized}::{task}::{method}"
|
|
1433
|
+
|
|
1434
|
+
def store(
|
|
1435
|
+
self,
|
|
1436
|
+
model: str,
|
|
1437
|
+
task: str,
|
|
1438
|
+
layer: int,
|
|
1439
|
+
strength: float,
|
|
1440
|
+
method: str = "CAA",
|
|
1441
|
+
token_aggregation: str = "average",
|
|
1442
|
+
prompt_strategy: str = "question_only",
|
|
1443
|
+
score: float = 0.0,
|
|
1444
|
+
metric: str = "accuracy",
|
|
1445
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
1446
|
+
set_as_default: bool = False,
|
|
1447
|
+
) -> str:
|
|
1448
|
+
"""Store an optimization result."""
|
|
1449
|
+
return store_optimization(
|
|
1450
|
+
model=model,
|
|
1451
|
+
task=task,
|
|
1452
|
+
layer=layer,
|
|
1453
|
+
strength=strength,
|
|
1454
|
+
method=method,
|
|
1455
|
+
token_aggregation=token_aggregation,
|
|
1456
|
+
prompt_strategy=prompt_strategy,
|
|
1457
|
+
score=score,
|
|
1458
|
+
metric=metric,
|
|
1459
|
+
metadata=metadata,
|
|
1460
|
+
set_as_default=set_as_default,
|
|
1461
|
+
)
|
|
1462
|
+
|
|
1463
|
+
def get(
|
|
1464
|
+
self,
|
|
1465
|
+
model: str,
|
|
1466
|
+
task: str,
|
|
1467
|
+
method: str = "CAA",
|
|
1468
|
+
) -> Optional[OptimizationResult]:
|
|
1469
|
+
"""Get a cached optimization result."""
|
|
1470
|
+
return get_cached_optimization(model, task, method, use_default=False)
|
|
1471
|
+
|
|
1472
|
+
def get_default(self, model: str, task: str) -> Optional[OptimizationResult]:
|
|
1473
|
+
"""Get the default optimization result for a model/task."""
|
|
1474
|
+
return get_cached_optimization(model, task, "*", use_default=True)
|
|
1475
|
+
|
|
1476
|
+
def set_default(self, model: str, task: str, method: str = "CAA") -> bool:
|
|
1477
|
+
"""Set a cached result as the default."""
|
|
1478
|
+
# Get the existing steering config
|
|
1479
|
+
steering = get_steering_config(model, task)
|
|
1480
|
+
if steering is None:
|
|
1481
|
+
return False
|
|
1482
|
+
|
|
1483
|
+
# Save it again with set_as_default=True
|
|
1484
|
+
save_steering_config(
|
|
1485
|
+
model_name=model,
|
|
1486
|
+
task_name=task,
|
|
1487
|
+
layer=steering.layer,
|
|
1488
|
+
strength=steering.strength,
|
|
1489
|
+
method=steering.method,
|
|
1490
|
+
token_aggregation=steering.token_aggregation,
|
|
1491
|
+
prompt_strategy=steering.prompt_strategy,
|
|
1492
|
+
score=steering.score,
|
|
1493
|
+
metric=steering.metric,
|
|
1494
|
+
set_as_default=True,
|
|
1495
|
+
)
|
|
1496
|
+
return True
|
|
1497
|
+
|
|
1498
|
+
def exists(self, model: str, task: str, method: str = "CAA") -> bool:
|
|
1499
|
+
"""Check if a cached result exists."""
|
|
1500
|
+
result = get_cached_optimization(model, task, method, use_default=False)
|
|
1501
|
+
return result is not None
|
|
1502
|
+
|
|
1503
|
+
def list_cached(
|
|
1504
|
+
self,
|
|
1505
|
+
model: Optional[str] = None,
|
|
1506
|
+
task: Optional[str] = None,
|
|
1507
|
+
) -> List[OptimizationResult]:
|
|
1508
|
+
"""List cached results, optionally filtered."""
|
|
1509
|
+
results = []
|
|
1510
|
+
|
|
1511
|
+
# If model specified, only look at that model
|
|
1512
|
+
if model:
|
|
1513
|
+
models = [model]
|
|
1514
|
+
else:
|
|
1515
|
+
models = self._manager.list_models()
|
|
1516
|
+
|
|
1517
|
+
for m in models:
|
|
1518
|
+
config = self._manager.get_model_config(m)
|
|
1519
|
+
|
|
1520
|
+
# Check default steering
|
|
1521
|
+
if config.default_steering:
|
|
1522
|
+
if not task: # No task filter, include default
|
|
1523
|
+
results.append(OptimizationResult(
|
|
1524
|
+
model=m,
|
|
1525
|
+
task="(default)",
|
|
1526
|
+
layer=config.default_steering.layer,
|
|
1527
|
+
strength=config.default_steering.strength,
|
|
1528
|
+
method=config.default_steering.method,
|
|
1529
|
+
token_aggregation=config.default_steering.token_aggregation,
|
|
1530
|
+
prompt_strategy=config.default_steering.prompt_strategy,
|
|
1531
|
+
score=config.default_steering.score,
|
|
1532
|
+
metric=config.default_steering.metric,
|
|
1533
|
+
))
|
|
1534
|
+
|
|
1535
|
+
# Check task-specific steering
|
|
1536
|
+
for task_name, task_config in config.tasks.items():
|
|
1537
|
+
if task and task_name != task:
|
|
1538
|
+
continue
|
|
1539
|
+
if task_config.steering:
|
|
1540
|
+
results.append(OptimizationResult(
|
|
1541
|
+
model=m,
|
|
1542
|
+
task=task_name,
|
|
1543
|
+
layer=task_config.steering.layer,
|
|
1544
|
+
strength=task_config.steering.strength,
|
|
1545
|
+
method=task_config.steering.method,
|
|
1546
|
+
token_aggregation=task_config.steering.token_aggregation,
|
|
1547
|
+
prompt_strategy=task_config.steering.prompt_strategy,
|
|
1548
|
+
score=task_config.steering.score,
|
|
1549
|
+
metric=task_config.steering.metric,
|
|
1550
|
+
))
|
|
1551
|
+
|
|
1552
|
+
return results
|
|
1553
|
+
|
|
1554
|
+
def delete(self, model: str, task: str, method: str = "CAA") -> bool:
|
|
1555
|
+
"""Delete a cached result."""
|
|
1556
|
+
config = self._manager.get_model_config(model)
|
|
1557
|
+
|
|
1558
|
+
if task in config.tasks and config.tasks[task].steering:
|
|
1559
|
+
config.tasks[task].steering = None
|
|
1560
|
+
self._manager._save_model_config(config)
|
|
1561
|
+
return True
|
|
1562
|
+
return False
|
|
1563
|
+
|
|
1564
|
+
def clear(self) -> int:
|
|
1565
|
+
"""Clear all cached steering results."""
|
|
1566
|
+
count = 0
|
|
1567
|
+
for model in self._manager.list_models():
|
|
1568
|
+
config = self._manager.get_model_config(model)
|
|
1569
|
+
|
|
1570
|
+
if config.default_steering:
|
|
1571
|
+
config.default_steering = None
|
|
1572
|
+
count += 1
|
|
1573
|
+
|
|
1574
|
+
for task_config in config.tasks.values():
|
|
1575
|
+
if task_config.steering:
|
|
1576
|
+
task_config.steering = None
|
|
1577
|
+
count += 1
|
|
1578
|
+
|
|
1579
|
+
self._manager._save_model_config(config)
|
|
1580
|
+
|
|
1581
|
+
return count
|
|
1582
|
+
|
|
1583
|
+
def _save(self) -> None:
|
|
1584
|
+
"""No-op for compatibility - config manager auto-saves."""
|
|
1585
|
+
pass
|
|
1586
|
+
|
|
1587
|
+
|
|
1588
|
+
# Global cache instance for backward compatibility
|
|
1589
|
+
_legacy_cache: Optional[OptimizationCache] = None
|
|
1590
|
+
|
|
1591
|
+
|
|
1592
|
+
def get_cache() -> OptimizationCache:
|
|
1593
|
+
"""Get the global optimization cache instance (backward compatible)."""
|
|
1594
|
+
global _legacy_cache
|
|
1595
|
+
if _legacy_cache is None:
|
|
1596
|
+
_legacy_cache = OptimizationCache()
|
|
1597
|
+
return _legacy_cache
|
|
1598
|
+
|
|
1599
|
+
|
|
1600
|
+
# ========== ModelConfigManager Backward Compatibility ==========
|
|
1601
|
+
# Provides compatibility with the old ModelConfigManager class API
|
|
1602
|
+
|
|
1603
|
+
class ModelConfigManager:
|
|
1604
|
+
"""
|
|
1605
|
+
Backward-compatible wrapper class for the unified config manager.
|
|
1606
|
+
Provides the same interface as the old ModelConfigManager class.
|
|
1607
|
+
"""
|
|
1608
|
+
|
|
1609
|
+
def __init__(self, config_dir: Optional[str] = None):
|
|
1610
|
+
self._manager = get_config_manager()
|
|
1611
|
+
# Ignore config_dir parameter since unified manager has fixed location
|
|
1612
|
+
|
|
1613
|
+
def _sanitize_model_name(self, model_name: str) -> str:
|
|
1614
|
+
return self._manager._sanitize_model_name(model_name)
|
|
1615
|
+
|
|
1616
|
+
def _get_config_path(self, model_name: str) -> str:
|
|
1617
|
+
return str(self._manager._get_config_path(model_name))
|
|
1618
|
+
|
|
1619
|
+
def save_model_config(
|
|
1620
|
+
self,
|
|
1621
|
+
model_name: str,
|
|
1622
|
+
classification_layer: int,
|
|
1623
|
+
steering_layer: Optional[int] = None,
|
|
1624
|
+
token_aggregation: str = "average",
|
|
1625
|
+
detection_threshold: float = 0.6,
|
|
1626
|
+
optimization_method: str = "manual",
|
|
1627
|
+
optimization_metrics: Optional[Dict[str, Any]] = None,
|
|
1628
|
+
task_specific_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
|
1629
|
+
) -> str:
|
|
1630
|
+
"""Save model configuration using unified config manager."""
|
|
1631
|
+
if steering_layer is None:
|
|
1632
|
+
steering_layer = classification_layer
|
|
1633
|
+
|
|
1634
|
+
# Save classification config
|
|
1635
|
+
save_classification_config(
|
|
1636
|
+
model_name=model_name,
|
|
1637
|
+
layer=classification_layer,
|
|
1638
|
+
token_aggregation=token_aggregation,
|
|
1639
|
+
detection_threshold=detection_threshold,
|
|
1640
|
+
optimization_method=optimization_method,
|
|
1641
|
+
set_as_default=True,
|
|
1642
|
+
)
|
|
1643
|
+
|
|
1644
|
+
# Save steering config with same layer
|
|
1645
|
+
save_steering_config(
|
|
1646
|
+
model_name=model_name,
|
|
1647
|
+
layer=steering_layer,
|
|
1648
|
+
optimization_method=optimization_method,
|
|
1649
|
+
set_as_default=True,
|
|
1650
|
+
)
|
|
1651
|
+
|
|
1652
|
+
return str(self._manager._get_config_path(model_name))
|
|
1653
|
+
|
|
1654
|
+
def load_model_config(self, model_name: str) -> Optional[Dict[str, Any]]:
|
|
1655
|
+
"""Load model configuration in legacy format."""
|
|
1656
|
+
config = self._manager.get_model_config(model_name)
|
|
1657
|
+
|
|
1658
|
+
# Return None if no configs exist
|
|
1659
|
+
if not config.default_classification and not config.default_steering:
|
|
1660
|
+
return None
|
|
1661
|
+
|
|
1662
|
+
# Build legacy format
|
|
1663
|
+
result = {
|
|
1664
|
+
"model_name": model_name,
|
|
1665
|
+
"optimal_parameters": {},
|
|
1666
|
+
"task_specific_overrides": {},
|
|
1667
|
+
"optimization_metrics": {},
|
|
1668
|
+
"config_version": "2.0",
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
if config.default_classification:
|
|
1672
|
+
result["optimal_parameters"]["classification_layer"] = config.default_classification.layer
|
|
1673
|
+
result["optimal_parameters"]["token_aggregation"] = config.default_classification.token_aggregation
|
|
1674
|
+
result["optimal_parameters"]["detection_threshold"] = config.default_classification.detection_threshold
|
|
1675
|
+
|
|
1676
|
+
if config.default_steering:
|
|
1677
|
+
result["optimal_parameters"]["steering_layer"] = config.default_steering.layer
|
|
1678
|
+
|
|
1679
|
+
return result
|
|
1680
|
+
|
|
1681
|
+
def has_model_config(self, model_name: str) -> bool:
|
|
1682
|
+
"""Check if a model has a saved configuration."""
|
|
1683
|
+
return self._manager.has_config(model_name)
|
|
1684
|
+
|
|
1685
|
+
def get_optimal_parameters(
|
|
1686
|
+
self,
|
|
1687
|
+
model_name: str,
|
|
1688
|
+
task_name: Optional[str] = None,
|
|
1689
|
+
) -> Optional[Dict[str, Any]]:
|
|
1690
|
+
"""Get optimal parameters for a model."""
|
|
1691
|
+
config = self.load_model_config(model_name)
|
|
1692
|
+
if not config:
|
|
1693
|
+
return None
|
|
1694
|
+
return config.get("optimal_parameters", {})
|
|
1695
|
+
|
|
1696
|
+
def list_model_configs(self) -> List[Dict[str, Any]]:
|
|
1697
|
+
"""List all available model configurations."""
|
|
1698
|
+
configs = []
|
|
1699
|
+
for model_name in self._manager.list_models():
|
|
1700
|
+
config = self.load_model_config(model_name)
|
|
1701
|
+
if config:
|
|
1702
|
+
configs.append({
|
|
1703
|
+
"model_name": model_name,
|
|
1704
|
+
"classification_layer": config.get("optimal_parameters", {}).get("classification_layer"),
|
|
1705
|
+
"steering_layer": config.get("optimal_parameters", {}).get("steering_layer"),
|
|
1706
|
+
})
|
|
1707
|
+
return configs
|
|
1708
|
+
|
|
1709
|
+
def remove_model_config(self, model_name: str) -> bool:
|
|
1710
|
+
"""Remove a model configuration."""
|
|
1711
|
+
return self._manager.delete_config(model_name)
|
|
1712
|
+
|
|
1713
|
+
|
|
1714
|
+
def get_default_manager() -> ModelConfigManager:
|
|
1715
|
+
"""Get a default ModelConfigManager instance (backward compatible)."""
|
|
1716
|
+
return ModelConfigManager()
|
|
1717
|
+
|
|
1718
|
+
|
|
1719
|
+
def save_model_config(model_name: str, **kwargs) -> str:
|
|
1720
|
+
"""Save model configuration using default manager (backward compatible)."""
|
|
1721
|
+
return ModelConfigManager().save_model_config(model_name, **kwargs)
|
|
1722
|
+
|
|
1723
|
+
|
|
1724
|
+
def load_model_config(model_name: str) -> Optional[Dict[str, Any]]:
|
|
1725
|
+
"""Load model configuration using default manager (backward compatible)."""
|
|
1726
|
+
return ModelConfigManager().load_model_config(model_name)
|
|
1727
|
+
|
|
1728
|
+
|
|
1729
|
+
def get_optimal_parameters(model_name: str, task_name: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
|
1730
|
+
"""Get optimal parameters using default manager (backward compatible)."""
|
|
1731
|
+
return ModelConfigManager().get_optimal_parameters(model_name, task_name)
|