wisent 0.5.14__py3-none-any.whl → 0.5.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of wisent might be problematic. Click here for more details.
- wisent/__init__.py +1 -1
- wisent/cli.py +114 -0
- wisent/core/activations/activations_collector.py +19 -11
- wisent/core/cli/__init__.py +3 -1
- wisent/core/cli/create_steering_vector.py +60 -18
- wisent/core/cli/evaluate_responses.py +14 -8
- wisent/core/cli/generate_pairs_from_task.py +18 -5
- wisent/core/cli/get_activations.py +1 -1
- wisent/core/cli/multi_steer.py +108 -0
- wisent/core/cli/optimize_classification.py +187 -285
- wisent/core/cli/optimize_sample_size.py +78 -0
- wisent/core/cli/optimize_steering.py +354 -53
- wisent/core/cli/tasks.py +274 -9
- wisent/core/errors/__init__.py +0 -0
- wisent/core/errors/error_handler.py +134 -0
- wisent/core/evaluators/benchmark_specific/log_likelihoods_evaluator.py +152 -295
- wisent/core/evaluators/rotator.py +22 -8
- wisent/core/main.py +5 -1
- wisent/core/model_persistence.py +4 -19
- wisent/core/models/wisent_model.py +11 -3
- wisent/core/parser.py +4 -3
- wisent/core/parser_arguments/main_parser.py +1 -1
- wisent/core/parser_arguments/multi_steer_parser.py +4 -3
- wisent/core/parser_arguments/optimize_steering_parser.py +4 -0
- wisent/core/sample_size_optimizer_v2.py +1 -1
- wisent/core/steering_optimizer.py +2 -2
- wisent/tests/__init__.py +0 -0
- wisent/tests/examples/__init__.py +0 -0
- wisent/tests/examples/cli/__init__.py +0 -0
- wisent/tests/examples/cli/activations/__init__.py +0 -0
- wisent/tests/examples/cli/activations/test_get_activations.py +127 -0
- wisent/tests/examples/cli/classifier/__init__.py +0 -0
- wisent/tests/examples/cli/classifier/test_classifier_examples.py +141 -0
- wisent/tests/examples/cli/contrastive_pairs/__init__.py +0 -0
- wisent/tests/examples/cli/contrastive_pairs/test_generate_pairs.py +89 -0
- wisent/tests/examples/cli/evaluation/__init__.py +0 -0
- wisent/tests/examples/cli/evaluation/test_evaluation_examples.py +117 -0
- wisent/tests/examples/cli/generate/__init__.py +0 -0
- wisent/tests/examples/cli/generate/test_generate_with_classifier.py +146 -0
- wisent/tests/examples/cli/generate/test_generate_with_steering.py +149 -0
- wisent/tests/examples/cli/generate/test_only_generate.py +110 -0
- wisent/tests/examples/cli/multi_steering/__init__.py +0 -0
- wisent/tests/examples/cli/multi_steering/test_multi_steer_from_trained_vectors.py +210 -0
- wisent/tests/examples/cli/multi_steering/test_multi_steer_with_different_parameters.py +205 -0
- wisent/tests/examples/cli/multi_steering/test_train_and_multi_steer.py +174 -0
- wisent/tests/examples/cli/optimizer/__init__.py +0 -0
- wisent/tests/examples/cli/optimizer/test_optimize_sample_size.py +102 -0
- wisent/tests/examples/cli/optimizer/test_optimizer_examples.py +59 -0
- wisent/tests/examples/cli/steering/__init__.py +0 -0
- wisent/tests/examples/cli/steering/test_create_steering_vectors.py +135 -0
- wisent/tests/examples/cli/synthetic/__init__.py +0 -0
- wisent/tests/examples/cli/synthetic/test_synthetic_pairs.py +45 -0
- {wisent-0.5.14.dist-info → wisent-0.5.15.dist-info}/METADATA +3 -1
- {wisent-0.5.14.dist-info → wisent-0.5.15.dist-info}/RECORD +59 -29
- wisent/core/agent/diagnose/test_synthetic_classifier.py +0 -71
- /wisent/core/parser_arguments/{test_nonsense_parser.py → nonsense_parser.py} +0 -0
- {wisent-0.5.14.dist-info → wisent-0.5.15.dist-info}/WHEEL +0 -0
- {wisent-0.5.14.dist-info → wisent-0.5.15.dist-info}/entry_points.txt +0 -0
- {wisent-0.5.14.dist-info → wisent-0.5.15.dist-info}/licenses/LICENSE +0 -0
- {wisent-0.5.14.dist-info → wisent-0.5.15.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Tests for synthetic pairs generation example.
|
|
3
|
+
|
|
4
|
+
Validates synthetic contrastive pair creation.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import subprocess
|
|
8
|
+
import pytest
|
|
9
|
+
import tempfile
|
|
10
|
+
import os
|
|
11
|
+
import json
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_create_synthetic_pairs():
|
|
15
|
+
"""Test creating synthetic contrastive pairs for a trait."""
|
|
16
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
17
|
+
output_file = os.path.join(tmpdir, "synthetic_pairs.json")
|
|
18
|
+
|
|
19
|
+
result = subprocess.run(
|
|
20
|
+
[
|
|
21
|
+
"python", "-m", "wisent.core.main", "generate-pairs",
|
|
22
|
+
"--trait", "helpfulness",
|
|
23
|
+
"--output", output_file,
|
|
24
|
+
"--model", "meta-llama/Llama-3.2-1B-Instruct",
|
|
25
|
+
"--num-pairs", "5",
|
|
26
|
+
"--similarity-threshold", "0.8",
|
|
27
|
+
"--device", "cpu",
|
|
28
|
+
"--verbose"
|
|
29
|
+
],
|
|
30
|
+
capture_output=True,
|
|
31
|
+
text=True,
|
|
32
|
+
timeout=300
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
assert result.returncode == 0, f"Command failed: {result.stderr}"
|
|
36
|
+
assert os.path.exists(output_file), "Output file not created"
|
|
37
|
+
|
|
38
|
+
# Verify JSON format
|
|
39
|
+
with open(output_file, 'r') as f:
|
|
40
|
+
data = json.load(f)
|
|
41
|
+
assert isinstance(data, (list, dict)), "Output should be JSON"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
if __name__ == "__main__":
|
|
45
|
+
pytest.main([__file__, "-v"])
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wisent
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.15
|
|
4
4
|
Summary: Monitor and guard against harmful content in language models
|
|
5
5
|
Home-page: https://github.com/yourusername/wisent-activation-guardrails
|
|
6
6
|
Author: Wisent Team
|
|
@@ -26,6 +26,8 @@ Requires-Dist: sentence-transformers>=2.0.0
|
|
|
26
26
|
Requires-Dist: faiss-cpu>=1.7.0
|
|
27
27
|
Provides-Extra: harness
|
|
28
28
|
Requires-Dist: lm-eval==0.4.8; extra == "harness"
|
|
29
|
+
Provides-Extra: cuda
|
|
30
|
+
Requires-Dist: flash-attn>=2.5.0; extra == "cuda"
|
|
29
31
|
Dynamic: author
|
|
30
32
|
Dynamic: author-email
|
|
31
33
|
Dynamic: classifier
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
wisent/__init__.py,sha256=
|
|
1
|
+
wisent/__init__.py,sha256=L6LoKMlJx-n68Agaz17S4PydD7S5Z4lwW3o79dy3l1c,23
|
|
2
|
+
wisent/cli.py,sha256=XKzGIGstr38EowHYpr821c6YuV9Eaw3I1I3NvLztTO0,3960
|
|
2
3
|
wisent/core/__init__.py,sha256=yNBOdCXpkwMuo1H1sNAYgPf5_-Hhf7y4H-RZJPCiOpI,463
|
|
3
4
|
wisent/core/autonomous_agent.py,sha256=XszhMkONc8kClCMqQfM8ZRQPxiMKJCmZSF7hrY-hf8Q,52878
|
|
4
5
|
wisent/core/benchmark_extractors.py,sha256=E51DOJM-cbtlDy9VrIeoBKSEQ6AZ1jeyDM7OAw-vn94,10473
|
|
@@ -9,20 +10,20 @@ wisent/core/detection_handling.py,sha256=iiuKpzAbJfx_KFn2SFABQHOeeWblDJMXjzGwGDe
|
|
|
9
10
|
wisent/core/download_full_benchmarks.py,sha256=GylPUmd0JtrnqGaqyU5PVV8uQPvJyaRcCs_2QqHyO1I,58743
|
|
10
11
|
wisent/core/hyperparameter_optimizer.py,sha256=GN-IY27kkFKuXwHbvalfEW5UVf4NfZ0G9rMWK2ThEAY,17319
|
|
11
12
|
wisent/core/lm_eval_harness_ground_truth.py,sha256=Fb33sZ_nT0dhkzsEmWv3Ed40NCC-onWXGk379UpsEsA,64964
|
|
12
|
-
wisent/core/main.py,sha256
|
|
13
|
+
wisent/core/main.py,sha256=-EtcGAbzqrKeT0zzM-EyEMOYOfsBYmx2qvIGrdZCMFc,2419
|
|
13
14
|
wisent/core/managed_cached_benchmarks.py,sha256=JbvpZ1fgSuQQhyQVKEvqrQZRHGqfnjo9NFhgITFoFsE,22854
|
|
14
15
|
wisent/core/mixed_benchmark_sampler.py,sha256=tKQCHUXVuYeCyx4VZt8O1hGyB-TOY_SQ_SYi8cyApII,13585
|
|
15
16
|
wisent/core/model_config_manager.py,sha256=rQAdSmk3GFlZXyHp3fSV1bORxiZWhmzIz1uo3H4JtkA,12009
|
|
16
|
-
wisent/core/model_persistence.py,sha256=
|
|
17
|
+
wisent/core/model_persistence.py,sha256=Pr1A3E0MxqwICKsFXmjKVq9KgG8Ip5UbtG-gujGFShc,9870
|
|
17
18
|
wisent/core/multi_steering.py,sha256=YhVKmf08KacVEYZWLk6t2uNWSv-Pi_zBeLdDopo3QXk,13491
|
|
18
|
-
wisent/core/parser.py,sha256=
|
|
19
|
+
wisent/core/parser.py,sha256=GFyHJklf7ReFMKk0ugSL2-bC8_Lyi6GSlNpeXlU-umY,69201
|
|
19
20
|
wisent/core/representation.py,sha256=hBl_N9qbr5Gsa7GCQ0nMWRm82RqYEfhd9cyf0PPH5LY,195
|
|
20
21
|
wisent/core/sample_size_optimizer.py,sha256=6wegGXZpdGpiR4R0YJ1D2JqLr6yinMndEx2gB5FL80s,23666
|
|
21
|
-
wisent/core/sample_size_optimizer_v2.py,sha256=
|
|
22
|
+
wisent/core/sample_size_optimizer_v2.py,sha256=wPkqNhiDkVAKl5iekMynqS9kILz4sY4ygHNZ3PPvbjs,13291
|
|
22
23
|
wisent/core/save_results.py,sha256=PRwaA5qO6EOsvURvLBl3YhvanlC0D0G4iYqxYAQ7sw8,13737
|
|
23
24
|
wisent/core/steering.py,sha256=I1mSLOQ6MMdUcJ_MmfmaCUH0IKxeCYlcsMCO6x6wLg4,22651
|
|
24
25
|
wisent/core/steering_method.py,sha256=-hZqtvwRS7sGqQJUd36MoPm0rjbO1LrtPAYmcIk8BqQ,462
|
|
25
|
-
wisent/core/steering_optimizer.py,sha256=
|
|
26
|
+
wisent/core/steering_optimizer.py,sha256=opgcNILzzKJJu4VuQIoMm72kiD5-a2aLsjsEUs-ZHZ4,55861
|
|
26
27
|
wisent/core/task_interface.py,sha256=OlWdcxkprmZcOto-bXmg75kzUcWzH_kyW_e7w2FdPLM,4471
|
|
27
28
|
wisent/core/task_selector.py,sha256=QVgozUuiM74BMUJ8Ucb_sn6HQk5v0wL_QUsqKb55vJE,6224
|
|
28
29
|
wisent/core/time_estimator.py,sha256=DcgSzW-hr9BjmXJwBnGqE2dkFK0zgyz5WNF7934CJ9k,5778
|
|
@@ -30,7 +31,7 @@ wisent/core/timing_calibration.py,sha256=AVSHWetHa01omwGh4zp2WN2ByfCq8zpktfLdtqD
|
|
|
30
31
|
wisent/core/user_model_config.py,sha256=8optjLqf9wTDtOf0c705d5_Rr2zE67jV9BNqoY-TRvA,6735
|
|
31
32
|
wisent/core/activations/__init__.py,sha256=ZT3aU-eU5-RR_QDkmKZRweCltJTWVof8mgbL3ONAXKw,839
|
|
32
33
|
wisent/core/activations/activations.py,sha256=SS-WEds9AoTnxcpVnAUYEydGHjfRRzLhKEHgTgwJYmE,3976
|
|
33
|
-
wisent/core/activations/activations_collector.py,sha256=
|
|
34
|
+
wisent/core/activations/activations_collector.py,sha256=i354mWA7QNufxW2VN91uHjmO7-tbpJaQN40lsnfXY9w,16870
|
|
34
35
|
wisent/core/activations/prompt_construction_strategy.py,sha256=KpaAdj75D3t6JR47mx70vr82uCijsuaMIolIEXuciO8,1403
|
|
35
36
|
wisent/core/activations/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
36
37
|
wisent/core/activations/core/atoms.py,sha256=FeeQj-YKb-1rLX0gnzFc5VM47ikWwfFh3datRduj6mQ,9101
|
|
@@ -47,23 +48,24 @@ wisent/core/agent/diagnose/create_classifier.py,sha256=UJKFgjGvIsRKfc5fPkfh227bA
|
|
|
47
48
|
wisent/core/agent/diagnose/response_diagnostics.py,sha256=49P_J29nYaz4BW3vcs6SdI_kbVutKGSpq_neUPcWlro,10684
|
|
48
49
|
wisent/core/agent/diagnose/select_classifiers.py,sha256=Uv5dxEmZRjjkZKmVmYQ1jC_XJVyM2MF5dv-4QRuTWKY,18320
|
|
49
50
|
wisent/core/agent/diagnose/synthetic_classifier_option.py,sha256=SetaYBdgbAlbCGjpgbADq9MnCtbQogByJLhqi3ndiM4,32495
|
|
50
|
-
wisent/core/agent/diagnose/test_synthetic_classifier.py,sha256=EWPPHtoND797XfoUjNZcmcuBp8w3PYm4YIBu1JXhzPE,2688
|
|
51
51
|
wisent/core/agent/diagnose/tasks/__init__.py,sha256=rfImwPtKCAfz-ASOBQyF4DEhU6hgCuiZBcqh1AaUE80,704
|
|
52
52
|
wisent/core/agent/diagnose/tasks/task_manager.py,sha256=OXodauLfbAMawL8rR1KL-CfOHtNbH05Ai4LP9gSo9yU,62746
|
|
53
53
|
wisent/core/agent/diagnose/tasks/task_relevance.py,sha256=D4UBr0TqUNXkDZnNgA5wa4NYHSKtDaiugYeVg5zGQjs,3250
|
|
54
54
|
wisent/core/agent/diagnose/tasks/task_selector.py,sha256=ll34stireeqW-B_T4daf_91kujzVFQ8sOilk-JrxpHA,5414
|
|
55
|
-
wisent/core/cli/__init__.py,sha256=
|
|
56
|
-
wisent/core/cli/create_steering_vector.py,sha256=
|
|
57
|
-
wisent/core/cli/evaluate_responses.py,sha256=
|
|
55
|
+
wisent/core/cli/__init__.py,sha256=hTo_urwejrc28w_4EqoiFyxYGTT0673r6gk36iU9eOg,1241
|
|
56
|
+
wisent/core/cli/create_steering_vector.py,sha256=IYfwdxSiWpakkGE-VOFASbiHNwpZa7SMgQzb7Ggt1OY,7928
|
|
57
|
+
wisent/core/cli/evaluate_responses.py,sha256=I-OUgBJYOXon5ijTt6Lm1iO5nb93pxNhvU_8Ci-9ZIE,29655
|
|
58
58
|
wisent/core/cli/generate_pairs.py,sha256=6xhdkwnwKVvkSrSlQ0Zq-qNWBWPFVqhDqpg2YXiU9jA,5209
|
|
59
|
-
wisent/core/cli/generate_pairs_from_task.py,sha256=
|
|
59
|
+
wisent/core/cli/generate_pairs_from_task.py,sha256=plbD5z4urG-kdJxIMlGyDexBNLJGetpWCdNxUZ5EtW4,5203
|
|
60
60
|
wisent/core/cli/generate_responses.py,sha256=mV6wKGz1Ewyeph25tSaNzqjNuv1qtwwG2ktOwCkdtG0,3966
|
|
61
61
|
wisent/core/cli/generate_vector_from_synthetic.py,sha256=TjmRD2k-ybwruolFC_na6bM79CPsYMweLDGP-q4XnNc,5567
|
|
62
62
|
wisent/core/cli/generate_vector_from_task.py,sha256=ZzrddZvaskF1L83j6rl08AmTe-XTAo_5spX-3MuZHUo,5474
|
|
63
|
-
wisent/core/cli/get_activations.py,sha256=
|
|
64
|
-
wisent/core/cli/
|
|
65
|
-
wisent/core/cli/
|
|
66
|
-
wisent/core/cli/
|
|
63
|
+
wisent/core/cli/get_activations.py,sha256=LDtNksSxsf7zh_899r4ZoG-l3fQm1FXtBTCd6LQSfH4,7693
|
|
64
|
+
wisent/core/cli/multi_steer.py,sha256=-ofVmsrSpZjuNq_UCQWT7TurpRkMCQdHPNFqcqnu9uU,4276
|
|
65
|
+
wisent/core/cli/optimize_classification.py,sha256=VaboV8XLMoO4b7Ny9Za_EUBoMRdWWXi4k3sFuBadxPw,10095
|
|
66
|
+
wisent/core/cli/optimize_sample_size.py,sha256=CGuqxvU_YpslZUhE1gJpcusFS_wiAh1mU9PdTd3H7r0,3005
|
|
67
|
+
wisent/core/cli/optimize_steering.py,sha256=rJAhN1MEmOl8xlCkGOUNXQ-kZITkQ8mihkDZssGv0wg,31902
|
|
68
|
+
wisent/core/cli/tasks.py,sha256=WWbPjwkoj6c4A0l6HjxLdRQ4ST80uXL7qLZRLvFeDG4,21164
|
|
67
69
|
wisent/core/contrastive_pairs/__init__.py,sha256=AbaAf-t_nyVVy_vLjp8WAlMDmNun3KNp_GMWAK25r9g,429
|
|
68
70
|
wisent/core/contrastive_pairs/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
69
71
|
wisent/core/contrastive_pairs/core/atoms.py,sha256=_zghw6c8iisW_SqBIUCoAnzhc5q7t5EgZ4zzTPxeLwQ,1129
|
|
@@ -142,13 +144,15 @@ wisent/core/data_loaders/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
|
|
|
142
144
|
wisent/core/data_loaders/loaders/custom.py,sha256=Xe1sOHH3_dRjBnQg9vwMM-XA8ROn65dUr9TeT-nuNtQ,4144
|
|
143
145
|
wisent/core/data_loaders/loaders/lm_loader.py,sha256=Trdit9L--ZMqkUE_tT0S2OWrMEO4o9tjFkHx2r3LZVU,7858
|
|
144
146
|
wisent/core/data_loaders/loaders/task_interface_loader.py,sha256=QGDfqMvdTFfogF9P533b-Gw52uHe0GnsS64AfnVVlek,11771
|
|
147
|
+
wisent/core/errors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
148
|
+
wisent/core/errors/error_handler.py,sha256=6OE7pQHOrOm6ZtSYlrN3GjyeWgCzkTF1Znsl0nd2Yog,4435
|
|
145
149
|
wisent/core/evaluators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
146
|
-
wisent/core/evaluators/rotator.py,sha256=
|
|
150
|
+
wisent/core/evaluators/rotator.py,sha256=gjJwHKncWWfJeK_603d-UQ_8oE8CjIcE6DbokMv6wFo,6542
|
|
147
151
|
wisent/core/evaluators/benchmark_specific/__init__.py,sha256=Hjrh-p2bGp7Px2KpyPBxQLgYzlmFwBBk0MgofYkygHM,955
|
|
148
152
|
wisent/core/evaluators/benchmark_specific/exact_match_evaluator.py,sha256=8v-yRdW8PD9eRwwcJIcIl9MZRMvqLECueYAHdL3DGrA,2645
|
|
149
153
|
wisent/core/evaluators/benchmark_specific/f1_evaluator.py,sha256=E2nTdZGem1P4KwQJDH6l3p9CfcMqW_PfmAgpoU2ocKs,3153
|
|
150
154
|
wisent/core/evaluators/benchmark_specific/generation_evaluator.py,sha256=o_gu2fG5A8cLAhPfhyGUjvmLxg8OlFpNS-GYUWegu3Q,7019
|
|
151
|
-
wisent/core/evaluators/benchmark_specific/log_likelihoods_evaluator.py,sha256=
|
|
155
|
+
wisent/core/evaluators/benchmark_specific/log_likelihoods_evaluator.py,sha256=khHjdfb517-tfNRWe-K6NkfiUc2wIlanSofesjs5ffI,7305
|
|
152
156
|
wisent/core/evaluators/benchmark_specific/perplexity_evaluator.py,sha256=NUlpKaDwkQR8BB8gudC1qTdHqvSyBX3oGoKrtE5ZBVw,4743
|
|
153
157
|
wisent/core/evaluators/benchmark_specific/personalization_evaluator.py,sha256=R9c0UQrdEDKK6ZRHeoDANy2lw1zJE6A8i6uL-nJezok,8887
|
|
154
158
|
wisent/core/evaluators/benchmark_specific/coding/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -181,7 +185,7 @@ wisent/core/evaluators/oracles/interactive.py,sha256=f3v2_N17fKzGyeOxONRJbrbn8i5
|
|
|
181
185
|
wisent/core/evaluators/oracles/nlp_evaluator.py,sha256=KxbnF-I2IFbBQpoYyjQKGbYh4NErsEuhTCRYX_Tob8o,18220
|
|
182
186
|
wisent/core/evaluators/oracles/user_specified.py,sha256=V1dKrNj3Oq7UC_I7DT0WGnktP7R_DSW6UAwDdrA8SnE,2360
|
|
183
187
|
wisent/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
184
|
-
wisent/core/models/wisent_model.py,sha256=
|
|
188
|
+
wisent/core/models/wisent_model.py,sha256=oViMB8of-kGBEoKt4niwlVdZA2bfUhsT358Ju1ytW5Y,32410
|
|
185
189
|
wisent/core/models/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
186
190
|
wisent/core/models/core/atoms.py,sha256=_Bpz0Sfiq6_VswThIltUwNGj_ukl5MhAg8RrgMKwEBM,15756
|
|
187
191
|
wisent/core/opti/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -215,16 +219,16 @@ wisent/core/parser_arguments/generate_vector_from_synthetic_parser.py,sha256=gC0
|
|
|
215
219
|
wisent/core/parser_arguments/generate_vector_from_task_parser.py,sha256=pm_nfSN0UqL2QsFYvFg9R-14Ukjm4jEb1f6H40pQXCI,3518
|
|
216
220
|
wisent/core/parser_arguments/generate_vector_parser.py,sha256=v9oi1ggqAV8GpGQhECy6htw4U_ZVmBgW9zWjZlnus6o,3618
|
|
217
221
|
wisent/core/parser_arguments/get_activations_parser.py,sha256=ChOV21B3Jji-f8KnZW_8H97agJJJv-dYLv2a33OE0y4,2382
|
|
218
|
-
wisent/core/parser_arguments/main_parser.py,sha256=
|
|
222
|
+
wisent/core/parser_arguments/main_parser.py,sha256=lQkZ7IDFwGWZxPyiGuh2OTmfvXt8E_Ph-60jQ6Qssx4,8268
|
|
219
223
|
wisent/core/parser_arguments/model_config_parser.py,sha256=MBbn5lUVi5n1iTOwKQ1IzSsFgOwDmHSuPJA9Y1Qp5Po,3150
|
|
220
224
|
wisent/core/parser_arguments/monitor_parser.py,sha256=qo3vyyVmdZBkhGuhlHqbEzTiIKcdFIvG3IIuc-DIRQE,1234
|
|
221
|
-
wisent/core/parser_arguments/multi_steer_parser.py,sha256=
|
|
225
|
+
wisent/core/parser_arguments/multi_steer_parser.py,sha256=v1CCmkO_avVetYaxwS0ryUiTGyAluWVRalfFYxhzxNk,2145
|
|
226
|
+
wisent/core/parser_arguments/nonsense_parser.py,sha256=ALxU27Hn7ewB8IVe7bMS78MqukU4riouy492nlM4BUM,1096
|
|
222
227
|
wisent/core/parser_arguments/optimize_classification_parser.py,sha256=G3OtlPa5MrqAx10bD3PO7NeT0VehLk-WxY4z9g7ZHBY,2745
|
|
223
228
|
wisent/core/parser_arguments/optimize_sample_size_parser.py,sha256=pgX3pyYowsrpWhM3aNtdQCClsQaLAFX_bb3oqHltU-U,2816
|
|
224
|
-
wisent/core/parser_arguments/optimize_steering_parser.py,sha256=
|
|
229
|
+
wisent/core/parser_arguments/optimize_steering_parser.py,sha256=cAQ2XA3nBpidm0U9Dx4NIsdFojp-_3wmmtkBiv6DKEo,7451
|
|
225
230
|
wisent/core/parser_arguments/synthetic_parser.py,sha256=NYxmdGmqo0680dkoKw82pleq4v0B1q77t1N7IBng3qI,3606
|
|
226
231
|
wisent/core/parser_arguments/tasks_parser.py,sha256=oVLxDdUcQVDmJzSYYbULmSQNvSIdzSRWL_UxNAu_2F4,22719
|
|
227
|
-
wisent/core/parser_arguments/test_nonsense_parser.py,sha256=ALxU27Hn7ewB8IVe7bMS78MqukU4riouy492nlM4BUM,1096
|
|
228
232
|
wisent/core/parser_arguments/utils.py,sha256=E2s2VE14-iedjjcATuu60qnfYgxSKiYyGqeFeTwABUo,3654
|
|
229
233
|
wisent/core/prompts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
230
234
|
wisent/core/prompts/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -286,9 +290,35 @@ wisent/core/trainers/core/__init__.py,sha256=D0JX0-XCHdtLrCXhVDHNQafvyWCvJ4-o4UK
|
|
|
286
290
|
wisent/core/trainers/core/atoms.py,sha256=ycWk0G-7EIAEOnQL-o5_V5B8KTQ7CQUilGF4ibjighM,1536
|
|
287
291
|
wisent/core/utils/__init__.py,sha256=NavuBkpDSz6q1dN6m34-I-l8Aps1Sgcnx82FHxFufzY,457
|
|
288
292
|
wisent/core/utils/device.py,sha256=5brw9tclTU77NNAokzLXYrKUnjjYBzFfA7wNJPM9ytM,1609
|
|
289
|
-
wisent
|
|
290
|
-
wisent
|
|
291
|
-
wisent
|
|
292
|
-
wisent
|
|
293
|
-
wisent
|
|
294
|
-
wisent-0
|
|
293
|
+
wisent/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
294
|
+
wisent/tests/examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
295
|
+
wisent/tests/examples/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
296
|
+
wisent/tests/examples/cli/activations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
297
|
+
wisent/tests/examples/cli/activations/test_get_activations.py,sha256=sIEpXyN4O3pPiIiZmbDAnKnRG2IubGl1L3M3D0agP_U,4009
|
|
298
|
+
wisent/tests/examples/cli/classifier/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
299
|
+
wisent/tests/examples/cli/classifier/test_classifier_examples.py,sha256=kLSw-fxVl0auFFRW3J_KEEucndtwWefFkyRTn6z4G1o,4855
|
|
300
|
+
wisent/tests/examples/cli/contrastive_pairs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
301
|
+
wisent/tests/examples/cli/contrastive_pairs/test_generate_pairs.py,sha256=oQv44hEYEGtk0DimgDbXJGrvyw_bsdJ-MBKvZUdXYAo,2860
|
|
302
|
+
wisent/tests/examples/cli/evaluation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
303
|
+
wisent/tests/examples/cli/evaluation/test_evaluation_examples.py,sha256=6YxRGtbPb08xFTjaBn1twz8_8OP7zMNM7ArxF8w4CyU,3551
|
|
304
|
+
wisent/tests/examples/cli/generate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
305
|
+
wisent/tests/examples/cli/generate/test_generate_with_classifier.py,sha256=FfUgHbeaeuexk61o0to9Cn6F47UjtVQyM1pUWRSm4mg,5096
|
|
306
|
+
wisent/tests/examples/cli/generate/test_generate_with_steering.py,sha256=s-BmC-v8nkLFnCq42swo9gyFlT6Al3t_Dran3kswcoU,5183
|
|
307
|
+
wisent/tests/examples/cli/generate/test_only_generate.py,sha256=WPuww-w9XtUWTLYMVgka2TFhkZWZ2EFZmLjQ5FZVjtg,3581
|
|
308
|
+
wisent/tests/examples/cli/multi_steering/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
309
|
+
wisent/tests/examples/cli/multi_steering/test_multi_steer_from_trained_vectors.py,sha256=3MAD1aMGfP1s3Touczkl5mCbUmQuGuqnq8B7_y875dg,7537
|
|
310
|
+
wisent/tests/examples/cli/multi_steering/test_multi_steer_with_different_parameters.py,sha256=vfUl6R7UsW5wADR5mleO4uGexKLrKocoUyhcIRVZCs4,6857
|
|
311
|
+
wisent/tests/examples/cli/multi_steering/test_train_and_multi_steer.py,sha256=2nuvKhrIoCXxZfFoNWNs_V4Sifpuhiy2hAqmO8VJ2sg,6196
|
|
312
|
+
wisent/tests/examples/cli/optimizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
313
|
+
wisent/tests/examples/cli/optimizer/test_optimize_sample_size.py,sha256=3Matgc0vcuJFIL1pu7Tz_EVmmyOP77YeXDoo9viiMQM,3428
|
|
314
|
+
wisent/tests/examples/cli/optimizer/test_optimizer_examples.py,sha256=l6L4eZDjmGWyUdkTcJGOK533LM1vlV7u0QcjmvjP4_o,1795
|
|
315
|
+
wisent/tests/examples/cli/steering/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
316
|
+
wisent/tests/examples/cli/steering/test_create_steering_vectors.py,sha256=MJnUtRlsJqeteS33WU-_3fPW33PdLxT3tLBtRikmA34,4585
|
|
317
|
+
wisent/tests/examples/cli/synthetic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
318
|
+
wisent/tests/examples/cli/synthetic/test_synthetic_pairs.py,sha256=d-OhvfMLMu_69T_RRNREGeiC-0aHJTooJZ6J4yR9ZKc,1320
|
|
319
|
+
wisent-0.5.15.dist-info/licenses/LICENSE,sha256=wy0iaw8b2tyqZAfKHib3lP3PJ9o88FDCg92oUHh3sDQ,1073
|
|
320
|
+
wisent-0.5.15.dist-info/METADATA,sha256=HazdPEJUPt5DouANRksVe9_S2brv1qmXAegOEb1qVDI,2484
|
|
321
|
+
wisent-0.5.15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
322
|
+
wisent-0.5.15.dist-info/entry_points.txt,sha256=BM76j3xjtIcVZGk24iDf5w18s6SuqeOpaiAxfZhpnY8,49
|
|
323
|
+
wisent-0.5.15.dist-info/top_level.txt,sha256=2Ts9Iyldnb3auIN2HBBaHPknRy7nSRDm2f6RGzYgr8A,7
|
|
324
|
+
wisent-0.5.15.dist-info/RECORD,,
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
import sys
|
|
2
|
-
import time
|
|
3
|
-
import signal
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
|
|
6
|
-
# Add the project root to the path
|
|
7
|
-
project_root = Path(__file__).parent.parent.parent.parent
|
|
8
|
-
sys.path.insert(0, str(project_root))
|
|
9
|
-
|
|
10
|
-
from wisent.core.model import Model
|
|
11
|
-
from wisent.core.agent.diagnose.synthetic_classifier_option import (
|
|
12
|
-
create_classifiers_for_prompt,
|
|
13
|
-
apply_classifiers_to_response
|
|
14
|
-
)
|
|
15
|
-
from wisent.core.agent.budget import set_time_budget
|
|
16
|
-
|
|
17
|
-
class TimeoutError(Exception):
|
|
18
|
-
"""Raised when test exceeds time budget."""
|
|
19
|
-
pass
|
|
20
|
-
|
|
21
|
-
def timeout_handler(signum, frame):
|
|
22
|
-
raise TimeoutError("Test exceeded time budget!")
|
|
23
|
-
|
|
24
|
-
def main():
|
|
25
|
-
# Set budget and timeout separately
|
|
26
|
-
budget_minutes = 1.0 # 1 minute - internal budget for classifier creation
|
|
27
|
-
timeout_seconds = 120 # 2 minutes - hard timeout for the test process
|
|
28
|
-
set_time_budget(budget_minutes)
|
|
29
|
-
|
|
30
|
-
print(f"⏱️ Starting synthetic classifier test with {timeout_seconds}s timeout and {budget_minutes*60}s budget...")
|
|
31
|
-
|
|
32
|
-
# Set up timeout signal
|
|
33
|
-
signal.signal(signal.SIGALRM, timeout_handler)
|
|
34
|
-
signal.alarm(timeout_seconds)
|
|
35
|
-
|
|
36
|
-
start_time = time.time()
|
|
37
|
-
|
|
38
|
-
try:
|
|
39
|
-
model = Model(name="meta-llama/Llama-3.1-8B-Instruct")
|
|
40
|
-
prompt = "What is the capital of France?"
|
|
41
|
-
|
|
42
|
-
# Test the system
|
|
43
|
-
classifiers, trait_discovery = create_classifiers_for_prompt(model, prompt)
|
|
44
|
-
|
|
45
|
-
# Clear the alarm since we completed successfully
|
|
46
|
-
signal.alarm(0)
|
|
47
|
-
|
|
48
|
-
elapsed_time = time.time() - start_time
|
|
49
|
-
print(f"✅ SUCCESS: Created {len(classifiers)} classifiers for {len(trait_discovery.traits_discovered)} traits")
|
|
50
|
-
print(f"⏱️ Total time: {elapsed_time:.1f}s (timeout: {timeout_seconds}s, budget: {budget_minutes*60}s)")
|
|
51
|
-
|
|
52
|
-
if elapsed_time > timeout_seconds:
|
|
53
|
-
print(f"⚠️ WARNING: Test completed but exceeded timeout by {elapsed_time - timeout_seconds:.1f}s")
|
|
54
|
-
else:
|
|
55
|
-
print(f"🎉 Test completed within timeout with {timeout_seconds - elapsed_time:.1f}s to spare!")
|
|
56
|
-
|
|
57
|
-
except TimeoutError as e:
|
|
58
|
-
elapsed_time = time.time() - start_time
|
|
59
|
-
print(f"❌ ERROR: {e}")
|
|
60
|
-
print(f"❌ Test failed after {elapsed_time:.1f}s (timeout: {timeout_seconds}s, budget: {budget_minutes*60}s)")
|
|
61
|
-
print("❌ This indicates a performance issue that needs investigation.")
|
|
62
|
-
sys.exit(1)
|
|
63
|
-
except Exception as e:
|
|
64
|
-
signal.alarm(0) # Clear timeout
|
|
65
|
-
elapsed_time = time.time() - start_time
|
|
66
|
-
print(f"❌ ERROR: Test failed with exception: {e}")
|
|
67
|
-
print(f"❌ Time elapsed: {elapsed_time:.1f}s")
|
|
68
|
-
sys.exit(1)
|
|
69
|
-
|
|
70
|
-
if __name__ == "__main__":
|
|
71
|
-
main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|