benchkit-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. benchkit/__init__.py +1 -0
  2. benchkit/_pi_proxy.py +219 -0
  3. benchkit/answer_key_guard.ts +49 -0
  4. benchkit/artifacts.py +87 -0
  5. benchkit/benchmarks/__init__.py +105 -0
  6. benchkit/benchmarks/aider_polyglot.py +405 -0
  7. benchkit/benchmarks/arc.py +52 -0
  8. benchkit/benchmarks/banking77.py +149 -0
  9. benchkit/benchmarks/base.py +19 -0
  10. benchkit/benchmarks/boolq.py +49 -0
  11. benchkit/benchmarks/chartqa.py +133 -0
  12. benchkit/benchmarks/code_verifier.py +77 -0
  13. benchkit/benchmarks/downloads.py +166 -0
  14. benchkit/benchmarks/evalplus.py +288 -0
  15. benchkit/benchmarks/git_surgery.py +485 -0
  16. benchkit/benchmarks/gpqa.py +56 -0
  17. benchkit/benchmarks/gsm8k.py +97 -0
  18. benchkit/benchmarks/hellaswag.py +49 -0
  19. benchkit/benchmarks/humaneval.py +103 -0
  20. benchkit/benchmarks/ifeval.py +56 -0
  21. benchkit/benchmarks/ifeval_instructions.py +477 -0
  22. benchkit/benchmarks/mbpp.py +60 -0
  23. benchkit/benchmarks/mc_arena.py +479 -0
  24. benchkit/benchmarks/mcq.py +94 -0
  25. benchkit/benchmarks/medfailbench.py +537 -0
  26. benchkit/benchmarks/mmlu.py +54 -0
  27. benchkit/benchmarks/mmlu_pro.py +62 -0
  28. benchkit/benchmarks/mmvp.py +115 -0
  29. benchkit/benchmarks/ocrbench.py +96 -0
  30. benchkit/benchmarks/openbookqa.py +52 -0
  31. benchkit/benchmarks/patcheval.py +929 -0
  32. benchkit/benchmarks/piqa.py +52 -0
  33. benchkit/benchmarks/ruler.py +172 -0
  34. benchkit/benchmarks/ruler_generator.py +535 -0
  35. benchkit/benchmarks/sanity.py +174 -0
  36. benchkit/benchmarks/treejs_arena.py +268 -0
  37. benchkit/benchmarks/truthfulqa.py +52 -0
  38. benchkit/benchmarks/utils.py +25 -0
  39. benchkit/benchmarks/vision.py +230 -0
  40. benchkit/benchmarks/vstar.py +108 -0
  41. benchkit/benchmarks/winogrande.py +52 -0
  42. benchkit/benchmarks/xstest.py +116 -0
  43. benchkit/browser.py +429 -0
  44. benchkit/catalog.py +121 -0
  45. benchkit/cli.py +1000 -0
  46. benchkit/client.py +1413 -0
  47. benchkit/controls.py +87 -0
  48. benchkit/datasets/MMLU_LICENSE.txt +21 -0
  49. benchkit/datasets/README.md +142 -0
  50. benchkit/datasets/arc_challenge.jsonl +1172 -0
  51. benchkit/datasets/banking77.jsonl +3080 -0
  52. benchkit/datasets/banking77_intents.json +79 -0
  53. benchkit/datasets/boolq.jsonl +3270 -0
  54. benchkit/datasets/code_verifier.jsonl +500 -0
  55. benchkit/datasets/gpqa_diamond.jsonl +198 -0
  56. benchkit/datasets/gsm8k.jsonl +1319 -0
  57. benchkit/datasets/hellaswag.jsonl +1000 -0
  58. benchkit/datasets/humaneval.jsonl +164 -0
  59. benchkit/datasets/ifeval.jsonl +541 -0
  60. benchkit/datasets/mbpp.jsonl +500 -0
  61. benchkit/datasets/mc_blocks_1_20_1.jsonl +1003 -0
  62. benchkit/datasets/medfailbench_hard30.jsonl +30 -0
  63. benchkit/datasets/medfailbench_hard30.md +13 -0
  64. benchkit/datasets/openbookqa.jsonl +500 -0
  65. benchkit/datasets/piqa.jsonl +1838 -0
  66. benchkit/datasets/sanity_code.jsonl +6 -0
  67. benchkit/datasets/truthfulqa.jsonl +817 -0
  68. benchkit/datasets/winogrande.jsonl +1267 -0
  69. benchkit/datasets/xstest.csv +451 -0
  70. benchkit/decision.py +194 -0
  71. benchkit/demo.py +451 -0
  72. benchkit/dockerfiles.py +247 -0
  73. benchkit/engine.py +1270 -0
  74. benchkit/evaluation.py +195 -0
  75. benchkit/events.py +141 -0
  76. benchkit/executor.py +96 -0
  77. benchkit/git_surgery/backport-release-stack/setup.sh +328 -0
  78. benchkit/git_surgery/backport-release-stack/verify.sh +119 -0
  79. benchkit/git_surgery/bisect-the-regression/setup.sh +49 -0
  80. benchkit/git_surgery/bisect-the-regression/verify.sh +34 -0
  81. benchkit/git_surgery/rebase-conflict-chain/setup.sh +65 -0
  82. benchkit/git_surgery/rebase-conflict-chain/verify.sh +43 -0
  83. benchkit/git_surgery/recover-complex-stash/setup.sh +152 -0
  84. benchkit/git_surgery/recover-complex-stash/verify.sh +95 -0
  85. benchkit/git_surgery/recover-lost-work/setup.sh +40 -0
  86. benchkit/git_surgery/recover-lost-work/verify.sh +36 -0
  87. benchkit/git_surgery/repair-force-pushed-remote/setup.sh +171 -0
  88. benchkit/git_surgery/repair-force-pushed-remote/verify.sh +83 -0
  89. benchkit/git_surgery/revert-merge-with-followups/setup.sh +198 -0
  90. benchkit/git_surgery/revert-merge-with-followups/verify.sh +98 -0
  91. benchkit/git_surgery/secret-in-history/README.md +27 -0
  92. benchkit/git_surgery/secret-in-history/setup.sh +107 -0
  93. benchkit/git_surgery/secret-in-history/verify.sh +106 -0
  94. benchkit/git_surgery/split-the-mega-commit/setup.sh +50 -0
  95. benchkit/git_surgery/split-the-mega-commit/verify.sh +48 -0
  96. benchkit/git_surgery/untangle-nested-submodules/setup.sh +193 -0
  97. benchkit/git_surgery/untangle-nested-submodules/verify.sh +90 -0
  98. benchkit/history.py +474 -0
  99. benchkit/jobs.py +332 -0
  100. benchkit/leakage.py +51 -0
  101. benchkit/looping.py +873 -0
  102. benchkit/mc_render.py +274 -0
  103. benchkit/mc_viewer/dist/atlas.png +0 -0
  104. benchkit/mc_viewer/dist/blockStates.json.gz +0 -0
  105. benchkit/mc_viewer/dist/version.json +4 -0
  106. benchkit/mc_viewer/dist/viewer.html +23 -0
  107. benchkit/mc_viewer/dist/viewer.js +2 -0
  108. benchkit/mc_viewer/dist/viewer.js.LICENSE.txt +25 -0
  109. benchkit/mc_viewer/dist/worker.js +2 -0
  110. benchkit/mc_viewer/dist/worker.js.LICENSE.txt +19 -0
  111. benchkit/metrics.py +100 -0
  112. benchkit/ollama.py +60 -0
  113. benchkit/perf.py +353 -0
  114. benchkit/perf_report.py +207 -0
  115. benchkit/perturbations.py +208 -0
  116. benchkit/pi_agent.py +476 -0
  117. benchkit/pi_package/package-lock.json +1827 -0
  118. benchkit/pi_package/package.json +8 -0
  119. benchkit/report.py +142 -0
  120. benchkit/report_markdown.py +645 -0
  121. benchkit/results.py +688 -0
  122. benchkit/retry.py +163 -0
  123. benchkit/runner.py +769 -0
  124. benchkit/sandbox.py +1167 -0
  125. benchkit/template_check.py +291 -0
  126. benchkit/templates/arena.html +504 -0
  127. benchkit/templates/history.html +58 -0
  128. benchkit/templates/perf.html +509 -0
  129. benchkit/templates/report.html +769 -0
  130. benchkit/tui/__init__.py +5 -0
  131. benchkit/tui/app.py +87 -0
  132. benchkit/tui/app.tcss +512 -0
  133. benchkit/tui/formatting.py +84 -0
  134. benchkit/tui/screens/__init__.py +1 -0
  135. benchkit/tui/screens/connect.py +191 -0
  136. benchkit/tui/screens/detail.py +288 -0
  137. benchkit/tui/screens/help.py +84 -0
  138. benchkit/tui/screens/modals.py +544 -0
  139. benchkit/tui/screens/results.py +361 -0
  140. benchkit/tui/screens/run.py +988 -0
  141. benchkit/tui/screens/setup.py +733 -0
  142. benchkit/tui/theme.py +135 -0
  143. benchkit/tui/widgets.py +120 -0
  144. benchkit_cli-0.1.0.dist-info/METADATA +294 -0
  145. benchkit_cli-0.1.0.dist-info/RECORD +149 -0
  146. benchkit_cli-0.1.0.dist-info/WHEEL +5 -0
  147. benchkit_cli-0.1.0.dist-info/entry_points.txt +3 -0
  148. benchkit_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
  149. benchkit_cli-0.1.0.dist-info/top_level.txt +1 -0
benchkit/datasets/banking77_intents.json ADDED
@@ -0,0 +1,79 @@
1
+ [
2
+ "card_arrival",
3
+ "card_linking",
4
+ "exchange_rate",
5
+ "card_payment_wrong_exchange_rate",
6
+ "extra_charge_on_statement",
7
+ "pending_cash_withdrawal",
8
+ "fiat_currency_support",
9
+ "card_delivery_estimate",
10
+ "automatic_top_up",
11
+ "card_not_working",
12
+ "exchange_via_app",
13
+ "lost_or_stolen_card",
14
+ "age_limit",
15
+ "pin_blocked",
16
+ "contactless_not_working",
17
+ "top_up_by_bank_transfer_charge",
18
+ "pending_top_up",
19
+ "cancel_transfer",
20
+ "top_up_limits",
21
+ "wrong_amount_of_cash_received",
22
+ "card_payment_fee_charged",
23
+ "transfer_not_received_by_recipient",
24
+ "supported_cards_and_currencies",
25
+ "getting_virtual_card",
26
+ "card_acceptance",
27
+ "top_up_reverted",
28
+ "balance_not_updated_after_cheque_or_cash_deposit",
29
+ "card_payment_not_recognised",
30
+ "edit_personal_details",
31
+ "why_verify_identity",
32
+ "unable_to_verify_identity",
33
+ "get_physical_card",
34
+ "visa_or_mastercard",
35
+ "topping_up_by_card",
36
+ "disposable_card_limits",
37
+ "compromised_card",
38
+ "atm_support",
39
+ "direct_debit_payment_not_recognised",
40
+ "passcode_forgotten",
41
+ "declined_cash_withdrawal",
42
+ "pending_card_payment",
43
+ "lost_or_stolen_phone",
44
+ "request_refund",
45
+ "declined_transfer",
46
+ "Refund_not_showing_up",
47
+ "declined_card_payment",
48
+ "pending_transfer",
49
+ "terminate_account",
50
+ "card_swallowed",
51
+ "transaction_charged_twice",
52
+ "verify_source_of_funds",
53
+ "transfer_timing",
54
+ "reverted_card_payment?",
55
+ "change_pin",
56
+ "beneficiary_not_allowed",
57
+ "transfer_fee_charged",
58
+ "receiving_money",
59
+ "failed_transfer",
60
+ "transfer_into_account",
61
+ "verify_top_up",
62
+ "getting_spare_card",
63
+ "top_up_by_cash_or_cheque",
64
+ "order_physical_card",
65
+ "virtual_card_not_working",
66
+ "wrong_exchange_rate_for_cash_withdrawal",
67
+ "get_disposable_virtual_card",
68
+ "top_up_failed",
69
+ "balance_not_updated_after_bank_transfer",
70
+ "cash_withdrawal_not_recognised",
71
+ "exchange_charge",
72
+ "top_up_by_card_charge",
73
+ "activate_my_card",
74
+ "cash_withdrawal_charge",
75
+ "card_about_to_expire",
76
+ "apple_pay_or_google_pay",
77
+ "verify_my_identity",
78
+ "country_support"
79
+ ]