benchkit-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. benchkit/__init__.py +1 -0
  2. benchkit/_pi_proxy.py +219 -0
  3. benchkit/answer_key_guard.ts +49 -0
  4. benchkit/artifacts.py +87 -0
  5. benchkit/benchmarks/__init__.py +105 -0
  6. benchkit/benchmarks/aider_polyglot.py +405 -0
  7. benchkit/benchmarks/arc.py +52 -0
  8. benchkit/benchmarks/banking77.py +149 -0
  9. benchkit/benchmarks/base.py +19 -0
  10. benchkit/benchmarks/boolq.py +49 -0
  11. benchkit/benchmarks/chartqa.py +133 -0
  12. benchkit/benchmarks/code_verifier.py +77 -0
  13. benchkit/benchmarks/downloads.py +166 -0
  14. benchkit/benchmarks/evalplus.py +288 -0
  15. benchkit/benchmarks/git_surgery.py +485 -0
  16. benchkit/benchmarks/gpqa.py +56 -0
  17. benchkit/benchmarks/gsm8k.py +97 -0
  18. benchkit/benchmarks/hellaswag.py +49 -0
  19. benchkit/benchmarks/humaneval.py +103 -0
  20. benchkit/benchmarks/ifeval.py +56 -0
  21. benchkit/benchmarks/ifeval_instructions.py +477 -0
  22. benchkit/benchmarks/mbpp.py +60 -0
  23. benchkit/benchmarks/mc_arena.py +479 -0
  24. benchkit/benchmarks/mcq.py +94 -0
  25. benchkit/benchmarks/medfailbench.py +537 -0
  26. benchkit/benchmarks/mmlu.py +54 -0
  27. benchkit/benchmarks/mmlu_pro.py +62 -0
  28. benchkit/benchmarks/mmvp.py +115 -0
  29. benchkit/benchmarks/ocrbench.py +96 -0
  30. benchkit/benchmarks/openbookqa.py +52 -0
  31. benchkit/benchmarks/patcheval.py +929 -0
  32. benchkit/benchmarks/piqa.py +52 -0
  33. benchkit/benchmarks/ruler.py +172 -0
  34. benchkit/benchmarks/ruler_generator.py +535 -0
  35. benchkit/benchmarks/sanity.py +174 -0
  36. benchkit/benchmarks/treejs_arena.py +268 -0
  37. benchkit/benchmarks/truthfulqa.py +52 -0
  38. benchkit/benchmarks/utils.py +25 -0
  39. benchkit/benchmarks/vision.py +230 -0
  40. benchkit/benchmarks/vstar.py +108 -0
  41. benchkit/benchmarks/winogrande.py +52 -0
  42. benchkit/benchmarks/xstest.py +116 -0
  43. benchkit/browser.py +429 -0
  44. benchkit/catalog.py +121 -0
  45. benchkit/cli.py +1000 -0
  46. benchkit/client.py +1413 -0
  47. benchkit/controls.py +87 -0
  48. benchkit/datasets/MMLU_LICENSE.txt +21 -0
  49. benchkit/datasets/README.md +142 -0
  50. benchkit/datasets/arc_challenge.jsonl +1172 -0
  51. benchkit/datasets/banking77.jsonl +3080 -0
  52. benchkit/datasets/banking77_intents.json +79 -0
  53. benchkit/datasets/boolq.jsonl +3270 -0
  54. benchkit/datasets/code_verifier.jsonl +500 -0
  55. benchkit/datasets/gpqa_diamond.jsonl +198 -0
  56. benchkit/datasets/gsm8k.jsonl +1319 -0
  57. benchkit/datasets/hellaswag.jsonl +1000 -0
  58. benchkit/datasets/humaneval.jsonl +164 -0
  59. benchkit/datasets/ifeval.jsonl +541 -0
  60. benchkit/datasets/mbpp.jsonl +500 -0
  61. benchkit/datasets/mc_blocks_1_20_1.jsonl +1003 -0
  62. benchkit/datasets/medfailbench_hard30.jsonl +30 -0
  63. benchkit/datasets/medfailbench_hard30.md +13 -0
  64. benchkit/datasets/openbookqa.jsonl +500 -0
  65. benchkit/datasets/piqa.jsonl +1838 -0
  66. benchkit/datasets/sanity_code.jsonl +6 -0
  67. benchkit/datasets/truthfulqa.jsonl +817 -0
  68. benchkit/datasets/winogrande.jsonl +1267 -0
  69. benchkit/datasets/xstest.csv +451 -0
  70. benchkit/decision.py +194 -0
  71. benchkit/demo.py +451 -0
  72. benchkit/dockerfiles.py +247 -0
  73. benchkit/engine.py +1270 -0
  74. benchkit/evaluation.py +195 -0
  75. benchkit/events.py +141 -0
  76. benchkit/executor.py +96 -0
  77. benchkit/git_surgery/backport-release-stack/setup.sh +328 -0
  78. benchkit/git_surgery/backport-release-stack/verify.sh +119 -0
  79. benchkit/git_surgery/bisect-the-regression/setup.sh +49 -0
  80. benchkit/git_surgery/bisect-the-regression/verify.sh +34 -0
  81. benchkit/git_surgery/rebase-conflict-chain/setup.sh +65 -0
  82. benchkit/git_surgery/rebase-conflict-chain/verify.sh +43 -0
  83. benchkit/git_surgery/recover-complex-stash/setup.sh +152 -0
  84. benchkit/git_surgery/recover-complex-stash/verify.sh +95 -0
  85. benchkit/git_surgery/recover-lost-work/setup.sh +40 -0
  86. benchkit/git_surgery/recover-lost-work/verify.sh +36 -0
  87. benchkit/git_surgery/repair-force-pushed-remote/setup.sh +171 -0
  88. benchkit/git_surgery/repair-force-pushed-remote/verify.sh +83 -0
  89. benchkit/git_surgery/revert-merge-with-followups/setup.sh +198 -0
  90. benchkit/git_surgery/revert-merge-with-followups/verify.sh +98 -0
  91. benchkit/git_surgery/secret-in-history/README.md +27 -0
  92. benchkit/git_surgery/secret-in-history/setup.sh +107 -0
  93. benchkit/git_surgery/secret-in-history/verify.sh +106 -0
  94. benchkit/git_surgery/split-the-mega-commit/setup.sh +50 -0
  95. benchkit/git_surgery/split-the-mega-commit/verify.sh +48 -0
  96. benchkit/git_surgery/untangle-nested-submodules/setup.sh +193 -0
  97. benchkit/git_surgery/untangle-nested-submodules/verify.sh +90 -0
  98. benchkit/history.py +474 -0
  99. benchkit/jobs.py +332 -0
  100. benchkit/leakage.py +51 -0
  101. benchkit/looping.py +873 -0
  102. benchkit/mc_render.py +274 -0
  103. benchkit/mc_viewer/dist/atlas.png +0 -0
  104. benchkit/mc_viewer/dist/blockStates.json.gz +0 -0
  105. benchkit/mc_viewer/dist/version.json +4 -0
  106. benchkit/mc_viewer/dist/viewer.html +23 -0
  107. benchkit/mc_viewer/dist/viewer.js +2 -0
  108. benchkit/mc_viewer/dist/viewer.js.LICENSE.txt +25 -0
  109. benchkit/mc_viewer/dist/worker.js +2 -0
  110. benchkit/mc_viewer/dist/worker.js.LICENSE.txt +19 -0
  111. benchkit/metrics.py +100 -0
  112. benchkit/ollama.py +60 -0
  113. benchkit/perf.py +353 -0
  114. benchkit/perf_report.py +207 -0
  115. benchkit/perturbations.py +208 -0
  116. benchkit/pi_agent.py +476 -0
  117. benchkit/pi_package/package-lock.json +1827 -0
  118. benchkit/pi_package/package.json +8 -0
  119. benchkit/report.py +142 -0
  120. benchkit/report_markdown.py +645 -0
  121. benchkit/results.py +688 -0
  122. benchkit/retry.py +163 -0
  123. benchkit/runner.py +769 -0
  124. benchkit/sandbox.py +1167 -0
  125. benchkit/template_check.py +291 -0
  126. benchkit/templates/arena.html +504 -0
  127. benchkit/templates/history.html +58 -0
  128. benchkit/templates/perf.html +509 -0
  129. benchkit/templates/report.html +769 -0
  130. benchkit/tui/__init__.py +5 -0
  131. benchkit/tui/app.py +87 -0
  132. benchkit/tui/app.tcss +512 -0
  133. benchkit/tui/formatting.py +84 -0
  134. benchkit/tui/screens/__init__.py +1 -0
  135. benchkit/tui/screens/connect.py +191 -0
  136. benchkit/tui/screens/detail.py +288 -0
  137. benchkit/tui/screens/help.py +84 -0
  138. benchkit/tui/screens/modals.py +544 -0
  139. benchkit/tui/screens/results.py +361 -0
  140. benchkit/tui/screens/run.py +988 -0
  141. benchkit/tui/screens/setup.py +733 -0
  142. benchkit/tui/theme.py +135 -0
  143. benchkit/tui/widgets.py +120 -0
  144. benchkit_cli-0.1.0.dist-info/METADATA +294 -0
  145. benchkit_cli-0.1.0.dist-info/RECORD +149 -0
  146. benchkit_cli-0.1.0.dist-info/WHEEL +5 -0
  147. benchkit_cli-0.1.0.dist-info/entry_points.txt +3 -0
  148. benchkit_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
  149. benchkit_cli-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,152 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ seed="${1:?usage: setup.sh SEED [WORKSPACE]}"
4
+ workspace="${2:-/workspace/recover-complex-stash}"
5
+ slug="$(printf '%s' "$seed:stash" | sha256sum | cut -c1-12)"
6
+ mkdir -p "$workspace"
7
+ cd "$workspace"
8
+ git init -q -b main
9
+ git config user.name "BenchKit Generator"
10
+ git config user.email "generator@benchkit.invalid"
11
+ git config commit.gpgSign false
12
+ git config core.autocrlf false
13
+ export GIT_AUTHOR_NAME="BenchKit Generator" GIT_AUTHOR_EMAIL="generator@benchkit.invalid"
14
+ export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
15
+ export GIT_CONFIG_GLOBAL=/dev/null
16
+ clock=1700700000
17
+ tick() {
18
+ clock=$((clock + 60))
19
+ export GIT_AUTHOR_DATE="@$clock +0000" GIT_COMMITTER_DATE="@$clock +0000"
20
+ }
21
+ commit() {
22
+ tick
23
+ git add -A
24
+ git commit -q "$@"
25
+ }
26
+ stash() {
27
+ tick
28
+ git stash push -q --include-untracked -m "$1"
29
+ }
30
+ replace() {
31
+ python3 - "$@" <<'PY'
32
+ import sys
33
+ from pathlib import Path
34
+
35
+ path, old, new = Path(sys.argv[1]), sys.argv[2], sys.argv[3]
36
+ text = path.read_text()
37
+ if old not in text:
38
+ raise SystemExit(f"setup: {old!r} not found in {path}")
39
+ path.write_text(text.replace(old, new, 1))
40
+ PY
41
+ }
42
+ fixture() {
43
+ mkdir -p fixtures
44
+ printf '{\n "currency": "EUR",\n "rate_bp": %s,\n "source": "%s"\n}\n' "$1" "$2" > fixtures/eur.json
45
+ }
46
+
47
+ printf '__pycache__/\n' > .gitignore
48
+ cat > legacy_rates.py <<'PY'
49
+ """Exchange rates in basis points of one USD."""
50
+
51
+ RATES = {
52
+ "USD": 10000,
53
+ "EUR": 9200,
54
+ }
55
+
56
+
57
+ def convert(amount: int, currency: str) -> int:
58
+ return amount * RATES[currency] // 10000
59
+ PY
60
+ cat > settings.py <<'PY'
61
+ """Quote settings."""
62
+
63
+ DEFAULT_CURRENCY = "USD"
64
+ PRECISION = 2
65
+ PY
66
+ cat > app.py <<'PY'
67
+ from legacy_rates import convert
68
+ from settings import DEFAULT_CURRENCY, PRECISION
69
+
70
+
71
+ def quote(amount: int) -> str:
72
+ value = convert(amount, DEFAULT_CURRENCY)
73
+ return f"{value:.{PRECISION}f} {DEFAULT_CURRENCY}"
74
+ PY
75
+ cat > test_app.py <<'PY'
76
+ import unittest
77
+
78
+ from app import quote
79
+ from settings import DEFAULT_CURRENCY, PRECISION
80
+
81
+
82
+ class QuoteTests(unittest.TestCase):
83
+ def test_quote_follows_settings(self):
84
+ self.assertTrue(quote(100).endswith(f" {DEFAULT_CURRENCY}"))
85
+ self.assertEqual(len(quote(100).split()[0].split(".")[1]), PRECISION)
86
+ PY
87
+ printf '# Quote service %s\n\nRun tests with `python3 -m unittest discover -v`.\n' "$slug" > README.md
88
+ commit -m "bootstrap quote service $slug"
89
+ printf '\nQuotes are rendered in the configured default currency.\n' >> README.md
90
+ commit -m "Document quote currency"
91
+
92
+ # First attempt: the rename is staged, but the fixture is for the wrong currency.
93
+ git mv legacy_rates.py rates.py
94
+ replace app.py 'from legacy_rates import convert' 'from rates import convert'
95
+ git add app.py
96
+ mkdir -p fixtures
97
+ printf '{\n "currency": "GBP",\n "rate_bp": 7900\n}\n' > fixtures/gbp.json
98
+ stash "rates migration (first try)"
99
+
100
+ # The wanted stash: staged rename and import, unstaged edits, untracked files.
101
+ git mv legacy_rates.py rates.py
102
+ replace app.py 'from legacy_rates import convert' 'from rates import convert'
103
+ git add app.py
104
+ replace rates.py ' "EUR": 9200,
105
+ ' ' "EUR": 9200,
106
+ "GBP": 7900,
107
+ '
108
+ replace settings.py 'DEFAULT_CURRENCY = "USD"' 'DEFAULT_CURRENCY = "EUR"'
109
+ fixture 9200 "$slug"
110
+ cat > test_rates.py <<'PY'
111
+ import json
112
+ import unittest
113
+ from pathlib import Path
114
+
115
+ from rates import RATES, convert
116
+
117
+
118
+ class RatesTests(unittest.TestCase):
119
+ def test_fixture_matches_rates(self):
120
+ fixture = json.loads(Path("fixtures/eur.json").read_text())
121
+ self.assertEqual(RATES[fixture["currency"]], fixture["rate_bp"])
122
+
123
+ def test_gbp_conversion(self):
124
+ self.assertEqual(convert(1000, "GBP"), 790)
125
+ PY
126
+ stash "rates migration"
127
+
128
+ # Newest attempt: copies instead of renaming and ships a stale fixture.
129
+ cp legacy_rates.py rates.py
130
+ replace app.py 'from legacy_rates import convert' 'from rates import convert'
131
+ replace settings.py 'DEFAULT_CURRENCY = "USD"' 'DEFAULT_CURRENCY = "EUR"'
132
+ fixture 9100 "stale-$slug"
133
+ stash "rates migration v2"
134
+
135
+ # A stray snapshot commit of the migration with another stale fixture.
136
+ git switch -q -c snapshot
137
+ git mv legacy_rates.py rates.py
138
+ replace app.py 'from legacy_rates import convert' 'from rates import convert'
139
+ fixture 9150 "snapshot-$slug"
140
+ commit -m "WIP on main: rates migration snapshot"
141
+ git switch -q main
142
+ git branch -q -D snapshot
143
+ git stash clear
144
+
145
+ # main moves on after the stash was dropped.
146
+ replace settings.py 'PRECISION = 2' 'PRECISION = 4'
147
+ commit -m "Quote with four decimal places"
148
+ printf '# Changelog\n\n- Quotes now use four decimal places.\n' > CHANGELOG.md
149
+ commit -m "Start a changelog"
150
+ unset GIT_AUTHOR_DATE GIT_COMMITTER_DATE
151
+ git reflog expire --expire=now --all
152
+ git status --porcelain | grep -q . && exit 1 || true
@@ -0,0 +1,95 @@
1
+ #!/usr/bin/env bash
2
+ set -u
3
+ seed="${1:?usage: verify.sh SEED [WORKSPACE]}"
4
+ workspace="${2:-/workspace/recover-complex-stash}"
5
+ emit() { printf '%s\t%s\t%s\n' "$1" "$2" "$3"; }
6
+ ids="explored_objects two_commits staged_commit_exact remaining_commit_exact main_changes_kept tests_pass"
7
+ if ! git -C "$workspace" rev-parse --git-dir >/dev/null 2>&1; then
8
+ for id in $ids; do emit "$id" 0 "repository missing"; done
9
+ emit trap 1 "repository replaced"
10
+ exit 0
11
+ fi
12
+ reference_root="$(mktemp -d)"
13
+ reference="$reference_root/reference"
14
+ bash "$(dirname "$0")/setup.sh" "$seed" "$reference" >/dev/null
15
+ git_r() { git -C "$reference" "$@"; }
16
+ main_tip="$(git_r rev-parse main)"
17
+ target=""
18
+ decoy_blobs=""
19
+ for commit in $(git_r fsck --unreachable --no-reflogs 2>/dev/null | awk '/unreachable commit/{print $3}'); do
20
+ if git_r cat-file -e "$commit^3" 2>/dev/null \
21
+ && git_r cat-file -e "$commit^2:rates.py" 2>/dev/null \
22
+ && ! git_r cat-file -e "$commit^2:legacy_rates.py" 2>/dev/null \
23
+ && git_r cat-file -e "$commit^3:fixtures/eur.json" 2>/dev/null; then
24
+ target="$commit"
25
+ fi
26
+ done
27
+ for commit in $(git_r fsck --unreachable --no-reflogs 2>/dev/null | awk '/unreachable commit/{print $3}'); do
28
+ [[ "$commit" == "$target" || "$commit" == "$(git_r rev-parse "$target^2")" || "$commit" == "$(git_r rev-parse "$target^3")" ]] && continue
29
+ for path in fixtures/eur.json fixtures/gbp.json; do
30
+ blob="$(git_r rev-parse -q --verify "$commit:$path" 2>/dev/null || true)"
31
+ [[ -n "$blob" ]] && decoy_blobs+=" $blob"
32
+ done
33
+ done
34
+ target_fixture="$(git_r rev-parse "$target^3:fixtures/eur.json")"
35
+ decoy_blobs="$(tr ' ' '\n' <<<"$decoy_blobs" | grep -v "^$target_fixture$" | sort -u | tr '\n' ' ')"
36
+ export GIT_INDEX_FILE="$reference_root/index"
37
+ git_r read-tree "$main_tip"
38
+ git_r diff --binary "$target^1" "$target^2" | git_r apply --cached
39
+ expected_staged="$(git_r write-tree)"
40
+ git_r diff --binary "$target^2" "$target" -- . ':(exclude)settings.py' | git_r apply --cached
41
+ git_r ls-tree -r "$target^3" | git_r update-index --index-info
42
+ expected_final="$(git_r ls-tree -r "$(git_r write-tree)" | grep -v $'\tsettings.py$')"
43
+ unset GIT_INDEX_FILE
44
+ changelog="$(git_r rev-parse "$main_tip:CHANGELOG.md")"
45
+ rm -rf "$reference_root"
46
+
47
+ git_w() { git -C "$workspace" "$@" 2>/dev/null; }
48
+ head="$(git_w rev-parse -q --verify HEAD)"
49
+ branch="$(git_w symbolic-ref -q --short HEAD)"
50
+ emit explored_objects 1 "trace-verified"
51
+ new_commits=()
52
+ git_w merge-base --is-ancestor "$main_tip" "$head" && mapfile -t new_commits < <(git_w rev-list --reverse "$main_tip..$head")
53
+ merges="$(git_w rev-list --merges --count "$main_tip..$head" || printf 0)"
54
+ subjects=""
55
+ for commit in "${new_commits[@]}"; do subjects+="$(git_w log -1 --format=%s "$commit")|"; done
56
+ two=0
57
+ [[ "${#new_commits[@]}" == 2 && "$merges" == 0 && "$branch" == main && "$subjects" == "Migrate rates module|Finish rates migration|" ]] && two=1
58
+ emit two_commits "$two" "count=${#new_commits[@]} branch=$branch subjects=$subjects"
59
+
60
+ staged=0
61
+ remaining=0
62
+ if [[ "$two" == 1 ]]; then
63
+ [[ "$(git_w rev-parse "${new_commits[0]}^{tree}")" == "$expected_staged" ]] && staged=1
64
+ actual_final="$(git_w ls-tree -r "${new_commits[1]}" | grep -v $'\tsettings.py$')"
65
+ [[ "$actual_final" == "$expected_final" ]] && remaining=1
66
+ fi
67
+ emit staged_commit_exact "$staged" "staged_tree_match=$staged"
68
+ emit remaining_commit_exact "$remaining" "final_tree_match_except_settings=$remaining"
69
+
70
+ tree="$(mktemp -d)"
71
+ [[ -n "$head" ]] && git_w archive "$head" | tar -x -C "$tree" 2>/dev/null
72
+ kept=0
73
+ if [[ "$two" == 1 && "$(git_w rev-parse "$head:CHANGELOG.md")" == "$changelog" ]]; then
74
+ (cd "$tree" && python3 -B -c '
75
+ import settings
76
+ assert settings.DEFAULT_CURRENCY == "EUR"
77
+ assert settings.PRECISION == 4') >/dev/null 2>&1 && kept=1
78
+ fi
79
+ emit main_changes_kept "$kept" "settings_merged_and_changelog=$kept"
80
+ tests=0
81
+ if [[ "$two" == 1 && -f "$tree/test_rates.py" ]]; then
82
+ (cd "$tree" && python3 -B -m unittest discover) >/dev/null 2>&1 && tests=1
83
+ fi
84
+ rm -rf "$tree"
85
+ emit tests_pass "$tests" "recovered_and_existing_tests=$tests"
86
+
87
+ trap=0
88
+ git_w merge-base --is-ancestor "$main_tip" "$head" || trap=1
89
+ for commit in "${new_commits[@]}"; do
90
+ for path in fixtures/eur.json fixtures/gbp.json; do
91
+ blob="$(git_w rev-parse -q --verify "$commit:$path" || true)"
92
+ [[ -n "$blob" && " $decoy_blobs " == *" $blob "* ]] && trap=1
93
+ done
94
+ done
95
+ emit trap "$trap" "decoy_stash_used_or_main_rewritten=$trap"
@@ -0,0 +1,40 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ seed="${1:?usage: setup.sh SEED [WORKSPACE]}"
4
+ workspace="${2:-/workspace/recover-lost-work}"
5
+ slug="$(printf '%s' "$seed:recover" | sha256sum | cut -c1-12)"
6
+ mkdir -p "$workspace"
7
+ cd "$workspace"
8
+ git init -q -b main
9
+ git config user.name "BenchKit Generator"
10
+ git config user.email "generator@benchkit.invalid"
11
+ git config commit.gpgSign false
12
+ git config core.autocrlf false
13
+ export GIT_AUTHOR_NAME="BenchKit Generator" GIT_AUTHOR_EMAIL="generator@benchkit.invalid"
14
+ export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
15
+ printf '# Recovery exercise %s\n\nRun tests with `python3 -m unittest discover -v`.\n' "$slug" > README.md
16
+ git add README.md
17
+ GIT_AUTHOR_DATE='@1700300000 +0000' GIT_COMMITTER_DATE='@1700300000 +0000' git commit -q -m "bootstrap recovery workspace $slug"
18
+ git tag "benchkit-recover-baseline-$slug"
19
+ git switch -q -c abandoned-work
20
+ cat > recovery.py <<EOF
21
+ RECOVERY_TOKEN = "$slug"
22
+
23
+
24
+ def recovered_value() -> str:
25
+ return f"recovered:{RECOVERY_TOKEN}"
26
+ EOF
27
+ cat > test_recovery.py <<EOF
28
+ import unittest
29
+ from recovery import recovered_value
30
+
31
+
32
+ class RecoveryTests(unittest.TestCase):
33
+ def test_recovered_value(self):
34
+ self.assertEqual(recovered_value(), "recovered:$slug")
35
+ EOF
36
+ git add .
37
+ GIT_AUTHOR_DATE='@1700300060 +0000' GIT_COMMITTER_DATE='@1700300060 +0000' git commit -q -m "preserve recovered payload $slug"
38
+ git switch -q main
39
+ git branch -D abandoned-work >/dev/null
40
+ git reflog expire --expire=now --all
@@ -0,0 +1,36 @@
1
+ #!/usr/bin/env bash
2
+ set -u
3
+ seed="${1:?usage: verify.sh SEED [WORKSPACE]}"
4
+ workspace="${2:-/workspace/recover-lost-work}"
5
+ emit() { printf '%s\t%s\t%s\n' "$1" "$2" "$3"; }
6
+ if ! git -C "$workspace" rev-parse --git-dir >/dev/null 2>&1; then
7
+ for id in explored_objects branch_restored original_object recovered_tests history_preserved; do emit "$id" 0 "repository missing"; done
8
+ emit trap 1 "repository replaced"
9
+ exit 0
10
+ fi
11
+ reference_root="$(mktemp -d)"
12
+ reference="$reference_root/reference"
13
+ bash "$(dirname "$0")/setup.sh" "$seed" "$reference" >/dev/null
14
+ lost="$(git -C "$reference" fsck --unreachable --no-reflogs 2>/dev/null | awk '/unreachable commit/{print $3; exit}')"
15
+ baseline="$(git -C "$reference" rev-parse main)"
16
+ rm -rf "$reference_root"
17
+ tip="$(git -C "$workspace" rev-parse -q --verify refs/heads/recovered-work 2>/dev/null || true)"
18
+ [[ -n "$tip" ]] && branch=1 || branch=0
19
+ emit explored_objects 1 "trace-verified"
20
+ emit branch_restored "$branch" "tip=$tip"
21
+ [[ -n "$lost" && "$tip" == "$lost" ]] && original=1 || original=0
22
+ emit original_object "$original" "expected=$lost tip=$tip"
23
+ tests=0
24
+ if [[ "$original" == 1 ]]; then
25
+ tmp="$(mktemp -d)"
26
+ git -C "$workspace" archive "$tip" | tar -x -C "$tmp"
27
+ (cd "$tmp" && python3 -m unittest discover -v) >/dev/null 2>&1 && tests=1
28
+ rm -rf "$tmp"
29
+ fi
30
+ emit recovered_tests "$tests" "exit=$((1 - tests))"
31
+ slug="$(printf '%s' "$seed:recover" | sha256sum | cut -c1-12)"
32
+ tag="$(git -C "$workspace" rev-parse -q --verify "refs/tags/benchkit-recover-baseline-$slug" 2>/dev/null || true)"
33
+ [[ "$tag" == "$baseline" ]] && history=1 || history=0
34
+ emit history_preserved "$history" "baseline_match=$history"
35
+ [[ "$branch" == 1 && "$original" == 0 ]] && trap=1 || trap=0
36
+ emit trap "$trap" "retyped_commit=$trap"
@@ -0,0 +1,171 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ seed="${1:?usage: setup.sh SEED [WORKSPACE]}"
4
+ workspace="${2:-/workspace/repair-force-pushed-remote}"
5
+ slug="$(printf '%s' "$seed:force-push" | sha256sum | cut -c1-12)"
6
+ mkdir -p "$workspace"
7
+ cd "$workspace"
8
+ export GIT_AUTHOR_NAME="BenchKit Generator" GIT_AUTHOR_EMAIL="generator@benchkit.invalid"
9
+ export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
10
+ export GIT_CONFIG_GLOBAL=/dev/null
11
+ clock=1700800000
12
+ tick() {
13
+ clock=$((clock + 60))
14
+ export GIT_AUTHOR_DATE="@$clock +0000" GIT_COMMITTER_DATE="@$clock +0000"
15
+ }
16
+ configure() {
17
+ git -C "$1" config user.name "$2"
18
+ git -C "$1" config user.email "$(tr 'A-Z' 'a-z' <<<"$2")@benchkit.invalid"
19
+ git -C "$1" config commit.gpgSign false
20
+ git -C "$1" config core.autocrlf false
21
+ git -C "$1" config core.logAllRefUpdates always
22
+ }
23
+ commit() {
24
+ local clone="$1"
25
+ shift
26
+ tick
27
+ git -C "$clone" add -A
28
+ git -C "$clone" commit -q "$@"
29
+ }
30
+ push() {
31
+ tick
32
+ git -C "$1" push -q "${@:2}" 2>/dev/null
33
+ }
34
+
35
+ git init -q --bare -b main origin.git
36
+ git -C origin.git config core.logAllRefUpdates always
37
+ git -C origin.git config receive.denyNonFastForwards false
38
+
39
+ tick
40
+ git clone -q origin.git bob 2>/dev/null
41
+ configure bob Bob
42
+ printf '__pycache__/\n' > bob/.gitignore
43
+ cat > bob/invoice.py <<'PY'
44
+ """Invoice helpers in integer cents."""
45
+
46
+
47
+ def subtotal(lines: list[int]) -> int:
48
+ return sum(lines)
49
+ PY
50
+ cat > bob/test_invoice.py <<'PY'
51
+ import unittest
52
+
53
+ from invoice import subtotal
54
+
55
+
56
+ class InvoiceTests(unittest.TestCase):
57
+ def test_subtotal(self):
58
+ self.assertEqual(subtotal([100, 250]), 350)
59
+ PY
60
+ printf '# Invoicing %s\n\nRun tests with `python3 -m unittest discover -v`.\n' "$slug" > bob/README.md
61
+ printf '1.0.0\n' > bob/VERSION
62
+ commit bob -m "bootstrap invoicing $slug"
63
+ tick
64
+ git -C bob tag -a v1.0 -m "Invoicing 1.0"
65
+ push bob origin main v1.0
66
+
67
+ git -C bob switch -q -c release
68
+ cat >> bob/invoice.py <<'PY'
69
+
70
+
71
+ def tax(amount: int) -> int:
72
+ return amount * 20 // 100
73
+ PY
74
+ commit bob -m "release: add tax helper"
75
+ cat >> bob/test_invoice.py <<'PY'
76
+
77
+
78
+ class TaxTests(unittest.TestCase):
79
+ def test_tax(self):
80
+ from invoice import tax
81
+
82
+ self.assertEqual(tax(1000), 200)
83
+ PY
84
+ commit bob -m "release: cover tax helper"
85
+ printf '1.1.0\n' > bob/VERSION
86
+ commit bob -m "release: prepare 1.1"
87
+ tick
88
+ git -C bob tag -a v1.1 -m "Invoicing 1.1"
89
+ push bob -u origin release v1.1
90
+
91
+ git -C bob switch -q main
92
+ printf '\nSee CONTRIBUTING for the release process.\n' >> bob/README.md
93
+ commit bob -m "Document the release process"
94
+ push bob origin main
95
+
96
+ tick
97
+ git clone -q origin.git alice 2>/dev/null
98
+ configure alice Alice
99
+ git -C alice switch -q release
100
+ cat >> alice/invoice.py <<'PY'
101
+
102
+
103
+ def discount(amount: int, percent: int) -> int:
104
+ return amount - amount * percent // 100
105
+ PY
106
+ cat > alice/test_discount.py <<'PY'
107
+ import unittest
108
+
109
+ from invoice import discount
110
+
111
+
112
+ class DiscountTests(unittest.TestCase):
113
+ def test_discount(self):
114
+ self.assertEqual(discount(1000, 15), 850)
115
+ PY
116
+ commit alice -m "release: add percentage discounts"
117
+ cat >> alice/invoice.py <<'PY'
118
+
119
+
120
+ def grand_total(lines: list[int], percent: int = 0) -> int:
121
+ net = discount(subtotal(lines), percent)
122
+ return net + tax(net)
123
+ PY
124
+ cat > alice/test_grand_total.py <<'PY'
125
+ import unittest
126
+
127
+ from invoice import grand_total
128
+
129
+
130
+ class GrandTotalTests(unittest.TestCase):
131
+ def test_grand_total(self):
132
+ self.assertEqual(grand_total([600, 400], 10), 1080)
133
+ PY
134
+ printf '1.2.0-rc1\n' > alice/VERSION
135
+ commit alice -m "release: prepare 1.2"
136
+ printf '1.2.0\n' > alice/VERSION
137
+ tick
138
+ git -C alice add -A
139
+ git -C alice commit -q --amend -m "release: prepare 1.2"
140
+ push alice origin release
141
+
142
+ # A candidate that was pushed for review and then deleted again.
143
+ git -C alice switch -q -c candidate
144
+ printf '1.2.0-final\n' > alice/VERSION
145
+ commit alice -m "release: prepare 1.2 final"
146
+ push alice origin candidate
147
+ push alice origin --delete candidate
148
+ git -C alice switch -q release
149
+ git -C alice branch -q -D candidate
150
+
151
+ # Bob rewords his stale tip and force-pushes over Alice's work.
152
+ tick
153
+ git -C bob switch -q release
154
+ git -C bob commit -q --amend -m "release: prepare 1.1 (reworded)"
155
+ push bob --force origin release
156
+
157
+ # Alice syncs to the rewritten branch and pushes a legitimate fix on top.
158
+ tick
159
+ git -C alice fetch -q origin
160
+ tick
161
+ git -C alice reset -q --hard origin/release
162
+ printf '\nInvoices round tax down to whole cents.\n' >> alice/README.md
163
+ cat > alice/CHANGES.md <<'PY'
164
+ # Changes
165
+
166
+ - Tax is rounded down to whole cents.
167
+ PY
168
+ commit alice -m "release: document tax rounding"
169
+ push alice origin release
170
+ git -C bob fetch -q origin
171
+ unset GIT_AUTHOR_DATE GIT_COMMITTER_DATE
@@ -0,0 +1,83 @@
1
+ #!/usr/bin/env bash
2
+ set -u
3
+ seed="${1:?usage: verify.sh SEED [WORKSPACE]}"
4
+ workspace="${2:-/workspace/repair-force-pushed-remote}"
5
+ emit() { printf '%s\t%s\t%s\n' "$1" "$2" "$3"; }
6
+ ids="remote_intact lineage_restored followup_retained healthy_refs_untouched tests_pass"
7
+ origin="$workspace/origin.git"
8
+ if [[ "$(git -C "$origin" rev-parse --is-bare-repository 2>/dev/null)" != true ]]; then
9
+ for id in $ids; do emit "$id" 0 "bare origin missing"; done
10
+ emit trap 1 "origin replaced or removed"
11
+ exit 0
12
+ fi
13
+ reference_root="$(mktemp -d)"
14
+ reference="$reference_root/reference"
15
+ bash "$(dirname "$0")/setup.sh" "$seed" "$reference" >/dev/null
16
+ git_r() { git -C "$reference/origin.git" "$@"; }
17
+ mapfile -t history < <(git_r reflog show --format=%H refs/heads/release)
18
+ # Newest first: Alice's fix, Bob's force-push, Alice's 1.2 push, first publish.
19
+ published="${history[2]}"
20
+ rewritten="${history[1]}"
21
+ followup="${history[0]}"
22
+ first_publish="${history[3]}"
23
+ decoys=""
24
+ for commit in $(git_r fsck --unreachable --no-reflogs 2>/dev/null | awk '/unreachable commit/{print $3}'); do
25
+ git_r merge-base --is-ancestor "$commit" "$published" || decoys+=" $commit"
26
+ done
27
+ decoys+=" $(git -C "$reference/alice" reflog --format='%H %gs' | awk '/^[0-9a-f]+ commit: release: prepare 1.2$/{print $1}' | tr '\n' ' ')"
28
+ healthy_refs="$(git_r for-each-ref --format='%(objectname) %(refname)' | grep -v ' refs/heads/release$')"
29
+ export GIT_INDEX_FILE="$reference_root/index"
30
+ git_r read-tree "$published"
31
+ git_r diff --binary "$followup^" "$followup" | git_r apply --cached
32
+ expected_tree="$(git_r write-tree)"
33
+ unset GIT_INDEX_FILE
34
+ followup_patch="$(git_r diff "$followup^" "$followup" | git_r patch-id --stable | cut -d' ' -f1)"
35
+ rm -rf "$reference_root"
36
+
37
+ git_o() { git -C "$origin" "$@" 2>/dev/null; }
38
+ release="$(git_o rev-parse -q --verify refs/heads/release)"
39
+ changed=0
40
+ [[ -n "$release" && "$release" != "$followup" ]] && changed=1
41
+ reflog="$(git_o reflog show --format=%H refs/heads/release | tr '\n' ' ')"
42
+ intact=0
43
+ [[ "$changed" == 1 && "$reflog" == *"$first_publish"* && "$reflog" == *"$published"* ]] && git_o cat-file -e "$published" && intact=1
44
+ emit remote_intact "$intact" "bare=1 release_reflog_kept=$intact"
45
+
46
+ parent="$(git_o rev-parse -q --verify "$release^")"
47
+ lineage=0
48
+ [[ "$parent" == "$published" ]] && lineage=1
49
+ git_o merge-base --is-ancestor "$rewritten" "$release" && lineage=0
50
+ emit lineage_restored "$lineage" "release_parent=$parent expected=$published"
51
+
52
+ retained=0
53
+ if [[ "$lineage" == 1 ]]; then
54
+ parents="$(git_o rev-list --parents -n 1 "$release" | wc -w)"
55
+ patch="$(git_o diff "$release^" "$release" | git patch-id --stable | cut -d' ' -f1)"
56
+ tree="$(git_o rev-parse "$release^{tree}")"
57
+ [[ "$parents" == 2 && "$patch" == "$followup_patch" && "$tree" == "$expected_tree" ]] && retained=1
58
+ fi
59
+ emit followup_retained "$retained" "followup_reapplied_on_published_tip=$retained"
60
+
61
+ current_refs="$(git_o for-each-ref --format='%(objectname) %(refname)' | grep -v ' refs/heads/release$')"
62
+ healthy=0
63
+ [[ "$changed" == 1 && "$current_refs" == "$healthy_refs" ]] && healthy=1
64
+ emit healthy_refs_untouched "$healthy" "other_refs_match=$([[ "$current_refs" == "$healthy_refs" ]] && echo 1 || echo 0)"
65
+
66
+ tests=0
67
+ if [[ "$changed" == 1 ]]; then
68
+ tmp="$(mktemp -d)"
69
+ git_o archive "$release" | tar -x -C "$tmp" 2>/dev/null
70
+ (cd "$tmp" && python3 -B -m unittest discover) >/dev/null 2>&1 \
71
+ && [[ -f "$tmp/test_grand_total.py" && -f "$tmp/CHANGES.md" ]] && tests=1
72
+ rm -rf "$tmp"
73
+ fi
74
+ emit tests_pass "$tests" "release_tests=$tests"
75
+
76
+ trap=0
77
+ [[ "$current_refs" == "$healthy_refs" ]] || trap=1
78
+ [[ -n "$release" ]] || trap=1
79
+ [[ "$reflog" == *"$first_publish"* ]] || trap=1
80
+ for decoy in $decoys; do
81
+ [[ -n "$release" ]] && git_o merge-base --is-ancestor "$decoy" "$release" && trap=1
82
+ done
83
+ emit trap "$trap" "healthy_refs_changed_or_wrong_lineage=$trap"