benchkit-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- benchkit/__init__.py +1 -0
- benchkit/_pi_proxy.py +219 -0
- benchkit/answer_key_guard.ts +49 -0
- benchkit/artifacts.py +87 -0
- benchkit/benchmarks/__init__.py +105 -0
- benchkit/benchmarks/aider_polyglot.py +405 -0
- benchkit/benchmarks/arc.py +52 -0
- benchkit/benchmarks/banking77.py +149 -0
- benchkit/benchmarks/base.py +19 -0
- benchkit/benchmarks/boolq.py +49 -0
- benchkit/benchmarks/chartqa.py +133 -0
- benchkit/benchmarks/code_verifier.py +77 -0
- benchkit/benchmarks/downloads.py +166 -0
- benchkit/benchmarks/evalplus.py +288 -0
- benchkit/benchmarks/git_surgery.py +485 -0
- benchkit/benchmarks/gpqa.py +56 -0
- benchkit/benchmarks/gsm8k.py +97 -0
- benchkit/benchmarks/hellaswag.py +49 -0
- benchkit/benchmarks/humaneval.py +103 -0
- benchkit/benchmarks/ifeval.py +56 -0
- benchkit/benchmarks/ifeval_instructions.py +477 -0
- benchkit/benchmarks/mbpp.py +60 -0
- benchkit/benchmarks/mc_arena.py +479 -0
- benchkit/benchmarks/mcq.py +94 -0
- benchkit/benchmarks/medfailbench.py +537 -0
- benchkit/benchmarks/mmlu.py +54 -0
- benchkit/benchmarks/mmlu_pro.py +62 -0
- benchkit/benchmarks/mmvp.py +115 -0
- benchkit/benchmarks/ocrbench.py +96 -0
- benchkit/benchmarks/openbookqa.py +52 -0
- benchkit/benchmarks/patcheval.py +929 -0
- benchkit/benchmarks/piqa.py +52 -0
- benchkit/benchmarks/ruler.py +172 -0
- benchkit/benchmarks/ruler_generator.py +535 -0
- benchkit/benchmarks/sanity.py +174 -0
- benchkit/benchmarks/treejs_arena.py +268 -0
- benchkit/benchmarks/truthfulqa.py +52 -0
- benchkit/benchmarks/utils.py +25 -0
- benchkit/benchmarks/vision.py +230 -0
- benchkit/benchmarks/vstar.py +108 -0
- benchkit/benchmarks/winogrande.py +52 -0
- benchkit/benchmarks/xstest.py +116 -0
- benchkit/browser.py +429 -0
- benchkit/catalog.py +121 -0
- benchkit/cli.py +1000 -0
- benchkit/client.py +1413 -0
- benchkit/controls.py +87 -0
- benchkit/datasets/MMLU_LICENSE.txt +21 -0
- benchkit/datasets/README.md +142 -0
- benchkit/datasets/arc_challenge.jsonl +1172 -0
- benchkit/datasets/banking77.jsonl +3080 -0
- benchkit/datasets/banking77_intents.json +79 -0
- benchkit/datasets/boolq.jsonl +3270 -0
- benchkit/datasets/code_verifier.jsonl +500 -0
- benchkit/datasets/gpqa_diamond.jsonl +198 -0
- benchkit/datasets/gsm8k.jsonl +1319 -0
- benchkit/datasets/hellaswag.jsonl +1000 -0
- benchkit/datasets/humaneval.jsonl +164 -0
- benchkit/datasets/ifeval.jsonl +541 -0
- benchkit/datasets/mbpp.jsonl +500 -0
- benchkit/datasets/mc_blocks_1_20_1.jsonl +1003 -0
- benchkit/datasets/medfailbench_hard30.jsonl +30 -0
- benchkit/datasets/medfailbench_hard30.md +13 -0
- benchkit/datasets/openbookqa.jsonl +500 -0
- benchkit/datasets/piqa.jsonl +1838 -0
- benchkit/datasets/sanity_code.jsonl +6 -0
- benchkit/datasets/truthfulqa.jsonl +817 -0
- benchkit/datasets/winogrande.jsonl +1267 -0
- benchkit/datasets/xstest.csv +451 -0
- benchkit/decision.py +194 -0
- benchkit/demo.py +451 -0
- benchkit/dockerfiles.py +247 -0
- benchkit/engine.py +1270 -0
- benchkit/evaluation.py +195 -0
- benchkit/events.py +141 -0
- benchkit/executor.py +96 -0
- benchkit/git_surgery/backport-release-stack/setup.sh +328 -0
- benchkit/git_surgery/backport-release-stack/verify.sh +119 -0
- benchkit/git_surgery/bisect-the-regression/setup.sh +49 -0
- benchkit/git_surgery/bisect-the-regression/verify.sh +34 -0
- benchkit/git_surgery/rebase-conflict-chain/setup.sh +65 -0
- benchkit/git_surgery/rebase-conflict-chain/verify.sh +43 -0
- benchkit/git_surgery/recover-complex-stash/setup.sh +152 -0
- benchkit/git_surgery/recover-complex-stash/verify.sh +95 -0
- benchkit/git_surgery/recover-lost-work/setup.sh +40 -0
- benchkit/git_surgery/recover-lost-work/verify.sh +36 -0
- benchkit/git_surgery/repair-force-pushed-remote/setup.sh +171 -0
- benchkit/git_surgery/repair-force-pushed-remote/verify.sh +83 -0
- benchkit/git_surgery/revert-merge-with-followups/setup.sh +198 -0
- benchkit/git_surgery/revert-merge-with-followups/verify.sh +98 -0
- benchkit/git_surgery/secret-in-history/README.md +27 -0
- benchkit/git_surgery/secret-in-history/setup.sh +107 -0
- benchkit/git_surgery/secret-in-history/verify.sh +106 -0
- benchkit/git_surgery/split-the-mega-commit/setup.sh +50 -0
- benchkit/git_surgery/split-the-mega-commit/verify.sh +48 -0
- benchkit/git_surgery/untangle-nested-submodules/setup.sh +193 -0
- benchkit/git_surgery/untangle-nested-submodules/verify.sh +90 -0
- benchkit/history.py +474 -0
- benchkit/jobs.py +332 -0
- benchkit/leakage.py +51 -0
- benchkit/looping.py +873 -0
- benchkit/mc_render.py +274 -0
- benchkit/mc_viewer/dist/atlas.png +0 -0
- benchkit/mc_viewer/dist/blockStates.json.gz +0 -0
- benchkit/mc_viewer/dist/version.json +4 -0
- benchkit/mc_viewer/dist/viewer.html +23 -0
- benchkit/mc_viewer/dist/viewer.js +2 -0
- benchkit/mc_viewer/dist/viewer.js.LICENSE.txt +25 -0
- benchkit/mc_viewer/dist/worker.js +2 -0
- benchkit/mc_viewer/dist/worker.js.LICENSE.txt +19 -0
- benchkit/metrics.py +100 -0
- benchkit/ollama.py +60 -0
- benchkit/perf.py +353 -0
- benchkit/perf_report.py +207 -0
- benchkit/perturbations.py +208 -0
- benchkit/pi_agent.py +476 -0
- benchkit/pi_package/package-lock.json +1827 -0
- benchkit/pi_package/package.json +8 -0
- benchkit/report.py +142 -0
- benchkit/report_markdown.py +645 -0
- benchkit/results.py +688 -0
- benchkit/retry.py +163 -0
- benchkit/runner.py +769 -0
- benchkit/sandbox.py +1167 -0
- benchkit/template_check.py +291 -0
- benchkit/templates/arena.html +504 -0
- benchkit/templates/history.html +58 -0
- benchkit/templates/perf.html +509 -0
- benchkit/templates/report.html +769 -0
- benchkit/tui/__init__.py +5 -0
- benchkit/tui/app.py +87 -0
- benchkit/tui/app.tcss +512 -0
- benchkit/tui/formatting.py +84 -0
- benchkit/tui/screens/__init__.py +1 -0
- benchkit/tui/screens/connect.py +191 -0
- benchkit/tui/screens/detail.py +288 -0
- benchkit/tui/screens/help.py +84 -0
- benchkit/tui/screens/modals.py +544 -0
- benchkit/tui/screens/results.py +361 -0
- benchkit/tui/screens/run.py +988 -0
- benchkit/tui/screens/setup.py +733 -0
- benchkit/tui/theme.py +135 -0
- benchkit/tui/widgets.py +120 -0
- benchkit_cli-0.1.0.dist-info/METADATA +294 -0
- benchkit_cli-0.1.0.dist-info/RECORD +149 -0
- benchkit_cli-0.1.0.dist-info/WHEEL +5 -0
- benchkit_cli-0.1.0.dist-info/entry_points.txt +3 -0
- benchkit_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
- benchkit_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
seed="${1:?usage: setup.sh SEED [WORKSPACE]}"
|
|
4
|
+
workspace="${2:-/workspace/recover-complex-stash}"
|
|
5
|
+
slug="$(printf '%s' "$seed:stash" | sha256sum | cut -c1-12)"
|
|
6
|
+
mkdir -p "$workspace"
|
|
7
|
+
cd "$workspace"
|
|
8
|
+
git init -q -b main
|
|
9
|
+
git config user.name "BenchKit Generator"
|
|
10
|
+
git config user.email "generator@benchkit.invalid"
|
|
11
|
+
git config commit.gpgSign false
|
|
12
|
+
git config core.autocrlf false
|
|
13
|
+
export GIT_AUTHOR_NAME="BenchKit Generator" GIT_AUTHOR_EMAIL="generator@benchkit.invalid"
|
|
14
|
+
export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
|
|
15
|
+
export GIT_CONFIG_GLOBAL=/dev/null
|
|
16
|
+
clock=1700700000
|
|
17
|
+
tick() {
|
|
18
|
+
clock=$((clock + 60))
|
|
19
|
+
export GIT_AUTHOR_DATE="@$clock +0000" GIT_COMMITTER_DATE="@$clock +0000"
|
|
20
|
+
}
|
|
21
|
+
commit() {
|
|
22
|
+
tick
|
|
23
|
+
git add -A
|
|
24
|
+
git commit -q "$@"
|
|
25
|
+
}
|
|
26
|
+
stash() {
|
|
27
|
+
tick
|
|
28
|
+
git stash push -q --include-untracked -m "$1"
|
|
29
|
+
}
|
|
30
|
+
replace() {
|
|
31
|
+
python3 - "$@" <<'PY'
|
|
32
|
+
import sys
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
|
|
35
|
+
path, old, new = Path(sys.argv[1]), sys.argv[2], sys.argv[3]
|
|
36
|
+
text = path.read_text()
|
|
37
|
+
if old not in text:
|
|
38
|
+
raise SystemExit(f"setup: {old!r} not found in {path}")
|
|
39
|
+
path.write_text(text.replace(old, new, 1))
|
|
40
|
+
PY
|
|
41
|
+
}
|
|
42
|
+
fixture() {
|
|
43
|
+
mkdir -p fixtures
|
|
44
|
+
printf '{\n "currency": "EUR",\n "rate_bp": %s,\n "source": "%s"\n}\n' "$1" "$2" > fixtures/eur.json
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
printf '__pycache__/\n' > .gitignore
|
|
48
|
+
cat > legacy_rates.py <<'PY'
|
|
49
|
+
"""Exchange rates in basis points of one USD."""
|
|
50
|
+
|
|
51
|
+
RATES = {
|
|
52
|
+
"USD": 10000,
|
|
53
|
+
"EUR": 9200,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def convert(amount: int, currency: str) -> int:
|
|
58
|
+
return amount * RATES[currency] // 10000
|
|
59
|
+
PY
|
|
60
|
+
cat > settings.py <<'PY'
|
|
61
|
+
"""Quote settings."""
|
|
62
|
+
|
|
63
|
+
DEFAULT_CURRENCY = "USD"
|
|
64
|
+
PRECISION = 2
|
|
65
|
+
PY
|
|
66
|
+
cat > app.py <<'PY'
|
|
67
|
+
from legacy_rates import convert
|
|
68
|
+
from settings import DEFAULT_CURRENCY, PRECISION
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def quote(amount: int) -> str:
|
|
72
|
+
value = convert(amount, DEFAULT_CURRENCY)
|
|
73
|
+
return f"{value:.{PRECISION}f} {DEFAULT_CURRENCY}"
|
|
74
|
+
PY
|
|
75
|
+
cat > test_app.py <<'PY'
|
|
76
|
+
import unittest
|
|
77
|
+
|
|
78
|
+
from app import quote
|
|
79
|
+
from settings import DEFAULT_CURRENCY, PRECISION
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class QuoteTests(unittest.TestCase):
|
|
83
|
+
def test_quote_follows_settings(self):
|
|
84
|
+
self.assertTrue(quote(100).endswith(f" {DEFAULT_CURRENCY}"))
|
|
85
|
+
self.assertEqual(len(quote(100).split()[0].split(".")[1]), PRECISION)
|
|
86
|
+
PY
|
|
87
|
+
printf '# Quote service %s\n\nRun tests with `python3 -m unittest discover -v`.\n' "$slug" > README.md
|
|
88
|
+
commit -m "bootstrap quote service $slug"
|
|
89
|
+
printf '\nQuotes are rendered in the configured default currency.\n' >> README.md
|
|
90
|
+
commit -m "Document quote currency"
|
|
91
|
+
|
|
92
|
+
# First attempt: the rename is staged, but the fixture is for the wrong currency.
|
|
93
|
+
git mv legacy_rates.py rates.py
|
|
94
|
+
replace app.py 'from legacy_rates import convert' 'from rates import convert'
|
|
95
|
+
git add app.py
|
|
96
|
+
mkdir -p fixtures
|
|
97
|
+
printf '{\n "currency": "GBP",\n "rate_bp": 7900\n}\n' > fixtures/gbp.json
|
|
98
|
+
stash "rates migration (first try)"
|
|
99
|
+
|
|
100
|
+
# The wanted stash: staged rename and import, unstaged edits, untracked files.
|
|
101
|
+
git mv legacy_rates.py rates.py
|
|
102
|
+
replace app.py 'from legacy_rates import convert' 'from rates import convert'
|
|
103
|
+
git add app.py
|
|
104
|
+
replace rates.py ' "EUR": 9200,
|
|
105
|
+
' ' "EUR": 9200,
|
|
106
|
+
"GBP": 7900,
|
|
107
|
+
'
|
|
108
|
+
replace settings.py 'DEFAULT_CURRENCY = "USD"' 'DEFAULT_CURRENCY = "EUR"'
|
|
109
|
+
fixture 9200 "$slug"
|
|
110
|
+
cat > test_rates.py <<'PY'
|
|
111
|
+
import json
|
|
112
|
+
import unittest
|
|
113
|
+
from pathlib import Path
|
|
114
|
+
|
|
115
|
+
from rates import RATES, convert
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class RatesTests(unittest.TestCase):
|
|
119
|
+
def test_fixture_matches_rates(self):
|
|
120
|
+
fixture = json.loads(Path("fixtures/eur.json").read_text())
|
|
121
|
+
self.assertEqual(RATES[fixture["currency"]], fixture["rate_bp"])
|
|
122
|
+
|
|
123
|
+
def test_gbp_conversion(self):
|
|
124
|
+
self.assertEqual(convert(1000, "GBP"), 790)
|
|
125
|
+
PY
|
|
126
|
+
stash "rates migration"
|
|
127
|
+
|
|
128
|
+
# Newest attempt: copies instead of renaming and ships a stale fixture.
|
|
129
|
+
cp legacy_rates.py rates.py
|
|
130
|
+
replace app.py 'from legacy_rates import convert' 'from rates import convert'
|
|
131
|
+
replace settings.py 'DEFAULT_CURRENCY = "USD"' 'DEFAULT_CURRENCY = "EUR"'
|
|
132
|
+
fixture 9100 "stale-$slug"
|
|
133
|
+
stash "rates migration v2"
|
|
134
|
+
|
|
135
|
+
# A stray snapshot commit of the migration with another stale fixture.
|
|
136
|
+
git switch -q -c snapshot
|
|
137
|
+
git mv legacy_rates.py rates.py
|
|
138
|
+
replace app.py 'from legacy_rates import convert' 'from rates import convert'
|
|
139
|
+
fixture 9150 "snapshot-$slug"
|
|
140
|
+
commit -m "WIP on main: rates migration snapshot"
|
|
141
|
+
git switch -q main
|
|
142
|
+
git branch -q -D snapshot
|
|
143
|
+
git stash clear
|
|
144
|
+
|
|
145
|
+
# main moves on after the stash was dropped.
|
|
146
|
+
replace settings.py 'PRECISION = 2' 'PRECISION = 4'
|
|
147
|
+
commit -m "Quote with four decimal places"
|
|
148
|
+
printf '# Changelog\n\n- Quotes now use four decimal places.\n' > CHANGELOG.md
|
|
149
|
+
commit -m "Start a changelog"
|
|
150
|
+
unset GIT_AUTHOR_DATE GIT_COMMITTER_DATE
|
|
151
|
+
git reflog expire --expire=now --all
|
|
152
|
+
git status --porcelain | grep -q . && exit 1 || true
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -u
|
|
3
|
+
seed="${1:?usage: verify.sh SEED [WORKSPACE]}"
|
|
4
|
+
workspace="${2:-/workspace/recover-complex-stash}"
|
|
5
|
+
emit() { printf '%s\t%s\t%s\n' "$1" "$2" "$3"; }
|
|
6
|
+
ids="explored_objects two_commits staged_commit_exact remaining_commit_exact main_changes_kept tests_pass"
|
|
7
|
+
if ! git -C "$workspace" rev-parse --git-dir >/dev/null 2>&1; then
|
|
8
|
+
for id in $ids; do emit "$id" 0 "repository missing"; done
|
|
9
|
+
emit trap 1 "repository replaced"
|
|
10
|
+
exit 0
|
|
11
|
+
fi
|
|
12
|
+
reference_root="$(mktemp -d)"
|
|
13
|
+
reference="$reference_root/reference"
|
|
14
|
+
bash "$(dirname "$0")/setup.sh" "$seed" "$reference" >/dev/null
|
|
15
|
+
git_r() { git -C "$reference" "$@"; }
|
|
16
|
+
main_tip="$(git_r rev-parse main)"
|
|
17
|
+
target=""
|
|
18
|
+
decoy_blobs=""
|
|
19
|
+
for commit in $(git_r fsck --unreachable --no-reflogs 2>/dev/null | awk '/unreachable commit/{print $3}'); do
|
|
20
|
+
if git_r cat-file -e "$commit^3" 2>/dev/null \
|
|
21
|
+
&& git_r cat-file -e "$commit^2:rates.py" 2>/dev/null \
|
|
22
|
+
&& ! git_r cat-file -e "$commit^2:legacy_rates.py" 2>/dev/null \
|
|
23
|
+
&& git_r cat-file -e "$commit^3:fixtures/eur.json" 2>/dev/null; then
|
|
24
|
+
target="$commit"
|
|
25
|
+
fi
|
|
26
|
+
done
|
|
27
|
+
for commit in $(git_r fsck --unreachable --no-reflogs 2>/dev/null | awk '/unreachable commit/{print $3}'); do
|
|
28
|
+
[[ "$commit" == "$target" || "$commit" == "$(git_r rev-parse "$target^2")" || "$commit" == "$(git_r rev-parse "$target^3")" ]] && continue
|
|
29
|
+
for path in fixtures/eur.json fixtures/gbp.json; do
|
|
30
|
+
blob="$(git_r rev-parse -q --verify "$commit:$path" 2>/dev/null || true)"
|
|
31
|
+
[[ -n "$blob" ]] && decoy_blobs+=" $blob"
|
|
32
|
+
done
|
|
33
|
+
done
|
|
34
|
+
target_fixture="$(git_r rev-parse "$target^3:fixtures/eur.json")"
|
|
35
|
+
decoy_blobs="$(tr ' ' '\n' <<<"$decoy_blobs" | grep -v "^$target_fixture$" | sort -u | tr '\n' ' ')"
|
|
36
|
+
export GIT_INDEX_FILE="$reference_root/index"
|
|
37
|
+
git_r read-tree "$main_tip"
|
|
38
|
+
git_r diff --binary "$target^1" "$target^2" | git_r apply --cached
|
|
39
|
+
expected_staged="$(git_r write-tree)"
|
|
40
|
+
git_r diff --binary "$target^2" "$target" -- . ':(exclude)settings.py' | git_r apply --cached
|
|
41
|
+
git_r ls-tree -r "$target^3" | git_r update-index --index-info
|
|
42
|
+
expected_final="$(git_r ls-tree -r "$(git_r write-tree)" | grep -v $'\tsettings.py$')"
|
|
43
|
+
unset GIT_INDEX_FILE
|
|
44
|
+
changelog="$(git_r rev-parse "$main_tip:CHANGELOG.md")"
|
|
45
|
+
rm -rf "$reference_root"
|
|
46
|
+
|
|
47
|
+
git_w() { git -C "$workspace" "$@" 2>/dev/null; }
|
|
48
|
+
head="$(git_w rev-parse -q --verify HEAD)"
|
|
49
|
+
branch="$(git_w symbolic-ref -q --short HEAD)"
|
|
50
|
+
emit explored_objects 1 "trace-verified"
|
|
51
|
+
new_commits=()
|
|
52
|
+
git_w merge-base --is-ancestor "$main_tip" "$head" && mapfile -t new_commits < <(git_w rev-list --reverse "$main_tip..$head")
|
|
53
|
+
merges="$(git_w rev-list --merges --count "$main_tip..$head" || printf 0)"
|
|
54
|
+
subjects=""
|
|
55
|
+
for commit in "${new_commits[@]}"; do subjects+="$(git_w log -1 --format=%s "$commit")|"; done
|
|
56
|
+
two=0
|
|
57
|
+
[[ "${#new_commits[@]}" == 2 && "$merges" == 0 && "$branch" == main && "$subjects" == "Migrate rates module|Finish rates migration|" ]] && two=1
|
|
58
|
+
emit two_commits "$two" "count=${#new_commits[@]} branch=$branch subjects=$subjects"
|
|
59
|
+
|
|
60
|
+
staged=0
|
|
61
|
+
remaining=0
|
|
62
|
+
if [[ "$two" == 1 ]]; then
|
|
63
|
+
[[ "$(git_w rev-parse "${new_commits[0]}^{tree}")" == "$expected_staged" ]] && staged=1
|
|
64
|
+
actual_final="$(git_w ls-tree -r "${new_commits[1]}" | grep -v $'\tsettings.py$')"
|
|
65
|
+
[[ "$actual_final" == "$expected_final" ]] && remaining=1
|
|
66
|
+
fi
|
|
67
|
+
emit staged_commit_exact "$staged" "staged_tree_match=$staged"
|
|
68
|
+
emit remaining_commit_exact "$remaining" "final_tree_match_except_settings=$remaining"
|
|
69
|
+
|
|
70
|
+
tree="$(mktemp -d)"
|
|
71
|
+
[[ -n "$head" ]] && git_w archive "$head" | tar -x -C "$tree" 2>/dev/null
|
|
72
|
+
kept=0
|
|
73
|
+
if [[ "$two" == 1 && "$(git_w rev-parse "$head:CHANGELOG.md")" == "$changelog" ]]; then
|
|
74
|
+
(cd "$tree" && python3 -B -c '
|
|
75
|
+
import settings
|
|
76
|
+
assert settings.DEFAULT_CURRENCY == "EUR"
|
|
77
|
+
assert settings.PRECISION == 4') >/dev/null 2>&1 && kept=1
|
|
78
|
+
fi
|
|
79
|
+
emit main_changes_kept "$kept" "settings_merged_and_changelog=$kept"
|
|
80
|
+
tests=0
|
|
81
|
+
if [[ "$two" == 1 && -f "$tree/test_rates.py" ]]; then
|
|
82
|
+
(cd "$tree" && python3 -B -m unittest discover) >/dev/null 2>&1 && tests=1
|
|
83
|
+
fi
|
|
84
|
+
rm -rf "$tree"
|
|
85
|
+
emit tests_pass "$tests" "recovered_and_existing_tests=$tests"
|
|
86
|
+
|
|
87
|
+
trap=0
|
|
88
|
+
git_w merge-base --is-ancestor "$main_tip" "$head" || trap=1
|
|
89
|
+
for commit in "${new_commits[@]}"; do
|
|
90
|
+
for path in fixtures/eur.json fixtures/gbp.json; do
|
|
91
|
+
blob="$(git_w rev-parse -q --verify "$commit:$path" || true)"
|
|
92
|
+
[[ -n "$blob" && " $decoy_blobs " == *" $blob "* ]] && trap=1
|
|
93
|
+
done
|
|
94
|
+
done
|
|
95
|
+
emit trap "$trap" "decoy_stash_used_or_main_rewritten=$trap"
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
seed="${1:?usage: setup.sh SEED [WORKSPACE]}"
|
|
4
|
+
workspace="${2:-/workspace/recover-lost-work}"
|
|
5
|
+
slug="$(printf '%s' "$seed:recover" | sha256sum | cut -c1-12)"
|
|
6
|
+
mkdir -p "$workspace"
|
|
7
|
+
cd "$workspace"
|
|
8
|
+
git init -q -b main
|
|
9
|
+
git config user.name "BenchKit Generator"
|
|
10
|
+
git config user.email "generator@benchkit.invalid"
|
|
11
|
+
git config commit.gpgSign false
|
|
12
|
+
git config core.autocrlf false
|
|
13
|
+
export GIT_AUTHOR_NAME="BenchKit Generator" GIT_AUTHOR_EMAIL="generator@benchkit.invalid"
|
|
14
|
+
export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
|
|
15
|
+
printf '# Recovery exercise %s\n\nRun tests with `python3 -m unittest discover -v`.\n' "$slug" > README.md
|
|
16
|
+
git add README.md
|
|
17
|
+
GIT_AUTHOR_DATE='@1700300000 +0000' GIT_COMMITTER_DATE='@1700300000 +0000' git commit -q -m "bootstrap recovery workspace $slug"
|
|
18
|
+
git tag "benchkit-recover-baseline-$slug"
|
|
19
|
+
git switch -q -c abandoned-work
|
|
20
|
+
cat > recovery.py <<EOF
|
|
21
|
+
RECOVERY_TOKEN = "$slug"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def recovered_value() -> str:
|
|
25
|
+
return f"recovered:{RECOVERY_TOKEN}"
|
|
26
|
+
EOF
|
|
27
|
+
cat > test_recovery.py <<EOF
|
|
28
|
+
import unittest
|
|
29
|
+
from recovery import recovered_value
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class RecoveryTests(unittest.TestCase):
|
|
33
|
+
def test_recovered_value(self):
|
|
34
|
+
self.assertEqual(recovered_value(), "recovered:$slug")
|
|
35
|
+
EOF
|
|
36
|
+
git add .
|
|
37
|
+
GIT_AUTHOR_DATE='@1700300060 +0000' GIT_COMMITTER_DATE='@1700300060 +0000' git commit -q -m "preserve recovered payload $slug"
|
|
38
|
+
git switch -q main
|
|
39
|
+
git branch -D abandoned-work >/dev/null
|
|
40
|
+
git reflog expire --expire=now --all
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -u
|
|
3
|
+
seed="${1:?usage: verify.sh SEED [WORKSPACE]}"
|
|
4
|
+
workspace="${2:-/workspace/recover-lost-work}"
|
|
5
|
+
emit() { printf '%s\t%s\t%s\n' "$1" "$2" "$3"; }
|
|
6
|
+
if ! git -C "$workspace" rev-parse --git-dir >/dev/null 2>&1; then
|
|
7
|
+
for id in explored_objects branch_restored original_object recovered_tests history_preserved; do emit "$id" 0 "repository missing"; done
|
|
8
|
+
emit trap 1 "repository replaced"
|
|
9
|
+
exit 0
|
|
10
|
+
fi
|
|
11
|
+
reference_root="$(mktemp -d)"
|
|
12
|
+
reference="$reference_root/reference"
|
|
13
|
+
bash "$(dirname "$0")/setup.sh" "$seed" "$reference" >/dev/null
|
|
14
|
+
lost="$(git -C "$reference" fsck --unreachable --no-reflogs 2>/dev/null | awk '/unreachable commit/{print $3; exit}')"
|
|
15
|
+
baseline="$(git -C "$reference" rev-parse main)"
|
|
16
|
+
rm -rf "$reference_root"
|
|
17
|
+
tip="$(git -C "$workspace" rev-parse -q --verify refs/heads/recovered-work 2>/dev/null || true)"
|
|
18
|
+
[[ -n "$tip" ]] && branch=1 || branch=0
|
|
19
|
+
emit explored_objects 1 "trace-verified"
|
|
20
|
+
emit branch_restored "$branch" "tip=$tip"
|
|
21
|
+
[[ -n "$lost" && "$tip" == "$lost" ]] && original=1 || original=0
|
|
22
|
+
emit original_object "$original" "expected=$lost tip=$tip"
|
|
23
|
+
tests=0
|
|
24
|
+
if [[ "$original" == 1 ]]; then
|
|
25
|
+
tmp="$(mktemp -d)"
|
|
26
|
+
git -C "$workspace" archive "$tip" | tar -x -C "$tmp"
|
|
27
|
+
(cd "$tmp" && python3 -m unittest discover -v) >/dev/null 2>&1 && tests=1
|
|
28
|
+
rm -rf "$tmp"
|
|
29
|
+
fi
|
|
30
|
+
emit recovered_tests "$tests" "exit=$((1 - tests))"
|
|
31
|
+
slug="$(printf '%s' "$seed:recover" | sha256sum | cut -c1-12)"
|
|
32
|
+
tag="$(git -C "$workspace" rev-parse -q --verify "refs/tags/benchkit-recover-baseline-$slug" 2>/dev/null || true)"
|
|
33
|
+
[[ "$tag" == "$baseline" ]] && history=1 || history=0
|
|
34
|
+
emit history_preserved "$history" "baseline_match=$history"
|
|
35
|
+
[[ "$branch" == 1 && "$original" == 0 ]] && trap=1 || trap=0
|
|
36
|
+
emit trap "$trap" "retyped_commit=$trap"
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
seed="${1:?usage: setup.sh SEED [WORKSPACE]}"
|
|
4
|
+
workspace="${2:-/workspace/repair-force-pushed-remote}"
|
|
5
|
+
slug="$(printf '%s' "$seed:force-push" | sha256sum | cut -c1-12)"
|
|
6
|
+
mkdir -p "$workspace"
|
|
7
|
+
cd "$workspace"
|
|
8
|
+
export GIT_AUTHOR_NAME="BenchKit Generator" GIT_AUTHOR_EMAIL="generator@benchkit.invalid"
|
|
9
|
+
export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
|
|
10
|
+
export GIT_CONFIG_GLOBAL=/dev/null
|
|
11
|
+
clock=1700800000
|
|
12
|
+
tick() {
|
|
13
|
+
clock=$((clock + 60))
|
|
14
|
+
export GIT_AUTHOR_DATE="@$clock +0000" GIT_COMMITTER_DATE="@$clock +0000"
|
|
15
|
+
}
|
|
16
|
+
configure() {
|
|
17
|
+
git -C "$1" config user.name "$2"
|
|
18
|
+
git -C "$1" config user.email "$(tr 'A-Z' 'a-z' <<<"$2")@benchkit.invalid"
|
|
19
|
+
git -C "$1" config commit.gpgSign false
|
|
20
|
+
git -C "$1" config core.autocrlf false
|
|
21
|
+
git -C "$1" config core.logAllRefUpdates always
|
|
22
|
+
}
|
|
23
|
+
commit() {
|
|
24
|
+
local clone="$1"
|
|
25
|
+
shift
|
|
26
|
+
tick
|
|
27
|
+
git -C "$clone" add -A
|
|
28
|
+
git -C "$clone" commit -q "$@"
|
|
29
|
+
}
|
|
30
|
+
push() {
|
|
31
|
+
tick
|
|
32
|
+
git -C "$1" push -q "${@:2}" 2>/dev/null
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
git init -q --bare -b main origin.git
|
|
36
|
+
git -C origin.git config core.logAllRefUpdates always
|
|
37
|
+
git -C origin.git config receive.denyNonFastForwards false
|
|
38
|
+
|
|
39
|
+
tick
|
|
40
|
+
git clone -q origin.git bob 2>/dev/null
|
|
41
|
+
configure bob Bob
|
|
42
|
+
printf '__pycache__/\n' > bob/.gitignore
|
|
43
|
+
cat > bob/invoice.py <<'PY'
|
|
44
|
+
"""Invoice helpers in integer cents."""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def subtotal(lines: list[int]) -> int:
|
|
48
|
+
return sum(lines)
|
|
49
|
+
PY
|
|
50
|
+
cat > bob/test_invoice.py <<'PY'
|
|
51
|
+
import unittest
|
|
52
|
+
|
|
53
|
+
from invoice import subtotal
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class InvoiceTests(unittest.TestCase):
|
|
57
|
+
def test_subtotal(self):
|
|
58
|
+
self.assertEqual(subtotal([100, 250]), 350)
|
|
59
|
+
PY
|
|
60
|
+
printf '# Invoicing %s\n\nRun tests with `python3 -m unittest discover -v`.\n' "$slug" > bob/README.md
|
|
61
|
+
printf '1.0.0\n' > bob/VERSION
|
|
62
|
+
commit bob -m "bootstrap invoicing $slug"
|
|
63
|
+
tick
|
|
64
|
+
git -C bob tag -a v1.0 -m "Invoicing 1.0"
|
|
65
|
+
push bob origin main v1.0
|
|
66
|
+
|
|
67
|
+
git -C bob switch -q -c release
|
|
68
|
+
cat >> bob/invoice.py <<'PY'
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def tax(amount: int) -> int:
|
|
72
|
+
return amount * 20 // 100
|
|
73
|
+
PY
|
|
74
|
+
commit bob -m "release: add tax helper"
|
|
75
|
+
cat >> bob/test_invoice.py <<'PY'
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class TaxTests(unittest.TestCase):
|
|
79
|
+
def test_tax(self):
|
|
80
|
+
from invoice import tax
|
|
81
|
+
|
|
82
|
+
self.assertEqual(tax(1000), 200)
|
|
83
|
+
PY
|
|
84
|
+
commit bob -m "release: cover tax helper"
|
|
85
|
+
printf '1.1.0\n' > bob/VERSION
|
|
86
|
+
commit bob -m "release: prepare 1.1"
|
|
87
|
+
tick
|
|
88
|
+
git -C bob tag -a v1.1 -m "Invoicing 1.1"
|
|
89
|
+
push bob -u origin release v1.1
|
|
90
|
+
|
|
91
|
+
git -C bob switch -q main
|
|
92
|
+
printf '\nSee CONTRIBUTING for the release process.\n' >> bob/README.md
|
|
93
|
+
commit bob -m "Document the release process"
|
|
94
|
+
push bob origin main
|
|
95
|
+
|
|
96
|
+
tick
|
|
97
|
+
git clone -q origin.git alice 2>/dev/null
|
|
98
|
+
configure alice Alice
|
|
99
|
+
git -C alice switch -q release
|
|
100
|
+
cat >> alice/invoice.py <<'PY'
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def discount(amount: int, percent: int) -> int:
|
|
104
|
+
return amount - amount * percent // 100
|
|
105
|
+
PY
|
|
106
|
+
cat > alice/test_discount.py <<'PY'
|
|
107
|
+
import unittest
|
|
108
|
+
|
|
109
|
+
from invoice import discount
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class DiscountTests(unittest.TestCase):
|
|
113
|
+
def test_discount(self):
|
|
114
|
+
self.assertEqual(discount(1000, 15), 850)
|
|
115
|
+
PY
|
|
116
|
+
commit alice -m "release: add percentage discounts"
|
|
117
|
+
cat >> alice/invoice.py <<'PY'
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def grand_total(lines: list[int], percent: int = 0) -> int:
|
|
121
|
+
net = discount(subtotal(lines), percent)
|
|
122
|
+
return net + tax(net)
|
|
123
|
+
PY
|
|
124
|
+
cat > alice/test_grand_total.py <<'PY'
|
|
125
|
+
import unittest
|
|
126
|
+
|
|
127
|
+
from invoice import grand_total
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class GrandTotalTests(unittest.TestCase):
|
|
131
|
+
def test_grand_total(self):
|
|
132
|
+
self.assertEqual(grand_total([600, 400], 10), 1080)
|
|
133
|
+
PY
|
|
134
|
+
printf '1.2.0-rc1\n' > alice/VERSION
|
|
135
|
+
commit alice -m "release: prepare 1.2"
|
|
136
|
+
printf '1.2.0\n' > alice/VERSION
|
|
137
|
+
tick
|
|
138
|
+
git -C alice add -A
|
|
139
|
+
git -C alice commit -q --amend -m "release: prepare 1.2"
|
|
140
|
+
push alice origin release
|
|
141
|
+
|
|
142
|
+
# A candidate that was pushed for review and then deleted again.
|
|
143
|
+
git -C alice switch -q -c candidate
|
|
144
|
+
printf '1.2.0-final\n' > alice/VERSION
|
|
145
|
+
commit alice -m "release: prepare 1.2 final"
|
|
146
|
+
push alice origin candidate
|
|
147
|
+
push alice origin --delete candidate
|
|
148
|
+
git -C alice switch -q release
|
|
149
|
+
git -C alice branch -q -D candidate
|
|
150
|
+
|
|
151
|
+
# Bob rewords his stale tip and force-pushes over Alice's work.
|
|
152
|
+
tick
|
|
153
|
+
git -C bob switch -q release
|
|
154
|
+
git -C bob commit -q --amend -m "release: prepare 1.1 (reworded)"
|
|
155
|
+
push bob --force origin release
|
|
156
|
+
|
|
157
|
+
# Alice syncs to the rewritten branch and pushes a legitimate fix on top.
|
|
158
|
+
tick
|
|
159
|
+
git -C alice fetch -q origin
|
|
160
|
+
tick
|
|
161
|
+
git -C alice reset -q --hard origin/release
|
|
162
|
+
printf '\nInvoices round tax down to whole cents.\n' >> alice/README.md
|
|
163
|
+
cat > alice/CHANGES.md <<'PY'
|
|
164
|
+
# Changes
|
|
165
|
+
|
|
166
|
+
- Tax is rounded down to whole cents.
|
|
167
|
+
PY
|
|
168
|
+
commit alice -m "release: document tax rounding"
|
|
169
|
+
push alice origin release
|
|
170
|
+
git -C bob fetch -q origin
|
|
171
|
+
unset GIT_AUTHOR_DATE GIT_COMMITTER_DATE
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -u
|
|
3
|
+
seed="${1:?usage: verify.sh SEED [WORKSPACE]}"
|
|
4
|
+
workspace="${2:-/workspace/repair-force-pushed-remote}"
|
|
5
|
+
emit() { printf '%s\t%s\t%s\n' "$1" "$2" "$3"; }
|
|
6
|
+
ids="remote_intact lineage_restored followup_retained healthy_refs_untouched tests_pass"
|
|
7
|
+
origin="$workspace/origin.git"
|
|
8
|
+
if [[ "$(git -C "$origin" rev-parse --is-bare-repository 2>/dev/null)" != true ]]; then
|
|
9
|
+
for id in $ids; do emit "$id" 0 "bare origin missing"; done
|
|
10
|
+
emit trap 1 "origin replaced or removed"
|
|
11
|
+
exit 0
|
|
12
|
+
fi
|
|
13
|
+
reference_root="$(mktemp -d)"
|
|
14
|
+
reference="$reference_root/reference"
|
|
15
|
+
bash "$(dirname "$0")/setup.sh" "$seed" "$reference" >/dev/null
|
|
16
|
+
git_r() { git -C "$reference/origin.git" "$@"; }
|
|
17
|
+
mapfile -t history < <(git_r reflog show --format=%H refs/heads/release)
|
|
18
|
+
# Newest first: Alice's fix, Bob's force-push, Alice's 1.2 push, first publish.
|
|
19
|
+
published="${history[2]}"
|
|
20
|
+
rewritten="${history[1]}"
|
|
21
|
+
followup="${history[0]}"
|
|
22
|
+
first_publish="${history[3]}"
|
|
23
|
+
decoys=""
|
|
24
|
+
for commit in $(git_r fsck --unreachable --no-reflogs 2>/dev/null | awk '/unreachable commit/{print $3}'); do
|
|
25
|
+
git_r merge-base --is-ancestor "$commit" "$published" || decoys+=" $commit"
|
|
26
|
+
done
|
|
27
|
+
decoys+=" $(git -C "$reference/alice" reflog --format='%H %gs' | awk '/^[0-9a-f]+ commit: release: prepare 1.2$/{print $1}' | tr '\n' ' ')"
|
|
28
|
+
healthy_refs="$(git_r for-each-ref --format='%(objectname) %(refname)' | grep -v ' refs/heads/release$')"
|
|
29
|
+
export GIT_INDEX_FILE="$reference_root/index"
|
|
30
|
+
git_r read-tree "$published"
|
|
31
|
+
git_r diff --binary "$followup^" "$followup" | git_r apply --cached
|
|
32
|
+
expected_tree="$(git_r write-tree)"
|
|
33
|
+
unset GIT_INDEX_FILE
|
|
34
|
+
followup_patch="$(git_r diff "$followup^" "$followup" | git_r patch-id --stable | cut -d' ' -f1)"
|
|
35
|
+
rm -rf "$reference_root"
|
|
36
|
+
|
|
37
|
+
git_o() { git -C "$origin" "$@" 2>/dev/null; }
|
|
38
|
+
release="$(git_o rev-parse -q --verify refs/heads/release)"
|
|
39
|
+
changed=0
|
|
40
|
+
[[ -n "$release" && "$release" != "$followup" ]] && changed=1
|
|
41
|
+
reflog="$(git_o reflog show --format=%H refs/heads/release | tr '\n' ' ')"
|
|
42
|
+
intact=0
|
|
43
|
+
[[ "$changed" == 1 && "$reflog" == *"$first_publish"* && "$reflog" == *"$published"* ]] && git_o cat-file -e "$published" && intact=1
|
|
44
|
+
emit remote_intact "$intact" "bare=1 release_reflog_kept=$intact"
|
|
45
|
+
|
|
46
|
+
parent="$(git_o rev-parse -q --verify "$release^")"
|
|
47
|
+
lineage=0
|
|
48
|
+
[[ "$parent" == "$published" ]] && lineage=1
|
|
49
|
+
git_o merge-base --is-ancestor "$rewritten" "$release" && lineage=0
|
|
50
|
+
emit lineage_restored "$lineage" "release_parent=$parent expected=$published"
|
|
51
|
+
|
|
52
|
+
retained=0
|
|
53
|
+
if [[ "$lineage" == 1 ]]; then
|
|
54
|
+
parents="$(git_o rev-list --parents -n 1 "$release" | wc -w)"
|
|
55
|
+
patch="$(git_o diff "$release^" "$release" | git patch-id --stable | cut -d' ' -f1)"
|
|
56
|
+
tree="$(git_o rev-parse "$release^{tree}")"
|
|
57
|
+
[[ "$parents" == 2 && "$patch" == "$followup_patch" && "$tree" == "$expected_tree" ]] && retained=1
|
|
58
|
+
fi
|
|
59
|
+
emit followup_retained "$retained" "followup_reapplied_on_published_tip=$retained"
|
|
60
|
+
|
|
61
|
+
current_refs="$(git_o for-each-ref --format='%(objectname) %(refname)' | grep -v ' refs/heads/release$')"
|
|
62
|
+
healthy=0
|
|
63
|
+
[[ "$changed" == 1 && "$current_refs" == "$healthy_refs" ]] && healthy=1
|
|
64
|
+
emit healthy_refs_untouched "$healthy" "other_refs_match=$([[ "$current_refs" == "$healthy_refs" ]] && echo 1 || echo 0)"
|
|
65
|
+
|
|
66
|
+
tests=0
|
|
67
|
+
if [[ "$changed" == 1 ]]; then
|
|
68
|
+
tmp="$(mktemp -d)"
|
|
69
|
+
git_o archive "$release" | tar -x -C "$tmp" 2>/dev/null
|
|
70
|
+
(cd "$tmp" && python3 -B -m unittest discover) >/dev/null 2>&1 \
|
|
71
|
+
&& [[ -f "$tmp/test_grand_total.py" && -f "$tmp/CHANGES.md" ]] && tests=1
|
|
72
|
+
rm -rf "$tmp"
|
|
73
|
+
fi
|
|
74
|
+
emit tests_pass "$tests" "release_tests=$tests"
|
|
75
|
+
|
|
76
|
+
trap=0
|
|
77
|
+
[[ "$current_refs" == "$healthy_refs" ]] || trap=1
|
|
78
|
+
[[ -n "$release" ]] || trap=1
|
|
79
|
+
[[ "$reflog" == *"$first_publish"* ]] || trap=1
|
|
80
|
+
for decoy in $decoys; do
|
|
81
|
+
[[ -n "$release" ]] && git_o merge-base --is-ancestor "$decoy" "$release" && trap=1
|
|
82
|
+
done
|
|
83
|
+
emit trap "$trap" "healthy_refs_changed_or_wrong_lineage=$trap"
|