rerun-bench 0.1.0__py3-none-any.whl

rerun_bench/__init__.py

@@ -0,0 +1,3 @@
+"""rerun-bench: same task, run N times. How consistent and how expensive is your coding agent?"""
+
+__version__ = "0.1.0"

rerun_bench/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

rerun_bench/_bundled_tasks/add-cli-flag/solution/wc.py

@@ -0,0 +1,33 @@
+"""Count lines in files. Usage: python wc.py FILE [FILE ...]"""
+
+import argparse
+import sys
+
+
+def count_lines(text):
+    return len(text.splitlines())
+
+
+def count_words(text):
+    return len(text.split())
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description="Count lines in files.")
+    parser.add_argument("files", nargs="+")
+    parser.add_argument("--words", action="store_true", help="count words instead of lines")
+    args = parser.parse_args(argv)
+    counter = count_words if args.words else count_lines
+    total = 0
+    for name in args.files:
+        with open(name, encoding="utf-8") as fh:
+            n = counter(fh.read())
+        total += n
+        print(f"{n:>8} {name}")
+    if len(args.files) > 1:
+        print(f"{total:>8} total")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

rerun_bench/_bundled_tasks/add-cli-flag/task.toml

@@ -0,0 +1,9 @@
+id = "add-cli-flag"
+title = "Add a flag to a small CLI without changing its default output"
+prompt = """
+Add a `--words` flag to wc.py. When it is given, print word counts (whitespace-separated
+tokens) instead of line counts, in exactly the same output format. Without the flag the
+output must stay exactly as it is now.
+"""
+timeout = 600
+tags = ["cli", "feature", "python"]

rerun_bench/_bundled_tasks/add-cli-flag/verify.py

@@ -0,0 +1,48 @@
+"""Hidden checks for add-cli-flag."""
+
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+ws = Path.cwd()
+
+
+def fail(msg):
+    print("FAIL:", msg)
+    sys.exit(1)
+
+
+def run(*args, cwd):
+    p = subprocess.run(
+        [sys.executable, str(ws / "wc.py"), *args],
+        cwd=cwd,
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+    return p.returncode, p.stdout.replace("\r\n", "\n")
+
+
+with tempfile.TemporaryDirectory() as d:
+    d = Path(d)
+    (d / "a.txt").write_text("one two three\nfour\n\nfive  six\n", encoding="utf-8")
+    (d / "b.txt").write_text("  lonely   \n", encoding="utf-8")
+    (d / "c.txt").write_text("", encoding="utf-8")
+    expect = {
+        ("a.txt",): "       4 a.txt\n",
+        ("a.txt", "b.txt", "c.txt"): "       4 a.txt\n       1 b.txt\n       0 c.txt\n"
+        "       5 total\n",
+        ("--words", "a.txt"): "       6 a.txt\n",
+        ("a.txt", "--words"): "       6 a.txt\n",
+        ("--words", "a.txt", "b.txt", "c.txt"): "       6 a.txt\n       1 b.txt\n"
+        "       0 c.txt\n       7 total\n",
+    }
+    for args, want in expect.items():
+        code, out = run(*args, cwd=d)
+        if code != 0 or out != want:
+            fail(f"wc.py {' '.join(args)}: exit {code}, output {out!r}, want {want!r}")
+    code, _ = run("--words", cwd=d)
+    if code == 0:
+        fail("wc.py --words with no files should exit non-zero")
+print("PASS")

rerun_bench/_bundled_tasks/add-cli-flag/workspace/wc.py

@@ -0,0 +1,27 @@
+"""Count lines in files. Usage: python wc.py FILE [FILE ...]"""
+
+import argparse
+import sys
+
+
+def count_lines(text):
+    return len(text.splitlines())
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description="Count lines in files.")
+    parser.add_argument("files", nargs="+")
+    args = parser.parse_args(argv)
+    total = 0
+    for name in args.files:
+        with open(name, encoding="utf-8") as fh:
+            n = count_lines(fh.read())
+        total += n
+        print(f"{n:>8} {name}")
+    if len(args.files) > 1:
+        print(f"{total:>8} total")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

rerun_bench/_bundled_tasks/edit-config/solution/config/app.toml

@@ -0,0 +1,14 @@
+# Local development settings. Production values live in app.prod.toml.
+
+[server]
+host = "127.0.0.1"
+port = 8081
+workers = 4
+
+[logging]
+level = "debug"
+format = "json"
+
+[cors]
+allowed_origins = ["https://example.com", "https://example.org"]
+allow_credentials = false

rerun_bench/_bundled_tasks/edit-config/task.toml

@@ -0,0 +1,9 @@
+id = "edit-config"
+title = "Make three precise edits to a TOML config"
+prompt = """
+In config/app.toml: change the server port to 8081, set the logging level to "debug", and
+add "https://example.org" to the CORS allowed origins while keeping the existing origin.
+Leave every other setting, comment and file unchanged.
+"""
+timeout = 300
+tags = ["config", "toml", "precision"]

rerun_bench/_bundled_tasks/edit-config/verify.py

@@ -0,0 +1,36 @@
+"""Hidden checks for edit-config: exact target values, nothing else touched."""
+
+import sys
+import tomllib
+from pathlib import Path
+
+ws = Path.cwd()
+ORIGINAL = Path(__file__).resolve().parent / "workspace"
+
+
+def fail(msg):
+    print("FAIL:", msg)
+    sys.exit(1)
+
+
+try:
+    cfg = tomllib.loads((ws / "config/app.toml").read_text(encoding="utf-8"))
+except Exception as exc:  # noqa: BLE001
+    fail(f"config/app.toml does not parse: {exc}")
+want = {
+    "server": {"host": "127.0.0.1", "port": 8081, "workers": 4},
+    "logging": {"level": "debug", "format": "json"},
+    "cors": {
+        "allowed_origins": ["https://example.com", "https://example.org"],
+        "allow_credentials": False,
+    },
+}
+if cfg != want:
+    fail(f"config/app.toml values differ from the request:\n got  {cfg}\n want {want}")
+text = (ws / "config/app.toml").read_text(encoding="utf-8")
+if "# Local development settings. Production values live in app.prod.toml." not in text:
+    fail("the header comment in config/app.toml was removed or changed")
+prod_now = (ws / "config/app.prod.toml").read_bytes()
+if prod_now != (ORIGINAL / "config/app.prod.toml").read_bytes():
+    fail("config/app.prod.toml was modified")
+print("PASS")

rerun_bench/_bundled_tasks/edit-config/workspace/config/app.prod.toml

@@ -0,0 +1,14 @@
+# Production settings. Changes here need a change ticket.
+
+[server]
+host = "0.0.0.0"
+port = 443
+workers = 16
+
+[logging]
+level = "warning"
+format = "json"
+
+[cors]
+allowed_origins = ["https://example.com"]
+allow_credentials = true

rerun_bench/_bundled_tasks/edit-config/workspace/config/app.toml

@@ -0,0 +1,14 @@
+# Local development settings. Production values live in app.prod.toml.
+
+[server]
+host = "127.0.0.1"
+port = 8080
+workers = 4
+
+[logging]
+level = "info"
+format = "json"
+
+[cors]
+allowed_origins = ["https://example.com"]
+allow_credentials = false

rerun_bench/_bundled_tasks/fix-failing-test/solution/stats.py

@@ -0,0 +1,23 @@
+"""Small descriptive statistics helpers."""
+
+
+def mean(xs):
+    """Arithmetic mean. Raises ValueError on empty input."""
+    if not xs:
+        raise ValueError("mean of empty sequence")
+    return sum(xs) / len(xs)
+
+
+def median(xs):
+    """Median of a sequence of numbers. Raises ValueError on empty input.
+
+    For an even number of values the median is the mean of the two middle values.
+    The input sequence is not modified.
+    """
+    if not xs:
+        raise ValueError("median of empty sequence")
+    s = sorted(xs)
+    mid = len(s) // 2
+    if len(s) % 2:
+        return s[mid]
+    return (s[mid - 1] + s[mid]) / 2

rerun_bench/_bundled_tasks/fix-failing-test/task.toml

@@ -0,0 +1,8 @@
+id = "fix-failing-test"
+title = "Fix the bug behind a failing unit test"
+prompt = """
+Running `python -m unittest` in this directory shows a failing test.
+Fix the bug in the source code so that all tests pass. Do not modify the test file.
+"""
+timeout = 600
+tags = ["bugfix", "python", "tests"]

rerun_bench/_bundled_tasks/fix-failing-test/verify.py

@@ -0,0 +1,54 @@
+"""Hidden checks for fix-failing-test. Run with cwd = the agent's workspace."""
+
+import subprocess
+import sys
+from pathlib import Path
+
+ORIGINAL = Path(__file__).resolve().parent / "workspace"
+ws = Path.cwd()
+sys.path.insert(0, str(ws))
+
+
+def fail(msg):
+    print("FAIL:", msg)
+    sys.exit(1)
+
+
+if (ws / "test_stats.py").read_bytes() != (ORIGINAL / "test_stats.py").read_bytes():
+    fail("test_stats.py was modified")
+
+proc = subprocess.run(
+    [sys.executable, "-m", "unittest", "-q"], cwd=ws, capture_output=True, text=True, timeout=60
+)
+if proc.returncode != 0:
+    fail("visible tests fail:\n" + proc.stderr[-1500:])
+
+from stats import mean, median  # noqa: E402
+
+cases = [
+    ([5], 5),
+    ([3, 1, 2], 2),
+    ([4, 1, 3, 2], 2.5),
+    ([1.0, 2.0], 1.5),
+    ([10, -10], 0),
+    ([7, 7, 7, 7], 7),
+    ([1, 2, 3, 4, 5, 6], 3.5),
+]
+for xs, want in cases:
+    got = median(list(xs))
+    if got != want:
+        fail(f"median({xs}) = {got!r}, want {want!r}")
+data = [3, 1, 2, 4]
+median(data)
+if data != [3, 1, 2, 4]:
+    fail("median mutated its input")
+for fn in (mean, median):
+    try:
+        fn([])
+    except ValueError:
+        pass
+    else:
+        fail(f"{fn.__name__}([]) should raise ValueError")
+if mean([1, 2, 3, 4]) != 2.5:
+    fail("mean changed behavior")
+print("PASS")

rerun_bench/_bundled_tasks/fix-failing-test/workspace/stats.py

@@ -0,0 +1,21 @@
+"""Small descriptive statistics helpers."""
+
+
+def mean(xs):
+    """Arithmetic mean. Raises ValueError on empty input."""
+    if not xs:
+        raise ValueError("mean of empty sequence")
+    return sum(xs) / len(xs)
+
+
+def median(xs):
+    """Median of a sequence of numbers. Raises ValueError on empty input.
+
+    For an even number of values the median is the mean of the two middle values.
+    The input sequence is not modified.
+    """
+    if not xs:
+        raise ValueError("median of empty sequence")
+    s = sorted(xs)
+    mid = len(s) // 2
+    return s[mid]

rerun_bench/_bundled_tasks/fix-failing-test/workspace/test_stats.py

@@ -0,0 +1,22 @@
+import unittest
+
+from stats import mean, median
+
+
+class StatsTest(unittest.TestCase):
+    def test_mean(self):
+        self.assertEqual(mean([1, 2, 3]), 2)
+
+    def test_median_odd(self):
+        self.assertEqual(median([3, 1, 2]), 2)
+
+    def test_median_even(self):
+        self.assertEqual(median([4, 1, 3, 2]), 2.5)
+
+    def test_median_empty(self):
+        with self.assertRaises(ValueError):
+            median([])
+
+
+if __name__ == "__main__":
+    unittest.main()

rerun_bench/_bundled_tasks/follow-agents-md/solution/CHANGES.md

@@ -0,0 +1,10 @@
+# Changes
+
+## Unreleased
+
+- Added `is_palindrome`.
+
+## 0.3.0
+
+- Added `word_count`.
+- `shout` now strips surrounding whitespace.

rerun_bench/_bundled_tasks/follow-agents-md/solution/textutils.py

@@ -0,0 +1,19 @@
+"""Small text helpers."""
+
+__all__ = ["is_palindrome", "shout", "word_count"]
+
+
+def shout(text):
+    """Return ``text`` stripped and upper-cased with an exclamation mark."""
+    return text.strip().upper() + "!"
+
+
+def word_count(text):
+    """Number of whitespace-separated words in ``text``."""
+    return len(text.split())
+
+
+def is_palindrome(text):
+    """True if ``text`` reads the same both ways, ignoring case and non-alphanumerics."""
+    chars = [c.lower() for c in text if c.isalnum()]
+    return chars == chars[::-1]

rerun_bench/_bundled_tasks/follow-agents-md/task.toml

@@ -0,0 +1,9 @@
+id = "follow-agents-md"
+title = "Add a function while following the repo's AGENTS.md conventions"
+prompt = """
+Add a function `is_palindrome(text)` to textutils.py. It returns True when the text reads
+the same forwards and backwards, ignoring case and any character that is not a letter or
+digit, and False otherwise.
+"""
+timeout = 600
+tags = ["instructions", "agents-md", "python"]

rerun_bench/_bundled_tasks/follow-agents-md/verify.py

@@ -0,0 +1,55 @@
+"""Hidden checks for follow-agents-md: the feature works AND the AGENTS.md rules were followed."""
+
+import re
+import sys
+from pathlib import Path
+
+ws = Path.cwd()
+ORIGINAL = Path(__file__).resolve().parent / "workspace"
+sys.path.insert(0, str(ws))
+
+
+def fail(msg):
+    print("FAIL:", msg)
+    sys.exit(1)
+
+
+import textutils  # noqa: E402
+
+fn = getattr(textutils, "is_palindrome", None)
+if fn is None:
+    fail("is_palindrome not defined")
+for text, want in [
+    ("racecar", True),
+    ("A man, a plan, a canal: Panama", True),
+    ("", True),
+    ("ab", False),
+    ("No 'x' in Nixon", True),
+    ("12321", True),
+    ("123", False),
+    ("Was it a car or a cat I saw?", True),
+    ("hello", False),
+]:
+    if fn(text) is not want:
+        fail(f"is_palindrome({text!r}) returned {fn(text)!r}, want {want}")
+if not (fn.__doc__ or "").strip():
+    fail("AGENTS.md: is_palindrome needs a docstring")
+allx = list(getattr(textutils, "__all__", []))
+if "is_palindrome" not in allx:
+    fail("AGENTS.md: is_palindrome missing from __all__")
+if allx != sorted(allx):
+    fail(f"AGENTS.md: __all__ not alphabetical: {allx}")
+if textutils.shout(" hi ") != "HI!" or textutils.word_count("a b c") != 3:
+    fail("existing functions changed behavior")
+
+changes = (ws / "CHANGES.md").read_text(encoding="utf-8").replace("\r\n", "\n")
+m = re.search(r"^## Unreleased\n(.*?)(?=^## )", changes, re.S | re.M)
+if not m:
+    fail("CHANGES.md: ## Unreleased section missing")
+if not re.search(r"^[-*] .*`is_palindrome`", m.group(1), re.M):
+    fail("AGENTS.md: no bullet mentioning `is_palindrome` under ## Unreleased")
+orig = (ORIGINAL / "CHANGES.md").read_text(encoding="utf-8").replace("\r\n", "\n")
+released = orig[orig.index("## 0.3.0") :]
+if released not in changes:
+    fail("CHANGES.md: a released section was edited")
+print("PASS")

rerun_bench/_bundled_tasks/follow-agents-md/workspace/AGENTS.md

@@ -0,0 +1,8 @@
+# Conventions for this repository
+
+These rules apply to every change, however small.
+
+- Every public function in `textutils.py` has a docstring.
+- Every public function is listed in `__all__` in `textutils.py`, kept in alphabetical order.
+- Every user-visible change gets a bullet under `## Unreleased` in `CHANGES.md`.
+  Mention the function name in backticks. Never edit released sections.

rerun_bench/_bundled_tasks/follow-agents-md/workspace/CHANGES.md

@@ -0,0 +1,8 @@
+# Changes
+
+## Unreleased
+
+## 0.3.0
+
+- Added `word_count`.
+- `shout` now strips surrounding whitespace.

rerun_bench/_bundled_tasks/follow-agents-md/workspace/textutils.py

@@ -0,0 +1,13 @@
+"""Small text helpers."""
+
+__all__ = ["shout", "word_count"]
+
+
+def shout(text):
+    """Return ``text`` stripped and upper-cased with an exclamation mark."""
+    return text.strip().upper() + "!"
+
+
+def word_count(text):
+    """Number of whitespace-separated words in ``text``."""
+    return len(text.split())

rerun_bench/_bundled_tasks/implement-lru-cache/solution/lru.py

@@ -0,0 +1,44 @@
+"""A least-recently-used cache."""
+
+from collections import OrderedDict
+
+
+class LRUCache:
+    """Fixed-capacity mapping that evicts the least recently used key when full.
+
+    "Used" means read with ``get`` (a hit) or written with ``put``. ``len()`` and the
+    ``in`` operator do not count as uses.
+    """
+
+    def __init__(self, capacity):
+        """Create an empty cache. Raise ValueError unless ``capacity`` is an int >= 1."""
+        if not isinstance(capacity, int) or isinstance(capacity, bool) or capacity < 1:
+            raise ValueError("capacity must be an int >= 1")
+        self.capacity = capacity
+        self._data = OrderedDict()
+
+    def get(self, key, default=None):
+        """Return the value for ``key`` and mark it most recently used, else ``default``."""
+        if key not in self._data:
+            return default
+        self._data.move_to_end(key)
+        return self._data[key]
+
+    def put(self, key, value):
+        """Insert or update ``key`` and mark it most recently used.
+
+        If the cache then holds more than ``capacity`` keys, evict the least recently
+        used key and return it. Otherwise return None.
+        """
+        self._data[key] = value
+        self._data.move_to_end(key)
+        if len(self._data) > self.capacity:
+            evicted, _ = self._data.popitem(last=False)
+            return evicted
+        return None
+
+    def __len__(self):
+        return len(self._data)
+
+    def __contains__(self, key):
+        return key in self._data

rerun_bench/_bundled_tasks/implement-lru-cache/task.toml

@@ -0,0 +1,8 @@
+id = "implement-lru-cache"
+title = "Implement a small data structure to spec"
+prompt = """
+Implement the `LRUCache` class in lru.py according to its docstrings. Use only the Python
+standard library.
+"""
+timeout = 600
+tags = ["implement", "data-structure", "python"]

rerun_bench/_bundled_tasks/implement-lru-cache/verify.py

@@ -0,0 +1,58 @@
+"""Hidden checks for implement-lru-cache."""
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path.cwd()))
+
+
+def fail(msg):
+    print("FAIL:", msg)
+    sys.exit(1)
+
+
+try:
+    from lru import LRUCache
+
+    for bad in (0, -3, 1.5, "2", None):
+        try:
+            LRUCache(bad)
+        except ValueError:
+            continue
+        fail(f"LRUCache({bad!r}) should raise ValueError")
+
+    c = LRUCache(2)
+    if c.put("a", 1) is not None or c.put("b", 2) is not None:
+        fail("put below capacity should return None")
+    if c.get("a") != 1:
+        fail("get a")
+    if c.put("c", 3) != "b":
+        fail("expected b evicted after a was read")
+    if "b" in c or c.get("b", "miss") != "miss":
+        fail("b should be gone")
+    if len(c) != 2:
+        fail("len should be 2")
+    if c.put("a", 10) is not None or c.get("a") != 10:
+        fail("updating an existing key should not evict and should store the new value")
+    if c.put("d", 4) != "c":
+        fail("expected c evicted (a was used more recently)")
+    _ = "a" in c
+    _ = len(c)
+    if c.put("e", 5) != "a":
+        fail("`in` and len() must not count as uses")
+    if c.get("missing") is not None:
+        fail("missing key should return None by default")
+
+    one = LRUCache(1)
+    one.put(1, "x")
+    if one.put(2, "y") != 1 or one.get(2) != "y" or len(one) != 1:
+        fail("capacity-1 behavior")
+
+    big = LRUCache(100)
+    for i in range(250):
+        big.put(i, i * i)
+    if len(big) != 100 or 149 in big or big.get(150) != 22500:
+        fail("bulk insert behavior")
+except NotImplementedError:
+    fail("LRUCache is not implemented")
+print("PASS")

rerun_bench/_bundled_tasks/implement-lru-cache/workspace/lru.py

@@ -0,0 +1,31 @@
+"""A least-recently-used cache."""
+
+
+class LRUCache:
+    """Fixed-capacity mapping that evicts the least recently used key when full.
+
+    "Used" means read with ``get`` (a hit) or written with ``put``. ``len()`` and the
+    ``in`` operator do not count as uses.
+    """
+
+    def __init__(self, capacity):
+        """Create an empty cache. Raise ValueError unless ``capacity`` is an int >= 1."""
+        raise NotImplementedError
+
+    def get(self, key, default=None):
+        """Return the value for ``key`` and mark it most recently used, else ``default``."""
+        raise NotImplementedError
+
+    def put(self, key, value):
+        """Insert or update ``key`` and mark it most recently used.
+
+        If the cache then holds more than ``capacity`` keys, evict the least recently
+        used key and return it. Otherwise return None.
+        """
+        raise NotImplementedError
+
+    def __len__(self):
+        raise NotImplementedError
+
+    def __contains__(self, key):
+        raise NotImplementedError

rerun_bench/_bundled_tasks/implement-slugify/solution/textkit/slug.py

@@ -0,0 +1,30 @@
+"""URL slug generation."""
+
+import re
+import unicodedata
+
+
+def slugify(text: str, max_length: int = 50) -> str:
+    """Turn arbitrary text into a URL slug.
+
+    Rules, applied in this order:
+
+    1. Normalize with Unicode NFKD and drop every character that is not ASCII
+       (so "Cafe" with an accent becomes "Cafe").
+    2. Lowercase.
+    3. Replace every run of characters that are not ASCII letters or digits with a
+       single hyphen.
+    4. Strip leading and trailing hyphens.
+    5. If the result is longer than ``max_length``, cut it to ``max_length`` characters
+       and strip any trailing hyphen left by the cut.
+    6. If the result is empty, return "n-a".
+
+    ``max_length`` must be at least 1; otherwise raise ValueError.
+    """
+    if max_length < 1:
+        raise ValueError("max_length must be >= 1")
+    s = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode("ascii")
+    s = re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")
+    if len(s) > max_length:
+        s = s[:max_length].rstrip("-")
+    return s or "n-a"