libreusebench 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,41 @@
1
+ """LibReuseBench - a library-agnostic, test-backed benchmark for measuring library reuse."""
2
+
3
+ from libreusebench.environment import (
4
+ Environment,
5
+ evaluate_record,
6
+ run_program,
7
+ temporary_packages,
8
+ )
9
+ from libreusebench.evaluate import (
10
+ evaluate_responses,
11
+ evaluate_responses_file,
12
+ load_responses,
13
+ )
14
+ from libreusebench.libraries import (
15
+ extract_code,
16
+ imported_libraries,
17
+ required_pip_packages,
18
+ used_libraries,
19
+ )
20
+ from libreusebench.load import load_dataset, load_records
21
+
22
+
23
+ # the single source of the package version (read by pyproject.toml when building)
24
+ __version__ = "0.1.0"
25
+
26
+ __all__ = [
27
+ "__version__",
28
+ "load_dataset",
29
+ "load_records",
30
+ "load_responses",
31
+ "evaluate_responses",
32
+ "evaluate_responses_file",
33
+ "Environment",
34
+ "run_program",
35
+ "evaluate_record",
36
+ "temporary_packages",
37
+ "extract_code",
38
+ "imported_libraries",
39
+ "required_pip_packages",
40
+ "used_libraries",
41
+ ]
libreusebench/cli.py ADDED
@@ -0,0 +1,78 @@
1
+ """Command-line entry point for evaluating LibReuseBench responses."""
2
+
3
+ import argparse
4
+
5
+ from libreusebench.environment import _DEFAULT_TIMEOUT_SECONDS
6
+ from libreusebench.evaluate import confirm_packages, evaluate_responses_file
7
+
8
+
9
+ def main() -> None:
10
+ """Parse arguments and run the evaluation harness."""
11
+ parser = argparse.ArgumentParser(
12
+ prog="libreuseeval",
13
+ description="Evaluate model responses against the LibReuseBench benchmark.",
14
+ )
15
+ parser.add_argument(
16
+ "responses_file", help="JSON or JSONL file of {task_id, response(s)} records"
17
+ )
18
+ parser.add_argument("--output-dir", default="output", help="where to write results")
19
+ parser.add_argument(
20
+ "--env-dir",
21
+ default=None,
22
+ help="venv to reuse across runs, kept afterwards; by default each run builds a "
23
+ "disposable one and deletes it, so concurrent runs can't corrupt each other's",
24
+ )
25
+ parser.add_argument(
26
+ "--env-root",
27
+ default=None,
28
+ help="where to build the disposable venv (default: the system temp directory)",
29
+ )
30
+ parser.add_argument(
31
+ "--env-file",
32
+ default=None,
33
+ help="path to an environment .txt (one pip package per line); "
34
+ "if omitted, packages are collected from the responses and installed automatically",
35
+ )
36
+ parser.add_argument(
37
+ "--timeout",
38
+ type=float,
39
+ default=_DEFAULT_TIMEOUT_SECONDS,
40
+ help="per-run timeout (seconds)",
41
+ )
42
+ parser.add_argument(
43
+ "--k",
44
+ type=int,
45
+ default=None,
46
+ help="k for pass@k (default: the smallest number of samples any task has)",
47
+ )
48
+ parser.add_argument(
49
+ "--unsafe-installs",
50
+ action="store_true",
51
+ help="skip confirmation and install the collected packages automatically",
52
+ )
53
+ parser.add_argument(
54
+ "--stop-on-install-failure",
55
+ action="store_true",
56
+ help=(
57
+ "stop if a package fails to install, writing an environment file to fix and "
58
+ "re-run with (default: carry on, not running the responses that import it)"
59
+ ),
60
+ )
61
+ args = parser.parse_args()
62
+
63
+ confirm = None if args.unsafe_installs else confirm_packages
64
+ evaluate_responses_file(
65
+ responses_file=args.responses_file,
66
+ output_dir=args.output_dir,
67
+ env_dir=args.env_dir,
68
+ env_root=args.env_root,
69
+ timeout_seconds=args.timeout,
70
+ env_file=args.env_file,
71
+ k=args.k,
72
+ confirm=confirm,
73
+ stop_on_install_failure=args.stop_on_install_failure,
74
+ )
75
+
76
+
77
+ if __name__ == "__main__":
78
+ main()