programbench 1.0.0__tar.gz → 1.0.2__tar.gz

{programbench-1.0.0 → programbench-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: programbench
-Version: 1.0.0
+Version: 1.0.2
 Summary: Evaluate whether LM-based SWE-agents can reverse-engineer black-box software systems
 Author: Kilian Adriano Lieret, John Yang
 Author-email: Kilian Adriano Lieret <klieret@meta.com>, John Yang <johnby@meta.com>
@@ -26,7 +26,7 @@ Given only a compiled binary and its documentation, AI agents must architect and
 ## Links
 
 - [Website](https://programbench.com)
-- [Paper](https://programbench.com/static/paper.pdf)
+- [Paper](https://arxiv.org/abs/2605.03546)
 - [HuggingFace](https://huggingface.co/datasets/programbench/ProgramBench-Tests)
 - [Leaderboard](https://programbench.com)
 - [Usage Guide](docs/README.md)
@@ -62,10 +62,14 @@ uv sync  # installs editable + dev dependencies
 If our work was useful for you, please cite it:
 
 ```bibtex
-@preprint{yang2026programbench,
-  title={ProgramBench: Can Language Models Rebuild Programs From Scratch?},
-  author={John Yang and Kilian Lieret and Jeffrey Ma and Parth Thakkar and Dmitrii Pedchenko and Sten Sootla and Emily McMilin and Pengcheng Yin and Rui Hou and Gabriel Synnaeve and Diyi Yang and Ofir Press},
-  year={2026},
+@misc{yang2026programbenchlanguagemodelsrebuild,
+    title={ProgramBench: Can Language Models Rebuild Programs From Scratch?},
+    author={John Yang and Kilian Lieret and Jeffrey Ma and Parth Thakkar and Dmitrii Pedchenko and Sten Sootla and Emily McMilin and Pengcheng Yin and Rui Hou and Gabriel Synnaeve and Diyi Yang and Ofir Press},
+    year={2026},
+    eprint={2605.03546},
+    archivePrefix={arXiv},
+    primaryClass={cs.SE},
+    url={https://arxiv.org/abs/2605.03546},
 }
 ```
 

{programbench-1.0.0 → programbench-1.0.2}/README.md

@@ -9,7 +9,7 @@ Given only a compiled binary and its documentation, AI agents must architect and
 ## Links
 
 - [Website](https://programbench.com)
-- [Paper](https://programbench.com/static/paper.pdf)
+- [Paper](https://arxiv.org/abs/2605.03546)
 - [HuggingFace](https://huggingface.co/datasets/programbench/ProgramBench-Tests)
 - [Leaderboard](https://programbench.com)
 - [Usage Guide](docs/README.md)
@@ -45,10 +45,14 @@ uv sync  # installs editable + dev dependencies
 If our work was useful for you, please cite it:
 
 ```bibtex
-@preprint{yang2026programbench,
-  title={ProgramBench: Can Language Models Rebuild Programs From Scratch?},
-  author={John Yang and Kilian Lieret and Jeffrey Ma and Parth Thakkar and Dmitrii Pedchenko and Sten Sootla and Emily McMilin and Pengcheng Yin and Rui Hou and Gabriel Synnaeve and Diyi Yang and Ofir Press},
-  year={2026},
+@misc{yang2026programbenchlanguagemodelsrebuild,
+    title={ProgramBench: Can Language Models Rebuild Programs From Scratch?},
+    author={John Yang and Kilian Lieret and Jeffrey Ma and Parth Thakkar and Dmitrii Pedchenko and Sten Sootla and Emily McMilin and Pengcheng Yin and Rui Hou and Gabriel Synnaeve and Diyi Yang and Ofir Press},
+    year={2026},
+    eprint={2605.03546},
+    archivePrefix={arXiv},
+    primaryClass={cs.SE},
+    url={https://arxiv.org/abs/2605.03546},
 }
 ```
 

{programbench-1.0.0 → programbench-1.0.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "programbench"
-version = "1.0.0"
+version = "1.0.2"
 description = "Evaluate whether LM-based SWE-agents can reverse-engineer black-box software systems"
 readme = "README.md"
 authors = [

{programbench-1.0.0 → programbench-1.0.2}/src/programbench/data/tasks/abishekvashok__cmatrix.5c082c6/tests.json

@@ -724,7 +724,17 @@
         "eval.tests.test_args_parsing.test_version_flag_short_circuits_other_validation[args2]",
         "eval.tests.test_args_parsing.test_version_flag_short_circuits_other_validation[args3]"
       ],
-      "ignored_tests": []
+      "ignored_tests": [
+        {
+          "name": "eval.tests.test_args_parsing.test_u_out_of_range_or_overflow_values_still_enter_render_loop[args2]",
+          "reasons": [
+            {
+              "id": "ignored_manual",
+              "description": "This test passes -u 999999999999999999999999, which overflows C's 32-bit int via atoi(), wrapping to a small/negative value so the original binary enters the render loop. This is undefined behavior in C, not intentional design. A correct reimplementation in a language without integer overflow (e.g. Python) would reasonably reject or hang on this value. The test enshrines accidental C overflow semantics rather than meaningful program behavior."
+            }
+          ]
+        }
+      ]
     },
     "1d30a9bd67f8": {
       "ignored": false,

{programbench-1.0.0 → programbench-1.0.2}/src/programbench/data/tasks/alecthomas__chroma.8d04def/tests.json

@@ -587,6 +587,32 @@
             }
           ]
         },
+        {
+          "name": "tests.test_gap_filling.test_large_stdin_with_fail_flag",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep__gemini_3_1_pro__gemini-3-1-pro-preview-genai",
+              "timestamp": 1778451059,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_gap_filling.test_large_stdin_without_fail_flag",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep__gemini_3_flash__gemini-3-flash-preview-genai",
+              "timestamp": 1778451067,
+              "user": "kilian"
+            }
+          ]
+        },
         {
           "name": "tests.test_harvest.test_lexer_analysis[bash]",
           "reasons": [
@@ -643,6 +669,123 @@
             }
           ]
         },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[fsharp/fsharp.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=569.1s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4__gpt-5-4-fair",
+              "timestamp": 1778451185,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[objectpascal.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=445.1s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4_responses__gpt-5-4-genai-responses",
+              "timestamp": 1778451205,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[openedgeabl.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=1007.3s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__muse_spark_direct__muse_spark_direct_5_10",
+              "timestamp": 1778451233,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[python/test_fstring_35b.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep__opus_4_7_max__claude-4-7-opus-genai",
+              "timestamp": 1778451144,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[raku/raku.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=1056.3s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4_responses__gpt-5-4-genai-responses",
+              "timestamp": 1778451205,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[tal.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4_responses__gpt-5-4-genai-responses",
+              "timestamp": 1778451205,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[v/v.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=1385.0s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4__gpt-5-4-fair",
+              "timestamp": 1778451185,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[wat.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=463.8s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4__gpt-5-4-fair",
+              "timestamp": 1778451185,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[zig.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4__gpt-5-4-fair",
+              "timestamp": 1778451185,
+              "user": "kilian"
+            }
+          ]
+        },
         {
           "name": "tests.test_integration.test_html_prevent_surrounding_pre",
           "reasons": [
@@ -663,6 +806,19 @@
             }
           ]
         },
+        {
+          "name": "tests.test_special_modes.test_large_input_processed_correctly",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep__gpt_5_5_responses_high__gpt-5-5-genai-responses",
+              "timestamp": 1778343057,
+              "user": "kilian"
+            }
+          ]
+        },
         {
           "name": "tests.test_special_modes.test_trace_output_json_structure_complete",
           "reasons": [