programbench 1.0.1__tar.gz → 1.0.2__tar.gz

{programbench-1.0.1 → programbench-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: programbench
-Version: 1.0.1
+Version: 1.0.2
 Summary: Evaluate whether LM-based SWE-agents can reverse-engineer black-box software systems
 Author: Kilian Adriano Lieret, John Yang
 Author-email: Kilian Adriano Lieret <klieret@meta.com>, John Yang <johnby@meta.com>

{programbench-1.0.1 → programbench-1.0.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "programbench"
-version = "1.0.1"
+version = "1.0.2"
 description = "Evaluate whether LM-based SWE-agents can reverse-engineer black-box software systems"
 readme = "README.md"
 authors = [

{programbench-1.0.1 → programbench-1.0.2}/src/programbench/data/tasks/abishekvashok__cmatrix.5c082c6/tests.json

@@ -724,7 +724,17 @@
         "eval.tests.test_args_parsing.test_version_flag_short_circuits_other_validation[args2]",
         "eval.tests.test_args_parsing.test_version_flag_short_circuits_other_validation[args3]"
       ],
-      "ignored_tests": []
+      "ignored_tests": [
+        {
+          "name": "eval.tests.test_args_parsing.test_u_out_of_range_or_overflow_values_still_enter_render_loop[args2]",
+          "reasons": [
+            {
+              "id": "ignored_manual",
+              "description": "This test passes -u 999999999999999999999999, which overflows C's 32-bit int via atoi(), wrapping to a small/negative value so the original binary enters the render loop. This is undefined behavior in C, not intentional design. A correct reimplementation in a language without integer overflow (e.g. Python) would reasonably reject or hang on this value. The test enshrines accidental C overflow semantics rather than meaningful program behavior."
+            }
+          ]
+        }
+      ]
     },
     "1d30a9bd67f8": {
       "ignored": false,

{programbench-1.0.1 → programbench-1.0.2}/src/programbench/data/tasks/alecthomas__chroma.8d04def/tests.json

@@ -587,6 +587,32 @@
             }
           ]
         },
+        {
+          "name": "tests.test_gap_filling.test_large_stdin_with_fail_flag",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep__gemini_3_1_pro__gemini-3-1-pro-preview-genai",
+              "timestamp": 1778451059,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_gap_filling.test_large_stdin_without_fail_flag",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep__gemini_3_flash__gemini-3-flash-preview-genai",
+              "timestamp": 1778451067,
+              "user": "kilian"
+            }
+          ]
+        },
         {
           "name": "tests.test_harvest.test_lexer_analysis[bash]",
           "reasons": [
@@ -643,6 +669,123 @@
             }
           ]
         },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[fsharp/fsharp.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=569.1s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4__gpt-5-4-fair",
+              "timestamp": 1778451185,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[objectpascal.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=445.1s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4_responses__gpt-5-4-genai-responses",
+              "timestamp": 1778451205,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[openedgeabl.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=1007.3s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__muse_spark_direct__muse_spark_direct_5_10",
+              "timestamp": 1778451233,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[python/test_fstring_35b.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep__opus_4_7_max__claude-4-7-opus-genai",
+              "timestamp": 1778451144,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[raku/raku.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=1056.3s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4_responses__gpt-5-4-genai-responses",
+              "timestamp": 1778451205,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[tal.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4_responses__gpt-5-4-genai-responses",
+              "timestamp": 1778451205,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[v/v.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=1385.0s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4__gpt-5-4-fair",
+              "timestamp": 1778451185,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[wat.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "total=463.8s phases=['call', 'setup', 'teardown']",
+              "kind": "slow",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4__gpt-5-4-fair",
+              "timestamp": 1778451185,
+              "user": "kilian"
+            }
+          ]
+        },
+        {
+          "name": "tests.test_harvest.test_lexer_fixture[zig.actual]",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep_diff_lang__gpt_5_4__gpt-5-4-fair",
+              "timestamp": 1778451185,
+              "user": "kilian"
+            }
+          ]
+        },
         {
           "name": "tests.test_integration.test_html_prevent_surrounding_pre",
           "reasons": [
@@ -663,6 +806,19 @@
             }
           ]
         },
+        {
+          "name": "tests.test_special_modes.test_large_input_processed_correctly",
+          "reasons": [
+            {
+              "id": "slow_or_hang",
+              "info": "phases=['call', 'setup'] run_tests rc=-1",
+              "kind": "hung",
+              "run": "infer_no_int_no_dep__gpt_5_5_responses_high__gpt-5-5-genai-responses",
+              "timestamp": 1778343057,
+              "user": "kilian"
+            }
+          ]
+        },
         {
           "name": "tests.test_special_modes.test_trace_output_json_structure_complete",
           "reasons": [