ttsem 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. {ttsem-0.1.0 → ttsem-0.2.0}/CITATION.cff +2 -2
  2. ttsem-0.1.0/README.md → ttsem-0.2.0/PKG-INFO +69 -6
  3. ttsem-0.1.0/PKG-INFO → ttsem-0.2.0/README.md +34 -39
  4. {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/device.py +2 -1
  5. {ttsem-0.1.0 → ttsem-0.2.0}/docs/JUDGE.md +1 -1
  6. ttsem-0.2.0/docs/REWARD.md +358 -0
  7. {ttsem-0.1.0 → ttsem-0.2.0}/pyproject.toml +6 -2
  8. ttsem-0.2.0/tests/fixtures/reward/kb_shift_answer.py +30 -0
  9. ttsem-0.2.0/tests/fixtures/reward/kb_shift_task.py +26 -0
  10. ttsem-0.2.0/tests/fixtures/reward/shift_right_answer.py +22 -0
  11. ttsem-0.2.0/tests/fixtures/reward/shift_right_task.py +21 -0
  12. ttsem-0.2.0/tests/fixtures/triton39_cache_policy.py +43 -0
  13. ttsem-0.2.0/tests/fixtures/triton39_cache_policy.ttir.generic +25 -0
  14. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_audits.py +2 -1
  15. ttsem-0.2.0/tests/test_compat.py +38 -0
  16. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_judge.py +6 -2
  17. ttsem-0.2.0/tests/test_reward.py +635 -0
  18. ttsem-0.2.0/tests/test_reward_cap.py +169 -0
  19. ttsem-0.2.0/tests/test_reward_hygiene.py +67 -0
  20. ttsem-0.2.0/tests/test_reward_integration.py +187 -0
  21. ttsem-0.2.0/tests/test_reward_verl.py +81 -0
  22. ttsem-0.2.0/tests/test_triton39.py +73 -0
  23. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/__init__.py +1 -1
  24. ttsem-0.2.0/ttsem/_compat.py +51 -0
  25. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_judge_runner.py +4 -0
  26. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/interp.py +2 -1
  27. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/interp2.py +2 -1
  28. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/judge.py +7 -2
  29. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/judge_tritonbench.py +12 -2
  30. ttsem-0.2.0/ttsem/reward/__init__.py +15 -0
  31. ttsem-0.2.0/ttsem/reward/cap.py +509 -0
  32. ttsem-0.2.0/ttsem/reward/core.py +766 -0
  33. ttsem-0.2.0/ttsem/reward/verl_adapter.py +135 -0
  34. ttsem-0.2.0/ttsem/reward/worker.py +263 -0
  35. {ttsem-0.1.0 → ttsem-0.2.0}/.gitignore +0 -0
  36. {ttsem-0.1.0 → ttsem-0.2.0}/CONTRIBUTING.md +0 -0
  37. {ttsem-0.1.0 → ttsem-0.2.0}/LEDGER.md +0 -0
  38. {ttsem-0.1.0 → ttsem-0.2.0}/LICENSE +0 -0
  39. {ttsem-0.1.0 → ttsem-0.2.0}/audits/kernelbench/README.md +0 -0
  40. {ttsem-0.1.0 → ttsem-0.2.0}/audits/kernelbench/prepare.py +0 -0
  41. {ttsem-0.1.0 → ttsem-0.2.0}/audits/kernelbench/table.py +0 -0
  42. {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/README.md +0 -0
  43. {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/agree.py +0 -0
  44. {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/device_runner.py +0 -0
  45. {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/manifest.py +0 -0
  46. {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/prepare.py +0 -0
  47. {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/table.py +0 -0
  48. {ttsem-0.1.0 → ttsem-0.2.0}/docs/AGENTS.md +0 -0
  49. {ttsem-0.1.0 → ttsem-0.2.0}/docs/DESIGN.md +0 -0
  50. {ttsem-0.1.0 → ttsem-0.2.0}/docs/INDUCTOR.md +0 -0
  51. {ttsem-0.1.0 → ttsem-0.2.0}/docs/OVERVIEW.md +0 -0
  52. {ttsem-0.1.0 → ttsem-0.2.0}/docs/RESULTS.md +0 -0
  53. {ttsem-0.1.0 → ttsem-0.2.0}/docs/SEMANTICS.md +0 -0
  54. {ttsem-0.1.0 → ttsem-0.2.0}/examples/ci/test_scale.py +0 -0
  55. {ttsem-0.1.0 → ttsem-0.2.0}/examples/e11601_min.mlir +0 -0
  56. {ttsem-0.1.0 → ttsem-0.2.0}/examples/e11601_repro.py +0 -0
  57. {ttsem-0.1.0 → ttsem-0.2.0}/examples/e11612_kernel_min.mlir +0 -0
  58. {ttsem-0.1.0 → ttsem-0.2.0}/examples/e11612_repro_min.mlir +0 -0
  59. {ttsem-0.1.0 → ttsem-0.2.0}/examples/e15_11519_main_e1f944a7a.json +0 -0
  60. {ttsem-0.1.0 → ttsem-0.2.0}/examples/e15_11519_min.mlir +0 -0
  61. {ttsem-0.1.0 → ttsem-0.2.0}/examples/e15_repro.py +0 -0
  62. {ttsem-0.1.0 → ttsem-0.2.0}/examples/e2_repro.py +0 -0
  63. {ttsem-0.1.0 → ttsem-0.2.0}/examples/inductor_models.py +0 -0
  64. {ttsem-0.1.0 → ttsem-0.2.0}/examples/inductor_models2.py +0 -0
  65. {ttsem-0.1.0 → ttsem-0.2.0}/examples/witness_e8m0_nan.py +0 -0
  66. {ttsem-0.1.0 → ttsem-0.2.0}/examples/witness_interp_bf16_add.py +0 -0
  67. {ttsem-0.1.0 → ttsem-0.2.0}/tests/conftest.py +0 -0
  68. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/accumulate_into_empty.py +0 -0
  69. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/add_no_load_mask.py +0 -0
  70. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/add_no_store_mask.py +0 -0
  71. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/add_ok.py +0 -0
  72. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/cuda_script.py +0 -0
  73. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/suite_big_launch.py +0 -0
  74. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/suite_big_tiled.py +0 -0
  75. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/suite_no_mask.py +0 -0
  76. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/gluon_copy.py +0 -0
  77. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/inductor/channel_share_output_code.py +0 -0
  78. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/inductor/safe_share_output_code.py +0 -0
  79. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/inductor/split_scan_output_code.py +0 -0
  80. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/inductor/user_kernel_clone_removed_output_code.py +0 -0
  81. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_config_asserts.py +0 -0
  82. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_config_faults.py +0 -0
  83. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_config_pruned.py +0 -0
  84. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_forges.py +0 -0
  85. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_lookback_helper.py +0 -0
  86. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_nested_grid.py +0 -0
  87. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_no_config_fits.py +0 -0
  88. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_no_kernel.py +0 -0
  89. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_ok.py +0 -0
  90. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_patch_restore.py +0 -0
  91. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_prewrites.py +0 -0
  92. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_protocol_big.py +0 -0
  93. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_protocol_marker.py +0 -0
  94. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_race_unchecked.py +0 -0
  95. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_swallows.py +0 -0
  96. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_tampers.py +0 -0
  97. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_thread_flip.py +0 -0
  98. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_unsafe.py +0 -0
  99. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_wrong.py +0 -0
  100. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/task_matmul.py +0 -0
  101. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/task_relu.py +0 -0
  102. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/membar_11325.generic +0 -0
  103. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.npy +0 -0
  104. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.py +0 -0
  105. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.ttgir +0 -0
  106. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.ttgir.generic +0 -0
  107. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.ttir +0 -0
  108. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.ttir.generic +0 -0
  109. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.npy +0 -0
  110. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.py +0 -0
  111. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.ttgir +0 -0
  112. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.ttgir.generic +0 -0
  113. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.ttir +0 -0
  114. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.ttir.generic +0 -0
  115. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.npy +0 -0
  116. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.py +0 -0
  117. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.ttgir +0 -0
  118. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.ttgir.generic +0 -0
  119. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.ttir +0 -0
  120. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.ttir.generic +0 -0
  121. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.npy +0 -0
  122. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.py +0 -0
  123. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.ttgir +0 -0
  124. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.ttgir.generic +0 -0
  125. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.ttir +0 -0
  126. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.ttir.generic +0 -0
  127. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_cheats.py +0 -0
  128. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_no_kernel.py +0 -0
  129. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_ok.py +0 -0
  130. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_peeks.py +0 -0
  131. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_plants.py +0 -0
  132. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_prewrites.py +0 -0
  133. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_protocol_marker.py +0 -0
  134. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_race_unchecked.py +0 -0
  135. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_raises.py +0 -0
  136. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_swallows.py +0 -0
  137. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_unsafe.py +0 -0
  138. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_wrong.py +0 -0
  139. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/task_relu.py +0 -0
  140. {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/ws_desc.py +0 -0
  141. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_bot.py +0 -0
  142. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_demo.py +0 -0
  143. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_gluon_compile.py +0 -0
  144. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_harness.py +0 -0
  145. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_interp.py +0 -0
  146. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_judge_integrity.py +0 -0
  147. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_judge_tritonbench.py +0 -0
  148. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_layout_checks.py +0 -0
  149. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_layouts.py +0 -0
  150. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_lost_updates.py +0 -0
  151. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_minimize.py +0 -0
  152. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_mlir.py +0 -0
  153. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_nvws.py +0 -0
  154. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_ops.py +0 -0
  155. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_parser_scope.py +0 -0
  156. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_pidraces.py +0 -0
  157. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_plugin_warnings.py +0 -0
  158. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_races.py +0 -0
  159. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_sanitize.py +0 -0
  160. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_scaled_dot.py +0 -0
  161. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_scheduler.py +0 -0
  162. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_torch_demo.py +0 -0
  163. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_torch_demo_ledger.py +0 -0
  164. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_validate.py +0 -0
  165. {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_validate_corpus.py +0 -0
  166. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/__main__.py +0 -0
  167. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_compile_runner.py +0 -0
  168. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_fakedriver.py +0 -0
  169. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_guard.py +0 -0
  170. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_install.py +0 -0
  171. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_interp_runner.py +0 -0
  172. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_tritonbench_runner.py +0 -0
  173. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/bot.py +0 -0
  174. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/bot.sh +0 -0
  175. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/campaign_target.py +0 -0
  176. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/check_fixtures.py +0 -0
  177. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/defuse.py +0 -0
  178. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/demo.py +0 -0
  179. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/dump_corpus.py +0 -0
  180. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/harness.py +0 -0
  181. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/inductor_trace.py +0 -0
  182. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/ir_types.py +0 -0
  183. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/judge_io.py +0 -0
  184. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/judge_shapes.py +0 -0
  185. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/layout_checks.py +0 -0
  186. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/layouts.py +0 -0
  187. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/linear_layout.py +0 -0
  188. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/memory.py +0 -0
  189. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/minimize.py +0 -0
  190. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/mlir.py +0 -0
  191. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/ops.py +0 -0
  192. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/pass_stress.py +0 -0
  193. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/pidraces.py +0 -0
  194. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/py.typed +0 -0
  195. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/pytest_ttsem.py +0 -0
  196. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/races.py +0 -0
  197. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/races_corpus.py +0 -0
  198. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/sanitize.py +0 -0
  199. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/torch_demo.py +0 -0
  200. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/validate.py +0 -0
  201. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/validate_corpus.py +0 -0
  202. {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/values.py +0 -0
@@ -5,8 +5,8 @@ title: "ttsem: an executable semantics of Triton's TTIR and TTGIR, with a per-pa
5
5
  authors:
6
6
  - family-names: Potenza
7
7
  given-names: Alessandro
8
- version: 0.1.0
9
- # date-released is set when 0.1.0 is tagged and published
8
+ version: 0.2.0
9
+ # date-released is set when 0.2.0 is tagged and published
10
10
  license: MIT
11
11
  repository-code: "https://github.com/alepot55/ttsem"
12
12
  url: "https://github.com/alepot55/ttsem"
@@ -1,3 +1,38 @@
1
+ Metadata-Version: 2.5
2
+ Name: ttsem
3
+ Version: 0.2.0
4
+ Summary: Run Triton kernel tests without a GPU: out-of-bounds accesses and races at the kernel's line
5
+ Project-URL: Homepage, https://github.com/alepot55/ttsem
6
+ Project-URL: Source, https://github.com/alepot55/ttsem
7
+ Project-URL: Issues, https://github.com/alepot55/ttsem/issues
8
+ Author: Alessandro Potenza
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: compiler,gpu,interpreter,mlir,semantics,triton
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: Operating System :: POSIX :: Linux
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3 :: Only
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Software Development :: Compilers
23
+ Classifier: Topic :: Software Development :: Testing
24
+ Classifier: Typing :: Typed
25
+ Requires-Python: >=3.10
26
+ Requires-Dist: numpy
27
+ Provides-Extra: test
28
+ Requires-Dist: pytest; extra == 'test'
29
+ Requires-Dist: pytest-xdist; extra == 'test'
30
+ Provides-Extra: triton
31
+ Requires-Dist: pytest; extra == 'triton'
32
+ Requires-Dist: torch==2.14.0; extra == 'triton'
33
+ Requires-Dist: triton==3.8.0; extra == 'triton'
34
+ Description-Content-Type: text/markdown
35
+
1
36
  # ttsem
2
37
 
3
38
  [![CI](https://github.com/alepot55/ttsem/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/alepot55/ttsem/actions/workflows/ci.yml)
@@ -11,12 +46,13 @@ the CPU.
11
46
 
12
47
  ```console
13
48
  $ pip install --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple \
14
- 'ttsem[triton] @ git+https://github.com/alepot55/ttsem'
49
+ 'ttsem[triton]'
15
50
  $ python -m ttsem.sanitize pytest tests/ -q # your suite, written for device="cuda"
16
51
  ```
17
52
 
18
- The first command installs ttsem with Triton 3.8.0 and the CPU build of torch 2.14.0 (Linux,
19
- Python 3.12 or 3.13); once ttsem is on PyPI, `'ttsem[triton]'` replaces the git requirement. The
53
+ The first command installs ttsem from PyPI with Triton 3.8.0 and the CPU build of torch 2.14.0
54
+ (Linux, Python 3.10 to 3.13); `pip install ttsem` alone, numpy its only dependency, runs
55
+ `python -m ttsem demo`. The
20
56
  second runs a pytest suite written for `device="cuda"`, unchanged. On [suite_no_mask.py][nomask], a
21
57
  test file whose kernel forgets the mask on its store, the size that is a multiple of the block
22
58
  passes and the other one fails at line 15 (from a clone; the last lines of the output):
@@ -31,10 +67,10 @@ FAILED tests/fixtures/agents/suite_no_mask.py::test_add[1000] - KernelFault: ...
31
67
  ```
32
68
 
33
69
  `python -m ttsem.sanitize script.py` does the same for a script. In CI, the action runs your suite
34
- this way on a CPU runner (pin it to `@v0.1.0` once that release is tagged):
70
+ this way on a CPU runner:
35
71
 
36
72
  ```yaml
37
- - uses: alepot55/ttsem@main
73
+ - uses: alepot55/ttsem@v0.2.0
38
74
  with:
39
75
  install: pip install -e . # your project
40
76
  args: tests/kernels -q # pytest arguments
@@ -273,11 +309,36 @@ right and holds against an adversarial answer, since the comparison rests on a r
273
309
  cannot touch (`verified` means right values; a kernel launched with the work then done in PyTorch
274
310
  still verifies). `unsafe` and `no_kernel` rest on two stop-gaps, a source scan and a run-time
275
311
  integrity check of the whole checker, that block every attack in the test suite but are not a
276
- proof: until the answer and the checker run in separate processes (planned for 0.2), use the judge
312
+ proof: until the answer and the checker run in separate processes (planned, not in 0.2), use the judge
277
313
  as an auditor and an evaluation tool, and as an RL reward against adversarial optimisation with that
278
314
  caveat. The rules, the batch mode, the classes, the sandbox and what each verdict guarantees are in
279
315
  [docs/JUDGE.md][judge].
280
316
 
317
+ ## Reward
318
+
319
+ New in 0.2, prototype-grade: `ttsem.reward` turns the judge's verdict into a reward for training
320
+ models that write Triton kernels, with no GPU in the reward loop. `Judge.score(reference, answer)`
321
+ returns a verdict: a score of 1.0, 0.0 or none (an abstention), and a paragraph for the model that
322
+ names the kernel, the buffer and the line. In gate mode a reward you already have is kept for a
323
+ verified answer and lowered to the fail value for a rejected one, never raised; in score mode the
324
+ score is the reward. The trainer's process imports neither torch nor Triton: the passes run
325
+ in a worker in the judge's own venv, every answer and every reference in its own `bwrap` sandbox,
326
+ and nothing runs without one. A verl adapter is included, tested against a fake worker and not yet
327
+ against a verl release.
328
+
329
+ ```python
330
+ from ttsem.reward import Judge, extract_answer
331
+
332
+ judge = Judge(cache_dir=cache, python=judge_venv_python, mode="gate")
333
+ verdict = judge.score(reference_src, extract_answer(completion))
334
+ reward = judge.gate(gpu_reward, verdict)
335
+ ```
336
+
337
+ It carries the judge's own caveat (`unsafe` and `no_kernel` are not proofs against an optimiser free
338
+ to write anything), prices no speed, and abstains where it cannot say whose fault an outcome is, which
339
+ an answer can sometimes provoke. What it does and does not catch, its API, the adapter and how the
340
+ worker runs are in [docs/REWARD.md][reward].
341
+
281
342
  ## How it was built
282
343
 
283
344
  Built by Alessandro Potenza with Claude Code (Anthropic's coding agent) as the engineering agent
@@ -312,6 +373,7 @@ after the system was found to exploit its evaluation harness ([TechCrunch][sakan
312
373
  | [docs/DESIGN.md][design] | the three levels, the interfaces between them, and why they are drawn there |
313
374
  | [docs/RESULTS.md][results] | the corpora, the counts, and the calibration of the semantics against the device |
314
375
  | [docs/JUDGE.md][judge] | the judge of model-written kernels: rules, verdicts, batches, the sandbox |
376
+ | [docs/REWARD.md][reward] | the judge as an RL reward: gate and score mode, abstentions, the API, the verl adapter, the worker, what it does not catch |
315
377
  | [docs/INDUCTOR.md][inductor] | the Triton code of `torch.compile` under ttsem |
316
378
  | [docs/AGENTS.md][agents] | for coding agents that write Triton kernels |
317
379
  | [audits/](https://github.com/alepot55/ttsem/tree/main/audits) | the commands that redo the TritonBench and KernelBench audits from a clone |
@@ -326,6 +388,7 @@ this repository with the metadata in `CITATION.cff` and the commit you used. MIT
326
388
  [agents]: https://github.com/alepot55/ttsem/blob/main/docs/AGENTS.md
327
389
  [skill]: https://github.com/alepot55/ttsem/blob/main/skills/triton-verify/SKILL.md
328
390
  [judge]: https://github.com/alepot55/ttsem/blob/main/docs/JUDGE.md
391
+ [reward]: https://github.com/alepot55/ttsem/blob/main/docs/REWARD.md
329
392
  [ledger]: https://github.com/alepot55/ttsem/blob/main/LEDGER.md
330
393
  [overview]: https://github.com/alepot55/ttsem/blob/main/docs/OVERVIEW.md
331
394
  [semantics]: https://github.com/alepot55/ttsem/blob/main/docs/SEMANTICS.md
@@ -1,36 +1,3 @@
1
- Metadata-Version: 2.5
2
- Name: ttsem
3
- Version: 0.1.0
4
- Summary: Run Triton kernel tests without a GPU: out-of-bounds accesses and races at the kernel's line
5
- Project-URL: Homepage, https://github.com/alepot55/ttsem
6
- Project-URL: Source, https://github.com/alepot55/ttsem
7
- Project-URL: Issues, https://github.com/alepot55/ttsem/issues
8
- Author: Alessandro Potenza
9
- License-Expression: MIT
10
- License-File: LICENSE
11
- Keywords: compiler,gpu,interpreter,mlir,semantics,triton
12
- Classifier: Development Status :: 3 - Alpha
13
- Classifier: Intended Audience :: Developers
14
- Classifier: Intended Audience :: Science/Research
15
- Classifier: Operating System :: POSIX :: Linux
16
- Classifier: Programming Language :: Python :: 3
17
- Classifier: Programming Language :: Python :: 3 :: Only
18
- Classifier: Programming Language :: Python :: 3.12
19
- Classifier: Programming Language :: Python :: 3.13
20
- Classifier: Topic :: Software Development :: Compilers
21
- Classifier: Topic :: Software Development :: Testing
22
- Classifier: Typing :: Typed
23
- Requires-Python: >=3.12
24
- Requires-Dist: numpy
25
- Provides-Extra: test
26
- Requires-Dist: pytest; extra == 'test'
27
- Requires-Dist: pytest-xdist; extra == 'test'
28
- Provides-Extra: triton
29
- Requires-Dist: pytest; extra == 'triton'
30
- Requires-Dist: torch==2.14.0; extra == 'triton'
31
- Requires-Dist: triton==3.8.0; extra == 'triton'
32
- Description-Content-Type: text/markdown
33
-
34
1
  # ttsem
35
2
 
36
3
  [![CI](https://github.com/alepot55/ttsem/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/alepot55/ttsem/actions/workflows/ci.yml)
@@ -44,12 +11,13 @@ the CPU.
44
11
 
45
12
  ```console
46
13
  $ pip install --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple \
47
- 'ttsem[triton] @ git+https://github.com/alepot55/ttsem'
14
+ 'ttsem[triton]'
48
15
  $ python -m ttsem.sanitize pytest tests/ -q # your suite, written for device="cuda"
49
16
  ```
50
17
 
51
- The first command installs ttsem with Triton 3.8.0 and the CPU build of torch 2.14.0 (Linux,
52
- Python 3.12 or 3.13); once ttsem is on PyPI, `'ttsem[triton]'` replaces the git requirement. The
18
+ The first command installs ttsem from PyPI with Triton 3.8.0 and the CPU build of torch 2.14.0
19
+ (Linux, Python 3.10 to 3.13); `pip install ttsem` alone, numpy its only dependency, runs
20
+ `python -m ttsem demo`. The
53
21
  second runs a pytest suite written for `device="cuda"`, unchanged. On [suite_no_mask.py][nomask], a
54
22
  test file whose kernel forgets the mask on its store, the size that is a multiple of the block
55
23
  passes and the other one fails at line 15 (from a clone; the last lines of the output):
@@ -64,10 +32,10 @@ FAILED tests/fixtures/agents/suite_no_mask.py::test_add[1000] - KernelFault: ...
64
32
  ```
65
33
 
66
34
  `python -m ttsem.sanitize script.py` does the same for a script. In CI, the action runs your suite
67
- this way on a CPU runner (pin it to `@v0.1.0` once that release is tagged):
35
+ this way on a CPU runner:
68
36
 
69
37
  ```yaml
70
- - uses: alepot55/ttsem@main
38
+ - uses: alepot55/ttsem@v0.2.0
71
39
  with:
72
40
  install: pip install -e . # your project
73
41
  args: tests/kernels -q # pytest arguments
@@ -306,11 +274,36 @@ right and holds against an adversarial answer, since the comparison rests on a r
306
274
  cannot touch (`verified` means right values; a kernel launched with the work then done in PyTorch
307
275
  still verifies). `unsafe` and `no_kernel` rest on two stop-gaps, a source scan and a run-time
308
276
  integrity check of the whole checker, that block every attack in the test suite but are not a
309
- proof: until the answer and the checker run in separate processes (planned for 0.2), use the judge
277
+ proof: until the answer and the checker run in separate processes (planned, not in 0.2), use the judge
310
278
  as an auditor and an evaluation tool, and as an RL reward against adversarial optimisation with that
311
279
  caveat. The rules, the batch mode, the classes, the sandbox and what each verdict guarantees are in
312
280
  [docs/JUDGE.md][judge].
313
281
 
282
+ ## Reward
283
+
284
+ New in 0.2, prototype-grade: `ttsem.reward` turns the judge's verdict into a reward for training
285
+ models that write Triton kernels, with no GPU in the reward loop. `Judge.score(reference, answer)`
286
+ returns a verdict: a score of 1.0, 0.0 or none (an abstention), and a paragraph for the model that
287
+ names the kernel, the buffer and the line. In gate mode a reward you already have is kept for a
288
+ verified answer and lowered to the fail value for a rejected one, never raised; in score mode the
289
+ score is the reward. The trainer's process imports neither torch nor Triton: the passes run
290
+ in a worker in the judge's own venv, every answer and every reference in its own `bwrap` sandbox,
291
+ and nothing runs without one. A verl adapter is included, tested against a fake worker and not yet
292
+ against a verl release.
293
+
294
+ ```python
295
+ from ttsem.reward import Judge, extract_answer
296
+
297
+ judge = Judge(cache_dir=cache, python=judge_venv_python, mode="gate")
298
+ verdict = judge.score(reference_src, extract_answer(completion))
299
+ reward = judge.gate(gpu_reward, verdict)
300
+ ```
301
+
302
+ It carries the judge's own caveat (`unsafe` and `no_kernel` are not proofs against an optimiser free
303
+ to write anything), prices no speed, and abstains where it cannot say whose fault an outcome is, which
304
+ an answer can sometimes provoke. What it does and does not catch, its API, the adapter and how the
305
+ worker runs are in [docs/REWARD.md][reward].
306
+
314
307
  ## How it was built
315
308
 
316
309
  Built by Alessandro Potenza with Claude Code (Anthropic's coding agent) as the engineering agent
@@ -345,6 +338,7 @@ after the system was found to exploit its evaluation harness ([TechCrunch][sakan
345
338
  | [docs/DESIGN.md][design] | the three levels, the interfaces between them, and why they are drawn there |
346
339
  | [docs/RESULTS.md][results] | the corpora, the counts, and the calibration of the semantics against the device |
347
340
  | [docs/JUDGE.md][judge] | the judge of model-written kernels: rules, verdicts, batches, the sandbox |
341
+ | [docs/REWARD.md][reward] | the judge as an RL reward: gate and score mode, abstentions, the API, the verl adapter, the worker, what it does not catch |
348
342
  | [docs/INDUCTOR.md][inductor] | the Triton code of `torch.compile` under ttsem |
349
343
  | [docs/AGENTS.md][agents] | for coding agents that write Triton kernels |
350
344
  | [audits/](https://github.com/alepot55/ttsem/tree/main/audits) | the commands that redo the TritonBench and KernelBench audits from a clone |
@@ -359,6 +353,7 @@ this repository with the metadata in `CITATION.cff` and the commit you used. MIT
359
353
  [agents]: https://github.com/alepot55/ttsem/blob/main/docs/AGENTS.md
360
354
  [skill]: https://github.com/alepot55/ttsem/blob/main/skills/triton-verify/SKILL.md
361
355
  [judge]: https://github.com/alepot55/ttsem/blob/main/docs/JUDGE.md
356
+ [reward]: https://github.com/alepot55/ttsem/blob/main/docs/REWARD.md
362
357
  [ledger]: https://github.com/alepot55/ttsem/blob/main/LEDGER.md
363
358
  [overview]: https://github.com/alepot55/ttsem/blob/main/docs/OVERVIEW.md
364
359
  [semantics]: https://github.com/alepot55/ttsem/blob/main/docs/SEMANTICS.md
@@ -36,6 +36,7 @@ from typing import Any
36
36
 
37
37
  from ttsem import judge
38
38
  from ttsem import judge_tritonbench as tb
39
+ from ttsem._compat import python_script
39
40
  from ttsem.judge_io import printable, read_json, run_sandboxed
40
41
 
41
42
  RUNNER = Path(__file__).resolve().parent / "device_runner.py"
@@ -86,7 +87,7 @@ def run_file(
86
87
  env["PYTHONPATH"] = os.environ["PYTHONPATH"]
87
88
  cmd = judge.sandbox_prefix(work, cache, [script], env, hide) + gpu_nodes() + list(extra)
88
89
  cmd += ["--ro-bind", str(RUNNER), str(RUNNER)] # whatever /tmp or `hide` hide
89
- cmd += [sys.executable, "-P", str(RUNNER), str(script.resolve())]
90
+ cmd += [*python_script(sys.executable, str(RUNNER)), str(script.resolve())]
90
91
  cmd += [str(work / "run.json"), str(work / "run.npz")]
91
92
  return tb.run_report(run_sandboxed(cmd, env, work, timeout), work, script, timeout)
92
93
 
@@ -145,7 +145,7 @@ into the checker. What each verdict holds against is therefore worth stating pla
145
145
  race; the 32-bit helper packs value and flag into one word updated only atomically, which is
146
146
  never a race. None of the 7,752 published TritonBench answers calls either.
147
147
  - Full separation of the answer from the checker (running the semantics in a process the answer's
148
- code cannot reach at all) is planned for 0.2. Until then the judge is fit for auditing published
148
+ code cannot reach at all) is planned, and is not in 0.2. Until then the judge is fit for auditing published
149
149
  answers and for evaluation, where the answers were not written to defeat it. Used as an RL reward
150
150
  against an optimiser that is free to write anything, treat `unsafe` and `no_kernel` with this
151
151
  caveat: the two stop-gaps block every attack in the test suite, but they are not a proof, and
@@ -0,0 +1,358 @@
1
+ # The judge as a reward
2
+
3
+ `ttsem.reward` turns the judge's verdict on a model-written Triton kernel into a reward for
4
+ training: it runs the answer on a CPU, in the judge's sandbox, against the task's reference, and
5
+ says whether the kernel is right, whether it reads or writes outside its buffers or races, and
6
+ where, in a paragraph the model can be shown. No GPU is needed in the reward loop.
7
+
8
+ **Status: new in 0.2, prototype-grade.** The API may still change. It is tested against a fake
9
+ worker (every verdict-to-reward rule, the tiers, the cap), against the judge's own fixtures with
10
+ the real worker and sandbox, and as a module loaded by path; it has not been tested inside a verl
11
+ release, and this release shows no training run that used it. [What it does not
12
+ catch](#what-it-does-not-catch) is part of the status, not a footnote.
13
+
14
+ It judges two kinds of task, the ones [docs/JUDGE.md](JUDGE.md) describes: a KernelBench-style task
15
+ (the reference defines `Model`, `get_inputs` and `get_init_inputs`; the answer defines `ModelNew`)
16
+ and, with `format="tritonbench"`, a TritonBench task (the reference, a line of 146 `#`, the test).
17
+
18
+ ## A first call
19
+
20
+ ```python
21
+ from pathlib import Path
22
+
23
+ from ttsem.reward import Judge, extract_answer
24
+
25
+ judge = Judge(
26
+ cache_dir=Path("~/.cache/ttsem_reward").expanduser(), # on disk, with room: see "Cache"
27
+ python=Path("~/venvs/judge/bin/python").expanduser(), # the judge's own venv: see "The worker"
28
+ workers=4, # passes judged at once
29
+ mode="gate",
30
+ )
31
+ verdict = judge.score(reference_src, extract_answer(completion))
32
+ print(verdict.score, verdict.verdict, verdict.kind)
33
+ print(verdict.feedback) # one paragraph, for the model's next turn
34
+ reward = judge.gate(gpu_reward, verdict) # never above gpu_reward
35
+ judge.close()
36
+ ```
37
+
38
+ ## Gate mode and score mode
39
+
40
+ `mode` says what `Verdict.score` is.
41
+
42
+ - **`mode="gate"`** (the default) is for a reward you already have, usually a GPU reward that
43
+ measures speed. `score` is a multiplier: 1.0 for a kernel the judge verified, 0.0 for one it
44
+ rejected, `None` for an abstention. `Judge.gate(base, verdict)` applies it and can only lower the
45
+ reward: it returns `base` for a verified answer, `min(base, fail_value)` for a rejected one
46
+ (`fail_value` is 0.0 unless you set it), and for an abstention `base` (`on_abstain="keep"`, the
47
+ default) or `min(base, fail_value)` (`on_abstain="zero"`). The worst an answer that fools the
48
+ judge can get is the reward it would have had without it.
49
+ - **`mode="cpu"`**, score mode, is for a trainer with no GPU reward: `score` is the reward, 1.0 for
50
+ verified, 0.0 for rejected, and `None` for an abstention, which a trainer that needs a float
51
+ replaces by `abstain_value` (the verl adapter uses 0.0).
52
+
53
+ The two modes differ only where a failure may be the judge's environment rather than the answer
54
+ (next section): score mode gives 0.0 where gate mode abstains.
55
+
56
+ ## What a verdict becomes
57
+
58
+ `scores(verdict, kind)` is the whole table, `(score-mode reward, gate multiplier)`; `kind_of`
59
+ derives `kind`, the finer reason, from the judge's record.
60
+
61
+ | judge verdict | `kind` | score mode | gate mode |
62
+ |---|---|---|---|
63
+ | `verified` | none | 1.0 | 1.0 |
64
+ | `wrong` | `values`, `unwritten` (memory nobody wrote shows as NaN), `autotune` (right under the first config only), `weights`, `shape`, `not_tensor` | 0.0 | 0.0 |
65
+ | `unsafe` | `oob_read`, `oob_write`, `race`, `poison` | 0.0 | 0.0 |
66
+ | `no_kernel` | none (no Triton launch: the work was done in PyTorch) | 0.0 | 0.0 |
67
+ | `not_judged` | `tampered`, `forged` (the answer reached into the checker; `Verdict.hack`) | 0.0 | 0.0 |
68
+ | `error` | `raised` (the answer raised an exception), `no_code` (the completion holds no code) | 0.0 | 0.0 |
69
+ | `not_judged` | `native_code` (the answer builds C++ or CUDA of its own) | 0.0 | abstain |
70
+ | `error` | `version` (a compile or import error that may be drift between your Triton and the judge's), `shared_memory` | 0.0 | abstain |
71
+ | `error` | `crash`, `configs` (the judge's sweep forced a config the autotuner would skip), `compare_dtype` | abstain | abstain |
72
+ | `too_slow` | `timeout` | abstain | abstain |
73
+ | `not_judged` | `races_unchecked`, `not_at_task_shape`, `tool_error`, unsupported operations, others | abstain | abstain |
74
+
75
+ ### Abstention
76
+
77
+ `score is None` means the judge cannot say whose fault the outcome is. It is not a pass and not a
78
+ fail, and `Verdict.feedback` then starts with `(the judge abstains)` and names why. An abstention
79
+ leaves the reward to the other terms: the base reward in gate mode (or zero with
80
+ `on_abstain="zero"`), `abstain_value` in score mode.
81
+
82
+ Some abstentions can be provoked by the answer itself: a launch too large for the race check
83
+ (`races_unchecked`), inline C++ or CUDA (`native_code`), an output dtype the judge cannot compare
84
+ under the v3 rule (`compare_dtype`), a run that does not finish in `timeout_s`. In gate mode with
85
+ `on_abstain="keep"` such an answer keeps its base reward, so a policy that learns to provoke one
86
+ escapes the gate. Count the abstentions per step (the verl adapter logs `ttsem_abstain`), and use
87
+ `on_abstain="zero"` where an abstention should not pay, at the price of zeroing the abstentions
88
+ that are not the answer's doing.
89
+
90
+ ## The API
91
+
92
+ ### `Judge`
93
+
94
+ `Judge(*, cache_dir, python=None, workers=None, mode="gate", tiers=("scale", "medium"),
95
+ contract="operator", rule="kernelbench", precision="fp32", gpu="", timeout_s=120, cap=16,
96
+ on_abstain="keep", abstain_value=None, fail_value=0.0, strict_fs=True, real_max=2**20,
97
+ ttsem_root=None, mem_gb=None)`
98
+
99
+ | argument | meaning |
100
+ |---|---|
101
+ | `cache_dir` | where the tasks, the answers, the reference runs and every pass's record are kept (see "Cache") |
102
+ | `python` | the interpreter of the judge's venv (default: this interpreter); see "The worker" |
103
+ | `workers` | passes the worker runs at once (default: the CPU count) |
104
+ | `mode` | `gate` or `cpu` |
105
+ | `tiers` | `scale` always, then any of `medium`, `real` |
106
+ | `contract` | `operator` (default) or `task`, see "Shapes" |
107
+ | `rule`, `precision`, `gpu` | the judge's own: `kernelbench` (`allclose` at 1e-2) or `v3` (the KernelBench-v3 release's rule); `fp32` casts every tensor input to float32 as KernelBench does, `keep` leaves the task's dtypes; `gpu` names the device whose shared-memory limit applies. `score` takes them per call too |
108
+ | `timeout_s` | wall seconds for each pass's run |
109
+ | `cap` | the block size the race cap uses; `None` turns the cap off |
110
+ | `on_abstain`, `abstain_value`, `fail_value` | see "Gate mode and score mode" |
111
+ | `strict_fs` | the sandbox shows a run only what it needs (the judge's `--strict`); on by default |
112
+ | `real_max` | largest tensor, in elements, the `real` tier will run |
113
+ | `ttsem_root` | a checkout of ttsem the worker imports ahead of the venv's own |
114
+ | `mem_gb` | the address-space cap of every run (`TTSEM_MEM_GB` of the judge) |
115
+
116
+ Methods:
117
+
118
+ - `score(reference_src, answer_src, **overrides) -> Verdict`: one answer against one task's
119
+ reference. `answer_src` may be `None` or empty: that is `error` / `no_code`, scored 0.0 without a
120
+ run. `overrides`: `rule`, `precision`, `gpu`, `format` (`kernelbench` or `tritonbench`),
121
+ `contract`. A repeated `(task, answer)` pair is judged once and read back; a failure of the
122
+ worker itself (it died, a request raised) is an abstention with `kind="tool_error"`, not an
123
+ exception. Two things do raise: an unknown option or setting (`ValueError`), and a machine that
124
+ cannot make the sandbox (`RuntimeError`, see "The sandbox").
125
+ - `score_many(items, **overrides) -> list[Verdict]` for `(reference_src, answer_src)` pairs, and
126
+ `await ascore(...)`, which runs `score` in a thread.
127
+ - `gate(base, verdict) -> float`, above.
128
+ - `close()` ends the worker process. `Judge.from_env(**overrides)` reads the variables listed at the
129
+ end of this page. `judge.info` holds what the worker reported on its first call (versions, the
130
+ judge's root).
131
+
132
+ ### `Verdict`
133
+
134
+ A frozen dataclass.
135
+
136
+ | field | meaning |
137
+ |---|---|
138
+ | `score` | the mode's score: 1.0, 0.0 or `None` |
139
+ | `verdict` | `verified`, `wrong`, `unsafe`, `no_kernel`, `error`, `too_slow` or `not_judged` |
140
+ | `kind` | the finer reason (the table above), or `None` |
141
+ | `feedback` | one paragraph for the model: the judge's own message (kernel, buffer, line, source line, what to check), the sizes it was found at and why those sizes count |
142
+ | `tier` | the tier that decided: `scale`, `scale_cap`, `medium`, `real` (`test`, `test_cap` for TritonBench) |
143
+ | `seconds` | CPU seconds the answer's sandboxed runs cost, the references' excluded |
144
+ | `record` | the judge's full record of every pass, for logs |
145
+
146
+ `Verdict.abstain` is `score is None`; `Verdict.hack` is true for `tampered` and `forged`, which a
147
+ policy that learns to attack the checker produces first: log it per step.
148
+
149
+ ### `extract_answer(completion) -> str | None`
150
+
151
+ The answer's code in a model's completion: the last fenced Python block that defines `ModelNew`,
152
+ else the last fenced Python block, else the whole completion if it is Python that defines
153
+ `ModelNew`; `None` when there is none.
154
+
155
+ ### `scores(verdict, kind)` and `kind_of(record, fmt="kernelbench")`
156
+
157
+ The table above as functions, for code that has the judge's record and not a `Verdict`.
158
+
159
+ ## Shapes: what a verdict at a small size means
160
+
161
+ The judge runs the task's program with every size constant divided by the same power of two, so
162
+ that an interpreter finishes, and the reward builds its tiers on that.
163
+
164
+ 1. `scale`: the largest tensor is shrunk to at most 2**13 elements, one trial, small enough that
165
+ the race check can usually cover every launch.
166
+ 2. `medium`, only after `scale` verified: 2**17 elements. It catches what grows with the shape.
167
+ 3. `real`, optional (`tiers=("scale", "medium", "real")`): the task's own sizes, when its largest
168
+ tensor holds at most `real_max` elements.
169
+
170
+ `contract="operator"` (default) holds an answer to compute the operator at the task's shape and at
171
+ the scaled ones, so a kernel specialised to the benchmark's shape (no bound on a dimension that is
172
+ a multiple of the block there) is rejected, with feedback that says the sizes. That is a choice
173
+ about what to pay for and it changes the reward. `contract="task"` holds it to the task's shape
174
+ only: a rejection at a scaled shape stands only if the task's shape confirms it, and where that
175
+ shape is too large to run the verdict is an abstention (`not_at_task_shape`).
176
+
177
+ ### The cap
178
+
179
+ A kernel launched as one program instance has nothing to race with. When `scale` verifies and the
180
+ answer has a launch whose grid is a ceiling division over a block constant (`cap.py` lists the
181
+ forms it recognises), the pass runs again on a copy whose blocks are capped at `cap`, so that the
182
+ small test launches many instances; only an `unsafe` verdict of that copy counts (`scale_cap`). A
183
+ launch whose autotune configs all list blocks above the cap is not capped, since no config the
184
+ answer's own tuner can pick launches a block that small. The cap is insurance against an optimiser
185
+ that learns a race, not a claim that models write them.
186
+
187
+ ## The verl adapter
188
+
189
+ `ttsem.reward.verl_adapter` is a custom reward function in the shape verl documents: four
190
+ arguments in, a dict with a `score` key out. `ground_truth` is the task's reference module as a
191
+ string. The judge is configured by environment variables ([the list](#environment-variables)), so
192
+ the trainer needs no code of ours.
193
+
194
+ Score mode, the reward being the judge's:
195
+
196
+ ```yaml
197
+ reward:
198
+ custom_reward_function:
199
+ path: /path/to/site-packages/ttsem/reward/verl_adapter.py # python -c "import ttsem.reward.verl_adapter as m; print(m.__file__)"
200
+ name: compute_score # acompute_score under verl's async reward loop
201
+ ```
202
+
203
+ That is the key verl reads from 0.7.1 on (verl 0.9.1 was the latest release when this was
204
+ written); in 0.6 it was a top-level `custom_reward_function`, and a trainer on 0.6 needs the old
205
+ form.
206
+
207
+ Gate mode over a reward of your own, in a file of yours that `custom_reward_function.path` names:
208
+
209
+ ```python
210
+ from ttsem.reward.verl_adapter import gated
211
+ from my_rewards import compute_score as base # the GPU reward, verl's signature
212
+
213
+ compute_score = gated(base) # gated(base, lazy=True): judge only what base accepts
214
+ ```
215
+
216
+ By default the base reward and the judge run at once; with `lazy=True` the judge runs only when the
217
+ base is above `fail_value`, which saves CPU and adds the two latencies.
218
+
219
+ Every call returns `{"score": float, "ttsem_verdict", "ttsem_kind", "ttsem_tier", "ttsem_abstain",
220
+ "ttsem_hack", "ttsem_seconds", "ttsem_feedback"}`, the same keys for a judged and a skipped sample
221
+ (verl stacks them across a batch); gate mode adds `base_score` and keeps the base's own keys.
222
+ `ttsem_feedback` is the paragraph for a multi-turn rollout's next turn. The judge judges one
223
+ sample per call, so use a reward manager that calls the function concurrently.
224
+
225
+ The function signatures and the dict return follow the source of verl 0.9.1; its documentation
226
+ page gives only the signature and an example that returns a number. This repository tests the
227
+ adapter against a fake worker and loads it by path as verl does, not against verl itself: check
228
+ the reward manager and the dict convention on the verl version you pin.
229
+
230
+ Two things to plan for on a real job, read from verl's source and not yet run: verl starts several
231
+ reward workers (`reward.num_workers`, 8 by default), each of which builds its own `Judge`, so the
232
+ real ceiling on concurrent passes is that number times `TTSEM_REWARD_WORKERS`; and the workers are
233
+ spread over the nodes of the job, so ttsem, the judge's venv, `bwrap` and the `TTSEM_REWARD_*`
234
+ variables must be present on each of them.
235
+
236
+ ## The worker: the judge runs in its own venv
237
+
238
+ The judge needs Triton and torch (CPU), pinned in ttsem's `triton` extra, and a trainer usually
239
+ has its own CUDA builds of both. So the trainer's process imports neither: `Judge` starts one
240
+ long-lived worker process, `python -I .../ttsem/reward/worker.py`, with the interpreter you give
241
+ as `python` (`TTSEM_REWARD_PYTHON`), and talks to it in JSON lines over a pipe. The worker hands
242
+ each pass to `ttsem.judge`, which runs the answer and the reference each in a `bwrap` sandbox of
243
+ its own, and sends back the judge's record. A worker that dies fails its pending requests (they
244
+ become abstentions, `tool_error`) and the next call starts another; a forked reward process starts
245
+ its own.
246
+
247
+ Make the judge's venv once, with the pinned wheels:
248
+
249
+ ```console
250
+ $ python -m venv ~/venvs/judge
251
+ $ ~/venvs/judge/bin/pip install --index-url https://download.pytorch.org/whl/cpu \
252
+ --extra-index-url https://pypi.org/simple 'ttsem[triton]'
253
+ $ export TTSEM_REWARD_PYTHON=~/venvs/judge/bin/python
254
+ ```
255
+
256
+ Make it from an interpreter that sits outside your home directory (the system Python, or one whose
257
+ real path is not under `$HOME`). The sandbox hides the home, so a venv whose `python` is a symlink
258
+ into a Python that a tool keeps there does not start inside it; `Judge` checks this on its first
259
+ call and raises instead of letting every pass abstain.
260
+
261
+ The trainer's side needs ttsem and numpy only. Keep the two ttsem installs at one version: the
262
+ worker file is the trainer's own and talks to the `ttsem.judge` of the venv. `ttsem_root`
263
+ (`TTSEM_REWARD_TTSEM_ROOT`) names a checkout the worker imports ahead of the venv's, for working on
264
+ ttsem itself. No new extra is needed: the worker's requirements are the `triton` extra.
265
+
266
+ Each pass costs CPU seconds, not GPU: the sandboxed run imports torch and Triton, runs the
267
+ reference once per task and tier (kept, see below) and runs the answer. `Verdict.seconds` is the
268
+ CPU time of the answer's runs, measured inside the sandbox on the process tree, so it stays honest
269
+ on a machine shared with other work. It is telemetry from the sandbox: an answer could forge it,
270
+ never the verdict.
271
+
272
+ ### Cache
273
+
274
+ `cache_dir` holds the tasks and answers (named by the hash of their text), one reference run per
275
+ task and tier, reused across a GRPO group and across epochs, and every pass's record. It also holds
276
+ every model-written answer the run saw. Put it on disk, not on a RAM-backed `/tmp`, and clear it
277
+ when the run is done. No answer's sandbox sees it, except its own working directory.
278
+
279
+ ## The sandbox
280
+
281
+ The reward never runs an answer without a sandbox and has no switch for it. It needs Linux and
282
+ [`bwrap`](https://github.com/containers/bubblewrap) (bubblewrap) on `PATH`, able to create its
283
+ namespaces: unprivileged user namespaces enabled, or root. The first call asks the worker to make
284
+ one, and raises `RuntimeError` ("the judge cannot run here") instead of judging if it cannot. The
285
+ sandbox is the judge's, described in [docs/JUDGE.md](JUDGE.md#the-sandbox): a read-only filesystem
286
+ (with `strict_fs`, only what the run needs), the home directory and the host's sockets hidden, no
287
+ network, its own pid namespace, no capabilities.
288
+
289
+ Not inside a container whose seccomp profile forbids namespaces, which is Docker's default: `bwrap`
290
+ cannot start there. Run the judge on the host, or start the container with a profile that allows
291
+ them. Switching the sandbox off is not an answer: the answer's process could then overwrite the
292
+ reference's outputs on disk and turn `wrong` into `verified`.
293
+
294
+ ## What it does not catch
295
+
296
+ - **The judge's own caveat applies unchanged.** `wrong` versus `verified` holds against an
297
+ adversarial answer. `unsafe` and `no_kernel` rest on a source scan and a run-time integrity
298
+ check, not on a separation of the answer from the checker, which this release does not have
299
+ ([docs/JUDGE.md](JUDGE.md#what-the-verdicts-guarantee)). Against an optimiser free to write
300
+ anything, they bound the damage (gate mode never raises a reward) and do not prove anything.
301
+ Every `tampered` and `forged` verdict is counted as a hack attempt.
302
+ - **Speed.** Nothing is timed. The reward says a kernel is right and safe, not fast; timing hacks
303
+ are the job of the GPU reward it gates.
304
+ - **The device.** Triton's IR is interpreted on a CPU. Below the LLVM conversion (ptxas, the
305
+ hardware, the order of atomic updates, fast-math, a tensor core's accumulation) nothing runs, so a
306
+ kernel right here can differ on a device, and the comparison tolerance of the rule decides the
307
+ rest.
308
+ - **Sizes that did not run.** A verdict holds for the inputs and sizes that ran. With the default
309
+ tiers the task's own shape is not run; a fault only that shape reaches is missed, and under
310
+ `contract="task"` a kernel wrong only at a scaled shape is let through.
311
+ - **Races in large launches.** Races are looked for between program instances, and past a limit on
312
+ the accesses of one launch they cannot be: such a launch is an abstention (`races_unchecked`),
313
+ not a pass.
314
+ - **Anything outside Triton.** Inline C++ or CUDA is not run (`native_code`), and inline PTX and the
315
+ operations the semantics does not model stop as `unsupported`, by name: abstentions, not
316
+ verdicts.
317
+ - **A compile error under the judge's Triton.** It may be drift between the trainer's Triton and
318
+ the judge's, not the answer's fault: gate mode abstains, and score mode scores 0.0, which is
319
+ where score mode risks rejecting a good answer.
320
+ - **The v3 rule and an output of another dtype.** The judge's comparison calls `torch.allclose`
321
+ under every rule, which raises when the answer's output dtype is not the reference's, where the
322
+ v3 harness casts both to float32 first. Under `rule="v3"` that is an abstention
323
+ (`compare_dtype`), so an answer can return another dtype to escape a gate with
324
+ `on_abstain="keep"`; under `rule="kernelbench"` the error stands, as KernelBench's own comparison
325
+ raises it.
326
+ - **Tasks the precision cast breaks.** `precision="fp32"` casts every tensor input to float32, which
327
+ breaks a reference that takes integer indices (gather, embedding): its run fails and the pass is
328
+ an abstention. Use `precision="keep"` for those.
329
+ - **What it does to training is not shown here.** The rules are tested; whether a policy trained on
330
+ them improves, or learns around them, is not.
331
+
332
+ ## Environment variables
333
+
334
+ `Judge.from_env()` and the verl adapter read:
335
+
336
+ | variable | argument |
337
+ |---|---|
338
+ | `TTSEM_REWARD_CACHE` | `cache_dir` (default `~/.cache/ttsem_reward`) |
339
+ | `TTSEM_REWARD_PYTHON` | `python` |
340
+ | `TTSEM_REWARD_TTSEM_ROOT` | `ttsem_root` |
341
+ | `TTSEM_REWARD_WORKERS` | `workers` |
342
+ | `TTSEM_REWARD_MODE` | `mode` (the adapter sets it per function) |
343
+ | `TTSEM_REWARD_CONTRACT` | `contract` |
344
+ | `TTSEM_REWARD_CAP` | `cap`; `0` turns it off |
345
+ | `TTSEM_REWARD_TIERS` | `tiers`, comma-separated |
346
+ | `TTSEM_REWARD_TIMEOUT` | `timeout_s` |
347
+ | `TTSEM_REWARD_RULE`, `TTSEM_REWARD_PRECISION`, `TTSEM_REWARD_GPU` | `rule`, `precision`, `gpu` |
348
+ | `TTSEM_REWARD_MEM_GB` | `mem_gb` |
349
+
350
+ `on_abstain`, `abstain_value`, `fail_value`, `strict_fs` and `real_max` are not read from the
351
+ environment: pass them to `Judge.from_env(...)`.
352
+
353
+ ## Tests
354
+
355
+ `pytest tests/test_reward*.py` runs the unit tests with no GPU and no Triton, and the integration
356
+ tests where the judge's interpreter has torch and Triton and `bwrap` works (they skip otherwise).
357
+ `tests/test_reward_hygiene.py` fails if the package or this page names a path of the machine it was
358
+ written on or anything that does not belong in public text.