jtc-core 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (298) hide show
  1. jtc_core-0.3.0/PKG-INFO +366 -0
  2. jtc_core-0.3.0/README.md +319 -0
  3. jtc_core-0.3.0/jtc/__init__.py +5 -0
  4. jtc_core-0.3.0/jtc/common/__init__.py +259 -0
  5. jtc_core-0.3.0/jtc/common/asset_store.py +9 -0
  6. jtc_core-0.3.0/jtc/common/chat_completion.py +97 -0
  7. jtc_core-0.3.0/jtc/common/cli.py +151 -0
  8. jtc_core-0.3.0/jtc/common/code_execution.py +443 -0
  9. jtc_core-0.3.0/jtc/common/code_utils.py +252 -0
  10. jtc_core-0.3.0/jtc/common/generation_workflow.py +404 -0
  11. jtc_core-0.3.0/jtc/common/hub.py +26 -0
  12. jtc_core-0.3.0/jtc/common/ifeval/__init__.py +11 -0
  13. jtc_core-0.3.0/jtc/common/ifeval/functions.py +1161 -0
  14. jtc_core-0.3.0/jtc/common/jrecord_sampling.py +201 -0
  15. jtc_core-0.3.0/jtc/common/jrow_regrouping.py +274 -0
  16. jtc_core-0.3.0/jtc/common/jrow_sampling.py +222 -0
  17. jtc_core-0.3.0/jtc/common/parquet.py +500 -0
  18. jtc_core-0.3.0/jtc/common/podman_verifier.py +209 -0
  19. jtc_core-0.3.0/jtc/common/prime_rl_dataset.py +180 -0
  20. jtc_core-0.3.0/jtc/common/process.py +47 -0
  21. jtc_core-0.3.0/jtc/common/rejection_sampling.py +252 -0
  22. jtc_core-0.3.0/jtc/common/resources/__init__.py +1 -0
  23. jtc_core-0.3.0/jtc/common/resources/templates/answer-generation.json +14 -0
  24. jtc_core-0.3.0/jtc/common/resources/templates/articulated_harness_gptoss.json +18 -0
  25. jtc_core-0.3.0/jtc/common/resources/templates/code-check-generation.json +15 -0
  26. jtc_core-0.3.0/jtc/common/resources/templates/code-check-label-generation-gptoss.json +35 -0
  27. jtc_core-0.3.0/jtc/common/resources/templates/extraction-prompt-generation.json +15 -0
  28. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-complete.json +37 -0
  29. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-prompt.json +20 -0
  30. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-python-prompt-gptoss-harmony.json +20 -0
  31. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-repl-prompt-gptoss-harmony.json +19 -0
  32. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-repl-prompt-v4.json +19 -0
  33. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-simple-prompt-json.json +20 -0
  34. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-simple-prompt.json +20 -0
  35. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-terminal-prompt-gptoss-harmony.json +19 -0
  36. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-terminal-release-browse-search-xml-v1.json +56 -0
  37. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-terminal-release-browse-xml-v1.json +36 -0
  38. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-terminal-release-browse.json +36 -0
  39. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-terminal-release.json +19 -0
  40. jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-tool_response.json +31 -0
  41. jtc_core-0.3.0/jtc/common/resources/templates/multistep-final-warning-harmony.json +14 -0
  42. jtc_core-0.3.0/jtc/common/resources/templates/multistep-final-warning.json +13 -0
  43. jtc_core-0.3.0/jtc/common/resources/templates/multistep-tool-message.json +13 -0
  44. jtc_core-0.3.0/jtc/common/resources/templates/recordset-batch-rubric-judge.json +19 -0
  45. jtc_core-0.3.0/jtc/common/resources/templates/rubric-generation-from-rollouts.json +19 -0
  46. jtc_core-0.3.0/jtc/common/resources/templates/rubric-generation-from-unit-tests.json +15 -0
  47. jtc_core-0.3.0/jtc/common/resources/templates/search-agent-visit-judge.json +15 -0
  48. jtc_core-0.3.0/jtc/common/resources/templates/search-agent-visit.json +15 -0
  49. jtc_core-0.3.0/jtc/common/resources/templates/search-agent-webterminal-10k.json +17 -0
  50. jtc_core-0.3.0/jtc/common/resources/templates/search-agent.json +17 -0
  51. jtc_core-0.3.0/jtc/common/resources/templates/tmax-podman-trace-submit-binary-gptoss.v2.json +13 -0
  52. jtc_core-0.3.0/jtc/common/resources/templates/tmax-podman-trace-submit-gptoss.json +41 -0
  53. jtc_core-0.3.0/jtc/common/resources/templates/wildguard-instance-rubric-generation.json +21 -0
  54. jtc_core-0.3.0/jtc/common/rubric_execution.py +179 -0
  55. jtc_core-0.3.0/jtc/common/schema.py +455 -0
  56. jtc_core-0.3.0/jtc/common/scoring.py +62 -0
  57. jtc_core-0.3.0/jtc/common/sources/__init__.py +202 -0
  58. jtc_core-0.3.0/jtc/common/sources/base.py +27 -0
  59. jtc_core-0.3.0/jtc/common/sources/coding.py +234 -0
  60. jtc_core-0.3.0/jtc/common/sources/coding_transform.py +53 -0
  61. jtc_core-0.3.0/jtc/common/sources/common.py +236 -0
  62. jtc_core-0.3.0/jtc/common/sources/deepresearch_bench.py +103 -0
  63. jtc_core-0.3.0/jtc/common/sources/drtulu.py +107 -0
  64. jtc_core-0.3.0/jtc/common/sources/drtulu_code_checks.py +165 -0
  65. jtc_core-0.3.0/jtc/common/sources/gpqa.py +146 -0
  66. jtc_core-0.3.0/jtc/common/sources/healthbench.py +113 -0
  67. jtc_core-0.3.0/jtc/common/sources/ifeval.py +151 -0
  68. jtc_core-0.3.0/jtc/common/sources/ifeval_transform.py +54 -0
  69. jtc_core-0.3.0/jtc/common/sources/liveresearchbench.py +96 -0
  70. jtc_core-0.3.0/jtc/common/sources/llmeval_med.py +267 -0
  71. jtc_core-0.3.0/jtc/common/sources/medqa_usmle.py +121 -0
  72. jtc_core-0.3.0/jtc/common/sources/mind2web2.py +145 -0
  73. jtc_core-0.3.0/jtc/common/sources/mmlu_pro.py +118 -0
  74. jtc_core-0.3.0/jtc/common/sources/monaco.py +334 -0
  75. jtc_core-0.3.0/jtc/common/sources/multiple_choice.py +108 -0
  76. jtc_core-0.3.0/jtc/common/sources/natural.py +90 -0
  77. jtc_core-0.3.0/jtc/common/sources/natural_synth.py +186 -0
  78. jtc_core-0.3.0/jtc/common/sources/openrubrics.py +99 -0
  79. jtc_core-0.3.0/jtc/common/sources/prbench.py +153 -0
  80. jtc_core-0.3.0/jtc/common/sources/profbench.py +131 -0
  81. jtc_core-0.3.0/jtc/common/sources/prometheus.py +65 -0
  82. jtc_core-0.3.0/jtc/common/sources/quest.py +336 -0
  83. jtc_core-0.3.0/jtc/common/sources/researchqa.py +405 -0
  84. jtc_core-0.3.0/jtc/common/sources/rubricbench.py +111 -0
  85. jtc_core-0.3.0/jtc/common/sources/ruft26k.py +140 -0
  86. jtc_core-0.3.0/jtc/common/sources/ruft26k_answer_generations.py +302 -0
  87. jtc_core-0.3.0/jtc/common/sources/ruft26k_code_checks.py +156 -0
  88. jtc_core-0.3.0/jtc/common/sources/science_qwen35_judgments.py +80 -0
  89. jtc_core-0.3.0/jtc/common/sources/tbd.py +29 -0
  90. jtc_core-0.3.0/jtc/common/sources/wildbench.py +271 -0
  91. jtc_core-0.3.0/jtc/common/submission.py +19 -0
  92. jtc_core-0.3.0/jtc/common/submission_display.py +28 -0
  93. jtc_core-0.3.0/jtc/common/templates.py +158 -0
  94. jtc_core-0.3.0/jtc/common/tmax_replay.py +157 -0
  95. jtc_core-0.3.0/jtc/common/tool_clients.py +74 -0
  96. jtc_core-0.3.0/jtc/common/tool_specs.py +111 -0
  97. jtc_core-0.3.0/jtc/common/trace_inspection.py +69 -0
  98. jtc_core-0.3.0/jtc/common/utils.py +80 -0
  99. jtc_core-0.3.0/jtc/common/workflows/__init__.py +9 -0
  100. jtc_core-0.3.0/jtc/common/workflows/__main__.py +3 -0
  101. jtc_core-0.3.0/jtc/common/workflows/answer_generation_workflow.py +499 -0
  102. jtc_core-0.3.0/jtc/common/workflows/cli.py +32 -0
  103. jtc_core-0.3.0/jtc/common/workflows/code_addressability_workflow.py +186 -0
  104. jtc_core-0.3.0/jtc/common/workflows/code_check_generation_workflow.py +912 -0
  105. jtc_core-0.3.0/jtc/common/workflows/code_check_label_generation_workflow.py +403 -0
  106. jtc_core-0.3.0/jtc/common/workflows/extraction_prompt_generation_workflow.py +314 -0
  107. jtc_core-0.3.0/jtc/common/workflows/inference_workflow.py +1431 -0
  108. jtc_core-0.3.0/jtc/common/workflows/jrecord_batch_rubric_judgment_workflow.py +231 -0
  109. jtc_core-0.3.0/jtc/common/workflows/logprob_workflow.py +375 -0
  110. jtc_core-0.3.0/jtc/common/workflows/multi_step_generation_workflow.py +654 -0
  111. jtc_core-0.3.0/jtc/common/workflows/multi_step_workflow.py +810 -0
  112. jtc_core-0.3.0/jtc/common/workflows/process_message_workers.py +156 -0
  113. jtc_core-0.3.0/jtc/common/workflows/registry.py +56 -0
  114. jtc_core-0.3.0/jtc/common/workflows/science_bundles_batch_rubric_judgment_workflow.py +438 -0
  115. jtc_core-0.3.0/jtc/common/workflows/science_bundles_rubric_generation_workflow.py +376 -0
  116. jtc_core-0.3.0/jtc/common/workflows/search_agent_rollout_workflow.py +891 -0
  117. jtc_core-0.3.0/jtc/common/workflows/tmax_unit_test_rubric_generation_workflow.py +86 -0
  118. jtc_core-0.3.0/jtc/common/workflows/tmax_unit_test_rubric_generation_workflow_v2.py +121 -0
  119. jtc_core-0.3.0/jtc/common/workflows/tmax_unit_test_rubric_generation_workflow_v3.py +50 -0
  120. jtc_core-0.3.0/jtc/common/workflows/tool_use_generation_workflow.py +478 -0
  121. jtc_core-0.3.0/jtc/common/workflows/tool_use_workflow.py +955 -0
  122. jtc_core-0.3.0/jtc/common/workflows/verifier_articulated_grouped_submit_smoke.py +92 -0
  123. jtc_core-0.3.0/jtc/common/workflows/verifier_articulated_submit_tool_use_workflow.py +37 -0
  124. jtc_core-0.3.0/jtc/common/workflows/verifier_articulated_submit_tool_use_workflow_v2.py +37 -0
  125. jtc_core-0.3.0/jtc/common/workflows/verifier_submit_tool_use_workflow.py +164 -0
  126. jtc_core-0.3.0/jtc/common/workflows/verifier_tmax_podman_binary_submit_workflow.py +59 -0
  127. jtc_core-0.3.0/jtc/common/workflows/verifier_tmax_podman_binary_submit_workflow_v3.py +36 -0
  128. jtc_core-0.3.0/jtc/common/workflows/verifier_tmax_podman_submit_workflow.py +330 -0
  129. jtc_core-0.3.0/jtc/common/workflows/verifier_tmax_podman_submit_workflow_v3.py +47 -0
  130. jtc_core-0.3.0/jtc/common/workflows/verifier_tool_use_workflow.py +985 -0
  131. jtc_core-0.3.0/jtc/common/workflows/wildguard_rubric_generation_workflow.py +314 -0
  132. jtc_core-0.3.0/jtc/datadev/__init__.py +24 -0
  133. jtc_core-0.3.0/jtc/datadev/cli.py +220 -0
  134. jtc_core-0.3.0/jtc/datadev/data/__init__.py +29 -0
  135. jtc_core-0.3.0/jtc/datadev/data/materialize.py +1466 -0
  136. jtc_core-0.3.0/jtc/datadev/data/recipes.py +311 -0
  137. jtc_core-0.3.0/jtc/datadev/infra/__init__.py +3 -0
  138. jtc_core-0.3.0/jtc/datadev/infra/bm25_server.py +358 -0
  139. jtc_core-0.3.0/jtc/datadev/literegistry/__init__.py +1 -0
  140. jtc_core-0.3.0/jtc/datadev/literegistry/gateway.py +430 -0
  141. jtc_core-0.3.0/jtc/datadev/literegistry/jina_reader.py +222 -0
  142. jtc_core-0.3.0/jtc/datadev/opt/__init__.py +5 -0
  143. jtc_core-0.3.0/jtc/datadev/opt/weights.py +245 -0
  144. jtc_core-0.3.0/jtc/datadev/pot_sampling.py +303 -0
  145. jtc_core-0.3.0/jtc/datadev/pots.py +238 -0
  146. jtc_core-0.3.0/jtc/datadev/raw_token_prior.py +364 -0
  147. jtc_core-0.3.0/jtc/datadev/semaphore/__init__.py +23 -0
  148. jtc_core-0.3.0/jtc/datadev/semaphore/__main__.py +199 -0
  149. jtc_core-0.3.0/jtc/datadev/semaphore/core.py +825 -0
  150. jtc_core-0.3.0/jtc/datadev/sft_pot_batches.py +444 -0
  151. jtc_core-0.3.0/jtc/datadev/token_counts.py +41 -0
  152. jtc_core-0.3.0/jtc/datadev/training_config.py +560 -0
  153. jtc_core-0.3.0/jtc/datadev/workflows.py +88 -0
  154. jtc_core-0.3.0/jtc/eval/__init__.py +31 -0
  155. jtc_core-0.3.0/jtc/eval/__main__.py +5 -0
  156. jtc_core-0.3.0/jtc/eval/benchmark.py +264 -0
  157. jtc_core-0.3.0/jtc/eval/bon_config.py +242 -0
  158. jtc_core-0.3.0/jtc/eval/bon_plot.py +325 -0
  159. jtc_core-0.3.0/jtc/eval/cli.py +53 -0
  160. jtc_core-0.3.0/jtc/eval/dashboard.py +18 -0
  161. jtc_core-0.3.0/jtc/eval/eval.py +569 -0
  162. jtc_core-0.3.0/jtc/eval/evalboard/__init__.py +5 -0
  163. jtc_core-0.3.0/jtc/eval/evalboard/__main__.py +50 -0
  164. jtc_core-0.3.0/jtc/eval/evalboard/export.py +67 -0
  165. jtc_core-0.3.0/jtc/eval/evalboard/site/.openai/hosting.json +5 -0
  166. jtc_core-0.3.0/jtc/eval/evalboard/site/README.md +56 -0
  167. jtc_core-0.3.0/jtc/eval/evalboard/site/app/bon/bon.module.css +108 -0
  168. jtc_core-0.3.0/jtc/eval/evalboard/site/app/bon/page.tsx +234 -0
  169. jtc_core-0.3.0/jtc/eval/evalboard/site/app/chatgpt-auth.ts +90 -0
  170. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.css +56 -0
  171. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v10.css +14 -0
  172. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v11.css +12 -0
  173. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v12.css +12 -0
  174. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v13.css +49 -0
  175. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v14.css +3 -0
  176. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v15.css +19 -0
  177. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v16.css +18 -0
  178. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v17.css +20 -0
  179. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v18.css +7 -0
  180. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v19.css +17 -0
  181. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v20.css +42 -0
  182. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v6.css +2 -0
  183. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v7.css +15 -0
  184. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v8.css +15 -0
  185. jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v9.css +29 -0
  186. jtc_core-0.3.0/jtc/eval/evalboard/site/app/layout.tsx +11 -0
  187. jtc_core-0.3.0/jtc/eval/evalboard/site/app/page.tsx +12 -0
  188. jtc_core-0.3.0/jtc/eval/evalboard/site/build/sites-vite-plugin.ts +45 -0
  189. jtc_core-0.3.0/jtc/eval/evalboard/site/db/index.ts +13 -0
  190. jtc_core-0.3.0/jtc/eval/evalboard/site/db/schema.ts +4 -0
  191. jtc_core-0.3.0/jtc/eval/evalboard/site/drizzle.config.ts +7 -0
  192. jtc_core-0.3.0/jtc/eval/evalboard/site/eslint.config.mjs +41 -0
  193. jtc_core-0.3.0/jtc/eval/evalboard/site/next-env.d.ts +5 -0
  194. jtc_core-0.3.0/jtc/eval/evalboard/site/next.config.ts +7 -0
  195. jtc_core-0.3.0/jtc/eval/evalboard/site/package-lock.json +10279 -0
  196. jtc_core-0.3.0/jtc/eval/evalboard/site/package.json +47 -0
  197. jtc_core-0.3.0/jtc/eval/evalboard/site/postcss.config.mjs +7 -0
  198. jtc_core-0.3.0/jtc/eval/evalboard/site/public/favicon.svg +6 -0
  199. jtc_core-0.3.0/jtc/eval/evalboard/site/public/file.svg +1 -0
  200. jtc_core-0.3.0/jtc/eval/evalboard/site/public/globe.svg +1 -0
  201. jtc_core-0.3.0/jtc/eval/evalboard/site/public/og.png +0 -0
  202. jtc_core-0.3.0/jtc/eval/evalboard/site/public/window.svg +1 -0
  203. jtc_core-0.3.0/jtc/eval/evalboard/site/tests/rendered-html.test.mjs +91 -0
  204. jtc_core-0.3.0/jtc/eval/evalboard/site/tsconfig.json +29 -0
  205. jtc_core-0.3.0/jtc/eval/evalboard/site/vite.config.ts +59 -0
  206. jtc_core-0.3.0/jtc/eval/evalboard/site/worker/index.ts +47 -0
  207. jtc_core-0.3.0/jtc/eval/inference_scaling.py +203 -0
  208. jtc_core-0.3.0/jtc/eval/jina_reader.py +222 -0
  209. jtc_core-0.3.0/jtc/eval/launcher.py +54 -0
  210. jtc_core-0.3.0/jtc/eval/outcomes.py +262 -0
  211. jtc_core-0.3.0/jtc/eval/plot.py +159 -0
  212. jtc_core-0.3.0/jtc/eval/reconstruct_rubricbench_jrecords.py +578 -0
  213. jtc_core-0.3.0/jtc/eval/rejection_sampling.py +61 -0
  214. jtc_core-0.3.0/jtc/eval/rejection_sampling_worker.py +177 -0
  215. jtc_core-0.3.0/jtc/eval/rubric_judge_audit.py +224 -0
  216. jtc_core-0.3.0/jtc/eval/rubric_judge_audit_worker.py +145 -0
  217. jtc_core-0.3.0/jtc/eval/search_agent.py +63 -0
  218. jtc_core-0.3.0/jtc/eval/search_worker.py +75 -0
  219. jtc_core-0.3.0/jtc/eval/trace_rubric_recovery.py +298 -0
  220. jtc_core-0.3.0/jtc/eval/trace_rubric_sampling.py +594 -0
  221. jtc_core-0.3.0/jtc/eval/worker.py +52 -0
  222. jtc_core-0.3.0/jtc/eval/workflow.py +37 -0
  223. jtc_core-0.3.0/jtc/eval/workflow_worker.py +104 -0
  224. jtc_core-0.3.0/jtc_core.egg-info/PKG-INFO +366 -0
  225. jtc_core-0.3.0/jtc_core.egg-info/SOURCES.txt +296 -0
  226. jtc_core-0.3.0/jtc_core.egg-info/dependency_links.txt +1 -0
  227. jtc_core-0.3.0/jtc_core.egg-info/entry_points.txt +8 -0
  228. jtc_core-0.3.0/jtc_core.egg-info/requires.txt +42 -0
  229. jtc_core-0.3.0/jtc_core.egg-info/top_level.txt +1 -0
  230. jtc_core-0.3.0/pyproject.toml +71 -0
  231. jtc_core-0.3.0/setup.cfg +4 -0
  232. jtc_core-0.3.0/setup.py +4 -0
  233. jtc_core-0.3.0/tests/test_aggagent_rollouts_jrecord_sets.py +67 -0
  234. jtc_core-0.3.0/tests/test_articulated_harness_gptoss_template.py +54 -0
  235. jtc_core-0.3.0/tests/test_asset_store.py +64 -0
  236. jtc_core-0.3.0/tests/test_asset_store_core.py +122 -0
  237. jtc_core-0.3.0/tests/test_audit_judge_trace_terminal_errors.py +153 -0
  238. jtc_core-0.3.0/tests/test_audit_judge_trace_terminal_usage.py +141 -0
  239. jtc_core-0.3.0/tests/test_audit_judge_trace_tool_domains.py +30 -0
  240. jtc_core-0.3.0/tests/test_bake_all_aggagent_rollouts.py +11 -0
  241. jtc_core-0.3.0/tests/test_batch_rubric_chunked_repairs.py +55 -0
  242. jtc_core-0.3.0/tests/test_batch_rubric_json_repairs.py +72 -0
  243. jtc_core-0.3.0/tests/test_bm25_server.py +31 -0
  244. jtc_core-0.3.0/tests/test_browsecomp_rollouts_jrecord_sets.py +163 -0
  245. jtc_core-0.3.0/tests/test_browsecomp_rubric_generation_template.py +97 -0
  246. jtc_core-0.3.0/tests/test_build_tmax_question_unit_tests.py +97 -0
  247. jtc_core-0.3.0/tests/test_chat_completion_boundary.py +63 -0
  248. jtc_core-0.3.0/tests/test_code_server.py +156 -0
  249. jtc_core-0.3.0/tests/test_compare_judge_trace_terminal_usage.py +73 -0
  250. jtc_core-0.3.0/tests/test_config_layout.py +125 -0
  251. jtc_core-0.3.0/tests/test_datadev.py +77 -0
  252. jtc_core-0.3.0/tests/test_dataset.py +55 -0
  253. jtc_core-0.3.0/tests/test_execution.py +208 -0
  254. jtc_core-0.3.0/tests/test_export_mercor_eval_traces.py +118 -0
  255. jtc_core-0.3.0/tests/test_extract_tmax_rollout_commands.py +228 -0
  256. jtc_core-0.3.0/tests/test_jrecord_sampling.py +62 -0
  257. jtc_core-0.3.0/tests/test_jrecord_set_schema.py +44 -0
  258. jtc_core-0.3.0/tests/test_jrow_regrouping.py +85 -0
  259. jtc_core-0.3.0/tests/test_jrow_sampling.py +92 -0
  260. jtc_core-0.3.0/tests/test_judge_aggagent_researchrubrics_parquet.py +13 -0
  261. jtc_core-0.3.0/tests/test_judge_client.py +59 -0
  262. jtc_core-0.3.0/tests/test_judge_trace_hint_templates.py +59 -0
  263. jtc_core-0.3.0/tests/test_judge_trace_terminal_hint_catalog.py +36 -0
  264. jtc_core-0.3.0/tests/test_literegistry_tool_client_boundary.py +33 -0
  265. jtc_core-0.3.0/tests/test_llmeval_med_source.py +171 -0
  266. jtc_core-0.3.0/tests/test_load_jsonl.py +22 -0
  267. jtc_core-0.3.0/tests/test_load_parquet.py +24 -0
  268. jtc_core-0.3.0/tests/test_loading.py +81 -0
  269. jtc_core-0.3.0/tests/test_model.py +137 -0
  270. jtc_core-0.3.0/tests/test_monaco_source.py +87 -0
  271. jtc_core-0.3.0/tests/test_multi_step_workflow.py +394 -0
  272. jtc_core-0.3.0/tests/test_multiple_choice_benchmark_sources.py +86 -0
  273. jtc_core-0.3.0/tests/test_multistep_rubric_schema.py +56 -0
  274. jtc_core-0.3.0/tests/test_openai_tool_client.py +89 -0
  275. jtc_core-0.3.0/tests/test_packaging.py +173 -0
  276. jtc_core-0.3.0/tests/test_plot_judge_trace_terminal_usage_comparison.py +82 -0
  277. jtc_core-0.3.0/tests/test_plot_judge_trace_terminal_usage_multimodel.py +56 -0
  278. jtc_core-0.3.0/tests/test_podman_verifier.py +163 -0
  279. jtc_core-0.3.0/tests/test_prime_rl_dataset.py +85 -0
  280. jtc_core-0.3.0/tests/test_quest_source.py +135 -0
  281. jtc_core-0.3.0/tests/test_research_benchmark_sources.py +99 -0
  282. jtc_core-0.3.0/tests/test_researchqa_answer_generations.py +94 -0
  283. jtc_core-0.3.0/tests/test_safety_rubric_trajectories_jrecords.py +175 -0
  284. jtc_core-0.3.0/tests/test_science_bundles_batch_rubric_judgment_workflow.py +112 -0
  285. jtc_core-0.3.0/tests/test_search_agent_rollout_workflow.py +99 -0
  286. jtc_core-0.3.0/tests/test_search_client.py +860 -0
  287. jtc_core-0.3.0/tests/test_snippet.py +45 -0
  288. jtc_core-0.3.0/tests/test_split_safety_rubric_trajectories_jrecords.py +95 -0
  289. jtc_core-0.3.0/tests/test_submission.py +93 -0
  290. jtc_core-0.3.0/tests/test_terminal.py +3 -0
  291. jtc_core-0.3.0/tests/test_terminal_release_browse_xml_v1.py +93 -0
  292. jtc_core-0.3.0/tests/test_tmax_podman_fastforward.py +109 -0
  293. jtc_core-0.3.0/tests/test_tmax_podman_fastforward_smoke.py +57 -0
  294. jtc_core-0.3.0/tests/test_tmax_replay.py +277 -0
  295. jtc_core-0.3.0/tests/test_tmax_rubric_generation_templates.py +51 -0
  296. jtc_core-0.3.0/tests/test_trace_terminal_template.py +47 -0
  297. jtc_core-0.3.0/tests/test_verifier_submit_tool_use_workflow.py +76 -0
  298. jtc_core-0.3.0/tests/test_verifier_tool_use_workflow.py +110 -0
@@ -0,0 +1,366 @@
1
+ Metadata-Version: 2.4
2
+ Name: jtc-core
3
+ Version: 0.3.0
4
+ Summary: Unified JTC data, configuration, evaluation, and workflow toolkit.
5
+ Project-URL: Homepage, https://github.com/goncalorafaria/jtc
6
+ Project-URL: Repository, https://github.com/goncalorafaria/jtc
7
+ Project-URL: Issues, https://github.com/goncalorafaria/jtc/issues
8
+ Requires-Python: ~=3.12.0
9
+ Description-Content-Type: text/markdown
10
+ Requires-Dist: aiohttp
11
+ Requires-Dist: cvxpy
12
+ Requires-Dist: datasets
13
+ Requires-Dist: fastapi
14
+ Requires-Dist: fire
15
+ Requires-Dist: httpx
16
+ Requires-Dist: litellm
17
+ Requires-Dist: literegistry-tool-client==0.1.0
18
+ Requires-Dist: literegistry==1.0.48
19
+ Requires-Dist: PyYAML>=6
20
+ Requires-Dist: tqdm
21
+ Requires-Dist: markdown
22
+ Requires-Dist: matplotlib
23
+ Requires-Dist: jtcflow==0.1.0
24
+ Requires-Dist: scipy
25
+ Requires-Dist: transformers
26
+ Requires-Dist: uvicorn
27
+ Requires-Dist: wandb
28
+ Requires-Dist: msgpack
29
+ Requires-Dist: numpy
30
+ Requires-Dist: pydantic
31
+ Provides-Extra: search
32
+ Requires-Dist: pyserini; extra == "search"
33
+ Provides-Extra: trace
34
+ Requires-Dist: rltracer<0.2,>=0.1.0; extra == "trace"
35
+ Provides-Extra: evals
36
+ Requires-Dist: openai; extra == "evals"
37
+ Requires-Dist: pyarrow; extra == "evals"
38
+ Requires-Dist: rltracer<0.2,>=0.1.0; extra == "evals"
39
+ Provides-Extra: harness
40
+ Requires-Dist: verifiers[harbor]==0.2.1; extra == "harness"
41
+ Provides-Extra: all
42
+ Requires-Dist: pyserini; extra == "all"
43
+ Requires-Dist: openai; extra == "all"
44
+ Requires-Dist: pyarrow; extra == "all"
45
+ Requires-Dist: rltracer<0.2,>=0.1.0; extra == "all"
46
+ Requires-Dist: verifiers[harbor]==0.2.1; extra == "all"
47
+
48
+ # JTC
49
+
50
+ `jtc` is one Python distribution with one import namespace and three
51
+ cooperating subpackages:
52
+
53
+ ```text
54
+ jtc
55
+ ├── common # shared data models, IO, sources, templates, and workflows
56
+ ├── eval # scheduler-free evaluation workers, outcomes, and reporting
57
+ └── datadev # data recipes, materialization, mixtures, and training configs
58
+ ```
59
+
60
+ These are not independent packages. Installing `jtc` installs all three under
61
+ the same namespace. The historical top-level imports `jtc_data_commons`,
62
+ `jtceval`, and `datadev` are no longer part of the package API.
63
+
64
+ JTC implements data and evaluation work. It does not own training or
65
+ evaluation infrastructure. PrimeBeaker loads JTC's Python workers, creates the
66
+ LiteRegistry service topology, and schedules the jobs. RL trace inspection and
67
+ export come from the standalone
68
+ [`rltracer`](https://github.com/goncalorafaria/rltracer) package.
69
+ Checkpoint and evaluation provenance indexing comes from
70
+ `primebeaker watcher`.
71
+
72
+ ## Install
73
+
74
+ Install the released `jtc-core` distribution (Python imports and commands remain `jtc`):
75
+
76
+ ```bash
77
+ pip install jtc-core
78
+ ```
79
+
80
+ JTC requires Python 3.12.
81
+
82
+ For development from this checkout:
83
+
84
+ ```bash
85
+ pip install -e .
86
+ ```
87
+
88
+ Install every optional JTC stack for development with:
89
+
90
+ ```bash
91
+ pip install '.[all]'
92
+ ```
93
+
94
+ Available extras are:
95
+
96
+ - `evals`: OpenAI, PyArrow, and RLTracer support used by evaluation tooling.
97
+ - `harness`: Verifiers/Harbor support used by verifier workflows.
98
+ - `search`: Pyserini for local-search workloads.
99
+ - `trace`: RLTracer without the other optional stacks.
100
+ - `all`: all optional JTC dependencies.
101
+
102
+ PrimeBeaker is intentionally not a JTC dependency. A runtime that schedules
103
+ JTC work installs PrimeBeaker separately; the supported evaluation image pins
104
+ and installs released versions of both packages from PyPI.
105
+
106
+ ## Package boundaries
107
+
108
+ - `jtc.common`: schemas, parquet IO, rubric execution, source adapters,
109
+ shared tool integration, prompt templates, and registered Python workflows.
110
+ - `jtc.eval`: evaluation workload contracts and workers, outcome archives,
111
+ audits, plots, and Evalboard export.
112
+ - `jtc.datadev`: typed SFT/RL data recipes, materialization, Pot sampling and
113
+ optimization, and training-TOML generation.
114
+
115
+ Verifier environments and reward functions live in
116
+ `primebeaker.environments`. Scheduling, dynamic ports, LiteRegistry deployment,
117
+ readiness, supervision, resume, and shutdown coordination also live in
118
+ PrimeBeaker. Runtime records, datasets, checkpoints, and evaluation results
119
+ stay outside this repository.
120
+
121
+ ## Data and training-configuration flow
122
+
123
+ `jtc.datadev` turns versioned YAML data recipes into materialized SFT or RL
124
+ datasets and then renders a compatible training TOML. It does not launch
125
+ training:
126
+
127
+ ```text
128
+ data recipe YAML -> materialized dataset + manifest -> training TOML -> PrimeBeaker
129
+ ```
130
+
131
+ The CLI is implemented with Fire:
132
+
133
+ ```bash
134
+ # Validate without writing data.
135
+ datadev data sft examples/datadev/sft-data-smoke.yaml --dry-run
136
+
137
+ # Materialize data and render the corresponding training TOML.
138
+ datadev prepare rl \
139
+ examples/datadev/rl-data-smoke.yaml \
140
+ examples/datadev/rl.toml \
141
+ /weka/gfaria/generated/example-rl.toml
142
+ ```
143
+
144
+ Canonical recipe configs live under `examples/configs/data/{sft,rl}` and
145
+ training configs under `examples/configs/train/{sft,rl}`. The small files under
146
+ `examples/datadev` demonstrate the formats. Generated datasets and manifests
147
+ must use an external artifact directory; the resulting TOML is handed to
148
+ PrimeBeaker for SFT or RL scheduling.
149
+
150
+ The `jtc` command exposes common parquet inspection and sampling operations:
151
+
152
+ ```bash
153
+ jtc inspect /weka/gfaria/records/example.parquet
154
+ jtc sample-jrows INPUT.parquet OUTPUT.parquet --max-input-objects=100
155
+ ```
156
+
157
+ In Python, import through the unified namespace:
158
+
159
+ ```python
160
+ from jtc.common.schema import JRecord, JRow
161
+ from jtc.datadev import Pot, build_data
162
+ from jtc.eval import EvaluationArchive, EvaluationWorkloadConfig
163
+ ```
164
+
165
+ ## Python-only evaluation workloads
166
+
167
+ `jtc.eval` owns benchmark selection and evaluation logic. PrimeBeaker owns the
168
+ YAML descriptions and all infrastructure lifecycle behavior: Beaker scheduling,
169
+ native LiteRegistry deployment, endpoint readiness, signal forwarding, and
170
+ cleanup. The dependency direction is PrimeBeaker -> JTC.
171
+
172
+ The evaluator always runs in a Beaker image containing released PyPI wheels;
173
+ JTC source is never imported or copied into the image. Build and publish the
174
+ pinned image using JTC's `docker/Dockerfile.eval`, then put its immutable
175
+ `beaker://...` URI in the PrimeBeaker-owned eval YAML or pass it with `--image`:
176
+
177
+ ```yaml
178
+ image: beaker://IMMUTABLE_PRIMEBEAKER_JTC_IMAGE
179
+ workspace: ai2/oe-agents-holmes
180
+ budget: ai2/oe-omai
181
+ evaluation:
182
+ model: /weka/gfaria/models/policy/step_400
183
+ run_dir: /weka/gfaria/jtc-artifacts/evals/policy-step-400-smoke
184
+ datasets: [healthbench]
185
+ max_rows: 4
186
+ runs: 1
187
+ verifier_tools: terminal
188
+ verifier_prompt_chat_template_path: templates/jtc-io-terminal-release.json
189
+ ```
190
+
191
+ Lifecycle YAMLs live in PrimeBeaker under `examples/configs/eval`. Preview and
192
+ submit through PrimeBeaker's Fire CLI:
193
+
194
+ ```bash
195
+ primebeaker evaluation preview --config=examples/configs/eval/python_only_smoke.yaml
196
+ primebeaker evaluation submit --config=examples/configs/eval/python_only_smoke.yaml
197
+ ```
198
+
199
+ PrimeBeaker selects a packaged Python entry point for each workload:
200
+
201
+ - standard benchmark evaluation: `jtc.eval.worker`
202
+ - search-agent evaluation: `jtc.eval.search_worker`
203
+ - any registered JTC workflow: `jtc.eval.workflow_worker`
204
+ - iterative rejection sampling: `jtc.eval.rejection_sampling_worker`
205
+ - rubric-judge audit: `jtc.eval.rubric_judge_audit_worker`
206
+
207
+ The generic workflow worker accepts every stable name exported by
208
+ `jtc.common.workflows.registry`. This includes articulated/grouped submit,
209
+ TMAX/Podman and binary-submit workflows, inference, rubric generation, and the
210
+ other verifier workflows in the registry.
211
+
212
+ JTC validates the selected workflow and executes its Python `main()` while
213
+ PrimeBeaker injects managed registry and gateway endpoints. A newly registered
214
+ workflow therefore uses the existing generic worker rather than a new scheduler
215
+ module.
216
+
217
+ For rejection sampling, PrimeBeaker owns the rounds, service topology, resume,
218
+ and cleanup lifecycle. The JTC worker runs the selected verifier workflow; the
219
+ canonical filtering and accepted-trajectory compilation algorithms live in
220
+ `jtc.common.rejection_sampling`. No launcher or fixed-port infrastructure lives
221
+ in JTC.
222
+
223
+ Templates are runtime data, not image code. A PrimeBeaker YAML can use a newly
224
+ created JSON template by YAML-relative, checkout-relative, or absolute path.
225
+ PrimeBeaker validates it on the submitting machine and writes a content-addressed
226
+ snapshot under `<run_dir>/.jtceval-inputs/templates/` before it submits the
227
+ evaluator. The `/weka` mount exposes that snapshot to the PyPI-only evaluation
228
+ image without rebuilding JTC. A dry run reports the future path without writing.
229
+
230
+ Rubric-judge audits use the same boundary. JTC implements Python sampling and
231
+ judgment recovery while PrimeBeaker schedules the shadow model, registry, and
232
+ gateway. The `jtc.eval.rubric_judge_audit_worker` calls those implementations
233
+ directly:
234
+
235
+ ```bash
236
+ primebeaker evaluation preview \
237
+ --config=examples/configs/eval/rubric_judge_audit/qwen35_4b_grouped_vs_qwen36_35b_steps005_400.yaml
238
+ primebeaker evaluation submit \
239
+ --config=examples/configs/eval/rubric_judge_audit/qwen35_4b_grouped_vs_qwen36_35b_steps005_400.yaml
240
+ ```
241
+
242
+ The coordinator can resume from a complete sample or response stream and rejects
243
+ partial artifacts unless `--force=True` is explicitly supplied.
244
+
245
+ Prime-RL TOMLs point directly at PrimeBeaker's package-qualified environment
246
+ modules:
247
+
248
+ ```toml
249
+ [orchestrator.train.source.legacy]
250
+ id = "primebeaker.environments.jtc_tool_label_env"
251
+ ```
252
+
253
+ Prime-RL loads the dotted module path directly. No environment implementation
254
+ is copied into JTC, and there are no JTC-owned harness aliases; callers use the
255
+ PrimeBeaker module path or its environment registry.
256
+
257
+ ## Discover and run workflows
258
+
259
+ ```bash
260
+ jtc-workflow list
261
+ ```
262
+
263
+ Programmatically:
264
+
265
+ ```python
266
+ from jtc.common.workflows import load_workflow
267
+
268
+ workflow = load_workflow("verifier-tool-use")
269
+ workflow.main(...)
270
+ ```
271
+
272
+ Only import-tested workflows are registered. Broken historical variants are
273
+ kept in the repository for provenance but excluded from the wheel registry.
274
+
275
+ ## Local evaluation utilities
276
+
277
+ The `jtceval` CLI does not schedule work. It imports and summarizes existing
278
+ evaluation outputs; lifecycle operations stay in PrimeBeaker:
279
+
280
+ ```bash
281
+ jtceval import-evaluations SOURCE_DIRECTORY OUTPUT.json --dry-run
282
+ ```
283
+
284
+ The distribution also installs these evaluation-side entry points:
285
+
286
+ - `jtceval-evalboard`: Evalboard export and local site tooling.
287
+ - `jtceval-bon` and `jtceval-bon-plot`: best-of-N configuration and plotting.
288
+
289
+ All JTC CLIs use Fire. Scheduling commands remain under
290
+ `primebeaker evaluation`, `primebeaker sft`, and `primebeaker rl`.
291
+
292
+ ## Discover verifier environments
293
+
294
+ Programmatically:
295
+
296
+ ```python
297
+ from primebeaker.environments import load_environment_module
298
+
299
+ module = load_environment_module("jtc-tool-label")
300
+ environment = module.load_environment(dataset="/path/to/train.jsonl")
301
+ ```
302
+
303
+ The stable direct import for the same implementation is:
304
+
305
+ ```python
306
+ from primebeaker.environments.jtc_tool_label_env import (
307
+ load_environment,
308
+ )
309
+ ```
310
+
311
+ ## Runtime prompt assets
312
+
313
+ Workflow defaults still use familiar paths such as
314
+ `templates/search-agent.json`. `resolve_resource_path()` first honors an
315
+ existing caller-provided path and then falls back to templates bundled in the
316
+ wheel. This preserves repository overrides while allowing jobs to run from any
317
+ working directory.
318
+
319
+ ```python
320
+ from jtc.common import load_prompt_template, resolve_resource_path
321
+
322
+ path = resolve_resource_path("templates/search-agent.json")
323
+ template = load_prompt_template(path)
324
+ ```
325
+
326
+ ## Release and PyPI
327
+
328
+ `.github/workflows/publish.yml` uses PyPI Trusted Publishing. Publishing a
329
+ GitHub release builds both the source distribution and wheel with Python 3.12,
330
+ then uploads them using GitHub's short-lived OIDC identity; no API token is
331
+ stored in this repository.
332
+
333
+ Create the `pypi` environment in GitHub Settings → Environments. For the first
334
+ release, add a pending publisher in [PyPI account publishing](https://pypi.org/manage/account/publishing/)
335
+ with these values:
336
+
337
+ - PyPI project name: `jtc-core`
338
+ - GitHub owner: `goncalorafaria`
339
+ - GitHub repository: `jtc`
340
+ - Workflow: `publish.yml`
341
+ - Environment: `pypi`
342
+
343
+ Publish by updating the version in `pyproject.toml`, merging that commit, and
344
+ creating a GitHub release whose tag matches the version, such as `v0.3.0`.
345
+ The release event runs the publishing workflow. The tag must be `v<version>`
346
+ or `<version>` and match `pyproject.toml`; a mismatch stops the build.
347
+ The first successful upload creates the PyPI project automatically. For an
348
+ existing project, add the same publisher in its PyPI publishing settings.
349
+
350
+ Manual workflow runs build and validate artifacts only. Builds use strict
351
+ Twine metadata checks; publishing runs in a separate job with OIDC permission.
352
+ Draft releases and tag pushes alone do not publish a package.
353
+
354
+ Before creating the release, run:
355
+
356
+ ```bash
357
+ python -m pytest -q tests/test_packaging.py tests/test_config_layout.py
358
+ python -m build
359
+ python -m twine check dist/*
360
+ ```
361
+
362
+ The package tests import every registry entry, verify the unified layout and
363
+ packaged template fallback, and catch stale names before a Beaker rollout.
364
+
365
+ PrimeBeaker owns the watcher implementation and command. The standalone
366
+ `rltracer` project is versioned and released independently from JTC.
@@ -0,0 +1,319 @@
1
+ # JTC
2
+
3
+ `jtc` is one Python distribution with one import namespace and three
4
+ cooperating subpackages:
5
+
6
+ ```text
7
+ jtc
8
+ ├── common # shared data models, IO, sources, templates, and workflows
9
+ ├── eval # scheduler-free evaluation workers, outcomes, and reporting
10
+ └── datadev # data recipes, materialization, mixtures, and training configs
11
+ ```
12
+
13
+ These are not independent packages. Installing `jtc` installs all three under
14
+ the same namespace. The historical top-level imports `jtc_data_commons`,
15
+ `jtceval`, and `datadev` are no longer part of the package API.
16
+
17
+ JTC implements data and evaluation work. It does not own training or
18
+ evaluation infrastructure. PrimeBeaker loads JTC's Python workers, creates the
19
+ LiteRegistry service topology, and schedules the jobs. RL trace inspection and
20
+ export come from the standalone
21
+ [`rltracer`](https://github.com/goncalorafaria/rltracer) package.
22
+ Checkpoint and evaluation provenance indexing comes from
23
+ `primebeaker watcher`.
24
+
25
+ ## Install
26
+
27
+ Install the released `jtc-core` distribution (Python imports and commands remain `jtc`):
28
+
29
+ ```bash
30
+ pip install jtc-core
31
+ ```
32
+
33
+ JTC requires Python 3.12.
34
+
35
+ For development from this checkout:
36
+
37
+ ```bash
38
+ pip install -e .
39
+ ```
40
+
41
+ Install every optional JTC stack for development with:
42
+
43
+ ```bash
44
+ pip install '.[all]'
45
+ ```
46
+
47
+ Available extras are:
48
+
49
+ - `evals`: OpenAI, PyArrow, and RLTracer support used by evaluation tooling.
50
+ - `harness`: Verifiers/Harbor support used by verifier workflows.
51
+ - `search`: Pyserini for local-search workloads.
52
+ - `trace`: RLTracer without the other optional stacks.
53
+ - `all`: all optional JTC dependencies.
54
+
55
+ PrimeBeaker is intentionally not a JTC dependency. A runtime that schedules
56
+ JTC work installs PrimeBeaker separately; the supported evaluation image pins
57
+ and installs released versions of both packages from PyPI.
58
+
59
+ ## Package boundaries
60
+
61
+ - `jtc.common`: schemas, parquet IO, rubric execution, source adapters,
62
+ shared tool integration, prompt templates, and registered Python workflows.
63
+ - `jtc.eval`: evaluation workload contracts and workers, outcome archives,
64
+ audits, plots, and Evalboard export.
65
+ - `jtc.datadev`: typed SFT/RL data recipes, materialization, Pot sampling and
66
+ optimization, and training-TOML generation.
67
+
68
+ Verifier environments and reward functions live in
69
+ `primebeaker.environments`. Scheduling, dynamic ports, LiteRegistry deployment,
70
+ readiness, supervision, resume, and shutdown coordination also live in
71
+ PrimeBeaker. Runtime records, datasets, checkpoints, and evaluation results
72
+ stay outside this repository.
73
+
74
+ ## Data and training-configuration flow
75
+
76
+ `jtc.datadev` turns versioned YAML data recipes into materialized SFT or RL
77
+ datasets and then renders a compatible training TOML. It does not launch
78
+ training:
79
+
80
+ ```text
81
+ data recipe YAML -> materialized dataset + manifest -> training TOML -> PrimeBeaker
82
+ ```
83
+
84
+ The CLI is implemented with Fire:
85
+
86
+ ```bash
87
+ # Validate without writing data.
88
+ datadev data sft examples/datadev/sft-data-smoke.yaml --dry-run
89
+
90
+ # Materialize data and render the corresponding training TOML.
91
+ datadev prepare rl \
92
+ examples/datadev/rl-data-smoke.yaml \
93
+ examples/datadev/rl.toml \
94
+ /weka/gfaria/generated/example-rl.toml
95
+ ```
96
+
97
+ Canonical recipe configs live under `examples/configs/data/{sft,rl}` and
98
+ training configs under `examples/configs/train/{sft,rl}`. The small files under
99
+ `examples/datadev` demonstrate the formats. Generated datasets and manifests
100
+ must use an external artifact directory; the resulting TOML is handed to
101
+ PrimeBeaker for SFT or RL scheduling.
102
+
103
+ The `jtc` command exposes common parquet inspection and sampling operations:
104
+
105
+ ```bash
106
+ jtc inspect /weka/gfaria/records/example.parquet
107
+ jtc sample-jrows INPUT.parquet OUTPUT.parquet --max-input-objects=100
108
+ ```
109
+
110
+ In Python, import through the unified namespace:
111
+
112
+ ```python
113
+ from jtc.common.schema import JRecord, JRow
114
+ from jtc.datadev import Pot, build_data
115
+ from jtc.eval import EvaluationArchive, EvaluationWorkloadConfig
116
+ ```
117
+
118
+ ## Python-only evaluation workloads
119
+
120
+ `jtc.eval` owns benchmark selection and evaluation logic. PrimeBeaker owns the
121
+ YAML descriptions and all infrastructure lifecycle behavior: Beaker scheduling,
122
+ native LiteRegistry deployment, endpoint readiness, signal forwarding, and
123
+ cleanup. The dependency direction is PrimeBeaker -> JTC.
124
+
125
+ The evaluator always runs in a Beaker image containing released PyPI wheels;
126
+ JTC source is never imported or copied into the image. Build and publish the
127
+ pinned image using JTC's `docker/Dockerfile.eval`, then put its immutable
128
+ `beaker://...` URI in the PrimeBeaker-owned eval YAML or pass it with `--image`:
129
+
130
+ ```yaml
131
+ image: beaker://IMMUTABLE_PRIMEBEAKER_JTC_IMAGE
132
+ workspace: ai2/oe-agents-holmes
133
+ budget: ai2/oe-omai
134
+ evaluation:
135
+ model: /weka/gfaria/models/policy/step_400
136
+ run_dir: /weka/gfaria/jtc-artifacts/evals/policy-step-400-smoke
137
+ datasets: [healthbench]
138
+ max_rows: 4
139
+ runs: 1
140
+ verifier_tools: terminal
141
+ verifier_prompt_chat_template_path: templates/jtc-io-terminal-release.json
142
+ ```
143
+
144
+ Lifecycle YAMLs live in PrimeBeaker under `examples/configs/eval`. Preview and
145
+ submit through PrimeBeaker's Fire CLI:
146
+
147
+ ```bash
148
+ primebeaker evaluation preview --config=examples/configs/eval/python_only_smoke.yaml
149
+ primebeaker evaluation submit --config=examples/configs/eval/python_only_smoke.yaml
150
+ ```
151
+
152
+ PrimeBeaker selects a packaged Python entry point for each workload:
153
+
154
+ - standard benchmark evaluation: `jtc.eval.worker`
155
+ - search-agent evaluation: `jtc.eval.search_worker`
156
+ - any registered JTC workflow: `jtc.eval.workflow_worker`
157
+ - iterative rejection sampling: `jtc.eval.rejection_sampling_worker`
158
+ - rubric-judge audit: `jtc.eval.rubric_judge_audit_worker`
159
+
160
+ The generic workflow worker accepts every stable name exported by
161
+ `jtc.common.workflows.registry`. This includes articulated/grouped submit,
162
+ TMAX/Podman and binary-submit workflows, inference, rubric generation, and the
163
+ other verifier workflows in the registry.
164
+
165
+ JTC validates the selected workflow and executes its Python `main()` while
166
+ PrimeBeaker injects managed registry and gateway endpoints. A newly registered
167
+ workflow therefore uses the existing generic worker rather than a new scheduler
168
+ module.
169
+
170
+ For rejection sampling, PrimeBeaker owns the rounds, service topology, resume,
171
+ and cleanup lifecycle. The JTC worker runs the selected verifier workflow; the
172
+ canonical filtering and accepted-trajectory compilation algorithms live in
173
+ `jtc.common.rejection_sampling`. No launcher or fixed-port infrastructure lives
174
+ in JTC.
175
+
176
+ Templates are runtime data, not image code. A PrimeBeaker YAML can use a newly
177
+ created JSON template by YAML-relative, checkout-relative, or absolute path.
178
+ PrimeBeaker validates it on the submitting machine and writes a content-addressed
179
+ snapshot under `<run_dir>/.jtceval-inputs/templates/` before it submits the
180
+ evaluator. The `/weka` mount exposes that snapshot to the PyPI-only evaluation
181
+ image without rebuilding JTC. A dry run reports the future path without writing.
182
+
183
+ Rubric-judge audits use the same boundary. JTC implements Python sampling and
184
+ judgment recovery while PrimeBeaker schedules the shadow model, registry, and
185
+ gateway. The `jtc.eval.rubric_judge_audit_worker` calls those implementations
186
+ directly:
187
+
188
+ ```bash
189
+ primebeaker evaluation preview \
190
+ --config=examples/configs/eval/rubric_judge_audit/qwen35_4b_grouped_vs_qwen36_35b_steps005_400.yaml
191
+ primebeaker evaluation submit \
192
+ --config=examples/configs/eval/rubric_judge_audit/qwen35_4b_grouped_vs_qwen36_35b_steps005_400.yaml
193
+ ```
194
+
195
+ The coordinator can resume from a complete sample or response stream and rejects
196
+ partial artifacts unless `--force=True` is explicitly supplied.
197
+
198
+ Prime-RL TOMLs point directly at PrimeBeaker's package-qualified environment
199
+ modules:
200
+
201
+ ```toml
202
+ [orchestrator.train.source.legacy]
203
+ id = "primebeaker.environments.jtc_tool_label_env"
204
+ ```
205
+
206
+ Prime-RL loads the dotted module path directly. No environment implementation
207
+ is copied into JTC, and there are no JTC-owned harness aliases; callers use the
208
+ PrimeBeaker module path or its environment registry.
209
+
210
+ ## Discover and run workflows
211
+
212
+ ```bash
213
+ jtc-workflow list
214
+ ```
215
+
216
+ Programmatically:
217
+
218
+ ```python
219
+ from jtc.common.workflows import load_workflow
220
+
221
+ workflow = load_workflow("verifier-tool-use")
222
+ workflow.main(...)
223
+ ```
224
+
225
+ Only import-tested workflows are registered. Broken historical variants are
226
+ kept in the repository for provenance but excluded from the wheel registry.
227
+
228
+ ## Local evaluation utilities
229
+
230
+ The `jtceval` CLI does not schedule work. It imports and summarizes existing
231
+ evaluation outputs; lifecycle operations stay in PrimeBeaker:
232
+
233
+ ```bash
234
+ jtceval import-evaluations SOURCE_DIRECTORY OUTPUT.json --dry-run
235
+ ```
236
+
237
+ The distribution also installs these evaluation-side entry points:
238
+
239
+ - `jtceval-evalboard`: Evalboard export and local site tooling.
240
+ - `jtceval-bon` and `jtceval-bon-plot`: best-of-N configuration and plotting.
241
+
242
+ All JTC CLIs use Fire. Scheduling commands remain under
243
+ `primebeaker evaluation`, `primebeaker sft`, and `primebeaker rl`.
244
+
245
+ ## Discover verifier environments
246
+
247
+ Programmatically:
248
+
249
+ ```python
250
+ from primebeaker.environments import load_environment_module
251
+
252
+ module = load_environment_module("jtc-tool-label")
253
+ environment = module.load_environment(dataset="/path/to/train.jsonl")
254
+ ```
255
+
256
+ The stable direct import for the same implementation is:
257
+
258
+ ```python
259
+ from primebeaker.environments.jtc_tool_label_env import (
260
+ load_environment,
261
+ )
262
+ ```
263
+
264
+ ## Runtime prompt assets
265
+
266
+ Workflow defaults still use familiar paths such as
267
+ `templates/search-agent.json`. `resolve_resource_path()` first honors an
268
+ existing caller-provided path and then falls back to templates bundled in the
269
+ wheel. This preserves repository overrides while allowing jobs to run from any
270
+ working directory.
271
+
272
+ ```python
273
+ from jtc.common import load_prompt_template, resolve_resource_path
274
+
275
+ path = resolve_resource_path("templates/search-agent.json")
276
+ template = load_prompt_template(path)
277
+ ```
278
+
279
+ ## Release and PyPI
280
+
281
+ `.github/workflows/publish.yml` uses PyPI Trusted Publishing. Publishing a
282
+ GitHub release builds both the source distribution and wheel with Python 3.12,
283
+ then uploads them using GitHub's short-lived OIDC identity; no API token is
284
+ stored in this repository.
285
+
286
+ Create the `pypi` environment in GitHub Settings → Environments. For the first
287
+ release, add a pending publisher in [PyPI account publishing](https://pypi.org/manage/account/publishing/)
288
+ with these values:
289
+
290
+ - PyPI project name: `jtc-core`
291
+ - GitHub owner: `goncalorafaria`
292
+ - GitHub repository: `jtc`
293
+ - Workflow: `publish.yml`
294
+ - Environment: `pypi`
295
+
296
+ Publish by updating the version in `pyproject.toml`, merging that commit, and
297
+ creating a GitHub release whose tag matches the version, such as `v0.3.0`.
298
+ The release event runs the publishing workflow. The tag must be `v<version>`
299
+ or `<version>` and match `pyproject.toml`; a mismatch stops the build.
300
+ The first successful upload creates the PyPI project automatically. For an
301
+ existing project, add the same publisher in its PyPI publishing settings.
302
+
303
+ Manual workflow runs build and validate artifacts only. Builds use strict
304
+ Twine metadata checks; publishing runs in a separate job with OIDC permission.
305
+ Draft releases and tag pushes alone do not publish a package.
306
+
307
+ Before creating the release, run:
308
+
309
+ ```bash
310
+ python -m pytest -q tests/test_packaging.py tests/test_config_layout.py
311
+ python -m build
312
+ python -m twine check dist/*
313
+ ```
314
+
315
+ The package tests import every registry entry, verify the unified layout and
316
+ packaged template fallback, and catch stale names before a Beaker rollout.
317
+
318
+ PrimeBeaker owns the watcher implementation and command. The standalone
319
+ `rltracer` project is versioned and released independently from JTC.
@@ -0,0 +1,5 @@
1
+ """Unified JTC data, evaluation, and configuration toolkit."""
2
+
3
+ from __future__ import annotations
4
+
5
+ __all__ = ["common", "datadev", "eval"]