jtc-core 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jtc_core-0.3.0/PKG-INFO +366 -0
- jtc_core-0.3.0/README.md +319 -0
- jtc_core-0.3.0/jtc/__init__.py +5 -0
- jtc_core-0.3.0/jtc/common/__init__.py +259 -0
- jtc_core-0.3.0/jtc/common/asset_store.py +9 -0
- jtc_core-0.3.0/jtc/common/chat_completion.py +97 -0
- jtc_core-0.3.0/jtc/common/cli.py +151 -0
- jtc_core-0.3.0/jtc/common/code_execution.py +443 -0
- jtc_core-0.3.0/jtc/common/code_utils.py +252 -0
- jtc_core-0.3.0/jtc/common/generation_workflow.py +404 -0
- jtc_core-0.3.0/jtc/common/hub.py +26 -0
- jtc_core-0.3.0/jtc/common/ifeval/__init__.py +11 -0
- jtc_core-0.3.0/jtc/common/ifeval/functions.py +1161 -0
- jtc_core-0.3.0/jtc/common/jrecord_sampling.py +201 -0
- jtc_core-0.3.0/jtc/common/jrow_regrouping.py +274 -0
- jtc_core-0.3.0/jtc/common/jrow_sampling.py +222 -0
- jtc_core-0.3.0/jtc/common/parquet.py +500 -0
- jtc_core-0.3.0/jtc/common/podman_verifier.py +209 -0
- jtc_core-0.3.0/jtc/common/prime_rl_dataset.py +180 -0
- jtc_core-0.3.0/jtc/common/process.py +47 -0
- jtc_core-0.3.0/jtc/common/rejection_sampling.py +252 -0
- jtc_core-0.3.0/jtc/common/resources/__init__.py +1 -0
- jtc_core-0.3.0/jtc/common/resources/templates/answer-generation.json +14 -0
- jtc_core-0.3.0/jtc/common/resources/templates/articulated_harness_gptoss.json +18 -0
- jtc_core-0.3.0/jtc/common/resources/templates/code-check-generation.json +15 -0
- jtc_core-0.3.0/jtc/common/resources/templates/code-check-label-generation-gptoss.json +35 -0
- jtc_core-0.3.0/jtc/common/resources/templates/extraction-prompt-generation.json +15 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-complete.json +37 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-prompt.json +20 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-python-prompt-gptoss-harmony.json +20 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-repl-prompt-gptoss-harmony.json +19 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-repl-prompt-v4.json +19 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-simple-prompt-json.json +20 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-simple-prompt.json +20 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-terminal-prompt-gptoss-harmony.json +19 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-terminal-release-browse-search-xml-v1.json +56 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-terminal-release-browse-xml-v1.json +36 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-terminal-release-browse.json +36 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-terminal-release.json +19 -0
- jtc_core-0.3.0/jtc/common/resources/templates/jtc-io-tool_response.json +31 -0
- jtc_core-0.3.0/jtc/common/resources/templates/multistep-final-warning-harmony.json +14 -0
- jtc_core-0.3.0/jtc/common/resources/templates/multistep-final-warning.json +13 -0
- jtc_core-0.3.0/jtc/common/resources/templates/multistep-tool-message.json +13 -0
- jtc_core-0.3.0/jtc/common/resources/templates/recordset-batch-rubric-judge.json +19 -0
- jtc_core-0.3.0/jtc/common/resources/templates/rubric-generation-from-rollouts.json +19 -0
- jtc_core-0.3.0/jtc/common/resources/templates/rubric-generation-from-unit-tests.json +15 -0
- jtc_core-0.3.0/jtc/common/resources/templates/search-agent-visit-judge.json +15 -0
- jtc_core-0.3.0/jtc/common/resources/templates/search-agent-visit.json +15 -0
- jtc_core-0.3.0/jtc/common/resources/templates/search-agent-webterminal-10k.json +17 -0
- jtc_core-0.3.0/jtc/common/resources/templates/search-agent.json +17 -0
- jtc_core-0.3.0/jtc/common/resources/templates/tmax-podman-trace-submit-binary-gptoss.v2.json +13 -0
- jtc_core-0.3.0/jtc/common/resources/templates/tmax-podman-trace-submit-gptoss.json +41 -0
- jtc_core-0.3.0/jtc/common/resources/templates/wildguard-instance-rubric-generation.json +21 -0
- jtc_core-0.3.0/jtc/common/rubric_execution.py +179 -0
- jtc_core-0.3.0/jtc/common/schema.py +455 -0
- jtc_core-0.3.0/jtc/common/scoring.py +62 -0
- jtc_core-0.3.0/jtc/common/sources/__init__.py +202 -0
- jtc_core-0.3.0/jtc/common/sources/base.py +27 -0
- jtc_core-0.3.0/jtc/common/sources/coding.py +234 -0
- jtc_core-0.3.0/jtc/common/sources/coding_transform.py +53 -0
- jtc_core-0.3.0/jtc/common/sources/common.py +236 -0
- jtc_core-0.3.0/jtc/common/sources/deepresearch_bench.py +103 -0
- jtc_core-0.3.0/jtc/common/sources/drtulu.py +107 -0
- jtc_core-0.3.0/jtc/common/sources/drtulu_code_checks.py +165 -0
- jtc_core-0.3.0/jtc/common/sources/gpqa.py +146 -0
- jtc_core-0.3.0/jtc/common/sources/healthbench.py +113 -0
- jtc_core-0.3.0/jtc/common/sources/ifeval.py +151 -0
- jtc_core-0.3.0/jtc/common/sources/ifeval_transform.py +54 -0
- jtc_core-0.3.0/jtc/common/sources/liveresearchbench.py +96 -0
- jtc_core-0.3.0/jtc/common/sources/llmeval_med.py +267 -0
- jtc_core-0.3.0/jtc/common/sources/medqa_usmle.py +121 -0
- jtc_core-0.3.0/jtc/common/sources/mind2web2.py +145 -0
- jtc_core-0.3.0/jtc/common/sources/mmlu_pro.py +118 -0
- jtc_core-0.3.0/jtc/common/sources/monaco.py +334 -0
- jtc_core-0.3.0/jtc/common/sources/multiple_choice.py +108 -0
- jtc_core-0.3.0/jtc/common/sources/natural.py +90 -0
- jtc_core-0.3.0/jtc/common/sources/natural_synth.py +186 -0
- jtc_core-0.3.0/jtc/common/sources/openrubrics.py +99 -0
- jtc_core-0.3.0/jtc/common/sources/prbench.py +153 -0
- jtc_core-0.3.0/jtc/common/sources/profbench.py +131 -0
- jtc_core-0.3.0/jtc/common/sources/prometheus.py +65 -0
- jtc_core-0.3.0/jtc/common/sources/quest.py +336 -0
- jtc_core-0.3.0/jtc/common/sources/researchqa.py +405 -0
- jtc_core-0.3.0/jtc/common/sources/rubricbench.py +111 -0
- jtc_core-0.3.0/jtc/common/sources/ruft26k.py +140 -0
- jtc_core-0.3.0/jtc/common/sources/ruft26k_answer_generations.py +302 -0
- jtc_core-0.3.0/jtc/common/sources/ruft26k_code_checks.py +156 -0
- jtc_core-0.3.0/jtc/common/sources/science_qwen35_judgments.py +80 -0
- jtc_core-0.3.0/jtc/common/sources/tbd.py +29 -0
- jtc_core-0.3.0/jtc/common/sources/wildbench.py +271 -0
- jtc_core-0.3.0/jtc/common/submission.py +19 -0
- jtc_core-0.3.0/jtc/common/submission_display.py +28 -0
- jtc_core-0.3.0/jtc/common/templates.py +158 -0
- jtc_core-0.3.0/jtc/common/tmax_replay.py +157 -0
- jtc_core-0.3.0/jtc/common/tool_clients.py +74 -0
- jtc_core-0.3.0/jtc/common/tool_specs.py +111 -0
- jtc_core-0.3.0/jtc/common/trace_inspection.py +69 -0
- jtc_core-0.3.0/jtc/common/utils.py +80 -0
- jtc_core-0.3.0/jtc/common/workflows/__init__.py +9 -0
- jtc_core-0.3.0/jtc/common/workflows/__main__.py +3 -0
- jtc_core-0.3.0/jtc/common/workflows/answer_generation_workflow.py +499 -0
- jtc_core-0.3.0/jtc/common/workflows/cli.py +32 -0
- jtc_core-0.3.0/jtc/common/workflows/code_addressability_workflow.py +186 -0
- jtc_core-0.3.0/jtc/common/workflows/code_check_generation_workflow.py +912 -0
- jtc_core-0.3.0/jtc/common/workflows/code_check_label_generation_workflow.py +403 -0
- jtc_core-0.3.0/jtc/common/workflows/extraction_prompt_generation_workflow.py +314 -0
- jtc_core-0.3.0/jtc/common/workflows/inference_workflow.py +1431 -0
- jtc_core-0.3.0/jtc/common/workflows/jrecord_batch_rubric_judgment_workflow.py +231 -0
- jtc_core-0.3.0/jtc/common/workflows/logprob_workflow.py +375 -0
- jtc_core-0.3.0/jtc/common/workflows/multi_step_generation_workflow.py +654 -0
- jtc_core-0.3.0/jtc/common/workflows/multi_step_workflow.py +810 -0
- jtc_core-0.3.0/jtc/common/workflows/process_message_workers.py +156 -0
- jtc_core-0.3.0/jtc/common/workflows/registry.py +56 -0
- jtc_core-0.3.0/jtc/common/workflows/science_bundles_batch_rubric_judgment_workflow.py +438 -0
- jtc_core-0.3.0/jtc/common/workflows/science_bundles_rubric_generation_workflow.py +376 -0
- jtc_core-0.3.0/jtc/common/workflows/search_agent_rollout_workflow.py +891 -0
- jtc_core-0.3.0/jtc/common/workflows/tmax_unit_test_rubric_generation_workflow.py +86 -0
- jtc_core-0.3.0/jtc/common/workflows/tmax_unit_test_rubric_generation_workflow_v2.py +121 -0
- jtc_core-0.3.0/jtc/common/workflows/tmax_unit_test_rubric_generation_workflow_v3.py +50 -0
- jtc_core-0.3.0/jtc/common/workflows/tool_use_generation_workflow.py +478 -0
- jtc_core-0.3.0/jtc/common/workflows/tool_use_workflow.py +955 -0
- jtc_core-0.3.0/jtc/common/workflows/verifier_articulated_grouped_submit_smoke.py +92 -0
- jtc_core-0.3.0/jtc/common/workflows/verifier_articulated_submit_tool_use_workflow.py +37 -0
- jtc_core-0.3.0/jtc/common/workflows/verifier_articulated_submit_tool_use_workflow_v2.py +37 -0
- jtc_core-0.3.0/jtc/common/workflows/verifier_submit_tool_use_workflow.py +164 -0
- jtc_core-0.3.0/jtc/common/workflows/verifier_tmax_podman_binary_submit_workflow.py +59 -0
- jtc_core-0.3.0/jtc/common/workflows/verifier_tmax_podman_binary_submit_workflow_v3.py +36 -0
- jtc_core-0.3.0/jtc/common/workflows/verifier_tmax_podman_submit_workflow.py +330 -0
- jtc_core-0.3.0/jtc/common/workflows/verifier_tmax_podman_submit_workflow_v3.py +47 -0
- jtc_core-0.3.0/jtc/common/workflows/verifier_tool_use_workflow.py +985 -0
- jtc_core-0.3.0/jtc/common/workflows/wildguard_rubric_generation_workflow.py +314 -0
- jtc_core-0.3.0/jtc/datadev/__init__.py +24 -0
- jtc_core-0.3.0/jtc/datadev/cli.py +220 -0
- jtc_core-0.3.0/jtc/datadev/data/__init__.py +29 -0
- jtc_core-0.3.0/jtc/datadev/data/materialize.py +1466 -0
- jtc_core-0.3.0/jtc/datadev/data/recipes.py +311 -0
- jtc_core-0.3.0/jtc/datadev/infra/__init__.py +3 -0
- jtc_core-0.3.0/jtc/datadev/infra/bm25_server.py +358 -0
- jtc_core-0.3.0/jtc/datadev/literegistry/__init__.py +1 -0
- jtc_core-0.3.0/jtc/datadev/literegistry/gateway.py +430 -0
- jtc_core-0.3.0/jtc/datadev/literegistry/jina_reader.py +222 -0
- jtc_core-0.3.0/jtc/datadev/opt/__init__.py +5 -0
- jtc_core-0.3.0/jtc/datadev/opt/weights.py +245 -0
- jtc_core-0.3.0/jtc/datadev/pot_sampling.py +303 -0
- jtc_core-0.3.0/jtc/datadev/pots.py +238 -0
- jtc_core-0.3.0/jtc/datadev/raw_token_prior.py +364 -0
- jtc_core-0.3.0/jtc/datadev/semaphore/__init__.py +23 -0
- jtc_core-0.3.0/jtc/datadev/semaphore/__main__.py +199 -0
- jtc_core-0.3.0/jtc/datadev/semaphore/core.py +825 -0
- jtc_core-0.3.0/jtc/datadev/sft_pot_batches.py +444 -0
- jtc_core-0.3.0/jtc/datadev/token_counts.py +41 -0
- jtc_core-0.3.0/jtc/datadev/training_config.py +560 -0
- jtc_core-0.3.0/jtc/datadev/workflows.py +88 -0
- jtc_core-0.3.0/jtc/eval/__init__.py +31 -0
- jtc_core-0.3.0/jtc/eval/__main__.py +5 -0
- jtc_core-0.3.0/jtc/eval/benchmark.py +264 -0
- jtc_core-0.3.0/jtc/eval/bon_config.py +242 -0
- jtc_core-0.3.0/jtc/eval/bon_plot.py +325 -0
- jtc_core-0.3.0/jtc/eval/cli.py +53 -0
- jtc_core-0.3.0/jtc/eval/dashboard.py +18 -0
- jtc_core-0.3.0/jtc/eval/eval.py +569 -0
- jtc_core-0.3.0/jtc/eval/evalboard/__init__.py +5 -0
- jtc_core-0.3.0/jtc/eval/evalboard/__main__.py +50 -0
- jtc_core-0.3.0/jtc/eval/evalboard/export.py +67 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/.openai/hosting.json +5 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/README.md +56 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/bon/bon.module.css +108 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/bon/page.tsx +234 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/chatgpt-auth.ts +90 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.css +56 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v10.css +14 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v11.css +12 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v12.css +12 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v13.css +49 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v14.css +3 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v15.css +19 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v16.css +18 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v17.css +20 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v18.css +7 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v19.css +17 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v20.css +42 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v6.css +2 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v7.css +15 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v8.css +15 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/globals.v9.css +29 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/layout.tsx +11 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/app/page.tsx +12 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/build/sites-vite-plugin.ts +45 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/db/index.ts +13 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/db/schema.ts +4 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/drizzle.config.ts +7 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/eslint.config.mjs +41 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/next-env.d.ts +5 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/next.config.ts +7 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/package-lock.json +10279 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/package.json +47 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/postcss.config.mjs +7 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/public/favicon.svg +6 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/public/file.svg +1 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/public/globe.svg +1 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/public/og.png +0 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/public/window.svg +1 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/tests/rendered-html.test.mjs +91 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/tsconfig.json +29 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/vite.config.ts +59 -0
- jtc_core-0.3.0/jtc/eval/evalboard/site/worker/index.ts +47 -0
- jtc_core-0.3.0/jtc/eval/inference_scaling.py +203 -0
- jtc_core-0.3.0/jtc/eval/jina_reader.py +222 -0
- jtc_core-0.3.0/jtc/eval/launcher.py +54 -0
- jtc_core-0.3.0/jtc/eval/outcomes.py +262 -0
- jtc_core-0.3.0/jtc/eval/plot.py +159 -0
- jtc_core-0.3.0/jtc/eval/reconstruct_rubricbench_jrecords.py +578 -0
- jtc_core-0.3.0/jtc/eval/rejection_sampling.py +61 -0
- jtc_core-0.3.0/jtc/eval/rejection_sampling_worker.py +177 -0
- jtc_core-0.3.0/jtc/eval/rubric_judge_audit.py +224 -0
- jtc_core-0.3.0/jtc/eval/rubric_judge_audit_worker.py +145 -0
- jtc_core-0.3.0/jtc/eval/search_agent.py +63 -0
- jtc_core-0.3.0/jtc/eval/search_worker.py +75 -0
- jtc_core-0.3.0/jtc/eval/trace_rubric_recovery.py +298 -0
- jtc_core-0.3.0/jtc/eval/trace_rubric_sampling.py +594 -0
- jtc_core-0.3.0/jtc/eval/worker.py +52 -0
- jtc_core-0.3.0/jtc/eval/workflow.py +37 -0
- jtc_core-0.3.0/jtc/eval/workflow_worker.py +104 -0
- jtc_core-0.3.0/jtc_core.egg-info/PKG-INFO +366 -0
- jtc_core-0.3.0/jtc_core.egg-info/SOURCES.txt +296 -0
- jtc_core-0.3.0/jtc_core.egg-info/dependency_links.txt +1 -0
- jtc_core-0.3.0/jtc_core.egg-info/entry_points.txt +8 -0
- jtc_core-0.3.0/jtc_core.egg-info/requires.txt +42 -0
- jtc_core-0.3.0/jtc_core.egg-info/top_level.txt +1 -0
- jtc_core-0.3.0/pyproject.toml +71 -0
- jtc_core-0.3.0/setup.cfg +4 -0
- jtc_core-0.3.0/setup.py +4 -0
- jtc_core-0.3.0/tests/test_aggagent_rollouts_jrecord_sets.py +67 -0
- jtc_core-0.3.0/tests/test_articulated_harness_gptoss_template.py +54 -0
- jtc_core-0.3.0/tests/test_asset_store.py +64 -0
- jtc_core-0.3.0/tests/test_asset_store_core.py +122 -0
- jtc_core-0.3.0/tests/test_audit_judge_trace_terminal_errors.py +153 -0
- jtc_core-0.3.0/tests/test_audit_judge_trace_terminal_usage.py +141 -0
- jtc_core-0.3.0/tests/test_audit_judge_trace_tool_domains.py +30 -0
- jtc_core-0.3.0/tests/test_bake_all_aggagent_rollouts.py +11 -0
- jtc_core-0.3.0/tests/test_batch_rubric_chunked_repairs.py +55 -0
- jtc_core-0.3.0/tests/test_batch_rubric_json_repairs.py +72 -0
- jtc_core-0.3.0/tests/test_bm25_server.py +31 -0
- jtc_core-0.3.0/tests/test_browsecomp_rollouts_jrecord_sets.py +163 -0
- jtc_core-0.3.0/tests/test_browsecomp_rubric_generation_template.py +97 -0
- jtc_core-0.3.0/tests/test_build_tmax_question_unit_tests.py +97 -0
- jtc_core-0.3.0/tests/test_chat_completion_boundary.py +63 -0
- jtc_core-0.3.0/tests/test_code_server.py +156 -0
- jtc_core-0.3.0/tests/test_compare_judge_trace_terminal_usage.py +73 -0
- jtc_core-0.3.0/tests/test_config_layout.py +125 -0
- jtc_core-0.3.0/tests/test_datadev.py +77 -0
- jtc_core-0.3.0/tests/test_dataset.py +55 -0
- jtc_core-0.3.0/tests/test_execution.py +208 -0
- jtc_core-0.3.0/tests/test_export_mercor_eval_traces.py +118 -0
- jtc_core-0.3.0/tests/test_extract_tmax_rollout_commands.py +228 -0
- jtc_core-0.3.0/tests/test_jrecord_sampling.py +62 -0
- jtc_core-0.3.0/tests/test_jrecord_set_schema.py +44 -0
- jtc_core-0.3.0/tests/test_jrow_regrouping.py +85 -0
- jtc_core-0.3.0/tests/test_jrow_sampling.py +92 -0
- jtc_core-0.3.0/tests/test_judge_aggagent_researchrubrics_parquet.py +13 -0
- jtc_core-0.3.0/tests/test_judge_client.py +59 -0
- jtc_core-0.3.0/tests/test_judge_trace_hint_templates.py +59 -0
- jtc_core-0.3.0/tests/test_judge_trace_terminal_hint_catalog.py +36 -0
- jtc_core-0.3.0/tests/test_literegistry_tool_client_boundary.py +33 -0
- jtc_core-0.3.0/tests/test_llmeval_med_source.py +171 -0
- jtc_core-0.3.0/tests/test_load_jsonl.py +22 -0
- jtc_core-0.3.0/tests/test_load_parquet.py +24 -0
- jtc_core-0.3.0/tests/test_loading.py +81 -0
- jtc_core-0.3.0/tests/test_model.py +137 -0
- jtc_core-0.3.0/tests/test_monaco_source.py +87 -0
- jtc_core-0.3.0/tests/test_multi_step_workflow.py +394 -0
- jtc_core-0.3.0/tests/test_multiple_choice_benchmark_sources.py +86 -0
- jtc_core-0.3.0/tests/test_multistep_rubric_schema.py +56 -0
- jtc_core-0.3.0/tests/test_openai_tool_client.py +89 -0
- jtc_core-0.3.0/tests/test_packaging.py +173 -0
- jtc_core-0.3.0/tests/test_plot_judge_trace_terminal_usage_comparison.py +82 -0
- jtc_core-0.3.0/tests/test_plot_judge_trace_terminal_usage_multimodel.py +56 -0
- jtc_core-0.3.0/tests/test_podman_verifier.py +163 -0
- jtc_core-0.3.0/tests/test_prime_rl_dataset.py +85 -0
- jtc_core-0.3.0/tests/test_quest_source.py +135 -0
- jtc_core-0.3.0/tests/test_research_benchmark_sources.py +99 -0
- jtc_core-0.3.0/tests/test_researchqa_answer_generations.py +94 -0
- jtc_core-0.3.0/tests/test_safety_rubric_trajectories_jrecords.py +175 -0
- jtc_core-0.3.0/tests/test_science_bundles_batch_rubric_judgment_workflow.py +112 -0
- jtc_core-0.3.0/tests/test_search_agent_rollout_workflow.py +99 -0
- jtc_core-0.3.0/tests/test_search_client.py +860 -0
- jtc_core-0.3.0/tests/test_snippet.py +45 -0
- jtc_core-0.3.0/tests/test_split_safety_rubric_trajectories_jrecords.py +95 -0
- jtc_core-0.3.0/tests/test_submission.py +93 -0
- jtc_core-0.3.0/tests/test_terminal.py +3 -0
- jtc_core-0.3.0/tests/test_terminal_release_browse_xml_v1.py +93 -0
- jtc_core-0.3.0/tests/test_tmax_podman_fastforward.py +109 -0
- jtc_core-0.3.0/tests/test_tmax_podman_fastforward_smoke.py +57 -0
- jtc_core-0.3.0/tests/test_tmax_replay.py +277 -0
- jtc_core-0.3.0/tests/test_tmax_rubric_generation_templates.py +51 -0
- jtc_core-0.3.0/tests/test_trace_terminal_template.py +47 -0
- jtc_core-0.3.0/tests/test_verifier_submit_tool_use_workflow.py +76 -0
- jtc_core-0.3.0/tests/test_verifier_tool_use_workflow.py +110 -0
jtc_core-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: jtc-core
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Unified JTC data, configuration, evaluation, and workflow toolkit.
|
|
5
|
+
Project-URL: Homepage, https://github.com/goncalorafaria/jtc
|
|
6
|
+
Project-URL: Repository, https://github.com/goncalorafaria/jtc
|
|
7
|
+
Project-URL: Issues, https://github.com/goncalorafaria/jtc/issues
|
|
8
|
+
Requires-Python: ~=3.12.0
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
Requires-Dist: aiohttp
|
|
11
|
+
Requires-Dist: cvxpy
|
|
12
|
+
Requires-Dist: datasets
|
|
13
|
+
Requires-Dist: fastapi
|
|
14
|
+
Requires-Dist: fire
|
|
15
|
+
Requires-Dist: httpx
|
|
16
|
+
Requires-Dist: litellm
|
|
17
|
+
Requires-Dist: literegistry-tool-client==0.1.0
|
|
18
|
+
Requires-Dist: literegistry==1.0.48
|
|
19
|
+
Requires-Dist: PyYAML>=6
|
|
20
|
+
Requires-Dist: tqdm
|
|
21
|
+
Requires-Dist: markdown
|
|
22
|
+
Requires-Dist: matplotlib
|
|
23
|
+
Requires-Dist: jtcflow==0.1.0
|
|
24
|
+
Requires-Dist: scipy
|
|
25
|
+
Requires-Dist: transformers
|
|
26
|
+
Requires-Dist: uvicorn
|
|
27
|
+
Requires-Dist: wandb
|
|
28
|
+
Requires-Dist: msgpack
|
|
29
|
+
Requires-Dist: numpy
|
|
30
|
+
Requires-Dist: pydantic
|
|
31
|
+
Provides-Extra: search
|
|
32
|
+
Requires-Dist: pyserini; extra == "search"
|
|
33
|
+
Provides-Extra: trace
|
|
34
|
+
Requires-Dist: rltracer<0.2,>=0.1.0; extra == "trace"
|
|
35
|
+
Provides-Extra: evals
|
|
36
|
+
Requires-Dist: openai; extra == "evals"
|
|
37
|
+
Requires-Dist: pyarrow; extra == "evals"
|
|
38
|
+
Requires-Dist: rltracer<0.2,>=0.1.0; extra == "evals"
|
|
39
|
+
Provides-Extra: harness
|
|
40
|
+
Requires-Dist: verifiers[harbor]==0.2.1; extra == "harness"
|
|
41
|
+
Provides-Extra: all
|
|
42
|
+
Requires-Dist: pyserini; extra == "all"
|
|
43
|
+
Requires-Dist: openai; extra == "all"
|
|
44
|
+
Requires-Dist: pyarrow; extra == "all"
|
|
45
|
+
Requires-Dist: rltracer<0.2,>=0.1.0; extra == "all"
|
|
46
|
+
Requires-Dist: verifiers[harbor]==0.2.1; extra == "all"
|
|
47
|
+
|
|
48
|
+
# JTC
|
|
49
|
+
|
|
50
|
+
`jtc` is one Python distribution with one import namespace and three
|
|
51
|
+
cooperating subpackages:
|
|
52
|
+
|
|
53
|
+
```text
|
|
54
|
+
jtc
|
|
55
|
+
├── common # shared data models, IO, sources, templates, and workflows
|
|
56
|
+
├── eval # scheduler-free evaluation workers, outcomes, and reporting
|
|
57
|
+
└── datadev # data recipes, materialization, mixtures, and training configs
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
These are not independent packages. Installing `jtc` installs all three under
|
|
61
|
+
the same namespace. The historical top-level imports `jtc_data_commons`,
|
|
62
|
+
`jtceval`, and `datadev` are no longer part of the package API.
|
|
63
|
+
|
|
64
|
+
JTC implements data and evaluation work. It does not own training or
|
|
65
|
+
evaluation infrastructure. PrimeBeaker loads JTC's Python workers, creates the
|
|
66
|
+
LiteRegistry service topology, and schedules the jobs. RL trace inspection and
|
|
67
|
+
export come from the standalone
|
|
68
|
+
[`rltracer`](https://github.com/goncalorafaria/rltracer) package.
|
|
69
|
+
Checkpoint and evaluation provenance indexing comes from
|
|
70
|
+
`primebeaker watcher`.
|
|
71
|
+
|
|
72
|
+
## Install
|
|
73
|
+
|
|
74
|
+
Install the released `jtc-core` distribution (Python imports and commands remain `jtc`):
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
pip install jtc-core
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
JTC requires Python 3.12.
|
|
81
|
+
|
|
82
|
+
For development from this checkout:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
pip install -e .
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Install every optional JTC stack for development with:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
pip install '.[all]'
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Available extras are:
|
|
95
|
+
|
|
96
|
+
- `evals`: OpenAI, PyArrow, and RLTracer support used by evaluation tooling.
|
|
97
|
+
- `harness`: Verifiers/Harbor support used by verifier workflows.
|
|
98
|
+
- `search`: Pyserini for local-search workloads.
|
|
99
|
+
- `trace`: RLTracer without the other optional stacks.
|
|
100
|
+
- `all`: all optional JTC dependencies.
|
|
101
|
+
|
|
102
|
+
PrimeBeaker is intentionally not a JTC dependency. A runtime that schedules
|
|
103
|
+
JTC work installs PrimeBeaker separately; the supported evaluation image pins
|
|
104
|
+
and installs released versions of both packages from PyPI.
|
|
105
|
+
|
|
106
|
+
## Package boundaries
|
|
107
|
+
|
|
108
|
+
- `jtc.common`: schemas, parquet IO, rubric execution, source adapters,
|
|
109
|
+
shared tool integration, prompt templates, and registered Python workflows.
|
|
110
|
+
- `jtc.eval`: evaluation workload contracts and workers, outcome archives,
|
|
111
|
+
audits, plots, and Evalboard export.
|
|
112
|
+
- `jtc.datadev`: typed SFT/RL data recipes, materialization, Pot sampling and
|
|
113
|
+
optimization, and training-TOML generation.
|
|
114
|
+
|
|
115
|
+
Verifier environments and reward functions live in
|
|
116
|
+
`primebeaker.environments`. Scheduling, dynamic ports, LiteRegistry deployment,
|
|
117
|
+
readiness, supervision, resume, and shutdown coordination also live in
|
|
118
|
+
PrimeBeaker. Runtime records, datasets, checkpoints, and evaluation results
|
|
119
|
+
stay outside this repository.
|
|
120
|
+
|
|
121
|
+
## Data and training-configuration flow
|
|
122
|
+
|
|
123
|
+
`jtc.datadev` turns versioned YAML data recipes into materialized SFT or RL
|
|
124
|
+
datasets and then renders a compatible training TOML. It does not launch
|
|
125
|
+
training:
|
|
126
|
+
|
|
127
|
+
```text
|
|
128
|
+
data recipe YAML -> materialized dataset + manifest -> training TOML -> PrimeBeaker
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
The CLI is implemented with Fire:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
# Validate without writing data.
|
|
135
|
+
datadev data sft examples/datadev/sft-data-smoke.yaml --dry-run
|
|
136
|
+
|
|
137
|
+
# Materialize data and render the corresponding training TOML.
|
|
138
|
+
datadev prepare rl \
|
|
139
|
+
examples/datadev/rl-data-smoke.yaml \
|
|
140
|
+
examples/datadev/rl.toml \
|
|
141
|
+
/weka/gfaria/generated/example-rl.toml
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Canonical recipe configs live under `examples/configs/data/{sft,rl}` and
|
|
145
|
+
training configs under `examples/configs/train/{sft,rl}`. The small files under
|
|
146
|
+
`examples/datadev` demonstrate the formats. Generated datasets and manifests
|
|
147
|
+
must use an external artifact directory; the resulting TOML is handed to
|
|
148
|
+
PrimeBeaker for SFT or RL scheduling.
|
|
149
|
+
|
|
150
|
+
The `jtc` command exposes common parquet inspection and sampling operations:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
jtc inspect /weka/gfaria/records/example.parquet
|
|
154
|
+
jtc sample-jrows INPUT.parquet OUTPUT.parquet --max-input-objects=100
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
In Python, import through the unified namespace:
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
from jtc.common.schema import JRecord, JRow
|
|
161
|
+
from jtc.datadev import Pot, build_data
|
|
162
|
+
from jtc.eval import EvaluationArchive, EvaluationWorkloadConfig
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## Python-only evaluation workloads
|
|
166
|
+
|
|
167
|
+
`jtc.eval` owns benchmark selection and evaluation logic. PrimeBeaker owns the
|
|
168
|
+
YAML descriptions and all infrastructure lifecycle behavior: Beaker scheduling,
|
|
169
|
+
native LiteRegistry deployment, endpoint readiness, signal forwarding, and
|
|
170
|
+
cleanup. The dependency direction is PrimeBeaker -> JTC.
|
|
171
|
+
|
|
172
|
+
The evaluator always runs in a Beaker image containing released PyPI wheels;
|
|
173
|
+
JTC source is never imported or copied into the image. Build and publish the
|
|
174
|
+
pinned image using JTC's `docker/Dockerfile.eval`, then put its immutable
|
|
175
|
+
`beaker://...` URI in the PrimeBeaker-owned eval YAML or pass it with `--image`:
|
|
176
|
+
|
|
177
|
+
```yaml
|
|
178
|
+
image: beaker://IMMUTABLE_PRIMEBEAKER_JTC_IMAGE
|
|
179
|
+
workspace: ai2/oe-agents-holmes
|
|
180
|
+
budget: ai2/oe-omai
|
|
181
|
+
evaluation:
|
|
182
|
+
model: /weka/gfaria/models/policy/step_400
|
|
183
|
+
run_dir: /weka/gfaria/jtc-artifacts/evals/policy-step-400-smoke
|
|
184
|
+
datasets: [healthbench]
|
|
185
|
+
max_rows: 4
|
|
186
|
+
runs: 1
|
|
187
|
+
verifier_tools: terminal
|
|
188
|
+
verifier_prompt_chat_template_path: templates/jtc-io-terminal-release.json
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Lifecycle YAMLs live in PrimeBeaker under `examples/configs/eval`. Preview and
|
|
192
|
+
submit through PrimeBeaker's Fire CLI:
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
primebeaker evaluation preview --config=examples/configs/eval/python_only_smoke.yaml
|
|
196
|
+
primebeaker evaluation submit --config=examples/configs/eval/python_only_smoke.yaml
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
PrimeBeaker selects a packaged Python entry point for each workload:
|
|
200
|
+
|
|
201
|
+
- standard benchmark evaluation: `jtc.eval.worker`
|
|
202
|
+
- search-agent evaluation: `jtc.eval.search_worker`
|
|
203
|
+
- any registered JTC workflow: `jtc.eval.workflow_worker`
|
|
204
|
+
- iterative rejection sampling: `jtc.eval.rejection_sampling_worker`
|
|
205
|
+
- rubric-judge audit: `jtc.eval.rubric_judge_audit_worker`
|
|
206
|
+
|
|
207
|
+
The generic workflow worker accepts every stable name exported by
|
|
208
|
+
`jtc.common.workflows.registry`. This includes articulated/grouped submit,
|
|
209
|
+
TMAX/Podman and binary-submit workflows, inference, rubric generation, and the
|
|
210
|
+
other verifier workflows in the registry.
|
|
211
|
+
|
|
212
|
+
JTC validates the selected workflow and executes its Python `main()` while
|
|
213
|
+
PrimeBeaker injects managed registry and gateway endpoints. A newly registered
|
|
214
|
+
workflow therefore uses the existing generic worker rather than a new scheduler
|
|
215
|
+
module.
|
|
216
|
+
|
|
217
|
+
For rejection sampling, PrimeBeaker owns the rounds, service topology, resume,
|
|
218
|
+
and cleanup lifecycle. The JTC worker runs the selected verifier workflow; the
|
|
219
|
+
canonical filtering and accepted-trajectory compilation algorithms live in
|
|
220
|
+
`jtc.common.rejection_sampling`. No launcher or fixed-port infrastructure lives
|
|
221
|
+
in JTC.
|
|
222
|
+
|
|
223
|
+
Templates are runtime data, not image code. A PrimeBeaker YAML can use a newly
|
|
224
|
+
created JSON template by YAML-relative, checkout-relative, or absolute path.
|
|
225
|
+
PrimeBeaker validates it on the submitting machine and writes a content-addressed
|
|
226
|
+
snapshot under `<run_dir>/.jtceval-inputs/templates/` before it submits the
|
|
227
|
+
evaluator. The `/weka` mount exposes that snapshot to the PyPI-only evaluation
|
|
228
|
+
image without rebuilding JTC. A dry run reports the future path without writing.
|
|
229
|
+
|
|
230
|
+
Rubric-judge audits use the same boundary. JTC implements Python sampling and
|
|
231
|
+
judgment recovery while PrimeBeaker schedules the shadow model, registry, and
|
|
232
|
+
gateway. The `jtc.eval.rubric_judge_audit_worker` calls those implementations
|
|
233
|
+
directly:
|
|
234
|
+
|
|
235
|
+
```bash
|
|
236
|
+
primebeaker evaluation preview \
|
|
237
|
+
--config=examples/configs/eval/rubric_judge_audit/qwen35_4b_grouped_vs_qwen36_35b_steps005_400.yaml
|
|
238
|
+
primebeaker evaluation submit \
|
|
239
|
+
--config=examples/configs/eval/rubric_judge_audit/qwen35_4b_grouped_vs_qwen36_35b_steps005_400.yaml
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
The coordinator can resume from a complete sample or response stream and rejects
|
|
243
|
+
partial artifacts unless `--force=True` is explicitly supplied.
|
|
244
|
+
|
|
245
|
+
Prime-RL TOMLs point directly at PrimeBeaker's package-qualified environment
|
|
246
|
+
modules:
|
|
247
|
+
|
|
248
|
+
```toml
|
|
249
|
+
[orchestrator.train.source.legacy]
|
|
250
|
+
id = "primebeaker.environments.jtc_tool_label_env"
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
Prime-RL loads the dotted module path directly. No environment implementation
|
|
254
|
+
is copied into JTC, and there are no JTC-owned harness aliases; callers use the
|
|
255
|
+
PrimeBeaker module path or its environment registry.
|
|
256
|
+
|
|
257
|
+
## Discover and run workflows
|
|
258
|
+
|
|
259
|
+
```bash
|
|
260
|
+
jtc-workflow list
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
Programmatically:
|
|
264
|
+
|
|
265
|
+
```python
|
|
266
|
+
from jtc.common.workflows import load_workflow
|
|
267
|
+
|
|
268
|
+
workflow = load_workflow("verifier-tool-use")
|
|
269
|
+
workflow.main(...)
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
Only import-tested workflows are registered. Broken historical variants are
|
|
273
|
+
kept in the repository for provenance but excluded from the wheel registry.
|
|
274
|
+
|
|
275
|
+
## Local evaluation utilities
|
|
276
|
+
|
|
277
|
+
The `jtceval` CLI does not schedule work. It imports and summarizes existing
|
|
278
|
+
evaluation outputs; lifecycle operations stay in PrimeBeaker:
|
|
279
|
+
|
|
280
|
+
```bash
|
|
281
|
+
jtceval import-evaluations SOURCE_DIRECTORY OUTPUT.json --dry-run
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
The distribution also installs these evaluation-side entry points:
|
|
285
|
+
|
|
286
|
+
- `jtceval-evalboard`: Evalboard export and local site tooling.
|
|
287
|
+
- `jtceval-bon` and `jtceval-bon-plot`: best-of-N configuration and plotting.
|
|
288
|
+
|
|
289
|
+
All JTC CLIs use Fire. Scheduling commands remain under
|
|
290
|
+
`primebeaker evaluation`, `primebeaker sft`, and `primebeaker rl`.
|
|
291
|
+
|
|
292
|
+
## Discover verifier environments
|
|
293
|
+
|
|
294
|
+
Programmatically:
|
|
295
|
+
|
|
296
|
+
```python
|
|
297
|
+
from primebeaker.environments import load_environment_module
|
|
298
|
+
|
|
299
|
+
module = load_environment_module("jtc-tool-label")
|
|
300
|
+
environment = module.load_environment(dataset="/path/to/train.jsonl")
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
The stable direct import for the same implementation is:
|
|
304
|
+
|
|
305
|
+
```python
|
|
306
|
+
from primebeaker.environments.jtc_tool_label_env import (
|
|
307
|
+
load_environment,
|
|
308
|
+
)
|
|
309
|
+
```
|
|
310
|
+
|
|
311
|
+
## Runtime prompt assets
|
|
312
|
+
|
|
313
|
+
Workflow defaults still use familiar paths such as
|
|
314
|
+
`templates/search-agent.json`. `resolve_resource_path()` first honors an
|
|
315
|
+
existing caller-provided path and then falls back to templates bundled in the
|
|
316
|
+
wheel. This preserves repository overrides while allowing jobs to run from any
|
|
317
|
+
working directory.
|
|
318
|
+
|
|
319
|
+
```python
|
|
320
|
+
from jtc.common import load_prompt_template, resolve_resource_path
|
|
321
|
+
|
|
322
|
+
path = resolve_resource_path("templates/search-agent.json")
|
|
323
|
+
template = load_prompt_template(path)
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
## Release and PyPI
|
|
327
|
+
|
|
328
|
+
`.github/workflows/publish.yml` uses PyPI Trusted Publishing. Publishing a
|
|
329
|
+
GitHub release builds both the source distribution and wheel with Python 3.12,
|
|
330
|
+
then uploads them using GitHub's short-lived OIDC identity; no API token is
|
|
331
|
+
stored in this repository.
|
|
332
|
+
|
|
333
|
+
Create the `pypi` environment in GitHub Settings → Environments. For the first
|
|
334
|
+
release, add a pending publisher in [PyPI account publishing](https://pypi.org/manage/account/publishing/)
|
|
335
|
+
with these values:
|
|
336
|
+
|
|
337
|
+
- PyPI project name: `jtc-core`
|
|
338
|
+
- GitHub owner: `goncalorafaria`
|
|
339
|
+
- GitHub repository: `jtc`
|
|
340
|
+
- Workflow: `publish.yml`
|
|
341
|
+
- Environment: `pypi`
|
|
342
|
+
|
|
343
|
+
Publish by updating the version in `pyproject.toml`, merging that commit, and
|
|
344
|
+
creating a GitHub release whose tag matches the version, such as `v0.3.0`.
|
|
345
|
+
The release event runs the publishing workflow. The tag must be `v<version>`
|
|
346
|
+
or `<version>` and match `pyproject.toml`; a mismatch stops the build.
|
|
347
|
+
The first successful upload creates the PyPI project automatically. For an
|
|
348
|
+
existing project, add the same publisher in its PyPI publishing settings.
|
|
349
|
+
|
|
350
|
+
Manual workflow runs build and validate artifacts only. Builds use strict
|
|
351
|
+
Twine metadata checks; publishing runs in a separate job with OIDC permission.
|
|
352
|
+
Draft releases and tag pushes alone do not publish a package.
|
|
353
|
+
|
|
354
|
+
Before creating the release, run:
|
|
355
|
+
|
|
356
|
+
```bash
|
|
357
|
+
python -m pytest -q tests/test_packaging.py tests/test_config_layout.py
|
|
358
|
+
python -m build
|
|
359
|
+
python -m twine check dist/*
|
|
360
|
+
```
|
|
361
|
+
|
|
362
|
+
The package tests import every registry entry, verify the unified layout and
|
|
363
|
+
packaged template fallback, and catch stale names before a Beaker rollout.
|
|
364
|
+
|
|
365
|
+
PrimeBeaker owns the watcher implementation and command. The standalone
|
|
366
|
+
`rltracer` project is versioned and released independently from JTC.
|
jtc_core-0.3.0/README.md
ADDED
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
# JTC
|
|
2
|
+
|
|
3
|
+
`jtc` is one Python distribution with one import namespace and three
|
|
4
|
+
cooperating subpackages:
|
|
5
|
+
|
|
6
|
+
```text
|
|
7
|
+
jtc
|
|
8
|
+
├── common # shared data models, IO, sources, templates, and workflows
|
|
9
|
+
├── eval # scheduler-free evaluation workers, outcomes, and reporting
|
|
10
|
+
└── datadev # data recipes, materialization, mixtures, and training configs
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
These are not independent packages. Installing `jtc` installs all three under
|
|
14
|
+
the same namespace. The historical top-level imports `jtc_data_commons`,
|
|
15
|
+
`jtceval`, and `datadev` are no longer part of the package API.
|
|
16
|
+
|
|
17
|
+
JTC implements data and evaluation work. It does not own training or
|
|
18
|
+
evaluation infrastructure. PrimeBeaker loads JTC's Python workers, creates the
|
|
19
|
+
LiteRegistry service topology, and schedules the jobs. RL trace inspection and
|
|
20
|
+
export come from the standalone
|
|
21
|
+
[`rltracer`](https://github.com/goncalorafaria/rltracer) package.
|
|
22
|
+
Checkpoint and evaluation provenance indexing comes from
|
|
23
|
+
`primebeaker watcher`.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
Install the released `jtc-core` distribution (Python imports and commands remain `jtc`):
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install jtc-core
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
JTC requires Python 3.12.
|
|
34
|
+
|
|
35
|
+
For development from this checkout:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install -e .
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Install every optional JTC stack for development with:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install '.[all]'
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Available extras are:
|
|
48
|
+
|
|
49
|
+
- `evals`: OpenAI, PyArrow, and RLTracer support used by evaluation tooling.
|
|
50
|
+
- `harness`: Verifiers/Harbor support used by verifier workflows.
|
|
51
|
+
- `search`: Pyserini for local-search workloads.
|
|
52
|
+
- `trace`: RLTracer without the other optional stacks.
|
|
53
|
+
- `all`: all optional JTC dependencies.
|
|
54
|
+
|
|
55
|
+
PrimeBeaker is intentionally not a JTC dependency. A runtime that schedules
|
|
56
|
+
JTC work installs PrimeBeaker separately; the supported evaluation image pins
|
|
57
|
+
and installs released versions of both packages from PyPI.
|
|
58
|
+
|
|
59
|
+
## Package boundaries
|
|
60
|
+
|
|
61
|
+
- `jtc.common`: schemas, parquet IO, rubric execution, source adapters,
|
|
62
|
+
shared tool integration, prompt templates, and registered Python workflows.
|
|
63
|
+
- `jtc.eval`: evaluation workload contracts and workers, outcome archives,
|
|
64
|
+
audits, plots, and Evalboard export.
|
|
65
|
+
- `jtc.datadev`: typed SFT/RL data recipes, materialization, Pot sampling and
|
|
66
|
+
optimization, and training-TOML generation.
|
|
67
|
+
|
|
68
|
+
Verifier environments and reward functions live in
|
|
69
|
+
`primebeaker.environments`. Scheduling, dynamic ports, LiteRegistry deployment,
|
|
70
|
+
readiness, supervision, resume, and shutdown coordination also live in
|
|
71
|
+
PrimeBeaker. Runtime records, datasets, checkpoints, and evaluation results
|
|
72
|
+
stay outside this repository.
|
|
73
|
+
|
|
74
|
+
## Data and training-configuration flow
|
|
75
|
+
|
|
76
|
+
`jtc.datadev` turns versioned YAML data recipes into materialized SFT or RL
|
|
77
|
+
datasets and then renders a compatible training TOML. It does not launch
|
|
78
|
+
training:
|
|
79
|
+
|
|
80
|
+
```text
|
|
81
|
+
data recipe YAML -> materialized dataset + manifest -> training TOML -> PrimeBeaker
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The CLI is implemented with Fire:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
# Validate without writing data.
|
|
88
|
+
datadev data sft examples/datadev/sft-data-smoke.yaml --dry-run
|
|
89
|
+
|
|
90
|
+
# Materialize data and render the corresponding training TOML.
|
|
91
|
+
datadev prepare rl \
|
|
92
|
+
examples/datadev/rl-data-smoke.yaml \
|
|
93
|
+
examples/datadev/rl.toml \
|
|
94
|
+
/weka/gfaria/generated/example-rl.toml
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Canonical recipe configs live under `examples/configs/data/{sft,rl}` and
|
|
98
|
+
training configs under `examples/configs/train/{sft,rl}`. The small files under
|
|
99
|
+
`examples/datadev` demonstrate the formats. Generated datasets and manifests
|
|
100
|
+
must use an external artifact directory; the resulting TOML is handed to
|
|
101
|
+
PrimeBeaker for SFT or RL scheduling.
|
|
102
|
+
|
|
103
|
+
The `jtc` command exposes common parquet inspection and sampling operations:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
jtc inspect /weka/gfaria/records/example.parquet
|
|
107
|
+
jtc sample-jrows INPUT.parquet OUTPUT.parquet --max-input-objects=100
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
In Python, import through the unified namespace:
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from jtc.common.schema import JRecord, JRow
|
|
114
|
+
from jtc.datadev import Pot, build_data
|
|
115
|
+
from jtc.eval import EvaluationArchive, EvaluationWorkloadConfig
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Python-only evaluation workloads
|
|
119
|
+
|
|
120
|
+
`jtc.eval` owns benchmark selection and evaluation logic. PrimeBeaker owns the
|
|
121
|
+
YAML descriptions and all infrastructure lifecycle behavior: Beaker scheduling,
|
|
122
|
+
native LiteRegistry deployment, endpoint readiness, signal forwarding, and
|
|
123
|
+
cleanup. The dependency direction is PrimeBeaker -> JTC.
|
|
124
|
+
|
|
125
|
+
The evaluator always runs in a Beaker image containing released PyPI wheels;
|
|
126
|
+
JTC source is never imported or copied into the image. Build and publish the
|
|
127
|
+
pinned image using JTC's `docker/Dockerfile.eval`, then put its immutable
|
|
128
|
+
`beaker://...` URI in the PrimeBeaker-owned eval YAML or pass it with `--image`:
|
|
129
|
+
|
|
130
|
+
```yaml
|
|
131
|
+
image: beaker://IMMUTABLE_PRIMEBEAKER_JTC_IMAGE
|
|
132
|
+
workspace: ai2/oe-agents-holmes
|
|
133
|
+
budget: ai2/oe-omai
|
|
134
|
+
evaluation:
|
|
135
|
+
model: /weka/gfaria/models/policy/step_400
|
|
136
|
+
run_dir: /weka/gfaria/jtc-artifacts/evals/policy-step-400-smoke
|
|
137
|
+
datasets: [healthbench]
|
|
138
|
+
max_rows: 4
|
|
139
|
+
runs: 1
|
|
140
|
+
verifier_tools: terminal
|
|
141
|
+
verifier_prompt_chat_template_path: templates/jtc-io-terminal-release.json
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Lifecycle YAMLs live in PrimeBeaker under `examples/configs/eval`. Preview and
|
|
145
|
+
submit through PrimeBeaker's Fire CLI:
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
primebeaker evaluation preview --config=examples/configs/eval/python_only_smoke.yaml
|
|
149
|
+
primebeaker evaluation submit --config=examples/configs/eval/python_only_smoke.yaml
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
PrimeBeaker selects a packaged Python entry point for each workload:
|
|
153
|
+
|
|
154
|
+
- standard benchmark evaluation: `jtc.eval.worker`
|
|
155
|
+
- search-agent evaluation: `jtc.eval.search_worker`
|
|
156
|
+
- any registered JTC workflow: `jtc.eval.workflow_worker`
|
|
157
|
+
- iterative rejection sampling: `jtc.eval.rejection_sampling_worker`
|
|
158
|
+
- rubric-judge audit: `jtc.eval.rubric_judge_audit_worker`
|
|
159
|
+
|
|
160
|
+
The generic workflow worker accepts every stable name exported by
|
|
161
|
+
`jtc.common.workflows.registry`. This includes articulated/grouped submit,
|
|
162
|
+
TMAX/Podman and binary-submit workflows, inference, rubric generation, and the
|
|
163
|
+
other verifier workflows in the registry.
|
|
164
|
+
|
|
165
|
+
JTC validates the selected workflow and executes its Python `main()` while
|
|
166
|
+
PrimeBeaker injects managed registry and gateway endpoints. A newly registered
|
|
167
|
+
workflow therefore uses the existing generic worker rather than a new scheduler
|
|
168
|
+
module.
|
|
169
|
+
|
|
170
|
+
For rejection sampling, PrimeBeaker owns the rounds, service topology, resume,
|
|
171
|
+
and cleanup lifecycle. The JTC worker runs the selected verifier workflow; the
|
|
172
|
+
canonical filtering and accepted-trajectory compilation algorithms live in
|
|
173
|
+
`jtc.common.rejection_sampling`. No launcher or fixed-port infrastructure lives
|
|
174
|
+
in JTC.
|
|
175
|
+
|
|
176
|
+
Templates are runtime data, not image code. A PrimeBeaker YAML can use a newly
|
|
177
|
+
created JSON template by YAML-relative, checkout-relative, or absolute path.
|
|
178
|
+
PrimeBeaker validates it on the submitting machine and writes a content-addressed
|
|
179
|
+
snapshot under `<run_dir>/.jtceval-inputs/templates/` before it submits the
|
|
180
|
+
evaluator. The `/weka` mount exposes that snapshot to the PyPI-only evaluation
|
|
181
|
+
image without rebuilding JTC. A dry run reports the future path without writing.
|
|
182
|
+
|
|
183
|
+
Rubric-judge audits use the same boundary. JTC implements Python sampling and
|
|
184
|
+
judgment recovery while PrimeBeaker schedules the shadow model, registry, and
|
|
185
|
+
gateway. The `jtc.eval.rubric_judge_audit_worker` calls those implementations
|
|
186
|
+
directly:
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
primebeaker evaluation preview \
|
|
190
|
+
--config=examples/configs/eval/rubric_judge_audit/qwen35_4b_grouped_vs_qwen36_35b_steps005_400.yaml
|
|
191
|
+
primebeaker evaluation submit \
|
|
192
|
+
--config=examples/configs/eval/rubric_judge_audit/qwen35_4b_grouped_vs_qwen36_35b_steps005_400.yaml
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
The coordinator can resume from a complete sample or response stream and rejects
|
|
196
|
+
partial artifacts unless `--force=True` is explicitly supplied.
|
|
197
|
+
|
|
198
|
+
Prime-RL TOMLs point directly at PrimeBeaker's package-qualified environment
|
|
199
|
+
modules:
|
|
200
|
+
|
|
201
|
+
```toml
|
|
202
|
+
[orchestrator.train.source.legacy]
|
|
203
|
+
id = "primebeaker.environments.jtc_tool_label_env"
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Prime-RL loads the dotted module path directly. No environment implementation
|
|
207
|
+
is copied into JTC, and there are no JTC-owned harness aliases; callers use the
|
|
208
|
+
PrimeBeaker module path or its environment registry.
|
|
209
|
+
|
|
210
|
+
## Discover and run workflows
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
jtc-workflow list
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
Programmatically:
|
|
217
|
+
|
|
218
|
+
```python
|
|
219
|
+
from jtc.common.workflows import load_workflow
|
|
220
|
+
|
|
221
|
+
workflow = load_workflow("verifier-tool-use")
|
|
222
|
+
workflow.main(...)
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Only import-tested workflows are registered. Broken historical variants are
|
|
226
|
+
kept in the repository for provenance but excluded from the wheel registry.
|
|
227
|
+
|
|
228
|
+
## Local evaluation utilities
|
|
229
|
+
|
|
230
|
+
The `jtceval` CLI does not schedule work. It imports and summarizes existing
|
|
231
|
+
evaluation outputs; lifecycle operations stay in PrimeBeaker:
|
|
232
|
+
|
|
233
|
+
```bash
|
|
234
|
+
jtceval import-evaluations SOURCE_DIRECTORY OUTPUT.json --dry-run
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
The distribution also installs these evaluation-side entry points:
|
|
238
|
+
|
|
239
|
+
- `jtceval-evalboard`: Evalboard export and local site tooling.
|
|
240
|
+
- `jtceval-bon` and `jtceval-bon-plot`: best-of-N configuration and plotting.
|
|
241
|
+
|
|
242
|
+
All JTC CLIs use Fire. Scheduling commands remain under
|
|
243
|
+
`primebeaker evaluation`, `primebeaker sft`, and `primebeaker rl`.
|
|
244
|
+
|
|
245
|
+
## Discover verifier environments
|
|
246
|
+
|
|
247
|
+
Programmatically:
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
from primebeaker.environments import load_environment_module
|
|
251
|
+
|
|
252
|
+
module = load_environment_module("jtc-tool-label")
|
|
253
|
+
environment = module.load_environment(dataset="/path/to/train.jsonl")
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
The stable direct import for the same implementation is:
|
|
257
|
+
|
|
258
|
+
```python
|
|
259
|
+
from primebeaker.environments.jtc_tool_label_env import (
|
|
260
|
+
load_environment,
|
|
261
|
+
)
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
## Runtime prompt assets
|
|
265
|
+
|
|
266
|
+
Workflow defaults still use familiar paths such as
|
|
267
|
+
`templates/search-agent.json`. `resolve_resource_path()` first honors an
|
|
268
|
+
existing caller-provided path and then falls back to templates bundled in the
|
|
269
|
+
wheel. This preserves repository overrides while allowing jobs to run from any
|
|
270
|
+
working directory.
|
|
271
|
+
|
|
272
|
+
```python
|
|
273
|
+
from jtc.common import load_prompt_template, resolve_resource_path
|
|
274
|
+
|
|
275
|
+
path = resolve_resource_path("templates/search-agent.json")
|
|
276
|
+
template = load_prompt_template(path)
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
## Release and PyPI
|
|
280
|
+
|
|
281
|
+
`.github/workflows/publish.yml` uses PyPI Trusted Publishing. Publishing a
|
|
282
|
+
GitHub release builds both the source distribution and wheel with Python 3.12,
|
|
283
|
+
then uploads them using GitHub's short-lived OIDC identity; no API token is
|
|
284
|
+
stored in this repository.
|
|
285
|
+
|
|
286
|
+
Create the `pypi` environment in GitHub Settings → Environments. For the first
|
|
287
|
+
release, add a pending publisher in [PyPI account publishing](https://pypi.org/manage/account/publishing/)
|
|
288
|
+
with these values:
|
|
289
|
+
|
|
290
|
+
- PyPI project name: `jtc-core`
|
|
291
|
+
- GitHub owner: `goncalorafaria`
|
|
292
|
+
- GitHub repository: `jtc`
|
|
293
|
+
- Workflow: `publish.yml`
|
|
294
|
+
- Environment: `pypi`
|
|
295
|
+
|
|
296
|
+
Publish by updating the version in `pyproject.toml`, merging that commit, and
|
|
297
|
+
creating a GitHub release whose tag matches the version, such as `v0.3.0`.
|
|
298
|
+
The release event runs the publishing workflow. The tag must be `v<version>`
|
|
299
|
+
or `<version>` and match `pyproject.toml`; a mismatch stops the build.
|
|
300
|
+
The first successful upload creates the PyPI project automatically. For an
|
|
301
|
+
existing project, add the same publisher in its PyPI publishing settings.
|
|
302
|
+
|
|
303
|
+
Manual workflow runs build and validate artifacts only. Builds use strict
|
|
304
|
+
Twine metadata checks; publishing runs in a separate job with OIDC permission.
|
|
305
|
+
Draft releases and tag pushes alone do not publish a package.
|
|
306
|
+
|
|
307
|
+
Before creating the release, run:
|
|
308
|
+
|
|
309
|
+
```bash
|
|
310
|
+
python -m pytest -q tests/test_packaging.py tests/test_config_layout.py
|
|
311
|
+
python -m build
|
|
312
|
+
python -m twine check dist/*
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
The package tests import every registry entry, verify the unified layout and
|
|
316
|
+
packaged template fallback, and catch stale names before a Beaker rollout.
|
|
317
|
+
|
|
318
|
+
PrimeBeaker owns the watcher implementation and command. The standalone
|
|
319
|
+
`rltracer` project is versioned and released independently from JTC.
|