jtc-core 0.3.0__py3-none-any.whl

jtc/__init__.py

@@ -0,0 +1,5 @@
+"""Unified JTC data, evaluation, and configuration toolkit."""
+
+from __future__ import annotations
+
+__all__ = ["common", "datadev", "eval"]

jtc/common/__init__.py

@@ -0,0 +1,259 @@
+"""Common schema and IO helpers for Judges That Code datasets."""
+
+from importlib import import_module
+from typing import Any
+
+from jtcflow.model import Assertion, LLMResponse
+
+from jtc.common.schema import (
+    BINARY_RUBRIC_SCORES,
+    CodeExecution,
+    FeedbackExecutableRubricJudgment,
+    ExecutableRubricJudgment,
+    ExecutableRubric,
+    JRecordSet,
+    JRecord,
+    JRow,
+    JTask,
+    MultiStepCodeStep,
+    MultiStepRubricJudgment,
+    MultiStepRubricTurn,
+    RubricJudgment,
+    RubricItem,
+    SerializableRubric,
+    RubricDefinition,
+    RubricType,
+    SourceInfo,
+)
+from jtc.common.templates import (
+    HasFeedbackLabel,
+    coerce_chat_messages,
+    format_rubric_dict,
+    load_prompt_template,
+    parse_feedback_label,
+    render_chat_template,
+    resolve_resource_path,
+)
+from jtc.common.parquet import (
+    examples_to_dataset,
+    examples_to_rows,
+    jrow_from_dict,
+    load_parquet_objects,
+    read_parquet,
+    write_parquet,
+)
+from jtc.common.scoring import (
+    RubricScorer,
+    UniformScorer,
+)
+
+_LAZY_EXPORTS = {
+    "sample_jrecords": ("jtc.common.jrecord_sampling", "sample_jrecords"),
+    "sample_jrecords_file": (
+        "jtc.common.jrecord_sampling",
+        "sample_jrecords_file",
+    ),
+    "selected_output_indexes": (
+        "jtc.common.jrecord_sampling",
+        "selected_output_indexes",
+    ),
+    "sample_jrows": ("jtc.common.jrow_sampling", "sample_jrows"),
+    "sample_jrows_file": ("jtc.common.jrow_sampling", "sample_jrows_file"),
+    "selected_indexes": ("jtc.common.jrow_sampling", "selected_indexes"),
+    "AssetStore": ("literegistry_tool_client", "AssetStore"),
+    "WebAssetStore": ("literegistry_tool_client", "WebAssetStore"),
+    "CodeExecutionMap": ("jtc.common.code_execution", "CodeExecutionMap"),
+    "WebTerminalExecutionClient": (
+        "literegistry_tool_client",
+        "WebTerminalExecutionClient",
+    ),
+    "FetchClient": ("literegistry_tool_client", "FetchClient"),
+    "JudgeClient": ("literegistry_tool_client", "JudgeClient"),
+    "MultiToolExecutionMap": (
+        "jtc.common.code_execution",
+        "MultiToolExecutionMap",
+    ),
+    "ParallelMultiToolExecutionMap": (
+        "jtc.common.code_execution",
+        "ParallelMultiToolExecutionMap",
+    ),
+    "PodmanExecutionClient": (
+        "literegistry_tool_client",
+        "PodmanExecutionClient",
+    ),
+    "RemoteCodeExecutionClient": (
+        "jtc.common.code_execution",
+        "RemoteCodeExecutionClient",
+    ),
+    "SearchClient": ("literegistry_tool_client", "SearchClient"),
+    "TerminalExecutionClient": (
+        "literegistry_tool_client",
+        "TerminalExecutionClient",
+    ),
+    "ToolClient": ("literegistry_tool_client", "ToolClient"),
+    "SubmissionStore": ("literegistry_tool_client", "SubmissionStore"),
+    "SubmitToolClient": ("literegistry_tool_client", "SubmitToolClient"),
+    "ToolExecutionMap": ("jtc.common.code_execution", "ToolExecutionMap"),
+    "ToolOutputDisplay": ("literegistry_tool_client", "ToolOutputDisplay"),
+    "fast_forward_container_state": (
+        "jtc.common.tmax_replay",
+        "fast_forward_container_state",
+    ),
+    "ContextBoundPodmanTerminalClient": (
+        "jtc.common.podman_verifier",
+        "ContextBoundPodmanTerminalClient",
+    ),
+    "PodmanVerifierConfig": (
+        "jtc.common.podman_verifier",
+        "PodmanVerifierConfig",
+    ),
+    "PodmanVerifierHarness": (
+        "jtc.common.podman_verifier",
+        "PodmanVerifierHarness",
+    ),
+    "PodmanVerifierRun": (
+        "jtc.common.podman_verifier",
+        "PodmanVerifierRun",
+    ),
+    "FencedToolOutputDisplay": ("literegistry_tool_client", "FencedToolOutputDisplay"),
+    "PlainJsonToolOutputDisplay": ("literegistry_tool_client", "PlainJsonToolOutputDisplay"),
+    "ExecutableRowProcessor": ("jtc.common.process", "ExecutableRowProcessor"),
+    "GenerationWorkflowSpec": (
+        "jtc.common.generation_workflow",
+        "GenerationWorkflowSpec",
+    ),
+    "GenericGenerationWorkflow": (
+        "jtc.common.generation_workflow",
+        "GenericGenerationWorkflow",
+    ),
+    "execute_executable_rubric_async": (
+        "jtc.common.rubric_execution",
+        "execute_executable_rubric_async",
+    ),
+    "execute_executable_row_async": (
+        "jtc.common.rubric_execution",
+        "execute_executable_row_async",
+    ),
+    "execute_record_async": ("jtc.common.rubric_execution", "execute_record_async"),
+    "executable_judgment_from_code_output": (
+        "jtc.common.rubric_execution",
+        "executable_judgment_from_code_output",
+    ),
+    "get_verification_results": (
+        "jtc.common.rubric_execution",
+        "get_verification_results",
+    ),
+    "load_input_rows": ("jtc.common.generation_workflow", "load_input_rows"),
+    "process_executable_record_async": (
+        "jtc.common.process",
+        "process_executable_record_async",
+    ),
+    "process_executable_rows_async": (
+        "jtc.common.process",
+        "process_executable_rows_async",
+    ),
+    "run_generation_workflow": (
+        "jtc.common.generation_workflow",
+        "run_generation_workflow",
+    ),
+    "start_literegistry_gateway": (
+        "jtc.common.generation_workflow",
+        "start_literegistry_gateway",
+    ),
+    "workflow_inputs_from_rows": (
+        "jtc.common.generation_workflow",
+        "workflow_inputs_from_rows",
+    ),
+    "write_jsonl": ("jtc.common.generation_workflow", "write_jsonl"),
+}
+
+
+def __getattr__(name: str) -> Any:
+    if name not in _LAZY_EXPORTS:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    module_name, attr_name = _LAZY_EXPORTS[name]
+    value = getattr(import_module(module_name), attr_name)
+    globals()[name] = value
+    return value
+
+
+__all__ = [
+    "AssetStore",
+    "WebTerminalExecutionClient",
+    "BINARY_RUBRIC_SCORES",
+    "CodeExecution",
+    "CodeExecutionMap",
+    "FeedbackExecutableRubricJudgment",
+    "FetchClient",
+    "JudgeClient",
+    "ExecutableRubricJudgment",
+    "ExecutableRubric",
+    "JRecordSet",
+    "JRecord",
+    "JRow",
+    "JTask",
+    "MultiStepCodeStep",
+    "MultiStepRubricJudgment",
+    "MultiStepRubricTurn",
+    "MultiToolExecutionMap",
+    "ParallelMultiToolExecutionMap",
+    "PodmanExecutionClient",
+    "RubricJudgment",
+    "RubricItem",
+    "SerializableRubric",
+    "RubricDefinition",
+    "RubricType",
+    "SourceInfo",
+    "RemoteCodeExecutionClient",
+    "SearchClient",
+    "TerminalExecutionClient",
+    "ToolClient",
+    "SubmissionStore",
+    "SubmitToolClient",
+    "ToolExecutionMap",
+    "ToolOutputDisplay",
+    "fast_forward_container_state",
+    "ContextBoundPodmanTerminalClient",
+    "PodmanVerifierConfig",
+    "PodmanVerifierHarness",
+    "PodmanVerifierRun",
+    "FencedToolOutputDisplay",
+    "PlainJsonToolOutputDisplay",
+    "execute_executable_rubric_async",
+    "execute_executable_row_async",
+    "execute_record_async",
+    "executable_judgment_from_code_output",
+    "get_verification_results",
+    "ExecutableRowProcessor",
+    "process_executable_record_async",
+    "process_executable_rows_async",
+    "HasFeedbackLabel",
+    "coerce_chat_messages",
+    "format_rubric_dict",
+    "load_prompt_template",
+    "parse_feedback_label",
+    "render_chat_template",
+    "resolve_resource_path",
+    "GenerationWorkflowSpec",
+    "GenericGenerationWorkflow",
+    "load_input_rows",
+    "run_generation_workflow",
+    "workflow_inputs_from_rows",
+    "write_jsonl",
+    "examples_to_dataset",
+    "examples_to_rows",
+    "jrow_from_dict",
+    "load_parquet_objects",
+    "read_parquet",
+    "write_parquet",
+    "sample_jrecords",
+    "sample_jrecords_file",
+    "selected_output_indexes",
+    "sample_jrows",
+    "sample_jrows_file",
+    "selected_indexes",
+    "RubricScorer",
+    "UniformScorer",
+    "WebAssetStore",
+    "start_literegistry_gateway",
+]

jtc/common/asset_store.py

@@ -0,0 +1,9 @@
+"""Compatibility exports for the canonical LiteRegistry asset stores."""
+
+from literegistry_tool_client import (
+    AssetStore,
+    WebAssetStore,
+    normalize_web_page_content,
+)
+
+__all__ = ["AssetStore", "WebAssetStore", "normalize_web_page_content"]

jtc/common/chat_completion.py

@@ -0,0 +1,97 @@
+"""OpenAI-compatible chat-completions client for local and hosted models."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from typing import Any
+
+from jtcflow.model import ChatGPTClient
+
+
+def chat_base_url(server_url: str) -> str:
+    """Return an OpenAI API base URL without duplicating the ``/v1`` suffix."""
+    base = str(server_url).rstrip("/")
+    return base if base.endswith("/v1") else f"{base}/v1"
+
+
+class ChatCompletionClient(ChatGPTClient):
+    """JTCFlow chat client with the former local-client constructor shape.
+
+    Local LiteRegistry/vLLM deployments expose the OpenAI-compatible
+    ``/v1/chat/completions`` endpoint. Keeping this small adapter lets existing
+    workflows migrate without rendering prompts or parsing model-specific wire
+    formats in JTC.
+    """
+
+    def __init__(
+        self,
+        server_url: str,
+        model_path: str,
+        max_new_tokens: int = 1024,
+        max_prompt_length: int = 3072,
+        stop_tokens: list[str] | None = None,
+        temperature: float = 1.0,
+        timeout: float = 300,
+        max_retries: int = 15,
+        max_concurrent_requests: int = 256,
+        tokenizer_path: str | None = None,
+        enable_thinking: bool = False,
+        tools: list[dict[str, Any]] | None = None,
+        api_key: str | None = None,
+        reasoning_effort: str | None = None,
+    ) -> None:
+        # Retain these attributes for workflow diagnostics and compatibility.
+        self.server_url = str(server_url).rstrip("/")
+        self.max_prompt_length = max_prompt_length
+        self.tokenizer_path = tokenizer_path
+        self.temperature = temperature
+        self.enable_thinking = enable_thinking
+        super().__init__(
+            api_key=api_key or os.getenv("OPENAI_API_KEY") or "local",
+            model=model_path,
+            max_new_tokens=max_new_tokens,
+            stop_tokens=stop_tokens,
+            timeout=timeout,
+            max_retries=max_retries,
+            max_concurrent_requests=max_concurrent_requests,
+            base_url=chat_base_url(server_url),
+            tools=tools,
+            reasoning_effort=reasoning_effort,
+        )
+
+    async def _make_request_with_semaphore(
+        self,
+        semaphore: asyncio.Semaphore,
+        messages: list[dict[str, Any]],
+        n: int = 1,
+    ) -> Any:
+        """Issue one chat-completions request with local vLLM options."""
+        async with semaphore:
+            kwargs: dict[str, Any] = {
+                "model": self.model,
+                "messages": messages,
+                "max_completion_tokens": self.max_new_tokens,
+                "n": n,
+                "temperature": self.temperature,
+                "extra_body": {
+                    "chat_template_kwargs": {
+                        "enable_thinking": self.enable_thinking,
+                    }
+                },
+            }
+            if self.stop_tokens:
+                kwargs["stop"] = self.stop_tokens
+            if self.tools is not None:
+                kwargs["tools"] = self.tools
+                kwargs["tool_choice"] = "auto"
+            effort = self._chat_completions_reasoning_effort()
+            if effort is not None:
+                kwargs["reasoning_effort"] = effort
+            try:
+                return await self.client.chat.completions.create(**kwargs)
+            except Exception as exc:
+                raise RuntimeError(f"Chat completions request failed: {exc}") from exc
+
+
+__all__ = ["ChatCompletionClient", "chat_base_url"]

jtc/common/cli.py

@@ -0,0 +1,151 @@
+"""Command line helpers for inspecting JTC data files."""
+
+from __future__ import annotations
+
+import json
+import textwrap
+from pathlib import Path
+from typing import Any
+
+import fire
+
+from jtc.common.jrow_sampling import sample_jrows_file
+from jtc.common.jrecord_sampling import sample_jrecords_file
+from jtc.common.parquet import DataRecord, load_parquet_objects
+from jtc.common.schema import JRecord, JRow
+
+
+def _shorten(value: Any, *, width: int = 1000) -> str:
+    if value is None:
+        return ""
+    text = str(value)
+    if "\n" in text:
+        return text if len(text) <= width else text[: width - 4] + " ..."
+    return textwrap.shorten(
+        text,
+        width=width,
+        placeholder=" ...",
+    )
+
+
+def _first_jrow(objects: list[DataRecord]) -> JRow | None:
+    for obj in objects:
+        if isinstance(obj, JRow):
+            return obj
+        if isinstance(obj, JRecord):
+            rows = obj.to_rows()
+            if rows:
+                return rows[0]
+    return None
+
+
+def _print_field(name: str, value: Any, *, width: int = 1000) -> None:
+    text = _shorten(value, width=width)
+    if "\n" not in text:
+        print(f"{name}: {text}")
+        return
+    print(f"{name}:")
+    print(textwrap.indent(text, "  "))
+
+
+def _print_jrow(row: JRow) -> None:
+    rubric = row.rubric.to_dict()
+    code_execution = rubric.pop("code_execution", None)
+
+    print("\nFirst JRow")
+    print("-" * 40)
+    _print_field("id", row.id)
+    _print_field("record_id", row.record_id)
+    _print_field("rubric_index", row.rubric_index)
+    _print_field("source", json.dumps(row.source.to_dict(), ensure_ascii=False))
+    _print_field("input", row.input)
+    _print_field("output", row.output)
+    _print_field("reference", row.reference)
+
+    print("\nRubric")
+    print("-" * 40)
+    for key, value in rubric.items():
+        if isinstance(value, dict):
+            value = json.dumps(value, ensure_ascii=False, indent=2)
+        _print_field(key, value)
+
+    if code_execution is not None:
+        print("\nCode Execution")
+        print("-" * 40)
+        if hasattr(code_execution, "to_dict"):
+            code_execution = code_execution.to_dict()
+        for key, value in code_execution.items():
+            _print_field(key, value)
+
+
+def inspect(path: str) -> None:
+    """Print a brief summary and the first JRow in a parquet file."""
+    parquet_path = Path(path)
+    objects = load_parquet_objects(parquet_path)
+    print(f"path: {parquet_path}")
+    print(f"entries: {len(objects)}")
+
+    row = _first_jrow(objects)
+    if row is None:
+        print("\nNo JRow found in this file.")
+        return
+    _print_jrow(row)
+
+
+def sample_rows(
+    input_path: str,
+    output_path: str,
+    seed: int = 55,
+    max_input_objects: int = 0,
+    max_outputs_per_record: int = 1,
+    max_rubrics_per_record: int = 2,
+    keep_empty_outputs: bool = False,
+) -> None:
+    """Sample JRows from preserved JRecordSet and JRecord parquet objects."""
+    stats = sample_jrows_file(
+        input_path,
+        output_path,
+        seed=seed,
+        max_input_objects=max_input_objects,
+        max_outputs_per_record=max_outputs_per_record,
+        max_rubrics_per_record=max_rubrics_per_record,
+        keep_empty_outputs=keep_empty_outputs,
+    )
+    print(json.dumps(stats, indent=2))
+
+
+def sample_records(
+    input_path: str,
+    output_path: str,
+    seed: int = 55,
+    max_input_record_sets: int = 0,
+    max_outputs_per_record: int = 1,
+    selection: str = "random",
+    keep_empty_outputs: bool = False,
+) -> None:
+    """Sample single-output JRecords from preserved JRecordSet parquet objects."""
+    stats = sample_jrecords_file(
+        input_path,
+        output_path,
+        seed=seed,
+        max_input_record_sets=max_input_record_sets,
+        max_outputs_per_record=max_outputs_per_record,
+        selection=selection,
+        keep_empty_outputs=keep_empty_outputs,
+    )
+    print(json.dumps(stats, indent=2))
+
+
+def main(argv: list[str] | None = None) -> None:
+    fire.Fire(
+        {
+            "inspect": inspect,
+            "sample-jrecords": sample_records,
+            "sample-jrows": sample_rows,
+        },
+        command=argv,
+    )
+
+
+if __name__ == "__main__":
+    main()