marin-taskcompendium 0.2.160.dev38062449553__py3-none-any.whl

marin_taskcompendium-0.2.160.dev38062449553.dist-info/METADATA

@@ -0,0 +1,16 @@
+Metadata-Version: 2.5
+Name: marin-taskcompendium
+Version: 0.2.160.dev38062449553
+Summary: Semantic task contracts, submission extraction, and shared grading
+Requires-Python: <3.14,>=3.12
+Requires-Dist: marin-rigging==0.2.160.dev38062449553
+Requires-Dist: marin-shellbox==0.2.160.dev38062449553
+Requires-Dist: marin-verifyit[judge,math,reasoning-gym,schema]==0.2.160.dev38062449553
+Requires-Dist: pydantic>=2.0
+Requires-Dist: tomlkit>=0.13
+Provides-Extra: pipeline
+Requires-Dist: datasets<5.0.0,>=3.1.0; extra == 'pipeline'
+Requires-Dist: marin-finestore==0.2.160.dev38062449553; extra == 'pipeline'
+Requires-Dist: marin-zephyr==0.2.160.dev38062449553; extra == 'pipeline'
+Requires-Dist: numpy>=1.26; extra == 'pipeline'
+Requires-Dist: pyarrow>=15.0; extra == 'pipeline'

marin_taskcompendium-0.2.160.dev38062449553.dist-info/RECORD

@@ -0,0 +1,81 @@
+taskcompendium/__init__.py,sha256=YCa8ajV8oMVGzdM-0ADm_Y7gL7bP-E4DDjDz4V7pBA0,68
+taskcompendium/chat.py,sha256=IrkvB-hvvZzI4E39NKkJzJnjBJ5pTmzZux883o9_A4g,3194
+taskcompendium/direct_chat.py,sha256=BRQYWrXl7NTH9abN_NovNSgLv6zKj1S8Wjg5RcZFK5c,1788
+taskcompendium/grader.py,sha256=oOqvfBzw_vPOiFRvzfTtL0CEF1HZXKXerJVCUb-oFXE,1613
+taskcompendium/grading.py,sha256=AL0uK07DN4UO8_sb3VOUOWgxkNVH8z6VYaeGfZN1jyY,5497
+taskcompendium/grading_result.py,sha256=ZOc3_YsACp_IqnN2iVn9PMYQDh5jlFMWLJ7nKGul_q8,935
+taskcompendium/models.py,sha256=UIqffR2yzu0VjB1ynND-7zmx3crYZaJsNI6NHt6eYSI,34910
+taskcompendium/submission.py,sha256=l96cBGXc4qJysitVlUx4f6_VhMTuuMAnFWkEd0kLeZs,7539
+taskcompendium/convert/__init__.py,sha256=AuRWrOcOqcm2s5NNLnBcPkluFLRcSxv9UnmesSeEaXM,146
+taskcompendium/convert/answers.py,sha256=04G-n8VdVGeVntkr6D58hXF9imEG14vHSSnkweBe2XY,7965
+taskcompendium/convert/code.py,sha256=XK1X24br7hZeo8UPQSARkEVOZcn6-lDgV5fZwcKKtxs,3846
+taskcompendium/convert/conversation.py,sha256=55S2jGI52XFxGRQ_zPvyfP7O0YiH-HdSBov_8tQUgwM,1349
+taskcompendium/convert/delivery.py,sha256=dLyrMXlC5ZH-dkDvT4QDtk0UmvKiIqCNv0EuSujrdyc,1418
+taskcompendium/convert/environment.py,sha256=zKSh8aIzx4Y9CkyPVDhdpp2wRY8ngidZmsH00VI0nxU,618
+taskcompendium/convert/executable.py,sha256=oRaBaMaHwkNKree_ZCxS8aVkAv6csgdbCBNH2-7e4WA,14309
+taskcompendium/convert/json_schema.py,sha256=6jdI79YTVeX0jRAhFAEwtduR5J-63xIyJVk6TULVcSo,1332
+taskcompendium/convert/nemotron_ultra.py,sha256=GKNEojj4aoh8GaGrurqwh4chZaIR0cMtDaz-pctPsbQ,8961
+taskcompendium/convert/preference.py,sha256=QfqfjrX4EexIKtFfLCUMAZi0jAYSOBvlGWqeILzf3Bk,3942
+taskcompendium/convert/script_grader.py,sha256=3uwzujkg3glbzVBXXl5KKHh_ZrQT48dB99iKEIZsVng,2420
+taskcompendium/convert/tasktrove.py,sha256=zojE0PfC3Hd9EWpgLBeepB0cYZfvfIGL5rCPKmn8kLY,7537
+taskcompendium/convert/tasktrove_code_contests.py,sha256=x8cXcpeWnbWSR28fgSVfywXeo4nci0nc3i0-vBKBMKo,3786
+taskcompendium/convert/tasktrove_codeforces.py,sha256=1YH6x_LaVWFYoMEJ39x0aWzV8G5wG5KzvUOYlZHoFZc,4217
+taskcompendium/convert/tasktrove_converted_task.py,sha256=Aq2v5xF-LTore-9y9HqCfZU_C0aspMLT5rIWmokeqD0,3542
+taskcompendium/convert/tasktrove_json_schemas.py,sha256=eYgzfinpVg-TUY-XxD0ZHhdWyqJqrhtD0BwHmZA-JvA,5284
+taskcompendium/convert/tasktrove_nemotron_data.py,sha256=FM3fJXSo3Z6HCM8tea7IxguoWergmo7XSIYS7T5VhQk,827
+taskcompendium/convert/tasktrove_nemotron_structured_outputs.py,sha256=4Jm93oo1yeuZzvggyZU1cbxSUEtWsdz7SrerSNo0mE0,12239
+taskcompendium/convert/tasktrove_nl2bash.py,sha256=Eevf7G96sPA2MUdA-xZ7yOMLs8LAMv5UWbFwQejK3D4,7028
+taskcompendium/convert/tasktrove_python_unit_tests.py,sha256=b_tI3wfjz3vgORE0E4klemSYSQ3YpNNOv66Gxs8rZ7E,3841
+taskcompendium/convert/tasktrove_stdio_cases.py,sha256=EF4u7t-4eSoE2A7uk3AJ7PhlWuH-F6qFaFWFhWt9-DY,4146
+taskcompendium/convert/tasktrove_taco.py,sha256=e2ynxAbYBJZQO906fMJaGBy8tH9IDkaiQBZVP2rwvbY,3757
+taskcompendium/convert/verifyit_build.py,sha256=WYxrr7ycjNsym1nAiN5RXVmowhMz_0J7QkLDXsF61DE,2817
+taskcompendium/harbor/__init__.py,sha256=FGKy0KOvEejGv3g0Ov1LgJx1RHqPo8o8MYprTPahN8Q,118
+taskcompendium/harbor/compare.py,sha256=CafzeTfgPSkin8BNziETdoQQk3CKd9ruyEUdkZh3-WE,10771
+taskcompendium/harbor/export.py,sha256=j71GQtaSB9W9AAGK3gmbpd5nSsOaGm6XGTo1yOSwnkc,28192
+taskcompendium/harbor/records.py,sha256=KFO8XTnvblnOLXwV09ut4AiH-YFn7ziTBoMc_xVJvwk,3328
+taskcompendium/harbor/snapshots.py,sha256=C5mhctpFg4A6qkAkiMu_JdqKBk3oBXQ58yncyN29f5A,4011
+taskcompendium/harbor/tasktrove.py,sha256=8SW2PGEFOhKxIbSyBic9lExj_RVv-sac9y1BHRXXi_4,3426
+taskcompendium/importers/__init__.py,sha256=Sx3uJyho-XnsgGKm4f3oBceCu6N7NzB8lkY3-sH134Y,142
+taskcompendium/importers/nemo_predicted_action.py,sha256=CEPAEQzAdt5KKiQBT9dmqHR41xdL9PetYebBow4nenA,9627
+taskcompendium/importers/swe.py,sha256=KicdqOUbs8Y8Uz-VwclrpeOZm5Y4prVrZLwl7c4Wq4E,2730
+taskcompendium/importers/tasktrove/__init__.py,sha256=zIJ4TMWYbzc3cs7aDWIAaw4RjB-ikKwek5GN3Smf7f0,128
+taskcompendium/importers/tasktrove/convert.py,sha256=WN5nYZv1F1QzmyIFJW0con9wk4LU9IUHmHNzxSQ8B7s,3018
+taskcompendium/importers/tasktrove/mcqa.py,sha256=nGXGLgDeBh33i8UfNkZsC3DQqdW7iofoWgXYgf_XMEo,3611
+taskcompendium/importers/tasktrove/models.py,sha256=1BLxl8tn-Gjz_4QEx4sSowgAU2uKxXDfROxCS0tdLtg,755
+taskcompendium/pipeline/README.md,sha256=yXoaVqf0jvdDoGkN44FKV7AW1C1F_zPcDeVHDfGe9wQ,13711
+taskcompendium/pipeline/__init__.py,sha256=EVYEzJis0ke3RzUcOEYCC2O-am3_o5vdRMMkrDFzyVI,135
+taskcompendium/pipeline/audit_schema.py,sha256=5ignj28PQr-oVsqu91icVsVil2W-pz02l7euCCBS674,4781
+taskcompendium/pipeline/chat_requests.py,sha256=VUC3K_anr-gfOT_837KRI4TngETHOePeFxkPl-QhB_k,3875
+taskcompendium/pipeline/controls.py,sha256=qr1dQgrYKkv8y3Dk17pGtyx_cJkEluXvPDjvSh1t22I,10897
+taskcompendium/pipeline/conversion.py,sha256=Md2sMl49Y65fIeGwiOG8xTsWd2l_mPOOQqDL-ILMT4M,5920
+taskcompendium/pipeline/environment_inventory.py,sha256=67kkb0KxD8bFu5tv3DbPaUTt-UDLz-nfwLToyr9yRDk,1879
+taskcompendium/pipeline/execution_telemetry.py,sha256=bD5Kf_jgqKlz_qC0i2PdAlT4t9wj8c1REbAtHCboj58,4651
+taskcompendium/pipeline/filtering.py,sha256=WKIK_7MXHdjpkc4LMawRhjkfsmzXtYcQlqDcnz5ylU0,1817
+taskcompendium/pipeline/fingerprints.py,sha256=4kLN5XYuK8imZrrF7ZrdIdJyfrpOPVsNKBUh6DtJCWg,4531
+taskcompendium/pipeline/inputs.py,sha256=136QHYa4ZWU01uDArHUoVLfnRsu2Ifrw6Hf1mZ0eDgE,3625
+taskcompendium/pipeline/models.py,sha256=vm2HIZZhhWJSCqXIkuP6CjCaHnjq3IboLNV5NP507ZM,9205
+taskcompendium/pipeline/query_cache.py,sha256=1hld3ddGs60KWpHPuXSLDOEE0BtDv6-cbPmmrWjFgAk,9633
+taskcompendium/pipeline/review.py,sha256=eRB2pRf-HnI9d90yPyCxUkXCMIXh0CUBOg0HaBa-9jY,34450
+taskcompendium/pipeline/review_requests.py,sha256=P9x6k5ZiLe5hiht5Dpob2_CHMnzTO3D3GGuI8MLu-4U,10359
+taskcompendium/pipeline/sampling.py,sha256=Zxai94IJctH6QrpeCFzBYJoISkRHXBM0FPXrfy53G7U,1416
+taskcompendium/pipeline/shard_outputs.py,sha256=Iov0RWinu0MVG35KZfUIyATCUY5oM3U_UZR7dfG1wTY,2205
+taskcompendium/pipeline/source_processing.py,sha256=PLDKMuHrBnVR7NQfYIOnZnc-UKw4Ta-4wAM_DxEttQo,43793
+taskcompendium/pipeline/source_quality.py,sha256=QAtdeLt22wfXySIWug0vURwTH_VAhWI_8OhITEwlv68,11553
+taskcompendium/pipeline/source_verification.py,sha256=YLNADkr47D8dZ17qSrY72_axIJ56mR-7YeQ46EhQFqc,25644
+taskcompendium/pipeline/sources.py,sha256=irN4Rv66yOGAGmwEQb-1g09N9R2FybWaGhgPgHS2OvU,9026
+taskcompendium/pipeline/stages.py,sha256=nQXzG1OjWpYRExjad5B5h9ykBXKyT2w-gi9O4h8MHhA,34348
+taskcompendium/pipeline/transforms.py,sha256=KbnvSu6zfU6lM4qxe3V83adGztH-Y5smO8O2zbWF7lw,8334
+taskcompendium/pipeline/verification.py,sha256=UgfxdopckCkfcgXTJdo-c6qub09UFvzN4WIOFssOyqI,8480
+taskcompendium/runtime/__init__.py,sha256=o67I1ai4r_ZEZSVdlrHdxWzabhAXKCw5N_IqYCqnj38,229
+taskcompendium/runtime/environment.py,sha256=ts3uM0zp5DSqevXKiNneyD3-wdTaEorRTyA9He7O0Io,951
+taskcompendium/runtime/episode.py,sha256=iBO6-rtNbELTIVzqgCjHW9QUqqgmGy7hSQJ9NLH3stI,3452
+taskcompendium/runtime/grading.py,sha256=Adoc1qrIB86PuIaWfvRKNIQ4ssG8KkcGsdviPHBD4qc,28257
+taskcompendium/runtime/local.py,sha256=UMJ2sVxHMIHAjzV3n7WXA-H9AdqQ1-XVBUKGhQ6V3Sw,8518
+taskcompendium/runtime/models.py,sha256=4PeIzEMuNuwJgXpdHqGTqXjqwq4WiCFM3JSyMjYyduE,2258
+taskcompendium/runtime/output_capture.py,sha256=nsc60DTTrpZVdr7hqo5xtZPopxZXtEUS_4xOAC527PQ,5314
+taskcompendium/runtime/resources.py,sha256=-LgVikv0pjBLpu1y_BXEvCJC2trGqaEJgzGkcRqQDYk,618
+taskcompendium/runtime/shell.py,sha256=gMUBDBz9Whnr4qac62sdXxG13xIHIDdmIib_Z05esw8,11696
+taskcompendium/runtime/task_grading.py,sha256=GJKxkVShu2ZmZbKYhDt7b9zsl5HS4tMGJT_J40EPNLw,2252
+marin_taskcompendium-0.2.160.dev38062449553.dist-info/METADATA,sha256=1bBUIOzI8lGveU4g7_DUz-oZaWR9sUQ1nwB0i68nz6s,770
+marin_taskcompendium-0.2.160.dev38062449553.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+marin_taskcompendium-0.2.160.dev38062449553.dist-info/RECORD,,

marin_taskcompendium-0.2.160.dev38062449553.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

taskcompendium/__init__.py

@@ -0,0 +1,2 @@
+# Copyright The Marin Authors
+# SPDX-License-Identifier: Apache-2.0

taskcompendium/chat.py

@@ -0,0 +1,93 @@
+# Copyright The Marin Authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Convert OpenAI chat messages to typed conversation evidence."""
+
+import json
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator
+from verifyit.json_objects import unique_object
+
+from taskcompendium.models import (
+    AssistantToolCalls,
+    ConversationEvent,
+    ConversationInput,
+    ConversationToolCall,
+    ConversationTrace,
+    TextMessage,
+    ToolResult,
+)
+
+
+class ChatFunction(BaseModel):
+    model_config = ConfigDict(strict=True, allow_inf_nan=False)
+
+    name: str = Field(min_length=1)
+    arguments: dict[str, JsonValue]
+
+    @field_validator("arguments", mode="before")
+    @classmethod
+    def decode_arguments(cls, value: str) -> dict[str, JsonValue]:
+        if not isinstance(value, str):
+            raise ValueError("Provider tool arguments must be a JSON string")
+        return json.loads(value, object_pairs_hook=unique_object)
+
+
+class ChatToolCall(BaseModel):
+    model_config = ConfigDict(strict=True)
+
+    id: str = Field(min_length=1)
+    type: Literal["function"]
+    function: ChatFunction
+
+
+class ChatAssistantMessage(BaseModel):
+    model_config = ConfigDict(strict=True)
+
+    role: Literal["assistant"]
+    content: str | None = None
+    tool_calls: list[ChatToolCall] | None = None
+
+
+def assistant_message(message: dict[str, Any]) -> TextMessage | AssistantToolCalls:
+    """Validate chat wire data and return protocol-independent submission evidence.
+
+    Malformed protocol data raises at this ingestion boundary. A valid text reply,
+    wrong function name, or wrong arguments remain available for grading.
+    """
+    validated = ChatAssistantMessage.model_validate(message)
+    if validated.tool_calls:
+        return AssistantToolCalls(
+            calls=tuple(
+                ConversationToolCall(call_id=call.id, name=call.function.name, arguments=call.function.arguments)
+                for call in validated.tool_calls
+            ),
+            content=validated.content,
+        )
+    if validated.content is None:
+        raise ValueError("Chat response requires assistant content or function calls")
+    return TextMessage(role="assistant", content=validated.content)
+
+
+def _conversation_events(messages: list[dict[str, Any]]) -> tuple[ConversationEvent, ...]:
+    events: list[ConversationEvent] = []
+    for message in messages:
+        role = message.get("role")
+        if role == "assistant":
+            events.append(assistant_message(message))
+        elif role == "tool":
+            events.append(ToolResult(call_id=message["tool_call_id"], content=message["content"]))
+        else:
+            events.append(TextMessage.model_validate(message))
+    return tuple(events)
+
+
+def chat_input(messages: list[dict[str, Any]]) -> ConversationInput:
+    """Normalize a public task prefix before model inference."""
+    return ConversationInput(events=_conversation_events(messages))
+
+
+def chat_conversation(messages: list[dict[str, Any]]) -> ConversationTrace:
+    """Normalize a complete chat transcript for a TaskCompendium verifier."""
+    return ConversationTrace(events=_conversation_events(messages))

taskcompendium/convert/__init__.py

@@ -0,0 +1,4 @@
+# Copyright The Marin Authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Build task specifications, with their graders fixed, from source rows."""

taskcompendium/convert/answers.py

@@ -0,0 +1,210 @@
+# Copyright The Marin Authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Conversation tasks whose final answer a verifyit mode grades in process.
+
+Each helper takes the public prompt and reference values that a declaration extracts from its row,
+checks that they are usable, and returns a task with its grader set, or a rejection naming the
+defect. ``evidence`` is source material kept with the grader for review, never shown to the solver.
+"""
+
+import json
+from collections.abc import Mapping
+from typing import Any
+
+from verifyit.modes.extract import extract_boxed
+from verifyit.numeric import numeric_literal
+from verifyit.spec import (
+    Constraint,
+    ExactSpec,
+    IfevalSpec,
+    JsonSchemaSpec,
+    MathSpec,
+    MathType,
+    McqSpec,
+    NumericSpec,
+    SchemaFormat,
+    Spec,
+)
+
+from taskcompendium.grader import verifyit_package
+from taskcompendium.models import (
+    AnswerType,
+    ConversationInput,
+    EnvironmentRequirements,
+    PlainText,
+    ResourceGroups,
+    TaskResource,
+    TaskSpec,
+    TextMessage,
+)
+from taskcompendium.pipeline.models import ImportFailureKind, ImportRejection, RawRow
+from taskcompendium.runtime.resources import inline_resource
+
+EVIDENCE_PATH = "reference/source-evidence.json"
+
+
+def source_defect(reason: str, detail: str) -> ImportRejection:
+    return ImportRejection(kind=ImportFailureKind.SOURCE_DEFECT, reason=reason, detail=detail)
+
+
+def unsupported(reason: str, detail: str) -> ImportRejection:
+    return ImportRejection(kind=ImportFailureKind.UNSUPPORTED, reason=reason, detail=detail)
+
+
+def _text(value: Any) -> str | None:
+    return value.strip() if isinstance(value, str) and value.strip() else None
+
+
+def evidence_resource(evidence: Mapping[str, Any]) -> TaskResource:
+    """Source fields retained beside the grader for review and audit."""
+    return inline_resource(EVIDENCE_PATH, json.dumps(dict(evidence), ensure_ascii=False, sort_keys=True).encode())
+
+
+def answer_task(
+    row: RawRow,
+    *,
+    prompt: str,
+    spec: Spec,
+    answer_type: AnswerType = AnswerType.TEXT,
+    evidence: Mapping[str, Any] | None = None,
+    resources: tuple[TaskResource, ...] = (),
+) -> TaskSpec:
+    """A one-message conversation task graded in process by ``spec``."""
+    verifier = (*resources, *((evidence_resource(evidence),) if evidence else ()))
+    package = verifyit_package(spec, verifier)
+    return TaskSpec(
+        id=row.id,
+        source=row.source,
+        context=ConversationInput(events=(TextMessage(role="user", content=prompt),)),
+        environment_requirements=EnvironmentRequirements(),
+        resources=ResourceGroups(verifier=package.resources),
+        answer_type=answer_type,
+        answer_format=PlainText(),
+        grader=package.grader,
+    )
+
+
+def math_type(expected: str) -> MathType:
+    """Keep structured mathematical answers distinct from scalar values."""
+    if expected.startswith("[") and expected.endswith("]"):
+        return MathType.LIST
+    if expected.startswith(("[", "(")) and expected.endswith(("]", ")")) and "," in expected:
+        return MathType.TUPLE if expected.startswith("(") and expected.endswith(")") else MathType.INTERVAL
+    if expected.startswith((r"\{", "{")):
+        return MathType.SET
+    if "=" in expected or r"\approx" in expected:
+        return MathType.EQUATION
+    return MathType.SCALAR
+
+
+def math_answer_task(
+    row: RawRow, *, prompt: Any, answer: Any, evidence: Mapping[str, Any] | None = None
+) -> TaskSpec | ImportRejection:
+    """Grade a symbolic math answer; a boxed reference is unwrapped."""
+    problem, reference = _text(prompt), _text(answer)
+    if problem is None:
+        return source_defect("missing_prompt", "The problem must be a nonempty string")
+    if reference is None:
+        return source_defect("invalid_reference", "The reference answer must be a nonempty string")
+    expected = extract_boxed(reference) or reference
+    return answer_task(
+        row, prompt=problem, spec=MathSpec(expected=expected, math_type=math_type(expected)), evidence=evidence
+    )
+
+
+def numeric_answer_task(
+    row: RawRow,
+    *,
+    prompt: Any,
+    answer: Any,
+    tolerance_abs: float,
+    tolerance_rel: float,
+    evidence: Mapping[str, Any] | None = None,
+) -> TaskSpec | ImportRejection:
+    """Grade a numeric literal within absolute and relative tolerances."""
+    problem = _text(prompt)
+    if problem is None:
+        return source_defect("missing_prompt", "The problem must be a nonempty string")
+    reference = answer.strip() if isinstance(answer, str) else str(answer) if isinstance(answer, int | float) else None
+    try:
+        if reference is None:
+            raise ValueError("missing")
+        numeric_literal(reference)
+    except ValueError:
+        return source_defect("invalid_reference", f"The reference answer is not numeric: {answer!r}")
+    return answer_task(
+        row,
+        prompt=problem,
+        spec=NumericSpec(reference, tolerance_abs=tolerance_abs, tolerance_rel=tolerance_rel),
+        answer_type=AnswerType.NUMBER,
+        evidence=evidence,
+    )
+
+
+def mcq_task(
+    row: RawRow, *, prompt: Any, answer: Any, options: int, evidence: Mapping[str, Any] | None = None
+) -> TaskSpec | ImportRejection:
+    """Grade an option letter; ``prompt`` already lists the ``options`` labeled choices."""
+    question, letter = _text(prompt), _text(answer)
+    if question is None:
+        return source_defect("missing_prompt", "The question must be a nonempty string")
+    if letter is None or len(letter) != 1 or not "A" <= letter.upper() < chr(65 + options):
+        return source_defect("invalid_reference", f"The key must be one of {options} option letters: {answer!r}")
+    return answer_task(row, prompt=question, spec=McqSpec(letter.upper(), options=options), evidence=evidence)
+
+
+def exact_answer_task(
+    row: RawRow,
+    *,
+    prompt: Any,
+    answers: tuple[str, ...],
+    ignore_case: bool,
+    evidence: Mapping[str, Any] | None = None,
+) -> TaskSpec | ImportRejection:
+    """Grade an answer equal to one of ``answers`` after whitespace normalization."""
+    question = _text(prompt)
+    if question is None:
+        return source_defect("missing_prompt", "The question must be a nonempty string")
+    if not answers or not all(_text(item) for item in answers):
+        return source_defect("invalid_reference", "Accepted answers must be nonempty strings")
+    return answer_task(row, prompt=question, spec=ExactSpec(tuple(answers), ignore_case=ignore_case), evidence=evidence)
+
+
+def ifeval_task(
+    row: RawRow,
+    *,
+    prompt: Any,
+    constraints: tuple[Constraint, ...],
+    evidence: Mapping[str, Any] | None = None,
+) -> TaskSpec | ImportRejection:
+    """Grade the fraction of instruction-following constraints the reply satisfies."""
+    instruction = _text(prompt)
+    if instruction is None:
+        return source_defect("missing_prompt", "The instruction must be a nonempty string")
+    if not constraints:
+        return unsupported("invalid_constraints", "At least one constraint is required")
+    return answer_task(row, prompt=instruction, spec=IfevalSpec(constraints=constraints), evidence=evidence)
+
+
+def json_schema_task(
+    row: RawRow,
+    *,
+    prompt: Any,
+    schema: Any,
+    schema_format: SchemaFormat,
+    evidence: Mapping[str, Any] | None = None,
+) -> TaskSpec | ImportRejection:
+    """Grade a reply against ``schema`` (serialized schema text), shipped as the grader's ``schema.json``."""
+    instruction, document = _text(prompt), _text(schema)
+    if instruction is None:
+        return source_defect("missing_prompt", "The instruction must be a nonempty string")
+    if document is None:
+        return source_defect("invalid_schema", "The schema must be nonempty text")
+    return answer_task(
+        row,
+        prompt=instruction,
+        spec=JsonSchemaSpec(schema="schema.json", format=schema_format),
+        evidence=evidence,
+        resources=(inline_resource("schema.json", document.encode()),),
+    )

taskcompendium/convert/code.py

@@ -0,0 +1,78 @@
+# Copyright The Marin Authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Checks and grading settings for code tasks whose hidden tests use the LiveCodeBench test-case layout."""
+
+import ast
+import re
+from collections.abc import Mapping
+from typing import Any
+
+CODE_GRADER_MEMORY_MB = 5120
+"""The LiveCodeBench child process may use 4 GiB; the grading machine also needs room for its runtime."""
+THREAD_ENVIRONMENT = {"OPENBLAS_NUM_THREADS": "1", "OMP_NUM_THREADS": "1"}
+"""Run each test's numeric libraries on one thread."""
+CODE_BLOCK = re.compile(r"```(?:\w+)?\n(.*?)```", re.DOTALL)
+"""A fenced code block as the source scorers match one; they run the last block in a reply."""
+
+
+def validate_code_cases(cases: Mapping[str, Any] | list[Any]) -> None:
+    """Reject decoded test layouts the LiveCodeBench scorer cannot normalize."""
+    if isinstance(cases, Mapping) and "test_cases" in cases:
+        if cases.get("language") not in (None, "python"):
+            raise ValueError("LCB supports only Python verification specs")
+        cases = cases["test_cases"]
+    elif isinstance(cases, Mapping):
+        inputs, outputs = cases.get("inputs"), cases.get("outputs")
+        if not isinstance(inputs, list) or not isinstance(outputs, list) or not inputs or len(inputs) != len(outputs):
+            raise ValueError("LCB requires aligned nonempty input and output lists")
+        kind = "functional" if cases.get("fn_name") is not None else "stdin"
+        cases = [
+            {"input": item, "output": result, "testtype": kind, "fn_name": cases.get("fn_name")}
+            for item, result in zip(inputs, outputs, strict=True)
+        ]
+    if not isinstance(cases, list) or not cases or not all(isinstance(case, Mapping) for case in cases):
+        raise ValueError("LCB requires a nonempty list of test cases")
+    modes = set()
+    names = set()
+    for case in cases:
+        kind = case.get("testtype", case.get("type"))
+        if kind in ("stdin", "stdin_stdout"):
+            if not isinstance(case.get("input"), str) or not isinstance(case.get("output"), str):
+                raise ValueError("Standard-input LCB cases require string input and output")
+            modes.add("stdin")
+        elif kind in ("functional", "call_based"):
+            metadata = case.get("metadata")
+            name = case.get("fn_name") or (metadata.get("func_name") if isinstance(metadata, Mapping) else None)
+            if not isinstance(name, str) or not name:
+                raise ValueError("Functional LCB cases require a function name")
+            if not isinstance(case.get("input"), str | list):
+                raise ValueError("Functional LCB input must be a list or encoded string")
+            modes.add("functional")
+            names.add(name)
+        else:
+            raise ValueError("Unsupported LCB test-case type")
+    if len(modes) != 1 or len(names) > 1:
+        raise ValueError("LCB cases must share one execution mode and function name")
+
+
+def has_code_block(response: str) -> bool:
+    """Match a fenced code block the way the source scorers extract one."""
+    return CODE_BLOCK.search(response) is not None
+
+
+def python_reply(solution: Any) -> str | None:
+    """A source solution as a reply the scorers extract code from.
+
+    Returns ``None`` when there is no solution or when the program the scorers would run is not
+    Python 3. Some sources keep Python 2 solutions; the graders run Python 3, so such a solution
+    cannot serve as a known-correct submission.
+    """
+    if not isinstance(solution, str) or not solution.strip():
+        return None
+    reply = solution if has_code_block(solution) else f"```python\n{solution}\n```"
+    try:
+        ast.parse(CODE_BLOCK.findall(reply)[-1].strip())
+    except SyntaxError:  # Includes TabError from Python 2 tab indentation.
+        return None
+    return reply

taskcompendium/convert/conversation.py

@@ -0,0 +1,44 @@
+# Copyright The Marin Authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Conversation tasks whose final reply a packaged grader scores."""
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from taskcompendium.convert.answers import evidence_resource
+from taskcompendium.grader import GraderPackage
+from taskcompendium.models import (
+    AnswerType,
+    ConversationEvent,
+    ConversationInput,
+    EnvironmentRequirements,
+    PlainText,
+    ResourceGroups,
+    TaskSpec,
+)
+from taskcompendium.pipeline.models import RawRow
+
+
+def conversation_task(
+    row: RawRow,
+    *,
+    events: Sequence[ConversationEvent],
+    package: GraderPackage,
+    evidence: Mapping[str, Any] | None = None,
+) -> TaskSpec:
+    """A task whose plain-text reply to ``events`` the packaged grader scores.
+
+    ``evidence`` is source material kept beside the grader for review, never shown to the solver.
+    """
+    verifier = (*package.resources, *((evidence_resource(evidence),) if evidence else ()))
+    return TaskSpec(
+        id=row.id,
+        source=row.source,
+        context=ConversationInput(events=tuple(events)),
+        environment_requirements=EnvironmentRequirements(),
+        resources=ResourceGroups(verifier=verifier),
+        answer_type=AnswerType.TEXT,
+        answer_format=PlainText(),
+        grader=package.grader,
+    )

taskcompendium/convert/delivery.py

@@ -0,0 +1,34 @@
+# Copyright The Marin Authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Move a source's file-delivery wording into the assistant reply and record the rewrite.
+
+Many sources were written for a shell agent that writes its answer to a file. A conversation task
+asks for the same answer in its reply instead; the rewrite is kept as a normalization change, so the
+audit retains the source's original wording.
+"""
+
+from collections.abc import Sequence
+
+from taskcompendium.models import TaskSpec, TextMessage
+from taskcompendium.pipeline.models import NormalizationChange, NormalizedTask
+
+
+def replace_phrases(text: str, phrases: Sequence[tuple[str, str]]) -> str:
+    """Apply each ``(original, replacement)`` rewrite in order."""
+    for original, replacement in phrases:
+        text = text.replace(original, replacement)
+    return text
+
+
+def rewritten_task(task: TaskSpec, *, original: str, reason: str) -> TaskSpec | NormalizedTask:
+    """``task``, with a change record when its first message differs from the source's ``original``.
+
+    Stripping outer whitespace alone is not recorded as a change.
+    """
+    prompt = task.context.events[0]
+    assert isinstance(prompt, TextMessage)
+    if prompt.content.strip() == original.strip():
+        return task
+    change = NormalizationChange(field="instruction", reason=reason, original=original, replacement=prompt.content)
+    return NormalizedTask(task, (change,))

taskcompendium/convert/environment.py

@@ -0,0 +1,16 @@
+# Copyright The Marin Authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Environment requirements for graders that run an image in a fresh machine."""
+
+from shellbox.machine import Backend
+
+from taskcompendium.models import EnvironmentRequirements
+
+IMAGE_BACKENDS = (Backend.GVISOR, Backend.DOCKER)
+"""Backends that start a fresh machine from a digest-pinned image."""
+
+
+def grading_environment(image: str) -> EnvironmentRequirements:
+    """A grader environment that runs the digest-pinned ``image`` under gVisor or Docker."""
+    return EnvironmentRequirements(docker_image=image, compatible_backends=IMAGE_BACKENDS)