verifiers 0.4.1.dev62__tar.gz → 0.4.1.dev64__tar.gz

{verifiers-0.4.1.dev62 → verifiers-0.4.1.dev64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev62
+Version: 0.4.1.dev64
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -40,7 +40,7 @@ Requires-Dist: prime-sandboxes>=0.5.0
 Requires-Dist: prime-tunnel>=0.1.14
 Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0
-Requires-Dist: renderers[multimodal]>=0.1.12
+Requires-Dist: renderers[multimodal]>=0.1.13.dev16
 Requires-Dist: rich>=11.0.0
 Requires-Dist: tenacity>=8.5.0
 Requires-Dist: tomli-w>=1.0.0

{verifiers-0.4.1.dev62 → verifiers-0.4.1.dev64}/pyproject.toml

@@ -49,7 +49,7 @@ dependencies = [
     "prime-pydantic-config[toml]>=0.4.3",
     "loguru>=0.7.0",
     "tomli-w>=1.0.0",
-    "renderers[multimodal]>=0.1.12",
+    "renderers[multimodal]>=0.1.13.dev16",
     "typing_extensions>=4.12.2",
     "mcp==2.0.0",
 ]

verifiers-0.4.1.dev64/tests/unit/test_api.py

@@ -0,0 +1,66 @@
+"""The `verifiers.v1` namespace: its lazy names stay in sync with what type checkers
+see, and the config tree imports without the runtime."""
+
+import ast
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import verifiers.v1 as vf
+
+# What the config tree may load: a trainer validates its configs against it in an
+# environment that installs nothing else.
+CONFIG_TREE_PACKAGES = {
+    "annotated_types",
+    "pydantic",
+    "pydantic_config",
+    "pydantic_core",
+    "renderers",
+    "tomli",
+    "tomli_w",
+    "typing_extensions",
+    "typing_inspection",
+    "verifiers",
+}
+
+IMPORT_CONFIG_TREE = """
+import importlib, json, pkgutil, sys
+import verifiers.v1.configs as configs
+for module in pkgutil.walk_packages(configs.__path__, "verifiers.v1.configs."):
+    importlib.import_module(module.name)
+packages = {name.partition(".")[0] for name in sys.modules if not name.startswith("_")}
+print(json.dumps(sorted(packages - sys.stdlib_module_names)))
+"""
+
+
+def type_checking_name_to_module() -> dict[str, str]:
+    tree = ast.parse(Path(vf.__file__).read_text())
+    block = next(node for node in tree.body if isinstance(node, ast.If))
+    return {alias.name: node.module for node in block.body for alias in node.names}
+
+
+def test_lazy_names_match_type_checking_imports():
+    assert vf._NAME_TO_MODULE == type_checking_name_to_module()
+
+
+def test_all_exports_every_lazy_name():
+    assert sorted(vf.__all__) == sorted(vf._NAME_TO_MODULE)
+
+
+def test_every_lazy_name_resolves():
+    resolved = {name: getattr(vf, name) for name in vf.__all__}
+
+    assert sorted(resolved) == sorted(vf.__all__)
+
+
+def test_config_tree_imports_without_the_runtime():
+    result = subprocess.run(
+        [sys.executable, "-c", IMPORT_CONFIG_TREE],
+        capture_output=True,
+        check=True,
+        text=True,
+        timeout=60,
+    )
+
+    assert set(json.loads(result.stdout)) - CONFIG_TREE_PACKAGES == set()

verifiers-0.4.1.dev64/verifiers/v1/__init__.py

@@ -0,0 +1,599 @@
+"""The verifiers v1 API, imported as `import verifiers.v1 as vf`.
+
+Names resolve on first access (PEP 562), so importing one module, such as the
+config tree a trainer validates against, loads only that module's dependencies
+and not the whole runtime.
+"""
+
+import importlib
+import logging as _logging
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pydantic_config import BaseConfig
+
+    from verifiers.v1.acp import ACPConfig, ACPHarness, ACPTurn
+    from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
+    from verifiers.v1.clients import Client, ModelContext, resolve_client
+    from verifiers.v1.configs.agent import AgentConfig
+    from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field
+    from verifiers.v1.configs.client import (
+        BaseClientConfig,
+        ClientConfig,
+        EvalClientConfig,
+        TrainClientConfig,
+    )
+    from verifiers.v1.configs.deployment import DeploymentConfig
+    from verifiers.v1.configs.env import (
+        EnvConfig,
+        SharedEnvConfig,
+        SingleAgentEnvConfig,
+        default_agent_harness,
+    )
+    from verifiers.v1.configs.harness import HarnessConfig
+    from verifiers.v1.configs.judge import JudgeConfig, Judges
+    from verifiers.v1.configs.retries import RetryConfig, RetryRule
+    from verifiers.v1.configs.runtime import (
+        CustomRuntimeConfig,
+        DockerConfig,
+        NetworkPolicyConfig,
+        PrimeConfig,
+        RuntimeConfig,
+        SubprocessConfig,
+    )
+    from verifiers.v1.configs.scoring import ScoringConfig
+    from verifiers.v1.configs.select import (
+        SelectCLIConfig,
+        SelectConfig,
+        TaskMatchConfig,
+    )
+    from verifiers.v1.configs.serve import (
+        ElasticPoolConfig,
+        ServeConfig,
+        StaticPoolConfig,
+        pool_serve_kwargs,
+    )
+    from verifiers.v1.configs.task import (
+        DecoratedFunctionConfig,
+        RewardFunctionConfig,
+        TaskConfig,
+    )
+    from verifiers.v1.configs.taskset import SharedTasksetConfig, TasksetConfig
+    from verifiers.v1.env import Env
+    from verifiers.v1.envs.single_agent import SingleAgentEnv
+    from verifiers.v1.episode import (
+        EnvInfo,
+        Episode,
+        EvalRunInfo,
+        EvalWorkInfo,
+        GroupInfo,
+        PolicySpan,
+        RunInfo,
+        TrainRunInfo,
+        TrainWorkInfo,
+        WireEpisode,
+        WorkInfo,
+    )
+    from verifiers.v1.errors import (
+        EnvError,
+        HarnessError,
+        InterceptionError,
+        ProviderError,
+        RolloutError,
+        SandboxError,
+        TaskError,
+        ToolsetError,
+        TunnelError,
+        UnscoredError,
+    )
+    from verifiers.v1.grading import grade_submission
+    from verifiers.v1.graph import MessageNode
+    from verifiers.v1.harness import Harness, HarnessSession
+    from verifiers.v1.judge import Judge, JudgeResponse, JudgeView
+    from verifiers.v1.judges import (
+        Criterion,
+        DecisionJudge,
+        DecisionJudgeConfig,
+        ReferenceJudge,
+        ReferenceJudgeConfig,
+        RubricJudge,
+        RubricJudgeConfig,
+    )
+    from verifiers.v1.mcp import SharedToolsetConfig, Toolset, ToolsetConfig
+    from verifiers.v1.runtimes import (
+        ProgramResult,
+        Runtime,
+        RuntimeInfo,
+        RuntimeProcess,
+    )
+    from verifiers.v1.runtimes.compose import ComposeSpec
+    from verifiers.v1.runtimes.deployment import Deployment
+    from verifiers.v1.semantic import (
+        ACP_SEMANTIC_EDGES_METADATA_KEY,
+        ACPInfo,
+        ParentLink,
+        SemanticEdge,
+        SemanticEdgeSet,
+    )
+    from verifiers.v1.state import State, StateT
+    from verifiers.v1.task import (
+        Task,
+        TaskData,
+        TaskResources,
+        TaskTimeout,
+        WireTaskData,
+    )
+    from verifiers.v1.taskset import Taskset
+    from verifiers.v1.trace import (
+        TRACE_VERSION,
+        AgentInfo,
+        AgentSpan,
+        Branch,
+        Error,
+        InterceptRecord,
+        ModelCall,
+        PolicyEvent,
+        Reward,
+        TimeSpan,
+        TimeSplit,
+        Timing,
+        Trace,
+        TraceTask,
+        VersionInfo,
+        WireTrace,
+    )
+    from verifiers.v1.types import (
+        ID,
+        AssistantMessage,
+        ContentPart,
+        ImageUrlContentPart,
+        ImageUrlSource,
+        Message,
+        MessageContent,
+        Messages,
+        Request,
+        Response,
+        Sampling,
+        SamplingConfig,
+        SamplingMask,
+        SystemMessage,
+        TextContentPart,
+        Tool,
+        ToolCall,
+        ToolMessage,
+        TopLogprobs,
+        TurnTokens,
+        Usage,
+        UserMessage,
+    )
+    from verifiers.v1.utils.artifacts import ARTIFACTS_DIR, Artifact, collect, restore
+    from verifiers.v1.utils.decorators import intercept, metric, reward, stop, tool
+    from verifiers.v1.utils.generic import merge_defaults
+    from verifiers.v1.utils.git import PATCH_CAP_BYTES, capture_patch, resolve_head
+    from verifiers.v1.utils.loaders import (
+        default_harness_id,
+        env_config_type,
+        environment_class,
+        harness_config_type,
+        import_environment,
+        import_harness,
+        import_judge,
+        import_taskset,
+        judge_config_type,
+        load_environment,
+        load_harness,
+        load_judge,
+        load_taskset,
+        resolve_env_config,
+        task_type,
+        taskset_config_type,
+    )
+    from verifiers.v1.utils.score import (
+        compare_stdout_results,
+        extract_boxed_answer,
+        parse_judge_choice,
+        parse_pytest_outcomes,
+        read_answer_file_or_last_reply,
+        verify_boxed_math_answer,
+    )
+
+_NAME_TO_MODULE = {
+    "ACPConfig": "verifiers.v1.acp",
+    "ACPHarness": "verifiers.v1.acp",
+    "ACPInfo": "verifiers.v1.semantic",
+    "ACPTurn": "verifiers.v1.acp",
+    "ACP_SEMANTIC_EDGES_METADATA_KEY": "verifiers.v1.semantic",
+    "ARTIFACTS_DIR": "verifiers.v1.utils.artifacts",
+    "Agent": "verifiers.v1.agent",
+    "AgentConfig": "verifiers.v1.configs.agent",
+    "AgentInfo": "verifiers.v1.trace",
+    "AgentSpan": "verifiers.v1.trace",
+    "Agents": "verifiers.v1.agent",
+    "Artifact": "verifiers.v1.utils.artifacts",
+    "AssistantMessage": "verifiers.v1.types",
+    "BaseClientConfig": "verifiers.v1.configs.client",
+    "BaseConfig": "pydantic_config",
+    "Branch": "verifiers.v1.trace",
+    "Client": "verifiers.v1.clients",
+    "ClientConfig": "verifiers.v1.configs.client",
+    "ComposeSpec": "verifiers.v1.runtimes.compose",
+    "ContentPart": "verifiers.v1.types",
+    "Criterion": "verifiers.v1.judges",
+    "CustomRuntimeConfig": "verifiers.v1.configs.runtime",
+    "DecisionJudge": "verifiers.v1.judges",
+    "DecisionJudgeConfig": "verifiers.v1.judges",
+    "DecoratedFunctionConfig": "verifiers.v1.configs.task",
+    "Deployment": "verifiers.v1.runtimes.deployment",
+    "DeploymentConfig": "verifiers.v1.configs.deployment",
+    "DockerConfig": "verifiers.v1.configs.runtime",
+    "ElasticPoolConfig": "verifiers.v1.configs.serve",
+    "Env": "verifiers.v1.env",
+    "EnvConfig": "verifiers.v1.configs.env",
+    "EnvError": "verifiers.v1.errors",
+    "EnvInfo": "verifiers.v1.episode",
+    "Episode": "verifiers.v1.episode",
+    "Error": "verifiers.v1.trace",
+    "EvalClientConfig": "verifiers.v1.configs.client",
+    "EvalRunInfo": "verifiers.v1.episode",
+    "EvalWorkInfo": "verifiers.v1.episode",
+    "GroupInfo": "verifiers.v1.episode",
+    "Harness": "verifiers.v1.harness",
+    "HarnessConfig": "verifiers.v1.configs.harness",
+    "HarnessError": "verifiers.v1.errors",
+    "HarnessSession": "verifiers.v1.harness",
+    "ID": "verifiers.v1.types",
+    "ImageUrlContentPart": "verifiers.v1.types",
+    "ImageUrlSource": "verifiers.v1.types",
+    "Interaction": "verifiers.v1.agent",
+    "InterceptRecord": "verifiers.v1.trace",
+    "InterceptionError": "verifiers.v1.errors",
+    "Judge": "verifiers.v1.judge",
+    "JudgeConfig": "verifiers.v1.configs.judge",
+    "JudgeResponse": "verifiers.v1.judge",
+    "JudgeView": "verifiers.v1.judge",
+    "Judges": "verifiers.v1.configs.judge",
+    "Message": "verifiers.v1.types",
+    "MessageContent": "verifiers.v1.types",
+    "MessageNode": "verifiers.v1.graph",
+    "Messages": "verifiers.v1.types",
+    "ModelCall": "verifiers.v1.trace",
+    "ModelContext": "verifiers.v1.clients",
+    "NetworkPolicyConfig": "verifiers.v1.configs.runtime",
+    "PATCH_CAP_BYTES": "verifiers.v1.utils.git",
+    "ParentLink": "verifiers.v1.semantic",
+    "PolicyEvent": "verifiers.v1.trace",
+    "PolicySpan": "verifiers.v1.episode",
+    "PrimeConfig": "verifiers.v1.configs.runtime",
+    "ProgramResult": "verifiers.v1.runtimes",
+    "ProviderError": "verifiers.v1.errors",
+    "ReferenceJudge": "verifiers.v1.judges",
+    "ReferenceJudgeConfig": "verifiers.v1.judges",
+    "Request": "verifiers.v1.types",
+    "Response": "verifiers.v1.types",
+    "RetryConfig": "verifiers.v1.configs.retries",
+    "RetryRule": "verifiers.v1.configs.retries",
+    "Reward": "verifiers.v1.trace",
+    "RewardFunctionConfig": "verifiers.v1.configs.task",
+    "RolloutError": "verifiers.v1.errors",
+    "RubricJudge": "verifiers.v1.judges",
+    "RubricJudgeConfig": "verifiers.v1.judges",
+    "RunInfo": "verifiers.v1.episode",
+    "Runtime": "verifiers.v1.runtimes",
+    "RuntimeConfig": "verifiers.v1.configs.runtime",
+    "RuntimeInfo": "verifiers.v1.runtimes",
+    "RuntimeProcess": "verifiers.v1.runtimes",
+    "Sampling": "verifiers.v1.types",
+    "SamplingConfig": "verifiers.v1.types",
+    "SamplingMask": "verifiers.v1.types",
+    "SandboxError": "verifiers.v1.errors",
+    "ScoringConfig": "verifiers.v1.configs.scoring",
+    "Segment": "verifiers.v1.agent",
+    "SelectCLIConfig": "verifiers.v1.configs.select",
+    "SelectConfig": "verifiers.v1.configs.select",
+    "SemanticEdge": "verifiers.v1.semantic",
+    "SemanticEdgeSet": "verifiers.v1.semantic",
+    "ServeConfig": "verifiers.v1.configs.serve",
+    "SharedEnvConfig": "verifiers.v1.configs.env",
+    "SharedTasksetConfig": "verifiers.v1.configs.taskset",
+    "SharedToolsetConfig": "verifiers.v1.mcp",
+    "SingleAgentEnv": "verifiers.v1.envs.single_agent",
+    "SingleAgentEnvConfig": "verifiers.v1.configs.env",
+    "State": "verifiers.v1.state",
+    "StateT": "verifiers.v1.state",
+    "StaticPoolConfig": "verifiers.v1.configs.serve",
+    "SubprocessConfig": "verifiers.v1.configs.runtime",
+    "SystemMessage": "verifiers.v1.types",
+    "TRACE_VERSION": "verifiers.v1.trace",
+    "Task": "verifiers.v1.task",
+    "TaskConfig": "verifiers.v1.configs.task",
+    "TaskData": "verifiers.v1.task",
+    "TaskError": "verifiers.v1.errors",
+    "TaskMatchConfig": "verifiers.v1.configs.select",
+    "TaskResources": "verifiers.v1.task",
+    "TaskTimeout": "verifiers.v1.task",
+    "Taskset": "verifiers.v1.taskset",
+    "TasksetConfig": "verifiers.v1.configs.taskset",
+    "TextContentPart": "verifiers.v1.types",
+    "TimeSpan": "verifiers.v1.trace",
+    "TimeSplit": "verifiers.v1.trace",
+    "Timing": "verifiers.v1.trace",
+    "Tool": "verifiers.v1.types",
+    "ToolCall": "verifiers.v1.types",
+    "ToolMessage": "verifiers.v1.types",
+    "Toolset": "verifiers.v1.mcp",
+    "ToolsetConfig": "verifiers.v1.mcp",
+    "ToolsetError": "verifiers.v1.errors",
+    "TopLogprobs": "verifiers.v1.types",
+    "Trace": "verifiers.v1.trace",
+    "TraceTask": "verifiers.v1.trace",
+    "TrainClientConfig": "verifiers.v1.configs.client",
+    "TrainRunInfo": "verifiers.v1.episode",
+    "TrainWorkInfo": "verifiers.v1.episode",
+    "TunnelError": "verifiers.v1.errors",
+    "TurnTokens": "verifiers.v1.types",
+    "UnscoredError": "verifiers.v1.errors",
+    "Usage": "verifiers.v1.types",
+    "UserMessage": "verifiers.v1.types",
+    "VersionInfo": "verifiers.v1.trace",
+    "WireEpisode": "verifiers.v1.episode",
+    "WireTaskData": "verifiers.v1.task",
+    "WireTrace": "verifiers.v1.trace",
+    "WorkInfo": "verifiers.v1.episode",
+    "capture_patch": "verifiers.v1.utils.git",
+    "collect": "verifiers.v1.utils.artifacts",
+    "compare_stdout_results": "verifiers.v1.utils.score",
+    "default_agent_harness": "verifiers.v1.configs.env",
+    "default_harness_id": "verifiers.v1.utils.loaders",
+    "env_config_type": "verifiers.v1.utils.loaders",
+    "environment_class": "verifiers.v1.utils.loaders",
+    "extract_boxed_answer": "verifiers.v1.utils.score",
+    "grade_submission": "verifiers.v1.grading",
+    "harness_config_type": "verifiers.v1.utils.loaders",
+    "import_environment": "verifiers.v1.utils.loaders",
+    "import_harness": "verifiers.v1.utils.loaders",
+    "import_judge": "verifiers.v1.utils.loaders",
+    "import_taskset": "verifiers.v1.utils.loaders",
+    "intercept": "verifiers.v1.utils.decorators",
+    "judge_config_type": "verifiers.v1.utils.loaders",
+    "load_environment": "verifiers.v1.utils.loaders",
+    "load_harness": "verifiers.v1.utils.loaders",
+    "load_judge": "verifiers.v1.utils.loaders",
+    "load_taskset": "verifiers.v1.utils.loaders",
+    "make_agent": "verifiers.v1.agent",
+    "merge_defaults": "verifiers.v1.utils.generic",
+    "metric": "verifiers.v1.utils.decorators",
+    "narrowed_env_annotation": "verifiers.v1.configs.cli.env",
+    "parse_judge_choice": "verifiers.v1.utils.score",
+    "parse_pytest_outcomes": "verifiers.v1.utils.score",
+    "pool_serve_kwargs": "verifiers.v1.configs.serve",
+    "read_answer_file_or_last_reply": "verifiers.v1.utils.score",
+    "resolve_client": "verifiers.v1.clients",
+    "resolve_env_config": "verifiers.v1.utils.loaders",
+    "resolve_env_field": "verifiers.v1.configs.cli.env",
+    "resolve_head": "verifiers.v1.utils.git",
+    "restore": "verifiers.v1.utils.artifacts",
+    "reward": "verifiers.v1.utils.decorators",
+    "stop": "verifiers.v1.utils.decorators",
+    "task_type": "verifiers.v1.utils.loaders",
+    "taskset_config_type": "verifiers.v1.utils.loaders",
+    "tool": "verifiers.v1.utils.decorators",
+    "verify_boxed_math_answer": "verifiers.v1.utils.score",
+}
+
+__all__ = [  # noqa: RUF022 - grouped by public API area
+    # types
+    "ID",
+    "AssistantMessage",
+    "ContentPart",
+    "ImageUrlContentPart",
+    "ImageUrlSource",
+    "Message",
+    "MessageContent",
+    "Messages",
+    "Request",
+    "Response",
+    "Sampling",
+    "SamplingConfig",
+    "SystemMessage",
+    "TextContentPart",
+    "Tool",
+    "ToolCall",
+    "ToolMessage",
+    "Usage",
+    "UserMessage",
+    # task / trace / state
+    "Task",
+    "TaskData",
+    "WireTaskData",
+    "TaskResources",
+    "TaskTimeout",
+    "Trace",
+    "TraceTask",
+    "WireTrace",
+    "Reward",
+    "EnvInfo",
+    "Episode",
+    "WireEpisode",
+    "GroupInfo",
+    "TRACE_VERSION",
+    "AgentInfo",
+    "RunInfo",
+    "EvalRunInfo",
+    "EvalWorkInfo",
+    "ModelCall",
+    "ACPInfo",
+    "ACP_SEMANTIC_EDGES_METADATA_KEY",
+    "ParentLink",
+    "SemanticEdge",
+    "SemanticEdgeSet",
+    "PolicyEvent",
+    "TrainRunInfo",
+    "TrainWorkInfo",
+    "WorkInfo",
+    "PolicySpan",
+    "VersionInfo",
+    "State",
+    "StateT",
+    "MessageNode",
+    "Branch",
+    "TurnTokens",
+    "SamplingMask",
+    "TopLogprobs",
+    "Timing",
+    "TimeSpan",
+    "TimeSplit",
+    "AgentSpan",
+    "Error",
+    # decorators
+    "intercept",
+    "stop",
+    "tool",
+    "metric",
+    "reward",
+    # interception
+    "InterceptRecord",
+    # errors
+    "RolloutError",
+    "EnvError",
+    "ProviderError",
+    "HarnessError",
+    "ToolsetError",
+    "SandboxError",
+    "TaskError",
+    "UnscoredError",
+    "InterceptionError",
+    "TunnelError",
+    # clients
+    "Client",
+    "BaseClientConfig",
+    "ClientConfig",
+    "EvalClientConfig",
+    "TrainClientConfig",
+    "resolve_client",
+    # taskset / harness / runtime / environment
+    "Taskset",
+    "TaskConfig",
+    "SelectConfig",
+    "SelectCLIConfig",
+    "TaskMatchConfig",
+    "TasksetConfig",
+    "SharedTasksetConfig",
+    "DecoratedFunctionConfig",
+    "RewardFunctionConfig",
+    "BaseConfig",
+    "Harness",
+    "HarnessSession",
+    "HarnessConfig",
+    "ACPConfig",
+    "ACPHarness",
+    "ACPTurn",
+    "ModelContext",
+    "Runtime",
+    "RuntimeProcess",
+    "RuntimeConfig",
+    "RuntimeInfo",
+    "ProgramResult",
+    "SubprocessConfig",
+    "DockerConfig",
+    "CustomRuntimeConfig",
+    "NetworkPolicyConfig",
+    "PrimeConfig",
+    "Env",
+    "SingleAgentEnv",
+    "EnvConfig",
+    "SharedEnvConfig",
+    "ServeConfig",
+    "resolve_env_field",
+    "merge_defaults",
+    "narrowed_env_annotation",
+    "SingleAgentEnvConfig",
+    "AgentConfig",
+    "StaticPoolConfig",
+    "ElasticPoolConfig",
+    "default_agent_harness",
+    "pool_serve_kwargs",
+    "RetryConfig",
+    "RetryRule",
+    # agent
+    "Agent",
+    "Agents",
+    "make_agent",
+    # loaders
+    "import_taskset",
+    "import_harness",
+    "import_judge",
+    "import_environment",
+    "load_environment",
+    "load_taskset",
+    "load_harness",
+    "load_judge",
+    "environment_class",
+    "task_type",
+    "taskset_config_type",
+    "harness_config_type",
+    "judge_config_type",
+    "env_config_type",
+    "resolve_env_config",
+    "default_harness_id",
+    # judge
+    "Judge",
+    "JudgeConfig",
+    "Judges",
+    "JudgeResponse",
+    "JudgeView",
+    "DecisionJudge",
+    "DecisionJudgeConfig",
+    "ReferenceJudge",
+    "ReferenceJudgeConfig",
+    "RubricJudge",
+    "RubricJudgeConfig",
+    "Criterion",
+    # git patch capture
+    "PATCH_CAP_BYTES",
+    "capture_patch",
+    "resolve_head",
+    "grade_submission",
+    # grading artifacts
+    "ARTIFACTS_DIR",
+    "Artifact",
+    "collect",
+    "restore",
+    # scoring
+    "compare_stdout_results",
+    "extract_boxed_answer",
+    "parse_judge_choice",
+    "parse_pytest_outcomes",
+    "read_answer_file_or_last_reply",
+    "verify_boxed_math_answer",
+    # mcp
+    "Toolset",
+    "SharedToolsetConfig",
+    "ToolsetConfig",
+    # the user channel
+    "Interaction",
+    "Segment",
+    # deployment and scoring
+    "ComposeSpec",
+    "Deployment",
+    "DeploymentConfig",
+    "ScoringConfig",
+]
+
+
+def __getattr__(name: str):
+    module = _NAME_TO_MODULE.get(name)
+    if module is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    value = getattr(importlib.import_module(module), name)
+    globals()[name] = value
+    return value
+
+
+def __dir__() -> list[str]:
+    return sorted(set(globals()) | set(_NAME_TO_MODULE))
+
+
+# The library logs via stdlib logging (per-module `getLogger(__name__)`), but is
+# silent until an app opts in: a NullHandler on the package root absorbs records
+# so nothing is emitted (and no "no handler" warning) unless handlers are added.
+_logging.getLogger(__name__).addHandler(_logging.NullHandler())