karotte 3.0.4__py3-none-any.whl

karotte/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Preference Model
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

karotte/README.md

@@ -0,0 +1,270 @@
+# karotte
+
+karotte runs LLM agents on tasks and scores the results.
+
+You write an *environment*: a Python project with one or more tasks. A task is a
+list of steps. Each step gives the agent instructions and a judge that decides
+whether the agent succeeded. karotte builds the environment into a container
+image, lets the model work inside it through tools like `bash`, and records every
+message, tool call and score in a transcript.
+
+The agent runs as an unprivileged user with its own resource limits, a firewall,
+and a disk quota, so a task can hand it a real shell without trusting it.
+The model under test is called the *student*.
+
+## Install
+
+karotte needs Python 3.12+, [uv](https://docs.astral.sh/uv/), and docker with
+[buildx](https://github.com/docker/buildx#installing) (or
+podman with `--runtime podman`) for containerized runs. Ubuntu's `docker.io`
+lacks buildx; install `docker-buildx` too. The justfiles need
+[just](https://github.com/casey/just) 1.40 or newer; `uv sync --extra dev`
+installs one into the venv.
+
+```
+uv tool install karotte
+```
+
+## Quick start
+
+Create an environment from the `default` template and run its example task:
+
+```
+karotte create-env my_env
+cd my_env
+uv sync --extra dev
+uv run setup_data.py
+uv run karotte create-run-config --model claude-fable-5
+export ANTHROPIC_API_KEY=...
+uv run karotte run --config run_config.json
+```
+
+Commands that load your tasks (`create-run-config`, `run`, `tasks list`,
+`check`) have to run in the environment's venv, hence `uv run`.
+`setup_data.py` prepares data the image needs, such as model weights; the
+`default` template's version does nothing. `karotte check` loads every task and
+tool and fails if one doesn't load; the image build runs it too.
+
+`karotte run` builds the image, runs the task, and writes the transcript to
+`out/transcript.json`. Files the task saves as artifacts go to
+`out/<run_id>_artifacts/`. `karotte dashboard out/` shows the transcripts in `out/`.
+`karotte models list` shows which models karotte knows; model ids are passed to
+[litellm](https://docs.litellm.ai/). The API key is `model_api_key` in the run
+config. `create-run-config` sets it to the key variable of the model's
+provider, such as `$ANTHROPIC_API_KEY` or `$OPENAI_API_KEY`, which karotte reads
+from your environment when the run starts. `--model-api-key` sets another.
+
+Useful while you work on a task:
+
+- `uv run karotte run --config run_config.json --dev` skips the image build and
+  mounts your `src/environment/` into the container.
+- `uv run karotte tasks list` shows the tasks the environment defines;
+  `create-run-config --task <task-id>` picks one.
+
+### The container image
+
+Every `karotte run` without `--dev` rebuilds the image first. podman and docker
+reuse cached layers, so an unchanged environment builds quickly.
+
+`--dev` skips the build and runs the existing image, with your
+`src/environment/` mounted over the installed copy. Rebuild after changing
+dependencies, the `Containerfile` or data.
+
+`karotte build` builds the same image without running a task. `run` always uses
+the image tagged `karotte`, which is `build`'s default tag, and has no option to
+pick another. `build --tag` is for images you use elsewhere.
+
+### Model endpoints and proxies
+
+`karotte run --proxy <url>` sends the model calls to `<url>` instead of the
+provider. Pass the root URL, without `/v1`:
+
+- Claude models go to `<url>/v1/messages`. karotte sets `ANTHROPIC_BASE_URL` to
+  `<url>` in the container, and litellm adds `/v1/messages`.
+- Other models go to `<url>/v1`, so the endpoint needs an OpenAI-compatible API
+  there. `vertex_ai/` models ignore the proxy.
+
+The same flag points karotte at any endpoint that serves these paths. The URL is
+used inside the container, where `localhost` is the container itself. Variables
+like `ANTHROPIC_BASE_URL` in your shell don't reach the container. With
+`--no-containerized`, karotte sets `ANTHROPIC_BASE_URL` and `KAROTTE_PROXY_URL`
+in its own process instead, unless one of them is already set.
+`--no-containerized` only runs inside a karotte container image, where
+`KAROTTE_CONTAINERIZED` is set.
+
+A key is needed only if the endpoint asks for one. karotte still sends
+`model_api_key`; if that names a variable such as `$OPENAI_API_KEY` that is
+unset, `--proxy` fills in a placeholder.
+
+Without `--proxy`, karotte uses the URL a plugin registers under
+`karotte.default_proxy_url` (see [Plugins](#plugins)). Without such a plugin,
+calls go straight to the provider. `--no-proxy` ignores the plugin's URL.
+
+`KAROTTE_INFERENCE_SERVICE_TIER=priority` asks Fireworks and Vertex AI Gemini
+for their priority tier, `auto` only once the provider runs out of capacity.
+Unset, the default, uses the provider's default tier.
+
+## Writing tasks
+
+Each task is a package in `src/environment/tasks/`: a directory whose
+`__init__.py` defines the `Task` class. Plain files and directories starting with
+`_` are skipped. `uv run just create-task <task-id>` copies the task template
+(`just` comes with `uv sync --extra dev` and lives in the venv). A minimal task:
+
+```python
+import re
+
+from karotte import Step, Task
+from karotte.judges import RegexJudge
+
+
+class FindPython(Task):
+    id = "find-python"
+
+    @property
+    def system_prompt(self) -> str:
+        return "You are working in a Linux shell."
+
+    @property
+    def steps(self):
+        return [FindPythonStep(config=self.config)]
+
+    @property
+    def tools(self):
+        return ["bash"]
+
+
+class FindPythonStep(Step):
+    @property
+    def instructions(self) -> str:
+        return "Find the path to your Python executable. Answer with `path: <path>`."
+
+    @property
+    def judge(self):
+        return RegexJudge([re.compile(r"path: .*/python3?")])
+```
+
+Judges in `karotte.judges`: `RegexJudge` matches the final message,
+`ExecutableJudge` runs a scoring script, `RubricJudge` asks an LLM to grade
+against a rubric, and `Judge` is the base class for your own. The example task
+in the `default` template shows submissions, hints, and hooks that run before
+scoring.
+
+## Templates
+
+A template is a directory of files, rendered with
+[Jinja](https://jinja.palletsprojects.com/) into a new environment. karotte ships
+two:
+
+- `default`: a CPU environment with an example task. Every other template builds
+  on it.
+- `language-toolchains`: gives the agent exactly one language toolchain per task.
+
+`karotte templates list` shows every installed template. Templates stack:
+`karotte create-env my_env --template default --template language-toolchains`
+renders `default` and then `language-toolchains` on top. A later template can
+replace a file or override a Jinja block in it:
+
+```
+{% extends "default/CLAUDE.md" %}
+{% block claude_md -%}
+New content
+{% endblock %}
+```
+
+`karotte update` brings an existing environment up to the latest templates with
+a 3-way merge, so your own edits survive.
+
+### Writing a template package
+
+Templates can live in their own Python package. Put the template directories
+(each with a `template.toml`) under one directory and register it under the
+`karotte.templates` entry point:
+
+```toml
+[project.entry-points."karotte.templates"]
+my_templates = "my_package:TEMPLATES_DIR"
+```
+
+`TEMPLATES_DIR` is a path to that directory. A `template.toml` holds the fields of
+[`EnvironmentTemplate`](https://github.com/preferencemodel/karotte/blob/main/src/karotte/schemas/environment_template.py), for
+example:
+
+```toml
+description = "Adds a Rust toolchain."
+requires = ["default"]
+```
+
+Install the package next to karotte (`uvx --with my-package karotte create-env
+...`). karotte records it in the environment's manifest, so `karotte update`
+pulls it in again.
+
+## Plugins
+
+Besides templates, a package can extend karotte through these entry points:
+
+| Entry point | Points at | Effect |
+| --- | --- | --- |
+| `karotte.cli` | a Typer app | Adds a subcommand named after the entry point. |
+| `karotte.run_config_preprocessors` | `f(config) -> config` | Rewrites the run config before a run, in entry point name order. |
+| `karotte.default_proxy_url` | a string | Default for `karotte run --proxy`. |
+| `karotte.harness_secret_env` | a list of names | Environment variables hidden from the agent. |
+| `karotte.platform_tooling_dirs` | a list of paths | Directories where a platform mounts its own tooling into every container; hidden from graded toolchain runs. |
+| `karotte.age_delay_exemptions` | a list of package names | More packages exempt from uv's `exclude-newer` delay (karotte always is). |
+| `karotte.update_migrations` | an object with `prepare`, `tool` and `migrate` | Moves envs made by an older release to the current names during `karotte update`. |
+| `karotte.default_hardware` | a string | `required_hardware` of tasks that set none. karotte itself knows no hardware names. |
+| `karotte.hardware_limits` | `f(hardware) -> HardwareLimits \| None` | Memory and disk a sandbox on that hardware holds. Without it, the agent's memory limit is the sandbox's cgroup limit or RAM, less 1 GiB for the harness, and none where cgroups don't work. |
+| `karotte.container_run_args` | `f(task, runtime) -> list[str]` | Extra arguments for the container engine's `run` (docker, podman or nerdctl), e.g. to pass devices through, in entry point name order. Raising refuses the launch with the exception's message. |
+
+A plugin that fails to load is skipped with a warning, except a run config
+preprocessor: that one fails the run.
+
+## Connecting a backend
+
+Without a backend, karotte writes the transcript to a file. To collect runs
+centrally, set `backend_uri` in the run config. karotte then calls these HTTP
+endpoints on it, with `run_id` as a query parameter (presign has it in the body):
+
+| Request | Purpose |
+| --- | --- |
+| `POST /api/internal/create_transcript` | Start a transcript. |
+| `GET /api/internal/transcript_length` | Number of stored events, 404 if none. |
+| `POST /api/internal/append_transcript` | Body `{"event": ..., "seq": n}`. Writing the same `seq` again must overwrite, not append. |
+| `POST /api/internal/update_run_state` | Body with `status` (`running`, `passed`, `failed`, `error`), `score`, and token counts. |
+| `POST /api/artifacts/presign` | Body `{"run_id", "artifact_paths"}`, returns `{"presigned_urls": {path: url}}`. karotte PUTs each file gzipped. |
+
+With the `external` agent the backend also supplies the model's messages:
+`GET /api/internal/get_message` returns the next message (404 while there is
+none) and `POST /api/internal/delete_message` consumes it.
+
+Requests carry `Authorization: Bearer <token>` if the file at
+`KAROTTE_BACKEND_TOKEN_PATH` (default `/var/run/secrets/service-account-token`)
+exists. The events are the models in
+[`karotte.schemas.transcript`](https://github.com/preferencemodel/karotte/blob/main/src/karotte/schemas/transcript.py).
+
+## Development
+
+```
+just lint
+just test
+just test-template default
+```
+
+Every merge to `main` is released. The patch version is the commit count.
+
+## License
+
+karotte is under the [MIT license](https://github.com/preferencemodel/karotte/blob/main/LICENSE). The templates in
+`src/karotte/templates/` are under [MIT No Attribution](https://github.com/preferencemodel/karotte/blob/main/src/karotte/templates/LICENSE)
+(`MIT-0`), so environments created from them need no license notice.
+
+### Third-party software in built images
+
+Images built from the templates contain third-party software under its own
+licenses. They are based on Amazon Linux 2023. The `language-toolchains`
+template installs GPL and LGPL software (gcc, GnuCOBOL, GNU Prolog, Free Pascal,
+the libraries bundled with Julia, and others), Amazon Corretto (GPL-2.0 with the
+Classpath Exception) and Clojure (EPL-1.0) for the languages you enable. Mojo,
+off by default, is under Modular's proprietary license and has its own terms even
+if you never share the image. The rest only matters if you redistribute a built
+image, for example by pushing it to a public registry.

karotte/__init__.py

@@ -0,0 +1,53 @@
+"""karotte: A library for creating and running AI agent evaluation environments."""
+
+from karotte.confinement import Contract as Contract
+from karotte.confinement import FileLimit as FileLimit
+from karotte.confinement import ResourceLimits as ResourceLimits
+from karotte.confinement import get_resource_limits as get_resource_limits
+from karotte.confinement import limit_resources as limit_resources
+from karotte.judges.judge import Judge as Judge
+from karotte.process_utils import UnreapableCohortError as UnreapableCohortError
+from karotte.process_utils import kill_processes as kill_processes
+from karotte.protected_store import ProtectedStore as ProtectedStore
+from karotte.reclaim import ReclaimError as ReclaimError
+from karotte.reclaim import delete_files as delete_files
+from karotte.runtime import Runtime as Runtime
+from karotte.save_artifact import save_artifact as save_artifact
+from karotte.save_submission import save_submission as save_submission
+from karotte.schemas.evaluation_run_config import (
+    EvaluationRunConfig as EvaluationRunConfig,
+)
+from karotte.step import Step as Step
+from karotte.student_misbehavior import (
+    StudentMisbehaviorError as StudentMisbehaviorError,
+)
+from karotte.task import Task as Task
+from karotte.task_factory import StepConfig as StepConfig
+from karotte.task_factory import create_task as create_task
+from karotte.tool_base import ToolBase as ToolBase
+from karotte.tool_base import ToolConfigWriter as ToolConfigWriter
+
+__all__ = [
+    "Task",
+    "Step",
+    "Judge",
+    "EvaluationRunConfig",
+    "ToolBase",
+    "ToolConfigWriter",
+    "ProtectedStore",
+    "save_artifact",
+    "save_submission",
+    "create_task",
+    "StepConfig",
+    "Runtime",
+    "kill_processes",
+    "UnreapableCohortError",
+    "delete_files",
+    "ReclaimError",
+    "StudentMisbehaviorError",
+    "limit_resources",
+    "get_resource_limits",
+    "ResourceLimits",
+    "Contract",
+    "FileLimit",
+]

karotte/agents/__init__.py

@@ -0,0 +1,52 @@
+"""Pluggable agents for the evaluation runner.
+
+An `Agent` produces a run's transcript one step at a time; the runner keeps the
+task lifecycle (hooks, judging, artifacts, event bookkeeping). Builtin and
+external agents share `MessageLoopAgent`, differing only in their
+`MessageSource` -- builtin drives the litellm API loop itself, external fetches
+messages from the backend (training rollouts). Fake replays env-authored
+messages.
+"""
+
+from karotte.agents.agent import (
+    Agent,
+    EmptyTurnLimitReachedError,
+    RunContext,
+    TurnLimitReachedError,
+)
+from karotte.agents.backend_source import EXTERNAL_MESSAGE_TIMEOUT_S, BackendSource
+from karotte.agents.builtin import BuiltinAgent
+from karotte.agents.builtin_source import BuiltinSource
+from karotte.agents.cli_agent import (
+    CliAgent,
+    cli_agent_types,
+    get_cli_agent_type,
+    register_cli_agent,
+)
+from karotte.agents.external import ExternalAgent
+from karotte.agents.fake_source import FakeSource
+from karotte.agents.message_loop import MessageLoopAgent
+from karotte.agents.message_source import MessageSource
+
+# Import agent modules for their `register_cli_agent` side effects.
+from karotte.agents.mistral_vibe import MistralVibeAgent
+
+__all__ = [
+    "EXTERNAL_MESSAGE_TIMEOUT_S",
+    "Agent",
+    "BackendSource",
+    "BuiltinAgent",
+    "BuiltinSource",
+    "CliAgent",
+    "EmptyTurnLimitReachedError",
+    "ExternalAgent",
+    "FakeSource",
+    "MessageLoopAgent",
+    "MessageSource",
+    "MistralVibeAgent",
+    "RunContext",
+    "TurnLimitReachedError",
+    "cli_agent_types",
+    "get_cli_agent_type",
+    "register_cli_agent",
+]

karotte/agents/agent.py

@@ -0,0 +1,97 @@
+from collections.abc import AsyncGenerator
+from dataclasses import dataclass
+from typing import ClassVar, Literal, Protocol, final
+
+from fastmcp import Client
+from fastmcp.client import StreamableHttpTransport
+from litellm import ChatCompletionToolParam
+
+from karotte.schemas.evaluation_run_config import EvaluationRunConfig
+from karotte.schemas.transcript import Event, Transcript
+
+
+class TurnLimitReachedError(Exception):
+    """Raised when the evaluation run exceeds the configured turn limit."""
+
+
+class StepTimeLimitReachedError(Exception):
+    """Raised when a step exceeds its configured wall-clock time limit."""
+
+
+class StepContextWindowLimitReachedError(Exception):
+    """Raised when a step exceeds its configured context-window token limit."""
+
+
+class EmptyTurnLimitReachedError(Exception):
+    """Raised when the model produced neither text nor tool calls too many turns
+    in a row, so nudging it to continue has stopped working."""
+
+
+@final
+@dataclass
+class RunContext:
+    """What an agent needs to solve steps within a run.
+
+    The runner owns the transcript and applies event bookkeeping; the agent
+    reads `transcript.messages` for conversation history and executes tool calls
+    against `mcp_client`. One `RunContext` per run, passed to `Agent.start`.
+    """
+
+    config: EvaluationRunConfig
+    mcp_client: Client[StreamableHttpTransport]
+    tools: list[ChatCompletionToolParam]
+    transcript: Transcript
+
+
+class Agent(Protocol):
+    """Produces a run's transcript, one step at a time.
+
+    The runner keeps the task lifecycle (hooks, judging, artifacts, event
+    bookkeeping); the agent decides how messages and tool calls get produced.
+    An agent instance is stateful across steps within a run.
+    """
+
+    allows_student_mcp_access: ClassVar[bool]
+    """Whether the student user may reach the MCP port.
+
+    True only for agents that run in-container as the student and make native
+    MCP tool calls (CLI agents). Builtin/external agents keep the port blocked:
+    karotte executes their tool calls itself, so student access would only let
+    task code bypass transcript recording.
+    """
+
+    native_tool_names: ClassVar[frozenset[str]]
+    """Tools this agent supplies itself, and which the runner therefore
+    omits when registering the task's tools. A CLI agent with its own bash/file
+    tools lists them here so it uses its native ones instead of duplicates over
+    MCP. Empty for agents that rely entirely on MCP tools.
+    """
+
+    async def start(self, ctx: RunContext) -> None:
+        """Verify/configure the agent for the run."""
+        ...
+
+    def run_step(
+        self,
+        instructions: str,
+        time_limit_seconds: float | None = None,
+        on_time_limit: Literal["error", "score"] = "error",
+        context_window_limit: int | None = None,
+        on_context_window_limit: Literal["error", "score"] = "error",
+    ) -> AsyncGenerator[Event]:
+        """Solve one step, yielding raw transcript events.
+
+        `time_limit_seconds` bounds the step's wall-clock duration; when it
+        elapses the step either aborts (`on_time_limit="error"`) or ends so it
+        can be scored (`on_time_limit="score"`). `None` disables the limit.
+
+        `context_window_limit` bounds the step's context-window length (the input
+        tokens of the most recent turn); when it is exceeded the step either
+        aborts (`on_context_window_limit="error"`) or ends so it can be scored
+        (`on_context_window_limit="score"`). `None` disables the limit.
+        """
+        ...
+
+    async def stop(self) -> None:
+        """Release any resources held for the run."""
+        ...

karotte/agents/backend_source.py

@@ -0,0 +1,47 @@
+import time
+from collections.abc import AsyncGenerator
+from typing import final
+
+from litellm import ChatCompletionToolParam
+from loguru import logger
+
+from karotte.backend_client import BackendClient
+from karotte.schemas.chat import Message
+from karotte.schemas.transcript import Event
+
+# Maximum time to wait for a message from the model before giving up.
+EXTERNAL_MESSAGE_TIMEOUT_S = 120 * 60
+
+
+@final
+class BackendSource:
+    """Long-polls the backend for the next message produced by an external model
+    (training rollouts). karotte still drives the step loop and executes the tool
+    calls the message contains."""
+
+    def __init__(self, backend_client: BackendClient, run_id: str) -> None:
+        self.backend_client = backend_client
+        self.run_id = run_id
+
+    async def collect(
+        self,
+        messages: list[Message],  # pyright: ignore[reportUnusedParameter]
+        tools: list[ChatCompletionToolParam],  # pyright: ignore[reportUnusedParameter]
+    ) -> AsyncGenerator[Event]:
+        start = time.time()
+        last_log = time.time()
+        while True:
+            elapsed = time.time() - start
+            if elapsed > EXTERNAL_MESSAGE_TIMEOUT_S:
+                raise TimeoutError(
+                    f"Timed out after {elapsed:.0f}s waiting for a response from the model"
+                )
+            if time.time() - last_log > 30:
+                logger.warning(f"Still waiting for new message, {elapsed:.0f}s elapsed")
+                last_log = time.time()
+            # get_message is a long-poll operation so we don't need extra sleep
+            msg_added_evt = await self.backend_client.get_message(self.run_id)
+            if msg_added_evt is not None:
+                yield msg_added_evt
+                await self.backend_client.delete_message(self.run_id)
+                return

karotte/agents/builtin.py

@@ -0,0 +1,13 @@
+from typing import final
+
+from karotte.agents.builtin_source import BuiltinSource
+from karotte.agents.message_loop import MessageLoopAgent
+from karotte.schemas.evaluation_run_config import EvaluationRunConfig
+
+
+@final
+class BuiltinAgent(MessageLoopAgent):
+    """karotte drives the model itself via the litellm API loop."""
+
+    def __init__(self, config: EvaluationRunConfig) -> None:
+        super().__init__(BuiltinSource(config))