evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/bundled/contracts/span-mapping-v1.json

@@ -0,0 +1,314 @@
+{
+  "$defs": {
+    "AttributeRule": {
+      "additionalProperties": false,
+      "properties": {
+        "attribute": {
+          "description": "The span attribute key to read.",
+          "title": "Attribute",
+          "type": "string"
+        },
+        "json": {
+          "default": false,
+          "description": "Parse the value with json.loads when it is a string; keep the string on failure.",
+          "title": "Json",
+          "type": "boolean"
+        },
+        "values": {
+          "anyOf": [
+            {
+              "additionalProperties": {
+                "type": "string"
+              },
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Translate the read value through this table.",
+          "title": "Values"
+        },
+        "message_format": {
+          "anyOf": [
+            {
+              "enum": [
+                "openinference",
+                "gen_ai",
+                "openai"
+              ],
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Collect messages laid out per this format under the attribute key.",
+          "title": "Message Format"
+        }
+      },
+      "required": [
+        "attribute"
+      ],
+      "title": "AttributeRule",
+      "type": "object"
+    },
+    "TokenRules": {
+      "additionalProperties": false,
+      "properties": {
+        "prompt": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/AttributeRule"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null
+        },
+        "completion": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/AttributeRule"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null
+        },
+        "cached": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/AttributeRule"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null
+        },
+        "cache_write": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/AttributeRule"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null
+        }
+      },
+      "title": "TokenRules",
+      "type": "object"
+    },
+    "ToolRules": {
+      "additionalProperties": false,
+      "properties": {
+        "name": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/AttributeRule"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null
+        },
+        "call_id": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/AttributeRule"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null
+        },
+        "arguments": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/AttributeRule"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null
+        },
+        "result": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/AttributeRule"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null
+        }
+      },
+      "title": "ToolRules",
+      "type": "object"
+    }
+  },
+  "additionalProperties": false,
+  "properties": {
+    "schema_version": {
+      "const": "1",
+      "title": "Schema Version",
+      "type": "string"
+    },
+    "kind": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/AttributeRule"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null
+    },
+    "session_id": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/AttributeRule"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null
+    },
+    "model": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/AttributeRule"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null
+    },
+    "system_instructions": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/AttributeRule"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "System prompt kept outside the input messages: a string or a list of text parts. Read on llm spans and put first in the input messages."
+    },
+    "input_messages": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/AttributeRule"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null
+    },
+    "output_messages": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/AttributeRule"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null
+    },
+    "tokens": {
+      "$ref": "#/$defs/TokenRules"
+    },
+    "cost_usd": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/AttributeRule"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null
+    },
+    "tool": {
+      "$ref": "#/$defs/ToolRules"
+    },
+    "agent_name": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/AttributeRule"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "The agent's name on agent spans; names the subagent trajectory. The span name is used when it is absent."
+    },
+    "input": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/AttributeRule"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null
+    },
+    "output": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/AttributeRule"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null
+    },
+    "known_attributes": {
+      "description": "Attributes no rule reads that are expected and need nothing: not reported as unmapped. Glob patterns such as llm.tools.* are allowed.",
+      "items": {
+        "type": "string"
+      },
+      "title": "Known Attributes",
+      "type": "array"
+    },
+    "answer": {
+      "default": "root",
+      "description": "Where the run's answer comes from: root, a span name, or <span name>.<output key>.",
+      "title": "Answer",
+      "type": "string"
+    },
+    "exclude_spans": {
+      "description": "Span names left out of the trajectory.",
+      "items": {
+        "type": "string"
+      },
+      "title": "Exclude Spans",
+      "type": "array"
+    }
+  },
+  "required": [
+    "schema_version"
+  ],
+  "title": "MappingDocument",
+  "type": "object"
+}

evalkit/bundled/docs/concepts.md

@@ -0,0 +1,26 @@
+# Concepts
+
+The words you need before your first scorecard. Everything else is explained on the page that uses it.
+
+| Term                       | Meaning                                                                                                                              |
+| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
+| Case                       | One eval question, written in YAML: the input, the rubrics a right answer meets, and the checks to run.                              |
+| Dataset                    | A directory of cases, run and tracked together. The directory name is the dataset name, and subfolders become tags you can run alone. |
+| Sync                       | Copying a dataset directory to the server. Runs do it for you.                                                                       |
+| Workspace                  | Where datasets, agents and experiments live: your own (named after you), or a shared one such as `dev`. Every user can read every workspace; only the owner changes a personal one, and admins and writers change a shared one.                              |
+| Reference                  | `name@workspace`, for example `acme@dev`. A bare name means your own workspace.                                                      |
+| Agent                      | Your agent, registered once on the server. A code-driven agent is run by your own code; this is the usual way to start.             |
+| Task                       | The function `run_experiment` calls once per case. It calls your agent and returns the answer.                                       |
+| Experiment                 | One run of a dataset against an agent.                                                                                               |
+| Run                        | One case answered once.                                                                                                              |
+| Trace, span, session       | What your agent records as it works: one span per model call or tool call. One run is one session.                                  |
+| Mapping                    | Which span attribute holds the model, the messages, the tokens and each tool call.                                                   |
+| Trajectory                 | The step-by-step record of a run that evalkit builds from the spans.                                                                 |
+| Judge                      | The LLM that grades an answer against the case's rubrics.                                                                            |
+| Evaluator                  | One scoring check, for example the rubric check or "did the agent call this tool".                                                   |
+| Scorecard                  | The table of scores for an experiment, per case and per bucket (content, trajectory, presentation, efficiency, custom).              |
+| Unreviewed                 | A case in a shared dataset with no review decision at its current content. It runs like any other case; review decides promotion.  |
+| Promotion                  | An admin links a shared workspace to a target (`evalkit workspaces promote-to dev prod`). Approving a case in `dev` adds it, approved, to the same-named dataset in `prod`. Datasets in `prod` are fed only that way. |
+| `evalkit.yaml`             | The file in your repo that says where your cases are and which environment variables hold API keys.                                  |
+
+Next: [Try locally](try-locally.md) to see these on a running server, or the [Quickstart](quickstart.md) to connect your own agent.

evalkit/bundled/docs/config.md

@@ -0,0 +1,38 @@
+# Config: `evalkit.yaml`
+
+`evalkit.yaml` sits at the root of the repository that holds your eval cases or your agent. The CLI and the SDK look for it above the current directory. It names the case and fixture directories and the API keys the runs from this machine use.
+
+```yaml
+cases: eval/cases # optional: directory whose children are datasets
+fixtures: eval/fixtures # optional: fixture bytes, flat, keyed by filename
+
+credentials: # values are env:NAME, read when a run needs them
+  agents:
+    myagent: env:MYAGENT_KEY # a bare name means your own workspace
+  evaluators:
+    citation_check: env:EVALUATOR_KEY
+  llm_providers:
+    https://litellm.example.com: env:LITELLM_API_KEY # keyed by the judge's base URL
+```
+
+Only your personal Harness API agents use `credentials.agents` here. The server drives shared agents (`ouragent@dev`) with the keys stored in its database, so an entry for one in `evalkit.yaml` is never read. The same holds for shared remote evaluators.
+
+Put the values in a `.env` file next to `evalkit.yaml` (and keep it out of git), or in your shell. A variable set in the shell wins over `.env`.
+
+Agents, remote evaluators and the judge are not configured here:
+
+- **Agents** are created once on the server with `evalkit agents create` (or on the Agents page). A personal agent runs from your machine; a shared one is driven by the server. An agent without a harness URL is code-driven: only `run_experiment` can run it. See [SDK runs](sdk-runs.md).
+- **Remote evaluators** are created with `evalkit evaluators create`. A run uses the evaluators in its agent's workspace.
+- **The judge** of an experiment is the server's default judge (an admin sets it with `evalkit judge-default set`) unless you pick one: `evalkit run … --judge-model M --judge-base-url URL --judge-effort E`, the run dialog's "Judge provider" picker, or `run_experiment(judge=Judge(...))`. Re-evaluating an experiment always reuses its judge.
+
+Rules:
+
+- `env:NAME` values are read only when a command needs them. A missing variable is an error that names it.
+- An agent or evaluator with no `credentials` entry is called without a key, and the run prints one line saying so. A judge with no entry stops the run before its first case: `no credentials.llm_providers entry for <url>`.
+- `credentials.judges` is now `credentials.llm_providers`. A file that still uses the old name fails to load.
+- An `llm_providers` key may itself be `env:NAME` (`env:LLM_BASE_URL: env:LLM_API_KEY`), so the judge's base URL can come from the environment too. A key whose variable is unset is skipped.
+- `cases` and `fixtures` are optional. Leave them out when you only run shared datasets, for example from `run_experiment`. Commands that need them say so:
+  - `evalkit sync <name>` and `evalkit run <name>` on a dataset in your workspace need `cases` to find `<cases>/<name>`. Passing the dataset's directory instead (`evalkit sync eval/cases/finance`) works without it.
+  - A sync uploads fixture bytes from `fixtures`. Without it, a case whose fixture the server does not already hold fails the sync with `E-FIXTURE-MISSING`.
+  - `run_experiment` on a dataset in your workspace skips the sync without `cases` and runs the copy the server holds.
+  - Fixtures a run downloads go to `fixtures`, or to a temporary directory when it is not set.

evalkit/bundled/docs/harness-api.md

@@ -0,0 +1,44 @@
+# Harness API
+
+The Harness API is the HTTP contract a client's agent harness implements so evalkit can drive it: start a run, poll its status, serve its trajectory. Agents in a shared workspace are driven by this server; agents in a personal workspace by `evalkit run` from their owner's machine. The OpenAPI schema ships inside the CLI:
+
+```bash
+evalkit contracts export --target ./contracts   # harness-api-v1.json, atif-accepted.json, …
+```
+
+Each endpoint lets evalkit do one thing. Every Harness API has the first three; declare the rest in the capabilities response when you implement them. An agent on the SDK + traces path has no Harness API and none of these endpoints; evalkit sees what its task returns and the spans it exports.
+
+| Endpoint                                              | Lets evalkit                                                                          | Capability flag           | Without it                                                                                                     |
+| ----------------------------------------------------- | ------------------------------------------------------------------------------------- | ------------------------- | -------------------------------------------------------------------------------------------------------------- |
+| `GET /capabilities`                                   | see what the harness supports and gate evaluators on it (snapshotted per experiment)  | part of every Harness API | —                                                                                                              |
+| `POST /runs`                                          | invoke the agent: start one run in a fresh session with the case input and files      | part of every Harness API | —                                                                                                              |
+| `GET /runs/{id}`                                      | wait for the run and read its answer, structured output, model and artifact list      | part of every Harness API | —                                                                                                              |
+| `GET /runs/{id}/trajectory`                           | read the trajectory (ATIF): the run view, trajectory evaluators and token/cost gauges | `provides.trajectory`     | trajectory evaluators skip, unless the agent's trajectory source is `otel` and the trajectory comes from spans |
+| `GET /runs/{id}/files`, `GET /runs/{id}/files/{path}` | read the workspace: the run's Workspace tab, downloads and artifact copies            | `provides.workspace`      | no Workspace tab; artifacts the run reports are not copied                                                     |
+| `GET /runs`                                           | list the agent's sessions on the Runs page and for `evalkit runs list`/`runs pull`, including ones evalkit did not start; optional `since`/`until` filter by creation time | `provides.run_listing`    | the Runs page is empty for this agent                                                                          |
+| `POST /runs/{id}/stop`                                | cancel a run that hit its time limit                                                  | `accepts.stop`            | timed-out runs keep running on the harness                                                                     |
+
+`POST /runs` takes JSON, or multipart when the case has fixture files and the harness accepts `files` but not `file_urls`. Run status moves through `queued`, `running`, then one of `completed`, `failed`, `cancelled`.
+
+A code-driven agent (no `harness_url`) has none of these endpoints, so its runs have no Workspace tab. To see an agent's files in evalkit, give it a Harness API with at least the files endpoints; its trajectory can still come from traces.
+
+Every request carries `x-api-key`. Two rules matter more than the rest:
+
+- **Capabilities honesty.** Declare only what works. evalkit neutral-skips evaluators whose inputs the harness does not provide; a false `true` produces failing scores blamed on the agent.
+- **Implement stop.** evalkit calls stop on timeout when `accepts.stop` is true; a harness without it leaks resources on every timed-out run.
+
+## Already have a harness? Keep it
+
+A harness that already emits OpenTelemetry traces can skip the trajectory endpoint: create the agent with `--trajectory-source otel --mapping FILE` and export spans to the collector with the run id it returned from `POST /runs` in the attribute the mapping's `session_id` rule names (`session.id` for OpenInference, `gen_ai.conversation.id` for GenAI). Everything else stays: server runs, `evalkit run`, the Workspace tab. The harness declares `provides.trajectory: false`. See [Tracing](tracing.md).
+
+The session evalkit polls is the `run_id` that `POST /runs` returns; there is no separate session field. A queue-backed agent whose worker opens its own session can return that session id as `run_id`, or stamp the `run_id` it returned on the job's spans in the mapping's session id attribute. Either way the ids must also work with the other `/runs/{id}` endpoints.
+
+Moving to [SDK runs](sdk-runs.md) instead only makes sense to stop hosting the harness. A code-driven agent loses server-started runs and the Workspace tab, and `run_experiment` refuses an agent with a `harness_url`: keep one agent name and run it with `evalkit run`, rather than adding a second entry for `run_experiment`.
+
+## Checking
+
+```bash
+evalkit harness check ouragent@dev
+```
+
+runs a trivial call, exercises every declared capability and prints a pass/fail table. For a harness agent whose trajectory source is `otel` the trajectory row reads "from traces" and a session row reports the collected spans.

evalkit/bundled/docs/install.md

@@ -0,0 +1,60 @@
+# Install
+
+evalkit is one Python package on PyPI, `evalkit-py`. It holds the `evalkit` command (the CLI), the `evalkit.sdk` module for SDK runs and, with the `server` extra, the server itself.
+
+Your CLI and SDK must run the **same version as the evalkit server** you use: local runs and SDK runs refuse to start otherwise. The commands on this page pin version **{{version}}**. On your server's own `/docs/install` page, that is the server's version; `curl <server url>/api/cli` prints it anywhere.
+
+## The CLI, as a global tool
+
+```bash
+uv tool install --reinstall "evalkit-py=={{version}}"
+```
+
+With pipx instead: `pipx install --force "evalkit-py=={{version}}"`.
+
+`curl {{origin}}/api/cli` prints the server's version, for scripts and CI:
+
+```bash
+uv tool install --reinstall "evalkit-py==$(curl -fsSL {{origin}}/api/cli | jq -r .version)"
+```
+
+## The SDK, into a project
+
+`run_experiment` and `tracing.register` live in the same package as the CLI. The `otel` extra brings the OpenTelemetry SDK and the OTLP exporter. Pin the server's version; `uv.lock` records it:
+
+```bash
+uv add "evalkit-py[otel]=={{version}}"
+```
+
+With pip instead: `pip install "evalkit-py[otel]=={{version}}"`.
+
+To keep evalkit out of the app's dependencies, run the eval script with it instead:
+
+```bash
+uv run --with "evalkit-py[otel]=={{version}}" python eval/run.py
+```
+
+## The server
+
+`pip install "evalkit-py[server]"` installs the server too: the API, the web UI and migrations. To run one on your machine against a Postgres you already have:
+
+```bash
+evalkit init-server   # creates the database, writes config.yaml and .env in ./evalkit-try
+cd evalkit-try && evalkit serve
+```
+
+[Try locally](try-locally.md) walks through it with a sample agent; the evalkit-deployment skill covers running a server for a team.
+
+## Updating
+
+Every API response carries `X-Evalkit-Version`. The CLI warns when it differs from the server and refuses when the major versions differ. Local runs and SDK runs refuse any difference, so after a server deploy, from the project's root:
+
+```bash
+evalkit update
+```
+
+It reinstalls the CLI at the server's version and, when `pyproject.toml` pins `evalkit-py==…`, moves the pin and runs `uv sync`. It then refreshes the skills in every directory `evalkit skills install` wrote to on this machine (listed in `~/.evalkit/skills-targets`), and in `./.claude/skills` or `./.agents/skills` if they hold an earlier install; commit the changes in repos that track the skills. Pass `--no-skills` to skip that.
+
+A project that vendored an evalkit wheel before evalkit was on PyPI moves to PyPI once: `uv remove evalkit && uv add "evalkit-py[otel]=={{version}}"`.
+
+Next: [Login](login.md).

evalkit/bundled/docs/login.md

@@ -0,0 +1,36 @@
+# Login
+
+```bash
+evalkit login --url {{origin}}
+```
+
+The command asks for your username and password and stores the server URL and your token in `~/.evalkit/config.toml`. The CLI talks to one server at a time: logging in again, to the same server or another, replaces the stored login. Dev and prod are workspaces on one server (`finance@dev`, `finance@prod`), not separate logins.
+
+Tokens expire after the TTL the deployer configured (30 days by default). `evalkit whoami` shows who the token logs in as and when it expires, and exits 1 if the server rejects it; run it before a long run. Commands that talk to the server warn when the token expires within a day. When commands start failing with 401, log in again.
+
+## Scripts and CI
+
+Two environment variables bypass the config file:
+
+```bash
+export EVALKIT_SERVER_URL={{origin}}
+export EVALKIT_TOKEN=<token>
+```
+
+A stored token expires, so CI jobs should log in at the start of every run with a password from the secret store:
+
+```bash
+evalkit login --url "$EVALKIT_SERVER_URL" --username admin --password "$EVALKIT_ADMIN_PASSWORD"
+```
+
+## Timeout
+
+The CLI and the SDK wait up to 120 seconds for each server call. Set `EVALKIT_TIMEOUT` (seconds) to wait longer, for example on a slow network:
+
+```bash
+export EVALKIT_TIMEOUT=300
+```
+
+It covers calls to the evalkit server only. The server's own calls to an agent's harness keep their fixed limits, and so does `evalkit harness check`.
+
+The SDK uses the same login and the same two variables. A deployed agent that only exports traces uses an agent token instead; see [Tracing](tracing.md).

evalkit/bundled/docs/mappings.md

@@ -0,0 +1,136 @@
+# Mappings
+
+A mapping tells evalkit which span attribute holds each field of the evalkit span format (ESF): the span kind, the session id, the model, the messages, the token counts, the tool name and call id. It also says where the run's answer comes from (`answer`) and which spans to leave out (`exclude_spans`).
+
+Every agent whose trajectories come from traces has its own mapping. `evalkit agents create NAME --trajectory-source otel --mapping FILE` makes the first version active. Start from the `connect-traces` skill's `openinference.json` or `gen_ai.json`, which cover agents on the OpenInference or GenAI semantic conventions as they are. Spans that reach an agent with no active mapping are stored without a session id.
+
+An existing agent switches between ATIF and traces with `evalkit agents edit NAME --trajectory-source otel --mapping FILE` (or `--trajectory-source atif`), or from the agent's Edit dialog. Switching to traces without a file reuses the agent's last active mapping. The switch is refused while one of the agent's experiments is running, and experiments that already ran keep the source they ran with.
+
+A mapping applies to every span of the agent and is versioned: `evalkit traces mapping push` creates a draft, you review the draft against recent sessions on the agent's Mappings tab (Agents → the agent → Mappings), and activate it there. The `connect-traces` skill walks a coding agent through writing one.
+
+## The document
+
+```json
+{
+  "schema_version": "1",
+  "kind": {
+    "attribute": "openinference.span.kind",
+    "values": {
+      "LLM": "llm",
+      "TOOL": "tool",
+      "AGENT": "agent",
+      "CHAIN": "chain",
+      "RETRIEVER": "retriever"
+    }
+  },
+  "session_id": {
+    "attribute": "session.id"
+  },
+  "model": {
+    "attribute": "llm.model_name"
+  },
+  "input_messages": {
+    "attribute": "llm.input_messages",
+    "message_format": "openinference"
+  },
+  "output_messages": {
+    "attribute": "llm.output_messages",
+    "message_format": "openinference"
+  },
+  "tokens": {
+    "prompt": {
+      "attribute": "llm.token_count.prompt"
+    },
+    "completion": {
+      "attribute": "llm.token_count.completion"
+    },
+    "cached": {
+      "attribute": "llm.token_count.prompt_details.cache_read"
+    },
+    "cache_write": {
+      "attribute": "llm.token_count.prompt_details.cache_write"
+    }
+  },
+  "cost_usd": {
+    "attribute": "llm.cost.total"
+  },
+  "tool": {
+    "name": {
+      "attribute": "tool.name"
+    },
+    "call_id": {
+      "attribute": "tool.id"
+    },
+    "arguments": {
+      "attribute": "input.value",
+      "json": true
+    },
+    "result": {
+      "attribute": "output.value"
+    }
+  },
+  "agent_name": {
+    "attribute": "agent.name"
+  },
+  "input": {
+    "attribute": "input.value",
+    "json": true
+  },
+  "output": {
+    "attribute": "output.value",
+    "json": true
+  },
+  "known_attributes": [
+    "openinference.span.kind",
+    "session.id",
+    "user.id",
+    "metadata",
+    "tag.tags",
+    "input.mime_type",
+    "output.mime_type",
+    "llm.provider",
+    "llm.system",
+    "llm.invocation_parameters",
+    "llm.finish_reason",
+    "llm.tools.*",
+    "llm.token_count.total",
+    "llm.token_count.prompt_details.*",
+    "llm.token_count.completion_details.*",
+    "llm.cost.*",
+    "tool.description",
+    "tool.parameters",
+    "tool.json_schema",
+    "agent.name",
+    "graph.node.*"
+  ]
+}
+```
+
+That is the skill's `openinference.json`. Its `gen_ai.json` reads `gen_ai.operation.name` (`chat`, `execute_tool`, `invoke_agent`, …), `gen_ai.request.model`, `gen_ai.input.messages` / `gen_ai.output.messages` with `"message_format": "gen_ai"`, `gen_ai.usage.input_tokens` and friends, reads the system prompt from `gen_ai.system_instructions`, cache writes from `gen_ai.usage.cache_creation.input_tokens` and the agent's name from `gen_ai.agent.name`, and sets `"cost_usd": null` because the conventions define no cost attribute. Both list the other attributes of their convention under `known_attributes`, so `evalkit traces check` does not report them as unmapped; entries may be exact keys or globs such as `llm.tools.*`.
+
+Rule shapes:
+
+| Shape                                   | Result                                                                                                                                                                       |
+| --------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `{"attribute": K}`                      | the attribute's value, or null                                                                                                                                               |
+| `{"attribute": K, "json": true}`        | the value parsed as JSON when it is a string; the string stays when parsing fails                                                                                            |
+| `{"attribute": K, "values": {...}}`     | the value translated through the table; an unlisted kind becomes `other`                                                                                                     |
+| `{"attribute": K, "message_format": F}` | the messages laid out under `K` per format `F`: `openinference` (flattened indexed keys), `gen_ai` (JSON list of messages with parts), `openai` (JSON list of chat messages) |
+| `null`                                  | never read                                                                                                                                                                   |
+
+Two more fields shape the trajectory: `answer` (`root`, a span name, or `<span name>.<output key>`; default `root`) and `exclude_spans` (span names left out; default none).
+
+Every mapping is complete on its own; there is no inheritance. The schema ships as `span-mapping-v1.json` with `evalkit contracts export`.
+
+## The loop
+
+```bash
+evalkit traces check myagent <session id> --dump spans.json   # read the raw attributes
+evalkit traces mapping push myagent eval/span-mapping.json    # upload a draft
+evalkit traces check myagent <session id> --mapping draft     # preview with the draft
+evalkit traces mapping list myagent
+```
+
+Repeat until the unmapped list holds nothing that matters and the counts look right, then activate the draft on the agent's Mappings tab. `--activate` on push activates in the same step; a coding agent passes it only when the developer has approved the document.
+
+Runs record the mapping version they were converted with; a later mapping change never alters a scored run. The Runs page converts sessions on demand with the active mapping.