benchkit-cli 0.1.0__tar.gz → 0.1.2__tar.gz

{benchkit_cli-0.1.0/src/benchkit_cli.egg-info → benchkit_cli-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchkit-cli
-Version: 0.1.0
+Version: 0.1.2
 Summary: Benchmark your local LLMs with real eval suites
 License-Expression: Apache-2.0
 Requires-Python: >=3.11
@@ -69,8 +69,10 @@ cp .env.example .env
 uv run benchkit
 ```
 
-Set your server in a `.env` in the directory you run `benchkit` from (parent
-directories are not searched), or enter it on the Connect screen:
+Set your server in `~/.config/benchkit/.env` to use it from any folder, in a
+`.env` in the directory you run `benchkit` from (parent directories are not
+searched), or on the Connect screen, where "Save as default" writes the user
+config for you:
 
 ```env
 BENCHKIT_PROVIDER=openai
@@ -85,6 +87,8 @@ BENCHKIT_PROVIDER=ollama
 OLLAMA_HOST=http://localhost:11434
 ```
 
+`benchkit config` shows every setting and where it came from.
+
 No inference server available? Explore the complete UI offline:
 
 ```bash
@@ -132,6 +136,7 @@ uv run benchkit --help
 | Aider Polyglot | `aider-polyglot` | 225 | Repository editing across six languages with Pi |
 | HumanEval | `humaneval` | 164 | Python function completion |
 | HumanEval+ | `humaneval-plus` | 164 | HumanEval with expanded EvalPlus tests |
+| HumanEval+ TR | `humaneval-plus-tr` | 164 | HumanEval+ fully translated to Turkish, function and variable names included |
 | MBPP | `mbpp` | 500 | Short Python programming tasks |
 | MBPP+ | `mbpp-plus` | 378 | MBPP with expanded EvalPlus tests |
 | GSM8K | `gsm8k` | 1,319 | Multi-step grade-school math |

{benchkit_cli-0.1.0 → benchkit_cli-0.1.2}/README.md

@@ -51,8 +51,10 @@ cp .env.example .env
 uv run benchkit
 ```
 
-Set your server in a `.env` in the directory you run `benchkit` from (parent
-directories are not searched), or enter it on the Connect screen:
+Set your server in `~/.config/benchkit/.env` to use it from any folder, in a
+`.env` in the directory you run `benchkit` from (parent directories are not
+searched), or on the Connect screen, where "Save as default" writes the user
+config for you:
 
 ```env
 BENCHKIT_PROVIDER=openai
@@ -67,6 +69,8 @@ BENCHKIT_PROVIDER=ollama
 OLLAMA_HOST=http://localhost:11434
 ```
 
+`benchkit config` shows every setting and where it came from.
+
 No inference server available? Explore the complete UI offline:
 
 ```bash
@@ -114,6 +118,7 @@ uv run benchkit --help
 | Aider Polyglot | `aider-polyglot` | 225 | Repository editing across six languages with Pi |
 | HumanEval | `humaneval` | 164 | Python function completion |
 | HumanEval+ | `humaneval-plus` | 164 | HumanEval with expanded EvalPlus tests |
+| HumanEval+ TR | `humaneval-plus-tr` | 164 | HumanEval+ fully translated to Turkish, function and variable names included |
 | MBPP | `mbpp` | 500 | Short Python programming tasks |
 | MBPP+ | `mbpp-plus` | 378 | MBPP with expanded EvalPlus tests |
 | GSM8K | `gsm8k` | 1,319 | Multi-step grade-school math |

{benchkit_cli-0.1.0 → benchkit_cli-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchkit-cli"
-version = "0.1.0"
+version = "0.1.2"
 description = "Benchmark your local LLMs with real eval suites"
 readme = "README.md"
 requires-python = ">=3.11"

{benchkit_cli-0.1.0 → benchkit_cli-0.1.2}/src/benchkit/benchmarks/__init__.py

@@ -6,7 +6,7 @@ from benchkit.benchmarks.banking77 import Banking77
 from benchkit.benchmarks.boolq import BoolQ
 from benchkit.benchmarks.chartqa import ChartQA
 from benchkit.benchmarks.code_verifier import CodeVerifier
-from benchkit.benchmarks.evalplus import HumanEvalPlus, MBPPPlus
+from benchkit.benchmarks.evalplus import HumanEvalPlus, HumanEvalPlusTR, MBPPPlus
 from benchkit.benchmarks.git_surgery import GitSurgery
 from benchkit.benchmarks.gpqa import GPQA
 from benchkit.benchmarks.gsm8k import GSM8K
@@ -41,6 +41,7 @@ REGISTRY: dict[str, type] = {
     "treejs-arena": TreeJSArena,
     "humaneval": HumanEval,
     "humaneval-plus": HumanEvalPlus,
+    "humaneval-plus-tr": HumanEvalPlusTR,
     "mbpp": MBPP,
     "mbpp-plus": MBPPPlus,
     "gsm8k": GSM8K,
@@ -78,6 +79,7 @@ DESCRIPTIONS: dict[str, str] = {
     "sanity": "25 curated checks across code, math, instructions, science, and commonsense",
     "humaneval": "Python function completions with the original unit tests",
     "humaneval-plus": "HumanEval with 122k+ tougher EvalPlus test inputs",
+    "humaneval-plus-tr": "HumanEval+ fully translated to Turkish, names included",
     "mbpp": "short Python functions from natural-language specifications",
     "mbpp-plus": "sanitized MBPP tasks with 39k+ EvalPlus test inputs",
     "gsm8k": "multi-step grade-school math problems with exact numeric answers",

{benchkit_cli-0.1.0 → benchkit_cli-0.1.2}/src/benchkit/benchmarks/downloads.py

@@ -164,3 +164,14 @@ MMLU_PRO = RemoteFile(
     sha256="92ec532d9e2bae342ec7256005c20b6e5defbed76504280df775fb77068f077e",
     size=10_260_787,
 )
+
+# Pinned to a dataset commit on the Hub; the sha256 covers the exact bytes.
+HUMANEVAL_PLUS_TR = RemoteFile(
+    name="humanevalplus_tr.jsonl",
+    url=(
+        "https://huggingface.co/datasets/DogukanUrker/HumanEvalPlus-TR/resolve/"
+        "e424171b8934581819fcdeddace2af28a664966f/humanevalplus_tr.jsonl"
+    ),
+    sha256="072172a4d7934b8ce27d8ad0c83b139b80296876bd2f8a8afad1476bdaa97955",
+    size=7_635_249,
+)

{benchkit_cli-0.1.0 → benchkit_cli-0.1.2}/src/benchkit/benchmarks/evalplus.py

@@ -3,23 +3,33 @@
 from __future__ import annotations
 
 import contextlib
+import json
 import multiprocessing
 import os
 import platform
+import shutil
+import tempfile
 import textwrap
 from functools import cache, lru_cache
 from pathlib import Path
 from typing import Any
 
 from benchkit.benchmarks.base import Task
-from benchkit.benchmarks.downloads import Dataset
+from benchkit.benchmarks.downloads import HUMANEVAL_PLUS_TR, Dataset
 from benchkit.benchmarks.utils import strip_think_tags
+from benchkit.executor import sandbox_env
 
 HUMANEVAL_SYSTEM = (
     "Complete the following Python function. "
     "Output ONLY the function body. "
     "No markdown fences, no explanation, no examples."
 )
+# Full Turkish translation, ASCII only like the dataset itself.
+HUMANEVAL_TR_SYSTEM = (
+    "Asagidaki Python fonksiyonunu tamamla. "
+    "YALNIZCA fonksiyon govdesini yaz. "
+    "Markdown blogu, aciklama veya ornek ekleme."
+)
 MBPP_SYSTEM = (
     "Write a Python function to solve the following task. "
     "Output ONLY the function. "
@@ -43,10 +53,29 @@ def _fork_multiprocessing():
 
     EvalPlus's ``evalplus.eval`` does ``from multiprocessing import Array, Value``
     so those names are bound on that module and must be patched there as well.
+
+    The forked child runs model-written code, so it swaps its inherited
+    environment for a minimal one before running: API keys stay with the
+    parent. ``EVALPLUS_*`` is kept because the child reads its memory limit.
     """
     import evalplus.eval as _ev
 
     fork_ctx = multiprocessing.get_context("fork")
+
+    class _ScrubbedProcess(fork_ctx.Process):
+        def run(self) -> None:
+            # EvalPlus's reliability guard disables shutil.rmtree in the child.
+            rmtree = shutil.rmtree
+            home = tempfile.mkdtemp(prefix="benchkit-evalplus-")
+            env = sandbox_env(home, keep_prefixes=("EVALPLUS_",))
+            os.environ.clear()
+            os.environ.update(env)
+            tempfile.tempdir = None
+            try:
+                super().run()
+            finally:
+                rmtree(home, ignore_errors=True)
+
     mp_originals = (
         multiprocessing.Process,
         multiprocessing.Value,
@@ -54,7 +83,7 @@ def _fork_multiprocessing():
     )
     ev_originals = (_ev.Value, _ev.Array)
     try:
-        multiprocessing.Process = fork_ctx.Process
+        multiprocessing.Process = _ScrubbedProcess
         multiprocessing.Value = fork_ctx.Value
         multiprocessing.Array = fork_ctx.Array
         _ev.Value = fork_ctx.Value
@@ -96,9 +125,14 @@ def _evalplus_api():
     )
 
 
-@lru_cache(maxsize=2)
+@lru_cache(maxsize=3)
 def _problems(dataset: str) -> dict[str, dict[str, Any]]:
     get_human_eval_plus, get_mbpp_plus, *_ = _evalplus_api()
+    if dataset == "humaneval-tr":
+        # Same schema as HumanEvalPlus.jsonl: translated prompt, entry point,
+        # canonical solution and contract, original inputs and tolerances.
+        rows = HUMANEVAL_PLUS_TR.ensure().read_text(encoding="utf-8").splitlines()
+        return {row["task_id"]: row for row in map(json.loads, filter(None, rows))}
     if dataset == "humaneval":
         return get_human_eval_plus()
     if dataset == "mbpp":
@@ -253,14 +287,14 @@ class _EvalPlusBenchmark:
     def evaluate(self, task: Task, response: str) -> bool:
         *_, pass_status, _, check_correctness, _ = _evalplus_api()
         problem = _problems(self.dataset)[task.id]
-        if self.dataset == "humaneval":
+        if self.dataset.startswith("humaneval"):
             solution = _human_solution(problem, response)
         else:
             solution = _extract_code(response)
 
         with _fork_multiprocessing():
             result = check_correctness(
-                self.dataset,
+                "mbpp" if self.dataset == "mbpp" else "humaneval",
                 completion_id=0,
                 problem=problem,
                 solution=solution,
@@ -280,6 +314,16 @@ class HumanEvalPlus(_EvalPlusBenchmark):
     )
 
 
+class HumanEvalPlusTR(_EvalPlusBenchmark):
+    name = "humaneval-plus-tr"
+    dataset = "humaneval-tr"
+    task_count = 164
+    system = HUMANEVAL_TR_SYSTEM
+    data_source = HUMANEVAL_PLUS_TR.source(
+        "HumanEval+ fully translated to Turkish (docstrings and identifiers)"
+    )
+
+
 class MBPPPlus(_EvalPlusBenchmark):
     name = "mbpp-plus"
     dataset = "mbpp"

{benchkit_cli-0.1.0 → benchkit_cli-0.1.2}/src/benchkit/cli.py

@@ -9,13 +9,13 @@ from __future__ import annotations
 import argparse
 import sys
 
-from dotenv import load_dotenv
 from rich import box
 from rich.console import Console
+from rich.markup import escape
 from rich.table import Table
 from rich.text import Text
 
-from benchkit import catalog
+from benchkit import catalog, config
 from benchkit.benchmarks import DESCRIPTIONS, REGISTRY
 from benchkit.client import InferenceClient
 from benchkit.engine import (
@@ -60,11 +60,11 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
     parser.add_argument(
         "command",
         nargs="?",
-        choices=("perf", "history", "render-check", "datasets"),
+        choices=("perf", "history", "render-check", "datasets", "config"),
         help=(
             "Run a performance profile, browse historical results, check "
-            "that headless rendering works on this machine, or list and "
-            "install benchmark datasets"
+            "that headless rendering works on this machine, list and "
+            "install benchmark datasets, or show the effective configuration"
         ),
     )
     parser.add_argument(
@@ -73,7 +73,8 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
         metavar="MODEL",
         help=(
             "Model name for `benchkit perf MODEL`; `install` for "
-            "`benchkit datasets install [BENCHMARK ...]`"
+            "`benchkit datasets install [BENCHMARK ...]`; `path` for "
+            "`benchkit config path`"
         ),
     )
     parser.add_argument(
@@ -83,6 +84,15 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
         help="Datasets: benchmarks to install (default: every missing one)",
     )
     parser.add_argument("--host", help="Inference host, overrides BENCHKIT_HOST")
+    parser.add_argument(
+        "--env-file",
+        metavar="PATH",
+        help=(
+            "Read settings from PATH instead of ./.env (also BENCHKIT_ENV_FILE). "
+            "Precedence: flags > environment > --env-file or ./.env > "
+            "~/.config/benchkit/.env ($XDG_CONFIG_HOME) > defaults"
+        ),
+    )
     parser.add_argument(
         "--demo",
         action="store_true",
@@ -956,11 +966,40 @@ def _perf(args: argparse.Namespace) -> None:
     )
 
 
+def _config(args: argparse.Namespace) -> None:
+    if args.perf_model == "path":
+        for entry in config.loaded().files:
+            if entry.skipped:
+                state = f"[dim]{entry.skipped}[/dim]"
+            elif entry.exists:
+                state = "[green]found[/green]"
+            else:
+                state = "[dim]missing[/dim]"
+            console.print(f"{entry.role:<18} {escape(str(entry.path))}  {state}")
+        return
+    if args.perf_model is not None:
+        console.print(
+            f"[red]Unknown config action {escape(repr(args.perf_model))}.[/red]"
+        )
+        sys.exit(1)
+
+    table = Table(box=box.SIMPLE_HEAD, header_style="bold")
+    table.add_column("Setting")
+    table.add_column("Value")
+    table.add_column("Origin", style="dim")
+    for key, value, origin in config.resolve({"BENCHKIT_HOST": args.host}):
+        shown = Text(value) if value else Text("-", style="dim")
+        table.add_row(key, shown, Text(origin))
+    console.print(table)
+
+
 def main(argv: list[str] | None = None) -> None:
-    # Only the working directory's .env: walking up to parent directories would
-    # let a file in a shared parent such as /tmp redirect the host and API key.
-    load_dotenv(".env")
     args = _parse_args(sys.argv[1:] if argv is None else argv)
+    try:
+        config.load(args.env_file)
+    except config.ConfigError as exc:
+        console.print(f"[red]{exc}[/red]")
+        sys.exit(2)
 
     if args.list:
         _list_benchmarks()
@@ -974,6 +1013,10 @@ def main(argv: list[str] | None = None) -> None:
         _datasets(args)
         return
 
+    if args.command == "config":
+        _config(args)
+        return
+
     if args.command == "perf":
         _perf(args)
         return

benchkit_cli-0.1.2/src/benchkit/config.py

@@ -0,0 +1,251 @@
+"""Layered configuration: which settings apply, and where each one came from.
+
+Precedence, highest first: CLI flags, real environment variables,
+``--env-file PATH`` (or ``BENCHKIT_ENV_FILE``), ``./.env`` in the working
+directory, the user config ``$XDG_CONFIG_HOME/benchkit/.env`` (default
+``~/.config/benchkit/.env``), then built-in defaults. An explicit env file
+replaces ``./.env`` rather than stacking on it.
+
+Only BenchKit's own keys are applied from any file, so running inside another
+project never imports that project's secrets. Parent directories are never
+searched: a ``.env`` in a shared parent such as ``/tmp`` could otherwise
+redirect the host and API key.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import MutableMapping
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from dotenv import dotenv_values, set_key, unset_key
+
+from benchkit.client import DEFAULT_HOST
+
+ENV_FILE_VAR = "BENCHKIT_ENV_FILE"
+_PREFIXES = ("BENCHKIT_", "EVALPLUS_")
+_NAMES = frozenset({"OLLAMA_HOST", "OPENAI_API_KEY", "OPENAI_BASE_URL"})
+_SECRET_SUFFIXES = ("_API_KEY", "_TOKEN", "_SECRET", "_PASSWORD")
+
+# Settings `benchkit config` always lists, with the default each falls back to
+# ("" when unset simply means "not used"). Internal variables the harness sets
+# for its own child processes are left out.
+# fmt: off
+SETTINGS: tuple[tuple[str, str], ...] = (
+    ("BENCHKIT_PROVIDER",             "auto"),
+    ("BENCHKIT_HOST",                 DEFAULT_HOST),
+    ("OPENAI_BASE_URL",               ""),
+    ("OLLAMA_HOST",                   ""),
+    ("BENCHKIT_API_KEY",              ""),
+    ("OPENAI_API_KEY",                ""),
+    ("BENCHKIT_TIMEOUT",              "300"),
+    ("BENCHKIT_CONCURRENCY",          ""),
+    ("BENCHKIT_HARDWARE",             ""),
+    ("BENCHKIT_PERF_CONFIG",          ""),
+    ("BENCHKIT_RETRIES",              "3"),
+    ("BENCHKIT_CONNECT_RETRIES",      "6"),
+    ("BENCHKIT_RETRY_BASE_DELAY",     "0.5"),
+    ("BENCHKIT_RETRY_MAX_DELAY",      "8"),
+    ("BENCHKIT_RETRY_MAX_WAIT",       "60"),
+    ("BENCHKIT_LOOP_KILL",            "true"),
+    ("BENCHKIT_LOOP_KILL_PERCENT",    "80"),
+    ("BENCHKIT_LOOP_KILL_SECONDS",    "10"),
+    ("BENCHKIT_PI_TIMEOUT",           "900"),
+    ("BENCHKIT_DOCKER",               "docker"),
+    ("BENCHKIT_SANDBOX_MEMORY",       "2g"),
+    ("BENCHKIT_SANDBOX_CPUS",         "2"),
+    ("BENCHKIT_SANDBOX_PIDS",         ""),
+    ("BENCHKIT_MC_ARENA_MEMORY",      "512m"),
+    ("BENCHKIT_MC_ARENA_CPUS",        "1"),
+    ("BENCHKIT_MC_ARENA_PIDS",        "64"),
+    ("BENCHKIT_MC_ARENA_BASE_IMAGE",  ""),
+    ("BENCHKIT_MC_ARENA_IMAGE",       ""),
+    ("BENCHKIT_RENDER_SETTLE",        "4"),
+    ("BENCHKIT_RENDER_TIMEOUT",       "30"),
+    ("BENCHKIT_RENDER_ALLOWED_HOSTS", ""),
+    ("BENCHKIT_RENDER_OFFLINE",       "false"),
+    ("BENCHKIT_RENDER_CHROMIUM",      ""),
+    ("BENCHKIT_RENDER_NO_SANDBOX",    "false"),
+    ("BENCHKIT_DATASET_CACHE",        ""),
+    ("BENCHKIT_VISION_CACHE",         ""),
+    ("BENCHKIT_PATCHEVAL_DATASET",    ""),
+    ("BENCHKIT_AIDER_DATASET",        ""),
+)
+# fmt: on
+
+
+class ConfigError(Exception):
+    """A configuration source was named but cannot be used."""
+
+
+@dataclass(frozen=True)
+class EnvFile:
+    """One file BenchKit considered, in precedence order."""
+
+    path: Path
+    role: str
+    exists: bool
+    skipped: str = ""
+
+
+@dataclass
+class Loaded:
+    """The files considered and the origin of every BenchKit key applied."""
+
+    files: list[EnvFile] = field(default_factory=list)
+    origins: dict[str, str] = field(default_factory=dict)
+
+
+_loaded = Loaded()
+
+
+def is_benchkit_key(key: str) -> bool:
+    return key.startswith(_PREFIXES) or key in _NAMES
+
+
+def is_secret(key: str) -> bool:
+    return key.endswith(_SECRET_SUFFIXES)
+
+
+def mask(value: str) -> str:
+    """Hide a secret, keeping the last four characters of a long one."""
+    if not value:
+        return ""
+    if len(value) <= 8:
+        return "****"
+    return "****" + value[-4:]
+
+
+def user_config_dir(environ: MutableMapping[str, str] | None = None) -> Path:
+    environ = os.environ if environ is None else environ
+    base = environ.get("XDG_CONFIG_HOME") or ""
+    root = Path(base).expanduser() if base else Path("~/.config").expanduser()
+    return root / "benchkit"
+
+
+def user_env_path(environ: MutableMapping[str, str] | None = None) -> Path:
+    return user_config_dir(environ) / ".env"
+
+
+def load(
+    env_file: str | None = None,
+    *,
+    environ: MutableMapping[str, str] | None = None,
+    cwd: Path | None = None,
+) -> Loaded:
+    """Apply BenchKit keys from the config files into ``environ``.
+
+    A key already in the environment is never overwritten, and a file only
+    fills keys that no higher file set. Raises ``ConfigError`` when an
+    explicit env file does not exist.
+    """
+    global _loaded
+    environ = os.environ if environ is None else environ
+    cwd = Path.cwd() if cwd is None else cwd
+
+    files: list[EnvFile] = []
+    project = cwd / ".env"
+    explicit, role = env_file, "--env-file"
+    if not explicit and environ.get(ENV_FILE_VAR):
+        explicit, role = environ[ENV_FILE_VAR], ENV_FILE_VAR
+    if explicit:
+        path = Path(explicit).expanduser()
+        if not path.is_absolute():
+            path = cwd / path
+        if not path.is_file():
+            raise ConfigError(f"{role}: no such file: {path}")
+        files.append(EnvFile(path, role, True))
+        files.append(
+            EnvFile(
+                project, "project", project.is_file(), skipped=f"replaced by {role}"
+            )
+        )
+    else:
+        files.append(EnvFile(project, "project", project.is_file()))
+    user = user_env_path(environ)
+    files.append(EnvFile(user, "user", user.is_file()))
+
+    origins = {key: "env" for key in environ if is_benchkit_key(key)}
+    for entry in files:
+        if not entry.exists or entry.skipped:
+            continue
+        for key, value in dotenv_values(entry.path).items():
+            if value is None or key == ENV_FILE_VAR or not is_benchkit_key(key):
+                continue
+            if key in origins:
+                continue
+            environ[key] = value
+            origins[key] = str(entry.path)
+
+    _loaded = Loaded(files, origins)
+    return _loaded
+
+
+def loaded() -> Loaded:
+    """What the last ``load`` applied (empty before the first call)."""
+    return _loaded
+
+
+def resolve(
+    flags: dict[str, str | None] | None = None,
+    *,
+    environ: MutableMapping[str, str] | None = None,
+) -> list[tuple[str, str, str]]:
+    """Every known or set BenchKit key as ``(key, value, origin)``.
+
+    ``value`` is masked for secrets. ``origin`` is ``flag``, ``env``, the file
+    path the value came from, or ``default``.
+    """
+    environ = os.environ if environ is None else environ
+    flags = {key: value for key, value in (flags or {}).items() if value}
+    defaults = dict(SETTINGS)
+    extra = sorted(
+        key
+        for key in environ
+        if is_benchkit_key(key) and key not in defaults and key != ENV_FILE_VAR
+    )
+    rows = []
+    for key in [*defaults, *extra]:
+        if key in flags:
+            value, origin = flags[key], "flag"
+        elif key in environ:
+            value, origin = environ[key], _loaded.origins.get(key, "env")
+        else:
+            value, origin = defaults.get(key, ""), "default"
+        rows.append((key, mask(value) if is_secret(key) else value, origin))
+    return rows
+
+
+def save_user(
+    values: dict[str, str | None],
+    *,
+    environ: MutableMapping[str, str] | None = None,
+) -> Path:
+    """Write ``values`` to the user config; ``None`` or ``""`` removes a key.
+
+    The file can hold an API key, so it is created readable by its owner only.
+    """
+    path = user_env_path(environ)
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    if not path.exists():
+        path.touch(mode=0o600)
+    present = dotenv_values(path)
+    for key, value in values.items():
+        if not is_benchkit_key(key):
+            raise ValueError(f"not a BenchKit setting: {key}")
+        if value:
+            set_key(path, key, value, quote_mode="auto")
+        elif key in present:
+            unset_key(path, key, quote_mode="auto")
+    return path
+
+
+def overridden(keys: list[str]) -> dict[str, str]:
+    """Keys whose current value comes from somewhere above the user config."""
+    user = str(user_env_path())
+    return {
+        key: origin
+        for key in keys
+        if (origin := _loaded.origins.get(key)) is not None and origin != user
+    }