local-coding-agent 1.1.0__py3-none-any.whl

local_coding_agent/benchmark/_cases.py

@@ -0,0 +1,307 @@
+"""Benchmark case model and the fixed default case set."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable, Mapping
+
+from ..task import TaskEnvelope
+
+from ._oracles import (
+    _unique_oracle,
+    _limit_oracle,
+    _utf8_oracle,
+    _no_mutation_oracle,
+    _count_positives_oracle,
+    _max_value_oracle,
+    _abs_sum_oracle,
+    _reverse_str_oracle,
+    _filter_evens_oracle,
+    _count_words_oracle,
+    _dict_default_oracle,
+    _strip_text_oracle,
+    _join_words_oracle,
+    _last_element_oracle,
+    _sorted_copy_oracle,
+    _replace_dash_oracle,
+    _starts_with_oracle,
+    _dot_product_oracle,
+    _min_value_oracle,
+    _title_case_oracle,
+)
+
+
+@dataclass(frozen=True)
+class BenchmarkCase:
+    id: str
+    task: TaskEnvelope
+    fixture: Mapping[str, str]
+    expected_files: Mapping[str, str]
+    oracle: Callable[[Path], tuple[bool, str]] | None = None
+
+    def __post_init__(self) -> None:
+        if not self.id.strip():
+            raise ValueError("benchmark case id must be non-empty")
+        fixture_paths = set(self.fixture)
+        if fixture_paths != set(self.task.files):
+            raise ValueError("benchmark fixture paths must match task allowlist")
+        if not self.expected_files:
+            raise ValueError("benchmark case must have expected files")
+        if not set(self.expected_files).issubset(fixture_paths):
+            raise ValueError("expected files must be part of the fixture")
+
+
+def _edit_case(
+    case_id: str,
+    goal: str,
+    file_name: str,
+    buggy: str,
+    expected: str,
+    context: str,
+    acceptance: str,
+    oracle: Callable[[Path], tuple[bool, str]],
+) -> BenchmarkCase:
+    return BenchmarkCase(
+        id=case_id,
+        task=TaskEnvelope(
+            id=case_id,
+            goal=goal,
+            files=(f"src/{file_name}.py",),
+            context=context,
+            constraints=("сохранить имя функции", "не добавлять зависимости"),
+            acceptance=(acceptance,),
+        ),
+        fixture={f"src/{file_name}.py": buggy},
+        expected_files={f"src/{file_name}.py": expected},
+        oracle=oracle,
+    )
+
+
+def default_cases() -> tuple[BenchmarkCase, ...]:
+    """Return a small fixed task set with deterministic external oracles."""
+
+    return (
+        BenchmarkCase(
+            id="unique-preserve-order",
+            task=TaskEnvelope(
+                id="unique-preserve-order",
+                goal="убрать сортировку из unique и сохранить порядок первого появления",
+                files=("src/unique.py",),
+                context="Функция должна удалить повторы, но не менять порядок входных значений.",
+                constraints=("не менять публичную сигнатуру", "не добавлять зависимости"),
+                acceptance=("повторы удаляются", "порядок первого появления сохраняется"),
+            ),
+            fixture={"src/unique.py": "def unique(values):\n    return sorted(set(values))\n"},
+            expected_files={"src/unique.py": "def unique(values):\n    return list(dict.fromkeys(values))\n"},
+            oracle=_unique_oracle,
+        ),
+        BenchmarkCase(
+            id="limit-inclusive",
+            task=TaskEnvelope(
+                id="limit-inclusive",
+                goal="исправить off-by-one и вернуть ровно limit элементов",
+                files=("src/window.py",),
+                context="При положительном limit срез должен включать элемент с индексом limit-1.",
+                constraints=("изменить только выражение среза",),
+                acceptance=("limit=3 возвращает первые три элемента",),
+            ),
+            fixture={"src/window.py": "def take(values, limit):\n    return values[: limit - 1]\n"},
+            expected_files={"src/window.py": "def take(values, limit):\n    return values[:limit]\n"},
+            oracle=_limit_oracle,
+        ),
+        BenchmarkCase(
+            id="utf8-json",
+            task=TaskEnvelope(
+                id="utf8-json",
+                goal="сохранить русские символы при сериализации JSON",
+                files=("src/encoding.py",),
+                context="JSON должен оставаться валидным и не превращать Unicode-символы в escape-последовательности.",
+                constraints=("не менять имя функции", "использовать только стандартную библиотеку"),
+                acceptance=("ensure_ascii отключён",),
+            ),
+            fixture={
+                "src/encoding.py": "import json\n\ndef encode(value):\n    return json.dumps(value)\n"
+            },
+            expected_files={
+                "src/encoding.py": "import json\n\ndef encode(value):\n    return json.dumps(value, ensure_ascii=False)\n"
+            },
+            oracle=_utf8_oracle,
+        ),
+        BenchmarkCase(
+            id="avoid-input-mutation",
+            task=TaskEnvelope(
+                id="avoid-input-mutation",
+                goal="добавить flag без изменения входного списка",
+                files=("src/flags.py",),
+                context="Вызов не должен мутировать values: исходный список должен остаться прежним.",
+                constraints=("сохранить имя функции", "не добавлять зависимости"),
+                acceptance=("возвращается новый список с flag в конце",),
+            ),
+            fixture={"src/flags.py": "def append_flag(values, flag):\n    values.append(flag)\n    return values\n"},
+            expected_files={
+                "src/flags.py": "def append_flag(values, flag):\n    return [*values, flag]\n"
+            },
+            oracle=_no_mutation_oracle,
+        ),
+        _edit_case(
+            "count-positives",
+            "посчитать количество положительных элементов",
+            "count",
+            "def count_positives(values):\n    return len(values)\n",
+            "def count_positives(values):\n    return sum(1 for value in values if value > 0)\n",
+            "Считать нужно только элементы строго больше нуля.",
+            "счётчик учитывает только положительные элементы",
+            _count_positives_oracle,
+        ),
+        _edit_case(
+            "max-value",
+            "вернуть максимальное значение из списка",
+            "maxval",
+            "def max_value(values):\n    return values[0]\n",
+            "def max_value(values):\n    return max(values)\n",
+            "Функция должна возвращать наибольший элемент входного списка.",
+            "для [3,7,2,5] возвращается 7",
+            _max_value_oracle,
+        ),
+        _edit_case(
+            "abs-sum",
+            "вернуть сумму модулей элементов",
+            "abssum",
+            "def abs_sum(values):\n    return sum(values)\n",
+            "def abs_sum(values):\n    return sum(abs(value) for value in values)\n",
+            "Отрицательные числа должны входить в сумму как положительные.",
+            "для [-1,2,-3] возвращается 6",
+            _abs_sum_oracle,
+        ),
+        _edit_case(
+            "reverse-str",
+            "развернуть строку в обратном порядке",
+            "reverse",
+            "def reverse_str(value):\n    return value\n",
+            "def reverse_str(value):\n    return value[::-1]\n",
+            "Результат — та же строка в обратном порядке символов.",
+            "для 'мир' возвращается 'рим'",
+            _reverse_str_oracle,
+        ),
+        _edit_case(
+            "filter-evens",
+            "оставить только чётные элементы",
+            "evens",
+            "def evens(values):\n    return values\n",
+            "def evens(values):\n    return [value for value in values if value % 2 == 0]\n",
+            "Нужно вернуть новый список только с чётными числами.",
+            "для [1,2,3,4,6] возвращается [2,4,6]",
+            _filter_evens_oracle,
+        ),
+        _edit_case(
+            "count-words",
+            "посчитать слова в строке",
+            "words",
+            "def count_words(text):\n    return len(text)\n",
+            "def count_words(text):\n    return len(text.split())\n",
+            "Слова разделены пробелами; считать именно слова, а не символы.",
+            "для 'один два три' возвращается 3",
+            _count_words_oracle,
+        ),
+        _edit_case(
+            "dict-default",
+            "вернуть значение ключа или ноль",
+            "defval",
+            "def get_or_zero(mapping, key):\n    return mapping[key]\n",
+            "def get_or_zero(mapping, key):\n    return mapping.get(key, 0)\n",
+            "Отсутствующий ключ не должен выбрасывать ошибку.",
+            "для отсутствующего ключа возвращается 0",
+            _dict_default_oracle,
+        ),
+        _edit_case(
+            "strip-text",
+            "убрать пробелы по краям строки",
+            "strip",
+            "def normalize(text):\n    return text.replace(' ', '')\n",
+            "def normalize(text):\n    return text.strip()\n",
+            "Убирать пробелы нужно только по краям, а не внутри.",
+            "для '  hi  ' возвращается 'hi'",
+            _strip_text_oracle,
+        ),
+        _edit_case(
+            "join-words",
+            "склеить слова через пробел",
+            "join",
+            "def join_words(words):\n    return words\n",
+            "def join_words(words):\n    return ' '.join(words)\n",
+            "Результат — одна строка из слов, разделённых одиночным пробелом.",
+            "для ['a','b','c'] возвращается 'a b c'",
+            _join_words_oracle,
+        ),
+        _edit_case(
+            "last-element",
+            "вернуть последний элемент списка",
+            "last",
+            "def last(values):\n    return values[0]\n",
+            "def last(values):\n    return values[-1]\n",
+            "Нужен именно последний, а не первый элемент.",
+            "для [1,2,3] возвращается 3",
+            _last_element_oracle,
+        ),
+        _edit_case(
+            "sorted-copy",
+            "вернуть отсортированную копию без изменения входа",
+            "sortcopy",
+            "def sorted_copy(values):\n    values.sort()\n    return values\n",
+            "def sorted_copy(values):\n    return sorted(values)\n",
+            "Исходный список не должен изменяться.",
+            "вход [3,1,2] остаётся неизменным, результат [1,2,3]",
+            _sorted_copy_oracle,
+        ),
+        _edit_case(
+            "replace-dash",
+            "заменить дефисы на подчёркивания",
+            "replace",
+            "def replace_dash(text):\n    return text\n",
+            "def replace_dash(text):\n    return text.replace('-', '_')\n",
+            "Все символы '-' должны стать '_'.",
+            "для 'a-b-c' возвращается 'a_b_c'",
+            _replace_dash_oracle,
+        ),
+        _edit_case(
+            "starts-with",
+            "проверить начало строки",
+            "starts",
+            "def starts_with(text, prefix):\n    return prefix in text\n",
+            "def starts_with(text, prefix):\n    return text.startswith(prefix)\n",
+            "Совпадение должно быть именно в начале строки.",
+            "для 'hello','he' — True; для 'hello','xy' — False",
+            _starts_with_oracle,
+        ),
+        _edit_case(
+            "dot-product",
+            "посчитать скалярное произведение",
+            "dot",
+            "def dot(left, right):\n    return sum(left) * sum(right)\n",
+            "def dot(left, right):\n    return sum(a * b for a, b in zip(left, right))\n",
+            "Суммировать нужно попарные произведения элементов.",
+            "для [1,2,3] и [4,5,6] возвращается 32",
+            _dot_product_oracle,
+        ),
+        _edit_case(
+            "min-value",
+            "вернуть минимальное значение из списка",
+            "minval",
+            "def min_value(values):\n    return values[0]\n",
+            "def min_value(values):\n    return min(values)\n",
+            "Функция должна возвращать наименьший элемент входного списка.",
+            "для [3,7,2,5] возвращается 2",
+            _min_value_oracle,
+        ),
+        _edit_case(
+            "title-case",
+            "перевести каждое слово в регистр заголовка",
+            "title",
+            "def title_case(text):\n    return text.upper()\n",
+            "def title_case(text):\n    return text.title()\n",
+            "Первая буква каждого слова — заглавная, остальные — строчные.",
+            "для 'hello world' возвращается 'Hello World'",
+            _title_case_oracle,
+        ),
+    )

local_coding_agent/benchmark/_judge.py

@@ -0,0 +1,175 @@
+"""Patch judgement: validate, git-apply, and run external oracles on a proposal."""
+
+from __future__ import annotations
+
+import json
+import inspect
+import os
+import shutil
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+from typing import Any, Callable, Mapping
+
+from ..validators import validate_candidate
+
+from ._cases import BenchmarkCase
+
+
+def _judge_patch(
+    result: Mapping[str, Any],
+    case: BenchmarkCase,
+    workspace: Path,
+    *,
+    fallback_patch: str,
+    fallback_edits: list[dict[str, Any]] | None = None,
+) -> tuple[bool, bool, str, str]:
+    patch_source = "accepted_result" if result.get("status") == "accepted" else "tool_proposal"
+    patch = result.get("patch") if patch_source == "accepted_result" else fallback_patch
+    if (not isinstance(patch, str) or not patch.strip()) and fallback_patch:
+        patch_source = "tool_proposal"
+        patch = fallback_patch
+    edits = None
+    if not isinstance(patch, str) or not patch.strip():
+        if fallback_edits:
+            patch_source = "tool_proposal"
+            edits = fallback_edits
+    has_patch = isinstance(patch, str) and bool(patch.strip())
+    if not has_patch and not edits:
+        return False, False, "candidate did not contain a patch", "none"
+    validation = _validate_patch_for_case(patch, case, edits=edits, workspace=workspace)
+    if not validation.valid:
+        return False, False, "; ".join(validation.issues), patch_source
+    resolved = validation.resolved_patch or patch
+    if shutil.which("git") is None:
+        return False, False, "git executable is unavailable for isolated patch application", patch_source
+    apply_check = _git_apply(workspace, resolved, check=True)
+    if apply_check.returncode != 0:
+        return False, False, _process_error(apply_check), patch_source
+    applied = _git_apply(workspace, resolved, check=False)
+    if applied.returncode != 0:
+        return False, False, _process_error(applied), patch_source
+    try:
+        if case.oracle is not None:
+            correct, oracle_error = _run_oracle_in_restricted_process(case.oracle, workspace)
+        else:
+            correct, oracle_error = _exact_file_oracle(case, workspace)
+    except Exception as error:  # external oracle must turn malformed proposals into a score
+        correct, oracle_error = False, f"external oracle error: {error}"
+    if not correct:
+        return True, False, oracle_error, patch_source
+    if set(validation.changed_files) != set(case.expected_files):
+        return True, False, "external oracle changed-file set mismatch", patch_source
+    return True, True, "", patch_source
+
+
+def _validate_patch_for_case(patch: str, case: BenchmarkCase, *, edits=None, workspace=None):
+    candidate = {
+        "status": "candidate",
+        "summary": "benchmark proposal",
+        "patch": patch,
+        "checks": [],
+        "risks": [],
+    }
+    if edits is not None:
+        candidate["edits"] = edits
+    return validate_candidate(candidate, case.task, workspace_root=workspace)
+
+
+def _exact_file_oracle(case: BenchmarkCase, workspace: Path) -> tuple[bool, str]:
+    for raw_path, expected in case.expected_files.items():
+        actual = (workspace / raw_path).read_text(encoding="utf-8")
+        if actual != expected:
+            return False, f"external oracle mismatch: {raw_path}"
+    return True, ""
+
+
+def _git_apply(workspace: Path, patch: str, *, check: bool) -> subprocess.CompletedProcess[bytes]:
+    command = ["git", "apply", "--whitespace=nowarn"]
+    if check:
+        command.append("--check")
+    command.append("-")
+    return subprocess.run(
+        command,
+        cwd=workspace,
+        input=patch.encode("utf-8"),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=False,
+        timeout=10,
+    )
+
+
+def _process_error(process: subprocess.CompletedProcess[bytes]) -> str:
+    detail = process.stderr.decode("utf-8", errors="replace").strip()
+    return detail or f"git apply exited with code {process.returncode}"
+
+
+def _run_oracle_in_restricted_process(
+    oracle: Callable[[Path], tuple[bool, str]], workspace: Path
+) -> tuple[bool, str]:
+    """Run model-controlled fixture code outside the controller process."""
+
+    try:
+        oracle_source = textwrap.dedent(inspect.getsource(oracle))
+    except (OSError, TypeError) as error:
+        return False, f"external oracle source is unavailable: {error}"
+    payload = {
+        "workspace": str(workspace),
+        "oracle_name": getattr(oracle, "__name__", ""),
+        "oracle_source": oracle_source,
+    }
+    # the worker module lives one directory up from this package
+    worker = Path(__file__).resolve().parents[1] / "benchmark_oracle_worker.py"
+    try:
+        completed = subprocess.run(
+            [sys.executable, "-I", "-u", str(worker)],
+            cwd=workspace,
+            env=_benchmark_worker_environment(workspace),
+            input=(json.dumps(payload, ensure_ascii=False) + "\n").encode("utf-8"),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            timeout=10,
+            check=False,
+            **_benchmark_process_options(),
+        )
+    except (OSError, subprocess.TimeoutExpired) as error:
+        return False, f"external oracle process failed: {error}"
+    if completed.returncode != 0:
+        detail = completed.stderr.decode("utf-8", errors="replace").strip()
+        return False, detail[:2000] or f"external oracle exited with code {completed.returncode}"
+    try:
+        result = json.loads(completed.stdout.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as error:
+        return False, f"external oracle returned invalid JSON: {error}"
+    if not isinstance(result, Mapping) or result.get("ok") is not True:
+        detail = result.get("error") if isinstance(result, Mapping) else None
+        return False, f"external oracle error: {detail or 'unknown worker error'}"
+    correct = result.get("correct")
+    detail = result.get("detail", "")
+    if not isinstance(correct, bool) or not isinstance(detail, str):
+        return False, "external oracle returned an invalid result shape"
+    return correct, detail
+
+
+def _benchmark_worker_environment(workspace: Path) -> dict[str, str]:
+    python_dir = str(Path(sys.executable).resolve().parent)
+    environment = {
+        "PATH": python_dir,
+        "PYTHONIOENCODING": "utf-8",
+        "PYTHONUTF8": "1",
+        "TEMP": str(workspace),
+        "TMP": str(workspace),
+    }
+    for key in ("SystemRoot", "WINDIR"):
+        value = os.environ.get(key)
+        if value:
+            environment[key] = value
+    return environment
+
+
+def _benchmark_process_options() -> dict[str, Any]:
+    if os.name == "nt":
+        return {"creationflags": getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)}
+    return {"start_new_session": True}

local_coding_agent/benchmark/_oracles.py

@@ -0,0 +1,157 @@
+"""External correctness oracles for benchmark cases.
+
+Note: oracle bodies reference ``_load_function`` as a module-level global.
+That symbol is injected at call time by ``benchmark_oracle_worker.py``, which
+executes the oracle source in a restricted child process; it is never resolved
+in this package.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Callable
+
+
+def _unique_oracle(workspace: Path) -> tuple[bool, str]:
+    unique = _load_function(workspace / "src/unique.py", "unique")
+    if unique([3, 1, 2, 1, 3]) != [3, 1, 2]:
+        return False, "external oracle mismatch: unique does not preserve order"
+    return True, ""
+
+
+def _limit_oracle(workspace: Path) -> tuple[bool, str]:
+    take = _load_function(workspace / "src/window.py", "take")
+    if take([0, 1, 2, 3], 3) != [0, 1, 2]:
+        return False, "external oracle mismatch: limit is not inclusive"
+    return True, ""
+
+
+def _utf8_oracle(workspace: Path) -> tuple[bool, str]:
+    encode = _load_function(workspace / "src/encoding.py", "encode")
+    encoded = encode({"message": "мир"})
+    if not isinstance(encoded, str) or "\\u" in encoded or '"мир"' not in encoded:
+        return False, "external oracle mismatch: Unicode was escaped"
+    return True, ""
+
+
+def _no_mutation_oracle(workspace: Path) -> tuple[bool, str]:
+    append_flag = _load_function(workspace / "src/flags.py", "append_flag")
+    values = ["a"]
+    returned = append_flag(values, "b")
+    if values != ["a"] or returned != ["a", "b"] or returned is values:
+        return False, "external oracle mismatch: input list was mutated"
+    return True, ""
+
+
+def _count_positives_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/count.py", "count_positives")
+    if fn([-1, 2, -3, 4, 0]) != 2:
+        return False, "external oracle mismatch: count_positives"
+    return True, ""
+
+
+def _max_value_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/maxval.py", "max_value")
+    if fn([3, 7, 2, 5]) != 7:
+        return False, "external oracle mismatch: max_value"
+    return True, ""
+
+
+def _abs_sum_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/abssum.py", "abs_sum")
+    if fn([-1, 2, -3]) != 6:
+        return False, "external oracle mismatch: abs_sum"
+    return True, ""
+
+
+def _reverse_str_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/reverse.py", "reverse_str")
+    if fn("мир") != "рим":
+        return False, "external oracle mismatch: reverse_str"
+    return True, ""
+
+
+def _filter_evens_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/evens.py", "evens")
+    if fn([1, 2, 3, 4, 6]) != [2, 4, 6]:
+        return False, "external oracle mismatch: evens"
+    return True, ""
+
+
+def _count_words_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/words.py", "count_words")
+    if fn("один два три") != 3:
+        return False, "external oracle mismatch: count_words"
+    return True, ""
+
+
+def _dict_default_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/defval.py", "get_or_zero")
+    if fn({"a": 1}, "b") != 0 or fn({"a": 5}, "a") != 5:
+        return False, "external oracle mismatch: get_or_zero"
+    return True, ""
+
+
+def _strip_text_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/strip.py", "normalize")
+    if fn("  hi  ") != "hi":
+        return False, "external oracle mismatch: normalize"
+    return True, ""
+
+
+def _join_words_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/join.py", "join_words")
+    if fn(["a", "b", "c"]) != "a b c":
+        return False, "external oracle mismatch: join_words"
+    return True, ""
+
+
+def _last_element_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/last.py", "last")
+    if fn([1, 2, 3]) != 3:
+        return False, "external oracle mismatch: last"
+    return True, ""
+
+
+def _sorted_copy_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/sortcopy.py", "sorted_copy")
+    values = [3, 1, 2]
+    returned = fn(values)
+    if returned != [1, 2, 3] or values != [3, 1, 2]:
+        return False, "external oracle mismatch: sorted_copy"
+    return True, ""
+
+
+def _replace_dash_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/replace.py", "replace_dash")
+    if fn("a-b-c") != "a_b_c":
+        return False, "external oracle mismatch: replace_dash"
+    return True, ""
+
+
+def _starts_with_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/starts.py", "starts_with")
+    if not fn("hello", "he") or fn("hello", "xy"):
+        return False, "external oracle mismatch: starts_with"
+    return True, ""
+
+
+def _dot_product_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/dot.py", "dot")
+    if fn([1, 2, 3], [4, 5, 6]) != 32:
+        return False, "external oracle mismatch: dot"
+    return True, ""
+
+
+def _min_value_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/minval.py", "min_value")
+    if fn([3, 7, 2, 5]) != 2:
+        return False, "external oracle mismatch: min_value"
+    return True, ""
+
+
+def _title_case_oracle(workspace: Path) -> tuple[bool, str]:
+    fn = _load_function(workspace / "src/title.py", "title_case")
+    if fn("hello world") != "Hello World":
+        return False, "external oracle mismatch: title_case"
+    return True, ""