benchkit-cli 0.1.1__tar.gz → 0.1.2__tar.gz

{benchkit_cli-0.1.1 → benchkit_cli-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchkit-cli
-Version: 0.1.1
+Version: 0.1.2
 Summary: Benchmark your local LLMs with real eval suites
 License-Expression: Apache-2.0
 Requires-Python: >=3.11
@@ -136,6 +136,7 @@ uv run benchkit --help
 | Aider Polyglot | `aider-polyglot` | 225 | Repository editing across six languages with Pi |
 | HumanEval | `humaneval` | 164 | Python function completion |
 | HumanEval+ | `humaneval-plus` | 164 | HumanEval with expanded EvalPlus tests |
+| HumanEval+ TR | `humaneval-plus-tr` | 164 | HumanEval+ fully translated to Turkish, function and variable names included |
 | MBPP | `mbpp` | 500 | Short Python programming tasks |
 | MBPP+ | `mbpp-plus` | 378 | MBPP with expanded EvalPlus tests |
 | GSM8K | `gsm8k` | 1,319 | Multi-step grade-school math |

{benchkit_cli-0.1.1 → benchkit_cli-0.1.2}/README.md

@@ -118,6 +118,7 @@ uv run benchkit --help
 | Aider Polyglot | `aider-polyglot` | 225 | Repository editing across six languages with Pi |
 | HumanEval | `humaneval` | 164 | Python function completion |
 | HumanEval+ | `humaneval-plus` | 164 | HumanEval with expanded EvalPlus tests |
+| HumanEval+ TR | `humaneval-plus-tr` | 164 | HumanEval+ fully translated to Turkish, function and variable names included |
 | MBPP | `mbpp` | 500 | Short Python programming tasks |
 | MBPP+ | `mbpp-plus` | 378 | MBPP with expanded EvalPlus tests |
 | GSM8K | `gsm8k` | 1,319 | Multi-step grade-school math |

{benchkit_cli-0.1.1 → benchkit_cli-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchkit-cli"
-version = "0.1.1"
+version = "0.1.2"
 description = "Benchmark your local LLMs with real eval suites"
 readme = "README.md"
 requires-python = ">=3.11"

{benchkit_cli-0.1.1 → benchkit_cli-0.1.2}/src/benchkit/benchmarks/__init__.py

@@ -6,7 +6,7 @@ from benchkit.benchmarks.banking77 import Banking77
 from benchkit.benchmarks.boolq import BoolQ
 from benchkit.benchmarks.chartqa import ChartQA
 from benchkit.benchmarks.code_verifier import CodeVerifier
-from benchkit.benchmarks.evalplus import HumanEvalPlus, MBPPPlus
+from benchkit.benchmarks.evalplus import HumanEvalPlus, HumanEvalPlusTR, MBPPPlus
 from benchkit.benchmarks.git_surgery import GitSurgery
 from benchkit.benchmarks.gpqa import GPQA
 from benchkit.benchmarks.gsm8k import GSM8K
@@ -41,6 +41,7 @@ REGISTRY: dict[str, type] = {
     "treejs-arena": TreeJSArena,
     "humaneval": HumanEval,
     "humaneval-plus": HumanEvalPlus,
+    "humaneval-plus-tr": HumanEvalPlusTR,
     "mbpp": MBPP,
     "mbpp-plus": MBPPPlus,
     "gsm8k": GSM8K,
@@ -78,6 +79,7 @@ DESCRIPTIONS: dict[str, str] = {
     "sanity": "25 curated checks across code, math, instructions, science, and commonsense",
     "humaneval": "Python function completions with the original unit tests",
     "humaneval-plus": "HumanEval with 122k+ tougher EvalPlus test inputs",
+    "humaneval-plus-tr": "HumanEval+ fully translated to Turkish, names included",
     "mbpp": "short Python functions from natural-language specifications",
     "mbpp-plus": "sanitized MBPP tasks with 39k+ EvalPlus test inputs",
     "gsm8k": "multi-step grade-school math problems with exact numeric answers",

{benchkit_cli-0.1.1 → benchkit_cli-0.1.2}/src/benchkit/benchmarks/downloads.py

@@ -164,3 +164,14 @@ MMLU_PRO = RemoteFile(
     sha256="92ec532d9e2bae342ec7256005c20b6e5defbed76504280df775fb77068f077e",
     size=10_260_787,
 )
+
+# Pinned to a dataset commit on the Hub; the sha256 covers the exact bytes.
+HUMANEVAL_PLUS_TR = RemoteFile(
+    name="humanevalplus_tr.jsonl",
+    url=(
+        "https://huggingface.co/datasets/DogukanUrker/HumanEvalPlus-TR/resolve/"
+        "e424171b8934581819fcdeddace2af28a664966f/humanevalplus_tr.jsonl"
+    ),
+    sha256="072172a4d7934b8ce27d8ad0c83b139b80296876bd2f8a8afad1476bdaa97955",
+    size=7_635_249,
+)

{benchkit_cli-0.1.1 → benchkit_cli-0.1.2}/src/benchkit/benchmarks/evalplus.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import contextlib
+import json
 import multiprocessing
 import os
 import platform
@@ -14,7 +15,7 @@ from pathlib import Path
 from typing import Any
 
 from benchkit.benchmarks.base import Task
-from benchkit.benchmarks.downloads import Dataset
+from benchkit.benchmarks.downloads import HUMANEVAL_PLUS_TR, Dataset
 from benchkit.benchmarks.utils import strip_think_tags
 from benchkit.executor import sandbox_env
 
@@ -23,6 +24,12 @@ HUMANEVAL_SYSTEM = (
     "Output ONLY the function body. "
     "No markdown fences, no explanation, no examples."
 )
+# Full Turkish translation, ASCII only like the dataset itself.
+HUMANEVAL_TR_SYSTEM = (
+    "Asagidaki Python fonksiyonunu tamamla. "
+    "YALNIZCA fonksiyon govdesini yaz. "
+    "Markdown blogu, aciklama veya ornek ekleme."
+)
 MBPP_SYSTEM = (
     "Write a Python function to solve the following task. "
     "Output ONLY the function. "
@@ -118,9 +125,14 @@ def _evalplus_api():
     )
 
 
-@lru_cache(maxsize=2)
+@lru_cache(maxsize=3)
 def _problems(dataset: str) -> dict[str, dict[str, Any]]:
     get_human_eval_plus, get_mbpp_plus, *_ = _evalplus_api()
+    if dataset == "humaneval-tr":
+        # Same schema as HumanEvalPlus.jsonl: translated prompt, entry point,
+        # canonical solution and contract, original inputs and tolerances.
+        rows = HUMANEVAL_PLUS_TR.ensure().read_text(encoding="utf-8").splitlines()
+        return {row["task_id"]: row for row in map(json.loads, filter(None, rows))}
     if dataset == "humaneval":
         return get_human_eval_plus()
     if dataset == "mbpp":
@@ -275,14 +287,14 @@ class _EvalPlusBenchmark:
     def evaluate(self, task: Task, response: str) -> bool:
         *_, pass_status, _, check_correctness, _ = _evalplus_api()
         problem = _problems(self.dataset)[task.id]
-        if self.dataset == "humaneval":
+        if self.dataset.startswith("humaneval"):
             solution = _human_solution(problem, response)
         else:
             solution = _extract_code(response)
 
         with _fork_multiprocessing():
             result = check_correctness(
-                self.dataset,
+                "mbpp" if self.dataset == "mbpp" else "humaneval",
                 completion_id=0,
                 problem=problem,
                 solution=solution,
@@ -302,6 +314,16 @@ class HumanEvalPlus(_EvalPlusBenchmark):
     )
 
 
+class HumanEvalPlusTR(_EvalPlusBenchmark):
+    name = "humaneval-plus-tr"
+    dataset = "humaneval-tr"
+    task_count = 164
+    system = HUMANEVAL_TR_SYSTEM
+    data_source = HUMANEVAL_PLUS_TR.source(
+        "HumanEval+ fully translated to Turkish (docstrings and identifiers)"
+    )
+
+
 class MBPPPlus(_EvalPlusBenchmark):
     name = "mbpp-plus"
     dataset = "mbpp"

{benchkit_cli-0.1.1 → benchkit_cli-0.1.2}/src/benchkit_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchkit-cli
-Version: 0.1.1
+Version: 0.1.2
 Summary: Benchmark your local LLMs with real eval suites
 License-Expression: Apache-2.0
 Requires-Python: >=3.11
@@ -136,6 +136,7 @@ uv run benchkit --help
 | Aider Polyglot | `aider-polyglot` | 225 | Repository editing across six languages with Pi |
 | HumanEval | `humaneval` | 164 | Python function completion |
 | HumanEval+ | `humaneval-plus` | 164 | HumanEval with expanded EvalPlus tests |
+| HumanEval+ TR | `humaneval-plus-tr` | 164 | HumanEval+ fully translated to Turkish, function and variable names included |
 | MBPP | `mbpp` | 500 | Short Python programming tasks |
 | MBPP+ | `mbpp-plus` | 378 | MBPP with expanded EvalPlus tests |
 | GSM8K | `gsm8k` | 1,319 | Multi-step grade-school math |