wisent 0.1.1__py3-none-any.whl → 0.5.2__py3-none-any.whl

wisent/__init__.py

@@ -1,8 +1 @@
-"""
-Wisent - Client library for interacting with the Wisent backend services.
-"""
-
-from wisent.client import WisentClient
-from wisent.version import __version__
-
-__all__ = ["WisentClient", "__version__"]
+__version__ = "0.5.2"

wisent/benchmarks/coding/metrics/core/atoms.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import  Iterable, Protocol
+
+@dataclass(frozen=True)
+class SampleOutcome:
+    """
+    Result of executing a single sample (possibly after self-repair).
+
+    attributes:
+        task_id: 
+            The unique identifier for the task.
+        status:
+            One of "ok", "compile_error", "runtime_error", or "timeout".
+        passed:
+            True if the code passed all tests, False otherwise.
+        elapsed:
+            Time taken to execute the code in seconds.
+    """
+    task_id: str
+    status: str         
+    passed: bool
+    elapsed: float
+
+class Metric(Protocol):
+    """
+    Metric computes a score from an iterable of SampleOutcome.
+    """
+    def compute(self, outcomes: Iterable[SampleOutcome]) -> float: ...
+
+
+class Evaluator(Protocol):
+    """
+    Runs tasks end-to-end (codegen + optional self-repair) and yields SampleOutcome.
+    """
+    def evaluate(self) -> Iterable[SampleOutcome]: ...

wisent/benchmarks/coding/metrics/evaluator.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Callable, Iterable, Optional, TYPE_CHECKING
+
+from wisent.benchmarks.coding.safe_docker.core.runtime import DockerSandboxExecutor
+from wisent.benchmarks.coding.safe_docker.recipes import RECIPE_REGISTRY
+from wisent.benchmarks.coding.metrics.core.atoms import SampleOutcome, Evaluator
+
+from wisent.benchmarks.coding.output_sanitizer.core.atoms import TaskSchema
+from wisent.benchmarks.coding.output_sanitizer.python_sanitizer import PythonStandardizer
+from wisent.benchmarks.coding.output_sanitizer.cpp_sanitizer import CppStandardizer
+from wisent.benchmarks.coding.output_sanitizer.java_sanitizer import JavaStandardizer
+
+if TYPE_CHECKING:
+    from wisent.benchmarks.coding.safe_docker.core.atoms import Result
+    from wisent.benchmarks.coding.providers.core.atoms import Provider, CodingTask
+    from wisent.benchmarks.coding.output_sanitizer.core.atoms import CodeStandardizer
+
+RepairFn = Callable[[str, dict[str,str], str], dict[str,str]]  
+
+@dataclass
+class EvaluatorConfig:
+    """
+    Configuration for CodingEvaluator.
+    
+    attributes:
+        image:
+            Docker image to use for code execution (default: "coding/sandbox:polyglot-1.0").
+        runtime:
+            Optional Docker runtime (e.g., "runsc" for gVisor).
+        feedback_max_chars:
+            Maximum characters of feedback to pass to the repair function (default: 2000).
+        self_repair:
+            Whether to perform a single self-repair turn (default: True). It means the we provide feedback to the model for one iteration.
+        time_limit_s:
+            Time limit in seconds for each code execution (default: 8s).
+        cpu_limit_s:
+            CPU time limit in seconds for each code execution (default: 3s).
+        mem_limit_mb:
+            Memory limit in megabytes for each code execution (default: 768MB).
+        pre_sanitize:
+            Whether to run LLM output through a sanitizer before execution (default: True).
+    """
+    image: str = "coding/sandbox:polyglot-1.0"
+    runtime: Optional[str] = None        
+    feedback_max_chars: int = 2000
+    self_repair: bool = True             
+    time_limit_s: int = 8
+    cpu_limit_s: int = 3
+    mem_limit_mb: int = 768
+    pre_sanitize: bool = True            
+
+_SANITIZERS = {
+    "python": PythonStandardizer(),
+    "cpp":    CppStandardizer(),
+    "java":   JavaStandardizer(),
+}
+
+def _default_filename(lang: str) -> str:
+    """
+    Returns the default source file name for a given programming language.
+    
+    arguments:
+        lang:
+            Programming language ("python", "cpp", or "java").
+    
+    returns:
+        Default filename as a string.
+    """
+    return {"python":"solution.py","cpp":"solution.cpp","java":"Solution.java"}[lang]
+
+def _make_schema(task: CodingTask) -> TaskSchema:
+    """
+    Constructs a TaskSchema from a CodingTask, using task options or defaults.
+    
+    arguments:
+        task:
+            CodingTask containing language and options.
+    
+    returns:
+        TaskSchema with language, file_name, entry_point, java_class, prefer_rename,
+        and allow_wrapper set appropriately.
+        
+    example:
+        >>> from wisent.benchmarks.coding.providers.core.atoms import CodingTask
+        >>> task = CodingTask(language="python", files={}, options={"entry_point":"add","file_name":"my_solution.py"})
+        >>> schema = _make_schema(task)
+        >>> schema.language
+        'python'
+        >>> schema.file_name
+        'my_solution.py'
+        >>> schema.entry_point
+        'add'
+        >>> schema.java_class
+        'Solution'
+        >>> schema.prefer_rename
+        True
+        >>> schema.allow_wrapper
+        True
+    """
+    entry = str(task.options.get("entry_point", "solve"))
+    file_name = str(task.options.get("file_name", _default_filename(task.language)))
+    java_class = str(task.options.get("java_class", "Solution"))
+    return TaskSchema(language=task.language, file_name=file_name, entry_point=entry,
+                      java_class=java_class, prefer_rename=True, allow_wrapper=True)
+
+class CodingEvaluator(Evaluator):
+    """
+    Evaluator for coding tasks with optional self-repair.
+    """
+    def __init__(self, provider: Provider, model_fn: Callable[[CodingTask], dict[str,str]],
+                 repair_fn: Optional[RepairFn] = None, cfg: EvaluatorConfig = EvaluatorConfig()):
+        self.provider = provider
+        self.model_fn = model_fn
+        self.repair_fn = repair_fn
+        self.cfg = cfg
+        self.exec = DockerSandboxExecutor(image=cfg.image, runtime=cfg.runtime)
+
+    def _feedback(self, res: "Result") -> str:
+        """
+        Generates feedback text from a Result object for use in self-repair.
+        
+        arguments:
+            res:
+                Result object containing status, stdout, stderr, and elapsed time.
+                
+        returns:
+            Feedback string summarizing the result, truncated to cfg.feedback_max_chars.
+            
+        examples:
+            >>> from wisent.benchmarks.coding.safe_docker.core.atoms import Result
+            >>> res = Result(status="timeout", stdout="", stderr="", elapsed=10.0)
+            >>> evaluator = CodingEvaluator(provider=None, model_fn=lambda x: {}, cfg=EvaluatorConfig())
+            >>> evaluator._feedback(res)
+            'Timeout after 10.00s.'
+            >>> res = Result(status="compile_error", stdout="", stderr="error: something went wrong", elapsed=1.5)
+            >>> evaluator._feedback(res)
+            'Compilation failed:\nerror: something went wrong'
+            >>> res = Result(status="runtime_error", stdout="test failed", stderr="", elapsed=0.5)
+            >>> evaluator._feedback(res)
+            'Runtime error:\ntest failed'
+        """
+        if res.status == "timeout":
+            return f"Timeout after {res.elapsed:.2f}s."
+        body = (res.stdout or "") + ("\n" + res.stderr if res.stderr else "")
+        if res.status == "compile_error":
+            prefix = "Compilation failed:\n"
+        else:
+            prefix = "Tests failed:\n"
+        return (prefix + body)[: self.cfg.feedback_max_chars]
+
+    def _run_once(self, task: CodingTask, files: dict[str,str]) -> Result:
+        """
+        Runs a single evaluation job for the given task and files.
+
+        arguments:
+            task:
+                The coding task to evaluate.
+            files:
+                The files to include in the evaluation.
+
+        returns:
+            Result object containing the status, stdout, stderr, and elapsed time.
+
+        examples:
+            >>> from wisent.benchmarks.coding.providers.core.atoms import CodingTask
+            >>> from wisent.benchmarks.coding.safe_docker.core.atoms import Result
+            >>> task = CodingTask(language="python", files={}, options={})
+            >>> files = {"solution.py": "def add(a,b): return a + b", "tests.py": "from solution import add\ndef test_ok(): assert add(1,2)==3"}
+            >>> evaluator = CodingEvaluator(provider=None, model_fn=lambda x: {})
+            >>> res: Result = evaluator._run_once(task, files)
+            >>> res.status
+            'ok'
+            >>> res.exit_code
+            0
+            >>> res.stdout
+            'test_ok passed'
+            >>> res.stderr
+            ''
+            >>> round(res.elapsed, 2)
+            0.23
+        """
+        recipe = RECIPE_REGISTRY[task.language]
+        job = recipe.make_job(files, **task.options,
+                              time_limit_s=self.cfg.time_limit_s,
+                              cpu_limit_s=self.cfg.cpu_limit_s,
+                              mem_limit_mb=self.cfg.mem_limit_mb)
+        return self.exec.run(files, job)
+
+    def _maybe_sanitize(self, task: CodingTask, files: dict[str,str]) -> dict[str,str]:
+        """
+        Optionally sanitizes the generated files based on the task schema.
+
+        arguments:
+            task:
+                The coding task containing language and options.
+            files:
+                The generated files to potentially sanitize.
+
+        returns:
+            The sanitized files if pre_sanitize is True and a sanitizer exists for the language; otherwise, the original files.
+        
+        examples:
+            >>> from wisent.benchmarks.coding.providers.core.atoms import CodingTask
+            >>> task = CodingTask(language="python", files={}, options={"entry_point":"add","file_name":"my_solution.py"})
+            >>> files = {"my_solution.py": "def add(a,b): return a - b  # BUG"}
+            >>> evaluator = CodingEvaluator(provider=None, model_fn=lambda x: {}, cfg=EvaluatorConfig(pre_sanitize=True))
+            >>> sanitized_files = evaluator._maybe_sanitize(task, files)
+            >>> "my_solution.py" in sanitized_files
+            True
+            >>> sanitized_files["my_solution.py"]
+            'def add(a, b):\n    return a + b\n'
+        """
+        if not self.cfg.pre_sanitize:
+            return files
+        schema = _make_schema(task)
+        sanitizer: "CodeStandardizer" = _SANITIZERS.get(task.language)
+        if sanitizer is None:
+            return files
+
+        raw = files.get(schema.file_name) or files.get("__raw__")
+        if not raw:
+            return files
+
+        out = sanitizer.normalize(raw, schema)
+        files = {**files, schema.file_name: out.files.get(schema.file_name, raw)}
+        return files
+
+    def evaluate(self) -> Iterable[SampleOutcome]:
+        """
+        Evaluates all tasks from the provider, performing optional self-repair.
+
+        yields:
+            SampleOutcome for each task, indicating pass/fail status and elapsed time.
+
+        examples:
+            >>> from wisent.benchmarks.coding.providers.core.atoms import CodingTask, Provider
+            >>> class DummyProvider:
+            ...     name = "dummy"
+            ...     def iter_tasks(self):
+            ...         yield CodingTask(language="python", files={"tests.py":"from solution import add\ndef test_ok(): assert add(1,2)==3"},
+            ...              options={"entry_point":"add","file_name":"solution.py"})
+            >>> def model_fn(task: CodingTask) -> Dict[str,str]:
+            ...     return {"solution.py": "def add(a,b): return a - b  # BUG"}
+            >>> def repair_fn(lang: str, prev_files: Dict[str,str], feedback: str) -> Dict[str,str]:
+            ...     fixed = prev_files["solution.py"].replace("a - b", "a + b")
+            ...     return {"solution.py": fixed}
+            >>> evaluator = CodingEvaluator(provider=DummyProvider(), model_fn=model_fn, repair_fn=repair_fn, cfg=EvaluatorConfig(self_repair=True))
+            >>> outcomes = list(evaluator.evaluate())
+            >>> len(outcomes)
+            1
+            >>> outcomes[0].passed
+            True
+        """
+        for idx, task in enumerate(self.provider.iter_tasks()):
+            files0 = self.model_fn(task)
+            files0 = {**task.files, **files0}
+            files0 = self._maybe_sanitize(task, files0)
+
+            r0 = self._run_once(task, files0)
+            if r0.status == "ok":
+                yield SampleOutcome(task_id=f"{self.provider.name}:{idx}", status=r0.status, passed=True, elapsed=r0.elapsed)
+                continue
+
+            if not self.cfg.self_repair or self.repair_fn is None:
+                yield SampleOutcome(task_id=f"{self.provider.name}:{idx}", status=r0.status, passed=False, elapsed=r0.elapsed); continue
+
+            fb = self._feedback(r0)
+            files1 = self.repair_fn(task.language, files0, fb)
+            files1 = {**task.files, **files1}
+            files1 = self._maybe_sanitize(task, files1)
+
+            r1 = self._run_once(task, files1)
+            passed = (r0.status == "ok") or (r1.status == "ok")
+            yield SampleOutcome(task_id=f"{self.provider.name}:{idx}", status=r1.status, passed=passed, elapsed=(r0.elapsed + r1.elapsed))

wisent/benchmarks/coding/metrics/passk.py

@@ -0,0 +1,66 @@
+# coding/metrics/passk.py
+from __future__ import annotations
+from typing import Iterable
+import math
+from collections import defaultdict
+from .core.atoms import SampleOutcome, Metric
+
+class PassAtK(Metric):
+    """
+    Exact Pass@k for code generation.
+    """
+
+    def __init__(self, k: int = 1):
+        if k < 1:
+            raise ValueError("k must be >= 1")
+        self.k = k
+
+    def compute(self, outcomes: Iterable[SampleOutcome]) -> float:
+        """
+        Aggregate counts per task_id
+
+        arguments:
+            outcomes: Iterable of SampleOutcome objects
+
+        returns:
+            Average Pass@k score across tasks
+
+        intuition:
+            For each task, we have n samples, c of which pass.
+            We want the probability that at least one of k random picks from these n samples is a passing one.
+            This is 1 - (combinations of picking k from the n-c failing ones) / (combinations of picking k from all n).
+            We then average this score across all tasks. 
+        """
+        per_task_counts = defaultdict(lambda: {"n": 0, "c": 0})
+        for o in outcomes:
+            d = per_task_counts[o.task_id]
+            d["n"] += 1
+            d["c"] += 1 if o.passed else 0
+
+        if not per_task_counts:
+            return 0.0
+
+        scores_sum = 0.0
+        task_cnt = 0
+        for counts in per_task_counts.values():
+            n = counts["n"]
+            c = counts["c"]
+            if n <= 0:
+                continue  
+
+            k = min(self.k, n)
+            if c <= 0:
+                score = 0.0
+            elif k == 0:
+                score = 0.0
+            elif k == 1:
+                score = c / n
+            else:
+                denom = math.comb(n, k)
+                num = math.comb(n - c, k) if k <= (n - c) else 0
+                score = 1.0 - (num / denom if denom > 0 else 0.0)
+
+            scores_sum += score
+            task_cnt += 1
+
+        return 0.0 if task_cnt == 0 else scores_sum / task_cnt

wisent/benchmarks/coding/output_sanitizer/core/atoms.py

@@ -0,0 +1,27 @@
+# coding/llm_sanitizer/core/atoms.py
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Dict, Protocol, Literal, Optional
+
+Language = Literal["python", "cpp", "java"]
+
+@dataclass(frozen=True)
+class TaskSchema:
+    """What the sandbox expects for this task."""
+    language: Language
+    file_name: str               # e.g., "solution.py" | "solution.cpp" | "Solution.java"
+    entry_point: str             # function/method name tests will call (e.g., "add", "solve")
+    java_class: str = "Solution" # only for Java; expected public class name
+    # Optional hints:
+    allow_wrapper: bool = True   # may synthesize thin wrapper instead of renaming
+    prefer_rename: bool = False  # if True and safe, rename single top-level function to entry_point
+
+@dataclass(frozen=True)
+class NormalizeResult:
+    files: Dict[str, str]        # filename -> normalized source
+    notes: str                   # human-readable log of what was done
+    ok: bool                     # True if we think it’s valid / parseable
+
+class CodeStandardizer(Protocol):
+    language: Language
+    def normalize(self, raw: str, schema: TaskSchema) -> NormalizeResult: ...

wisent/benchmarks/coding/output_sanitizer/cpp_sanitizer.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+import re
+from typing import List
+from wisent.benchmarks.coding.output_sanitizer.core.atoms import TaskSchema, NormalizeResult, CodeStandardizer
+from wisent.benchmarks.coding.output_sanitizer.utils import extract_code_block, normalize_whitespace
+
+FUNC_RE = re.compile(r"^\s*(?:template<[^>]+>\s*)?(?:[\w:\s*&<>,]+)\s+(\w+)\s*\(", re.MULTILINE)
+CLASS_RE = re.compile(r"^\s*class\s+(\w+)\s*[{:]", re.MULTILINE)
+
+class CppStandardizer(CodeStandardizer):
+    language = "cpp"
+
+    def normalize(self, raw: str, schema: TaskSchema) -> NormalizeResult:
+        notes: List[str] = []
+        code = normalize_whitespace(extract_code_block(raw, prefer_langs=("cpp","c++","cc","c")))
+        code = re.sub(r"^```.*?\n|\n```$", "", code, flags=re.DOTALL)
+
+        if re.search(rf"\b{re.escape(schema.entry_point)}\s*\(", code):
+            notes.append(f"found function '{schema.entry_point}'")
+            return NormalizeResult(files={schema.file_name: code}, notes="\n".join(notes), ok=True)
+
+        classes = CLASS_RE.findall(code)
+        for cls in classes:
+            if re.search(rf"\b{re.escape(cls)}\s*::\s*{re.escape(schema.entry_point)}\s*\(", code) or \
+               re.search(rf"class\s+{re.escape(cls)}.*?\b{re.escape(schema.entry_point)}\s*\(", code, flags=re.S):
+                notes.append(f"found {cls}::{schema.entry_point}; adding free-function shim")
+                shim = (
+                    f"\n\ntemplate <typename... Args>\n"
+                    f"auto {schema.entry_point}(Args&&... args)\n"
+                    f"    -> decltype({cls}().{schema.entry_point}(std::forward<Args>(args)...)) {{\n"
+                    f"    return {cls}().{schema.entry_point}(std::forward<Args>(args)...);\n"
+                    f"}}\n"
+                )
+                if "#include <utility>" not in code:
+                    code = "#include <utility>\n" + code
+                return NormalizeResult(files={schema.file_name: code + shim}, notes="\n".join(notes), ok=True)
+
+        candidates = [m.group(1) for m in FUNC_RE.finditer(code)]
+        if schema.prefer_rename and len(candidates) == 1:
+            old = candidates[0]
+            if old != schema.entry_point:
+                notes.append(f"renaming free function '{old}' -> '{schema.entry_point}'")
+                code2 = re.sub(rf"(\b){re.escape(old)}(\s*\()", rf"\1{schema.entry_point}\2", code)
+                return NormalizeResult(files={schema.file_name: code2}, notes="\n".join(notes), ok=True)
+
+        if candidates:
+            target = candidates[0]
+            if target != schema.entry_point:
+                notes.append(f"adding forwarding wrapper {schema.entry_point} -> {target}")
+                shim = (
+                    f"\n\ntemplate <typename... Args>\n"
+                    f"auto {schema.entry_point}(Args&&... args)\n"
+                    f"    -> decltype({target}(std::forward<Args>(args)...)) {{\n"
+                    f"    return {target}(std::forward<Args>(args)...);\n"
+                    f"}}\n"
+                )
+                if "#include <utility>" not in code:
+                    code = "#include <utility>\n" + code
+                return NormalizeResult(files={schema.file_name: code + shim}, notes="\n".join(notes), ok=True)
+
+        notes.append("no obvious function; returned normalized source only")
+        return NormalizeResult(files={schema.file_name: code}, notes="\n".join(notes), ok=False)

wisent/benchmarks/coding/output_sanitizer/java_sanitizer.py

@@ -0,0 +1,78 @@
+# coding/llm_sanitizer/java_sanitizer.py
+from __future__ import annotations
+import re
+from typing import List
+from wisent.benchmarks.coding.output_sanitizer.core.atoms import TaskSchema, NormalizeResult, CodeStandardizer
+from wisent.benchmarks.coding.output_sanitizer.utils import extract_code_block, normalize_whitespace
+
+CLASS_RE = re.compile(r"\bclass\s+([A-Za-z_]\w*)")
+METHOD_RE = re.compile(r"(public\s+static\s+[\w\<\>\[\]]+\s+)(\w+)\s*\(")
+
+class JavaStandardizer(CodeStandardizer):
+    language = "java"
+
+    def normalize(self, raw: str, schema: TaskSchema) -> NormalizeResult:
+        notes: List[str] = []
+        code = normalize_whitespace(extract_code_block(raw, prefer_langs=("java")))
+        code = re.sub(r"^```.*?\n|\n```$", "", code, flags=re.DOTALL)
+
+        m = CLASS_RE.search(code)
+        if m:
+            found = m.group(1)
+            if found != schema.java_class:
+                notes.append(f"renaming class '{found}' -> '{schema.java_class}'")
+                code = re.sub(rf"\bclass\s+{re.escape(found)}\b", f"class {schema.java_class}", code, count=1)
+
+        if not CLASS_RE.search(code):
+            notes.append(f"wrapping code in class {schema.java_class}")
+            code = f"public class {schema.java_class} {{\n{indent(code)}\n}}\n"
+
+        static_methods = list(METHOD_RE.finditer(code))
+        if any(m.group(2) == schema.entry_point for m in static_methods):
+            notes.append(f"found public static '{schema.entry_point}'")
+            return NormalizeResult(files={schema.file_name: code}, notes="\n".join(notes), ok=True)
+
+        if len(static_methods) == 1 and schema.prefer_rename:
+            old = static_methods[0].group(2)
+            if old != schema.entry_point:
+                notes.append(f"renaming static method '{old}' -> '{schema.entry_point}'")
+                code = re.sub(rf"(\bpublic\s+static\s+[\w\<\>\[\]]+\s+){re.escape(old)}(\s*\()",
+                              rf"\1{schema.entry_point}\2", code, count=1)
+                return NormalizeResult(files={schema.file_name: code}, notes="\n".join(notes), ok=True)
+
+        if re.search(rf"\b{schema.entry_point}\s*\(", code):
+            notes.append(f"adding static wrapper for instance method '{schema.entry_point}'")
+            wrapper = (
+                f"\n    public static <T> Object {schema.entry_point}(Object... args) {{\n"
+                f"        {schema.java_class} _x = new {schema.java_class}();\n"
+                f"        try {{\n"
+                f"            // attempt reflective dispatch to instance method\n"
+                f"            Class<?>[] types = new Class<?>[args.length];\n"
+                f"            for (int i=0;i<args.length;i++) types[i] = args[i].getClass();\n"
+                f"            return {schema.java_class}.class.getMethod(\"{schema.entry_point}\", types).invoke(_x, args);\n"
+                f"        }} catch (Exception ex) {{ throw new RuntimeException(ex); }}\n"
+                f"    }}\n"
+            )
+            code = re.sub(rf"(class\s+{schema.java_class}\s*{{)", r"\1" + wrapper, code, count=1)
+            return NormalizeResult(files={schema.file_name: code}, notes="\n".join(notes), ok=True)
+
+        notes.append("no suitable method; adding delegating static method to first public static or instance method via reflection")
+        fallback = (
+            f"\n    public static Object {schema.entry_point}(Object... args) {{\n"
+            f"        try {{\n"
+            f"            // try any public method first\n"
+            f"            for (var m : {schema.java_class}.class.getMethods()) {{\n"
+            f"                if (m.getName().equals(\"{schema.entry_point}\")) continue;\n"
+            f"                try {{ return m.invoke(m.getParameterCount()==0? new {schema.java_class}(): new {schema.java_class}(), args); }}\n"
+            f"                catch (Exception ignored) {{}}\n"
+            f"            }}\n"
+            f"        }} catch (Exception e) {{ throw new RuntimeException(e); }}\n"
+            f"        throw new RuntimeException(\"No suitable method for entry point\");\n"
+            f"    }}\n"
+        )
+        code = re.sub(rf"(class\s+{schema.java_class}\s*{{)", r"\1" + fallback, code, count=1)
+        return NormalizeResult(files={schema.file_name: code}, notes="\n".join(notes), ok=True)
+
+def indent(s: str, n: int = 4) -> str:
+    pad = " " * n
+    return "\n".join(pad + line if line.strip() else line for line in s.splitlines())

wisent/benchmarks/coding/output_sanitizer/python_sanitizer.py

@@ -0,0 +1,94 @@
+# coding/llm_sanitizer/python_sanitizer.py
+from __future__ import annotations
+import ast, re
+from typing import List
+from wisent.benchmarks.coding.output_sanitizer.core.atoms import TaskSchema, NormalizeResult, CodeStandardizer
+from wisent.benchmarks.coding.output_sanitizer.utils import extract_code_block, normalize_whitespace, maybe_black
+
+class PythonStandardizer(CodeStandardizer):
+    language = "python"
+
+    def normalize(self, raw: str, schema: TaskSchema) -> NormalizeResult:
+        notes: List[str] = []
+        code = extract_code_block(raw, prefer_langs=("python","py"))
+        code = normalize_whitespace(code)
+        code = re.sub(r"^```.*?\n|\n```$", "", code, flags=re.DOTALL)
+
+        try:
+            tree = ast.parse(code)
+        except SyntaxError as e:
+            notes.append(f"parse failed: {e}; returning raw after whitespace normalize")
+            return NormalizeResult(files={schema.file_name: code}, notes="\n".join(notes), ok=False)
+
+        fn_names = [n.name for n in tree.body if isinstance(n, ast.FunctionDef)]
+        cls_nodes = [n for n in tree.body if isinstance(n, ast.ClassDef)]
+        has_entry_top = schema.entry_point in fn_names
+
+        if has_entry_top:
+            notes.append(f"top-level function '{schema.entry_point}' found")
+            cleaned = maybe_black(code)
+            return NormalizeResult(files={schema.file_name: cleaned}, notes="\n".join(notes), ok=True)
+
+        if schema.prefer_rename and len(fn_names) == 1:
+            old = fn_names[0]
+            notes.append(f"renaming single function '{old}' -> '{schema.entry_point}'")
+            class Renamer(ast.NodeTransformer):
+                def visit_FunctionDef(self, node: ast.FunctionDef):
+                    if node.name == old:
+                        node.name = schema.entry_point
+                    return self.generic_visit(node)
+            tree2 = Renamer().visit(tree)
+            ast.fix_missing_locations(tree2)
+            try:
+                new_code = ast.unparse(tree2)  
+            except Exception:
+                new_code = code.replace(f"def {old}(", f"def {schema.entry_point}(")
+            new_code = maybe_black(new_code)
+            return NormalizeResult(files={schema.file_name: new_code}, notes="\n".join(notes), ok=True)
+
+        for cls in cls_nodes:
+            method_names = [n.name for n in cls.body if isinstance(n, ast.FunctionDef)]
+            if schema.entry_point in method_names:
+                notes.append(f"found method {cls.name}.{schema.entry_point}; adding thin adapter")
+                adapter = (
+                    f"\n\ndef {schema.entry_point}(*args, **kwargs):\n"
+                    f"    return {cls.name}().{schema.entry_point}(*args, **kwargs)\n"
+                )
+                final = code + adapter
+                final = maybe_black(final)
+                return NormalizeResult(files={schema.file_name: final}, notes="\n".join(notes), ok=True)
+
+        candidates = [n for n in fn_names if n in {"solve","solution","func","function","answer"}]
+        if candidates:
+            old = candidates[0]
+            notes.append(f"renaming fallback '{old}' -> '{schema.entry_point}'")
+            try:
+                class Renamer(ast.NodeTransformer):
+                    def visit_FunctionDef(self, node: ast.FunctionDef):
+                        if node.name == old: node.name = schema.entry_point
+                        return self.generic_visit(node)
+                tree2 = Renamer().visit(tree); ast.fix_missing_locations(tree2)
+                new_code = ast.unparse(tree2)
+            except Exception:
+                new_code = code.replace(f"def {old}(", f"def {schema.entry_point}(")
+            new_code = maybe_black(new_code)
+            return NormalizeResult(files={schema.file_name: new_code}, notes="\n".join(notes), ok=True)
+
+        if schema.allow_wrapper:
+            notes.append("no entry found; appending dynamic-dispatch adapter to call first callable")
+            adapter = (
+                f"\n\ndef {schema.entry_point}(*args, **kwargs):\n"
+                f"    # fallback: try first callable in module\n"
+                f"    import inspect\n"
+                f"    for _name, _obj in globals().items():\n"
+                f"        if callable(_obj) and _name not in ('{schema.entry_point}',):\n"
+                f"            try:\n"
+                f"                return _obj(*args, **kwargs)\n"
+                f"            except TypeError:\n"
+                f"                continue\n"
+                f"    raise NameError('No suitable function for entry point')\n"
+            )
+            final = maybe_black(code + adapter)
+            return NormalizeResult(files={schema.file_name: final}, notes="\n".join(notes), ok=True)
+
+        return NormalizeResult(files={schema.file_name: code}, notes="\n".join(notes), ok=False)