wisent 0.1.1__tar.gz → 0.5.2__tar.gz

{wisent-0.1.1 → wisent-0.5.2}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Wisent Team
+Copyright (c) 2023-2024 Wisent Team
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE. 

wisent-0.5.2/PKG-INFO

@@ -0,0 +1,67 @@
+Metadata-Version: 2.4
+Name: wisent
+Version: 0.5.2
+Summary: Monitor and guard against harmful content in language models
+Home-page: https://github.com/yourusername/wisent-activation-guardrails
+Author: Wisent Team
+Author-email: your.email@example.com
+Keywords: nlp,machine learning,language models,safety,guardrails,lm-evaluation-harness
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=1.9.0
+Requires-Dist: transformers>=4.20.0
+Requires-Dist: tqdm>=4.50.0
+Requires-Dist: scikit-learn>=0.24.0
+Requires-Dist: pandas>=1.2.0
+Requires-Dist: numpy>=1.21.0
+Requires-Dist: datasets>=2.0.0
+Requires-Dist: sentence-transformers>=2.0.0
+Requires-Dist: faiss-cpu>=1.7.0
+Provides-Extra: harness
+Requires-Dist: lm-eval==0.4.8; extra == "harness"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Wisent-Guard
+
+<p align="center">
+  <a href="https://github.com/wisent-ai/wisent-guard/stargazers">
+    <img src="https://img.shields.io/github/stars/wisent-ai/wisent-guard" alt="stars" />
+  </a>
+  <a href="https://pypi.org/project/wisent-guard">
+    <img src="https://static.pepy.tech/badge/wisent-guard" alt="PyPI - Downloads" />
+  </a>
+  <br />
+</p>
+
+<p align="center">
+  <img src="wisent-guard-logo.png" alt="Wisent Guard" width="200">
+</p>
+
+A Python package for latent space monitoring and guardrails. Delivered to you by the [Wisent](https://wisent.ai) team led by [Lukasz Bartoszcze](https://lukaszbartoszcze.com).
+
+## Overview
+
+Wisent-Guard allows you to control your AI by identifying brain patterns corresponding to responses you don't like, like hallucinations or harmful outputs. We use contrastive pairs of representations to detect when a model might be generating harmful content or hallucinating. Learn more at https://www.wisent.ai/wisent-guard.  
+
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details. 

wisent-0.5.2/README.md

@@ -0,0 +1,26 @@
+# Wisent-Guard
+
+<p align="center">
+  <a href="https://github.com/wisent-ai/wisent-guard/stargazers">
+    <img src="https://img.shields.io/github/stars/wisent-ai/wisent-guard" alt="stars" />
+  </a>
+  <a href="https://pypi.org/project/wisent-guard">
+    <img src="https://static.pepy.tech/badge/wisent-guard" alt="PyPI - Downloads" />
+  </a>
+  <br />
+</p>
+
+<p align="center">
+  <img src="wisent-guard-logo.png" alt="Wisent Guard" width="200">
+</p>
+
+A Python package for latent space monitoring and guardrails. Delivered to you by the [Wisent](https://wisent.ai) team led by [Lukasz Bartoszcze](https://lukaszbartoszcze.com).
+
+## Overview
+
+Wisent-Guard allows you to control your AI by identifying brain patterns corresponding to responses you don't like, like hallucinations or harmful outputs. We use contrastive pairs of representations to detect when a model might be generating harmful content or hallucinating. Learn more at https://www.wisent.ai/wisent-guard.  
+
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details. 

wisent-0.5.2/pyproject.toml

@@ -0,0 +1,85 @@
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = ["-v", "--tb=short", "--ignore=evaluation", "-m", "not slow and not heavy and not docker and not bigcode_required"]
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "heavy: marks tests as very resource-intensive with large model downloads (deselect with '-m \"not heavy\"')",
+    "model_download: marks tests that download models from HuggingFace (deselect with '-m \"not model_download\"')",
+    "integration: marks tests as integration tests",
+    "unit: marks tests as unit tests",
+    "docker: marks tests for Docker-based execution",
+    "performance: marks tests for performance evaluation",
+    "cli: marks tests for CLI integration",
+    "sandbox_required: marks tests that require sandbox environment (includes coding tasks and trust_remote_code)",
+    "bigcode_required: marks tests that require bigcode-evaluation-harness installation",
+]
+filterwarnings = [
+    "ignore::DeprecationWarning",
+    "ignore::PendingDeprecationWarning",
+]
+
+[tool.ruff]
+line-length = 120
+target-version = "py39"
+exclude = [
+    ".bzr",
+    ".direnv", 
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+    "evaluation",
+    "bigcode-evaluation-harness",
+]
+
+[tool.ruff.lint]
+# Enable common rule sets
+select = ["E4", "E7", "E9", "F", "I", "N", "UP", "B", "A", "C4", "ISC", "ICN", "PIE", "T20", "PT", "Q", "RSE", "RET", "SLF", "SIM", "TCH", "ARG", "ERA", "PGH", "RUF"]
+ignore = [
+    "PLR0913", # Too many arguments in function definition
+    "PLR0912", # Too many branches
+    "PLR0915", # Too many statements
+    "C901",    # Function is too complex
+    "PLR2004", # Magic value used in comparison
+    "TRY003",  # Avoid specifying long messages outside exception class
+    "PLW2901", # Redefined loop variable
+    "B008",    # Do not perform function calls in argument defaults
+    "ARG002",  # Unused method argument
+    "PIE800",  # Unnecessary spread operator,
+    "T201" # print
+]
+fixable = ["ALL"]
+unfixable = []
+
+[tool.ruff.format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"
+
+[tool.ruff.lint.isort]
+known-first-party = ["wisent"]
+force-single-line = false
+combine-as-imports = true

wisent-0.5.2/setup.py

@@ -0,0 +1,51 @@
+import os
+from setuptools import setup, find_packages
+
+# Read the contents of the README file
+with open("README.md", encoding="utf-8") as f:
+    long_description = f.read()
+
+# Read version from __init__.py
+with open(os.path.join("wisent", "__init__.py"), encoding="utf-8") as f:
+    for line in f:
+        if line.startswith("__version__"):
+            version = line.split("=")[1].strip().strip('"').strip("'")
+            break
+
+setup(
+    name="wisent",
+    version=version,
+    author="Wisent Team",
+    author_email="your.email@example.com",  # Replace with your email
+    description="Monitor and guard against harmful content in language models",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/yourusername/wisent-activation-guardrails",  # Replace with your GitHub repo
+    packages=find_packages(exclude=["patches", "patches.*"]),  # Exclude patches directory
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+        "Development Status :: 4 - Beta",
+        "Intended Audience :: Science/Research",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    ],
+    python_requires=">=3.8",
+    install_requires=[
+        "torch>=1.9.0",
+        "transformers>=4.20.0",
+        "tqdm>=4.50.0",
+        "scikit-learn>=0.24.0",
+        "pandas>=1.2.0",
+        "numpy>=1.21.0",
+        "datasets>=2.0.0",
+        "sentence-transformers>=2.0.0",
+        "faiss-cpu>=1.7.0",
+    ],
+    extras_require={
+        "harness": [
+            "lm-eval==0.4.8",
+        ],
+    },
+    keywords="nlp, machine learning, language models, safety, guardrails, lm-evaluation-harness",
+) 

wisent-0.5.2/wisent/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.2"

wisent-0.5.2/wisent/benchmarks/coding/metrics/core/atoms.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import  Iterable, Protocol
+
+@dataclass(frozen=True)
+class SampleOutcome:
+    """
+    Result of executing a single sample (possibly after self-repair).
+
+    attributes:
+        task_id: 
+            The unique identifier for the task.
+        status:
+            One of "ok", "compile_error", "runtime_error", or "timeout".
+        passed:
+            True if the code passed all tests, False otherwise.
+        elapsed:
+            Time taken to execute the code in seconds.
+    """
+    task_id: str
+    status: str         
+    passed: bool
+    elapsed: float
+
+class Metric(Protocol):
+    """
+    Metric computes a score from an iterable of SampleOutcome.
+    """
+    def compute(self, outcomes: Iterable[SampleOutcome]) -> float: ...
+
+
+class Evaluator(Protocol):
+    """
+    Runs tasks end-to-end (codegen + optional self-repair) and yields SampleOutcome.
+    """
+    def evaluate(self) -> Iterable[SampleOutcome]: ...

wisent-0.5.2/wisent/benchmarks/coding/metrics/evaluator.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Callable, Iterable, Optional, TYPE_CHECKING
+
+from wisent.benchmarks.coding.safe_docker.core.runtime import DockerSandboxExecutor
+from wisent.benchmarks.coding.safe_docker.recipes import RECIPE_REGISTRY
+from wisent.benchmarks.coding.metrics.core.atoms import SampleOutcome, Evaluator
+
+from wisent.benchmarks.coding.output_sanitizer.core.atoms import TaskSchema
+from wisent.benchmarks.coding.output_sanitizer.python_sanitizer import PythonStandardizer
+from wisent.benchmarks.coding.output_sanitizer.cpp_sanitizer import CppStandardizer
+from wisent.benchmarks.coding.output_sanitizer.java_sanitizer import JavaStandardizer
+
+if TYPE_CHECKING:
+    from wisent.benchmarks.coding.safe_docker.core.atoms import Result
+    from wisent.benchmarks.coding.providers.core.atoms import Provider, CodingTask
+    from wisent.benchmarks.coding.output_sanitizer.core.atoms import CodeStandardizer
+
+RepairFn = Callable[[str, dict[str,str], str], dict[str,str]]  
+
+@dataclass
+class EvaluatorConfig:
+    """
+    Configuration for CodingEvaluator.
+    
+    attributes:
+        image:
+            Docker image to use for code execution (default: "coding/sandbox:polyglot-1.0").
+        runtime:
+            Optional Docker runtime (e.g., "runsc" for gVisor).
+        feedback_max_chars:
+            Maximum characters of feedback to pass to the repair function (default: 2000).
+        self_repair:
+            Whether to perform a single self-repair turn (default: True). It means the we provide feedback to the model for one iteration.
+        time_limit_s:
+            Time limit in seconds for each code execution (default: 8s).
+        cpu_limit_s:
+            CPU time limit in seconds for each code execution (default: 3s).
+        mem_limit_mb:
+            Memory limit in megabytes for each code execution (default: 768MB).
+        pre_sanitize:
+            Whether to run LLM output through a sanitizer before execution (default: True).
+    """
+    image: str = "coding/sandbox:polyglot-1.0"
+    runtime: Optional[str] = None        
+    feedback_max_chars: int = 2000
+    self_repair: bool = True             
+    time_limit_s: int = 8
+    cpu_limit_s: int = 3
+    mem_limit_mb: int = 768
+    pre_sanitize: bool = True            
+
+_SANITIZERS = {
+    "python": PythonStandardizer(),
+    "cpp":    CppStandardizer(),
+    "java":   JavaStandardizer(),
+}
+
+def _default_filename(lang: str) -> str:
+    """
+    Returns the default source file name for a given programming language.
+    
+    arguments:
+        lang:
+            Programming language ("python", "cpp", or "java").
+    
+    returns:
+        Default filename as a string.
+    """
+    return {"python":"solution.py","cpp":"solution.cpp","java":"Solution.java"}[lang]
+
+def _make_schema(task: CodingTask) -> TaskSchema:
+    """
+    Constructs a TaskSchema from a CodingTask, using task options or defaults.
+    
+    arguments:
+        task:
+            CodingTask containing language and options.
+    
+    returns:
+        TaskSchema with language, file_name, entry_point, java_class, prefer_rename,
+        and allow_wrapper set appropriately.
+        
+    example:
+        >>> from wisent.benchmarks.coding.providers.core.atoms import CodingTask
+        >>> task = CodingTask(language="python", files={}, options={"entry_point":"add","file_name":"my_solution.py"})
+        >>> schema = _make_schema(task)
+        >>> schema.language
+        'python'
+        >>> schema.file_name
+        'my_solution.py'
+        >>> schema.entry_point
+        'add'
+        >>> schema.java_class
+        'Solution'
+        >>> schema.prefer_rename
+        True
+        >>> schema.allow_wrapper
+        True
+    """
+    entry = str(task.options.get("entry_point", "solve"))
+    file_name = str(task.options.get("file_name", _default_filename(task.language)))
+    java_class = str(task.options.get("java_class", "Solution"))
+    return TaskSchema(language=task.language, file_name=file_name, entry_point=entry,
+                      java_class=java_class, prefer_rename=True, allow_wrapper=True)
+
+class CodingEvaluator(Evaluator):
+    """
+    Evaluator for coding tasks with optional self-repair.
+    """
+    def __init__(self, provider: Provider, model_fn: Callable[[CodingTask], dict[str,str]],
+                 repair_fn: Optional[RepairFn] = None, cfg: EvaluatorConfig = EvaluatorConfig()):
+        self.provider = provider
+        self.model_fn = model_fn
+        self.repair_fn = repair_fn
+        self.cfg = cfg
+        self.exec = DockerSandboxExecutor(image=cfg.image, runtime=cfg.runtime)
+
+    def _feedback(self, res: "Result") -> str:
+        """
+        Generates feedback text from a Result object for use in self-repair.
+        
+        arguments:
+            res:
+                Result object containing status, stdout, stderr, and elapsed time.
+                
+        returns:
+            Feedback string summarizing the result, truncated to cfg.feedback_max_chars.
+            
+        examples:
+            >>> from wisent.benchmarks.coding.safe_docker.core.atoms import Result
+            >>> res = Result(status="timeout", stdout="", stderr="", elapsed=10.0)
+            >>> evaluator = CodingEvaluator(provider=None, model_fn=lambda x: {}, cfg=EvaluatorConfig())
+            >>> evaluator._feedback(res)
+            'Timeout after 10.00s.'
+            >>> res = Result(status="compile_error", stdout="", stderr="error: something went wrong", elapsed=1.5)
+            >>> evaluator._feedback(res)
+            'Compilation failed:\nerror: something went wrong'
+            >>> res = Result(status="runtime_error", stdout="test failed", stderr="", elapsed=0.5)
+            >>> evaluator._feedback(res)
+            'Runtime error:\ntest failed'
+        """
+        if res.status == "timeout":
+            return f"Timeout after {res.elapsed:.2f}s."
+        body = (res.stdout or "") + ("\n" + res.stderr if res.stderr else "")
+        if res.status == "compile_error":
+            prefix = "Compilation failed:\n"
+        else:
+            prefix = "Tests failed:\n"
+        return (prefix + body)[: self.cfg.feedback_max_chars]
+
+    def _run_once(self, task: CodingTask, files: dict[str,str]) -> Result:
+        """
+        Runs a single evaluation job for the given task and files.
+
+        arguments:
+            task:
+                The coding task to evaluate.
+            files:
+                The files to include in the evaluation.
+
+        returns:
+            Result object containing the status, stdout, stderr, and elapsed time.
+
+        examples:
+            >>> from wisent.benchmarks.coding.providers.core.atoms import CodingTask
+            >>> from wisent.benchmarks.coding.safe_docker.core.atoms import Result
+            >>> task = CodingTask(language="python", files={}, options={})
+            >>> files = {"solution.py": "def add(a,b): return a + b", "tests.py": "from solution import add\ndef test_ok(): assert add(1,2)==3"}
+            >>> evaluator = CodingEvaluator(provider=None, model_fn=lambda x: {})
+            >>> res: Result = evaluator._run_once(task, files)
+            >>> res.status
+            'ok'
+            >>> res.exit_code
+            0
+            >>> res.stdout
+            'test_ok passed'
+            >>> res.stderr
+            ''
+            >>> round(res.elapsed, 2)
+            0.23
+        """
+        recipe = RECIPE_REGISTRY[task.language]
+        job = recipe.make_job(files, **task.options,
+                              time_limit_s=self.cfg.time_limit_s,
+                              cpu_limit_s=self.cfg.cpu_limit_s,
+                              mem_limit_mb=self.cfg.mem_limit_mb)
+        return self.exec.run(files, job)
+
+    def _maybe_sanitize(self, task: CodingTask, files: dict[str,str]) -> dict[str,str]:
+        """
+        Optionally sanitizes the generated files based on the task schema.
+
+        arguments:
+            task:
+                The coding task containing language and options.
+            files:
+                The generated files to potentially sanitize.
+
+        returns:
+            The sanitized files if pre_sanitize is True and a sanitizer exists for the language; otherwise, the original files.
+        
+        examples:
+            >>> from wisent.benchmarks.coding.providers.core.atoms import CodingTask
+            >>> task = CodingTask(language="python", files={}, options={"entry_point":"add","file_name":"my_solution.py"})
+            >>> files = {"my_solution.py": "def add(a,b): return a - b  # BUG"}
+            >>> evaluator = CodingEvaluator(provider=None, model_fn=lambda x: {}, cfg=EvaluatorConfig(pre_sanitize=True))
+            >>> sanitized_files = evaluator._maybe_sanitize(task, files)
+            >>> "my_solution.py" in sanitized_files
+            True
+            >>> sanitized_files["my_solution.py"]
+            'def add(a, b):\n    return a + b\n'
+        """
+        if not self.cfg.pre_sanitize:
+            return files
+        schema = _make_schema(task)
+        sanitizer: "CodeStandardizer" = _SANITIZERS.get(task.language)
+        if sanitizer is None:
+            return files
+
+        raw = files.get(schema.file_name) or files.get("__raw__")
+        if not raw:
+            return files
+
+        out = sanitizer.normalize(raw, schema)
+        files = {**files, schema.file_name: out.files.get(schema.file_name, raw)}
+        return files
+
+    def evaluate(self) -> Iterable[SampleOutcome]:
+        """
+        Evaluates all tasks from the provider, performing optional self-repair.
+
+        yields:
+            SampleOutcome for each task, indicating pass/fail status and elapsed time.
+
+        examples:
+            >>> from wisent.benchmarks.coding.providers.core.atoms import CodingTask, Provider
+            >>> class DummyProvider:
+            ...     name = "dummy"
+            ...     def iter_tasks(self):
+            ...         yield CodingTask(language="python", files={"tests.py":"from solution import add\ndef test_ok(): assert add(1,2)==3"},
+            ...              options={"entry_point":"add","file_name":"solution.py"})
+            >>> def model_fn(task: CodingTask) -> Dict[str,str]:
+            ...     return {"solution.py": "def add(a,b): return a - b  # BUG"}
+            >>> def repair_fn(lang: str, prev_files: Dict[str,str], feedback: str) -> Dict[str,str]:
+            ...     fixed = prev_files["solution.py"].replace("a - b", "a + b")
+            ...     return {"solution.py": fixed}
+            >>> evaluator = CodingEvaluator(provider=DummyProvider(), model_fn=model_fn, repair_fn=repair_fn, cfg=EvaluatorConfig(self_repair=True))
+            >>> outcomes = list(evaluator.evaluate())
+            >>> len(outcomes)
+            1
+            >>> outcomes[0].passed
+            True
+        """
+        for idx, task in enumerate(self.provider.iter_tasks()):
+            files0 = self.model_fn(task)
+            files0 = {**task.files, **files0}
+            files0 = self._maybe_sanitize(task, files0)
+
+            r0 = self._run_once(task, files0)
+            if r0.status == "ok":
+                yield SampleOutcome(task_id=f"{self.provider.name}:{idx}", status=r0.status, passed=True, elapsed=r0.elapsed)
+                continue
+
+            if not self.cfg.self_repair or self.repair_fn is None:
+                yield SampleOutcome(task_id=f"{self.provider.name}:{idx}", status=r0.status, passed=False, elapsed=r0.elapsed); continue
+
+            fb = self._feedback(r0)
+            files1 = self.repair_fn(task.language, files0, fb)
+            files1 = {**task.files, **files1}
+            files1 = self._maybe_sanitize(task, files1)
+
+            r1 = self._run_once(task, files1)
+            passed = (r0.status == "ok") or (r1.status == "ok")
+            yield SampleOutcome(task_id=f"{self.provider.name}:{idx}", status=r1.status, passed=passed, elapsed=(r0.elapsed + r1.elapsed))

wisent-0.5.2/wisent/benchmarks/coding/metrics/passk.py

@@ -0,0 +1,66 @@
+# coding/metrics/passk.py
+from __future__ import annotations
+from typing import Iterable
+import math
+from collections import defaultdict
+from .core.atoms import SampleOutcome, Metric
+
+class PassAtK(Metric):
+    """
+    Exact Pass@k for code generation.
+    """
+
+    def __init__(self, k: int = 1):
+        if k < 1:
+            raise ValueError("k must be >= 1")
+        self.k = k
+
+    def compute(self, outcomes: Iterable[SampleOutcome]) -> float:
+        """
+        Aggregate counts per task_id
+
+        arguments:
+            outcomes: Iterable of SampleOutcome objects
+
+        returns:
+            Average Pass@k score across tasks
+
+        intuition:
+            For each task, we have n samples, c of which pass.
+            We want the probability that at least one of k random picks from these n samples is a passing one.
+            This is 1 - (combinations of picking k from the n-c failing ones) / (combinations of picking k from all n).
+            We then average this score across all tasks. 
+        """
+        per_task_counts = defaultdict(lambda: {"n": 0, "c": 0})
+        for o in outcomes:
+            d = per_task_counts[o.task_id]
+            d["n"] += 1
+            d["c"] += 1 if o.passed else 0
+
+        if not per_task_counts:
+            return 0.0
+
+        scores_sum = 0.0
+        task_cnt = 0
+        for counts in per_task_counts.values():
+            n = counts["n"]
+            c = counts["c"]
+            if n <= 0:
+                continue  
+
+            k = min(self.k, n)
+            if c <= 0:
+                score = 0.0
+            elif k == 0:
+                score = 0.0
+            elif k == 1:
+                score = c / n
+            else:
+                denom = math.comb(n, k)
+                num = math.comb(n - c, k) if k <= (n - c) else 0
+                score = 1.0 - (num / denom if denom > 0 else 0.0)
+
+            scores_sum += score
+            task_cnt += 1
+
+        return 0.0 if task_cnt == 0 else scores_sum / task_cnt

wisent-0.5.2/wisent/benchmarks/coding/output_sanitizer/core/atoms.py

@@ -0,0 +1,27 @@
+# coding/llm_sanitizer/core/atoms.py
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Dict, Protocol, Literal, Optional
+
+Language = Literal["python", "cpp", "java"]
+
+@dataclass(frozen=True)
+class TaskSchema:
+    """What the sandbox expects for this task."""
+    language: Language
+    file_name: str               # e.g., "solution.py" | "solution.cpp" | "Solution.java"
+    entry_point: str             # function/method name tests will call (e.g., "add", "solve")
+    java_class: str = "Solution" # only for Java; expected public class name
+    # Optional hints:
+    allow_wrapper: bool = True   # may synthesize thin wrapper instead of renaming
+    prefer_rename: bool = False  # if True and safe, rename single top-level function to entry_point
+
+@dataclass(frozen=True)
+class NormalizeResult:
+    files: Dict[str, str]        # filename -> normalized source
+    notes: str                   # human-readable log of what was done
+    ok: bool                     # True if we think it’s valid / parseable
+
+class CodeStandardizer(Protocol):
+    language: Language
+    def normalize(self, raw: str, schema: TaskSchema) -> NormalizeResult: ...