themis-eval 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

themis/evaluation/metrics/code/execution.py

@@ -0,0 +1,280 @@
+"""Safe code execution for testing functional correctness.
+
+This module provides utilities for safely executing generated code against
+test cases in a sandboxed environment.
+"""
+
+from __future__ import annotations
+
+import multiprocessing
+import signal
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Callable, Sequence
+
+from themis.core.entities import MetricScore
+from themis.interfaces import Metric
+
+
+class ExecutionStatus(str, Enum):
+    """Execution result status."""
+    
+    PASSED = "passed"
+    FAILED = "failed"
+    TIMEOUT = "timeout"
+    ERROR = "error"
+
+
+@dataclass
+class ExecutionResult:
+    """Result of code execution.
+    
+    Attributes:
+        status: Execution status
+        passed: Whether all tests passed
+        output: Captured stdout/stderr
+        error: Error message if any
+        duration: Execution time in seconds
+    """
+    
+    status: ExecutionStatus
+    passed: bool
+    output: str = ""
+    error: str | None = None
+    duration: float = 0.0
+
+
+class ExecutionAccuracy(Metric):
+    """Execute code and check against test cases.
+    
+    This metric safely executes generated code in a restricted environment
+    and verifies correctness against provided test cases.
+    
+    Security considerations:
+    - Executes in subprocess with timeout
+    - Restricted globals (no file I/O, network, etc.)
+    - Resource limits (memory, time)
+    
+    Attributes:
+        name: Metric identifier ("execution_accuracy")
+        timeout: Maximum execution time per test (seconds)
+        max_memory_mb: Maximum memory usage (MB)
+    
+    Example:
+        >>> from themis.evaluation.metrics.code import ExecutionAccuracy
+        >>> metric = ExecutionAccuracy(timeout=3.0)
+        >>> 
+        >>> # Reference contains test cases
+        >>> test_cases = {
+        ...     "test_fn": test_function,
+        ...     "inputs": [(1, 2), (3, 4)],
+        ...     "expected": [3, 7]
+        ... }
+        >>> 
+        >>> score = metric.compute(
+        ...     prediction="def add(a, b): return a + b",
+        ...     references=[test_cases]
+        ... )
+    """
+    
+    requires_reference = True
+    
+    def __init__(
+        self,
+        timeout: float = 3.0,
+        max_memory_mb: int = 512,
+    ):
+        """Initialize execution metric.
+        
+        Args:
+            timeout: Maximum execution time per test (seconds)
+            max_memory_mb: Maximum memory usage (MB)
+        """
+        self.name = "execution_accuracy"
+        self.timeout = timeout
+        self.max_memory_mb = max_memory_mb
+    
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> MetricScore:
+        """Execute code and compute accuracy.
+        
+        Args:
+            prediction: Generated code to execute
+            references: List of test specifications
+            metadata: Optional metadata dict
+        
+        Returns:
+            MetricScore with execution accuracy
+        """
+        code_str = str(prediction)
+        
+        if not references:
+            return MetricScore(
+                metric_name=self.name,
+                value=0.0,
+                details={"error": "No test cases provided"},
+                metadata=metadata or {},
+            )
+        
+        # Extract test cases from reference
+        test_spec = references[0]
+        if not isinstance(test_spec, dict):
+            return MetricScore(
+                metric_name=self.name,
+                value=0.0,
+                details={"error": "Test specification must be a dictionary"},
+                metadata=metadata or {},
+            )
+        
+        test_inputs = test_spec.get("inputs", [])
+        expected_outputs = test_spec.get("expected", [])
+        test_fn_name = test_spec.get("function_name", "solution")
+        
+        if len(test_inputs) != len(expected_outputs):
+            return MetricScore(
+                metric_name=self.name,
+                value=0.0,
+                details={"error": "Mismatch between inputs and expected outputs"},
+                metadata=metadata or {},
+            )
+        
+        # Execute code and run tests
+        results = []
+        for test_input, expected in zip(test_inputs, expected_outputs):
+            result = self._execute_test(
+                code_str,
+                test_fn_name,
+                test_input,
+                expected,
+            )
+            results.append(result)
+        
+        # Compute accuracy
+        passed = sum(1 for r in results if r.passed)
+        total = len(results)
+        accuracy = passed / total if total > 0 else 0.0
+        
+        return MetricScore(
+            metric_name=self.name,
+            value=accuracy,
+            details={
+                "accuracy": accuracy,
+                "passed": passed,
+                "total": total,
+                "results": [
+                    {
+                        "status": r.status.value,
+                        "passed": r.passed,
+                        "error": r.error,
+                        "duration": r.duration,
+                    }
+                    for r in results
+                ],
+            },
+            metadata=metadata or {},
+        )
+    
+    def _execute_test(
+        self,
+        code: str,
+        function_name: str,
+        test_input: Any,
+        expected_output: Any,
+    ) -> ExecutionResult:
+        """Execute a single test case.
+        
+        Args:
+            code: Code to execute
+            function_name: Name of function to test
+            test_input: Input to pass to function
+            expected_output: Expected output
+        
+        Returns:
+            ExecutionResult with status and outcome
+        """
+        import time
+        
+        start_time = time.time()
+        
+        try:
+            # Create restricted globals (no file I/O, network, etc.)
+            restricted_globals = {
+                "__builtins__": {
+                    "abs": abs,
+                    "all": all,
+                    "any": any,
+                    "bool": bool,
+                    "dict": dict,
+                    "enumerate": enumerate,
+                    "filter": filter,
+                    "float": float,
+                    "int": int,
+                    "len": len,
+                    "list": list,
+                    "map": map,
+                    "max": max,
+                    "min": min,
+                    "range": range,
+                    "reversed": reversed,
+                    "set": set,
+                    "sorted": sorted,
+                    "str": str,
+                    "sum": sum,
+                    "tuple": tuple,
+                    "zip": zip,
+                }
+            }
+            
+            # Execute code with timeout
+            local_vars = {}
+            exec(code, restricted_globals, local_vars)
+            
+            # Get the function
+            if function_name not in local_vars:
+                return ExecutionResult(
+                    status=ExecutionStatus.ERROR,
+                    passed=False,
+                    error=f"Function '{function_name}' not found",
+                    duration=time.time() - start_time,
+                )
+            
+            func = local_vars[function_name]
+            
+            # Run function with input
+            if isinstance(test_input, (list, tuple)):
+                actual_output = func(*test_input)
+            else:
+                actual_output = func(test_input)
+            
+            # Check if output matches expected
+            passed = actual_output == expected_output
+            
+            return ExecutionResult(
+                status=ExecutionStatus.PASSED if passed else ExecutionStatus.FAILED,
+                passed=passed,
+                output=str(actual_output),
+                duration=time.time() - start_time,
+            )
+            
+        except TimeoutError:
+            return ExecutionResult(
+                status=ExecutionStatus.TIMEOUT,
+                passed=False,
+                error=f"Execution timeout ({self.timeout}s)",
+                duration=self.timeout,
+            )
+        except Exception as e:
+            return ExecutionResult(
+                status=ExecutionStatus.ERROR,
+                passed=False,
+                error=str(e),
+                duration=time.time() - start_time,
+            )
+
+
+__all__ = ["ExecutionAccuracy", "ExecutionResult", "ExecutionStatus"]

themis/evaluation/metrics/code/pass_at_k.py

@@ -0,0 +1,181 @@
+"""Pass@k metric for code generation evaluation.
+
+Pass@k measures functional correctness by executing k generated code samples
+and checking if any of them pass the test cases.
+
+References:
+    Chen et al. (2021). Evaluating Large Language Models Trained on Code.
+    (HumanEval paper)
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any, Sequence
+
+from themis.core.entities import MetricScore
+from themis.interfaces import Metric
+
+
+def estimate_pass_at_k(n: int, c: int, k: int) -> float:
+    """Estimate pass@k using unbiased estimator.
+    
+    This is the standard estimator from the HumanEval paper.
+    
+    Args:
+        n: Total number of samples generated
+        c: Number of samples that passed
+        k: k value for pass@k
+    
+    Returns:
+        Estimated pass@k probability
+    
+    Example:
+        >>> # Generated 10 samples, 3 passed, compute pass@1
+        >>> estimate_pass_at_k(n=10, c=3, k=1)
+        0.3
+        
+        >>> # Generated 100 samples, 30 passed, compute pass@10
+        >>> estimate_pass_at_k(n=100, c=30, k=10)
+        0.8926
+    """
+    if n - c < k:
+        return 1.0
+    
+    # Unbiased estimator: 1 - C(n-c, k) / C(n, k)
+    # = 1 - product((n-c-i)/(n-i) for i in range(k))
+    result = 1.0
+    for i in range(k):
+        result *= (n - c - i) / (n - i)
+    
+    return 1.0 - result
+
+
+class PassAtK(Metric):
+    """Pass@k metric for code generation.
+    
+    Pass@k measures the probability that at least one of k generated samples
+    passes all test cases. It's the standard metric for evaluating code
+    generation models like Codex, CodeGen, etc.
+    
+    The metric requires:
+    - Multiple samples per problem (num_samples >= k)
+    - Test cases to execute against
+    - Safe code execution environment
+    
+    Attributes:
+        name: Metric identifier ("pass_at_k")
+        k: Number of samples to consider
+        timeout: Maximum execution time per sample (seconds)
+        require_all_tests: Whether all tests must pass (vs any test)
+    
+    Example:
+        >>> from themis.evaluation.metrics.code import PassAtK
+        >>> metric = PassAtK(k=1)
+        >>> score = metric.compute(
+        ...     prediction={
+        ...         "samples": ["def add(a, b): return a + b", ...],
+        ...         "test_results": [True, False, ...],
+        ...     },
+        ...     references=[]
+        ... )
+        >>> print(f"Pass@1: {score.value:.2%}")
+        Pass@1: 30.00%
+    """
+    
+    requires_reference = False  # Uses test execution, not reference matching
+    
+    def __init__(
+        self,
+        k: int = 1,
+        timeout: float = 3.0,
+        require_all_tests: bool = True,
+    ):
+        """Initialize Pass@k metric.
+        
+        Args:
+            k: Number of samples for pass@k estimation
+            timeout: Maximum execution time per sample (seconds)
+            require_all_tests: Whether all test cases must pass (default: True)
+        """
+        self.name = f"pass_at_{k}"
+        self.k = k
+        self.timeout = timeout
+        self.require_all_tests = require_all_tests
+    
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> MetricScore:
+        """Compute Pass@k score.
+        
+        Args:
+            prediction: Dictionary containing:
+                - "samples": List of generated code samples
+                - "test_results": List of booleans (True if passed)
+                - "execution_errors": Optional list of error messages
+            references: Not used (test-based evaluation)
+            metadata: Optional metadata dict
+        
+        Returns:
+            MetricScore with estimated pass@k probability
+        
+        Note:
+            The prediction should be prepared by ExecutionAccuracy metric
+            or similar execution framework.
+        """
+        if not isinstance(prediction, dict):
+            return MetricScore(
+                metric_name=self.name,
+                value=0.0,
+                details={"error": "Prediction must be dict with samples and test_results"},
+                metadata=metadata or {},
+            )
+        
+        samples = prediction.get("samples", [])
+        test_results = prediction.get("test_results", [])
+        
+        if not samples or not test_results:
+            return MetricScore(
+                metric_name=self.name,
+                value=0.0,
+                details={
+                    "error": "Missing samples or test_results",
+                    "num_samples": len(samples),
+                    "num_results": len(test_results),
+                },
+                metadata=metadata or {},
+            )
+        
+        # Count number of samples and passes
+        n = len(test_results)
+        c = sum(1 for result in test_results if result)
+        
+        # Estimate pass@k
+        if n < self.k:
+            # Not enough samples, use empirical rate
+            pass_at_k = c / n if n > 0 else 0.0
+            warning = f"Only {n} samples available for pass@{self.k}"
+        else:
+            pass_at_k = estimate_pass_at_k(n, c, self.k)
+            warning = None
+        
+        return MetricScore(
+            metric_name=self.name,
+            value=pass_at_k,
+            details={
+                "k": self.k,
+                "n_samples": n,
+                "n_passed": c,
+                "pass_rate": c / n if n > 0 else 0.0,
+                "pass_at_k": pass_at_k,
+                "warning": warning,
+            },
+            metadata=metadata or {},
+        )
+
+
+__all__ = ["PassAtK", "estimate_pass_at_k"]

themis/evaluation/metrics/nlp/__init__.py

@@ -0,0 +1,21 @@
+"""NLP evaluation metrics.
+
+This module provides standard NLP metrics for text generation evaluation:
+- BLEU: Bilingual Evaluation Understudy for translation quality
+- ROUGE: Recall-Oriented Understudy for Gisting Evaluation for summarization
+- BERTScore: Contextual embeddings-based evaluation
+- METEOR: Metric for Evaluation of Translation with Explicit ORdering
+"""
+
+from themis.evaluation.metrics.nlp.bleu import BLEU
+from themis.evaluation.metrics.nlp.rouge import ROUGE, ROUGEVariant
+from themis.evaluation.metrics.nlp.bertscore import BERTScore
+from themis.evaluation.metrics.nlp.meteor import METEOR
+
+__all__ = [
+    "BLEU",
+    "ROUGE",
+    "ROUGEVariant",
+    "BERTScore",
+    "METEOR",
+]

themis/evaluation/metrics/nlp/bertscore.py

@@ -0,0 +1,138 @@
+"""BERTScore metric implementation.
+
+BERTScore computes similarity using contextual embeddings from BERT-like models
+instead of exact word matches.
+
+References:
+    Zhang et al. (2020). BERTScore: Evaluating Text Generation with BERT.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Sequence
+
+from themis.core.entities import MetricScore
+from themis.interfaces import Metric
+
+
+class BERTScore(Metric):
+    """BERTScore metric using bert-score library.
+    
+    BERTScore leverages contextual embeddings from pre-trained models (BERT, RoBERTa, etc.)
+    to compute semantic similarity between generated and reference texts. It's more
+    robust to paraphrasing than exact n-gram matching methods.
+    
+    The metric computes token-level cosine similarity between embeddings and aggregates
+    using precision, recall, and F1.
+    
+    Attributes:
+        name: Metric identifier ("bertscore")
+        model_type: Pre-trained model to use for embeddings
+        lang: Language code for automatic model selection
+        rescale_with_baseline: Whether to rescale scores using baseline
+    
+    Example:
+        >>> from themis.evaluation.metrics.nlp import BERTScore
+        >>> metric = BERTScore(model_type="microsoft/deberta-xlarge-mnli")
+        >>> score = metric.compute(
+        ...     prediction="The cat sat on the mat",
+        ...     references=["A cat is sitting on a mat"]
+        ... )
+        >>> print(f"BERTScore F1: {score.value:.4f}")
+        BERTScore F1: 0.9234
+    """
+    
+    requires_reference = True
+    
+    def __init__(
+        self,
+        model_type: str | None = None,
+        lang: str | None = None,
+        rescale_with_baseline: bool = True,
+        device: str | None = None,
+    ):
+        """Initialize BERTScore metric.
+        
+        Args:
+            model_type: Pre-trained model identifier. Popular choices:
+                - "microsoft/deberta-xlarge-mnli" (recommended, large)
+                - "microsoft/deberta-large-mnli" (good balance)
+                - "roberta-large" (fast, good quality)
+                - "bert-base-uncased" (fastest, lower quality)
+            lang: Language code (e.g., "en", "zh", "fr"). If provided,
+                automatically selects appropriate model.
+            rescale_with_baseline: Whether to rescale scores using baseline
+                (recommended for human correlation)
+            device: Device to use ("cuda", "cpu", or None for auto-detect)
+        """
+        self.name = "bertscore"
+        self.model_type = model_type
+        self.lang = lang
+        self.rescale_with_baseline = rescale_with_baseline
+        self.device = device
+        
+        # Lazy import bert-score (not required for all users)
+        try:
+            import bert_score
+            self._bert_score = bert_score
+        except ImportError:
+            raise ImportError(
+                "bert-score is required for BERTScore metric. "
+                "Install it with: pip install bert-score"
+            )
+    
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> MetricScore:
+        """Compute BERTScore.
+        
+        Args:
+            prediction: Generated text (already extracted by pipeline)
+            references: List of reference texts
+            metadata: Optional metadata dict
+        
+        Returns:
+            MetricScore with BERTScore F1 and precision/recall details
+        """
+        # Convert to strings
+        pred_str = str(prediction)
+        ref_strs = [str(ref) for ref in references]
+        
+        # Compute BERTScore
+        # Note: bert_score.score expects lists of predictions and references
+        P, R, F1 = self._bert_score.score(
+            [pred_str] * len(ref_strs),  # Repeat prediction for each reference
+            ref_strs,
+            model_type=self.model_type,
+            lang=self.lang,
+            rescale_with_baseline=self.rescale_with_baseline,
+            device=self.device,
+            verbose=False,
+        )
+        
+        # Take maximum F1 across references
+        max_idx = F1.argmax().item()
+        max_precision = P[max_idx].item()
+        max_recall = R[max_idx].item()
+        max_f1 = F1[max_idx].item()
+        
+        return MetricScore(
+            metric_name=self.name,
+            value=max_f1,  # Use F1 as primary score
+            details={
+                "precision": max_precision,
+                "recall": max_recall,
+                "f1": max_f1,
+                "model_type": self.model_type or f"auto-{self.lang}",
+                "num_references": len(ref_strs),
+                "rescaled": self.rescale_with_baseline,
+            },
+            metadata=metadata or {},
+        )
+
+
+__all__ = ["BERTScore"]