eval-ai-library 0.2.2__py3-none-any.whl → 0.3.1__py3-none-any.whl

eval_lib/utils.py

@@ -4,7 +4,7 @@ Utility functions for metrics evaluation
 import re
 import json
 from typing import List
-from math import exp
+import math
 
 
 """
@@ -12,47 +12,62 @@ Utility functions for metrics evaluation
 """
 
 
+def _map_temperature_to_p(
+    temperature: float,
+    t_min: float = 0.1,
+    t_max: float = 1.0,
+    p_min: float = -8.0,
+    p_max: float = 12.25,  # chosen so that t=0.5 -> p=1
+) -> float:
+    """
+    Map temperature in [t_min, t_max] linearly to power exponent p, with:
+      t=0.1 -> p=-8 (very strict)
+      t=0.5 -> p=+1 (arithmetic mean)
+      t=1.0 -> p=+12.25 (very lenient)
+    """
+    t = max(t_min, min(t_max, temperature))
+    alpha = (t - t_min) / (t_max - t_min)  # in [0,1]
+    return p_min + alpha * (p_max - p_min)
+
+
 def score_agg(
     scores: List[float],
     temperature: float = 0.5,
-    penalty: float = 0.1
+    penalty: float = 0.1,
+    eps_for_neg_p: float = 1e-9
 ) -> float:
     """
-    Compute a softmax-weighted aggregate of scores with penalty for low-scoring items.
-
-    This function applies softmax weighting (higher scores get more weight) and then
-    applies a penalty proportional to the number of low-scoring items.
+    Aggregate verdict scores with temperature-controlled strictness via power mean.
 
-    Args:
-        scores: List of scores (0.0 to 1.0) to aggregate
-        temperature: Controls strictness of aggregation
-            - Lower (0.1-0.3): Strict - high scores dominate
-            - Medium (0.4-0.6): Balanced - default behavior
-            - Higher (0.8-1.5): Lenient - closer to arithmetic mean
-        penalty: Penalty factor for low-scoring items (default 0.1)
-            - Applied to scores <= 0.4
-
-    Returns:
-        Aggregated score between 0.0 and 1.0
+    - Low temperature (~0.1): strict (p negative) -> close to min
+    - Medium temperature (=0.5): balanced (p=1) -> arithmetic mean
+    - High temperature (=1.0): lenient (large positive p) -> close to max
 
-    Example:
-        >>> scores = [1.0, 0.9, 0.7, 0.3, 0.0]
-        >>> score_agg(scores, temperature=0.5)
-        0.73
+    Applies a penalty for "none" verdicts (0.0) only.
     """
     if not scores:
         return 0.0
 
-    # Compute softmax weights
-    exp_scores = [exp(s / temperature) for s in scores]
-    total = sum(exp_scores)
-    softmax_score = sum(s * e / total for s, e in zip(scores, exp_scores))
+    p = _map_temperature_to_p(temperature)
+
+    # For negative p, clamp zeros to a small epsilon to avoid 0**p blowing up
+    base = [(s if s > 0.0 else eps_for_neg_p)
+            for s in scores] if p < 0 else scores
+
+    # Power mean: M_p = ( (Σ s_i^p) / n )^(1/p)
+    if abs(p) < 1e-12:
+        # Limit p -> 0 is geometric mean
+        logs = [math.log(s if s > 0 else eps_for_neg_p) for s in base]
+        agg = math.exp(sum(logs) / len(logs))
+    else:
+        mean_pow = sum(s ** p for s in base) / len(base)
+        agg = mean_pow ** (1.0 / p)
 
-    # Apply penalty if many statements have low scores (≤ 0.4)
-    irrelevant = sum(1 for s in scores if s <= 0.4)
-    penalty_factor = max(0.0, 1 - penalty * irrelevant)
+    # Apply penalty for "none" verdicts only
+    none_count = sum(1 for s in scores if s == 0.0)
+    penalty_factor = max(0.0, 1 - penalty * none_count)
 
-    return round(softmax_score * penalty_factor, 4)
+    return round(agg * penalty_factor, 4)
 
 
 def extract_json_block(text: str) -> str: