themis-eval 0.2.2__tar.gz → 0.2.3__tar.gz

{themis_eval-0.2.2/themis_eval.egg-info → themis_eval-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: themis-eval
-Version: 0.2.2
+Version: 0.2.3
 Summary: Lightweight evaluation platform for LLM experiments
 Author: Pittawat Taveekitworachai
 License: MIT

{themis_eval-0.2.2 → themis_eval-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "themis-eval"
-version = "0.2.2"
+version = "0.2.3"
 description = "Lightweight evaluation platform for LLM experiments"
 readme = "README.md"
 requires-python = ">=3.12"

{themis_eval-0.2.2 → themis_eval-0.2.3}/themis/_version.py

@@ -9,7 +9,7 @@ def _detect_version() -> str:
     try:
         return metadata.version("themis-eval")
     except metadata.PackageNotFoundError:  # pragma: no cover - local dev only
-        return "0.2.2"  # Fallback for development
+        return "0.2.3"  # Fallback for development
 
 
 __version__ = _detect_version()

{themis_eval-0.2.2 → themis_eval-0.2.3}/themis/evaluation/pipelines/standard_pipeline.py

@@ -233,7 +233,8 @@ class EvaluationPipeline:
                             if reference is not None
                             else []
                         )
-                        metadata = {"sample_id": sample_id}
+                        # Preserve all task metadata for metrics, add sample_id
+                        metadata = {**record.task.metadata, "sample_id": sample_id}
                         extract_start = time.perf_counter()
                         item_scores_for_item: list[core_entities.MetricScore] = []
                         for metric in self._metrics:

{themis_eval-0.2.2 → themis_eval-0.2.3}/themis/evaluation/strategies/attempt_aware_evaluation_strategy.py

@@ -37,13 +37,16 @@ class AttemptAwareEvaluationStrategy:
             grouped.setdefault(score.metric_name, []).append(score)
         for metric_name, group in grouped.items():
             value = sum(item.value for item in group) / len(group)
+            # Preserve original metadata from first score
+            base_metadata = group[0].metadata.copy() if group[0].metadata else {}
             aggregated.append(
                 core_entities.MetricScore(
                     metric_name=metric_name,
                     value=value,
                     metadata={
-                        "attempts": len(group),
-                        "sample_id": group[0].metadata.get("sample_id"),
+                        **base_metadata,  # Preserve all original metadata
+                        "attempts": len(group),  # Add aggregation-specific field
+                        "sample_id": base_metadata.get("sample_id"),
                     },
                     details={},
                 )

{themis_eval-0.2.2 → themis_eval-0.2.3}/themis/evaluation/strategies/judge_evaluation_strategy.py

@@ -49,6 +49,8 @@ class JudgeEvaluationStrategy:
                 counts = Counter(labels)
                 agreement = max(counts.values()) / max(1, len(labels))
 
+            # Preserve original metadata from first score
+            base_metadata = group[0].metadata.copy() if group[0].metadata else {}
             aggregated.append(
                 core_entities.MetricScore(
                     metric_name=metric_name,
@@ -58,7 +60,10 @@ class JudgeEvaluationStrategy:
                         "agreement": agreement,
                         "labels": labels,
                     },
-                    metadata={"sample_id": group[0].metadata.get("sample_id")},
+                    metadata={
+                        **base_metadata,  # Preserve all original metadata
+                        "sample_id": base_metadata.get("sample_id"),
+                    },
                 )
             )
         return aggregated

{themis_eval-0.2.2 → themis_eval-0.2.3}/themis/generation/plan.py

@@ -88,9 +88,20 @@ class GenerationPlan:
         }
         if dataset_id is not None:
             metadata["dataset_id"] = dataset_id
-        for field_name in self.metadata_fields:
-            if field_name in row:
-                metadata[field_name] = row[field_name]
+        
+        # If metadata_fields is explicitly specified, use it as a filter (existing behavior)
+        # Otherwise, include all fields by default (new behavior for custom metrics)
+        if self.metadata_fields:
+            # Explicit filter - only include specified fields
+            for field_name in self.metadata_fields:
+                if field_name in row:
+                    metadata[field_name] = row[field_name]
+        else:
+            # No filter - include all fields except those used for other purposes
+            for field_name, field_value in row.items():
+                if field_name not in (self.dataset_id_field, self.reference_field):
+                    metadata[field_name] = field_value
+        
         return metadata
 
     def _build_reference(
@@ -209,9 +220,20 @@ class CartesianExpansionStrategy:
         }
         if dataset_id is not None:
             metadata["dataset_id"] = dataset_id
-        for field_name in context.metadata_fields:
-            if field_name in row:
-                metadata[field_name] = row[field_name]
+        
+        # If metadata_fields is explicitly specified, use it as a filter (existing behavior)
+        # Otherwise, include all fields by default (new behavior for custom metrics)
+        if context.metadata_fields:
+            # Explicit filter - only include specified fields
+            for field_name in context.metadata_fields:
+                if field_name in row:
+                    metadata[field_name] = row[field_name]
+        else:
+            # No filter - include all fields except those used for other purposes
+            for field_name, field_value in row.items():
+                if field_name not in (context.dataset_id_field, context.reference_field):
+                    metadata[field_name] = field_value
+        
         return metadata
 
     def _build_reference(

{themis_eval-0.2.2 → themis_eval-0.2.3/themis_eval.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: themis-eval
-Version: 0.2.2
+Version: 0.2.3
 Summary: Lightweight evaluation platform for LLM experiments
 Author: Pittawat Taveekitworachai
 License: MIT