latch-eval-tools 0.1.0__py3-none-any.whl

latch_eval_tools/graders/distribution.py

@@ -0,0 +1,102 @@
+from .base import BinaryGrader, GraderResult
+
+
+class DistributionComparisonGrader(BinaryGrader):
+    def evaluate_answer(self, agent_answer: dict, config: dict) -> GraderResult:
+        ground_truth = config.get("ground_truth", {})
+        tolerances = config.get("tolerances", {})
+
+        gt_total_cells = ground_truth.get("total_cells")
+        gt_distribution = ground_truth.get("cell_type_distribution", {})
+
+        total_cells_tolerance = tolerances.get("total_cells", {})
+        pct_tolerance_config = tolerances.get("cell_type_percentages", {})
+        pct_tolerance = pct_tolerance_config.get("value", 3.0)
+
+        if "cell_type_distribution" not in agent_answer:
+            return GraderResult(
+                passed=False,
+                metrics={},
+                reasoning="Agent answer missing required field: cell_type_distribution",
+                agent_answer=agent_answer
+            )
+
+        agent_total_cells = agent_answer.get("total_cells")
+        agent_distribution = agent_answer["cell_type_distribution"]
+
+        metrics = {}
+        all_pass = True
+        failures = []
+
+        if gt_total_cells is not None and agent_total_cells is not None:
+            total_cells_tol_value = total_cells_tolerance.get("value", 0)
+            total_cells_diff = abs(agent_total_cells - gt_total_cells)
+            total_cells_pass = total_cells_diff <= total_cells_tol_value
+
+            metrics["total_cells_actual"] = agent_total_cells
+            metrics["total_cells_expected"] = gt_total_cells
+            metrics["total_cells_diff"] = total_cells_diff
+            metrics["total_cells_pass"] = total_cells_pass
+
+            if not total_cells_pass:
+                all_pass = False
+                failures.append(f"total_cells: {agent_total_cells} vs {gt_total_cells} (diff: {total_cells_diff})")
+
+        distribution_failures = []
+        for cell_type, expected_pct in gt_distribution.items():
+            if cell_type not in agent_distribution:
+                all_pass = False
+                failures.append(f"Missing cell type: {cell_type}")
+                distribution_failures.append(cell_type)
+                metrics[f"{cell_type}_actual"] = None
+                metrics[f"{cell_type}_expected"] = expected_pct
+                metrics[f"{cell_type}_diff"] = None
+                metrics[f"{cell_type}_pass"] = False
+                continue
+
+            actual_pct = agent_distribution[cell_type]
+            diff = abs(actual_pct - expected_pct)
+            within_tolerance = diff <= pct_tolerance
+
+            metrics[f"{cell_type}_actual"] = actual_pct
+            metrics[f"{cell_type}_expected"] = expected_pct
+            metrics[f"{cell_type}_diff"] = diff
+            metrics[f"{cell_type}_pass"] = within_tolerance
+
+            if not within_tolerance:
+                all_pass = False
+                failures.append(f"{cell_type}: {actual_pct:.2f}% vs {expected_pct:.2f}% (diff: {diff:.2f}%)")
+                distribution_failures.append(cell_type)
+
+        extra_types = set(agent_distribution.keys()) - set(gt_distribution.keys())
+        if extra_types:
+            metrics["extra_cell_types"] = sorted(list(extra_types))
+
+        lines = [
+            f"Distribution Comparison: {'PASS' if all_pass else 'FAIL'}",
+            "",
+            f"Cell type percentages (tolerance: +/-{pct_tolerance}%):"
+        ]
+
+        for cell_type in sorted(gt_distribution.keys()):
+            expected = gt_distribution[cell_type]
+            if cell_type not in agent_distribution:
+                lines.append(f"  x {cell_type}: MISSING vs {expected:.2f}%")
+            else:
+                actual = agent_distribution[cell_type]
+                diff = abs(actual - expected)
+                within_tol = diff <= pct_tolerance
+                check = "+" if within_tol else "x"
+                lines.append(f"  {check} {cell_type}: {actual:.2f}% vs {expected:.2f}% (diff: {diff:.2f}%)")
+
+        if failures:
+            lines.extend(["", "Failures:"])
+            for failure in failures:
+                lines.append(f"  - {failure}")
+
+        return GraderResult(
+            passed=all_pass,
+            metrics=metrics,
+            reasoning="\n".join(lines),
+            agent_answer=agent_answer
+        )

latch_eval_tools/graders/label_set.py

@@ -0,0 +1,75 @@
+from .base import BinaryGrader, GraderResult
+
+
+class LabelSetJaccardGrader(BinaryGrader):
+    def evaluate_answer(self, agent_answer: dict, config: dict) -> GraderResult:
+        ground_truth_labels = set(config.get("ground_truth_labels", []))
+        scoring = config.get("scoring", {})
+        pass_threshold = scoring.get("pass_threshold", 0.90)
+        answer_field = config.get("answer_field", "cell_types_predicted")
+
+        if answer_field not in agent_answer:
+            return GraderResult(
+                passed=False,
+                metrics={},
+                reasoning=f"Agent answer missing required field: {answer_field}",
+                agent_answer=agent_answer
+            )
+
+        predicted_labels = set(agent_answer[answer_field])
+
+        intersection = ground_truth_labels & predicted_labels
+        union = ground_truth_labels | predicted_labels
+
+        jaccard_index = len(intersection) / len(union) if len(union) > 0 else 0.0
+        passed = jaccard_index >= pass_threshold
+
+        true_positives = intersection
+        false_positives = predicted_labels - ground_truth_labels
+        false_negatives = ground_truth_labels - predicted_labels
+
+        metrics = {
+            "jaccard_index": jaccard_index,
+            "pass_threshold": pass_threshold,
+            "answer_field": answer_field,
+            "true_positives": sorted(list(true_positives)),
+            "false_positives": sorted(list(false_positives)),
+            "false_negatives": sorted(list(false_negatives)),
+            "predicted_count": len(predicted_labels),
+            "ground_truth_count": len(ground_truth_labels),
+        }
+
+        lines = [
+            f"Label Set Comparison: {'PASS' if passed else 'FAIL'}",
+            "",
+            f"  {'+'if passed else 'x'} Jaccard Index: {jaccard_index:.3f} (threshold: {pass_threshold:.3f})",
+            "",
+            f"Correct Labels ({len(true_positives)}):"
+        ]
+
+        if true_positives:
+            for label in sorted(true_positives):
+                lines.append(f"  + {label}")
+        else:
+            lines.append("  None")
+
+        lines.extend(["", f"Missing Labels ({len(false_negatives)}):"])
+        if false_negatives:
+            for label in sorted(false_negatives):
+                lines.append(f"  - {label}")
+        else:
+            lines.append("  None")
+
+        lines.extend(["", f"Extra Labels ({len(false_positives)}):"])
+        if false_positives:
+            for label in sorted(false_positives):
+                lines.append(f"  ? {label}")
+        else:
+            lines.append("  None")
+
+        return GraderResult(
+            passed=passed,
+            metrics=metrics,
+            reasoning="\n".join(lines),
+            agent_answer=agent_answer
+        )

latch_eval_tools/graders/marker_gene.py

@@ -0,0 +1,317 @@
+from .base import BinaryGrader, GraderResult
+
+
+class MarkerGenePrecisionRecallGrader(BinaryGrader):
+    def evaluate_answer(self, agent_answer: dict, config: dict) -> GraderResult:
+        canonical_markers = config.get("canonical_markers", config.get("ground_truth_labels", []))
+        scoring = config.get("scoring", {})
+        thresholds = scoring.get("pass_thresholds", {})
+        answer_field = config.get("answer_field", "top_marker_genes")
+
+        if answer_field not in agent_answer:
+            for key in agent_answer.keys():
+                if isinstance(agent_answer[key], (list, dict)):
+                    answer_field = key
+                    break
+
+        if answer_field not in agent_answer:
+            return GraderResult(
+                passed=False,
+                metrics={},
+                reasoning=f"Agent answer missing required field. Available keys: {list(agent_answer.keys())}",
+                agent_answer=agent_answer
+            )
+
+        predicted = agent_answer[answer_field]
+
+        if isinstance(canonical_markers, dict) and isinstance(predicted, dict):
+            return self._evaluate_per_celltype(predicted, canonical_markers, thresholds, answer_field, agent_answer)
+
+        if isinstance(canonical_markers, dict) and answer_field in canonical_markers:
+            canonical_markers = canonical_markers[answer_field]
+
+        if not isinstance(predicted, list):
+            return GraderResult(
+                passed=False,
+                metrics={},
+                reasoning=f"{answer_field} must be a list, got {type(predicted).__name__}",
+                agent_answer=agent_answer
+            )
+
+        if not isinstance(canonical_markers, list):
+            return GraderResult(
+                passed=False,
+                metrics={},
+                reasoning=f"canonical_markers must be a list for flat evaluation, got {type(canonical_markers).__name__}",
+                agent_answer=agent_answer
+            )
+
+        return self._evaluate_flat_list(predicted, canonical_markers, thresholds, answer_field, agent_answer)
+
+    def _evaluate_per_celltype(self, predicted: dict, canonical_markers: dict, thresholds: dict, answer_field: str, agent_answer: dict) -> GraderResult:
+        min_recall = thresholds.get("min_recall_per_celltype", thresholds.get("recall_at_k", 0.50))
+        min_celltypes_passing = thresholds.get("min_celltypes_passing", len(canonical_markers))
+
+        celltype_results = {}
+        celltypes_passing = 0
+        total_celltypes = len(canonical_markers)
+
+        for celltype, canonical_genes in canonical_markers.items():
+            predicted_genes = predicted.get(celltype, [])
+            if not isinstance(predicted_genes, list):
+                celltype_results[celltype] = {
+                    "pass": False,
+                    "recall": 0.0,
+                    "error": f"Expected list, got {type(predicted_genes).__name__}"
+                }
+                continue
+
+            predicted_genes = [str(g) for g in predicted_genes]
+            canonical_set = set(str(gene).lower() for gene in canonical_genes)
+            predicted_set = set(gene.lower() for gene in predicted_genes)
+
+            true_positives = canonical_set & predicted_set
+            false_negatives = canonical_set - predicted_set
+
+            recall = len(true_positives) / len(canonical_set) if len(canonical_set) > 0 else 1.0
+            celltype_pass = recall >= min_recall
+
+            if celltype_pass:
+                celltypes_passing += 1
+
+            celltype_results[celltype] = {
+                "pass": celltype_pass,
+                "recall": recall,
+                "num_predicted": len(predicted_genes),
+                "num_canonical": len(canonical_set),
+                "true_positives": sorted(true_positives),
+                "false_negatives": sorted(false_negatives),
+            }
+
+        passed = celltypes_passing >= min_celltypes_passing
+
+        metrics = {
+            "celltypes_passing": celltypes_passing,
+            "total_celltypes": total_celltypes,
+            "min_celltypes_passing": min_celltypes_passing,
+            "min_recall_per_celltype": min_recall,
+            "per_celltype": celltype_results,
+            "answer_field_used": answer_field,
+        }
+
+        lines = [
+            f"Marker Gene Per-Celltype: {'PASS' if passed else 'FAIL'}",
+            f"Celltypes passing: {celltypes_passing}/{total_celltypes} (required: {min_celltypes_passing})",
+            ""
+        ]
+        for celltype, result in celltype_results.items():
+            check = "+" if result["pass"] else "x"
+            lines.append(f"  {check} {celltype}: recall={result['recall']:.2f} (threshold: {min_recall:.2f})")
+
+        return GraderResult(
+            passed=passed,
+            metrics=metrics,
+            reasoning="\n".join(lines),
+            agent_answer=agent_answer
+        )
+
+    def _evaluate_flat_list(self, predicted_genes: list, canonical_markers: list, thresholds: dict, answer_field: str, agent_answer: dict) -> GraderResult:
+        precision_threshold = thresholds.get("precision_at_k", 0.60)
+        recall_threshold = thresholds.get("recall_at_k", 0.50)
+
+        predicted_genes = [str(g) for g in predicted_genes]
+        k = len(predicted_genes)
+
+        canonical_set = set(str(gene).lower() for gene in canonical_markers)
+        predicted_set = set(gene.lower() for gene in predicted_genes)
+
+        true_positives = canonical_set & predicted_set
+        false_positives = predicted_set - canonical_set
+        false_negatives = canonical_set - predicted_set
+
+        precision_at_k = len(true_positives) / k if k > 0 else 0.0
+        recall_at_k = len(true_positives) / len(canonical_set) if len(canonical_set) > 0 else 0.0
+
+        precision_pass = precision_at_k >= precision_threshold
+        recall_pass = recall_at_k >= recall_threshold
+        passed = precision_pass and recall_pass
+
+        original_case_map = {gene.lower(): gene for gene in predicted_genes}
+        canonical_case_map = {str(gene).lower(): str(gene) for gene in canonical_markers}
+
+        true_positive_genes = [original_case_map.get(g, canonical_case_map.get(g, g)) for g in true_positives]
+        false_positive_genes = [original_case_map.get(g, g) for g in false_positives]
+        false_negative_genes = [canonical_case_map.get(g, g) for g in false_negatives]
+
+        metrics = {
+            "k": k,
+            "precision_at_k": precision_at_k,
+            "recall_at_k": recall_at_k,
+            "precision_threshold": precision_threshold,
+            "recall_threshold": recall_threshold,
+            "true_positives": sorted(true_positive_genes),
+            "false_positives": sorted(false_positive_genes),
+            "false_negatives": sorted(false_negative_genes),
+            "num_true_positives": len(true_positives),
+            "num_false_positives": len(false_positives),
+            "num_false_negatives": len(false_negatives),
+            "num_canonical_markers": len(canonical_set),
+            "precision_pass": precision_pass,
+            "recall_pass": recall_pass,
+            "answer_field_used": answer_field,
+        }
+
+        reasoning = self._format_reasoning(
+            k, precision_at_k, recall_at_k, precision_threshold, recall_threshold,
+            true_positive_genes, false_positive_genes, false_negative_genes,
+            precision_pass, recall_pass, passed, answer_field
+        )
+
+        return GraderResult(
+            passed=passed,
+            metrics=metrics,
+            reasoning=reasoning,
+            agent_answer=agent_answer
+        )
+
+    def _format_reasoning(self, k, precision, recall, precision_threshold, recall_threshold,
+                          true_positives, false_positives, false_negatives,
+                          precision_pass, recall_pass, passed, answer_field):
+        lines = [
+            f"Marker Gene Precision/Recall: {'PASS' if passed else 'FAIL'}",
+            f"Answer field: {answer_field}",
+            "",
+            f"  {'+'if precision_pass else 'x'} Precision@{k}: {precision:.3f} (threshold: {precision_threshold:.3f})",
+            f"  {'+'if recall_pass else 'x'} Recall@{k}: {recall:.3f} (threshold: {recall_threshold:.3f})",
+            "",
+            f"True Positives ({len(true_positives)}):"
+        ]
+
+        if true_positives:
+            for gene in sorted(true_positives):
+                lines.append(f"  + {gene}")
+        else:
+            lines.append("  None")
+
+        lines.extend(["", f"False Negatives ({len(false_negatives)}):"])
+        if false_negatives:
+            for gene in sorted(false_negatives):
+                lines.append(f"  - {gene}")
+        else:
+            lines.append("  None")
+
+        if not passed:
+            lines.append("")
+            failures = []
+            if not precision_pass:
+                failures.append(f"Precision {precision:.3f} < {precision_threshold:.3f}")
+            if not recall_pass:
+                failures.append(f"Recall {recall:.3f} < {recall_threshold:.3f}")
+            lines.append(f"Failure: {'; '.join(failures)}")
+
+        return "\n".join(lines)
+
+
+class MarkerGeneSeparationGrader(BinaryGrader):
+    def evaluate_answer(self, agent_answer: dict, config: dict) -> GraderResult:
+        scoring = config.get("scoring", {})
+        thresholds = scoring.get("pass_thresholds", {})
+        mean_auroc_threshold = thresholds.get("mean_auroc", 0.85)
+        fraction_high_threshold = thresholds.get("fraction_high", 0.70)
+        per_gene_cutoff = thresholds.get("per_gene_cutoff", 0.80)
+
+        if "per_gene_stats" not in agent_answer:
+            return GraderResult(
+                passed=False,
+                metrics={},
+                reasoning="Agent answer missing required field: per_gene_stats",
+                agent_answer=agent_answer
+            )
+
+        if "mean_auroc" not in agent_answer:
+            return GraderResult(
+                passed=False,
+                metrics={},
+                reasoning="Agent answer missing required field: mean_auroc",
+                agent_answer=agent_answer
+            )
+
+        per_gene_stats = agent_answer["per_gene_stats"]
+        agent_mean_auroc = agent_answer["mean_auroc"]
+
+        if not isinstance(per_gene_stats, list):
+            return GraderResult(
+                passed=False,
+                metrics={},
+                reasoning="per_gene_stats must be a list",
+                agent_answer=agent_answer
+            )
+
+        num_genes = len(per_gene_stats)
+        if num_genes == 0:
+            return GraderResult(
+                passed=False,
+                metrics={},
+                reasoning="per_gene_stats is empty",
+                agent_answer=agent_answer
+            )
+
+        gene_aurocs = {}
+        for stat in per_gene_stats:
+            if not isinstance(stat, dict) or "gene" not in stat or "auroc" not in stat:
+                return GraderResult(
+                    passed=False,
+                    metrics={},
+                    reasoning="Each element in per_gene_stats must have 'gene' and 'auroc' fields",
+                    agent_answer=agent_answer
+                )
+            gene_aurocs[stat["gene"]] = stat["auroc"]
+
+        computed_mean_auroc = sum(gene_aurocs.values()) / len(gene_aurocs)
+
+        high_auroc_genes = [gene for gene, auroc in gene_aurocs.items() if auroc >= per_gene_cutoff]
+        low_auroc_genes = [gene for gene, auroc in gene_aurocs.items() if auroc < per_gene_cutoff]
+        fraction_high = len(high_auroc_genes) / num_genes
+
+        mean_auroc_pass = agent_mean_auroc >= mean_auroc_threshold
+        fraction_high_pass = fraction_high >= fraction_high_threshold
+        passed = mean_auroc_pass and fraction_high_pass
+
+        metrics = {
+            "num_genes": num_genes,
+            "mean_auroc_agent": agent_mean_auroc,
+            "mean_auroc_computed": computed_mean_auroc,
+            "mean_auroc_threshold": mean_auroc_threshold,
+            "fraction_high": fraction_high,
+            "fraction_high_threshold": fraction_high_threshold,
+            "per_gene_cutoff": per_gene_cutoff,
+            "num_high_auroc_genes": len(high_auroc_genes),
+            "num_low_auroc_genes": len(low_auroc_genes),
+            "high_auroc_genes": sorted(high_auroc_genes),
+            "low_auroc_genes": sorted(low_auroc_genes),
+            "mean_auroc_pass": mean_auroc_pass,
+            "fraction_high_pass": fraction_high_pass,
+            "per_gene_aurocs": gene_aurocs,
+        }
+
+        lines = [
+            f"Marker Gene Separation: {'PASS' if passed else 'FAIL'}",
+            "",
+            f"  {'+'if mean_auroc_pass else 'x'} Mean AUROC: {agent_mean_auroc:.3f} (threshold: {mean_auroc_threshold:.3f})",
+            f"  {'+'if fraction_high_pass else 'x'} Fraction High (>={per_gene_cutoff:.2f}): {fraction_high:.3f} ({len(high_auroc_genes)}/{num_genes})",
+        ]
+
+        if not passed:
+            failures = []
+            if not mean_auroc_pass:
+                failures.append(f"Mean AUROC {agent_mean_auroc:.3f} < {mean_auroc_threshold:.3f}")
+            if not fraction_high_pass:
+                failures.append(f"Fraction high {fraction_high:.3f} < {fraction_high_threshold:.3f}")
+            lines.append(f"\nFailure: {'; '.join(failures)}")
+
+        return GraderResult(
+            passed=passed,
+            metrics=metrics,
+            reasoning="\n".join(lines),
+            agent_answer=agent_answer
+        )

latch_eval_tools/graders/multiple_choice.py

@@ -0,0 +1,38 @@
+from .base import BinaryGrader, GraderResult
+
+
+class MultipleChoiceGrader(BinaryGrader):
+    def evaluate_answer(self, agent_answer: dict, config: dict) -> GraderResult:
+        if "correct_answers" in config:
+            correct_answers = [a.strip().upper() for a in config["correct_answers"]]
+        else:
+            correct_answers = [config.get("correct_answer", "").strip().upper()]
+
+        if "answer" not in agent_answer:
+            return GraderResult(
+                passed=False,
+                metrics={},
+                reasoning="Agent answer missing required field: answer",
+                agent_answer=agent_answer
+            )
+
+        agent_choice = str(agent_answer["answer"]).strip().upper()
+        passed = agent_choice in correct_answers
+
+        display_correct = correct_answers[0] if len(correct_answers) == 1 else correct_answers
+        metrics = {
+            "correct_answers": correct_answers,
+            "agent_answer": agent_choice,
+        }
+
+        if passed:
+            reasoning = f"Multiple Choice: PASS\n\n  + Agent answered: {agent_choice} (correct)"
+        else:
+            reasoning = f"Multiple Choice: FAIL\n\n  x Agent answered: {agent_choice}\n    Correct answer(s): {display_correct}"
+
+        return GraderResult(
+            passed=passed,
+            metrics=metrics,
+            reasoning=reasoning,
+            agent_answer=agent_answer
+        )

latch_eval_tools/graders/numeric.py

@@ -0,0 +1,137 @@
+from .base import BinaryGrader, GraderResult, get_nested_value
+
+
+class NumericToleranceGrader(BinaryGrader):
+    def evaluate_answer(self, agent_answer: dict, config: dict) -> GraderResult:
+        ground_truth = config.get("ground_truth", {})
+        tolerances = config.get("tolerances", config.get("tolerance", {}))
+
+        metrics = {}
+        all_pass = True
+        failures = []
+
+        for field, expected_value in ground_truth.items():
+            actual_value, found = get_nested_value(agent_answer, field)
+            if not found:
+                all_pass = False
+                failures.append(f"Missing field: {field}")
+                continue
+
+            if isinstance(actual_value, str):
+                try:
+                    actual_value = float(actual_value)
+                except ValueError:
+                    all_pass = False
+                    failures.append(f"{field}: cannot parse '{actual_value}' as number")
+                    continue
+
+            if isinstance(actual_value, bool):
+                actual_value = int(actual_value)
+
+            if actual_value is None:
+                all_pass = False
+                failures.append(f"{field}: got null/None value")
+                metrics[f"{field}_actual"] = None
+                metrics[f"{field}_expected"] = expected_value
+                metrics[f"{field}_error"] = float('inf')
+                metrics[f"{field}_pass"] = False
+                continue
+
+            tolerance_config = tolerances.get(field, {"type": "absolute", "value": 0})
+            if isinstance(tolerances, dict) and "type" in tolerances and "value" not in tolerances.get(field, {}):
+                tolerance_config = tolerances
+            tolerance_type = tolerance_config.get("type", "absolute")
+            has_asymmetric = "lower" in tolerance_config and "upper" in tolerance_config
+            tolerance_value = tolerance_config.get("value", 0)
+            tolerance_lower = tolerance_config.get("lower", tolerance_value)
+            tolerance_upper = tolerance_config.get("upper", tolerance_value)
+
+            try:
+                if tolerance_type == "absolute":
+                    if has_asymmetric:
+                        within_tolerance = (expected_value - tolerance_lower) <= actual_value <= (expected_value + tolerance_upper)
+                        error = actual_value - expected_value
+                    else:
+                        within_tolerance = abs(actual_value - expected_value) <= tolerance_value
+                        error = abs(actual_value - expected_value)
+                elif tolerance_type == "relative":
+                    relative_error = abs(actual_value - expected_value) / abs(expected_value) if expected_value != 0 else float('inf')
+                    within_tolerance = relative_error <= tolerance_value
+                    error = relative_error
+                elif tolerance_type == "min":
+                    threshold = tolerance_value
+                    within_tolerance = actual_value >= threshold
+                    error = threshold - actual_value if actual_value < threshold else 0
+                elif tolerance_type == "max":
+                    threshold = tolerance_value
+                    within_tolerance = actual_value <= threshold
+                    error = actual_value - threshold if actual_value > threshold else 0
+                else:
+                    within_tolerance = False
+                    error = float('inf')
+            except TypeError:
+                all_pass = False
+                failures.append(f"{field}: invalid type {type(actual_value).__name__}, expected numeric")
+                metrics[f"{field}_actual"] = actual_value
+                metrics[f"{field}_expected"] = expected_value
+                metrics[f"{field}_error"] = float('inf')
+                metrics[f"{field}_pass"] = False
+                continue
+
+            metrics[f"{field}_actual"] = actual_value
+            metrics[f"{field}_expected"] = expected_value
+            metrics[f"{field}_error"] = error
+            metrics[f"{field}_pass"] = within_tolerance
+
+            if not within_tolerance:
+                all_pass = False
+                if tolerance_type == "min":
+                    failures.append(f"{field}: {actual_value} (minimum required: {tolerance_value})")
+                elif tolerance_type == "max":
+                    failures.append(f"{field}: {actual_value} (maximum allowed: {tolerance_value})")
+                elif has_asymmetric:
+                    failures.append(f"{field}: {actual_value} vs {expected_value} (allowed: -{tolerance_lower}/+{tolerance_upper})")
+                else:
+                    failures.append(f"{field}: {actual_value} vs {expected_value} (error: {error:.2f}, tolerance: {tolerance_value})")
+
+        reasoning = self._format_reasoning(ground_truth, tolerances, metrics, failures, all_pass)
+
+        return GraderResult(
+            passed=all_pass,
+            metrics=metrics,
+            reasoning=reasoning,
+            agent_answer=agent_answer
+        )
+
+    def _format_reasoning(self, ground_truth, tolerances, metrics, failures, passed):
+        lines = [f"Numeric Tolerance Check: {'PASS' if passed else 'FAIL'}", ""]
+
+        for field in ground_truth.keys():
+            if f"{field}_actual" in metrics:
+                actual = metrics[f"{field}_actual"]
+                expected = metrics[f"{field}_expected"]
+                error = metrics[f"{field}_error"]
+                field_pass = metrics[f"{field}_pass"]
+                check = "+" if field_pass else "x"
+                tolerance_config = tolerances.get(field, {}) if isinstance(tolerances, dict) else {}
+                tolerance_type = tolerance_config.get("type", "absolute")
+                has_asymmetric = "lower" in tolerance_config and "upper" in tolerance_config
+                if tolerance_type == "min":
+                    tol_val = tolerance_config.get("value", expected)
+                    lines.append(f"  {check} {field}: {actual} (minimum: {tol_val})")
+                elif tolerance_type == "max":
+                    tol_val = tolerance_config.get("value", expected)
+                    lines.append(f"  {check} {field}: {actual} (maximum: {tol_val})")
+                elif has_asymmetric:
+                    lower = tolerance_config["lower"]
+                    upper = tolerance_config["upper"]
+                    lines.append(f"  {check} {field}: {actual} vs {expected} (allowed: -{lower}/+{upper})")
+                else:
+                    lines.append(f"  {check} {field}: {actual} vs {expected} (error: {error:.4f})")
+
+        if not passed and failures:
+            lines.extend(["", "Failures:"])
+            for failure in failures:
+                lines.append(f"  - {failure}")
+
+        return "\n".join(lines)