clear-eval 2.0.3__tar.gz → 2.0.4__tar.gz

{clear_eval-2.0.3 → clear_eval-2.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clear_eval
-Version: 2.0.3
+Version: 2.0.4
 Summary: A python API sdk facilitating Error Analysis via LLM-as-a-Judge
 Author-email: Lilach Eden <lilache@il.ibm.com>, Asaf Yehudai <asaf.yehudai@ibm.com>
 License: Apache-2.0

{clear_eval-2.0.3 → clear_eval-2.0.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "clear_eval"
-version = "2.0.3"
+version = "2.0.4"
 description = "A python API sdk facilitating Error Analysis via LLM-as-a-Judge"
 readme = "README.md"
 authors = [{ name = "Lilach Eden", email = "lilache@il.ibm.com"},

{clear_eval-2.0.3 → clear_eval-2.0.4}/src/clear_eval/agentic/dashboard/generate_static_dashboard.py

@@ -42,7 +42,7 @@ def _maybe_parse_json_text(value):
     return value
 
 
-def load_json_data(json_path, include_examples=True):
+def load_json_data(json_path, num_examples=3):
     """Load all dashboard data from clear_results.json and map it to the HTML data shape."""
     json_path = Path(json_path)
 
@@ -79,8 +79,8 @@ def load_json_data(json_path, include_examples=True):
             for issue in issues:
                 issue_text = issue.get("issue_text") or issue.get("issue_id") or ""
                 examples = []
-                if include_examples:
-                    for occurrence in issue.get("occurrences", [])[:3]:
+                if num_examples:
+                    for occurrence in issue.get("occurrences", [])[:num_examples]:
                         tasks_list.add(occurrence.get("trace_id"))
                         span_reference = occurrence.get("span_reference", {})
                         input_output_pair = occurrence.get("input_output_pair", {})
@@ -121,13 +121,22 @@ def load_json_data(json_path, include_examples=True):
                 key=lambda row: -row["count"],
             ) + [row for row in issues_table if row["is_no_issue"]]
 
-            agents_data[agent_label] = {
+            entry = {
                 "total_evals": int(summary.get("total_interactions", 0)),
                 "avg_score": round(float(summary["avg_score"]), 2) if summary.get("avg_score") is not None else None,
                 "unique_issues": len(issues),
                 "unique_tasks": node_stats.get(agent_name, {}).get("unique_tasks", 0),
                 "issues_table": issues_table,
+                "recommendations": None,
             }
+            if agent_type == "tools":
+                # Recommendations now travel inside the per-agent payload
+                # produced by build_json_results._aggregate_recommendations,
+                # so the dashboard just reads the key off the JSON.
+                recs = agent_type_payload.get("recommendations")
+                if recs and (recs.get("by_tool") or recs.get("by_system_prompt")):
+                    entry["recommendations"] = recs
+            agents_data[agent_label] = entry
 
     return {
         "unique_tasks": int(stats.get("total_traces", 0)),
@@ -234,6 +243,26 @@ body{font-family:'Inter',-apple-system,BlinkMacSystemFont,sans-serif;background:
 .chat-message-content{font-size:14px;color:#1E293B;white-space:pre-wrap;word-break:break-word;line-height:1.6}
 .chat-message-tools{margin-top:10px;padding-top:10px;border-top:1px dashed #CBD5E1}
 .expand-indicator{display:block;margin-top:8px;color:#6366F1;font-weight:800;font-size:13px}
+.recs-block{margin-top:24px}
+.recs-subblock{margin-top:14px}
+.recs-subblock-title{font-size:14px;font-weight:800;color:#4338CA;text-transform:uppercase;letter-spacing:.08em;margin-bottom:10px}
+.rec-card{background:#fff;border:1px solid #D8E1EE;border-radius:12px;padding:14px 16px;margin-bottom:12px;box-shadow:0 1px 3px rgba(15,23,42,.04)}
+.rec-card-head{display:flex;align-items:center;gap:10px;flex-wrap:wrap;margin-bottom:8px}
+.rec-target-pill{font-size:11px;font-weight:700;text-transform:uppercase;letter-spacing:.06em;padding:2px 8px;border-radius:999px;background:#EEF2FF;color:#4338CA}
+.rec-target-pill.system{background:#DBEAFE;color:#1D4ED8}
+.rec-target-pill.tool{background:#DCFCE7;color:#166534}
+.rec-target-pill.param{background:#FEF3C7;color:#92400E}
+.rec-scope{font-size:13px;color:#334155;font-weight:600}
+.rec-importance{margin-left:auto;font-size:12px;font-weight:700;color:#6366F1}
+.rec-count{font-size:12px;color:#64748B}
+.rec-rationale{font-size:13px;color:#475569;line-height:1.6;margin-bottom:10px}
+.rec-diff{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;font-size:12.5px;background:#0F172A;color:#E2E8F0;padding:12px 14px;border-radius:10px;overflow:auto;max-height:260px;white-space:pre;line-height:1.5}
+.rec-diff .rec-line-add{color:#86EFAC}
+.rec-diff .rec-line-del{color:#FCA5A5}
+.rec-diff .rec-line-hunk{color:#93C5FD}
+.rec-tool-header{display:flex;align-items:baseline;gap:12px;margin:14px 0 6px}
+.rec-tool-name{font-size:15px;font-weight:800;color:#0F172A}
+.rec-tool-count{font-size:12px;color:#64748B}
 @media(max-width:768px){.container{padding:16px}.dashboard-header{padding:24px}.agent-metrics-grid{grid-template-columns:repeat(2,1fr)}}
 .info-tooltip{position:relative;display:inline-block}
 .info-tooltip-trigger{display:inline-flex;align-items:center;justify-content:center;width:16px;height:16px;margin-left:6px;border-radius:50%;background:#E2E8F0;color:#475569;font-size:11px;font-weight:700;cursor:help;vertical-align:middle}
@@ -567,7 +596,8 @@ function heatFg(val, min, max) {
         <div class="agent-metric"><div class="agent-metric-label">#Traces</div><div class="agent-metric-value">${ad.unique_tasks}</div></div>
       </div></div>
       <div class="section-header" style="margin-top:0"><div class="section-title">Discovered Issues</div><div class="section-subtitle">Recurring problems identified in this agent's outputs <span class="info-tooltip"><span class="info-tooltip-trigger" tabindex="0" aria-label="Issue matching explanation">i</span><span class="info-tooltip-bubble">Each llm call can be matched to any number of discovered issues, or to no issues.</span></span></div></div>
-      ${tableHtml}`;
+      ${tableHtml}
+      ${renderRecommendations(ad.recommendations)}`;
     container.appendChild(sec);
   }
 })();
@@ -579,14 +609,98 @@ function toggleIssueExamples(id, rowEl){
   const indicator=rowEl.querySelector('.expand-indicator');
   if(indicator) indicator.textContent=isHidden?'▾ Examples':'▸ Examples';
 }
+
+// ─── SPARC recommendations renderer ─────────────────────────────────────
+function escHtml(s){return esc(s||'');}
+function renderDiff(raw){
+  if(!raw) return '';
+  const lines=String(raw).split('\n');
+  const out=lines.map(line=>{
+    const safe=escHtml(line);
+    if(line.startsWith('+++')||line.startsWith('---')) return `<span class="rec-line-hunk">${safe}</span>`;
+    if(line.startsWith('@@')) return `<span class="rec-line-hunk">${safe}</span>`;
+    if(line.startsWith('+')) return `<span class="rec-line-add">${safe}</span>`;
+    if(line.startsWith('-')) return `<span class="rec-line-del">${safe}</span>`;
+    return safe;
+  });
+  return `<div class="rec-diff">${out.join('\n')}</div>`;
+}
+function targetPillClass(target){
+  if(target==='system_prompt') return 'system';
+  if(target==='tool_description') return 'tool';
+  if(target==='parameter_description'||target==='parameter_examples') return 'param';
+  return '';
+}
+function renderRecCard(rec, scope){
+  const imp=(typeof rec.importance_mean==='number')?rec.importance_mean:(rec.importance||0);
+  const count=rec.count||1;
+  const cls=targetPillClass(rec.target||'system_prompt');
+  const rationale=rec.rationale?`<div class="rec-rationale">${escHtml(rec.rationale)}</div>`:'';
+  const scopeLabel=scope?`<span class="rec-scope">${escHtml(scope)}</span>`:'';
+  return `<div class="rec-card">
+    <div class="rec-card-head">
+      <span class="rec-target-pill ${cls}">${escHtml(rec.target||'')}</span>
+      ${scopeLabel}
+      <span class="rec-count">×${count}</span>
+      <span class="rec-importance">importance ${imp.toFixed(2)}</span>
+    </div>
+    ${rationale}
+    ${renderDiff(rec.diff)}
+  </div>`;
+}
+function renderRecommendations(recs){
+  if(!recs) return '';
+  const byTool=recs.by_tool||{};
+  const bySystem=recs.by_system_prompt||[];
+  const toolEntries=Object.entries(byTool);
+  if(!toolEntries.length && !bySystem.length) return '';
+
+  const header=`<div class="section-header" style="margin-top:28px">
+    <div class="section-title">SPARC Recommendations</div>
+    <div class="section-subtitle">Unified-diff suggestions derived from the call judgments (${recs.total||0} recommendation occurrence(s))</div>
+  </div>`;
+
+  let body='<div class="recs-block">';
+
+  if(bySystem.length){
+    body+='<div class="recs-subblock"><div class="recs-subblock-title">System Prompt</div>';
+    for(const r of bySystem){ body+=renderRecCard(r, null); }
+    body+='</div>';
+  }
+
+  if(toolEntries.length){
+    body+='<div class="recs-subblock"><div class="recs-subblock-title">Per-Tool Suggestions</div>';
+    // Sort tools by total recommendations descending
+    toolEntries.sort((a,b)=>{
+      const ca=a[1].reduce((s,x)=>s+(x.count||1),0);
+      const cb=b[1].reduce((s,x)=>s+(x.count||1),0);
+      return cb-ca || a[0].localeCompare(b[0]);
+    });
+    for(const [toolName, items] of toolEntries){
+      const total=items.reduce((s,x)=>s+(x.count||1),0);
+      body+=`<div class="rec-tool-header">
+        <span class="rec-tool-name">${escHtml(toolName)}</span>
+        <span class="rec-tool-count">${items.length} distinct suggestion(s), ${total} occurrence(s)</span>
+      </div>`;
+      for(const r of items){
+        const scope=r.parameter_name?`${toolName} · ${r.parameter_name}`:toolName;
+        body+=renderRecCard(r, scope);
+      }
+    }
+    body+='</div>';
+  }
+
+  body+='</div>';
+  return header+body;
+}
 </script>
 </body>
 </html>"""
 
 
-def generate_html(json_path, output_path=None, include_examples=False):
+def generate_html(json_path, output_path=None, num_examples=3):
     """Generate static HTML dashboard from a clear_results.json file."""
-    data = load_json_data(json_path, include_examples=include_examples)
+    data = load_json_data(json_path, num_examples=num_examples)
     logging.info(f"Generating Static HTML report  from {json_path}")
 
     input_path = Path(json_path)

{clear_eval-2.0.3 → clear_eval-2.0.4}/src/clear_eval/agentic/pipeline/build_json_results.py

@@ -9,10 +9,12 @@ Creates a structured JSON with all issues mapped to spans, including:
 """
 
 import ast
+import hashlib
 import json
 import logging
 import math
 import os
+import re
 from collections import defaultdict
 from datetime import datetime
 from pathlib import Path
@@ -28,6 +30,9 @@ logger = logging.getLogger(__name__)
 # Must match the subdir name used in run_clear_step_analysis.run_analysis_for_agent.
 TOOL_CALLS_SUBDIR = "tool_calls"
 
+# SPARC recommendation targets that scope to a specific tool (vs. system prompt)
+_TOOL_SCOPED_TARGETS = ("tool_description", "parameter_description", "parameter_examples")
+
 
 def build_workflow_graph(traj_df):
     """Build workflow graph data from trajectory dataframe."""
@@ -142,6 +147,126 @@ def _parse_meta_data(meta_data_str):
         return {}
 
 
+_WHITESPACE_RE = re.compile(r"\s+")
+
+
+def _normalize_diff(diff: str) -> str:
+    """Whitespace-normalize a unified diff for stable hashing/dedup."""
+    if not isinstance(diff, str):
+        return ""
+    return _WHITESPACE_RE.sub(" ", diff).strip()
+
+
+def _diff_hash(diff: str) -> str:
+    """Stable short hash of the normalized diff text for grouping."""
+    return hashlib.sha1(_normalize_diff(diff).encode("utf-8")).hexdigest()[:16]
+
+
+def _parse_recs(cell: Any) -> List[Dict[str, Any]]:
+    """Parse a ``sparc_recommendations`` cell into a list of rec dicts.
+
+    Accepts JSON arrays (the canonical form), already-parsed lists, or
+    ``"[]"``/empty/NaN. Anything else returns an empty list."""
+    if cell is None or (isinstance(cell, float) and pd.isna(cell)):
+        return []
+    if isinstance(cell, list):
+        return [r for r in cell if isinstance(r, dict)]
+    if isinstance(cell, str) and cell.strip():
+        try:
+            parsed = json.loads(cell)
+        except (json.JSONDecodeError, TypeError):
+            return []
+        if isinstance(parsed, list):
+            return [r for r in parsed if isinstance(r, dict)]
+    return []
+
+
+def _aggregate_recommendations(results_df: pd.DataFrame) -> Optional[Dict[str, Any]]:
+    """Group per-row SPARC recommendations into ``by_tool`` and
+    ``by_system_prompt`` buckets.
+
+    Returns ``None`` if the DataFrame has no ``sparc_recommendations`` column
+    or if no row produced any recommendation (runtime mode / static-only)."""
+    if "sparc_recommendations" not in results_df.columns:
+        return None
+
+    # by_tool: {tool_name: {hash: accumulator}}
+    by_tool: Dict[str, Dict[str, Dict[str, Any]]] = defaultdict(dict)
+    # by_system_prompt: {hash: accumulator}
+    by_system: Dict[str, Dict[str, Any]] = {}
+    total = 0
+
+    for _, row in results_df.iterrows():
+        for rec in _parse_recs(row.get("sparc_recommendations")):
+            target = rec.get("target")
+            diff = rec.get("diff") or ""
+            if not target or not diff:
+                continue
+            try:
+                importance = max(0.0, min(1.0, float(rec.get("importance", 0.0))))
+            except (TypeError, ValueError):
+                importance = 0.0
+            h = _diff_hash(diff)
+            entry_template = {
+                "hash": h,
+                "target": target,
+                "diff": diff,
+                "rationale": rec.get("rationale", ""),
+                "tool_name": rec.get("tool_name"),
+                "parameter_name": rec.get("parameter_name"),
+                "count": 0,
+                "importance_sum": 0.0,
+                "importance_max": 0.0,
+            }
+
+            if target == "system_prompt":
+                bucket = by_system.setdefault(h, dict(entry_template))
+            elif target in _TOOL_SCOPED_TARGETS:
+                tool_name = rec.get("tool_name") or "<unknown>"
+                bucket = by_tool[tool_name].setdefault(h, dict(entry_template))
+            else:
+                continue
+
+            bucket["count"] += 1
+            bucket["importance_sum"] += importance
+            if importance > bucket["importance_max"]:
+                bucket["importance_max"] = importance
+            total += 1
+
+    if total == 0:
+        return None
+
+    def _finalize(entries: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        out = []
+        for e in entries:
+            mean = e["importance_sum"] / e["count"] if e["count"] else 0.0
+            out.append({
+                "hash": e["hash"],
+                "target": e["target"],
+                "diff": e["diff"],
+                "rationale": e["rationale"],
+                "tool_name": e["tool_name"],
+                "parameter_name": e["parameter_name"],
+                "count": e["count"],
+                "importance_mean": round(mean, 4),
+                "importance_max": round(e["importance_max"], 4),
+            })
+        out.sort(key=lambda r: (-r["importance_mean"], -r["count"], r["target"]))
+        return out
+
+    by_tool_out: Dict[str, List[Dict[str, Any]]] = {
+        tool: _finalize(list(entries.values()))
+        for tool, entries in by_tool.items()
+    }
+    by_system_out = _finalize(list(by_system.values()))
+
+    return {
+        "by_tool": by_tool_out,
+        "by_system_prompt": by_system_out,
+        "total": total,
+    }
+
+
 def _parse_issues_list(recurring_issues_str):
     """Parse recurring issues string into a list."""
     if pd.isna(recurring_issues_str) or not recurring_issues_str:
@@ -315,6 +440,12 @@ def _process_results_dir(
 
     result["no_issues"] = no_issue_spans
 
+    # Aggregate SPARC actionable recommendations (tool-call results only carry
+    # the column; reasoning_eval CSVs don't, so this is a no-op there).
+    recs = _aggregate_recommendations(results_df)
+    if recs is not None:
+        result["recommendations"] = recs
+
     # Attach scores and weighted severity for pass/fail calculation
     result["_scores"] = [_safe_float(row.get('score', 0)) for _, row in results_df.iterrows()]
     result["_weighted_severity"] = sum(

{clear_eval-2.0.3 → clear_eval-2.0.4}/src/clear_eval/agentic/pipeline/full_traces_evaluation/run_trajectory_evaluation_pipeline.py

@@ -721,6 +721,9 @@ def main():
         config.get('run_name')
     )
 
+    # Update config with resolved run_name so downstream components use the same name
+    config['run_name'] = run_name
+
     # Convert paths
     traj_input_dir = Path(config['data_dir'])
     rubric_dir = Path(config['rubric_dir']) if config.get('rubric_dir') else None

{clear_eval-2.0.3 → clear_eval-2.0.4}/src/clear_eval/agentic/pipeline/full_traces_evaluation/trace_evaluation/base_evaluator.py

@@ -95,6 +95,7 @@ class TrajectoryEvaluator(ABC):
         overwrite: bool = False,
         max_workers: int = 7,
         max_files: int | None = None,
+        checkpoint_every: int = 0,
         full_trace_evaluation_criteria: dict | None = None,
     ):
         """
@@ -110,6 +111,7 @@ class TrajectoryEvaluator(ABC):
             max_files: Maximum number of files to process (for testing)
             full_trace_evaluation_criteria: Custom flat evaluation criteria dict
                 that replaces all default dimensions when set
+            checkpoint_every: Save intermediate results every N items (0 = disabled)
         """
         self.inference_config = inference_config
         self.traj_input_dir = Path(traj_input_dir)
@@ -118,6 +120,7 @@ class TrajectoryEvaluator(ABC):
         self.overwrite = overwrite
         self.max_workers = max_workers
         self.max_files = max_files
+        self.checkpoint_every = checkpoint_every
         self.full_trace_evaluation_criteria = full_trace_evaluation_criteria
 
         # Create results directory: output_dir/evaluation_type[/model_subdir]
@@ -543,14 +546,20 @@ class TrajectoryEvaluator(ABC):
             for entry in entries
         ]
 
+        # Caching params
+        item_ids = [entry["traj_name"] for entry in entries] if self.checkpoint_every else None
+        cache_path = str(self.results_dir / "run_cache.jsonl") if self.checkpoint_every else None
+
         # Use pipeline's parallel execution with progress bar (always async)
         start = time.time()
         parallel_results = run_parallel(
             func=self.evaluate_single,
             inputs=inputs,
-            use_async=True,
             max_workers=self.max_workers,
-            progress_desc=f"Evaluating trajectories ({self.__class__.__name__})"
+            progress_desc=f"Evaluating trajectories ({self.__class__.__name__})",
+            checkpoint_every=self.checkpoint_every,
+            checkpoint_path=cache_path,
+            item_ids=item_ids,
         )
         elapsed = time.time() - start
 

{clear_eval-2.0.3 → clear_eval-2.0.4}/src/clear_eval/agentic/pipeline/run_clear_agentic_eval.py

@@ -337,6 +337,9 @@ def main():
         config.get('run_name')
     )
 
+    # Update config with resolved run_name so downstream components use the same name
+    config['run_name'] = run_name
+
     # Extract parameters
     data_dir = Path(config['data_dir'])
     from_raw_traces = config.get('from_raw_traces')

{clear_eval-2.0.3 → clear_eval-2.0.4}/src/clear_eval/pipeline/constants.py

@@ -11,4 +11,14 @@ SHORTCOMING_LIST_FILE_PREFIX = "shortcoming_list"
 MAPPING_FILE_PREFIX = "mapping_results"
 MAPPING_NO_ISSUES = "NO_ISSUES"
 ANALYSIS_SKIPPED = "Analysis Skipped"
-DEFAULT_ISSUES_FORMAT_MODE = "shortcomings"
+DEFAULT_ISSUES_FORMAT_MODE = "shortcomings"
+
+# SPARC-specific per-row columns produced by the tool-call use case. They are
+# carried through analysis CSVs unchanged so downstream aggregation
+# (build_json_results) and dashboards can read per-row judgments without
+# re-running SPARC.
+_SPARC_COLUMNS_PASSTHROUGH = (
+    "sparc_decision",
+    "sparc_score_1_to_5",
+    "sparc_recommendations",
+)

{clear_eval-2.0.3 → clear_eval-2.0.4}/src/clear_eval/pipeline/eval_utils.py

@@ -5,13 +5,15 @@ import numpy as np
 from clear_eval.pipeline.caching_utils import save_dataframe_to_cache
 import pandas as pd
 from clear_eval.pipeline.constants import IDENTIFIED_SHORTCOMING_COL, EVALUATION_TEXT_COL, EVALUATION_SUMMARY_COL, \
-    SHORTCOMING_PREFIX, SCORE_COL, ERROR_COL, MAPPING_NO_ISSUES, ANALYSIS_SKIPPED, DEFAULT_ISSUES_FORMAT_MODE
+    SHORTCOMING_PREFIX, SCORE_COL, ERROR_COL, MAPPING_NO_ISSUES, ANALYSIS_SKIPPED, DEFAULT_ISSUES_FORMAT_MODE, \
+    _SPARC_COLUMNS_PASSTHROUGH
 from clear_eval.pipeline.propmts import get_summarization_prompt, \
      get_shortcomings_clustering_prompt, get_issues_mapping_system_prompt, get_issues_mapping_human_prompt, get_synthesis_prompt, get_synthesis_prompt_cont
 import re
 from clear_eval.pipeline.inference_utils.llm_client import run_parallel, run_async, get_llm_client
 logger = logging.getLogger(__name__)
 
+
 def is_missing_or_error(eval_text):
     if pd.isna(eval_text) or not eval_text or not eval_text.strip() or \
         eval_text.startswith(ANALYSIS_SKIPPED) or eval_text.startswith("Error:"):
@@ -32,6 +34,16 @@ async def evaluate_row(row, config, llm, generate_evaluation_model_prompt_func):
     return parse_evaluation_response(content)
 
 
+def _get_checkpoint_cache_path(config, stage_suffix):
+    """Build a cache file path for run_parallel checkpointing."""
+    checkpoint_path = config.get('checkpoint_path', '')
+    if checkpoint_path:
+        # Derive cache path from the main checkpoint CSV path
+        return checkpoint_path.replace('.csv', f'_cache_{stage_suffix}.jsonl')
+    output_dir = config.get('output_dir', '.')
+    return os.path.join(output_dir, f"cache_{stage_suffix}.jsonl")
+
+
 def evaluate_single_records(df, llm, config, get_evaluation_prompt_func, score_col=SCORE_COL):
     """Evaluates predictions and adds scores."""
     logger.info(f"\n--- Evaluating Predictions ---")
@@ -52,13 +64,19 @@ def evaluate_single_records(df, llm, config, get_evaluation_prompt_func, score_c
     for idx, row in df.iterrows():
         inputs.append((row, config, llm, get_evaluation_prompt_func))
 
+    checkpoint_every = config.get('checkpoint_every', 0)
+    item_ids = [str(row[config['qid_column']]) for _, row in df.iterrows()] if checkpoint_every else None
+    cache_path = _get_checkpoint_cache_path(config, "eval") if checkpoint_every else None
+
     results = run_parallel(
         func=evaluate_row,
         inputs=inputs,
-        use_async=True,
         max_workers=config['max_workers'],
         error_prefix="Evaluation: ",
-        progress_desc="Evaluating predictions"
+        progress_desc="Evaluating predictions",
+        checkpoint_every=checkpoint_every,
+        checkpoint_path=cache_path,
+        item_ids=item_ids,
     )
 
     for i, result in enumerate(results):
@@ -83,19 +101,25 @@ def produce_summaries_per_record(df, llm, config):
 
     logger.info(f"Generating evaluation summaries for {len(inputs)} items ...")
 
+    df[EVALUATION_SUMMARY_COL] = ""
+    if ERROR_COL not in df.columns:
+        df[ERROR_COL] = [[] for _ in range(len(df))]
+
+    checkpoint_every = config.get('checkpoint_every', 0)
+    item_ids = [str(row[config['qid_column']]) for _, row in df.iterrows()] if checkpoint_every else None
+    cache_path = _get_checkpoint_cache_path(config, "summary") if checkpoint_every else None
+
     results = run_parallel(
         func=generate_evaluation_summary,
         inputs=inputs,
-        use_async=True,
         max_workers=config['max_workers'],
         error_prefix="Summary: ",
-        progress_desc="Generating evaluation summaries"
+        progress_desc="Generating evaluation summaries",
+        checkpoint_every=checkpoint_every,
+        checkpoint_path=cache_path,
+        item_ids=item_ids,
     )
 
-    df[EVALUATION_SUMMARY_COL] = ""
-    if ERROR_COL not in df.columns:
-        df[ERROR_COL] = [[] for _ in range(len(df))]
-
     for i, result in enumerate(results):
         if result.is_success:
             df.at[df.index[i], EVALUATION_SUMMARY_COL] = result.result
@@ -383,7 +407,9 @@ async def analyze_shortcoming_row(eval_text, question_id, shortcomings_list, llm
 
 
 def map_shortcomings_to_records(df, llm, shortcomings_list,
-                                use_full_text, qid_col, max_workers, high_score_threshold, score_col = SCORE_COL, format_mode=DEFAULT_ISSUES_FORMAT_MODE):
+                                use_full_text, qid_col, max_workers, high_score_threshold,
+                                score_col = SCORE_COL, format_mode=DEFAULT_ISSUES_FORMAT_MODE,
+                                checkpoint_every = 0, cache_path = None):
     """Analyzes evaluation text for the dynamically generated shortcomings."""
     logger.info(f"\n--- Analyzing Shortcomings based on Synthesized List ---")
     df[IDENTIFIED_SHORTCOMING_COL] = ""
@@ -420,13 +446,17 @@ def map_shortcomings_to_records(df, llm, shortcomings_list,
             inputs.append((str(row[evaluation_text_col]), row.get(qid_col, f"row_{idx}"), shortcomings_list, llm, system_prompt, format_mode))
     logger.info(f"Mapping {n_records_to_map}/{len(df)} records to {len(shortcomings_list)} discovered shortcomings.")
 
+    item_ids = [row.get(qid_col, f"row_{idx}") for idx, row in df.iterrows()] if checkpoint_every else None
+
     results = run_parallel(
         func=analyze_shortcoming_row,
         inputs=inputs,
-        use_async=True,
         max_workers=max_workers,
         error_prefix="Mapping: ",
-        progress_desc="Analyzing shortcomings"
+        progress_desc="Analyzing shortcomings",
+        checkpoint_every = checkpoint_every,
+        checkpoint_path = cache_path,
+        item_ids = item_ids,
     )
 
     for i, result in enumerate(results):
@@ -509,13 +539,19 @@ def generate_model_predictions(df, llm, config):
     for i, row in df.iterrows():
         inputs.append((llm, row[config['model_input_column']], row[config['qid_column']]))
 
+    checkpoint_every = config.get('checkpoint_every', 0)
+    item_ids = [str(row[config['qid_column']]) for _, row in df.iterrows()] if checkpoint_every else None
+    cache_path = _get_checkpoint_cache_path(config, "gen") if checkpoint_every else None
+
     results = run_parallel(
         func=predict_row,
         inputs=inputs,
-        use_async=True,
         max_workers=config["max_workers"],
         error_prefix="Prediction: ",
-        progress_desc="Generating predictions"
+        progress_desc="Generating predictions",
+        checkpoint_every=checkpoint_every,
+        checkpoint_path=cache_path,
+        item_ids=item_ids,
     )
 
     for i, result in enumerate(results):
@@ -606,10 +642,17 @@ def convert_results_to_ui_input(df, config, task_data):
         df.loc[:, "recurring_issues_str"] = df.apply(lambda r: get_recurring_issues_list(r), axis=1)
         custom_output_df["recurring_issues_str"] = df["recurring_issues_str"]
 
+        # Carry SPARC per-row columns through unchanged. Aggregation happens
+        # in build_json_results._process_results_dir, which reads these.
+        for col in _SPARC_COLUMNS_PASSTHROUGH:
+            if col in df.columns:
+                custom_output_df[col] = df[col]
+
         required_cols =[config.get(r, r) for r in required_input_fields] + config.get("input_columns", []) + \
                          ["question_id", 'model_input', 'response',
                          'score', 'evaluation_text', 'evaluation_summary',
-                         'recurring_issues', 'recurring_issues_str', 'ground_truth', 'error']
+                         'recurring_issues', 'recurring_issues_str', 'ground_truth', 'error'] + \
+                         [c for c in _SPARC_COLUMNS_PASSTHROUGH if c in df.columns]
         required_cols = list(dict.fromkeys(required_cols))
 
         for col in required_cols:

{clear_eval-2.0.3 → clear_eval-2.0.4}/src/clear_eval/pipeline/full_pipeline.py

@@ -16,10 +16,11 @@ from clear_eval.pipeline.constants import (GENERATION_FILE_PREFIX, SHORTCOMING_L
 from clear_eval.pipeline.caching_utils import load_dataframe_from_cache, save_dataframe_to_cache, save_json_to_cache, \
     ensure_dir, \
     load_json_from_cache, resolve_data_path
-from clear_eval.pipeline.eval_utils import map_shortcomings_to_records, get_model_name_for_file, convert_results_to_ui_input, \
+from clear_eval.pipeline.eval_utils import map_shortcomings_to_records, get_model_name_for_file, \
+    convert_results_to_ui_input, \
     load_inputs, synthesize_shortcomings_from_df, \
     remove_duplicates_shortcomings, run_predictions_generation_save_results, produce_summaries_per_record, \
-    generate_model_predictions
+    generate_model_predictions, _get_checkpoint_cache_path
 from clear_eval.pipeline.inference_utils.llm_client import get_llm_client
 from clear_eval.pipeline.config_loader import load_yaml
 
@@ -101,6 +102,7 @@ def convert_to_ui_format(mapped_data_df, output_dir, config, file_name_info):
     logger.info(f"Custom formatted analysis results saved to {output_path}")
     save_ui_input_results(output_df, output_path, config)
 
+
 def save_ui_input_results(output_df, output_path, config):
     parquet_bytes = get_parquet_bytes(output_df)
     json_bytes = json.dumps(config, indent=2).encode()
@@ -223,9 +225,13 @@ def resolve_issues_and_map(df, config, eval_llm, resume_enabled, checkpoint_path
     use_full_text = config['use_full_text_for_analysis']
     qid_col = config['qid_column']
     max_workers = config['max_workers']
+    checkpoint_every = config.get("checkpoint_every", 0)
+    cache_path =_get_checkpoint_cache_path(config, "map") if checkpoint_every else None
+
     high_score_threshold = config.get("high_score_threshold", 1)
     df = map_shortcomings_to_records(df, eval_llm, shortcoming_list, use_full_text,
-                                     qid_col, max_workers, high_score_threshold, format_mode=format_mode)
+                                     qid_col, max_workers, high_score_threshold, format_mode=format_mode,
+                                     checkpoint_every = checkpoint_every, cache_path=cache_path)
     save_dataframe_to_cache(df, checkpoint_path)
     return df
 
@@ -240,6 +246,7 @@ def run_aggregation_from_df(config, df, file_name_info, eval_llm=None):
     resume_enabled = config['resume_enabled']
     format_mode = get_issues_format(config)
     checkpoint_path = f"{output_dir}/{CHECKPOINT_FILE_PREFIX}_{file_name_info}.csv"
+    config['checkpoint_path'] = checkpoint_path
     shortcoming_list_output_path = f"{output_dir}/{SHORTCOMING_LIST_FILE_PREFIX}_{file_name_info}.json"
     deduplicated_shortcomings_list_output_path = f"{output_dir}/{SHORTCOMING_LIST_FILE_PREFIX}_{file_name_info}_dedup.json"
     zip_path = f"{output_dir}/analysis_results_{file_name_info}.zip"
@@ -270,6 +277,7 @@ def run_eval_pipeline(config):
 
     generate_issues = config.get("generate_issues", True)
     checkpoint_path = f"{output_dir}/{CHECKPOINT_FILE_PREFIX}_{run_info}.csv"
+    config['checkpoint_path'] = checkpoint_path
     zip_path = f"{output_dir}/analysis_results_{run_info}.zip"
 
     # If final output exists, nothing to do