clear-eval 2.0.2__tar.gz → 2.0.4__tar.gz

{clear_eval-2.0.2 → clear_eval-2.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clear_eval
-Version: 2.0.2
+Version: 2.0.4
 Summary: A python API sdk facilitating Error Analysis via LLM-as-a-Judge
 Author-email: Lilach Eden <lilache@il.ibm.com>, Asaf Yehudai <asaf.yehudai@ibm.com>
 License: Apache-2.0

{clear_eval-2.0.2 → clear_eval-2.0.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "clear_eval"
-version = "2.0.2"
+version = "2.0.4"
 description = "A python API sdk facilitating Error Analysis via LLM-as-a-Judge"
 readme = "README.md"
 authors = [{ name = "Lilach Eden", email = "lilache@il.ibm.com"},

{clear_eval-2.0.2 → clear_eval-2.0.4}/src/clear_eval/agentic/dashboard/agentic_workflow_dashboard.py

@@ -2174,7 +2174,8 @@ def main_page():
                 # Try to get dimensions from parsed_evaluation first (preferred structure)
                 step_quality = parsed_eval.get("step_quality_dimensions", {})
                 trajectory_dims = parsed_eval.get("trajectory_dimensions", {})
-                
+                custom_dims = parsed_eval.get("dimensions", {})
+
                 # Collect step quality dimension scores
                 for criterion, details in step_quality.items():
                     score = details.get("score")
@@ -2183,7 +2184,7 @@ def main_page():
                             all_dimension_scores[f"Step Quality: {criterion}"].append(float(score))
                         except (ValueError, TypeError):
                             print(f"⚠️  Warning: Could not convert score to float for {criterion}: {score}")
-                
+
                 # Collect trace dimension scores
                 for criterion, details in trajectory_dims.items():
                     score = details.get("score")
@@ -2192,9 +2193,19 @@ def main_page():
                             all_dimension_scores[f"Trace: {criterion}"].append(float(score))
                         except (ValueError, TypeError):
                             print(f"⚠️  Warning: Could not convert score to float for {criterion}: {score}")
-                
+
+                # Collect custom flat dimension scores
+                if custom_dims and not step_quality and not trajectory_dims:
+                    for criterion, details in custom_dims.items():
+                        score = details.get("score")
+                        if score is not None:
+                            try:
+                                all_dimension_scores[criterion].append(float(score))
+                            except (ValueError, TypeError):
+                                print(f"⚠️  Warning: Could not convert score to float for {criterion}: {score}")
+
                 # Fallback: if no dimensions found in parsed_evaluation, try dimension_scores directly
-                if not step_quality and not trajectory_dims and dimension_scores:
+                if not step_quality and not trajectory_dims and not custom_dims and dimension_scores:
                     for criterion, score in dimension_scores.items():
                         if score is not None:
                             try:
@@ -3083,7 +3094,8 @@ def main_page():
                 # Separate step quality and trace dimensions
                 step_quality = parsed_eval.get("step_quality_dimensions", {})
                 trajectory_dims = parsed_eval.get("trajectory_dimensions", {})
-                
+                custom_dims = parsed_eval.get("dimensions", {})
+
                 if step_quality:
                     ui.html('<div style="font-weight: 600; color: #1E293B; margin: 12px 0 8px 0; font-size: 15px;">Step Quality Dimensions</div>')
                     for criterion, details in step_quality.items():
@@ -3091,7 +3103,7 @@ def main_page():
                         justification = details.get("justification", "No justification provided")
                         score_color = score_to_hex(score)
                         definition = get_dimension_definition(criterion)
-                        
+
                         with ui.row().classes("w-full items-center gap-2 mb-2"):
                             with ui.expansion(f"{criterion}: {score:.2f}", icon="check_circle").classes("flex-1").style(f"border-left: 4px solid {score_color};"):
                                 ui.html(f"""
@@ -3101,7 +3113,7 @@ def main_page():
                                 """)
                             with ui.button(icon="info", on_click=lambda: None).props("flat dense round size=sm").classes("text-blue-500"):
                                 ui.tooltip(definition).classes("bg-slate-800 text-white text-sm max-w-md")
-                
+
                 if trajectory_dims:
                     ui.html('<div style="font-weight: 600; color: #1E293B; margin: 20px 0 8px 0; font-size: 15px;">Trace-Level Dimensions</div>')
                     for criterion, details in trajectory_dims.items():
@@ -3109,7 +3121,7 @@ def main_page():
                         justification = details.get("justification", "No justification provided")
                         score_color = score_to_hex(score)
                         definition = get_dimension_definition(criterion)
-                        
+
                         with ui.row().classes("w-full items-center gap-2 mb-2"):
                             with ui.expansion(f"{criterion}: {score:.2f}", icon="check_circle").classes("flex-1").style(f"border-left: 4px solid {score_color};"):
                                 ui.html(f"""
@@ -3120,6 +3132,21 @@ def main_page():
                             with ui.button(icon="info", on_click=lambda: None).props("flat dense round size=sm").classes("text-blue-500"):
                                 ui.tooltip(definition).classes("bg-slate-800 text-white text-sm max-w-md")
 
+                if custom_dims and not step_quality and not trajectory_dims:
+                    ui.html('<div style="font-weight: 600; color: #1E293B; margin: 12px 0 8px 0; font-size: 15px;">Evaluation Dimensions</div>')
+                    for criterion, details in custom_dims.items():
+                        score = float(details.get("score", 0))
+                        justification = details.get("justification", "No justification provided")
+                        score_color = score_to_hex(score)
+
+                        with ui.row().classes("w-full items-center gap-2 mb-2"):
+                            with ui.expansion(f"{criterion}: {score:.2f}", icon="check_circle").classes("flex-1").style(f"border-left: 4px solid {score_color};"):
+                                ui.html(f"""
+                                    <div style="padding: 8px; background: #FAFAFA; border-radius: 6px; color: #475569; font-size: 13px; line-height: 1.5;">
+                                        {justification}
+                                    </div>
+                                """)
+
     def render_rubric_evaluation(rubric_eval):
         """Render the rubric-based trace evaluation."""
         # Overall score card

{clear_eval-2.0.2 → clear_eval-2.0.4}/src/clear_eval/agentic/dashboard/generate_static_dashboard.py

@@ -42,7 +42,7 @@ def _maybe_parse_json_text(value):
     return value
 
 
-def load_json_data(json_path, include_examples=True):
+def load_json_data(json_path, num_examples=3):
     """Load all dashboard data from clear_results.json and map it to the HTML data shape."""
     json_path = Path(json_path)
 
@@ -79,8 +79,8 @@ def load_json_data(json_path, include_examples=True):
             for issue in issues:
                 issue_text = issue.get("issue_text") or issue.get("issue_id") or ""
                 examples = []
-                if include_examples:
-                    for occurrence in issue.get("occurrences", [])[:3]:
+                if num_examples:
+                    for occurrence in issue.get("occurrences", [])[:num_examples]:
                         tasks_list.add(occurrence.get("trace_id"))
                         span_reference = occurrence.get("span_reference", {})
                         input_output_pair = occurrence.get("input_output_pair", {})
@@ -121,13 +121,22 @@ def load_json_data(json_path, include_examples=True):
                 key=lambda row: -row["count"],
             ) + [row for row in issues_table if row["is_no_issue"]]
 
-            agents_data[agent_label] = {
+            entry = {
                 "total_evals": int(summary.get("total_interactions", 0)),
                 "avg_score": round(float(summary["avg_score"]), 2) if summary.get("avg_score") is not None else None,
                 "unique_issues": len(issues),
                 "unique_tasks": node_stats.get(agent_name, {}).get("unique_tasks", 0),
                 "issues_table": issues_table,
+                "recommendations": None,
             }
+            if agent_type == "tools":
+                # Recommendations now travel inside the per-agent payload
+                # produced by build_json_results._aggregate_recommendations,
+                # so the dashboard just reads the key off the JSON.
+                recs = agent_type_payload.get("recommendations")
+                if recs and (recs.get("by_tool") or recs.get("by_system_prompt")):
+                    entry["recommendations"] = recs
+            agents_data[agent_label] = entry
 
     return {
         "unique_tasks": int(stats.get("total_traces", 0)),
@@ -234,6 +243,26 @@ body{font-family:'Inter',-apple-system,BlinkMacSystemFont,sans-serif;background:
 .chat-message-content{font-size:14px;color:#1E293B;white-space:pre-wrap;word-break:break-word;line-height:1.6}
 .chat-message-tools{margin-top:10px;padding-top:10px;border-top:1px dashed #CBD5E1}
 .expand-indicator{display:block;margin-top:8px;color:#6366F1;font-weight:800;font-size:13px}
+.recs-block{margin-top:24px}
+.recs-subblock{margin-top:14px}
+.recs-subblock-title{font-size:14px;font-weight:800;color:#4338CA;text-transform:uppercase;letter-spacing:.08em;margin-bottom:10px}
+.rec-card{background:#fff;border:1px solid #D8E1EE;border-radius:12px;padding:14px 16px;margin-bottom:12px;box-shadow:0 1px 3px rgba(15,23,42,.04)}
+.rec-card-head{display:flex;align-items:center;gap:10px;flex-wrap:wrap;margin-bottom:8px}
+.rec-target-pill{font-size:11px;font-weight:700;text-transform:uppercase;letter-spacing:.06em;padding:2px 8px;border-radius:999px;background:#EEF2FF;color:#4338CA}
+.rec-target-pill.system{background:#DBEAFE;color:#1D4ED8}
+.rec-target-pill.tool{background:#DCFCE7;color:#166534}
+.rec-target-pill.param{background:#FEF3C7;color:#92400E}
+.rec-scope{font-size:13px;color:#334155;font-weight:600}
+.rec-importance{margin-left:auto;font-size:12px;font-weight:700;color:#6366F1}
+.rec-count{font-size:12px;color:#64748B}
+.rec-rationale{font-size:13px;color:#475569;line-height:1.6;margin-bottom:10px}
+.rec-diff{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;font-size:12.5px;background:#0F172A;color:#E2E8F0;padding:12px 14px;border-radius:10px;overflow:auto;max-height:260px;white-space:pre;line-height:1.5}
+.rec-diff .rec-line-add{color:#86EFAC}
+.rec-diff .rec-line-del{color:#FCA5A5}
+.rec-diff .rec-line-hunk{color:#93C5FD}
+.rec-tool-header{display:flex;align-items:baseline;gap:12px;margin:14px 0 6px}
+.rec-tool-name{font-size:15px;font-weight:800;color:#0F172A}
+.rec-tool-count{font-size:12px;color:#64748B}
 @media(max-width:768px){.container{padding:16px}.dashboard-header{padding:24px}.agent-metrics-grid{grid-template-columns:repeat(2,1fr)}}
 .info-tooltip{position:relative;display:inline-block}
 .info-tooltip-trigger{display:inline-flex;align-items:center;justify-content:center;width:16px;height:16px;margin-left:6px;border-radius:50%;background:#E2E8F0;color:#475569;font-size:11px;font-weight:700;cursor:help;vertical-align:middle}
@@ -567,7 +596,8 @@ function heatFg(val, min, max) {
         <div class="agent-metric"><div class="agent-metric-label">#Traces</div><div class="agent-metric-value">${ad.unique_tasks}</div></div>
       </div></div>
       <div class="section-header" style="margin-top:0"><div class="section-title">Discovered Issues</div><div class="section-subtitle">Recurring problems identified in this agent's outputs <span class="info-tooltip"><span class="info-tooltip-trigger" tabindex="0" aria-label="Issue matching explanation">i</span><span class="info-tooltip-bubble">Each llm call can be matched to any number of discovered issues, or to no issues.</span></span></div></div>
-      ${tableHtml}`;
+      ${tableHtml}
+      ${renderRecommendations(ad.recommendations)}`;
     container.appendChild(sec);
   }
 })();
@@ -579,14 +609,98 @@ function toggleIssueExamples(id, rowEl){
   const indicator=rowEl.querySelector('.expand-indicator');
   if(indicator) indicator.textContent=isHidden?'▾ Examples':'▸ Examples';
 }
+
+// ─── SPARC recommendations renderer ─────────────────────────────────────
+function escHtml(s){return esc(s||'');}
+function renderDiff(raw){
+  if(!raw) return '';
+  const lines=String(raw).split('\n');
+  const out=lines.map(line=>{
+    const safe=escHtml(line);
+    if(line.startsWith('+++')||line.startsWith('---')) return `<span class="rec-line-hunk">${safe}</span>`;
+    if(line.startsWith('@@')) return `<span class="rec-line-hunk">${safe}</span>`;
+    if(line.startsWith('+')) return `<span class="rec-line-add">${safe}</span>`;
+    if(line.startsWith('-')) return `<span class="rec-line-del">${safe}</span>`;
+    return safe;
+  });
+  return `<div class="rec-diff">${out.join('\n')}</div>`;
+}
+function targetPillClass(target){
+  if(target==='system_prompt') return 'system';
+  if(target==='tool_description') return 'tool';
+  if(target==='parameter_description'||target==='parameter_examples') return 'param';
+  return '';
+}
+function renderRecCard(rec, scope){
+  const imp=(typeof rec.importance_mean==='number')?rec.importance_mean:(rec.importance||0);
+  const count=rec.count||1;
+  const cls=targetPillClass(rec.target||'system_prompt');
+  const rationale=rec.rationale?`<div class="rec-rationale">${escHtml(rec.rationale)}</div>`:'';
+  const scopeLabel=scope?`<span class="rec-scope">${escHtml(scope)}</span>`:'';
+  return `<div class="rec-card">
+    <div class="rec-card-head">
+      <span class="rec-target-pill ${cls}">${escHtml(rec.target||'')}</span>
+      ${scopeLabel}
+      <span class="rec-count">×${count}</span>
+      <span class="rec-importance">importance ${imp.toFixed(2)}</span>
+    </div>
+    ${rationale}
+    ${renderDiff(rec.diff)}
+  </div>`;
+}
+function renderRecommendations(recs){
+  if(!recs) return '';
+  const byTool=recs.by_tool||{};
+  const bySystem=recs.by_system_prompt||[];
+  const toolEntries=Object.entries(byTool);
+  if(!toolEntries.length && !bySystem.length) return '';
+
+  const header=`<div class="section-header" style="margin-top:28px">
+    <div class="section-title">SPARC Recommendations</div>
+    <div class="section-subtitle">Unified-diff suggestions derived from the call judgments (${recs.total||0} recommendation occurrence(s))</div>
+  </div>`;
+
+  let body='<div class="recs-block">';
+
+  if(bySystem.length){
+    body+='<div class="recs-subblock"><div class="recs-subblock-title">System Prompt</div>';
+    for(const r of bySystem){ body+=renderRecCard(r, null); }
+    body+='</div>';
+  }
+
+  if(toolEntries.length){
+    body+='<div class="recs-subblock"><div class="recs-subblock-title">Per-Tool Suggestions</div>';
+    // Sort tools by total recommendations descending
+    toolEntries.sort((a,b)=>{
+      const ca=a[1].reduce((s,x)=>s+(x.count||1),0);
+      const cb=b[1].reduce((s,x)=>s+(x.count||1),0);
+      return cb-ca || a[0].localeCompare(b[0]);
+    });
+    for(const [toolName, items] of toolEntries){
+      const total=items.reduce((s,x)=>s+(x.count||1),0);
+      body+=`<div class="rec-tool-header">
+        <span class="rec-tool-name">${escHtml(toolName)}</span>
+        <span class="rec-tool-count">${items.length} distinct suggestion(s), ${total} occurrence(s)</span>
+      </div>`;
+      for(const r of items){
+        const scope=r.parameter_name?`${toolName} · ${r.parameter_name}`:toolName;
+        body+=renderRecCard(r, scope);
+      }
+    }
+    body+='</div>';
+  }
+
+  body+='</div>';
+  return header+body;
+}
 </script>
 </body>
 </html>"""
 
 
-def generate_html(json_path, output_path=None, include_examples=False):
+def generate_html(json_path, output_path=None, num_examples=3):
     """Generate static HTML dashboard from a clear_results.json file."""
-    data = load_json_data(json_path, include_examples=include_examples)
+    data = load_json_data(json_path, num_examples=num_examples)
     logging.info(f"Generating Static HTML report  from {json_path}")
 
     input_path = Path(json_path)

{clear_eval-2.0.2 → clear_eval-2.0.4}/src/clear_eval/agentic/pipeline/argument_definitions.py

@@ -10,7 +10,7 @@ reusable argument groups that can be composed based on each script's needs.
 """
 
 import argparse
-from clear_eval.args import str2bool
+from clear_eval.args import str2bool, parse_dict
 
 
 def add_agentic_pipeline_args(parser: argparse.ArgumentParser) -> None:
@@ -132,6 +132,14 @@ def add_full_trajectory_args(parser: argparse.ArgumentParser) -> None:
         type=int,
         help="Model context window size (for full trajectory)"
     )
+    group.add_argument(
+        "--full-trace-evaluation-criteria",
+        type=parse_dict,
+        default=None,
+        help="JSON dict of evaluation criteria for full trace evaluation. "
+             "Replaces all default dimensions when set. "
+             "Example: '{\"Correctness\": \"Accurate results\", \"Efficiency\": \"No redundant steps\"}'"
+    )
     # Note: --max-files is defined in add_agentic_pipeline_args() (shared across pipelines)
     # Do not add it here to avoid conflicts
 

{clear_eval-2.0.2 → clear_eval-2.0.4}/src/clear_eval/agentic/pipeline/build_json_results.py

@@ -9,10 +9,12 @@ Creates a structured JSON with all issues mapped to spans, including:
 """
 
 import ast
+import hashlib
 import json
 import logging
 import math
 import os
+import re
 from collections import defaultdict
 from datetime import datetime
 from pathlib import Path
@@ -28,6 +30,9 @@ logger = logging.getLogger(__name__)
 # Must match the subdir name used in run_clear_step_analysis.run_analysis_for_agent.
 TOOL_CALLS_SUBDIR = "tool_calls"
 
+# SPARC recommendation targets that scope to a specific tool (vs. system prompt)
+_TOOL_SCOPED_TARGETS = ("tool_description", "parameter_description", "parameter_examples")
+
 
 def build_workflow_graph(traj_df):
     """Build workflow graph data from trajectory dataframe."""
@@ -142,6 +147,126 @@ def _parse_meta_data(meta_data_str):
         return {}
 
 
+_WHITESPACE_RE = re.compile(r"\s+")
+
+
+def _normalize_diff(diff: str) -> str:
+    """Whitespace-normalize a unified diff for stable hashing/dedup."""
+    if not isinstance(diff, str):
+        return ""
+    return _WHITESPACE_RE.sub(" ", diff).strip()
+
+
+def _diff_hash(diff: str) -> str:
+    """Stable short hash of the normalized diff text for grouping."""
+    return hashlib.sha1(_normalize_diff(diff).encode("utf-8")).hexdigest()[:16]
+
+
+def _parse_recs(cell: Any) -> List[Dict[str, Any]]:
+    """Parse a ``sparc_recommendations`` cell into a list of rec dicts.
+
+    Accepts JSON arrays (the canonical form), already-parsed lists, or
+    ``"[]"``/empty/NaN. Anything else returns an empty list."""
+    if cell is None or (isinstance(cell, float) and pd.isna(cell)):
+        return []
+    if isinstance(cell, list):
+        return [r for r in cell if isinstance(r, dict)]
+    if isinstance(cell, str) and cell.strip():
+        try:
+            parsed = json.loads(cell)
+        except (json.JSONDecodeError, TypeError):
+            return []
+        if isinstance(parsed, list):
+            return [r for r in parsed if isinstance(r, dict)]
+    return []
+
+
+def _aggregate_recommendations(results_df: pd.DataFrame) -> Optional[Dict[str, Any]]:
+    """Group per-row SPARC recommendations into ``by_tool`` and
+    ``by_system_prompt`` buckets.
+
+    Returns ``None`` if the DataFrame has no ``sparc_recommendations`` column
+    or if no row produced any recommendation (runtime mode / static-only)."""
+    if "sparc_recommendations" not in results_df.columns:
+        return None
+
+    # by_tool: {tool_name: {hash: accumulator}}
+    by_tool: Dict[str, Dict[str, Dict[str, Any]]] = defaultdict(dict)
+    # by_system_prompt: {hash: accumulator}
+    by_system: Dict[str, Dict[str, Any]] = {}
+    total = 0
+
+    for _, row in results_df.iterrows():
+        for rec in _parse_recs(row.get("sparc_recommendations")):
+            target = rec.get("target")
+            diff = rec.get("diff") or ""
+            if not target or not diff:
+                continue
+            try:
+                importance = max(0.0, min(1.0, float(rec.get("importance", 0.0))))
+            except (TypeError, ValueError):
+                importance = 0.0
+            h = _diff_hash(diff)
+            entry_template = {
+                "hash": h,
+                "target": target,
+                "diff": diff,
+                "rationale": rec.get("rationale", ""),
+                "tool_name": rec.get("tool_name"),
+                "parameter_name": rec.get("parameter_name"),
+                "count": 0,
+                "importance_sum": 0.0,
+                "importance_max": 0.0,
+            }
+
+            if target == "system_prompt":
+                bucket = by_system.setdefault(h, dict(entry_template))
+            elif target in _TOOL_SCOPED_TARGETS:
+                tool_name = rec.get("tool_name") or "<unknown>"
+                bucket = by_tool[tool_name].setdefault(h, dict(entry_template))
+            else:
+                continue
+
+            bucket["count"] += 1
+            bucket["importance_sum"] += importance
+            if importance > bucket["importance_max"]:
+                bucket["importance_max"] = importance
+            total += 1
+
+    if total == 0:
+        return None
+
+    def _finalize(entries: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        out = []
+        for e in entries:
+            mean = e["importance_sum"] / e["count"] if e["count"] else 0.0
+            out.append({
+                "hash": e["hash"],
+                "target": e["target"],
+                "diff": e["diff"],
+                "rationale": e["rationale"],
+                "tool_name": e["tool_name"],
+                "parameter_name": e["parameter_name"],
+                "count": e["count"],
+                "importance_mean": round(mean, 4),
+                "importance_max": round(e["importance_max"], 4),
+            })
+        out.sort(key=lambda r: (-r["importance_mean"], -r["count"], r["target"]))
+        return out
+
+    by_tool_out: Dict[str, List[Dict[str, Any]]] = {
+        tool: _finalize(list(entries.values()))
+        for tool, entries in by_tool.items()
+    }
+    by_system_out = _finalize(list(by_system.values()))
+
+    return {
+        "by_tool": by_tool_out,
+        "by_system_prompt": by_system_out,
+        "total": total,
+    }
+
+
 def _parse_issues_list(recurring_issues_str):
     """Parse recurring issues string into a list."""
     if pd.isna(recurring_issues_str) or not recurring_issues_str:
@@ -315,6 +440,12 @@ def _process_results_dir(
 
     result["no_issues"] = no_issue_spans
 
+    # Aggregate SPARC actionable recommendations (tool-call results only carry
+    # the column; reasoning_eval CSVs don't, so this is a no-op there).
+    recs = _aggregate_recommendations(results_df)
+    if recs is not None:
+        result["recommendations"] = recs
+
     # Attach scores and weighted severity for pass/fail calculation
     result["_scores"] = [_safe_float(row.get('score', 0)) for _, row in results_df.iterrows()]
     result["_weighted_severity"] = sum(

{clear_eval-2.0.2 → clear_eval-2.0.4}/src/clear_eval/agentic/pipeline/full_traces_evaluation/clear_analysis/base_clear_runner.py

@@ -44,6 +44,7 @@ class BaseClearRunner(ABC):
         output_dir: Path,
         inference_config: InferenceConfig,
         overwrite: bool = False,
+        predefined_issues: list = None,
     ):
         """
         Initialize CLEAR runner.
@@ -53,11 +54,13 @@ class BaseClearRunner(ABC):
             output_dir: Base directory for CLEAR analysis outputs
             inference_config: LLM inference configuration
             overwrite: Whether to overwrite existing CLEAR results
+            predefined_issues: Predefined issues list to skip issue discovery
         """
         self.eval_results_dir = Path(eval_results_dir)
         self.output_dir = Path(output_dir)
         self.inference_config = inference_config
         self.overwrite = overwrite
+        self.predefined_issues = predefined_issues
 
     @abstractmethod
     def get_source_name(self) -> str:
@@ -219,6 +222,9 @@ class BaseClearRunner(ABC):
                 "endpoint_url": self.inference_config.endpoint_url,
             }
 
+            if self.predefined_issues:
+                analysis_kwargs["predefined_issues"] = self.predefined_issues
+
             run_clear_eval_aggregation(**analysis_kwargs)
             logger.info(f"Completed: {group_key}")
             return True

{clear_eval-2.0.2 → clear_eval-2.0.4}/src/clear_eval/agentic/pipeline/full_traces_evaluation/run_trajectory_evaluation_pipeline.py

@@ -99,6 +99,9 @@ Arguments:
         --overwrite: Re-run even if results exist
         --max-workers: Number of parallel workers (default: 7)
         --max-files: Limit files to process (for testing)
+        --full-trace-evaluation-criteria: JSON dict of custom evaluation criteria
+                                         Replaces all default dimensions when set
+        --predefined-issues: Predefined issues list for CLEAR analysis
 
 Output Structure:
     output_dir/
@@ -273,6 +276,7 @@ def run_full_trajectory_evaluation(
     overwrite: bool,
     max_workers: int,
     max_files: Optional[int],
+    full_trace_evaluation_criteria: Optional[dict] = None,
 ) -> bool:
     """Run full trajectory evaluation."""
     logger.info("=" * 80)
@@ -288,6 +292,7 @@ def run_full_trajectory_evaluation(
             overwrite=overwrite,
             max_workers=max_workers,
             max_files=max_files,
+            full_trace_evaluation_criteria=full_trace_evaluation_criteria,
         )
         evaluator.run_pipeline()
         logger.info("Full trajectory evaluation completed successfully")
@@ -378,8 +383,19 @@ def run_clear_analysis(
     clear_types: List[str],
     inference_config: InferenceConfig,
     overwrite: bool,
+    predefined_issues: Optional[list] = None,
 ) -> bool:
-    """Run CLEAR analysis on evaluation results."""
+    """
+    Run CLEAR analysis on evaluation results.
+
+    Args:
+        eval_results_dir: Directory containing evaluation result files
+        clear_output_dir: Output directory for CLEAR analysis
+        clear_types: List of CLEAR analysis types to run
+        inference_config: LLM inference configuration
+        overwrite: Whether to overwrite existing results
+        predefined_issues: If set, skip issue discovery and use these issues directly
+    """
     if not clear_types:
         logger.info("Skipping CLEAR analysis (none requested)")
         return True
@@ -400,6 +416,7 @@ def run_clear_analysis(
                 output_dir=clear_output_dir,
                 inference_config=inference_config,
                 overwrite=overwrite,
+                predefined_issues=predefined_issues,
             )
             runner.run_analysis()
             logger.info("Root cause CLEAR analysis completed successfully")
@@ -416,6 +433,7 @@ def run_clear_analysis(
                 output_dir=clear_output_dir,
                 inference_config=inference_config,
                 overwrite=overwrite,
+                predefined_issues=predefined_issues,
             )
             runner.run_analysis()
             logger.info("Issues CLEAR analysis completed successfully")
@@ -517,6 +535,8 @@ def run_trajectory_evaluation_pipeline(
     overwrite: bool = False,
     max_workers: int = 10,
     max_files: Optional[int] = None,
+    predefined_issues: Optional[list] = None,
+    full_trace_evaluation_criteria: Optional[dict] = None,
 ) -> tuple[List[str], List[str]]:
     """
     Run trajectory evaluation pipeline on CSV trajectory data.
@@ -536,6 +556,8 @@ def run_trajectory_evaluation_pipeline(
         overwrite: Re-run even if results exist
         max_workers: Number of parallel workers
         max_files: Limit files to process
+        predefined_issues: Predefined issues list to skip issue discovery in CLEAR analysis
+        full_trace_evaluation_criteria: Custom flat evaluation criteria dict for full trajectory eval
 
     Returns:
         Tuple of (completed_evals, failed_evals)
@@ -634,7 +656,10 @@ def run_trajectory_evaluation_pipeline(
             failed_evals.append(EVAL_TYPE_TASK_SUCCESS)
 
     if EVAL_TYPE_FULL_TRAJECTORY in eval_types:
-        if run_full_trajectory_evaluation(**eval_kwargs):
+        if run_full_trajectory_evaluation(
+            **eval_kwargs,
+            full_trace_evaluation_criteria=full_trace_evaluation_criteria,
+        ):
             completed_evals.append(EVAL_TYPE_FULL_TRAJECTORY)
         else:
             failed_evals.append(EVAL_TYPE_FULL_TRAJECTORY)
@@ -662,6 +687,7 @@ def run_trajectory_evaluation_pipeline(
                 clear_types=clear_types,
                 inference_config=inference_config,
                 overwrite=overwrite,
+                predefined_issues=predefined_issues,
             )
 
     # Print summary
@@ -695,6 +721,9 @@ def main():
         config.get('run_name')
     )
 
+    # Update config with resolved run_name so downstream components use the same name
+    config['run_name'] = run_name
+
     # Convert paths
     traj_input_dir = Path(config['data_dir'])
     rubric_dir = Path(config['rubric_dir']) if config.get('rubric_dir') else None
@@ -744,6 +773,8 @@ def main():
         overwrite=config.get('overwrite'),
         max_workers=config.get('max_workers'),
         max_files=config.get('max_files'),
+        predefined_issues=config.get('predefined_issues'),
+        full_trace_evaluation_criteria=config.get('full_trace_evaluation_criteria'),
     )
 
     if failed_evals: