clear-eval 2.0.2__tar.gz → 2.0.3__tar.gz

{clear_eval-2.0.2 → clear_eval-2.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clear_eval
-Version: 2.0.2
+Version: 2.0.3
 Summary: A python API sdk facilitating Error Analysis via LLM-as-a-Judge
 Author-email: Lilach Eden <lilache@il.ibm.com>, Asaf Yehudai <asaf.yehudai@ibm.com>
 License: Apache-2.0

{clear_eval-2.0.2 → clear_eval-2.0.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "clear_eval"
-version = "2.0.2"
+version = "2.0.3"
 description = "A python API sdk facilitating Error Analysis via LLM-as-a-Judge"
 readme = "README.md"
 authors = [{ name = "Lilach Eden", email = "lilache@il.ibm.com"},

{clear_eval-2.0.2 → clear_eval-2.0.3}/src/clear_eval/agentic/dashboard/agentic_workflow_dashboard.py

@@ -2174,7 +2174,8 @@ def main_page():
                 # Try to get dimensions from parsed_evaluation first (preferred structure)
                 step_quality = parsed_eval.get("step_quality_dimensions", {})
                 trajectory_dims = parsed_eval.get("trajectory_dimensions", {})
-                
+                custom_dims = parsed_eval.get("dimensions", {})
+
                 # Collect step quality dimension scores
                 for criterion, details in step_quality.items():
                     score = details.get("score")
@@ -2183,7 +2184,7 @@ def main_page():
                             all_dimension_scores[f"Step Quality: {criterion}"].append(float(score))
                         except (ValueError, TypeError):
                             print(f"⚠️  Warning: Could not convert score to float for {criterion}: {score}")
-                
+
                 # Collect trace dimension scores
                 for criterion, details in trajectory_dims.items():
                     score = details.get("score")
@@ -2192,9 +2193,19 @@ def main_page():
                             all_dimension_scores[f"Trace: {criterion}"].append(float(score))
                         except (ValueError, TypeError):
                             print(f"⚠️  Warning: Could not convert score to float for {criterion}: {score}")
-                
+
+                # Collect custom flat dimension scores
+                if custom_dims and not step_quality and not trajectory_dims:
+                    for criterion, details in custom_dims.items():
+                        score = details.get("score")
+                        if score is not None:
+                            try:
+                                all_dimension_scores[criterion].append(float(score))
+                            except (ValueError, TypeError):
+                                print(f"⚠️  Warning: Could not convert score to float for {criterion}: {score}")
+
                 # Fallback: if no dimensions found in parsed_evaluation, try dimension_scores directly
-                if not step_quality and not trajectory_dims and dimension_scores:
+                if not step_quality and not trajectory_dims and not custom_dims and dimension_scores:
                     for criterion, score in dimension_scores.items():
                         if score is not None:
                             try:
@@ -3083,7 +3094,8 @@ def main_page():
                 # Separate step quality and trace dimensions
                 step_quality = parsed_eval.get("step_quality_dimensions", {})
                 trajectory_dims = parsed_eval.get("trajectory_dimensions", {})
-                
+                custom_dims = parsed_eval.get("dimensions", {})
+
                 if step_quality:
                     ui.html('<div style="font-weight: 600; color: #1E293B; margin: 12px 0 8px 0; font-size: 15px;">Step Quality Dimensions</div>')
                     for criterion, details in step_quality.items():
@@ -3091,7 +3103,7 @@ def main_page():
                         justification = details.get("justification", "No justification provided")
                         score_color = score_to_hex(score)
                         definition = get_dimension_definition(criterion)
-                        
+
                         with ui.row().classes("w-full items-center gap-2 mb-2"):
                             with ui.expansion(f"{criterion}: {score:.2f}", icon="check_circle").classes("flex-1").style(f"border-left: 4px solid {score_color};"):
                                 ui.html(f"""
@@ -3101,7 +3113,7 @@ def main_page():
                                 """)
                             with ui.button(icon="info", on_click=lambda: None).props("flat dense round size=sm").classes("text-blue-500"):
                                 ui.tooltip(definition).classes("bg-slate-800 text-white text-sm max-w-md")
-                
+
                 if trajectory_dims:
                     ui.html('<div style="font-weight: 600; color: #1E293B; margin: 20px 0 8px 0; font-size: 15px;">Trace-Level Dimensions</div>')
                     for criterion, details in trajectory_dims.items():
@@ -3109,7 +3121,7 @@ def main_page():
                         justification = details.get("justification", "No justification provided")
                         score_color = score_to_hex(score)
                         definition = get_dimension_definition(criterion)
-                        
+
                         with ui.row().classes("w-full items-center gap-2 mb-2"):
                             with ui.expansion(f"{criterion}: {score:.2f}", icon="check_circle").classes("flex-1").style(f"border-left: 4px solid {score_color};"):
                                 ui.html(f"""
@@ -3120,6 +3132,21 @@ def main_page():
                             with ui.button(icon="info", on_click=lambda: None).props("flat dense round size=sm").classes("text-blue-500"):
                                 ui.tooltip(definition).classes("bg-slate-800 text-white text-sm max-w-md")
 
+                if custom_dims and not step_quality and not trajectory_dims:
+                    ui.html('<div style="font-weight: 600; color: #1E293B; margin: 12px 0 8px 0; font-size: 15px;">Evaluation Dimensions</div>')
+                    for criterion, details in custom_dims.items():
+                        score = float(details.get("score", 0))
+                        justification = details.get("justification", "No justification provided")
+                        score_color = score_to_hex(score)
+
+                        with ui.row().classes("w-full items-center gap-2 mb-2"):
+                            with ui.expansion(f"{criterion}: {score:.2f}", icon="check_circle").classes("flex-1").style(f"border-left: 4px solid {score_color};"):
+                                ui.html(f"""
+                                    <div style="padding: 8px; background: #FAFAFA; border-radius: 6px; color: #475569; font-size: 13px; line-height: 1.5;">
+                                        {justification}
+                                    </div>
+                                """)
+
     def render_rubric_evaluation(rubric_eval):
         """Render the rubric-based trace evaluation."""
         # Overall score card

{clear_eval-2.0.2 → clear_eval-2.0.3}/src/clear_eval/agentic/pipeline/argument_definitions.py

@@ -10,7 +10,7 @@ reusable argument groups that can be composed based on each script's needs.
 """
 
 import argparse
-from clear_eval.args import str2bool
+from clear_eval.args import str2bool, parse_dict
 
 
 def add_agentic_pipeline_args(parser: argparse.ArgumentParser) -> None:
@@ -132,6 +132,14 @@ def add_full_trajectory_args(parser: argparse.ArgumentParser) -> None:
         type=int,
         help="Model context window size (for full trajectory)"
     )
+    group.add_argument(
+        "--full-trace-evaluation-criteria",
+        type=parse_dict,
+        default=None,
+        help="JSON dict of evaluation criteria for full trace evaluation. "
+             "Replaces all default dimensions when set. "
+             "Example: '{\"Correctness\": \"Accurate results\", \"Efficiency\": \"No redundant steps\"}'"
+    )
     # Note: --max-files is defined in add_agentic_pipeline_args() (shared across pipelines)
     # Do not add it here to avoid conflicts
 

{clear_eval-2.0.2 → clear_eval-2.0.3}/src/clear_eval/agentic/pipeline/full_traces_evaluation/clear_analysis/base_clear_runner.py

@@ -44,6 +44,7 @@ class BaseClearRunner(ABC):
         output_dir: Path,
         inference_config: InferenceConfig,
         overwrite: bool = False,
+        predefined_issues: list = None,
     ):
         """
         Initialize CLEAR runner.
@@ -53,11 +54,13 @@ class BaseClearRunner(ABC):
             output_dir: Base directory for CLEAR analysis outputs
             inference_config: LLM inference configuration
             overwrite: Whether to overwrite existing CLEAR results
+            predefined_issues: Predefined issues list to skip issue discovery
         """
         self.eval_results_dir = Path(eval_results_dir)
         self.output_dir = Path(output_dir)
         self.inference_config = inference_config
         self.overwrite = overwrite
+        self.predefined_issues = predefined_issues
 
     @abstractmethod
     def get_source_name(self) -> str:
@@ -219,6 +222,9 @@ class BaseClearRunner(ABC):
                 "endpoint_url": self.inference_config.endpoint_url,
             }
 
+            if self.predefined_issues:
+                analysis_kwargs["predefined_issues"] = self.predefined_issues
+
             run_clear_eval_aggregation(**analysis_kwargs)
             logger.info(f"Completed: {group_key}")
             return True

{clear_eval-2.0.2 → clear_eval-2.0.3}/src/clear_eval/agentic/pipeline/full_traces_evaluation/run_trajectory_evaluation_pipeline.py

@@ -99,6 +99,9 @@ Arguments:
         --overwrite: Re-run even if results exist
         --max-workers: Number of parallel workers (default: 7)
         --max-files: Limit files to process (for testing)
+        --full-trace-evaluation-criteria: JSON dict of custom evaluation criteria
+                                         Replaces all default dimensions when set
+        --predefined-issues: Predefined issues list for CLEAR analysis
 
 Output Structure:
     output_dir/
@@ -273,6 +276,7 @@ def run_full_trajectory_evaluation(
     overwrite: bool,
     max_workers: int,
     max_files: Optional[int],
+    full_trace_evaluation_criteria: Optional[dict] = None,
 ) -> bool:
     """Run full trajectory evaluation."""
     logger.info("=" * 80)
@@ -288,6 +292,7 @@ def run_full_trajectory_evaluation(
             overwrite=overwrite,
             max_workers=max_workers,
             max_files=max_files,
+            full_trace_evaluation_criteria=full_trace_evaluation_criteria,
         )
         evaluator.run_pipeline()
         logger.info("Full trajectory evaluation completed successfully")
@@ -378,8 +383,19 @@ def run_clear_analysis(
     clear_types: List[str],
     inference_config: InferenceConfig,
     overwrite: bool,
+    predefined_issues: Optional[list] = None,
 ) -> bool:
-    """Run CLEAR analysis on evaluation results."""
+    """
+    Run CLEAR analysis on evaluation results.
+
+    Args:
+        eval_results_dir: Directory containing evaluation result files
+        clear_output_dir: Output directory for CLEAR analysis
+        clear_types: List of CLEAR analysis types to run
+        inference_config: LLM inference configuration
+        overwrite: Whether to overwrite existing results
+        predefined_issues: If set, skip issue discovery and use these issues directly
+    """
     if not clear_types:
         logger.info("Skipping CLEAR analysis (none requested)")
         return True
@@ -400,6 +416,7 @@ def run_clear_analysis(
                 output_dir=clear_output_dir,
                 inference_config=inference_config,
                 overwrite=overwrite,
+                predefined_issues=predefined_issues,
             )
             runner.run_analysis()
             logger.info("Root cause CLEAR analysis completed successfully")
@@ -416,6 +433,7 @@ def run_clear_analysis(
                 output_dir=clear_output_dir,
                 inference_config=inference_config,
                 overwrite=overwrite,
+                predefined_issues=predefined_issues,
             )
             runner.run_analysis()
             logger.info("Issues CLEAR analysis completed successfully")
@@ -517,6 +535,8 @@ def run_trajectory_evaluation_pipeline(
     overwrite: bool = False,
     max_workers: int = 10,
     max_files: Optional[int] = None,
+    predefined_issues: Optional[list] = None,
+    full_trace_evaluation_criteria: Optional[dict] = None,
 ) -> tuple[List[str], List[str]]:
     """
     Run trajectory evaluation pipeline on CSV trajectory data.
@@ -536,6 +556,8 @@ def run_trajectory_evaluation_pipeline(
         overwrite: Re-run even if results exist
         max_workers: Number of parallel workers
         max_files: Limit files to process
+        predefined_issues: Predefined issues list to skip issue discovery in CLEAR analysis
+        full_trace_evaluation_criteria: Custom flat evaluation criteria dict for full trajectory eval
 
     Returns:
         Tuple of (completed_evals, failed_evals)
@@ -634,7 +656,10 @@ def run_trajectory_evaluation_pipeline(
             failed_evals.append(EVAL_TYPE_TASK_SUCCESS)
 
     if EVAL_TYPE_FULL_TRAJECTORY in eval_types:
-        if run_full_trajectory_evaluation(**eval_kwargs):
+        if run_full_trajectory_evaluation(
+            **eval_kwargs,
+            full_trace_evaluation_criteria=full_trace_evaluation_criteria,
+        ):
             completed_evals.append(EVAL_TYPE_FULL_TRAJECTORY)
         else:
             failed_evals.append(EVAL_TYPE_FULL_TRAJECTORY)
@@ -662,6 +687,7 @@ def run_trajectory_evaluation_pipeline(
                 clear_types=clear_types,
                 inference_config=inference_config,
                 overwrite=overwrite,
+                predefined_issues=predefined_issues,
             )
 
     # Print summary
@@ -744,6 +770,8 @@ def main():
         overwrite=config.get('overwrite'),
         max_workers=config.get('max_workers'),
         max_files=config.get('max_files'),
+        predefined_issues=config.get('predefined_issues'),
+        full_trace_evaluation_criteria=config.get('full_trace_evaluation_criteria'),
     )
 
     if failed_evals:

{clear_eval-2.0.2 → clear_eval-2.0.3}/src/clear_eval/agentic/pipeline/full_traces_evaluation/trace_evaluation/base_evaluator.py

@@ -73,6 +73,9 @@ class TrajectoryEvaluator(ABC):
         - Parallel execution
         - Error handling and logging
     
+    Supports optional full_trace_evaluation_criteria for custom evaluation
+    dimensions (used by FullTrajectoryEvaluator).
+
     Subclasses implement evaluation-specific logic via abstract methods:
         - prepare_evaluation_data: Load/prepare evaluation-specific data
         - prepare_context: Build context dict for prompt
@@ -92,6 +95,7 @@ class TrajectoryEvaluator(ABC):
         overwrite: bool = False,
         max_workers: int = 7,
         max_files: int | None = None,
+        full_trace_evaluation_criteria: dict | None = None,
     ):
         """
         Initialize evaluator with common configuration.
@@ -104,6 +108,8 @@ class TrajectoryEvaluator(ABC):
             overwrite: Whether to overwrite existing evaluation results
             max_workers: Number of parallel workers
             max_files: Maximum number of files to process (for testing)
+            full_trace_evaluation_criteria: Custom flat evaluation criteria dict
+                that replaces all default dimensions when set
         """
         self.inference_config = inference_config
         self.traj_input_dir = Path(traj_input_dir)
@@ -112,6 +118,7 @@ class TrajectoryEvaluator(ABC):
         self.overwrite = overwrite
         self.max_workers = max_workers
         self.max_files = max_files
+        self.full_trace_evaluation_criteria = full_trace_evaluation_criteria
 
         # Create results directory: output_dir/evaluation_type[/model_subdir]
         eval_type = self.get_evaluation_type().replace(" ", "_").replace("/", "_")