clear-eval 1.0.0__tar.gz

clear_eval-1.0.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

clear_eval-1.0.0/PKG-INFO

@@ -0,0 +1,223 @@
+Metadata-Version: 2.4
+Name: clear_eval
+Version: 1.0.0
+Summary: A python API sdk facilitating Error Analysis via LLM-as-a-Judge
+Author-email: Lilach Eden <lilache@il.ibm.com>, Asaf Yehudai <asaf.yehudai@ibm.com>
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: python-dotenv
+Requires-Dist: langchain
+Requires-Dist: langgraph
+Requires-Dist: langchain_openai
+Requires-Dist: langchain_ibm
+Requires-Dist: ibm_watsonx_ai>=1.2.8
+Requires-Dist: openai
+Requires-Dist: pyyaml
+Requires-Dist: pandas
+Requires-Dist: tqdm
+Requires-Dist: pyyaml
+Requires-Dist: streamlit
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+Requires-Dist: langchain_community
+Requires-Dist: numpy
+Requires-Dist: argparse
+Dynamic: license-file
+
+# CLEAR: Error Analysis via LLM-as-a-Judge Made Easy
+
+**CLEAR (Comprehensive LLM Error Analysis and Reporting)** is an interactive, open-source package for **LLM-based error analysis**. It helps surface meaningful, recurring issues in model outputs by combining automated evaluation with powerful visualization tools.
+
+The workflow consists of two main phases:
+
+1. **Analysis**  
+    Generates textual feedback for each instance; Identifies system-level error categories from these critiques and quantifies their frequencies.
+
+2. **Interactive Dashboard**  
+   An intuitive dashboard provides a comprehensive view of model behavior. Users can:  
+   - Explore aggregate visualizations of identified issues  
+   - Apply dynamic filters to focus on specific error types or score ranges  
+   - Drill down into individual examples that illustrate specific failure patterns
+
+CLEAR makes it easier to diagnose model shortcomings and prioritize targeted improvements.
+
+You can run CLEAR as a full pipeline, or reuse specific stages (generation, evaluation, or just UI).
+
+
+
+## 🚀 Quickstart
+
+Requires Python 3.10+ and the necessary credentials for a supported provider.
+
+1. ### **Clone the repo and set up a virtual environment:**
+
+```bash
+git clone https://github.com/IBM/CLEAR.git
+cd CLEAR
+python3 -m venv .venv
+source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+pip install -e .
+```
+
+ 2. ### Set provider type and credentials
+CLEAR requires a supported LLM provider and credentials to run analysis. [See supported providers ↓](#supported-providers-and-credentials)
+> ⚠️ Using a private proxy or openai deployment? You must configure your model names explicitly (see below). Otherwise, default model names will be used automatically for supported providers.
+
+3. ### **Run on sample data:**
+
+The sample dataset is a small subset of the **GSM8K math problems**.
+For running on the sample data and default configuration, you simpy have to set your provider and run
+```bash
+run-clear-eval-analysis --provider=openai # or rits, watsonx
+```
+
+This will:
+- Run the full CLEAR pipeline
+- Save results under: `results/gsm8k/sample_output/`
+
+4. ###  **View results in the interactive dashboard:**
+
+```bash
+run-clear-eval-dashboard
+```
+
+Or set the port with 
+```bash
+run-clear-eval-dashboard --port <port>
+```
+Then:
+- Upload the generated ZIP file from `results/gsm8k/sample_output/`
+- Explore issues, scores, filters, and drill into examples
+
+5. ###  **To explore the dashboard without running any analysis:**
+Run the dashboard:
+```bash
+run-clear-eval-dashboard
+```
+
+Then you can load the pre-generated sample output zip from [here](results/input_for_ui), without running any analysis. 
+
+
+---
+
+
+## 📂 Analyzing your own data
+
+### 📄 Input Data Format
+
+CLEAR takes a **CSV file** as input, with each row representing a single instance to be evaluated.
+
+#### Required Columns
+
+| Column         | Used When                           | Description                                                       |
+|----------------|-------------------------------------|-------------------------------------------------------------------|
+| `id`           | Always                              | Unique identifier for the instance                                |
+| `model_input`  | Always                              | Prompt provided to the generation model                           |
+| `response`     | Using pre-generated responses       | Pre-generated model response (ignored if generation is enabled)   |
+| `ground_truth` | Performing reference based analysis | Ground-truth answer for evaluation (optional)                     |
+| _others_       | `--input_columns` is used           | Additional input columns to show in dashboard (e.g. `question`)   |
+
+---
+
+### 🚀 Running the analysis
+
+CLEAR can be run via the CLI or Python API.
+
+#### Option 1: CLI commands
+
+Each stage has its own entry point:
+
+```bash
+run-clear-eval-analysis --config_path path/to/config.yaml    # run full pypeline
+run-clear-eval-generation --config_path path/to/config.yaml  # run generation only
+run-clear-eval-evaluation --config_path path/to/config.yaml  # Assume generation responses are given, run evaluation
+```
+
+- If `--config_path` is specified, **all parameters are taken from the config** unless explicitly overridden
+- CLI flags passed directly override corresponding config values
+
+#### Option 2: Python API
+
+```python
+from clear_eval.analysis_runner import run_clear_eval_analysis, run_clear_eval_generation, run_clear_eval_evaluation
+
+run_clear_eval_analysis(
+    config_path="configs/sample_run_config.yaml"
+)
+```
+
+You may also pass overrides instead of using a config file:
+
+```python
+from clear_eval.analysis_runner import run_clear_eval_analysis
+
+run_clear_eval_analysis(
+    run_name="my_data",
+    provider="openai",
+    data_path="my_data.csv",
+    gen_model_name="gpt-3.5-turbo",
+    eval_model_name="gpt-4",
+    output_dir="results/gsm8k/",
+    perform_generation=False,
+    input_columns=["question"]
+)
+```
+### 📊 Launching the Dashboard
+
+```bash
+run-clear-eval-dashboard
+```
+
+Upload the ZIP file generated in your `--output-dir` when prompted.
+
+### 🎛 Supported CLI Arguments
+
+Arguments can be provided via:
+- A YAML config file (`--config_path`)
+- CLI flags
+- Python function parameters (when using the API)
+
+> ⚠️ **Boolean arguments** (`perform_generation`, `is_reference_based`, `resume_enabled`)  
+> These must be set explicitly to `true` or `false` in YAML, CLI, or Python.  
+> On the CLI, use `--flag True` or `--flag False` (case-insensitive).
+
+> ⚠️ **Naming Convention**  
+> Parameter names use `snake_case` in YAML and Python, but use `--kebab-case` in CLI.  
+> For example:  
+> - YAML: `perform_generation: true`  
+> - Python: `perform_generation=True`  
+> - CLI: `--perform-generation True`
+
+| Argument             | Description                                                                                                                                | Default                    |
+|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------|----------------------------|
+| `--config_path`      | Path to a YAML config file (all values loaded unless overridden by CLI args)                                                               |           |
+| `--run_name`         | Unique run name (used in result file names)                                                                                                |        |
+| `--data_path`        | Path to input CSV file                                                                                                                     |           |
+| `--output_dir`       | Output directory to write results                                                                                                          |           |
+| `--provider`         | Model provider: `openai`, `watsonx`, `rits`                                                                                                |           |
+| `--eval_model_name`  | Name of judge model (e.g. `gpt-4o`)                                                                                                        |                            |
+| `--gen_model_name`   | Name of the generator model to evaluate. If not running generations - the generator name to display.                                       |                            |
+| `--perform_generation` | Whether to generate responses or use existing `response` column                                                                            | True                       |  
+| `--is_reference_based` | Use reference-based evaluation (requires `ground_truth` column in input)                                                                   | False                      |
+| `--resume_enabled`   | Whether to reuse intermediate outputs from previous runs stored in output_dir                                                              | True                       |
+| `--evaluation_criteria` | Custom criteria dictionary for scoring individual records: `{"criteria_name1":"criteria_desc1", ...}`supported for yaml config and python. | None                       |
+| `--input_columns`    | Comma-separated list of additional input fields (other than `model_input`) to appear in the results and dashboard (e.g. `question`)        | None                       | 
+
+---
+
+## 🔑Supported providers and credentials
+
+Depending on your selected `--provider`:
+
+| Provider   | Required Environment Variables                                      |
+|------------|---------------------------------------------------------------------|
+| `openai`   | `OPENAI_API_KEY`,  [`OPENAI_API_BASE` if using proxy ]              |                                   |
+| `watsonx`  | `WATSONX_APIKEY`, `WATSONX_URL`, `WATSONX_SPACE_ID` or `PROJECT_ID` |
+| `rits`     | `RITS_API_KEY`                                                      |
+
+---
+

clear_eval-1.0.0/README.md

@@ -0,0 +1,193 @@
+# CLEAR: Error Analysis via LLM-as-a-Judge Made Easy
+
+**CLEAR (Comprehensive LLM Error Analysis and Reporting)** is an interactive, open-source package for **LLM-based error analysis**. It helps surface meaningful, recurring issues in model outputs by combining automated evaluation with powerful visualization tools.
+
+The workflow consists of two main phases:
+
+1. **Analysis**  
+    Generates textual feedback for each instance; Identifies system-level error categories from these critiques and quantifies their frequencies.
+
+2. **Interactive Dashboard**  
+   An intuitive dashboard provides a comprehensive view of model behavior. Users can:  
+   - Explore aggregate visualizations of identified issues  
+   - Apply dynamic filters to focus on specific error types or score ranges  
+   - Drill down into individual examples that illustrate specific failure patterns
+
+CLEAR makes it easier to diagnose model shortcomings and prioritize targeted improvements.
+
+You can run CLEAR as a full pipeline, or reuse specific stages (generation, evaluation, or just UI).
+
+
+
+## 🚀 Quickstart
+
+Requires Python 3.10+ and the necessary credentials for a supported provider.
+
+1. ### **Clone the repo and set up a virtual environment:**
+
+```bash
+git clone https://github.com/IBM/CLEAR.git
+cd CLEAR
+python3 -m venv .venv
+source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+pip install -e .
+```
+
+ 2. ### Set provider type and credentials
+CLEAR requires a supported LLM provider and credentials to run analysis. [See supported providers ↓](#supported-providers-and-credentials)
+> ⚠️ Using a private proxy or openai deployment? You must configure your model names explicitly (see below). Otherwise, default model names will be used automatically for supported providers.
+
+3. ### **Run on sample data:**
+
+The sample dataset is a small subset of the **GSM8K math problems**.
+For running on the sample data and default configuration, you simpy have to set your provider and run
+```bash
+run-clear-eval-analysis --provider=openai # or rits, watsonx
+```
+
+This will:
+- Run the full CLEAR pipeline
+- Save results under: `results/gsm8k/sample_output/`
+
+4. ###  **View results in the interactive dashboard:**
+
+```bash
+run-clear-eval-dashboard
+```
+
+Or set the port with 
+```bash
+run-clear-eval-dashboard --port <port>
+```
+Then:
+- Upload the generated ZIP file from `results/gsm8k/sample_output/`
+- Explore issues, scores, filters, and drill into examples
+
+5. ###  **To explore the dashboard without running any analysis:**
+Run the dashboard:
+```bash
+run-clear-eval-dashboard
+```
+
+Then you can load the pre-generated sample output zip from [here](results/input_for_ui), without running any analysis. 
+
+
+---
+
+
+## 📂 Analyzing your own data
+
+### 📄 Input Data Format
+
+CLEAR takes a **CSV file** as input, with each row representing a single instance to be evaluated.
+
+#### Required Columns
+
+| Column         | Used When                           | Description                                                       |
+|----------------|-------------------------------------|-------------------------------------------------------------------|
+| `id`           | Always                              | Unique identifier for the instance                                |
+| `model_input`  | Always                              | Prompt provided to the generation model                           |
+| `response`     | Using pre-generated responses       | Pre-generated model response (ignored if generation is enabled)   |
+| `ground_truth` | Performing reference based analysis | Ground-truth answer for evaluation (optional)                     |
+| _others_       | `--input_columns` is used           | Additional input columns to show in dashboard (e.g. `question`)   |
+
+---
+
+### 🚀 Running the analysis
+
+CLEAR can be run via the CLI or Python API.
+
+#### Option 1: CLI commands
+
+Each stage has its own entry point:
+
+```bash
+run-clear-eval-analysis --config_path path/to/config.yaml    # run full pypeline
+run-clear-eval-generation --config_path path/to/config.yaml  # run generation only
+run-clear-eval-evaluation --config_path path/to/config.yaml  # Assume generation responses are given, run evaluation
+```
+
+- If `--config_path` is specified, **all parameters are taken from the config** unless explicitly overridden
+- CLI flags passed directly override corresponding config values
+
+#### Option 2: Python API
+
+```python
+from clear_eval.analysis_runner import run_clear_eval_analysis, run_clear_eval_generation, run_clear_eval_evaluation
+
+run_clear_eval_analysis(
+    config_path="configs/sample_run_config.yaml"
+)
+```
+
+You may also pass overrides instead of using a config file:
+
+```python
+from clear_eval.analysis_runner import run_clear_eval_analysis
+
+run_clear_eval_analysis(
+    run_name="my_data",
+    provider="openai",
+    data_path="my_data.csv",
+    gen_model_name="gpt-3.5-turbo",
+    eval_model_name="gpt-4",
+    output_dir="results/gsm8k/",
+    perform_generation=False,
+    input_columns=["question"]
+)
+```
+### 📊 Launching the Dashboard
+
+```bash
+run-clear-eval-dashboard
+```
+
+Upload the ZIP file generated in your `--output-dir` when prompted.
+
+### 🎛 Supported CLI Arguments
+
+Arguments can be provided via:
+- A YAML config file (`--config_path`)
+- CLI flags
+- Python function parameters (when using the API)
+
+> ⚠️ **Boolean arguments** (`perform_generation`, `is_reference_based`, `resume_enabled`)  
+> These must be set explicitly to `true` or `false` in YAML, CLI, or Python.  
+> On the CLI, use `--flag True` or `--flag False` (case-insensitive).
+
+> ⚠️ **Naming Convention**  
+> Parameter names use `snake_case` in YAML and Python, but use `--kebab-case` in CLI.  
+> For example:  
+> - YAML: `perform_generation: true`  
+> - Python: `perform_generation=True`  
+> - CLI: `--perform-generation True`
+
+| Argument             | Description                                                                                                                                | Default                    |
+|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------|----------------------------|
+| `--config_path`      | Path to a YAML config file (all values loaded unless overridden by CLI args)                                                               |           |
+| `--run_name`         | Unique run name (used in result file names)                                                                                                |        |
+| `--data_path`        | Path to input CSV file                                                                                                                     |           |
+| `--output_dir`       | Output directory to write results                                                                                                          |           |
+| `--provider`         | Model provider: `openai`, `watsonx`, `rits`                                                                                                |           |
+| `--eval_model_name`  | Name of judge model (e.g. `gpt-4o`)                                                                                                        |                            |
+| `--gen_model_name`   | Name of the generator model to evaluate. If not running generations - the generator name to display.                                       |                            |
+| `--perform_generation` | Whether to generate responses or use existing `response` column                                                                            | True                       |  
+| `--is_reference_based` | Use reference-based evaluation (requires `ground_truth` column in input)                                                                   | False                      |
+| `--resume_enabled`   | Whether to reuse intermediate outputs from previous runs stored in output_dir                                                              | True                       |
+| `--evaluation_criteria` | Custom criteria dictionary for scoring individual records: `{"criteria_name1":"criteria_desc1", ...}`supported for yaml config and python. | None                       |
+| `--input_columns`    | Comma-separated list of additional input fields (other than `model_input`) to appear in the results and dashboard (e.g. `question`)        | None                       | 
+
+---
+
+## 🔑Supported providers and credentials
+
+Depending on your selected `--provider`:
+
+| Provider   | Required Environment Variables                                      |
+|------------|---------------------------------------------------------------------|
+| `openai`   | `OPENAI_API_KEY`,  [`OPENAI_API_BASE` if using proxy ]              |                                   |
+| `watsonx`  | `WATSONX_APIKEY`, `WATSONX_URL`, `WATSONX_SPACE_ID` or `PROJECT_ID` |
+| `rits`     | `RITS_API_KEY`                                                      |
+
+---
+

clear_eval-1.0.0/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "clear_eval"
+version = "1.0.0"
+description = "A python API sdk facilitating Error Analysis via LLM-as-a-Judge"
+readme = "README.md"
+authors = [{ name = "Lilach Eden", email = "lilache@il.ibm.com"},
+            {name = "Asaf Yehudai", email = "asaf.yehudai@ibm.com"}]
+license = { text = "Apache-2.0" }
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "python-dotenv",
+    "langchain",
+    "langgraph",
+    "langchain_openai",
+    "langchain_ibm",
+    "ibm_watsonx_ai>=1.2.8",
+    "openai",
+    "pyyaml",
+    "pandas",
+    "tqdm",
+    "pyyaml",
+    "streamlit",
+    "matplotlib",
+    "seaborn",
+    "langchain_community",
+    "numpy",
+    "argparse"
+]
+requires-python = ">=3.10"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+packages = ["clear_eval"]
+
+[tool.setuptools.package-data]
+clear_eval = ["*.yaml"]
+where = ["src"]
+
+[project.scripts]
+run-analysis = "clear_eval.cli:main"
+run-clear-eval-analysis = "clear_eval.cli:main"
+run-clear-eval-generation = "clear_eval.cli:run_generation_cli"
+run-clear-eval-evaluation = "clear_eval.cli:run_evaluation_cli"
+run-clear-eval-aggregation = "clear_eval.cli:run_aggregation_cli"
+run-clear-eval-dashboard = "clear_eval.cli:run_dashboard_cli"

clear_eval-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

clear_eval-1.0.0/src/clear_eval/analysis_runner.py

@@ -0,0 +1,29 @@
+import os
+from clear_eval.pipeline.full_pipeline import run_eval_pipeline, run_generation_pipeline, run_aggregation_pipeline
+from clear_eval.pipeline.config_loader import load_config
+from clear_eval.logging_config import setup_logging
+setup_logging()
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+
+DEFAULT_CONFIG_PATH = os.path.join(script_dir, "pipeline", "setup", "default_config.yaml")
+
+def run_clear_eval_analysis(config_path=None, **kwargs):
+    config_dict = load_config(DEFAULT_CONFIG_PATH, config_path, **kwargs)
+    run_eval_pipeline(config_dict)
+
+def run_analysis_pipeline(config_path=None, **kwargs):
+    run_clear_eval_analysis(config_path, **kwargs)
+
+def run_clear_eval_evaluation(config_path=None, **kwargs):
+    config_dict = load_config(DEFAULT_CONFIG_PATH, config_path, **kwargs)
+    config_dict["perform_generation"] = False
+    run_eval_pipeline(config_dict)
+
+def run_clear_eval_generation(config_path=None, **kwargs):
+    config_dict = load_config(DEFAULT_CONFIG_PATH, config_path, **kwargs)
+    run_generation_pipeline(config_dict)
+
+def run_clear_eval_aggregation(config_path=None, **kwargs):
+    config_dict = load_config(DEFAULT_CONFIG_PATH, config_path, **kwargs)
+    run_aggregation_pipeline(config_dict)

clear_eval-1.0.0/src/clear_eval/args.py

@@ -0,0 +1,51 @@
+import argparse
+import json
+
+
+def parse_dict(arg: str) -> dict:
+    try:
+        return json.loads(arg)
+    except json.JSONDecodeError as e:
+        raise argparse.ArgumentTypeError(f"Invalid JSON format: {e}")
+
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ("yes", "true", "t", "1"):
+        return True
+    elif v.lower() in ("no", "false", "f", "0"):
+        return False
+    raise argparse.ArgumentTypeError("Boolean value expected.")
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--data-path", help="Path to the data csv file")
+    parser.add_argument("--output-dir", default=None, help="Output directory")
+    parser.add_argument("--provider", choices=["azure", "openai", "watsonx", "rits"])
+    parser.add_argument("--eval-model-name", help="Name of the model used by CLEAR for evaluating and analyzing outputs")
+    parser.add_argument("--gen-model-name", help="Name of the generator model whose responses are evaluated (e.g. gpt-3.5-turbo)",
+                        default=None)
+
+    parser.add_argument("--config-path", default=None, help="Optional: path to the config file")
+    parser.add_argument("--perform-generation", type=str2bool, default=True, help="Whether to perform generations or"
+                                                                    "use existing generations")
+    parser.add_argument("--is-reference-based", type=str2bool, default=False,
+                        help="Whether to use use references for the evaluations (if true, references must be stored in the 'reference' column of the input.")
+    parser.add_argument("--resume-enabled", type=str2bool, default=True,
+                        help="Whether to use use intermediate results found in the output dir")
+    parser.add_argument("--run-name", default=None,
+                        help="Unique identifier for the run")
+    parser.add_argument("--evaluation-criteria", type=parse_dict, help="Json of a dictionary of evaluation criteria for"
+                                                "the judge. Example: --evaluation-criteria '{\"correction\": \"Response is factually correct\"}'")
+    parser.add_argument("--max-examples-to-analyze", type=int, help="Analyze only the specified number of examples")
+    parser.add_argument("--input-columns", nargs='+', help="List of column names to present in the ui")
+
+    args = parser.parse_args()
+
+    # Only keep explicitly passed args (ignore None)
+    overrides = {
+        k: v for k, v in vars(args).items()
+        if v is not None
+    }
+    return overrides

clear_eval-1.0.0/src/clear_eval/cli.py

@@ -0,0 +1,41 @@
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+import streamlit.web.cli as stcli
+from clear_eval.analysis_runner import run_clear_eval_analysis, run_clear_eval_generation, run_clear_eval_aggregation
+from clear_eval.args import parse_args
+
+
+def main():
+    overrides = parse_args()
+    run_clear_eval_analysis(**overrides)
+
+def run_generation_cli():
+    overrides = parse_args()
+    run_clear_eval_generation(**overrides)
+
+def run_evaluation_cli():
+    overrides = parse_args()
+    overrides["perform_generation"] = False
+    run_clear_eval_analysis(**overrides)
+
+def run_aggregation_cli():
+    overrides = parse_args()
+    run_clear_eval_aggregation(**overrides)
+
+def run_dashboard_cli():
+    parser = argparse.ArgumentParser(description="Run the dashboard.")
+    parser.add_argument("--port", type=int, help="Optional port to run the dashboard on.")
+    args = parser.parse_args()
+
+    streamlit_app = Path(__file__).parent / "load_ui.py"
+    sys.argv = ["streamlit", "run", str(streamlit_app)]
+    if args.port:
+        sys.argv += ["--server.port", str(args.port)]
+
+    stcli.main()
+
+if __name__ == "__main__":
+    main()

clear_eval-1.0.0/src/clear_eval/load_ui.py

@@ -0,0 +1,3 @@
+from clear_eval.dashboard.show_analysis_dashboard import show_dashboard
+
+show_dashboard()

clear_eval-1.0.0/src/clear_eval/logging_config.py

@@ -0,0 +1,10 @@
+import logging
+
+def setup_logging(level=logging.INFO):
+    if not logging.getLogger().hasHandlers():
+        logging.basicConfig(
+            format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+            level=logging.INFO,
+        )
+    for noisy_lib in ["requests", "urllib3", "openai", "ibm_watsonx_ai", "httpx"]:
+        logging.getLogger(noisy_lib).setLevel(logging.WARNING)

clear_eval-1.0.0/src/clear_eval.egg-info/PKG-INFO

@@ -0,0 +1,223 @@
+Metadata-Version: 2.4
+Name: clear_eval
+Version: 1.0.0
+Summary: A python API sdk facilitating Error Analysis via LLM-as-a-Judge
+Author-email: Lilach Eden <lilache@il.ibm.com>, Asaf Yehudai <asaf.yehudai@ibm.com>
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: python-dotenv
+Requires-Dist: langchain
+Requires-Dist: langgraph
+Requires-Dist: langchain_openai
+Requires-Dist: langchain_ibm
+Requires-Dist: ibm_watsonx_ai>=1.2.8
+Requires-Dist: openai
+Requires-Dist: pyyaml
+Requires-Dist: pandas
+Requires-Dist: tqdm
+Requires-Dist: pyyaml
+Requires-Dist: streamlit
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+Requires-Dist: langchain_community
+Requires-Dist: numpy
+Requires-Dist: argparse
+Dynamic: license-file
+
+# CLEAR: Error Analysis via LLM-as-a-Judge Made Easy
+
+**CLEAR (Comprehensive LLM Error Analysis and Reporting)** is an interactive, open-source package for **LLM-based error analysis**. It helps surface meaningful, recurring issues in model outputs by combining automated evaluation with powerful visualization tools.
+
+The workflow consists of two main phases:
+
+1. **Analysis**  
+    Generates textual feedback for each instance; Identifies system-level error categories from these critiques and quantifies their frequencies.
+
+2. **Interactive Dashboard**  
+   An intuitive dashboard provides a comprehensive view of model behavior. Users can:  
+   - Explore aggregate visualizations of identified issues  
+   - Apply dynamic filters to focus on specific error types or score ranges  
+   - Drill down into individual examples that illustrate specific failure patterns
+
+CLEAR makes it easier to diagnose model shortcomings and prioritize targeted improvements.
+
+You can run CLEAR as a full pipeline, or reuse specific stages (generation, evaluation, or just UI).
+
+
+
+## 🚀 Quickstart
+
+Requires Python 3.10+ and the necessary credentials for a supported provider.
+
+1. ### **Clone the repo and set up a virtual environment:**
+
+```bash
+git clone https://github.com/IBM/CLEAR.git
+cd CLEAR
+python3 -m venv .venv
+source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+pip install -e .
+```
+
+ 2. ### Set provider type and credentials
+CLEAR requires a supported LLM provider and credentials to run analysis. [See supported providers ↓](#supported-providers-and-credentials)
+> ⚠️ Using a private proxy or openai deployment? You must configure your model names explicitly (see below). Otherwise, default model names will be used automatically for supported providers.
+
+3. ### **Run on sample data:**
+
+The sample dataset is a small subset of the **GSM8K math problems**.
+For running on the sample data and default configuration, you simpy have to set your provider and run
+```bash
+run-clear-eval-analysis --provider=openai # or rits, watsonx
+```
+
+This will:
+- Run the full CLEAR pipeline
+- Save results under: `results/gsm8k/sample_output/`
+
+4. ###  **View results in the interactive dashboard:**
+
+```bash
+run-clear-eval-dashboard
+```
+
+Or set the port with 
+```bash
+run-clear-eval-dashboard --port <port>
+```
+Then:
+- Upload the generated ZIP file from `results/gsm8k/sample_output/`
+- Explore issues, scores, filters, and drill into examples
+
+5. ###  **To explore the dashboard without running any analysis:**
+Run the dashboard:
+```bash
+run-clear-eval-dashboard
+```
+
+Then you can load the pre-generated sample output zip from [here](results/input_for_ui), without running any analysis. 
+
+
+---
+
+
+## 📂 Analyzing your own data
+
+### 📄 Input Data Format
+
+CLEAR takes a **CSV file** as input, with each row representing a single instance to be evaluated.
+
+#### Required Columns
+
+| Column         | Used When                           | Description                                                       |
+|----------------|-------------------------------------|-------------------------------------------------------------------|
+| `id`           | Always                              | Unique identifier for the instance                                |
+| `model_input`  | Always                              | Prompt provided to the generation model                           |
+| `response`     | Using pre-generated responses       | Pre-generated model response (ignored if generation is enabled)   |
+| `ground_truth` | Performing reference based analysis | Ground-truth answer for evaluation (optional)                     |
+| _others_       | `--input_columns` is used           | Additional input columns to show in dashboard (e.g. `question`)   |
+
+---
+
+### 🚀 Running the analysis
+
+CLEAR can be run via the CLI or Python API.
+
+#### Option 1: CLI commands
+
+Each stage has its own entry point:
+
+```bash
+run-clear-eval-analysis --config_path path/to/config.yaml    # run full pypeline
+run-clear-eval-generation --config_path path/to/config.yaml  # run generation only
+run-clear-eval-evaluation --config_path path/to/config.yaml  # Assume generation responses are given, run evaluation
+```
+
+- If `--config_path` is specified, **all parameters are taken from the config** unless explicitly overridden
+- CLI flags passed directly override corresponding config values
+
+#### Option 2: Python API
+
+```python
+from clear_eval.analysis_runner import run_clear_eval_analysis, run_clear_eval_generation, run_clear_eval_evaluation
+
+run_clear_eval_analysis(
+    config_path="configs/sample_run_config.yaml"
+)
+```
+
+You may also pass overrides instead of using a config file:
+
+```python
+from clear_eval.analysis_runner import run_clear_eval_analysis
+
+run_clear_eval_analysis(
+    run_name="my_data",
+    provider="openai",
+    data_path="my_data.csv",
+    gen_model_name="gpt-3.5-turbo",
+    eval_model_name="gpt-4",
+    output_dir="results/gsm8k/",
+    perform_generation=False,
+    input_columns=["question"]
+)
+```
+### 📊 Launching the Dashboard
+
+```bash
+run-clear-eval-dashboard
+```
+
+Upload the ZIP file generated in your `--output-dir` when prompted.
+
+### 🎛 Supported CLI Arguments
+
+Arguments can be provided via:
+- A YAML config file (`--config_path`)
+- CLI flags
+- Python function parameters (when using the API)
+
+> ⚠️ **Boolean arguments** (`perform_generation`, `is_reference_based`, `resume_enabled`)  
+> These must be set explicitly to `true` or `false` in YAML, CLI, or Python.  
+> On the CLI, use `--flag True` or `--flag False` (case-insensitive).
+
+> ⚠️ **Naming Convention**  
+> Parameter names use `snake_case` in YAML and Python, but use `--kebab-case` in CLI.  
+> For example:  
+> - YAML: `perform_generation: true`  
+> - Python: `perform_generation=True`  
+> - CLI: `--perform-generation True`
+
+| Argument             | Description                                                                                                                                | Default                    |
+|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------|----------------------------|
+| `--config_path`      | Path to a YAML config file (all values loaded unless overridden by CLI args)                                                               |           |
+| `--run_name`         | Unique run name (used in result file names)                                                                                                |        |
+| `--data_path`        | Path to input CSV file                                                                                                                     |           |
+| `--output_dir`       | Output directory to write results                                                                                                          |           |
+| `--provider`         | Model provider: `openai`, `watsonx`, `rits`                                                                                                |           |
+| `--eval_model_name`  | Name of judge model (e.g. `gpt-4o`)                                                                                                        |                            |
+| `--gen_model_name`   | Name of the generator model to evaluate. If not running generations - the generator name to display.                                       |                            |
+| `--perform_generation` | Whether to generate responses or use existing `response` column                                                                            | True                       |  
+| `--is_reference_based` | Use reference-based evaluation (requires `ground_truth` column in input)                                                                   | False                      |
+| `--resume_enabled`   | Whether to reuse intermediate outputs from previous runs stored in output_dir                                                              | True                       |
+| `--evaluation_criteria` | Custom criteria dictionary for scoring individual records: `{"criteria_name1":"criteria_desc1", ...}`supported for yaml config and python. | None                       |
+| `--input_columns`    | Comma-separated list of additional input fields (other than `model_input`) to appear in the results and dashboard (e.g. `question`)        | None                       | 
+
+---
+
+## 🔑Supported providers and credentials
+
+Depending on your selected `--provider`:
+
+| Provider   | Required Environment Variables                                      |
+|------------|---------------------------------------------------------------------|
+| `openai`   | `OPENAI_API_KEY`,  [`OPENAI_API_BASE` if using proxy ]              |                                   |
+| `watsonx`  | `WATSONX_APIKEY`, `WATSONX_URL`, `WATSONX_SPACE_ID` or `PROJECT_ID` |
+| `rits`     | `RITS_API_KEY`                                                      |
+
+---
+

clear_eval-1.0.0/src/clear_eval.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+LICENSE
+README.md
+pyproject.toml
+src/clear_eval/__init__.py
+src/clear_eval/analysis_runner.py
+src/clear_eval/args.py
+src/clear_eval/cli.py
+src/clear_eval/load_ui.py
+src/clear_eval/logging_config.py
+src/clear_eval.egg-info/PKG-INFO
+src/clear_eval.egg-info/SOURCES.txt
+src/clear_eval.egg-info/dependency_links.txt
+src/clear_eval.egg-info/entry_points.txt
+src/clear_eval.egg-info/requires.txt
+src/clear_eval.egg-info/top_level.txt

clear_eval-1.0.0/src/clear_eval.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

clear_eval-1.0.0/src/clear_eval.egg-info/entry_points.txt

@@ -0,0 +1,7 @@
+[console_scripts]
+run-analysis = clear_eval.cli:main
+run-clear-eval-aggregation = clear_eval.cli:run_aggregation_cli
+run-clear-eval-analysis = clear_eval.cli:main
+run-clear-eval-dashboard = clear_eval.cli:run_dashboard_cli
+run-clear-eval-evaluation = clear_eval.cli:run_evaluation_cli
+run-clear-eval-generation = clear_eval.cli:run_generation_cli

clear_eval-1.0.0/src/clear_eval.egg-info/requires.txt

@@ -0,0 +1,17 @@
+python-dotenv
+langchain
+langgraph
+langchain_openai
+langchain_ibm
+ibm_watsonx_ai>=1.2.8
+openai
+pyyaml
+pandas
+tqdm
+pyyaml
+streamlit
+matplotlib
+seaborn
+langchain_community
+numpy
+argparse

clear_eval-1.0.0/src/clear_eval.egg-info/top_level.txt

@@ -0,0 +1 @@
+clear_eval