text2sql-eval-toolkit 1.0.0__tar.gz → 1.4.0__tar.gz

text2sql_eval_toolkit-1.4.0/MANIFEST.in

@@ -0,0 +1,7 @@
+# The wheel's frontend is copied from here by setup.py, so the sdist must carry
+# it -- otherwise `pip install <sdist>` builds a wheel with no dashboard UI.
+graft dashboard/dist
+
+exclude .env
+prune dashboard/node_modules
+prune data/results

{text2sql_eval_toolkit-1.0.0/src/text2sql_eval_toolkit.egg-info → text2sql_eval_toolkit-1.4.0}/PKG-INFO

@@ -1,10 +1,14 @@
 Metadata-Version: 2.4
 Name: text2sql-eval-toolkit
-Version: 1.0.0
+Version: 1.4.0
 Summary: A toolkit for evaluating text-to-SQL pipelines
 Author-email: Oktie Hassanzadeh <lastname@us.ibm.com>
 License: MIT
 Project-URL: Homepage, https://github.com/IBM/text2sql-eval-toolkit
+Project-URL: Documentation, https://text2sql-eval-toolkit.readthedocs.io/
+Project-URL: Repository, https://github.com/IBM/text2sql-eval-toolkit
+Project-URL: Changelog, https://github.com/IBM/text2sql-eval-toolkit/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/IBM/text2sql-eval-toolkit/issues
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -16,7 +20,6 @@ Requires-Dist: pandas>=2.2.3
 Requires-Dist: requests>=2.32.4
 Requires-Dist: sqlalchemy>=2.0.41
 Requires-Dist: sqlparse>=0.5.3
-Requires-Dist: unitxt>=1.26.1
 Requires-Dist: psycopg2-binary>=2.9.10
 Requires-Dist: sqlglot>=27.0.0
 Requires-Dist: asyncpg>=0.30.0
@@ -29,6 +32,14 @@ Requires-Dist: python-dotenv>=1.1.1
 Requires-Dist: langgraph>=0.2.0
 Requires-Dist: langchain-core>=0.3.0
 Requires-Dist: openai>=1.0.0
+Requires-Dist: google-genai>=1.0.0
+Requires-Dist: huggingface_hub<1.0,>=0.30
+Provides-Extra: litellm
+Requires-Dist: litellm>=1.50; extra == "litellm"
+Provides-Extra: docs
+Requires-Dist: mkdocs>=1.6; extra == "docs"
+Requires-Dist: mkdocs-material>=9.5; extra == "docs"
+Requires-Dist: mkdocstrings[python]>=0.26; extra == "docs"
 Provides-Extra: db2
 Requires-Dist: ibm-db>=3.2.6; extra == "db2"
 Requires-Dist: ibm-db-sa>=0.4.0; extra == "db2"
@@ -44,6 +55,24 @@ Requires-Dist: presto-python-client>=0.8.4; extra == "presto"
 Provides-Extra: notebook
 Requires-Dist: jupyter>=1.0.0; extra == "notebook"
 Requires-Dist: nbconvert>=7.0.0; extra == "notebook"
+Provides-Extra: dashboard
+Requires-Dist: fastapi>=0.115.0; extra == "dashboard"
+Requires-Dist: uvicorn[standard]>=0.30.0; extra == "dashboard"
+Requires-Dist: authlib>=1.3; extra == "dashboard"
+Requires-Dist: itsdangerous>=2.1; extra == "dashboard"
+Requires-Dist: pymysql>=1.1.1; extra == "dashboard"
+Requires-Dist: aiomysql>=0.2.0; extra == "dashboard"
+Requires-Dist: greenlet>=3.0.0; extra == "dashboard"
+Requires-Dist: cryptography>=45.0.6; extra == "dashboard"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.24.0; extra == "dev"
+Requires-Dist: ruff>=0.6.0; extra == "dev"
+Requires-Dist: black>=24.0.0; extra == "dev"
+Requires-Dist: mypy>=1.11.0; extra == "dev"
+Requires-Dist: types-PyYAML>=6.0; extra == "dev"
+Requires-Dist: types-requests>=2.32; extra == "dev"
 Dynamic: license-file
 
 # Text-to-SQL Evaluation Toolkit
@@ -51,6 +80,10 @@ Dynamic: license-file
 > **A modular framework for evaluating natural language to SQL systems**  
 > Supports execution-based metrics, multiple ground truths, LLM-as-judge, and rich error analysis.
 
+<p align="center">
+  <img src="https://raw.githubusercontent.com/IBM/text2sql-eval-toolkit/main/dashboard-screenshot.png" alt="Text2SQL Evaluation Dashboard: benchmark overview with tiles for each dataset" width="560"/>
+</p>
+
 Text-to-SQL systems aim to translate natural language questions into executable SQL queries, enabling intuitive access to structured data. While recent advances in large language models have significantly improved generation quality, evaluating these systems remains a complex and critical challenge.
 
 *Why an Evaluation Toolkit?*
@@ -76,17 +109,33 @@ Assessing the correctness and utility of generated SQL queries requires more tha
 
 Whether you're building new models, comparing existing ones, or diagnosing performance bottlenecks, this toolkit provides the resources needed to evaluate text-to-SQL systems with rigor and clarity.
 
+## Documentation
+
+- **[Documentation site](https://text2sql-eval-toolkit.readthedocs.io/)** — the
+  guide (installation, the five stages, the data model, benchmarks, models,
+  LLM-as-judge, the CLI and configuration) and the
+  [API reference](https://text2sql-eval-toolkit.readthedocs.io/en/latest/reference/) for
+  every exported function and class.
+- **[docs/dashboard/](https://github.com/IBM/text2sql-eval-toolkit/tree/main/docs/dashboard)** — the evaluation dashboard: features,
+  shareable links, the query index, capability tiers, and deployment.
+- **[data/benchmarks/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks/README.md)** — benchmark
+  definitions and configuration.
+- **[data/benchmarks/dbs/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks/dbs/README.md)** — database
+  setup for each benchmark.
+- **[CONTRIBUTING.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/CONTRIBUTING.md)** — toolchain, tests, and coding standards.
+
 ## Components
 
 <p align="center">
-  <img src="toolkit.png" alt="Text-to-SQL Toolkit Components" width="500"/>
+  <img src="https://raw.githubusercontent.com/IBM/text2sql-eval-toolkit/main/toolkit.png" alt="Text-to-SQL Toolkit Components" width="500"/>
 </p>
 
-- **Evaluation** ([`scripts/evaluation`](scripts/evaluation)/`src/evaluation.py`): Includes a script and library for evaluating text-to-SQL model outputs using various metrics. See [`scripts/evaluation/README.md`](scripts/evaluation/README.md) for details.
-- **Text-to-SQL Inference** ([`scripts/inference`](scripts/inference)): Provides a simple LLM inference pipeline for generating SQL queries from natural language inputs. Run `python scripts/inference/run_inference.py -h` for more information.
-- **SQL Execution** ([`scripts/execution`](scripts/execution)): Runs the ground truth and predicted SQLs for a given benchmark and saves the dataframes for evaluation and error analysis. Run `python scripts/execution/run_execution.py -h` for more information.
-- **Results and Error Analysis** ([`scripts/analysis`](`scripts/analysis`)): Contains scripts and utilities for analyzing evaluation results, identifying common error patterns, and generating summary statistics and visualizations. Useful for debugging and improving model performance.
-- **SQL Profiling** ([`scripts/profiling`](`scripts/profiling`)) tools to profile SQL queries to gather query characteristics to facilitate better analysis of results and errors.
+- **Evaluation** ([`scripts/evaluation`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/scripts/evaluation)/`src/evaluation.py`): Includes a script and library for evaluating text-to-SQL model outputs using various metrics. See [`scripts/evaluation/README.md`](https://github.com/IBM/text2sql-eval-toolkit/blob/main/scripts/evaluation/README.md) for details.
+- **Text-to-SQL Inference** ([`scripts/inference`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/scripts/inference)): Provides a simple LLM inference pipeline for generating SQL queries from natural language inputs. Run `python scripts/inference/run_inference.py -h` for more information.
+- **SQL Execution** ([`scripts/execution`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/scripts/execution)): Runs the ground truth and predicted SQLs for a given benchmark and saves the dataframes for evaluation and error analysis. Run `python scripts/execution/run_execution.py -h` for more information.
+- **Results and Error Analysis** ([`scripts/analysis`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/scripts/analysis)): Contains scripts and utilities for analyzing evaluation results, identifying common error patterns, and generating summary statistics and visualizations. Useful for debugging and improving model performance.
+- **SQL Profiling** ([`scripts/profiling`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/scripts/profiling)) tools to profile SQL queries to gather query characteristics to facilitate better analysis of results and errors.
+- **Evaluation Dashboard** ([`dashboard`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/dashboard)): Optional FastAPI + React web UI for browsing benchmarks and pipeline metrics, error analysis (search, filters, cross-pipeline disagreement), side-by-side comparison of result summaries, editing LLM-as-judge YAML, and launching evaluations with job status. See [docs/dashboard/](https://github.com/IBM/text2sql-eval-toolkit/tree/main/docs/dashboard) for features, deployment, and development.
 
 ## Setup
 
@@ -110,6 +159,32 @@ pip install uv
 brew install uv
 ```
 
+## Pre-computed results
+
+Zero-shot and agentic baseline results for all packaged benchmarks are
+hosted on the [Hugging Face Hub](https://huggingface.co/datasets/text2sql-eval-toolkit/text2sql-eval-results).
+To download them (~4 GB) into `${TEXT2SQL_DATA_ROOT:-./data}/results/`:
+
+```bash
+text2sql-eval-toolkit results fetch
+```
+
+After this completes, the dashboard and analysis scripts will work
+against the downloaded artefacts. See [docs/dashboard/](https://github.com/IBM/text2sql-eval-toolkit/tree/main/docs/dashboard) to launch
+the UI.
+
+To fetch only a specific benchmark:
+
+```bash
+text2sql-eval-toolkit results fetch --benchmarks bird_mini_dev_sqlite
+```
+
+To list what is available without downloading:
+
+```bash
+text2sql-eval-toolkit results list
+```
+
 ## Installation
 
 ### From PyPI
@@ -156,13 +231,13 @@ pip install -e .
 
 ## Additional Requirements
 
-To use features that rely on LLM inference, you need to set the required environment variables for the relevant services. For watsonx.ai, this includes setting the following environment variables: `WATSONX_APIKEY`, `WATSONX_API_BASE`, and `WATSONX_PROJECTID`. See [env.example](env.example) for all available configuration options.
+To use features that rely on LLM inference, you need to set the required environment variables for the relevant services. For watsonx.ai, this includes setting the following environment variables: `WATSONX_APIKEY`, `WATSONX_API_BASE`, and `WATSONX_PROJECTID`. See [env.example](https://github.com/IBM/text2sql-eval-toolkit/blob/main/env.example) for all available configuration options.
 
 ### Benchmarks
 
 The toolkit comes with pre-defined public benchmarks including BIRD-SQL, Spider, Beaver, and Archer. These benchmarks require initial setup such as downloading databases or configuring connection strings. The toolkit also includes smaller test benchmarks (subsets of the full benchmarks) for quick validation and CI/CD pipelines.
 
-**📚 See [data/benchmarks/README.md](data/benchmarks/README.md) for complete benchmark details, setup instructions, and configuration.**
+**📚 See [data/benchmarks/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks/README.md) for complete benchmark details, setup instructions, and configuration.**
 
 ## Usage
 
@@ -241,7 +316,7 @@ agentic.run_pipeline(
 )
 ```
 
-See the docstrings of the exported functions/classes in `text2sql_eval_toolkit.__init__` for the full list of public APIs.
+Every exported function and class is documented in the **[API reference](https://text2sql-eval-toolkit.readthedocs.io/en/latest/reference/)**, generated from the docstrings.
 
 ### Running Experiments
 
@@ -250,7 +325,7 @@ See the docstrings of the exported functions/classes in `text2sql_eval_toolkit._
 python scripts/run_experiment.py [benchmark_name]
 ```
 
-This will initialize a pipeline, perform inference to generate SQLs and output dataframes, and run the evaluation script to gather accuracy metrics. Benchmark names and configurations are defined in [`data/benchmarks.json`](data/benchmarks.json) (for full benchmarks) or [`data/test-benchmarks.json`](data/test-benchmarks.json) (for test benchmarks). See [data/benchmarks/README.md](data/benchmarks/README.md) for more details. Run `python scripts/run_experiment.py -h` for details on how to set different input arguments for the script.
+This will initialize a pipeline, perform inference to generate SQLs and output dataframes, and run the evaluation script to gather accuracy metrics. Benchmark names and configurations are defined in [`data/benchmarks.json`](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks.json) (for full benchmarks) or [`data/test-benchmarks.json`](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/test-benchmarks.json) (for test benchmarks). See [data/benchmarks/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks/README.md) for more details. Run `python scripts/run_experiment.py -h` for details on how to set different input arguments for the script.
 
 **Multiple Benchmarks:**
 ```bash
@@ -264,7 +339,7 @@ python scripts/run_all_benchmarks.py
 python scripts/run_all_benchmarks.py --test --run_all_baselines
 ```
 
-The `run_all_benchmarks.py` script supports separate model configurations for standard and agentic baselines to optimize runtime. See [data/benchmarks/README.md](data/benchmarks/README.md#model-configuration) for detailed configuration options.
+The `run_all_benchmarks.py` script supports separate model configurations for standard and agentic baselines to optimize runtime. See [data/benchmarks/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks/README.md#model-configuration) for detailed configuration options.
 
 ### Running Individual Components
 
@@ -274,7 +349,7 @@ To run evaluation only:
 python scripts/evaluation/run_evaluation.py [-h] [--output_file OUTPUT_FILE] [--summary_file SUMMARY_FILE] [--csv_summary_file CSV_SUMMARY_FILE] [--use_llm_judge] input_file
 ```
 
-See [`scripts/evaluation/README.md`](scripts/evaluation/README.md) for details.
+See [`scripts/evaluation/README.md`](https://github.com/IBM/text2sql-eval-toolkit/blob/main/scripts/evaluation/README.md) for details.
 
 To run inference only:
 
@@ -311,12 +386,33 @@ To produce the results dashboard along with results summary and error analysis M
 python scripts/analysis/make_summary_report.py
 ```
 
-The output will be written to [data/results/README.md](data/results/README.md).
+The output will be written to [data/results/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/results/README.md).
+
+### Evaluation dashboard
+
+A web UI for browsing results, comparing pipelines and doing error analysis:
+
+```bash
+uv pip install -e ".[dashboard]"
+text2sql-eval-dashboard --open-browser
+```
+
+Every view has its own URL, so a benchmark, pipeline, filtered query or
+individual record can be linked to directly.
+
+The dashboard runs as a local tool with every capability enabled, or as a
+shared read-only site with optional sign-in for a small allowlist. See
+**[docs/dashboard/](https://github.com/IBM/text2sql-eval-toolkit/tree/main/docs/dashboard)** for features, the URL scheme, the query
+index, capability tiers, and deployment.
 
 ## Project Structure
 
 ```
 text2sql-eval-toolkit
+├── docs/                       # Dashboard guides, deployment, development history
+├── deploy/                     # Container, Compose stack, and data provisioning
+├── dashboard/                  # React frontend (production build committed to dist/)
+├── tests/                      # Test suite
 ├── notebooks/                  # Jupyter notebooks showcasing the use of the toolkit functions
 ├── data/                       # Benchmark datasets and evaluation results
 │   ├── benchmarks/             # Benchmark data and schema files
@@ -340,10 +436,12 @@ text2sql-eval-toolkit
 │       ├── evaluation/         # Evaluation module
 │       ├── execution/          # SQL execution module
 │       ├── inference/          # LLM inference (baseline) module
-│       └── profiling/          # SQL profiling module
+│       ├── indexing/           # SQLite query index over the result artifacts
+│       ├── profiling/          # SQL profiling module
+│       └── ui/                 # Dashboard backend (FastAPI)
 ├── pyproject.toml              # Build system and project metadata
 ├── README.md                   # Project documentation
-├── requirements.txt            # Python dependencies
+├── requirements.txt            # Pinned export of uv.lock (generated; see its header)
 └── LICENSE                     # License file
 
 ```
@@ -358,7 +456,7 @@ A good way to familiarize yourself with the codebase and contribution process is
 
 **Note: We appreciate your effort, and want to avoid a situation where a contribution requires extensive rework (by you or by us), sits in backlog for a long time, or cannot be accepted at all!**
 
-For detailed contribution guidelines, please see [CONTRIBUTING.md](CONTRIBUTING.md).
+For detailed contribution guidelines, please see [CONTRIBUTING.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/CONTRIBUTING.md).
 
 ### Pull Request Process
 
@@ -370,13 +468,31 @@ For detailed contribution guidelines, please see [CONTRIBUTING.md](CONTRIBUTING.
 
 ## License
 
-This project is licensed under the Apache License 2.0. See the [LICENSE](LICENSE) file for more details.
+This project is licensed under the Apache License 2.0. See the [LICENSE](https://github.com/IBM/text2sql-eval-toolkit/tree/main/LICENSE) file for more details.
+
+## Citation
+
+If you use this toolkit in research, please cite:
+
+```bibtex
+@article{HassanzadehPPKZVGSPR26,
+  title   = {Text-to-{SQL} Evaluation Toolkit},
+  volume  = {19},
+  url     = {https://doi.org/10.14778/3827998.3828071},
+  doi     = {10.14778/3827998.3828071},
+  number  = {12},
+  journal = {Proc. VLDB Endow.},
+  author  = {Hassanzadeh, Oktie and Perlitz, Yotam and Pham, Nhan and Kaple, Tanvi and \.{Z}r\'{o}bek, Karolina and Vu, Long and Glass, Michael and Subramanian, Dharmashankar and Pourreza, Mohammadreza and Rafiei, Davood},
+  year    = {2026},
+  pages   = {4582--4585},
+}
+```
 
 ## Additional Resources
 
-- [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) - Community guidelines
-- [SECURITY.md](SECURITY.md) - Security policy and vulnerability reporting
-- [MAINTAINERS.md](MAINTAINERS.md) - Project maintainers
-- [CHANGELOG.md](CHANGELOG.md) - Project changelog
+- [CODE_OF_CONDUCT.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/CODE_OF_CONDUCT.md) - Community guidelines
+- [SECURITY.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/SECURITY.md) - Security policy and vulnerability reporting
+- [MAINTAINERS.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/MAINTAINERS.md) - Project maintainers
+- [CHANGELOG.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/CHANGELOG.md) - Project changelog
 
 [issues]: https://github.com/IBM/text2sql-eval-toolkit/issues/new

text2sql_eval_toolkit-1.0.0/PKG-INFO → text2sql_eval_toolkit-1.4.0/README.md

@@ -1,56 +1,12 @@
-Metadata-Version: 2.4
-Name: text2sql-eval-toolkit
-Version: 1.0.0
-Summary: A toolkit for evaluating text-to-SQL pipelines
-Author-email: Oktie Hassanzadeh <lastname@us.ibm.com>
-License: MIT
-Project-URL: Homepage, https://github.com/IBM/text2sql-eval-toolkit
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.11
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: ibm_watsonx_ai>=1.3.30
-Requires-Dist: pandas>=2.2.3
-Requires-Dist: requests>=2.32.4
-Requires-Dist: sqlalchemy>=2.0.41
-Requires-Dist: sqlparse>=0.5.3
-Requires-Dist: unitxt>=1.26.1
-Requires-Dist: psycopg2-binary>=2.9.10
-Requires-Dist: sqlglot>=27.0.0
-Requires-Dist: asyncpg>=0.30.0
-Requires-Dist: setuptools>=77.0.3
-Requires-Dist: loguru>=0.7.3
-Requires-Dist: func-timeout>=4.3.5
-Requires-Dist: tqdm>=4.67.1
-Requires-Dist: matplotlib>=3.10.5
-Requires-Dist: python-dotenv>=1.1.1
-Requires-Dist: langgraph>=0.2.0
-Requires-Dist: langchain-core>=0.3.0
-Requires-Dist: openai>=1.0.0
-Provides-Extra: db2
-Requires-Dist: ibm-db>=3.2.6; extra == "db2"
-Requires-Dist: ibm-db-sa>=0.4.0; extra == "db2"
-Requires-Dist: sqlalchemy>=2.0; extra == "db2"
-Provides-Extra: mysql
-Requires-Dist: pymysql>=1.1.1; extra == "mysql"
-Requires-Dist: aiomysql>=0.2.0; extra == "mysql"
-Requires-Dist: greenlet>=3.0.0; extra == "mysql"
-Requires-Dist: cryptography>=45.0.6; extra == "mysql"
-Provides-Extra: presto
-Requires-Dist: pyhive>=0.7.0; extra == "presto"
-Requires-Dist: presto-python-client>=0.8.4; extra == "presto"
-Provides-Extra: notebook
-Requires-Dist: jupyter>=1.0.0; extra == "notebook"
-Requires-Dist: nbconvert>=7.0.0; extra == "notebook"
-Dynamic: license-file
-
 # Text-to-SQL Evaluation Toolkit
 
 > **A modular framework for evaluating natural language to SQL systems**  
 > Supports execution-based metrics, multiple ground truths, LLM-as-judge, and rich error analysis.
 
+<p align="center">
+  <img src="https://raw.githubusercontent.com/IBM/text2sql-eval-toolkit/main/dashboard-screenshot.png" alt="Text2SQL Evaluation Dashboard: benchmark overview with tiles for each dataset" width="560"/>
+</p>
+
 Text-to-SQL systems aim to translate natural language questions into executable SQL queries, enabling intuitive access to structured data. While recent advances in large language models have significantly improved generation quality, evaluating these systems remains a complex and critical challenge.
 
 *Why an Evaluation Toolkit?*
@@ -76,17 +32,33 @@ Assessing the correctness and utility of generated SQL queries requires more tha
 
 Whether you're building new models, comparing existing ones, or diagnosing performance bottlenecks, this toolkit provides the resources needed to evaluate text-to-SQL systems with rigor and clarity.
 
+## Documentation
+
+- **[Documentation site](https://text2sql-eval-toolkit.readthedocs.io/)** — the
+  guide (installation, the five stages, the data model, benchmarks, models,
+  LLM-as-judge, the CLI and configuration) and the
+  [API reference](https://text2sql-eval-toolkit.readthedocs.io/en/latest/reference/) for
+  every exported function and class.
+- **[docs/dashboard/](https://github.com/IBM/text2sql-eval-toolkit/tree/main/docs/dashboard)** — the evaluation dashboard: features,
+  shareable links, the query index, capability tiers, and deployment.
+- **[data/benchmarks/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks/README.md)** — benchmark
+  definitions and configuration.
+- **[data/benchmarks/dbs/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks/dbs/README.md)** — database
+  setup for each benchmark.
+- **[CONTRIBUTING.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/CONTRIBUTING.md)** — toolchain, tests, and coding standards.
+
 ## Components
 
 <p align="center">
-  <img src="toolkit.png" alt="Text-to-SQL Toolkit Components" width="500"/>
+  <img src="https://raw.githubusercontent.com/IBM/text2sql-eval-toolkit/main/toolkit.png" alt="Text-to-SQL Toolkit Components" width="500"/>
 </p>
 
-- **Evaluation** ([`scripts/evaluation`](scripts/evaluation)/`src/evaluation.py`): Includes a script and library for evaluating text-to-SQL model outputs using various metrics. See [`scripts/evaluation/README.md`](scripts/evaluation/README.md) for details.
-- **Text-to-SQL Inference** ([`scripts/inference`](scripts/inference)): Provides a simple LLM inference pipeline for generating SQL queries from natural language inputs. Run `python scripts/inference/run_inference.py -h` for more information.
-- **SQL Execution** ([`scripts/execution`](scripts/execution)): Runs the ground truth and predicted SQLs for a given benchmark and saves the dataframes for evaluation and error analysis. Run `python scripts/execution/run_execution.py -h` for more information.
-- **Results and Error Analysis** ([`scripts/analysis`](`scripts/analysis`)): Contains scripts and utilities for analyzing evaluation results, identifying common error patterns, and generating summary statistics and visualizations. Useful for debugging and improving model performance.
-- **SQL Profiling** ([`scripts/profiling`](`scripts/profiling`)) tools to profile SQL queries to gather query characteristics to facilitate better analysis of results and errors.
+- **Evaluation** ([`scripts/evaluation`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/scripts/evaluation)/`src/evaluation.py`): Includes a script and library for evaluating text-to-SQL model outputs using various metrics. See [`scripts/evaluation/README.md`](https://github.com/IBM/text2sql-eval-toolkit/blob/main/scripts/evaluation/README.md) for details.
+- **Text-to-SQL Inference** ([`scripts/inference`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/scripts/inference)): Provides a simple LLM inference pipeline for generating SQL queries from natural language inputs. Run `python scripts/inference/run_inference.py -h` for more information.
+- **SQL Execution** ([`scripts/execution`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/scripts/execution)): Runs the ground truth and predicted SQLs for a given benchmark and saves the dataframes for evaluation and error analysis. Run `python scripts/execution/run_execution.py -h` for more information.
+- **Results and Error Analysis** ([`scripts/analysis`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/scripts/analysis)): Contains scripts and utilities for analyzing evaluation results, identifying common error patterns, and generating summary statistics and visualizations. Useful for debugging and improving model performance.
+- **SQL Profiling** ([`scripts/profiling`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/scripts/profiling)) tools to profile SQL queries to gather query characteristics to facilitate better analysis of results and errors.
+- **Evaluation Dashboard** ([`dashboard`](https://github.com/IBM/text2sql-eval-toolkit/tree/main/dashboard)): Optional FastAPI + React web UI for browsing benchmarks and pipeline metrics, error analysis (search, filters, cross-pipeline disagreement), side-by-side comparison of result summaries, editing LLM-as-judge YAML, and launching evaluations with job status. See [docs/dashboard/](https://github.com/IBM/text2sql-eval-toolkit/tree/main/docs/dashboard) for features, deployment, and development.
 
 ## Setup
 
@@ -110,6 +82,32 @@ pip install uv
 brew install uv
 ```
 
+## Pre-computed results
+
+Zero-shot and agentic baseline results for all packaged benchmarks are
+hosted on the [Hugging Face Hub](https://huggingface.co/datasets/text2sql-eval-toolkit/text2sql-eval-results).
+To download them (~4 GB) into `${TEXT2SQL_DATA_ROOT:-./data}/results/`:
+
+```bash
+text2sql-eval-toolkit results fetch
+```
+
+After this completes, the dashboard and analysis scripts will work
+against the downloaded artefacts. See [docs/dashboard/](https://github.com/IBM/text2sql-eval-toolkit/tree/main/docs/dashboard) to launch
+the UI.
+
+To fetch only a specific benchmark:
+
+```bash
+text2sql-eval-toolkit results fetch --benchmarks bird_mini_dev_sqlite
+```
+
+To list what is available without downloading:
+
+```bash
+text2sql-eval-toolkit results list
+```
+
 ## Installation
 
 ### From PyPI
@@ -156,13 +154,13 @@ pip install -e .
 
 ## Additional Requirements
 
-To use features that rely on LLM inference, you need to set the required environment variables for the relevant services. For watsonx.ai, this includes setting the following environment variables: `WATSONX_APIKEY`, `WATSONX_API_BASE`, and `WATSONX_PROJECTID`. See [env.example](env.example) for all available configuration options.
+To use features that rely on LLM inference, you need to set the required environment variables for the relevant services. For watsonx.ai, this includes setting the following environment variables: `WATSONX_APIKEY`, `WATSONX_API_BASE`, and `WATSONX_PROJECTID`. See [env.example](https://github.com/IBM/text2sql-eval-toolkit/blob/main/env.example) for all available configuration options.
 
 ### Benchmarks
 
 The toolkit comes with pre-defined public benchmarks including BIRD-SQL, Spider, Beaver, and Archer. These benchmarks require initial setup such as downloading databases or configuring connection strings. The toolkit also includes smaller test benchmarks (subsets of the full benchmarks) for quick validation and CI/CD pipelines.
 
-**📚 See [data/benchmarks/README.md](data/benchmarks/README.md) for complete benchmark details, setup instructions, and configuration.**
+**📚 See [data/benchmarks/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks/README.md) for complete benchmark details, setup instructions, and configuration.**
 
 ## Usage
 
@@ -241,7 +239,7 @@ agentic.run_pipeline(
 )
 ```
 
-See the docstrings of the exported functions/classes in `text2sql_eval_toolkit.__init__` for the full list of public APIs.
+Every exported function and class is documented in the **[API reference](https://text2sql-eval-toolkit.readthedocs.io/en/latest/reference/)**, generated from the docstrings.
 
 ### Running Experiments
 
@@ -250,7 +248,7 @@ See the docstrings of the exported functions/classes in `text2sql_eval_toolkit._
 python scripts/run_experiment.py [benchmark_name]
 ```
 
-This will initialize a pipeline, perform inference to generate SQLs and output dataframes, and run the evaluation script to gather accuracy metrics. Benchmark names and configurations are defined in [`data/benchmarks.json`](data/benchmarks.json) (for full benchmarks) or [`data/test-benchmarks.json`](data/test-benchmarks.json) (for test benchmarks). See [data/benchmarks/README.md](data/benchmarks/README.md) for more details. Run `python scripts/run_experiment.py -h` for details on how to set different input arguments for the script.
+This will initialize a pipeline, perform inference to generate SQLs and output dataframes, and run the evaluation script to gather accuracy metrics. Benchmark names and configurations are defined in [`data/benchmarks.json`](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks.json) (for full benchmarks) or [`data/test-benchmarks.json`](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/test-benchmarks.json) (for test benchmarks). See [data/benchmarks/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks/README.md) for more details. Run `python scripts/run_experiment.py -h` for details on how to set different input arguments for the script.
 
 **Multiple Benchmarks:**
 ```bash
@@ -264,7 +262,7 @@ python scripts/run_all_benchmarks.py
 python scripts/run_all_benchmarks.py --test --run_all_baselines
 ```
 
-The `run_all_benchmarks.py` script supports separate model configurations for standard and agentic baselines to optimize runtime. See [data/benchmarks/README.md](data/benchmarks/README.md#model-configuration) for detailed configuration options.
+The `run_all_benchmarks.py` script supports separate model configurations for standard and agentic baselines to optimize runtime. See [data/benchmarks/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/benchmarks/README.md#model-configuration) for detailed configuration options.
 
 ### Running Individual Components
 
@@ -274,7 +272,7 @@ To run evaluation only:
 python scripts/evaluation/run_evaluation.py [-h] [--output_file OUTPUT_FILE] [--summary_file SUMMARY_FILE] [--csv_summary_file CSV_SUMMARY_FILE] [--use_llm_judge] input_file
 ```
 
-See [`scripts/evaluation/README.md`](scripts/evaluation/README.md) for details.
+See [`scripts/evaluation/README.md`](https://github.com/IBM/text2sql-eval-toolkit/blob/main/scripts/evaluation/README.md) for details.
 
 To run inference only:
 
@@ -311,12 +309,33 @@ To produce the results dashboard along with results summary and error analysis M
 python scripts/analysis/make_summary_report.py
 ```
 
-The output will be written to [data/results/README.md](data/results/README.md).
+The output will be written to [data/results/README.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/data/results/README.md).
+
+### Evaluation dashboard
+
+A web UI for browsing results, comparing pipelines and doing error analysis:
+
+```bash
+uv pip install -e ".[dashboard]"
+text2sql-eval-dashboard --open-browser
+```
+
+Every view has its own URL, so a benchmark, pipeline, filtered query or
+individual record can be linked to directly.
+
+The dashboard runs as a local tool with every capability enabled, or as a
+shared read-only site with optional sign-in for a small allowlist. See
+**[docs/dashboard/](https://github.com/IBM/text2sql-eval-toolkit/tree/main/docs/dashboard)** for features, the URL scheme, the query
+index, capability tiers, and deployment.
 
 ## Project Structure
 
 ```
 text2sql-eval-toolkit
+├── docs/                       # Dashboard guides, deployment, development history
+├── deploy/                     # Container, Compose stack, and data provisioning
+├── dashboard/                  # React frontend (production build committed to dist/)
+├── tests/                      # Test suite
 ├── notebooks/                  # Jupyter notebooks showcasing the use of the toolkit functions
 ├── data/                       # Benchmark datasets and evaluation results
 │   ├── benchmarks/             # Benchmark data and schema files
@@ -340,10 +359,12 @@ text2sql-eval-toolkit
 │       ├── evaluation/         # Evaluation module
 │       ├── execution/          # SQL execution module
 │       ├── inference/          # LLM inference (baseline) module
-│       └── profiling/          # SQL profiling module
+│       ├── indexing/           # SQLite query index over the result artifacts
+│       ├── profiling/          # SQL profiling module
+│       └── ui/                 # Dashboard backend (FastAPI)
 ├── pyproject.toml              # Build system and project metadata
 ├── README.md                   # Project documentation
-├── requirements.txt            # Python dependencies
+├── requirements.txt            # Pinned export of uv.lock (generated; see its header)
 └── LICENSE                     # License file
 
 ```
@@ -358,7 +379,7 @@ A good way to familiarize yourself with the codebase and contribution process is
 
 **Note: We appreciate your effort, and want to avoid a situation where a contribution requires extensive rework (by you or by us), sits in backlog for a long time, or cannot be accepted at all!**
 
-For detailed contribution guidelines, please see [CONTRIBUTING.md](CONTRIBUTING.md).
+For detailed contribution guidelines, please see [CONTRIBUTING.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/CONTRIBUTING.md).
 
 ### Pull Request Process
 
@@ -370,13 +391,31 @@ For detailed contribution guidelines, please see [CONTRIBUTING.md](CONTRIBUTING.
 
 ## License
 
-This project is licensed under the Apache License 2.0. See the [LICENSE](LICENSE) file for more details.
+This project is licensed under the Apache License 2.0. See the [LICENSE](https://github.com/IBM/text2sql-eval-toolkit/tree/main/LICENSE) file for more details.
+
+## Citation
+
+If you use this toolkit in research, please cite:
+
+```bibtex
+@article{HassanzadehPPKZVGSPR26,
+  title   = {Text-to-{SQL} Evaluation Toolkit},
+  volume  = {19},
+  url     = {https://doi.org/10.14778/3827998.3828071},
+  doi     = {10.14778/3827998.3828071},
+  number  = {12},
+  journal = {Proc. VLDB Endow.},
+  author  = {Hassanzadeh, Oktie and Perlitz, Yotam and Pham, Nhan and Kaple, Tanvi and \.{Z}r\'{o}bek, Karolina and Vu, Long and Glass, Michael and Subramanian, Dharmashankar and Pourreza, Mohammadreza and Rafiei, Davood},
+  year    = {2026},
+  pages   = {4582--4585},
+}
+```
 
 ## Additional Resources
 
-- [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) - Community guidelines
-- [SECURITY.md](SECURITY.md) - Security policy and vulnerability reporting
-- [MAINTAINERS.md](MAINTAINERS.md) - Project maintainers
-- [CHANGELOG.md](CHANGELOG.md) - Project changelog
+- [CODE_OF_CONDUCT.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/CODE_OF_CONDUCT.md) - Community guidelines
+- [SECURITY.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/SECURITY.md) - Security policy and vulnerability reporting
+- [MAINTAINERS.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/MAINTAINERS.md) - Project maintainers
+- [CHANGELOG.md](https://github.com/IBM/text2sql-eval-toolkit/blob/main/CHANGELOG.md) - Project changelog
 
 [issues]: https://github.com/IBM/text2sql-eval-toolkit/issues/new

text2sql_eval_toolkit-1.4.0/dashboard/dist/assets/BenchmarkDetail-gEcRuzmW.js

@@ -0,0 +1,4 @@
+import{R as p,u as Ne,a as Be,r as l,b as Oe,F as Ke,c as We,d as ee,m as He,i as qe,_ as te,W as Ue,e as Ve,C as Qe,f as Xe,s as Ge,g as Je,h as Ye,A as Ze,P as i,j as ue,k as s,I as ve,l as et,D as tt,B as se,n as st,T as at,o as it,p as nt,q as je,t as rt,v as lt,w as ot,x as dt,y as ct,z as ut}from"./index-DLyduwv0.js";import{d as _e,u as $e,L as ae,a as mt,b as pt}from"./defaultItemToString-ClUBp4QC.js";const{ItemMouseMove:ht,MenuMouseLeave:ft,ToggleButtonBlur:gt,FunctionCloseMenu:bt}=$e.stateChangeTypes;function Ie(o,E){const{changes:h,type:S}=E;switch(S){case ht:return o;case ft:return h.highlightedIndex===o.highlightedIndex?o:h;case gt:case bt:return{...h,selectedItem:o.selectedItem};default:return h}}const ie=p.forwardRef(({autoAlign:o=!1,className:E,decorator:h,disabled:S=!1,direction:z="bottom",items:R,label:A,["aria-label"]:I,ariaLabel:Q,itemToString:b=_e,itemToElement:L=null,renderSelectedItem:N,type:X="default",size:D,onChange:y,id:F,titleText:B="",hideLabel:m,helperText:O="",translateWithId:M,light:K,invalid:G,invalidText:$,warn:ne,warnText:P,initialSelectedItem:W,selectedItem:H,downshiftProps:J,readOnly:w,slug:v,...re},Y)=>{const C=Ne("enable-v12-dynamic-floating-styles"),{refs:j,floatingStyles:q,middlewareData:le}=Be(C||o?{placement:z,strategy:"fixed",middleware:[Ge({apply({rects:a,elements:c}){Object.assign(c.floating.style,{width:`${a.reference.width}px`})}}),o&&Je(),o&&Ye()],whileElementsMounted:Xe}:{});l.useEffect(()=>{var a;if(C||o){const c={...q,visibility:(a=le.hide)!=null&&a.referenceHidden?"hidden":"visible"};Object.keys(c).forEach(_=>{j.floating.current&&(j.floating.current.style[_]=c[_])})}},[q,o,j.floating]);const e=Oe(),{isFluid:t}=l.useContext(Ke),n=l.useCallback(({selectedItem:a})=>{y&&y({selectedItem:a??null})},[y]),u=l.useCallback((a,c)=>a!==null&&typeof a=="object"&&"disabled"in a&&a.disabled===!0,[]),d=l.useCallback(a=>{const{highlightedIndex:c}=a;if(c!==void 0&&c>-1){const Z=document.querySelectorAll(`li.${e}--list-box__menu-item[role="option"]`)[c];Z&&Z.scrollIntoView({behavior:"smooth",block:"nearest"})}},[e]),f=l.useMemo(()=>R,[R]),k=l.useMemo(()=>({items:f,itemToString:b,initialSelectedItem:W,onSelectedItemChange:n,stateReducer:Ie,isItemDisabled:u,onHighlightedIndexChange:d,...J}),[f,b,W,n,Ie,u,d,J]);H!==void 0&&(k.selectedItem=H);const{isOpen:g,getToggleButtonProps:Ce,getLabelProps:ke,getMenuProps:me,getItemProps:Te,selectedItem:x,highlightedIndex:Ee}=$e(k),T=X==="inline",r=We({id:F,readOnly:w,disabled:S??!1,invalid:G??!1,invalidText:$,warn:ne??!1,warnText:P}),[Se,oe]=l.useState(!1),De=ee(`${e}--dropdown`,{[`${e}--dropdown--invalid`]:r.invalid,[`${e}--dropdown--warning`]:r.warn,[`${e}--dropdown--open`]:g,[`${e}--dropdown--focus`]:Se,[`${e}--dropdown--inline`]:T,[`${e}--dropdown--disabled`]:r.disabled,[`${e}--dropdown--light`]:K,[`${e}--dropdown--readonly`]:w,[`${e}--dropdown--${D}`]:D,[`${e}--list-box--up`]:z==="top",[`${e}--autoalign`]:o}),Fe=ee(`${e}--label`,{[`${e}--label--disabled`]:r.disabled,[`${e}--visually-hidden`]:m}),Me=ee(`${e}--form__helper-text`,{[`${e}--form__helper-text--disabled`]:r.disabled}),Pe=ee(`${e}--dropdown__wrapper`,`${e}--list-box__wrapper`,E,{[`${e}--dropdown__wrapper--inline`]:T,[`${e}--list-box__wrapper--inline`]:T,[`${e}--dropdown__wrapper--inline--invalid`]:T&&r.invalid,[`${e}--list-box__wrapper--inline--invalid`]:T&&r.invalid,[`${e}--list-box__wrapper--fluid--invalid`]:t&&r.invalid,[`${e}--list-box__wrapper--slug`]:v,[`${e}--list-box__wrapper--decorator`]:h}),U=Ce({"aria-label":I||Q}),pe=O&&!t?p.createElement("div",{id:r.helperId,className:Me},O):null,he=a=>{oe(a.type==="focus"&&!x)},fe=l.useRef(null),ze=He(U.ref,Y,fe),[V,ge]=l.useState(),[be,de]=l.useState(!1),xe=l.useCallback(a=>{["ArrowDown","ArrowUp"," ","Enter"].includes(a.key)?be&&a.key===" "&&(V&&clearTimeout(V),ge(setTimeout(()=>{de(!1)},3e3))):(de(!0),V&&clearTimeout(V),ge(setTimeout(()=>{de(!1)},3e3))),["ArrowDown"].includes(a.key)&&oe(!1),["Enter"].includes(a.key)&&!x&&!g&&oe(!0),U.onKeyDown&&(a.key!=="ArrowUp"||g&&a.key==="ArrowUp")&&U.onKeyDown(a)},[be,V,U]),Re=l.useMemo(()=>w?{onClick:a=>{var c;a.preventDefault(),(c=fe.current)==null||c.focus()},onKeyDown:a=>{["ArrowDown","ArrowUp"," ","Enter"].includes(a.key)&&a.preventDefault()}}:{onKeyDown:xe},[w,xe]),Ae=l.useMemo(()=>me({ref:C||o?j.setFloating:null}),[o,me,j.setFloating,C]),ce=v??h,ye=qe(ce,Ze)?l.cloneElement(ce,{size:"mini"}):ce,we=ke(),Le=l.isValidElement(B)?{id:we.id}:we;return p.createElement("div",te({className:Pe},re),B&&p.createElement("label",te({className:Fe},Le),B),p.createElement(ae,{onFocus:he,onBlur:he,size:D,className:De,invalid:r.invalid,invalidText:$,invalidTextId:r.invalidId,warn:r.warn,warnText:P,warnTextId:r.warnId,light:K,isOpen:g,ref:C||o?j.setReference:null,id:F},r.invalid&&p.createElement(Ue,{className:`${e}--list-box__invalid-icon`}),r.warn&&p.createElement(Ve,{className:`${e}--list-box__invalid-icon ${e}--list-box__invalid-icon--warning`}),p.createElement("button",te({type:"button",className:`${e}--list-box__field`,disabled:r.disabled,"aria-disabled":w?!0:void 0,"aria-describedby":!T&&!r.invalid&&!r.warn&&pe?r.helperId:r.invalid?r.invalidId:r.warn?r.warnId:void 0,title:x&&b!==void 0?b(x):_e(A)},U,Re,{ref:ze}),p.createElement("span",{className:`${e}--list-box__label`},x?N?N(x):b(x):A),p.createElement(ae.MenuIcon,{isOpen:g,translateWithId:M})),v?ye:h?p.createElement("div",{className:`${e}--list-box__inner-wrapper--decorator`},ye):"",p.createElement(ae.Menu,Ae,g&&f.map((a,c)=>{const _=Te({item:a,index:c}),Z=b(a);return p.createElement(ae.MenuItem,te({key:_.id,isActive:x===a,isHighlighted:Ee===c,title:Z,disabled:_["aria-disabled"]},_),L?L(a):b(a),x===a&&p.createElement(Qe,{className:`${e}--list-box__menu-item__selected-icon`}))}))),!T&&!t&&!r.validation&&pe)});ie.displayName="Dropdown";ie.propTypes={"aria-label":i.string,ariaLabel:ue(i.string),autoAlign:i.bool,className:i.string,decorator:i.node,direction:i.oneOf(["top","bottom"]),disabled:i.bool,downshiftProps:i.object,helperText:i.node,hideLabel:i.bool,id:i.string.isRequired,initialSelectedItem:i.oneOfType([i.object,i.string,i.number]),invalid:i.bool,invalidText:i.node,itemToElement:i.func,itemToString:i.func,items:i.array.isRequired,label:i.node.isRequired,light:ue(i.bool),onChange:i.func,readOnly:i.bool,renderSelectedItem:i.func,selectedItem:i.oneOfType([i.object,i.string,i.number]),size:pt,slug:ue(i.node),titleText:i.node.isRequired,translateWithId:i.func,type:mt,warn:i.bool,warnText:i.node};const vt=({benchmarkId:o,style:E,onSelectPipeline:h,onOpenToolkitInsights:S,onOpenPipelineCompare:z,onOpenProfileCompare:R,onOpenErrorAnalysis:A})=>{const[I,Q]=l.useState(null),[b,L]=l.useState(!0),[N,X]=l.useState(null),[D,y]=l.useState(1),[F,B]=l.useState(10),[m,O]=l.useState(null),[M,K]=l.useState("overall");if(l.useEffect(()=>{(async()=>{try{L(!0),X(null),Q(null);const n=await(await ct(ut(`/api/benchmarks/${o}/summary/by-category`))).json();Q(n),O(n.default_sort_metric),K("overall"),y(1)}catch(t){X(t.message||"Failed to load benchmark summary")}finally{L(!1)}})()},[o]),N)return s.jsx(ve,{kind:"error",title:`Error loading summary for ${o}`,subtitle:N,lowContrast:!0});if(b)return s.jsxs("div",{style:{...E,display:"flex",flexDirection:"column",gap:"1rem"},children:[s.jsxs("h3",{style:{margin:0},children:[o," – Summary"]}),s.jsx(et,{description:`Loading benchmark summary for ${o}…`,status:"active"}),s.jsx(tt,{role:"progressbar",columnCount:5,rowCount:8})]});if(!I)return null;const G=M==="overall"?I.overall:I.categories[M]||[],$=new Set;G.forEach(e=>{Object.keys(e.metrics).forEach(t=>{t!=="num_records"&&t!=="num_evaluated"&&$.add(t)})});const ne=[{key:"name",header:"Pipeline"},...Array.from($).map(e=>({key:e,header:e}))],P=[...G].sort((e,t)=>{var d,f,k,g;if(!m)return 0;const n=((f=(d=e.metrics)==null?void 0:d[m])==null?void 0:f.average)??0;return(((g=(k=t.metrics)==null?void 0:k[m])==null?void 0:g.average)??0)-n}),W=e=>{var n;if(!m)return 0;const t=(n=e.metrics)==null?void 0:n[m];return typeof t=="number"?t:t&&typeof t=="object"&&typeof t.average=="number"?t.average:0},H=e=>{var n;if(!m)return 0;const t=(n=e.metrics)==null?void 0:n[m];return t&&typeof t=="object"&&typeof t.stddev=="number"?Math.max(0,t.stddev):0},J=e=>{var d;if(!m)return{low:0,high:0,hasCI:!1};const t=(d=e.metrics)==null?void 0:d[m];if(t&&typeof t=="object"&&typeof t.ci95_low=="number"&&typeof t.ci95_high=="number")return{low:t.ci95_low,high:t.ci95_high,hasCI:!0};const n=W(e),u=H(e);return{low:Math.max(0,n-u),high:n+u,hasCI:!1}},w=P.map(e=>({name:e.name,value:W(e),stddev:H(e),ci:J(e)})),v=w.reduce((e,t)=>Math.max(e,t.value,t.ci.high),0)||1,re=P.length,Y=(D-1)*F,C=Y+F,j=P.slice(Y,C),q=Array.from($),le=["overall",...Object.keys(I.categories)];return s.jsxs("div",{style:{...E,display:"flex",flexDirection:"column",gap:"0.5rem"},children:[I.has_full_results===!1&&s.jsx(ve,{kind:"info",title:"Summary-only mode",subtitle:`The full evaluation results file (${o}-predictions_eval.json) was not found. Overall pipeline metrics are shown from the summary file, but category breakdown and error analysis are unavailable. Download pre-computed results with: text2sql-eval-toolkit results fetch --benchmarks ${o} or run the evaluation pipeline locally (or fix TEXT2SQL_DATA_ROOT to a directory that already contains this file).`,lowContrast:!0}),s.jsxs("div",{style:{display:"flex",gap:"1rem",alignItems:"center"},children:[s.jsxs("h3",{style:{margin:0},children:[o," – Summary"]}),s.jsxs("div",{style:{marginLeft:"auto",display:"flex",gap:"0.75rem",alignItems:"center"},children:[S&&s.jsx(se,{kind:"ghost",size:"sm",onClick:()=>S(),children:"Metric Insights"}),z&&s.jsx(se,{kind:"ghost",size:"sm",onClick:()=>z(),children:"Pipeline Compare"}),R&&s.jsx(se,{kind:"ghost",size:"sm",onClick:()=>R(),children:"Profile Compare"}),A&&s.jsx(se,{kind:"ghost",size:"sm",onClick:()=>A(),children:"Error Analysis"}),s.jsx("div",{style:{minWidth:"220px"},children:s.jsx(ie,{id:"category-dropdown",titleText:"Query Category",label:M,items:le,selectedItem:M,onChange:e=>{K(e.selectedItem),y(1)}})}),q.length>0&&s.jsx("div",{style:{minWidth:"260px"},children:s.jsx(ie,{id:"sort-metric-dropdown",titleText:"Sort by metric",label:m||"Select metric",items:q,selectedItem:m,onChange:e=>{O(e.selectedItem),y(1)}})})]})]}),s.jsxs("div",{style:{border:"1px solid rgba(255,255,255,0.12)",borderRadius:"6px",padding:"0.75rem",background:"rgba(255,255,255,0.02)"},children:[s.jsxs("div",{style:{marginBottom:"0.5rem",fontWeight:600},children:["Pipeline performance by ",m||"selected metric"]}),s.jsx("div",{style:{maxHeight:"260px",overflow:"auto",display:"flex",flexDirection:"column",gap:"0.45rem"},children:w.map(e=>{const t=Math.max(2,e.value/v*100),n=Math.max(0,e.ci.low),u=Math.min(v,e.ci.high),d=n/v*100,f=u/v*100,k=Math.max(0,f-d),g=f>=99.9;return s.jsxs("div",{title:e.ci.hasCI?`${e.name}
+Mean: ${e.value.toFixed(3)}
+95% CI: [${e.ci.low.toFixed(3)}, ${e.ci.high.toFixed(3)}]`:`${e.name}
+Mean: ${e.value.toFixed(3)}`,style:{display:"grid",gridTemplateColumns:"260px 1fr 56px",gap:"0.6rem",alignItems:"center"},children:[s.jsx("div",{title:e.name,style:{whiteSpace:"nowrap",overflow:"hidden",textOverflow:"ellipsis",fontSize:"0.82rem",opacity:.9},children:e.name}),s.jsxs("div",{style:{height:"12px",background:"rgba(255,255,255,0.1)",borderRadius:"999px",overflow:"hidden",position:"relative"},children:[s.jsx("div",{style:{width:`${t}%`,height:"100%",background:"linear-gradient(90deg, #0f62fe, #78a9ff)",borderRadius:"999px"}}),u>n&&s.jsxs(s.Fragment,{children:[s.jsx("div",{style:{position:"absolute",left:`${d}%`,width:`${k}%`,top:"5.5px",height:"1px",background:"rgba(255,255,255,0.7)"}}),s.jsx("div",{style:{position:"absolute",left:`${d}%`,top:"3px",width:"1px",height:"6px",background:"rgba(255,255,255,0.7)"}}),s.jsx("div",{style:{position:"absolute",...g?{right:0}:{left:`${f}%`},top:"3px",width:"1px",height:"6px",background:"rgba(255,255,255,0.7)"}})]})]}),s.jsx("div",{style:{textAlign:"right",fontSize:"0.82rem"},children:e.value.toFixed(3)})]},`chart-${e.name}`)})})]}),s.jsxs("div",{style:{maxHeight:"360px",overflow:"auto"},children:[s.jsx("div",{style:{fontSize:"0.82rem",opacity:.85,marginBottom:"0.45rem"},children:"Tip: click a pipeline row in the table below for detailed pipeline analysis."}),s.jsx(st,{rows:j.map(e=>({id:e.name,name:e.name,...Object.fromEntries(Array.from($).map(t=>{const n=e.metrics[t];return n&&typeof n=="object"&&"average"in n?[t,n.average.toFixed(3)]:[t,n??""]}))})),headers:ne,size:"sm",children:({rows:e,headers:t,getHeaderProps:n})=>s.jsx(at,{children:s.jsxs(it,{"aria-label":"Pipeline summary",children:[s.jsx(nt,{children:s.jsx(je,{children:t.map(u=>{const{key:d,...f}=n({header:u});return s.jsx(rt,{...f,children:u.header},d)})})}),s.jsx(lt,{children:e.map(u=>s.jsx(je,{style:{cursor:h?"pointer":"default"},onClick:()=>h==null?void 0:h(u.id),children:u.cells.map(d=>s.jsx(ot,{children:d.value},d.id))},u.id))})]})})})]}),s.jsx(dt,{page:D,pageSize:F,pageSizes:[10,25,50],totalItems:re,onChange:({page:e,pageSize:t})=>{y(e),B(t)}})]})};export{vt as BenchmarkDetail};