evalsuite-python 0.3.1__tar.gz → 0.4.1__tar.gz

{evalsuite_python-0.3.1 → evalsuite_python-0.4.1}/CHANGELOG.md

@@ -6,6 +6,63 @@ All notable changes to this project are documented here. The format follows
 
 ## [Unreleased]
 
+## [0.4.1] - 2026-10-09
+
+Benchmarks for every metric. No API changed.
+
+### Added
+- `run_benchmarks(suite="metrics")` and `evalsuite benchmark --suite metrics`: one row for each of the 147
+  registered metrics and statistics functions, compared with a reference library (scikit-learn, SciPy,
+  statsmodels, pycocotools, sacreBLEU, rouge-score, NLTK, pycocoevalcap, ranx, krippendorff, choix, POT,
+  jsonschema) where one exists, otherwise with an independent NumPy / standard-library implementation of
+  the textbook formula; learned, judge-dependent and randomised procedures are timed alone. All 128
+  comparisons agree to 1e-9. A test keeps the suite in step with the registry.
+- `scripts/bench_metrics_report.py` renders the per-metric table for BENCHMARKS.md and the website.
+
+### Changed
+- `net_benefit` at a single threshold computes in one pass instead of sorting (about 3× faster).
+- `perplexity` and `cross_entropy` validate the pooled tokens once instead of each sequence separately
+  (5–9× faster for many short sequences); errors still name the offending sequence.
+
+## [0.4.0] - 2026-10-09
+
+LLM evaluation (the v0.4.0 roadmap). No existing API changed.
+
+### Added
+- `es.text` and top-level functions for language-model evaluation; 69 new registered metrics, each checked
+  against a reference implementation where one exists:
+  - Text generation: BLEU, sentence BLEU, chrF/chrF++, TER (sacreBLEU), ROUGE-1/2/L/Lsum (rouge-score),
+    METEOR (NLTK, synonyms pluggable), CIDEr-D (pycocoevalcap), perplexity, cross-entropy, Distinct-n,
+    Self-BLEU, MAUVE (mauve-text divergence frontier, NumPy k-means).
+  - Semantic: BERTScore, embedding cosine / Euclidean / Manhattan, MoverScore (exact transport, = POT), and
+    `model_score` to run COMET, BLEURT, BARTScore, AlignScore or any scorer with intervals and comparison.
+  - Factuality: faithfulness, hallucination (unsupported-claim) rate, groundedness, citation precision and
+    recall (ALCE), claim-verification accuracy, knowledge consistency, answer correctness and relevance
+    (RAGAS), abstention accuracy, exact match and token F1 (SQuAD).
+  - LLM-as-a-judge: win rate, Bradley–Terry (= choix), Elo, Krippendorff's alpha (= krippendorff), Fleiss'
+    kappa (= statsmodels), judge–human agreement, position consistency, verbosity and self-preference bias,
+    rubric scores.
+  - Reasoning: pass@k, majority-vote accuracy, answer extraction and benchmark accuracy (GSM8K, MATH,
+    multiple choice).
+  - RAG: Precision/Recall/Hit rate@k, MRR, MAP, NDCG (= ranx), context precision, recall and relevance,
+    latency summary, task success rate, failure attribution.
+  - Structured output: JSON validity, JSON Schema compliance (built-in validator, agrees with jsonschema),
+    XML validity, required-field accuracy, tool selection / argument accuracy and tool-call F1, API-call
+    success, instruction compliance, constraint satisfaction, format compliance, multi-turn retention,
+    extra-content rate.
+- `es.text_report` and `evalsuite text` (CLI) for the reference-based text metrics in one report.
+- `bootstrap_ci`, `compare` and the paired tests resample whole items (texts, reference lists, rankings) for
+  these metrics, so corpus metrics such as BLEU get proper intervals.
+- Benchmark suite `llm` against sacreBLEU, rouge-score, NLTK, ranx, krippendorff and jsonschema.
+- `llm` extra (`nltk`) for METEOR's default Porter stemmer.
+- Plots: `es.plot.ratings` (Bradley–Terry or Elo leaderboard with bootstrap intervals), `es.plot.win_matrix`
+  (pairwise win rates) and `es.plot.text_scores` (per-example score distributions by system).
+- Benchmark overall summary counts cases without their size suffixes.
+
+### Not included
+- SPICE (needs a Java scene-graph parser); BLEURT, COMET, BARTScore and AlignScore run through
+  `es.model_score` with your model rather than being bundled.
+
 ## [0.3.1] - 2026-10-09
 
 Quality-assurance release: release pipeline, packaging checks, edge-case fixes and an overall benchmark.

{evalsuite_python-0.3.1 → evalsuite_python-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: evalsuite-python
-Version: 0.3.1
+Version: 0.4.1
 Summary: A unified Python framework for machine-learning, clinical, statistical, segmentation, object-detection, uncertainty, and model evaluation.
 Project-URL: Homepage, https://evalsuite-nine.vercel.app
 Project-URL: Documentation, https://evalsuite-nine.vercel.app/docs
@@ -36,22 +36,34 @@ Requires-Dist: pandas>=1.4
 Requires-Dist: scipy>=1.8
 Provides-Extra: all
 Requires-Dist: matplotlib>=3.5; extra == 'all'
+Requires-Dist: nltk>=3.8; extra == 'all'
 Requires-Dist: pillow>=9; extra == 'all'
 Provides-Extra: dev
 Requires-Dist: build; extra == 'dev'
+Requires-Dist: choix>=0.3; extra == 'dev'
 Requires-Dist: hypothesis>=6.80; extra == 'dev'
+Requires-Dist: jsonschema>=4.18; extra == 'dev'
+Requires-Dist: krippendorff>=0.6; extra == 'dev'
 Requires-Dist: matplotlib>=3.5; extra == 'dev'
 Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: nltk>=3.8; extra == 'dev'
 Requires-Dist: pandas-stubs; extra == 'dev'
 Requires-Dist: pillow>=9; extra == 'dev'
 Requires-Dist: pip-audit; extra == 'dev'
+Requires-Dist: pot>=0.9; (python_version < '3.14') and extra == 'dev'
+Requires-Dist: pycocoevalcap>=1.2; extra == 'dev'
 Requires-Dist: pycocotools>=2.0.7; extra == 'dev'
 Requires-Dist: pytest-cov>=4; extra == 'dev'
 Requires-Dist: pytest>=7; extra == 'dev'
+Requires-Dist: ranx>=0.3.19; (python_version < '3.14') and extra == 'dev'
+Requires-Dist: rouge-score>=0.1.2; extra == 'dev'
 Requires-Dist: ruff>=0.6; extra == 'dev'
+Requires-Dist: sacrebleu>=2.4; extra == 'dev'
 Requires-Dist: scikit-learn>=1.4; extra == 'dev'
 Requires-Dist: statsmodels>=0.13; extra == 'dev'
 Requires-Dist: twine; extra == 'dev'
+Provides-Extra: llm
+Requires-Dist: nltk>=3.8; extra == 'llm'
 Provides-Extra: plot
 Requires-Dist: matplotlib>=3.5; extra == 'plot'
 Provides-Extra: vision
@@ -65,12 +77,14 @@ Description-Content-Type: text/markdown
 [![Python](https://img.shields.io/pypi/pyversions/evalsuite-python)](https://pypi.org/project/evalsuite-python/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
 
-**Unified, reproducible evaluation for machine learning and research.**
+**Unified, reproducible evaluation for machine learning, LLM and research.**
 
 EvalSuite brings classification, regression, clinical, statistical, segmentation and object-detection
 evaluation into one consistent, validated, documented framework.
 
-> **Status: stable (0.3.1).** Every item on the 0.1.0, 0.2.0 and 0.3.0 roadmaps is implemented and verified.
+> **Status: stable (0.4.1).** Every item on the 0.1.0, 0.2.0, 0.3.0 and 0.4.0 roadmaps is implemented and verified
+> (except SPICE, which needs a Java scene-graph parser).
+> **LLM evaluation is available from 0.4.0**; LLM systems, safety and operations are planned for 0.5.0.
 
 ## Installation
 
@@ -226,6 +240,22 @@ Models can be compared over the same images with intervals and paired tests, as
 `es.compare(y_true, {"unet": masks_a, "deeplab": masks_b})` resamples images; for detection it compares
 mAP.
 
+## LLM evaluation
+
+```python
+es.bleu(references, predictions)  # sacreBLEU-identical; also chrf, ter, rouge_l, meteor, cider
+es.text_report(references, predictions)  # BLEU, chrF(++), TER, ROUGE, METEOR, EM, token F1 at once
+es.bertscore(ref_token_embs, pred_token_embs)  # from your encoder's embeddings; also mauve, moverscore
+es.model_score(refs, preds, scorer=comet_fn)  # any learned metric or judge, with intervals and compare
+es.faithfulness(claim_verdicts)  # also hallucination_rate, citation_recall, answer_correctness
+es.bradley_terry(comparisons, scale="elo")  # also win_rate, elo_ratings, krippendorff_alpha
+es.ndcg_at_k(relevant, retrieved, k=10)  # also mrr, context_precision, context_recall
+es.json_schema_compliance(outputs, schema)  # also tool_call_f1, instruction_compliance_rate
+es.pass_at_k(n_samples, n_correct, k=10)  # also benchmark_accuracy(..., style="gsm8k")
+es.compare(references, {"a": preds_a, "b": preds_b}, metrics=["bleu", "rouge_l"])  # paired over examples
+es.plot.ratings(comparisons)  # Bradley–Terry leaderboard with intervals; also win_matrix, text_scores
+```
+
 ## Classification report
 
 ```python
@@ -283,28 +313,41 @@ Input files can be CSV, TSV, Parquet or JSON. Output format follows `--format` o
 ## Performance
 
 Benchmarked against reference implementations on the same data (fastest of 5 runs; Python 3.12, NumPy 2.5,
-Linux x86_64). **Overall: 45 of 45 rows agree with the reference** (largest
-difference 2.8e-14), 32 are faster, and the geometric-mean speed-up across all
-15 cases and three sizes is **2.22×**. At 1,000,000 samples (pixels for
-segmentation; 1,000 images for detection and 50 for Hausdorff), in alphabetical order:
+Linux x86_64). **Overall: 63 of 63 rows agree with the reference** (largest
+difference 1.8e-14), 42 are faster, and the geometric-mean speed-up across all
+21 cases and three sizes is **2.12×**. At 1,000,000 samples (pixels for
+segmentation; 1,000 images for detection and 50 for Hausdorff; 10,000 examples for the LLM cases), in
+alphabetical order:
 
 | Case | Reference | EvalSuite (ms) | Reference (ms) | Speed-up |
 | --- | --- | ---: | ---: | ---: |
-| 10 classes: macro F1 | scikit-learn | 29.3 | 130.8 | **4.5×** |
-| binary: 8 label metrics via evaluate() | scikit-learn | 39.5 | 1084.1 | **27.4×** |
-| binary: ROC AUC | scikit-learn | 195.0 | 348.3 | **1.8×** |
-| calibration: slope and intercept | statsmodels | 193.0 | 1068.0 | **5.5×** |
-| clinical: diagnostic report (7 CIs) | statsmodels | 23.5 | 4.7 | 0.20× |
-| clinical: sensitivity, specificity, LR+, LR− | scikit-learn | 69.1 | 414.0 | **6.0×** |
-| decision curve: 99 thresholds | NumPy loop | 156.3 | 181.6 | **1.2×** |
-| detection: COCO evaluation (1000 images) | pycocotools | 817.3 | 983.7 | **1.2×** |
-| multiple testing: Hochberg (n p-values) | statsmodels | 82.0 | 108.0 | **1.3×** |
-| regression: MAE, MSE, RMSE, R² via evaluate() | scikit-learn | 23.1 | 10.8 | 0.47× |
-| segmentation: Dice and IoU per class (n = pixels) | scikit-learn | 43.0 | 290.0 | **6.8×** |
-| segmentation: Hausdorff distance (50 images) | SciPy | 27.0 | 20.2 | 0.75× |
-| statistics: Cramér's V (5×5 table) | SciPy | 0.5 | 0.4 | 0.88× |
-| statistics: Mann–Whitney U | SciPy | 349.7 | 350.2 | **1.0×** |
-| statistics: Welch t-test | SciPy | 14.5 | 7.2 | 0.49× |
+| 10 classes: macro F1 | scikit-learn | 23.8 | 135.5 | **5.7×** |
+| agreement: Krippendorff's alpha, interval (4 raters × 10000 items) | krippendorff | 3.5 | 3.4 | 0.96× |
+| binary: 8 label metrics via evaluate() | scikit-learn | 28.5 | 1023.8 | **36.0×** |
+| binary: ROC AUC | scikit-learn | 187.4 | 327.3 | **1.7×** |
+| calibration: slope and intercept | statsmodels | 205.7 | 1173.7 | **5.7×** |
+| clinical: diagnostic report (7 CIs) | statsmodels | 19.6 | 4.9 | 0.25× |
+| clinical: sensitivity, specificity, LR+, LR− | scikit-learn | 75.2 | 411.5 | **5.5×** |
+| decision curve: 99 thresholds | NumPy loop | 165.0 | 201.9 | **1.2×** |
+| detection: COCO evaluation (1000 images) | pycocotools | 838.7 | 848.6 | **1.0×** |
+| multiple testing: Hochberg (n p-values) | statsmodels | 72.1 | 79.0 | **1.1×** |
+| regression: MAE, MSE, RMSE, R² via evaluate() | scikit-learn | 22.9 | 10.6 | 0.47× |
+| retrieval: MRR, MAP@20, NDCG@10 (10000 queries) | ranx | 170.4 | 861.0 | **5.1×** |
+| segmentation: Dice and IoU per class (n = pixels) | scikit-learn | 34.3 | 256.0 | **7.5×** |
+| segmentation: Hausdorff distance (50 images) | SciPy | 24.2 | 19.8 | 0.82× |
+| statistics: Cramér's V (5×5 table) | SciPy | 0.2 | 0.2 | **1.0×** |
+| statistics: Mann–Whitney U | SciPy | 353.5 | 340.6 | 0.96× |
+| statistics: Welch t-test | SciPy | 16.1 | 8.3 | 0.52× |
+| structured: JSON Schema compliance (10000 documents) | jsonschema | 144.6 | 330.5 | **2.3×** |
+| text: corpus BLEU and chrF (10000 sentences) | sacreBLEU | 2789.8 | 3051.8 | **1.1×** |
+| text: METEOR, exact and stem matches (10000 sentences) | NLTK | 488.1 | 626.7 | **1.3×** |
+| text: ROUGE-1, ROUGE-2, ROUGE-L (10000 sentences) | rouge-score | 1290.1 | 1095.6 | 0.85× |
+
+**Every metric is benchmarked too** (`evalsuite benchmark --suite metrics`): all 147 registered metrics and
+statistics functions at 10,000 and 100,000 samples. 79 are compared with a reference library, 49 with an
+independent textbook formula and 19 (learned, judge-dependent or randomised) are timed alone; **all 128
+comparisons agree**, and against the libraries the geometric-mean speed-up is **2.76×** (123 of 158
+measurements faster). Per-metric table in BENCHMARKS.md.
 
 `evaluate()` validates inputs once and builds the confusion matrix once for all metrics, which is where most
 of the speed-up comes from. Hypothesis tests use SciPy underneath, so they match its speed at best; rows

{evalsuite_python-0.3.1 → evalsuite_python-0.4.1}/README.md

@@ -5,12 +5,14 @@
 [![Python](https://img.shields.io/pypi/pyversions/evalsuite-python)](https://pypi.org/project/evalsuite-python/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
 
-**Unified, reproducible evaluation for machine learning and research.**
+**Unified, reproducible evaluation for machine learning, LLM and research.**
 
 EvalSuite brings classification, regression, clinical, statistical, segmentation and object-detection
 evaluation into one consistent, validated, documented framework.
 
-> **Status: stable (0.3.1).** Every item on the 0.1.0, 0.2.0 and 0.3.0 roadmaps is implemented and verified.
+> **Status: stable (0.4.1).** Every item on the 0.1.0, 0.2.0, 0.3.0 and 0.4.0 roadmaps is implemented and verified
+> (except SPICE, which needs a Java scene-graph parser).
+> **LLM evaluation is available from 0.4.0**; LLM systems, safety and operations are planned for 0.5.0.
 
 ## Installation
 
@@ -166,6 +168,22 @@ Models can be compared over the same images with intervals and paired tests, as
 `es.compare(y_true, {"unet": masks_a, "deeplab": masks_b})` resamples images; for detection it compares
 mAP.
 
+## LLM evaluation
+
+```python
+es.bleu(references, predictions)  # sacreBLEU-identical; also chrf, ter, rouge_l, meteor, cider
+es.text_report(references, predictions)  # BLEU, chrF(++), TER, ROUGE, METEOR, EM, token F1 at once
+es.bertscore(ref_token_embs, pred_token_embs)  # from your encoder's embeddings; also mauve, moverscore
+es.model_score(refs, preds, scorer=comet_fn)  # any learned metric or judge, with intervals and compare
+es.faithfulness(claim_verdicts)  # also hallucination_rate, citation_recall, answer_correctness
+es.bradley_terry(comparisons, scale="elo")  # also win_rate, elo_ratings, krippendorff_alpha
+es.ndcg_at_k(relevant, retrieved, k=10)  # also mrr, context_precision, context_recall
+es.json_schema_compliance(outputs, schema)  # also tool_call_f1, instruction_compliance_rate
+es.pass_at_k(n_samples, n_correct, k=10)  # also benchmark_accuracy(..., style="gsm8k")
+es.compare(references, {"a": preds_a, "b": preds_b}, metrics=["bleu", "rouge_l"])  # paired over examples
+es.plot.ratings(comparisons)  # Bradley–Terry leaderboard with intervals; also win_matrix, text_scores
+```
+
 ## Classification report
 
 ```python
@@ -223,28 +241,41 @@ Input files can be CSV, TSV, Parquet or JSON. Output format follows `--format` o
 ## Performance
 
 Benchmarked against reference implementations on the same data (fastest of 5 runs; Python 3.12, NumPy 2.5,
-Linux x86_64). **Overall: 45 of 45 rows agree with the reference** (largest
-difference 2.8e-14), 32 are faster, and the geometric-mean speed-up across all
-15 cases and three sizes is **2.22×**. At 1,000,000 samples (pixels for
-segmentation; 1,000 images for detection and 50 for Hausdorff), in alphabetical order:
+Linux x86_64). **Overall: 63 of 63 rows agree with the reference** (largest
+difference 1.8e-14), 42 are faster, and the geometric-mean speed-up across all
+21 cases and three sizes is **2.12×**. At 1,000,000 samples (pixels for
+segmentation; 1,000 images for detection and 50 for Hausdorff; 10,000 examples for the LLM cases), in
+alphabetical order:
 
 | Case | Reference | EvalSuite (ms) | Reference (ms) | Speed-up |
 | --- | --- | ---: | ---: | ---: |
-| 10 classes: macro F1 | scikit-learn | 29.3 | 130.8 | **4.5×** |
-| binary: 8 label metrics via evaluate() | scikit-learn | 39.5 | 1084.1 | **27.4×** |
-| binary: ROC AUC | scikit-learn | 195.0 | 348.3 | **1.8×** |
-| calibration: slope and intercept | statsmodels | 193.0 | 1068.0 | **5.5×** |
-| clinical: diagnostic report (7 CIs) | statsmodels | 23.5 | 4.7 | 0.20× |
-| clinical: sensitivity, specificity, LR+, LR− | scikit-learn | 69.1 | 414.0 | **6.0×** |
-| decision curve: 99 thresholds | NumPy loop | 156.3 | 181.6 | **1.2×** |
-| detection: COCO evaluation (1000 images) | pycocotools | 817.3 | 983.7 | **1.2×** |
-| multiple testing: Hochberg (n p-values) | statsmodels | 82.0 | 108.0 | **1.3×** |
-| regression: MAE, MSE, RMSE, R² via evaluate() | scikit-learn | 23.1 | 10.8 | 0.47× |
-| segmentation: Dice and IoU per class (n = pixels) | scikit-learn | 43.0 | 290.0 | **6.8×** |
-| segmentation: Hausdorff distance (50 images) | SciPy | 27.0 | 20.2 | 0.75× |
-| statistics: Cramér's V (5×5 table) | SciPy | 0.5 | 0.4 | 0.88× |
-| statistics: Mann–Whitney U | SciPy | 349.7 | 350.2 | **1.0×** |
-| statistics: Welch t-test | SciPy | 14.5 | 7.2 | 0.49× |
+| 10 classes: macro F1 | scikit-learn | 23.8 | 135.5 | **5.7×** |
+| agreement: Krippendorff's alpha, interval (4 raters × 10000 items) | krippendorff | 3.5 | 3.4 | 0.96× |
+| binary: 8 label metrics via evaluate() | scikit-learn | 28.5 | 1023.8 | **36.0×** |
+| binary: ROC AUC | scikit-learn | 187.4 | 327.3 | **1.7×** |
+| calibration: slope and intercept | statsmodels | 205.7 | 1173.7 | **5.7×** |
+| clinical: diagnostic report (7 CIs) | statsmodels | 19.6 | 4.9 | 0.25× |
+| clinical: sensitivity, specificity, LR+, LR− | scikit-learn | 75.2 | 411.5 | **5.5×** |
+| decision curve: 99 thresholds | NumPy loop | 165.0 | 201.9 | **1.2×** |
+| detection: COCO evaluation (1000 images) | pycocotools | 838.7 | 848.6 | **1.0×** |
+| multiple testing: Hochberg (n p-values) | statsmodels | 72.1 | 79.0 | **1.1×** |
+| regression: MAE, MSE, RMSE, R² via evaluate() | scikit-learn | 22.9 | 10.6 | 0.47× |
+| retrieval: MRR, MAP@20, NDCG@10 (10000 queries) | ranx | 170.4 | 861.0 | **5.1×** |
+| segmentation: Dice and IoU per class (n = pixels) | scikit-learn | 34.3 | 256.0 | **7.5×** |
+| segmentation: Hausdorff distance (50 images) | SciPy | 24.2 | 19.8 | 0.82× |
+| statistics: Cramér's V (5×5 table) | SciPy | 0.2 | 0.2 | **1.0×** |
+| statistics: Mann–Whitney U | SciPy | 353.5 | 340.6 | 0.96× |
+| statistics: Welch t-test | SciPy | 16.1 | 8.3 | 0.52× |
+| structured: JSON Schema compliance (10000 documents) | jsonschema | 144.6 | 330.5 | **2.3×** |
+| text: corpus BLEU and chrF (10000 sentences) | sacreBLEU | 2789.8 | 3051.8 | **1.1×** |
+| text: METEOR, exact and stem matches (10000 sentences) | NLTK | 488.1 | 626.7 | **1.3×** |
+| text: ROUGE-1, ROUGE-2, ROUGE-L (10000 sentences) | rouge-score | 1290.1 | 1095.6 | 0.85× |
+
+**Every metric is benchmarked too** (`evalsuite benchmark --suite metrics`): all 147 registered metrics and
+statistics functions at 10,000 and 100,000 samples. 79 are compared with a reference library, 49 with an
+independent textbook formula and 19 (learned, judge-dependent or randomised) are timed alone; **all 128
+comparisons agree**, and against the libraries the geometric-mean speed-up is **2.76×** (123 of 158
+measurements faster). Per-metric table in BENCHMARKS.md.
 
 `evaluate()` validates inputs once and builds the confusion matrix once for all metrics, which is where most
 of the speed-up comes from. Hypothesis tests use SciPy underneath, so they match its speed at best; rows

{evalsuite_python-0.3.1 → evalsuite_python-0.4.1}/pyproject.toml

@@ -38,7 +38,8 @@ dependencies = ["numpy>=1.22", "scipy>=1.8", "pandas>=1.4"]
 [project.optional-dependencies]
 plot = ["matplotlib>=3.5"]
 vision = ["pillow>=9"]
-all = ["matplotlib>=3.5", "pillow>=9"]
+llm = ["nltk>=3.8"]
+all = ["matplotlib>=3.5", "pillow>=9", "nltk>=3.8"]
 dev = [
   "pytest>=7",
   "pytest-cov>=4",
@@ -48,6 +49,15 @@ dev = [
   "matplotlib>=3.5",
   "pillow>=9",
   "pycocotools>=2.0.7",
+  "nltk>=3.8",
+  "sacrebleu>=2.4",
+  "rouge-score>=0.1.2",
+  "pycocoevalcap>=1.2",
+  "jsonschema>=4.18",
+  "choix>=0.3",
+  "krippendorff>=0.6",
+  "pot>=0.9; python_version < '3.14'",
+  "ranx>=0.3.19; python_version < '3.14'",
   "ruff>=0.6",
   "mypy>=1.10",
   "pandas-stubs",
@@ -99,6 +109,12 @@ files = ["src/evalsuite"]
 module = ["scipy.*", "matplotlib.*"]
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+# optional reference libraries used only by the benchmarks and METEOR's default stemmer
+module = ["sacrebleu.*", "rouge_score.*", "nltk.*", "ranx.*", "krippendorff.*", "jsonschema.*", "pycocotools.*"]
+ignore_missing_imports = true
+follow_untyped_imports = false
+
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 addopts = "-q --strict-markers"

{evalsuite_python-0.3.1 → evalsuite_python-0.4.1}/src/evalsuite/__init__.py

@@ -5,7 +5,7 @@
 >>> print(result.summary())  # doctest: +SKIP
 """
 
-from . import calibration, classification, clinical, plot, regression, stats, vision
+from . import calibration, classification, clinical, plot, regression, stats, text, vision
 from .api import evaluate
 from .calibration import (
     CalibrationReport,
@@ -110,6 +110,8 @@ from .stats import (
     t_test,
     wilcoxon_test,
 )
+from .text import *  # noqa: F403  (every LLM-evaluation function, listed in text.__all__)
+from .text import __all__ as _text_all
 from .version import __version__
 from .vision import (
     DetectionReport,
@@ -134,6 +136,7 @@ from .vision import (
 )
 
 __all__ = [
+    "text",
     "SegmentationReport",
     "segmentation_report",
     "detection_pr_curve",
@@ -258,3 +261,4 @@ __all__ = [
     "specificity",
     "top_k_accuracy",
 ]
+__all__ += list(_text_all)