crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

{crfm_helm-0.5.6.dist-info → crfm_helm-0.5.10.dist-info}/METADATA

@@ -1,19 +1,15 @@
 Metadata-Version: 2.4
 Name: crfm-helm
-Version: 0.5.6
+Version: 0.5.10
 Summary: Benchmark for language models
-Home-page: https://github.com/stanford-crfm/helm
-Author: Stanford CRFM
-Author-email: contact-crfm@stanford.edu
+Author-email: Stanford CRFM <contact-crfm@stanford.edu>
 License: Apache License 2.0
-Keywords: language models benchmarking
+Project-URL: Homepage, https://github.com/stanford-crfm/helm
+Keywords: language,models,benchmarking
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: License :: OSI Approved :: Apache Software License
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: cattrs~=22.2
@@ -30,7 +26,7 @@ Requires-Dist: tqdm~=4.64
 Requires-Dist: zstandard~=0.18.0
 Requires-Dist: sqlitedict<3.0,>=2.1.0
 Requires-Dist: bottle~=0.12.23
-Requires-Dist: datasets~=2.17
+Requires-Dist: datasets~=3.1
 Requires-Dist: pyarrow>=11.0.0
 Requires-Dist: pyarrow-hotfix~=0.6
 Requires-Dist: nltk!=3.9.0,~=3.7
@@ -38,24 +34,31 @@ Requires-Dist: rouge-score~=0.1.2
 Requires-Dist: scipy>=1.10
 Requires-Dist: uncertainty-calibration~=0.1.4
 Requires-Dist: scikit-learn>=1.1
-Requires-Dist: transformers~=4.40
+Requires-Dist: transformers<4.53.0,~=4.40
 Requires-Dist: torch<3.0.0,>=1.13.1
 Requires-Dist: torchvision<3.0.0,>=0.14.1
 Provides-Extra: proxy-server
 Requires-Dist: gunicorn>=20.1; extra == "proxy-server"
 Provides-Extra: human-evaluation
-Requires-Dist: scaleapi~=2.13.0; extra == "human-evaluation"
-Requires-Dist: surge-api~=1.1.0; extra == "human-evaluation"
+Requires-Dist: scaleapi~=2.13; extra == "human-evaluation"
+Requires-Dist: surge-api~=1.1; extra == "human-evaluation"
+Provides-Extra: dspy
+Requires-Dist: dspy~=3.0.3; extra == "dspy"
+Requires-Dist: fastapi~=0.118.0; extra == "dspy"
+Requires-Dist: apscheduler~=3.11.0; extra == "dspy"
+Requires-Dist: cryptography~=43.0.3; extra == "dspy"
+Requires-Dist: python-multipart~=0.0.20; extra == "dspy"
+Requires-Dist: email-validator~=2.3.0; extra == "dspy"
+Requires-Dist: fastapi_sso~=0.18.0; extra == "dspy"
 Provides-Extra: scenarios
 Requires-Dist: gdown~=5.1; extra == "scenarios"
-Requires-Dist: sympy~=1.11.1; extra == "scenarios"
-Requires-Dist: xlrd~=2.0.1; extra == "scenarios"
+Requires-Dist: xlrd~=2.0; extra == "scenarios"
 Provides-Extra: metrics
 Requires-Dist: google-api-python-client~=2.64; extra == "metrics"
 Requires-Dist: numba~=0.56; extra == "metrics"
-Requires-Dist: sacrebleu~=2.2.1; extra == "metrics"
-Requires-Dist: langdetect~=1.0.9; extra == "metrics"
-Requires-Dist: immutabledict~=4.2.0; extra == "metrics"
+Requires-Dist: sacrebleu~=2.2; extra == "metrics"
+Requires-Dist: langdetect~=1.0; extra == "metrics"
+Requires-Dist: immutabledict~=4.2; extra == "metrics"
 Requires-Dist: gradio_client~=1.3; extra == "metrics"
 Provides-Extra: ranking
 Requires-Dist: pytrec_eval==0.5; extra == "ranking"
@@ -63,7 +66,7 @@ Provides-Extra: summarization
 Requires-Dist: summ-eval~=0.892; extra == "summarization"
 Requires-Dist: bert-score~=0.3; extra == "summarization"
 Provides-Extra: plots
-Requires-Dist: colorcet~=3.0.1; extra == "plots"
+Requires-Dist: colorcet~=3.0; extra == "plots"
 Requires-Dist: matplotlib>=3.6.0; extra == "plots"
 Requires-Dist: seaborn>=0.11.0; extra == "plots"
 Provides-Extra: decodingtrust
@@ -86,22 +89,22 @@ Requires-Dist: evaluate~=0.4.1; extra == "unitxt"
 Provides-Extra: seahelm
 Requires-Dist: pythainlp==5.0.0; extra == "seahelm"
 Requires-Dist: pyonmttok==1.37.0; extra == "seahelm"
-Requires-Dist: sacrebleu~=2.2.1; extra == "seahelm"
+Requires-Dist: sacrebleu~=2.2; extra == "seahelm"
 Requires-Dist: python-crfsuite~=0.9.11; extra == "seahelm"
 Provides-Extra: accelerate
 Requires-Dist: accelerate~=0.25; extra == "accelerate"
 Provides-Extra: aleph-alpha
-Requires-Dist: aleph-alpha-client~=2.14.0; extra == "aleph-alpha"
+Requires-Dist: aleph-alpha-client~=2.14; extra == "aleph-alpha"
 Requires-Dist: tokenizers>=0.13.3; extra == "aleph-alpha"
 Provides-Extra: allenai
 Requires-Dist: ai2-olmo~=0.2; extra == "allenai"
 Provides-Extra: amazon
-Requires-Dist: boto3~=1.34.131; extra == "amazon"
-Requires-Dist: awscli~=1.33.13; extra == "amazon"
-Requires-Dist: botocore~=1.34.131; extra == "amazon"
+Requires-Dist: boto3~=1.34; extra == "amazon"
+Requires-Dist: awscli~=1.33; extra == "amazon"
+Requires-Dist: botocore~=1.34; extra == "amazon"
 Provides-Extra: anthropic
 Requires-Dist: anthropic~=0.48; extra == "anthropic"
-Requires-Dist: websocket-client~=1.3.2; extra == "anthropic"
+Requires-Dist: websocket-client~=1.3; extra == "anthropic"
 Requires-Dist: httpx<0.28.0; extra == "anthropic"
 Provides-Extra: cohere
 Requires-Dist: cohere~=5.3; extra == "cohere"
@@ -136,7 +139,7 @@ Requires-Dist: crfm-helm[yandex]; extra == "models"
 Requires-Dist: crfm-helm[writer]; extra == "models"
 Requires-Dist: crfm-helm[ibm-enterprise-scenarios]; extra == "models"
 Provides-Extra: reka
-Requires-Dist: reka-api~=2.0.0; extra == "reka"
+Requires-Dist: reka-api~=2.0; extra == "reka"
 Provides-Extra: vlm
 Requires-Dist: crfm-helm[openai]; extra == "vlm"
 Requires-Dist: einops~=0.7.0; extra == "vlm"
@@ -150,63 +153,65 @@ Requires-Dist: crfm-helm[reka]; extra == "vlm"
 Requires-Dist: crfm-helm[images]; extra == "vlm"
 Requires-Dist: crfm-helm[image2struct]; extra == "vlm"
 Requires-Dist: pycocoevalcap~=1.2; extra == "vlm"
-Requires-Dist: transformers~=4.45.2; extra == "vlm"
+Requires-Dist: transformers~=4.45; extra == "vlm"
 Requires-Dist: qwen-vl-utils~=0.0.8; extra == "vlm"
 Provides-Extra: ibm-enterprise-scenarios
 Requires-Dist: openpyxl~=3.1; extra == "ibm-enterprise-scenarios"
 Provides-Extra: ibm
-Requires-Dist: ibm-watsonx-ai~=1.2.6; extra == "ibm"
+Requires-Dist: ibm-watsonx-ai~=1.2; extra == "ibm"
 Provides-Extra: image2struct
 Requires-Dist: crfm-helm[images]; extra == "image2struct"
 Requires-Dist: latex~=0.7.0; extra == "image2struct"
-Requires-Dist: pdf2image~=1.16.3; extra == "image2struct"
-Requires-Dist: selenium~=4.17.2; extra == "image2struct"
+Requires-Dist: pdf2image~=1.16; extra == "image2struct"
+Requires-Dist: selenium~=4.17; extra == "image2struct"
 Requires-Dist: html2text~=2024.2.26; extra == "image2struct"
-Requires-Dist: opencv-python<4.8.2.0,>=4.7.0.68; extra == "image2struct"
+Requires-Dist: opencv-python-headless<=4.11.0.86,>=4.7.0.68; extra == "image2struct"
 Requires-Dist: lpips~=0.1.4; extra == "image2struct"
-Requires-Dist: imagehash~=4.3.1; extra == "image2struct"
+Requires-Dist: imagehash~=4.3; extra == "image2struct"
 Provides-Extra: heim
 Requires-Dist: gdown~=5.1; extra == "heim"
-Requires-Dist: diffusers~=0.24.0; extra == "heim"
+Requires-Dist: diffusers~=0.34.0; extra == "heim"
 Requires-Dist: icetk~=0.0.4; extra == "heim"
-Requires-Dist: jax~=0.4.13; extra == "heim"
-Requires-Dist: jaxlib~=0.4.13; extra == "heim"
+Requires-Dist: jax~=0.6.2; python_version >= "3.10" and extra == "heim"
+Requires-Dist: jax~=0.4.30; python_version < "3.10" and extra == "heim"
+Requires-Dist: jaxlib~=0.6.2; python_version >= "3.10" and extra == "heim"
+Requires-Dist: jaxlib~=0.4.30; python_version < "3.10" and extra == "heim"
 Requires-Dist: crfm-helm[openai]; extra == "heim"
 Requires-Dist: einops~=0.7.0; extra == "heim"
-Requires-Dist: omegaconf~=2.3.0; extra == "heim"
-Requires-Dist: pytorch-lightning~=2.0.5; extra == "heim"
-Requires-Dist: flax~=0.6.11; extra == "heim"
-Requires-Dist: ftfy~=6.1.1; extra == "heim"
-Requires-Dist: Unidecode~=1.3.6; extra == "heim"
+Requires-Dist: omegaconf~=2.3; extra == "heim"
+Requires-Dist: pytorch-lightning~=2.0; extra == "heim"
+Requires-Dist: flax~=0.10.7; python_version >= "3.10" and extra == "heim"
+Requires-Dist: flax~=0.8.5; python_version < "3.10" and extra == "heim"
+Requires-Dist: ftfy~=6.1; extra == "heim"
+Requires-Dist: Unidecode~=1.3; extra == "heim"
 Requires-Dist: wandb~=0.16; extra == "heim"
-Requires-Dist: google-cloud-translate~=3.11.2; extra == "heim"
-Requires-Dist: autokeras~=1.0.20; extra == "heim"
-Requires-Dist: clip-anytorch~=2.5.0; extra == "heim"
+Requires-Dist: google-cloud-translate~=3.11; extra == "heim"
+Requires-Dist: autokeras~=1.0; extra == "heim"
+Requires-Dist: clip-anytorch~=2.5; extra == "heim"
 Requires-Dist: google-cloud-storage~=2.9; extra == "heim"
 Requires-Dist: lpips~=0.1.4; extra == "heim"
-Requires-Dist: multilingual-clip~=1.0.10; extra == "heim"
-Requires-Dist: NudeNet~=2.0.9; extra == "heim"
-Requires-Dist: opencv-python<4.8.2.0,>=4.7.0.68; extra == "heim"
+Requires-Dist: multilingual-clip~=1.0; extra == "heim"
+Requires-Dist: NudeNet~=2.0; extra == "heim"
+Requires-Dist: numpy<2,>=1.26; extra == "heim"
+Requires-Dist: opencv-python<4.8.2.0,>=4.7.0.68; python_version >= "3.10" and extra == "heim"
+Requires-Dist: opencv-python-headless<=4.11.0.86,>=4.7.0.68; python_version < "3.10" and extra == "heim"
 Requires-Dist: pytorch-fid~=0.3.0; extra == "heim"
 Requires-Dist: tensorflow~=2.11; extra == "heim"
 Requires-Dist: timm~=0.6.12; extra == "heim"
 Requires-Dist: torch-fidelity~=0.3.0; extra == "heim"
 Requires-Dist: torchmetrics~=0.11.1; extra == "heim"
-Requires-Dist: scikit-image!=0.23.*,==0.*,>=0.22; extra == "heim"
+Requires-Dist: scikit-image==0.*,>=0.22; extra == "heim"
 Requires-Dist: crfm-helm[images]; extra == "heim"
 Provides-Extra: medhelm
-Requires-Dist: crfm-helm[accelerate]; extra == "medhelm"
+Requires-Dist: accelerate~=0.25; extra == "medhelm"
 Requires-Dist: crfm-helm[openai]; extra == "medhelm"
-Requires-Dist: crfm-helm[summarization]; extra == "medhelm"
 Requires-Dist: crfm-helm[yandex]; extra == "medhelm"
+Requires-Dist: crfm-helm[scenarios]; extra == "medhelm"
 Requires-Dist: bert_score~=0.3.13; extra == "medhelm"
-Requires-Dist: langchain~=0.3.9; extra == "medhelm"
-Requires-Dist: langchain-community~=0.3.8; extra == "medhelm"
-Requires-Dist: lxml~=5.3.0; extra == "medhelm"
+Requires-Dist: lxml~=5.3; extra == "medhelm"
 Requires-Dist: openpyxl~=3.1; extra == "medhelm"
-Requires-Dist: python-docx~=1.1.2; extra == "medhelm"
-Requires-Dist: torch~=2.2.2; extra == "medhelm"
-Requires-Dist: torchvision~=0.17.2; extra == "medhelm"
+Requires-Dist: python-docx~=1.1; extra == "medhelm"
+Requires-Dist: transformers<4.50,~=4.45; extra == "medhelm"
 Provides-Extra: audiolm
 Requires-Dist: crfm-helm[openai]; extra == "audiolm"
 Requires-Dist: crfm-helm[google]; extra == "audiolm"
@@ -216,16 +221,21 @@ Requires-Dist: soundfile~=0.12; extra == "audiolm"
 Requires-Dist: librosa~=0.10; extra == "audiolm"
 Requires-Dist: einops~=0.7.0; extra == "audiolm"
 Requires-Dist: openai-whisper==20240930; extra == "audiolm"
-Requires-Dist: transformers~=4.48.0; extra == "audiolm"
+Requires-Dist: transformers~=4.48; extra == "audiolm"
 Requires-Dist: transformers_stream_generator~=0.0.4; extra == "audiolm"
-Requires-Dist: av~=14.3.0; extra == "audiolm"
+Requires-Dist: av~=14.3; extra == "audiolm"
 Requires-Dist: scipy~=1.10; extra == "audiolm"
 Requires-Dist: torchvision<3.0.0,>=0.14.1; extra == "audiolm"
-Requires-Dist: flash-attn~=2.7.3; extra == "audiolm"
+Requires-Dist: flash-attn~=2.7; extra == "audiolm"
 Requires-Dist: pycocoevalcap~=1.2; extra == "audiolm"
 Requires-Dist: jiwer~=3.0; extra == "audiolm"
 Requires-Dist: rapidfuzz~=3.10; extra == "audiolm"
 Requires-Dist: jieba~=0.42.1; extra == "audiolm"
+Provides-Extra: codeinsights
+Requires-Dist: clang~=20.1; extra == "codeinsights"
+Requires-Dist: Levenshtein~=0.27; extra == "codeinsights"
+Provides-Extra: lmkt
+Requires-Dist: sentence_transformers~=4.1; extra == "lmkt"
 Provides-Extra: all
 Requires-Dist: crfm-helm[proxy-server]; extra == "all"
 Requires-Dist: crfm-helm[human-evaluation]; extra == "all"
@@ -240,8 +250,11 @@ Requires-Dist: crfm-helm[models]; extra == "all"
 Requires-Dist: crfm-helm[mongo]; extra == "all"
 Requires-Dist: crfm-helm[heim]; extra == "all"
 Requires-Dist: crfm-helm[vlm]; extra == "all"
+Requires-Dist: crfm-helm[codeinsights]; extra == "all"
+Requires-Dist: crfm-helm[lmkt]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: pytest~=7.2.0; extra == "dev"
+Requires-Dist: xdoctest~=1.2.0; extra == "dev"
 Requires-Dist: pre-commit~=2.20.0; extra == "dev"
 Requires-Dist: black==24.3.0; extra == "dev"
 Requires-Dist: mypy==1.16.0; extra == "dev"
@@ -334,6 +347,7 @@ The HELM framework was used in the following papers for evaluating models.
 - **The Mighty ToRR: A Benchmark for Table Reasoning and Robustness** - [paper](https://arxiv.org/abs/2502.19412), [leaderboard](https://crfm.stanford.edu/helm/torr/latest/)
 - **Reliable and Efficient Amortized Model-based Evaluation** - [paper](https://arxiv.org/abs/2503.13335), [documentation](https://crfm-helm.readthedocs.io/en/latest/reeval/)
 - **MedHELM** - paper in progress, [leaderboard](https://crfm.stanford.edu/helm/medhelm/latest/), [documentation](https://crfm-helm.readthedocs.io/en/latest/reeval/)
+- **Holistic Evaluation of Audio-Language Models** - [paper](https://arxiv.org/abs/2508.21376), [leaderboard](https://crfm.stanford.edu/helm/audio/latest/)
 
 The HELM framework can be used to reproduce the published model evaluation results from these papers. To get started, refer to the documentation links above for the corresponding paper, or the [main Reproducing Leaderboards documentation](https://crfm-helm.readthedocs.io/en/latest/reproducing_leaderboards/).
 
@@ -353,75 +367,3 @@ url={https://openreview.net/forum?id=iO4LZibEqW},
 note={Featured Certification, Expert Certification}
 }
 ```
-
-# Tutorial
-
-This tutorial will explain how to use the HELM command line tools to run benchmarks, aggregate statistics, and visualize results.
-
-We will run two runs using the `mmlu` scenario on the `openai/gpt2` model. The `mmlu` scenario implements the **Massive Multitask Language (MMLU)** benchmark from [this paper](https://arxiv.org/pdf/2009.03300.pdf), and consists of a Question Answering (QA) task using a dataset with questions from 57 subjects such as elementary mathematics, US history, computer science, law, and more. Note that GPT-2 performs poorly on MMLU, so this is just a proof of concept. We will run two runs: the first using questions about anatomy, and the second using questions about philosophy.
-
-## Using `helm-run`
-
-`helm-run` is a command line tool for running benchmarks.
-
-To run this benchmark using the HELM command-line tools, we need to specify **run entries** that describes the desired runs. For this example, the run entries are `mmlu:subject=anatomy,model=openai/gpt2` (for anatomy) and `mmlu:subject=philosophy,model=openai/gpt2` (for philosophy).
-
-We will now use `helm-run` to execute the runs. Run this command:
-
-```sh
-helm-run --run-entries mmlu:subject=anatomy,model=openai/gpt2 mmlu:subject=philosophy,model=openai/gpt2 --suite my-suite --max-eval-instances 10
-```
-
-The meaning of the arguments are as follows:
-
-- `--run-entries` specifies the run entries from the desired runs.
-- `--suite` specifies a subdirectory under the output directory in which all the output will be placed.
-- `--max-eval-instances` limits evaluation to only *N* instances (i.e. items) from the benchmark, using a randomly shuffled order of instances.
-
-`helm-run` creates an environment directory environment and an output directory by default.
-
--  The environment directory is `prod_env/` by default and can be set using `--local-path`. Credentials for making API calls should be added to a `credentials.conf` file in this directory.
--  The output directory is `benchmark_output/` by default and can be set using `--output-path`.
-
-After running this command, navigate to the `benchmark_output/runs/my-suite/` directory. This should contain a two sub-directories named `mmlu:subject=anatomy,model=openai_gpt2` and `mmlu:subject=philosophy,model=openai_gpt2`. Note that the names of these sub-directories is based on the run entries we used earlier, but with `/` replaced with `_`.
-
-Each output sub-directory will contain several JSON files that were generated during the corresponding run:
-
-- `run_spec.json` contains the `RunSpec`, which specifies the scenario, adapter and metrics for the run.
-- `scenario.json` contains a serialized `Scenario`, which contains the scenario for the run and specifies the instances (i.e. inputs) used.
-- `scenario_state.json` contains a serialized `ScenarioState`, which contains every request to and response from the model.
-- `per_instance_stats.json` contains a serialized list of `PerInstanceStats`, which contains the statistics produced for the metrics for each instance (i.e. input).
-- `stats.json` contains a serialized list of `PerInstanceStats`, which contains the statistics produced for the metrics, aggregated across all instances (i.e. inputs).
-
-## Using `helm-summarize`
-
-The `helm-summarize` reads the output files of `helm-run` and computes aggregate statistics across runs. Run the following:
-
-```sh
-helm-summarize --suite my-suite
-```
-
-This reads the pre-existing files in `benchmark_output/runs/my-suite/` that were written by `helm-run` previously, and writes the following new files back to `benchmark_output/runs/my-suite/`:
-
-- `summary.json` contains a serialized `ExecutiveSummary` with a date and suite name.
-- `run_specs.json` contains the run entries for all the runs.
-- `runs.json` contains serialized list of `Run`, which contains the run path, run spec and adapter spec and statistics for each run.
-- `groups.json` contains a serialized list of `Table`, each containing information about groups in a group category.
-- `groups_metadata.json` contains a list of all the groups along with a human-readable description and a taxonomy.
-
-Additionally, for each group and group-relavent metric, it will output a pair of files: `benchmark_output/runs/my-suite/groups/latex/<group_name>_<metric_name>.tex` and `benchmark_output/runs/my-suite/groups/json/<group_name>_<metric_name>.json`. These files contain the statistics for that metric from each run within the group.
-
-## Using `helm-server`
-
-Finally, the `helm-server` command launches a web server to visualize the output files of `helm-run` and `helm-benchmark`. Run:
-
-```sh
-helm-server --suite my-suite
-```
-
-Open a browser and go to http://localhost:8000/ to view the visualization. You should see a similar view as [live website for the paper](https://crfm.stanford.edu/helm/classic/latest/), but for the data from your benchmark runs. The website has the following sections accessible from the top menu bar:
-
-- **Leaderboards** contains the leaderboards with aggregate metrics.
-- **Models** contains a list of models and their descriptions
-- **Scenarios** contains a list of scenarios and their descriptions.
-- **Predictions** contains a searchable list of runs.