crfm-helm 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

{crfm_helm-0.5.6.dist-info → crfm_helm-0.5.8.dist-info}/METADATA

@@ -1,12 +1,11 @@
 Metadata-Version: 2.4
 Name: crfm-helm
-Version: 0.5.6
+Version: 0.5.8
 Summary: Benchmark for language models
-Home-page: https://github.com/stanford-crfm/helm
-Author: Stanford CRFM
-Author-email: contact-crfm@stanford.edu
+Author-email: Stanford CRFM <contact-crfm@stanford.edu>
 License: Apache License 2.0
-Keywords: language models benchmarking
+Project-URL: Homepage, https://github.com/stanford-crfm/helm
+Keywords: language,models,benchmarking
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.9
@@ -30,7 +29,7 @@ Requires-Dist: tqdm~=4.64
 Requires-Dist: zstandard~=0.18.0
 Requires-Dist: sqlitedict<3.0,>=2.1.0
 Requires-Dist: bottle~=0.12.23
-Requires-Dist: datasets~=2.17
+Requires-Dist: datasets~=3.1
 Requires-Dist: pyarrow>=11.0.0
 Requires-Dist: pyarrow-hotfix~=0.6
 Requires-Dist: nltk!=3.9.0,~=3.7
@@ -38,24 +37,23 @@ Requires-Dist: rouge-score~=0.1.2
 Requires-Dist: scipy>=1.10
 Requires-Dist: uncertainty-calibration~=0.1.4
 Requires-Dist: scikit-learn>=1.1
-Requires-Dist: transformers~=4.40
+Requires-Dist: transformers<4.53.0,~=4.40
 Requires-Dist: torch<3.0.0,>=1.13.1
 Requires-Dist: torchvision<3.0.0,>=0.14.1
 Provides-Extra: proxy-server
 Requires-Dist: gunicorn>=20.1; extra == "proxy-server"
 Provides-Extra: human-evaluation
-Requires-Dist: scaleapi~=2.13.0; extra == "human-evaluation"
-Requires-Dist: surge-api~=1.1.0; extra == "human-evaluation"
+Requires-Dist: scaleapi~=2.13; extra == "human-evaluation"
+Requires-Dist: surge-api~=1.1; extra == "human-evaluation"
 Provides-Extra: scenarios
 Requires-Dist: gdown~=5.1; extra == "scenarios"
-Requires-Dist: sympy~=1.11.1; extra == "scenarios"
-Requires-Dist: xlrd~=2.0.1; extra == "scenarios"
+Requires-Dist: xlrd~=2.0; extra == "scenarios"
 Provides-Extra: metrics
 Requires-Dist: google-api-python-client~=2.64; extra == "metrics"
 Requires-Dist: numba~=0.56; extra == "metrics"
-Requires-Dist: sacrebleu~=2.2.1; extra == "metrics"
-Requires-Dist: langdetect~=1.0.9; extra == "metrics"
-Requires-Dist: immutabledict~=4.2.0; extra == "metrics"
+Requires-Dist: sacrebleu~=2.2; extra == "metrics"
+Requires-Dist: langdetect~=1.0; extra == "metrics"
+Requires-Dist: immutabledict~=4.2; extra == "metrics"
 Requires-Dist: gradio_client~=1.3; extra == "metrics"
 Provides-Extra: ranking
 Requires-Dist: pytrec_eval==0.5; extra == "ranking"
@@ -63,7 +61,7 @@ Provides-Extra: summarization
 Requires-Dist: summ-eval~=0.892; extra == "summarization"
 Requires-Dist: bert-score~=0.3; extra == "summarization"
 Provides-Extra: plots
-Requires-Dist: colorcet~=3.0.1; extra == "plots"
+Requires-Dist: colorcet~=3.0; extra == "plots"
 Requires-Dist: matplotlib>=3.6.0; extra == "plots"
 Requires-Dist: seaborn>=0.11.0; extra == "plots"
 Provides-Extra: decodingtrust
@@ -86,22 +84,22 @@ Requires-Dist: evaluate~=0.4.1; extra == "unitxt"
 Provides-Extra: seahelm
 Requires-Dist: pythainlp==5.0.0; extra == "seahelm"
 Requires-Dist: pyonmttok==1.37.0; extra == "seahelm"
-Requires-Dist: sacrebleu~=2.2.1; extra == "seahelm"
+Requires-Dist: sacrebleu~=2.2; extra == "seahelm"
 Requires-Dist: python-crfsuite~=0.9.11; extra == "seahelm"
 Provides-Extra: accelerate
 Requires-Dist: accelerate~=0.25; extra == "accelerate"
 Provides-Extra: aleph-alpha
-Requires-Dist: aleph-alpha-client~=2.14.0; extra == "aleph-alpha"
+Requires-Dist: aleph-alpha-client~=2.14; extra == "aleph-alpha"
 Requires-Dist: tokenizers>=0.13.3; extra == "aleph-alpha"
 Provides-Extra: allenai
 Requires-Dist: ai2-olmo~=0.2; extra == "allenai"
 Provides-Extra: amazon
-Requires-Dist: boto3~=1.34.131; extra == "amazon"
-Requires-Dist: awscli~=1.33.13; extra == "amazon"
-Requires-Dist: botocore~=1.34.131; extra == "amazon"
+Requires-Dist: boto3~=1.34; extra == "amazon"
+Requires-Dist: awscli~=1.33; extra == "amazon"
+Requires-Dist: botocore~=1.34; extra == "amazon"
 Provides-Extra: anthropic
 Requires-Dist: anthropic~=0.48; extra == "anthropic"
-Requires-Dist: websocket-client~=1.3.2; extra == "anthropic"
+Requires-Dist: websocket-client~=1.3; extra == "anthropic"
 Requires-Dist: httpx<0.28.0; extra == "anthropic"
 Provides-Extra: cohere
 Requires-Dist: cohere~=5.3; extra == "cohere"
@@ -136,7 +134,7 @@ Requires-Dist: crfm-helm[yandex]; extra == "models"
 Requires-Dist: crfm-helm[writer]; extra == "models"
 Requires-Dist: crfm-helm[ibm-enterprise-scenarios]; extra == "models"
 Provides-Extra: reka
-Requires-Dist: reka-api~=2.0.0; extra == "reka"
+Requires-Dist: reka-api~=2.0; extra == "reka"
 Provides-Extra: vlm
 Requires-Dist: crfm-helm[openai]; extra == "vlm"
 Requires-Dist: einops~=0.7.0; extra == "vlm"
@@ -150,43 +148,47 @@ Requires-Dist: crfm-helm[reka]; extra == "vlm"
 Requires-Dist: crfm-helm[images]; extra == "vlm"
 Requires-Dist: crfm-helm[image2struct]; extra == "vlm"
 Requires-Dist: pycocoevalcap~=1.2; extra == "vlm"
-Requires-Dist: transformers~=4.45.2; extra == "vlm"
+Requires-Dist: transformers~=4.45; extra == "vlm"
 Requires-Dist: qwen-vl-utils~=0.0.8; extra == "vlm"
 Provides-Extra: ibm-enterprise-scenarios
 Requires-Dist: openpyxl~=3.1; extra == "ibm-enterprise-scenarios"
 Provides-Extra: ibm
-Requires-Dist: ibm-watsonx-ai~=1.2.6; extra == "ibm"
+Requires-Dist: ibm-watsonx-ai~=1.2; extra == "ibm"
 Provides-Extra: image2struct
 Requires-Dist: crfm-helm[images]; extra == "image2struct"
 Requires-Dist: latex~=0.7.0; extra == "image2struct"
-Requires-Dist: pdf2image~=1.16.3; extra == "image2struct"
-Requires-Dist: selenium~=4.17.2; extra == "image2struct"
+Requires-Dist: pdf2image~=1.16; extra == "image2struct"
+Requires-Dist: selenium~=4.17; extra == "image2struct"
 Requires-Dist: html2text~=2024.2.26; extra == "image2struct"
-Requires-Dist: opencv-python<4.8.2.0,>=4.7.0.68; extra == "image2struct"
+Requires-Dist: opencv-python-headless<=4.11.0.86,>=4.7.0.68; extra == "image2struct"
 Requires-Dist: lpips~=0.1.4; extra == "image2struct"
-Requires-Dist: imagehash~=4.3.1; extra == "image2struct"
+Requires-Dist: imagehash~=4.3; extra == "image2struct"
 Provides-Extra: heim
 Requires-Dist: gdown~=5.1; extra == "heim"
-Requires-Dist: diffusers~=0.24.0; extra == "heim"
+Requires-Dist: diffusers~=0.34.0; extra == "heim"
 Requires-Dist: icetk~=0.0.4; extra == "heim"
-Requires-Dist: jax~=0.4.13; extra == "heim"
-Requires-Dist: jaxlib~=0.4.13; extra == "heim"
+Requires-Dist: jax~=0.6.2; python_version >= "3.10" and extra == "heim"
+Requires-Dist: jax~=0.4.30; python_version < "3.10" and extra == "heim"
+Requires-Dist: jaxlib~=0.6.2; python_version >= "3.10" and extra == "heim"
+Requires-Dist: jaxlib~=0.4.30; python_version < "3.10" and extra == "heim"
 Requires-Dist: crfm-helm[openai]; extra == "heim"
 Requires-Dist: einops~=0.7.0; extra == "heim"
-Requires-Dist: omegaconf~=2.3.0; extra == "heim"
-Requires-Dist: pytorch-lightning~=2.0.5; extra == "heim"
-Requires-Dist: flax~=0.6.11; extra == "heim"
-Requires-Dist: ftfy~=6.1.1; extra == "heim"
-Requires-Dist: Unidecode~=1.3.6; extra == "heim"
+Requires-Dist: omegaconf~=2.3; extra == "heim"
+Requires-Dist: pytorch-lightning~=2.0; extra == "heim"
+Requires-Dist: flax~=0.10.7; python_version >= "3.10" and extra == "heim"
+Requires-Dist: flax~=0.8.5; python_version < "3.10" and extra == "heim"
+Requires-Dist: ftfy~=6.1; extra == "heim"
+Requires-Dist: Unidecode~=1.3; extra == "heim"
 Requires-Dist: wandb~=0.16; extra == "heim"
-Requires-Dist: google-cloud-translate~=3.11.2; extra == "heim"
-Requires-Dist: autokeras~=1.0.20; extra == "heim"
-Requires-Dist: clip-anytorch~=2.5.0; extra == "heim"
+Requires-Dist: google-cloud-translate~=3.11; extra == "heim"
+Requires-Dist: autokeras~=1.0; extra == "heim"
+Requires-Dist: clip-anytorch~=2.5; extra == "heim"
 Requires-Dist: google-cloud-storage~=2.9; extra == "heim"
 Requires-Dist: lpips~=0.1.4; extra == "heim"
-Requires-Dist: multilingual-clip~=1.0.10; extra == "heim"
-Requires-Dist: NudeNet~=2.0.9; extra == "heim"
-Requires-Dist: opencv-python<4.8.2.0,>=4.7.0.68; extra == "heim"
+Requires-Dist: multilingual-clip~=1.0; extra == "heim"
+Requires-Dist: NudeNet~=2.0; extra == "heim"
+Requires-Dist: opencv-python<4.8.2.0,>=4.7.0.68; python_version >= "3.10" and extra == "heim"
+Requires-Dist: opencv-python-headless<=4.11.0.86,>=4.7.0.68; python_version < "3.10" and extra == "heim"
 Requires-Dist: pytorch-fid~=0.3.0; extra == "heim"
 Requires-Dist: tensorflow~=2.11; extra == "heim"
 Requires-Dist: timm~=0.6.12; extra == "heim"
@@ -195,18 +197,15 @@ Requires-Dist: torchmetrics~=0.11.1; extra == "heim"
 Requires-Dist: scikit-image!=0.23.*,==0.*,>=0.22; extra == "heim"
 Requires-Dist: crfm-helm[images]; extra == "heim"
 Provides-Extra: medhelm
-Requires-Dist: crfm-helm[accelerate]; extra == "medhelm"
+Requires-Dist: accelerate~=0.25; extra == "medhelm"
 Requires-Dist: crfm-helm[openai]; extra == "medhelm"
-Requires-Dist: crfm-helm[summarization]; extra == "medhelm"
 Requires-Dist: crfm-helm[yandex]; extra == "medhelm"
+Requires-Dist: crfm-helm[scenarios]; extra == "medhelm"
 Requires-Dist: bert_score~=0.3.13; extra == "medhelm"
-Requires-Dist: langchain~=0.3.9; extra == "medhelm"
-Requires-Dist: langchain-community~=0.3.8; extra == "medhelm"
-Requires-Dist: lxml~=5.3.0; extra == "medhelm"
+Requires-Dist: lxml~=5.3; extra == "medhelm"
 Requires-Dist: openpyxl~=3.1; extra == "medhelm"
-Requires-Dist: python-docx~=1.1.2; extra == "medhelm"
-Requires-Dist: torch~=2.2.2; extra == "medhelm"
-Requires-Dist: torchvision~=0.17.2; extra == "medhelm"
+Requires-Dist: python-docx~=1.1; extra == "medhelm"
+Requires-Dist: transformers<4.50,~=4.45; extra == "medhelm"
 Provides-Extra: audiolm
 Requires-Dist: crfm-helm[openai]; extra == "audiolm"
 Requires-Dist: crfm-helm[google]; extra == "audiolm"
@@ -216,16 +215,21 @@ Requires-Dist: soundfile~=0.12; extra == "audiolm"
 Requires-Dist: librosa~=0.10; extra == "audiolm"
 Requires-Dist: einops~=0.7.0; extra == "audiolm"
 Requires-Dist: openai-whisper==20240930; extra == "audiolm"
-Requires-Dist: transformers~=4.48.0; extra == "audiolm"
+Requires-Dist: transformers~=4.48; extra == "audiolm"
 Requires-Dist: transformers_stream_generator~=0.0.4; extra == "audiolm"
-Requires-Dist: av~=14.3.0; extra == "audiolm"
+Requires-Dist: av~=14.3; extra == "audiolm"
 Requires-Dist: scipy~=1.10; extra == "audiolm"
 Requires-Dist: torchvision<3.0.0,>=0.14.1; extra == "audiolm"
-Requires-Dist: flash-attn~=2.7.3; extra == "audiolm"
+Requires-Dist: flash-attn~=2.7; extra == "audiolm"
 Requires-Dist: pycocoevalcap~=1.2; extra == "audiolm"
 Requires-Dist: jiwer~=3.0; extra == "audiolm"
 Requires-Dist: rapidfuzz~=3.10; extra == "audiolm"
 Requires-Dist: jieba~=0.42.1; extra == "audiolm"
+Provides-Extra: codeinsights
+Requires-Dist: clang~=20.1; extra == "codeinsights"
+Requires-Dist: Levenshtein~=0.27; extra == "codeinsights"
+Provides-Extra: lmkt
+Requires-Dist: sentence_transformers~=4.1; extra == "lmkt"
 Provides-Extra: all
 Requires-Dist: crfm-helm[proxy-server]; extra == "all"
 Requires-Dist: crfm-helm[human-evaluation]; extra == "all"
@@ -240,8 +244,11 @@ Requires-Dist: crfm-helm[models]; extra == "all"
 Requires-Dist: crfm-helm[mongo]; extra == "all"
 Requires-Dist: crfm-helm[heim]; extra == "all"
 Requires-Dist: crfm-helm[vlm]; extra == "all"
+Requires-Dist: crfm-helm[codeinsights]; extra == "all"
+Requires-Dist: crfm-helm[lmkt]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: pytest~=7.2.0; extra == "dev"
+Requires-Dist: xdoctest~=1.2.0; extra == "dev"
 Requires-Dist: pre-commit~=2.20.0; extra == "dev"
 Requires-Dist: black==24.3.0; extra == "dev"
 Requires-Dist: mypy==1.16.0; extra == "dev"
@@ -353,75 +360,3 @@ url={https://openreview.net/forum?id=iO4LZibEqW},
 note={Featured Certification, Expert Certification}
 }
 ```
-
-# Tutorial
-
-This tutorial will explain how to use the HELM command line tools to run benchmarks, aggregate statistics, and visualize results.
-
-We will run two runs using the `mmlu` scenario on the `openai/gpt2` model. The `mmlu` scenario implements the **Massive Multitask Language (MMLU)** benchmark from [this paper](https://arxiv.org/pdf/2009.03300.pdf), and consists of a Question Answering (QA) task using a dataset with questions from 57 subjects such as elementary mathematics, US history, computer science, law, and more. Note that GPT-2 performs poorly on MMLU, so this is just a proof of concept. We will run two runs: the first using questions about anatomy, and the second using questions about philosophy.
-
-## Using `helm-run`
-
-`helm-run` is a command line tool for running benchmarks.
-
-To run this benchmark using the HELM command-line tools, we need to specify **run entries** that describes the desired runs. For this example, the run entries are `mmlu:subject=anatomy,model=openai/gpt2` (for anatomy) and `mmlu:subject=philosophy,model=openai/gpt2` (for philosophy).
-
-We will now use `helm-run` to execute the runs. Run this command:
-
-```sh
-helm-run --run-entries mmlu:subject=anatomy,model=openai/gpt2 mmlu:subject=philosophy,model=openai/gpt2 --suite my-suite --max-eval-instances 10
-```
-
-The meaning of the arguments are as follows:
-
-- `--run-entries` specifies the run entries from the desired runs.
-- `--suite` specifies a subdirectory under the output directory in which all the output will be placed.
-- `--max-eval-instances` limits evaluation to only *N* instances (i.e. items) from the benchmark, using a randomly shuffled order of instances.
-
-`helm-run` creates an environment directory environment and an output directory by default.
-
--  The environment directory is `prod_env/` by default and can be set using `--local-path`. Credentials for making API calls should be added to a `credentials.conf` file in this directory.
--  The output directory is `benchmark_output/` by default and can be set using `--output-path`.
-
-After running this command, navigate to the `benchmark_output/runs/my-suite/` directory. This should contain a two sub-directories named `mmlu:subject=anatomy,model=openai_gpt2` and `mmlu:subject=philosophy,model=openai_gpt2`. Note that the names of these sub-directories is based on the run entries we used earlier, but with `/` replaced with `_`.
-
-Each output sub-directory will contain several JSON files that were generated during the corresponding run:
-
-- `run_spec.json` contains the `RunSpec`, which specifies the scenario, adapter and metrics for the run.
-- `scenario.json` contains a serialized `Scenario`, which contains the scenario for the run and specifies the instances (i.e. inputs) used.
-- `scenario_state.json` contains a serialized `ScenarioState`, which contains every request to and response from the model.
-- `per_instance_stats.json` contains a serialized list of `PerInstanceStats`, which contains the statistics produced for the metrics for each instance (i.e. input).
-- `stats.json` contains a serialized list of `PerInstanceStats`, which contains the statistics produced for the metrics, aggregated across all instances (i.e. inputs).
-
-## Using `helm-summarize`
-
-The `helm-summarize` reads the output files of `helm-run` and computes aggregate statistics across runs. Run the following:
-
-```sh
-helm-summarize --suite my-suite
-```
-
-This reads the pre-existing files in `benchmark_output/runs/my-suite/` that were written by `helm-run` previously, and writes the following new files back to `benchmark_output/runs/my-suite/`:
-
-- `summary.json` contains a serialized `ExecutiveSummary` with a date and suite name.
-- `run_specs.json` contains the run entries for all the runs.
-- `runs.json` contains serialized list of `Run`, which contains the run path, run spec and adapter spec and statistics for each run.
-- `groups.json` contains a serialized list of `Table`, each containing information about groups in a group category.
-- `groups_metadata.json` contains a list of all the groups along with a human-readable description and a taxonomy.
-
-Additionally, for each group and group-relavent metric, it will output a pair of files: `benchmark_output/runs/my-suite/groups/latex/<group_name>_<metric_name>.tex` and `benchmark_output/runs/my-suite/groups/json/<group_name>_<metric_name>.json`. These files contain the statistics for that metric from each run within the group.
-
-## Using `helm-server`
-
-Finally, the `helm-server` command launches a web server to visualize the output files of `helm-run` and `helm-benchmark`. Run:
-
-```sh
-helm-server --suite my-suite
-```
-
-Open a browser and go to http://localhost:8000/ to view the visualization. You should see a similar view as [live website for the paper](https://crfm.stanford.edu/helm/classic/latest/), but for the data from your benchmark runs. The website has the following sections accessible from the top menu bar:
-
-- **Leaderboards** contains the leaderboards with aggregate metrics.
-- **Models** contains a list of models and their descriptions
-- **Scenarios** contains a list of scenarios and their descriptions.
-- **Predictions** contains a searchable list of runs.