PyPI - crfm-helm - Versions diffs - 0.5.2__py3-none-any.whl → 0.5.4__py3-none-any.whl - Mend

crfm-helm 0.5.2py3-none-any.whl → 0.5.4py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of crfm-helm might be problematic. Click here for more details.

Files changed (209) hide show

helm/benchmark/static/config.js DELETED Viewed

@@ -1,3 +0,0 @@
-window.BENCHMARK_OUTPUT_BASE_URL = "benchmark_output";
-window.SUITE = "latest";
-window.RELEASE = null;

helm/benchmark/static/general.js DELETED Viewed

@@ -1,122 +0,0 @@
-function assert(condition, message) {
-  if (!condition) {
-    throw message || "Assertion failed";
-  }
-}
-function encodeUrlParams(params) {
-  let s = "";
-  for (let k in params) {
-    if (params[k] != null) {
-      s += (s === "" ? "?" : "&") + k + "=" + encodeURIComponent(params[k]);
-    }
-  }
-  return s;
-}
-function decodeUrlParams(str) {
-  const params = {};
-  if (str === "") return params;
-  const items = str.substring(1).split(/&/);
-  for (let i = 0; i < items.length; i++) {
-    const pair = items[i].split(/=/);
-    params[pair[0]] = decodeURIComponent(pair[1]);
-  }
-  return params;
-}
-function updateBrowserLocation(params) {
-  // Update the address bar
-  window.history.pushState(
-    {},
-    "",
-    window.location.pathname + encodeUrlParams(params),
-  );
-}
-function createCookie(key, value, days) {
-  let expires = "";
-  if (days) {
-    const date = new Date();
-    date.setTime(date.getTime() + days * 24 * 60 * 60 * 1000);
-    expires = "; expires=" + date.toUTCString();
-  }
-  document.cookie = key + "=" + value + expires + "; path=/";
-}
-function readCookie(key) {
-  let tokens = document.cookie.split(";");
-  for (let i = 0; i < tokens.length; i++) {
-    const [k, v] = tokens[i].trim().split("=", 2);
-    if (key === k) return v;
-  }
-  return null;
-}
-function eraseCookie(key) {
-  createCookie(key, "", -1);
-}
-function renderTimestamp(timestamp) {
-  if (!timestamp) return null;
-  const d = new Date(timestamp * 1000);
-  return d.toLocaleString();
-}
-function renderDict(data) {
-  return JSON.stringify(data).substring(0, 10000);
-}
-function loadScript(src, onload, onerror) {
-  // Using jquery doesn't work, so do it in with our bare hands.
-  const s = document.createElement("script");
-  s.src = src;
-  s.onload = onload;
-  s.onerror = onerror;
-  document.head.appendChild(s);
-}
-function getRandomString() {
-  const vocab =
-    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
-  let text = "";
-  for (let i = 0; i < 6; i++)
-    text += vocab.charAt(Math.floor(Math.random() * vocab.length));
-  return text;
-}
-function round(x, n) {
-  const base = Math.pow(10, n);
-  return Math.round(x * base) / base;
-}
-function multilineHtml(s) {
-  return s.replace(/\n/g, "<br>");
-}
-function renderError(e) {
-  return $("<div>").addClass("alert alert-danger").append(multilineHtml(e));
-}
-function helpIcon(help, link) {
-  // Show a ?
-  return $("<a>", { href: link, target: "blank_", class: "help-icon" }).append(
-    $("<img>", { src: "info-icon.png", width: 15, title: help }),
-  );
-}
-const markdownConverter = new showdown.Converter({ optionKey: "value" });
-function renderMarkdown(markdown) {
-  return markdown && markdownConverter.makeHtml(markdown);
-}
-function refreshHashLocation() {
-  // If we request a hash location (URL contains #foo), the problem is #foo
-  // might not exist (since it's generated).  Call this function to jump to the
-  // hash location once all the anchors are generated.
-  if (location.hash) {
-    const hash = location.hash;
-    location.hash = "";
-    location.hash = hash;
-  }
-}

helm/benchmark/static/images/crfm-logo.png DELETED Viewed

Binary file

helm/benchmark/static/images/helm-logo-simple.png DELETED Viewed

Binary file

helm/benchmark/static/images/helm-logo.png DELETED Viewed

Binary file

helm/benchmark/static/images/language-model-helm.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/ai21.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/anthropic.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/bigscience.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/cohere.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/eleutherai.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/google.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/meta.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/microsoft.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/nvidia.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/openai.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/together.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/tsinghua-keg.png DELETED Viewed

Binary file

helm/benchmark/static/images/organizations/yandex.png DELETED Viewed

Binary file

helm/benchmark/static/images/scenarios-by-metrics.png DELETED Viewed

Binary file

helm/benchmark/static/images/taxonomy-scenarios.png DELETED Viewed

Binary file

helm/benchmark/static/index.html DELETED Viewed

@@ -1,68 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <title>Holistic Evaluation of Language Models (HELM)</title>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/css/bootstrap.min.css" integrity="sha384-/Y6pD6FV/Vv2HJnA6t+vslU6fwYXjCFtcEpHbNJ0lyAFsXTsjBbfaDjzALeQsN6M" crossorigin="anonymous">
-    <link rel="stylesheet" type="text/css" href="benchmarking.css">
-  </head>
-  <body>
-    <div class="container-fluid">
-      <nav class="navbar navbar-expand-sm navbar-light bg-faded">
-        <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#nav-content" aria-controls="nav-content" aria-expanded="false" aria-label="Toggle navigation">
-          <span class="navbar-toggler-icon"></span>
-        </button>
-            <a class="nav-link active" href="https://crfm.stanford.edu"><img src="images/crfm-logo.png" width="100"/></a>
-        <div class="collapse navbar-collapse" id="nav-content">
-          <ul class="navbar-nav">
-            <li class="nav-item"><a class="nav-link active" href="?"><img src="images/helm-logo-simple.png" width="80"/></a></li>
-            <li class="nav-item"><a class="nav-link active" href="?models=1">Models</a></li>
-            <li class="nav-item"><a class="nav-link active" href="?scenarios=1">Scenarios</a></li>
-            <li class="nav-item"><a class="nav-link active" href="?groups=1">Results</a></li>
-            <!--
-            TODO(#1441): Enable plots.
-            <li class="nav-item"><a class="nav-link active" href="?plots=1">Plots</a></li>
-            -->
-            <li class="nav-item"><a class="nav-link active" href="?runs=1">Raw runs</a></li>
-          </ul>
-        </div>
-        <div class="text-right" id="summary" style="white-space: nowrap">
-        </div>
-      </nav>
-      <div class="row">
-        <div class="col-sm-12" id="main">
-        </div>
-      </div>
-    </div>
-    <script src="https://code.jquery.com/jquery-3.2.1.min.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.11.0/umd/popper.min.js" integrity="sha384-b/U6ypiBEHpOf/4+1nzFpr53nxSS+GLCkfwBdFNTxtclqqenISfwAzpKaMNFNmj4" crossorigin="anonymous"></script>
-    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/js/bootstrap.min.js" integrity="sha384-h0AbiXch4ZDo7tp9hKZ4TsHbi047NrKGLO3SEJAg45jXxnGIfYzk4Si90RDIqNm1" crossorigin="anonymous"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.tablesorter/2.29.0/js/jquery.tablesorter.min.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/js-yaml/4.1.0/js-yaml.min.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/showdown/2.0.3/showdown.min.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/handlebars.js/4.7.7/handlebars.min.js"></script>
-    <!-- *GTAG*
-    <script async src="https://www.googletagmanager.com/gtag/js?id=G-T0MW28MP3W"></script>
-    <script>
-        window.dataLayer = window.dataLayer || [];
-            function gtag() {
-              dataLayer.push(arguments);
-            }
-            gtag('js', new Date());
-            gtag('config', 'G-T0MW28MP3W');
-    </script>
-      *GTAG* -->
-    <script src="config.js"></script>
-    <script src="general.js"></script>
-    <script src="utils.js"></script>
-    <script src="json-urls.js"></script>
-    <script src="benchmarking.js"></script>
-    <script src="plot-captions.js"></script>
-  </body>
-</html>

helm/benchmark/static/info-icon.png DELETED Viewed

Binary file

helm/benchmark/static/json-urls.js DELETED Viewed

@@ -1,69 +0,0 @@
-////////////////////////////////////////////////////////////
-// Helper functions for getting URLs of JSON files
-function versionBaseUrl() {
-  if (window.RELEASE) {
-    return `${BENCHMARK_OUTPUT_BASE_URL}/releases/${window.RELEASE}`;
-  } else {
-    return `${BENCHMARK_OUTPUT_BASE_URL}/runs/${window.SUITE}`;
-  }
-}
-function schemaJsonUrl() {
-  return `${versionBaseUrl()}/schema.json`;
-}
-function summaryJsonUrl() {
-  return `${versionBaseUrl()}/summary.json`;
-}
-function runsToRunSuitesJsonUrl() {
-  return `${versionBaseUrl()}/runs_to_run_suites.json`;
-}
-function runSpecsJsonUrl() {
-  return `${versionBaseUrl()}/run_specs.json`;
-}
-function groupsMetadataJsonUrl() {
-  return `${versionBaseUrl()}/groups_metadata.json`;
-}
-function groupsJsonUrl() {
-  return `${versionBaseUrl()}/groups.json`;
-}
-function groupJsonUrl(groupName) {
-  return `${versionBaseUrl()}/groups/${groupName}.json`;
-}
-function runSpecJsonUrl(suite, runSpecName) {
-  return `${BENCHMARK_OUTPUT_BASE_URL}/runs/${suite}/${runSpecName}/run_spec.json`;
-}
-function scenarioJsonUrl(suite, runSpecName) {
-  return `${BENCHMARK_OUTPUT_BASE_URL}/runs/${suite}/${runSpecName}/scenario.json`;
-}
-function scenarioStateJsonUrl(suite, runSpecName) {
-  return `${BENCHMARK_OUTPUT_BASE_URL}/runs/${suite}/${runSpecName}/scenario_state.json`;
-}
-function statsJsonUrl(suite, runSpecName) {
-  return `${BENCHMARK_OUTPUT_BASE_URL}/runs/${suite}/${runSpecName}/stats.json`;
-}
-function instancesJsonUrl(suite, runSpecName) {
-  return `${BENCHMARK_OUTPUT_BASE_URL}/runs/${suite}/${runSpecName}/instances.json`;
-}
-function predictionsJsonUrl(suite, runSpecName) {
-  return `${BENCHMARK_OUTPUT_BASE_URL}/runs/${suite}/${runSpecName}/display_predictions.json`;
-}
-function requestsJsonUrl(suite, runSpecName) {
-  return `${BENCHMARK_OUTPUT_BASE_URL}/runs/${suite}/${runSpecName}/display_requests.json`;
-}
-function plotUrl(suite, plotName) {
-  return `${BENCHMARK_OUTPUT_BASE_URL}/runs/${suite}/plots/${plotName}.png`;
-}

helm/benchmark/static/plot-captions.js DELETED Viewed

@@ -1,27 +0,0 @@
-////////////////////////////////////////////////////////////
-// Dictionary of plot captions
-const plotCaptions = {
-  generic_summary:
-    "Metrics for every model on every core scenario as a means for indicating the spread on a per-metric basis.",
-  model_ranking_all:
-    "The fraction of head-to-head comparisons between the given model and all other models, across all scenarios, where the given model is higher along the metric (e.g. more accurate in the accuracy subfigure). If a model was the highest for the given metric for every scenario, it would receive a score of 1.0; if a model received a score of 0.5, then if a scenario and second model were chosen at random, the outcome of the comparison would be a coin flip. For calibration error, we measure ECE-10; for bias, we measure bias in gender representation.",
-  accuracy_v_x:
-    "The relationship between accuracy (x-axis) and each of the 6 metrics (calibration, robustness, fairness, social bias, toxicity, efficiency) we study in this work across all core scenarios and for all models. For calibration error, we measure ECE-10; for bias, we measure bias in gender representation; and for efficiency, we measure denoised inference time.",
-  metric_correlation:
-    "The Pearson correlation between each metric and every other metric (x-axis). The small blue dots denote the correlation on each individual scenario, while the larger orange dots average the correlation across scenarios. Trends are qualitatively similarly for other correlation measures (e.g. Spearman correlation). For calibration error, we measure ECE-10; for bias, we measure bias in gender representation; and for efficiency, we measure denoised inference time.",
-  accuracy_v_access:
-    "The relationship between access (open vs. limited vs.  closed) and model accuracy for each of the core scenarios. Shaded bars indicate the performance of the best model for that scenario, whereas the solid bars indicate the performance of the overall most accurate model across all core scenarios.",
-  accuracy_over_num_parameters:
-    "Cumulative plot, depicting the accuracy of the most accurate model up to a given size across all core scenarios.",
-  accuracy_over_release_date:
-    "The relationship between time (x-axis) and the accuracy of models (y-axis) across the core scenarios.",
-  accuracy_over_the_pile_perplexity:
-    "The relationship between log bits-per-byte (BPB) on The Pile and the accuracy on each core scenario.",
-  targeted_evals:
-    "Model accuracy on scenario targeting specific performance components (language, knowledge, reasoning).",
-  in_context_ablations:
-    "For each model, we set the maximum number of in-context examples to [0, 1, 2, 4, 8, 16] and fit as many in-context examples as possible within the context window. We plot performance as a function of the average number of in-context examples actually used.",
-  mc_ablations:
-    "For each adaptation method (joint, separate, and separate calibrated), we compare models across scenarios.",
-};

helm/benchmark/static/utils.js DELETED Viewed

@@ -1,285 +0,0 @@
-////////////////////////////////////////////////////////////
-// Helper functions for visualizing benchmarking
-function describeField(field) {
-  let result = field.name + ": " + field.description;
-  if (field.values) {
-    result +=
-      "\nPossible values:\n" +
-      field.values
-        .map((value) => `- ${value.name}: ${value.description}`)
-        .join("\n");
-  }
-  return result;
-}
-function renderStopSequence(value) {
-  return JSON.stringify(value);
-}
-function renderFieldValue(field, value) {
-  if (!field.values) {
-    if (field.name === "stop_sequences") {
-      return renderStopSequence(value);
-    }
-    return value;
-  }
-  const valueField = field.values.find(
-    (valueField) => valueField.name === value,
-  );
-  return $("<a>", {
-    title: valueField ? valueField.description : "(no description)",
-  }).append(value);
-}
-function perturbationEquals(perturbation1, perturbation2) {
-  if (perturbation1 == null) {
-    return perturbation2 == null;
-  }
-  if (perturbation2 == null) {
-    return perturbation1 == null;
-  }
-  return renderDict(perturbation1) === renderDict(perturbation2);
-}
-function metricNameEquals(name1, name2) {
-  return (
-    name1.name === name2.name &&
-    name1.split === name2.split &&
-    name1.sub_split === name2.sub_split &&
-    perturbationEquals(name1.perturbation, name2.perturbation)
-  );
-}
-function renderPerturbation(perturbation) {
-  if (!perturbation) {
-    return "original";
-  }
-  // The perturbation field must have the "name" subfield
-  const verbose = false;
-  if (verbose) {
-    const fields_str = Object.keys(perturbation)
-      .filter((key) => key !== "name")
-      .map((key) => `${key}=${perturbation[key]}`)
-      .join(", ");
-    return perturbation.name + (fields_str ? "(" + fields_str + ")" : "");
-  } else {
-    return perturbation.name;
-  }
-}
-function renderMetricName(name) {
-  // Return a short name (suitable for a cell of a table)
-  // Example: name = {name: 'exact_match'}
-  let result = name.name.bold();
-  if (name.split) {
-    result +=
-      " on " + name.split + (name.sub_split ? "/" + name.sub_split : "");
-  }
-  if (name.perturbation) {
-    result += " with " + renderPerturbation(name.perturbation);
-  }
-  return result;
-}
-function describeMetricName(field, name) {
-  // Return a longer description that explains the name
-  let result = describeField(field);
-  if (name.split) {
-    result += `\n* on ${name.split}: evaluated on the subset of ${name.split} instances`;
-  }
-  if (name.perturbation) {
-    result += `\n* with ${renderPerturbation(
-      name.perturbation,
-    )}: applied this perturbation`;
-  }
-  return result;
-}
-function renderStatName(schema, statName) {
-  const metric = schema.metricsField(statName);
-  if (metric.display_name) {
-    return metric.display_name;
-  } else {
-    const formattedName = statName.replaceAll("_", " ");
-    const capitalizedName =
-      formattedName.charAt(0).toUpperCase() + formattedName.slice(1);
-    return capitalizedName;
-  }
-}
-function renderPerturbationName(schema, perturbationName) {
-  return schema.perturbationsField(perturbationName).display_name;
-}
-function renderScenarioDisplayName(scenario, scenarioSpec) {
-  // Describe the scenario
-  const name = scenario.name; // e.g. mmlu
-  const args = scenarioSpec.args; // e.g., {subject: 'philosophy'}
-  if (Object.keys(args).length > 0) {
-    return name + " (" + renderDict(args) + ")";
-  } else {
-    return name;
-  }
-}
-////////////////////////////////////////////////////////////
-// Generic utility functions
-function renderHeader(header, body) {
-  return $("<div>").append($("<h4>").append(header)).append(body);
-}
-function getJSONList(paths, callback, defaultValue) {
-  // Fetch the JSON files `paths`, and pass the list of results into `callback`.
-  const responses = {};
-  const deferred = $.Deferred();
-  $.when(
-    ...paths.map((path) =>
-      $.getJSON(path, {}, (response) => {
-        responses[path] = response;
-      }),
-    ),
-  ).then(
-    () => {
-      const result = paths.map((path) => responses[path] || defaultValue);
-      if (callback) {
-        callback(result);
-      }
-      deferred.resolve(result);
-    },
-    (error) => {
-      console.error(
-        "Failed to load / parse:",
-        paths.filter((path) => !(path in responses)),
-      );
-      console.error(error.responseText);
-      const result = paths.map((path) => responses[path] || defaultValue);
-      if (callback) {
-        callback(result);
-      }
-      deferred.resolve(result);
-    },
-  );
-  return deferred.promise();
-}
-function getLast(l) {
-  return l[l.length - 1];
-}
-function sortListWithReferenceOrder(list, referenceOrder) {
-  // Return items in `list` based on referenceOrder.
-  // Example:
-  // - list = [3, 5, 2], referenceOrder = [2, 5]
-  // - Returns [2, 5, 3]
-  function getKey(x) {
-    const i = referenceOrder.indexOf(x);
-    return i === -1 ? 9999 : i; // Put unknown items at the end
-  }
-  list.sort(([a, b]) => getKey(a) - getKey(b));
-}
-function canonicalizeList(lists, compare) {
-  // Takes as input a list of lists and optional compare function,
-  // and returns the list of unique elements (preserving order).
-  // compare(a, b) should return 0 if and only if a equals b.
-  // Example: lists = [[1, 2, 3], [2, 3, 4]]
-  //   => [1, 2, 3, 4]
-  // Example: lists = [[1, 2, 3], [2, 3, 4]], compare = (a, b) => a % 2 - b % 2
-  //   => [1, 2]
-  const result = [];
-  lists.forEach((list) => {
-    list.forEach((elem) => {
-      const inResult = compare
-        ? result.some((resultElem) => compare(resultElem, elem) === 0)
-        : result.indexOf(elem) >= 0;
-      if (!inResult) {
-        result.push(elem);
-      }
-    });
-  });
-  return result;
-}
-function dict(entries) {
-  // Make a dictionary (object) out of the key/value `entries`
-  const obj = {};
-  entries.forEach(([key, value]) => {
-    obj[key] = value;
-  });
-  return obj;
-}
-function findDiff(items) {
-  // `items` is a list of dictionaries.
-  // Return a corresponding list of dictionaries where all the common keys have been removed.
-  const commonKeys = Object.keys(items[0]).filter((key) =>
-    items.every(
-      (item) => JSON.stringify(item[key]) === JSON.stringify(items[0][key]),
-    ),
-  );
-  return items.map((item) => {
-    return dict(
-      Object.entries(item).filter(
-        (entry) => commonKeys.indexOf(entry[0]) === -1,
-      ),
-    );
-  });
-}
-function renderDict(obj) {
-  return Object.entries(obj)
-    .map(([key, value]) => `${key}=${value}`)
-    .join(",");
-}
-function truncateMiddle(text, border) {
-  // Return `text` with only `border` characters at the beginning and at the end
-  // Example: "this is a test", border=4 ==> "this...(6 characters)...test"
-  if (text.length <= border * 2) {
-    return text;
-  }
-  const numRemoved = text.length - 2 * border;
-  return (
-    text.substring(0, border) +
-    ' <span style="color: lightgray">...(' +
-    numRemoved +
-    " characters)...</span> " +
-    text.substring(text.length - border)
-  );
-}
-function substitute(str, environment) {
-  if (!str) {
-    return str;
-  }
-  for (let key in environment) {
-    str = str.replace("${" + key + "}", environment[key]);
-  }
-  return str;
-}
-function renderAccess(access) {
-  return $("<span>", { class: "access-" + access + " btn" }).append(access);
-}
-function renderItems(items) {
-  // [1, 2, 3] => "[1 | 2 | 3]"
-  const $result = $("<div>");
-  $result.append("[ ");
-  items.forEach((item, index) => {
-    if (index > 0) {
-      $result.append(" | ");
-    }
-    $result.append(item);
-  });
-  $result.append(" ]");
-  return $result;
-}
-function getSuiteForRun(runNameToSuite, runName) {
-  suite = window.RELEASE ? runNameToSuite[runName] : window.SUITE;
-  return suite;
-}

crfm-helm 0.5.2__py3-none-any.whl → 0.5.4__py3-none-any.whl

Potentially problematic release.

crfm-helm 0.5.2py3-none-any.whl → 0.5.4py3-none-any.whl