crfm-helm 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

helm/benchmark/static_build/assets/index-e439d5e1.js

@@ -1,10 +0,0 @@
-import{r as o,a as De,L as b,O as Oe,d as Fe,u as ae,f as ge,h as Be,H as Ue,i as ze,j as C,R as We}from"./react-f82877fd.js";import{g as O,b as z,m as ee,s as je,a as qe,d as de,y as Ve,c as he,e as ne,l as re}from"./tremor-38a10867.js";import"./recharts-4037aff0.js";(function(){const t=document.createElement("link").relList;if(t&&t.supports&&t.supports("modulepreload"))return;for(const r of document.querySelectorAll('link[rel="modulepreload"]'))n(r);new MutationObserver(r=>{for(const l of r)if(l.type==="childList")for(const i of l.addedNodes)i.tagName==="LINK"&&i.rel==="modulepreload"&&n(i)}).observe(document,{childList:!0,subtree:!0});function a(r){const l={};return r.integrity&&(l.integrity=r.integrity),r.referrerPolicy&&(l.referrerPolicy=r.referrerPolicy),r.crossOrigin==="use-credentials"?l.credentials="include":r.crossOrigin==="anonymous"?l.credentials="omit":l.credentials="same-origin",l}function n(r){if(r.ep)return;r.ep=!0;const l=a(r);fetch(r.href,l)}})();var be={exports:{}},K={};/**
- * @license React
- * react-jsx-runtime.production.min.js
- *
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */var Je=o,Ge=Symbol.for("react.element"),Qe=Symbol.for("react.fragment"),Ke=Object.prototype.hasOwnProperty,Ze=Je.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED.ReactCurrentOwner,Xe={key:!0,ref:!0,__self:!0,__source:!0};function ve(s,t,a){var n,r={},l=null,i=null;a!==void 0&&(l=""+a),t.key!==void 0&&(l=""+t.key),t.ref!==void 0&&(i=t.ref);for(n in t)Ke.call(t,n)&&!Xe.hasOwnProperty(n)&&(r[n]=t[n]);if(s&&s.defaultProps)for(n in t=s.defaultProps,t)r[n]===void 0&&(r[n]=t[n]);return{$$typeof:Ge,type:s,key:l,ref:i,props:r,_owner:Ze.current}}K.Fragment=Qe;K.jsx=ve;K.jsxs=ve;be.exports=K;var e=be.exports,se={},me=De;se.createRoot=me.createRoot,se.hydrateRoot=me.hydrateRoot;function Ye({title:s,titleId:t,...a},n){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:1.5,stroke:"currentColor","aria-hidden":"true",ref:n,"aria-labelledby":t},a),s?o.createElement("title",{id:t},s):null,o.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M3.75 6.75h16.5M3.75 12h16.5m-16.5 5.25h16.5"}))}const es=o.forwardRef(Ye),we=es;function ss({title:s,titleId:t,...a},n){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:1.5,stroke:"currentColor","aria-hidden":"true",ref:n,"aria-labelledby":t},a),s?o.createElement("title",{id:t},s):null,o.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M9 12.75L11.25 15 15 9.75M21 12a9 9 0 11-18 0 9 9 0 0118 0z"}))}const ts=o.forwardRef(ss),as=ts;function ns({title:s,titleId:t,...a},n){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:1.5,stroke:"currentColor","aria-hidden":"true",ref:n,"aria-labelledby":t},a),s?o.createElement("title",{id:t},s):null,o.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M9.75 9.75l4.5 4.5m0-4.5l-4.5 4.5M21 12a9 9 0 11-18 0 9 9 0 0118 0z"}))}const rs=o.forwardRef(ns),is=rs,ye=""+new URL("crfm-logo-74391ab8.png",import.meta.url).href,Ne=""+new URL("helm-logo-simple-2ed5400b.png",import.meta.url).href;function ls({title:s,titleId:t,...a},n){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor","aria-hidden":"true",ref:n,"aria-labelledby":t},a),s?o.createElement("title",{id:t},s):null,o.createElement("path",{fillRule:"evenodd",d:"M12 2.25a.75.75 0 01.75.75v11.69l3.22-3.22a.75.75 0 111.06 1.06l-4.5 4.5a.75.75 0 01-1.06 0l-4.5-4.5a.75.75 0 111.06-1.06l3.22 3.22V3a.75.75 0 01.75-.75zm-9 13.5a.75.75 0 01.75.75v2.25a1.5 1.5 0 001.5 1.5h13.5a1.5 1.5 0 001.5-1.5V16.5a.75.75 0 011.5 0v2.25a3 3 0 01-3 3H5.25a3 3 0 01-3-3V16.5a.75.75 0 01.75-.75z",clipRule:"evenodd"}))}const os=o.forwardRef(ls),cs=os;function ds({title:s,titleId:t,...a},n){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor","aria-hidden":"true",ref:n,"aria-labelledby":t},a),s?o.createElement("title",{id:t},s):null,o.createElement("path",{fillRule:"evenodd",d:"M15.75 2.25H21a.75.75 0 01.75.75v5.25a.75.75 0 01-1.5 0V4.81L8.03 17.03a.75.75 0 01-1.06-1.06L19.19 3.75h-3.44a.75.75 0 010-1.5zm-10.5 4.5a1.5 1.5 0 00-1.5 1.5v10.5a1.5 1.5 0 001.5 1.5h10.5a1.5 1.5 0 001.5-1.5V10.5a.75.75 0 011.5 0v8.25a3 3 0 01-3 3H5.25a3 3 0 01-3-3V8.25a3 3 0 013-3h8.25a.75.75 0 010 1.5H5.25z",clipRule:"evenodd"}))}const hs=o.forwardRef(ds),ms=hs;function us({title:s,titleId:t,...a},n){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor","aria-hidden":"true",ref:n,"aria-labelledby":t},a),s?o.createElement("title",{id:t},s):null,o.createElement("path",{fillRule:"evenodd",d:"M15 3.75a.75.75 0 01.75-.75h4.5a.75.75 0 01.75.75v4.5a.75.75 0 01-1.5 0V5.56l-3.97 3.97a.75.75 0 11-1.06-1.06l3.97-3.97h-2.69a.75.75 0 01-.75-.75zm-12 0A.75.75 0 013.75 3h4.5a.75.75 0 010 1.5H5.56l3.97 3.97a.75.75 0 01-1.06 1.06L4.5 5.56v2.69a.75.75 0 01-1.5 0v-4.5zm11.47 11.78a.75.75 0 111.06-1.06l3.97 3.97v-2.69a.75.75 0 011.5 0v4.5a.75.75 0 01-.75.75h-4.5a.75.75 0 010-1.5h2.69l-3.97-3.97zm-4.94-1.06a.75.75 0 010 1.06L5.56 19.5h2.69a.75.75 0 010 1.5h-4.5a.75.75 0 01-.75-.75v-4.5a.75.75 0 011.5 0v2.69l3.97-3.97a.75.75 0 011.06 0z",clipRule:"evenodd"}))}const xs=o.forwardRef(us),fs=xs;function ps({title:s,titleId:t,...a},n){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor","aria-hidden":"true",ref:n,"aria-labelledby":t},a),s?o.createElement("title",{id:t},s):null,o.createElement("path",{fillRule:"evenodd",d:"M12.53 16.28a.75.75 0 01-1.06 0l-7.5-7.5a.75.75 0 011.06-1.06L12 14.69l6.97-6.97a.75.75 0 111.06 1.06l-7.5 7.5z",clipRule:"evenodd"}))}const gs=o.forwardRef(ps),Le=gs;function js({title:s,titleId:t,...a},n){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor","aria-hidden":"true",ref:n,"aria-labelledby":t},a),s?o.createElement("title",{id:t},s):null,o.createElement("path",{fillRule:"evenodd",d:"M11.47 4.72a.75.75 0 011.06 0l3.75 3.75a.75.75 0 01-1.06 1.06L12 6.31 8.78 9.53a.75.75 0 01-1.06-1.06l3.75-3.75zm-3.75 9.75a.75.75 0 011.06 0L12 17.69l3.22-3.22a.75.75 0 111.06 1.06l-3.75 3.75a.75.75 0 01-1.06 0l-3.75-3.75a.75.75 0 010-1.06z",clipRule:"evenodd"}))}const bs=o.forwardRef(js),Me=bs;function vs({title:s,titleId:t,...a},n){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor","aria-hidden":"true",ref:n,"aria-labelledby":t},a),s?o.createElement("title",{id:t},s):null,o.createElement("path",{fillRule:"evenodd",d:"M10.5 3.75a6.75 6.75 0 100 13.5 6.75 6.75 0 000-13.5zM2.25 10.5a8.25 8.25 0 1114.59 5.28l4.69 4.69a.75.75 0 11-1.06 1.06l-4.69-4.69A8.25 8.25 0 012.25 10.5z",clipRule:"evenodd"}))}const ws=o.forwardRef(vs),ys=ws;function ie(s,t){return t?t==="home"?"https://crfm.stanford.edu/helm/":s?`https://crfm.stanford.edu/helm/${t}/${s}/`:`https://crfm.stanford.edu/helm/${t}/latest/`:"#"}function Ee(){const[s,t]=o.useState([]),[a,n]=o.useState();return o.useEffect(()=>{if(a&&a.title&&a.title!=="All Leaderboards"){const r=a.title==="Lite"||a.title==="Classic"?"HELM "+a.title:a.title;document.title=r+" - Holistic Evaluation of Language Models (HELM)"}},[a]),o.useEffect(()=>{fetch("https://crfm.stanford.edu/helm/project_metadata.json").then(r=>r.json()).then(r=>{if(t(r),window.PROJECT_ID){const l=r.find(i=>i.id===window.PROJECT_ID);n(l)}else{const l=r.find(i=>i.id==="lite");n(l)}}).catch(r=>{console.error("Error fetching JSON:",r)})},[]),a===void 0||a.title===void 0?null:e.jsxs("div",{className:"dropdown z-50",children:[e.jsxs("div",{tabIndex:0,role:"button",className:"btn normal-case bg-white font-bold p-2 border-0 text-lg block whitespace-nowrap z-40","aria-haspopup":"true","aria-controls":"menu",children:[a.title," ",e.jsx(Le,{fill:"black",color:"black",className:"text w-4 h-4 inline"})]}),e.jsx("ul",{tabIndex:0,className:"-translate-x-36 dropdown-content z-[1] menu p-1 shadow-lg bg-base-100 rounded-box w-max text-base",role:"menu",children:s.map((r,l)=>e.jsx("li",{className:"z-40",children:e.jsxs("a",{href:ie(void 0,r.id),className:"block",role:"menuitem",children:[e.jsx("strong",{className:a.title===r.title?"underline":"",children:r.title}),": ",r.description]})},l))})]})}function A(s){return s.startsWith("http://")||s.startsWith("https://")?s:`${window.BENCHMARK_OUTPUT_BASE_URL.replace(/\/$/,"")}/${s.replace(/^\//,"").split("/").map(t=>encodeURIComponent(t)).join("/")}`}function W(){return window.RELEASE?`/releases/${window.RELEASE}`:`/runs/${window.SUITE}`}async function Ns(s){try{return await(await fetch(A(`${W()}/summary.json`),{signal:s})).json()}catch(t){return console.log(t),{release:void 0,suites:void 0,suite:void 0,date:""}}}function Ls(){const[s,t]=o.useState({release:void 0,suites:void 0,suite:void 0,date:""}),[a,n]=o.useState();o.useEffect(()=>{fetch("https://crfm.stanford.edu/helm/project_metadata.json").then(m=>m.json()).then(m=>{if(window.PROJECT_ID){const x=m.find(p=>p.id===window.PROJECT_ID);n(x)}else{const x=m.find(p=>p.id==="lite");n(x)}}).catch(m=>{console.error("Error fetching JSON:",m)})},[]),o.useEffect(()=>{const m=new AbortController;async function x(){const p=await Ns(m.signal);t(p)}return x(),()=>m.abort()},[]);const r=a!==void 0&&a.releases!==void 0?a.releases:["v1.0.0"],l=s.release||s.suite||null;if(!l)return null;const i=`Release ${l} (${s.date})`;if(r.length<=1)return e.jsx("div",{children:i});const u=r.indexOf(l),h=u<0?e.jsx(O,{color:"blue",children:"preview"}):u===0?e.jsx(O,{color:"blue",children:"latest"}):e.jsx(O,{color:"yellow",children:"stale"});return e.jsxs("div",{className:"dropdown",children:[e.jsxs("div",{tabIndex:0,role:"button",className:"normal-case bg-white border-0 block whitespace-nowrap","aria-haspopup":"true","aria-controls":"menu",children:[i," ",h," ",e.jsx(Le,{fill:"black",color:"black",className:"inline text w-4 h-4"})]}),e.jsx("ul",{tabIndex:0,className:"dropdown-content z-[50] menu p-1 shadow-lg bg-base-100 rounded-box w-max text-base",role:"menu",children:["latest"].concat(r).map(m=>e.jsx("li",{children:e.jsx("a",{href:ie(m,a?a.id:"lite"),className:"block",role:"menuitem",children:m})},m))})]})}function Ms(){return e.jsxs("nav",{className:"navbar h-24 px-8 md:px-12 bg-base-100 max-w[1500]px",children:[e.jsx("div",{children:e.jsxs("div",{className:"dropdown md:hidden mr-4",children:[e.jsx("label",{tabIndex:0,className:"btn btn-ghost hover:bg-transparent btn-lg px-0",children:e.jsx(we,{className:"w-16 h-16"})}),e.jsxs("ul",{tabIndex:0,className:"menu menu-lg dropdown-content mt-3 z-50 p-2 bg-base-100 shadow",children:[e.jsx("li",{children:e.jsx(b,{to:"leaderboard",children:"Leaderboard"})}),e.jsx("li",{children:e.jsx(b,{to:"models",children:"Models"})}),e.jsx("li",{children:e.jsx(b,{to:"scenarios",children:"Scenarios"})}),e.jsx("li",{children:e.jsx(b,{to:"runs",className:"whitespace-nowrap",children:"Predictions"})}),e.jsx("li",{children:e.jsx(b,{to:"https://github.com/stanford-crfm/helm",children:"GitHub"})})]})]})}),e.jsxs("div",{className:"flex-1 items-center",children:[e.jsx("a",{href:"https://crfm.stanford.edu/",className:"w-24",children:e.jsx("img",{src:ye,className:"object-contain"})}),e.jsx(b,{to:"/",className:"mx-2 w-32",children:e.jsx("img",{src:Ne,className:"object-contain"})}),e.jsx(Ee,{})]}),e.jsx("div",{className:"flex-none hidden md:block",children:e.jsxs("ul",{className:"flex flex-row gap-6 px-1",children:[e.jsx("li",{children:e.jsx(b,{to:"leaderboard",children:"Leaderboard"})}),e.jsx("li",{children:e.jsx(b,{to:"models",children:"Models"})}),e.jsx("li",{children:e.jsx(b,{to:"scenarios",children:"Scenarios"})}),e.jsx("li",{children:e.jsx(b,{to:"runs",children:"Predictions"})}),e.jsx("li",{children:e.jsx(b,{to:"https://github.com/stanford-crfm/helm",children:"GitHub"})}),e.jsx("li",{className:"hidden lg:flex",children:e.jsx(Ls,{})})]})})]})}function Es(){return e.jsxs("nav",{className:"navbar h-24 px-8 md:px-12 bg-base-100 max-w[1500]px",children:[e.jsx("div",{children:e.jsx("div",{className:"dropdown md:hidden mr-4",children:e.jsx("label",{tabIndex:0,className:"btn btn-ghost hover:bg-transparent btn-lg px-0",children:e.jsx(we,{className:"w-16 h-16"})})})}),e.jsxs("div",{className:"flex-1 items-center",children:[e.jsx(b,{to:"https://crfm.stanford.edu/",className:"w-24",children:e.jsx("img",{src:ye,className:"object-contain"})}),e.jsx(b,{to:"/",className:"mx-2 w-32",children:e.jsx("img",{src:Ne,className:"object-contain"})}),e.jsx(Ee,{})]})]})}function Ss(){return e.jsxs(e.Fragment,{children:[window.PROJECT_ID==="home"?e.jsx(Es,{}):e.jsx(Ms,{}),e.jsx("main",{className:"p-8 pt-0",children:e.jsx("div",{className:"mx-auto max-w-[1500]px",children:e.jsx(Oe,{})})})]})}async function D(s){try{return await(await fetch(A(`${W()}/schema.json`),{signal:s})).json()}catch(t){return t instanceof Error&&t.name!=="AbortError"&&console.log(t),{adapter:[],metric_groups:[],metrics:[],models:[],perturbations:[],run_groups:[]}}}function ks({href:s,children:t}){return e.jsx("a",{href:s,className:"link link-primary link-hover",target:"_blank",rel:"noreferrer",children:t})}function F({value:s}){return e.jsx("span",{children:e.jsx(Fe,{components:{a:ks},children:s})})}function $({title:s,subtitle:t,markdown:a=!1}){return e.jsxs("header",{className:"m-4 ml-0",children:[e.jsx("h1",{className:"text-4xl",children:s}),a&&t!==void 0?e.jsx("h2",{className:"mt-2 text-neutral",children:e.jsx(F,{value:t})}):t!==void 0&&e.jsx("h2",{className:"mt-2 text-neutral",children:t})]})}const Rs={open:"green",limited:"yellow",closed:"red"},As={open:"Open",limited:"Limited",closed:"Closed"};function _s({level:s}){return e.jsx(O,{color:Rs[s],children:As[s]})}function H(){return e.jsx("div",{className:"w-full",children:e.jsx("div",{className:"block mx-auto my-24 loading loading-spinner loading-lg"})})}function Ts(){const[s,t]=o.useState([]);o.useEffect(()=>{const i=new AbortController;async function u(){const h=await D(i.signal);t(h.models)}return u(),()=>i.abort()},[]);const[a,n,r]=s.reduce((i,u)=>{switch(u.access){case"open":i[0]+=1;break;case"limited":i[1]+=1;break;case"closed":i[2]+=1;break}return i},[0,0,0]),l=Object.values(s.reduce((i,u)=>{const h=u.creator_organization;return i[h]===void 0?(i[h]={name:h,models:1},i):(i[h].models+=1,i)},{}));return s.length===0?e.jsx(H,{}):e.jsxs(e.Fragment,{children:[e.jsx($,{title:"Models"}),e.jsxs("div",{className:"overflow-x-auto mt-12",children:[e.jsxs("table",{className:"table",children:[e.jsx("thead",{children:e.jsxs("tr",{children:[e.jsx("th",{children:"Creator"}),e.jsx("th",{children:"Model"}),e.jsx("th",{children:"Description"}),e.jsx("th",{children:"Access"})]})}),e.jsx("tbody",{children:s.map(i=>e.jsxs("tr",{children:[e.jsx("td",{className:"text-lg",children:i.creator_organization}),e.jsxs("td",{children:[e.jsx("span",{className:"text-xl",children:i.display_name}),e.jsx("br",{}),e.jsx("span",{children:i.name})]}),e.jsx("td",{children:e.jsx(F,{value:i.description})}),e.jsx("td",{children:e.jsx(_s,{level:i.access})})]}))})]}),e.jsx($,{title:"Analysis"}),e.jsxs("div",{className:"grid md:grid-cols-3 grid-cols-1 gap-8",children:[e.jsxs(z,{className:"flex flex-col justify-between",children:[e.jsx(ee,{children:"Models"}),e.jsx(je,{className:"text-6xl md:!text-[96px]",children:s.length}),e.jsx(qe,{values:[a,n,r],colors:["green","yellow","red"]}),e.jsx(de,{categories:["Open","Limited","Closed"],colors:["green","yellow","red"]})]}),e.jsxs(z,{className:"md:col-span-2",children:[e.jsx(ee,{children:"Creator Organizations"}),e.jsxs("div",{className:"flex justify-between mt-4",children:[e.jsx(Ve,{data:l,category:"models",index:"name",variant:"pie",className:"basis-5/12"}),e.jsx(de,{categories:l.map(i=>i.name),className:"basis-7/12"})]})]})]})]})]})}function J({to:s,children:t,inTable:a=!1,title:n=""}){return a?e.jsx(b,{className:"link link-hover",to:s,title:n,children:t}):e.jsx(b,{className:"link link-primary link-hover",to:s,children:t})}function Cs(){const[s,t]=o.useState([]),[a,n]=o.useState({});o.useEffect(()=>{const i=new AbortController;async function u(){const h=await D(i.signal);t(h.run_groups.filter(m=>!m.todo&&m.taxonomy&&!m.display_name.includes("CLEVA"))),n(Object.fromEntries(h.metrics.map(m=>[m.name,m])))}return u(),()=>i.abort()},[]);const r=i=>{var x;const u=(x=i.environment)==null?void 0:x.main_name,h=u?a[u]:void 0;if(h===void 0)return"";const m=h.display_name||h.short_display_name||h.name;return h.description?`${m} – ${h.description}`:m},l=Object.values(s.reduce((i,u)=>{var m;const h=((m=u.taxonomy)==null?void 0:m.task)||"Unknown";return i[h]===void 0?(i[h]={name:h,value:1},i):(i[h].value+=1,i)},{}));return s.length===0?e.jsx(H,{}):(console.log(s),e.jsxs(e.Fragment,{children:[e.jsx($,{title:"Scenarios",subtitle:"A scenario represents a use case and consists of a dataset of instances."}),e.jsxs("div",{className:"overflow-x-auto mt-12",children:[e.jsxs("table",{className:"table",children:[e.jsx("thead",{children:e.jsxs("tr",{children:[e.jsx("th",{children:"Scenario"}),e.jsx("th",{children:"Task"}),e.jsx("th",{children:"What"}),e.jsx("th",{children:"Who"}),e.jsx("th",{children:"When"}),e.jsx("th",{children:"Language"}),e.jsx("th",{children:"Description"}),e.jsx("th",{children:"Metric"})]})}),e.jsx("tbody",{children:s.map(i=>{var u,h,m,x,p;return e.jsxs("tr",{children:[e.jsxs("td",{children:[e.jsx(J,{to:`/leaderboard/${i.name}`,children:e.jsx("span",{className:"text-lg",children:i.display_name})}),e.jsx("span",{className:"block",children:i.name})]}),e.jsx("td",{children:((u=i.taxonomy)==null?void 0:u.task)||""}),e.jsx("td",{children:((h=i.taxonomy)==null?void 0:h.what)||""}),e.jsx("td",{children:((m=i.taxonomy)==null?void 0:m.who)||""}),e.jsx("td",{children:((x=i.taxonomy)==null?void 0:x.when)||""}),e.jsx("td",{children:((p=i.taxonomy)==null?void 0:p.language)||""}),e.jsx("td",{children:e.jsx(F,{value:i.short_description||i.description})}),e.jsx("td",{children:e.jsx(F,{value:r(i)})})]})})})]}),e.jsx($,{title:"Analysis"}),e.jsxs("div",{className:"grid md:grid-cols-4 gap-8",children:[e.jsxs(z,{className:"flex flex-col",children:[e.jsx(ee,{children:"Total scenarios"}),e.jsx(je,{className:"mx-auto my-6 md:mt-16 !text-[72px] md:!text-[96px]",children:s.length})]}),e.jsx(z,{className:"col-span-3",children:e.jsxs("div",{className:"grid md:grid-cols-2 gap-x-12",children:[e.jsx(he,{data:l.slice(0,Math.floor(l.length/2))}),e.jsx(he,{data:l.slice(Math.ceil(l.length/2))})]})})]})]})]}))}function Is(){return A(`${W()}/groups.json`)}async function Se(s){try{return await(await fetch(Is(),{signal:s})).json()}catch(t){return t instanceof Error&&t.name!=="AbortError"&&console.log(t),[]}}function le({children:s}){return e.jsx("div",{role:"navigation",className:"tabs flex-nowrap border-b-2 border-gray-2 overflow-x-auto overflow-y-hidden",children:s})}function G({active:s=!1,onClick:t=()=>{},size:a="md",children:n}){return e.jsx("div",{onClick:t,className:`whitespace-nowrap text-${a} mb-[-2px] text-md tab tab-bordered${s?" border-2 border-grey-500 rounded":" border-none"}`,children:n})}function Ps({title:s,titleId:t,...a},n){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",fill:"currentColor","aria-hidden":"true",ref:n,"aria-labelledby":t},a),s?o.createElement("title",{id:t},s):null,o.createElement("path",{fillRule:"evenodd",d:"M4.25 5.5a.75.75 0 00-.75.75v8.5c0 .414.336.75.75.75h8.5a.75.75 0 00.75-.75v-4a.75.75 0 011.5 0v4A2.25 2.25 0 0112.75 17h-8.5A2.25 2.25 0 012 14.75v-8.5A2.25 2.25 0 014.25 4h5a.75.75 0 010 1.5h-5z",clipRule:"evenodd"}),o.createElement("path",{fillRule:"evenodd",d:"M6.194 12.753a.75.75 0 001.06.053L16.5 4.44v2.81a.75.75 0 001.5 0v-4.5a.75.75 0 00-.75-.75h-4.5a.75.75 0 000 1.5h2.553l-9.056 8.194a.75.75 0 00-.053 1.06z",clipRule:"evenodd"}))}const Hs=o.forwardRef(Ps),ue=Hs;function B(s){return Number.isNaN(Number(s))?String(s):String(Math.round(Number(s)*1e3)/1e3)}function Q({value:s,title:t,hideIcon:a}){if(typeof s.value=="string"&&s.value.includes("⚠")&&(s.value=s.value.replace("⚠","")),s.value===void 0)return"-";if(s.run_spec_names){const n=(()=>{if(s.run_spec_names.length==1)return"/runs/"+s.run_spec_names[0];if(s.run_spec_names.length>1){const r="/runs/?q="+s.run_spec_names.map(i=>`^${i}$`).join("|");return encodeURI(r)}})();return n?e.jsx(J,{to:n,inTable:!0,title:t,children:e.jsxs("div",{className:"flex items-center ",children:[B(s.value),!a&&e.jsx(ue,{className:"w-3 h-3 ml-1",stroke:"#cbcbcb",fill:"#cbcbcb"})]})}):t?e.jsx("a",{title:t,children:B(s.value)}):e.jsx(e.Fragment,{children:B(s.value)})}return s.href?e.jsx(J,{to:s.href,inTable:!0,title:t,children:e.jsxs("div",{className:"flex items-center",children:[B(s.value),!a&&e.jsx(ue,{className:"w-3 h-3 ml-1",stroke:"#cbcbcb",fill:"#cbcbcb"})]})}):s.markdown?e.jsx(F,{value:String(s.value)}):t?e.jsx("a",{title:t,children:B(s.value)}):e.jsx(e.Fragment,{children:B(s.value)})}function oe({schema:s,groupTable:t,numRowsToDisplay:a,sortColumnIndex:n=1,sortable:r=!0,displayColumnIndexes:l=void 0,miniStyle:i=!1}){const[u,h]=o.useState(1),[m,x]=o.useState(n);function p(c){return c.length>30?c.substring(0,27)+"...":c}const N=c=>{const f=["AIRBench 2024 -","-book"];if(c.value==="Model/adapter")return"Model";if(f.some(d=>c.value.includes(d))){let d=c.value;return f.forEach(w=>{d=d.replace(w,"")}),p(d)}else return p(c.value)},v=c=>{if(s){const f=s.models.find(d=>d.display_name===c);if(f){let d=f.description;return d.includes("/")&&(d=d.replace("/","_")),d}}return""},k=c=>{h(c===m?u*-1:c===0?-1:1),x(c)},j=c=>{if(s){const f=s.models.find(d=>d.display_name===c);if(f){let d=f.name;return d.includes("/")&&(d=d.replace("/","_")),d}}return""},_=()=>{const c=t.header[m].lower_is_better,f=u*(c?1:-1),d=t.rows.slice();return d.sort((w,M)=>{var L,R;const E=(L=w[m])==null?void 0:L.value,y=(R=M[m])==null?void 0:R.value;return E!==void 0&&y===void 0?-1:y!==void 0&&E===void 0?1:typeof E=="number"&&typeof y=="number"?(E-y)*f:typeof E=="string"&&typeof y=="string"?f===1?E.localeCompare(y):y.localeCompare(E):0}),a>0?d.slice(0,a):d};function g(c){const f=c.lastIndexOf(" - ");return f===-1?c:c.substring(0,f)+"*"+c.substring(f+1)}const T=c=>{const d=g(c).split("*")[0].trim();if(s){const w=s.run_groups.find(M=>M.display_name===d||M.short_display_name===d);if(w)return w.name}return""};return e.jsxs("table",{className:i?"table w-full":"rounded-lg shadow-md table",children:[e.jsx("thead",{children:e.jsx("tr",{children:t.header.filter((c,f)=>l===void 0||l.includes(f)).map((c,f)=>e.jsx("th",{className:`${f===m?"bg-gray-100":"bg-white"} ${f===0?"left-0 z-40":""} ${c.description?"underline decoration-dashed decoration-gray-300	":""}  whitespace-nowrap px-4 sticky top-0`,title:c.description?c.description:"",children:e.jsxs("div",{className:i?"flex gap-2 items-center":"z-20 flex justify-between items-center min-w-48 w-48 max-w-48 text-wrap",children:[e.jsx("span",{className:"inline-block w-full break-words",children:N(c)}),r?e.jsx("button",{className:"link",onClick:()=>k(f),children:e.jsx(Me,{className:"w-6 h-6"})}):null]})},`$${f}`))})}),e.jsx("tbody",{children:_().map((c,f)=>e.jsx("tr",{children:c.filter((d,w)=>l===void 0||l.includes(w)).map((d,w)=>e.jsx("td",{className:`${w===0?"z-20 text-lg sticky left-0":"z-0"} ${f%2===0?"bg-gray-50":"bg-white"}`,children:w==1?e.jsx("div",{className:`${d&&d.style&&d.style["font-weight"]&&d.style["font-weight"]==="bold"?"font-bold":""}`,children:e.jsx(Q,{value:{...d,href:"/runs/?q="+j(String(c[0].value))},title:`Click value to see all predictions for: ${j(String(c[0].value))}`})}):e.jsx("div",{className:`${d&&d.style&&d.style["font-weight"]&&d.style["font-weight"]==="bold"?"font-bold":""} ${w===0?"underline decoration-dashed decoration-gray-300 z-10":"z-0"}`,children:e.jsx(Q,{value:{...d},title:String(c[0].value)===d.value?v(String(c[0].value)):`Click value to see predictions for ${String(c[0].value)} for ${T(N(t.header[w]))}: ${j(String(c[0].value))}`})})},`${w}`))},`$${c[0].value}`))})]})}function $s(){const[s,t]=o.useState(0),[a,n]=o.useState(),[r,l]=o.useState();return o.useEffect(()=>{const i=new AbortController;async function u(){const h=D(i.signal),m=Se(i.signal),x=await h;l(x);const p=await m;n(p)}return u(),()=>i.abort()},[]),a===void 0||r===void 0?e.jsx(H,{}):a.length===0?e.jsxs("div",{children:[e.jsx($,{title:"Results",subtitle:"Groupings of the processes, methods, and metrics involved in evaluating models, particularly in the context of natural language understanding and question answering.",className:"mb-16"}),e.jsx("div",{children:"No groups found."})]}):e.jsxs("div",{children:[e.jsx($,{title:"Results",subtitle:"Groupings of the processes, methods, and metrics involved in evaluating models, particularly in the context of natural language understanding and question answering.",className:"mb-16"}),e.jsxs("div",{children:[a.length>1?e.jsx(le,{children:a.map((i,u)=>e.jsx(G,{active:u===s,onClick:()=>t(u),children:i.title},u))}):null,e.jsx(oe,{schema:r,groupTable:a[s],numRowsToDisplay:-1,sortColumnIndex:1,sortable:!0},`${s}`)]})]})}async function Ds(s){try{return await(await fetch(A(`${W()}/run_specs.json`),{signal:s})).json()}catch(t){return t instanceof Error&&t.name!=="AbortError"&&console.log(t),[]}}function ce({currentPage:s,totalPages:t,onNextPage:a,onPrevPage:n,className:r}){let l="join";return r!==void 0&&(l=`join ${r}`),e.jsxs("div",{className:l,children:[e.jsx("button",{onClick:n,className:"join-item btn",children:"«"}),e.jsxs("button",{className:"join-item btn",children:["Page ",s," of ",t]}),e.jsx("button",{onClick:a,className:"join-item btn",children:"»"})]})}const X=100;function Os(){const[s,t]=ae(),[a,n]=o.useState(),[r,l]=o.useState(Number(s.get("page")||1)),[i,u]=o.useState(!0),[h,m]=o.useState(s.get("q")||"");o.useEffect(()=>{const j=new AbortController;async function _(){const g=await Ds(j.signal);n(g)}return _(),()=>j.abort()},[]);const x=j=>{j.preventDefault();const g=j.target.q.value;m(g),t({q:g,page:"1"})};if(a===void 0)return e.jsx(H,{});const p=i?new RegExp(h):null,N=a.filter(j=>p?p.test(j.name):j.name.includes(h)),v=N.slice((r-1)*X,r*X),k=Math.ceil(N.length/X);return e.jsxs(e.Fragment,{children:[e.jsx($,{title:"Predictions",subtitle:"All benchmark predictions"}),e.jsxs("form",{className:"flex mb-8",onSubmit:x,children:[e.jsxs("div",{className:"form-control",children:[e.jsx("input",{type:"text",name:"q",placeholder:"Search",className:"input input-bordered",value:h,onChange:j=>m(j.target.value)}),e.jsxs("label",{className:"label",children:[e.jsxs("span",{className:"label-text-alt flex item-center",children:[e.jsx("input",{type:"checkbox",className:"toggle toggle-xs",checked:i,onChange:()=>u(!i)}),e.jsx("span",{className:"ml-2",children:"Regex"})]}),e.jsx("span",{className:"label-text-alt",children:`${N.length} results`})]})]}),e.jsx("div",{className:"form-control ml-4",children:e.jsx("button",{className:"btn",children:e.jsx(ys,{className:"w-6 h-6"})})})]}),e.jsx("div",{className:"overflow-x-auto",children:e.jsxs("table",{className:"table",children:[e.jsx("thead",{children:e.jsxs("tr",{children:[e.jsx("th",{children:"Run"}),e.jsx("th",{children:"Model"}),e.jsx("th",{children:"Groups"}),e.jsx("th",{children:"Adapter method"}),e.jsx("th",{children:"Subject / Task"})]})}),e.jsx("tbody",{children:v.map((j,_)=>e.jsxs("tr",{children:[e.jsx("td",{children:e.jsx(J,{to:`/runs/${j.name}`,children:j.name})}),e.jsx("td",{children:j.adapter_spec.model}),e.jsx("td",{children:j.groups.join(", ")}),e.jsx("td",{children:j.adapter_spec.method}),e.jsx("td",{children:j.scenario_spec.args.subject||j.scenario_spec.args.task||"-"})]},`${j.name}-${_}`))})]})}),k>0?e.jsx(ce,{className:"flex justify-center my-8",onNextPage:()=>{const j=Math.min(r+1,k);l(j),s.set("page",String(j)),t(s)},onPrevPage:()=>{const j=Math.max(r-1,1);l(j),s.set("page",String(j)),t(s)},currentPage:r,totalPages:k}):e.jsx("div",{className:"my-8 text-center",children:"No results"})]})}function P(){return window.SUITE!==void 0?window.SUITE:void 0}async function Fs(s,t,a){try{return await(await fetch(A(`/runs/${a||P()}/${s}/scenario.json`),{signal:t})).json()}catch(n){n instanceof Error&&n.name!=="AbortError"&&console.log(n);return}}function ke(s,t){return A(`/runs/${t||P()}/${s}/run_spec.json`)}async function Bs(s,t,a){try{return await(await fetch(ke(s,a),{signal:t})).json()}catch(n){n instanceof Error&&n.name!=="AbortError"&&console.log(n);return}}function Us(s,t){return A(`/runs/${t||P()}/${s}/scenario_state.json`)}function Re(){return window.RELEASE!==void 0?window.RELEASE:void 0}async function zs(s){try{return await(await fetch(A(`/releases/${Re()}/runs_to_run_suites.json`),{signal:s})).json()}catch(t){return t instanceof Error&&t.name!=="AbortError"&&console.log(t),{}}}function Ws(s,t){return Re()?s[t]:window.SUITE}function qs(s){const a={quasi_exact_match:!1,toxic_frac:!0,safety_score:!1,exact_match:!1},n=Object.keys(s);for(const r of n)if(s[r]!==void 0&&a[r]!==void 0)return a[r]?s[r]<.5?[r,!0]:[r,!1]:s[r]>=.5?[r,!0]:[r,!1];return["",!1]}function Vs(s){const[t,a]=qs(s.stats);return t===""?null:a?e.jsx(Js,{value:`${t.replace(/_/g," ")}: ${s.stats[t]}`}):e.jsx(Gs,{value:`${t.replace(/_/g," ")}: ${s.stats[t]}`})}function Js({value:s}){return e.jsx(O,{icon:as,color:"green",children:s})}function Gs({value:s}){return e.jsx(O,{icon:is,color:"red",children:s})}function I({value:s}){const[t,a]=o.useState(!1),[n,r]=o.useState(!1);return e.jsxs(e.Fragment,{children:[e.jsxs("div",{onMouseOver:()=>a(!0),onMouseOut:()=>a(!1),className:"relative",children:[e.jsx("div",{className:"bg-base-200 p-2 block overflow-auto w-full max-h-[36rem] mb-2 whitespace-pre-wrap",children:s}),t?e.jsx("button",{className:"bg-white absolute p-2 leading-none height-fit min-h-none right-1 bottom-1 shadow",onClick:()=>r(!0),children:e.jsx(fs,{fill:"black",color:"black",className:"text w-4 h-4"})}):null]}),e.jsx("dialog",{open:n,className:"modal p-16 bg-opacity-80 bg-white",onClick:()=>r(!1),children:e.jsx("div",{className:"modal-box max-w-none p-4 whitespace-pre-wrap bg-base-200",children:s})})]})}function Ae({messages:s}){return e.jsx("div",{children:s.map((t,a)=>e.jsxs("div",{children:[e.jsx("div",{children:t.role}),e.jsx(I,{value:t.content})]},a))})}function _e({mediaObject:s}){if(s.content_type.includes("image")){if(s.location===void 0)return null;const t=A(s.location.replace("benchmark_output/","").replace("prod_env/","../"));return e.jsxs("div",{children:[e.jsx("img",{src:t}),e.jsx("br",{})]})}else if(s.content_type.includes("audio")){if(s.location===void 0)return null;const t=A(s.location.replace(/^.*benchmark_output\//,"").replace("prod_env/","../"));return e.jsx("div",{children:e.jsx("audio",{controls:!0,src:t})})}else if(s.content_type.includes("video")){if(s.location===void 0)return null;const t=A(s.location.replace(/^.*benchmark_output\//,"").replace("prod_env/","../"));return e.jsxs("div",{children:[e.jsx("video",{controls:!0,style:{width:"720px"},children:e.jsx("source",{src:t,type:s.content_type})}),e.jsx("br",{})]})}else return s.text&&s.content_type&&s.content_type==="text/plain"&&s.text.length>1?e.jsxs("div",{children:[s.text,e.jsx("br",{}),e.jsx("br",{})]}):e.jsx("div",{})}function Te({multimediaObject:s}){return e.jsx("div",{children:s.media_objects.map(t=>e.jsx(_e,{mediaObject:t}))})}function Qs(s){return Array.isArray(s)?s.length==0?"[]":`[${s.map(t=>String(t).replace(/\n/,"\\n")).join(", ")}]`:String(s)}function Ks({request:s}){return e.jsxs("div",{children:[s.request.prompt.length>0?e.jsxs("div",{children:[e.jsxs("h3",{className:"block text text-gray-400",children:["Prompt (",s.request.prompt.length," Chars)"]}),e.jsx(I,{value:s.request.prompt})]}):s.request.multimodal_prompt?e.jsxs("div",{children:[e.jsx("h3",{className:"block text text-gray-400",children:"Prompt"}),e.jsx(Te,{multimediaObject:s.request.multimodal_prompt})]}):s.request.messages&&s.request.messages.length?e.jsxs("div",{children:[e.jsx("h3",{className:"block text text-gray-400",children:"Prompt"}),e.jsx(Ae,{messages:s.request.messages})]}):e.jsx("h3",{className:"block text text-gray-400",children:"Empty Prompt"}),e.jsx(ne,{children:Object.keys(s.request).filter(t=>t!=="prompt").map((t,a)=>e.jsxs(re,{children:[e.jsxs("span",{children:[t,":"]}),s.request&&s.request[t]?e.jsx("span",{children:Qs(s.request[t])}):"null"]},a+1))})]})}function Zs(s){return e.jsx("div",{children:s.map((t,a)=>e.jsxs("div",{children:[t.error&&e.jsxs("div",{children:[e.jsx("h3",{className:"ml-1",children:"Error"}),e.jsx(I,{value:t.error})," "]}),t.text&&e.jsxs("div",{children:[e.jsx("h3",{className:"ml-1",children:"Text"}),e.jsx(I,{value:t.text})," "]}),t.media_object&&e.jsx(_e,{mediaObject:t.media_object})]},a))})}function te(s,t){return Array.isArray(s)?s.flatMap((a,n)=>te(a,`${t||""}[${n}]`)):s instanceof Object&&s.constructor===Object?Object.entries(s).flatMap(([a,n])=>te(n,t?`${t}.${a}`:a)):[[t||"",typeof s=="string"?s:JSON.stringify(s)]]}function Xs(s){return e.jsx("div",{children:te(s).map(([t,a])=>e.jsxs("div",{children:[e.jsx("h3",{className:"ml-1",children:t}),e.jsx(I,{value:a})]},t))})}function Ys({predictionAnnotations:s}){return e.jsx("div",{children:s&&s!==void 0?Object.entries(s).map(([t,a])=>e.jsxs("details",{className:"collapse collapse-arrow border rounded-md bg-white my-2",children:[e.jsx("summary",{className:"collapse-title",children:e.jsx(e.Fragment,{children:"View "+t+" annotations"})}),e.jsx("div",{className:"collapse-content",children:Array.isArray(a)?Zs(a):Xs(a)})]},t)):null})}function et({predictions:s,requests:t,metricFieldMap:a}){return s.length<1?null:e.jsx("div",{children:e.jsx("div",{className:"flex flex-wrap justify-start items-start",children:s.map((n,r)=>e.jsxs("div",{className:"w-full",children:[s.length>1?e.jsxs("h2",{children:["Trial ",n.train_trial_index]}):null,e.jsxs("div",{className:"mt-2 w-full",children:[n.thinking_text?e.jsxs(e.Fragment,{children:[e.jsx("h3",{children:e.jsx("span",{className:"mr-4",children:"Thinking"})}),e.jsx(I,{value:n.thinking_text})]}):null,n.base64_images&&n.base64_images.length>0?e.jsxs(e.Fragment,{children:[e.jsx("h3",{className:"mr-4",children:"Prediction image"}),n.base64_images.map(l=>e.jsx("img",{src:"data:image;base64,"+l,alt:"Base64 Image"}))]}):e.jsxs(e.Fragment,{children:[e.jsxs("h3",{children:[e.jsx("span",{className:"mr-4",children:"Prediction raw text"}),e.jsx(Vs,{stats:n.stats})]}),e.jsx(I,{value:n.predicted_text}),n.mapped_output?e.jsxs(e.Fragment,{children:[e.jsx("h3",{children:"Prediction mapped output"}),e.jsx(I,{value:String(n.mapped_output)})]}):null]})]}),e.jsx(Ys,{predictionAnnotations:n.annotations}),e.jsxs("div",{className:"mx-1",children:[e.jsx("h3",{children:"Metrics"}),e.jsx(ne,{children:Object.keys(n.stats).map((l,i)=>e.jsxs(re,{children:[a[l]?e.jsx("span",{title:a[l].description,children:a[l].display_name}):e.jsx("span",{children:l}),e.jsx("span",{children:String(n.stats[l])})]},i))})]}),e.jsxs("details",{className:"collapse collapse-arrow border rounded-md bg-white",children:[e.jsx("summary",{className:"collapse-title",children:"Request details"}),e.jsx("div",{className:"collapse-content",children:e.jsx(Ks,{request:t[r]})})]})]},r))})})}const st="correct";function tt({references:s}){return e.jsxs("span",{children:[e.jsx("h3",{children:"References"}),e.jsx("ul",{children:s.map((t,a)=>e.jsxs("li",{className:"bg-base-200 p-2 block overflow-auto w-full max-h-72 mb-2 whitespace-pre-wrap",children:[t.output.text,t.tags.map(n=>e.jsx(O,{className:"mx-2",color:n===st?"green":void 0,children:n}))]},a))})]})}function Ce(s){return e.jsx(I,{value:s===null?"null":typeof s=="object"?JSON.stringify(s):s.toString()})}function at(s){return e.jsx("div",{children:s.map(t=>Ce(t))})}function nt({extraData:s}){return e.jsxs("details",{className:"collapse collapse-arrow border rounded-md bg-white my-2",children:[e.jsx("summary",{className:"collapse-title",children:"View instance extra data"}),e.jsx("div",{className:"collapse-content",children:Object.entries(s).map(([t,a])=>e.jsxs("div",{children:[e.jsx("h3",{className:"ml-1",children:t}),Array.isArray(a)?at(a):Ce(a)]}))})]})}function rt({instance:s,requests:t,predictions:a,metricFieldMap:n}){return e.jsxs("div",{children:[e.jsx("h3",{children:"Input"}),s.input.multimedia_content!==void 0?e.jsx(Te,{multimediaObject:s.input.multimedia_content}):s.input.text.includes('<br><img src="data:image;base64')?e.jsx("div",{dangerouslySetInnerHTML:{__html:s.input.text}}):s.input.messages!==void 0&&s.input.messages.length?e.jsx(Ae,{messages:s.input.messages}):e.jsx(I,{value:s.input.text}),e.jsx("div",{children:s.references&&s.references.length>0?e.jsx(tt,{references:s.references}):null}),s.extra_data&&s.extra_data.length?e.jsx(nt,{extraData:s.extra_data}):null,e.jsx("div",{children:a&&t?e.jsx(et,{predictions:a,requests:t,metricFieldMap:n}):null})]})}function Z(s){return s.includes("gpqa")||s.includes("ewok")}async function xe(s,t,a,n){const r=m=>Uint8Array.from(atob(m),x=>x.charCodeAt(0)),l=await window.crypto.subtle.importKey("raw",r(t),"AES-GCM",!0,["decrypt"]),i=new Uint8Array([...r(s),...r(n)]),u=r(a),h=await window.crypto.subtle.decrypt({name:"AES-GCM",iv:u},l,i);return new TextDecoder().decode(h)}async function it(s,t,a,n){try{const l=await(await fetch(A(`/runs/${a||P()}/${s}/instances.json`),{signal:t})).json();if(Z(s)&&n){const u=await(await fetch(A(`/runs/${a||P()}/${s}/encryption_data.json`),{signal:t})).json();for(const h of l){const m=u[h.input.text];m&&(h.input.text="encrypted",h.input.text=await xe(m.ciphertext,m.key,m.iv,m.tag));for(const x of h.references){const p=u[x.output.text];p&&(x.output.text=await xe(p.ciphertext,p.key,p.iv,p.tag))}}}return l}catch(r){return r instanceof Error&&r.name!=="AbortError"&&console.log(r),[]}}async function lt(s,t,a,n){const r=m=>Uint8Array.from(atob(m),x=>x.charCodeAt(0)),l=await window.crypto.subtle.importKey("raw",r(t),"AES-GCM",!0,["decrypt"]),i=new Uint8Array([...r(s),...r(n)]),u=r(a),h=await window.crypto.subtle.decrypt({name:"AES-GCM",iv:u},l,i);return new TextDecoder().decode(h)}async function ot(s,t,a,n){try{const l=await(await fetch(A(`/runs/${a||P()}/${s}/display_predictions.json`),{signal:t})).json();if(Z(s)&&n){const u=await(await fetch(A(`/runs/${a||P()}/${s}/encryption_data.json`),{signal:t})).json();for(const h of l){const m=h.predicted_text,x=u[m];if(x)try{h.predicted_text=await lt(x.ciphertext,x.key,x.iv,x.tag)}catch(p){console.error(`Failed to decrypt predicted_text for instance_id: ${h.instance_id}`,p)}}}return l}catch(r){return r instanceof Error&&r.name==="AbortError"&&console.log(r),[]}}async function ct(s,t,a,n){const r=m=>Uint8Array.from(atob(m),x=>x.charCodeAt(0)),l=await window.crypto.subtle.importKey("raw",r(t),"AES-GCM",!0,["decrypt"]),i=new Uint8Array([...r(s),...r(n)]),u=r(a),h=await window.crypto.subtle.decrypt({name:"AES-GCM",iv:u},l,i);return new TextDecoder().decode(h)}async function dt(s,t,a,n){try{const l=await(await fetch(A(`/runs/${a||P()}/${s}/display_requests.json`),{signal:t})).json();if(Z(s)&&n){const u=await(await fetch(A(`/runs/${a||P()}/${s}/encryption_data.json`),{signal:t})).json();for(const h of l){const m=h.request.prompt,x=u[m];if(x)try{h.request.prompt=await ct(x.ciphertext,x.key,x.iv,x.tag)}catch(p){console.error(`Failed to decrypt prompt for instance_id: ${h.instance_id}`,p)}}}return l}catch(r){return r instanceof Error&&r.name!=="AbortError"&&console.log(r),[]}}const q=10;function ht({runName:s,suite:t,metricFieldMap:a,userAgreed:n}){const[r,l]=ae(),[i,u]=o.useState([]),[h,m]=o.useState(),[x,p]=o.useState(),[N,v]=o.useState(1);o.useEffect(()=>{const g=new AbortController;async function T(){const c=g.signal,[f,d,w]=await Promise.all([it(s,c,t,n),ot(s,c,t,n),dt(s,c,t,n)]);u(f);const M={};w.forEach(y=>{var U;const L=y.instance_id,R=((U=y.perturbation)==null?void 0:U.name)||"";M[L]===void 0&&(M[L]={}),M[L][R]===void 0&&(M[L][R]=[]),M[L][R].push(y)}),p(M);const E={};d.forEach(y=>{var U;const L=y.instance_id,R=((U=y.perturbation)==null?void 0:U.name)||"";E[L]===void 0&&(E[L]={}),E[L][R]===void 0&&(E[L][R]=[]),E[L][R].push(y)}),m(E)}return T(),()=>g.abort()},[s,t,n]);const k=i.slice((N-1)*q,(N-1)*q+q),j=Math.ceil(i.length/q);o.useEffect(()=>{const g=r.get("instance");if(g&&!window.helmHasScrolledToInstance&&k.length>0){if(k.findIndex(c=>c.id===g)===-1)return;requestAnimationFrame(()=>{const c=document.getElementById(`instance-${g}`);c&&c.scrollIntoView({behavior:"smooth"})}),window.helmHasScrolledToInstance=!0}},[r,N,l,k]);const _=g=>g.perturbation===void 0?`Instance id: ${g.id} [split: ${g.split}]`:`Instance id: ${g.id} [split: ${g.split}][perturbation: ${g.perturbation.name}]`;return h===void 0||x===void 0?e.jsx(H,{}):e.jsxs(e.Fragment,{children:[e.jsx("div",{className:"grid gap-8",children:k.map((g,T)=>{var c,f;return e.jsxs("div",{id:"instance-"+g.id,className:"border p-4",children:[e.jsxs("div",{className:"flex items-center justify-between",children:[e.jsx("h3",{className:"text-xl mb-4",children:_(g)}),e.jsx("button",{className:"btn btn-sm normal-case px-2 py-1",onClick:()=>{const d=window.location.href+(window.location.href.includes("?")?"&instance=":"?instance=")+g.id;navigator.clipboard.writeText(d)},children:"Copy Link"})]}),e.jsx(rt,{instance:g,requests:x[g.id][((c=g.perturbation)==null?void 0:c.name)||""],predictions:h[g.id][((f=g.perturbation)==null?void 0:f.name)||""],metricFieldMap:a},`${g.id}-${T}`)]})})}),e.jsx(ce,{className:"flex justify-center my-8",onNextPage:()=>{const g=Math.min(N+1,j);v(g),r.set("instancesPage",String(g)),l(r)},onPrevPage:()=>{const g=Math.max(N-1,1);v(g),r.set("instancesPage",String(g)),l(r)},currentPage:N,totalPages:j})]})}async function mt(s,t,a){try{return await(await fetch(A(`/runs/${a||P()}/${s}/stats.json`),{signal:t})).json()}catch(n){return n instanceof Error&&n.name!=="AbortError"&&console.log(n),[]}}function ut({stat:s,metricFieldMap:t}){const a=`${s.name.split!==void 0?` on ${s.name.split}`:""}${s.name.sub_split!==void 0?`/${s.name.sub_split}`:""}${s.name.perturbation!==void 0?` with ${s.name.perturbation.name}`:" original"}`;return t[s.name.name]?e.jsxs("span",{title:t[s.name.name].description,children:[e.jsx("strong",{children:t[s.name.name].display_name||s.name.name}),a]}):e.jsxs("span",{children:[e.jsx("strong",{children:s.name.name}),a]})}const V=50,fe=["name","mean","min","max","sum","sum_squared","variance","stddev"];function xt({runName:s,suite:t,metricFieldMap:a}){const[n,r]=ae(),[l,i]=o.useState(),[u,h]=o.useState(1),[m,x]=o.useState("");if(o.useEffect(()=>{const v=new AbortController;async function k(){const j=v.signal,_=await mt(s,j,t);i(_)}return k(),()=>v.abort()},[s,t]),l===void 0||l.length===0)return e.jsx(H,{});const p=Math.ceil(l.length/V),N=l.slice((u-1)*V,(u-1)*V+V);return e.jsxs("div",{children:[e.jsx("div",{className:"flex justify-start my-4",children:e.jsx("input",{type:"text",className:"input input-bordered w-full max-w-xs",placeholder:"Search for a metric",onChange:v=>x(v.target.value)})}),e.jsx("div",{className:"overflow-x-auto",children:e.jsxs("table",{className:"table",children:[e.jsx("thead",{children:e.jsx("tr",{children:fe.map(v=>e.jsx("th",{children:v},v))})}),e.jsx("tbody",{children:N.filter(v=>!m||v.name.name.toLowerCase().includes(m.toLowerCase())).map(v=>e.jsx("tr",{children:fe.map(k=>{const j=v[k];return typeof j=="number"?e.jsx("td",{children:j}):e.jsx("td",{children:e.jsx(ut,{stat:v,metricFieldMap:a})},k)})}))})]})}),e.jsx(ce,{className:"flex justify-center my-8",onNextPage:()=>{const v=Math.min(u+1,p);h(v),n.set("metricsPage",String(v)),r(n)},onPrevPage:()=>{const v=Math.max(u-1,1);h(v),n.set("metricsPage",String(v)),r(n)},currentPage:u,totalPages:p})]})}function ft({runName:s,onAgree:t}){const a=o.useRef(null),n=()=>{a.current!==null&&a.current.value.trim()==="Yes, I agree"?t():alert("Please type 'Yes, I agree' exactly.")},r=s.includes("gpqa")?e.jsx(pt,{}):s.includes("ewok")?e.jsx(gt,{}):null;return e.jsxs("div",{className:"mb-8",children:[r,e.jsxs("p",{className:"mb-4",children:["If you agree to this condition, please type"," ",e.jsx("strong",{children:'"Yes, I agree"'})," in the box below and then click"," ",e.jsx("strong",{children:"Decrypt"}),"."]}),e.jsxs("div",{className:"flex gap-2 mt-2",children:[e.jsx("input",{type:"text",ref:a,className:"input input-bordered",placeholder:'Type "Yes, I agree"'}),e.jsx("button",{onClick:n,className:"btn btn-primary",children:"Decrypt"})]}),e.jsx("hr",{className:"my-4"})]})}function pt(){return e.jsxs("div",{children:[e.jsx("p",{className:"mb-4",children:"The GPQA dataset instances are encrypted by default to comply with the following request:"}),e.jsx("blockquote",{className:"italic border-l-4 border-gray-300 pl-4 text-gray-700 mb-4",children:"“We ask that you do not reveal examples from this dataset in plain text or images online, to minimize the risk of these instances being included in foundation model training corpora.”"})]})}function gt(){return e.jsxs("div",{children:[e.jsx("p",{className:"mb-4",children:"The EWoK dataset instances are encrypted by default to comply with the following request:"}),e.jsx("blockquote",{className:"italic border-l-4 border-gray-300 pl-4 text-gray-700 mb-4",children:"“PLEASE DO NOT distribute any of the EWoK materials or derivatives publicly in plain-text! Any materials should appear in password-protected ZIP files or behind gated authentication mechanisms such as Huggingface datasets.”"})]})}function jt(){const{runName:s}=ge(),[t,a]=o.useState(0),[n,r]=o.useState(),[l,i]=o.useState(),[u,h]=o.useState(),[m,x]=o.useState(),[p,N]=o.useState(),[v,k]=o.useState({}),[j,_]=o.useState({}),[g,T]=o.useState(!1);return o.useEffect(()=>{const c=new AbortController;async function f(){const d=c.signal;if(s===void 0)return()=>c.abort();const w=window.SUITE?window.SUITE:Ws(await zs(d),s);i(w);const[M,E,y]=await Promise.all([Bs(s,d,w),Fs(s,d,w),D(d)]);r(M),x(E),_(y.metrics.reduce((L,R)=>(L[R.name]=R,L),{})),k(y.adapter.reduce((L,R)=>(L[R.name]=R,L),{})),N(y.run_groups.find(L=>M==null?void 0:M.groups.includes(L.name))),h(y.models.find(L=>L.name===(M==null?void 0:M.adapter_spec.model)))}return f(),()=>c.abort()},[s]),n===void 0||m===void 0||s===void 0||l===void 0||j===void 0?e.jsx(H,{}):e.jsxs(e.Fragment,{children:[e.jsx("div",{className:"flex justify-between gap-8 mb-12",children:e.jsxs("div",{children:[e.jsxs("h1",{className:"text-3xl flex items-center",children:[(p==null?void 0:p.display_name)||m.name,p?e.jsx("a",{href:"/#/groups/"+p.name,children:e.jsx(ms,{className:"w-6 h-6 ml-2"})}):null]}),e.jsx("h3",{className:"text-xl",children:e.jsx(F,{value:(p==null?void 0:p.description)||m.description})}),e.jsx("h1",{className:"text-3xl mt-2",children:(u==null?void 0:u.display_name)||n.adapter_spec.model}),e.jsx("h3",{className:"text-xl",children:e.jsx(F,{value:(u==null?void 0:u.description)||""})}),e.jsx("div",{className:"mt-2 flex gap-2",children:m.tags.map(c=>e.jsx(O,{size:"xs",color:"gray",children:e.jsx("span",{className:"text text-md",children:c})}))})]})}),e.jsxs(z,{children:[e.jsxs("div",{className:"flex justify-between",children:[e.jsx("h3",{className:"text-lg mb-1",children:"Adapter Specification"}),e.jsxs("div",{className:"flex gap-2",children:[e.jsx(cs,{className:"w-6 h-6 mr-1 text text-primary"}),e.jsx("a",{className:"link link-primary link-hover",href:ke(n.name,l),download:"true",target:"_blank",children:"Spec JSON"}),e.jsx("a",{className:"link link-primary link-hover",href:Us(n.name,l),download:"true",target:"_blank",children:"Full JSON"})]})]}),e.jsx("div",{children:e.jsx(ne,{className:"grid md:grid-cols-2 lg:grid-cols-3 gap-x-8",children:Object.entries(n.adapter_spec).map(([c,f],d)=>e.jsxs(re,{className:d<3?"!border-0":"",children:[e.jsx("strong",{className:"mr-1",title:v[c]?v[c].description:void 0,children:`${c}: `}),e.jsx("span",{className:"overflow-x-auto",children:f})]}))})})]}),e.jsx("div",{className:"mt-16 mb-8",children:e.jsxs(le,{children:[e.jsx(G,{size:"lg",active:t===0,onClick:()=>a(0),children:"Instances + Predictions"}),e.jsx(G,{size:"lg",active:t===1,onClick:()=>a(1),children:"All metrics"})]})}),t===0&&Z(s)&&!g&&e.jsx(ft,{runName:s,onAgree:()=>T(!0)}),t===0?e.jsx(ht,{runName:s,suite:l,metricFieldMap:j,userAgreed:g},g?"instances-agreed":"instances-not-agreed"):e.jsx(xt,{runName:s,suite:l,metricFieldMap:j})]})}function bt({schema:s,groupTable:t,numRowsToDisplay:a,sortColumnIndex:n=1,sortable:r=!0,miniStyle:l=!1}){const[i,u]=o.useState(1),[h,m]=o.useState(n),x=c=>c.value,p=c=>{if(s){const f=s.models.find(d=>d.display_name===c);if(f){let d=f.description;return d.includes("/")&&(d=d.replace("/","_")),d}}return""},N=c=>{u(c===h?i*-1:c===0?-1:1),m(c)},v=c=>{if(s){const f=s.models.find(d=>d.display_name===c);if(f){let d=f.name;return d.includes("/")&&(d=d.replace("/","_")),d}}return""},k=()=>{const c=t.header[h].lower_is_better,f=i*(c?1:-1),d=t.rows.slice();return d.sort((w,M)=>{var L,R;const E=(L=w[h])==null?void 0:L.value,y=(R=M[h])==null?void 0:R.value;return E!==void 0&&y===void 0?-1:y!==void 0&&E===void 0?1:typeof E=="number"&&typeof y=="number"?(E-y)*f:typeof E=="string"&&typeof y=="string"?f===1?E.localeCompare(y):y.localeCompare(E):0}),a>0?d.slice(0,a):d};function j(c){const f=c.lastIndexOf(" - ");return f===-1?c:c.substring(0,f)+"*"+c.substring(f+1)}const _=c=>{const d=j(c).split("*")[0].trim();if(s){const w=s.run_groups.find(M=>M.display_name===d||M.short_display_name===d);if(w)return w.name}return""},g=[];g.push(t.header.map((c,f)=>e.jsx("td",{className:`${f===h?"bg-gray-100":f%2===0?"bg-gray-50":"bg-white"} ${f===0?"top-0 z-20":"z-10"} ${c.description?"underline decoration-dashed decoration-gray-300":""} sticky left-0 justify-between items-center w-48`,title:c.description?c.description:"",children:e.jsxs("div",{className:l?"flex gap-2":"z-10 flex w-full",children:[e.jsx("span",{className:"inline-block w-full break-words",children:x(c)}),r?e.jsx("button",{className:"link",onClick:()=>N(f),children:e.jsx(Me,{className:"w-6 h-6"})}):null]})},`scenario-${f}`))),k().forEach((c,f)=>g.push(c.map((d,w)=>e.jsx("td",{className:`${w%2===0?"bg-gray-50":"bg-white"} ${w===0?"px-4 sticky top-0":""} w-32`,children:w==1?e.jsx("div",{className:`${d&&d.style&&d.style["font-weight"]&&d.style["font-weight"]==="bold"?"font-bold":""}`,children:e.jsx(Q,{value:{...d,href:"/runs/?q="+v(String(c[0].value))},title:`Click value to see all predictions for: ${v(String(c[0].value))}`})}):e.jsx("div",{className:`${d&&d.style&&d.style["font-weight"]&&d.style["font-weight"]==="bold"?"font-bold":""} ${w===0?"underline decoration-dashed decoration-gray-300 z-20":"z-0"}`,children:e.jsx(Q,{value:{...d},title:String(c[0].value)===d.value?p(String(c[0].value)):`Click value to see predictions for ${String(c[0].value)} for ${_(x(t.header[w]))}: ${v(String(c[0].value))}`})})},`${f}-${w}`))));const T=g[0].map((c,f)=>g.map(d=>d[f]));return e.jsxs("table",{className:l?"table table-fixed w-full":"rounded-lg shadow-md table table-fixed",children:[e.jsx("thead",{children:e.jsx("tr",{children:T[0]})}),e.jsx("tbody",{children:T.slice(1).map((c,f)=>e.jsx("tr",{children:c},f))})]})}async function Ie(s,t){try{return await(await fetch(A(`${W()}/groups/${s}.json`),{signal:t})).json()}catch(a){return a instanceof Error&&a.name!=="AbortError"&&console.log(a),[]}}function vt({schema:s,runGroupName:t,numRowsToDisplay:a=-1}){const[n,r]=o.useState(),[l,i]=o.useState(0);if(o.useEffect(()=>{const h=new AbortController;async function m(){const x=await Ie(t,h.signal);r(x)}return m(),()=>h.abort()},[s,t]),n===void 0||n.length===0)return e.jsx(H,{});if(n.length===0)return e.jsx("div",{children:"Group currently has no tables."});const u=s.run_groups[0].name=="medhelm_scenarios";return e.jsxs("div",{children:[n.length>1?e.jsx(le,{children:n.map((h,m)=>e.jsx(G,{active:m===l,onClick:()=>i(m),children:h.title},m))}):null,u?e.jsx(bt,{schema:s,groupTable:n[l],numRowsToDisplay:a,sortColumnIndex:1},`${t}-${l}`):e.jsx(oe,{schema:s,groupTable:n[l],numRowsToDisplay:a,sortColumnIndex:1},`${t}-${l}`)]})}function Y(){const{groupName:s}=ge(),t=Be(),[a,n]=o.useState(void 0),[r,l]=o.useState(),[i,u]=o.useState();o.useEffect(()=>{const x=new AbortController;async function p(){const N=D(x.signal),v=Se(x.signal),k=await N;n(k);let j;const _=await v,g={};_.forEach(T=>{g[T.title]=[],T.rows.forEach(c=>{const f=c[0].href.replace("?group=","");j===void 0&&(j=f),g[T.title].push({title:String(c[0].value),name:f})})}),l(g),u(j)}return p(),()=>x.abort()},[]);const h=s||i;if(a===void 0||r===void 0||h===void 0)return e.jsx(H,{});let m;for(const x of a.run_groups)x.name===h&&(m=x);return e.jsxs(e.Fragment,{children:[e.jsxs("div",{className:"flex flex-row",children:[e.jsx("div",{className:"w-3/4",children:m?e.jsx($,{title:"Leaderboard: "+m.display_name,subtitle:m.description,markdown:!0}):e.jsx($,{title:"Leaderboard",subtitle:"The HELM leaderboard shows how the various models perform across different scenarios and metrics.",markdown:!0})}),e.jsx("div",{className:"w-1/4 pt-8",children:e.jsxs("label",{htmlFor:"group",className:"block text-sm font-medium text-gray-700",children:["Select a group:",e.jsx("select",{id:"group",name:"group",onChange:x=>t("/leaderboard/"+x.target.value),className:"mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring focus:border-blue-300 rounded-md",value:h,children:Object.entries(r).map(([x,p])=>e.jsx("optgroup",{label:x,children:p.map(N=>e.jsx("option",{value:N.name,children:N.title},N.name))},x))})]})})]}),e.jsx(vt,{schema:a,runGroupName:h},h)]})}const wt=""+new URL("instruct-flowchart-48854f7c.svg",import.meta.url).href,yt=""+new URL("instruct-graph-0a57d7d2.svg",import.meta.url).href;function Nt(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl mt-16 font-bold text-center",children:"HELM Instruct: A Multidimensional Instruction Following Evaluation Framework with Absolute Ratings"}),e.jsxs("div",{className:"flex flex-col sm:flex-row justify-center gap-2 sm:gap-8 md:gap-32 my-8",children:[e.jsx("a",{className:"px-10 btn rounded-md",href:"https://crfm.stanford.edu/2024/02/18/helm-instruct.html",children:"Blog Post"}),e.jsx("a",{className:"px-10 btn rounded-md",href:"https://github.com/stanford-crfm/helm",children:"Github"})]}),e.jsxs("p",{children:["We introduce ",e.jsx("em",{children:"HELM Instruct"}),", a multidimensional evaluation framework for instruction-following LLMs with absolute ratings. The framework takes an instruction, a model, an evaluator, and a criterion to generate a score. In our study, we use HELM Instruct to compare 4 instruction-following models on 7 scenarios based on 4 Human/LM evaluators and 5 criteria. Check out the blog post for more details."]}),e.jsxs("div",{className:"grid my-16 grid-cols-1 md:mx-32 md:grid-cols-2 md:gap-2",children:[e.jsx("img",{src:wt,alt:"Evaluation flowchart",className:"mx-auto block",sizes:"100vw"}),e.jsx("img",{src:yt,alt:"7 scenarios, 4 models, 4 evaluators and 5 criteria",className:"mx-auto block",sizes:"100vw"})]}),e.jsxs("table",{className:"rounded-lg shadow-md table",children:[e.jsx("thead",{children:e.jsxs("tr",{children:[e.jsx("th",{children:"Model"}),e.jsx("th",{children:"Average"}),e.jsx("th",{children:"Helpfulness"}),e.jsx("th",{children:"Understandability"}),e.jsx("th",{children:"Completeness"}),e.jsx("th",{children:"Conciseness"}),e.jsx("th",{children:"Harmlessness"})]})}),e.jsxs("tbody",{children:[e.jsxs("tr",{children:[e.jsx("td",{children:"openai_gpt-4-0314"}),e.jsx("td",{children:"4.63"}),e.jsx("td",{children:"4.42"}),e.jsx("td",{children:"4.85"}),e.jsx("td",{children:"4.50"}),e.jsx("td",{children:"4.42"}),e.jsx("td",{children:"4.95"})]}),e.jsxs("tr",{children:[e.jsx("td",{children:"openai_gpt-3.5-turbo-0613"}),e.jsx("td",{children:"4.60"}),e.jsx("td",{children:"4.34"}),e.jsx("td",{children:"4.86"}),e.jsx("td",{children:"4.42"}),e.jsx("td",{children:"4.41"}),e.jsx("td",{children:"4.97"})]}),e.jsxs("tr",{children:[e.jsx("td",{children:"anthropic_claude-v1.3"}),e.jsx("td",{children:"4.56"}),e.jsx("td",{children:"4.25"}),e.jsx("td",{children:"4.87"}),e.jsx("td",{children:"4.32"}),e.jsx("td",{children:"4.40"}),e.jsx("td",{children:"4.97"})]}),e.jsxs("tr",{children:[e.jsx("td",{children:"cohere_command-xlarge-beta"}),e.jsx("td",{children:"4.31"}),e.jsx("td",{children:"3.90"}),e.jsx("td",{children:"4.73"}),e.jsx("td",{children:"3.88"}),e.jsx("td",{children:"4.31"}),e.jsx("td",{children:"4.72"})]})]})]})]})}function Pe({models:s}){return e.jsxs("section",{children:[e.jsxs("h3",{className:"text-3xl",children:[s.length," models"]}),e.jsx("ul",{children:s.map((t,a)=>t.todo?e.jsxs("li",{className:"text-slate-300 mt-1",children:[t.creator_organization," / ",t.display_name]},a):e.jsx(b,{to:"models",children:e.jsxs("li",{className:"text-black mt-1",children:[t.creator_organization," / ",t.display_name]},a)}))})]})}function He({runGroups:s}){const t=new Map(s.filter(r=>r.metric_groups!==void 0&&(r.subgroups===void 0||r.subgroups.length===0)).map(r=>[r.name,r])),a=new Set,n=[];return s.forEach(r=>{const l=r.subgroups?r.subgroups:[],i=[];l.forEach(u=>{const h=t.get(u);h&&(i.push(h),a.add(h.name))}),i.length>0&&n.push([r,i])}),e.jsxs("section",{children:[e.jsxs("h3",{className:"text-3xl",children:[a.size," scenarios"]}),e.jsx("ul",{children:n.map(([r,l])=>e.jsxs("li",{className:"my-3",children:[e.jsx(b,{className:"text-black",to:"groups/"+r.name,children:e.jsx("h2",{children:r.display_name})}),e.jsx("ul",{className:"list-disc list-inside",children:l.map(i=>i.todo?e.jsx("li",{className:`${i.todo?"ml-4 text-slate-300":"ml-4"}`,children:i.display_name},i.name):e.jsx(b,{className:"text-black",to:"groups/"+i.name,children:e.jsx("li",{className:`${i.todo?"ml-4 text-slate-300":"ml-4"}`,children:i.display_name},i.name)}))})]},r.name))})]})}const $e=""+new URL("helmhero-28e90f4d.png",import.meta.url).href;function S({runGroupName:s=void 0,tableIndexToDisplay:t=0,numRowsToDisplay:a=10,sortColumnIndex:n=1}){const[r,l]=o.useState(void 0),[i,u]=o.useState(void 0);return o.useEffect(()=>{const h=new AbortController;async function m(){const x=await D(h.signal);l(x);const p=x.run_groups;if(p.length===0)return;const N=s||p[0].name,v=await Ie(N,h.signal);u(v[t])}return m(),()=>h.abort()},[s,t]),r===void 0||i===void 0?e.jsx(H,{}):e.jsx("div",{className:"rounded-2xl overflow-hidden border-2 bg-white p-1 mx-2 my-0 overflow-x-auto",style:{overflow:"auto",justifyContent:"space-between"},children:e.jsx(oe,{schema:r,groupTable:i,numRowsToDisplay:a,sortColumnIndex:n,displayColumnIndexes:[0,1],sortable:!1,miniStyle:!0})})}function Lt(){return e.jsxs("div",{className:"flex flex-col px-4 sm:px-6 py-100 sm:py-10 sm:mb-96 md:mb-96 lg:mb-0 xl:mb-0 2xl:mb-0",children:[e.jsx("div",{className:"flex flex-col text-center mb-10 justify-start",children:e.jsx("h1",{className:"text-3xl sm:text-4xl mb-3 sm:mb-4 mx-2 mt-2",children:e.jsx("strong",{children:"A holistic framework for evaluating foundation models."})})}),e.jsxs("div",{className:"flex flex-col md:flex-col lg:flex-row lg:justify-center",children:[e.jsx("div",{className:"w-full lg:w-1/2 flex justify-center mb-4 lg:mb-0 h-full py-10",children:e.jsx("img",{src:$e,alt:"HELM Hero",className:"object-contain w-96"})}),e.jsxs("div",{className:"py-2 rounded-xl bg-gray-100 h-full",children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-2 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function pe(){const[s,t]=o.useState(void 0);return o.useEffect(()=>{const a=new AbortController;async function n(){const r=await D(a.signal);t(r)}return n(),()=>a.abort()},[]),s?e.jsxs(e.Fragment,{children:[e.jsx(Lt,{}),e.jsx("div",{className:"mx-auto text-lg px-16",children:e.jsx("div",{className:"container mx-auto",children:e.jsxs("div",{className:"grid grid-cols-1 sm:grid-cols-2 gap-8",children:[e.jsx(Pe,{models:s.models}),e.jsx(He,{runGroups:s.run_groups})]})})})]}):null}function Mt(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl mt-16 my-8 font-bold text-center",children:"Massive Multitask Language Understanding (MMLU) on HELM"}),e.jsxs("div",{className:"flex flex-col lg:flex-row items-center gap-8",children:[e.jsxs("div",{className:"flex-1 text-xl",children:[e.jsxs("p",{children:[e.jsx("strong",{children:"Massive Multitask Language Understanding (MMLU)"})," ",e.jsx("a",{href:"https://arxiv.org/pdf/2009.03300.pdf",className:"link",children:"(Hendrycks et al, 2020)"})," ","is a multiple-choice question answering test that covers 57 tasks including elementary mathematics, US history, computer science, law, and more. We publish evaluation results from evaluating various models on MMLU using HELM. Our evaluation results include the following:"]}),e.jsxs("ul",{className:"my-2 list-disc list-inside",children:[e.jsx("li",{children:"Simple, standardized prompts"}),e.jsx("li",{children:"Accuracy breakdown for each of the 57 subjects"}),e.jsx("li",{children:"Full transparency of all raw prompts and predictions"})]}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://crfm.stanford.edu/2024/05/01/helm-mmlu.html",children:"Blog Post"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})]})]}),e.jsx("div",{className:"flex-1",children:e.jsx(S,{})})]})]})}const Et=""+new URL("air-overview-d2e6c49f.png",import.meta.url).href;function St(){const s={fontVariant:"small-caps",fontWeight:"bold"},t=e.jsx("span",{style:s,children:"AIR-Bench 2024"});return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl mt-16 my-8 font-bold text-center",children:t}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsx("img",{src:Et,alt:"AIR 2024 Categories",className:"mx-auto my-4 block w-3/4",sizes:"100vw"}),e.jsxs("p",{children:["We introduce ",t,", the first AI safety benchmark aligned with emerging government regulations and company policies, following the regulation-based safety categories grounded in our AI Risks study, AIR 2024. AIR 2024 decomposes 8 government regulations and 16 company policies into a four-tiered safety taxonomy with 314 granular risk categories in the lowest tier. ",t," contains 5,694 diverse prompts spanning these categories, with manual curation and human auditing to ensure quality. We evaluate leading language models on ",t,", uncovering insights into their alignment with specified safety concerns. By bridging the gap between public benchmarks and practical AI risks, ",t," ","provides a foundation for assessing model safety across jurisdictions, fostering the development of safer and more responsible AI systems."]}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://arxiv.org/abs/2407.17436",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function kt(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"ThaiExam"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{children:["In collaboration with"," ",e.jsx("a",{href:"https://www.scbx.com/",className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",children:"SCBX"})," ","and"," ",e.jsx("a",{href:"https://www.scb10x.com/",className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",children:"SCB 10X"}),", we introduce the ThaiExam HELM leaderboard. ThaiExam is a Thai language benchmark based on examinations for high school students and investment professionals in Thailand. The ThaiExam leaderboard is the first public leaderboard for large language models on Thai language scenarios, and features evaluations of leading language models. Like all other HELM leaderboards, the ThaiExam leaderboard provides full prompt-level transparency, and the results can be fully reproduced using the HELM framework. We hope that this leaderboard will encourage further work in multilingual language model evaluation."]}),e.jsx("div",{className:"flex flex-row justify-center my-4",children:e.jsx(b,{to:"leaderboard",className:"px-10 btn rounded-md mx-4",children:"Full Leaderboard"})})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function Rt(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"HELM Finance"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{className:"my-4",children:["LLMs show great potential for applications in the financial domain, yet there is a lack of financial-domain evaluations for LLMs. To help address this, we introduce the"," ",e.jsx("strong",{className:"font-bold",children:"HELM Finance"})," leaderboard. The HELM Finance leaderboard evaluates leading LLMs on three financial benchmarks (i.e.,"," ",e.jsx("a",{className:"underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://arxiv.org/abs/2109.00122/",children:"FinQA"}),","," ",e.jsx("a",{className:"underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://arxiv.org/abs/2311.11944/",children:"FinanceBench"}),","," ",e.jsx("a",{className:"underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://aclanthology.org/2020.nlp4convai-1.5/",children:"BANKING77"}),") that utilize real financial documents. Like all other HELM leaderboards, the HELM Finance leaderboard provides full prompt-level transparency, and the results can be fully reproduced using the open-source HELM framework. We hope that this leaderboard provides valuable insights for financial practitioners."]}),e.jsxs("p",{className:"my-4",children:["This leaderboard was produced through a collaboration with"," ",e.jsx("a",{className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://www.wellsfargo.com/",children:"Wells Fargo"}),", and was funded by the"," ",e.jsx("a",{className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://hai.stanford.edu/corporate-affiliate-program",children:"HAI Corporate Affiliate Program"}),"."]}),e.jsx("div",{className:"flex flex-row justify-center my-4",children:e.jsx(b,{to:"leaderboard",className:"px-10 btn rounded-md mx-4",children:"Full Leaderboard"})})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}const At=""+new URL("heim-logo-3e5e3aa4.png",import.meta.url).href;function _t({metricFieldMap:s,metricGroups:t}){const a=new Set,n=[];return t.forEach(r=>{const l=[];r.metrics.forEach(i=>{const u=s[i.name];u&&(l.push(u),a.add(u.name))}),l.length>0&&n.push([r,l])}),e.jsxs("section",{children:[e.jsxs("h3",{className:"text-3xl",children:[a.size," metrics"]}),e.jsx("ul",{children:n.map(([r,l])=>e.jsxs("li",{className:"my-3",children:[e.jsx("h4",{children:r.display_name}),e.jsx("ul",{className:"list-disc list-inside",children:l.map(i=>e.jsx("li",{className:"ml-4",children:i.display_name},i.name))})]},r.name))})]})}function Tt(){const[s,t]=o.useState(void 0);o.useEffect(()=>{const n=new AbortController;async function r(){const l=await D(n.signal);t(l)}return r(),()=>n.abort()},[]);const a=s?s.metrics.reduce((n,r)=>(n[r.name]=r,n),{}):void 0;return e.jsxs("div",{className:"container mx-auto px-16 text-base",children:[e.jsx("div",{className:"container max-w-screen-lg mx-auto",children:e.jsx("img",{className:"mx-auto w-96",src:At,alt:"HEIM Logo"})}),e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"Holistic Evaluation of Text-To-Image Models"}),e.jsxs("div",{className:"flex flex-col sm:flex-row justify-center gap-2 sm:gap-8 md:gap-32 my-8",children:[e.jsx("a",{className:"px-10 btn rounded-md",href:"https://arxiv.org/abs/2311.04287",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md",href:"https://github.com/stanford-crfm/helm",children:"Github"})]}),e.jsxs("p",{className:"my-2",children:["Significant effort has recently been made in developing text-to-image generation models, which take textual prompts as input and generate images. As these models are widely used in real-world applications, there is an urgent need to comprehensively understand their capabilities and risks. However, existing evaluations primarily focus on image-text alignment and image quality. To address this limitation, we introduce a new benchmark,"," ",e.jsx("strong",{children:"Holistic Evaluation of Text-To-Image Models (HEIM)"}),"."]}),e.jsx("p",{className:"my-2",children:"We identify 12 different aspects that are important in real-world model deployment, including:"}),e.jsxs("ul",{className:"my-2 list-disc list-inside unreset",children:[e.jsx("li",{children:"image-text alignment"}),e.jsx("li",{children:"image quality"}),e.jsx("li",{children:"aesthetics"}),e.jsx("li",{children:"originality"}),e.jsx("li",{children:"reasoning"}),e.jsx("li",{children:"knowledge"}),e.jsx("li",{children:"bias"}),e.jsx("li",{children:"toxicity"}),e.jsx("li",{children:"fairness"}),e.jsx("li",{children:"robustness"}),e.jsx("li",{children:"multilinguality"}),e.jsx("li",{children:"efficiency"})]}),e.jsx("p",{className:"my-2",children:"By curating scenarios encompassing these aspects, we evaluate state-of-the-art text-to-image models using this benchmark. Unlike previous evaluations that focused on alignment and quality, HEIM significantly improves coverage by evaluating all models across all aspects. Our results reveal that no single model excels in all aspects, with different models demonstrating strengths in different aspects."}),e.jsx("p",{className:"my-2",children:"For full transparency, this website contains all the prompts, generated images and the results for the automated and human evaluation metrics."}),e.jsx("p",{className:"my-2",children:"Inspired by HELM, we decompose the model evaluation into four key components: aspect, scenario, adaptation, and metric:"}),e.jsx("div",{className:"container max-w-screen-lg mx-auto my-8",children:e.jsx("img",{src:"https://crfm.stanford.edu/heim/latest/images/heim-main.png",alt:"HEIM scenarios, prompts, images and metrics"})}),s&&a?e.jsxs("div",{className:"grid grid-cols-1 md:grid-cols-3 gap-8",children:[e.jsx(Pe,{models:s.models}),e.jsx(He,{runGroups:s.run_groups}),e.jsx(_t,{metricFieldMap:a,metricGroups:s.metric_groups})]}):null]})}const Ct=""+new URL("vhelm-framework-a1ca3f3f.png",import.meta.url).href,It=""+new URL("vhelm-model-8afb7616.png",import.meta.url).href,Pt=""+new URL("vhelm-aspects-1437d673.png",import.meta.url).href;function Ht(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl mt-16 my-8 font-bold text-center",children:"Holistic Evaluation of Vision-Language Models"}),e.jsxs("div",{className:"flex flex-col sm:flex-row justify-center gap-2 sm:gap-8 md:gap-32 my-8",children:[e.jsx("a",{className:"px-10 btn rounded-md",href:"https://arxiv.org/abs/2410.07112",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md",href:"#/leaderboard",children:"Leaderboard"}),e.jsx("a",{className:"px-10 btn rounded-md",href:"https://github.com/stanford-crfm/helm",children:"Github"})]}),e.jsxs("p",{className:"my-4",children:["Current benchmarks for assessing vision-language models (VLMs) often focus on their perception or problem-solving capabilities and neglect other critical aspects such as fairness, multilinguality, or toxicity. Furthermore, they differ in their evaluation procedures and the scope of the evaluation, making it difficult to compare models. To address these issues, we extend the HELM framework to VLMs to present the Holistic Evaluation of Vision Language Models (VHELM). To address these issues, we introduce VHELM, built on HELM for language models. VHELM aggregates various datasets to cover one or more of the 9 aspects:"," ",e.jsx("b",{children:"visual perception"}),", ",e.jsx("b",{children:"bias"}),", ",e.jsx("b",{children:"fairness"}),", ",e.jsx("b",{children:"knowledge"}),", ",e.jsx("b",{children:"multilinguality"}),", ",e.jsx("b",{children:"reasoning"}),", ",e.jsx("b",{children:"robustness"}),","," ",e.jsx("b",{children:"safety"}),", and ",e.jsx("b",{children:"toxicity"}),". In doing so, we produce a comprehensive, multi-dimensional view of the capabilities of the VLMs across these important factors. In addition, we standardize the standard inference parameters, methods of prompting, and evaluation metrics to enable fair comparisons across models. Our framework is designed to be lightweight and automatic so that evaluation runs are cheap and fast. For transparency, we release the raw model generations and complete results on this website."]}),e.jsx("p",{className:"my-4 font-bold",children:"VHELM is intended to be a living benchmark. We hope to continue adding new datasets, models and metrics over time, so please stay tuned!"}),e.jsxs("div",{className:"my-16 flex flex-col lg:flex-row items-center gap-8",children:[e.jsxs("div",{className:"flex-1 text-xl",children:[e.jsx("img",{src:It,alt:"A vision-lanuage model (VLM) takes in an image and a text prompt and generates text.",className:""}),e.jsx("img",{src:Ct,alt:"An example of an evaluation for an Aspect (Knowledge) - a Scenario (MMMU) undergoes Adaptation (multimodal multiple choice) for a Model (GPT-4 Omni), then Metrics (Exact match) are computed",className:""})]}),e.jsxs("div",{className:"flex-1",children:[e.jsx(S,{}),e.jsx(b,{to:"leaderboard",className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})]})]}),e.jsx("div",{className:"container max-w-screen-lg mx-auto my-8",children:e.jsx("img",{src:Pt,alt:"An example of each aspect in VHELM: Visual Perception, Bias, Fairness, Knowledge, Multilinguality, Reasoning, Robustness, Toxicity Mitigation and Safety. ",className:""})})]})}function $t(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"HELM Call Center Leaderboard"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{className:"my-4",children:["LLMs show great potential for applications for the call center, yet there is a lack of domain-specific and ecologically-valid evaluations in this domain. To address this, we introduce the"," ",e.jsx("strong",{className:"font-bold",children:"HELM Call Center leaderboard"}),". The HELM Call Center leaderboard evaluates leading LLMs on a summarization task over a dataset of real helpdesk call transcripts provided by Accenture. The quality of the summaries is evaluated using LLM-as-judge with an ensemble of 3 models. We hope that this leaderboard provides some initial insights into the potential of LLMs in this domain."]}),e.jsxs("p",{className:"my-4",children:["This leaderboard was produced through research collaboration with"," ",e.jsx("a",{className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://www.accenture.com/",children:"Accenture"}),", and was funded by the"," ",e.jsx("a",{className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://hai.stanford.edu/corporate-affiliate-program",children:"HAI Corporate Affiliate Program"}),"."]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function Dt(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"HELM Call Transcript Summarization Leaderboard"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-[3] text-l",children:[e.jsx("img",{src:"https://storage.googleapis.com/crfm-helm-public/assets/images/call-center-banner-narrow.jpg",className:"shadow-xl",width:"1200",height:"400"}),e.jsxs("p",{className:"my-4",children:["Large language models (LLMs) show great potential for call center applications, yet there is a lack of domain-specific and ecologically valid evaluations in this domain. To address this, we introduce the"," ",e.jsx("strong",{className:"font-bold",children:"HELM Call Transcript Summarization"})," ","leaderboard, which evaluates leading LLMs on a summarization task over a dataset of real call transcripts provided by Accenture."]}),e.jsx("p",{className:"my-4",children:"This dataset consists of 162 transcribed calls to an internal corporate IT helpdesk. The calls were transcribed using an automatic speech recognition (ASR) model. Transcription errors were deliberately left uncorrected to reflect the nature of real-life transcripts. The transcripts were anonymized using a semi-automated process with human verification."}),e.jsx("p",{className:"my-4",children:"To evaluate the LLMs, summaries of the transcripts were generated using 17 LLMs. The quality of the generated summaries were then evaluated using LLM-as-judge with an ensemble of 3 models."}),e.jsx("p",{className:"my-4",children:"As with all HELM leaderboards, this leaderboard provides full transparency into all LLM requests and responses, and the results are reproducible using the HELM open source framework. We hope that this leaderboard offers initial insights into the potential of LLMs for this task."}),e.jsxs("p",{className:"my-4",children:["This leaderboard was produced through research collaboration with"," ",e.jsx("a",{className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://www.accenture.com/",children:"Accenture"}),", and was funded by the"," ",e.jsx("a",{className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://hai.stanford.edu/corporate-affiliate-program",children:"HAI Corporate Affiliate Program"}),"."]})]}),e.jsxs("div",{className:"flex-[2] py-2 rounded-3xl bg-gray-100 h-full",children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function Ot(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"Chinese Language Models EVAluation Platform (CLEVA)"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{children:["In collaboration with the"," ",e.jsx("a",{href:"https://lwwangcse.github.io/",className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",children:"LaVi Lab"})," ","team from"," ",e.jsx("a",{href:"https://www.cuhk.edu.hk/",className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",children:"The Chinese University of Hong Kong (CUHK)"}),", we introduce the"," ",e.jsx("strong",{className:"font-bold",children:"Chinese Language Models EVAluation Platform (CLEVA)"})," ","leaderboard on HELM. CLEVA is a comprehensive Chinese-language benchmark for holistic evaluation of Chinese-language LLMs, and employs a standardized workflow to assess LLMs' performance across various dimensions."]}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://arxiv.org/abs/2308.04813",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function Ft(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"The Mighty ToRR: A Benchmark for Table Reasoning and Robustness"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{className:"mb-4 italic",children:["This leaderboard is a collaboration with"," ",e.jsx("a",{href:"https://research.ibm.com/",className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",children:"IBM Research"}),"."]}),e.jsxs("p",{className:"my-4",children:["Despite its real-world significance, model performance on tabular data remains underexplored, leaving uncertainty about which model to rely on and which prompt configuration to adopt. To address this gap, we create ",e.jsx("strong",{className:"font-bold",children:"ToRR"}),", a benchmark for Table Reasoning and Robustness, a varied benchmark that measures model performance and robustness on table-related tasks. The benchmark includes 10 datasets that cover different types of table reasoning capabilities across varied domains. ToRR goes beyond model performance rankings, and is designed to reflect whether models can handle tabular data consistently and robustly, across a variety of common table representation formats. We present a leaderboard as well as comprehensive analyses of the results of leading models over ToRR. Our results reveal a striking pattern of brittle model behavior, where even strong models are unable to perform robustly on tabular data tasks. Although no specific table format leads to consistently better performance, we show that testing over multiple formats is crucial for reliably estimating model capabilities. Moreover, we show that the reliability boost from testing multiple prompts can be equivalent to adding more test examples. Overall, our findings show that reasoning over table tasks remains a significant challenge."]}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4 hidden",href:"#",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}const Bt=({id:s,title:t,text:a})=>((t==="Classic"||t==="Lite"||t==="Instruct")&&(t="HELM "+t),e.jsx("div",{className:"max-w-sm rounded overflow-hidden bg-gray-100 hover:scale-105 transition-transform duration-300",children:e.jsx("a",{href:ie(void 0,s),children:e.jsxs("div",{className:"px-6 py-4",children:[e.jsxs("div",{className:"font-bold text-xl mb-2",children:[e.jsx("div",{className:"py-3",children:e.jsx("svg",{fill:"#000000",width:"20px",height:"20px",viewBox:"0 0 24 24",xmlns:"http://www.w3.org/2000/svg",children:e.jsx("path",{d:"M22,7H16.333V4a1,1,0,0,0-1-1H8.667a1,1,0,0,0-1,1v7H2a1,1,0,0,0-1,1v8a1,1,0,0,0,1,1H22a1,1,0,0,0,1-1V8A1,1,0,0,0,22,7ZM7.667,19H3V13H7.667Zm6.666,0H9.667V5h4.666ZM21,19H16.333V9H21Z"})})}),t+" →"]}),e.jsx("p",{className:"text-gray-700 text-base",children:a})]})})}));function Ut(){const[s,t]=o.useState();return o.useEffect(()=>{fetch("https://crfm.stanford.edu/helm/project_metadata.json").then(a=>a.json()).then(a=>{t(a)}).catch(a=>{console.error("Error fetching JSON:",a)})},[]),e.jsx("div",{className:"p-10 mb-20",children:e.jsx("div",{className:"grid grid-cols-2 lg:grid-cols-3 gap-4",children:s&&s.map((a,n)=>a.id==="home"?null:e.jsx(Bt,{id:a.id,title:a.title,text:a.description},n))})})}function zt(){return e.jsxs("div",{className:"flex flex-col md:flex-row px-6 py-32",children:[e.jsxs("div",{className:"flex-1 p-4 flex flex-col justify-center",children:[e.jsx("div",{className:"flex justify-start",children:e.jsxs("div",{children:[e.jsx("h1",{className:"text-4xl mb-4 mx-4 mt-2",children:e.jsx("strong",{children:"A reproducible and transparent framework for evaluating foundation models."})}),e.jsx("h3",{className:`text-xl
-             mb-4 mx-4 mt-2`,children:"Find leaderboards with many scenarios, metrics, and models with support for multimodality and model-graded evaluation."})]})}),e.jsxs("div",{className:"flex flex-col md:flex-row justify-start mt-6 ml-4",children:[e.jsx("button",{className:"px-6 btn btn-grey rounded-md mb-4 md:mb-0",onClick:()=>window.scrollTo({top:760,behavior:"smooth"}),children:e.jsx("div",{children:"Leaderboards ↓"})}),e.jsx("button",{className:"px-6 btn btn-grey rounded-md md:ml-4",children:e.jsx("a",{href:"https://github.com/stanford-crfm/helm",children:"Github"})})]})]}),e.jsx("div",{className:"mx-4 mt-6 md:mt-0 md:w-1/3",children:e.jsx("img",{src:$e,alt:"HELM Hero",className:"object-cover w-full h-full"})})]})}function Wt(){const[s,t]=o.useState(void 0);return o.useEffect(()=>{const a=new AbortController;async function n(){const r=await D(a.signal);t(r)}return n(),()=>a.abort()},[]),s?e.jsxs(e.Fragment,{children:[e.jsx(zt,{}),e.jsx("div",{className:"container py-5 mx-auto text-lg",children:e.jsx("div",{className:"flex flex-col sm:flex-row justify-center mb-10 flex sm:gap-8 md:gap-32",children:e.jsx("h1",{className:"text-4xl mx-4 ",children:e.jsx("strong",{children:"HELM Leaderboards"})})})}),e.jsx(Ut,{})]}):null}const qt=""+new URL("overview-74aea3d8.png",import.meta.url).href,Vt=""+new URL("process-flow-bd2eba96.png",import.meta.url).href;function Jt(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl mt-16 my-8 font-bold text-center",children:"Image2Struct: A Benchmark for Evaluating Vision-Language Models in Extracting Structured Information from Images"}),e.jsxs("div",{className:"flex flex-col sm:flex-row justify-center gap-2 sm:gap-2 md:gap-8 my-8",children:[e.jsx("a",{className:"px-10 btn rounded-md",href:"https://arxiv.org/abs/2410.22456",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md",href:"https://github.com/stanford-crfm/helm",children:"Github"}),e.jsx("a",{className:"px-10 btn rounded-md",href:"https://huggingface.co/datasets/stanford-crfm/i2s-latex",children:"Latex dataset"}),e.jsx("a",{className:"px-5 btn rounded-md",href:"https://huggingface.co/datasets/stanford-crfm/i2s-webpage",children:"Webpage dataset"}),e.jsx("a",{className:"px-10 btn rounded-md",href:"https://huggingface.co/datasets/stanford-crfm/i2s-musicsheet",children:"Music sheet dataset"})]}),e.jsxs("div",{className:"flex flex-col lg:flex-row items-center gap-8",children:[e.jsxs("div",{className:"flex-1 text-xl",children:[e.jsxs("p",{children:[e.jsx("strong",{children:"Image2Struct"})," is a benchmark for evaluating vision-language models in practical tasks of extracting structured information from images."]}),e.jsx("br",{}),e.jsx("p",{children:"In our tasks, VLMs are prompted to generate the underlying structured information (i.e., code) from an input image. The code can be compiled, and the output image is evaluated against the input image to produce a score. This round-trip evaluation allows us to quantitatively evaluate VLMs on complex tasks with multiple correct answers. We create a pipeline that downloads fresh, user-submitted data from active online communities upon execution, evaluates the VLMs shortly, and produces a leaderboard."}),e.jsx("br",{}),e.jsx("img",{src:qt,alt:"Evaluation flowchart",className:"mx-auto block w-full",sizes:"100vw"}),e.jsx("br",{}),e.jsx("p",{children:"We introduce 3 tasks:"}),e.jsxs("ul",{className:"my-2 list-disc list-inside",children:[e.jsx("li",{children:"LaTex: equations, tables, plots and algorithms from ArXiV papers."}),e.jsx("li",{children:"Webpages: pages from GitHub written in HTML, CSS and Javascript. ..."}),e.jsx("li",{children:"Music sheets: crops of measures from IMSLP music sheets."})]}),e.jsx("div",{className:"flex flex-row justify-center mt-8",children:e.jsx("a",{className:"px-10 btn rounded-md",href:"#/leaderboard",children:"Full Leaderboard"})})]}),e.jsx("div",{className:"flex-1",children:e.jsx(S,{numRowsToDisplay:12})})]}),e.jsx("br",{}),e.jsxs("div",{className:"flex flex-col lg:flex-row items-center gap-8",children:[e.jsxs("div",{className:"flex-1 text-xl",children:[e.jsx("p",{children:"We provide an automated process for collecting new fresh data from online communities, evaluating the models and producing a leaderboard. The pipeline is designed to be executed on a regular basis to keep the leaderboard up-to-date."}),e.jsx("br",{}),e.jsxs("p",{children:["In addition to the automated data collection, we also provide a"," ",e.jsx("i",{children:"wild"})," subset for the LaTeX and webpage tasks that are collected from Wikipedia and various popular websites. These instances do not have a corresponding code, and the evaluation is done by our proposed metric: block EMD (Earth Mover Distance)."]})]}),e.jsx("div",{className:"flex-1",children:e.jsx("img",{src:Vt,alt:"7 scenarios, 4 models, 4 evaluators and 5 criteria",className:"mx-auto block w-full",sizes:"200vw"})})]})]})}function Gt(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"Elements of World Knowledge (EWoK)"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{children:["We present the"," ",e.jsx("a",{className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://arxiv.org/abs/2405.09605",children:"Elements of World Knowledge (EWoK)"})," ","leaderboard in collaboration with the EWoK team. EWoK is a benchmark for evaluating world modeling in language models by testing their ability to use knowledge of a concept to match a target text with a plausible/implausible context. EWoK targets specific concepts from multiple knowledge domains known to be vital for world modeling in humans, including social interactions and spatial relations."]}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://arxiv.org/abs/2405.09605",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}const Qt=""+new URL("medhelm-overview-eac29843.png",import.meta.url).href;function Kt(){const s=e.jsx("strong",{className:"font-bold",children:"MedHELM"});return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-4 font-bold text-center",children:"MedHELM"}),e.jsx("p",{className:"text-xl my-4 italic text-center",children:"Holistic Evaluation of Large Language Models for Medical Tasks"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-[7] text-l",children:[e.jsxs("p",{className:"my-",children:["We introduce ",s,", an extensible evaluation framework for assessing LLM performance for medical tasks. Building on the HELM framework, ",s," comprises a structured taxonomy with 5 categories, 22 subcategories, and 121 distinct clinical tasks as well as 35 distinct benchmarks (14 private, 7 gated-access, and 14 public). The benchmarks represent a spectrum of healthcare scenarios, from diagnostic decision-making to patient communication, providing a more nuanced and medically relevant assessment of AI capabilities in healthcare settings."]}),e.jsx("img",{src:Qt,alt:"MedHELM Task Categories",className:"mx-auto my-4 block w-7/8",sizes:"100vw"}),e.jsxs("p",{className:"my-4",children:[s," establishes a foundation for testing and evaluation of the real-world applicability of language models in healthcare. It is made possible by a unique collaboration between the Center for Research on Foundation Models, Technology and Digital Solutions at Stanford Healthcare, and Microsoft Healthcare and Life Sciences in partnership with faculty in the Departments of Medicine, Computer Science, Anesthesiology, Dermatology, Pediatrics and Biomedical Data Science as well as trainees from the MCiM program at the Clinical Excellence Research Center. The effort is coordinated by the Center for Biomedical Informatics Research."]}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://arxiv.org/abs/2505.23802",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://crfm-helm.readthedocs.io/en/latest/medhelm/",children:"Documentation"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://hai.stanford.edu/news/holistic-evaluation-of-large-language-models-for-medical-applications",children:"Blog Post"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Leaderboard"})]})]}),e.jsxs("div",{className:"flex-[3] py-2 rounded-3xl bg-gray-100 h-full",children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}const Zt=""+new URL("medhelm-v1-overview-3ddfcd65.png",import.meta.url).href;function Xt(){const s=e.jsx("strong",{className:"font-bold",children:"MedHELM"});return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-4 font-bold text-center",children:"MedHELM"}),e.jsx("p",{className:"text-xl my-4 italic text-center",children:"Medical and AI experts build a benchmark for evaluation of LLMs grounded in real-world healthcare needs."}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{className:"my-4",children:["We introduce ",s,", a comprehensive healthcare benchmark to evaluate language models on real-world clinical tasks using real electronic health records. Building on the HELM framework,"," ",s," comprises a structured taxonomy with 5 categories, 22 subcategories, and 121 distinct clinical tasks as well as 31 diverse datasets (12 private, 6 gated-access, and 13 public). The datasets represent a spectrum of healthcare scenarios, from diagnostic decision-making to patient communication, providing a more nuanced and clinically relevant assessment of AI capabilities in healthcare settings."]}),e.jsx("img",{src:Zt,alt:"MedHELM Task Categories",className:"mx-auto my-4 block w-7/8",sizes:"100vw"}),e.jsxs("p",{className:"my-4",children:[s," establishes a foundation for testing and evaluation of the real-world applicability of language models in healthcare. It is made possible by a unique collaboration between the Center for Research on Foundation Models, Technology and Digital Solutions at Stanford Healthcare, and Microsoft Healthcare and Life Sciences in partnership with faculty in the Departments of Medicine, Computer Science, Anesthesiology, Dermatology, Pediatrics and Biomedical Data Science as well as trainees from the MCiM program at the Clinical Excellence Research Center. The effort is coordinated by the Center for Biomedical Informatics Research."]}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://arxiv.org/abs/2505.23802",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://crfm-helm.readthedocs.io/en/latest/medhelm/",children:"Documentation"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://hai.stanford.edu/news/holistic-evaluation-of-large-language-models-for-medical-applications",children:"Blog Post"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}const Yt=""+new URL("helm-safety-2907a7b6.png",import.meta.url).href;function ea(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl mt-16 my-8 font-bold text-center",children:"HELM Safety"}),e.jsxs("div",{className:"flex flex-col lg:flex-row items-center gap-8",children:[e.jsxs("div",{className:"flex-1 text-xl",children:[e.jsx("img",{src:Yt,alt:"Logo",className:"mx-auto p-0 block",style:{width:"300px"}}),e.jsx("p",{children:"Language models demonstrate powerful capabilities and pose significant risks. Given their widespread deployment, standardized public benchmarking of such models is vital. While language models are routinely evaluated on standard capability benchmarks, comparable standardization for benchmarking safety risks lags behind. To address this gap, we introduce HELM-Safety as a collection of 5 safety benchmarks that span 6 risk categories (e.g. violence, fraud, discrimination, sexual, harassment, deception). We present evaluation results for recent leading open weights and closed models."}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://crfm.stanford.edu/2024/11/08/helm-safety.html",children:"Blog Post"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})]})]}),e.jsx("div",{className:"flex-1",children:e.jsx(S,{})})]})]})}function sa(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"HELM Capabilities"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{children:["Building on the HELM framework, we introduce"," ",e.jsx("strong",{className:"font-bold",children:"HELM Capabilities"})," to capture our latest thinking on the evaluation of general capabilities. HELM Capabilities is a new benchmark and leaderboard that consists of a curated set of scenarios for measuring various capabilities of language models. Like all other HELM leaderboards, the HELM Capabilities leaderboard provides full prompt-level transparency, and the results can be fully reproduced using the HELM framework."]}),e.jsxs("div",{className:"flex flex-row justify-center my-4",children:[e.jsx("a",{href:"https://crfm.stanford.edu/2025/03/20/helm-capabilities.html",className:"px-10 btn rounded-md mx-4",children:"Blog Post"}),e.jsx(b,{to:"leaderboard",className:"px-10 btn rounded-md mx-4",children:"Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function ta(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"MMLU-Winogrande-Afr: Clinical MMLU and Winogrande in 11 low-resource African languages"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{className:"mb-4 italic",children:["This leaderboard is a collaboration with"," ",e.jsx("a",{href:"https://ghamut.com/",className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",children:"Ghamut Corporation"})," ","and the"," ",e.jsx("a",{href:"https://www.gatesfoundation.org/",className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",children:"Gates Foundation"}),"."]}),e.jsx("p",{className:"my-4",children:"Large Language Models (LLMs) have shown remarkable performance across various tasks, yet significant disparities remain for non-English languages, and especially native African languages. This paper addresses these disparities by creating approximately 1 million human-translated words of new benchmark data in 8 low-resource African languages, covering a population of over 160 million speakers of: Amharic, Bambara, Igbo, Sepedi (Northern Sotho), Shona, Sesotho (Southern Sotho), Setswana, and Tsonga. Our benchmarks are translations of Winogrande and three sections of MMLU: college medicine, clinical knowledge, and virology. Using the translated benchmarks, we report previously unknown performance gaps between state-of-the-art (SOTA) LLMs in English and African languages. The publicly available benchmarks, translations, and code from this study support further research and development aimed at creating more inclusive and effective language technologies."}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://arxiv.org/abs/2412.12417",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function aa(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"SEA-HELM: Southeast Asian Holistic Evaluation of Language Models"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{className:"mb-4 italic",children:["This leaderboard is a collaboration with"," ",e.jsx("a",{href:"https://aisingapore.org/",className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",children:"AI Singapore"}),"."]}),e.jsxs("p",{className:"my-4",children:["With the rapid emergence of novel capabilities in Large Language Models (LLMs), the need for rigorous multilingual and multicultural benchmarks that are integrated has become more pronounced. Though existing LLM benchmarks are capable of evaluating specific capabilities of LLMs in English as well as in various mid- to low-resource languages, including those in the Southeast Asian (SEA) region, a comprehensive and authentic evaluation suite for the SEA languages has not been developed thus far. Here, we present"," ",e.jsx("strong",{className:"font-bold",children:"SEA-HELM"}),", a holistic linguistic and cultural LLM evaluation suite that emphasizes SEA languages, comprising five core pillars: (1) NLP Classics, (2) LLM-specifics, (3) SEA Linguistics, (4) SEA Culture, (5) Safety. SEA-HELM currently supports Filipino, Indonesian, Tamil, Thai, and Vietnamese. We also introduce the SEA-HELM leaderboard, which allows users to understand models' multilingual and multicultural performance in a systematic and user-friendly manner."]}),e.jsxs("p",{className:"mb-4 italic",children:["Additional evaluation results are available on the external"," ",e.jsx("a",{href:"https://leaderboard.sea-lion.ai/",className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",children:"AI Singapore (AISG) SEA-HELM leaderboard"}),"."]}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://arxiv.org/abs/2502.14301",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"HELM Leaderboard"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://leaderboard.sea-lion.ai/",children:"AISG Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function na(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"HELM Speech"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsx("p",{className:"my-4",children:"We present a HELM leaderboard for speech tasks."}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function ra(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-4 font-bold text-center",children:"HELM Long Context"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-[1] text-l",children:[e.jsxs("p",{className:"my-4",children:["Recent Large Language Models (LLMs) claim to have powerful long-context capabilities (i.e. the ability to process and reason over long inputs), but these claims have not been rigorously evaluated. To address this, we introduce the"," ",e.jsx("strong",{className:"font-bold",children:"HELM Long Context"})," ","leaderboard, which evaluates leading long context LLMs on a broad set of long context tasks."]}),e.jsxs("p",{className:"my-4",children:["The benchmark consists of the following tasks: question answering over a set of retrieved passages (",e.jsx("a",{className:"underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"#/leaderboard/ruler_hotpotqa",children:"RULER HotPotQA"}),","," ",e.jsx("a",{className:"underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"#/leaderboard/ruler_squad",children:"RULER SQuAD"}),"), question answering over a long passage (",e.jsx("a",{className:"underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"#/leaderboard/infinite_bench_en_qa",children:"∞Bench En.QA"}),"), summarization of a long passage (",e.jsx("a",{className:"underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"#/leaderboard/infinite_bench_en_sum",children:"∞Bench En.Sum"}),"), and multi-round co-reference resolution over a long multi-turn conversation (",e.jsx("a",{className:"underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"#/leaderboard/openai_mrcr",children:"OpenAI MRCR"}),"). The leaderboard demonstrates that even though significant progress has been made on long context capabilities, there is still considerable room for improvement."]}),e.jsx("p",{className:"my-4",children:"As with all HELM leaderboards, this leaderboard provides full transparency into all LLM requests and responses, and the results are reproducible using the HELM open source framework."}),e.jsxs("p",{className:"my-4",children:["This leaderboard was produced through research collaboration with"," ",e.jsx("a",{className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://www.lvmh.com/",children:"LVMH"}),", and was funded by the"," ",e.jsx("a",{className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://hai.stanford.edu/corporate-affiliate-program",children:"HAI Corporate Affiliate Program"}),"."]}),e.jsx("div",{className:"flex flex-row justify-center mt-4",children:e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})})]}),e.jsxs("div",{className:"flex-[1] py-2 rounded-3xl bg-gray-100 h-full",children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function ia(){return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"HELM SQL"}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{className:"mb-4",children:["Text-to-SQL is the task of converting natural language instructions into SQL code. There has been increasing interest in text-to-SQL for applications by data scientists in various domains. Thus, we introduce the ",e.jsx("strong",{className:"font-bold",children:"HELM SQL"})," ","leaderboard for text-to-SQL evaluations. The HELM SQL leaderboard evaluates leading LLMs on two existing text-to-SQL benchmarks (Spider, BIRD-SQL) that cover a range of professional domains. In addition, we introduce a new benchmark,"," ",e.jsx("strong",{className:"font-bold",children:"CzechBankQA"}),", a text-to-SQL benchmark based on a real public bank customer relational database, to address the lack of coverage of text-to-SQL in the financial domain. CzechBankQA consists of text-to-SQL queries and gold labels provided by professionals at Wells Fargo. We hope that this leaderboard provides useful insights for data science practitioners."]}),e.jsxs("p",{className:"my-4",children:["This leaderboard was produced through research collaboration with"," ",e.jsx("a",{className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://www.wellsfargo.com/",children:"Wells Fargo"}),", and was funded by the"," ",e.jsx("a",{className:"font-bold underline text-blue-600 hover:text-blue-800 visited:text-purple-600",href:"https://hai.stanford.edu/corporate-affiliate-program",children:"HAI Corporate Affiliate Program"}),"."]}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4 hidden",href:"#",children:"Blog Post"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function la(){const s=e.jsx("strong",{className:"font-bold",children:"ViLLM"});return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"ViLLM: Crossing Linguistic Horizon"}),e.jsxs("p",{className:"text-xl my-4 italic text-center",children:[s," is a comprehensive benchmark suite for evaluating the performance of language models in ",e.jsx("strong",{children:"Vietnamese"}),"."]}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{className:"my-4",children:["As multilingual large language models (LLMs) continue to advance natural language processing, bridging communication across diverse cultures and languages, their effectiveness in lower-resourced languages like Vietnamese remains limited. Despite being trained on large multilingual corpora, most open-source LLMs struggle with Vietnamese understanding and generation.",e.jsx("strong",{children:" ViLLM"})," addresses this gap by providing a robust evaluation framework tailored specifically for Vietnamese. It includes ",e.jsx("strong",{children:"11 essential scenarios"}),", each targeting a core capability of Vietnamese LLMs:"]}),e.jsxs("p",{className:"my-4",children:[e.jsx("strong",{children:"ViLLM"})," includes 11 carefully designed evaluation scenarios, each addressing a core language modeling capability:",e.jsxs("ul",{className:"list-disc list-inside mt-2",children:[e.jsxs("li",{children:[e.jsx("strong",{children:"Question Answering:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/juletxara/xquad_xtreme",children:"XQuAD"}),","," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/facebook/mlqa",children:"MLQA"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Summarization:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/Yuhthe/vietnews",children:"VietNews"}),","," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/GEM/wiki_lingua",children:"WikiLingua"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Sentiment Analysis:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/ura-hcmut/vlsp2016",children:"VLSP2016"}),","," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/ura-hcmut/UIT-VSFC",children:"UiT-VSFC"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Text Classification:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/ura-hcmut/PhoATIS",children:"PhoATIS"}),","," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/ura-hcmut/UIT-VSMEC",children:"UiT-VSMEC"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Knowledge:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/ura-hcmut/zalo_e2eqa",children:"ZaloE2E"}),","," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/ura-hcmut/ViMMRC",children:"UiT-ViMMRC"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Toxic Detection:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/ura-hcmut/UIT-ViHSD",children:"UiT-VIHSD"}),","," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/tarudesu/ViCTSD",children:"UiT-ViCTSD"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Information Retrieval:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/unicamp-dl/mmarco",children:"mMARCO"}),","," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/unicamp-dl/mrobust",children:"mRobust04"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Language Modeling:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/ura-hcmut/MLQA",children:"MLQA"}),","," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/ura-hcmut/VSEC",children:"VSEC"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Reasoning:"})," ",e.jsx("a",{className:"link-primary",href:"",children:"Synthetic reasoning"}),","," ",e.jsx("a",{className:"link-primary",href:"",children:"Natural synthetic reasoning"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Mathematic:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/ura-hcmut/Vietnamese-MATH",children:"MATH"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Translation:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/vietgpt/opus100_envi",children:"OPUS100"}),","," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/ura-hcmut/PhoMT",children:"PhoMT"})]})]})]}),e.jsxs("p",{className:"my-4",children:[e.jsx("strong",{children:"ViLLM"})," also includes tools to promote the ethical and responsible use of language models:",e.jsxs("ul",{className:"list-disc list-inside mt-2",children:[e.jsxs("li",{children:[e.jsx("strong",{children:"Bias Assessment:"})," Detects and mitigates biased patterns in model outputs."]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Toxicity Assessment:"})," Monitors and controls the generation of harmful or offensive content."]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Fairness Evaluation:"})," Ensures equitable performance across demographic groups and languages."]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Robustness Analysis:"})," Evaluates model stability against noisy or adversarial inputs in real-world scenarios."]})]})]}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"https://aclanthology.org/2024.findings-naacl.182",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function oa(){const s=e.jsx("strong",{className:"font-bold",children:"SLPHelm"});return e.jsxs("div",{className:"container mx-auto px-16",children:[e.jsx("h1",{className:"text-3xl my-8 font-bold text-center",children:"SLPHelm: Advancing Speech Language Processing"}),e.jsxs("p",{className:"text-xl my-4 italic text-center",children:[s," is a comprehensive benchmark suite for evaluating the performance of speech language models across multiple languages and tasks."]}),e.jsxs("div",{className:"flex flex-col lg:flex-row gap-8",children:[e.jsxs("div",{className:"flex-1 text-l",children:[e.jsxs("p",{className:"my-4",children:["As speech language models continue to evolve and transform how we interact with technology, there is a growing need for standardized evaluation frameworks that can assess their capabilities across diverse languages and tasks. Despite significant advances in speech processing, many models struggle with multilingual understanding, accent recognition, and domain-specific speech tasks."," ",e.jsx("strong",{children:"SLPHelm"})," addresses these challenges by providing a robust evaluation framework that includes",e.jsx("strong",{children:" 5 key scenarios"})," across ",e.jsx("strong",{children:"15 models"}),", each targeting essential speech processing capabilities:"]}),e.jsxs("p",{className:"my-4",children:[e.jsx("strong",{children:"SLPHelm"})," includes 5 carefully designed evaluation scenarios, each addressing a core speech processing capability:",e.jsxs("ul",{className:"list-disc list-inside mt-2",children:[e.jsx("li",{children:e.jsx("strong",{children:"Disorder Diagnosis"})}),e.jsx("li",{children:e.jsx("strong",{children:"Transcription Accuracy"})}),e.jsx("li",{children:e.jsx("strong",{children:"Disorder Type Diagnosis"})}),e.jsx("li",{children:e.jsx("strong",{children:"Disorder Symptom Diagnosis"})}),e.jsx("li",{children:e.jsx("strong",{children:"Disorder Diagnosis via Transcription"})})]})]}),e.jsxs("p",{className:"my-4",children:[e.jsx("strong",{children:"SLPHelm"})," evaluates models using comprehensive datasets:",e.jsxs("ul",{className:"list-disc list-inside mt-2",children:[e.jsxs("li",{children:[e.jsx("strong",{children:"UltraSuite:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/SAA-Lab/SLPHelmUltraSuite",children:"UltraSuite"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"ENNI:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/SAA-Lab/SLPHelmDataset/tree/main/ENNI",children:"ENNI"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"LeNormand:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/SAA-Lab/SLPHelmDataset/tree/main/LeNormand",children:"LeNormand"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"PERCEPT-GFTA:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/SAA-Lab/SLPHelmDataset/tree/main/PERCEPT-GFTA",children:"PERCEPT-GFTA"})]}),e.jsxs("li",{children:[e.jsx("strong",{children:"UltraSuite w/ Manual Labels:"})," ",e.jsx("a",{className:"link-primary",href:"https://huggingface.co/datasets/SAA-Lab/SLPHelmManualLabels",children:"SLPHelmManualLabels"})]})]})]}),e.jsxs("div",{className:"flex flex-row justify-center mt-4",children:[e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#",children:"Paper"}),e.jsx("a",{className:"px-10 btn rounded-md mx-4",href:"#/leaderboard",children:"Full Leaderboard"})]})]}),e.jsxs("div",{className:"py-2 pb-6 rounded-3xl bg-gray-100 h-full",style:{maxWidth:"100%"},children:[e.jsx(S,{}),e.jsx("div",{className:"flex justify-end",children:e.jsx(b,{to:"leaderboard",children:e.jsx("button",{className:"px-4 mx-3 mt-1 btn bg-white rounded-md",children:e.jsx("span",{children:"See More"})})})})]})]})]})}function ca(){return window.PROJECT_ID==="lite"?e.jsx(pe,{}):window.PROJECT_ID==="instruct"?e.jsx(Nt,{}):window.PROJECT_ID==="image2struct"?e.jsx(Jt,{}):window.PROJECT_ID==="heim"?e.jsx(Tt,{}):window.PROJECT_ID==="mmlu"?e.jsx(Mt,{}):window.PROJECT_ID==="vhelm"?e.jsx(Ht,{}):window.PROJECT_ID==="air-bench"?e.jsx(St,{}):window.PROJECT_ID==="thaiexam"?e.jsx(kt,{}):window.PROJECT_ID==="finance"?e.jsx(Rt,{}):window.PROJECT_ID==="call-center"?e.jsx($t,{}):window.PROJECT_ID==="call-transcript-summarization"?e.jsx(Dt,{}):window.PROJECT_ID==="cleva"?e.jsx(Ot,{}):window.PROJECT_ID==="torr"?e.jsx(Ft,{}):window.PROJECT_ID==="ewok"?e.jsx(Gt,{}):window.PROJECT_ID==="medhelm"&&window.RELEASE=="v1.0.0"?e.jsx(Xt,{}):window.PROJECT_ID==="medhelm"?e.jsx(Kt,{}):window.PROJECT_ID==="safety"?e.jsx(ea,{}):window.PROJECT_ID==="capabilities"?e.jsx(sa,{}):window.PROJECT_ID==="mmlu-winogrande-afr"?e.jsx(ta,{}):window.PROJECT_ID==="seahelm"?e.jsx(aa,{}):window.PROJECT_ID==="speech"?e.jsx(na,{}):window.PROJECT_ID==="sql"?e.jsx(ia,{}):window.PROJECT_ID==="long-context"?e.jsx(ra,{}):window.PROJECT_ID==="villm"?e.jsx(la,{}):window.PROJECT_ID==="slphelm"?e.jsx(oa,{}):window.PROJECT_ID==="home"?e.jsx(Wt,{}):e.jsx(pe,{})}function da(){return e.jsx(Ue,{children:e.jsx(ze,{children:e.jsxs(C,{path:"/",element:e.jsx(Ss,{}),children:[e.jsx(C,{index:!0,element:e.jsx(ca,{})}),e.jsx(C,{path:"leaderboard",element:e.jsx(Y,{})}),e.jsx(C,{path:"leaderboard/:groupName",element:e.jsx(Y,{})}),e.jsx(C,{path:"models",element:e.jsx(Ts,{})}),e.jsx(C,{path:"scenarios",element:e.jsx(Cs,{})}),e.jsx(C,{path:"groups",element:e.jsx($s,{})}),e.jsx(C,{path:"groups/:groupName",element:e.jsx(Y,{})}),e.jsx(C,{path:"runs",element:e.jsx(Os,{})}),e.jsx(C,{path:"runs/:runName",element:e.jsx(jt,{})})]})})})}se.createRoot(document.getElementById("root")).render(e.jsx(We.StrictMode,{children:e.jsx(da,{})}));