token-vocabs 0.2.2 → 0.4.0

package/chunks/main.js

@@ -1,2 +1,2 @@
-import{n as e,r as t}from"./rolldown-runtime.js";var n;try{n=new TextDecoder}catch{}var r,i,a=0;const o=[];var s=o,c=0,l={},u,d,f=0,p=0,m,h,g=[],_,ee={useRecords:!1,mapsAsObjects:!0};const te=new class{};te.name=`MessagePack 0xC1`;var v=!1,ne=2,re=class e{constructor(e){e&&(e.useRecords===!1&&e.mapsAsObjects===void 0&&(e.mapsAsObjects=!0),e.sequential&&e.trusted!==!1&&(e.trusted=!0,!e.structures&&e.useRecords!=0&&(e.structures=[],e.maxSharedStructures||=0)),e.structures?e.structures.sharedLength=e.structures.length:e.getStructures&&((e.structures=[]).uninitialized=!0,e.structures.sharedLength=0),e.int64AsNumber&&(e.int64AsType=`number`)),Object.assign(this,e)}unpack(t,n){if(r)return Ee(()=>(De(),this?this.unpack(t,n):e.prototype.unpack.call(ee,t,n)));!t.buffer&&t.constructor===ArrayBuffer&&(t=typeof Buffer<`u`?Buffer.from(t):new Uint8Array(t)),typeof n==`object`?(i=n.end||t.length,a=n.start||0):(a=0,i=n>-1?n:t.length),c=0,p=0,d=null,s=o,m=null,r=t;try{_=t.dataView||=new DataView(t.buffer,t.byteOffset,t.byteLength)}catch(e){throw r=null,t instanceof Uint8Array?e:Error(`Source must be a Uint8Array or Buffer but was a `+(t&&typeof t==`object`?t.constructor.name:typeof t))}if(this instanceof e){if(l=this,this.structures)return u=this.structures,y(n);(!u||u.length>0)&&(u=[])}else l=ee,(!u||u.length>0)&&(u=[]);return y(n)}unpackMultiple(e,t){let n,r=0;try{v=!0;let i=e.length,o=this?this.unpack(e,i):ke.unpack(e,i);if(t){if(t(o,r,a)===!1)return;for(;a<i;)if(r=a,t(y(),r,a)===!1)return}else{for(n=[o];a<i;)r=a,n.push(y());return n}}catch(e){throw e.lastPosition=r,e.values=n,e}finally{v=!1,De()}}_mergeStructures(e,t){this._onLoadedStructures&&(e=this._onLoadedStructures(e)),e||=[],Object.isFrozen(e)&&(e=e.map(e=>e.slice(0)));for(let t=0,n=e.length;t<n;t++){let n=e[t];n&&(n.isShared=!0,t>=32&&(n.highByte=t-32>>5))}e.sharedLength=e.length;for(let n in t||[])if(n>=0){let r=e[n],i=t[n];i&&(r&&((e.restoreStructures||=[])[n]=r),e[n]=i)}return this.structures=e}decode(e,t){return this.unpack(e,t)}};function y(e){try{if(!l.trusted&&!v){let e=u.sharedLength||0;e<u.length&&(u.length=e)}let t;if(l._readStruct&&r[a]<64&&r[a]>=32?(t=l._readStruct(r,a,i),r=null,!(e&&e.lazy)&&t&&(t=t.toJSON()),a=i):t=b(),m&&=(a=m.postBundlePosition,null),v&&(u.restoreStructures=null),a==i)u&&u.restoreStructures&&ie(),u=null,r=null,h&&=null;else if(a>i)throw Error(`Unexpected end of MessagePack data`);else if(!v){let e;try{e=JSON.stringify(t,(e,t)=>typeof t==`bigint`?`${t}n`:t).slice(0,100)}catch(t){e=`(JSON view not available `+t+`)`}throw Error(`Data read, but end of buffer not reached `+e)}return t}catch(e){throw u&&u.restoreStructures&&ie(),De(),(e instanceof RangeError||e.message.startsWith(`Unexpected end of buffer`)||a>i)&&(e.incomplete=!0),e}}function ie(){for(let e in u.restoreStructures)u[e]=u.restoreStructures[e];u.restoreStructures=null}function b(){let e=r[a++];if(e<160)if(e<128){if(e<64)return e;{let t=u[e&63]||l.getStructures&&se()[e&63];return t?(t.read||=x(t,e&63),t.read()):e}}else if(e<144)if(e-=128,l.mapsAsObjects){let t={};for(let n=0;n<e;n++){let e=ye();e===`__proto__`&&(e=`__proto_`),t[e]=b()}return t}else{let t=new Map;for(let n=0;n<e;n++)t.set(b(),b());return t}else{e-=144;let t=Array(e);for(let n=0;n<e;n++)t[n]=b();return l.freezeData?Object.freeze(t):t}else if(e<192){let t=e-160;if(p>=a)return d.slice(a-f,(a+=t)-f);if(p==0&&i<140){let e=t<16?he(t):me(t);if(e!=null)return e}return ce(t)}else{let t;switch(e){case 192:return null;case 193:return m?(t=b(),t>0?m[1].slice(m.position1,m.position1+=t):m[0].slice(m.position0,m.position0-=t)):te;case 194:return!1;case 195:return!0;case 196:if(t=r[a++],t===void 0)throw Error(`Unexpected end of buffer`);return _e(t);case 197:return t=_.getUint16(a),a+=2,_e(t);case 198:return t=_.getUint32(a),a+=4,_e(t);case 199:return w(r[a++]);case 200:return t=_.getUint16(a),a+=2,w(t);case 201:return t=_.getUint32(a),a+=4,w(t);case 202:if(t=_.getFloat32(a),l.useFloat32>2){let e=Oe[(r[a]&127)<<1|r[a+1]>>7];return a+=4,(e*t+(t>0?.5:-.5)>>0)/e}return a+=4,t;case 203:return t=_.getFloat64(a),a+=8,t;case 204:return r[a++];case 205:return t=_.getUint16(a),a+=2,t;case 206:return t=_.getUint32(a),a+=4,t;case 207:return l.int64AsType===`number`?(t=_.getUint32(a)*4294967296,t+=_.getUint32(a+4)):l.int64AsType===`string`?t=_.getBigUint64(a).toString():l.int64AsType===`auto`?(t=_.getBigUint64(a),t<=BigInt(2)<<BigInt(52)&&(t=Number(t))):t=_.getBigUint64(a),a+=8,t;case 208:return _.getInt8(a++);case 209:return t=_.getInt16(a),a+=2,t;case 210:return t=_.getInt32(a),a+=4,t;case 211:return l.int64AsType===`number`?(t=_.getInt32(a)*4294967296,t+=_.getUint32(a+4)):l.int64AsType===`string`?t=_.getBigInt64(a).toString():l.int64AsType===`auto`?(t=_.getBigInt64(a),t>=BigInt(-2)<<BigInt(52)&&t<=BigInt(2)<<BigInt(52)&&(t=Number(t))):t=_.getBigInt64(a),a+=8,t;case 212:if(t=r[a++],t==114)return xe(r[a++]&63);{let e=g[t];if(e)return e.read?(a++,e.read(b())):e.noBuffer?(a++,e()):e(r.subarray(a,++a));throw Error(`Unknown extension `+t)}case 213:return t=r[a],t==114?(a++,xe(r[a++]&63,r[a++])):w(2);case 214:return w(4);case 215:return w(8);case 216:return w(16);case 217:return t=r[a++],p>=a?d.slice(a-f,(a+=t)-f):le(t);case 218:return t=_.getUint16(a),a+=2,p>=a?d.slice(a-f,(a+=t)-f):ue(t);case 219:return t=_.getUint32(a),a+=4,p>=a?d.slice(a-f,(a+=t)-f):de(t);case 220:return t=_.getUint16(a),a+=2,fe(t);case 221:return t=_.getUint32(a),a+=4,fe(t);case 222:return t=_.getUint16(a),a+=2,pe(t);case 223:return t=_.getUint32(a),a+=4,pe(t);default:if(e>=224)return e-256;if(e===void 0){let e=Error(`Unexpected end of MessagePack data`);throw e.incomplete=!0,e}throw Error(`Unknown MessagePack token `+e)}}}const ae=/^[a-zA-Z_$][a-zA-Z\d_$]*$/;function x(e,t){function n(){if(n.count++>ne){let r;try{r=e.read=Function(`r`,`return function(){return `+(l.freezeData?`Object.freeze`:``)+`({`+e.map(e=>e===`__proto__`?`__proto_:r()`:ae.test(e)?e+`:r()`:`[`+JSON.stringify(e)+`]:r()`).join(`,`)+`})}`)(b)}catch{return ne=1/0,n()}return e.highByte===0&&(e.read=oe(t,e.read)),r()}let r={};for(let t=0,n=e.length;t<n;t++){let n=e[t];n===`__proto__`&&(n=`__proto_`),r[n]=b()}return l.freezeData?Object.freeze(r):r}return n.count=0,e.highByte===0?oe(t,n):n}const oe=(e,t)=>function(){let n=r[a++];if(n===0)return t();let i=e<32?-(e+(n<<5)):e+(n<<5),o=u[i]||se()[i];if(!o)throw Error(`Record id is not defined for `+i);return o.read||=x(o,e),o.read()};function se(){let e=Ee(()=>(r=null,l.getStructures()));return u=l._mergeStructures(e,u)}var ce=S,le=S,ue=S,de=S;function S(e){let t;if(e<16&&(t=he(e)))return t;if(e>64&&n)return n.decode(r.subarray(a,a+=e));let i=a+e,o=[];for(t=``;a<i;){let e=r[a++];if(!(e&128))o.push(e);else if((e&224)==192){let t=r[a++]&63,n=(e&31)<<6|t;n<128?o.push(65533):o.push(n)}else if((e&240)==224){let t=r[a++]&63,n=r[a++]&63,i=(e&31)<<12|t<<6|n;i<2048||i>=55296&&i<=57343?o.push(65533):o.push(i)}else if((e&248)==240){let t=r[a++]&63,n=r[a++]&63,i=r[a++]&63,s=(e&7)<<18|t<<12|n<<6|i;s<65536||s>1114111?o.push(65533):s>65535?(s-=65536,o.push(s>>>10&1023|55296),s=56320|s&1023,o.push(s)):o.push(s)}else o.push(65533);o.length>=4096&&(t+=C.apply(String,o),o.length=0)}return o.length>0&&(t+=C.apply(String,o)),t}function fe(e){let t=Array(e);for(let n=0;n<e;n++)t[n]=b();return l.freezeData?Object.freeze(t):t}function pe(e){if(l.mapsAsObjects){let t={};for(let n=0;n<e;n++){let e=ye();e===`__proto__`&&(e=`__proto_`),t[e]=b()}return t}else{let t=new Map;for(let n=0;n<e;n++)t.set(b(),b());return t}}var C=String.fromCharCode;function me(e){let t=a,n=Array(e);for(let i=0;i<e;i++){let e=r[a++];if((e&128)>0){a=t;return}n[i]=e}return C.apply(String,n)}function he(e){if(e<4)if(e<2){if(e===0)return``;{let e=r[a++];if((e&128)>1){--a;return}return C(e)}}else{let t=r[a++],n=r[a++];if((t&128)>0||(n&128)>0){a-=2;return}if(e<3)return C(t,n);let i=r[a++];if((i&128)>0){a-=3;return}return C(t,n,i)}else{let t=r[a++],n=r[a++],i=r[a++],o=r[a++];if((t&128)>0||(n&128)>0||(i&128)>0||(o&128)>0){a-=4;return}if(e<6){if(e===4)return C(t,n,i,o);{let e=r[a++];if((e&128)>0){a-=5;return}return C(t,n,i,o,e)}}else if(e<8){let s=r[a++],c=r[a++];if((s&128)>0||(c&128)>0){a-=6;return}if(e<7)return C(t,n,i,o,s,c);let l=r[a++];if((l&128)>0){a-=7;return}return C(t,n,i,o,s,c,l)}else{let s=r[a++],c=r[a++],l=r[a++],u=r[a++];if((s&128)>0||(c&128)>0||(l&128)>0||(u&128)>0){a-=8;return}if(e<10){if(e===8)return C(t,n,i,o,s,c,l,u);{let e=r[a++];if((e&128)>0){a-=9;return}return C(t,n,i,o,s,c,l,u,e)}}else if(e<12){let d=r[a++],f=r[a++];if((d&128)>0||(f&128)>0){a-=10;return}if(e<11)return C(t,n,i,o,s,c,l,u,d,f);let p=r[a++];if((p&128)>0){a-=11;return}return C(t,n,i,o,s,c,l,u,d,f,p)}else{let d=r[a++],f=r[a++],p=r[a++],m=r[a++];if((d&128)>0||(f&128)>0||(p&128)>0||(m&128)>0){a-=12;return}if(e<14){if(e===12)return C(t,n,i,o,s,c,l,u,d,f,p,m);{let e=r[a++];if((e&128)>0){a-=13;return}return C(t,n,i,o,s,c,l,u,d,f,p,m,e)}}else{let h=r[a++],g=r[a++];if((h&128)>0||(g&128)>0){a-=14;return}if(e<15)return C(t,n,i,o,s,c,l,u,d,f,p,m,h,g);let _=r[a++];if((_&128)>0){a-=15;return}return C(t,n,i,o,s,c,l,u,d,f,p,m,h,g,_)}}}}}function ge(){let e=r[a++],t;if(e<192)t=e-160;else switch(e){case 217:t=r[a++];break;case 218:t=_.getUint16(a),a+=2;break;case 219:t=_.getUint32(a),a+=4;break;default:throw Error(`Expected string`)}return S(t)}function _e(e){return l.copyBuffers?Uint8Array.prototype.slice.call(r,a,a+=e):r.subarray(a,a+=e)}function w(e){let t=r[a++];if(g[t]){let n;return g[t](r.subarray(a,n=a+=e),e=>{a=e;try{return b()}finally{a=n}})}else throw Error(`Unknown extension type `+t)}var ve=Array(4096);function ye(){let e=r[a++];if(e>=160&&e<192){if(e-=160,p>=a)return d.slice(a-f,(a+=e)-f);if(!(p==0&&i<180))return ce(e)}else return a--,be(b());let t=(e<<5^(e>1?_.getUint16(a):e>0?r[a]:0))&4095,n=ve[t],o=a,s=a+e-3,c,l=0;if(n&&n.bytes==e){for(;o<s;){if(c=_.getUint32(o),c!=n[l++]){o=1879048192;break}o+=4}for(s+=3;o<s;)if(c=r[o++],c!=n[l++]){o=1879048192;break}if(o===s)return a=o,n.string;s-=3,o=a}for(n=[],ve[t]=n,n.bytes=e;o<s;)c=_.getUint32(o),n.push(c),o+=4;for(s+=3;o<s;)c=r[o++],n.push(c);let u=e<16?he(e):me(e);return u==null?n.string=ce(e):n.string=u}function be(e){if(typeof e==`string`)return e;if(typeof e==`number`||typeof e==`boolean`||typeof e==`bigint`)return e.toString();if(e==null)return e+``;if(l.allowArraysInMapKeys&&Array.isArray(e)&&e.flat().every(e=>[`string`,`number`,`boolean`,`bigint`].includes(typeof e)))return e.flat().toString();throw Error(`Invalid property type for record: ${typeof e}`)}const xe=(e,t)=>{let n=b().map(be),r=e;t!==void 0&&(e=e<32?-((t<<5)+e):(t<<5)+e,n.highByte=t);let i=u[e];return i&&(i.isShared||v)&&((u.restoreStructures||=[])[e]=i),u[e]=n,n.read=x(n,r),n.read()};g[0]=()=>{},g[0].noBuffer=!0,g[66]=e=>{let t=e.byteLength%8||8,n=BigInt(e[0]&128?e[0]-256:e[0]);for(let r=1;r<t;r++)n<<=BigInt(8),n+=BigInt(e[r]);if(e.byteLength!==t){let r=new DataView(e.buffer,e.byteOffset,e.byteLength),i=(e,t)=>{let n=t-e;if(n<=40){let n=r.getBigUint64(e);for(let i=e+8;i<t;i+=8)n<<=BigInt(64),n|=r.getBigUint64(i);return n}let a=e+(n>>4<<3),o=i(e,a),s=i(a,t);return o<<BigInt((t-a)*8)|s};n=n<<BigInt((r.byteLength-t)*8)|i(t,r.byteLength)}return n};let Se={Error,EvalError,RangeError,ReferenceError,SyntaxError,TypeError,URIError,AggregateError:typeof AggregateError==`function`?AggregateError:null};g[101]=()=>{let e=b();if(!Se[e[0]]){let t=Error(e[1],{cause:e[2]});return t.name=e[0],t}return Se[e[0]](e[1],{cause:e[2]})},g[105]=e=>{if(l.structuredClone===!1)throw Error(`Structured clone extension is disabled`);let t=_.getUint32(a-4);h||=new Map;let n=r[a],i;i=n>=144&&n<160||n==220||n==221?[]:n>=128&&n<144||n==222||n==223?new Map:(n>=199&&n<=201||n>=212&&n<=216)&&r[a+1]===115?new Set:{};let o={target:i};h.set(t,o);let s=b();if(o.used)Object.assign(i,s);else return o.target=s;if(i instanceof Map)for(let[e,t]of s.entries())i.set(e,t);if(i instanceof Set)for(let e of Array.from(s))i.add(e);return i},g[112]=e=>{if(l.structuredClone===!1)throw Error(`Structured clone extension is disabled`);let t=_.getUint32(a-4),n=h.get(t);return n.used=!0,n.target},g[115]=()=>new Set(b());const Ce=[`Int8`,`Uint8`,`Uint8Clamped`,`Int16`,`Uint16`,`Int32`,`Uint32`,`Float32`,`Float64`,`BigInt64`,`BigUint64`].map(e=>e+`Array`);let we=typeof globalThis==`object`?globalThis:window;g[116]=e=>{let t=e[0],n=Uint8Array.prototype.slice.call(e,1).buffer,r=Ce[t];if(!r){if(t===16)return n;if(t===17)return new DataView(n);throw Error(`Could not find typed array for code `+t)}return new we[r](n)},g[120]=()=>{let e=b();return new RegExp(e[0],e[1])};const Te=[];g[98]=e=>{let t=(e[0]<<24)+(e[1]<<16)+(e[2]<<8)+e[3],n=a;return a+=t-e.length,m=Te,m=[ge(),ge()],m.position0=0,m.position1=0,m.postBundlePosition=a,a=n,b()},g[255]=e=>e.length==4?new Date((e[0]*16777216+(e[1]<<16)+(e[2]<<8)+e[3])*1e3):e.length==8?new Date(((e[0]<<22)+(e[1]<<14)+(e[2]<<6)+(e[3]>>2))/1e6+((e[3]&3)*4294967296+e[4]*16777216+(e[5]<<16)+(e[6]<<8)+e[7])*1e3):e.length==12?new Date(((e[0]<<24)+(e[1]<<16)+(e[2]<<8)+e[3])/1e6+((e[4]&128?-281474976710656:0)+e[6]*1099511627776+e[7]*4294967296+e[8]*16777216+(e[9]<<16)+(e[10]<<8)+e[11])*1e3):new Date(`invalid`);function Ee(e){l&&l._onSaveState&&l._onSaveState();let t=i,n=a,o=c,g=f,ee=p,te=d,ne=s,re=h,y=m,ie=new Uint8Array(r.slice(0,i)),b=u,ae=u.slice(0,u.length),x=l,oe=v,se=e();return i=t,a=n,c=o,f=g,p=ee,d=te,s=ne,h=re,m=y,r=ie,v=oe,u=b,u.splice(0,u.length,...ae),l=x,_=new DataView(r.buffer,r.byteOffset,r.byteLength),se}function De(){r=null,h=null,u=null}const Oe=Array(147);for(let e=0;e<256;e++)Oe[e]=+(`1e`+Math.floor(45.15-e*.30103));var ke=new re({useRecords:!1});ke.unpack,ke.unpackMultiple,ke.unpack,new Uint8Array(new Float32Array(1).buffer,0,4),re.SUPPORTS_STRUCT_HOOKS=!0;const Ae=new Set([9,10,11,12,13,32]),je=`z`.codePointAt(0),Me=`!`.codePointAt(0),Ne=`u`.codePointAt(0),Pe=e=>Ae.has(e),Fe=e=>e>=Me&&e<=Ne,Ie=e=>{let t=0,n=0;for(let r=0;r<e.length;r++){let i=e.codePointAt(r);if(!Pe(i)){if(i===je){if(n!==0)throw Error(`Invalid ASCII85 payload: “z” may only appear at the start of a group.`);t+=4;continue}if(!Fe(i))throw Error(`Invalid ASCII85 payload: unexpected character ${JSON.stringify(e[r])}.`);n++,n===5&&(t+=4,n=0)}}if(n===1)throw Error(`Invalid ASCII85 payload: trailing group must contain at least 2 characters.`);return n>1&&(t+=n-1),t},Le=(e,t,n,r=4)=>{r>=1&&(e[t]=n>>>24),r>=2&&(e[t+1]=n>>>16&255),r>=3&&(e[t+2]=n>>>8&255),r>=4&&(e[t+3]=n&255)},Re=e=>{let t=new Uint8Array(Ie(e)),n=0,r=0,i=0;for(let a=0;a<e.length;a++){let o=e.codePointAt(a);if(!Pe(o)){if(o===je){Le(t,i,0),i+=4;continue}r=r*85+o-Me,n++,n===5&&(Le(t,i,r),i+=4,n=0,r=0)}}if(n>1){for(let e=n;e<5;e++)r=r*85+84;Le(t,i,r,n-1)}return t},ze={gpt:{encoding:`o200k_base`,kind:`tiktoken-builtin`,openrouter:`openai/gpt-5.5`,source:{encodingJsonUrl:`https://tiktoken.pages.dev/js/o200k_base.json`},title:`GPT-5.5`},gemma:{kind:`huggingface`,openrouter:`google/gemma-4-31b-it`,source:{tokenizerConfigUrl:`https://huggingface.co/google/gemma-4-31B-it/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/google/gemma-4-31B-it/resolve/main/tokenizer.json`},title:`Gemma 4 31B it`},qwen:{kind:`huggingface`,openrouter:`qwen/qwen3.6-27b`,source:{tokenizerConfigUrl:`https://huggingface.co/Qwen/Qwen3.6-27B/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/Qwen/Qwen3.6-27B/resolve/main/tokenizer.json`},title:`Qwen 3.6 27B`},kimi:{kind:`tiktoken-custom`,openrouter:`moonshotai/kimi-k2.6`,source:{modelUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tiktoken.model`,tokenizerConfigUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tokenizer_config.json`,tokenizerImplementationUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tokenization_kimi.py`},title:`Kimi K2.6`},deepseek:{kind:`huggingface`,openrouter:`deepseek/deepseek-v4-pro`,source:{tokenizerConfigUrl:`https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/main/tokenizer.json`},title:`DeepSeek V4 Pro`},mimo:{kind:`huggingface`,openrouter:`xiaomi/mimo-v2.5-pro`,source:{tokenizerConfigUrl:`https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/main/tokenizer.json`},title:`MiMo V2.5 Pro`},sdxl:{kind:`clip-bpe`,source:{mergesUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/merges.txt`,specialTokensMapUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/special_tokens_map.json`,tokenizerConfigUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/tokenizer_config.json`,vocabUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/vocab.json`},title:`Stable Diffusion XL`},glm:{kind:`huggingface`,openrouter:`zai-org/glm-5.1`,source:{tokenizerConfigUrl:`https://huggingface.co/zai-org/GLM-5.1/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/zai-org/GLM-5.1/resolve/main/tokenizer.json`},title:`GLM 5.1`},minimax:{kind:`huggingface`,openrouter:`minimax/minimax-m2.7`,source:{tokenizerConfigUrl:`https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/main/tokenizer.json`},title:`MiniMax M2.7`}},T=Object.keys(ze),Be=new re({mapsAsObjects:!1}),Ve=new TextDecoder,He=Object.create(null),E=new Map,D=new Map,O=new Map,Ue=(e,t)=>`${e}/${t}`,We=e=>{let t=`${e}/`;for(let e of E.keys())e.startsWith(t)&&E.delete(e);for(let e of O.keys())e.startsWith(t)&&O.delete(e);for(let e of D.keys())e.startsWith(t)&&D.delete(e)},Ge=e=>{let t=He[e];if(!t)throw Error(`Missing tokenizer assets for model ${JSON.stringify(e)}. Run “bun run fetch” first or load the vocabulary chunk before tokenizing.`);return t},Ke=(e,t)=>{let n=Ge(e)[t];if(!n)throw Error(`Missing tokenizer asset ${JSON.stringify(t)} for model ${JSON.stringify(e)}. Run “bun run fetch” first.`);return n},qe=(e,t)=>{let n=Ue(e,t),r=E.get(n);if(r)return r;let i=Ke(e,t);if(i instanceof Uint8Array)return E.set(n,i),i;let a=Re(i);return E.set(n,a),a},Je=e=>{if(e.endsWith(`.msgpack`))return e;if(e.endsWith(`.json`))return`${e.slice(0,-5)}.msgpack`;throw TypeError(`Expected a MessagePack or JSON file name, got ${JSON.stringify(e)}.`)},Ye=e=>Object.hasOwn(He,e),Xe=(e,t)=>{He[e]=t,We(e)},Ze=e=>{for(let t of T){let n=e[t];n&&Xe(t,n)}},Qe=(e,t)=>{if(t.endsWith(`.json`)||t.endsWith(`.msgpack`))throw TypeError(`Cannot read structured tokenizer asset ${JSON.stringify(t)} as text.`);let n=Ue(e,t),r=O.get(n);if(r!==void 0)return r;let i=Ve.decode(qe(e,t));return O.set(n,i),i},k=(e,t)=>{let n=Je(t),r=Ue(e,n),i=D.get(r);if(i!==void 0)return i;let a=Be.unpack(qe(e,n));return D.set(r,a),a},$e=()=>T.filter(e=>Ye(e));let et;const tt=()=>(et??=(async()=>(await import(`./decompress.js`).then(e=>t(e.default,1))).default)(),et),nt=async e=>{if(typeof DecompressionStream==`function`)try{let t=new Blob([e]).stream().pipeThrough(new DecompressionStream(`brotli`));return new Uint8Array(await new Response(t).arrayBuffer())}catch{}return(await tt())(e)},rt=e=>e.endsWith(`.msgpack.br`),it=e=>rt(e)?e.slice(0,-3):e,at=async e=>{let t=await Promise.all(Object.entries(e).map(async([e,t])=>{let n=Re(t);return rt(e)?[it(e),await nt(n)]:[e,n]}));return Object.fromEntries(t)},ot=e=>{let t=[...T];e!==void 0&&(t.length=0,typeof e==`string`?t.push(e):t.push(...e));for(let e of t)if(!T.includes(e))throw Error(`Unknown model ${JSON.stringify(e)}. Supported models: ${T.join(`, `)}`);return t};var st=class{#e=!1;#t;encode(e){return this.encodeWithState(e,this.getState())}getState(){return this.#e||=(this.#t=this.createState(),!0),this.#t}getTokenCount(e){return this.encode(e).length}};const ct=/<\|startoftext\|>|<\|endoftext\|>|'d|'ll|'m|'re|'s|'t|'ve|\p{L}+|\p{N}|[^\s\p{L}\p{N}]+/gu,lt=/\s+/gu,ut=new TextEncoder,dt=e=>{if(typeof e==`string`)return e;if(e instanceof Map){let t=e.get(`content`);if(typeof t==`string`)return t}if(e&&typeof e==`object`&&`content`in e&&typeof e.content==`string`)return e.content},ft=()=>{let e=Array.from({length:256},()=>``),t=[...Array.from({length:94},(e,t)=>t+33),...Array.from({length:12},(e,t)=>t+161),...Array.from({length:82},(e,t)=>t+174)],n=[...t],r=256;for(let e=0;e<256;e++)t.includes(e)||(t.push(e),n.push(r),r++);for(let[r,i]of t.entries())e[i]=String.fromCodePoint(n[r]);return e},pt=e=>{let t=new Map;for(let[n,r]of e.split(/\r?\n/u).filter(Boolean).filter(e=>!e.startsWith(`#`)).entries())t.set(r,n);return t},mt=e=>e.normalize(`NFC`).replaceAll(lt,` `).trim().toLowerCase();var ht=class extends st{modelId;#e=new Map;constructor(e){super(),this.modelId=e}createState(){let e=k(this.modelId,`vocab.msgpack`),t=k(this.modelId,`tokenizer_config.msgpack`),n=new Map,r=dt(t.get(`unk_token`))??`<|endoftext|>`,i=e.get(r);if(i===void 0)throw Error(`Could not find CLIP unknown token ${JSON.stringify(r)} in ${JSON.stringify(this.modelId)} vocabulary.`);for(let r of[t.get(`bos_token`),t.get(`eos_token`),t.get(`unk_token`)]){let t=dt(r);if(!t)continue;let i=e.get(t);i!==void 0&&n.set(t,i)}return{byteEncoder:ft(),mergeRanks:pt(Qe(this.modelId,`merges.txt`)),specialTokenIds:n,unknownTokenId:i,vocabulary:e}}encodeWithState(e,t){let n=mt(e);if(!n)return[];let r=[];for(let e of n.matchAll(ct)){let n=e[0],i=t.specialTokenIds.get(n);if(i!==void 0){r.push(i);continue}let a=Array.from(ut.encode(n),e=>t.byteEncoder[e]).join(``);for(let e of this.#t(a,t.mergeRanks))r.push(t.vocabulary.get(e)??t.unknownTokenId)}return r}#t(e,t){let n=this.#e.get(e);if(n)return n;let r=[...e];if(!r.length){let t=[];return this.#e.set(e,t),t}let i=[...r.slice(0,-1),`${r.at(-1)}</w>`];for(;i.length>1;){let e=-1,n=1/0;for(let r=0;r<i.length-1;r++){let a=t.get(`${i[r]} ${i[r+1]}`);a!==void 0&&a<n&&(n=a,e=r)}if(e<0)break;let r=[];for(let t=0;t<i.length;t++){if(t===e){r.push(`${i[t]}${i[t+1]}`),t++;continue}r.push(i[t])}i=r}return this.#e.set(e,i),i}},gt=class{constructor(e){this.trie=this._build_trie(e)}_build_trie(e){let t=Object.create(null);for(let n of e){let e=t;for(let t=0;t<n.length;++t){let r=n[t];e=e[r]??=Object.create(null)}e.end=n}return t}split(e){let t=[],n=e.length,r=0,i=0;for(;i<n;){let a=this.trie,o=null,s=i;for(;s<n&&(a=a[e[s]]);)a.end&&(o=a.end),++s;o?(i>r&&t.push(e.slice(r,i)),t.push(o),i+=o.length,r=i):++i}return r<n&&t.push(e.slice(r)),t}},_t=class{constructor(e){this.content=e.content,this.id=e.id,this.single_word=e.single_word??!1,this.lstrip=e.lstrip??!1,this.rstrip=e.rstrip??!1,this.special=e.special??!1,this.normalized=e.normalized??!this.special}},vt=(()=>{let e=[...Array.from({length:94},(e,t)=>t+33),...Array.from({length:12},(e,t)=>t+161),...Array.from({length:82},(e,t)=>t+174)],t=e.slice(),n=0;for(let r=0;r<256;++r)e.includes(r)||(e.push(r),t.push(256+n),n+=1);let r=t.map(e=>String.fromCharCode(e));return Object.fromEntries(e.map((e,t)=>[e,r[t]]))})(),yt=(e=>Object.fromEntries(Object.entries(e).map(([e,t])=>[t,e])))(vt),bt=`.,!?…。，、।۔،`,xt=new Map([[`(?i:'s|'t|'re|'ve|'m|'ll|'d)`,`(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))`],[`(?i:[sdmt]|ll|ve|re)`,`(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`],[`[^\\r\\n\\p{L}\\p{N}]?+`,`[^\\r\\n\\p{L}\\p{N}]?`],[`[^\\s\\p{L}\\p{N}]++`,`[^\\s\\p{L}\\p{N}]+`],[`(?>\\p{Nd}{510})`,`(?:\\p{Nd}{510})`],[`\\p{Nd}{3}+`,`(?:\\p{Nd}{3})+`],[`\\G`,``],[` ?[^(\\s|[${bt}])]+`,` ?[^\\s${bt}]+`]]),A=`\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E`,St=e=>e.replace(/ \./g,`.`).replace(/ \?/g,`?`).replace(/ \!/g,`!`).replace(/ ,/g,`,`).replace(/ \' /g,`'`).replace(/ n't/g,`n't`).replace(/ 'm/g,`'m`).replace(/ 's/g,`'s`).replace(/ 've/g,`'ve`).replace(/ 're/g,`'re`),j=(e,t=!0)=>{if(e.Regex!==void 0){let t=e.Regex.replace(/\\([#&~])/g,`$1`);t=t.replace(/\\A/g,`^`).replace(/\\z/g,`$`).replace(/\\Z/g,`(?=\\r?\\n?$)`);for(let[e,n]of xt)t=t.replaceAll(e,n);try{return new RegExp(t,`gu`)}catch(e){if(!(e instanceof SyntaxError)||!e.message.toLowerCase().includes(`invalid property name`))throw e;let n=!1,r=t.replace(/(\\[pP])\{([^}=]+)\}/g,(e,t,r)=>{try{return RegExp(`\\p{${r}}`,`u`),`${t}{${r}}`}catch{return n=!0,`${t}{Script=${r}}`}});if(!n)throw e;try{return new RegExp(r,`gu`)}catch{throw e}}}else if(e.String!==void 0){let n=Ct(e.String);return new RegExp(t?n:`(${n})`,`gu`)}else return console.warn(`Unknown pattern type:`,e),null},Ct=e=>e.replace(/[.*+?^${}()|[\]\\]/g,`\\$&`),wt=(e,t,n)=>{let r=[],i=0;for(;i<e.length;){if(r.push(e[i]),(t.get(e[i])??n)!==n){++i;continue}for(;++i<e.length&&(t.get(e[i])??n)===n;)t.get(r.at(-1))!==n&&(r[r.length-1]+=e[i])}return r},Tt=e=>e>=19968&&e<=40959||e>=13312&&e<=19903||e>=131072&&e<=173791||e>=173824&&e<=177983||e>=177984&&e<=178207||e>=178208&&e<=183983||e>=63744&&e<=64255||e>=194560&&e<=195103,Et=e=>Number.isInteger(e)||typeof e==`bigint`,Dt=e=>{let t=0;for(let n of e)++t;return t},Ot=e=>jt(e.toLowerCase()),M=(...e)=>Array.prototype.concat.apply([],e),kt=e=>new Map(Object.entries(e)),At=(e,t)=>{let n=[],r=0;for(let i of e.matchAll(t)){let t=i[0];r<i.index&&n.push(e.slice(r,i.index)),t.length>0&&n.push(t),r=i.index+t.length}return r<e.length&&n.push(e.slice(r)),n},jt=e=>e.replace(/\p{M}/gu,``),Mt=(e,t,n=[])=>{if(!e||Array.isArray(e)||typeof e!=`object`)return`${t} must be a valid object`;for(let r of n)if(!(r in e))return`${t} must contain a "${r}" property`;return null},Nt=e=>e.match(/\S+/g)||[],N=class{constructor(){let e=function(...t){return e._call(...t)};return Object.setPrototypeOf(e,new.target.prototype)}},P=class extends N{constructor(e){super(),this.config=e}_call(e){return this.normalize(e)}},Pt=class extends P{tokenize_chinese_chars(e){let t=[];for(let n=0;n<e.length;++n){let r=e[n];Tt(r.charCodeAt(0))?(t.push(` `),t.push(r),t.push(` `)):t.push(r)}return t.join(``)}strip_accents(e){return e.normalize(`NFD`).replace(/\p{Mn}/gu,``)}is_control(e){switch(e){case`	`:case`
-`:case`\r`:return!1;default:return/^\p{Cc}|\p{Cf}|\p{Co}|\p{Cs}$/u.test(e)}}clean_text(e){let t=[];for(let n of e){let e=n.charCodeAt(0);e===0||e===65533||this.is_control(n)||(/^\s$/.test(n)?t.push(` `):t.push(n))}return t.join(``)}normalize(e){return this.config.clean_text&&(e=this.clean_text(e)),this.config.handle_chinese_chars&&(e=this.tokenize_chinese_chars(e)),this.config.lowercase?(e=e.toLowerCase(),this.config.strip_accents!==!1&&(e=this.strip_accents(e))):this.config.strip_accents&&(e=this.strip_accents(e)),e}},Ft=class extends P{constructor(e){super(e),this.charsmap=e.precompiled_charsmap??null}normalize(e){return e=e.replace(/[\u0001-\u0008\u000B\u000E-\u001F\u007F\u008F\u009F]/gm,``),e=e.replace(/[\u0009\u000A\u000C\u000D\u00A0\u1680\u2000-\u200F\u2028\u2029\u202F\u205F\u2581\u3000\uFEFF\uFFFD]/gm,` `),e=e.includes(`～`)?e.split(`～`).map(e=>e.normalize(`NFKC`)).join(`～`):e.normalize(`NFKC`),e}},It=class extends P{constructor(e){super(e),this.normalizers=(e.normalizers??[]).map(e=>qt(e))}normalize(e){return this.normalizers.reduce((e,t)=>t?t.normalize(e):e,e)}},Lt=class extends P{normalize(e){let t=j(this.config.pattern??{});return t===null?e:e.replaceAll(t,this.config.content??``)}},F=class extends P{constructor(){super(...arguments),this.form=`NFC`}normalize(e){return e=e.normalize(this.form),e}},Rt=class extends F{constructor(){super(...arguments),this.form=`NFC`}},zt=class extends F{constructor(){super(...arguments),this.form=`NFD`}},Bt=class extends F{constructor(){super(...arguments),this.form=`NFKC`}},Vt=class extends F{constructor(){super(...arguments),this.form=`NFKD`}},Ht=class extends P{normalize(e){return this.config.strip_left&&this.config.strip_right?e=e.trim():(this.config.strip_left&&(e=e.trimStart()),this.config.strip_right&&(e=e.trimEnd())),e}},Ut=class extends P{normalize(e){return jt(e)}},Wt=class extends P{normalize(e){return e.toLowerCase()}},Gt=class extends P{normalize(e){return e=this.config.prepend+e,e}};function Kt(e){if(e===null)return null;switch(e.type){case`BertNormalizer`:return new Pt(e);case`Precompiled`:return new Ft(e);case`Sequence`:return new It(e);case`Replace`:return new Lt(e);case`NFC`:return new Rt(e);case`NFD`:return new zt(e);case`NFKC`:return new Bt(e);case`NFKD`:return new Vt(e);case`Strip`:return new Ht(e);case`StripAccents`:return new Ut(e);case`Lowercase`:return new Wt(e);case`Prepend`:return new Gt(e);default:throw Error(`Unknown Normalizer type: ${e.type}`)}}var qt=Kt,I=class extends N{pre_tokenize(e,t){return(Array.isArray(e)?e.map(e=>this.pre_tokenize_text(e,t)):this.pre_tokenize_text(e,t)).flat()}_call(e,t){return this.pre_tokenize(e,t)}},Jt=class extends I{constructor(e){super(),this.config=e,this.add_prefix_space=this.config.add_prefix_space??!1,this.trim_offsets=this.config.trim_offsets??!1,this.use_regex=this.config.use_regex??!0,this.pattern=/'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu,this.byte_encoder=vt,this.text_encoder=new TextEncoder}pre_tokenize_text(e,t){return this.add_prefix_space&&!e.startsWith(` `)&&(e=` `+e),(this.use_regex?e.match(this.pattern)||[]:[e]).map(e=>Array.from(this.text_encoder.encode(e),e=>this.byte_encoder[e]).join(``))}},Yt=class extends I{pre_tokenize_text(e,t){return e.match(/\w+|[^\w\s]+/g)||[]}},Xt=class extends I{constructor(e){super(),this.replacement=e.replacement??`▁`,this.str_rep=e.str_rep||this.replacement,this.prepend_scheme=e.prepend_scheme??`always`}pre_tokenize_text(e,t){let{section_index:n=void 0}=t??{},r=e.replaceAll(` `,this.str_rep);return!r.startsWith(this.replacement)&&(this.prepend_scheme===`always`||this.prepend_scheme===`first`&&n===0)&&(r=this.str_rep+r),[r]}},Zt=class extends I{constructor(e){super(),this.config=e,this.pattern=j(this.config.pattern??{},this.config.invert??!0)}pre_tokenize_text(e){return this.pattern===null?[]:this.config.invert?e.match(this.pattern)||[]:this.config.behavior?.toLowerCase()===`removed`?e.split(this.pattern).filter(e=>e):At(e,this.pattern)}},Qt=class extends I{constructor(e){super(),this.config=e,this.pattern=RegExp(`[^${A}]+|[${A}]+`,`gu`)}pre_tokenize_text(e){return e.match(this.pattern)||[]}},$t=class extends I{constructor(e){super(),this.config=e;let t=`[^\\d]+|\\d${this.config.individual_digits?``:`+`}`;this.pattern=new RegExp(t,`gu`)}pre_tokenize_text(e){return e.match(this.pattern)||[]}},en=class extends I{constructor(){super(),this.pattern=RegExp(`[^\\s${A}]+|[${A}]`,`gu`)}pre_tokenize_text(e,t){return e.trim().match(this.pattern)||[]}},tn=class extends I{constructor(e){super(),this.config=e,this.pattern=j(this.config.pattern??{}),this.content=this.config.content??``}pre_tokenize_text(e){return this.pattern===null?[e]:[e.replaceAll(this.pattern,this.config.content??``)]}},nn=class extends I{constructor(e){super(),this.tokenizers=(e.pretokenizers??[]).map(e=>sn(e))}pre_tokenize_text(e,t){return this.tokenizers.reduce((e,n)=>n?n.pre_tokenize(e,t):e,[e])}},rn=class extends I{pre_tokenize_text(e){return Nt(e)}},an=class extends I{constructor(e){super(),this.config=e,this._length=e.length}pre_tokenize_text(e){let t=[];for(let n=0;n<e.length;n+=this._length)t.push(e.slice(n,n+this._length));return t}};function on(e){if(e===null)return null;switch(e.type){case`BertPreTokenizer`:return new en;case`Sequence`:return new nn(e);case`Whitespace`:return new Yt;case`WhitespaceSplit`:return new rn;case`Metaspace`:return new Xt(e);case`ByteLevel`:return new Jt(e);case`Split`:return new Zt(e);case`Punctuation`:return new Qt(e);case`Digits`:return new $t(e);case`Replace`:return new tn(e);case`FixedLength`:return new an(e);default:throw Error(`Unknown PreTokenizer type: ${e.type}`)}}var sn=on,L=class extends N{constructor(e){super(),this.config=e,this.vocab=[],this.tokens_to_ids=new Map,this.unk_token_id=void 0,this.unk_token=void 0,this.end_of_word_suffix=void 0,this.fuse_unk=this.config.fuse_unk??!1}_call(e){let t=this.encode(e);return this.fuse_unk&&(t=wt(t,this.tokens_to_ids,this.unk_token_id)),t}},cn=class extends L{constructor(e){super(e),this.max_input_chars_per_word=100,this.tokens_to_ids=kt(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.max_input_chars_per_word=e.max_input_chars_per_word??100,this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e}encode(e){let t=[];for(let n of e){let e=[...n];if(e.length>this.max_input_chars_per_word){t.push(this.unk_token);continue}let r=!1,i=0,a=[];for(;i<e.length;){let t=e.length,n=null;for(;i<t;){let r=e.slice(i,t).join(``);if(i>0&&(r=this.config.continuing_subword_prefix+r),this.tokens_to_ids.has(r)){n=r;break}--t}if(n===null){r=!0;break}a.push(n),i=t}r?t.push(this.unk_token):t.push(...a)}return t}},ln=class e{constructor(e,t){this.is_leaf=e,this.children=t}static default(){return new e(!1,new Map)}},un=class{constructor(){this.root=ln.default()}extend(e){for(let t of e)this.push(t)}push(e){let t=this.root;for(let n of e){let e=t.children.get(n);e===void 0&&(e=ln.default(),t.children.set(n,e)),t=e}t.is_leaf=!0}*common_prefix_search(e){let t=this.root;if(t===void 0)return;let n=``;for(let r of e){if(n+=r,t=t.children.get(r),t===void 0)return;t.is_leaf&&(yield n)}}},dn=class e{constructor(e,t,n,r,i){this.token_id=e,this.node_id=t,this.pos=n,this.length=r,this.score=i,this.prev=null,this.backtrace_score=0}clone(){let t=new e(this.token_id,this.node_id,this.pos,this.length,this.score);return t.prev=this.prev,t.backtrace_score=this.backtrace_score,t}},fn=class{constructor(e,t,n){this.chars=Array.from(e),this.len=this.chars.length,this.bos_token_id=t,this.eos_token_id=n,this.nodes=[],this.begin_nodes=Array.from({length:this.len+1},()=>[]),this.end_nodes=Array.from({length:this.len+1},()=>[]);let r=new dn(this.bos_token_id??0,0,0,0,0),i=new dn(this.eos_token_id??0,1,this.len,0,0);this.nodes.push(r.clone()),this.nodes.push(i.clone()),this.begin_nodes[this.len].push(i),this.end_nodes[0].push(r)}insert(e,t,n,r){let i=this.nodes.length,a=new dn(r,i,e,t,n);this.begin_nodes[e].push(a),this.end_nodes[e+t].push(a),this.nodes.push(a)}viterbi(){let e=this.len,t=0;for(;t<=e;){if(this.begin_nodes[t].length==0)return[];for(let e of this.begin_nodes[t]){e.prev=null;let n=0,r=null;for(let i of this.end_nodes[t]){let t=i.backtrace_score+e.score;(r===null||t>n)&&(r=i.clone(),n=t)}if(r!==null)e.prev=r,e.backtrace_score=n;else return[]}++t}let n=[],r=this.begin_nodes[e][0].prev;if(r===null)return[];let i=r.clone();for(;i.prev!==null;)n.push(i.clone()),i=i.clone().prev.clone();return n.reverse(),n}piece(e){return this.chars.slice(e.pos,e.pos+e.length).join(``)}tokens(){return this.viterbi().map(e=>this.piece(e))}token_ids(){return this.viterbi().map(e=>e.token_id)}};function pn(e){if(e.length===0)throw Error(`Array must not be empty`);let t=e[0],n=0;for(let r=1;r<e.length;++r)e[r]<t&&(t=e[r],n=r);return[t,n]}var mn=class extends L{constructor(e,t){super(e);let n=e.vocab.length;this.vocab=Array(n),this.scores=Array(n);for(let t=0;t<n;++t)[this.vocab[t],this.scores[t]]=e.vocab[t];this.unk_token_id=e.unk_id,this.unk_token=this.vocab[e.unk_id],this.tokens_to_ids=new Map(this.vocab.map((e,t)=>[e,t])),this.bos_token=` `,this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.unk_token=this.vocab[this.unk_token_id],this.min_score=pn(this.scores)[0],this.unk_score=this.min_score-10,this.scores[this.unk_token_id]=this.unk_score,this.trie=new un,this.trie.extend(this.vocab),this.fuse_unk=!0}populate_nodes(e){let t=e.chars,n=0;for(;n<t.length;){let r=!1,i=[],a=t.slice(n).join(``),o=this.trie.common_prefix_search(a);for(let t of o){i.push(t);let a=this.tokens_to_ids.get(t),o=this.scores[a],s=Dt(t);e.insert(n,s,o,a),!r&&s===1&&(r=!0)}r||e.insert(n,1,this.unk_score,this.unk_token_id),n+=1}}tokenize(e){let t=new fn(e,this.bos_token_id,this.eos_token_id);return this.populate_nodes(t),t.tokens()}encode(e){let t=[];for(let n of e){let e=this.tokenize(n);t.push(...e)}return t}},hn=class{constructor(e=(e,t)=>e>t,t=1/0){this._heap=[],this._comparator=e,this._max_size=t}get size(){return this._heap.length}is_empty(){return this.size===0}peek(){return this._heap[0]}push(...e){return this.extend(e)}extend(e){for(let t of e)if(this.size<this._max_size)this._heap.push(t),this._sift_up();else{let e=this._smallest();this._comparator(t,this._heap[e])&&(this._heap[e]=t,this._sift_up_from(e))}return this.size}pop(){let e=this.peek(),t=this.size-1;return t>0&&this._swap(0,t),this._heap.pop(),this._sift_down(),e}replace(e){let t=this.peek();return this._heap[0]=e,this._sift_down(),t}_parent(e){return(e+1>>>1)-1}_left(e){return(e<<1)+1}_right(e){return e+1<<1}_greater(e,t){return this._comparator(this._heap[e],this._heap[t])}_swap(e,t){let n=this._heap[e];this._heap[e]=this._heap[t],this._heap[t]=n}_sift_up(){this._sift_up_from(this.size-1)}_sift_up_from(e){for(;e>0&&this._greater(e,this._parent(e));)this._swap(e,this._parent(e)),e=this._parent(e)}_sift_down(){let e=0;for(;this._left(e)<this.size&&this._greater(this._left(e),e)||this._right(e)<this.size&&this._greater(this._right(e),e);){let t=this._right(e)<this.size&&this._greater(this._right(e),this._left(e))?this._right(e):this._left(e);this._swap(e,t),e=t}}_smallest(){return 2**Math.floor(Math.log2(this.size))-1}},gn=class{constructor(e){this.capacity=e,this.cache=new Map}get(e){if(!this.cache.has(e))return;let t=this.cache.get(e);return this.cache.delete(e),this.cache.set(e,t),t}put(e,t){this.cache.has(e)&&this.cache.delete(e),this.cache.set(e,t),this.cache.size>this.capacity&&this.cache.delete(this.cache.keys().next().value)}clear(){this.cache.clear()}},_n=class extends L{constructor(e){super(e),this.tokens_to_ids=kt(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e;let t=Array.isArray(e.merges[0]);this.merges=t?e.merges:e.merges.map(e=>e.split(` `,2)),this.bpe_ranks=new Map(this.merges.map((e,t)=>[JSON.stringify(e),t])),this.end_of_word_suffix=e.end_of_word_suffix,this.continuing_subword_suffix=e.continuing_subword_suffix??null,this.byte_fallback=this.config.byte_fallback??!1,this.byte_fallback&&(this.text_encoder=new TextEncoder),this.ignore_merges=this.config.ignore_merges??!1,this.max_length_to_cache=256,this.cache_capacity=1e4,this.cache=new gn(this.cache_capacity)}clear_cache(){this.cache.clear()}bpe(e){if(e.length===0)return[];let t=this.cache.get(e);if(t!==void 0)return t;let n=Array.from(e);this.end_of_word_suffix&&(n[n.length-1]+=this.end_of_word_suffix);let r=[];if(n.length>1){let e=new hn((e,t)=>e.score<t.score),t={token:n[0],bias:0,prev:null,next:null},i=t;for(let t=1;t<n.length;++t){let r={bias:t/n.length,token:n[t],prev:i,next:null};i.next=r,this.add_node(e,i),i=r}for(;!e.is_empty();){let n=e.pop();if(n.deleted||!n.next||n.next.deleted)continue;if(n.deleted=!0,n.next.deleted=!0,n.prev){let e={...n.prev};n.prev.deleted=!0,n.prev=e,e.prev?e.prev.next=e:t=e}let r={token:n.token+n.next.token,bias:n.bias,prev:n.prev,next:n.next.next};r.prev?(r.prev.next=r,this.add_node(e,r.prev)):t=r,r.next&&(r.next.prev=r,this.add_node(e,r))}for(let e=t;e!==null;e=e.next)r.push(e.token)}else r=n;if(this.continuing_subword_suffix)for(let e=0;e<r.length-1;++e)r[e]+=this.continuing_subword_suffix;return e.length<this.max_length_to_cache&&this.cache.put(e,r),r}add_node(e,t){let n=this.bpe_ranks.get(JSON.stringify([t.token,t.next.token]));n!==void 0&&(t.score=n+t.bias,e.push(t))}encode(e){let t=[];for(let n of e){if(this.ignore_merges&&this.tokens_to_ids.has(n)){t.push(n);continue}let e=this.bpe(n);for(let n of e)if(this.tokens_to_ids.has(n))t.push(n);else if(this.byte_fallback){let e=Array.from(this.text_encoder.encode(n)).map(e=>`<0x${e.toString(16).toUpperCase().padStart(2,`0`)}>`);e.every(e=>this.tokens_to_ids.has(e))?t.push(...e):this.unk_token!=null&&t.push(this.unk_token)}else this.unk_token!=null&&t.push(this.unk_token)}return t}},vn=class extends L{constructor(e,t){super(e);let n=e.vocab;this.tokens_to_ids=kt(t.target_lang?n[t.target_lang]:n),this.bos_token=t.bos_token,this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t.eos_token,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.pad_token=t.pad_token,this.pad_token_id=this.tokens_to_ids.get(this.pad_token),this.unk_token=t.unk_token,this.unk_token_id=this.tokens_to_ids.get(this.unk_token),this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e}encode(e){return e}};function yn(e,t){switch(e.type){case`WordPiece`:return new cn(e);case`Unigram`:return new mn(e,t.eos_token);case`BPE`:return new _n(e);default:if(e.vocab)return Array.isArray(e.vocab)?new mn(e,t.eos_token):Object.hasOwn(e,`continuing_subword_prefix`)&&Object.hasOwn(e,`unk_token`)?Object.hasOwn(e,`merges`)?new _n(e):new cn(e):new vn(e,{target_lang:t.target_lang,bos_token:t.bos_token,eos_token:t.eos_token,pad_token:t.pad_token,unk_token:t.unk_token});throw Error(`Unknown TokenizerModel type: ${e?.type}`)}}var bn=yn,R=class extends N{constructor(e){super(),this.config=e}_call(e,...t){return this.post_process(e,...t)}},xn=class extends R{post_process(e,t=null,n=!0){let r=t===null?this.config.single:this.config.pair,i=[],a=[];for(let o of r)`SpecialToken`in o?n&&(i.push(o.SpecialToken.id),a.push(o.SpecialToken.type_id)):`Sequence`in o&&(o.Sequence.id===`A`?(i=M(i,e),a=M(a,Array(e.length).fill(o.Sequence.type_id))):o.Sequence.id===`B`&&(i=M(i,t),a=M(a,Array(t.length).fill(o.Sequence.type_id))));return{tokens:i,token_type_ids:a}}},Sn=class extends R{post_process(e,t=null){return{tokens:e,tokens_pair:t}}},Cn=class extends R{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t=null,n=!0){n&&(e=M([this.cls[0]],e,[this.sep[0]]));let r=Array(e.length).fill(0);if(t){let i=[],a=n?[this.sep[0]]:[];e=M(e,i,t,a),r=M(r,Array(t.length+i.length+a.length).fill(1))}return{tokens:e,token_type_ids:r}}},wn=class extends R{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t,n=!0){n&&(e=M([this.cls[0]],e,[this.sep[0]]));let r=Array(e.length).fill(0);if(t){let i=n?[this.sep[0]]:[],a=n?[this.sep[0]]:[];e=M(e,i,t,a),r=M(r,Array(t.length+i.length+a.length).fill(1))}return{tokens:e,token_type_ids:r}}},Tn=class extends R{constructor(e){super(e),this.processors=(e.processors??[]).map(e=>Dn(e))}post_process(e,t=null,n=!0){let r={tokens:e,tokens_pair:t};for(let e of this.processors)r=e.post_process(r.tokens,r.tokens_pair,n);return r}};function En(e){if(e===null)return null;switch(e.type){case`TemplateProcessing`:return new xn(e);case`ByteLevel`:return new Sn(e);case`BertProcessing`:return new Cn(e);case`RobertaProcessing`:return new wn(e);case`Sequence`:return new Tn(e);default:throw Error(`Unknown PostProcessor type: ${e.type}`)}}var Dn=En,z=class extends N{constructor(e){super(),this.config=e,this.added_tokens=[],this.end_of_word_suffix=null,this.trim_offsets=`trim_offsets`in e?e.trim_offsets:!1}_call(e){return this.decode(e)}decode(e){return this.decode_chain(e).join(``)}},On=class extends z{constructor(e){super(e),this.byte_decoder=yt,this.text_decoder=new TextDecoder(`utf-8`,{fatal:!1,ignoreBOM:!0}),this.end_of_word_suffix=null}convert_tokens_to_string(e){let t=e.join(``),n=new Uint8Array([...t].map(e=>this.byte_decoder[e]));return this.text_decoder.decode(n)}decode_chain(e){let t=[],n=[];for(let r of e)this.added_tokens.find(e=>e.content===r)===void 0?n.push(r):(n.length>0&&(t.push(this.convert_tokens_to_string(n)),n=[]),t.push(r));return n.length>0&&t.push(this.convert_tokens_to_string(n)),t}},kn=class extends z{constructor(e){super(e),this.cleanup=e.cleanup}decode_chain(e){return e.map((e,t)=>{if(t!==0){let t=this.config.prefix;e=t&&e.startsWith(t)?e.replace(t,``):` `+e}return this.cleanup&&(e=St(e)),e})}},An=class extends z{constructor(e){super(e),this.replacement=e.replacement??`▁`}decode_chain(e){let t=[];for(let n=0;n<e.length;++n){let r=e[n].replaceAll(this.replacement,` `);n==0&&r.startsWith(` `)&&(r=r.substring(1)),t.push(r)}return t}},jn=class extends z{constructor(e){super(e),this.suffix=e.suffix??``}decode_chain(e){return e.map((t,n)=>t.replaceAll(this.suffix,n===e.length-1?``:` `))}},Mn=class extends z{constructor(e){super(e),this.pad_token=e.pad_token??``,this.word_delimiter_token=e.word_delimiter_token??``,this.cleanup=e.cleanup}convert_tokens_to_string(e){if(e.length===0)return``;let t=[e[0]];for(let n=1;n<e.length;++n)e[n]!==t.at(-1)&&t.push(e[n]);let n=t.filter(e=>e!==this.pad_token).join(``);return this.cleanup&&(n=St(n).replaceAll(this.word_delimiter_token,` `).trim()),n}decode_chain(e){return[this.convert_tokens_to_string(e)]}},Nn=class extends z{constructor(e){super(e),this.decoders=(e.decoders??[]).map(e=>zn(e))}decode_chain(e){return this.decoders.reduce((e,t)=>t.decode_chain(e),e)}},Pn=class extends z{decode_chain(e){let t=j(this.config.pattern),n=this.config.content??``;return t===null?e:e.map(e=>e.replaceAll(t,n))}},Fn=class extends z{decode_chain(e){return[e.join(``)]}},In=class extends z{constructor(e){super(e),this.content=e.content??``,this.start=e.start??0,this.stop=e.stop??0}decode_chain(e){return e.map(e=>{let t=0;for(let n=0;n<this.start&&e[n]===this.content;++n)t=n+1;let n=e.length;for(let t=0;t<this.stop;++t){let r=e.length-t-1;if(e[r]===this.content){n=r;continue}else break}return e.slice(t,n)})}},Ln=class extends z{constructor(e){super(e),this.text_decoder=new TextDecoder}decode_chain(e){let t=[],n=[];for(let r of e){let e=null;if(r.length===6&&r.startsWith(`<0x`)&&r.endsWith(`>`)){let t=parseInt(r.slice(3,5),16);isNaN(t)||(e=t)}if(e!==null)n.push(e);else{if(n.length>0){let e=this.text_decoder.decode(Uint8Array.from(n));t.push(e),n=[]}t.push(r)}}if(n.length>0){let e=this.text_decoder.decode(Uint8Array.from(n));t.push(e),n=[]}return t}};function Rn(e){if(e===null)return null;switch(e.type){case`ByteLevel`:return new On(e);case`WordPiece`:return new kn(e);case`Metaspace`:return new An(e);case`BPEDecoder`:return new jn(e);case`CTC`:return new Mn(e);case`Sequence`:return new Nn(e);case`Replace`:return new Pn(e);case`Fuse`:return new Fn(e);case`Strip`:return new In(e);case`ByteFallback`:return new Ln(e);default:throw Error(`Unknown Decoder type: ${e.type}`)}}var zn=Rn,Bn=class{constructor(e,t){let n=Mt(e,`Tokenizer`,[`model`,`decoder`,`post_processor`,`pre_tokenizer`,`normalizer`]);if(n)throw Error(n);let r=Mt(t,`Config`);if(r)throw Error(r);this.tokenizer=e,this.config=t,this.normalizer=qt(this.tokenizer.normalizer),this.pre_tokenizer=sn(this.tokenizer.pre_tokenizer),this.model=bn(this.tokenizer.model,this.config),this.post_processor=Dn(this.tokenizer.post_processor),this.decoder=zn(this.tokenizer.decoder),this.special_tokens=[],this.all_special_ids=[],this.added_tokens=[];let i=[],a=[];this.added_tokens_map=new Map;for(let e of this.tokenizer.added_tokens){let t=new _t(e);if(this.added_tokens.push(t),this.model.tokens_to_ids.set(t.content,t.id),this.model.vocab[t.id]=t.content,t.special&&(this.special_tokens.push(t.content),this.all_special_ids.push(t.id)),this.added_tokens_map.set(t.content,t),t.normalized&&this.normalizer!==null){let e=this.normalizer(t.content);a.push(e),this.added_tokens_map.set(e,t)}else i.push(t.content)}(this.config.additional_special_tokens??[]).forEach(e=>{this.special_tokens.includes(e)||this.special_tokens.push(e)}),this.decoder&&(this.decoder.added_tokens=this.added_tokens,this.decoder.end_of_word_suffix=this.model.end_of_word_suffix),this.splitter_unnormalized=new gt(i),this.splitter_normalized=new gt(a),this.remove_space=this.config.remove_space,this.clean_up_tokenization_spaces=this.config.clean_up_tokenization_spaces??!0,this.do_lowercase_and_remove_accent=this.config.do_lowercase_and_remove_accent??!1}encode(e,{text_pair:t=null,add_special_tokens:n=!0,return_token_type_ids:r=null}={}){let{tokens:i,token_type_ids:a}=this.tokenize_helper(e,{text_pair:t,add_special_tokens:n}),o=i.map(e=>this.added_tokens_map.get(e)?.id??this.model.tokens_to_ids.get(e)??this.model.unk_token_id),s={ids:o,tokens:i,attention_mask:Array(o.length).fill(1)};return r&&a&&(s.token_type_ids=a),s}decode(e,t={}){if(!Array.isArray(e)||e.length===0||!Et(e[0]))throw Error(`token_ids must be a non-empty array of integers.`);let n=e.map(e=>this.model.vocab[Number(e)]??this.model.unk_token);t.skip_special_tokens&&(n=n.filter(e=>!this.special_tokens.includes(e)));let r=this.decoder?this.decoder(n):n.join(` `);return this.decoder&&this.decoder.end_of_word_suffix&&(r=r.replaceAll(this.decoder.end_of_word_suffix,` `),t.skip_special_tokens&&(r=r.trim())),(t.clean_up_tokenization_spaces??this.clean_up_tokenization_spaces)&&(r=St(r)),r}tokenize(e,{text_pair:t=null,add_special_tokens:n=!1}={}){return this.tokenize_helper(e,{text_pair:t,add_special_tokens:n}).tokens}encode_text(e){if(e===null)return null;let t=this.splitter_unnormalized.split(e);return t.forEach((e,n)=>{let r=this.added_tokens_map.get(e);r&&(r.lstrip&&n>0&&(t[n-1]=t[n-1].trimEnd()),r.rstrip&&n<t.length-1&&(t[n+1]=t[n+1].trimStart()))}),t.flatMap((e,t)=>{if(e.length===0)return[];if(this.added_tokens_map.has(e))return[e];if(this.remove_space===!0&&(e=e.trim().split(/\s+/).join(` `)),this.do_lowercase_and_remove_accent&&(e=Ot(e)),this.normalizer!==null&&(e=this.normalizer(e)),e.length===0)return[];let n=this.splitter_normalized.split(e);return n.forEach((e,t)=>{let r=this.added_tokens_map.get(e);r&&(r.lstrip&&t>0&&(n[t-1]=n[t-1].trimEnd()),r.rstrip&&t<n.length-1&&(n[t+1]=n[t+1].trimStart()))}),n.flatMap(e=>{if(e.length===0)return[];if(this.added_tokens_map.has(e))return[e];let n=this.pre_tokenizer===null?[e]:this.pre_tokenizer(e,{section_index:t});return this.model(n)})})}tokenize_helper(e,{text_pair:t=null,add_special_tokens:n=!0}){let r=this.encode_text(e),i=this.encode_text(t||null);return this.post_processor?this.post_processor(r,i,n):{tokens:M(r??[],i??[])}}token_to_id(e){return this.model.tokens_to_ids.get(e)}id_to_token(e){return this.model.vocab[e]}get_added_tokens_decoder(){let e=new Map;for(let t of this.added_tokens)e.set(t.id,t);return e}get_vocab(e=!0){let t=new Map;for(let n=0;n<this.model.vocab.length;++n){let r=this.model.vocab[n];(e||!this.added_tokens_map.has(r))&&t.set(r,n)}return t}};const Vn=(e,t)=>{if(!e.has(t))throw Error(`Missing required key ${JSON.stringify(t)} in structured data.`);return e.get(t)},B=e=>{if(Array.isArray(e))return e.map(e=>B(e));if(e instanceof Map){let t=Object.create(null);for(let[n,r]of e.entries()){if(typeof n!=`string`)throw TypeError(`Expected string structured-data key, got ${typeof n}.`);t[n]=B(r)}return t}return e},Hn=Bn;var Un=class extends st{modelId;constructor(e){super(),this.modelId=e}createState(){return{tokenizer:new Hn(B(k(this.modelId,`tokenizer.msgpack`)),B(k(this.modelId,`tokenizer_config.msgpack`)))}}encodeWithState(e,t){return[...t.tokenizer.encode(e).ids]}},Wn=e({Tiktoken:()=>cr,__wbg_parse_def2e24ef1252aff:()=>lr,__wbg_set_wasm:()=>Gn,__wbg_stringify_f7ed6987935b4a24:()=>ur,__wbindgen_error_new:()=>dr,__wbindgen_is_undefined:()=>fr,__wbindgen_object_drop_ref:()=>pr,__wbindgen_string_get:()=>mr,__wbindgen_throw:()=>hr,encoding_for_model:()=>ar,get_encoding:()=>ir,get_encoding_name_for_model:()=>or});let V;function Gn(e){V=e}let Kn=new(typeof TextDecoder>`u`?(0,module.require)(`util`).TextDecoder:TextDecoder)(`utf-8`,{ignoreBOM:!0,fatal:!0});Kn.decode();let H=null;function U(){return(H===null||H.byteLength===0)&&(H=new Uint8Array(V.memory.buffer)),H}function W(e,t){return e>>>=0,Kn.decode(U().subarray(e,e+t))}const G=Array(128).fill(void 0);G.push(void 0,null,!0,!1);let K=G.length;function q(e){K===G.length&&G.push(G.length+1);let t=K;return K=G[t],G[t]=e,t}function qn(e,t){try{return e.apply(this,t)}catch(e){V.__wbindgen_export_0(q(e))}}function Jn(e){return G[e]}function Yn(e){e<132||(G[e]=K,K=e)}function J(e){let t=Jn(e);return Yn(e),t}let Y=0,X=new(typeof TextEncoder>`u`?(0,module.require)(`util`).TextEncoder:TextEncoder)(`utf-8`);const Xn=typeof X.encodeInto==`function`?function(e,t){return X.encodeInto(e,t)}:function(e,t){let n=X.encode(e);return t.set(n),{read:e.length,written:n.length}};function Z(e,t,n){if(n===void 0){let n=X.encode(e),r=t(n.length,1)>>>0;return U().subarray(r,r+n.length).set(n),Y=n.length,r}let r=e.length,i=t(r,1)>>>0,a=U(),o=0;for(;o<r;o++){let t=e.charCodeAt(o);if(t>127)break;a[i+o]=t}if(o!==r){o!==0&&(e=e.slice(o)),i=n(i,r,r=o+e.length*3,1)>>>0;let t=U().subarray(i+o,i+r),a=Xn(e,t);o+=a.written,i=n(i,r,o,1)>>>0}return Y=o,i}function Zn(e){return e==null}let Q=null;function $(){return(Q===null||Q.buffer.detached===!0||Q.buffer.detached===void 0&&Q.buffer!==V.memory.buffer)&&(Q=new DataView(V.memory.buffer)),Q}let Qn=null;function $n(){return(Qn===null||Qn.byteLength===0)&&(Qn=new Uint32Array(V.memory.buffer)),Qn}function er(e,t){return e>>>=0,$n().subarray(e/4,e/4+t)}function tr(e,t){let n=t(e.length*1,1)>>>0;return U().set(e,n/1),Y=e.length,n}function nr(e,t){let n=t(e.length*4,4)>>>0;return $n().set(e,n/4),Y=e.length,n}function rr(e,t){return e>>>=0,U().subarray(e/1,e/1+t)}function ir(e,t){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=Z(e,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y;V.get_encoding(i,a,o,q(t));var n=$().getInt32(i+0,!0),r=$().getInt32(i+4,!0);if($().getInt32(i+8,!0))throw J(r);return cr.__wrap(n)}finally{V.__wbindgen_add_to_stack_pointer(16)}}function ar(e,t){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=Z(e,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y;V.encoding_for_model(i,a,o,q(t));var n=$().getInt32(i+0,!0),r=$().getInt32(i+4,!0);if($().getInt32(i+8,!0))throw J(r);return cr.__wrap(n)}finally{V.__wbindgen_add_to_stack_pointer(16)}}function or(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let t,n;try{let l=V.__wbindgen_add_to_stack_pointer(-16),u=Z(e,V.__wbindgen_export_1,V.__wbindgen_export_2),d=Y;V.get_encoding_name_for_model(l,u,d);var r=$().getInt32(l+0,!0),i=$().getInt32(l+4,!0),a=$().getInt32(l+8,!0),o=$().getInt32(l+12,!0),s=r,c=i;if(o)throw s=0,c=0,J(a);return t=s,n=c,W(s,c)}finally{V.__wbindgen_add_to_stack_pointer(16),V.__wbindgen_export_3(t,n,1)}}const sr=typeof FinalizationRegistry>`u`?{register:()=>{},unregister:()=>{}}:new FinalizationRegistry(e=>V.__wbg_tiktoken_free(e>>>0,1));var cr=class e{static __wrap(t){t>>>=0;let n=Object.create(e.prototype);return n.__wbg_ptr=t,sr.register(n,n.__wbg_ptr,n),n}__destroy_into_raw(){let e=this.__wbg_ptr;return this.__wbg_ptr=0,sr.unregister(this),e}free(){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let e=this.__destroy_into_raw();V.__wbg_tiktoken_free(e,0)}constructor(e,t,n){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let r=Z(e,V.__wbindgen_export_1,V.__wbindgen_export_2),i=Y,a=Z(n,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y,s=V.tiktoken_new(r,i,q(t),a,o);return this.__wbg_ptr=s>>>0,sr.register(this,this.__wbg_ptr,this),this}get name(){try{let n=V.__wbindgen_add_to_stack_pointer(-16);V.tiktoken_name(n,this.__wbg_ptr);var e=$().getInt32(n+0,!0),t=$().getInt32(n+4,!0);let r;return e!==0&&(r=W(e,t).slice(),V.__wbindgen_export_3(e,t*1,1)),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode(e,t,n){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let s=V.__wbindgen_add_to_stack_pointer(-16),c=Z(e,V.__wbindgen_export_1,V.__wbindgen_export_2),l=Y;V.tiktoken_encode(s,this.__wbg_ptr,c,l,q(t),q(n));var r=$().getInt32(s+0,!0),i=$().getInt32(s+4,!0),a=$().getInt32(s+8,!0);if($().getInt32(s+12,!0))throw J(a);var o=er(r,i).slice();return V.__wbindgen_export_3(r,i*4,4),o}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode_ordinary(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=Z(e,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y;V.tiktoken_encode_ordinary(i,this.__wbg_ptr,a,o);var t=$().getInt32(i+0,!0),n=$().getInt32(i+4,!0),r=er(t,n).slice();return V.__wbindgen_export_3(t,n*4,4),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode_with_unstable(e,t,n){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let a=V.__wbindgen_add_to_stack_pointer(-16),o=Z(e,V.__wbindgen_export_1,V.__wbindgen_export_2),s=Y;V.tiktoken_encode_with_unstable(a,this.__wbg_ptr,o,s,q(t),q(n));var r=$().getInt32(a+0,!0),i=$().getInt32(a+4,!0);if($().getInt32(a+8,!0))throw J(i);return J(r)}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode_single_token(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let t=tr(e,V.__wbindgen_export_1),n=Y;return V.tiktoken_encode_single_token(this.__wbg_ptr,t,n)>>>0}decode(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=nr(e,V.__wbindgen_export_1),o=Y;V.tiktoken_decode(i,this.__wbg_ptr,a,o);var t=$().getInt32(i+0,!0),n=$().getInt32(i+4,!0),r=rr(t,n).slice();return V.__wbindgen_export_3(t,n*1,1),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}decode_single_token_bytes(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16);V.tiktoken_decode_single_token_bytes(i,this.__wbg_ptr,e);var t=$().getInt32(i+0,!0),n=$().getInt32(i+4,!0),r=rr(t,n).slice();return V.__wbindgen_export_3(t,n*1,1),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}token_byte_values(){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);return J(V.tiktoken_token_byte_values(this.__wbg_ptr))}};function lr(){return qn(function(e,t){return q(JSON.parse(W(e,t)))},arguments)}function ur(){return qn(function(e){return q(JSON.stringify(Jn(e)))},arguments)}function dr(e,t){return q(Error(W(e,t)))}function fr(e){return Jn(e)===void 0}function pr(e){J(e)}function mr(e,t){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let n=Jn(t),r=typeof n==`string`?n:void 0;var i=Zn(r)?0:Z(r,V.__wbindgen_export_1,V.__wbindgen_export_2),a=Y;$().setInt32(e+4,a,!0),$().setInt32(e+0,i,!0)}function hr(e,t){throw Error(W(e,t))}let gr=!1;async function _r(e){if(gr)return Wn;let t=await e({"./tiktoken_bg.js":Wn}),n=`instance`in t&&t.instance instanceof WebAssembly.Instance?t.instance:t instanceof WebAssembly.Instance?t:null;if(n==null)throw Error(`Missing instance`);return Gn(n.exports),gr=!0,Wn}const vr=new URL(`../tiktoken_bg.wasm`,import.meta.url).href;await(async()=>{await _r(async e=>{let t=await fetch(vr);if(!t.ok)throw Error(`Failed to load tiktoken WASM from ${JSON.stringify(vr)}: ${t.status} ${t.statusText}`);if(typeof WebAssembly.instantiateStreaming==`function`)try{return await WebAssembly.instantiateStreaming(t.clone(),e)}catch{}return WebAssembly.instantiate(await t.arrayBuffer(),e)})})();var yr=class extends st{modelId;constructor(e){super(),this.modelId=e}createState(){return ir(Vn(k(this.modelId,`config.msgpack`),`encoding`))}encodeWithState(e,t){return[...t.encode(e)]}},br=class extends st{modelId;constructor(e){super(),this.modelId=e}createState(){let e=k(this.modelId,`config.msgpack`);return new cr(Qe(this.modelId,`tiktoken.model`),B(Vn(e,`special_tokens`)),Vn(e,`pat_str`))}encodeWithState(e,t){return[...t.encode(e)]}};const xr=new Map,Sr=e=>{let t=xr.get(e);if(t)return t;let n=ze[e],r;switch(n.kind){case`tiktoken-builtin`:r=new yr(e);break;case`tiktoken-custom`:r=new br(e);break;case`huggingface`:r=new Un(e);break;case`clip-bpe`:r=new ht(e);break}return xr.set(e,r),r},Cr=e=>{if(typeof e==`string`){let[t]=ot(e);return{isSingleModel:!0,models:[t]}}let t=e?.model;return{isSingleModel:typeof t==`string`,models:ot(t)}},wr=(e,t)=>{let{isSingleModel:n,models:r}=Cr(t);if(n){let[t]=r;return Sr(t).encode(e)}return Object.fromEntries(r.map(t=>[t,Sr(t).encode(e)]))};function Tr(e,t){return wr(e,t)}function Er(e,t){let n=wr(e,t);return Array.isArray(n)?n.length:Object.fromEntries(Object.entries(n).map(([e,t])=>[e,t.length]))}const Dr={deepseek:()=>import(`./deepseek.js`),gemma:()=>import(`./gemma.js`),glm:()=>import(`./glm.js`),gpt:()=>import(`./gpt.js`),kimi:()=>import(`./kimi.js`),mimo:()=>import(`./mimo.js`),minimax:()=>import(`./minimax.js`),qwen:()=>import(`./qwen.js`),sdxl:()=>import(`./sdxl.js`)},Or=e=>Ye(e),kr=()=>$e(),Ar=async e=>(Or(e)||Xe(e,await at((await Dr[e]()).default)),e),jr=async e=>{let t=ot(e);return await Promise.all(t.map(e=>Ar(e))),t};export{Er as a,Ze as c,jr as i,T as l,Or as n,Tr as o,Ar as r,at as s,kr as t,ze as u};
+import{n as e,r as t}from"./chunk.js";var n;try{n=new TextDecoder}catch{}var r,i,a=0;const o=[];var s=o,c=0,l={},u,d,f=0,p=0,m,h,g=[],_,ee={useRecords:!1,mapsAsObjects:!0};const te=new class{};te.name=`MessagePack 0xC1`;var v=!1,ne=2,y=class e{constructor(e){e&&(e.useRecords===!1&&e.mapsAsObjects===void 0&&(e.mapsAsObjects=!0),e.sequential&&e.trusted!==!1&&(e.trusted=!0,!e.structures&&e.useRecords!=0&&(e.structures=[],e.maxSharedStructures||=0)),e.structures?e.structures.sharedLength=e.structures.length:e.getStructures&&((e.structures=[]).uninitialized=!0,e.structures.sharedLength=0),e.int64AsNumber&&(e.int64AsType=`number`)),Object.assign(this,e)}unpack(t,n){if(r)return Ee(()=>(De(),this?this.unpack(t,n):e.prototype.unpack.call(ee,t,n)));!t.buffer&&t.constructor===ArrayBuffer&&(t=typeof Buffer<`u`?Buffer.from(t):new Uint8Array(t)),typeof n==`object`?(i=n.end||t.length,a=n.start||0):(a=0,i=n>-1?n:t.length),c=0,p=0,d=null,s=o,m=null,r=t;try{_=t.dataView||=new DataView(t.buffer,t.byteOffset,t.byteLength)}catch(e){throw r=null,t instanceof Uint8Array?e:Error(`Source must be a Uint8Array or Buffer but was a `+(t&&typeof t==`object`?t.constructor.name:typeof t))}if(this instanceof e){if(l=this,this.structures)return u=this.structures,b(n);(!u||u.length>0)&&(u=[])}else l=ee,(!u||u.length>0)&&(u=[]);return b(n)}unpackMultiple(e,t){let n,r=0;try{v=!0;let i=e.length,o=this?this.unpack(e,i):T.unpack(e,i);if(t){if(t(o,r,a)===!1)return;for(;a<i;)if(r=a,t(b(),r,a)===!1)return}else{for(n=[o];a<i;)r=a,n.push(b());return n}}catch(e){throw e.lastPosition=r,e.values=n,e}finally{v=!1,De()}}_mergeStructures(e,t){this._onLoadedStructures&&(e=this._onLoadedStructures(e)),e||=[],Object.isFrozen(e)&&(e=e.map(e=>e.slice(0)));for(let t=0,n=e.length;t<n;t++){let n=e[t];n&&(n.isShared=!0,t>=32&&(n.highByte=t-32>>5))}e.sharedLength=e.length;for(let n in t||[])if(n>=0){let r=e[n],i=t[n];i&&(r&&((e.restoreStructures||=[])[n]=r),e[n]=i)}return this.structures=e}decode(e,t){return this.unpack(e,t)}};function b(e){try{if(!l.trusted&&!v){let e=u.sharedLength||0;e<u.length&&(u.length=e)}let t;if(l._readStruct&&r[a]<64&&r[a]>=32?(t=l._readStruct(r,a,i),r=null,!(e&&e.lazy)&&t&&(t=t.toJSON()),a=i):t=x(),m&&=(a=m.postBundlePosition,null),v&&(u.restoreStructures=null),a==i)u&&u.restoreStructures&&re(),u=null,r=null,h&&=null;else if(a>i)throw Error(`Unexpected end of MessagePack data`);else if(!v){let e;try{e=JSON.stringify(t,(e,t)=>typeof t==`bigint`?`${t}n`:t).slice(0,100)}catch(t){e=`(JSON view not available `+t+`)`}throw Error(`Data read, but end of buffer not reached `+e)}return t}catch(e){throw u&&u.restoreStructures&&re(),De(),(e instanceof RangeError||e.message.startsWith(`Unexpected end of buffer`)||a>i)&&(e.incomplete=!0),e}}function re(){for(let e in u.restoreStructures)u[e]=u.restoreStructures[e];u.restoreStructures=null}function x(){let e=r[a++];if(e<160)if(e<128){if(e<64)return e;{let t=u[e&63]||l.getStructures&&se()[e&63];return t?(t.read||=ae(t,e&63),t.read()):e}}else if(e<144)if(e-=128,l.mapsAsObjects){let t={};for(let n=0;n<e;n++){let e=ye();e===`__proto__`&&(e=`__proto_`),t[e]=x()}return t}else{let t=new Map;for(let n=0;n<e;n++)t.set(x(),x());return t}else{e-=144;let t=Array(e);for(let n=0;n<e;n++)t[n]=x();return l.freezeData?Object.freeze(t):t}else if(e<192){let t=e-160;if(p>=a)return d.slice(a-f,(a+=t)-f);if(p==0&&i<140){let e=t<16?he(t):me(t);if(e!=null)return e}return ce(t)}else{let t;switch(e){case 192:return null;case 193:return m?(t=x(),t>0?m[1].slice(m.position1,m.position1+=t):m[0].slice(m.position0,m.position0-=t)):te;case 194:return!1;case 195:return!0;case 196:if(t=r[a++],t===void 0)throw Error(`Unexpected end of buffer`);return _e(t);case 197:return t=_.getUint16(a),a+=2,_e(t);case 198:return t=_.getUint32(a),a+=4,_e(t);case 199:return w(r[a++]);case 200:return t=_.getUint16(a),a+=2,w(t);case 201:return t=_.getUint32(a),a+=4,w(t);case 202:if(t=_.getFloat32(a),l.useFloat32>2){let e=Oe[(r[a]&127)<<1|r[a+1]>>7];return a+=4,(e*t+(t>0?.5:-.5)>>0)/e}return a+=4,t;case 203:return t=_.getFloat64(a),a+=8,t;case 204:return r[a++];case 205:return t=_.getUint16(a),a+=2,t;case 206:return t=_.getUint32(a),a+=4,t;case 207:return l.int64AsType===`number`?(t=_.getUint32(a)*4294967296,t+=_.getUint32(a+4)):l.int64AsType===`string`?t=_.getBigUint64(a).toString():l.int64AsType===`auto`?(t=_.getBigUint64(a),t<=BigInt(2)<<BigInt(52)&&(t=Number(t))):t=_.getBigUint64(a),a+=8,t;case 208:return _.getInt8(a++);case 209:return t=_.getInt16(a),a+=2,t;case 210:return t=_.getInt32(a),a+=4,t;case 211:return l.int64AsType===`number`?(t=_.getInt32(a)*4294967296,t+=_.getUint32(a+4)):l.int64AsType===`string`?t=_.getBigInt64(a).toString():l.int64AsType===`auto`?(t=_.getBigInt64(a),t>=BigInt(-2)<<BigInt(52)&&t<=BigInt(2)<<BigInt(52)&&(t=Number(t))):t=_.getBigInt64(a),a+=8,t;case 212:if(t=r[a++],t==114)return xe(r[a++]&63);{let e=g[t];if(e)return e.read?(a++,e.read(x())):e.noBuffer?(a++,e()):e(r.subarray(a,++a));throw Error(`Unknown extension `+t)}case 213:return t=r[a],t==114?(a++,xe(r[a++]&63,r[a++])):w(2);case 214:return w(4);case 215:return w(8);case 216:return w(16);case 217:return t=r[a++],p>=a?d.slice(a-f,(a+=t)-f):le(t);case 218:return t=_.getUint16(a),a+=2,p>=a?d.slice(a-f,(a+=t)-f):ue(t);case 219:return t=_.getUint32(a),a+=4,p>=a?d.slice(a-f,(a+=t)-f):de(t);case 220:return t=_.getUint16(a),a+=2,fe(t);case 221:return t=_.getUint32(a),a+=4,fe(t);case 222:return t=_.getUint16(a),a+=2,pe(t);case 223:return t=_.getUint32(a),a+=4,pe(t);default:if(e>=224)return e-256;if(e===void 0){let e=Error(`Unexpected end of MessagePack data`);throw e.incomplete=!0,e}throw Error(`Unknown MessagePack token `+e)}}}const ie=/^[a-zA-Z_$][a-zA-Z\d_$]*$/;function ae(e,t){function n(){if(n.count++>ne){let r;try{r=e.read=Function(`r`,`return function(){return `+(l.freezeData?`Object.freeze`:``)+`({`+e.map(e=>e===`__proto__`?`__proto_:r()`:ie.test(e)?e+`:r()`:`[`+JSON.stringify(e)+`]:r()`).join(`,`)+`})}`)(x)}catch{return ne=1/0,n()}return e.highByte===0&&(e.read=oe(t,e.read)),r()}let r={};for(let t=0,n=e.length;t<n;t++){let n=e[t];n===`__proto__`&&(n=`__proto_`),r[n]=x()}return l.freezeData?Object.freeze(r):r}return n.count=0,e.highByte===0?oe(t,n):n}const oe=(e,t)=>function(){let n=r[a++];if(n===0)return t();let i=e<32?-(e+(n<<5)):e+(n<<5),o=u[i]||se()[i];if(!o)throw Error(`Record id is not defined for `+i);return o.read||=ae(o,e),o.read()};function se(){let e=Ee(()=>(r=null,l.getStructures()));return u=l._mergeStructures(e,u)}var ce=S,le=S,ue=S,de=S;function S(e){let t;if(e<16&&(t=he(e)))return t;if(e>64&&n)return n.decode(r.subarray(a,a+=e));let i=a+e,o=[];for(t=``;a<i;){let e=r[a++];if(!(e&128))o.push(e);else if((e&224)==192){let t=r[a++]&63,n=(e&31)<<6|t;n<128?o.push(65533):o.push(n)}else if((e&240)==224){let t=r[a++]&63,n=r[a++]&63,i=(e&31)<<12|t<<6|n;i<2048||i>=55296&&i<=57343?o.push(65533):o.push(i)}else if((e&248)==240){let t=r[a++]&63,n=r[a++]&63,i=r[a++]&63,s=(e&7)<<18|t<<12|n<<6|i;s<65536||s>1114111?o.push(65533):s>65535?(s-=65536,o.push(s>>>10&1023|55296),s=56320|s&1023,o.push(s)):o.push(s)}else o.push(65533);o.length>=4096&&(t+=C.apply(String,o),o.length=0)}return o.length>0&&(t+=C.apply(String,o)),t}function fe(e){let t=Array(e);for(let n=0;n<e;n++)t[n]=x();return l.freezeData?Object.freeze(t):t}function pe(e){if(l.mapsAsObjects){let t={};for(let n=0;n<e;n++){let e=ye();e===`__proto__`&&(e=`__proto_`),t[e]=x()}return t}else{let t=new Map;for(let n=0;n<e;n++)t.set(x(),x());return t}}var C=String.fromCharCode;function me(e){let t=a,n=Array(e);for(let i=0;i<e;i++){let e=r[a++];if((e&128)>0){a=t;return}n[i]=e}return C.apply(String,n)}function he(e){if(e<4)if(e<2){if(e===0)return``;{let e=r[a++];if((e&128)>1){--a;return}return C(e)}}else{let t=r[a++],n=r[a++];if((t&128)>0||(n&128)>0){a-=2;return}if(e<3)return C(t,n);let i=r[a++];if((i&128)>0){a-=3;return}return C(t,n,i)}else{let t=r[a++],n=r[a++],i=r[a++],o=r[a++];if((t&128)>0||(n&128)>0||(i&128)>0||(o&128)>0){a-=4;return}if(e<6){if(e===4)return C(t,n,i,o);{let e=r[a++];if((e&128)>0){a-=5;return}return C(t,n,i,o,e)}}else if(e<8){let s=r[a++],c=r[a++];if((s&128)>0||(c&128)>0){a-=6;return}if(e<7)return C(t,n,i,o,s,c);let l=r[a++];if((l&128)>0){a-=7;return}return C(t,n,i,o,s,c,l)}else{let s=r[a++],c=r[a++],l=r[a++],u=r[a++];if((s&128)>0||(c&128)>0||(l&128)>0||(u&128)>0){a-=8;return}if(e<10){if(e===8)return C(t,n,i,o,s,c,l,u);{let e=r[a++];if((e&128)>0){a-=9;return}return C(t,n,i,o,s,c,l,u,e)}}else if(e<12){let d=r[a++],f=r[a++];if((d&128)>0||(f&128)>0){a-=10;return}if(e<11)return C(t,n,i,o,s,c,l,u,d,f);let p=r[a++];if((p&128)>0){a-=11;return}return C(t,n,i,o,s,c,l,u,d,f,p)}else{let d=r[a++],f=r[a++],p=r[a++],m=r[a++];if((d&128)>0||(f&128)>0||(p&128)>0||(m&128)>0){a-=12;return}if(e<14){if(e===12)return C(t,n,i,o,s,c,l,u,d,f,p,m);{let e=r[a++];if((e&128)>0){a-=13;return}return C(t,n,i,o,s,c,l,u,d,f,p,m,e)}}else{let h=r[a++],g=r[a++];if((h&128)>0||(g&128)>0){a-=14;return}if(e<15)return C(t,n,i,o,s,c,l,u,d,f,p,m,h,g);let _=r[a++];if((_&128)>0){a-=15;return}return C(t,n,i,o,s,c,l,u,d,f,p,m,h,g,_)}}}}}function ge(){let e=r[a++],t;if(e<192)t=e-160;else switch(e){case 217:t=r[a++];break;case 218:t=_.getUint16(a),a+=2;break;case 219:t=_.getUint32(a),a+=4;break;default:throw Error(`Expected string`)}return S(t)}function _e(e){return l.copyBuffers?Uint8Array.prototype.slice.call(r,a,a+=e):r.subarray(a,a+=e)}function w(e){let t=r[a++];if(g[t]){let n;return g[t](r.subarray(a,n=a+=e),e=>{a=e;try{return x()}finally{a=n}})}else throw Error(`Unknown extension type `+t)}var ve=Array(4096);function ye(){let e=r[a++];if(e>=160&&e<192){if(e-=160,p>=a)return d.slice(a-f,(a+=e)-f);if(!(p==0&&i<180))return ce(e)}else return a--,be(x());let t=(e<<5^(e>1?_.getUint16(a):e>0?r[a]:0))&4095,n=ve[t],o=a,s=a+e-3,c,l=0;if(n&&n.bytes==e){for(;o<s;){if(c=_.getUint32(o),c!=n[l++]){o=1879048192;break}o+=4}for(s+=3;o<s;)if(c=r[o++],c!=n[l++]){o=1879048192;break}if(o===s)return a=o,n.string;s-=3,o=a}for(n=[],ve[t]=n,n.bytes=e;o<s;)c=_.getUint32(o),n.push(c),o+=4;for(s+=3;o<s;)c=r[o++],n.push(c);let u=e<16?he(e):me(e);return u==null?n.string=ce(e):n.string=u}function be(e){if(typeof e==`string`)return e;if(typeof e==`number`||typeof e==`boolean`||typeof e==`bigint`)return e.toString();if(e==null)return e+``;if(l.allowArraysInMapKeys&&Array.isArray(e)&&e.flat().every(e=>[`string`,`number`,`boolean`,`bigint`].includes(typeof e)))return e.flat().toString();throw Error(`Invalid property type for record: ${typeof e}`)}const xe=(e,t)=>{let n=x().map(be),r=e;t!==void 0&&(e=e<32?-((t<<5)+e):(t<<5)+e,n.highByte=t);let i=u[e];return i&&(i.isShared||v)&&((u.restoreStructures||=[])[e]=i),u[e]=n,n.read=ae(n,r),n.read()};g[0]=()=>{},g[0].noBuffer=!0,g[66]=e=>{let t=e.byteLength%8||8,n=BigInt(e[0]&128?e[0]-256:e[0]);for(let r=1;r<t;r++)n<<=BigInt(8),n+=BigInt(e[r]);if(e.byteLength!==t){let r=new DataView(e.buffer,e.byteOffset,e.byteLength),i=(e,t)=>{let n=t-e;if(n<=40){let n=r.getBigUint64(e);for(let i=e+8;i<t;i+=8)n<<=BigInt(64),n|=r.getBigUint64(i);return n}let a=e+(n>>4<<3),o=i(e,a),s=i(a,t);return o<<BigInt((t-a)*8)|s};n=n<<BigInt((r.byteLength-t)*8)|i(t,r.byteLength)}return n};let Se={Error,EvalError,RangeError,ReferenceError,SyntaxError,TypeError,URIError,AggregateError:typeof AggregateError==`function`?AggregateError:null};g[101]=()=>{let e=x();if(!Se[e[0]]){let t=Error(e[1],{cause:e[2]});return t.name=e[0],t}return Se[e[0]](e[1],{cause:e[2]})},g[105]=e=>{if(l.structuredClone===!1)throw Error(`Structured clone extension is disabled`);let t=_.getUint32(a-4);h||=new Map;let n=r[a],i;i=n>=144&&n<160||n==220||n==221?[]:n>=128&&n<144||n==222||n==223?new Map:(n>=199&&n<=201||n>=212&&n<=216)&&r[a+1]===115?new Set:{};let o={target:i};h.set(t,o);let s=x();if(o.used)Object.assign(i,s);else return o.target=s;if(i instanceof Map)for(let[e,t]of s.entries())i.set(e,t);if(i instanceof Set)for(let e of Array.from(s))i.add(e);return i},g[112]=e=>{if(l.structuredClone===!1)throw Error(`Structured clone extension is disabled`);let t=_.getUint32(a-4),n=h.get(t);return n.used=!0,n.target},g[115]=()=>new Set(x());const Ce=[`Int8`,`Uint8`,`Uint8Clamped`,`Int16`,`Uint16`,`Int32`,`Uint32`,`Float32`,`Float64`,`BigInt64`,`BigUint64`].map(e=>e+`Array`);let we=typeof globalThis==`object`?globalThis:window;g[116]=e=>{let t=e[0],n=Uint8Array.prototype.slice.call(e,1).buffer,r=Ce[t];if(!r){if(t===16)return n;if(t===17)return new DataView(n);throw Error(`Could not find typed array for code `+t)}return new we[r](n)},g[120]=()=>{let e=x();return new RegExp(e[0],e[1])};const Te=[];g[98]=e=>{let t=(e[0]<<24)+(e[1]<<16)+(e[2]<<8)+e[3],n=a;return a+=t-e.length,m=Te,m=[ge(),ge()],m.position0=0,m.position1=0,m.postBundlePosition=a,a=n,x()},g[255]=e=>e.length==4?new Date((e[0]*16777216+(e[1]<<16)+(e[2]<<8)+e[3])*1e3):e.length==8?new Date(((e[0]<<22)+(e[1]<<14)+(e[2]<<6)+(e[3]>>2))/1e6+((e[3]&3)*4294967296+e[4]*16777216+(e[5]<<16)+(e[6]<<8)+e[7])*1e3):e.length==12?new Date(((e[0]<<24)+(e[1]<<16)+(e[2]<<8)+e[3])/1e6+((e[4]&128?-281474976710656:0)+e[6]*1099511627776+e[7]*4294967296+e[8]*16777216+(e[9]<<16)+(e[10]<<8)+e[11])*1e3):new Date(`invalid`);function Ee(e){l&&l._onSaveState&&l._onSaveState();let t=i,n=a,o=c,g=f,ee=p,te=d,ne=s,y=h,b=m,re=new Uint8Array(r.slice(0,i)),x=u,ie=u.slice(0,u.length),ae=l,oe=v,se=e();return i=t,a=n,c=o,f=g,p=ee,d=te,s=ne,h=y,m=b,r=re,v=oe,u=x,u.splice(0,u.length,...ie),l=ae,_=new DataView(r.buffer,r.byteOffset,r.byteLength),se}function De(){r=null,h=null,u=null}const Oe=Array(147);for(let e=0;e<256;e++)Oe[e]=+(`1e`+Math.floor(45.15-e*.30103));var T=new y({useRecords:!1});T.unpack,T.unpackMultiple,T.unpack,new Uint8Array(new Float32Array(1).buffer,0,4),y.SUPPORTS_STRUCT_HOOKS=!0;const ke={gpt:{encoding:`o200k_base`,kind:`tiktoken-builtin`,openrouter:`openai/gpt-5.5`,source:{encodingJsonUrl:`https://tiktoken.pages.dev/js/o200k_base.json`},title:`GPT-5.5`},gemma:{kind:`huggingface`,openrouter:`google/gemma-4-31b-it`,source:{tokenizerConfigUrl:`https://huggingface.co/google/gemma-4-31B-it/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/google/gemma-4-31B-it/resolve/main/tokenizer.json`},title:`Gemma 4 31B it`},qwen:{kind:`huggingface`,openrouter:`qwen/qwen3.6-27b`,source:{tokenizerConfigUrl:`https://huggingface.co/Qwen/Qwen3.6-27B/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/Qwen/Qwen3.6-27B/resolve/main/tokenizer.json`},title:`Qwen 3.6 27B`},kimi:{kind:`tiktoken-custom`,openrouter:`moonshotai/kimi-k2.6`,source:{modelUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tiktoken.model`,tokenizerConfigUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tokenizer_config.json`,tokenizerImplementationUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tokenization_kimi.py`},title:`Kimi K2.6`},deepseek:{kind:`huggingface`,openrouter:`deepseek/deepseek-v4-pro`,source:{tokenizerConfigUrl:`https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/main/tokenizer.json`},title:`DeepSeek V4 Pro`},mimo:{kind:`huggingface`,openrouter:`xiaomi/mimo-v2.5-pro`,source:{tokenizerConfigUrl:`https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/main/tokenizer.json`},title:`MiMo V2.5 Pro`},sdxl:{kind:`clip-bpe`,source:{mergesUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/merges.txt`,specialTokensMapUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/special_tokens_map.json`,tokenizerConfigUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/tokenizer_config.json`,vocabUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/vocab.json`},title:`Stable Diffusion XL`},glm:{kind:`huggingface`,openrouter:`zai-org/glm-5.1`,source:{tokenizerConfigUrl:`https://huggingface.co/zai-org/GLM-5.1/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/zai-org/GLM-5.1/resolve/main/tokenizer.json`},title:`GLM 5.1`},minimax:{kind:`huggingface`,openrouter:`minimax/minimax-m2.7`,source:{tokenizerConfigUrl:`https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/main/tokenizer.json`},title:`MiniMax M2.7`},hy:{kind:`huggingface`,openrouter:`tencent/hy3-preview`,title:`Hy3 Preview`,source:{tokenizerConfigUrl:`https://huggingface.co/tencent/Hy3-preview/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/tencent/Hy3-preview/resolve/main/tokenizer.json`}},step:{kind:`huggingface`,openrouter:`stepfun/step-3.7-flash`,source:{specialTokensMapUrl:`https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/special_tokens_map.json`,tokenizerConfigUrl:`https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/tokenizer.json`},title:`Step 3.7 Flash`}},E=Object.keys(ke),Ae=new y({mapsAsObjects:!1}),je=new TextDecoder,D=Object.create(null),O=new Map,Me=Object.create(null),k=new Map,A=new Map;let Ne;const Pe=(e,t)=>`${e}/${t}`,Fe=e=>Me[e]??0,Ie=e=>{let t=Fe(e)+1;return Me[e]=t,t},Le=e=>{let t=`${e}/`;for(let e of A.keys())e.startsWith(t)&&A.delete(e);for(let e of k.keys())e.startsWith(t)&&k.delete(e)},Re=e=>{let t=D[e];if(!t)throw Error(`Missing tokenizer assets for model ${JSON.stringify(e)}. Call load() first or use tokenize()/count() to auto-load it.`);return t},ze=()=>{if(!Ne)throw Error(`No tokenizer asset loader is registered for the current entry.`);return Ne},Be=(e,t)=>{let n=Re(e);if(!Object.hasOwn(n,t))throw Error(`Missing tokenizer asset ${JSON.stringify(t)} for model ${JSON.stringify(e)}. Run “bun run fetch” first.`);return n[t]},Ve=(e,t)=>Be(e,t),He=e=>{if(e.endsWith(`.msgpack`))return e;if(e.endsWith(`.json`))return`${e.slice(0,-5)}.msgpack`;throw TypeError(`Expected a MessagePack or JSON file name, got ${JSON.stringify(e)}.`)},Ue=e=>Object.hasOwn(D,e),We=e=>{Ne=e},Ge=(e,t)=>{D[e]=t,Le(e)},Ke=async e=>{await Promise.all(e.map(async e=>{if(Ue(e))return;let t=O.get(e);if(t){await t;return}let n=Fe(e),r=(async()=>{let t=await ze().load(e);Fe(e)===n&&Ge(e,t)})().finally(()=>{O.get(e)===r&&O.delete(e)});O.set(e,r),await r}))},qe=e=>{Ie(e),delete D[e],O.delete(e),Le(e)},Je=e=>{if(e){qe(e);return}for(let e of E)qe(e)},Ye=(e,t)=>{if(t.endsWith(`.json`)||t.endsWith(`.msgpack`))throw TypeError(`Cannot read structured tokenizer asset ${JSON.stringify(t)} as text.`);let n=Pe(e,t),r=A.get(n);if(r!==void 0)return r;let i=je.decode(Ve(e,t));return A.set(n,i),i},j=(e,t)=>{let n=He(t),r=Pe(e,n),i=k.get(r);if(i!==void 0)return i;let a=Ae.unpack(Ve(e,n));return k.set(r,a),a},Xe={gpt:new URL(`../gpt.bin`,import.meta.url),gemma:new URL(`../gemma.bin`,import.meta.url),qwen:new URL(`../qwen.bin`,import.meta.url),kimi:new URL(`../kimi.bin`,import.meta.url),deepseek:new URL(`../deepseek.bin`,import.meta.url),mimo:new URL(`../mimo.bin`,import.meta.url),sdxl:new URL(`../sdxl.bin`,import.meta.url),glm:new URL(`../glm.bin`,import.meta.url),minimax:new URL(`../minimax.bin`,import.meta.url),hy:new URL(`../hy.bin`,import.meta.url),step:new URL(`../step.bin`,import.meta.url)};let Ze;const Qe=()=>(Ze??=(async()=>(await import(`./decompress.js`).then(e=>t(e.default,1))).default)(),Ze),$e=async e=>{if(typeof DecompressionStream==`function`)try{let t=new Blob([e]).stream().pipeThrough(new DecompressionStream(`brotli`));return new Uint8Array(await new Response(t).arrayBuffer())}catch{}return(await Qe())(e)},et=new y({mapsAsObjects:!1}),tt=e=>{if(e instanceof Uint8Array)return e;if(e instanceof ArrayBuffer)return new Uint8Array(e);if(ArrayBuffer.isView(e))return new Uint8Array(e.buffer,e.byteOffset,e.byteLength);throw TypeError(`Expected bundled tokenizer asset contents to be binary, got ${typeof e}.`)},nt=e=>{if(!(e instanceof Map))throw TypeError(`Expected a tokenizer asset bundle to unpack into a Map.`);let t=Object.create(null);for(let[n,r]of e.entries()){if(typeof n!=`string`)throw TypeError(`Expected tokenizer asset file names to be strings, got ${typeof n}.`);t[n]=tt(r)}return t};var rt=class{async load(e){let t=await $e(await this.readCompressedBundle(e));return nt(et.unpack(t))}};We(new class extends rt{bundleUrls;constructor(e){super(),this.bundleUrls=e}async readCompressedBundle(e){let t=this.bundleUrls[e],n=await fetch(t);if(!n.ok)throw Error(`Failed to fetch tokenizer assets for model ${JSON.stringify(e)} from ${JSON.stringify(t.href)}: ${n.status} ${n.statusText}`);return new Uint8Array(await n.arrayBuffer())}}(Xe));const it=e=>{let t=[...E];e!==void 0&&(t.length=0,typeof e==`string`?t.push(e):t.push(...e));for(let e of t)if(!E.includes(e))throw Error(`Unknown model ${JSON.stringify(e)}. Supported models: ${E.join(`, `)}`);return t},at=new TextDecoder(`utf-8`,{fatal:!0}),ot=new TextEncoder,st=e=>{let t=[],n=0;for(let r of e)t.push(n),n+=r;return t},ct=e=>{if(typeof e==`string`)return e;try{return at.decode(e)}catch{throw TypeError(`Tokenizer input bytes must be valid UTF-8.`)}},lt=e=>ot.encode(e).byteLength,ut=(e,t,n)=>{if(e.length!==t.length)throw TypeError(`Expected ${e.length} token start offsets, got ${t.length}.`);let r={offsets:t.slice(1),tokens:[...e]};return n===void 0?r:{...r,processedInput:n}},dt=(e,t,n)=>{if(e.length!==t.length)throw TypeError(`Expected ${e.length} token byte lengths, got ${t.length}.`);return ut(e,st(t),n)},ft=new TextEncoder;var pt=class{#e=!1;#t;disposeState(e){}encode(e){return this.encodeWithState(ct(e),this.getState())}free(){this.#e&&=(this.disposeState(this.#t),this.#t=void 0,!1)}getState(){return this.#e||=(this.#t=this.createState(),!0),this.#t}getTokenCount(e){return this.encode(e).length}tokenize(e){let t=this.tokenizeWithState(ct(e),this.getState());return typeof e==`string`||t.processedInput===void 0?t:{...t,processedInput:ft.encode(t.processedInput)}}};const mt=/<\|startoftext\|>|<\|endoftext\|>|'d|'ll|'m|'re|'s|'t|'ve|\p{L}+|\p{N}|[^\s\p{L}\p{N}]+/gu,ht=/<\/w>$/u,gt=/\s+/gu,_t=new TextEncoder,vt=e=>{if(typeof e==`string`)return e;if(e instanceof Map){let t=e.get(`content`);if(typeof t==`string`)return t}if(e&&typeof e==`object`&&`content`in e&&typeof e.content==`string`)return e.content},yt=()=>{let e=Array.from({length:256},()=>``),t=[...Array.from({length:94},(e,t)=>t+33),...Array.from({length:12},(e,t)=>t+161),...Array.from({length:82},(e,t)=>t+174)],n=[...t],r=256;for(let e=0;e<256;e++)t.includes(e)||(t.push(e),n.push(r),r++);for(let[r,i]of t.entries())e[i]=String.fromCodePoint(n[r]);return e},bt=e=>{let t=new Map;for(let[n,r]of e.split(/\r?\n/u).filter(Boolean).filter(e=>!e.startsWith(`#`)).entries())t.set(r,n);return t},xt=e=>e.normalize(`NFC`).replaceAll(gt,` `).trim().toLowerCase(),St=e=>[...e.replace(ht,``)].length,Ct=(e,t,n)=>n===t?e:{...e,processedInput:n};var wt=class extends pt{modelId;#e=new Map;constructor(e){super(),this.modelId=e}createState(){let e=j(this.modelId,`vocab.msgpack`),t=j(this.modelId,`tokenizer_config.msgpack`),n=new Map,r=vt(t.get(`unk_token`))??`<|endoftext|>`,i=e.get(r);if(i===void 0)throw Error(`Could not find CLIP unknown token ${JSON.stringify(r)} in ${JSON.stringify(this.modelId)} vocabulary.`);for(let r of[t.get(`bos_token`),t.get(`eos_token`),t.get(`unk_token`)]){let t=vt(r);if(!t)continue;let i=e.get(t);i!==void 0&&n.set(t,i)}return{byteEncoder:yt(),mergeRanks:bt(Ye(this.modelId,`merges.txt`)),specialTokenIds:n,unknownTokenId:i,vocabulary:e}}encodeWithState(e,t){return this.tokenizeWithState(e,t).tokens}tokenizeWithState(e,t){let n=xt(e);if(!n)return Ct({offsets:[],tokens:[]},e,n);let r=[],i=[],a=0,o=0;for(let e of n.matchAll(mt)){let s=e[0],c=e.index;a+=lt(n.slice(o,c));let l=a,u=t.specialTokenIds.get(s);if(u!==void 0){r.push(u),i.push(l),a+=lt(s),o=c+s.length;continue}let d=Array.from(_t.encode(s),e=>t.byteEncoder[e]).join(``),f=0;for(let e of this.#t(d,t.mergeRanks))r.push(t.vocabulary.get(e)??t.unknownTokenId),i.push(l+f),f+=St(e);a+=lt(s),o=c+s.length}return Ct(ut(r,i),e,n)}#t(e,t){let n=this.#e.get(e);if(n)return n;let r=[...e];if(!r.length){let t=[];return this.#e.set(e,t),t}let i=[...r.slice(0,-1),`${r.at(-1)}</w>`];for(;i.length>1;){let e=-1,n=1/0;for(let r=0;r<i.length-1;r++){let a=t.get(`${i[r]} ${i[r+1]}`);a!==void 0&&a<n&&(n=a,e=r)}if(e<0)break;let r=[];for(let t=0;t<i.length;t++){if(t===e){r.push(`${i[t]}${i[t+1]}`),t++;continue}r.push(i[t])}i=r}return this.#e.set(e,i),i}},Tt=class{constructor(e){this.trie=this._build_trie(e)}_build_trie(e){let t=Object.create(null);for(let n of e){let e=t;for(let t=0;t<n.length;++t){let r=n[t];e=e[r]??=Object.create(null)}e.end=n}return t}split(e){let t=[],n=e.length,r=0,i=0;for(;i<n;){let a=this.trie,o=null,s=i;for(;s<n&&(a=a[e[s]]);)a.end&&(o=a.end),++s;o?(i>r&&t.push(e.slice(r,i)),t.push(o),i+=o.length,r=i):++i}return r<n&&t.push(e.slice(r)),t}},Et=class{constructor(e){this.content=e.content,this.id=e.id,this.single_word=e.single_word??!1,this.lstrip=e.lstrip??!1,this.rstrip=e.rstrip??!1,this.special=e.special??!1,this.normalized=e.normalized??!this.special}},Dt=(()=>{let e=[...Array.from({length:94},(e,t)=>t+33),...Array.from({length:12},(e,t)=>t+161),...Array.from({length:82},(e,t)=>t+174)],t=e.slice(),n=0;for(let r=0;r<256;++r)e.includes(r)||(e.push(r),t.push(256+n),n+=1);let r=t.map(e=>String.fromCharCode(e));return Object.fromEntries(e.map((e,t)=>[e,r[t]]))})(),Ot=(e=>Object.fromEntries(Object.entries(e).map(([e,t])=>[t,e])))(Dt),kt=`.,!?…。，、।۔،`,At=new Map([[`(?i:'s|'t|'re|'ve|'m|'ll|'d)`,`(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))`],[`(?i:[sdmt]|ll|ve|re)`,`(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`],[`[^\\r\\n\\p{L}\\p{N}]?+`,`[^\\r\\n\\p{L}\\p{N}]?`],[`[^\\s\\p{L}\\p{N}]++`,`[^\\s\\p{L}\\p{N}]+`],[`(?>\\p{Nd}{510})`,`(?:\\p{Nd}{510})`],[`\\p{Nd}{3}+`,`(?:\\p{Nd}{3})+`],[`\\G`,``],[` ?[^(\\s|[${kt}])]+`,` ?[^\\s${kt}]+`]]),jt=`\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E`,Mt=e=>e.replace(/ \./g,`.`).replace(/ \?/g,`?`).replace(/ \!/g,`!`).replace(/ ,/g,`,`).replace(/ \' /g,`'`).replace(/ n't/g,`n't`).replace(/ 'm/g,`'m`).replace(/ 's/g,`'s`).replace(/ 've/g,`'ve`).replace(/ 're/g,`'re`),Nt=(e,t=!0)=>{if(e.Regex!==void 0){let t=e.Regex.replace(/\\([#&~])/g,`$1`);t=t.replace(/\\A/g,`^`).replace(/\\z/g,`$`).replace(/\\Z/g,`(?=\\r?\\n?$)`);for(let[e,n]of At)t=t.replaceAll(e,n);try{return new RegExp(t,`gu`)}catch(e){if(!(e instanceof SyntaxError)||!e.message.toLowerCase().includes(`invalid property name`))throw e;let n=!1,r=t.replace(/(\\[pP])\{([^}=]+)\}/g,(e,t,r)=>{try{return RegExp(`\\p{${r}}`,`u`),`${t}{${r}}`}catch{return n=!0,`${t}{Script=${r}}`}});if(!n)throw e;try{return new RegExp(r,`gu`)}catch{throw e}}}else if(e.String!==void 0){let n=Pt(e.String);return new RegExp(t?n:`(${n})`,`gu`)}else return console.warn(`Unknown pattern type:`,e),null},Pt=e=>e.replace(/[.*+?^${}()|[\]\\]/g,`\\$&`),Ft=(e,t,n)=>{let r=[],i=0;for(;i<e.length;){if(r.push(e[i]),(t.get(e[i])??n)!==n){++i;continue}for(;++i<e.length&&(t.get(e[i])??n)===n;)t.get(r.at(-1))!==n&&(r[r.length-1]+=e[i])}return r},It=e=>e>=19968&&e<=40959||e>=13312&&e<=19903||e>=131072&&e<=173791||e>=173824&&e<=177983||e>=177984&&e<=178207||e>=178208&&e<=183983||e>=63744&&e<=64255||e>=194560&&e<=195103,Lt=e=>Number.isInteger(e)||typeof e==`bigint`,Rt=e=>{let t=0;for(let n of e)++t;return t},zt=e=>Ht(e.toLowerCase()),M=(...e)=>Array.prototype.concat.apply([],e),Bt=e=>new Map(Object.entries(e)),Vt=(e,t)=>{let n=[],r=0;for(let i of e.matchAll(t)){let t=i[0];r<i.index&&n.push(e.slice(r,i.index)),t.length>0&&n.push(t),r=i.index+t.length}return r<e.length&&n.push(e.slice(r)),n},Ht=e=>e.replace(/\p{M}/gu,``),Ut=(e,t,n=[])=>{if(!e||Array.isArray(e)||typeof e!=`object`)return`${t} must be a valid object`;for(let r of n)if(!(r in e))return`${t} must contain a "${r}" property`;return null},Wt=e=>e.match(/\S+/g)||[],N=class{constructor(){let e=function(...t){return e._call(...t)};return Object.setPrototypeOf(e,new.target.prototype)}},P=class extends N{constructor(e){super(),this.config=e}_call(e){return this.normalize(e)}},Gt=class extends P{tokenize_chinese_chars(e){let t=[];for(let n=0;n<e.length;++n){let r=e[n];It(r.charCodeAt(0))?(t.push(` `),t.push(r),t.push(` `)):t.push(r)}return t.join(``)}strip_accents(e){return e.normalize(`NFD`).replace(/\p{Mn}/gu,``)}is_control(e){switch(e){case`	`:case`
+`:case`\r`:return!1;default:return/^\p{Cc}|\p{Cf}|\p{Co}|\p{Cs}$/u.test(e)}}clean_text(e){let t=[];for(let n of e){let e=n.charCodeAt(0);e===0||e===65533||this.is_control(n)||(/^\s$/.test(n)?t.push(` `):t.push(n))}return t.join(``)}normalize(e){return this.config.clean_text&&(e=this.clean_text(e)),this.config.handle_chinese_chars&&(e=this.tokenize_chinese_chars(e)),this.config.lowercase?(e=e.toLowerCase(),this.config.strip_accents!==!1&&(e=this.strip_accents(e))):this.config.strip_accents&&(e=this.strip_accents(e)),e}},Kt=class extends P{constructor(e){super(e),this.charsmap=e.precompiled_charsmap??null}normalize(e){return e=e.replace(/[\u0001-\u0008\u000B\u000E-\u001F\u007F\u008F\u009F]/gm,``),e=e.replace(/[\u0009\u000A\u000C\u000D\u00A0\u1680\u2000-\u200F\u2028\u2029\u202F\u205F\u2581\u3000\uFEFF\uFFFD]/gm,` `),e=e.includes(`～`)?e.split(`～`).map(e=>e.normalize(`NFKC`)).join(`～`):e.normalize(`NFKC`),e}},qt=class extends P{constructor(e){super(e),this.normalizers=(e.normalizers??[]).map(e=>an(e))}normalize(e){return this.normalizers.reduce((e,t)=>t?t.normalize(e):e,e)}},Jt=class extends P{normalize(e){let t=Nt(this.config.pattern??{});return t===null?e:e.replaceAll(t,this.config.content??``)}},F=class extends P{constructor(){super(...arguments),this.form=`NFC`}normalize(e){return e=e.normalize(this.form),e}},Yt=class extends F{constructor(){super(...arguments),this.form=`NFC`}},Xt=class extends F{constructor(){super(...arguments),this.form=`NFD`}},Zt=class extends F{constructor(){super(...arguments),this.form=`NFKC`}},Qt=class extends F{constructor(){super(...arguments),this.form=`NFKD`}},$t=class extends P{normalize(e){return this.config.strip_left&&this.config.strip_right?e=e.trim():(this.config.strip_left&&(e=e.trimStart()),this.config.strip_right&&(e=e.trimEnd())),e}},en=class extends P{normalize(e){return Ht(e)}},tn=class extends P{normalize(e){return e.toLowerCase()}},nn=class extends P{normalize(e){return e=this.config.prepend+e,e}};function rn(e){if(e===null)return null;switch(e.type){case`BertNormalizer`:return new Gt(e);case`Precompiled`:return new Kt(e);case`Sequence`:return new qt(e);case`Replace`:return new Jt(e);case`NFC`:return new Yt(e);case`NFD`:return new Xt(e);case`NFKC`:return new Zt(e);case`NFKD`:return new Qt(e);case`Strip`:return new $t(e);case`StripAccents`:return new en(e);case`Lowercase`:return new tn(e);case`Prepend`:return new nn(e);default:throw Error(`Unknown Normalizer type: ${e.type}`)}}var an=rn,I=class extends N{pre_tokenize(e,t){return(Array.isArray(e)?e.map(e=>this.pre_tokenize_text(e,t)):this.pre_tokenize_text(e,t)).flat()}_call(e,t){return this.pre_tokenize(e,t)}},on=class extends I{constructor(e){super(),this.config=e,this.add_prefix_space=this.config.add_prefix_space??!1,this.trim_offsets=this.config.trim_offsets??!1,this.use_regex=this.config.use_regex??!0,this.pattern=/'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu,this.byte_encoder=Dt,this.text_encoder=new TextEncoder}pre_tokenize_text(e,t){return this.add_prefix_space&&!e.startsWith(` `)&&(e=` `+e),(this.use_regex?e.match(this.pattern)||[]:[e]).map(e=>Array.from(this.text_encoder.encode(e),e=>this.byte_encoder[e]).join(``))}},sn=class extends I{pre_tokenize_text(e,t){return e.match(/\w+|[^\w\s]+/g)||[]}},cn=class extends I{constructor(e){super(),this.replacement=e.replacement??`▁`,this.str_rep=e.str_rep||this.replacement,this.prepend_scheme=e.prepend_scheme??`always`}pre_tokenize_text(e,t){let{section_index:n=void 0}=t??{},r=e.replaceAll(` `,this.str_rep);return!r.startsWith(this.replacement)&&(this.prepend_scheme===`always`||this.prepend_scheme===`first`&&n===0)&&(r=this.str_rep+r),[r]}},ln=class extends I{constructor(e){super(),this.config=e,this.pattern=Nt(this.config.pattern??{},this.config.invert??!0)}pre_tokenize_text(e){return this.pattern===null?[]:this.config.invert?e.match(this.pattern)||[]:this.config.behavior?.toLowerCase()===`removed`?e.split(this.pattern).filter(e=>e):Vt(e,this.pattern)}},un=class extends I{constructor(e){super(),this.config=e,this.pattern=RegExp(`[^${jt}]+|[${jt}]+`,`gu`)}pre_tokenize_text(e){return e.match(this.pattern)||[]}},dn=class extends I{constructor(e){super(),this.config=e;let t=`[^\\d]+|\\d${this.config.individual_digits?``:`+`}`;this.pattern=new RegExp(t,`gu`)}pre_tokenize_text(e){return e.match(this.pattern)||[]}},fn=class extends I{constructor(){super(),this.pattern=RegExp(`[^\\s${jt}]+|[${jt}]`,`gu`)}pre_tokenize_text(e,t){return e.trim().match(this.pattern)||[]}},pn=class extends I{constructor(e){super(),this.config=e,this.pattern=Nt(this.config.pattern??{}),this.content=this.config.content??``}pre_tokenize_text(e){return this.pattern===null?[e]:[e.replaceAll(this.pattern,this.config.content??``)]}},mn=class extends I{constructor(e){super(),this.tokenizers=(e.pretokenizers??[]).map(e=>vn(e))}pre_tokenize_text(e,t){return this.tokenizers.reduce((e,n)=>n?n.pre_tokenize(e,t):e,[e])}},hn=class extends I{pre_tokenize_text(e){return Wt(e)}},gn=class extends I{constructor(e){super(),this.config=e,this._length=e.length}pre_tokenize_text(e){let t=[];for(let n=0;n<e.length;n+=this._length)t.push(e.slice(n,n+this._length));return t}};function _n(e){if(e===null)return null;switch(e.type){case`BertPreTokenizer`:return new fn;case`Sequence`:return new mn(e);case`Whitespace`:return new sn;case`WhitespaceSplit`:return new hn;case`Metaspace`:return new cn(e);case`ByteLevel`:return new on(e);case`Split`:return new ln(e);case`Punctuation`:return new un(e);case`Digits`:return new dn(e);case`Replace`:return new pn(e);case`FixedLength`:return new gn(e);default:throw Error(`Unknown PreTokenizer type: ${e.type}`)}}var vn=_n,L=class extends N{constructor(e){super(),this.config=e,this.vocab=[],this.tokens_to_ids=new Map,this.unk_token_id=void 0,this.unk_token=void 0,this.end_of_word_suffix=void 0,this.fuse_unk=this.config.fuse_unk??!1}_call(e){let t=this.encode(e);return this.fuse_unk&&(t=Ft(t,this.tokens_to_ids,this.unk_token_id)),t}},yn=class extends L{constructor(e){super(e),this.max_input_chars_per_word=100,this.tokens_to_ids=Bt(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.max_input_chars_per_word=e.max_input_chars_per_word??100,this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e}encode(e){let t=[];for(let n of e){let e=[...n];if(e.length>this.max_input_chars_per_word){t.push(this.unk_token);continue}let r=!1,i=0,a=[];for(;i<e.length;){let t=e.length,n=null;for(;i<t;){let r=e.slice(i,t).join(``);if(i>0&&(r=this.config.continuing_subword_prefix+r),this.tokens_to_ids.has(r)){n=r;break}--t}if(n===null){r=!0;break}a.push(n),i=t}r?t.push(this.unk_token):t.push(...a)}return t}},bn=class e{constructor(e,t){this.is_leaf=e,this.children=t}static default(){return new e(!1,new Map)}},xn=class{constructor(){this.root=bn.default()}extend(e){for(let t of e)this.push(t)}push(e){let t=this.root;for(let n of e){let e=t.children.get(n);e===void 0&&(e=bn.default(),t.children.set(n,e)),t=e}t.is_leaf=!0}*common_prefix_search(e){let t=this.root;if(t===void 0)return;let n=``;for(let r of e){if(n+=r,t=t.children.get(r),t===void 0)return;t.is_leaf&&(yield n)}}},Sn=class e{constructor(e,t,n,r,i){this.token_id=e,this.node_id=t,this.pos=n,this.length=r,this.score=i,this.prev=null,this.backtrace_score=0}clone(){let t=new e(this.token_id,this.node_id,this.pos,this.length,this.score);return t.prev=this.prev,t.backtrace_score=this.backtrace_score,t}},Cn=class{constructor(e,t,n){this.chars=Array.from(e),this.len=this.chars.length,this.bos_token_id=t,this.eos_token_id=n,this.nodes=[],this.begin_nodes=Array.from({length:this.len+1},()=>[]),this.end_nodes=Array.from({length:this.len+1},()=>[]);let r=new Sn(this.bos_token_id??0,0,0,0,0),i=new Sn(this.eos_token_id??0,1,this.len,0,0);this.nodes.push(r.clone()),this.nodes.push(i.clone()),this.begin_nodes[this.len].push(i),this.end_nodes[0].push(r)}insert(e,t,n,r){let i=this.nodes.length,a=new Sn(r,i,e,t,n);this.begin_nodes[e].push(a),this.end_nodes[e+t].push(a),this.nodes.push(a)}viterbi(){let e=this.len,t=0;for(;t<=e;){if(this.begin_nodes[t].length==0)return[];for(let e of this.begin_nodes[t]){e.prev=null;let n=0,r=null;for(let i of this.end_nodes[t]){let t=i.backtrace_score+e.score;(r===null||t>n)&&(r=i.clone(),n=t)}if(r!==null)e.prev=r,e.backtrace_score=n;else return[]}++t}let n=[],r=this.begin_nodes[e][0].prev;if(r===null)return[];let i=r.clone();for(;i.prev!==null;)n.push(i.clone()),i=i.clone().prev.clone();return n.reverse(),n}piece(e){return this.chars.slice(e.pos,e.pos+e.length).join(``)}tokens(){return this.viterbi().map(e=>this.piece(e))}token_ids(){return this.viterbi().map(e=>e.token_id)}};function wn(e){if(e.length===0)throw Error(`Array must not be empty`);let t=e[0],n=0;for(let r=1;r<e.length;++r)e[r]<t&&(t=e[r],n=r);return[t,n]}var Tn=class extends L{constructor(e,t){super(e);let n=e.vocab.length;this.vocab=Array(n),this.scores=Array(n);for(let t=0;t<n;++t)[this.vocab[t],this.scores[t]]=e.vocab[t];this.unk_token_id=e.unk_id,this.unk_token=this.vocab[e.unk_id],this.tokens_to_ids=new Map(this.vocab.map((e,t)=>[e,t])),this.bos_token=` `,this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.unk_token=this.vocab[this.unk_token_id],this.min_score=wn(this.scores)[0],this.unk_score=this.min_score-10,this.scores[this.unk_token_id]=this.unk_score,this.trie=new xn,this.trie.extend(this.vocab),this.fuse_unk=!0}populate_nodes(e){let t=e.chars,n=0;for(;n<t.length;){let r=!1,i=[],a=t.slice(n).join(``),o=this.trie.common_prefix_search(a);for(let t of o){i.push(t);let a=this.tokens_to_ids.get(t),o=this.scores[a],s=Rt(t);e.insert(n,s,o,a),!r&&s===1&&(r=!0)}r||e.insert(n,1,this.unk_score,this.unk_token_id),n+=1}}tokenize(e){let t=new Cn(e,this.bos_token_id,this.eos_token_id);return this.populate_nodes(t),t.tokens()}encode(e){let t=[];for(let n of e){let e=this.tokenize(n);t.push(...e)}return t}},En=class{constructor(e=(e,t)=>e>t,t=1/0){this._heap=[],this._comparator=e,this._max_size=t}get size(){return this._heap.length}is_empty(){return this.size===0}peek(){return this._heap[0]}push(...e){return this.extend(e)}extend(e){for(let t of e)if(this.size<this._max_size)this._heap.push(t),this._sift_up();else{let e=this._smallest();this._comparator(t,this._heap[e])&&(this._heap[e]=t,this._sift_up_from(e))}return this.size}pop(){let e=this.peek(),t=this.size-1;return t>0&&this._swap(0,t),this._heap.pop(),this._sift_down(),e}replace(e){let t=this.peek();return this._heap[0]=e,this._sift_down(),t}_parent(e){return(e+1>>>1)-1}_left(e){return(e<<1)+1}_right(e){return e+1<<1}_greater(e,t){return this._comparator(this._heap[e],this._heap[t])}_swap(e,t){let n=this._heap[e];this._heap[e]=this._heap[t],this._heap[t]=n}_sift_up(){this._sift_up_from(this.size-1)}_sift_up_from(e){for(;e>0&&this._greater(e,this._parent(e));)this._swap(e,this._parent(e)),e=this._parent(e)}_sift_down(){let e=0;for(;this._left(e)<this.size&&this._greater(this._left(e),e)||this._right(e)<this.size&&this._greater(this._right(e),e);){let t=this._right(e)<this.size&&this._greater(this._right(e),this._left(e))?this._right(e):this._left(e);this._swap(e,t),e=t}}_smallest(){return 2**Math.floor(Math.log2(this.size))-1}},Dn=class{constructor(e){this.capacity=e,this.cache=new Map}get(e){if(!this.cache.has(e))return;let t=this.cache.get(e);return this.cache.delete(e),this.cache.set(e,t),t}put(e,t){this.cache.has(e)&&this.cache.delete(e),this.cache.set(e,t),this.cache.size>this.capacity&&this.cache.delete(this.cache.keys().next().value)}clear(){this.cache.clear()}},On=class extends L{constructor(e){super(e),this.tokens_to_ids=Bt(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e;let t=Array.isArray(e.merges[0]);this.merges=t?e.merges:e.merges.map(e=>e.split(` `,2)),this.bpe_ranks=new Map(this.merges.map((e,t)=>[JSON.stringify(e),t])),this.end_of_word_suffix=e.end_of_word_suffix,this.continuing_subword_suffix=e.continuing_subword_suffix??null,this.byte_fallback=this.config.byte_fallback??!1,this.byte_fallback&&(this.text_encoder=new TextEncoder),this.ignore_merges=this.config.ignore_merges??!1,this.max_length_to_cache=256,this.cache_capacity=1e4,this.cache=new Dn(this.cache_capacity)}clear_cache(){this.cache.clear()}bpe(e){if(e.length===0)return[];let t=this.cache.get(e);if(t!==void 0)return t;let n=Array.from(e);this.end_of_word_suffix&&(n[n.length-1]+=this.end_of_word_suffix);let r=[];if(n.length>1){let e=new En((e,t)=>e.score<t.score),t={token:n[0],bias:0,prev:null,next:null},i=t;for(let t=1;t<n.length;++t){let r={bias:t/n.length,token:n[t],prev:i,next:null};i.next=r,this.add_node(e,i),i=r}for(;!e.is_empty();){let n=e.pop();if(n.deleted||!n.next||n.next.deleted)continue;if(n.deleted=!0,n.next.deleted=!0,n.prev){let e={...n.prev};n.prev.deleted=!0,n.prev=e,e.prev?e.prev.next=e:t=e}let r={token:n.token+n.next.token,bias:n.bias,prev:n.prev,next:n.next.next};r.prev?(r.prev.next=r,this.add_node(e,r.prev)):t=r,r.next&&(r.next.prev=r,this.add_node(e,r))}for(let e=t;e!==null;e=e.next)r.push(e.token)}else r=n;if(this.continuing_subword_suffix)for(let e=0;e<r.length-1;++e)r[e]+=this.continuing_subword_suffix;return e.length<this.max_length_to_cache&&this.cache.put(e,r),r}add_node(e,t){let n=this.bpe_ranks.get(JSON.stringify([t.token,t.next.token]));n!==void 0&&(t.score=n+t.bias,e.push(t))}encode(e){let t=[];for(let n of e){if(this.ignore_merges&&this.tokens_to_ids.has(n)){t.push(n);continue}let e=this.bpe(n);for(let n of e)if(this.tokens_to_ids.has(n))t.push(n);else if(this.byte_fallback){let e=Array.from(this.text_encoder.encode(n)).map(e=>`<0x${e.toString(16).toUpperCase().padStart(2,`0`)}>`);e.every(e=>this.tokens_to_ids.has(e))?t.push(...e):this.unk_token!=null&&t.push(this.unk_token)}else this.unk_token!=null&&t.push(this.unk_token)}return t}},kn=class extends L{constructor(e,t){super(e);let n=e.vocab;this.tokens_to_ids=Bt(t.target_lang?n[t.target_lang]:n),this.bos_token=t.bos_token,this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t.eos_token,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.pad_token=t.pad_token,this.pad_token_id=this.tokens_to_ids.get(this.pad_token),this.unk_token=t.unk_token,this.unk_token_id=this.tokens_to_ids.get(this.unk_token),this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e}encode(e){return e}};function An(e,t){switch(e.type){case`WordPiece`:return new yn(e);case`Unigram`:return new Tn(e,t.eos_token);case`BPE`:return new On(e);default:if(e.vocab)return Array.isArray(e.vocab)?new Tn(e,t.eos_token):Object.hasOwn(e,`continuing_subword_prefix`)&&Object.hasOwn(e,`unk_token`)?Object.hasOwn(e,`merges`)?new On(e):new yn(e):new kn(e,{target_lang:t.target_lang,bos_token:t.bos_token,eos_token:t.eos_token,pad_token:t.pad_token,unk_token:t.unk_token});throw Error(`Unknown TokenizerModel type: ${e?.type}`)}}var jn=An,R=class extends N{constructor(e){super(),this.config=e}_call(e,...t){return this.post_process(e,...t)}},Mn=class extends R{post_process(e,t=null,n=!0){let r=t===null?this.config.single:this.config.pair,i=[],a=[];for(let o of r)`SpecialToken`in o?n&&(i.push(o.SpecialToken.id),a.push(o.SpecialToken.type_id)):`Sequence`in o&&(o.Sequence.id===`A`?(i=M(i,e),a=M(a,Array(e.length).fill(o.Sequence.type_id))):o.Sequence.id===`B`&&(i=M(i,t),a=M(a,Array(t.length).fill(o.Sequence.type_id))));return{tokens:i,token_type_ids:a}}},Nn=class extends R{post_process(e,t=null){return{tokens:e,tokens_pair:t}}},Pn=class extends R{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t=null,n=!0){n&&(e=M([this.cls[0]],e,[this.sep[0]]));let r=Array(e.length).fill(0);if(t){let i=[],a=n?[this.sep[0]]:[];e=M(e,i,t,a),r=M(r,Array(t.length+i.length+a.length).fill(1))}return{tokens:e,token_type_ids:r}}},Fn=class extends R{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t,n=!0){n&&(e=M([this.cls[0]],e,[this.sep[0]]));let r=Array(e.length).fill(0);if(t){let i=n?[this.sep[0]]:[],a=n?[this.sep[0]]:[];e=M(e,i,t,a),r=M(r,Array(t.length+i.length+a.length).fill(1))}return{tokens:e,token_type_ids:r}}},In=class extends R{constructor(e){super(e),this.processors=(e.processors??[]).map(e=>Rn(e))}post_process(e,t=null,n=!0){let r={tokens:e,tokens_pair:t};for(let e of this.processors)r=e.post_process(r.tokens,r.tokens_pair,n);return r}};function Ln(e){if(e===null)return null;switch(e.type){case`TemplateProcessing`:return new Mn(e);case`ByteLevel`:return new Nn(e);case`BertProcessing`:return new Pn(e);case`RobertaProcessing`:return new Fn(e);case`Sequence`:return new In(e);default:throw Error(`Unknown PostProcessor type: ${e.type}`)}}var Rn=Ln,z=class extends N{constructor(e){super(),this.config=e,this.added_tokens=[],this.end_of_word_suffix=null,this.trim_offsets=`trim_offsets`in e?e.trim_offsets:!1}_call(e){return this.decode(e)}decode(e){return this.decode_chain(e).join(``)}},zn=class extends z{constructor(e){super(e),this.byte_decoder=Ot,this.text_decoder=new TextDecoder(`utf-8`,{fatal:!1,ignoreBOM:!0}),this.end_of_word_suffix=null}convert_tokens_to_string(e){let t=e.join(``),n=new Uint8Array([...t].map(e=>this.byte_decoder[e]));return this.text_decoder.decode(n)}decode_chain(e){let t=[],n=[];for(let r of e)this.added_tokens.find(e=>e.content===r)===void 0?n.push(r):(n.length>0&&(t.push(this.convert_tokens_to_string(n)),n=[]),t.push(r));return n.length>0&&t.push(this.convert_tokens_to_string(n)),t}},Bn=class extends z{constructor(e){super(e),this.cleanup=e.cleanup}decode_chain(e){return e.map((e,t)=>{if(t!==0){let t=this.config.prefix;e=t&&e.startsWith(t)?e.replace(t,``):` `+e}return this.cleanup&&(e=Mt(e)),e})}},Vn=class extends z{constructor(e){super(e),this.replacement=e.replacement??`▁`}decode_chain(e){let t=[];for(let n=0;n<e.length;++n){let r=e[n].replaceAll(this.replacement,` `);n==0&&r.startsWith(` `)&&(r=r.substring(1)),t.push(r)}return t}},Hn=class extends z{constructor(e){super(e),this.suffix=e.suffix??``}decode_chain(e){return e.map((t,n)=>t.replaceAll(this.suffix,n===e.length-1?``:` `))}},Un=class extends z{constructor(e){super(e),this.pad_token=e.pad_token??``,this.word_delimiter_token=e.word_delimiter_token??``,this.cleanup=e.cleanup}convert_tokens_to_string(e){if(e.length===0)return``;let t=[e[0]];for(let n=1;n<e.length;++n)e[n]!==t.at(-1)&&t.push(e[n]);let n=t.filter(e=>e!==this.pad_token).join(``);return this.cleanup&&(n=Mt(n).replaceAll(this.word_delimiter_token,` `).trim()),n}decode_chain(e){return[this.convert_tokens_to_string(e)]}},Wn=class extends z{constructor(e){super(e),this.decoders=(e.decoders??[]).map(e=>Xn(e))}decode_chain(e){return this.decoders.reduce((e,t)=>t.decode_chain(e),e)}},Gn=class extends z{decode_chain(e){let t=Nt(this.config.pattern),n=this.config.content??``;return t===null?e:e.map(e=>e.replaceAll(t,n))}},Kn=class extends z{decode_chain(e){return[e.join(``)]}},qn=class extends z{constructor(e){super(e),this.content=e.content??``,this.start=e.start??0,this.stop=e.stop??0}decode_chain(e){return e.map(e=>{let t=0;for(let n=0;n<this.start&&e[n]===this.content;++n)t=n+1;let n=e.length;for(let t=0;t<this.stop;++t){let r=e.length-t-1;if(e[r]===this.content){n=r;continue}else break}return e.slice(t,n)})}},Jn=class extends z{constructor(e){super(e),this.text_decoder=new TextDecoder}decode_chain(e){let t=[],n=[];for(let r of e){let e=null;if(r.length===6&&r.startsWith(`<0x`)&&r.endsWith(`>`)){let t=parseInt(r.slice(3,5),16);isNaN(t)||(e=t)}if(e!==null)n.push(e);else{if(n.length>0){let e=this.text_decoder.decode(Uint8Array.from(n));t.push(e),n=[]}t.push(r)}}if(n.length>0){let e=this.text_decoder.decode(Uint8Array.from(n));t.push(e),n=[]}return t}};function Yn(e){if(e===null)return null;switch(e.type){case`ByteLevel`:return new zn(e);case`WordPiece`:return new Bn(e);case`Metaspace`:return new Vn(e);case`BPEDecoder`:return new Hn(e);case`CTC`:return new Un(e);case`Sequence`:return new Wn(e);case`Replace`:return new Gn(e);case`Fuse`:return new Kn(e);case`Strip`:return new qn(e);case`ByteFallback`:return new Jn(e);default:throw Error(`Unknown Decoder type: ${e.type}`)}}var Xn=Yn,Zn=class{constructor(e,t){let n=Ut(e,`Tokenizer`,[`model`,`decoder`,`post_processor`,`pre_tokenizer`,`normalizer`]);if(n)throw Error(n);let r=Ut(t,`Config`);if(r)throw Error(r);this.tokenizer=e,this.config=t,this.normalizer=an(this.tokenizer.normalizer),this.pre_tokenizer=vn(this.tokenizer.pre_tokenizer),this.model=jn(this.tokenizer.model,this.config),this.post_processor=Rn(this.tokenizer.post_processor),this.decoder=Xn(this.tokenizer.decoder),this.special_tokens=[],this.all_special_ids=[],this.added_tokens=[];let i=[],a=[];this.added_tokens_map=new Map;for(let e of this.tokenizer.added_tokens){let t=new Et(e);if(this.added_tokens.push(t),this.model.tokens_to_ids.set(t.content,t.id),this.model.vocab[t.id]=t.content,t.special&&(this.special_tokens.push(t.content),this.all_special_ids.push(t.id)),this.added_tokens_map.set(t.content,t),t.normalized&&this.normalizer!==null){let e=this.normalizer(t.content);a.push(e),this.added_tokens_map.set(e,t)}else i.push(t.content)}(this.config.additional_special_tokens??[]).forEach(e=>{this.special_tokens.includes(e)||this.special_tokens.push(e)}),this.decoder&&(this.decoder.added_tokens=this.added_tokens,this.decoder.end_of_word_suffix=this.model.end_of_word_suffix),this.splitter_unnormalized=new Tt(i),this.splitter_normalized=new Tt(a),this.remove_space=this.config.remove_space,this.clean_up_tokenization_spaces=this.config.clean_up_tokenization_spaces??!0,this.do_lowercase_and_remove_accent=this.config.do_lowercase_and_remove_accent??!1}encode(e,{text_pair:t=null,add_special_tokens:n=!0,return_token_type_ids:r=null}={}){let{tokens:i,token_type_ids:a}=this.tokenize_helper(e,{text_pair:t,add_special_tokens:n}),o=i.map(e=>this.added_tokens_map.get(e)?.id??this.model.tokens_to_ids.get(e)??this.model.unk_token_id),s={ids:o,tokens:i,attention_mask:Array(o.length).fill(1)};return r&&a&&(s.token_type_ids=a),s}decode(e,t={}){if(!Array.isArray(e)||e.length===0||!Lt(e[0]))throw Error(`token_ids must be a non-empty array of integers.`);let n=e.map(e=>this.model.vocab[Number(e)]??this.model.unk_token);t.skip_special_tokens&&(n=n.filter(e=>!this.special_tokens.includes(e)));let r=this.decoder?this.decoder(n):n.join(` `);return this.decoder&&this.decoder.end_of_word_suffix&&(r=r.replaceAll(this.decoder.end_of_word_suffix,` `),t.skip_special_tokens&&(r=r.trim())),(t.clean_up_tokenization_spaces??this.clean_up_tokenization_spaces)&&(r=Mt(r)),r}tokenize(e,{text_pair:t=null,add_special_tokens:n=!1}={}){return this.tokenize_helper(e,{text_pair:t,add_special_tokens:n}).tokens}encode_text(e){if(e===null)return null;let t=this.splitter_unnormalized.split(e);return t.forEach((e,n)=>{let r=this.added_tokens_map.get(e);r&&(r.lstrip&&n>0&&(t[n-1]=t[n-1].trimEnd()),r.rstrip&&n<t.length-1&&(t[n+1]=t[n+1].trimStart()))}),t.flatMap((e,t)=>{if(e.length===0)return[];if(this.added_tokens_map.has(e))return[e];if(this.remove_space===!0&&(e=e.trim().split(/\s+/).join(` `)),this.do_lowercase_and_remove_accent&&(e=zt(e)),this.normalizer!==null&&(e=this.normalizer(e)),e.length===0)return[];let n=this.splitter_normalized.split(e);return n.forEach((e,t)=>{let r=this.added_tokens_map.get(e);r&&(r.lstrip&&t>0&&(n[t-1]=n[t-1].trimEnd()),r.rstrip&&t<n.length-1&&(n[t+1]=n[t+1].trimStart()))}),n.flatMap(e=>{if(e.length===0)return[];if(this.added_tokens_map.has(e))return[e];let n=this.pre_tokenizer===null?[e]:this.pre_tokenizer(e,{section_index:t});return this.model(n)})})}tokenize_helper(e,{text_pair:t=null,add_special_tokens:n=!0}){let r=this.encode_text(e),i=this.encode_text(t||null);return this.post_processor?this.post_processor(r,i,n):{tokens:M(r??[],i??[])}}token_to_id(e){return this.model.tokens_to_ids.get(e)}id_to_token(e){return this.model.vocab[e]}get_added_tokens_decoder(){let e=new Map;for(let t of this.added_tokens)e.set(t.id,t);return e}get_vocab(e=!0){let t=new Map;for(let n=0;n<this.model.vocab.length;++n){let r=this.model.vocab[n];(e||!this.added_tokens_map.has(r))&&t.set(r,n)}return t}};const Qn=(e,t)=>{if(!e.has(t))throw Error(`Missing required key ${JSON.stringify(t)} in structured data.`);return e.get(t)},B=e=>{if(Array.isArray(e))return e.map(e=>B(e));if(e instanceof Map){let t=Object.create(null);for(let[n,r]of e.entries()){if(typeof n!=`string`)throw TypeError(`Expected string structured-data key, got ${typeof n}.`);t[n]=B(r)}return t}return e},$n=Zn,er={add_special_tokens:!1},tr={clean_up_tokenization_spaces:!1,skip_special_tokens:!1};var nr=class extends pt{modelId;constructor(e){super(),this.modelId=e}createState(){return{tokenizer:new $n(B(j(this.modelId,`tokenizer.msgpack`)),B(j(this.modelId,`tokenizer_config.msgpack`)))}}encodeWithState(e,t){return[...t.tokenizer.encode(e,er).ids]}tokenizeWithState(e,t){let n=this.encodeWithState(e,t);return dt(n,n.map(e=>lt(t.tokenizer.decode([e],tr))))}},rr=e({Tiktoken:()=>Sr,__wbg_parse_def2e24ef1252aff:()=>Cr,__wbg_set_wasm:()=>ir,__wbg_stringify_f7ed6987935b4a24:()=>wr,__wbindgen_error_new:()=>Tr,__wbindgen_is_undefined:()=>Er,__wbindgen_object_drop_ref:()=>Dr,__wbindgen_string_get:()=>Or,__wbindgen_throw:()=>kr,encoding_for_model:()=>yr,get_encoding:()=>vr,get_encoding_name_for_model:()=>br});let V;function ir(e){V=e}let ar=new(typeof TextDecoder>`u`?(0,module.require)(`util`).TextDecoder:TextDecoder)(`utf-8`,{ignoreBOM:!0,fatal:!0});ar.decode();let H=null;function U(){return(H===null||H.byteLength===0)&&(H=new Uint8Array(V.memory.buffer)),H}function W(e,t){return e>>>=0,ar.decode(U().subarray(e,e+t))}const G=Array(128).fill(void 0);G.push(void 0,null,!0,!1);let K=G.length;function q(e){K===G.length&&G.push(G.length+1);let t=K;return K=G[t],G[t]=e,t}function or(e,t){try{return e.apply(this,t)}catch(e){V.__wbindgen_export_0(q(e))}}function sr(e){return G[e]}function cr(e){e<132||(G[e]=K,K=e)}function J(e){let t=sr(e);return cr(e),t}let Y=0,lr=new(typeof TextEncoder>`u`?(0,module.require)(`util`).TextEncoder:TextEncoder)(`utf-8`);const ur=typeof lr.encodeInto==`function`?function(e,t){return lr.encodeInto(e,t)}:function(e,t){let n=lr.encode(e);return t.set(n),{read:e.length,written:n.length}};function X(e,t,n){if(n===void 0){let n=lr.encode(e),r=t(n.length,1)>>>0;return U().subarray(r,r+n.length).set(n),Y=n.length,r}let r=e.length,i=t(r,1)>>>0,a=U(),o=0;for(;o<r;o++){let t=e.charCodeAt(o);if(t>127)break;a[i+o]=t}if(o!==r){o!==0&&(e=e.slice(o)),i=n(i,r,r=o+e.length*3,1)>>>0;let t=U().subarray(i+o,i+r),a=ur(e,t);o+=a.written,i=n(i,r,o,1)>>>0}return Y=o,i}function dr(e){return e==null}let Z=null;function Q(){return(Z===null||Z.buffer.detached===!0||Z.buffer.detached===void 0&&Z.buffer!==V.memory.buffer)&&(Z=new DataView(V.memory.buffer)),Z}let fr=null;function pr(){return(fr===null||fr.byteLength===0)&&(fr=new Uint32Array(V.memory.buffer)),fr}function mr(e,t){return e>>>=0,pr().subarray(e/4,e/4+t)}function hr(e,t){let n=t(e.length*1,1)>>>0;return U().set(e,n/1),Y=e.length,n}function gr(e,t){let n=t(e.length*4,4)>>>0;return pr().set(e,n/4),Y=e.length,n}function _r(e,t){return e>>>=0,U().subarray(e/1,e/1+t)}function vr(e,t){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y;V.get_encoding(i,a,o,q(t));var n=Q().getInt32(i+0,!0),r=Q().getInt32(i+4,!0);if(Q().getInt32(i+8,!0))throw J(r);return Sr.__wrap(n)}finally{V.__wbindgen_add_to_stack_pointer(16)}}function yr(e,t){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y;V.encoding_for_model(i,a,o,q(t));var n=Q().getInt32(i+0,!0),r=Q().getInt32(i+4,!0);if(Q().getInt32(i+8,!0))throw J(r);return Sr.__wrap(n)}finally{V.__wbindgen_add_to_stack_pointer(16)}}function br(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let t,n;try{let l=V.__wbindgen_add_to_stack_pointer(-16),u=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),d=Y;V.get_encoding_name_for_model(l,u,d);var r=Q().getInt32(l+0,!0),i=Q().getInt32(l+4,!0),a=Q().getInt32(l+8,!0),o=Q().getInt32(l+12,!0),s=r,c=i;if(o)throw s=0,c=0,J(a);return t=s,n=c,W(s,c)}finally{V.__wbindgen_add_to_stack_pointer(16),V.__wbindgen_export_3(t,n,1)}}const xr=typeof FinalizationRegistry>`u`?{register:()=>{},unregister:()=>{}}:new FinalizationRegistry(e=>V.__wbg_tiktoken_free(e>>>0,1));var Sr=class e{static __wrap(t){t>>>=0;let n=Object.create(e.prototype);return n.__wbg_ptr=t,xr.register(n,n.__wbg_ptr,n),n}__destroy_into_raw(){let e=this.__wbg_ptr;return this.__wbg_ptr=0,xr.unregister(this),e}free(){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let e=this.__destroy_into_raw();V.__wbg_tiktoken_free(e,0)}constructor(e,t,n){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let r=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),i=Y,a=X(n,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y,s=V.tiktoken_new(r,i,q(t),a,o);return this.__wbg_ptr=s>>>0,xr.register(this,this.__wbg_ptr,this),this}get name(){try{let n=V.__wbindgen_add_to_stack_pointer(-16);V.tiktoken_name(n,this.__wbg_ptr);var e=Q().getInt32(n+0,!0),t=Q().getInt32(n+4,!0);let r;return e!==0&&(r=W(e,t).slice(),V.__wbindgen_export_3(e,t*1,1)),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode(e,t,n){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let s=V.__wbindgen_add_to_stack_pointer(-16),c=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),l=Y;V.tiktoken_encode(s,this.__wbg_ptr,c,l,q(t),q(n));var r=Q().getInt32(s+0,!0),i=Q().getInt32(s+4,!0),a=Q().getInt32(s+8,!0);if(Q().getInt32(s+12,!0))throw J(a);var o=mr(r,i).slice();return V.__wbindgen_export_3(r,i*4,4),o}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode_ordinary(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y;V.tiktoken_encode_ordinary(i,this.__wbg_ptr,a,o);var t=Q().getInt32(i+0,!0),n=Q().getInt32(i+4,!0),r=mr(t,n).slice();return V.__wbindgen_export_3(t,n*4,4),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode_with_unstable(e,t,n){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let a=V.__wbindgen_add_to_stack_pointer(-16),o=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),s=Y;V.tiktoken_encode_with_unstable(a,this.__wbg_ptr,o,s,q(t),q(n));var r=Q().getInt32(a+0,!0),i=Q().getInt32(a+4,!0);if(Q().getInt32(a+8,!0))throw J(i);return J(r)}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode_single_token(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let t=hr(e,V.__wbindgen_export_1),n=Y;return V.tiktoken_encode_single_token(this.__wbg_ptr,t,n)>>>0}decode(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=gr(e,V.__wbindgen_export_1),o=Y;V.tiktoken_decode(i,this.__wbg_ptr,a,o);var t=Q().getInt32(i+0,!0),n=Q().getInt32(i+4,!0),r=_r(t,n).slice();return V.__wbindgen_export_3(t,n*1,1),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}decode_single_token_bytes(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16);V.tiktoken_decode_single_token_bytes(i,this.__wbg_ptr,e);var t=Q().getInt32(i+0,!0),n=Q().getInt32(i+4,!0),r=_r(t,n).slice();return V.__wbindgen_export_3(t,n*1,1),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}token_byte_values(){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);return J(V.tiktoken_token_byte_values(this.__wbg_ptr))}};function Cr(){return or(function(e,t){return q(JSON.parse(W(e,t)))},arguments)}function wr(){return or(function(e){return q(JSON.stringify(sr(e)))},arguments)}function Tr(e,t){return q(Error(W(e,t)))}function Er(e){return sr(e)===void 0}function Dr(e){J(e)}function Or(e,t){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let n=sr(t),r=typeof n==`string`?n:void 0;var i=dr(r)?0:X(r,V.__wbindgen_export_1,V.__wbindgen_export_2),a=Y;Q().setInt32(e+4,a,!0),Q().setInt32(e+0,i,!0)}function kr(e,t){throw Error(W(e,t))}let Ar=!1;async function jr(e){if(Ar)return rr;let t=await e({"./tiktoken_bg.js":rr}),n=`instance`in t&&t.instance instanceof WebAssembly.Instance?t.instance:t instanceof WebAssembly.Instance?t:null;if(n==null)throw Error(`Missing instance`);return ir(n.exports),Ar=!0,rr}const Mr=new URL(`../tiktoken_bg.wasm`,import.meta.url).href;await(async()=>{await jr(async e=>{let t=e,n=await fetch(Mr);if(!n.ok)throw Error(`Failed to load tiktoken WASM from ${JSON.stringify(Mr)}: ${n.status} ${n.statusText}`);if(typeof WebAssembly.instantiateStreaming==`function`)try{return await WebAssembly.instantiateStreaming(n.clone(),t)}catch{}return WebAssembly.instantiate(await n.arrayBuffer(),t)})})();var Nr=class extends pt{disposeState(e){e.free()}tokenizeWithState(e,t){let n=this.encodeWithState(e,t);return dt(n,n.map(e=>t.decode_single_token_bytes(e).length))}},Pr=class extends Nr{modelId;constructor(e){super(),this.modelId=e}createState(){return vr(Qn(j(this.modelId,`config.msgpack`),`encoding`))}encodeWithState(e,t){return[...t.encode(e)]}},Fr=class extends Nr{modelId;constructor(e){super(),this.modelId=e}createState(){let e=j(this.modelId,`config.msgpack`);return new Sr(Ye(this.modelId,`tiktoken.model`),B(Qn(e,`special_tokens`)),Qn(e,`pat_str`))}encodeWithState(e,t){return[...t.encode(e)]}};const $=new Map,Ir=e=>{let t=$.get(e);if(t)return t;let n=ke[e],r;switch(n.kind){case`tiktoken-builtin`:r=new Pr(e);break;case`tiktoken-custom`:r=new Fr(e);break;case`huggingface`:r=new nr(e);break;case`clip-bpe`:r=new wt(e);break}return $.set(e,r),r},Lr=e=>{if(e){$.get(e)?.free(),$.delete(e);return}for(let e of $.values())e.free();$.clear()},Rr=(e,t)=>{if(typeof t==`string`){let[e]=it(t);return e}let n=t?.model;if(typeof n!=`string`)throw TypeError(`${e}() requires a single model ID as the second argument.`);let[r]=it(n);return r};function zr(e,t){return Ir(Rr(`tokenizeLoaded`,t)).tokenize(e)}function Br(e,t){return Ir(Rr(`countLoaded`,t)).getTokenCount(e)}async function Vr(e){let t=it(e);return await Ke(t),typeof e==`string`?t[0]:t}async function Hr(e,t){let n=Rr(`tokenize`,t);return await Vr(n),zr(e,n)}async function Ur(e,t){let n=Rr(`count`,t);return await Vr(n),Br(e,n)}const Wr=e=>{Je(e),Lr(e)};export{Hr as a,ke as c,Vr as i,Br as n,zr as o,Wr as r,E as s,Ur as t};

package/gpt.bin

@@ -0,0 +1 @@
+���config.msgpack���encoding�o200k_base

package/lib/api.d.ts

@@ -1,19 +1,29 @@
 import type { ModelId } from './models.js';
 import type { ModelSelection } from './modelSelection.js';
+import type { RawTokenizeResult, TokenizeInput } from './tokenization.js';
 import { modelIds, models } from './models.js';
-export type TokenizeResult = Record<ModelId, Array<number>>;
-export type CountTokensResult = Record<ModelId, number>;
-export type CountTokensOptions<TModel extends ModelSelection | undefined = undefined> = {
-    model?: TModel;
+export type { ModelSelection } from './modelSelection.js';
+export type CountResult = number;
+export type CountTokensResult = CountResult;
+export type TokenizeResult<InputGeneric extends TokenizeInput = TokenizeInput> = RawTokenizeResult<InputGeneric>;
+export type TokenizeOptions<TModel extends ModelId = ModelId> = SingleModelOptions<TModel>;
+export type CountOptions<TModel extends ModelId = ModelId> = SingleModelOptions<TModel>;
+export type CountTokensOptions<TModel extends ModelId = ModelId> = CountOptions<TModel>;
+type SingleModelOptions<TModel extends ModelId = ModelId> = {
+    model: TModel;
 };
-export declare function tokenize<TModel extends ModelId>(text: string, model: TModel): Array<number>;
-export declare function tokenize<TModels extends ReadonlyArray<ModelId>>(text: string, options: CountTokensOptions<TModels>): Pick<TokenizeResult, TModels[number]>;
-export declare function tokenize<TModel extends ModelId>(text: string, options: CountTokensOptions<TModel>): Array<number>;
-export declare function tokenize(text: string, options?: CountTokensOptions<ReadonlyArray<ModelId>>): TokenizeResult;
-export declare function countTokens<TModel extends ModelId>(text: string, model: TModel): number;
-export declare function countTokens<TModels extends ReadonlyArray<ModelId>>(text: string, options: CountTokensOptions<TModels>): Pick<CountTokensResult, TModels[number]>;
-export declare function countTokens<TModel extends ModelId>(text: string, options: CountTokensOptions<TModel>): number;
-export declare function countTokens(text: string, options?: CountTokensOptions<ReadonlyArray<ModelId>>): CountTokensResult;
+export declare function tokenizeLoaded<InputGeneric extends TokenizeInput, TModel extends ModelId>(input: InputGeneric, model: TModel): RawTokenizeResult<InputGeneric>;
+export declare function tokenizeLoaded<InputGeneric extends TokenizeInput, TModel extends ModelId>(input: InputGeneric, options: TokenizeOptions<TModel>): RawTokenizeResult<InputGeneric>;
+export declare function countLoaded<TModel extends ModelId>(input: TokenizeInput, model: TModel): CountResult;
+export declare function countLoaded<TModel extends ModelId>(input: TokenizeInput, options: CountOptions<TModel>): CountResult;
+export declare function load<TModel extends ModelId>(model: TModel): Promise<TModel>;
+export declare function load(model?: ModelSelection): Promise<Array<ModelId>>;
+export declare function tokenize<InputGeneric extends TokenizeInput, TModel extends ModelId>(input: InputGeneric, model: TModel): Promise<RawTokenizeResult<InputGeneric>>;
+export declare function tokenize<InputGeneric extends TokenizeInput, TModel extends ModelId>(input: InputGeneric, options: TokenizeOptions<TModel>): Promise<RawTokenizeResult<InputGeneric>>;
+export declare function count<TModel extends ModelId>(input: TokenizeInput, model: TModel): Promise<CountResult>;
+export declare function count<TModel extends ModelId>(input: TokenizeInput, options: CountOptions<TModel>): Promise<CountResult>;
+export declare const free: (modelId?: ModelId) => void;
 export { modelIds, models };
-export default countTokens;
-export type { ModelId, ModelSelection };
+export default tokenize;
+export type { ModelId };
+export type { RawTokenizeResult, TokenizeInput } from './tokenization.js';

package/lib/data.d.ts

@@ -1,9 +1,8 @@
-import type { ModelAssetFiles } from './modelAssets.js';
+import type { ModelAssetBundleLoader, ModelAssetFiles } from './modelAssets/base/ModelAssetBundleLoader.js';
 import type { ModelId } from './models.js';
 export type ModelAssetMap = Partial<Record<ModelId, ModelAssetFiles>>;
-export declare const hasModelAssets: (modelId: ModelId) => boolean;
-export declare const registerModelAssets: (modelId: ModelId, files: ModelAssetFiles) => void;
-export declare const registerModelAssetMap: (assets: ModelAssetMap) => void;
+export declare const registerModelAssetBundleLoader: (loader: ModelAssetBundleLoader) => void;
+export declare const loadModelAssets: (selectedModels: ReadonlyArray<ModelId>) => Promise<void>;
+export declare const freeModelAssets: (modelId?: ModelId) => void;
 export declare const readModelTextFile: (modelId: ModelId, fileName: string) => string;
 export declare const readModelMsgpackFile: <T>(modelId: ModelId, fileName: string) => T;
-export declare const getAvailableModelIds: () => ModelId[];

package/lib/modelAssets/base/ModelAssetBundleLoader.d.ts

@@ -0,0 +1,6 @@
+import type { ModelId } from '../../models.js';
+export type ModelAssetFiles = Record<string, Uint8Array>;
+export declare abstract class ModelAssetBundleLoader {
+    load(modelId: ModelId): Promise<ModelAssetFiles>;
+    protected abstract readCompressedBundle(modelId: ModelId): Promise<Uint8Array>;
+}

package/lib/models.d.ts

@@ -1,4 +1,4 @@
-export type ModelId = 'deepseek' | 'gemma' | 'glm' | 'gpt' | 'kimi' | 'mimo' | 'minimax' | 'qwen' | 'sdxl';
+export type ModelId = 'deepseek' | 'gemma' | 'glm' | 'gpt' | 'hy' | 'kimi' | 'mimo' | 'minimax' | 'qwen' | 'sdxl' | 'step';
 export type ModelDefinition = BuiltinTiktokenModelDefinition | ClipBpeModelDefinition | CustomTiktokenModelDefinition | HuggingFaceModelDefinition;
 type BaseModelDefinition = {
     openrouter?: string;
@@ -22,6 +22,7 @@ type CustomTiktokenModelDefinition = BaseModelDefinition & {
 type HuggingFaceModelDefinition = BaseModelDefinition & {
     kind: 'huggingface';
     source: {
+        specialTokensMapUrl?: string;
         tokenizerConfigUrl: string;
         tokenizerJsonUrl: string;
     };
@@ -119,6 +120,25 @@ export declare const models: {
         };
         readonly title: "MiniMax M2.7";
     };
+    readonly hy: {
+        readonly kind: "huggingface";
+        readonly openrouter: "tencent/hy3-preview";
+        readonly title: "Hy3 Preview";
+        readonly source: {
+            readonly tokenizerConfigUrl: "https://huggingface.co/tencent/Hy3-preview/resolve/main/tokenizer_config.json";
+            readonly tokenizerJsonUrl: "https://huggingface.co/tencent/Hy3-preview/resolve/main/tokenizer.json";
+        };
+    };
+    readonly step: {
+        readonly kind: "huggingface";
+        readonly openrouter: "stepfun/step-3.7-flash";
+        readonly source: {
+            readonly specialTokensMapUrl: "https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/special_tokens_map.json";
+            readonly tokenizerConfigUrl: "https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/tokenizer_config.json";
+            readonly tokenizerJsonUrl: "https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/tokenizer.json";
+        };
+        readonly title: "Step 3.7 Flash";
+    };
 };
 export declare const modelIds: Array<ModelId>;
 export {};

package/lib/tiktoken.d.ts

@@ -0,0 +1,4 @@
+import type { TiktokenEncoding } from 'tiktoken';
+import { get_encoding, Tiktoken } from 'tiktoken';
+export { get_encoding, Tiktoken };
+export type { TiktokenEncoding };

package/lib/tokenization.d.ts

@@ -0,0 +1,38 @@
+export type TokenizeInput = Uint8Array | string;
+export type RawTokenizeResultBase = {
+    /**
+     * array of byte starting offsets for every token after the first one
+     *
+     * Offsets are measured against the tokenizer’s effective UTF-8 text representation after its built-in normalization and preprocessing. That usually matches the provided input, but tokenizers that normalize or discard characters can shift the reported positions.
+     *
+     * The array length is equal to the number of tokens minus one, since the first token always starts at byte offset `0` within that effective tokenizer input.
+     */
+    offsets: Array<number>;
+    /**
+     * array of token IDs in token order
+     */
+    tokens: Array<number>;
+};
+export type RawTextTokenizeResultExtension = {
+    /**
+     * the actual input passed to the tokenizer after applying all preprocessing/normalization steps
+     *
+     * If no such steps are defined by the corresponding tokenizer or they didn’t modify the input, this property is absent.
+     */
+    processedInput?: string;
+};
+export type RawBinaryTokenizeResultExtension = {
+    /**
+     * the actual input passed to the tokenizer after applying all preprocessing/normalization steps
+     *
+     * If no such steps are defined by the corresponding tokenizer or they didn’t modify the input, this property is absent.
+     */
+    processedInput?: Uint8Array;
+};
+export type RawTokenizeResult<InputGeneric extends TokenizeInput = TokenizeInput> = RawTokenizeResultBase & (InputGeneric extends Uint8Array ? RawBinaryTokenizeResultExtension : RawTextTokenizeResultExtension);
+export type RawBinaryTokenizeResult = RawTokenizeResult<Uint8Array>;
+export type RawTextTokenizeResult = RawTokenizeResult<string>;
+export declare const toTokenizeText: (input: TokenizeInput) => string;
+export declare const getUtf8ByteLength: (text: string) => number;
+export declare const toRawTokenizeResult: <InputGeneric extends TokenizeInput = string>(tokens: ReadonlyArray<number>, tokenStartOffsets: ReadonlyArray<number>, processedInput?: InputGeneric) => RawTokenizeResult<InputGeneric>;
+export declare const toRawTokenizeResultFromTokenByteLengths: <InputGeneric extends TokenizeInput = string>(tokens: ReadonlyArray<number>, tokenByteLengths: ReadonlyArray<number>, processedInput?: InputGeneric) => RawTokenizeResult<InputGeneric>;

package/lib/tokenizers/ClipTokenizer.d.ts

@@ -1,4 +1,5 @@
 import type { ModelId } from '../models.js';
+import type { RawTextTokenizeResult } from '../tokenization.js';
 import { BaseTokenizer } from './base/BaseTokenizer.js';
 type ClipTokenizerState = {
     byteEncoder: Array<string>;
@@ -19,5 +20,6 @@ export declare class ClipTokenizer extends BaseTokenizer<ClipTokenizerState> {
         vocabulary: Map<string, number>;
     };
     protected encodeWithState(text: string, state: ClipTokenizerState): number[];
+    protected tokenizeWithState(text: string, state: ClipTokenizerState): RawTextTokenizeResult;
 }
 export {};

package/lib/tokenizers/HuggingFaceTokenizer.d.ts

@@ -3,8 +3,16 @@ import { BaseTokenizer } from './base/BaseTokenizer.js';
 type HfEncodeResult = {
     ids: Array<number>;
 };
+type HfEncodeOptions = {
+    add_special_tokens?: boolean;
+};
+type HfDecodeOptions = {
+    clean_up_tokenization_spaces?: boolean;
+    skip_special_tokens?: boolean;
+};
 type HfTokenizer = {
-    encode: (text: string) => HfEncodeResult;
+    decode: (tokenIds: Array<number>, options?: HfDecodeOptions) => string;
+    encode: (text: string, options?: HfEncodeOptions) => HfEncodeResult;
 };
 type TokenizerState = {
     tokenizer: HfTokenizer;
@@ -16,5 +24,6 @@ export declare class HuggingFaceTokenizer extends BaseTokenizer<TokenizerState>
         tokenizer: HfTokenizer;
     };
     protected encodeWithState(text: string, state: TokenizerState): number[];
+    protected tokenizeWithState(text: string, state: TokenizerState): import("../tokenization.js").RawTokenizeResult<string>;
 }
 export {};

package/lib/tokenizers/TiktokenTokenizer.d.ts

@@ -1,15 +1,20 @@
 import type { ModelId } from '../models.js';
-import { Tiktoken } from 'tiktoken';
+import { Tiktoken } from '#tiktoken';
 import { BaseTokenizer } from './base/BaseTokenizer.js';
-export declare class BuiltinTiktokenTokenizer extends BaseTokenizer<Tiktoken> {
+declare abstract class BaseTiktokenTokenizer extends BaseTokenizer<Tiktoken> {
+    protected disposeState(state: Tiktoken): void;
+    protected tokenizeWithState(text: string, state: Tiktoken): import("../tokenization.js").RawTokenizeResult<string>;
+}
+export declare class BuiltinTiktokenTokenizer extends BaseTiktokenTokenizer {
     readonly modelId: ModelId;
     constructor(modelId: ModelId);
     protected createState(): Tiktoken;
     protected encodeWithState(text: string, state: Tiktoken): number[];
 }
-export declare class CustomTiktokenTokenizer extends BaseTokenizer<Tiktoken> {
+export declare class CustomTiktokenTokenizer extends BaseTiktokenTokenizer {
     readonly modelId: ModelId;
     constructor(modelId: ModelId);
     protected createState(): Tiktoken;
     protected encodeWithState(text: string, state: Tiktoken): number[];
 }
+export {};

package/lib/tokenizers/base/BaseTokenizer.d.ts

@@ -1,8 +1,13 @@
+import type { RawTextTokenizeResult, RawTokenizeResult, TokenizeInput } from '../../tokenization.js';
 export declare abstract class BaseTokenizer<TState> {
     #private;
     protected abstract createState(): TState;
-    encode(text: string): number[];
+    protected disposeState(_state: TState): void;
+    encode(input: TokenizeInput): number[];
     protected abstract encodeWithState(text: string, state: TState): Array<number>;
+    free(): void;
     protected getState(): TState;
-    getTokenCount(text: string): number;
+    getTokenCount(input: TokenizeInput): number;
+    tokenize<InputGeneric extends TokenizeInput>(input: InputGeneric): RawTokenizeResult<InputGeneric>;
+    protected abstract tokenizeWithState(text: string, state: TState): RawTextTokenizeResult;
 }

package/lib/tokenizers/index.d.ts

@@ -1,7 +1,11 @@
 import type { ModelId } from '../models.js';
+import type { RawTokenizeResult, TokenizeInput } from '../tokenization.js';
 type TokenizerLike = {
-    encode: (text: string) => Array<number>;
-    getTokenCount: (text: string) => number;
+    encode: (input: TokenizeInput) => Array<number>;
+    free: () => void;
+    getTokenCount: (input: TokenizeInput) => number;
+    tokenize: (input: TokenizeInput) => RawTokenizeResult;
 };
 export declare const getTokenizer: (modelId: ModelId) => TokenizerLike;
+export declare const freeTokenizers: (modelId?: ModelId) => void;
 export {};

package/main.d.ts

@@ -1,10 +1,3 @@
-import type {ModelId, ModelSelection} from './lib/api.js'
-
-export {countTokens, modelIds, models, tokenize} from './lib/api.js'
+export {count, countLoaded, free, load, modelIds, models, tokenize, tokenizeLoaded} from './lib/api.js'
 export {default} from './lib/api.js'
-export type {CountTokensOptions, CountTokensResult, ModelId, ModelSelection, TokenizeResult} from './lib/api.js'
-
-export declare const isModelLoaded: (modelId: ModelId) => boolean
-export declare const getLoadedModelIds: () => Array<ModelId>
-export declare const loadModel: (modelId: ModelId) => Promise<ModelId>
-export declare const loadModels: (model?: ModelSelection) => Promise<Array<ModelId>>
+export type {CountOptions, CountResult, CountTokensOptions, CountTokensResult, ModelId, ModelSelection, RawTokenizeResult, TokenizeInput, TokenizeOptions, TokenizeResult} from './lib/api.js'

package/main.js

@@ -1 +1 @@
-import{a as e,i as t,l as n,n as r,o as i,r as a,t as o,u as s}from"./chunks/main.js";export{e as countTokens,e as default,o as getLoadedModelIds,r as isModelLoaded,a as loadModel,t as loadModels,n as modelIds,s as models,i as tokenize};
+import{a as e,c as t,i as n,n as r,o as i,r as a,s as o,t as s}from"./chunks/main.js";export{s as count,r as countLoaded,e as default,e as tokenize,a as free,n as load,o as modelIds,t as models,i as tokenizeLoaded};